diff --git a/docSite/content/zh-cn/docs/development/upgrading/4820.md b/docSite/content/zh-cn/docs/development/upgrading/4820.md index 07f86b464..ecf160a9b 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4820.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4820.md @@ -35,4 +35,5 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4820' \ 3. 新增 - markdown 语法扩展,支持音视频(代码块 audio 和 video)。 4. 优化 - 页面组件抽离,减少页面组件路由。 5. 优化 - 全文检索,忽略大小写。 -6. 优化 - 问答生成和增强索引改成流输出,避免部分模型超时。 \ No newline at end of file +6. 优化 - 问答生成和增强索引改成流输出,避免部分模型超时。 +7. 优化 - 自动给 assistant 空 content,补充 null,同时合并连续的 text assistant,避免部分模型抛错。 \ No newline at end of file diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts index ed35466e8..130b894e8 100644 --- a/packages/service/core/chat/utils.ts +++ b/packages/service/core/chat/utils.ts @@ -1,6 +1,9 @@ import { countGptMessagesTokens } from '../../common/string/tiktoken/index'; import type { + ChatCompletionAssistantMessageParam, ChatCompletionContentPart, + ChatCompletionContentPartRefusal, + ChatCompletionContentPartText, ChatCompletionMessageParam, SdkChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d'; @@ -102,223 +105,324 @@ export const loadRequestMessages = async ({ useVision?: boolean; origin?: string; }) => { - // Load image to base64 - const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => { - return Promise.all( - messages.map(async (item) => { - if (item.type === 'image_url') { - // Remove url origin - const imgUrl = (() => { - if (origin && item.image_url.url.startsWith(origin)) { - return item.image_url.url.replace(origin, ''); - } - return item.image_url.url; - })(); - - // base64 image - if (imgUrl.startsWith('data:image/')) { - return item; - } - - try { - // If imgUrl is a local path, load image from local, and set url to base64 - if (imgUrl.startsWith('/') || process.env.MULTIPLE_DATA_TO_BASE64 === 'true') { - addLog.debug('Load image from local server', { - baseUrl: serverRequestBaseUrl, - requestUrl: imgUrl - }); - const response = await axios.get(imgUrl, { - baseURL: serverRequestBaseUrl, - responseType: 'arraybuffer', - proxy: false - }); - const base64 = Buffer.from(response.data, 'binary').toString('base64'); - const imageType = - getFileContentTypeFromHeader(response.headers['content-type']) || - guessBase64ImageType(base64); - - return { - ...item, - image_url: { - ...item.image_url, - url: `data:${imageType};base64,${base64}` - } - }; - } - - // 检查下这个图片是否可以被访问,如果不行的话,则过滤掉 - const response = await axios.head(imgUrl, { - timeout: 10000 - }); - if (response.status < 200 || response.status >= 400) { - addLog.info(`Filter invalid image: ${imgUrl}`); - return; - } - } catch (error) { - return; - } - } - return item; - }) - ).then((res) => res.filter(Boolean) as ChatCompletionContentPart[]); + const replaceLinkUrl = (text: string) => { + const baseURL = process.env.FE_DOMAIN; + if (!baseURL) return text; + // 匹配 /api/system/img/xxx.xx 的图片链接,并追加 baseURL + return text.replace(/(\/api\/system\/img\/[^\s.]*\.[^\s]*)/g, (match, p1) => `${baseURL}${p1}`); }; - // Split question text and image - const parseStringWithImages = (input: string): ChatCompletionContentPart[] => { - if (!useVision || input.length > 500) { - return [{ type: 'text', text: input || '' }]; + const parseSystemMessage = ( + content: string | ChatCompletionContentPartText[] + ): string | ChatCompletionContentPartText[] | undefined => { + if (typeof content === 'string') { + if (!content) return; + return replaceLinkUrl(content); } - // 正则表达式匹配图片URL - const imageRegex = - /(https?:\/\/[^\s/$.?#].[^\s]*\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg|ico|heic|avif))/gi; - - const result: ChatCompletionContentPart[] = []; - - // 提取所有HTTPS图片URL并添加到result开头 - const httpsImages = [...new Set(Array.from(input.matchAll(imageRegex), (m) => m[0]))]; - httpsImages.forEach((url) => { - result.push({ - type: 'image_url', - image_url: { - url: url - } - }); - }); - - // Too many images return text - if (httpsImages.length > 4) { - return [{ type: 'text', text: input || '' }]; - } - - // 添加原始input作为文本 - result.push({ type: 'text', text: input }); - return result; + const arrayContent = content + .filter((item) => item.text) + .map((item) => ({ ...item, text: replaceLinkUrl(item.text) })); + if (arrayContent.length === 0) return; + return arrayContent; }; // Parse user content(text and img) Store history => api messages const parseUserContent = async (content: string | ChatCompletionContentPart[]) => { - if (typeof content === 'string') { - return loadImageToBase64(parseStringWithImages(content)); - } - - const result = await Promise.all( - content.map(async (item) => { - if (item.type === 'text') return parseStringWithImages(item.text); - if (item.type === 'file_url') return; // LLM not support file_url - - if (!item.image_url.url) return item; - - return item; - }) - ); - - return loadImageToBase64(result.flat().filter(Boolean) as ChatCompletionContentPart[]); - }; - - // format GPT messages, concat text messages - const clearInvalidMessages = (messages: ChatCompletionMessageParam[]) => { - return messages - .map((item) => { - if (item.role === ChatCompletionRequestMessageRoleEnum.System && !item.content) { - return; - } - if (item.role === ChatCompletionRequestMessageRoleEnum.User) { - if (item.content === undefined) return; - - if (typeof item.content === 'string') { - return { - ...item, - content: item.content.trim() - }; - } - - // array - if (item.content.length === 0) return; - if (item.content.length === 1 && item.content[0].type === 'text') { - return { - ...item, - content: item.content[0].text - }; - } - } - if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) { - if (item.content === undefined && !item.tool_calls && !item.function_call) return; - } - - return item; - }) - .filter(Boolean) as ChatCompletionMessageParam[]; - }; - /* - Merge data for some consecutive roles - 1. Contiguous assistant and both have content, merge content - */ - const mergeConsecutiveMessages = ( - messages: ChatCompletionMessageParam[] - ): ChatCompletionMessageParam[] => { - return messages.reduce((mergedMessages: ChatCompletionMessageParam[], currentMessage) => { - const lastMessage = mergedMessages[mergedMessages.length - 1]; - - if ( - lastMessage && - currentMessage.role === ChatCompletionRequestMessageRoleEnum.Assistant && - lastMessage.role === ChatCompletionRequestMessageRoleEnum.Assistant && - typeof lastMessage.content === 'string' && - typeof currentMessage.content === 'string' - ) { - lastMessage.content += currentMessage ? `\n${currentMessage.content}` : ''; - } else { - mergedMessages.push(currentMessage); + // Split question text and image + const parseStringWithImages = (input: string): ChatCompletionContentPart[] => { + if (!useVision || input.length > 500) { + return [{ type: 'text', text: input }]; } - return mergedMessages; - }, []); + // 正则表达式匹配图片URL + const imageRegex = + /(https?:\/\/[^\s/$.?#].[^\s]*\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg|ico|heic|avif))/gi; + + const result: ChatCompletionContentPart[] = []; + + // 提取所有HTTPS图片URL并添加到result开头 + const httpsImages = [...new Set(Array.from(input.matchAll(imageRegex), (m) => m[0]))]; + httpsImages.forEach((url) => { + result.push({ + type: 'image_url', + image_url: { + url: url + } + }); + }); + + // Too many images return text + if (httpsImages.length > 4) { + return [{ type: 'text', text: input }]; + } + + // 添加原始input作为文本 + result.push({ type: 'text', text: input }); + return result; + }; + // Load image to base64 + const loadUserContentImage = async (content: ChatCompletionContentPart[]) => { + return Promise.all( + content.map(async (item) => { + if (item.type === 'image_url') { + // Remove url origin + const imgUrl = (() => { + if (origin && item.image_url.url.startsWith(origin)) { + return item.image_url.url.replace(origin, ''); + } + return item.image_url.url; + })(); + + // base64 image + if (imgUrl.startsWith('data:image/')) { + return item; + } + + try { + // If imgUrl is a local path, load image from local, and set url to base64 + if (imgUrl.startsWith('/') || process.env.MULTIPLE_DATA_TO_BASE64 === 'true') { + addLog.debug('Load image from local server', { + baseUrl: serverRequestBaseUrl, + requestUrl: imgUrl + }); + const response = await axios.get(imgUrl, { + baseURL: serverRequestBaseUrl, + responseType: 'arraybuffer', + proxy: false + }); + const base64 = Buffer.from(response.data, 'binary').toString('base64'); + const imageType = + getFileContentTypeFromHeader(response.headers['content-type']) || + guessBase64ImageType(base64); + + return { + ...item, + image_url: { + ...item.image_url, + url: `data:${imageType};base64,${base64}` + } + }; + } + + // 检查下这个图片是否可以被访问,如果不行的话,则过滤掉 + const response = await axios.head(imgUrl, { + timeout: 10000 + }); + if (response.status < 200 || response.status >= 400) { + addLog.info(`Filter invalid image: ${imgUrl}`); + return; + } + } catch (error) { + return; + } + } + return item; + }) + ).then((res) => res.filter(Boolean) as ChatCompletionContentPart[]); + }; + + if (content === undefined) return; + if (typeof content === 'string') { + if (content === '') return; + + const loadImageContent = await loadUserContentImage(parseStringWithImages(content)); + if (loadImageContent.length === 0) return; + return loadImageContent; + } + + const result = ( + await Promise.all( + content.map(async (item) => { + if (item.type === 'text') { + if (item.text) return parseStringWithImages(item.text); + return; + } + if (item.type === 'file_url') return; // LLM not support file_url + if (item.type === 'image_url') { + // close vision, remove image_url + if (!useVision) return; + // remove empty image_url + if (!item.image_url.url) return; + } + + return item; + }) + ) + ) + .flat() + .filter(Boolean) as ChatCompletionContentPart[]; + + const loadImageContent = await loadUserContentImage(result); + + if (loadImageContent.length === 0) return; + return loadImageContent; + }; + + const formatAssistantItem = (item: ChatCompletionAssistantMessageParam) => { + return { + role: item.role, + content: item.content, + function_call: item.function_call, + name: item.name, + refusal: item.refusal, + tool_calls: item.tool_calls + }; + }; + const parseAssistantContent = ( + content: + | string + | (ChatCompletionContentPartText | ChatCompletionContentPartRefusal)[] + | null + | undefined + ) => { + if (typeof content === 'string') { + return content || ''; + } + // 交互节点 + if (!content) return ''; + + const result = content.filter((item) => item?.type === 'text'); + if (result.length === 0) return ''; + + return result.map((item) => item.text).join('\n'); }; if (messages.length === 0) { return Promise.reject(i18nT('common:core.chat.error.Messages empty')); } - // filter messages file - const filterMessages = messages.map((item) => { - // If useVision=false, only retain text. - if ( - item.role === ChatCompletionRequestMessageRoleEnum.User && - Array.isArray(item.content) && - !useVision - ) { - return { - ...item, - content: item.content.filter((item) => item.type === 'text') - }; - } + // 合并相邻 role 的内容,只保留一个 role, content 变成数组。 assistant 的话,工具调用不合并。 + const mergeMessages = ((messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[] => { + return messages.reduce((mergedMessages: ChatCompletionMessageParam[], currentMessage) => { + const lastMessage = mergedMessages[mergedMessages.length - 1]; - return item; - }); - - const loadMessages = (await Promise.all( - filterMessages.map(async (item) => { - if (item.role === ChatCompletionRequestMessageRoleEnum.User) { - return { - ...item, - content: await parseUserContent(item.content) - }; - } else if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) { - // remove invalid field - return { - role: item.role, - content: item.content, - function_call: item.function_call, - name: item.name, - refusal: item.refusal, - tool_calls: item.tool_calls - }; - } else { - return item; + if (!lastMessage) { + return [currentMessage]; } - }) - )) as ChatCompletionMessageParam[]; - return mergeConsecutiveMessages( - clearInvalidMessages(loadMessages) - ) as SdkChatCompletionMessageParam[]; + if ( + lastMessage.role === ChatCompletionRequestMessageRoleEnum.System && + currentMessage.role === ChatCompletionRequestMessageRoleEnum.System + ) { + const lastContent: ChatCompletionContentPartText[] = Array.isArray(lastMessage.content) + ? lastMessage.content + : [{ type: 'text', text: lastMessage.content || '' }]; + const currentContent: ChatCompletionContentPartText[] = Array.isArray( + currentMessage.content + ) + ? currentMessage.content + : [{ type: 'text', text: currentMessage.content || '' }]; + lastMessage.content = [...lastContent, ...currentContent]; + } // Handle user messages + else if ( + lastMessage.role === ChatCompletionRequestMessageRoleEnum.User && + currentMessage.role === ChatCompletionRequestMessageRoleEnum.User + ) { + const lastContent: ChatCompletionContentPart[] = Array.isArray(lastMessage.content) + ? lastMessage.content + : [{ type: 'text', text: lastMessage.content }]; + const currentContent: ChatCompletionContentPart[] = Array.isArray(currentMessage.content) + ? currentMessage.content + : [{ type: 'text', text: currentMessage.content }]; + lastMessage.content = [...lastContent, ...currentContent]; + } else if ( + lastMessage.role === ChatCompletionRequestMessageRoleEnum.Assistant && + currentMessage.role === ChatCompletionRequestMessageRoleEnum.Assistant + ) { + // Content 不为空的对象,或者是交互节点 + if ( + (typeof lastMessage.content === 'string' || + Array.isArray(lastMessage.content) || + lastMessage.interactive) && + (typeof currentMessage.content === 'string' || + Array.isArray(currentMessage.content) || + currentMessage.interactive) + ) { + const lastContent: (ChatCompletionContentPartText | ChatCompletionContentPartRefusal)[] = + Array.isArray(lastMessage.content) + ? lastMessage.content + : [{ type: 'text', text: lastMessage.content || '' }]; + const currentContent: ( + | ChatCompletionContentPartText + | ChatCompletionContentPartRefusal + )[] = Array.isArray(currentMessage.content) + ? currentMessage.content + : [{ type: 'text', text: currentMessage.content || '' }]; + + lastMessage.content = [...lastContent, ...currentContent]; + } else { + // 有其中一个没有 content,说明不是连续的文本输出 + mergedMessages.push(currentMessage); + } + } else { + mergedMessages.push(currentMessage); + } + + return mergedMessages; + }, []); + })(messages); + + const loadMessages = ( + await Promise.all( + mergeMessages.map(async (item, i) => { + if (item.role === ChatCompletionRequestMessageRoleEnum.System) { + const content = parseSystemMessage(item.content); + if (!content) return; + return { + ...item, + content + }; + } else if (item.role === ChatCompletionRequestMessageRoleEnum.User) { + const content = await parseUserContent(item.content); + if (!content) { + return { + ...item, + content: 'null' + }; + } + + const formatContent = (() => { + if (Array.isArray(content) && content.length === 1 && content[0].type === 'text') { + return content[0].text; + } + return content; + })(); + + return { + ...item, + content: formatContent + }; + } else if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) { + if (item.tool_calls || item.function_call) { + return formatAssistantItem(item); + } + + const parseContent = parseAssistantContent(item.content); + + // 如果内容为空,且前后不再是 assistant,需要补充成 null,避免丢失 user-assistant 的交互 + const formatContent = (() => { + const lastItem = mergeMessages[i - 1]; + const nextItem = mergeMessages[i + 1]; + if ( + parseContent === '' && + (lastItem?.role === ChatCompletionRequestMessageRoleEnum.Assistant || + nextItem?.role === ChatCompletionRequestMessageRoleEnum.Assistant) + ) { + return; + } + return parseContent || 'null'; + })(); + if (!formatContent) return; + + return { + ...formatAssistantItem(item), + content: formatContent + }; + } else { + return item; + } + }) + ) + ).filter(Boolean) as ChatCompletionMessageParam[]; + + return loadMessages as SdkChatCompletionMessageParam[]; }; diff --git a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts index 71d5f4021..2b0f7d457 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts @@ -272,7 +272,7 @@ export const runToolWithToolChoice = async ( }, toolModel ); - // console.log(JSON.stringify(requestBody, null, 2), '==requestBody'); + // console.log(JSON.stringify(requestMessages, null, 2), '==requestBody'); /* Run llm */ const { response: aiResponse,