perf: push chat log (#3109)

docSite/content/zh-cn/docs/development/upgrading/4813.md

@@ -30,12 +30,13 @@ weight: 811
 5. 新增 - 循环节点增加下标值。 
 6. 新增 - 部分对话错误提醒增加翻译。 
 7. 新增 - 对话输入框支持拖拽文件上传，可直接拖文件到输入框中。  
-8. 优化 - 合并多个 system 提示词成 1 个，避免部分模型不支持多个 system 提示词。  
+8. 新增 - 对话日志，来源可显示分享链接/API具体名称
-9. 优化 - 知识库上传文件，优化报错提示。  
+9. 优化 - 合并多个 system 提示词成 1 个，避免部分模型不支持多个 system 提示词。  
-10. 优化 - 全文检索语句，减少一轮子查询。  
+10. 优化 - 知识库上传文件，优化报错提示。  
-11. 优化 - 修改 findLast 为 [...array].reverse().find，适配旧版浏览器。  
+11. 优化 - 全文检索语句，减少一轮子查询。  
-12. 优化 - Markdown 组件自动空格，避免分割 url 中的中文。  
+12. 优化 - 修改 findLast 为 [...array].reverse().find，适配旧版浏览器。  
-13. 优化 - 工作流上下文拆分，性能优化。 
+13. 优化 - Markdown 组件自动空格，避免分割 url 中的中文。  
-14. 优化 - 语音播报，不支持 mediaSource 的浏览器可等待完全生成语音后输出。 
+14. 优化 - 工作流上下文拆分，性能优化。 
-15. 修复 - Dockerfile pnpm install 支持代理。 
+15. 优化 - 语音播报，不支持 mediaSource 的浏览器可等待完全生成语音后输出。 
-16. 修复 - BI 图表生成无法写入文件。 
+16. 修复 - Dockerfile pnpm install 支持代理。 
+17. 修复 - BI 图表生成无法写入文件。 

packages/service/core/chat/chatSchema.ts

@@ -52,7 +52,6 @@ const ChatSchema = new Schema({
   },
   source: {
     type: String,
-    enum: Object.keys(ChatSourceMap),
     required: true
   },
   shareId: {

packages/service/core/chat/pushChatLog.ts

@@ -2,7 +2,8 @@ import { addLog } from '../../common/system/log';
 import { MongoChatItem } from './chatItemSchema';
 import { MongoChat } from './chatSchema';
 import axios from 'axios';
-import { AIChatItemType, ChatItemType } from '@fastgpt/global/core/chat/type';
+import { AIChatItemType, ChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type';
+import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
 
 export type Metadata = {
   [key: string]: {
@@ -26,8 +27,8 @@ export const pushChatLog = ({
 }) => {
   const interval = Number(process.env.CHAT_LOG_INTERVAL);
   const url = process.env.CHAT_LOG_URL;
-  if (interval > 0 && url) {
+  if (!isNaN(interval) && interval > 0 && url) {
-    addLog.info(`[ChatLogPush] push chat log after ${interval}ms`, {
+    addLog.debug(`[ChatLogPush] push chat log after ${interval}ms`, {
       appId,
       chatItemIdHuman,
       chatItemIdAi
@@ -82,7 +83,7 @@ const pushChatLogInternal = async ({
 }) => {
   try {
     const [chatItemHuman, chatItemAi] = await Promise.all([
-      MongoChatItem.findById(chatItemIdHuman).lean(),
+      MongoChatItem.findById(chatItemIdHuman).lean() as Promise<UserChatItemType>,
       MongoChatItem.findById(chatItemIdAi).lean() as Promise<AIChatItemType>
     ]);
 
@@ -103,8 +104,52 @@ const pushChatLogInternal = async ({
 
     const uid = chat.outLinkUid || chat.tmbId;
     // Pop last two items
-    const question = chatItemHuman.value[chatItemHuman.value.length - 1]?.text?.content;
+    const question = chatItemHuman.value
-    const answer = chatItemAi.value[chatItemAi.value.length - 1]?.text?.content;
+      .map((item) => {
+        if (item.type === ChatItemValueTypeEnum.text) {
+          return item.text?.content;
+        } else if (item.type === ChatItemValueTypeEnum.file) {
+          if (item.file?.type === 'image') {
+            return `![${item.file?.name}](${item.file?.url})`;
+          }
+          return `[${item.file?.name}](${item.file?.url})`;
+        }
+        return '';
+      })
+      .join('\n');
+    const answer = chatItemAi.value
+      .map((item) => {
+        const text = [];
+        if (item.text?.content) {
+          text.push(item.text?.content);
+        }
+        if (item.tools) {
+          text.push(
+            item.tools.map(
+              (tool) =>
+                `\`\`\`json
+${JSON.stringify(
+  {
+    name: tool.toolName,
+    params: tool.params,
+    response: tool.response
+  },
+  null,
+  2
+)}
+\`\`\``
+            )
+          );
+        }
+        if (item.interactive) {
+          text.push(`\`\`\`json
+${JSON.stringify(item.interactive, null, 2)}
+            \`\`\``);
+        }
+        return text.join('\n');
+      })
+      .join('\n');
+
     if (!question || !answer) {
       addLog.error('[ChatLogPush] question or answer is empty', {
         question: chatItemHuman.value,
@@ -112,11 +157,13 @@ const pushChatLogInternal = async ({
       });
       return;
     }
+
+    // computed response time
     const responseData = chatItemAi.responseData;
     const responseTime =
       responseData?.reduce((acc, item) => acc + (item?.runningTime ?? 0), 0) || 0;
 
-    const sourceIdPrefix = process.env.SOURCE_ID_PREFIX ?? '';
+    const sourceIdPrefix = process.env.CHAT_LOG_SOURCE_ID_PREFIX ?? 'fastgpt-';
 
     const chatLog: ChatLog = {
       title: chat.title,
@@ -141,14 +188,7 @@ const pushChatLogInternal = async ({
       createdAt: new Date(chatItemAi.time).getTime(),
       sourceId: `${sourceIdPrefix}${appId}`
     };
-    await axios
+    await axios.post(`${url}/api/chat/push`, chatLog);
-      .post(`${url}/api/chat/push`, chatLog)
-      .then((res) => {
-        addLog.info('[ChatLogPush] push success', res.data);
-      })
-      .catch((e) => {
-        addLog.error('[ChatLogPush] push failed', { e, resData: e.response?.data });
-      });
   } catch (e) {
     addLog.error('[ChatLogPush] error', e);
   }

packages/service/core/chat/saveChat.ts

@@ -30,7 +30,7 @@ type Props = {
   variables?: Record<string, any>;
   isUpdateUseTime: boolean;
   newTitle: string;
-  source: `${ChatSourceEnum}`;
+  source: string;
   shareId?: string;
   outLinkUid?: string;
   content: [UserChatItemType & { dataId?: string }, AIChatItemType & { dataId?: string }];

packages/service/support/openapi/auth.ts

@@ -11,7 +11,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
     return Promise.reject(ERROR_ENUM.unAuthApiKey);
   }
   try {
-    const openApi = await MongoOpenApi.findOne({ apiKey: apikey.trim() });
+    const openApi = await MongoOpenApi.findOne({ apiKey: apikey.trim() }).lean();
     if (!openApi) {
       return Promise.reject(ERROR_ENUM.unAuthApiKey);
     }
@@ -20,7 +20,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
     // @ts-ignore
     if (global.feConfigs?.isPlus) {
       await POST('/support/openapi/authLimit', {
-        openApi: openApi.toObject()
+        openApi
       } as AuthOpenApiLimitProps);
     }
 
@@ -30,7 +30,8 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
       apikey,
       teamId: String(openApi.teamId),
       tmbId: String(openApi.tmbId),
-      appId: openApi.appId || ''
+      appId: openApi.appId || '',
+      sourceName: openApi.name
     };
   } catch (error) {
     return Promise.reject(error);

packages/service/support/permission/controller.ts

@@ -313,14 +313,15 @@ export async function parseHeaderCert({
     })();
 
     // auth apikey
-    const { teamId, tmbId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
+    const { teamId, tmbId, appId: apiKeyAppId = '', sourceName } = await authOpenApiKey({ apikey });
 
     return {
       uid: '',
       teamId,
       tmbId,
       apikey,
-      appId: apiKeyAppId || authorizationAppid
+      appId: apiKeyAppId || authorizationAppid,
+      sourceName
     };
   }
   // root user
@@ -332,48 +333,50 @@ export async function parseHeaderCert({
 
   const { cookie, token, rootkey, authorization } = (req.headers || {}) as ReqHeaderAuthType;
 
-  const { uid, teamId, tmbId, appId, openApiKey, authType, isRoot } = await (async () => {
+  const { uid, teamId, tmbId, appId, openApiKey, authType, isRoot, sourceName } =
-    if (authApiKey && authorization) {
+    await (async () => {
-      // apikey from authorization
+      if (authApiKey && authorization) {
-      const authResponse = await parseAuthorization(authorization);
+        // apikey from authorization
-      return {
+        const authResponse = await parseAuthorization(authorization);
-        uid: authResponse.uid,
+        return {
-        teamId: authResponse.teamId,
+          uid: authResponse.uid,
-        tmbId: authResponse.tmbId,
+          teamId: authResponse.teamId,
-        appId: authResponse.appId,
+          tmbId: authResponse.tmbId,
-        openApiKey: authResponse.apikey,
+          appId: authResponse.appId,
-        authType: AuthUserTypeEnum.apikey
+          openApiKey: authResponse.apikey,
-      };
+          authType: AuthUserTypeEnum.apikey,
-    }
+          sourceName: authResponse.sourceName
-    if (authToken && (token || cookie)) {
+        };
-      // user token(from fastgpt web)
+      }
-      const res = await authCookieToken(cookie, token);
+      if (authToken && (token || cookie)) {
-      return {
+        // user token(from fastgpt web)
-        uid: res.userId,
+        const res = await authCookieToken(cookie, token);
-        teamId: res.teamId,
+        return {
-        tmbId: res.tmbId,
+          uid: res.userId,
-        appId: '',
+          teamId: res.teamId,
-        openApiKey: '',
+          tmbId: res.tmbId,
-        authType: AuthUserTypeEnum.token,
+          appId: '',
-        isRoot: res.isRoot
+          openApiKey: '',
-      };
+          authType: AuthUserTypeEnum.token,
-    }
+          isRoot: res.isRoot
-    if (authRoot && rootkey) {
+        };
-      await parseRootKey(rootkey);
+      }
-      // root user
+      if (authRoot && rootkey) {
-      return {
+        await parseRootKey(rootkey);
-        uid: '',
+        // root user
-        teamId: '',
+        return {
-        tmbId: '',
+          uid: '',
-        appId: '',
+          teamId: '',
-        openApiKey: '',
+          tmbId: '',
-        authType: AuthUserTypeEnum.root,
+          appId: '',
-        isRoot: true
+          openApiKey: '',
-      };
+          authType: AuthUserTypeEnum.root,
-    }
+          isRoot: true
+        };
+      }
 
-    return Promise.reject(ERROR_ENUM.unAuthorization);
+      return Promise.reject(ERROR_ENUM.unAuthorization);
-  })();
+    })();
 
   if (!authRoot && (!teamId || !tmbId)) {
     return Promise.reject(ERROR_ENUM.unAuthorization);
@@ -385,6 +388,7 @@ export async function parseHeaderCert({
     tmbId: String(tmbId),
     appId,
     authType,
+    sourceName,
     apikey: openApiKey,
     isRoot: !!isRoot
   };

projects/app/.env.template

@@ -53,5 +53,5 @@ WORKFLOW_MAX_LOOP_TIMES=50
 # CHAT_LOG_URL=http://localhost:8080
 # # 日志推送间隔
 # CHAT_LOG_INTERVAL=10000
-# # 日志来源前缀
+# # 日志来源ID前缀
-# SOURCE_ID_PREFIX=fastgpt-
+# CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-

projects/app/src/pages/api/v1/chat/completions.ts

@@ -87,6 +87,7 @@ type AuthResponseType = {
   apikey?: string;
   canWrite: boolean;
   outLinkUserId?: string;
+  sourceName?: string;
 };
 
 async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -156,6 +157,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       app,
       responseDetail,
       authType,
+      sourceName,
       apikey,
       canWrite,
       outLinkUserId = customUid
@@ -343,7 +345,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
           newTitle,
           shareId,
           outLinkUid: outLinkUserId,
-          source,
+          source: sourceName || source,
           content: [userQuestion, aiResponse],
           metadata: {
             originIp,
@@ -459,7 +461,7 @@ const authShareChat = async ({
   shareId: string;
   chatId?: string;
 }): Promise<AuthResponseType> => {
-  const { teamId, tmbId, user, appId, authType, responseDetail, uid } =
+  const { teamId, tmbId, user, appId, authType, responseDetail, uid, sourceName } =
     await authOutLinkChatStart(data);
   const app = await MongoApp.findById(appId).lean();
 
@@ -474,6 +476,7 @@ const authShareChat = async ({
   }
 
   return {
+    sourceName,
     teamId,
     tmbId,
     user,
@@ -541,6 +544,7 @@ const authHeaderRequest = async ({
     teamId,
     tmbId,
     authType,
+    sourceName,
     apikey
   } = await authCert({
     req,
@@ -607,6 +611,7 @@ const authHeaderRequest = async ({
     responseDetail: true,
     apikey,
     authType,
+    sourceName,
     canWrite: true
   };
 };

projects/app/src/pages/app/detail/components/Logs/index.tsx

@@ -129,15 +129,16 @@ const Logs = () => {
                   onClick={() => setDetailLogsId(item.id)}
                 >
                   <Td>
-                    <Box>{t(ChatSourceMap[item.source]?.name || ('UnKnow' as any))}</Box>
+                    {/* @ts-ignore */}
+                    <Box>{t(ChatSourceMap[item.source]?.name) || item.source}</Box>
                     <Box color={'myGray.500'}>{dayjs(item.time).format('YYYY/MM/DD HH:mm')}</Box>
                   </Td>
                   <Td>
                     <Box>
-                      {item.source === 'share' ? (
+                      {!!item.outLinkUid ? (
                         item.outLinkUid
                       ) : (
-                        <Tag key={item._id} type={'fill'} colorSchema="white">
+                        <HStack>
                           <Avatar
                             src={teamMembers.find((v) => v.tmbId === item.tmbId)?.avatar}
                             w="1.25rem"
@@ -145,7 +146,7 @@ const Logs = () => {
                           <Box fontSize={'sm'} ml={1}>
                             {teamMembers.find((v) => v.tmbId === item.tmbId)?.memberName}
                           </Box>
-                        </Tag>
+                        </HStack>
                       )}
                     </Box>
                   </Td>

projects/app/src/pages/chat/share.tsx

@@ -57,6 +57,7 @@ const OutLink = ({
     showHistory = '1',
     showHead = '1',
     authToken,
+    customUid,
     ...customVariables
   } = router.query as {
     shareId: string;

projects/app/src/service/support/permission/auth/outLink.ts

@@ -65,6 +65,7 @@ export async function authOutLinkChatStart({
   ]);
 
   return {
+    sourceName: shareChat.name,
     teamId: shareChat.teamId,
     tmbId: shareChat.tmbId,
     authType: AuthUserTypeEnum.token,

projects/app/src/types/app.d.ts

@@ -35,7 +35,7 @@ export type AppItemType = {
 export type AppLogsListItemType = {
   _id: string;
   id: string;
-  source: ChatSchema['source'];
+  source: string;
   time: Date;
   title: string;
   messageCount: number;