perf: push chat log (#3109)

2024-11-10 11:22:03 +08:00
parent e9f0d5dad5
commit 0201e63cfd
12 changed files with 134 additions and 81 deletions
--- a/docSite/content/zh-cn/docs/development/upgrading/4813.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/4813.md
@@ -30,12 +30,13 @@ weight: 811
 5. 新增 - 循环节点增加下标值。 
 6. 新增 - 部分对话错误提醒增加翻译。 
 7. 新增 - 对话输入框支持拖拽文件上传，可直接拖文件到输入框中。  
-8. 优化 - 合并多个 system 提示词成 1 个，避免部分模型不支持多个 system 提示词。  
-9. 优化 - 知识库上传文件，优化报错提示。  
-10. 优化 - 全文检索语句，减少一轮子查询。  
-11. 优化 - 修改 findLast 为 [...array].reverse().find，适配旧版浏览器。  
-12. 优化 - Markdown 组件自动空格，避免分割 url 中的中文。  
-13. 优化 - 工作流上下文拆分，性能优化。 
-14. 优化 - 语音播报，不支持 mediaSource 的浏览器可等待完全生成语音后输出。 
-15. 修复 - Dockerfile pnpm install 支持代理。 
-16. 修复 - BI 图表生成无法写入文件。 
+8. 新增 - 对话日志，来源可显示分享链接/API具体名称
+9. 优化 - 合并多个 system 提示词成 1 个，避免部分模型不支持多个 system 提示词。  
+10. 优化 - 知识库上传文件，优化报错提示。  
+11. 优化 - 全文检索语句，减少一轮子查询。  
+12. 优化 - 修改 findLast 为 [...array].reverse().find，适配旧版浏览器。  
+13. 优化 - Markdown 组件自动空格，避免分割 url 中的中文。  
+14. 优化 - 工作流上下文拆分，性能优化。 
+15. 优化 - 语音播报，不支持 mediaSource 的浏览器可等待完全生成语音后输出。 
+16. 修复 - Dockerfile pnpm install 支持代理。 
+17. 修复 - BI 图表生成无法写入文件。 
--- a/packages/service/core/chat/chatSchema.ts
+++ b/packages/service/core/chat/chatSchema.ts
@@ -52,7 +52,6 @@ const ChatSchema = new Schema({
  },
  source: {
    type: String,
-    enum: Object.keys(ChatSourceMap),
    required: true
  },
  shareId: {
--- a/packages/service/core/chat/pushChatLog.ts
+++ b/packages/service/core/chat/pushChatLog.ts
@@ -2,7 +2,8 @@ import { addLog } from '../../common/system/log';
 import { MongoChatItem } from './chatItemSchema';
 import { MongoChat } from './chatSchema';
 import axios from 'axios';
-import { AIChatItemType, ChatItemType } from '@fastgpt/global/core/chat/type';
+import { AIChatItemType, ChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type';
+import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';

 export type Metadata = {
  [key: string]: {
@@ -26,8 +27,8 @@ export const pushChatLog = ({
 }) => {
  const interval = Number(process.env.CHAT_LOG_INTERVAL);
  const url = process.env.CHAT_LOG_URL;
-  if (interval > 0 && url) {
-    addLog.info(`[ChatLogPush] push chat log after ${interval}ms`, {
+  if (!isNaN(interval) && interval > 0 && url) {
+    addLog.debug(`[ChatLogPush] push chat log after ${interval}ms`, {
      appId,
      chatItemIdHuman,
      chatItemIdAi
@@ -82,7 +83,7 @@ const pushChatLogInternal = async ({
 }) => {
  try {
    const [chatItemHuman, chatItemAi] = await Promise.all([
-      MongoChatItem.findById(chatItemIdHuman).lean(),
+      MongoChatItem.findById(chatItemIdHuman).lean() as Promise<UserChatItemType>,
      MongoChatItem.findById(chatItemIdAi).lean() as Promise<AIChatItemType>
    ]);

@@ -103,8 +104,52 @@ const pushChatLogInternal = async ({

    const uid = chat.outLinkUid || chat.tmbId;
    // Pop last two items
-    const question = chatItemHuman.value[chatItemHuman.value.length - 1]?.text?.content;
-    const answer = chatItemAi.value[chatItemAi.value.length - 1]?.text?.content;
+    const question = chatItemHuman.value
+      .map((item) => {
+        if (item.type === ChatItemValueTypeEnum.text) {
+          return item.text?.content;
+        } else if (item.type === ChatItemValueTypeEnum.file) {
+          if (item.file?.type === 'image') {
+            return `![${item.file?.name}](${item.file?.url})`;
+          }
+          return `[${item.file?.name}](${item.file?.url})`;
+        }
+        return '';
+      })
+      .join('\n');
+    const answer = chatItemAi.value
+      .map((item) => {
+        const text = [];
+        if (item.text?.content) {
+          text.push(item.text?.content);
+        }
+        if (item.tools) {
+          text.push(
+            item.tools.map(
+              (tool) =>
+                `\`\`\`json
+${JSON.stringify(
+  {
+    name: tool.toolName,
+    params: tool.params,
+    response: tool.response
+  },
+  null,
+  2
+)}
+\`\`\``
+            )
+          );
+        }
+        if (item.interactive) {
+          text.push(`\`\`\`json
+${JSON.stringify(item.interactive, null, 2)}
+            \`\`\``);
+        }
+        return text.join('\n');
+      })
+      .join('\n');
+
    if (!question || !answer) {
      addLog.error('[ChatLogPush] question or answer is empty', {
        question: chatItemHuman.value,
@@ -112,11 +157,13 @@ const pushChatLogInternal = async ({
      });
      return;
    }
+
+    // computed response time
    const responseData = chatItemAi.responseData;
    const responseTime =
      responseData?.reduce((acc, item) => acc + (item?.runningTime ?? 0), 0) || 0;

-    const sourceIdPrefix = process.env.SOURCE_ID_PREFIX ?? '';
+    const sourceIdPrefix = process.env.CHAT_LOG_SOURCE_ID_PREFIX ?? 'fastgpt-';

    const chatLog: ChatLog = {
      title: chat.title,
@@ -141,14 +188,7 @@ const pushChatLogInternal = async ({
      createdAt: new Date(chatItemAi.time).getTime(),
      sourceId: `${sourceIdPrefix}${appId}`
    };
-    await axios
-      .post(`${url}/api/chat/push`, chatLog)
-      .then((res) => {
-        addLog.info('[ChatLogPush] push success', res.data);
-      })
-      .catch((e) => {
-        addLog.error('[ChatLogPush] push failed', { e, resData: e.response?.data });
-      });
+    await axios.post(`${url}/api/chat/push`, chatLog);
  } catch (e) {
    addLog.error('[ChatLogPush] error', e);
  }
--- a/packages/service/core/chat/saveChat.ts
+++ b/packages/service/core/chat/saveChat.ts
@@ -30,7 +30,7 @@ type Props = {
  variables?: Record<string, any>;
  isUpdateUseTime: boolean;
  newTitle: string;
-  source: `${ChatSourceEnum}`;
+  source: string;
  shareId?: string;
  outLinkUid?: string;
  content: [UserChatItemType & { dataId?: string }, AIChatItemType & { dataId?: string }];
--- a/packages/service/support/openapi/auth.ts
+++ b/packages/service/support/openapi/auth.ts
@@ -11,7 +11,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
    return Promise.reject(ERROR_ENUM.unAuthApiKey);
  }
  try {
-    const openApi = await MongoOpenApi.findOne({ apiKey: apikey.trim() });
+    const openApi = await MongoOpenApi.findOne({ apiKey: apikey.trim() }).lean();
    if (!openApi) {
      return Promise.reject(ERROR_ENUM.unAuthApiKey);
    }
@@ -20,7 +20,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
    // @ts-ignore
    if (global.feConfigs?.isPlus) {
      await POST('/support/openapi/authLimit', {
-        openApi: openApi.toObject()
+        openApi
      } as AuthOpenApiLimitProps);
    }

@@ -30,7 +30,8 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
      apikey,
      teamId: String(openApi.teamId),
      tmbId: String(openApi.tmbId),
-      appId: openApi.appId || ''
+      appId: openApi.appId || '',
+      sourceName: openApi.name
    };
  } catch (error) {
    return Promise.reject(error);
--- a/packages/service/support/permission/controller.ts
+++ b/packages/service/support/permission/controller.ts
@@ -313,14 +313,15 @@ export async function parseHeaderCert({
    })();

    // auth apikey
-    const { teamId, tmbId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
+    const { teamId, tmbId, appId: apiKeyAppId = '', sourceName } = await authOpenApiKey({ apikey });

    return {
      uid: '',
      teamId,
      tmbId,
      apikey,
-      appId: apiKeyAppId || authorizationAppid
+      appId: apiKeyAppId || authorizationAppid,
+      sourceName
    };
  }
  // root user
@@ -332,48 +333,50 @@ export async function parseHeaderCert({

  const { cookie, token, rootkey, authorization } = (req.headers || {}) as ReqHeaderAuthType;

-  const { uid, teamId, tmbId, appId, openApiKey, authType, isRoot } = await (async () => {
-    if (authApiKey && authorization) {
-      // apikey from authorization
-      const authResponse = await parseAuthorization(authorization);
-      return {
-        uid: authResponse.uid,
-        teamId: authResponse.teamId,
-        tmbId: authResponse.tmbId,
-        appId: authResponse.appId,
-        openApiKey: authResponse.apikey,
-        authType: AuthUserTypeEnum.apikey
-      };
-    }
-    if (authToken && (token || cookie)) {
-      // user token(from fastgpt web)
-      const res = await authCookieToken(cookie, token);
-      return {
-        uid: res.userId,
-        teamId: res.teamId,
-        tmbId: res.tmbId,
-        appId: '',
-        openApiKey: '',
-        authType: AuthUserTypeEnum.token,
-        isRoot: res.isRoot
-      };
-    }
-    if (authRoot && rootkey) {
-      await parseRootKey(rootkey);
-      // root user
-      return {
-        uid: '',
-        teamId: '',
-        tmbId: '',
-        appId: '',
-        openApiKey: '',
-        authType: AuthUserTypeEnum.root,
-        isRoot: true
-      };
-    }
+  const { uid, teamId, tmbId, appId, openApiKey, authType, isRoot, sourceName } =
+    await (async () => {
+      if (authApiKey && authorization) {
+        // apikey from authorization
+        const authResponse = await parseAuthorization(authorization);
+        return {
+          uid: authResponse.uid,
+          teamId: authResponse.teamId,
+          tmbId: authResponse.tmbId,
+          appId: authResponse.appId,
+          openApiKey: authResponse.apikey,
+          authType: AuthUserTypeEnum.apikey,
+          sourceName: authResponse.sourceName
+        };
+      }
+      if (authToken && (token || cookie)) {
+        // user token(from fastgpt web)
+        const res = await authCookieToken(cookie, token);
+        return {
+          uid: res.userId,
+          teamId: res.teamId,
+          tmbId: res.tmbId,
+          appId: '',
+          openApiKey: '',
+          authType: AuthUserTypeEnum.token,
+          isRoot: res.isRoot
+        };
+      }
+      if (authRoot && rootkey) {
+        await parseRootKey(rootkey);
+        // root user
+        return {
+          uid: '',
+          teamId: '',
+          tmbId: '',
+          appId: '',
+          openApiKey: '',
+          authType: AuthUserTypeEnum.root,
+          isRoot: true
+        };
+      }

-    return Promise.reject(ERROR_ENUM.unAuthorization);
-  })();
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    })();

  if (!authRoot && (!teamId || !tmbId)) {
    return Promise.reject(ERROR_ENUM.unAuthorization);
@@ -385,6 +388,7 @@ export async function parseHeaderCert({
    tmbId: String(tmbId),
    appId,
    authType,
+    sourceName,
    apikey: openApiKey,
    isRoot: !!isRoot
  };
--- a/projects/app/.env.template
+++ b/projects/app/.env.template
@@ -53,5 +53,5 @@ WORKFLOW_MAX_LOOP_TIMES=50
 # CHAT_LOG_URL=http://localhost:8080
 # # 日志推送间隔
 # CHAT_LOG_INTERVAL=10000
-# # 日志来源前缀
-# SOURCE_ID_PREFIX=fastgpt-
+# # 日志来源ID前缀
+# CHAT_LOG_SOURCE_ID_PREFIX=fastgpt-
--- a/projects/app/src/pages/api/v1/chat/completions.ts
+++ b/projects/app/src/pages/api/v1/chat/completions.ts
@@ -87,6 +87,7 @@ type AuthResponseType = {
  apikey?: string;
  canWrite: boolean;
  outLinkUserId?: string;
+  sourceName?: string;
 };

 async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -156,6 +157,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
      app,
      responseDetail,
      authType,
+      sourceName,
      apikey,
      canWrite,
      outLinkUserId = customUid
@@ -343,7 +345,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
          newTitle,
          shareId,
          outLinkUid: outLinkUserId,
-          source,
+          source: sourceName || source,
          content: [userQuestion, aiResponse],
          metadata: {
            originIp,
@@ -459,7 +461,7 @@ const authShareChat = async ({
  shareId: string;
  chatId?: string;
 }): Promise<AuthResponseType> => {
-  const { teamId, tmbId, user, appId, authType, responseDetail, uid } =
+  const { teamId, tmbId, user, appId, authType, responseDetail, uid, sourceName } =
    await authOutLinkChatStart(data);
  const app = await MongoApp.findById(appId).lean();

@@ -474,6 +476,7 @@ const authShareChat = async ({
  }

  return {
+    sourceName,
    teamId,
    tmbId,
    user,
@@ -541,6 +544,7 @@ const authHeaderRequest = async ({
    teamId,
    tmbId,
    authType,
+    sourceName,
    apikey
  } = await authCert({
    req,
@@ -607,6 +611,7 @@ const authHeaderRequest = async ({
    responseDetail: true,
    apikey,
    authType,
+    sourceName,
    canWrite: true
  };
 };
--- a/projects/app/src/pages/app/detail/components/Logs/index.tsx
+++ b/projects/app/src/pages/app/detail/components/Logs/index.tsx
@@ -129,15 +129,16 @@ const Logs = () => {
                  onClick={() => setDetailLogsId(item.id)}
                >
                  <Td>
-                    <Box>{t(ChatSourceMap[item.source]?.name || ('UnKnow' as any))}</Box>
+                    {/* @ts-ignore */}
+                    <Box>{t(ChatSourceMap[item.source]?.name) || item.source}</Box>
                    <Box color={'myGray.500'}>{dayjs(item.time).format('YYYY/MM/DD HH:mm')}</Box>
                  </Td>
                  <Td>
                    <Box>
-                      {item.source === 'share' ? (
+                      {!!item.outLinkUid ? (
                        item.outLinkUid
                      ) : (
-                        <Tag key={item._id} type={'fill'} colorSchema="white">
+                        <HStack>
                          <Avatar
                            src={teamMembers.find((v) => v.tmbId === item.tmbId)?.avatar}
                            w="1.25rem"
@@ -145,7 +146,7 @@ const Logs = () => {
                          <Box fontSize={'sm'} ml={1}>
                            {teamMembers.find((v) => v.tmbId === item.tmbId)?.memberName}
                          </Box>
-                        </Tag>
+                        </HStack>
                      )}
                    </Box>
                  </Td>
--- a/projects/app/src/pages/chat/share.tsx
+++ b/projects/app/src/pages/chat/share.tsx
@@ -57,6 +57,7 @@ const OutLink = ({
    showHistory = '1',
    showHead = '1',
    authToken,
+    customUid,
    ...customVariables
  } = router.query as {
    shareId: string;
--- a/projects/app/src/service/support/permission/auth/outLink.ts
+++ b/projects/app/src/service/support/permission/auth/outLink.ts
@@ -65,6 +65,7 @@ export async function authOutLinkChatStart({
  ]);

  return {
+    sourceName: shareChat.name,
    teamId: shareChat.teamId,
    tmbId: shareChat.tmbId,
    authType: AuthUserTypeEnum.token,
--- a/projects/app/src/types/app.d.ts
+++ b/projects/app/src/types/app.d.ts
@@ -35,7 +35,7 @@ export type AppItemType = {
 export type AppLogsListItemType = {
  _id: string;
  id: string;
-  source: ChatSchema['source'];
+  source: string;
  time: Date;
  title: string;
  messageCount: number;