perf: ery extension and fix filter same embedding result (#3833)

* perf: ery extension and fix filter same embedding result

* fix: extract node too long

* perf: ui

* perf: not chatId will auto save

* fix: laf

* fix: member load

* feat: add completions unstream error response

* feat: add completions unstream error response

* updat emodel provider

projects/app/src/pages/api/admin/initv4822.ts

@@ -11,6 +11,9 @@ async function handler(req: NextApiRequest, _res: NextApiResponse) {
   await authCert({ req, authRoot: true });
   const users = await MongoUser.find();
   const teams = await MongoTeam.find();
+
+  console.log('Total users:', users.length);
+  let success = 0;
   for await (const user of users) {
     try {
       const team = teams.find((team) => String(team.ownerId) === String(user._id));
@@ -18,6 +21,7 @@ async function handler(req: NextApiRequest, _res: NextApiResponse) {
         user.contact = team.notificationAccount;
       }
       await user.save();
+      console.log('Success:', ++success);
     } catch (error) {
       console.error(error);
     }

projects/app/src/pages/api/core/dataset/collection/sync.ts

@@ -6,12 +6,12 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { syncCollection } from '@fastgpt/service/core/dataset/collection/utils';
 
 /* 
-    Collection sync
-    1. Check collection type: link, api dataset collection
-    2. Get collection and raw text
-    3. Check whether the original text is the same: skip if same
-    4. Create new collection
-    5. Delete old collection
+  Collection sync
+  1. Check collection type: link, api dataset collection
+  2. Get collection and raw text
+  3. Check whether the original text is the same: skip if same
+  4. Create new collection
+  5. Delete old collection
 */
 export type CollectionSyncBody = {
   collectionId: string;
@@ -27,6 +27,7 @@ async function handler(req: ApiRequestProps<CollectionSyncBody>) {
   const { collection } = await authDatasetCollection({
     req,
     authToken: true,
+    authApiKey: true,
     collectionId,
     per: WritePermissionVal
   });

projects/app/src/pages/api/support/user/account/tokenLogin.ts

@@ -16,13 +16,13 @@ async function handler(
   const user = await getUserDetail({ tmbId });
 
   // Remove sensitive information
-  if (user.team.lafAccount) {
-    user.team.lafAccount = {
-      appid: user.team.lafAccount.appid,
-      token: '',
-      pat: ''
-    };
-  }
+  // if (user.team.lafAccount) {
+  //   user.team.lafAccount = {
+  //     appid: user.team.lafAccount.appid,
+  //     token: '',
+  //     pat: ''
+  //   };
+  // }
   if (user.team.openaiAccount) {
     user.team.openaiAccount = {
       key: '',

projects/app/src/pages/api/v1/chat/completions.ts

@@ -302,65 +302,64 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
     })();
 
     // save chat
-    if (chatId) {
-      const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
-      const source = (() => {
-        if (shareId) {
-          return ChatSourceEnum.share;
-        }
-        if (authType === 'apikey') {
-          return ChatSourceEnum.api;
-        }
-        if (spaceTeamId) {
-          return ChatSourceEnum.team;
-        }
-        return ChatSourceEnum.online;
-      })();
-
-      const isInteractiveRequest = !!getLastInteractiveValue(histories);
-      const { text: userInteractiveVal } = chatValue2RuntimePrompt(userQuestion.value);
-
-      const newTitle = isPlugin
-        ? variables.cTime ?? getSystemTime(timezone)
-        : getChatTitleFromChatMessage(userQuestion);
-
-      const aiResponse: AIChatItemType & { dataId?: string } = {
-        dataId: responseChatItemId,
-        obj: ChatRoleEnum.AI,
-        value: assistantResponses,
-        [DispatchNodeResponseKeyEnum.nodeResponse]: flowResponses
-      };
-
-      if (isInteractiveRequest) {
-        await updateInteractiveChat({
-          chatId,
-          appId: app._id,
-          userInteractiveVal,
-          aiResponse,
-          newVariables
-        });
-      } else {
-        await saveChat({
-          chatId,
-          appId: app._id,
-          teamId,
-          tmbId: tmbId,
-          nodes,
-          appChatConfig: chatConfig,
-          variables: newVariables,
-          isUpdateUseTime: isOwnerUse && source === ChatSourceEnum.online, // owner update use time
-          newTitle,
-          shareId,
-          outLinkUid: outLinkUserId,
-          source: source,
-          sourceName: sourceName || '',
-          content: [userQuestion, aiResponse],
-          metadata: {
-            originIp,
-            ...metadata
-          }
-        });
+    const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
+    const source = (() => {
+      if (shareId) {
+        return ChatSourceEnum.share;
       }
+      if (authType === 'apikey') {
+        return ChatSourceEnum.api;
+      }
+      if (spaceTeamId) {
+        return ChatSourceEnum.team;
+      }
+      return ChatSourceEnum.online;
+    })();
+
+    const isInteractiveRequest = !!getLastInteractiveValue(histories);
+    const { text: userInteractiveVal } = chatValue2RuntimePrompt(userQuestion.value);
+
+    const newTitle = isPlugin
+      ? variables.cTime ?? getSystemTime(timezone)
+      : getChatTitleFromChatMessage(userQuestion);
+
+    const aiResponse: AIChatItemType & { dataId?: string } = {
+      dataId: responseChatItemId,
+      obj: ChatRoleEnum.AI,
+      value: assistantResponses,
+      [DispatchNodeResponseKeyEnum.nodeResponse]: flowResponses
+    };
+
+    const saveChatId = chatId || getNanoid(24);
+    if (isInteractiveRequest) {
+      await updateInteractiveChat({
+        chatId: saveChatId,
+        appId: app._id,
+        userInteractiveVal,
+        aiResponse,
+        newVariables
+      });
+    } else {
+      await saveChat({
+        chatId: saveChatId,
+        appId: app._id,
+        teamId,
+        tmbId: tmbId,
+        nodes,
+        appChatConfig: chatConfig,
+        variables: newVariables,
+        isUpdateUseTime: isOwnerUse && source === ChatSourceEnum.online, // owner update use time
+        newTitle,
+        shareId,
+        outLinkUid: outLinkUserId,
+        source,
+        sourceName: sourceName || '',
+        content: [userQuestion, aiResponse],
+        metadata: {
+          originIp,
+          ...metadata
+        }
+      });
     }
 
     addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
@@ -407,9 +406,11 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
 
         return assistantResponses;
       })();
+      const error = flowResponses[flowResponses.length - 1]?.error;
 
       res.json({
         ...(detail ? { responseData: feResponseData, newVariables } : {}),
+        error,
         id: chatId || '',
         model: '',
         usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 },