v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/pages/api/admin/initv469.ts

@@ -4,6 +4,11 @@ import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoUsage } from '@fastgpt/service/support/wallet/usage/schema';
 import { connectionMongo } from '@fastgpt/service/common/mongo';
+import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
+import { addHours } from 'date-fns';
+import { checkInvalid as checkInvalidImg } from '../timerTask/dataset/checkInvalidDatasetImage';
+import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
+import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
 
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -21,6 +26,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
     }
 
+    (async () => {
+      try {
+        console.log('执行脏数据清理任务');
+        const end = addHours(new Date(), -1);
+        const start = addHours(new Date(), -360 * 24);
+        await checkFiles(start, end);
+        await checkInvalidImg(start, end);
+        await checkInvalidCollection(start, end);
+        await checkInvalidVector(start, end);
+        console.log('执行脏数据清理任务完毕');
+      } catch (error) {
+        console.log('执行脏数据清理任务出错了');
+      }
+    })();
+
     jsonRes(res, {
       message: 'success'
     });

projects/app/src/pages/api/common/file/uploadImage.ts

@@ -1,16 +1,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
+import { authChatCert } from '@/service/support/permission/auth/chat';
 import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { shareId, ...body } = req.body as UploadImgProps;
+    const body = req.body as UploadImgProps;
 
-    const { teamId } = await authCertOrShareId({ req, shareId, authToken: true });
+    const { teamId } = await authChatCert({ req, authToken: true });
 
     const data = await uploadMongoImg({
       teamId,

projects/app/src/pages/api/common/system/getInitData.ts

@@ -4,8 +4,6 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { readFileSync, readdirSync } from 'fs';
 import type { InitDateResponse } from '@/global/common/api/systemRes';
 import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
-import { getTikTokenEnc } from '@fastgpt/global/common/string/tiktoken';
-import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
 import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
 import { getFastGPTConfigFromDB } from '@fastgpt/service/common/system/config/controller';
 import { connectToDatabase } from '@/service/mongo';
@@ -63,7 +61,6 @@ export async function getInitConfig() {
     await connectToDatabase();
 
     await Promise.all([
-      initGlobal(),
       initSystemConfig(),
       // getSimpleModeTemplates(),
       getSystemVersion(),
@@ -84,18 +81,6 @@ export async function getInitConfig() {
   }
 }
 
-export function initGlobal() {
-  if (global.communityPlugins) return;
-
-  global.communityPlugins = [];
-  global.simpleModeTemplates = [];
-  global.qaQueueLen = global.qaQueueLen ?? 0;
-  global.vectorQueueLen = global.vectorQueueLen ?? 0;
-  // init tikToken
-  getTikTokenEnc();
-  initHttpAgent();
-}
-
 export async function initSystemConfig() {
   // load config
   const [dbConfig, fileConfig] = await Promise.all([
@@ -125,7 +110,6 @@ export async function initSystemConfig() {
 
   // set config
   initFastGPTConfig(config);
-  global.systemEnv = config.systemEnv;
 
   console.log({
     feConfigs: global.feConfigs,

projects/app/src/pages/api/common/system/unlockTask.ts

@@ -1,14 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { startQueue } from '@/service/utils/tools';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
     await authCert({ req, authToken: true });
-    startQueue();
+    startTrainingQueue();
   } catch (error) {}
   jsonRes(res);
 }

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -4,22 +4,21 @@ import { connectToDatabase } from '@/service/mongo';
 import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
 import { pushQuestionGuideUsage } from '@/service/support/wallet/usage/push';
 import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
-import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
+import { authChatCert } from '@/service/support/permission/auth/chat';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { messages, shareId } = req.body as CreateQuestionGuideParams;
+    const { messages } = req.body as CreateQuestionGuideParams;
 
-    const { tmbId, teamId } = await authCertOrShareId({
+    const { tmbId, teamId } = await authChatCert({
       req,
-      authToken: true,
-      shareId
+      authToken: true
     });
 
     const qgModel = global.llmModels[0];
 
-    const { result, charsLength } = await createQuestionGuide({
+    const { result, tokens } = await createQuestionGuide({
       messages,
       model: qgModel.model
     });
@@ -29,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
 
     pushQuestionGuideUsage({
-      charsLength,
+      tokens,
       teamId,
       tmbId
     });

projects/app/src/pages/api/core/chat/clearHistories.ts

@@ -7,12 +7,13 @@ import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { ClearHistoriesProps } from '@/global/core/chat/api';
 import { authOutLink } from '@/service/support/permission/auth/outLink';
 import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
+import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
 
 /* clear chat history */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { appId, shareId, outLinkUid } = req.query as ClearHistoriesProps;
+    const { appId, shareId, outLinkUid, teamId, teamToken } = req.query as ClearHistoriesProps;
 
     let chatAppId = appId;
 
@@ -26,6 +27,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
           outLinkUid: uid
         };
       }
+      if (teamId && teamToken) {
+        const { uid } = await authTeamSpaceToken({ teamId, teamToken });
+        return {
+          teamId,
+          appId,
+          outLinkUid: uid
+        };
+      }
       if (appId) {
         const { tmbId } = await authCert({ req, authToken: true });
 

projects/app/src/pages/api/core/chat/getHistories.ts

@@ -4,14 +4,15 @@ import { connectToDatabase } from '@/service/mongo';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import type { ChatHistoryItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
-import { getHistoriesProps } from '@/global/core/chat/api';
+import { GetHistoriesProps } from '@/global/core/chat/api';
 import { authOutLink } from '@/service/support/permission/auth/outLink';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { appId, shareId, outLinkUid } = req.body as getHistoriesProps;
+    const { appId, shareId, outLinkUid, teamId, teamToken } = req.body as GetHistoriesProps;
 
     const limit = shareId && outLinkUid ? 20 : 30;
 
@@ -28,10 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
           }
         };
       }
-      if (appId && outLinkUid) {
+      if (appId && teamId && teamToken) {
+        const { uid } = await authTeamSpaceToken({ teamId, teamToken });
         return {
-          shareId,
-          outLinkUid: outLinkUid,
+          teamId,
+          appId,
+          outLinkUid: uid,
           source: ChatSourceEnum.team
         };
       }

projects/app/src/pages/api/core/chat/item/getSpeech.ts

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechUsage } from '@/service/support/wallet/usage/push';
-import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
+import { authChatCert } from '@/service/support/permission/auth/chat';
 import { authType2UsageSource } from '@/service/support/wallet/usage/utils';
 import { getAudioSpeechModel } from '@fastgpt/service/core/ai/model';
 import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
@@ -19,13 +19,13 @@ import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { ttsConfig, input, shareId } = req.body as GetChatSpeechProps;
+    const { ttsConfig, input } = req.body as GetChatSpeechProps;
 
     if (!ttsConfig.model || !ttsConfig.voice) {
       throw new Error('model or voice not found');
     }
 
-    const { teamId, tmbId, authType } = await authCertOrShareId({ req, authToken: true, shareId });
+    const { teamId, tmbId, authType } = await authChatCert({ req, authToken: true });
 
     const ttsModel = getAudioSpeechModel(ttsConfig.model);
     const voiceData = ttsModel.voices?.find((item) => item.value === ttsConfig.voice);

projects/app/src/pages/api/core/chat/outLink/getInforByTeamId.ts

@@ -1,37 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import type { chatByTeamProps } from '@/global/core/chat/api.d';
-import axios from 'axios';
-import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
-import { getChatItems } from '@fastgpt/service/core/chat/controller';
-import { selectShareResponse } from '@/utils/service/core/chat';
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await connectToDatabase();
-
-    let { teamId, appId, outLinkUid } = req.query as chatByTeamProps;
-
-    const history = await MongoChatItem.find({
-      appId: appId,
-      outLinkUid: outLinkUid,
-      teamId: teamId
-    });
-
-    jsonRes(res, {
-      data: history
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      data: req.query,
-      error: err
-    });
-  }
-}
-
-export const config = {
-  api: {
-    responseLimit: '10mb'
-  }
-};

projects/app/src/pages/api/core/chat/outLink/init.ts

@@ -9,7 +9,7 @@ import { getChatItems } from '@fastgpt/service/core/chat/controller';
 import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
 import { authOutLink } from '@/service/support/permission/auth/outLink';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
-import { selectShareResponse } from '@/utils/service/core/chat';
+import { selectSimpleChatResponse } from '@/utils/service/core/chat';
 import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
@@ -50,7 +50,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     // pick share response field
     history.forEach((item) => {
-      item.responseData = selectShareResponse({ responseData: item.responseData });
+      item.responseData = selectSimpleChatResponse({ responseData: item.responseData });
     });
 
     jsonRes<InitChatResponse>(res, {

projects/app/src/pages/api/core/chat/team/init.ts

@@ -4,59 +4,57 @@ import { connectToDatabase } from '@/service/mongo';
 import { getGuideModule } from '@fastgpt/global/core/module/utils';
 import { getChatModelNameListByModules } from '@/service/core/app/module';
 import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
-import type { InitChatProps, InitChatResponse } from '@/global/core/chat/api.d';
+import type { InitChatResponse, InitTeamChatProps } from '@/global/core/chat/api.d';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 import { getChatItems } from '@fastgpt/service/core/chat/controller';
 import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
+import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
+import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
+import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
+import { selectSimpleChatResponse } from '@/utils/service/core/chat';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
 
-    let { appId, chatId, outLinkUid } = req.query as {
-      chatId?: string;
-      appId?: string;
-      outLinkUid?: string;
-    };
+    let { teamId, appId, chatId, teamToken } = req.query as InitTeamChatProps;
 
-    if (!appId) {
-      return jsonRes(res, {
-        code: 501,
-        message: "You don't have an app yet"
-      });
+    if (!teamId || !appId || !teamToken) {
+      throw new Error('teamId, appId, teamToken are required');
     }
 
-    // auth app permission
-    const [chat, app] = await Promise.all([
-      //   authApp({
-      //     req,
-      //     authToken: false,
-      //     appId,
-      //     per: 'r'
-      //   }),
-      chatId ? MongoChat.findOne({ appId, chatId }) : undefined,
+    const { uid } = await authTeamSpaceToken({
+      teamId,
+      teamToken
+    });
+
+    const [team, chat, app] = await Promise.all([
+      MongoTeam.findById(teamId, 'name avatar').lean(),
+      MongoChat.findOne({ teamId, appId, chatId }).lean(),
       MongoApp.findById(appId).lean()
     ]);
+
     if (!app) {
       throw new Error(AppErrEnum.unExist);
     }
 
     // auth chat permission
-    // if (chat && chat.outLinkUid !== outLinkUid) {
-    //   throw new Error(ChatErrEnum.unAuthChat);
-    // }
-    // // auth chat permission
-    // if (chat && !app.canWrite && String(tmbId) !== String(chat?.tmbId)) {
-    //   throw new Error(ChatErrEnum.unAuthChat);
-    // }
+    if (chat && chat.outLinkUid !== uid) {
+      throw new Error(ChatErrEnum.unAuthChat);
+    }
 
     // get app and history
     const { history } = await getChatItems({
       appId,
       chatId,
       limit: 30,
-      field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${ModuleOutputKeyEnum.responseData}`
+      field: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${ModuleOutputKeyEnum.responseData}`
+    });
+
+    // pick share response field
+    history.forEach((item) => {
+      item.responseData = selectSimpleChatResponse({ responseData: item.responseData });
     });
 
     jsonRes<InitChatResponse>(res, {
@@ -64,7 +62,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         chatId,
         appId,
         title: chat?.title || '新对话',
-        userAvatar: undefined,
+        userAvatar: team?.avatar,
         variables: chat?.variables || {},
         history,
         app: {

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -8,9 +8,9 @@ import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
 import { PagingData } from '@/types';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
-import { startQueue } from '@/service/utils/tools';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
+import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -158,7 +158,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     );
 
     if (data.find((item) => item.trainingAmount > 0)) {
-      startQueue();
+      startTrainingQueue();
     }
 
     // count collections

projects/app/src/pages/api/core/dataset/data/insertData.ts

@@ -75,7 +75,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       a: formatA
     });
 
-    const { insertId, charsLength } = await insertData2Dataset({
+    const { insertId, tokens } = await insertData2Dataset({
       teamId,
       tmbId,
       datasetId,
@@ -90,7 +90,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     pushGenerateVectorUsage({
       teamId,
       tmbId,
-      charsLength,
+      tokens,
       model: vectorModelData.model
     });
 

projects/app/src/pages/api/core/dataset/data/update.ts

@@ -34,7 +34,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       insertLen: 1
     });
 
-    const { charsLength } = await updateData2Dataset({
+    const { tokens } = await updateData2Dataset({
       dataId: id,
       q,
       a,
@@ -45,7 +45,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     pushGenerateVectorUsage({
       teamId,
       tmbId,
-      charsLength,
+      tokens,
       model: vectorModel
     });
 

projects/app/src/pages/api/core/dataset/searchTest.ts

@@ -58,7 +58,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       extensionBg: datasetSearchExtensionBg
     });
 
-    const { searchRes, charsLength, ...result } = await searchDatasetData({
+    const { searchRes, tokens, ...result } = await searchDatasetData({
       teamId,
       reRankQuery: rewriteQuery,
       queries: concatQueries,
@@ -74,14 +74,14 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     const { totalPoints } = pushGenerateVectorUsage({
       teamId,
       tmbId,
-      charsLength,
+      tokens,
       model: dataset.vectorModel,
       source: apikey ? UsageSourceEnum.api : UsageSourceEnum.fastgpt,
 
       ...(aiExtensionResult &&
         extensionModel && {
           extensionModel: extensionModel.name,
-          extensionCharsLength: aiExtensionResult.charsLength
+          extensionTokens: aiExtensionResult.tokens
         })
     });
     if (apikey) {

projects/app/src/pages/api/timerTask/dataset/checkInValidDatasetFiles.ts

@@ -0,0 +1,91 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import {
+  delFileByFileIdList,
+  getGFSCollection
+} from '@fastgpt/service/common/file/gridfs/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { addHours } from 'date-fns';
+
+/* 
+  check dataset.files data. If there is no match in dataset.collections, delete it
+  可能异常情况
+  1. 上传了文件，未成功创建集合
+*/
+let deleteFileAmount = 0;
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { startHour = 24, endHour = 1 } = req.body as {
+      startHour?: number;
+      endHour?: number;
+      limit?: number;
+    };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    // start: now - maxDay, end: now - 3 day
+    const start = addHours(new Date(), -startHour);
+    const end = addHours(new Date(), -endHour);
+    deleteFileAmount = 0;
+    console.log(start, end);
+
+    await checkFiles(start, end);
+
+    jsonRes(res, {
+      data: deleteFileAmount,
+      message: 'success'
+    });
+  } catch (error) {
+    addLog.error(`check valid dataset files error`, error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export async function checkFiles(start: Date, end: Date) {
+  const collection = getGFSCollection('dataset');
+  const where = {
+    uploadDate: { $gte: start, $lte: end }
+  };
+
+  // 1. get all file _id
+  const files = await collection
+    .find(where, {
+      projection: {
+        metadata: 1,
+        _id: 1
+      }
+    })
+    .toArray();
+  console.log('total files', files.length);
+
+  let index = 0;
+  for await (const file of files) {
+    try {
+      // 2. find fileId in dataset.collections
+      const hasCollection = await MongoDatasetCollection.countDocuments({
+        teamId: file.metadata.teamId,
+        fileId: file._id
+      });
+
+      // 3. if not found, delete file
+      if (hasCollection === 0) {
+        await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(file._id)] });
+        console.log('delete file', file._id);
+        deleteFileAmount++;
+      }
+      index++;
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+  console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
+}

projects/app/src/pages/api/timerTask/dataset/checkInvalidDatasetImage.ts

@@ -0,0 +1,88 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { addHours } from 'date-fns';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoImage } from '@fastgpt/service/common/file/image/schema';
+
+/* 
+  检测无效的数据集图片
+
+  可能异常情况：
+  1. 上传文件过程中，上传了图片，但是最终没有创建数据集。
+*/
+
+let deleteImageAmount = 0;
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const {
+      startHour = 72,
+      endHour = 24,
+      limit = 10
+    } = req.body as { startHour?: number; endHour?: number; limit?: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    // start: now - maxDay, end: now - 3 day
+    const start = addHours(new Date(), -startHour);
+    const end = addHours(new Date(), -endHour);
+    deleteImageAmount = 0;
+
+    await checkInvalid(start, end, limit);
+
+    jsonRes(res, {
+      data: deleteImageAmount
+    });
+  } catch (error) {
+    addLog.error(`check Invalid user error`, error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export async function checkInvalid(start: Date, end: Date, limit = 50) {
+  const images = await MongoImage.find(
+    {
+      createTime: {
+        $gte: start,
+        $lte: end
+      },
+      'metadata.relatedId': { $exists: true }
+    },
+    '_id teamId metadata'
+  );
+  console.log('total images', images.length);
+  let index = 0;
+
+  for await (const image of images) {
+    try {
+      // 1. 检测是否有对应的集合
+      const collection = await MongoDatasetCollection.findOne(
+        {
+          teamId: image.teamId,
+          'metadata.relatedImgId': image.metadata?.relatedId
+        },
+        '_id'
+      );
+
+      if (!collection) {
+        await image.deleteOne();
+        deleteImageAmount++;
+      }
+
+      index++;
+
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  console.log(`检测完成，共删除 ${deleteImageAmount} 个无效图片`);
+}

projects/app/src/pages/api/timerTask/dataset/checkInvalidMongoCollection.ts

@@ -0,0 +1,96 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { addHours } from 'date-fns';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+
+/* 
+  检测无效的 Mongo 数据
+  异常情况：
+  1. 训练过程删除知识库，可能导致还会有新的数据插入，导致无效。
+*/
+
+let deleteAmount = 0;
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { startHour = 3, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    // start: now - maxDay, end: now - endHour
+    const start = addHours(new Date(), -startHour);
+    const end = addHours(new Date(), -endHour);
+    deleteAmount = 0;
+
+    await checkInvalidCollection(start, end);
+
+    jsonRes(res, {
+      data: deleteAmount,
+      message: 'success'
+    });
+  } catch (error) {
+    addLog.error(`check Invalid user error`, error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export async function checkInvalidCollection(start: Date, end: Date) {
+  // 1. 获取时间范围的所有data
+  const rows = await MongoDatasetData.find(
+    {
+      updateTime: {
+        $gte: start,
+        $lte: end
+      }
+    },
+    '_id teamId collectionId'
+  ).lean();
+
+  // 2. 合并所有的collectionId
+  const map = new Map<string, { teamId: string; collectionId: string }>();
+  for (const item of rows) {
+    const collectionId = String(item.collectionId);
+    if (!map.has(collectionId)) {
+      map.set(collectionId, { teamId: item.teamId, collectionId });
+    }
+  }
+  const list = Array.from(map.values());
+  console.log('total collections', list.length);
+  let index = 0;
+
+  for await (const item of list) {
+    try {
+      // 3. 查看该collection是否存在，不存在，则删除对应的数据
+      const collection = await MongoDatasetCollection.findOne({ _id: item.collectionId });
+      if (!collection) {
+        const result = await Promise.all([
+          MongoDatasetTraining.deleteMany({
+            teamId: item.teamId,
+            collectionId: item.collectionId
+          }),
+          MongoDatasetData.deleteMany({
+            teamId: item.teamId,
+            collectionId: item.collectionId
+          }),
+          deleteDatasetDataVector({
+            teamId: item.teamId,
+            collectionIds: [String(item.collectionId)]
+          })
+        ]);
+        console.log(result);
+        console.log('collection is not found', item);
+        continue;
+      }
+    } catch (error) {}
+    console.log(++index);
+  }
+}

projects/app/src/pages/api/timerTask/dataset/checkInvalidVector.ts

@@ -0,0 +1,86 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addLog } from '@fastgpt/service/common/system/log';
+import {
+  deleteDatasetDataVector,
+  getVectorDataByTime
+} from '@fastgpt/service/common/vectorStore/controller';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { addHours } from 'date-fns';
+
+/* 
+  检测无效的 Vector 数据. 
+  异常情况：
+  1. 插入数据时，vector成功，mongo失败
+  2. 更新数据，也会有插入 vector
+*/
+
+let deletedVectorAmount = 0;
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { startHour = 5, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    // start: now - maxDay, end: now - endHour
+    const start = addHours(new Date(), -startHour);
+    const end = addHours(new Date(), -endHour);
+    deletedVectorAmount = 0;
+
+    await checkInvalidVector(start, end);
+
+    jsonRes(res, {
+      data: deletedVectorAmount,
+      message: 'success'
+    });
+  } catch (error) {
+    addLog.error(`check Invalid user error`, error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export async function checkInvalidVector(start: Date, end: Date) {
+  // 1. get all vector data
+  const rows = await getVectorDataByTime(start, end);
+  console.log('total data', rows.length);
+
+  let index = 0;
+
+  for await (const item of rows) {
+    if (!item.teamId || !item.datasetId || !item.id) {
+      console.log('error data', item);
+      continue;
+    }
+    try {
+      // 2. find dataset.data
+      const hasData = await MongoDatasetData.countDocuments({
+        teamId: item.teamId,
+        datasetId: item.datasetId,
+        'indexes.dataId': item.id
+      });
+
+      // 3. if not found, delete vector
+      if (hasData === 0) {
+        await deleteDatasetDataVector({
+          teamId: item.teamId,
+          id: item.id
+        });
+        console.log('delete vector data', item.id);
+        deletedVectorAmount++;
+      }
+
+      index++;
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  console.log(`检测完成，共删除 ${deletedVectorAmount} 个无效 向量 数据`);
+}

projects/app/src/pages/api/v1/audio/transcriptions.ts

@@ -1,12 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { getUploadModel } from '@fastgpt/service/common/file/multer';
 import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
 import fs from 'fs';
 import { getAIApi } from '@fastgpt/service/core/ai/config';
 import { pushWhisperUsage } from '@/service/support/wallet/usage/push';
+import { authChatCert } from '@/service/support/permission/auth/chat';
 
 const upload = getUploadModel({
   maxSize: 2
@@ -18,12 +18,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
   try {
     const {
       file,
-      data: { duration }
-    } = await upload.doUpload<{ duration: number; shareId?: string }>(req, res);
+      data: { duration, teamId: spaceTeamId, teamToken }
+    } = await upload.doUpload<{
+      duration: number;
+      shareId?: string;
+      teamId?: string;
+      teamToken?: string;
+    }>(req, res);
+
+    req.body.teamId = spaceTeamId;
+    req.body.teamToken = teamToken;
 
     filePaths = [file.path];
 
-    const { teamId, tmbId } = await authCert({ req, authToken: true });
+    const { teamId, tmbId } = await authChatCert({ req, authToken: true });
 
     if (!global.whisperModel) {
       throw new Error('whisper model not found');

projects/app/src/pages/api/v1/chat/completions.ts

@@ -18,31 +18,28 @@ import { authOutLinkChatStart } from '@/service/support/permission/auth/outLink'
 import { pushResult2Remote, addOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
 import requestIp from 'request-ip';
 import { getUsageSourceByAuthType } from '@fastgpt/global/support/wallet/usage/tools';
-import { authTeamShareChatStart } from '@/service/support/permission/auth/teamChat';
-import { selectShareResponse } from '@/utils/service/core/chat';
+import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
+import { selectSimpleChatResponse } from '@/utils/service/core/chat';
 import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { connectToDatabase } from '@/service/mongo';
 import { getUserChatInfoAndAuthTeamPoints } from '@/service/support/permission/auth/team';
 import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
-import { autChatCrud } from '@/service/support/permission/auth/chat';
+import { UserModelSchema } from '@fastgpt/global/support/user/type';
+import { AppSchema } from '@fastgpt/global/core/app/type';
+import { AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api';
+import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
+import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
+import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
 
 type FastGptWebChatProps = {
   chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
   appId?: string;
 };
-type FastGptShareChatProps = {
-  shareId?: string;
-  outLinkUid?: string;
-};
-type FastGptTeamShareChatProps = {
-  shareTeamId?: string;
-  outLinkUid?: string;
-};
+
 export type Props = ChatCompletionCreateParams &
   FastGptWebChatProps &
-  FastGptShareChatProps &
-  FastGptTeamShareChatProps & {
+  OutLinkChatAuthProps & {
     messages: ChatMessageItemType[];
     stream?: boolean;
     detail?: boolean;
@@ -53,6 +50,18 @@ export type ChatResponseType = {
   quoteLen?: number;
 };
 
+type AuthResponseType = {
+  teamId: string;
+  tmbId: string;
+  user: UserModelSchema;
+  app: AppSchema;
+  responseDetail?: boolean;
+  authType: `${AuthUserTypeEnum}`;
+  apikey?: string;
+  canWrite: boolean;
+  outLinkUserId?: string;
+};
+
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
   res.on('close', () => {
     res.end();
@@ -65,9 +74,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
   const {
     chatId,
     appId,
-    shareTeamId,
+    // share chat
     shareId,
     outLinkUid,
+    // team chat
+    teamId: spaceTeamId,
+    teamToken,
     stream = false,
     detail = false,
     messages = [],
@@ -100,136 +112,44 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     if (!question) {
       throw new Error('Question is empty');
     }
-    /* auth app permission */
+
+    /* 
+      1. auth app permission
+      2. auth balance
+      3. get app
+      4. parse outLink token
+    */
     const { teamId, tmbId, user, app, responseDetail, authType, apikey, canWrite, outLinkUserId } =
       await (async () => {
+        // share chat
         if (shareId && outLinkUid) {
-          const { teamId, tmbId, user, appId, authType, responseDetail, uid } =
-            await authOutLinkChatStart({
-              shareId,
-              ip: originIp,
-              outLinkUid,
-              question: question.value
-            });
-          const app = await MongoApp.findById(appId);
-
-          if (!app) {
-            return Promise.reject('app is empty');
-          }
-
-          return {
-            teamId,
-            tmbId,
-            user,
-            app,
-            responseDetail,
-            apikey: '',
-            authType,
-            canWrite: false,
-            outLinkUserId: uid
-          };
-        }
-        // team Apps share
-        if (shareTeamId && appId && outLinkUid) {
-          const { user, uid, tmbId } = await authTeamShareChatStart({
-            teamId: shareTeamId,
-            ip: originIp,
+          return authShareChat({
+            shareId,
             outLinkUid,
+            chatId,
+            ip: originIp,
             question: question.value
           });
-          const app = await MongoApp.findById(appId);
-          if (!app) {
-            return Promise.reject('app is empty');
-          }
-
-          return {
-            teamId: shareTeamId,
-            tmbId,
-            user,
-            app,
-            responseDetail: detail,
-            authType: AuthUserTypeEnum.token,
-            apikey: '',
-            canWrite: false,
-            outLinkUserId: uid
-          };
+        }
+        // team space chat
+        if (spaceTeamId && appId && teamToken) {
+          return authTeamSpaceChat({
+            teamId: spaceTeamId,
+            teamToken,
+            appId,
+            chatId
+          });
         }
 
-        const {
-          appId: apiKeyAppId,
-          teamId,
-          tmbId,
-          authType,
-          apikey
-        } = await authCert({
+        /* parse req: api or token */
+        return authHeaderRequest({
           req,
-          authToken: true,
-          authApiKey: true
-        });
-
-        const { user } = await getUserChatInfoAndAuthTeamPoints(tmbId);
-
-        // openapi key
-        if (authType === AuthUserTypeEnum.apikey) {
-          if (!apiKeyAppId) {
-            return Promise.reject(
-              'Key is error. You need to use the app key rather than the account key.'
-            );
-          }
-          const app = await MongoApp.findById(apiKeyAppId);
-
-          if (!app) {
-            return Promise.reject('app is empty');
-          }
-
-          return {
-            teamId,
-            tmbId,
-            user,
-            app,
-            responseDetail: detail,
-            apikey,
-            authType,
-            canWrite: true
-          };
-        }
-
-        // token auth
-        if (!appId) {
-          return Promise.reject('appId is empty');
-        }
-        const { app, canWrite } = await authApp({
-          req,
-          authToken: true,
           appId,
-          per: 'r'
+          chatId,
+          detail
         });
-
-        return {
-          teamId,
-          tmbId,
-          user,
-          app,
-          responseDetail: detail,
-          apikey,
-          authType,
-          canWrite: canWrite || false
-        };
       })();
 
-    // auth chat permission
-    await autChatCrud({
-      req,
-      authToken: true,
-      authApiKey: true,
-      appId: app._id,
-      chatId,
-      shareId,
-      shareTeamId,
-      outLinkUid,
-      per: 'w'
-    });
-
     // get and concat history
     const { history } = await getChatItems({
       appId: app._id,
@@ -237,7 +157,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       limit: 30,
       field: `dataId obj value`
     });
-
     const concatHistories = history.concat(chatMessages);
     const responseChatItemId: string | undefined = messages[messages.length - 1].dataId;
 
@@ -263,13 +182,14 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 
     // save chat
     if (chatId) {
+      const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
       await saveChat({
         chatId,
         appId: app._id,
         teamId,
         tmbId: tmbId,
         variables,
-        updateUseTime: !shareId && String(tmbId) === String(app.tmbId), // owner update use time
+        updateUseTime: isOwnerUse, // owner update use time
         shareId,
         outLinkUid: outLinkUserId,
         source: (() => {
@@ -279,6 +199,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
           if (authType === 'apikey') {
             return ChatSourceEnum.api;
           }
+          if (spaceTeamId) {
+            return ChatSourceEnum.team;
+          }
           return ChatSourceEnum.online;
         })(),
         content: [
@@ -299,7 +222,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
 
     /* select fe response field */
-    const feResponseData = canWrite ? responseData : selectShareResponse({ responseData });
+    const feResponseData = canWrite ? responseData : selectSimpleChatResponse({ responseData });
 
     if (stream) {
       responseWrite({
@@ -382,3 +305,162 @@ export const config = {
     responseLimit: '20mb'
   }
 };
+
+const authShareChat = async ({
+  chatId,
+  ...data
+}: AuthOutLinkChatProps & {
+  shareId: string;
+  chatId?: string;
+}): Promise<AuthResponseType> => {
+  const { teamId, tmbId, user, appId, authType, responseDetail, uid } =
+    await authOutLinkChatStart(data);
+  const app = await MongoApp.findById(appId).lean();
+
+  if (!app) {
+    return Promise.reject('app is empty');
+  }
+
+  // get chat
+  const chat = await MongoChat.findOne({ appId, chatId }).lean();
+  if (chat && (chat.shareId !== data.shareId || chat.outLinkUid !== uid)) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    teamId,
+    tmbId,
+    user,
+    app,
+    responseDetail,
+    apikey: '',
+    authType,
+    canWrite: false,
+    outLinkUserId: uid
+  };
+};
+const authTeamSpaceChat = async ({
+  appId,
+  teamId,
+  teamToken,
+  chatId
+}: {
+  appId: string;
+  teamId: string;
+  teamToken: string;
+  chatId?: string;
+}): Promise<AuthResponseType> => {
+  const { uid } = await authTeamSpaceToken({
+    teamId,
+    teamToken
+  });
+
+  const app = await MongoApp.findById(appId).lean();
+  if (!app) {
+    return Promise.reject('app is empty');
+  }
+
+  const [chat, { user }] = await Promise.all([
+    MongoChat.findOne({ appId, chatId }).lean(),
+    getUserChatInfoAndAuthTeamPoints(app.tmbId)
+  ]);
+
+  if (chat && (String(chat.teamId) !== teamId || chat.outLinkUid !== uid)) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    teamId,
+    tmbId: app.tmbId,
+    user,
+    app,
+    responseDetail: true,
+    authType: AuthUserTypeEnum.outLink,
+    apikey: '',
+    canWrite: false,
+    outLinkUserId: uid
+  };
+};
+const authHeaderRequest = async ({
+  req,
+  appId,
+  chatId,
+  detail
+}: {
+  req: NextApiRequest;
+  appId?: string;
+  chatId?: string;
+  detail?: boolean;
+}): Promise<AuthResponseType> => {
+  const {
+    appId: apiKeyAppId,
+    teamId,
+    tmbId,
+    authType,
+    apikey,
+    canWrite: apiKeyCanWrite
+  } = await authCert({
+    req,
+    authToken: true,
+    authApiKey: true
+  });
+
+  const { app, canWrite } = await (async () => {
+    if (authType === AuthUserTypeEnum.apikey) {
+      if (!apiKeyAppId) {
+        return Promise.reject(
+          'Key is error. You need to use the app key rather than the account key.'
+        );
+      }
+      const app = await MongoApp.findById(apiKeyAppId);
+
+      if (!app) {
+        return Promise.reject('app is empty');
+      }
+
+      appId = String(app._id);
+
+      return {
+        app,
+        canWrite: apiKeyCanWrite
+      };
+    } else {
+      // token auth
+      if (!appId) {
+        return Promise.reject('appId is empty');
+      }
+      const { app, canWrite } = await authApp({
+        req,
+        authToken: true,
+        appId,
+        per: 'r'
+      });
+
+      return {
+        app,
+
+        canWrite: canWrite
+      };
+    }
+  })();
+
+  const [{ user }, chat] = await Promise.all([
+    getUserChatInfoAndAuthTeamPoints(tmbId),
+    MongoChat.findOne({ appId, chatId }).lean()
+  ]);
+
+  if (chat && (String(chat.teamId) !== teamId || String(chat.tmbId) !== tmbId)) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    teamId,
+    tmbId,
+    user,
+    app,
+    responseDetail: detail,
+    apikey,
+    authType,
+    canWrite
+  };
+};

projects/app/src/pages/api/v1/embeddings.ts

@@ -36,7 +36,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 
     await checkTeamAIPoints(teamId);
 
-    const { charsLength, vectors } = await getVectorsByText({
+    const { tokens, vectors } = await getVectorsByText({
       input: query,
       model: getVectorModel(model)
     });
@@ -50,15 +50,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       })),
       model,
       usage: {
-        prompt_tokens: charsLength,
-        total_tokens: charsLength
+        prompt_tokens: tokens,
+        total_tokens: tokens
       }
     });
 
     const { totalPoints } = pushGenerateVectorUsage({
       teamId,
       tmbId,
-      charsLength,
+      tokens,
       model,
       billId,
       source: getUsageSourceByAuthType({ authType })