4.6.4-alpha (#582)

2023-12-08 15:01:11 +08:00
parent 54d52d8d25
commit b58249fc3a
66 changed files with 962 additions and 527 deletions
--- a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
+++ b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
@@ -2,7 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { delFileById, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
+import {
+  delFileByFileIdList,
+  getGFSCollection
+} from '@fastgpt/service/common/file/gridfs/controller';
 import { addLog } from '@fastgpt/service/common/mongo/controller';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { delay } from '@fastgpt/global/common/system/utils';
@@ -77,7 +80,7 @@ export async function checkFiles(start: Date, end: Date, limit: number) {

      // 3. if not found, delete file
      if (hasCollection === 0) {
-        await delFileById({ bucketName: 'dataset', fileId: String(_id) });
+        await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] });
        console.log('delete file', _id);
        deleteFileAmount++;
      }
--- a/projects/app/src/pages/api/common/file/read.ts
+++ b/projects/app/src/pages/api/common/file/read.ts
@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
 import { detect } from 'jschardet';
 import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
+import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -22,6 +23,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      getDownloadStream({ bucketName, fileId })
    ]);

+    if (!file) {
+      return Promise.reject(CommonErrEnum.fileNotFound);
+    }
+
    // get encoding
    let buffers: Buffer = Buffer.from([]);
    for await (const chunk of encodeStream) {
--- a/projects/app/src/pages/api/common/file/uploadImage.ts
+++ b/projects/app/src/pages/api/common/file/uploadImage.ts
@@ -1,21 +1,22 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
 import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
-
-type Props = { base64Img: string; expiredTime?: Date };
+import { UploadImgProps } from '@fastgpt/global/common/file/api';

 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
    await connectToDatabase();
-    const { teamId } = await authCert({ req, authToken: true });
-    const { base64Img, expiredTime } = req.body as Props;
+    const { base64Img, expiredTime, metadata, shareId } = req.body as UploadImgProps;
+
+    const { teamId } = await authCertOrShareId({ req, shareId, authToken: true });

    const data = await uploadMongoImg({
      teamId,
      base64Img,
-      expiredTime
+      expiredTime,
+      metadata
    });

    jsonRes(res, { data });
--- a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
+++ b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
@@ -4,13 +4,14 @@ import { connectToDatabase } from '@/service/mongo';
 import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
 import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push';
 import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
-import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
+import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    await connectToDatabase();
    const { messages, shareId } = req.body as CreateQuestionGuideParams;
-    const { tmbId, teamId } = await authCertAndShareId({
+
+    const { tmbId, teamId } = await authCertOrShareId({
      req,
      authToken: true,
      shareId
--- a/projects/app/src/pages/api/core/app/update.ts
+++ b/projects/app/src/pages/api/core/app/update.ts
@@ -4,6 +4,9 @@ import { connectToDatabase } from '@/service/mongo';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
 import { authApp } from '@fastgpt/service/support/permission/auth/app';
+import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
+import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
+import { getChatModel } from '@/service/core/ai/model';

 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -20,6 +23,36 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // 凭证校验
    await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });

+    // check modules
+    // 1. dataset search limit, less than model quoteMaxToken
+    if (modules) {
+      let maxTokens = 3000;
+
+      modules.forEach((item) => {
+        if (item.flowType === FlowNodeTypeEnum.chatNode) {
+          const model =
+            item.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || '';
+          const chatModel = getChatModel(model);
+          const quoteMaxToken = chatModel.quoteMaxToken || 3000;
+
+          maxTokens = Math.max(maxTokens, quoteMaxToken);
+        }
+      });
+
+      modules.forEach((item) => {
+        if (item.flowType === FlowNodeTypeEnum.datasetSearchNode) {
+          item.inputs.forEach((input) => {
+            if (input.key === ModuleInputKeyEnum.datasetLimit) {
+              const val = input.value as number;
+              if (val > maxTokens) {
+                input.value = maxTokens;
+              }
+            }
+          });
+        }
+      });
+    }
+
    // 更新模型
    await MongoApp.updateOne(
      {
--- a/projects/app/src/pages/api/core/chat/item/delete.ts
+++ b/projects/app/src/pages/api/core/chat/item/delete.ts
@@ -10,6 +10,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    await connectToDatabase();
    const { chatId, contentId, shareId, outLinkUid } = req.query as DeleteChatItemProps;

+    if (!contentId || !chatId) {
+      return jsonRes(res);
+    }
+
    await autChatCrud({
      req,
      authToken: true,
--- a/projects/app/src/pages/api/core/chat/item/getSpeech.ts
+++ b/projects/app/src/pages/api/core/chat/item/getSpeech.ts
@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push';
-import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common';
+import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common';
 import { authType2BillSource } from '@/service/support/wallet/bill/utils';
 import { getAudioSpeechModel } from '@/service/core/ai/model';
 import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
@@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      throw new Error('model or voice not found');
    }

-    const { teamId, tmbId, authType } = await authCertAndShareId({ req, authToken: true, shareId });
+    const { teamId, tmbId, authType } = await authCertOrShareId({ req, authToken: true, shareId });

    const ttsModel = getAudioSpeechModel(ttsConfig.model);
    const voiceData = ttsModel.voices?.find((item) => item.value === ttsConfig.voice);
--- a/projects/app/src/pages/api/core/dataset/collection/delete.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts
@@ -24,13 +24,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    });

    // find all delete id
-    const collections = await findCollectionAndChild(collectionId, '_id metadata');
+    const collections = await findCollectionAndChild(collectionId, '_id fileId');
    const delIdList = collections.map((item) => item._id);

    // delete
    await delCollectionRelevantData({
      collectionIds: delIdList,
-      fileIds: collections.map((item) => item.metadata?.fileId).filter(Boolean)
+      fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
    });

    // delete collection
--- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts
@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 import { loadingOneChunkCollection } from '@fastgpt/service/core/dataset/collection/utils';
 import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
-import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
--- a/projects/app/src/pages/api/core/dataset/update.ts
+++ b/projects/app/src/pages/api/core/dataset/update.ts
@@ -15,8 +15,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      throw new Error('缺少参数');
    }

-    // 凭证校验
-    await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
+    if (permission) {
+      await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
+    } else {
+      await authDataset({ req, authToken: true, datasetId: id, per: 'w' });
+    }

    await MongoDataset.findOneAndUpdate(
      {