4.6.3-alpha1 (#529)

2023-11-29 20:45:36 +08:00
parent 007fce2deb
commit b916183848
43 changed files with 515 additions and 184 deletions
--- a/projects/app/src/pages/api/admin/initv463.ts
+++ b/projects/app/src/pages/api/admin/initv463.ts
@@ -0,0 +1,55 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+
+let success = 0;
+/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { limit = 50 } = req.body as { limit: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+    success = 0;
+
+    await MongoDatasetCollection.updateMany({}, [
+      {
+        $set: {
+          createTime: '$updateTime',
+          trainingType: {
+            $cond: {
+              if: { $ifNull: ['$a', false] },
+              then: TrainingModeEnum.qa,
+              else: TrainingModeEnum.chunk
+            }
+          },
+          chunkSize: 0,
+          fileId: '$metadata.fileId',
+          rawLink: '$metadata.rawLink'
+        }
+      }
+    ]);
+
+    await MongoDatasetData.updateMany(
+      {},
+      {
+        chunkIndex: 0,
+        updateTime: new Date()
+      }
+    );
+
+    jsonRes(res, {
+      message: 'success'
+    });
+  } catch (error) {
+    console.log(error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
--- a/projects/app/src/pages/api/core/dataset/collection/create.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/create.ts
@@ -6,8 +6,11 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
-import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+import {
+  TrainingModeEnum,
+  DatasetCollectionTypeEnum,
+  DatasetCollectionTrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';

@@ -45,7 +48,10 @@ export async function createOneCollection({
  parentId,
  datasetId,
  type,
-  metadata = {},
+  trainingType = DatasetCollectionTrainingModeEnum.manual,
+  chunkSize = 0,
+  fileId,
+  rawLink,
  teamId,
  tmbId
 }: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
@@ -56,8 +62,10 @@ export async function createOneCollection({
    datasetId,
    parentId: parentId || null,
    type,
-    metadata,
-    updateTime: getCollectionUpdateTime({ name })
+    trainingType,
+    chunkSize,
+    fileId,
+    rawLink
  });

  // create default collection
@@ -94,7 +102,8 @@ export function createDefaultCollection({
    datasetId,
    parentId,
    type: DatasetCollectionTypeEnum.virtual,
-    updateTime: new Date('2099'),
-    metadata: {}
+    trainingType: DatasetCollectionTrainingModeEnum.manual,
+    chunkSize: 0,
+    updateTime: new Date('2099')
  });
 }
--- a/projects/app/src/pages/api/core/dataset/collection/delById.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/delById.ts
@@ -42,10 +42,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // delete file
    await Promise.all(
      collections.map((collection) => {
-        if (!collection.metadata?.fileId) return;
+        if (!collection?.fileId) return;
        return delFileById({
          bucketName: BucketNameEnum.dataset,
-          fileId: collection.metadata.fileId
+          fileId: collection.fileId
        });
      })
    );
--- a/projects/app/src/pages/api/core/dataset/collection/detail.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/detail.ts
@@ -6,6 +6,8 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { getFileById } from '@fastgpt/service/common/file/gridfs/controller';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -24,12 +26,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      per: 'r'
    });

+    // get file
+    const file = collection?.fileId
+      ? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
+      : undefined;
+
    jsonRes<DatasetCollectionItemType>(res, {
      data: {
        ...collection,
        canWrite,
        sourceName: collection?.name,
-        sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
+        sourceId: collection?.fileId || collection?.rawLink,
+        file
      }
    });
  } catch (err) {
--- a/projects/app/src/pages/api/core/dataset/collection/list.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/list.ts
@@ -115,9 +115,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
            name: 1,
            type: 1,
            updateTime: 1,
-            trainingAmount: { $size: '$trainings' },
            dataAmount: { $size: '$datas' },
-            metadata: 1
+            trainingAmount: { $size: '$trainings' },
+            fileId: 1,
+            rawLink: 1
          }
        },
        {
--- a/projects/app/src/pages/api/core/dataset/collection/update.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/update.ts
@@ -9,7 +9,7 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    await connectToDatabase();
-    const { id, parentId, name, metadata = {} } = req.body as UpdateDatasetCollectionParams;
+    const { id, parentId, name } = req.body as UpdateDatasetCollectionParams;

    if (!id) {
      throw new Error('缺少参数');
@@ -23,11 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      ...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
    };

-    // 将metadata的每个字段添加到updateFields中
-    for (const [key, value] of Object.entries(metadata)) {
-      updateFields[`metadata.${key}`] = value;
-    }
-
    await MongoDatasetCollection.findByIdAndUpdate(id, {
      $set: updateFields
    });
--- a/projects/app/src/pages/api/core/dataset/data/insertData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts
@@ -76,6 +76,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      collectionId,
      q: formatQ,
      a: formatA,
+      chunkIndex: 0,
      model: vectorModelData.model,
      indexes: formatIndexes
    });
--- a/projects/app/src/pages/api/core/dataset/data/list.ts
+++ b/projects/app/src/pages/api/core/dataset/data/list.ts
@@ -32,8 +32,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    };

    const [data, total] = await Promise.all([
-      MongoDatasetData.find(match, '_id datasetId collectionId q a  indexes')
-        .sort({ _id: -1 })
+      MongoDatasetData.find(match, '_id datasetId collectionId q a indexes')
+        .sort({ chunkIndex: 1, updateTime: -1 })
        .skip((pageNum - 1) * pageSize)
        .limit(pageSize)
        .lean(),
--- a/projects/app/src/pages/api/core/dataset/data/pushData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts
@@ -125,7 +125,7 @@ export async function pushDataToDatasetCollection({

  // 插入记录
  const insertRes = await MongoDatasetTraining.insertMany(
-    filterResult.success.map((item) => ({
+    filterResult.success.map((item, i) => ({
      teamId,
      tmbId,
      datasetId,
@@ -136,6 +136,7 @@ export async function pushDataToDatasetCollection({
      model,
      q: item.q,
      a: item.a,
+      chunkIndex: i,
      indexes: item.indexes
    }))
  );
--- a/projects/app/src/pages/api/core/dataset/delete.ts
+++ b/projects/app/src/pages/api/core/dataset/delete.ts
@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller';
-import { Types } from '@fastgpt/service/common/mongo';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { delDataByDatasetId } from '@/service/core/dataset/data/controller';
--- a/projects/app/src/pages/api/core/plugin/templates.ts
+++ b/projects/app/src/pages/api/core/plugin/templates.ts
@@ -16,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<

    const [userPlugins, plusPlugins] = await Promise.all([
      MongoPlugin.find({ teamId }).lean(),
-      GET<PluginTemplateType[]>('/core/plugin/getTemplates')
+      global.systemEnv.pluginBaseUrl ? GET<PluginTemplateType[]>('/core/plugin/getTemplates') : []
    ]);

    const data: FlowModuleTemplateType[] = [
--- a/projects/app/src/pages/api/v1/chat/completions.ts
+++ b/projects/app/src/pages/api/v1/chat/completions.ts
@@ -142,7 +142,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
          responseDetail: detail,
          apikey,
          authType,
-          canWrite: false
+          canWrite: true
        };
      }