4.6.2-alpha (#517)

projects/app/src/pages/api/admin/initv46.ts

@@ -295,7 +295,7 @@ async function initPgData() {
     ]);
   } catch (error) {
     console.log(error);
-    console.log('column exits');
+    console.log('column exists');
   }
 
   const { rows } = await PgClient.query<{ user_id: string }>(`

projects/app/src/pages/api/admin/initv462.ts

@@ -2,18 +2,9 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { delay } from '@/utils/tools';
-import { PgClient } from '@fastgpt/service/common/pg';
-import {
-  DatasetDataIndexTypeEnum,
-  PgDatasetTableName
-} from '@fastgpt/global/core/dataset/constant';
-
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
-import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
-import { defaultQAModels } from '@fastgpt/global/core/ai/model';
-import { MongoApp } from '@fastgpt/service/core/app/schema';
+import { jiebaSplit } from '@/service/core/dataset/utils';
 
 let success = 0;
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
@@ -22,6 +13,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const { limit = 50 } = req.body as { limit: number };
     await authCert({ req, authRoot: true });
     await connectToDatabase();
+    success = 0;
+
+    console.log(
+      'total',
+      await MongoDatasetData.countDocuments({ fullTextToken: { $exists: false } })
+    );
 
     await initFullTextToken(limit);
 
@@ -37,4 +34,31 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
   }
 }
-export async function initFullTextToken(limit = 50) {}
+export async function initFullTextToken(limit = 50): Promise<any> {
+  try {
+    const dataList = await MongoDatasetData.find({ fullTextToken: { $exists: false } }, '_id q a')
+      .limit(limit)
+      .lean();
+    if (dataList.length === 0) return;
+
+    const result = await Promise.allSettled(
+      dataList.map((item) => {
+        const text = item.q + (item.a || '');
+        const tokens = jiebaSplit({ text });
+
+        return MongoDatasetData.findByIdAndUpdate(item._id, {
+          $set: {
+            fullTextToken: tokens
+          }
+        });
+      })
+    );
+
+    success += result.filter((item) => item.status === 'fulfilled').length;
+    console.log(`success: ${success}`);
+    return initFullTextToken(limit);
+  } catch (error) {
+    await delay(1000);
+    return initFullTextToken(limit);
+  }
+}

projects/app/src/pages/api/core/app/create.ts

@@ -6,6 +6,7 @@ import type { CreateAppParams } from '@fastgpt/global/core/app/api.d';
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
+import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -39,7 +40,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       teamId,
       tmbId,
       modules,
-      type
+      type,
+      simpleTemplateId: SimpleModeTemplate_FastGPT_Universal.id
     });
 
     jsonRes(res, {

projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts

@@ -41,7 +41,7 @@ function simpleChatTemplate({
     {
       moduleId: 'userChatInput',
       name: '用户问题(对话入口)',
-      logo: '/imgs/module/userChatInput.png',
+      avatar: '/imgs/module/userChatInput.png',
       flowType: 'questionInput',
       position: {
         x: 464.32198615344566,
@@ -73,7 +73,7 @@ function simpleChatTemplate({
     {
       moduleId: 'history',
       name: '聊天记录',
-      logo: '/imgs/module/history.png',
+      avatar: '/imgs/module/history.png',
       flowType: 'historyNode',
       position: {
         x: 452.5466249541586,
@@ -114,7 +114,7 @@ function simpleChatTemplate({
     {
       moduleId: 'chatModule',
       name: 'AI 对话',
-      logo: '/imgs/module/AI.png',
+      avatar: '/imgs/module/AI.png',
       flowType: 'chatNode',
       showStatus: true,
       position: {
@@ -284,7 +284,7 @@ function datasetTemplate({
     {
       moduleId: 'userChatInput',
       name: '用户问题(对话入口)',
-      logo: '/imgs/module/userChatInput.png',
+      avatar: '/imgs/module/userChatInput.png',
       flowType: 'questionInput',
       position: {
         x: 464.32198615344566,
@@ -320,7 +320,7 @@ function datasetTemplate({
     {
       moduleId: 'history',
       name: '聊天记录',
-      logo: '/imgs/module/history.png',
+      avatar: '/imgs/module/history.png',
       flowType: 'historyNode',
       position: {
         x: 452.5466249541586,
@@ -361,7 +361,7 @@ function datasetTemplate({
     {
       moduleId: 'datasetSearch',
       name: '知识库搜索',
-      logo: '/imgs/module/db.png',
+      avatar: '/imgs/module/db.png',
       flowType: 'datasetSearchNode',
       showStatus: true,
       position: {
@@ -454,7 +454,7 @@ function datasetTemplate({
     {
       moduleId: 'chatModule',
       name: 'AI 对话',
-      logo: '/imgs/module/AI.png',
+      avatar: '/imgs/module/AI.png',
       flowType: 'chatNode',
       showStatus: true,
       position: {

projects/app/src/pages/api/core/dataset/data/insertData.ts

@@ -15,6 +15,7 @@ import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controll
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
 import { InsertOneDatasetDataProps } from '@/global/core/dataset/api';
+import { simpleText } from '@fastgpt/global/common/string/tools';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -46,8 +47,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     ] = await Promise.all([getCollectionWithDataset(collectionId), authTeamBalance(teamId)]);
 
     // format data
-    const formatQ = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-    const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
+    const formatQ = simpleText(q);
+    const formatA = simpleText(a);
+    const formatIndexes = indexes?.map((item) => ({
+      ...item,
+      text: simpleText(item.text)
+    }));
 
     // token check
     const token = countPromptTokens(formatQ, 'system');
@@ -72,7 +77,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       q: formatQ,
       a: formatA,
       model: vectorModelData.model,
-      indexes
+      indexes: formatIndexes
     });
 
     pushGenerateVectorBill({

projects/app/src/pages/api/core/dataset/data/pushData.ts

@@ -13,6 +13,7 @@ import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
 import { getQAModel, getVectorModel } from '@/service/core/ai/model';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
 import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
+import { simpleText } from '@fastgpt/global/common/string/tools';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -72,6 +73,21 @@ export async function pushDataToDatasetCollection({
     collectionId
   });
 
+  // format q and a, remove empty char
+  data.forEach((item) => {
+    item.q = simpleText(item.q);
+    item.a = simpleText(item.a);
+
+    item.indexes = item.indexes
+      ?.map((index) => {
+        return {
+          ...index,
+          text: simpleText(index.text)
+        };
+      })
+      .filter(Boolean);
+  });
+
   // filter repeat or equal content
   const set = new Set();
   const filterResult: Record<string, PushDatasetDataChunkProps[]> = {

projects/app/src/pages/api/core/dataset/searchTest.ts

@@ -1,9 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
-import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
+import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
 import { connectToDatabase } from '@/service/mongo';
-import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
@@ -22,6 +21,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       throw new Error('缺少参数');
     }
 
+    const start = Date.now();
+
     // auth dataset role
     const { dataset, teamId, tmbId, apikey } = await authDataset({
       req,
@@ -61,8 +62,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       });
     }
 
-    jsonRes<SearchDataResponseItemType[]>(res, {
-      data: searchRes
+    jsonRes<SearchTestResponse>(res, {
+      data: {
+        list: searchRes,
+        duration: `${((Date.now() - start) / 1000).toFixed(3)}s`
+      }
     });
   } catch (err) {
     jsonRes(res, {

projects/app/src/pages/api/core/plugin/detail.ts

@@ -1,17 +1,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
 import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     const { id } = req.query as { id: string };
     await connectToDatabase();
-    await authPluginCrud({ req, authToken: true, id, per: 'r' });
+    const { plugin } = await authPluginCrud({ req, authToken: true, id, per: 'r' });
 
     jsonRes(res, {
-      data: await MongoPlugin.findOne({ id })
+      data: plugin
     });
   } catch (err) {
     jsonRes(res, {