4.6.4 (#588)

2023-12-11 15:12:14 +08:00
parent 84cf6b5658
commit d2d7eac9e0
105 changed files with 1091 additions and 801 deletions
--- a/projects/app/src/pages/api/admin/init464.ts
+++ b/projects/app/src/pages/api/admin/init464.ts
@@ -0,0 +1,45 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { PgClient } from '@fastgpt/service/common/pg';
+import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
+import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
+import { connectToDatabase } from '@/service/mongo';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    try {
+      await PgClient.query(
+        `ALTER TABLE ${PgDatasetTableName} ADD COLUMN createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP;`
+      );
+    } catch (error) {
+      console.log(error);
+    }
+
+    try {
+      const result = await MongoChatItem.updateMany(
+        { userFeedback: { $exists: true } },
+        { $rename: { userFeedback: 'userBadFeedback' } }
+      );
+      console.log(result);
+    } catch (error) {
+      console.log(error);
+    }
+
+    jsonRes(res, {
+      data: {}
+    });
+  } catch (error) {
+    console.log(error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
--- a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
+++ b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
@@ -6,7 +6,7 @@ import {
  delFileByFileIdList,
  getGFSCollection
 } from '@fastgpt/service/common/file/gridfs/controller';
-import { addLog } from '@fastgpt/service/common/mongo/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { delay } from '@fastgpt/global/common/system/utils';

--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
@@ -379,14 +379,14 @@ function datasetTemplate({
        },
        {
          key: 'similarity',
-          value: 0.4,
+          value: 0.1,
          type: FlowNodeInputTypeEnum.slider,
          label: '相关度',
          connected: true
        },
        {
          key: 'limit',
-          value: 8,
+          value: 2000,
          type: FlowNodeInputTypeEnum.slider,
          label: '单次搜索上限',
          connected: true
--- a/projects/app/src/pages/api/core/app/getChatLogs.ts
+++ b/projects/app/src/pages/api/core/app/getChatLogs.ts
@@ -60,12 +60,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
        },
        {
          $addFields: {
-            feedbackCount: {
+            userGoodFeedbackCount: {
              $size: {
                $filter: {
                  input: '$chatitems',
                  as: 'item',
-                  cond: { $ifNull: ['$$item.userFeedback', false] }
+                  cond: { $ifNull: ['$$item.userGoodFeedback', false] }
+                }
+              }
+            },
+            userBadFeedbackCount: {
+              $size: {
+                $filter: {
+                  input: '$chatitems',
+                  as: 'item',
+                  cond: { $ifNull: ['$$item.userBadFeedback', false] }
+                }
+              }
+            },
+            robotBadFeedbackCount: {
+              $size: {
+                $filter: {
+                  input: '$chatitems',
+                  as: 'item',
+                  cond: { $ifNull: ['$$item.robotBadFeedback', false] }
                }
              }
            },
@@ -80,7 +98,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
            }
          }
        },
-        { $sort: { feedbackCount: -1, updateTime: -1 } },
+        {
+          $sort: {
+            userBadFeedbackCount: -1,
+            userGoodFeedbackCount: -1,
+            robotBadFeedbackCount: -1,
+            updateTime: -1
+          }
+        },
        { $skip: (pageNum - 1) * pageSize },
        { $limit: pageSize },
        {
@@ -91,7 +116,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
            source: 1,
            time: '$updateTime',
            messageCount: { $size: '$chatitems' },
-            feedbackCount: 1,
+            userGoodFeedbackCount: 1,
+            userBadFeedbackCount: 1,
+            robotBadFeedbackCount: 1,
            markCount: 1
          }
        }
--- a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts
+++ b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts
@@ -0,0 +1,52 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
+import { UpdateChatFeedbackProps } from '@fastgpt/global/core/chat/api';
+import { autChatCrud } from '@/service/support/permission/auth/chat';
+
+/* 初始化我的聊天框，需要身份验证 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  const { chatId, chatItemId, shareId, outLinkUid, userBadFeedback, userGoodFeedback } =
+    req.body as UpdateChatFeedbackProps;
+
+  try {
+    await connectToDatabase();
+
+    await autChatCrud({
+      req,
+      authToken: true,
+      chatId,
+      shareId,
+      outLinkUid,
+      per: 'r'
+    });
+
+    if (!chatItemId) {
+      throw new Error('chatItemId is required');
+    }
+
+    await MongoChatItem.findOneAndUpdate(
+      {
+        dataId: chatItemId
+      },
+      {
+        $unset: {
+          ...(userBadFeedback === undefined && { userBadFeedback: '' }),
+          ...(userGoodFeedback === undefined && { userGoodFeedback: '' })
+        },
+        $set: {
+          ...(userBadFeedback !== undefined && { userBadFeedback }),
+          ...(userGoodFeedback !== undefined && { userGoodFeedback })
+        }
+      }
+    );
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
--- a/projects/app/src/pages/api/core/chat/feedback/userUpdate.ts
+++ b/projects/app/src/pages/api/core/chat/feedback/userUpdate.ts
@@ -1,35 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
-
-/* 初始化我的聊天框，需要身份验证 */
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await connectToDatabase();
-    const { chatItemId, userFeedback = undefined } = req.body as {
-      chatItemId: string;
-      userFeedback?: string;
-    };
-
-    if (!chatItemId) {
-      throw new Error('chatItemId is required');
-    }
-
-    await MongoChatItem.findOneAndUpdate(
-      {
-        dataId: chatItemId
-      },
-      {
-        ...(userFeedback ? { userFeedback } : { $unset: { userFeedback: '' } })
-      }
-    );
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
--- a/projects/app/src/pages/api/core/chat/init.ts
+++ b/projects/app/src/pages/api/core/chat/init.ts
@@ -43,7 +43,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { history } = await getChatItems({
      chatId,
      limit: 30,
-      field: `dataId obj value adminFeedback userFeedback ${ModuleOutputKeyEnum.responseData}`
+      field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback robotBadFeedback ${ModuleOutputKeyEnum.responseData}`
    });

    jsonRes<InitChatResponse>(res, {
--- a/projects/app/src/pages/api/core/chat/outLink/init.ts
+++ b/projects/app/src/pages/api/core/chat/outLink/init.ts
@@ -42,7 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { history } = await getChatItems({
      chatId,
      limit: 30,
-      field: `dataId obj value userFeedback ${
+      field: `dataId obj value userGoodFeedback userBadFeedback ${
        shareChat.responseDetail ? `adminFeedback ${ModuleOutputKeyEnum.responseData}` : ''
      } `
    });
--- a/projects/app/src/pages/api/core/dataset/checkExportLimit.ts
+++ b/projects/app/src/pages/api/core/dataset/checkExportLimit.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { addLog } from '@fastgpt/service/common/mongo/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
--- a/projects/app/src/pages/api/core/dataset/collection/delete.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts
@@ -33,11 +33,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
    });

-    // delete collection
-    await MongoDatasetCollection.deleteMany({
-      _id: { $in: delIdList }
-    });
-
    jsonRes(res);
  } catch (err) {
    jsonRes(res, {
--- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts
@@ -68,13 +68,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    });

    // delete old collection
-    await Promise.all([
-      delCollectionRelevantData({
-        collectionIds: [collection._id],
-        fileIds: collection.fileId ? [collection.fileId] : []
-      }),
-      MongoDatasetCollection.findByIdAndRemove(collection._id)
-    ]);
+    await delCollectionRelevantData({
+      collectionIds: [collection._id],
+      fileIds: collection.fileId ? [collection.fileId] : []
+    });

    jsonRes(res);
  } catch (err) {
--- a/projects/app/src/pages/api/core/dataset/data/pushData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts
@@ -136,7 +136,7 @@ export async function pushDataToDatasetCollection({
      model,
      q: item.q,
      a: item.a,
-      chunkIndex: i,
+      chunkIndex: item.chunkIndex ?? i,
      indexes: item.indexes
    }))
  );
--- a/projects/app/src/pages/api/core/dataset/exportAll.ts
+++ b/projects/app/src/pages/api/core/dataset/exportAll.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes, responseWriteController } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { addLog } from '@fastgpt/service/common/mongo/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
--- a/projects/app/src/pages/api/core/dataset/searchTest.ts
+++ b/projects/app/src/pages/api/core/dataset/searchTest.ts
@@ -6,10 +6,8 @@ import { connectToDatabase } from '@/service/mongo';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
-import { countModelPrice } from '@/service/support/wallet/bill/utils';
 import { searchDatasetData } from '@/service/core/dataset/data/pg';
 import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
-import { ModelTypeEnum } from '@/service/core/ai/model';
 import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -38,7 +36,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    const { searchRes, tokenLen } = await searchDatasetData({
      text,
      model: dataset.vectorModel,
-      limit: Math.min(limit, 50),
+      limit: Math.min(limit * 800, 30000),
      datasetIds: [datasetId],
      searchMode
    });
--- a/projects/app/src/pages/api/system/getInitData.ts
+++ b/projects/app/src/pages/api/system/getInitData.ts
@@ -20,6 +20,7 @@ import {
 import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
 import { getSimpleTemplatesFromPlus } from '@/service/core/app/utils';
 import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
+import { getFastGPTFeConfig } from '@fastgpt/service/common/system/config/controller';

 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  await getInitConfig();
@@ -73,7 +74,17 @@ export async function getInitConfig() {
      process.env.NODE_ENV === 'development' ? 'data/config.local.json' : '/app/data/config.json';
    const res = JSON.parse(readFileSync(filename, 'utf-8')) as ConfigFileType;

-    setDefaultData(res);
+    // get config from database
+    const dbFeConfig = await getFastGPTFeConfig();
+    const concatConfig: ConfigFileType = {
+      ...res,
+      FeConfig: {
+        ...res.FeConfig,
+        ...dbFeConfig
+      }
+    };
+
+    setDefaultData(concatConfig);
  } catch (error) {
    setDefaultData();
    console.log('get init config error, set default', error);
@@ -83,6 +94,23 @@ export async function getInitConfig() {
  getSystemVersion();
  getModelPrice();
  getSystemPlugin();
+
+  console.log({
+    FeConfig: global.feConfigs,
+    SystemParams: global.systemEnv,
+    ChatModels: global.chatModels,
+    QAModels: global.qaModels,
+    CQModels: global.cqModels,
+    ExtractModels: global.extractModels,
+    QGModels: global.qgModels,
+    VectorModels: global.vectorModels,
+    ReRankModels: global.reRankModels,
+    AudioSpeechModels: global.reRankModels,
+    WhisperModel: global.whisperModel,
+    price: global.priceMd,
+    simpleModeTemplates: global.simpleModeTemplates,
+    communityPlugins: global.communityPlugins
+  });
 }

 export function initGlobal() {
@@ -125,8 +153,6 @@ export function setDefaultData(res?: ConfigFileType) {
  global.whisperModel = res?.WhisperModel || defaultWhisperModel;

  global.priceMd = '';
-
-  console.log(res);
 }

 export function getSystemVersion() {
@@ -173,7 +199,6 @@ ${global.audioSpeechModels
  .join('\n')}
 ${`| 语音输入-${global.whisperModel.name} | ${global.whisperModel.price}/分钟 |`}
 `;
-  console.log(global.priceMd);
 }

 async function getSimpleModeTemplates() {
@@ -209,8 +234,6 @@ async function getSimpleModeTemplates() {
  } catch (error) {
    global.simpleModeTemplates = [SimpleModeTemplate_FastGPT_Universal];
  }
-  console.log('simple mode templates: ');
-  console.log(global.simpleModeTemplates);
 }

 function getSystemPlugin() {
@@ -236,6 +259,4 @@ function getSystemPlugin() {
  });

  global.communityPlugins = fileTemplates;
-  console.log('community plugins: ');
-  console.log(fileTemplates);
 }
--- a/projects/app/src/pages/api/user/account/loginByPassword.ts
+++ b/projects/app/src/pages/api/user/account/loginByPassword.ts
@@ -5,6 +5,7 @@ import { createJWT, setCookie } from '@fastgpt/service/support/permission/contro
 import { connectToDatabase } from '@/service/mongo';
 import { getUserDetail } from '@fastgpt/service/support/user/controller';
 import type { PostLoginProps } from '@fastgpt/global/support/user/api.d';
+import { UserStatusEnum } from '@fastgpt/global/support/user/constant';

 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
@@ -16,13 +17,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    }

    // 检测用户是否存在
-    const authCert = await MongoUser.findOne({
-      username
-    });
+    const authCert = await MongoUser.findOne(
+      {
+        username
+      },
+      'status'
+    );
    if (!authCert) {
      throw new Error('用户未注册');
    }

+    if (authCert.status === UserStatusEnum.forbidden) {
+      throw new Error('账号已停用，无法登录');
+    }
+
    const user = await MongoUser.findOne({
      username,
      password
--- a/projects/app/src/pages/api/v1/chat/completions.ts
+++ b/projects/app/src/pages/api/v1/chat/completions.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { authApp } from '@fastgpt/service/support/permission/auth/app';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
-import { addLog } from '@fastgpt/service/common/mongo/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
 import { sseResponseEventEnum } from '@fastgpt/service/common/response/constant';