Record scroll test (#2783)

* perf: history add scrollList (#2696)

* perf: chatHistorySlider add virtualList

* perf: chat records add scrollList

* delete console

* perf: ScrollData add ref props

* 优化代码

* optimize code && add line breaks

* add total records display

* finish test

* perf: ScrollComponent load data

* perf: Scroll components load

* perf: scroll code

---------

Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>

projects/app/src/pages/api/core/app/version/detail.ts

@@ -1,7 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { NextAPI } from '@/service/middleware/entry';
 import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
-import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import { authApp } from '@fastgpt/service/support/permission/app/auth';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 

projects/app/src/pages/api/core/app/version/list.ts

@@ -3,6 +3,7 @@ import { NextAPI } from '@/service/middleware/entry';
 import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
 import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
 
 type Props = PaginationProps<{
   appId: string;
@@ -10,8 +11,8 @@ type Props = PaginationProps<{
 
 type Response = PaginationResponse<AppVersionSchemaType>;
 
-async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<Response> {
-  const { current, pageSize, appId } = req.body as Props;
+async function handler(req: ApiRequestProps<Props>, res: NextApiResponse<any>): Promise<Response> {
+  const { offset, pageSize, appId } = req.body;
 
   const [result, total] = await Promise.all([
     MongoAppVersion.find({
@@ -20,7 +21,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<
       .sort({
         time: -1
       })
-      .skip((current - 1) * pageSize)
+      .skip(offset)
       .limit(pageSize),
     MongoAppVersion.countDocuments({ appId })
   ]);

projects/app/src/pages/api/core/app/version/listWorkflow.tsx

@@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { NextAPI } from '@/service/middleware/entry';
 import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
 import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
 
 type Props = PaginationProps<{
   appId: string;
@@ -18,8 +19,8 @@ export type versionListResponse = {
 
 type Response = PaginationResponse<versionListResponse>;
 
-async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<Response> {
-  const { current, pageSize, appId } = req.body as Props;
+async function handler(req: ApiRequestProps<Props>, res: NextApiResponse<any>): Promise<Response> {
+  const { offset, pageSize, appId } = req.body;
 
   const [result, total] = await Promise.all([
     MongoAppVersion.find(
@@ -31,7 +32,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<
       .sort({
         time: -1
       })
-      .skip((current - 1) * pageSize)
+      .skip(offset)
       .limit(pageSize),
     MongoAppVersion.countDocuments({ appId })
   ]);

projects/app/src/pages/api/core/chat/getHistories.ts

@@ -1,4 +1,3 @@
-import { connectToDatabase } from '@/service/mongo';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
 import { authOutLink } from '@/service/support/permission/auth/outLink';
@@ -18,7 +17,7 @@ async function handler(
   req: ApiRequestProps<getHistoriesBody, getHistoriesQuery>,
   res: ApiResponseType<any>
 ): Promise<PaginationResponse<getHistoriesResponse>> {
-  const { appId, shareId, outLinkUid, teamId, teamToken, current, pageSize } =
+  const { appId, shareId, outLinkUid, teamId, teamToken, offset, pageSize } =
     req.body as getHistoriesBody;
 
   const match = await (async () => {
@@ -63,7 +62,7 @@ async function handler(
   const [data, total] = await Promise.all([
     await MongoChat.find(match, 'chatId title top customTitle appId updateTime')
       .sort({ top: -1, updateTime: -1 })
-      .skip((current - 1) * pageSize)
+      .skip(offset)
       .limit(pageSize),
     MongoChat.countDocuments(match)
   ]);

projects/app/src/pages/api/core/chat/getPaginationRecords.ts

@@ -0,0 +1,93 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { GetChatRecordsProps } from '@/global/core/chat/api';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+import { transformPreviewHistories } from '@/global/core/chat/utils';
+import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
+import { getChatItems } from '@fastgpt/service/core/chat/controller';
+import { authChatCrud } from '@/service/support/permission/auth/chat';
+import { MongoApp } from '@fastgpt/service/core/app/schema';
+import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
+import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
+import { authOutLink } from '@/service/support/permission/auth/outLink';
+import { GetChatTypeEnum } from '@/global/core/chat/constants';
+import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import { ChatItemType } from '@fastgpt/global/core/chat/type';
+
+export type getPaginationRecordsQuery = {};
+
+export type getPaginationRecordsBody = PaginationProps & GetChatRecordsProps;
+
+export type getPaginationRecordsResponse = PaginationResponse<ChatItemType>;
+
+async function handler(
+  req: ApiRequestProps<getPaginationRecordsBody, getPaginationRecordsQuery>,
+  res: ApiResponseType<any>
+): Promise<getPaginationRecordsResponse> {
+  const { chatId, appId, offset, pageSize = 10, loadCustomFeedbacks, type } = req.body;
+
+  if (!appId || !chatId) {
+    return {
+      list: [],
+      total: 0
+    };
+  }
+
+  const [app] = await Promise.all([
+    MongoApp.findById(appId, 'type').lean(),
+    authChatCrud({
+      req,
+      authToken: true,
+      ...req.body,
+      per: ReadPermissionVal
+    })
+  ]);
+  if (!app) {
+    return Promise.reject(AppErrEnum.unExist);
+  }
+  const isPlugin = app.type === AppTypeEnum.plugin;
+
+  const shareChat = await (async () => {
+    if (type === GetChatTypeEnum.outLink)
+      return await authOutLink({
+        shareId: req.body.shareId,
+        outLinkUid: req.body.outLinkUid
+      }).then((result) => result.shareChat);
+  })();
+
+  const fieldMap = {
+    [GetChatTypeEnum.normal]: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${
+      DispatchNodeResponseKeyEnum.nodeResponse
+    } ${loadCustomFeedbacks ? 'customFeedbacks' : ''}`,
+    [GetChatTypeEnum.outLink]: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${
+      shareChat?.responseDetail || isPlugin ? `${DispatchNodeResponseKeyEnum.nodeResponse}` : ''
+    } `,
+    [GetChatTypeEnum.team]: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
+  };
+
+  const { total, histories } = await getChatItems({
+    appId,
+    chatId,
+    field: fieldMap[type],
+    offset,
+    limit: pageSize
+  });
+
+  // Remove important information
+  if (type === 'outLink' && app.type !== AppTypeEnum.plugin) {
+    histories.forEach((item) => {
+      if (item.obj === ChatRoleEnum.AI) {
+        item.responseData = filterPublicNodeResponseData({ flowResponses: item.responseData });
+      }
+    });
+  }
+
+  return {
+    list: isPlugin ? histories : transformPreviewHistories(histories),
+    total
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/chat/init.ts

@@ -5,15 +5,11 @@ import { getGuideModule, getAppChatConfig } from '@fastgpt/global/core/workflow/
 import { getChatModelNameListByModules } from '@/service/core/app/workflow';
 import type { InitChatProps, InitChatResponse } from '@/global/core/chat/api.d';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
-import { getChatItems } from '@fastgpt/service/core/chat/controller';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
-import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
 import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { transformPreviewHistories } from '@/global/core/chat/utils';
-import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 
 async function handler(
   req: NextApiRequest,
@@ -45,17 +41,8 @@ async function handler(
   }
 
   // get app and history
-  const [{ histories }, { nodes, chatConfig }] = await Promise.all([
-    getChatItems({
-      appId,
-      chatId,
-      limit: 30,
-      field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${
-        DispatchNodeResponseKeyEnum.nodeResponse
-      } ${loadCustomFeedbacks ? 'customFeedbacks' : ''}`
-    }),
-    getAppLatestVersion(app._id, app)
-  ]);
+  const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
+
   const pluginInputs =
     app?.modules?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)?.inputs ?? [];
 
@@ -65,7 +52,6 @@ async function handler(
     title: chat?.title,
     userAvatar: undefined,
     variables: chat?.variables || {},
-    history: app.type === AppTypeEnum.plugin ? histories : transformPreviewHistories(histories),
     app: {
       chatConfig: getAppChatConfig({
         chatConfig,

projects/app/src/pages/api/core/chat/inputGuide/list.ts

@@ -14,10 +14,10 @@ export type ChatInputGuideProps = PaginationProps<{
 export type ChatInputGuideResponse = PaginationResponse<ChatInputGuideSchemaType>;
 
 async function handler(
-  req: ApiRequestProps<{}, ChatInputGuideProps>,
+  req: ApiRequestProps<ChatInputGuideProps>,
   res: NextApiResponse<any>
 ): Promise<ChatInputGuideResponse> {
-  const { appId, pageSize, current, searchKey } = req.query;
+  const { appId, pageSize, offset, searchKey } = req.body;
 
   await authApp({ req, appId, authToken: true, per: ReadPermissionVal });
 
@@ -27,10 +27,7 @@ async function handler(
   };
 
   const [result, total] = await Promise.all([
-    MongoChatInputGuide.find(params)
-      .sort({ _id: -1 })
-      .skip(pageSize * (current - 1))
-      .limit(pageSize),
+    MongoChatInputGuide.find(params).sort({ _id: -1 }).skip(offset).limit(pageSize),
     MongoChatInputGuide.countDocuments(params)
   ]);
 

projects/app/src/pages/api/core/chat/outLink/init.ts

@@ -3,8 +3,6 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import type { InitChatResponse, InitOutLinkChatProps } from '@/global/core/chat/api.d';
 import { getGuideModule, getAppChatConfig } from '@fastgpt/global/core/workflow/utils';
 import { getChatModelNameListByModules } from '@/service/core/app/workflow';
-import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import { getChatItems } from '@fastgpt/service/core/chat/controller';
 import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
 import { authOutLink } from '@/service/support/permission/auth/outLink';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
@@ -13,8 +11,6 @@ import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
 import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
-import { transformPreviewHistories } from '@/global/core/chat/utils';
 import { NextAPI } from '@/service/middleware/entry';
 
 async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -39,19 +35,8 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
     throw new Error(ChatErrEnum.unAuthChat);
   }
 
-  const [{ histories }, { nodes, chatConfig }] = await Promise.all([
-    getChatItems({
-      appId: app._id,
-      chatId,
-      limit: 30,
-      field: `dataId obj value userGoodFeedback userBadFeedback ${
-        shareChat.responseDetail || app.type === AppTypeEnum.plugin
-          ? `adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
-          : ''
-      } `
-    }),
-    getAppLatestVersion(app._id, app)
-  ]);
+  const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
+  // pick share response field
 
   jsonRes<InitChatResponse>(res, {
     data: {
@@ -61,7 +46,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       //@ts-ignore
       userAvatar: tmb?.userId?.avatar,
       variables: chat?.variables || {},
-      history: app.type === AppTypeEnum.plugin ? histories : transformPreviewHistories(histories),
       app: {
         chatConfig: getAppChatConfig({
           chatConfig,

projects/app/src/pages/api/core/chat/team/init.ts

@@ -2,19 +2,15 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { getGuideModule, getAppChatConfig } from '@fastgpt/global/core/workflow/utils';
 import { getChatModelNameListByModules } from '@/service/core/app/workflow';
-import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import type { InitChatResponse, InitTeamChatProps } from '@/global/core/chat/api.d';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
-import { getChatItems } from '@fastgpt/service/core/chat/controller';
 import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
 import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
 import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
 import { getAppLatestVersion } from '@fastgpt/service/core/app/controller';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
-import { transformPreviewHistories } from '@/global/core/chat/utils';
 import { NextAPI } from '@/service/middleware/entry';
 
 async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -45,15 +41,9 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
   }
 
   // get app and history
-  const [{ histories }, { nodes, chatConfig }] = await Promise.all([
-    getChatItems({
-      appId,
-      chatId,
-      limit: 30,
-      field: `dataId obj value userGoodFeedback userBadFeedback adminFeedback ${DispatchNodeResponseKeyEnum.nodeResponse}`
-    }),
-    getAppLatestVersion(app._id, app)
-  ]);
+  const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
+
+  // pick share response field
 
   jsonRes<InitChatResponse>(res, {
     data: {
@@ -62,7 +52,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       title: chat?.title,
       userAvatar: team?.avatar,
       variables: chat?.variables || {},
-      history: app.type === AppTypeEnum.plugin ? histories : transformPreviewHistories(histories),
       app: {
         chatConfig: getAppChatConfig({
           chatConfig,

projects/app/src/pages/api/core/dataset/collection/scrollList.ts

@@ -22,21 +22,22 @@ export type GetScrollCollectionsProps = PaginationProps<{
 }>;
 
 async function handler(
-  req: ApiRequestProps<{}, GetScrollCollectionsProps>
+  req: ApiRequestProps<GetScrollCollectionsProps, {}>
 ): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
   let {
     datasetId,
     pageSize = 10,
-    current = 1,
+    offset,
     parentId = null,
     searchText = '',
     selectFolder = false,
     filterTags = [],
     simple = false
-  } = req.query;
+  } = req.body;
   if (!datasetId) {
     return Promise.reject(CommonErrEnum.missingParams);
   }
+
   searchText = searchText?.replace(/'/g, '');
   pageSize = Math.min(pageSize, 30);
 
@@ -84,7 +85,7 @@ async function handler(
       .sort({
         updateTime: -1
       })
-      .skip(pageSize * (current - 1))
+      .skip(offset)
       .limit(pageSize)
       .lean();
 
@@ -110,7 +111,7 @@ async function handler(
         $sort: { updateTime: -1 }
       },
       {
-        $skip: (current - 1) * pageSize
+        $skip: offset
       },
       {
         $limit: pageSize

projects/app/src/pages/api/core/dataset/data/list.ts

@@ -3,19 +3,20 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { replaceRegChars } from '@fastgpt/global/common/string/tools';
 import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
-import { PagingData, RequestPaging } from '@/types';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { DatasetDataListItemType } from '@/global/core/dataset/type';
+import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 
-export type GetDatasetDataListProps = RequestPaging & {
+export type GetDatasetDataListProps = PaginationProps & {
   searchText?: string;
   collectionId: string;
 };
+export type GetDatasetDataListRes = PaginationResponse<DatasetDataListItemType>;
 
 async function handler(
   req: ApiRequestProps<GetDatasetDataListProps>
-): Promise<PagingData<DatasetDataListItemType>> {
-  let { pageNum = 1, pageSize = 10, searchText = '', collectionId } = req.body;
+): Promise<GetDatasetDataListRes> {
+  let { offset, pageSize = 10, searchText = '', collectionId } = req.body;
 
   pageSize = Math.min(pageSize, 30);
 
@@ -40,19 +41,17 @@ async function handler(
       : {})
   };
 
-  const [data, total] = await Promise.all([
+  const [list, total] = await Promise.all([
     MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex')
       .sort({ chunkIndex: 1, updateTime: -1 })
-      .skip((pageNum - 1) * pageSize)
+      .skip(offset)
       .limit(pageSize)
       .lean(),
     MongoDatasetData.countDocuments(match)
   ]);
 
   return {
-    pageNum,
-    pageSize,
-    data,
+    list,
     total
   };
 }

projects/app/src/pages/api/v1/chat/completions.ts

@@ -208,6 +208,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       getChatItems({
         appId: app._id,
         chatId,
+        offset: 0,
         limit,
         field: `dataId obj value nodeOutputs`
       }),
@@ -555,6 +556,7 @@ const authHeaderRequest = async ({
       };
     } else {
       // token_auth
+
       if (!appId) {
         return Promise.reject('appId is empty');
       }