4.7.1-alpha (#1120)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/components/core/module/Flow/FlowProvider.tsx

@@ -214,9 +214,6 @@ export const FlowProvider = ({
         if (source?.flowType === FlowNodeTypeEnum.classifyQuestion && !type) {
           return ModuleIOValueTypeEnum.boolean;
         }
-        if (source?.flowType === FlowNodeTypeEnum.tools) {
-          return ModuleIOValueTypeEnum.tools;
-        }
         if (source?.flowType === FlowNodeTypeEnum.pluginInput) {
           return source?.inputs.find((input) => input.key === connect.sourceHandle)?.valueType;
         }

projects/app/src/components/core/module/Flow/ModuleTemplateList.tsx

@@ -52,6 +52,7 @@ const ModuleTemplateList = ({ isOpen, onClose }: ModuleTemplateListProps) => {
   const router = useRouter();
   const [currentParent, setCurrentParent] = useState<RenderListProps['currentParent']>();
   const [searchKey, setSearchKey] = useState('');
+  const { feConfigs } = useSystemStore();
 
   const {
     basicNodeTemplates,
@@ -64,7 +65,12 @@ const ModuleTemplateList = ({ isOpen, onClose }: ModuleTemplateListProps) => {
 
   const templates = useMemo(() => {
     const map = {
-      [TemplateTypeEnum.basic]: basicNodeTemplates,
+      [TemplateTypeEnum.basic]: basicNodeTemplates.filter((item) => {
+        if (item.flowType === FlowNodeTypeEnum.lafModule && !feConfigs.lafEnv) {
+          return false;
+        }
+        return true;
+      }),
       [TemplateTypeEnum.systemPlugin]: systemNodeTemplates,
       [TemplateTypeEnum.teamPlugin]: teamPluginNodeTemplates.filter((item) =>
         searchKey ? item.pluginType !== PluginTypeEnum.folder : true

projects/app/src/components/core/module/Flow/components/nodes/NodeHttp/index.tsx

@@ -88,7 +88,7 @@ enum TabEnum {
   headers = 'headers',
   body = 'body'
 }
-type PropsArrType = {
+export type PropsArrType = {
   key: string;
   type: string;
   value: string;
@@ -245,7 +245,7 @@ const RenderHttpMethodAndUrl = React.memo(function RenderHttpMethodAndUrl({
   );
 });
 
-function RenderHttpProps({
+export function RenderHttpProps({
   moduleId,
   inputs
 }: {

projects/app/src/components/core/module/Flow/components/nodes/NodeLaf.tsx

@@ -0,0 +1,260 @@
+import React, { useCallback, useMemo } from 'react';
+import { NodeProps } from 'reactflow';
+import NodeCard from '../render/NodeCard';
+import { FlowModuleItemType } from '@fastgpt/global/core/module/type.d';
+import Container from '../modules/Container';
+import { Box, Button, Center, Flex, useDisclosure } from '@chakra-ui/react';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
+import { onChangeNode, useFlowProviderStore } from '../../FlowProvider';
+import { useTranslation } from 'next-i18next';
+import { getLafAppDetail } from '@/web/support/laf/api';
+import MySelect from '@fastgpt/web/components/common/MySelect';
+import { getApiSchemaByUrl } from '@/web/core/plugin/api';
+import { str2OpenApiSchema } from '@fastgpt/global/core/plugin/httpPlugin/utils';
+import { useUserStore } from '@/web/support/user/useUserStore';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { ChevronRightIcon } from '@chakra-ui/icons';
+import { useQuery } from '@tanstack/react-query';
+import dynamic from 'next/dynamic';
+import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant';
+import { useToast } from '@fastgpt/web/hooks/useToast';
+import Divider from '../modules/Divider';
+import RenderToolInput from '../render/RenderToolInput';
+import RenderInput from '../render/RenderInput';
+import RenderOutput from '../render/RenderOutput';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+
+const LafAccountModal = dynamic(() => import('@/components/support/laf/LafAccountModal'));
+
+const NodeLaf = (props: NodeProps<FlowModuleItemType>) => {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  const { feConfigs } = useSystemStore();
+  const { data, selected } = props;
+  const { moduleId, inputs } = data;
+
+  const requestUrl = inputs.find((item) => item.key === ModuleInputKeyEnum.httpReqUrl);
+
+  const { userInfo } = useUserStore();
+
+  const token = userInfo?.team.lafAccount?.token;
+  const appid = userInfo?.team.lafAccount?.appid;
+
+  // not config laf
+  if (!token || !appid) {
+    return (
+      <NodeCard minW={'350px'} selected={selected} {...data}>
+        <ConfigLaf />
+      </NodeCard>
+    );
+  }
+
+  const { data: lafData, isLoading: isLoadingFunctions } = useQuery(
+    ['getLafFunctionList'],
+    async () => {
+      // load laf app detail
+      const appDetail = await getLafAppDetail(appid);
+
+      // load laf app functions
+      const schemaUrl = `https://${appDetail?.domain.domain}/_/api-docs?token=${appDetail?.openapi_token}`;
+
+      const schema = await getApiSchemaByUrl(schemaUrl);
+      const openApiSchema = await str2OpenApiSchema(JSON.stringify(schema));
+      const filterPostSchema = openApiSchema.pathData.filter((item) => item.method === 'post');
+
+      return {
+        lafApp: appDetail,
+        lafFunctions: filterPostSchema.map((item) => ({
+          ...item,
+          requestUrl: `https://${appDetail?.domain.domain}${item.path}`
+        }))
+      };
+    },
+    {
+      onError(err) {
+        toast({
+          status: 'error',
+          title: getErrText(err, '获取Laf函数列表失败')
+        });
+      }
+    }
+  );
+
+  const lafFunctionSelectList = useMemo(
+    () =>
+      lafData?.lafFunctions.map((item) => ({
+        label: item.description ? `${item.name} (${item.description})` : item.name,
+        value: item.requestUrl
+      })) || [],
+    [lafData?.lafFunctions]
+  );
+
+  const selectedFunction = useMemo(
+    () => lafFunctionSelectList.find((item) => item.value === requestUrl?.value)?.value,
+    [lafFunctionSelectList, requestUrl?.value]
+  );
+
+  const onSyncParams = useCallback(() => {
+    const lafFunction = lafData?.lafFunctions.find((item) => item.requestUrl === selectedFunction);
+
+    if (!lafFunction) return;
+
+    const bodyParams =
+      lafFunction?.request?.content?.['application/json']?.schema?.properties || {};
+
+    const requiredParams =
+      lafFunction?.request?.content?.['application/json']?.schema?.required || [];
+
+    const allParams = [
+      ...Object.keys(bodyParams).map((key) => ({
+        name: key,
+        desc: bodyParams[key].description,
+        required: requiredParams?.includes(key) || false,
+        value: `{{${key}}}`,
+        type: 'string'
+      }))
+    ].filter((item) => !inputs.find((input) => input.key === item.name));
+
+    // add params
+    allParams.forEach((param) => {
+      onChangeNode({
+        moduleId,
+        type: 'addInput',
+        key: param.name,
+        value: {
+          key: param.name,
+          valueType: ModuleIOValueTypeEnum.string,
+          label: param.name,
+          type: FlowNodeInputTypeEnum.target,
+          required: param.required,
+          description: param.desc || '',
+          toolDescription: param.desc || '未设置参数描述',
+          edit: true,
+          editField: {
+            key: true,
+            name: true,
+            description: true,
+            required: true,
+            dataType: true,
+            inputType: true,
+            isToolInput: true
+          },
+          connected: false
+        }
+      });
+    });
+
+    toast({
+      status: 'success',
+      title: t('common.Sync success')
+    });
+  }, [inputs, lafData?.lafFunctions, moduleId, selectedFunction, t, toast]);
+
+  return (
+    <NodeCard minW={'350px'} selected={selected} {...data}>
+      <Container>
+        {/* select function */}
+        <MySelect
+          isLoading={isLoadingFunctions}
+          list={lafFunctionSelectList}
+          placeholder={t('core.module.laf.Select laf function')}
+          onchange={(e) => {
+            onChangeNode({
+              moduleId,
+              type: 'updateInput',
+              key: ModuleInputKeyEnum.httpReqUrl,
+              value: {
+                ...requestUrl,
+                value: e
+              }
+            });
+          }}
+          value={selectedFunction}
+        />
+        {/* auto set params and go to edit */}
+        {!!selectedFunction && (
+          <Flex justifyContent={'flex-end'} mt={2} gap={2}>
+            {/* <Button variant={'whiteBase'} size={'sm'} onClick={onSyncParams}>
+              {t('core.module.Laf sync params')}
+            </Button> */}
+            <Button
+              variant={'grayBase'}
+              size={'sm'}
+              onClick={() => {
+                const lafFunction = lafData?.lafFunctions.find(
+                  (item) => item.requestUrl === selectedFunction
+                );
+
+                if (!lafFunction) return;
+                const url = `${feConfigs.lafEnv}/app/${lafData?.lafApp?.appid}/function${lafFunction?.path}`;
+                window.open(url, '_blank');
+              }}
+            >
+              {t('plugin.go to laf')}
+            </Button>
+          </Flex>
+        )}
+      </Container>
+      {!!selectedFunction && <RenderIO {...props} />}
+    </NodeCard>
+  );
+};
+export default React.memo(NodeLaf);
+
+const ConfigLaf = () => {
+  const { t } = useTranslation();
+  const { userInfo } = useUserStore();
+  const { feConfigs } = useSystemStore();
+  const {
+    isOpen: isOpenLafConfig,
+    onOpen: onOpenLafConfig,
+    onClose: onCloseLafConfig
+  } = useDisclosure();
+
+  return !!feConfigs?.lafEnv ? (
+    <Center minH={150}>
+      <Button onClick={onOpenLafConfig} variant={'whitePrimary'}>
+        {t('plugin.Please bind laf accout first')} <ChevronRightIcon />
+      </Button>
+
+      {isOpenLafConfig && feConfigs?.lafEnv && (
+        <LafAccountModal defaultData={userInfo?.team.lafAccount} onClose={onCloseLafConfig} />
+      )}
+    </Center>
+  ) : (
+    <Box>系统未配置Laf环境</Box>
+  );
+};
+
+const RenderIO = ({ data, selected }: NodeProps<FlowModuleItemType>) => {
+  const { t } = useTranslation();
+  const { moduleId, inputs, outputs } = data;
+  const { splitToolInputs, hasToolNode } = useFlowProviderStore();
+  const { commonInputs, toolInputs } = splitToolInputs(inputs, moduleId);
+
+  return (
+    <>
+      {hasToolNode && (
+        <>
+          <Divider text={t('core.module.tool.Tool input')} />
+          <Container>
+            <RenderToolInput moduleId={moduleId} inputs={toolInputs} canEdit />
+          </Container>
+        </>
+      )}
+      <>
+        <Divider text={t('common.Input')} />
+        <Container>
+          <Box mb={3}>自定义Body参数</Box>
+          <RenderInput moduleId={moduleId} flowInputList={commonInputs} />
+        </Container>
+      </>
+      <>
+        <Divider text={t('common.Output')} />
+        <Container>
+          <RenderOutput moduleId={moduleId} flowOutputList={outputs} />
+        </Container>
+      </>
+    </>
+  );
+};

projects/app/src/components/core/module/Flow/components/render/NodeCard.tsx

@@ -164,7 +164,7 @@ const NodeCard = (props: Props) => {
             top={'-20px'}
             right={0}
             transform={'translateX(90%)'}
-            pl={'17px'}
+            pl={'20px'}
             pr={'10px'}
             pb={'20px'}
             pt={'20px'}

projects/app/src/components/core/module/Flow/components/render/RenderToolInput/constants.ts

@@ -1,11 +1,25 @@
 import { FlowNodeInputItemType } from '@fastgpt/global/core/module/node/type';
 import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant';
+import { ModuleIOValueTypeEnum } from '@fastgpt/global/core/module/constants';
 
 export const defaultEditFormData: FlowNodeInputItemType = {
   valueType: 'string',
-  type: FlowNodeInputTypeEnum.hidden,
+  type: FlowNodeInputTypeEnum.target,
   key: '',
   label: '',
   toolDescription: '',
-  required: true
+  required: true,
+  edit: true,
+  editField: {
+    key: true,
+    description: true,
+    dataType: true
+  },
+  defaultEditField: {
+    label: '',
+    key: '',
+    description: '',
+    inputType: FlowNodeInputTypeEnum.target,
+    valueType: ModuleIOValueTypeEnum.string
+  }
 };

projects/app/src/components/core/module/Flow/index.tsx

@@ -44,7 +44,8 @@ const nodeTypes: Record<`${FlowNodeTypeEnum}`, any> = {
   [FlowNodeTypeEnum.tools]: dynamic(() => import('./components/nodes/NodeTools')),
   [FlowNodeTypeEnum.stopTool]: (data: NodeProps<FlowModuleItemType>) => (
     <NodeSimple {...data} minW={'100px'} maxW={'300px'} />
-  )
+  ),
+  [FlowNodeTypeEnum.lafModule]: dynamic(() => import('./components/nodes/NodeLaf'))
 };
 const edgeTypes = {
   [EDGE_TYPE]: ButtonEdge

projects/app/src/components/support/laf/LafAccountModal.tsx

@@ -0,0 +1,185 @@
+import React, { useCallback } from 'react';
+import { ModalBody, Box, Flex, Input, ModalFooter, Button, Link } from '@chakra-ui/react';
+import MyModal from '@fastgpt/web/components/common/MyModal';
+import { useTranslation } from 'next-i18next';
+import { useForm } from 'react-hook-form';
+import { useRequest } from '@fastgpt/web/hooks/useRequest';
+import { useQuery } from '@tanstack/react-query';
+import MySelect from '@fastgpt/web/components/common/MySelect';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { useToast } from '@fastgpt/web/hooks/useToast';
+import { putUpdateTeam } from '@/web/support/user/team/api';
+import { useUserStore } from '@/web/support/user/useUserStore';
+import type { LafAccountType } from '@fastgpt/global/support/user/team/type.d';
+import { postLafPat2Token, getLafApplications } from '@/web/support/laf/api';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+
+const LafAccountModal = ({
+  defaultData = {
+    token: '',
+    appid: ''
+  },
+  onClose
+}: {
+  defaultData?: LafAccountType;
+  onClose: () => void;
+}) => {
+  const { t } = useTranslation();
+  const { register, handleSubmit, setValue, getValues, watch, reset } = useForm({
+    defaultValues: {
+      ...defaultData,
+      pat: ''
+    }
+  });
+
+  const lafToken = watch('token');
+  const pat = watch('pat');
+  const appid = watch('appid');
+
+  const { feConfigs } = useSystemStore();
+  const { toast } = useToast();
+  const { userInfo, initUserInfo } = useUserStore();
+
+  const onResetForm = useCallback(() => {
+    reset({
+      token: '',
+      appid: '',
+      pat: ''
+    });
+  }, [reset]);
+
+  const { mutate: authLafPat, isLoading: isPatLoading } = useRequest({
+    mutationFn: async (pat) => {
+      const token = await postLafPat2Token(pat);
+      setValue('token', token);
+    },
+    errorToast: t('plugin.Invalid Env')
+  });
+
+  const { data: appListData = [] } = useQuery(
+    ['appList', lafToken],
+    () => {
+      return getLafApplications(lafToken);
+    },
+    {
+      enabled: !!lafToken,
+      onSuccess: (data) => {
+        if (!getValues('appid') && data.length > 0) {
+          setValue('appid', data[0].appid);
+        }
+      },
+      onError: (err) => {
+        onResetForm();
+        toast({
+          title: getErrText(err, '获取应用列表失败'),
+          status: 'error'
+        });
+      }
+    }
+  );
+
+  const { mutate: onSubmit, isLoading: isUpdating } = useRequest({
+    mutationFn: async (data: LafAccountType) => {
+      if (!userInfo?.team.teamId) return;
+      return putUpdateTeam({
+        teamId: userInfo?.team.teamId,
+        lafAccount: data
+      });
+    },
+    onSuccess() {
+      initUserInfo();
+      onClose();
+    },
+    successToast: t('common.Update Success'),
+    errorToast: t('common.Update Failed')
+  });
+
+  return (
+    <MyModal isOpen iconSrc="/imgs/module/laf.png" title={t('user.Laf Account Setting')}>
+      <ModalBody>
+        <Box fontSize={'sm'} color={'myGray.500'}>
+          <Box>{t('support.user.Laf account intro')}</Box>
+          <Box textDecoration={'underline'}>
+            <Link href={`https://doc.laf.run/zh/`} isExternal>
+              {t('support.user.Laf account course')}
+            </Link>
+          </Box>
+          <Box>
+            <Link textDecoration={'underline'} href={`${feConfigs.lafEnv}/`} isExternal>
+              {t('support.user.Go laf env')}
+            </Link>
+          </Box>
+        </Box>
+        <Flex alignItems={'center'} mt={5}>
+          <Box flex={'0 0 70px'}>PAT:</Box>
+          {!lafToken ? (
+            <>
+              <Input
+                flex={'1 0 0'}
+                size={'sm'}
+                {...register('pat')}
+                placeholder={t('plugin.Enter PAT')}
+              />
+              <Button
+                ml={2}
+                variant={'whitePrimary'}
+                isDisabled={!pat}
+                onClick={() => {
+                  authLafPat(pat);
+                }}
+                isLoading={isPatLoading}
+              >
+                验证
+              </Button>
+            </>
+          ) : (
+            <Button
+              variant={'whitePrimary'}
+              onClick={() => {
+                onResetForm();
+                putUpdateTeam({
+                  teamId: userInfo?.team.teamId || '',
+                  lafAccount: { token: '', appid: '' }
+                });
+              }}
+            >
+              已验证，点击取消绑定
+            </Button>
+          )}
+        </Flex>
+        {!!lafToken && (
+          <Flex alignItems={'center'} mt={5}>
+            <Box flex={'0 0 70px'}>{t('plugin.Currentapp')}</Box>
+            <MySelect
+              minW={'200px'}
+              list={
+                appListData
+                  .filter((app) => app.state === 'Running')
+                  .map((app) => ({
+                    label: `${app.name}`,
+                    value: app.appid
+                  })) || []
+              }
+              placeholder={t('plugin.App')}
+              value={watch('appid')}
+              onchange={(e) => {
+                setValue('appid', e);
+              }}
+              {...(register('appid'), { required: true })}
+            />
+          </Flex>
+        )}
+      </ModalBody>
+      <ModalFooter>
+        <Button mr={3} variant={'whiteBase'} onClick={onClose}>
+          {t('common.Close')}
+        </Button>
+        <Button isLoading={isUpdating} onClick={handleSubmit((data) => onSubmit(data))}>
+          {t('common.Update')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export default LafAccountModal;

projects/app/src/pages/account/components/Info.tsx

@@ -8,7 +8,8 @@ import {
   Input,
   Link,
   Progress,
-  Grid
+  Grid,
+  Image
 } from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
 import { UserUpdateParams } from '@/types/user';
@@ -41,12 +42,14 @@ import {
 } from '@/web/support/wallet/sub/constants';
 
 import StandardPlanContentList from '@/components/support/wallet/StandardPlanContentList';
+import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 
 const StandDetailModal = dynamic(() => import('./standardDetailModal'));
 const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu'));
 const PayModal = dynamic(() => import('./PayModal'));
 const UpdatePswModal = dynamic(() => import('./UpdatePswModal'));
 const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal'));
+const LafAccountModal = dynamic(() => import('@/components/support/laf/LafAccountModal'));
 const CommunityModal = dynamic(() => import('@/components/CommunityModal'));
 
 const Account = () => {
@@ -518,7 +521,7 @@ const Other = () => {
   const { reset } = useForm<UserUpdateParams>({
     defaultValues: userInfo as UserType
   });
-
+  const { isOpen: isOpenLaf, onClose: onCloseLaf, onOpen: onOpenLaf } = useDisclosure();
   const { isOpen: isOpenOpenai, onClose: onCloseOpenai, onOpen: onOpenOpenai } = useDisclosure();
   const { isOpen: isOpenConcat, onClose: onCloseConcat, onOpen: onOpenConcat } = useDisclosure();
 
@@ -537,7 +540,6 @@ const Other = () => {
     },
     [reset, toast, updateUserInfo]
   );
-
   return (
     <Box>
       <Grid gridGap={4} mt={3}>
@@ -582,6 +584,32 @@ const Other = () => {
           </Box>
         </Link>
 
+        {feConfigs?.lafEnv && userInfo?.team.role === TeamMemberRoleEnum.owner && (
+          <Flex
+            bg={'white'}
+            py={4}
+            px={6}
+            border={theme.borders.sm}
+            borderWidth={'1.5px'}
+            borderRadius={'md'}
+            alignItems={'center'}
+            cursor={'pointer'}
+            userSelect={'none'}
+            onClick={onOpenLaf}
+          >
+            <Image src="/imgs/module/laf.png" w={'18px'} alt="laf" />
+            <Box ml={2} flex={1}>
+              laf 账号
+            </Box>
+            <Box
+              w={'9px'}
+              h={'9px'}
+              borderRadius={'50%'}
+              bg={userInfo?.team.lafAccount?.token ? '#67c13b' : 'myGray.500'}
+            />
+          </Flex>
+        )}
+
         {feConfigs?.show_openai_account && (
           <Flex
             bg={'white'}
@@ -620,6 +648,9 @@ const Other = () => {
         )}
       </Grid>
 
+      {isOpenLaf && userInfo && (
+        <LafAccountModal defaultData={userInfo?.team.lafAccount} onClose={onCloseLaf} />
+      )}
       {isOpenOpenai && userInfo && (
         <OpenAIAccountModal
           defaultData={userInfo?.openaiAccount}

projects/app/src/pages/api/admin/clearInvalidData.ts

@@ -2,12 +2,14 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
 import { addHours } from 'date-fns';
-import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
-import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
 import { MongoImage } from '@fastgpt/service/common/file/image/schema';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import {
+  checkInvalidDatasetFiles,
+  checkInvalidDatasetData,
+  checkInvalidVector
+} from '@/service/common/system/cronTask';
 
 let deleteImageAmount = 0;
 async function checkInvalidImg(start: Date, end: Date, limit = 50) {
@@ -60,11 +62,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     (async () => {
       try {
         console.log('执行脏数据清理任务');
-        const end = addHours(new Date(), -1);
+        // 360天 ~ 2小时前
+        const end = addHours(new Date(), -2);
         const start = addHours(new Date(), -360 * 24);
-        await checkFiles(start, end);
+        await checkInvalidDatasetFiles(start, end);
         await checkInvalidImg(start, end);
-        await checkInvalidCollection(start, end);
+        await checkInvalidDatasetData(start, end);
         await checkInvalidVector(start, end);
         console.log('执行脏数据清理任务完毕');
       } catch (error) {

projects/app/src/pages/api/admin/initv469.ts

@@ -1,98 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { MongoUsage } from '@fastgpt/service/support/wallet/usage/schema';
-import { connectionMongo } from '@fastgpt/service/common/mongo';
-import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
-import { addHours } from 'date-fns';
-import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
-import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
-import { MongoImage } from '@fastgpt/service/common/file/image/schema';
-import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-
-let deleteImageAmount = 0;
-export async function checkInvalidImg(start: Date, end: Date, limit = 50) {
-  const images = await MongoImage.find(
-    {
-      createTime: {
-        $gte: start,
-        $lte: end
-      },
-      'metadata.relatedId': { $exists: true }
-    },
-    '_id teamId metadata'
-  );
-  console.log('total images', images.length);
-  let index = 0;
-
-  for await (const image of images) {
-    try {
-      // 1. 检测是否有对应的集合
-      const collection = await MongoDatasetCollection.findOne(
-        {
-          teamId: image.teamId,
-          'metadata.relatedImgId': image.metadata?.relatedId
-        },
-        '_id'
-      );
-
-      if (!collection) {
-        await image.deleteOne();
-        deleteImageAmount++;
-      }
-
-      index++;
-
-      index % 100 === 0 && console.log(index);
-    } catch (error) {
-      console.log(error);
-    }
-  }
-
-  console.log(`检测完成，共删除 ${deleteImageAmount} 个无效图片`);
-}
-
-/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await connectToDatabase();
-    await authCert({ req, authRoot: true });
-
-    // 检查 usage 是否有记录
-    const totalUsages = await MongoUsage.countDocuments();
-    if (totalUsages === 0) {
-      // 重命名 bills 集合成 usages
-      await connectionMongo.connection.db.renameCollection('bills', 'usages', {
-        // 强制
-        dropTarget: true
-      });
-    }
-
-    (async () => {
-      try {
-        console.log('执行脏数据清理任务');
-        const end = addHours(new Date(), -1);
-        const start = addHours(new Date(), -360 * 24);
-        await checkFiles(start, end);
-        await checkInvalidImg(start, end);
-        await checkInvalidCollection(start, end);
-        await checkInvalidVector(start, end);
-        console.log('执行脏数据清理任务完毕');
-      } catch (error) {
-        console.log('执行脏数据清理任务出错了');
-      }
-    })();
-
-    jsonRes(res, {
-      message: 'success'
-    });
-  } catch (error) {
-    console.log(error);
-
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}

projects/app/src/pages/api/common/file/previewContent.ts

@@ -4,7 +4,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
+import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
 import { authFile } from '@fastgpt/service/support/permission/auth/file';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 
@@ -19,7 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const { teamId } = await authFile({ req, authToken: true, fileId });
 
-    const { rawText } = await readFileContent({
+    const { rawText } = await readFileContentFromMongo({
       teamId,
       bucketName: BucketNameEnum.dataset,
       fileId,

projects/app/src/pages/api/common/system/getInitData.ts

@@ -90,6 +90,7 @@ export async function initSystemConfig() {
   // get config from database
   const config: FastGPTConfigFileType = {
     feConfigs: {
+      ...fileRes?.feConfigs,
       ...defaultFeConfigs,
       ...(dbConfig.feConfigs || {}),
       isPlus: !!FastGPTProUrl

projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
+import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
 import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
@@ -36,7 +36,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
 
     // 1. read file
-    const { rawText, filename } = await readFileContent({
+    const { rawText, filename } = await readFileContentFromMongo({
       teamId,
       bucketName: BucketNameEnum.dataset,
       fileId

projects/app/src/pages/api/core/dataset/collection/create/file.ts

@@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import {
   delFileByFileIdList,
-  readFileContent
+  readFileContentFromMongo
 } from '@fastgpt/service/common/file/gridfs/controller';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
@@ -47,7 +47,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
 
     // 1. read file
-    const { rawText, filename } = await readFileContent({
+    const { rawText, filename } = await readFileContentFromMongo({
       teamId,
       bucketName: BucketNameEnum.dataset,
       fileId

projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { authFile } from '@fastgpt/service/support/permission/auth/file';
 import { PostPreviewFilesChunksProps } from '@/global/core/dataset/api';
-import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
+import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
 import { parseCsvTable2Chunks } from '@fastgpt/service/core/dataset/training/utils';
@@ -28,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         const { file, teamId } = await authFile({ req, authToken: true, fileId: sourceId });
         const fileId = String(file._id);
 
-        const { rawText } = await readFileContent({
+        const { rawText } = await readFileContentFromMongo({
           teamId,
           bucketName: BucketNameEnum.dataset,
           fileId,
@@ -53,7 +53,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       if (type === ImportDataSourceEnum.csvTable) {
         const { file, teamId } = await authFile({ req, authToken: true, fileId: sourceId });
         const fileId = String(file._id);
-        const { rawText } = await readFileContent({
+        const { rawText } = await readFileContentFromMongo({
           teamId,
           bucketName: BucketNameEnum.dataset,
           fileId,

projects/app/src/pages/api/lafApi/[...path].ts

@@ -0,0 +1,69 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { request } from 'https';
+import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
+import url from 'url';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    await connectToDatabase();
+    const { path = [], ...query } = req.query as any;
+
+    const queryStr = new URLSearchParams(query).toString();
+    const requestPath = queryStr
+      ? `/${path?.join('/')}?${new URLSearchParams(query).toString()}`
+      : `/${path?.join('/')}`;
+
+    if (!requestPath) {
+      throw new Error('url is empty');
+    }
+
+    const lafEnv = global.feConfigs?.lafEnv;
+
+    if (!lafEnv) {
+      throw new Error('lafEnv is empty');
+    }
+
+    const parsedUrl = url.parse(lafEnv);
+    delete req.headers?.cookie;
+    delete req.headers?.host;
+    delete req.headers?.origin;
+
+    const requestResult = request({
+      protocol: parsedUrl.protocol,
+      hostname: parsedUrl.hostname,
+      port: parsedUrl.port,
+      path: requestPath,
+      method: req.method,
+      headers: req.headers,
+      timeout: 30000
+    });
+
+    req.pipe(requestResult);
+
+    requestResult.on('response', (response) => {
+      Object.keys(response.headers).forEach((key) => {
+        // @ts-ignore
+        res.setHeader(key, response.headers[key]);
+      });
+      response.statusCode && res.writeHead(response.statusCode);
+      response.pipe(res);
+    });
+    requestResult.on('error', (e) => {
+      res.send(e);
+      res.end();
+    });
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export const config = {
+  api: {
+    bodyParser: false
+  }
+};

projects/app/src/pages/api/support/user/account/update.ts

@@ -11,7 +11,7 @@ import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSc
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams;
+    const { avatar, timezone, openaiAccount, lafAccount } = req.body as UserUpdateParams;
 
     const { tmbId } = await authCert({ req, authToken: true });
     const tmb = await MongoTeamMember.findById(tmbId);
@@ -47,7 +47,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       {
         ...(avatar && { avatar }),
         ...(timezone && { timezone }),
-        openaiAccount: openaiAccount?.key ? openaiAccount : null
+        openaiAccount: openaiAccount?.key ? openaiAccount : null,
+        lafAccount: lafAccount?.token ? lafAccount : null
       }
     );
 

projects/app/src/pages/api/timerTask/dataset/checkInValidDatasetFiles.ts

@@ -1,91 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import {
-  delFileByFileIdList,
-  getGFSCollection
-} from '@fastgpt/service/common/file/gridfs/controller';
-import { addLog } from '@fastgpt/service/common/system/log';
-import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { addHours } from 'date-fns';
-
-/* 
-  check dataset.files data. If there is no match in dataset.collections, delete it
-  可能异常情况
-  1. 上传了文件，未成功创建集合
-*/
-let deleteFileAmount = 0;
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { startHour = 24, endHour = 1 } = req.body as {
-      startHour?: number;
-      endHour?: number;
-      limit?: number;
-    };
-    await authCert({ req, authRoot: true });
-    await connectToDatabase();
-
-    // start: now - maxDay, end: now - 3 day
-    const start = addHours(new Date(), -startHour);
-    const end = addHours(new Date(), -endHour);
-    deleteFileAmount = 0;
-    console.log(start, end);
-
-    await checkFiles(start, end);
-
-    jsonRes(res, {
-      data: deleteFileAmount,
-      message: 'success'
-    });
-  } catch (error) {
-    addLog.error(`check valid dataset files error`, error);
-
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}
-
-export async function checkFiles(start: Date, end: Date) {
-  const collection = getGFSCollection('dataset');
-  const where = {
-    uploadDate: { $gte: start, $lte: end }
-  };
-
-  // 1. get all file _id
-  const files = await collection
-    .find(where, {
-      projection: {
-        metadata: 1,
-        _id: 1
-      }
-    })
-    .toArray();
-  console.log('total files', files.length);
-
-  let index = 0;
-  for await (const file of files) {
-    try {
-      // 2. find fileId in dataset.collections
-      const hasCollection = await MongoDatasetCollection.countDocuments({
-        teamId: file.metadata.teamId,
-        fileId: file._id
-      });
-
-      // 3. if not found, delete file
-      if (hasCollection === 0) {
-        await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(file._id)] });
-        console.log('delete file', file._id);
-        deleteFileAmount++;
-      }
-      index++;
-      index % 100 === 0 && console.log(index);
-    } catch (error) {
-      console.log(error);
-    }
-  }
-  console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
-}

projects/app/src/pages/api/timerTask/dataset/checkInvalidMongoCollection.ts

@@ -1,96 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { addLog } from '@fastgpt/service/common/system/log';
-import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
-import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { addHours } from 'date-fns';
-import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
-
-/* 
-  检测无效的 Mongo 数据
-  异常情况：
-  1. 训练过程删除知识库，可能导致还会有新的数据插入，导致无效。
-*/
-
-let deleteAmount = 0;
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { startHour = 3, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
-    await authCert({ req, authRoot: true });
-    await connectToDatabase();
-
-    // start: now - maxDay, end: now - endHour
-    const start = addHours(new Date(), -startHour);
-    const end = addHours(new Date(), -endHour);
-    deleteAmount = 0;
-
-    await checkInvalidCollection(start, end);
-
-    jsonRes(res, {
-      data: deleteAmount,
-      message: 'success'
-    });
-  } catch (error) {
-    addLog.error(`check Invalid user error`, error);
-
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}
-
-export async function checkInvalidCollection(start: Date, end: Date) {
-  // 1. 获取时间范围的所有data
-  const rows = await MongoDatasetData.find(
-    {
-      updateTime: {
-        $gte: start,
-        $lte: end
-      }
-    },
-    '_id teamId collectionId'
-  ).lean();
-
-  // 2. 合并所有的collectionId
-  const map = new Map<string, { teamId: string; collectionId: string }>();
-  for (const item of rows) {
-    const collectionId = String(item.collectionId);
-    if (!map.has(collectionId)) {
-      map.set(collectionId, { teamId: item.teamId, collectionId });
-    }
-  }
-  const list = Array.from(map.values());
-  console.log('total collections', list.length);
-  let index = 0;
-
-  for await (const item of list) {
-    try {
-      // 3. 查看该collection是否存在，不存在，则删除对应的数据
-      const collection = await MongoDatasetCollection.findOne({ _id: item.collectionId });
-      if (!collection) {
-        const result = await Promise.all([
-          MongoDatasetTraining.deleteMany({
-            teamId: item.teamId,
-            collectionId: item.collectionId
-          }),
-          MongoDatasetData.deleteMany({
-            teamId: item.teamId,
-            collectionId: item.collectionId
-          }),
-          deleteDatasetDataVector({
-            teamId: item.teamId,
-            collectionIds: [String(item.collectionId)]
-          })
-        ]);
-        console.log(result);
-        console.log('collection is not found', item);
-        continue;
-      }
-    } catch (error) {}
-    console.log(++index);
-  }
-}

projects/app/src/pages/api/timerTask/dataset/checkInvalidVector.ts

@@ -1,86 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { addLog } from '@fastgpt/service/common/system/log';
-import {
-  deleteDatasetDataVector,
-  getVectorDataByTime
-} from '@fastgpt/service/common/vectorStore/controller';
-import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { addHours } from 'date-fns';
-
-/* 
-  检测无效的 Vector 数据. 
-  异常情况：
-  1. 插入数据时，vector成功，mongo失败
-  2. 更新数据，也会有插入 vector
-*/
-
-let deletedVectorAmount = 0;
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { startHour = 5, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
-    await authCert({ req, authRoot: true });
-    await connectToDatabase();
-
-    // start: now - maxDay, end: now - endHour
-    const start = addHours(new Date(), -startHour);
-    const end = addHours(new Date(), -endHour);
-    deletedVectorAmount = 0;
-
-    await checkInvalidVector(start, end);
-
-    jsonRes(res, {
-      data: deletedVectorAmount,
-      message: 'success'
-    });
-  } catch (error) {
-    addLog.error(`check Invalid user error`, error);
-
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}
-
-export async function checkInvalidVector(start: Date, end: Date) {
-  // 1. get all vector data
-  const rows = await getVectorDataByTime(start, end);
-  console.log('total data', rows.length);
-
-  let index = 0;
-
-  for await (const item of rows) {
-    if (!item.teamId || !item.datasetId || !item.id) {
-      console.log('error data', item);
-      continue;
-    }
-    try {
-      // 2. find dataset.data
-      const hasData = await MongoDatasetData.countDocuments({
-        teamId: item.teamId,
-        datasetId: item.datasetId,
-        'indexes.dataId': item.id
-      });
-
-      // 3. if not found, delete vector
-      if (hasData === 0) {
-        await deleteDatasetDataVector({
-          teamId: item.teamId,
-          id: item.id
-        });
-        console.log('delete vector data', item.id);
-        deletedVectorAmount++;
-      }
-
-      index++;
-      index % 100 === 0 && console.log(index);
-    } catch (error) {
-      console.log(error);
-    }
-  }
-
-  console.log(`检测完成，共删除 ${deletedVectorAmount} 个无效 向量 数据`);
-}

projects/app/src/pages/dataset/detail/components/Import/components/RenderFiles.tsx

@@ -117,3 +117,5 @@ export const RenderUploadFiles = ({
     </>
   ) : null;
 };
+
+export default RenderUploadFiles;

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -249,7 +249,7 @@ const InputDataModal = ({
                 return openConfirm(onDeleteData)();
               }
               if (e === TabEnum.doc) {
-                return window.open(getDocPath('/docs/course/datasetengine'), '_blank');
+                return window.open(getDocPath('/docs/course/dataset_engine'), '_blank');
               }
               setCurrentTab(e);
             }}

projects/app/src/service/common/system/cron.ts

@@ -1,22 +1,61 @@
 import { setCron } from '@fastgpt/service/common/system/cron';
 import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 import { clearTmpUploadFiles } from '@fastgpt/service/common/file/utils';
+import { checkInvalidDatasetFiles, checkInvalidDatasetData, checkInvalidVector } from './cronTask';
+import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
+import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
+import { addHours } from 'date-fns';
 
-export const startCron = () => {
-  setTrainingQueueCron();
-  setClearTmpUploadFilesCron();
-};
-
-export const setTrainingQueueCron = () => {
+const setTrainingQueueCron = () => {
   setCron('*/1 * * * *', () => {
     startTrainingQueue();
   });
 };
 
-export const setClearTmpUploadFilesCron = () => {
-  clearTmpUploadFiles();
+const setClearTmpUploadFilesCron = () => {
   // Clear tmp upload files every ten minutes
   setCron('*/10 * * * *', () => {
     clearTmpUploadFiles();
   });
 };
+
+const clearInvalidDataCron = () => {
+  setCron('0 */1 * * *', async () => {
+    if (
+      await checkTimerLock({
+        timerId: TimerIdEnum.checkInValidDatasetFiles,
+        lockMinuted: 59
+      })
+    ) {
+      checkInvalidDatasetFiles(addHours(new Date(), 2), addHours(new Date(), 6));
+    }
+  });
+
+  setCron('10 */1 * * *', async () => {
+    if (
+      await checkTimerLock({
+        timerId: TimerIdEnum.checkInvalidDatasetData,
+        lockMinuted: 59
+      })
+    ) {
+      checkInvalidDatasetData(addHours(new Date(), 2), addHours(new Date(), 6));
+    }
+  });
+
+  setCron('30 */1 * * *', async () => {
+    if (
+      await checkTimerLock({
+        timerId: TimerIdEnum.checkInvalidVector,
+        lockMinuted: 59
+      })
+    ) {
+      checkInvalidVector(addHours(new Date(), 2), addHours(new Date(), 6));
+    }
+  });
+};
+
+export const startCron = () => {
+  setTrainingQueueCron();
+  setClearTmpUploadFilesCron();
+  clearInvalidDataCron();
+};

projects/app/src/service/common/system/cronTask.ts

@@ -0,0 +1,157 @@
+import {
+  delFileByFileIdList,
+  getGFSCollection
+} from '@fastgpt/service/common/file/gridfs/controller';
+import { addLog } from '@fastgpt/service/common/system/log';
+import {
+  deleteDatasetDataVector,
+  getVectorDataByTime
+} from '@fastgpt/service/common/vectorStore/controller';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+
+/* 
+  check dataset.files data. If there is no match in dataset.collections, delete it
+  可能异常情况
+  1. 上传了文件，未成功创建集合
+*/
+export async function checkInvalidDatasetFiles(start: Date, end: Date) {
+  let deleteFileAmount = 0;
+  const collection = getGFSCollection('dataset');
+  const where = {
+    uploadDate: { $gte: start, $lte: end }
+  };
+
+  // 1. get all file _id
+  const files = await collection
+    .find(where, {
+      projection: {
+        metadata: 1,
+        _id: 1
+      }
+    })
+    .toArray();
+  addLog.info(`Clear invalid dataset files, total files: ${files.length}`);
+
+  let index = 0;
+  for await (const file of files) {
+    try {
+      // 2. find fileId in dataset.collections
+      const hasCollection = await MongoDatasetCollection.countDocuments({
+        teamId: file.metadata.teamId,
+        fileId: file._id
+      });
+
+      // 3. if not found, delete file
+      if (hasCollection === 0) {
+        await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(file._id)] });
+        console.log('delete file', file._id);
+        deleteFileAmount++;
+      }
+      index++;
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+  addLog.info(`Clear invalid dataset files finish, remove ${deleteFileAmount} files`);
+}
+
+/* 
+  检测无效的 Mongo 数据
+  异常情况：
+  1. 训练过程删除知识库，可能导致还会有新的数据继续插入，导致无效。
+*/
+export async function checkInvalidDatasetData(start: Date, end: Date) {
+  // 1. 获取时间范围的所有data
+  const rows = await MongoDatasetData.find(
+    {
+      updateTime: {
+        $gte: start,
+        $lte: end
+      }
+    },
+    '_id teamId collectionId'
+  ).lean();
+
+  // 2. 合并所有的collectionId
+  const map = new Map<string, { teamId: string; collectionId: string }>();
+  for (const item of rows) {
+    const collectionId = String(item.collectionId);
+    if (!map.has(collectionId)) {
+      map.set(collectionId, { teamId: item.teamId, collectionId });
+    }
+  }
+  const list = Array.from(map.values());
+  addLog.info(`Clear invalid dataset data, total collections: ${list.length}`);
+  let index = 0;
+
+  for await (const item of list) {
+    try {
+      // 3. 查看该collection是否存在，不存在，则删除对应的数据
+      const collection = await MongoDatasetCollection.findOne({ _id: item.collectionId });
+      if (!collection) {
+        const result = await Promise.all([
+          MongoDatasetTraining.deleteMany({
+            teamId: item.teamId,
+            collectionId: item.collectionId
+          }),
+          MongoDatasetData.deleteMany({
+            teamId: item.teamId,
+            collectionId: item.collectionId
+          }),
+          deleteDatasetDataVector({
+            teamId: item.teamId,
+            collectionIds: [String(item.collectionId)]
+          })
+        ]);
+        console.log(result);
+        console.log('collection is not found', item);
+        continue;
+      }
+    } catch (error) {}
+    console.log(++index);
+  }
+}
+
+export async function checkInvalidVector(start: Date, end: Date) {
+  let deletedVectorAmount = 0;
+  // 1. get all vector data
+  const rows = await getVectorDataByTime(start, end);
+  addLog.info(`Clear invalid vector, total vector data: ${rows.length}`);
+
+  let index = 0;
+
+  for await (const item of rows) {
+    if (!item.teamId || !item.datasetId || !item.id) {
+      addLog.error('error data', item);
+      continue;
+    }
+    try {
+      // 2. find dataset.data
+      const hasData = await MongoDatasetData.countDocuments({
+        teamId: item.teamId,
+        datasetId: item.datasetId,
+        'indexes.dataId': item.id
+      });
+
+      // 3. if not found, delete vector
+      if (hasData === 0) {
+        await deleteDatasetDataVector({
+          teamId: item.teamId,
+          id: item.id
+        });
+        console.log('delete vector data', item.id);
+        deletedVectorAmount++;
+      }
+
+      index++;
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  addLog.info(`Clear invalid vector finish, remove ${deletedVectorAmount} data`);
+}

projects/app/src/service/core/dataset/data/controller.ts

@@ -44,8 +44,7 @@ export async function insertData2Dataset({
   indexes =
     Array.isArray(indexes) && indexes.length > 0
       ? indexes.map((index) => ({
-          // @ts-ignore
-          ...index.toObject(),
+          text: index.text,
           dataId: undefined,
           defaultIndex: index.text.trim() === qaStr
         }))

projects/app/src/types/user.d.ts

@@ -1,9 +1,11 @@
 import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
 import type { UserModelSchema } from '@fastgpt/global/support/user/type';
+import { LafAccountType } from '@fastgpt/global/support/user/team/type.d';
 
 export interface UserUpdateParams {
   balance?: number;
   avatar?: string;
   timezone?: string;
   openaiAccount?: UserModelSchema['openaiAccount'];
+  lafAccount?: LafAccountType;
 }

projects/app/src/web/common/api/lafRequest.ts

@@ -0,0 +1,169 @@
+import axios, {
+  Method,
+  InternalAxiosRequestConfig,
+  AxiosResponse,
+  AxiosProgressEvent
+} from 'axios';
+import { useUserStore } from '@/web/support/user/useUserStore';
+
+interface ConfigType {
+  headers?: { [key: string]: string };
+  timeout?: number;
+  onUploadProgress?: (progressEvent: AxiosProgressEvent) => void;
+  cancelToken?: AbortController;
+  maxQuantity?: number;
+}
+interface ResponseDataType {
+  error: string | null;
+  data: any;
+}
+
+const maxQuantityMap: Record<
+  string,
+  {
+    amount: number;
+    sign: AbortController;
+  }
+> = {};
+
+function requestStart({ url, maxQuantity }: { url: string; maxQuantity?: number }) {
+  if (!maxQuantity) return;
+  const item = maxQuantityMap[url];
+
+  if (item) {
+    if (item.amount >= maxQuantity && item.sign) {
+      item.sign.abort();
+      delete maxQuantityMap[url];
+    }
+  } else {
+    maxQuantityMap[url] = {
+      amount: 1,
+      sign: new AbortController()
+    };
+  }
+}
+function requestFinish({ url }: { url: string }) {
+  const item = maxQuantityMap[url];
+  if (item) {
+    item.amount--;
+    if (item.amount <= 0) {
+      delete maxQuantityMap[url];
+    }
+  }
+}
+
+/**
+ * 请求开始
+ */
+function startInterceptors(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
+  if (config.headers && !config.headers.Authorization) {
+    config.headers.Authorization = `Bearer ${useUserStore.getState().userInfo?.team?.lafAccount?.token || ''}`;
+  }
+
+  return config;
+}
+
+/**
+ * 请求成功,检查请求头
+ */
+function responseSuccess(response: AxiosResponse<ResponseDataType>) {
+  return response;
+}
+/**
+ * 响应数据检查
+ */
+function checkRes(data: ResponseDataType) {
+  if (data === undefined) {
+    console.log('error->', data, 'data is empty');
+    return Promise.reject('服务器异常');
+  } else if (data.error) {
+    return responseError(data.error);
+  }
+
+  return data.data;
+}
+
+/**
+ * 响应错误
+ */
+function responseError(err: any) {
+  console.log('error->', '请求错误', err);
+
+  if (!err) {
+    return Promise.reject({ message: '未知错误' });
+  }
+  if (typeof err === 'string') {
+    return Promise.reject({ message: err });
+  }
+
+  if (err?.response?.data) {
+    return Promise.reject(err?.response?.data);
+  }
+  return Promise.reject(err);
+}
+
+/* 创建请求实例 */
+const instance = axios.create({
+  timeout: 60000, // 超时时间
+  headers: {
+    'content-type': 'application/json'
+  }
+});
+
+/* 请求拦截 */
+instance.interceptors.request.use(startInterceptors, (err) => Promise.reject(err));
+/* 响应拦截 */
+instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
+
+function request(
+  url: string,
+  data: any,
+  { cancelToken, maxQuantity, ...config }: ConfigType,
+  method: Method
+): any {
+  /* 去空 */
+  for (const key in data) {
+    if (data[key] === null || data[key] === undefined) {
+      delete data[key];
+    }
+  }
+
+  requestStart({ url, maxQuantity });
+
+  return instance
+    .request({
+      baseURL: '/api/lafApi',
+      url,
+      method,
+      data: ['POST', 'PUT'].includes(method) ? data : null,
+      params: !['POST', 'PUT'].includes(method) ? data : null,
+      signal: cancelToken?.signal,
+      ...config // 用户自定义配置，可以覆盖前面的配置
+    })
+    .then((res) => checkRes(res.data))
+    .catch((err) => responseError(err))
+    .finally(() => requestFinish({ url }));
+}
+
+/**
+ * api请求方式
+ * @param {String} url
+ * @param {Any} params
+ * @param {Object} config
+ * @returns
+ */
+export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, params, config, 'GET');
+}
+
+export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'POST');
+}
+
+export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'PUT');
+}
+
+export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'DELETE');
+}

projects/app/src/web/support/laf/api.ts

@@ -0,0 +1,34 @@
+import { GET, POST, PUT } from '@/web/common/api/lafRequest';
+
+export const postLafPat2Token = (pat: string) => POST<string>(`/v1/auth/pat2token`, { pat });
+
+export const getLafApplications = (token: string) =>
+  GET<
+    {
+      appid: string;
+      name: string;
+      state: 'Running' | 'Failed' | 'Stopped';
+    }[]
+  >(
+    `/v1/applications`,
+    {},
+    {
+      headers: {
+        Authorization: `Bearer ${token}`
+      }
+    }
+  );
+
+export const getLafAppDetail = (appid: string) =>
+  GET<{
+    appid: string;
+    name: string;
+    openapi_token: string;
+    domain: {
+      _id: string;
+      appid: string;
+      domain: string;
+      state: string;
+      phase: string;
+    };
+  }>(`/v1/applications/${appid}`);