add env to check internal ip (#4187)

* fix: ts

* update jieba package

* add env to check internal ip

* package

* fix: jieba

* reset package

* update config

* fix: jieba package

* init shell

* init version

* change team reload

projects/app/.env.template

@@ -52,6 +52,8 @@ USE_IP_LIMIT=false
 WORKFLOW_MAX_RUN_TIMES=500
 # 循环最大运行次数，避免极端的死循环情况
 WORKFLOW_MAX_LOOP_TIMES=50
+# 启用内网 IP 检查
+CHECK_INTERNAL_IP=false
 
 # 对话日志推送服务
 # # 日志服务地址

projects/app/next.config.js

@@ -84,7 +84,8 @@ const nextConfig = {
       'mongoose',
       'pg',
       '@zilliz/milvus2-sdk-node',
-      "tiktoken"
+      "tiktoken",
+      "@node-rs/jieba"
     ],
     outputFileTracingRoot: path.join(__dirname, '../../'),
     instrumentationHook: true

projects/app/src/pageComponents/account/TeamSelector.tsx

@@ -1,14 +1,12 @@
 import React, { useMemo } from 'react';
-import { Box, ButtonProps, Flex } from '@chakra-ui/react';
+import { Box, ButtonProps } from '@chakra-ui/react';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { useTranslation } from 'next-i18next';
-import Avatar from '@fastgpt/web/components/common/Avatar';
 import { getTeamList, putSwitchTeam } from '@/web/support/user/team/api';
 import { TeamMemberStatusEnum } from '@fastgpt/global/support/user/team/constant';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import MySelect from '@fastgpt/web/components/common/MySelect';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useRouter } from 'next/router';
 
 const TeamSelector = ({
@@ -21,7 +19,7 @@ const TeamSelector = ({
 }) => {
   const { t } = useTranslation();
   const router = useRouter();
-  const { userInfo, initUserInfo } = useUserStore();
+  const { userInfo } = useUserStore();
   const { setLoading } = useSystemStore();
 
   const { data: myTeams = [] } = useRequest2(() => getTeamList(TeamMemberStatusEnum.active), {
@@ -33,12 +31,11 @@ const TeamSelector = ({
     async (teamId: string) => {
       setLoading(true);
       await putSwitchTeam(teamId);
-      return initUserInfo();
     },
     {
       onFinally: () => {
+        router.reload();
         setLoading(false);
-        onChange?.();
       },
       errorToast: t('common:user.team.Switch Team Failed')
     }

projects/app/src/pages/account/info/index.tsx

@@ -277,7 +277,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
           <Flex mt={6} alignItems={'center'}>
             <Box {...labelStyles}>{t('account_info:user_team_team_name')}:&nbsp;</Box>
             <Flex flex={'1 0 0'} w={0} align={'center'}>
-              <TeamSelector height={'28px'} w={'100%'} showManage onChange={initUserInfo} />
+              <TeamSelector height={'28px'} w={'100%'} showManage />
             </Flex>
           </Flex>
         )}

projects/app/src/pages/account/team/index.tsx

@@ -48,10 +48,7 @@ const Team = () => {
   const { t } = useTranslation();
   const { userInfo } = useUserStore();
 
-  const { setEditTeamData, isLoading, teamSize, refetchMembers } = useContextSelector(
-    TeamContext,
-    (v) => v
-  );
+  const { setEditTeamData, isLoading, teamSize } = useContextSelector(TeamContext, (v) => v);
 
   const Tabs = useMemo(
     () => (

projects/app/src/pages/api/admin/initv4818.ts

@@ -1,7 +1,7 @@
 import { NextAPI } from '@/service/middleware/entry';
 import { delay } from '@fastgpt/global/common/system/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
-import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
+import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
 import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';

projects/app/src/pages/api/admin/initv491.ts

@@ -0,0 +1,64 @@
+import { NextAPI } from '@/service/middleware/entry';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { NextApiRequest, NextApiResponse } from 'next';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { delay } from '@fastgpt/global/common/system/utils';
+import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
+import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+
+const updateData = async () => {
+  let success = 0;
+  while (true) {
+    try {
+      const data = await MongoDatasetData.find({ initJieba: { $exists: false } }).limit(100);
+      if (data.length === 0) {
+        console.log('更新分词完成');
+        break;
+      }
+
+      await Promise.allSettled(
+        data.map(async (item) => {
+          const text = `${item.q} ${item.a}`.trim();
+
+          try {
+            await mongoSessionRun(async (session) => {
+              await MongoDatasetDataText.updateOne(
+                {
+                  dataId: item._id
+                },
+                {
+                  fullTextToken: await jiebaSplit({ text })
+                },
+                {
+                  session
+                }
+              );
+              // @ts-ignore
+              item.initJieba = true;
+              await item.save({ session });
+            });
+          } catch (error) {
+            console.log(error);
+          }
+        })
+      );
+      success += data.length;
+      console.log(`成功 ${success}`);
+    } catch (error) {
+      addLog.error('更新所有旧的 jieba 分词失败', error);
+      await delay(1000);
+    }
+  }
+};
+
+async function handler(req: NextApiRequest, _res: NextApiResponse) {
+  await authCert({ req, authRoot: true });
+
+  console.log('更新所有旧的 jieba 分词');
+  updateData();
+  return { success: true };
+}
+
+export default NextAPI(handler);

projects/app/src/service/core/dataset/data/controller.ts

@@ -6,7 +6,7 @@ import {
 } from '@fastgpt/global/core/dataset/controller';
 import { insertDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
 import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
-import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
+import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
 import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
 import { DatasetDataIndexItemType, DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
 import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';