perf: async read file (#3531)

docSite/content/zh-cn/docs/development/upgrading/4818.md

@@ -15,4 +15,5 @@ weight: 806
 4. 优化 - Mongo 全文索引表分离。
 5. 优化 - 知识库检索查询语句合并，同时减少查库数量。
 6. 优化 - 文件编码检测，减少 CSV 文件乱码概率。
-7. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。
+7. 优化 - 异步读取文件内容，减少进程阻塞。
+8. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。

packages/plugins/register.ts

@@ -35,24 +35,26 @@ export const list = [...staticPluginList, ...packagePluginList];
 
 /* Get plugins */
 export const getCommunityPlugins = () => {
-  return list.map<SystemPluginTemplateItemType>((name) => {
-    const config = require(`./src/${name}/template.json`);
+  return Promise.all(
+    list.map<Promise<SystemPluginTemplateItemType>>(async (name) => {
+      const config = (await import(`./src/${name}/template.json`))?.default;
 
-    const isFolder = list.find((item) => item.startsWith(`${name}/`));
+      const isFolder = list.find((item) => item.startsWith(`${name}/`));
 
-    const parentIdList = name.split('/').slice(0, -1);
-    const parentId =
-      parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null;
+      const parentIdList = name.split('/').slice(0, -1);
+      const parentId =
+        parentIdList.length > 0 ? `${PluginSourceEnum.community}-${parentIdList.join('/')}` : null;
 
-    return {
-      ...config,
-      id: `${PluginSourceEnum.community}-${name}`,
-      isFolder,
-      parentId,
-      isActive: true,
-      isOfficial: true
-    };
-  });
+      return {
+        ...config,
+        id: `${PluginSourceEnum.community}-${name}`,
+        isFolder,
+        parentId,
+        isActive: true,
+        isOfficial: true
+      };
+    })
+  );
 };
 
 export const getSystemPluginTemplates = () => {

packages/service/common/file/read/utils.ts

@@ -7,7 +7,6 @@ import type { ReadFileResponse } from '../../../worker/readFile/type';
 import axios from 'axios';
 import { addLog } from '../../system/log';
 import { batchRun } from '@fastgpt/global/common/fn/utils';
-import { addHours } from 'date-fns';
 import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
 
 export type readRawTextByLocalFileParams = {
@@ -21,7 +20,7 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
 
   const extension = path?.split('.')?.pop()?.toLowerCase() || '';
 
-  const buffer = fs.readFileSync(path);
+  const buffer = await fs.promises.readFile(path);
 
   const { rawText } = await readRawContentByFileBuffer({
     extension,

packages/service/worker/readFile/parseOffice.ts

@@ -44,13 +44,15 @@ const parsePowerPoint = async ({
   }
 
   // Returning an array of all the xml contents read using fs.readFileSync
-  const xmlContentArray = files.map((file) => {
-    try {
-      return fs.readFileSync(`${decompressPath}/${file.path}`, encoding);
-    } catch (err) {
-      return fs.readFileSync(`${decompressPath}/${file.path}`, 'utf-8');
-    }
-  });
+  const xmlContentArray = await Promise.all(
+    files.map((file) => {
+      try {
+        return fs.promises.readFile(`${decompressPath}/${file.path}`, encoding);
+      } catch (err) {
+        return fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8');
+      }
+    })
+  );
 
   let responseArr: string[] = [];
 

packages/templates/register.ts

@@ -12,24 +12,24 @@ const getTemplateNameList = () => {
   return fs.readdirSync(templatesPath) as string[];
 };
 
-const getFileTemplates = (): AppTemplateSchemaType[] => {
+const getFileTemplates = async (): Promise<AppTemplateSchemaType[]> => {
   const templateNames = getTemplateNameList();
 
-  const appMarketTemplates = templateNames.map((name) => {
-    const fileContent = require(`./src/${name}/template.json`);
+  return Promise.all(
+    templateNames.map<Promise<AppTemplateSchemaType>>(async (name) => {
+      const fileContent = (await import(`./src/${name}/template.json`))?.default;
 
-    return {
-      ...fileContent,
-      templateId: `${PluginSourceEnum.community}-${name}`,
-      isActive: true
-    };
-  });
-
-  return appMarketTemplates;
+      return {
+        ...fileContent,
+        templateId: `${PluginSourceEnum.community}-${name}`,
+        isActive: true
+      };
+    })
+  );
 };
 
 const getAppTemplates = async () => {
-  const communityTemplates = getFileTemplates();
+  const communityTemplates = await getFileTemplates();
 
   const dbTemplates = await MongoAppTemplate.find();
 

projects/app/src/service/common/system/index.ts

@@ -1,5 +1,5 @@
 import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
-import { existsSync, readdirSync, readFileSync } from 'fs';
+import fs, { existsSync, readdirSync } from 'fs';
 import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';
 import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
 import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
@@ -13,7 +13,7 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c
 import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
 import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema';
 
-export const readConfigData = (name: string) => {
+export const readConfigData = async (name: string) => {
   const splitName = name.split('.');
   const devName = `${splitName[0]}.local.${splitName[1]}`;
 
@@ -30,7 +30,7 @@ export const readConfigData = (name: string) => {
     return `/app/data/${name}`;
   })();
 
-  const content = readFileSync(filename, 'utf-8');
+  const content = await fs.promises.readFile(filename, 'utf-8');
 
   return content;
 };
@@ -120,13 +120,13 @@ export async function initSystemConfig() {
   });
 }
 
-function getSystemVersion() {
+async function getSystemVersion() {
   if (global.systemVersion) return;
   try {
     if (process.env.NODE_ENV === 'development') {
       global.systemVersion = process.env.npm_package_version || '0.0.0';
     } else {
-      const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8'));
+      const packageJson = json5.parse(await fs.promises.readFile('/app/package.json', 'utf-8'));
 
       global.systemVersion = packageJson?.version;
     }
@@ -138,7 +138,7 @@ function getSystemVersion() {
   }
 }
 
-function getSystemPlugin() {
+async function getSystemPlugin() {
   if (global.communityPlugins && global.communityPlugins.length > 0) return;
 
   const basePath =
@@ -149,15 +149,17 @@ function getSystemPlugin() {
   const filterFiles = files.filter((item) => item.endsWith('.json'));
 
   // read json file
-  const fileTemplates = filterFiles.map<SystemPluginTemplateItemType>((filename) => {
-    const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
-    return {
-      ...json5.parse(content),
-      originCost: 0,
-      currentCost: 0,
-      id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
-    };
-  });
+  const fileTemplates = await Promise.all(
+    filterFiles.map<Promise<SystemPluginTemplateItemType>>(async (filename) => {
+      const content = await fs.promises.readFile(`${basePath}/${filename}`, 'utf-8');
+      return {
+        ...json5.parse(content),
+        originCost: 0,
+        currentCost: 0,
+        id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
+      };
+    })
+  );
 
   fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0));
 

projects/app/src/service/core/app/plugin.ts

@@ -21,7 +21,9 @@ export const getSystemPlugins = async (refresh = false) => {
       global.systemPlugins = [];
     }
 
-    global.systemPlugins = FastGPTProUrl ? await getCommercialPlugins() : getCommunityPlugins();
+    global.systemPlugins = FastGPTProUrl
+      ? await getCommercialPlugins()
+      : await getCommunityPlugins();
 
     addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);