gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add env to check internal ip (#4187)

Browse Source 
* fix: ts

* update jieba package

* add env to check internal ip

* package

* fix: jieba

* reset package

* update config

* fix: jieba package

* init shell

* init version

* change team reload
This commit is contained in:
Archer
2025-03-17 18:21:27 +08:00
committed by archer
parent 9d43edb75c
commit 02813f3a47
17 changed files with 121 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/service/common/string/jieba/dict.json Normal file
View File

File diff suppressed because one or more lines are too long

18
packages/service/common/string/jieba.ts → packages/service/common/string/jieba/index.ts
View File

@@ -1,13 +1,13 @@
import { Jieba } from '@node-rs/jieba';
import fs from 'fs';
import path from 'path';
// 使用 require.resolve 获取包的路径，然后拼接字典文件路径
const jiebaPath = path.dirname(require.resolve('@node-rs/jieba/package.json'));
const dictPath = path.join(jiebaPath, 'dict.txt');
let jieba: Jieba | undefined;
// 使用正确的文件路径加载字典
const jieba = Jieba.withDict(fs.readFileSync(dictPath));
(async () => {
const dictData = await import('./dict.json');
// @ts-ignore
const dictBuffer = Buffer.from(dictData.dict?.replace(/\\n/g, '\n'), 'utf-8');
jieba = Jieba.withDict(dictBuffer);
})();
const stopWords = new Set([
'--',
@@ -1519,7 +1519,9 @@ const stopWords = new Set([
]);
export async function jiebaSplit({ text }: { text: string }) {
const tokens = (await jieba.cutAsync(text, true)) as string[];
text = text.replace(/[#*`_~>[\](){}|]/g, '').replace(/\S*https?\S*/gi, '');
const tokens = (await jieba!.cutAsync(text, true)) as string[];
return (
tokens