This commit is contained in:
Archer
2023-10-17 10:00:32 +08:00
committed by GitHub
parent dd8f2744bf
commit 3b776b6639
98 changed files with 1525 additions and 983 deletions

View File

@@ -12,11 +12,12 @@ export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen:
const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
const stepReg: Record<number, RegExp> = {
0: /(\n)/g,
1: /([。]|\.\s)/g,
2: /([]|!\s|\?\s)/g,
3: /([]|;\s)/g,
4: /([]|,\s)/g
0: /(\n\n)/g,
1: /([\n])/g,
2: /([。]|\.\s)/g,
3: /([]|!\s|\?\s)/g,
4: /([]|;\s)/g,
5: /([]|,\s)/g
};
const splitTextRecursively = ({ text = '', step }: { text: string; step: number }) => {
@@ -43,7 +44,6 @@ export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen:
.filter((part) => part);
let chunks: string[] = [];
let preChunk = '';
let chunk = '';
for (let i = 0; i < splitTexts.length; i++) {