4.6.3-website dataset (#532)

This commit is contained in:
Archer
2023-12-03 20:45:57 +08:00
committed by GitHub
parent b916183848
commit a9ae270335
122 changed files with 3793 additions and 1360 deletions

View File

@@ -1,6 +1,7 @@
import mammoth from 'mammoth';
import Papa from 'papaparse';
import { compressBase64ImgAndUpload } from './controller';
import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
/**
* 读取 txt 文件内容
@@ -182,9 +183,9 @@ export const formatMarkdown = async (rawText: string = '') => {
try {
const str = await compressBase64ImgAndUpload({
base64,
maxW: 800,
maxH: 800,
maxSize: 1024 * 1024 * 2
maxW: 4329,
maxH: 4329,
maxSize: 1024 * 1024 * 5
});
rawText = rawText.replace(base64, str);
} catch (error) {
@@ -199,14 +200,7 @@ export const formatMarkdown = async (rawText: string = '') => {
rawText = rawText.replace(/\s*(!\[.*\]\(.*\))\s*/g, '$1');
}
// replace \
const reg1 = /\\([-.!`_(){}\[\]])/g;
if (reg1.test(rawText)) {
rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
}
rawText = rawText.replace(/\\\\n/g, '\\n');
return rawText;
return simpleMarkdownText(rawText);
};
/**