gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize the project structure and introduce DDD design (#394)

Browse Source
This commit is contained in:
Archer
2023-10-12 17:46:37 +08:00
committed by GitHub
parent 76ac5238b6
commit ad7a17bf40
193 changed files with 1169 additions and 1084 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
projects/app/src/utils/app.ts
View File

@@ -1,5 +1,5 @@
import type { AppModuleItemType, VariableItemType } from '@/types/app';
import { chatModelList } from '@/store/static';
import { chatModelList } from '@/web/common/store/static';
import {
FlowInputItemTypeEnum,
FlowModuleTypeEnum,
@@ -203,7 +203,7 @@ const chatModelInput = (formData: EditFormType): FlowInputItemType[] => [
key: 'switch',
type: 'target',
label: '触发器',
connected: formData.kb.list.length > 0
connected: formData.kb.list.length > 0 && !!formData.kb.searchEmptyText
},
{
key: 'quoteQA',
@@ -466,21 +466,18 @@ const kbTemplate = (formData: EditFormType): AppModuleItemType[] => [
key: 'switch'
}
]
: [
: []
},
{
key: 'unEmpty',
targets: formData.kb.searchEmptyText
? [
{
moduleId: 'chatModule',
key: 'switch'
}
]
},
{
key: 'unEmpty',
targets: [
{
moduleId: 'chatModule',
key: 'switch'
}
]
: []
},
{
key: 'quoteQA',

18
projects/app/src/utils/plugin/eventbus.ts
View File

@@ -1,18 +0,0 @@
export enum EventNameEnum {
guideClick = 'guideClick'
}
type EventNameType = `${EventNameEnum}`;
export const event = {
list: new Map<EventNameType, Function>(),
on: function (name: EventNameType, fn: Function) {
this.list.set(name, fn);
},
emit: function (name: EventNameType, data: Record<string, any> = {}) {
const fn = this.list.get(name);
fn && fn(data);
},
off: function (name: EventNameType) {
this.list.delete(name);
}
};

2
projects/app/src/utils/user.ts
View File

@@ -1,4 +1,4 @@
import { loginOut } from '@/api/user';
import { loginOut } from '@/web/support/api/user';
import timezones from 'timezones-list';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';

53
projects/app/src/utils/web/core/dataset.ts
View File

@@ -1,53 +0,0 @@
import { postCreateTrainingBill } from '@/api/common/bill';
import { postChunks2Dataset } from '@/api/core/dataset/data';
import { TrainingModeEnum } from '@/constants/plugin';
import type { DatasetDataItemType } from '@/types/core/dataset/data';
import { delay } from '@/utils/tools';
export async function chunksUpload({
kbId,
mode,
chunks,
prompt,
rate = 150,
onUploading
}: {
kbId: string;
mode: `${TrainingModeEnum}`;
chunks: DatasetDataItemType[];
prompt?: string;
rate?: number;
onUploading?: (insertLen: number, total: number) => void;
}) {
// create training bill
const billId = await postCreateTrainingBill({ name: 'dataset.Training Name' });
async function upload(data: DatasetDataItemType[]) {
return postChunks2Dataset({
kbId,
data,
mode,
prompt,
billId
});
}
let successInsert = 0;
let retryTimes = 10;
for (let i = 0; i < chunks.length; i += rate) {
try {
const { insertLen } = await upload(chunks.slice(i, i + rate));
onUploading && onUploading(i + rate, chunks.length);
successInsert += insertLen;
} catch (error) {
if (retryTimes === 0) {
return Promise.reject(error);
}
await delay(1000);
retryTimes--;
i -= rate;
}
}
return { insertLen: successInsert };
}

261
projects/app/src/utils/web/file.ts
View File

@@ -1,261 +0,0 @@
import mammoth from 'mammoth';
import Papa from 'papaparse';
import { uploadImg, postUploadFiles, getFileViewUrl } from '@/api/support/file';
/**
* upload file to mongo gridfs
*/
export const uploadFiles = (
files: File[],
metadata: Record<string, any> = {},
percentListen?: (percent: number) => void
) => {
const form = new FormData();
form.append('metadata', JSON.stringify(metadata));
files.forEach((file) => {
form.append('file', file, encodeURIComponent(file.name));
});
return postUploadFiles(form, (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
percentListen && percentListen(percent);
});
};
/**
* 读取 txt 文件内容
*/
export const readTxtContent = (file: File) => {
return new Promise((resolve: (_: string) => void, reject) => {
try {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result as string);
};
reader.onerror = (err) => {
console.log('error txt read:', err);
reject('读取 txt 文件失败');
};
reader.readAsText(file);
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
};
/**
* 读取 pdf 内容
*/
export const readPdfContent = (file: File) =>
new Promise<string>((resolve, reject) => {
try {
const pdfjsLib = window['pdfjs-dist/build/pdf'];
pdfjsLib.workerSrc = '/js/pdf.worker.js';
const readPDFPage = async (doc: any, pageNo: number) => {
const page = await doc.getPage(pageNo);
const tokenizedText = await page.getTextContent();
const pageText = tokenizedText.items
.map((token: any) => token.str)
.filter((item: string) => item)
.join('');
return pageText;
};
let reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = async (event) => {
if (!event?.target?.result) return reject('解析 PDF 失败');
try {
const doc = await pdfjsLib.getDocument(event.target.result).promise;
const pageTextPromises = [];
for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
pageTextPromises.push(readPDFPage(doc, pageNo));
}
const pageTexts = await Promise.all(pageTextPromises);
resolve(pageTexts.join('\n'));
} catch (err) {
console.log(err, 'pdf load error');
reject('解析 PDF 失败');
}
};
reader.onerror = (err) => {
console.log(err, 'pdf load error');
reject('解析 PDF 失败');
};
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
/**
* 读取doc
*/
export const readDocContent = (file: File) =>
new Promise<string>((resolve, reject) => {
try {
const reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = async ({ target }) => {
if (!target?.result) return reject('读取 doc 文件失败');
try {
const res = await mammoth.extractRawText({
arrayBuffer: target.result as ArrayBuffer
});
resolve(res?.value);
} catch (error) {
window.umami?.track('wordReadError', {
err: error?.toString()
});
console.log('error doc read:', error);
reject('读取 doc 文件失败, 请转换成 PDF');
}
};
reader.onerror = (err) => {
window.umami?.track('wordReadError', {
err: err?.toString()
});
console.log('error doc read:', err);
reject('读取 doc 文件失败');
};
} catch (error) {
reject('浏览器不支持文件内容读取');
}
});
/**
* 读取csv
*/
export const readCsvContent = async (file: File) => {
try {
const textArr = await readTxtContent(file);
const csvArr = Papa.parse(textArr).data as string[][];
if (csvArr.length === 0) {
throw new Error('csv 解析失败');
}
return {
header: csvArr.shift() as string[],
data: csvArr.map((item) => item)
};
} catch (error) {
return Promise.reject('解析 csv 文件失败');
}
};
/**
* file download
*/
export const fileDownload = ({
text,
type,
filename
}: {
text: string;
type: string;
filename: string;
}) => {
// 导出为文件
const blob = new Blob([`\uFEFF${text}`], { type: `${type};charset=utf-8;` });
// 创建下载链接
const downloadLink = document.createElement('a');
downloadLink.href = window.URL.createObjectURL(blob);
downloadLink.download = filename;
// 添加链接到页面并触发下载
document.body.appendChild(downloadLink);
downloadLink.click();
document.body.removeChild(downloadLink);
};
export async function getFileAndOpen(fileId: string) {
const url = await getFileViewUrl(fileId);
const asPath = `${location.origin}${url}`;
window.open(asPath, '_blank');
}
export const fileToBase64 = (file: File) => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = (error) => reject(error);
});
};
/**
* compress image. response base64
* @param maxSize The max size of the compressed image
*/
export const compressImg = ({
file,
maxW = 200,
maxH = 200,
maxSize = 1024 * 100
}: {
file: File;
maxW?: number;
maxH?: number;
maxSize?: number;
}) =>
new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = async () => {
const img = new Image();
// @ts-ignore
img.src = reader.result;
img.onload = async () => {
let width = img.width;
let height = img.height;
if (width > height) {
if (width > maxW) {
height *= maxW / width;
width = maxW;
}
} else {
if (height > maxH) {
width *= maxH / height;
height = maxH;
}
}
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
if (!ctx) {
return reject('压缩图片异常');
}
ctx.drawImage(img, 0, 0, width, height);
const compressedDataUrl = canvas.toDataURL(file.type, 0.8);
// 移除 canvas 元素
canvas.remove();
if (compressedDataUrl.length > maxSize) {
return reject('图片太大了');
}
const src = await (async () => {
try {
const src = await uploadImg(compressedDataUrl);
return src;
} catch (error) {
return compressedDataUrl;
}
})();
resolve(src);
};
};
reader.onerror = (err) => {
console.log(err);
reject('压缩图片异常');
};
});

37
projects/app/src/utils/web/i18n.ts
View File

@@ -1,37 +0,0 @@
import { serverSideTranslations } from 'next-i18next/serverSideTranslations';
import Cookies from 'js-cookie';
export const LANG_KEY = 'NEXT_LOCALE_LANG';
export enum LangEnum {
'zh' = 'zh',
'en' = 'en'
}
export const langMap = {
[LangEnum.en]: {
label: 'English',
icon: 'language_en'
},
[LangEnum.zh]: {
label: '简体中文',
icon: 'language_zh'
}
};
export const setLangStore = (value: `${LangEnum}`) => {
return Cookies.set(LANG_KEY, value, { expires: 7, sameSite: 'None', secure: true });
};
export const getLangStore = () => {
return (Cookies.get(LANG_KEY) as `${LangEnum}`) || LangEnum.zh;
};
export const serviceSideProps = (content: any) => {
const acceptLanguage = (content.req.headers['accept-language'] as string) || '';
const acceptLanguageList = acceptLanguage.split(/,|;/g);
// @ts-ignore
const firstLang = acceptLanguageList.find((lang) => langMap[lang]);
const language = content.req.cookies[LANG_KEY] || firstLang || 'zh';
return serverSideTranslations(language, undefined, null, content.locales);
};

129
projects/app/src/utils/web/voice.ts
View File

@@ -1,129 +0,0 @@
import { useState, useCallback, useEffect, useMemo } from 'react';
import { useToast } from '@/hooks/useToast';
import { getErrText } from '../tools';
export const useAudioPlay = (props?: { ttsUrl?: string }) => {
const { ttsUrl } = props || {};
const { toast } = useToast();
const [audio, setAudio] = useState<HTMLAudioElement>();
const [audioLoading, setAudioLoading] = useState(false);
const [audioPlaying, setAudioPlaying] = useState(false);
const hasAudio = useMemo(() => {
if (ttsUrl) return true;
const voices = window.speechSynthesis?.getVoices?.() || []; // 获取语言包
const voice = voices.find((item) => {
return item.lang === 'zh-CN';
});
return !!voice;
}, [ttsUrl]);
const playAudio = useCallback(
async (text: string) => {
text = text.replace(/\\n/g, '\n');
try {
if (audio && ttsUrl) {
setAudioLoading(true);
const response = await fetch(ttsUrl, {
method: 'POST',
headers: {
'content-type': 'application/json'
},
body: JSON.stringify({
text
})
}).then((res) => res.blob());
const audioUrl = URL.createObjectURL(response);
audio.src = audioUrl;
audio.play();
} else {
// window speech
window.speechSynthesis?.cancel();
const msg = new SpeechSynthesisUtterance(text);
const voices = window.speechSynthesis?.getVoices?.() || []; // 获取语言包
const voice = voices.find((item) => {
return item.lang === 'zh-CN';
});
if (voice) {
msg.onstart = () => {
setAudioPlaying(true);
};
msg.onend = () => {
setAudioPlaying(false);
msg.onstart = null;
msg.onend = null;
};
msg.voice = voice;
window.speechSynthesis?.speak(msg);
}
}
} catch (error) {
toast({
status: 'error',
title: getErrText(error, '语音播报异常')
});
}
setAudioLoading(false);
},
[audio, toast, ttsUrl]
);
const cancelAudio = useCallback(() => {
if (audio) {
audio.pause();
audio.src = '';
}
window.speechSynthesis?.cancel();
setAudioPlaying(false);
}, [audio]);
useEffect(() => {
if (ttsUrl) {
setAudio(new Audio());
} else {
setAudio(undefined);
}
}, [ttsUrl]);
useEffect(() => {
if (audio) {
audio.onplay = () => {
setAudioPlaying(true);
};
audio.onended = () => {
setAudioPlaying(false);
};
audio.onerror = () => {
setAudioPlaying(false);
};
}
const listen = () => {
cancelAudio();
};
window.addEventListener('beforeunload', listen);
return () => {
if (audio) {
audio.onplay = null;
audio.onended = null;
audio.onerror = null;
}
cancelAudio();
window.removeEventListener('beforeunload', listen);
};
}, [audio, cancelAudio]);
useEffect(() => {
return () => {
setAudio(undefined);
};
}, []);
return {
audioPlaying,
audioLoading,
hasAudio,
playAudio,
cancelAudio
};
};