V4.9.4 feature (#4470)
* Training status (#4424) * dataset data training state (#4311) * dataset data training state * fix * fix ts * fix * fix api format * fix * fix * perf: count training * format * fix: dataset training state (#4417) * fix * add test * fix * fix * fix test * fix test * perf: training count * count * loading status --------- Co-authored-by: heheer <heheer@sealos.io> * doc * website sync feature (#4429) * perf: introduce BullMQ for website sync (#4403) * perf: introduce BullMQ for website sync * feat: new redis module * fix: remove graceful shutdown * perf: improve UI in dataset detail - Updated the "change" icon SVG file. - Modified i18n strings. - Added new i18n string "immediate_sync". - Improved UI in dataset detail page, including button icons and background colors. * refactor: Add chunkSettings to DatasetSchema * perf: website sync ux * env template * fix: clean up website dataset when updating chunk settings (#4420) * perf: check setting updated * perf: worker currency * feat: init script for website sync refactor (#4425) * website feature doc --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> * pro migration (#4388) (#4433) * pro migration * reuse customPdfParseType Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> * perf: remove loading ui * feat: config chat file expired time * Redis cache (#4436) * perf: add Redis cache for vector counting (#4432) * feat: cache * perf: get cache key --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> * perf: mobile voice input (#4437) * update:Mobile voice interaction (#4362) * Add files via upload * Add files via upload * Update ollama.md * Update ollama.md * Add files via upload * Update useSpeech.ts * Update ChatInput.tsx * Update useSpeech.ts * Update ChatInput.tsx * Update useSpeech.ts * Update constants.ts * Add files via upload * Update ChatInput.tsx * Update useSpeech.ts * Update useSpeech.ts * Update useSpeech.ts * Update ChatInput.tsx * Add files via upload * Update common.json * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update useSpeech.ts * Update useSpeech.ts * Update common.json * Update common.json * Update common.json * Update VoiceInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update useSpeech.ts * Update common.json * Update chat.json * Update common.json * Update chat.json * Update common.json * Update chat.json * Update VoiceInput.tsx * Update ChatInput.tsx * Update useSpeech.ts * Update VoiceInput.tsx * speech ui * 优化语音输入组件，调整输入框显示逻辑，修复语音输入遮罩层样式，更新画布背景透明度，增强用户交互体验。 (#4435) * perf: mobil voice input --------- Co-authored-by: dreamer6680 <1468683855@qq.com> * Test completion v2 (#4438) * add v2 completions (#4364) * add v2 completions * completion config * config version * fix * frontend * doc * fix * fix: completions v2 api --------- Co-authored-by: heheer <heheer@sealos.io> * package * Test mongo log (#4443) * feat: mongodb-log (#4426) * perf: mongo log * feat: completions stop reasoner * mongo db log --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> * update doc * Update doc * fix external var ui (#4444) * action * fix: ts (#4458) * preview doc action add docs preview permission update preview action udpate action * update doc (#4460) * update preview action * update doc * remove * update * schema * update mq export;perf: redis cache (#4465) * perf: redis cache * update mq export * perf: website sync error tip * add error worker * website sync ui (#4466) * Updated the dynamic display of the voice input pop-up (#4469) * Update VoiceInput.tsx * Update VoiceInput.tsx * Update VoiceInput.tsx * fix: voice input --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: dreamer6680 <1468683855@qq.com> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
Archer
@@ -28,10 +28,11 @@ const setClearTmpUploadFilesCron = () => {
|
||||
};
|
||||
|
||||
const clearInvalidDataCron = () => {
|
||||
// Clear files
|
||||
setCron('0 */1 * * *', async () => {
|
||||
if (
|
||||
await checkTimerLock({
|
||||
timerId: TimerIdEnum.checkInValidDatasetFiles,
|
||||
timerId: TimerIdEnum.checkExpiredFiles,
|
||||
lockMinuted: 59
|
||||
})
|
||||
) {
|
||||
|
||||
@@ -71,8 +71,10 @@ export async function checkInvalidDatasetFiles(start: Date, end: Date) {
|
||||
export const removeExpiredChatFiles = async () => {
|
||||
let deleteFileAmount = 0;
|
||||
const collection = getGFSCollection(BucketNameEnum.chat);
|
||||
|
||||
const expireTime = Number(process.env.CHAT_FILE_EXPIRE_TIME || 7);
|
||||
const where = {
|
||||
uploadDate: { $lte: addDays(new Date(), -7) }
|
||||
uploadDate: { $lte: addDays(new Date(), -expireTime) }
|
||||
};
|
||||
|
||||
// get all file _id
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
chunkAutoChunkSize,
|
||||
getLLMMaxChunkSize
|
||||
} from '@fastgpt/global/core/dataset/training/utils';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
|
||||
const reduceQueue = () => {
|
||||
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
|
||||
@@ -50,7 +51,7 @@ export async function generateQA(): Promise<any> {
|
||||
const data = await MongoDatasetTraining.findOneAndUpdate(
|
||||
{
|
||||
mode: TrainingModeEnum.qa,
|
||||
retryCount: { $gte: 0 },
|
||||
retryCount: { $gt: 0 },
|
||||
lockTime: { $lte: addMinutes(new Date(), -10) }
|
||||
},
|
||||
{
|
||||
@@ -176,7 +177,16 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
|
||||
generateQA();
|
||||
} catch (err: any) {
|
||||
addLog.error(`[QA Queue] Error`, err);
|
||||
reduceQueue();
|
||||
await MongoDatasetTraining.updateOne(
|
||||
{
|
||||
teamId: data.teamId,
|
||||
datasetId: data.datasetId,
|
||||
_id: data._id
|
||||
},
|
||||
{
|
||||
errorMsg: getErrText(err, 'unknown error')
|
||||
}
|
||||
);
|
||||
|
||||
setTimeout(() => {
|
||||
generateQA();
|
||||
|
||||
@@ -14,6 +14,7 @@ import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
|
||||
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
|
||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { Document } from '@fastgpt/service/common/mongo';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
|
||||
const reduceQueue = () => {
|
||||
global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
|
||||
@@ -48,7 +49,7 @@ export async function generateVector(): Promise<any> {
|
||||
const data = await MongoDatasetTraining.findOneAndUpdate(
|
||||
{
|
||||
mode: TrainingModeEnum.chunk,
|
||||
retryCount: { $gte: 0 },
|
||||
retryCount: { $gt: 0 },
|
||||
lockTime: { $lte: addMinutes(new Date(), -3) }
|
||||
},
|
||||
{
|
||||
@@ -117,6 +118,16 @@ export async function generateVector(): Promise<any> {
|
||||
return reduceQueueAndReturn();
|
||||
} catch (err: any) {
|
||||
addLog.error(`[Vector Queue] Error`, err);
|
||||
await MongoDatasetTraining.updateOne(
|
||||
{
|
||||
teamId: data.teamId,
|
||||
datasetId: data.datasetId,
|
||||
_id: data._id
|
||||
},
|
||||
{
|
||||
errorMsg: getErrText(err, 'unknown error')
|
||||
}
|
||||
);
|
||||
return reduceQueueAndReturn(1000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,9 @@
|
||||
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/constants';
|
||||
import { MongoUser } from '@fastgpt/service/support/user/schema';
|
||||
import { connectMongo } from '@fastgpt/service/common/mongo/init';
|
||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||
import { createDefaultTeam } from '@fastgpt/service/support/user/team/controller';
|
||||
import { exit } from 'process';
|
||||
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
|
||||
|
||||
/**
|
||||
* This function is equivalent to the entry to the service
|
||||
* connect MongoDB and init data
|
||||
*/
|
||||
export function connectToDatabase() {
|
||||
return connectMongo();
|
||||
}
|
||||
|
||||
export async function initRootUser(retry = 3): Promise<any> {
|
||||
try {
|
||||
const rootUser = await MongoUser.findOne({
|
||||
|
||||
Reference in New Issue
Block a user