gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.9.2 feature (#4354)

Browse Source 
* feat: custom dataset split sign (#4221)

* feat: custom dataset split sign

* feat: custom dataset split sign

* add external variable debug (#4204)

* add external variable debug

* fix ui

* plugin variables

* perf: custom varialbe (#4225)

* fix: invite link (#4229)

* fix: invite link

* feat: create invite link and copy it directly

* feat: sync api collection will refresh title;perf: invite link ux (#4237)

* update queue

* feat: sync api collection will refresh title

* sync collection

* remove lock

* perf: invite link ux

* fix ts (#4239)

* sync collection

* remove lock

* fix ts

* fix: ts

* Sso (#4235)

* feat: redirect url can be inner url (#4138)

* fix: update new user sync api (#4145)

* feat: post all params to backend (#4151)

* pref: sso getauthurl api (#4172)

* pref: sso getauthurl api

* pref: sso

* solve the rootorglist (#4234)

---------

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* fix variable sync & popover button height (#4227)

* fix variable sync & popover button height

* required

* feat: node prompt version (#4141)

* feat: node prompt version

* fix

* delete unused code

* fix

* fix code

* update prompt version (#4242)

* sync collection

* remove lock

* update prompt version

* perf: ai proxy (#4265)

* sync collection

* remove lock

* perf: ai proxy

* fix: member count (#4269)

* feat: chunk index independent config (#4271)

* sync collection

* remove lock

* feat: chunk index independent config

* feat: add max chunksize to split chunk function

* remove log

* update doc

* remove

* remove log

* fix input form label overflow (#4266)

* add model test log (#4272)

* sync collection

* remove lock

* add model test log

* update ui

* update log

* fix: channel test

* preview chunk ui

* test model ux

* test model log

* perf: dataset selector

* fix: system plugin auth

* update nextjs

* perf: ai proxy log remove retry log；perf: workflow type auto parse；add chunk spliter test (#4296)

* sync collection

* remove lock

* perf: workflow type auto parse

* add chunk spliter test

* perf: ai proxy log remove retry log

* udpate ai proxy field

* pref: member/org/gourp list (#4295)

* refactor: org api

* refactor: org api

* pref: member/org/group list

* feat: change group owner api

* fix: manage org member

* pref: member search

* tmp org api rewrite (#4304)

* sync collection

* remove lock

* tmp org api rewrite

* perf: text splitter (#4313)

* sync collection

* remove lock

* perf: text splitter

* update comment

* update search filter code (#4317)

* sync collection

* remove lock

* update search filter code

* pref: member/group/org (#4316)

* feat: change group owner api

* pref: member/org/group

* fix: member modal select clb

* fix: search member when change owner

* fix: member list, login button (#4322)

* perf: member group (#4324)

* sync collection

* remove lock

* perf: member group

* fix: ts   (#4325)

* sync collection

* remove lock

* fix: ts

* fix: group (#4330)

* perf: intro wrap (#4346)

* sync collection

* remove lock

* perf: intro wrap

* pref: member list (#4344)

* chore: search member new api

* chore: permission

* fix: ts error

* fix: member modal

* perf: long org name ui (#4347)

* sync collection

* remove lock

* perf: long org name ui

* perf: member tableui (#4353)

* fix: ts (#4357)

* docs: Add SSO Markdown Doc (#4334)

* add sso doc

* fix comment

* update sso doc (#4358)

* pref: useScrollPagination support debounce and throttle. (#4355)

* pref: useScrollPagination support debounce and throttle.

* fix: useScrollPagination loading

* fix: isloading

* fix: org search path hide

* fix: simple app all_app button (#4365)

* add qwen long (#4363)

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
This commit is contained in:
Archer
2025-03-27 16:54:08 +08:00
committed by GitHub
parent f483832749 fed04f0b5d
commit 05b3062204
209 changed files with 5156 additions and 2466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
projects/app/src/pages/account/info/index.tsx
View File

@@ -294,7 +294,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
title={t('account_info:click_modify_nickname')}
borderColor={'transparent'}
transform={'translateX(-11px)'}
maxLength={20}
maxLength={100}
onBlur={async (e) => {
const val = e.target.value;
if (val === userInfo?.team?.memberName) return;

4
projects/app/src/pages/account/team/index.tsx
View File

@@ -48,7 +48,7 @@ const Team = () => {
const { t } = useTranslation();
const { userInfo } = useUserStore();
const { setEditTeamData, isLoading, teamSize } = useContextSelector(TeamContext, (v) => v);
const { setEditTeamData, teamSize } = useContextSelector(TeamContext, (v) => v);
const Tabs = useMemo(
() => (
@@ -75,7 +75,7 @@ const Team = () => {
);
return (
<AccountContainer isLoading={isLoading}>
<AccountContainer>
<Flex h={'100%'} flexDirection={'column'}>
{/* header */}
<Flex

13
projects/app/src/pages/api/core/ai/model/test.ts
View File

@@ -35,11 +35,17 @@ async function handler(
if (!modelData) return Promise.reject('Model not found');
if (channelId) {
delete modelData.requestUrl;
delete modelData.requestAuth;
}
const headers: Record<string, string> = channelId
? {
'Aiproxy-Channel': String(channelId)
}
: {};
addLog.debug(`Test model`, modelData);
if (modelData.type === 'llm') {
return testLLMModel(modelData, headers);
@@ -63,10 +69,6 @@ async function handler(
export default NextAPI(handler);
const testLLMModel = async (model: LLMModelItemType, headers: Record<string, string>) => {
const ai = getAIApi({
timeout: 10000
});
const requestBody = llmCompletionsBodyFormat(
{
model: model.model,
@@ -75,6 +77,7 @@ const testLLMModel = async (model: LLMModelItemType, headers: Record<string, str
},
model
);
const { response, isStreamResponse } = await createChatCompletion({
body: requestBody,
options: {
@@ -144,7 +147,7 @@ const testTTSModel = async (model: TTSModelType, headers: Record<string, string>
const testSTTModel = async (model: STTModelType, headers: Record<string, string>) => {
const path = isProduction ? '/app/data/test.mp3' : 'data/test.mp3';
const { text } = await aiTranscriptions({
model: model.model,
model,
fileStream: fs.createReadStream(path),
headers
});

6
projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts
View File

@@ -41,7 +41,7 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
return Promise.reject(DatasetErrEnum.sameApiCollection);
}
const content = await readApiServerFileContent({
const { title, rawText } = await readApiServerFileContent({
apiServer,
feishuServer,
yuqueServer,
@@ -53,14 +53,14 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
const { collectionId, insertResults } = await createCollectionAndInsertData({
dataset,
rawText: content,
rawText,
relatedId: apiFileId,
createCollectionParams: {
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.apiFile,
name,
name: title || name,
apiFileId,
metadata: {
relatedImgId: apiFileId

7
projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts
View File

@@ -2,8 +2,7 @@ import { reTrainingDatasetFileCollectionParams } from '@fastgpt/global/core/data
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
import {
DatasetCollectionTypeEnum,
DatasetSourceReadTypeEnum,
TrainingModeEnum
DatasetSourceReadTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { hashStr } from '@fastgpt/global/common/string/tools';
@@ -77,7 +76,7 @@ async function handler(
return Promise.reject(i18nT('dataset:collection_not_support_retraining'));
})();
const rawText = await readDatasetSourceRawText({
const { title, rawText } = await readDatasetSourceRawText({
teamId,
tmbId,
customPdfParse,
@@ -100,7 +99,7 @@ async function handler(
teamId: collection.teamId,
tmbId: collection.tmbId,
datasetId: collection.dataset._id,
name: collection.name,
name: title || collection.name,
type: collection.type,
customPdfParse,

13
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -4,7 +4,7 @@
*/
import type { NextApiRequest } from 'next';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken/index';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
@@ -16,6 +16,7 @@ import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils';
async function handler(req: NextApiRequest) {
const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps;
@@ -45,7 +46,7 @@ async function handler(req: NextApiRequest) {
// auth collection and get dataset
const [
{
dataset: { _id: datasetId, vectorModel }
dataset: { _id: datasetId, vectorModel, agentModel }
}
] = await Promise.all([getCollectionWithDataset(collectionId)]);
@@ -60,9 +61,11 @@ async function handler(req: NextApiRequest) {
// token check
const token = await countPromptTokens(formatQ + formatA, '');
const vectorModelData = getEmbeddingModel(vectorModel);
const llmModelData = getLLMModel(agentModel);
const maxChunkSize = getLLMMaxChunkSize(llmModelData);
if (token > vectorModelData.maxToken) {
return Promise.reject('Q Over Tokens');
if (token > maxChunkSize) {
return Promise.reject(`Content over max chunk size: ${maxChunkSize}`);
}
// Duplicate data check
@@ -82,7 +85,7 @@ async function handler(req: NextApiRequest) {
q: formatQ,
a: formatA,
chunkIndex: 0,
model: vectorModelData.model,
embeddingModel: vectorModelData.model,
indexes: formatIndexes
});

135
projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
View File

@@ -1,4 +1,9 @@
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
import {
ChunkSettingModeEnum,
DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum,
DatasetSourceReadTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -8,100 +13,130 @@ import {
} from '@fastgpt/global/support/permission/constant';
import { authCollectionFile } from '@fastgpt/service/support/permission/auth/file';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import {
computeChunkSize,
computeChunkSplitter,
getLLMMaxChunkSize
} from '@fastgpt/global/core/dataset/training/utils';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { getLLMModel } from '@fastgpt/service/core/ai/model';
export type PostPreviewFilesChunksProps = {
datasetId: string;
type: DatasetSourceReadTypeEnum;
sourceId: string;
chunkSize: number;
overlapRatio: number;
customSplitChar?: string;
customPdfParse?: boolean;
trainingType: DatasetCollectionDataProcessModeEnum;
// Chunk settings
chunkSettingMode: ChunkSettingModeEnum;
chunkSplitMode: DataChunkSplitModeEnum;
chunkSize: number;
chunkSplitter?: string;
overlapRatio: number;
// Read params
selector?: string;
isQAImport?: boolean;
externalFileId?: string;
};
export type PreviewChunksResponse = {
q: string;
a: string;
}[];
chunks: {
q: string;
a: string;
}[];
total: number;
};
async function handler(
req: ApiRequestProps<PostPreviewFilesChunksProps>
): Promise<PreviewChunksResponse> {
const {
let {
type,
sourceId,
customPdfParse = false,
trainingType,
chunkSettingMode,
chunkSplitMode,
chunkSize,
customSplitChar,
chunkSplitter,
overlapRatio,
selector,
isQAImport,
datasetId,
externalFileId,
customPdfParse = false
externalFileId
} = req.body;
if (!sourceId) {
throw new Error('sourceId is empty');
}
if (chunkSize > 30000) {
throw new Error('chunkSize is too large, should be less than 30000');
const fileAuthRes =
type === DatasetSourceReadTypeEnum.fileLocal
? await authCollectionFile({
req,
authToken: true,
authApiKey: true,
fileId: sourceId,
per: OwnerPermissionVal
})
: undefined;
const { dataset, teamId, tmbId } = await authDataset({
req,
authApiKey: true,
authToken: true,
datasetId,
per: WritePermissionVal
});
if (fileAuthRes && String(fileAuthRes.tmbId) !== String(tmbId) && !fileAuthRes.isRoot) {
return Promise.reject(CommonErrEnum.unAuthFile);
}
const { teamId, tmbId, apiServer, feishuServer, yuqueServer } = await (async () => {
if (type === DatasetSourceReadTypeEnum.fileLocal) {
const res = await authCollectionFile({
req,
authToken: true,
authApiKey: true,
fileId: sourceId,
per: OwnerPermissionVal
});
return {
teamId: res.teamId,
tmbId: res.tmbId
};
}
const { dataset, teamId, tmbId } = await authDataset({
req,
authApiKey: true,
authToken: true,
datasetId,
per: WritePermissionVal
});
return {
teamId,
tmbId,
apiServer: dataset.apiServer,
feishuServer: dataset.feishuServer,
yuqueServer: dataset.yuqueServer
};
})();
chunkSize = computeChunkSize({
trainingType,
chunkSettingMode,
chunkSplitMode,
chunkSize,
llmModel: getLLMModel(dataset.agentModel)
});
const rawText = await readDatasetSourceRawText({
chunkSplitter = computeChunkSplitter({
chunkSettingMode,
chunkSplitMode,
chunkSplitter
});
const { rawText } = await readDatasetSourceRawText({
teamId,
tmbId,
type,
sourceId,
selector,
isQAImport,
apiServer,
feishuServer,
yuqueServer,
apiServer: dataset.apiServer,
feishuServer: dataset.feishuServer,
yuqueServer: dataset.yuqueServer,
externalFileId,
customPdfParse
});
return rawText2Chunks({
const chunks = rawText2Chunks({
rawText,
chunkLen: chunkSize,
chunkSize,
maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : [],
customReg: chunkSplitter ? [chunkSplitter] : [],
isQAImport: isQAImport
}).slice(0, 10);
});
return {
chunks: chunks.slice(0, 10),
total: chunks.length
};
}
export default NextAPI(handler);

26
projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
View File

@@ -1,27 +1,31 @@
import type { NextApiRequest } from 'next';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { GetTrainingQueueProps } from '@/global/core/dataset/api';
import { NextAPI } from '@/service/middleware/entry';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
export type GetQueueLenResponse = {
vectorTrainingCount: number;
qaTrainingCount: number;
autoTrainingCount: number;
imageTrainingCount: number;
};
async function handler(req: NextApiRequest) {
await authCert({ req, authToken: true });
const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate(
[
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
lockTime: { $lt: new Date('2040/1/1') }
}
},
{
$group: {
_id: '$model',
_id: '$mode',
count: { $sum: 1 }
}
}
@@ -31,12 +35,16 @@ async function handler(req: NextApiRequest) {
}
);
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
const vectorTrainingCount = data.find((item) => item._id === TrainingModeEnum.chunk)?.count || 0;
const qaTrainingCount = data.find((item) => item._id === TrainingModeEnum.qa)?.count || 0;
const autoTrainingCount = data.find((item) => item._id === TrainingModeEnum.auto)?.count || 0;
const imageTrainingCount = data.find((item) => item._id === TrainingModeEnum.image)?.count || 0;
return {
vectorTrainingCount,
agentTrainingCount
qaTrainingCount,
autoTrainingCount,
imageTrainingCount
};
}

2
projects/app/src/pages/api/v1/audio/transcriptions.ts
View File

@@ -66,7 +66,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
// }
const result = await aiTranscriptions({
model: getDefaultSTTModel().model,
model: getDefaultSTTModel(),
fileStream: fs.createReadStream(file.path)
});

16
projects/app/src/pages/login/provider.tsx
View File

@@ -19,7 +19,7 @@ const provider = () => {
const { initd, loginStore, setLoginStore } = useSystemStore();
const { setUserInfo } = useUserStore();
const router = useRouter();
const { code, state, error } = router.query as { code: string; state: string; error?: string };
const { state, error, ...props } = router.query as Record<string, string>;
const { toast } = useToast();
const loginSuccess = useCallback(
@@ -31,12 +31,12 @@ const provider = () => {
[setUserInfo, router, loginStore?.lastRoute]
);
const authCode = useCallback(
async (code: string) => {
const authProps = useCallback(
async (props: Record<string, string>) => {
try {
const res = await oauthLogin({
type: loginStore?.provider || OAuthEnum.sso,
code,
props,
callbackUrl: `${location.origin}/login/provider`,
inviterId: localStorage.getItem('inviterId') || undefined,
bd_vid: sessionStorage.getItem('bd_vid') || undefined,
@@ -86,8 +86,8 @@ const provider = () => {
return;
}
console.log('SSO', { initd, loginStore, code, state });
if (!code || !initd) return;
console.log('SSO', { initd, loginStore, props, state });
if (!props || !initd) return;
if (isOauthLogging) return;
@@ -107,10 +107,10 @@ const provider = () => {
}, 1000);
return;
} else {
authCode(code);
authProps(props);
}
})();
}, [initd, authCode, code, error, loginStore, loginStore?.state, router, state, t, toast]);
}, [initd, authProps, error, loginStore, loginStore?.state, router, state, t, toast, props]);
return <Loading />;
};