gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.8.18 feature (#3565)

Browse Source 
* feat: org CRUD (#3380)

* feat: add org schema

* feat: org manage UI

* feat: OrgInfoModal

* feat: org tree view

* feat: org management

* fix: init root org

* feat: org permission for app

* feat: org support for dataset

* fix: disable org role control

* styles: opt type signatures

* fix: remove unused permission

* feat: delete org collaborator

* perf: Team org ui (#3499)

* perf: org ui

* perf: org ui

* feat: org auth for app & dataset (#3498)

* feat: auth org resource permission

* feat: org auth support for app & dataset

* perf: org permission check (#3500)

* i18n (#3501)

* name

* i18n

* feat: support dataset changeOwner (#3483)

* feat: support dataset changeOwner

* chore: update dataset change owner api

* feat: permission manage UI for org (#3503)

* perf: password check;perf: image upload check;perf: sso login check (#3509)

* perf: password check

* perf: image upload check

* perf: sso login check

* force show update notification modal & fix login page text (#3512)

* fix login page English text

* update notification modal

* perf: notify account (#3515)

* perf(plugin): improve searXNG empty result handling and documentation (#3507)

* perf(plugin): improve searXNG empty result handling and documentation

* 修改了文档和代码部分无搜索的结果的反馈

* refactor: org pathId (#3516)

* optimize payment process (#3517)

* feat: support wecom sso (#3518)

* feat: support wecom sso

* chore: remove unused wecom js-sdk dependency

* fix qrcode script (#3520)

* fix qrcode script

* i18n

* perf: full text collection and search code;perf: rename function (#3519)

* perf: full text collection and search code

* perf: rename function

* perf: notify modal

* remove invalid code

* perf: sso login

* perf: pay process

* 4.8.18 test (#3524)

* perf: remove local token

* perf: index

* perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528)

* perf: text encoding

* perf: leave team code

* perf: full text search code

* fix: http status

* perf: embedding search and vector avatar

* perf: async read file (#3531)

* refactor: team permission  manager (#3535)

* perf: classify org, group and member

* refactor: team per manager

* fix: missing functions

* 4.8.18 test (#3543)

* perf: login check

* doc

* perf: llm model config

* perf: team clb config

* fix: MemberModal UI (#3553)

* fix: adapt MemberModal title and icon

* fix: adapt member modal

* fix: search input placeholder

* fix: add button text

* perf: org permission (#3556)

* docs:用户答疑的官方文档补充 (#3540)

* docs:用户答疑的官方文档补充

* 问题回答的内容修补

* share link random avatar (#3541)

* share link random avatar

* fix

* delete unused code

* share page avatar (#3558)

* feat: init 4818

* share page avatar

* feat: tmp upgrade code (#3559)

* feat: tmp upgrade code

* fulltext search test

* update action

* full text tmp code (#3561)

* full text tmp code

* fix: init

* fix: init

* remove tmp code

* remove tmp code

* 4818-alpha

* 4.8.18 test (#3562)

* full text tmp code

* fix: init

* upgrade code

* account log

* account log

* perf: dockerfile

* upgrade code

* chore: update docs app template submission (#3564)

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
This commit is contained in:
Archer
2025-01-11 15:15:38 +08:00
committed by GitHub
parent bb669ca3ff
commit 10d8c56e23
205 changed files with 5305 additions and 2428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
projects/app/src/service/common/system/index.ts
View File

@@ -1,5 +1,5 @@
import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
import { existsSync, readdirSync, readFileSync } from 'fs';
import fs, { existsSync, readdirSync } from 'fs';
import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';
import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
@@ -13,7 +13,7 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c
import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema';
export const readConfigData = (name: string) => {
export const readConfigData = async (name: string) => {
const splitName = name.split('.');
const devName = `${splitName[0]}.local.${splitName[1]}`;
@@ -30,7 +30,7 @@ export const readConfigData = (name: string) => {
return `/app/data/${name}`;
})();
const content = readFileSync(filename, 'utf-8');
const content = await fs.promises.readFile(filename, 'utf-8');
return content;
};
@@ -120,13 +120,13 @@ export async function initSystemConfig() {
});
}
function getSystemVersion() {
async function getSystemVersion() {
if (global.systemVersion) return;
try {
if (process.env.NODE_ENV === 'development') {
global.systemVersion = process.env.npm_package_version || '0.0.0';
} else {
const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8'));
const packageJson = json5.parse(await fs.promises.readFile('/app/package.json', 'utf-8'));
global.systemVersion = packageJson?.version;
}
@@ -138,7 +138,7 @@ function getSystemVersion() {
}
}
function getSystemPlugin() {
async function getSystemPlugin() {
if (global.communityPlugins && global.communityPlugins.length > 0) return;
const basePath =
@@ -149,15 +149,17 @@ function getSystemPlugin() {
const filterFiles = files.filter((item) => item.endsWith('.json'));
// read json file
const fileTemplates = filterFiles.map<SystemPluginTemplateItemType>((filename) => {
const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
return {
...json5.parse(content),
originCost: 0,
currentCost: 0,
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
};
});
const fileTemplates = await Promise.all(
filterFiles.map<Promise<SystemPluginTemplateItemType>>(async (filename) => {
const content = await fs.promises.readFile(`${basePath}/${filename}`, 'utf-8');
return {
...json5.parse(content),
originCost: 0,
currentCost: 0,
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
};
})
);
fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0));

5
projects/app/src/service/core/ai/apiproxy.d.ts vendored Normal file
View File

@@ -0,0 +1,5 @@
export type CreateModelParams = {
name: string;
description: string;
prompt: string;
};

66
projects/app/src/service/core/ai/apiproxy.ts Normal file
View File

@@ -0,0 +1,66 @@
import { addLog } from '@fastgpt/service/common/system/log';
import axios, { Method } from 'axios';
const url = process.env.API_PROXY_URL;
const token = process.env.API_PROXY_TOKEN;
const instance = axios.create({
baseURL: url,
timeout: 60000, // 超时时间
headers: {
Authorization: `Bearer ${token}`
}
});
/**
* 响应数据检查
*/
const checkRes = (data: any) => {
if (data === undefined) {
addLog.info('api proxy data is empty');
return Promise.reject('服务器异常');
}
return data.data;
};
const responseError = (err: any) => {
console.log('error->', '请求错误', err);
if (!err) {
return Promise.reject({ message: '未知错误' });
}
if (typeof err === 'string') {
return Promise.reject({ message: err });
}
if (typeof err.message === 'string') {
return Promise.reject({ message: err.message });
}
if (typeof err.data === 'string') {
return Promise.reject({ message: err.data });
}
if (err?.response?.data) {
return Promise.reject(err?.response?.data);
}
return Promise.reject(err);
};
const request = <T>(url: string, data: any, method: Method): Promise<T> => {
/* 去空 */
for (const key in data) {
if (data[key] === undefined) {
delete data[key];
}
}
return instance
.request({
url,
method,
data: ['POST', 'PUT'].includes(method) ? data : undefined,
params: !['POST', 'PUT'].includes(method) ? data : undefined
})
.then((res) => checkRes(res.data))
.catch((err) => responseError(err));
};
// TODO: channel crud
export const ApiProxy = {};

4
projects/app/src/service/core/app/plugin.ts
View File

@@ -21,7 +21,9 @@ export const getSystemPlugins = async (refresh = false) => {
global.systemPlugins = [];
}
global.systemPlugins = FastGPTProUrl ? await getCommercialPlugins() : getCommunityPlugins();
global.systemPlugins = FastGPTProUrl
? await getCommercialPlugins()
: await getCommunityPlugins();
addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);

33
projects/app/src/service/core/dataset/data/controller.ts
View File

@@ -12,6 +12,7 @@ import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { ClientSession } from '@fastgpt/service/common/mongo';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
/* insert data.
* 1. create data id
@@ -42,7 +43,8 @@ export async function insertData2Dataset({
const qaStr = getDefaultIndex({ q, a }).text;
// empty indexes check, if empty, create default index
// 1. Get vector indexes and insert
// Empty indexes check, if empty, create default index
indexes =
Array.isArray(indexes) && indexes.length > 0
? indexes.map((index) => ({
@@ -77,7 +79,7 @@ export async function insertData2Dataset({
)
);
// create mongo data
// 2. Create mongo data
const [{ _id }] = await MongoDatasetData.create(
[
{
@@ -87,7 +89,8 @@ export async function insertData2Dataset({
collectionId,
q,
a,
fullTextToken: jiebaSplit({ text: qaStr }),
// FullText tmp
// fullTextToken: jiebaSplit({ text: qaStr }),
chunkIndex,
indexes: indexes?.map((item, i) => ({
...item,
@@ -98,6 +101,20 @@ export async function insertData2Dataset({
{ session }
);
// 3. Create mongo data text
await MongoDatasetDataText.create(
[
{
teamId,
datasetId,
collectionId,
dataId: _id,
fullTextToken: jiebaSplit({ text: qaStr })
}
],
{ session }
);
return {
insertId: _id,
tokens: result.reduce((acc, cur) => acc + cur.tokens, 0)
@@ -225,11 +242,19 @@ export async function updateData2Dataset({
// update mongo other data
mongoData.q = q || mongoData.q;
mongoData.a = a ?? mongoData.a;
mongoData.fullTextToken = jiebaSplit({ text: mongoData.q + mongoData.a });
// FullText tmp
// mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
// @ts-ignore
mongoData.indexes = newIndexes;
await mongoData.save({ session });
// update mongo data text
await MongoDatasetDataText.updateOne(
{ dataId: mongoData._id },
{ fullTextToken: jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
{ session }
);
// delete vector
const deleteIdList = patchResult
.filter((item) => item.type === 'delete' || item.type === 'update')

4
projects/app/src/service/events/generateVector.ts
View File

@@ -166,9 +166,9 @@ const rebuildData = async ({
// get new mongoData insert to training
const newRebuildingData = await MongoDatasetData.findOneAndUpdate(
{
rebuilding: true,
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
rebuilding: true
datasetId: mongoData.datasetId
},
{
$unset: {

2
projects/app/src/service/mongo.ts
View File

@@ -42,7 +42,7 @@ export async function initRootUser(retry = 3): Promise<any> {
rootId = _id;
}
// init root team
await createDefaultTeam({ userId: rootId, balance: 9999 * PRICE_SCALE, session });
await createDefaultTeam({ userId: rootId, session });
});
console.log(`root user init:`, {