gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528)

Browse Source 
* perf: text encoding

* perf: leave team code

* perf: full text search code

* fix: http status

* perf: embedding search and vector avatar
This commit is contained in:
Archer
2025-01-05 14:40:02 +08:00
committed by archer
parent 5465ca642f
commit 2bf1fce32a
24 changed files with 345 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docSite/content/zh-cn/docs/development/upgrading/4818.md
View File

@@ -13,4 +13,6 @@ weight: 806
2. 新增 - 支持部门架构权限模式
3. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。
4. 优化 - Mongo 全文索引表分离。
5. 优化 - 知识库检索查询语句合并，同时减少查库数量。
5. 优化 - 知识库检索查询语句合并，同时减少查库数量。
6. 优化 - 文件编码检测，减少 CSV 文件乱码概率。
7. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。

17
packages/global/common/file/tools.ts
View File

@@ -2,6 +2,7 @@ import { detect } from 'jschardet';
import { documentFileType, imageFileType } from './constants';
import { ChatFileTypeEnum } from '../../core/chat/constants';
import { UserChatItemValueItemType } from '../../core/chat/type';
import * as fs from 'fs';
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
@@ -16,6 +17,22 @@ export const formatFileSize = (bytes: number): string => {
export const detectFileEncoding = (buffer: Buffer) => {
return detect(buffer.slice(0, 200))?.encoding?.toLocaleLowerCase();
};
export const detectFileEncodingByPath = async (path: string) => {
// Get 64KB file head
const MAX_BYTES = 64 * 1024;
const buffer = Buffer.alloc(MAX_BYTES);
const fd = await fs.promises.open(path, 'r');
try {
// Read file head
const { bytesRead } = await fd.read(buffer, 0, MAX_BYTES, 0);
const actualBuffer = buffer.slice(0, bytesRead);
return detect(actualBuffer)?.encoding?.toLocaleLowerCase();
} finally {
await fd.close();
}
};
// Url => user upload file type
export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file'] | undefined => {

6
packages/service/common/file/gridfs/controller.ts
View File

@@ -4,7 +4,7 @@ import fsp from 'fs/promises';
import fs from 'fs';
import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
import { readRawContentByFileBuffer } from '../read/utils';
@@ -36,7 +36,6 @@ export async function uploadFile({
path,
filename,
contentType,
encoding,
metadata = {}
}: {
bucketName: `${BucketNameEnum}`;
@@ -45,7 +44,6 @@ export async function uploadFile({
path: string;
filename: string;
contentType?: string;
encoding: string;
metadata?: Record<string, any>;
}) {
if (!path) return Promise.reject(`filePath is empty`);
@@ -59,7 +57,7 @@ export async function uploadFile({
// Add default metadata
metadata.teamId = teamId;
metadata.uid = uid;
metadata.encoding = encoding;
metadata.encoding = await detectFileEncodingByPath(path);
// create a gridfs bucket
const bucket = getGridBucket(bucketName);

1
packages/service/common/middle/reqFrequencyLimit.ts
View File

@@ -22,7 +22,6 @@ export function useReqFrequencyLimit(seconds: number, limit: number, force = fal
expiredTime: addSeconds(new Date(), seconds)
});
} catch (_) {
res.status(429);
jsonRes(res, {
code: 429,
error: ERROR_ENUM.tooManyRequest

3
packages/service/common/response/index.ts
View File

@@ -33,8 +33,7 @@ export const jsonRes = <T = any>(
addLog.error(`Api response error: ${url}`, ERROR_RESPONSE[errResponseKey]);
res.status(ERROR_RESPONSE[errResponseKey].code);
return res.json(ERROR_RESPONSE[errResponseKey]);
return res.status(code).json(ERROR_RESPONSE[errResponseKey]);
}
// another error

4
packages/service/core/dataset/data/dataTextSchema.ts
View File

@@ -25,7 +25,7 @@ const DatasetDataTextSchema = new Schema({
required: true
},
dataId: {
type: String,
type: Schema.Types.ObjectId,
ref: DatasetDataCollectionName,
required: true
},
@@ -37,7 +37,7 @@ const DatasetDataTextSchema = new Schema({
try {
DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
DatasetDataTextSchema.index({ dataId: 'hashed' });
DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
} catch (error) {
console.log(error);
}

17
packages/service/core/dataset/data/schema.ts
View File

@@ -39,10 +39,6 @@ const DatasetDataSchema = new Schema({
type: String,
default: ''
},
fullTextToken: {
type: String,
default: ''
},
indexes: {
type: [
{
@@ -72,7 +68,13 @@ const DatasetDataSchema = new Schema({
default: 0
},
rebuilding: Boolean,
inited: Boolean
// Abandon
fullTextToken: {
type: String,
default: ''
},
initFullText: Boolean
});
try {
@@ -85,13 +87,14 @@ try {
updateTime: -1
});
// full text index
DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
// DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
// Recall vectors after data matching
DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
DatasetDataSchema.index({ updateTime: 1 });
// rebuild data
DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
DatasetDataSchema.index({ inited: 'hashed' });
DatasetDataSchema.index({ initFullText: 1 });
} catch (error) {
console.log(error);
}

174
packages/service/core/dataset/search/controller.ts
View File

@@ -289,20 +289,22 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
).lean()
]);
const formatResult = dataList
.map((data, index) => {
const collection = collections.find((col) => String(col._id) === String(data.collectionId));
const formatResult = results
.map((item, index) => {
const collection = collections.find((col) => String(col._id) === String(item.collectionId));
if (!collection) {
console.log('Collection is not found', data);
console.log('Collection is not found', item);
return;
}
const data = dataList.find((data) =>
data.indexes.some((index) => index.dataId === item.id)
);
if (!data) {
console.log('Data is not found', item);
return;
}
// add score to data(It's already sorted. The first one is the one with the most points)
const dataIdList = data.indexes.map((item) => item.dataId);
const maxScoreResult = results.find((item) => {
return dataIdList.includes(item.id);
});
const score = maxScoreResult?.score || 0;
const score = item?.score || 0;
const result: SearchDataResponseItemType = {
id: String(data._id),
@@ -320,8 +322,6 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
})
.filter(Boolean) as SearchDataResponseItemType[];
formatResult.sort((a, b) => b.score[0].value - a.score[0].value);
return {
embeddingRecallResults: formatResult,
tokens
@@ -411,22 +411,6 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
'_id name fileId rawLink externalFileId externalFileUrl',
{ ...readFromSecondary }
).lean();
// const [dataList, collections] = await Promise.all([
// MongoDatasetData.find(
// {
// _id: { $in: searchResults.map((item) => item.dataId) }
// },
// '_id datasetId collectionId updateTime q a chunkIndex indexes',
// { ...readFromSecondary }
// ).lean(),
// MongoDatasetCollection.find(
// {
// _id: { $in: searchResults.map((item) => item.collectionId) }
// },
// '_id name fileId rawLink externalFileId externalFileUrl',
// { ...readFromSecondary }
// ).lean()
// ]);
return {
fullTextRecallResults: searchResults
@@ -439,9 +423,6 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
return;
}
// const score =
// searchResults.find((item) => String(item.dataId) === String(data._id))?.score || 0;
return {
id: String(data._id),
datasetId: String(data.datasetId),
@@ -459,6 +440,135 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
tokenLen: 0
};
};
const fullTextRecall2 = async ({
query,
limit,
filterCollectionIdList,
forbidCollectionIdList
}: {
query: string;
limit: number;
filterCollectionIdList?: string[];
forbidCollectionIdList: string[];
}): Promise<{
fullTextRecallResults: SearchDataResponseItemType[];
tokenLen: number;
}> => {
if (limit === 0) {
return {
fullTextRecallResults: [],
tokenLen: 0
};
}
const searchResults = (
await Promise.all(
datasetIds.map(async (id) => {
return MongoDatasetDataText.aggregate(
[
{
$match: {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(id),
$text: { $search: jiebaSplit({ text: query }) },
...(filterCollectionIdList
? {
collectionId: {
$in: filterCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {}),
...(forbidCollectionIdList && forbidCollectionIdList.length > 0
? {
collectionId: {
$nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {})
}
},
{
$sort: {
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
}
],
{
...readFromSecondary
}
);
})
)
).flat() as (DatasetDataTextSchemaType & { score: number })[];
// Get data and collections
const [dataList, collections] = await Promise.all([
MongoDatasetData.find(
{
_id: { $in: searchResults.map((item) => item.dataId) }
},
'_id datasetId collectionId updateTime q a chunkIndex indexes',
{ ...readFromSecondary }
).lean(),
MongoDatasetCollection.find(
{
_id: { $in: searchResults.map((item) => item.collectionId) }
},
'_id name fileId rawLink externalFileId externalFileUrl',
{ ...readFromSecondary }
).lean()
]);
return {
fullTextRecallResults: searchResults
.map((item, index) => {
const collection = collections.find(
(col) => String(col._id) === String(item.collectionId)
);
if (!collection) {
console.log('Collection is not found', item);
return;
}
const data = dataList.find((data) => String(data._id) === String(item.dataId));
if (!data) {
console.log('Data is not found', item);
return;
}
return {
id: String(data._id),
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
updateTime: data.updateTime,
q: data.q,
a: data.a,
chunkIndex: data.chunkIndex,
indexes: data.indexes,
...getCollectionSourceData(collection),
score: [
{
type: SearchScoreTypeEnum.fullText,
value: item.score || 0,
index
}
]
};
})
.filter(Boolean) as SearchDataResponseItemType[],
tokenLen: 0
};
};
const reRankSearchResult = async ({
data,
query
@@ -526,7 +636,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
forbidCollectionIdList,
filterCollectionIdList
}),
fullTextRecall({
fullTextRecall2({
query,
limit: fullTextLimit,
filterCollectionIdList,

22
packages/service/support/permission/memberGroup/controllers.ts
View File

@@ -47,26 +47,32 @@ export const getTeamDefaultGroup = async ({
export const getGroupsByTmbId = async ({
tmbId,
teamId,
role
role,
session
}: {
tmbId: string;
teamId: string;
role?: `${GroupMemberRole}`[];
session?: ClientSession;
}) =>
(
await Promise.all([
(
await MongoGroupMemberModel.find({
tmbId,
groupId: {
$exists: true
await MongoGroupMemberModel.find(
{
tmbId,
groupId: {
$exists: true
},
...(role ? { role: { $in: role } } : {})
},
...(role ? { role: { $in: role } } : {})
})
undefined,
{ session }
)
.populate<{ group: MemberGroupSchemaType }>('group')
.lean()
).map((item) => item.group),
role ? [] : getTeamDefaultGroup({ teamId })
role ? [] : getTeamDefaultGroup({ teamId, session })
])
).flat();

1
packages/service/support/permission/schema.ts
View File

@@ -115,6 +115,7 @@ try {
}
);
// Delete tmb permission
ResourcePermissionSchema.index({
resourceType: 1,
teamId: 1,

3
packages/service/support/user/schema.ts
View File

@@ -78,9 +78,6 @@ const UserSchema = new Schema({
});
try {
// login
UserSchema.index({ username: 1 });
// Admin charts
UserSchema.index({ createTime: -1 });
} catch (error) {

10
packages/service/support/user/team/teamMemberSchema.ts
View File

@@ -23,10 +23,6 @@ const TeamMemberSchema = new Schema({
type: String,
default: 'Member'
},
role: {
type: String
// enum: Object.keys(TeamMemberRoleMap) // disable enum validation for old data
},
status: {
type: String,
enum: Object.keys(TeamMemberStatusMap)
@@ -38,6 +34,12 @@ const TeamMemberSchema = new Schema({
defaultTeam: {
type: Boolean,
default: false
},
// Abandoned
role: {
type: String
// enum: Object.keys(TeamMemberRoleMap) // disable enum validation for old data
}
});

27
packages/service/worker/htmlStr2Md/utils.ts
View File

@@ -1,9 +1,27 @@
import TurndownService from 'turndown';
import { ImageType } from '../readFile/type';
import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
import { getNanoid } from '@fastgpt/global/common/string/tools';
// @ts-ignore
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
const processBase64Images = (htmlContent: string) => {
const base64Regex = /src="data:([^;]+);base64,([^"]+)"/g;
const images: ImageType[] = [];
const processedHtml = htmlContent.replace(base64Regex, (match, mime, base64Data) => {
const uuid = `IMAGE_${getNanoid(12)}_IMAGE`;
images.push({
uuid,
base64: base64Data,
mime
});
return `src="${uuid}"`;
});
return { processedHtml, images };
};
export const html2md = (
html: string
): {
@@ -25,11 +43,14 @@ export const html2md = (
turndownService.remove(['i', 'script', 'iframe', 'style']);
turndownService.use(turndownPluginGfm.gfm);
const { text, imageList } = matchMdImgTextAndUpload(html);
// Base64 img to id, otherwise it will occupy memory when going to md
const { processedHtml, images } = processBase64Images(html);
const md = turndownService.turndown(processedHtml);
const { text, imageList } = matchMdImgTextAndUpload(md);
return {
rawText: turndownService.turndown(text),
imageList
rawText: text,
imageList: [...images, ...imageList]
};
} catch (error) {
console.log('html 2 markdown error', error);

6
packages/service/worker/readFile/extension/rawText.ts
View File

@@ -24,7 +24,11 @@ export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): Read
return buffer.toString(encoding as BufferEncoding);
}
return iconv.decode(buffer, encoding);
if (encoding) {
return iconv.decode(buffer, encoding);
}
return buffer.toString('utf-8');
} catch (error) {
return buffer.toString('utf-8');
}

26
packages/web/i18n/en/account_team.json
View File

@@ -1,31 +1,31 @@
{
"action": "operate",
"confirm_delete_group": "Confirm to delete group?",
"confirm_leave_team": "Confirmed to leave the team? \n \nAfter you log out, all your resources in the team (applications, knowledge bases, folders, managed groups, etc.) will be transferred to the team owner.",
"confirm_delete_org": "Confirm to delete organization?",
"confirm_delete_member": "Confirm to delete member?",
"confirm_delete_org": "Confirm to delete organization?",
"confirm_leave_team": "Confirmed to leave the team? \nAfter exiting, all your resources in the team are transferred to the team owner.",
"create_group": "Create group",
"delete": "delete",
"edit_info": "Edit information",
"group": "group",
"group_name": "Group name",
"org": "organization",
"org_name": "Organization name",
"org_description": "Organization description",
"create_org": "Create organization",
"create_sub_org": "Create sub-organization",
"edit_org_info": "Edit organization information",
"move_org": "Move organization",
"move_member": "Move member",
"delete": "delete",
"delete_org": "Delete organization",
"remark": "remark",
"edit_info": "Edit information",
"edit_org_info": "Edit organization information",
"group": "group",
"group_name": "Group name",
"label_sync": "Tag sync",
"leave_team_failed": "Leaving the team exception",
"manage_member": "Managing members",
"member": "member",
"member_group": "Belonging to member group",
"move_member": "Move member",
"move_org": "Move organization",
"org": "organization",
"org_description": "Organization description",
"org_name": "Organization name",
"owner": "owner",
"permission": "Permissions",
"remark": "remark",
"remove_tip": "Confirm to remove {{username}} from the team?",
"retain_admin_permissions": "Keep administrator rights",
"search_member_group_name": "Search member/group name",

2
packages/web/i18n/zh-CN/account_team.json
View File

@@ -3,7 +3,7 @@
"confirm_delete_group": "确认删除群组？",
"confirm_delete_member": "确认删除成员？",
"confirm_delete_org": "确认删除该部门？",
"confirm_leave_team": "确认离开该团队？ \n 退出后，您在该团队所有的资源（ 应用、知识库、文件夹、管理的群组等）均转让给团队所有者。",
"confirm_leave_team": "确认离开该团队？ \n退出后，您在该团队所有的资源均转让给团队所有者。",
"create_group": "创建群组",
"create_org": "创建部门",
"create_sub_org": "创建子部门",

2
packages/web/i18n/zh-Hant/account_team.json
View File

@@ -3,7 +3,7 @@
"confirm_delete_group": "確認刪除群組？",
"confirm_delete_member": "確認刪除成員？",
"confirm_delete_org": "確認刪除該部門？",
"confirm_leave_team": "確認離開該團隊？ \n \n退出後，您在該團隊所有的資源（ 應用程式、知識庫、資料夾、管理的群組等）均轉讓給團隊所有者。",
"confirm_leave_team": "確認離開該團隊？ \n退出後，您在該團隊所有的資源轉讓給團隊所有者。",
"create_group": "建立群組",
"create_org": "創建部門",
"create_sub_org": "創建子部門",

12
projects/app/src/pages/account/team/index.tsx
View File

@@ -76,14 +76,13 @@ const Team = () => {
onClose: onCloseManageGroupMember
} = useDisclosure();
const { runAsync: onLeaveTeam, loading: isLoadingLeaveTeam } = useRequest2(
async (teamId?: string) => {
if (!teamId) return;
const { runAsync: onLeaveTeam } = useRequest2(
async () => {
const defaultTeam = myTeams.find((item) => item.defaultTeam) || myTeams[0];
// change to personal team
// get members
onSwitchTeam(defaultTeam.teamId);
return delLeaveTeam(teamId);
return delLeaveTeam();
},
{
onSuccess() {
@@ -242,10 +241,7 @@ const Team = () => {
borderRadius={'md'}
ml={3}
leftIcon={<MyIcon name={'support/account/loginoutLight'} w={'14px'} />}
isLoading={isLoadingLeaveTeam}
onClick={() => {
openLeaveConfirm(() => onLeaveTeam(userInfo?.team?.teamId))();
}}
onClick={() => openLeaveConfirm(onLeaveTeam)()}
>
{t('account_team:user_team_leave_team')}
</Button>

94
projects/app/src/pages/api/admin/initv4818.ts
View File

@@ -1,14 +1,104 @@
import { NextAPI } from '@/service/middleware/entry';
import { delay } from '@fastgpt/global/common/system/utils';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
import { NextApiRequest, NextApiResponse } from 'next';
/*
简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。
无缝迁移：
1. 先用 4.8.18-tmp 版本，会同时有 MongoDatasetData 和 MongoDatasetDataText 两个表和索引，依然是 MongoDatasetData 生效。会同步更新两张表数据。
2. 执行升级脚本，不要删除 MongoDatasetData 里的数据。
3. 切换正式版镜像，让 MongoDatasetDataText 生效。
4. 删除 MongoDatasetData 里的索引和多余字段。
*/
let success = 0;
async function handler(req: NextApiRequest, res: NextApiResponse) {
await authCert({ req, authRoot: true });
const batchSize = req.body.batchSize || 500;
success = 0;
const start = Date.now();
await initData(batchSize);
console.log('Init data time:', Date.now() - start);
success = 0;
await batchUpdateFields();
return { success: true };
}
export default NextAPI(handler);
const initData = async (batchSize: number) => {
try {
// 找到没有初始化的数据
const dataList = await MongoDatasetData.find(
{
initFullText: { $exists: false }
},
'_id teamId datasetId collectionId fullTextToken'
)
.limit(batchSize)
.lean();
if (dataList.length === 0) return;
await mongoSessionRun(async (session) => {
// 插入新数据
const result = await MongoDatasetDataText.insertMany(
dataList.map((item) => ({
teamId: item.teamId,
datasetId: item.datasetId,
collectionId: item.collectionId,
dataId: item._id,
fullTextToken: item.fullTextToken
})),
{ ordered: false, session, lean: true }
);
// 把成功插入的新数据的 dataId 更新为已初始化
await MongoDatasetData.updateMany(
{ _id: { $in: result.map((item) => item.dataId) } },
{ $set: { initFullText: true }, $unset: { fullTextToken: 1 } },
{ session }
);
success += result.length;
console.log('Success:', success);
});
await initData(batchSize);
} catch (error) {
console.log(error, '---');
await delay(500);
await initData(batchSize);
}
};
const batchUpdateFields = async (batchSize = 2000) => {
// Find documents that still have these fields
const documents = await MongoDatasetData.find({ initFullText: { $exists: true } }, '_id')
.limit(batchSize)
.lean();
if (documents.length === 0) return;
// Update in batches
await MongoDatasetData.updateMany(
{ _id: { $in: documents.map((doc) => doc._id) } },
{
$unset: {
initFullText: 1
// fullTextToken: 1
}
}
);
success += documents.length;
console.log('Delete success:', success);
await batchUpdateFields(batchSize);
};

1
projects/app/src/pages/api/common/file/upload.ts
View File

@@ -89,7 +89,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
encoding: file.encoding,
metadata: metadata
});

1
projects/app/src/pages/api/core/dataset/collection/create/localFile.ts
View File

@@ -64,7 +64,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
encoding: file.encoding,
metadata: fileMetadata
});

5
projects/app/src/pages/dataset/list/component/List.tsx
View File

@@ -31,6 +31,7 @@ import { useTranslation } from 'next-i18next';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import SideTag from './SideTag';
import { getModelProvider } from '@fastgpt/global/core/ai/provider';
const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));
@@ -156,6 +157,8 @@ function List() {
>
{formatDatasets.map((dataset, index) => {
const owner = members.find((v) => v.tmbId === dataset.tmbId);
const vectorModelAvatar = getModelProvider(dataset.vectorModel.provider)?.avatar;
return (
<MyTooltip
key={dataset._id}
@@ -281,7 +284,7 @@ function List() {
<HStack>
{isPc && dataset.type !== DatasetTypeEnum.folder && (
<HStack spacing={1} className="time">
<Avatar src={dataset.vectorModel.avatar} w={'0.85rem'} />
<Avatar src={vectorModelAvatar} w={'0.85rem'} />
<Box color={'myGray.500'} fontSize={'mini'}>
{dataset.vectorModel.name}
</Box>

4
projects/app/src/service/core/dataset/data/controller.ts
View File

@@ -89,7 +89,7 @@ export async function insertData2Dataset({
collectionId,
q,
a,
fullTextToken: jiebaSplit({ text: qaStr }),
// fullTextToken: jiebaSplit({ text: qaStr }),
chunkIndex,
indexes: indexes?.map((item, i) => ({
...item,
@@ -241,7 +241,7 @@ export async function updateData2Dataset({
// update mongo other data
mongoData.q = q || mongoData.q;
mongoData.a = a ?? mongoData.a;
mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
// mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
// @ts-ignore
mongoData.indexes = newIndexes;
await mongoData.save({ session });

3
projects/app/src/web/support/user/team/api.ts
View File

@@ -37,8 +37,7 @@ export const delRemoveMember = (tmbId: string) =>
DELETE(`/proApi/support/user/team/member/delete`, { tmbId });
export const updateInviteResult = (data: UpdateInviteProps) =>
PUT('/proApi/support/user/team/member/updateInvite', data);
export const delLeaveTeam = (teamId: string) =>
DELETE('/proApi/support/user/team/member/leave', { teamId });
export const delLeaveTeam = () => DELETE('/proApi/support/user/team/member/leave');
export const getTeamClbs = () =>
GET<ResourcePermissionType[]>(`/proApi/support/user/team/collaborator/list`);