diff --git a/docSite/content/zh-cn/docs/development/upgrading/4818.md b/docSite/content/zh-cn/docs/development/upgrading/4818.md index ae7a5f015..c0e244562 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4818.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4818.md @@ -11,4 +11,6 @@ weight: 806 1. 2. 新增 - 支持部门架构权限模式 -3. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。 \ No newline at end of file +3. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。 +4. 优化 - Mongo 全文索引表分离。 +5. 优化 - 知识库检索查询语句合并，同时减少查库数量。 \ No newline at end of file diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 1b4dae45c..7a772f78d 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -112,6 +112,15 @@ export type DatasetDataSchemaType = { rebuilding?: boolean; }; +export type DatasetDataTextSchemaType = { + _id: string; + teamId: string; + datasetId: string; + collectionId: string; + dataId: string; + fullTextToken: string; +}; + export type DatasetTrainingSchemaType = { _id: string; userId: string; diff --git a/packages/service/support/user/wecom.ts b/packages/global/support/user/login/constants.ts similarity index 53% rename from packages/service/support/user/wecom.ts rename to packages/global/support/user/login/constants.ts index 0e939409e..0093f9367 100644 --- a/packages/service/support/user/wecom.ts +++ b/packages/global/support/user/login/constants.ts @@ -1,3 +1,3 @@ -export function isWecomTerminal() { +export function checkIsWecomTerminal() { return /wxwork/i.test(navigator.userAgent); } diff --git a/packages/global/support/user/team/org/constant.ts b/packages/global/support/user/team/org/constant.ts index 93d3f020a..370cdad82 100644 --- a/packages/global/support/user/team/org/constant.ts +++ b/packages/global/support/user/team/org/constant.ts @@ -3,7 +3,7 @@ import { OrgSchemaType } from './type'; export const OrgCollectionName = 'team_orgs'; export const OrgMemberCollectionName = 'team_org_members'; -export const getChildrenPath = (org: OrgSchemaType) => `${org.path}/${org.pathId}`; +export const getOrgChildrenPath = (org: OrgSchemaType) => `${org.path}/${org.pathId}`; // export enum OrgMemberRole { // owner = 'owner', diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index be569b748..a18065330 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -86,24 +86,21 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ dataId: 1 }, { background: true }); + ChatItemSchema.index({ dataId: 1 }); /* delete by app; delete by chat id; get chat list; get chat logs; close custom feedback; */ - ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true }); + ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }); // admin charts - ChatItemSchema.index({ time: -1, obj: 1 }, { background: true }); + ChatItemSchema.index({ time: -1, obj: 1 }); // timer, clear history - ChatItemSchema.index({ teamId: 1, time: -1 }, { background: true }); + ChatItemSchema.index({ teamId: 1, time: -1 }); // Admin charts - ChatItemSchema.index( - { obj: 1, time: -1 }, - { background: true, partialFilterExpression: { obj: 'Human' } } - ); + ChatItemSchema.index({ obj: 1, time: -1 }, { partialFilterExpression: { obj: 'Human' } }); } catch (error) { console.log(error); } diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 5f5a2f403..53a6569ee 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -81,19 +81,19 @@ const ChatSchema = new Schema({ }); try { - ChatSchema.index({ chatId: 1 }, { background: true }); + ChatSchema.index({ chatId: 1 }); // get user history - ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }, { background: true }); + ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }); // delete by appid; clear history; init chat; update chat; auth chat; get chat; - ChatSchema.index({ appId: 1, chatId: 1 }, { background: true }); + ChatSchema.index({ appId: 1, chatId: 1 }); // get chat logs; - ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }); // get share chat history - ChatSchema.index({ shareId: 1, outLinkUid: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ shareId: 1, outLinkUid: 1, updateTime: -1 }); // timer, clear history - ChatSchema.index({ teamId: 1, updateTime: -1 }, { background: true }); + ChatSchema.index({ teamId: 1, updateTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 6875fb546..061543274 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -24,6 +24,7 @@ import { pushDataListToTrainingQueue } from '../training/controller'; import { MongoImage } from '../../../common/file/image/schema'; import { hashStr } from '@fastgpt/global/common/string/tools'; import { addDays } from 'date-fns'; +import { MongoDatasetDataText } from '../data/dataTextSchema'; export const createCollectionAndInsertData = async ({ dataset, @@ -240,12 +241,12 @@ export const delCollectionRelatedSource = async ({ .map((item) => item?.metadata?.relatedImgId || '') .filter(Boolean); - // delete files + // Delete files await delFileByFileIdList({ bucketName: BucketNameEnum.dataset, fileIdList }); - // delete images + // Delete images await delImgByRelatedId({ teamId, relateIds: relatedImageIds, @@ -273,7 +274,7 @@ export async function delCollection({ const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId)))); const collectionIds = collections.map((item) => String(item._id)); - // delete training data + // Delete training data await MongoDatasetTraining.deleteMany({ teamId, datasetIds: { $in: datasetIds }, @@ -285,11 +286,16 @@ export async function delCollection({ await delCollectionRelatedSource({ collections, session }); } - // delete dataset.datas + // Delete dataset_datas await MongoDatasetData.deleteMany( { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }, { session } ); + // Delete dataset_data_texts + await MongoDatasetDataText.deleteMany( + { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } }, + { session } + ); // delete collections await MongoDatasetCollection.deleteMany( diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index 8ed6a3539..96f6523e7 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -6,6 +6,7 @@ import { ClientSession } from '../../common/mongo'; import { MongoDatasetTraining } from './training/schema'; import { MongoDatasetData } from './data/schema'; import { deleteDatasetDataVector } from '../../common/vectorStore/controller'; +import { MongoDatasetDataText } from './data/dataTextSchema'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ @@ -92,7 +93,7 @@ export async function delDatasetRelevantData({ { session } ).lean(); - // image and file + // Delete Image and file await delCollectionRelatedSource({ collections, session }); // delete collections @@ -101,9 +102,15 @@ export async function delDatasetRelevantData({ datasetId: { $in: datasetIds } }).session(session); - // delete dataset.datas(Not need session) + // No session delete: + // Delete dataset_data_texts + await MongoDatasetDataText.deleteMany({ + teamId, + datasetId: { $in: datasetIds } + }); + // delete dataset_datas await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }); - // no session delete: delete files, vector data + // Delete vector data await deleteDatasetDataVector({ teamId, datasetIds }); } diff --git a/packages/service/core/dataset/data/dataTextSchema.ts b/packages/service/core/dataset/data/dataTextSchema.ts new file mode 100644 index 000000000..5332309d8 --- /dev/null +++ b/packages/service/core/dataset/data/dataTextSchema.ts @@ -0,0 +1,48 @@ +import { connectionMongo, getMongoModel } from '../../../common/mongo'; +const { Schema } = connectionMongo; +import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { DatasetCollectionName } from '../schema'; +import { DatasetColCollectionName } from '../collection/schema'; +import { DatasetDataCollectionName } from './schema'; + +export const DatasetDataTextCollectionName = 'dataset_data_texts'; + +const DatasetDataTextSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + collectionId: { + type: Schema.Types.ObjectId, + ref: DatasetColCollectionName, + required: true + }, + dataId: { + type: String, + ref: DatasetDataCollectionName, + required: true + }, + fullTextToken: { + type: String, + required: true + } +}); + +try { + DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); + DatasetDataTextSchema.index({ dataId: 'hashed' }); +} catch (error) { + console.log(error); +} + +export const MongoDatasetDataText = getMongoModel( + DatasetDataTextCollectionName, + DatasetDataTextSchema +); diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 241f14285..546b481e3 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -71,17 +71,8 @@ const DatasetDataSchema = new Schema({ type: Number, default: 0 }, - inited: { - type: Boolean - }, - rebuilding: Boolean -}); - -DatasetDataSchema.virtual('collection', { - ref: DatasetColCollectionName, - localField: 'collectionId', - foreignField: '_id', - justOne: true + rebuilding: Boolean, + inited: Boolean }); try { @@ -100,6 +91,7 @@ try { DatasetDataSchema.index({ updateTime: 1 }); // rebuild data DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); + DatasetDataSchema.index({ inited: 'hashed' }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index 4b87d2ca7..d6ec87464 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -8,8 +8,8 @@ import { getVectorsByText } from '../../ai/embedding'; import { getVectorModel } from '../../ai/model'; import { MongoDatasetData } from '../data/schema'; import { - DatasetCollectionSchemaType, DatasetDataSchemaType, + DatasetDataTextSchemaType, SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from '../collection/schema'; @@ -23,6 +23,7 @@ import { Types } from '../../../common/mongo'; import json5 from 'json5'; import { MongoDatasetCollectionTags } from '../tag/schema'; import { readFromSecondary } from '../../../common/mongo/utils'; +import { MongoDatasetDataText } from '../data/dataTextSchema'; type SearchDatasetDataProps = { teamId: string; @@ -266,57 +267,60 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { filterCollectionIdList }); - // get q and a - const dataList = await MongoDatasetData.find( - { - teamId, - datasetId: { $in: datasetIds }, - collectionId: { $in: Array.from(new Set(results.map((item) => item.collectionId))) }, - 'indexes.dataId': { $in: results.map((item) => item.id?.trim()) } - }, - 'datasetId collectionId updateTime q a chunkIndex indexes' - ) - .populate<{ collection: DatasetCollectionSchemaType }>( - 'collection', - 'name fileId rawLink externalFileId externalFileUrl' - ) - .lean(); + // Get data and collections + const collectionIdList = Array.from(new Set(results.map((item) => item.collectionId))); + const [dataList, collections] = await Promise.all([ + MongoDatasetData.find( + { + teamId, + datasetId: { $in: datasetIds }, + collectionId: { $in: collectionIdList }, + 'indexes.dataId': { $in: results.map((item) => item.id?.trim()) } + }, + '_id datasetId collectionId updateTime q a chunkIndex indexes', + { ...readFromSecondary } + ).lean(), + MongoDatasetCollection.find( + { + _id: { $in: collectionIdList } + }, + '_id name fileId rawLink externalFileId externalFileUrl', + { ...readFromSecondary } + ).lean() + ]); - // add score to data(It's already sorted. The first one is the one with the most points) - const concatResults = dataList.map((data) => { - const dataIdList = data.indexes.map((item) => item.dataId); + const formatResult = dataList + .map((data, index) => { + const collection = collections.find((col) => String(col._id) === String(data.collectionId)); + if (!collection) { + console.log('Collection is not found', data); + return; + } - const maxScoreResult = results.find((item) => { - return dataIdList.includes(item.id); - }); + // add score to data(It's already sorted. The first one is the one with the most points) + const dataIdList = data.indexes.map((item) => item.dataId); + const maxScoreResult = results.find((item) => { + return dataIdList.includes(item.id); + }); + const score = maxScoreResult?.score || 0; - return { - ...data, - score: maxScoreResult?.score || 0 - }; - }); + const result: SearchDataResponseItemType = { + id: String(data._id), + updateTime: data.updateTime, + q: data.q, + a: data.a, + chunkIndex: data.chunkIndex, + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + ...getCollectionSourceData(collection), + score: [{ type: SearchScoreTypeEnum.embedding, value: score, index }] + }; - concatResults.sort((a, b) => b.score - a.score); + return result; + }) + .filter(Boolean) as SearchDataResponseItemType[]; - const formatResult = concatResults.map((data, index) => { - if (!data.collectionId) { - console.log('Collection is not found', data); - } - - const result: SearchDataResponseItemType = { - id: String(data._id), - updateTime: data.updateTime, - q: data.q, - a: data.a, - chunkIndex: data.chunkIndex, - datasetId: String(data.datasetId), - collectionId: String(data.collectionId), - ...getCollectionSourceData(data.collection), - score: [{ type: SearchScoreTypeEnum.embedding, value: data.score, index }] - }; - - return result; - }); + formatResult.sort((a, b) => b.score[0].value - a.score[0].value); return { embeddingRecallResults: formatResult, @@ -344,88 +348,114 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { }; } - let searchResults = ( + const searchResults = ( await Promise.all( datasetIds.map(async (id) => { - return MongoDatasetData.aggregate([ - { - $match: { - teamId: new Types.ObjectId(teamId), - datasetId: new Types.ObjectId(id), - $text: { $search: jiebaSplit({ text: query }) }, - ...(filterCollectionIdList - ? { - collectionId: { - $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + return MongoDatasetData.aggregate( + [ + { + $match: { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(id), + $text: { $search: jiebaSplit({ text: query }) }, + ...(filterCollectionIdList + ? { + collectionId: { + $in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) + } } - } - : {}), - ...(forbidCollectionIdList && forbidCollectionIdList.length > 0 - ? { - collectionId: { - $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) + : {}), + ...(forbidCollectionIdList && forbidCollectionIdList.length > 0 + ? { + collectionId: { + $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) + } } - } - : {}) + : {}) + } + }, + { + $sort: { + score: { $meta: 'textScore' } + } + }, + { + $limit: limit + }, + { + $project: { + _id: 1, + datasetId: 1, + collectionId: 1, + updateTime: 1, + q: 1, + a: 1, + chunkIndex: 1, + score: { $meta: 'textScore' } + } } - }, + ], { - $addFields: { - score: { $meta: 'textScore' } - } - }, - { - $sort: { - score: { $meta: 'textScore' } - } - }, - { - $limit: limit - }, - { - $project: { - _id: 1, - datasetId: 1, - collectionId: 1, - updateTime: 1, - q: 1, - a: 1, - chunkIndex: 1, - score: 1 - } + ...readFromSecondary } - ]); + ); }) ) ).flat() as (DatasetDataSchemaType & { score: number })[]; - // resort - searchResults.sort((a, b) => b.score - a.score); - searchResults.slice(0, limit); - + // Get data and collections const collections = await MongoDatasetCollection.find( { _id: { $in: searchResults.map((item) => item.collectionId) } }, - '_id name fileId rawLink' - ); + '_id name fileId rawLink externalFileId externalFileUrl', + { ...readFromSecondary } + ).lean(); + // const [dataList, collections] = await Promise.all([ + // MongoDatasetData.find( + // { + // _id: { $in: searchResults.map((item) => item.dataId) } + // }, + // '_id datasetId collectionId updateTime q a chunkIndex indexes', + // { ...readFromSecondary } + // ).lean(), + // MongoDatasetCollection.find( + // { + // _id: { $in: searchResults.map((item) => item.collectionId) } + // }, + // '_id name fileId rawLink externalFileId externalFileUrl', + // { ...readFromSecondary } + // ).lean() + // ]); return { - fullTextRecallResults: searchResults.map((item, index) => { - const collection = collections.find((col) => String(col._id) === String(item.collectionId)); - return { - id: String(item._id), - datasetId: String(item.datasetId), - collectionId: String(item.collectionId), - updateTime: item.updateTime, - ...getCollectionSourceData(collection), - q: item.q, - a: item.a, - chunkIndex: item.chunkIndex, - indexes: item.indexes, - score: [{ type: SearchScoreTypeEnum.fullText, value: item.score, index }] - }; - }), + fullTextRecallResults: searchResults + .map((data, index) => { + const collection = collections.find( + (col) => String(col._id) === String(data.collectionId) + ); + if (!collection) { + console.log('Collection is not found', data); + return; + } + + // const score = + // searchResults.find((item) => String(item.dataId) === String(data._id))?.score || 0; + + return { + id: String(data._id), + datasetId: String(data.datasetId), + collectionId: String(data.collectionId), + updateTime: data.updateTime, + q: data.q, + a: data.a, + chunkIndex: data.chunkIndex, + indexes: data.indexes, + ...getCollectionSourceData(collection), + score: [{ type: SearchScoreTypeEnum.fullText, value: data.score ?? 0, index }] + }; + }) + .filter(Boolean) as SearchDataResponseItemType[], tokenLen: 0 }; }; diff --git a/packages/service/support/permission/org/controllers.ts b/packages/service/support/permission/org/controllers.ts index b58736b74..a19c0e7c6 100644 --- a/packages/service/support/permission/org/controllers.ts +++ b/packages/service/support/permission/org/controllers.ts @@ -3,7 +3,8 @@ import type { OrgSchemaType } from '@fastgpt/global/support/user/team/org/type'; import type { ClientSession } from 'mongoose'; import { MongoOrgModel } from './orgSchema'; import { MongoOrgMemberModel } from './orgMemberSchema'; -import { getChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; +import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; export const getOrgsByTmbId = async ({ teamId, tmbId }: { teamId: string; tmbId: string }) => MongoOrgMemberModel.find({ teamId, tmbId }, 'orgId').lean(); @@ -43,9 +44,13 @@ export const getChildrenByOrg = async ({ teamId: string; session?: ClientSession; }) => { - return MongoOrgModel.find({ teamId, path: { $regex: `^${getChildrenPath(org)}` } }, undefined, { - session - }).lean(); + return MongoOrgModel.find( + { teamId, path: { $regex: `^${getOrgChildrenPath(org)}` } }, + undefined, + { + session + } + ).lean(); }; export const getOrgAndChildren = async ({ diff --git a/packages/service/support/permission/org/orgSchema.ts b/packages/service/support/permission/org/orgSchema.ts index 18d3e1a83..45030c4a6 100644 --- a/packages/service/support/permission/org/orgSchema.ts +++ b/packages/service/support/permission/org/orgSchema.ts @@ -6,10 +6,6 @@ import { OrgMemberCollectionName } from './orgMemberSchema'; import { getNanoid } from '@fastgpt/global/common/string/tools'; const { Schema } = connectionMongo; -function requiredStringPath(this: OrgSchemaType) { - return typeof this.path !== 'string'; -} - export const OrgSchema = new Schema( { teamId: { @@ -25,7 +21,9 @@ export const OrgSchema = new Schema( }, path: { type: String, - required: requiredStringPath // allow empty string, but not null + required: function (this: OrgSchemaType) { + return typeof this.path !== 'string'; + } // allow empty string, but not null }, name: { type: String, diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index 89b5eb58d..c54245c9b 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -79,10 +79,10 @@ const UserSchema = new Schema({ try { // login - UserSchema.index({ username: 1, password: 1 }, { background: true }); + UserSchema.index({ username: 1 }); // Admin charts - UserSchema.index({ createTime: -1 }, { background: true }); + UserSchema.index({ createTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/support/wallet/usage/schema.ts b/packages/service/support/wallet/usage/schema.ts index fc1a3b500..7416d6b45 100644 --- a/packages/service/support/wallet/usage/schema.ts +++ b/packages/service/support/wallet/usage/schema.ts @@ -61,11 +61,11 @@ const UsageSchema = new Schema({ }); try { - UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }, { background: true }); + UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 }); // timer task. clear dead team - // UsageSchema.index({ teamId: 1, time: -1 }, { background: true }); + // UsageSchema.index({ teamId: 1, time: -1 }); - UsageSchema.index({ time: 1 }, { background: true, expireAfterSeconds: 360 * 24 * 60 * 60 }); + UsageSchema.index({ time: 1 }, { expireAfterSeconds: 360 * 24 * 60 * 60 }); } catch (error) { console.log(error); } diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 890e5b1bd..e7e87d5cd 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -31,6 +31,7 @@ export const iconPaths = { 'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'), 'common/data': () => import('./icons/common/data.svg'), 'common/dingtalkFill': () => import('./icons/common/dingtalkFill.svg'), + 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), @@ -73,7 +74,6 @@ export const iconPaths = { 'common/resultLight': () => import('./icons/common/resultLight.svg'), 'common/retryLight': () => import('./icons/common/retryLight.svg'), 'common/rightArrowFill': () => import('./icons/common/rightArrowFill.svg'), - 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), 'common/rightArrowLight': () => import('./icons/common/rightArrowLight.svg'), 'common/routePushLight': () => import('./icons/common/routePushLight.svg'), 'common/saveFill': () => import('./icons/common/saveFill.svg'), @@ -99,6 +99,7 @@ export const iconPaths = { 'common/wallet': () => import('./icons/common/wallet.svg'), 'common/warn': () => import('./icons/common/warn.svg'), 'common/wechatFill': () => import('./icons/common/wechatFill.svg'), + 'common/wecom': () => import('./icons/common/wecom.svg'), configmap: () => import('./icons/configmap.svg'), copy: () => import('./icons/copy.svg'), 'core/app/aiFill': () => import('./icons/core/app/aiFill.svg'), diff --git a/packages/web/components/common/Icon/icons/common/wecom.svg b/packages/web/components/common/Icon/icons/common/wecom.svg new file mode 100644 index 000000000..1345e42ca --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/wecom.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/projects/app/src/components/support/wallet/NotSufficientModal/index.tsx b/projects/app/src/components/support/wallet/NotSufficientModal/index.tsx index 12d49f04c..fa4590428 100644 --- a/projects/app/src/components/support/wallet/NotSufficientModal/index.tsx +++ b/projects/app/src/components/support/wallet/NotSufficientModal/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useMemo, useState } from 'react'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { useTranslation } from 'next-i18next'; import { Box, Button, Flex, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'; @@ -10,6 +10,7 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useUserStore } from '@/web/support/user/useUserStore'; import { standardSubLevelMap } from '@fastgpt/global/support/wallet/sub/constants'; import { TeamErrEnum } from '@fastgpt/global/common/error/code/team'; +import { useMount } from 'ahooks'; const NotSufficientModal = ({ type }: { type: NotSufficientModalType }) => { const { t } = useTranslation(); @@ -73,11 +74,9 @@ const RechargeModal = ({ const { t } = useTranslation(); const { teamPlanStatus, initTeamPlanStatus } = useUserStore(); - useEffect(() => { - (async () => { - await initTeamPlanStatus(); - })(); - }, [initTeamPlanStatus]); + useMount(() => { + initTeamPlanStatus(); + }); const planName = useMemo(() => { if (!teamPlanStatus?.standard?.currentSubLevel) return ''; @@ -94,8 +93,8 @@ const RechargeModal = ({ title={t('common:user.Pay')} onClose={onClose} isCentered - minW={['100%', '1200px']} - minH={['100%', '800px']} + minW={'90%'} + maxH={'90%'} > diff --git a/projects/app/src/components/support/wallet/QRCodePayModal.tsx b/projects/app/src/components/support/wallet/QRCodePayModal.tsx index 379f7d135..6b3fe5bd5 100644 --- a/projects/app/src/components/support/wallet/QRCodePayModal.tsx +++ b/projects/app/src/components/support/wallet/QRCodePayModal.tsx @@ -1,5 +1,5 @@ import MyModal from '@fastgpt/web/components/common/MyModal'; -import React, { useEffect, useRef } from 'react'; +import React, { useCallback, useEffect, useRef } from 'react'; import { useTranslation } from 'next-i18next'; import { Box, ModalBody } from '@chakra-ui/react'; import { checkBalancePayResult } from '@/web/support/wallet/bill/api'; @@ -25,25 +25,25 @@ const QRCodePayModal = ({ billId, onSuccess }: QRPayProps & { tip?: string; onSuccess?: () => any }) => { - const router = useRouter(); const { t } = useTranslation(); const { toast } = useToast(); const dom = useRef(null); + const drawCode = useCallback(() => { + if (dom.current && window.QRCode && !dom.current.innerHTML) { + new window.QRCode(dom.current, { + text: codeUrl, + width: qrCodeSize, + height: qrCodeSize, + colorDark: '#000000', + colorLight: '#ffffff', + correctLevel: window.QRCode.CorrectLevel.H + }); + } + }, [codeUrl]); + useEffect(() => { let timer: NodeJS.Timeout; - const drawCode = () => { - if (dom.current && window.QRCode && !dom.current.innerHTML) { - new window.QRCode(dom.current, { - text: codeUrl, - width: qrCodeSize, - height: qrCodeSize, - colorDark: '#000000', - colorLight: '#ffffff', - correctLevel: window.QRCode.CorrectLevel.H - }); - } - }; const check = async () => { try { const res = await checkBalancePayResult(billId); @@ -54,9 +54,6 @@ const QRCodePayModal = ({ title: res, status: 'success' }); - setTimeout(() => { - router.reload(); - }, 1000); return; } catch (error) { toast({ @@ -75,11 +72,15 @@ const QRCodePayModal = ({ check(); return () => clearTimeout(timer); - }, [billId, onSuccess, toast]); + }, [billId, drawCode, onSuccess, toast]); return ( <> - + diff --git a/projects/app/src/pages/account/components/TeamSelector.tsx b/projects/app/src/pages/account/components/TeamSelector.tsx index 27ef88953..c3cd03c2a 100644 --- a/projects/app/src/pages/account/components/TeamSelector.tsx +++ b/projects/app/src/pages/account/components/TeamSelector.tsx @@ -75,7 +75,7 @@ const TeamSelector = ({ key={'manage'} alignItems={'center'} borderRadius={'md'} - cursor={'default'} + cursor={'pointer'} gap={3} onClick={() => router.push('/account/team')} > diff --git a/projects/app/src/pages/account/team/components/OrgManage/OrgTree.tsx b/projects/app/src/pages/account/team/components/OrgManage/OrgTree.tsx index 2305f6bb4..bd7030c4a 100644 --- a/projects/app/src/pages/account/team/components/OrgManage/OrgTree.tsx +++ b/projects/app/src/pages/account/team/components/OrgManage/OrgTree.tsx @@ -4,7 +4,7 @@ import Avatar from '@fastgpt/web/components/common/Avatar'; import { useToggle } from 'ahooks'; import { useMemo } from 'react'; import IconButton from './IconButton'; -import { getChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; +import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; function OrgTreeNode({ org, @@ -20,7 +20,7 @@ function OrgTreeNode({ index?: number; }) { const children = useMemo( - () => list.filter((item) => item.path === getChildrenPath(org)), + () => list.filter((item) => item.path === getOrgChildrenPath(org)), [org, list] ); const [isExpanded, toggleIsExpanded] = useToggle(index === 0); diff --git a/projects/app/src/pages/account/team/components/OrgManage/index.tsx b/projects/app/src/pages/account/team/components/OrgManage/index.tsx index 3388df440..b4fadcc70 100644 --- a/projects/app/src/pages/account/team/components/OrgManage/index.tsx +++ b/projects/app/src/pages/account/team/components/OrgManage/index.tsx @@ -36,7 +36,7 @@ import dynamic from 'next/dynamic'; import MyBox from '@fastgpt/web/components/common/MyBox'; import Path from '@/components/common/folder/Path'; import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; -import { getChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; +import { getOrgChildrenPath } from '@fastgpt/global/support/user/team/org/constant'; const OrgInfoModal = dynamic(() => import('./OrgInfoModal')); const OrgMemberManageModal = dynamic(() => import('./OrgMemberManageModal')); @@ -88,17 +88,20 @@ function OrgTable() { }); const currentOrgs = useMemo(() => { if (orgs.length === 0) return []; + // Auto select the first org(root org is team) if (parentPath === '') { - setParentPath(`/${orgs[0].pathId}`); + setParentPath(getOrgChildrenPath(orgs[0])); return []; } + return orgs .filter((org) => org.path === parentPath) .map((item) => { return { ...item, + // Member + org count: - item.members.length + orgs.filter((org) => org.path === getChildrenPath(item)).length + item.members.length + orgs.filter((org) => org.path === getOrgChildrenPath(item)).length }; }); }, [orgs, parentPath]); @@ -109,6 +112,7 @@ function OrgTable() { return orgs.find((org) => org.pathId === currentOrgId); }, [orgs, parentPath]); + const paths = useMemo(() => { const splitPath = parentPath.split('/').filter(Boolean); return splitPath @@ -118,7 +122,7 @@ function OrgTable() { if (org.path === '') return; return { - parentId: getChildrenPath(org), + parentId: getOrgChildrenPath(org), parentName: org.name }; }) @@ -175,7 +179,10 @@ function OrgTable() { {currentOrgs.map((org) => ( - setParentPath(getChildrenPath(org))}> + setParentPath(getOrgChildrenPath(org))} + > {org.count} + setManageMemberOrg(currentOrg)} + /> {currentOrg?.path !== '' && ( <> - setManageMemberOrg(currentOrg)} - /> {item.text} - + {t(formatTimeToChatTime(item.time) as any).replace('#', ':')} - + { e.stopPropagation(); diff --git a/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx b/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx index b4ffd2dce..5a5a5cb33 100644 --- a/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx +++ b/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx @@ -3,17 +3,16 @@ import { useSystemStore } from '@/web/common/system/useSystemStore'; import { AbsoluteCenter, Box, Button, Flex } from '@chakra-ui/react'; import { LOGO_ICON } from '@fastgpt/global/common/system/constants'; import { OAuthEnum } from '@fastgpt/global/support/user/constant'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import { customAlphabet } from 'nanoid'; import { useRouter } from 'next/router'; import { Dispatch, useCallback, useEffect, useMemo, useRef } from 'react'; import { useTranslation } from 'next-i18next'; import I18nLngSelector from '@/components/Select/I18nLngSelector'; import { useSystem } from '@fastgpt/web/hooks/useSystem'; import MyImage from '@fastgpt/web/components/common/Image/MyImage'; -import { isWecomTerminal } from '@fastgpt/service/support/user/wecom'; -import { PUT } from '@fastgpt/service/common/api/plusRequest'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 8); +import { checkIsWecomTerminal } from '@fastgpt/global/support/user/login/constants'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import Avatar from '@fastgpt/web/components/common/Avatar'; +import dynamic from 'next/dynamic'; interface Props { children: React.ReactNode; @@ -21,18 +20,38 @@ interface Props { pageType: `${LoginPageTypeEnum}`; } +type OAuthItem = { + label: string; + provider: OAuthEnum | LoginPageTypeEnum; + icon: any; + pageType?: LoginPageTypeEnum; + redirectUrl?: string; +}; + const FormLayout = ({ children, setPageType, pageType }: Props) => { const { t } = useTranslation(); const router = useRouter(); const { setLoginStore, feConfigs } = useSystemStore(); - const { lastRoute = '/app/list' } = router.query as { lastRoute: string }; - const state = useRef(nanoid()); - const redirectUri = `${location.origin}/login/provider`; const { isPc } = useSystem(); - const isWecom = isWecomTerminal(); - const oAuthList = [ + const { lastRoute = '/app/list' } = router.query as { lastRoute: string }; + const state = useRef(getNanoid(8)); + const redirectUri = `${location.origin}/login/provider`; + + const isWecomWorkTerminal = checkIsWecomTerminal(); + + const oAuthList: OAuthItem[] = [ + ...(feConfigs?.sso?.url + ? [ + { + label: feConfigs.sso.title || 'Unknown', + provider: OAuthEnum.sso, + icon: feConfigs.sso.icon, + redirectUrl: `${feConfigs.sso.url}/login/oauth/authorize?redirect_uri=${encodeURIComponent(redirectUri)}&state=${state.current}` + } + ] + : []), ...(feConfigs?.oauth?.wechat && pageType !== LoginPageTypeEnum.wechat ? [ { @@ -90,7 +109,7 @@ const FormLayout = ({ children, setPageType, pageType }: Props) => { label: t('login:wecom'), provider: OAuthEnum.wecom, icon: 'common/wecom', - redirectUrl: isWecom + redirectUrl: isWecomWorkTerminal ? `https://open.weixin.qq.com/connect/oauth2/authorize?appid=${feConfigs?.oauth?.wecom?.corpid}&redirect_uri=${redirectUri}&response_type=code&scope=snsapi_base&agentid=${feConfigs?.oauth?.wecom?.agentid}&state=${state.current}#wechat_redirect` : `https://login.work.weixin.qq.com/wwlogin/sso/login?login_type=CorpApp&appid=${feConfigs?.oauth?.wecom?.corpid}&agentid=${feConfigs?.oauth?.wecom?.agentid}&redirect_uri=${redirectUri}&state=${state.current}` } @@ -114,39 +133,31 @@ const FormLayout = ({ children, setPageType, pageType }: Props) => { ); const onClickOauth = useCallback( - async (item: any) => { - item.redirectUrl && + async (item: OAuthItem) => { + if (item.redirectUrl) { setLoginStore({ - provider: item.provider, + provider: item.provider as OAuthEnum, lastRoute, state: state.current }); - item.redirectUrl && router.replace(item.redirectUrl, '_self'); + router.replace(item.redirectUrl, '_self'); + } item.pageType && setPageType(item.pageType); }, - [lastRoute, setLoginStore, setPageType] + [lastRoute, router, setLoginStore, setPageType] ); - const onClickSso = useCallback(() => { - if (!feConfigs?.sso?.url) return; - setLoginStore({ - provider: OAuthEnum.sso, - lastRoute, - state: state.current - }); - const url = `${feConfigs.sso.url}/login/oauth/authorize?redirect_uri=${encodeURIComponent(redirectUri)}&state=${state.current}`; - - window.open(url, '_self'); - }, [feConfigs?.sso?.url, lastRoute, redirectUri, setLoginStore]); - useEffect(() => { - if (feConfigs?.sso?.autoLogin) { - onClickSso(); + const sso = oAuthList.find((item) => item.provider === OAuthEnum.sso); + const wecom = oAuthList.find((item) => item.provider === OAuthEnum.wecom); + if (feConfigs?.sso?.autoLogin && sso) { + // sso auto + onClickOauth(sso); + } else if (isWecomWorkTerminal && wecom) { + // Auto wecom login + onClickOauth(wecom); } - if (isWecom) { - onClickOauth(oAuthList.find((item) => item.provider === OAuthEnum.wecom)); - } - }, [feConfigs?.sso?.autoLogin, isWecom]); + }, [feConfigs?.sso?.autoLogin, isWecomWorkTerminal, onClickOauth]); return ( @@ -189,28 +200,13 @@ const FormLayout = ({ children, setPageType, pageType }: Props) => { h={'40px'} borderRadius={'sm'} fontWeight={'medium'} - leftIcon={} + leftIcon={} onClick={() => onClickOauth(item)} > {item.label} ))} - - {feConfigs?.sso?.url && ( - - - - )} )} @@ -218,4 +214,6 @@ const FormLayout = ({ children, setPageType, pageType }: Props) => { ); }; -export default FormLayout; +export default dynamic(() => Promise.resolve(FormLayout), { + ssr: false +}); diff --git a/projects/app/src/pages/login/index.tsx b/projects/app/src/pages/login/index.tsx index 460252bd9..84807bdfa 100644 --- a/projects/app/src/pages/login/index.tsx +++ b/projects/app/src/pages/login/index.tsx @@ -16,7 +16,6 @@ import type { ResLogin } from '@/global/support/api/userRes.d'; import { useRouter } from 'next/router'; import { useUserStore } from '@/web/support/user/useUserStore'; import { useChatStore } from '@/web/core/chat/context/useChatStore'; -import LoginForm from './components/LoginForm/LoginForm'; import dynamic from 'next/dynamic'; import { serviceSideProps } from '@fastgpt/web/common/system/nextjs'; import { clearToken, setToken } from '@/web/support/user/auth'; @@ -29,6 +28,7 @@ import { useSystem } from '@fastgpt/web/hooks/useSystem'; import { GET } from '@/web/common/api/request'; import { getDocPath } from '@/web/common/system/doc'; import { getWebReqUrl } from '@fastgpt/web/common/system/utils'; +import LoginForm from './components/LoginForm/LoginForm'; const RegisterForm = dynamic(() => import('./components/RegisterForm')); const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm')); @@ -42,7 +42,7 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => { const { t } = useTranslation(); const { lastRoute = '' } = router.query as { lastRoute: string }; const { feConfigs } = useSystemStore(); - const [pageType, setPageType] = useState<`${LoginPageTypeEnum}`>(); + const [pageType, setPageType] = useState<`${LoginPageTypeEnum}`>(LoginPageTypeEnum.passwordLogin); const { setUserInfo } = useUserStore(); const { setLastChatAppId } = useChatStore(); const { isOpen, onOpen, onClose } = useDisclosure(); diff --git a/projects/app/src/pages/price/index.tsx b/projects/app/src/pages/price/index.tsx index c454c0dac..7084e775f 100644 --- a/projects/app/src/pages/price/index.tsx +++ b/projects/app/src/pages/price/index.tsx @@ -13,16 +13,24 @@ import { getToken } from '@/web/support/user/auth'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useRouter } from 'next/router'; const PriceBox = () => { const { userInfo } = useUserStore(); const { t } = useTranslation(); const { feConfigs } = useSystemStore(); + const router = useRouter(); const { data: teamSubPlan } = useQuery(['getTeamPlanStatus'], getTeamPlanStatus, { enabled: !!getToken() || !!userInfo }); + const onPaySuccess = () => { + setTimeout(() => { + router.reload(); + }, 1000); + }; + return ( { title: feConfigs?.systemTitle })} - + @@ -62,7 +70,7 @@ const PriceBox = () => { {t('common:support.wallet.subscription.Extra plan tip')} - + {/* points */} diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 1dd5ea863..24c6b5926 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -12,6 +12,7 @@ import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; import { getVectorModel } from '@fastgpt/service/core/ai/model'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { ClientSession } from '@fastgpt/service/common/mongo'; +import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema'; /* insert data. * 1. create data id @@ -42,7 +43,8 @@ export async function insertData2Dataset({ const qaStr = getDefaultIndex({ q, a }).text; - // empty indexes check, if empty, create default index + // 1. Get vector indexes and insert + // Empty indexes check, if empty, create default index indexes = Array.isArray(indexes) && indexes.length > 0 ? indexes.map((index) => ({ @@ -77,7 +79,7 @@ export async function insertData2Dataset({ ) ); - // create mongo data + // 2. Create mongo data const [{ _id }] = await MongoDatasetData.create( [ { @@ -98,6 +100,20 @@ export async function insertData2Dataset({ { session } ); + // 3. Create mongo data text + await MongoDatasetDataText.create( + [ + { + teamId, + datasetId, + collectionId, + dataId: _id, + fullTextToken: jiebaSplit({ text: qaStr }) + } + ], + { session } + ); + return { insertId: _id, tokens: result.reduce((acc, cur) => acc + cur.tokens, 0) @@ -225,11 +241,18 @@ export async function updateData2Dataset({ // update mongo other data mongoData.q = q || mongoData.q; mongoData.a = a ?? mongoData.a; - mongoData.fullTextToken = jiebaSplit({ text: mongoData.q + mongoData.a }); + mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }); // @ts-ignore mongoData.indexes = newIndexes; await mongoData.save({ session }); + // update mongo data text + await MongoDatasetDataText.updateOne( + { dataId: mongoData._id }, + { fullTextToken: jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) }, + { session } + ); + // delete vector const deleteIdList = patchResult .filter((item) => item.type === 'delete' || item.type === 'update') diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts index e17a51259..a84d80479 100644 --- a/projects/app/src/service/events/generateVector.ts +++ b/projects/app/src/service/events/generateVector.ts @@ -166,9 +166,9 @@ const rebuildData = async ({ // get new mongoData insert to training const newRebuildingData = await MongoDatasetData.findOneAndUpdate( { + rebuilding: true, teamId: mongoData.teamId, - datasetId: mongoData.datasetId, - rebuilding: true + datasetId: mongoData.datasetId }, { $unset: { diff --git a/projects/app/src/web/common/system/useSystemStore.ts b/projects/app/src/web/common/system/useSystemStore.ts index da6bbe367..656a762d1 100644 --- a/projects/app/src/web/common/system/useSystemStore.ts +++ b/projects/app/src/web/common/system/useSystemStore.ts @@ -42,8 +42,8 @@ type State = { gitStar: number; loadGitStar: () => Promise; - notSufficientModalType: NotSufficientModalType | undefined; - setNotSufficientModalType: (val: NotSufficientModalType | undefined) => void; + notSufficientModalType?: NotSufficientModalType; + setNotSufficientModalType: (val?: NotSufficientModalType) => void; initDataBufferId?: string; feConfigs: FastGPTFeConfigsType; @@ -115,7 +115,7 @@ export const useSystemStore = create()( }, notSufficientModalType: undefined, - setNotSufficientModalType(type: NotSufficientModalType | undefined) { + setNotSufficientModalType(type) { set((state) => { state.notSufficientModalType = type; });