perf: usages list；perf: move components (#3654)

* perf: usages list

* team sub plan load

* perf: usage dashboard code

* perf: dashboard ui

* perf: move components

packages/service/common/middle/reqFrequencyLimit.ts

@@ -9,7 +9,7 @@ import { jsonRes } from '../response';
 // unit: times/s
 // how to use?
 // export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip
-export function useReqFrequencyLimit(seconds: number, limit: number, force = false) {
+export function useIPFrequencyLimit(seconds: number, limit: number, force = false) {
   return async (req: ApiRequestProps, res: NextApiResponse) => {
     const ip = requestIp.getClientIp(req);
     if (!ip || (process.env.USE_IP_LIMIT !== 'true' && !force)) {

packages/service/core/dataset/search/controller.ts

@@ -348,119 +348,6 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
       };
     }
 
-    const searchResults = (
-      await Promise.all(
-        datasetIds.map(async (id) => {
-          return MongoDatasetData.aggregate(
-            [
-              {
-                $match: {
-                  teamId: new Types.ObjectId(teamId),
-                  datasetId: new Types.ObjectId(id),
-                  $text: { $search: jiebaSplit({ text: query }) },
-                  ...(filterCollectionIdList
-                    ? {
-                        collectionId: {
-                          $in: filterCollectionIdList.map((id) => new Types.ObjectId(id))
-                        }
-                      }
-                    : {}),
-                  ...(forbidCollectionIdList && forbidCollectionIdList.length > 0
-                    ? {
-                        collectionId: {
-                          $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
-                        }
-                      }
-                    : {})
-                }
-              },
-              {
-                $sort: {
-                  score: { $meta: 'textScore' }
-                }
-              },
-              {
-                $limit: limit
-              },
-              {
-                $project: {
-                  _id: 1,
-                  datasetId: 1,
-                  collectionId: 1,
-                  updateTime: 1,
-                  q: 1,
-                  a: 1,
-                  chunkIndex: 1,
-                  score: { $meta: 'textScore' }
-                }
-              }
-            ],
-            {
-              ...readFromSecondary
-            }
-          );
-        })
-      )
-    ).flat() as (DatasetDataSchemaType & { score: number })[];
-
-    // Get data and collections
-    const collections = await MongoDatasetCollection.find(
-      {
-        _id: { $in: searchResults.map((item) => item.collectionId) }
-      },
-      '_id name fileId rawLink externalFileId externalFileUrl',
-      { ...readFromSecondary }
-    ).lean();
-
-    return {
-      fullTextRecallResults: searchResults
-        .map((data, index) => {
-          const collection = collections.find(
-            (col) => String(col._id) === String(data.collectionId)
-          );
-          if (!collection) {
-            console.log('Collection is not found', data);
-            return;
-          }
-
-          return {
-            id: String(data._id),
-            datasetId: String(data.datasetId),
-            collectionId: String(data.collectionId),
-            updateTime: data.updateTime,
-            q: data.q,
-            a: data.a,
-            chunkIndex: data.chunkIndex,
-            indexes: data.indexes,
-            ...getCollectionSourceData(collection),
-            score: [{ type: SearchScoreTypeEnum.fullText, value: data.score ?? 0, index }]
-          };
-        })
-        .filter(Boolean) as SearchDataResponseItemType[],
-      tokenLen: 0
-    };
-  };
-  const fullTextRecall2 = async ({
-    query,
-    limit,
-    filterCollectionIdList,
-    forbidCollectionIdList
-  }: {
-    query: string;
-    limit: number;
-    filterCollectionIdList?: string[];
-    forbidCollectionIdList: string[];
-  }): Promise<{
-    fullTextRecallResults: SearchDataResponseItemType[];
-    tokenLen: number;
-  }> => {
-    if (limit === 0) {
-      return {
-        fullTextRecallResults: [],
-        tokenLen: 0
-      };
-    }
-
     const searchResults = (
       await Promise.all(
         datasetIds.map(async (id) => {
@@ -637,7 +524,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
             filterCollectionIdList
           }),
           // FullText tmp
-          fullTextRecall2({
+          fullTextRecall({
             query,
             limit: fullTextLimit,
             filterCollectionIdList,

packages/service/support/wallet/usage/schema.ts

@@ -61,7 +61,8 @@ const UsageSchema = new Schema({
 });
 
 try {
-  UsageSchema.index({ teamId: 1, tmbId: 1, source: 1, time: -1 });
+  UsageSchema.index({ teamId: 1, time: 1, tmbId: 1, source: 1 });
+  UsageSchema.index({ teamId: 1, time: 1, appName: 1 });
   // timer task. clear dead team
   // UsageSchema.index({ teamId: 1, time: -1 });