Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

packages/service/common/mongo/index.ts

@@ -1,17 +1,26 @@
 import { addLog } from '../../common/system/log';
-import mongoose, { Model } from 'mongoose';
+import mongoose, { Model, Mongoose } from 'mongoose';
 
 export default mongoose;
 export * from 'mongoose';
 
+export const MONGO_URL = process.env.MONGODB_URI as string;
+export const MONGO_LOG_URL = (process.env.MONGODB_LOG_URI ?? process.env.MONGODB_URI) as string;
+
 export const connectionMongo = (() => {
   if (!global.mongodb) {
-    global.mongodb = mongoose;
+    global.mongodb = new Mongoose();
   }
-
   return global.mongodb;
 })();
 
+export const connectionLogMongo = (() => {
+  if (!global.mongodbLog) {
+    global.mongodbLog = new Mongoose();
+  }
+  return global.mongodbLog;
+})();
+
 const addCommonMiddleware = (schema: mongoose.Schema) => {
   const operations = [
     /^find/,
@@ -71,6 +80,19 @@ export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
   return model;
 };
 
+export const getMongoLogModel = <T>(name: string, schema: mongoose.Schema) => {
+  if (connectionLogMongo.models[name]) return connectionLogMongo.models[name] as Model<T>;
+  console.log('Load model======', name);
+  addCommonMiddleware(schema);
+
+  const model = connectionLogMongo.model<T>(name, schema);
+
+  // Sync index
+  syncMongoIndex(model);
+
+  return model;
+};
+
 const syncMongoIndex = async (model: Model<any>) => {
   if (process.env.SYNC_INDEX !== '0' && process.env.NODE_ENV !== 'test') {
     try {

packages/service/common/mongo/init.ts

@@ -1,6 +1,5 @@
 import { delay } from '@fastgpt/global/common/system/utils';
 import { addLog } from '../system/log';
-import { connectionMongo } from './index';
 import type { Mongoose } from 'mongoose';
 
 const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
@@ -8,41 +7,41 @@ const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
 /**
  * connect MongoDB and init data
  */
-export async function connectMongo(): Promise<Mongoose> {
+export async function connectMongo(db: Mongoose, url: string): Promise<Mongoose> {
   /* Connecting, connected will return */
-  if (connectionMongo.connection.readyState !== 0) {
-    return connectionMongo;
+  if (db.connection.readyState !== 0) {
+    return db;
   }
 
-  console.log('mongo start connect');
+  console.log('MongoDB start connect');
   try {
     // Remove existing listeners to prevent duplicates
-    connectionMongo.connection.removeAllListeners('error');
-    connectionMongo.connection.removeAllListeners('disconnected');
-    connectionMongo.set('strictQuery', 'throw');
+    db.connection.removeAllListeners('error');
+    db.connection.removeAllListeners('disconnected');
+    db.set('strictQuery', 'throw');
 
-    connectionMongo.connection.on('error', async (error) => {
+    db.connection.on('error', async (error) => {
       console.log('mongo error', error);
       try {
-        if (connectionMongo.connection.readyState !== 0) {
-          await connectionMongo.disconnect();
+        if (db.connection.readyState !== 0) {
+          await db.disconnect();
           await delay(1000);
-          await connectMongo();
+          await connectMongo(db, url);
         }
       } catch (error) {}
     });
-    connectionMongo.connection.on('disconnected', async () => {
+    db.connection.on('disconnected', async () => {
       console.log('mongo disconnected');
       try {
-        if (connectionMongo.connection.readyState !== 0) {
-          await connectionMongo.disconnect();
+        if (db.connection.readyState !== 0) {
+          await db.disconnect();
           await delay(1000);
-          await connectMongo();
+          await connectMongo(db, url);
         }
       } catch (error) {}
     });
 
-    await connectionMongo.connect(process.env.MONGODB_URI as string, {
+    const options = {
       bufferCommands: true,
       maxConnecting: maxConnecting,
       maxPoolSize: maxConnecting,
@@ -53,18 +52,18 @@ export async function connectMongo(): Promise<Mongoose> {
       maxIdleTimeMS: 300000,
       retryWrites: true,
       retryReads: true
+    };
 
-      // readPreference: 'secondaryPreferred',
-      // readConcern: { level: 'local' },
-      // writeConcern: { w: 'majority', j: true }
-    });
+    db.connect(url, options);
 
     console.log('mongo connected');
-    return connectionMongo;
+    return db;
   } catch (error) {
-    addLog.error('mongo connect error', error);
-    await connectionMongo.disconnect();
+    addLog.error('Mongo connect error', error);
+
+    await db.disconnect();
+
     await delay(1000);
-    return connectMongo();
+    return connectMongo(db, url);
   }
 }

packages/service/common/mongo/type.d.ts

@@ -3,4 +3,5 @@ import type { Logger } from 'winston';
 
 declare global {
   var mongodb: Mongoose | undefined;
+  var mongodbLog: Mongoose | undefined;
 }

packages/service/common/system/log/schema.ts

@@ -1,4 +1,4 @@
-import { getMongoModel, Schema } from '../../../common/mongo';
+import { getMongoLogModel as getMongoModel, Schema } from '../../../common/mongo';
 import { SystemLogType } from './type';
 import { LogLevelEnum } from './constant';