4.8.8 test fix (#2143)

* perf: transcriptions api

* perf: variable picker tip

* perf: variable picker tip

* perf: chat select app

* feat: router to app detail

* perf: variable avoid space

* perf: variable picker

* perf: doc2x icon and params

* perf: sandbox support countToken

* feat: sandbox support delay and countToken

projects/sandbox/src/sandbox/jsFn/delay.ts

@@ -0,0 +1,10 @@
+export const timeDelay = (time: number) => {
+  return new Promise((resolve, reject) => {
+    if (time > 10000) {
+      reject('Delay time must be less than 10');
+    }
+    setTimeout(() => {
+      resolve('');
+    }, time);
+  });
+};

projects/sandbox/src/sandbox/jsFn/tiktoken/index.ts

@@ -0,0 +1,8 @@
+import { Tiktoken } from 'tiktoken/lite';
+const cl100k_base = require('tiktoken/encoders/cl100k_base');
+
+export const countToken = (text: string = '') => {
+  const enc = new Tiktoken(cl100k_base.bpe_ranks, cl100k_base.special_tokens, cl100k_base.pat_str);
+  const encodeText = enc.encode(text);
+  return encodeText.length;
+};

projects/sandbox/src/sandbox/sandbox.controller.ts

@@ -1,15 +1,14 @@
 import { Controller, Post, Body, HttpCode } from '@nestjs/common';
-import { SandboxService } from './sandbox.service';
-import { RunCodeDto, RunCodeResponse } from './dto/create-sandbox.dto';
-import { WorkerNameEnum, runWorker } from 'src/worker/utils';
+import { RunCodeDto } from './dto/create-sandbox.dto';
+import { runSandbox } from './utils';
 
 @Controller('sandbox')
 export class SandboxController {
-  constructor(private readonly sandboxService: SandboxService) {}
+  constructor() {}
 
   @Post('/js')
   @HttpCode(200)
   runJs(@Body() codeProps: RunCodeDto) {
-    return runWorker<RunCodeResponse>(WorkerNameEnum.runJs, codeProps);
+    return runSandbox(codeProps);
   }
 }

projects/sandbox/src/sandbox/sandbox.service.ts

@@ -1,10 +1,9 @@
 import { Injectable } from '@nestjs/common';
 import { RunCodeDto } from './dto/create-sandbox.dto';
-import { WorkerNameEnum, runWorker } from 'src/worker/utils';
 
 @Injectable()
 export class SandboxService {
   runJs(params: RunCodeDto) {
-    return runWorker(WorkerNameEnum.runJs, params);
+    return {};
   }
 }

projects/sandbox/src/sandbox/utils.ts

@@ -0,0 +1,102 @@
+import { RunCodeDto, RunCodeResponse } from 'src/sandbox/dto/create-sandbox.dto';
+import IsolatedVM, { ExternalCopy, Isolate, Reference } from 'isolated-vm';
+import { countToken } from './jsFn/tiktoken';
+import { timeDelay } from './jsFn/delay';
+
+const CustomLogStr = 'CUSTOM_LOG';
+
+/* 
+    Rewrite code to add custom functions: Promise function; Log.
+  */
+function getFnCode(code: string) {
+  const rewriteSystemFn = `
+    const thisDelay = (...args) => global_delay.applySyncPromise(undefined,args)
+`;
+
+  // rewrite delay
+  code = code.replace(/delay\((.*)\)/g, `thisDelay($1)`);
+
+  // rewrite log
+  code = code.replace(/console\.log/g, `${CustomLogStr}`);
+
+  const runCode = `
+    (async() => { 
+        try {
+            ${rewriteSystemFn}
+            ${code}
+
+            const res = await main(variables, {})
+            return JSON.stringify(res);
+        } catch(err) {
+            return JSON.stringify({ERROR: err?.message ?? err})
+        }
+    })
+`;
+  return runCode;
+}
+
+function registerSystemFn(jail: IsolatedVM.Reference<Record<string | number | symbol, any>>) {
+  return Promise.all([
+    // delay
+    jail.set('global_delay', new Reference(timeDelay)),
+    jail.set('countToken', countToken)
+  ]);
+}
+
+export const runSandbox = async ({
+  code,
+  variables = {}
+}: RunCodeDto): Promise<RunCodeResponse> => {
+  const logData = [];
+
+  const isolate = new Isolate({ memoryLimit: 32 });
+  const context = await isolate.createContext();
+  const jail = context.global;
+
+  try {
+    // Add global variables
+    await Promise.all([
+      jail.set('variables', new ExternalCopy(variables).copyInto()),
+      jail.set(CustomLogStr, function (...args) {
+        logData.push(
+          args
+            .map((item) => (typeof item === 'object' ? JSON.stringify(item, null, 2) : item))
+            .join(', ')
+        );
+      }),
+      registerSystemFn(jail)
+    ]);
+
+    // Run code
+    const fn = await context.eval(getFnCode(code), { reference: true, timeout: 10000 });
+
+    try {
+      // Get result and parse
+      const value = await fn.apply(undefined, [], { result: { promise: true } });
+      const result = JSON.parse(value.toLocaleString());
+
+      //   release memory
+      context.release();
+      isolate.dispose();
+
+      if (result.ERROR) {
+        return Promise.reject(result.ERROR);
+      }
+
+      return {
+        codeReturn: result,
+        log: logData.join('\n')
+      };
+    } catch (error) {
+      context.release();
+      isolate.dispose();
+      return Promise.reject('Not an invalid response.You must return an object');
+    }
+  } catch (err) {
+    console.log(err);
+
+    context.release();
+    isolate.dispose();
+    return Promise.reject(err);
+  }
+};

projects/sandbox/src/worker/runJs.ts

@@ -1,53 +0,0 @@
-import { RunCodeDto, RunCodeResponse } from 'src/sandbox/dto/create-sandbox.dto';
-import { parentPort } from 'worker_threads';
-import { workerResponse } from './utils';
-
-// @ts-ignore
-const ivm = require('isolated-vm');
-
-parentPort?.on('message', ({ code, variables = {} }: RunCodeDto) => {
-  const resolve = (data: RunCodeResponse) => workerResponse({ parentPort, type: 'success', data });
-  const reject = (error: any) => workerResponse({ parentPort, type: 'error', data: error });
-  try {
-    const isolate = new ivm.Isolate({ memoryLimit: 32 });
-    const context = isolate.createContextSync();
-    const jail = context.global;
-
-    // custom function
-    const logData = [];
-    const CustomLogStr = 'CUSTOM_LOG';
-    code = code.replace(/console\.log/g, `${CustomLogStr}`);
-    jail.setSync(CustomLogStr, function (...args) {
-      logData.push(
-        args
-          .map((item) => (typeof item === 'object' ? JSON.stringify(item, null, 2) : item))
-          .join(', ')
-      );
-    });
-
-    jail.setSync('responseData', function (args: any): any {
-      if (typeof args === 'object') {
-        resolve({
-          codeReturn: args,
-          log: logData.join('\n')
-        });
-      } else {
-        reject('Not an invalid response, must return an object');
-      }
-    });
-
-    // Add global variables
-    jail.setSync('variables', new ivm.ExternalCopy(variables).copyInto());
-
-    const scriptCode = `
-      ${code}
-      responseData(main(variables))`;
-
-    context.evalSync(scriptCode, { timeout: 6000 });
-  } catch (err) {
-    console.log(err);
-    reject(err);
-  }
-
-  process.exit();
-});

projects/sandbox/src/worker/utils.ts

@@ -1,47 +0,0 @@
-import { type MessagePort, Worker } from 'worker_threads';
-import * as path from 'path';
-
-export enum WorkerNameEnum {
-  runJs = 'runJs',
-  runPy = 'runPy'
-}
-
-type WorkerResponseType = { type: 'success' | 'error'; data: any };
-
-export const getWorker = (name: WorkerNameEnum) => {
-  const baseUrl =
-    process.env.NODE_ENV === 'production' ? 'projects/sandbox/dist/worker' : 'dist/worker';
-  const workerPath = path.join(process.cwd(), baseUrl, `${name}.js`);
-  return new Worker(workerPath);
-};
-
-export const runWorker = <T = any>(name: WorkerNameEnum, params?: Record<string, any>) => {
-  return new Promise<T>((resolve, reject) => {
-    const worker = getWorker(name);
-
-    worker.postMessage(params);
-
-    worker.on('message', (msg: WorkerResponseType) => {
-      if (msg.type === 'error') return reject(msg.data);
-
-      resolve(msg.data);
-      worker.terminate();
-    });
-
-    worker.on('error', (err) => {
-      reject(err);
-      worker.terminate();
-    });
-    worker.on('messageerror', (err) => {
-      reject(err);
-      worker.terminate();
-    });
-  });
-};
-
-export const workerResponse = ({
-  parentPort,
-  ...data
-}: WorkerResponseType & { parentPort?: MessagePort }) => {
-  parentPort?.postMessage(data);
-};