Sandbox (#1610) (#1611)

2024-05-28 14:47:10 +08:00
parent d9f5f4ede0
commit 9639139b52
58 changed files with 4715 additions and 283 deletions
--- a/packages/global/core/workflow/constants.ts
+++ b/packages/global/core/workflow/constants.ts
@@ -108,7 +108,11 @@ export enum NodeInputKeyEnum {
  ifElseList = 'ifElseList',

  // variable update
-  updateList = 'updateList'
+  updateList = 'updateList',
+
+  // code
+  code = 'code',
+  codeType = 'codeType' // js|py
 }

 export enum NodeOutputKeyEnum {
@@ -121,6 +125,7 @@ export enum NodeOutputKeyEnum {
  error = 'error',
  text = 'system_text',
  addOutputParam = 'system_addOutputParam',
+  rawResponse = 'system_rawResponse',

  // dataset
  datasetQuoteQA = 'quoteQA',
--- a/packages/global/core/workflow/node/constant.ts
+++ b/packages/global/core/workflow/node/constant.ts
@@ -113,7 +113,8 @@ export enum FlowNodeTypeEnum {
  stopTool = 'stopTool',
  lafModule = 'lafModule',
  ifElseNode = 'ifElseNode',
-  variableUpdate = 'variableUpdate'
+  variableUpdate = 'variableUpdate',
+  code = 'code'
 }

 export const EDGE_TYPE = 'default';
--- a/packages/global/core/workflow/runtime/type.d.ts
+++ b/packages/global/core/workflow/runtime/type.d.ts
@@ -31,6 +31,8 @@ export type DispatchNodeResponseType = {
  runningTime?: number;
  query?: string;
  textOutput?: string;
+  customInputs?: Record<string, any>;
+  customOutputs?: Record<string, any>;

  // bill
  tokens?: number;
--- a/packages/global/core/workflow/template/constants.ts
+++ b/packages/global/core/workflow/template/constants.ts
@@ -22,6 +22,7 @@ import type { FlowNodeTemplateType } from '../type';
 import { LafModule } from './system/laf';
 import { IfElseNode } from './system/ifElse/index';
 import { VariableUpdateNode } from './system/variableUpdate';
+import { CodeNode } from './system/sandbox';

 /* app flow module templates */
 export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -40,7 +41,8 @@ export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
  AiQueryExtension,
  LafModule,
  IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];
 /* plugin flow module templates */
 export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -59,7 +61,8 @@ export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
  AiQueryExtension,
  LafModule,
  IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];

 /* all module */
@@ -84,5 +87,6 @@ export const moduleTemplatesFlat: FlowNodeTemplateType[] = [
  AiQueryExtension,
  LafModule,
  IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];
--- a/packages/global/core/workflow/template/system/sandbox/constants.ts
+++ b/packages/global/core/workflow/template/system/sandbox/constants.ts
@@ -0,0 +1,7 @@
+export const JS_TEMPLATE = `function main({data1, data2}){
+    
+    return {
+        result: data1,
+        data2
+    }
+}`;
--- a/packages/global/core/workflow/template/system/sandbox/index.ts
+++ b/packages/global/core/workflow/template/system/sandbox/index.ts
@@ -0,0 +1,72 @@
+import {
+  FlowNodeTemplateTypeEnum,
+  NodeInputKeyEnum,
+  NodeOutputKeyEnum,
+  WorkflowIOValueTypeEnum
+} from '../../../constants';
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../../node/constant';
+import { FlowNodeTemplateType } from '../../../type';
+import { getHandleConfig } from '../../utils';
+import { Input_Template_DynamicInput } from '../../input';
+import { Output_Template_AddOutput } from '../../output';
+import { JS_TEMPLATE } from './constants';
+
+export const CodeNode: FlowNodeTemplateType = {
+  id: FlowNodeTypeEnum.code,
+  templateType: FlowNodeTemplateTypeEnum.tools,
+  flowNodeType: FlowNodeTypeEnum.code,
+  sourceHandle: getHandleConfig(true, true, true, true),
+  targetHandle: getHandleConfig(true, true, true, true),
+  avatar: '/imgs/workflow/code.svg',
+  name: '代码运行',
+  intro: '执行一段简单的脚本代码，通常用于进行复杂的数据处理。',
+  showStatus: true,
+  version: '482',
+  inputs: [
+    {
+      ...Input_Template_DynamicInput,
+      description: '这些变量会作为代码的运行的输入参数',
+      editField: {
+        key: true,
+        valueType: true
+      }
+    },
+    {
+      key: NodeInputKeyEnum.codeType,
+      renderTypeList: [FlowNodeInputTypeEnum.hidden],
+      label: '',
+      value: 'js'
+    },
+    {
+      key: NodeInputKeyEnum.code,
+      renderTypeList: [FlowNodeInputTypeEnum.custom],
+      label: '',
+      value: JS_TEMPLATE
+    }
+  ],
+  outputs: [
+    {
+      ...Output_Template_AddOutput,
+      description: '将代码中 return 的对象作为输出，传递给后续的节点'
+    },
+    {
+      id: NodeOutputKeyEnum.rawResponse,
+      key: NodeOutputKeyEnum.rawResponse,
+      label: '完整响应数据',
+      valueType: WorkflowIOValueTypeEnum.object,
+      type: FlowNodeOutputTypeEnum.static
+    },
+    {
+      id: NodeOutputKeyEnum.error,
+      key: NodeOutputKeyEnum.error,
+      label: '运行错误',
+      description: '代码运行错误信息，成功时返回空',
+      valueType: WorkflowIOValueTypeEnum.object,
+      type: FlowNodeOutputTypeEnum.static
+    }
+  ]
+};
--- a/packages/service/core/workflow/dispatch/code/isolatedvm.ts
+++ b/packages/service/core/workflow/dispatch/code/isolatedvm.ts
@@ -1,38 +0,0 @@
-// import { addLog } from '../../../../common/system/log';
-// const ivm = require('isolated-vm');
-
-// export const runJsCode = ({
-//   code,
-//   variables
-// }: {
-//   code: string;
-//   variables: Record<string, any>;
-// }) => {
-//   const isolate = new ivm.Isolate({ memoryLimit: 16 });
-//   const context = isolate.createContextSync();
-//   const jail = context.global;
-
-//   return new Promise((resolve, reject) => {
-//     // custom log function
-//     jail.setSync('responseData', function (args: any): any {
-//       if (typeof args === 'object') {
-//         resolve(args);
-//       } else {
-//         reject('Not an invalid response');
-//       }
-//     });
-
-//     // Add global variables
-//     jail.setSync('variables', new ivm.ExternalCopy(variables).copyInto());
-
-//     try {
-//       const scriptCode = `
-//         ${code}
-//         responseData(main(variables))`;
-//       context.evalSync(scriptCode, { timeout: 2000 });
-//     } catch (err) {
-//       addLog.error('Error during script execution:', err);
-//       reject(err);
-//     }
-//   });
-// };
--- a/packages/service/core/workflow/dispatch/code/run.ts
+++ b/packages/service/core/workflow/dispatch/code/run.ts
@@ -0,0 +1,51 @@
+import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
+import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
+import axios from 'axios';
+import { formatHttpError } from '../utils';
+import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+
+type RunCodeType = ModuleDispatchProps<{
+  [NodeInputKeyEnum.codeType]: 'js';
+  [NodeInputKeyEnum.code]: string;
+  [key: string]: any;
+}>;
+type RunCodeResponse = DispatchNodeResultType<{
+  [NodeOutputKeyEnum.error]?: any;
+  [NodeOutputKeyEnum.rawResponse]?: Record<string, any>;
+  [key: string]: any;
+}>;
+
+export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeResponse> => {
+  const {
+    params: { codeType, code, ...customVariables }
+  } = props;
+
+  const sandBoxRequestUrl = `${process.env.SANDBOX_URL}/sandbox/js`;
+  try {
+    const { data: runResult } = await axios.post<{
+      success: boolean;
+      data: Record<string, any>;
+    }>(sandBoxRequestUrl, {
+      code,
+      variables: customVariables
+    });
+
+    if (runResult.success) {
+      return {
+        [NodeOutputKeyEnum.rawResponse]: runResult.data,
+        [DispatchNodeResponseKeyEnum.nodeResponse]: {
+          customInputs: customVariables,
+          customOutputs: runResult.data
+        },
+        ...runResult.data
+      };
+    } else {
+      throw new Error('Run code failed');
+    }
+  } catch (error) {
+    return {
+      [NodeOutputKeyEnum.error]: formatHttpError(error)
+    };
+  }
+};
--- a/packages/service/core/workflow/dispatch/index.ts
+++ b/packages/service/core/workflow/dispatch/index.ts
@@ -46,6 +46,7 @@ import { dispatchSystemConfig } from './init/systemConfig';
 import { dispatchUpdateVariable } from './tools/runUpdateVar';
 import { addLog } from '../../../common/system/log';
 import { surrenderProcess } from '../../../common/system/tools';
+import { dispatchRunCode } from './code/run';

 const callbackMap: Record<FlowNodeTypeEnum, Function> = {
  [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -66,6 +67,7 @@ const callbackMap: Record<FlowNodeTypeEnum, Function> = {
  [FlowNodeTypeEnum.lafModule]: dispatchLafRequest,
  [FlowNodeTypeEnum.ifElseNode]: dispatchIfElse,
  [FlowNodeTypeEnum.variableUpdate]: dispatchUpdateVariable,
+  [FlowNodeTypeEnum.code]: dispatchRunCode,

  // none
  [FlowNodeTypeEnum.systemConfig]: dispatchSystemConfig,
--- a/packages/service/core/workflow/dispatch/tools/http468.ts
+++ b/packages/service/core/workflow/dispatch/tools/http468.ts
@@ -9,7 +9,7 @@ import {
  SseResponseEventEnum
 } from '@fastgpt/global/core/workflow/runtime/constants';
 import axios from 'axios';
-import { valueTypeFormat } from '../utils';
+import { formatHttpError, valueTypeFormat } from '../utils';
 import { SERVICE_LOCAL_HOST } from '../../../../common/system/tools';
 import { addLog } from '../../../../common/system/log';
 import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
@@ -310,14 +310,3 @@ function removeUndefinedSign(obj: Record<string, any>) {
  }
  return obj;
 }
-function formatHttpError(error: any) {
-  return {
-    message: error?.message,
-    name: error?.name,
-    method: error?.config?.method,
-    baseURL: error?.config?.baseURL,
-    url: error?.config?.url,
-    code: error?.code,
-    status: error?.status
-  };
-}
--- a/packages/service/core/workflow/dispatch/utils.ts
+++ b/packages/service/core/workflow/dispatch/utils.ts
@@ -3,12 +3,7 @@ import {
  WorkflowIOValueTypeEnum,
  NodeOutputKeyEnum
 } from '@fastgpt/global/core/workflow/constants';
-import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import {
-  RuntimeEdgeItemType,
-  RuntimeNodeItemType
-} from '@fastgpt/global/core/workflow/runtime/type';
-import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/index.d';
+import { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/runtime/type';

 export const filterToolNodeIdByEdges = ({
  nodeId,
@@ -91,3 +86,15 @@ export const removeSystemVariable = (variables: Record<string, any>) => {

  return copyVariables;
 };
+
+export const formatHttpError = (error: any) => {
+  return {
+    message: error?.message,
+    name: error?.name,
+    method: error?.config?.method,
+    baseURL: error?.config?.baseURL,
+    url: error?.config?.url,
+    code: error?.code,
+    status: error?.status
+  };
+};
--- a/packages/web/components/common/Textarea/CodeEditor/Editor.tsx
+++ b/packages/web/components/common/Textarea/CodeEditor/Editor.tsx
@@ -0,0 +1,160 @@
+import React, { useCallback, useRef, useState } from 'react';
+import Editor, { Monaco, loader } from '@monaco-editor/react';
+import { Box, BoxProps } from '@chakra-ui/react';
+import MyIcon from '../../Icon';
+
+loader.config({
+  paths: { vs: '/js/monaco-editor.0.45.0/vs' }
+});
+
+type EditorVariablePickerType = {
+  key: string;
+  label: string;
+};
+
+export type Props = Omit<BoxProps, 'resize' | 'onChange'> & {
+  height?: number;
+  resize?: boolean;
+  defaultValue?: string;
+  value?: string;
+  onChange?: (e: string) => void;
+  onOpenModal?: () => void;
+  variables?: EditorVariablePickerType[];
+  defaultHeight?: number;
+};
+
+const options = {
+  lineNumbers: 'on',
+  guides: {
+    indentation: false
+  },
+  automaticLayout: true,
+  minimap: {
+    enabled: false
+  },
+  scrollbar: {
+    verticalScrollbarSize: 4,
+    horizontalScrollbarSize: 8,
+    alwaysConsumeMouseWheel: false
+  },
+  lineNumbersMinChars: 0,
+  fontSize: 14,
+  scrollBeyondLastLine: false,
+  folding: true,
+  overviewRulerBorder: false,
+  tabSize: 2
+};
+
+const MyEditor = ({
+  defaultValue,
+  value,
+  onChange,
+  resize,
+  variables = [],
+  defaultHeight = 200,
+  onOpenModal,
+  ...props
+}: Props) => {
+  const [height, setHeight] = useState(defaultHeight);
+  const initialY = useRef(0);
+
+  const handleMouseDown = useCallback((e: React.MouseEvent) => {
+    initialY.current = e.clientY;
+
+    const handleMouseMove = (e: MouseEvent) => {
+      const deltaY = e.clientY - initialY.current;
+      initialY.current = e.clientY;
+      setHeight((prevHeight) => (prevHeight + deltaY < 100 ? 100 : prevHeight + deltaY));
+    };
+
+    const handleMouseUp = () => {
+      document.removeEventListener('mousemove', handleMouseMove);
+      document.removeEventListener('mouseup', handleMouseUp);
+    };
+
+    document.addEventListener('mousemove', handleMouseMove);
+    document.addEventListener('mouseup', handleMouseUp);
+  }, []);
+
+  const beforeMount = useCallback((monaco: Monaco) => {
+    monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
+      validate: false,
+      allowComments: false,
+      schemas: [
+        {
+          uri: 'http://myserver/foo-schema.json', // 一个假设的 URI
+          fileMatch: ['*'], // 匹配所有文件
+          schema: {} // 空的 Schema
+        }
+      ]
+    });
+
+    monaco.editor.defineTheme('JSONEditorTheme', {
+      base: 'vs', // 可以基于已有的主题进行定制
+      inherit: true, // 继承基础主题的设置
+      rules: [{ token: 'variable', foreground: '2B5FD9' }],
+      colors: {
+        'editor.background': '#ffffff00',
+        'editorLineNumber.foreground': '#aaa',
+        'editorOverviewRuler.border': '#ffffff00',
+        'editor.lineHighlightBackground': '#F7F8FA',
+        'scrollbarSlider.background': '#E8EAEC',
+        'editorIndentGuide.activeBackground': '#ddd',
+        'editorIndentGuide.background': '#eee'
+      }
+    });
+  }, []);
+
+  return (
+    <Box
+      borderWidth={'1px'}
+      borderRadius={'md'}
+      borderColor={'myGray.200'}
+      py={2}
+      height={height}
+      position={'relative'}
+      pl={2}
+      {...props}
+    >
+      <Editor
+        height={'100%'}
+        defaultLanguage="typescript"
+        options={options as any}
+        theme="JSONEditorTheme"
+        beforeMount={beforeMount}
+        defaultValue={defaultValue}
+        value={value}
+        onChange={(e) => {
+          onChange?.(e || '');
+        }}
+      />
+      {resize && (
+        <Box
+          position={'absolute'}
+          right={'-1'}
+          bottom={'-1'}
+          zIndex={10}
+          cursor={'ns-resize'}
+          px={'4px'}
+          onMouseDown={handleMouseDown}
+        >
+          <MyIcon name={'common/editor/resizer'} width={'16px'} height={'16px'} />
+        </Box>
+      )}
+      {!!onOpenModal && (
+        <Box
+          zIndex={10}
+          position={'absolute'}
+          bottom={0}
+          right={2}
+          cursor={'pointer'}
+          onClick={onOpenModal}
+        >
+          <MyIcon name={'common/fullScreenLight'} w={'14px'} color={'myGray.600'} />
+        </Box>
+      )}
+    </Box>
+  );
+};
+
+export default React.memo(MyEditor);
--- a/packages/web/components/common/Textarea/CodeEditor/index.tsx
+++ b/packages/web/components/common/Textarea/CodeEditor/index.tsx
@@ -0,0 +1,36 @@
+import React from 'react';
+import MyEditor, { type Props as EditorProps } from './Editor';
+import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react';
+import MyModal from '../../MyModal';
+import { useTranslation } from 'next-i18next';
+
+type Props = Omit<EditorProps, 'resize'> & {};
+
+const CodeEditor = (props: Props) => {
+  const { t } = useTranslation();
+  const { isOpen, onOpen, onClose } = useDisclosure();
+
+  return (
+    <>
+      <MyEditor {...props} resize onOpenModal={onOpen} />
+      <MyModal
+        isOpen={isOpen}
+        onClose={onClose}
+        iconSrc="modal/edit"
+        title={t('Code editor')}
+        w={'full'}
+      >
+        <ModalBody>
+          <MyEditor {...props} bg={'myGray.50'} defaultHeight={600} />
+        </ModalBody>
+        <ModalFooter>
+          <Button mr={2} onClick={onClose}>
+            {t('common.Confirm')}
+          </Button>
+        </ModalFooter>
+      </MyModal>
+    </>
+  );
+};
+
+export default React.memo(CodeEditor);