New file upload (#3058)

* feat: toolNode aiNode readFileNode adapt new version

* update docker-compose

* update tip

* feat: adapt new file version

* perf: file input

* fix: ts
This commit is contained in:
Archer
2024-11-04 10:44:45 +08:00
committed by archer
parent 7a929db0a5
commit 469858877e
55 changed files with 1147 additions and 475 deletions

View File

@@ -19,3 +19,5 @@ export const bucketNameMap = {
export const ReadFileBaseUrl = `${process.env.FE_DOMAIN || ''}/api/common/file/read`;
export const documentFileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
export const imageFileType =
'.jpg, .jpeg, .png, .gif, .bmp, .webp, .svg, .tiff, .tif, .ico, .heic, .heif, .avif';

View File

@@ -1,4 +1,7 @@
import { detect } from 'jschardet';
import { documentFileType, imageFileType } from './constants';
import { ChatFileTypeEnum } from '../../core/chat/constants';
import { UserChatItemValueItemType } from '../../core/chat/type';
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
@@ -13,3 +16,39 @@ export const formatFileSize = (bytes: number): string => {
export const detectFileEncoding = (buffer: Buffer) => {
return detect(buffer.slice(0, 200))?.encoding?.toLocaleLowerCase();
};
// Url => user upload file type
export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file'] | undefined => {
const parseUrl = new URL(url, 'https://locaohost:3000');
const filename = (() => {
// Old version file url: https://xxx.com/file/read?filename=xxx.pdf
const filenameQuery = parseUrl.searchParams.get('filename');
if (filenameQuery) return filenameQuery;
// Common file https://xxx.com/xxx.pdf?xxxx=xxx
const pathname = parseUrl.pathname;
if (pathname) return pathname.split('/').pop();
})();
if (!filename) return;
const extension = filename.split('.').pop()?.toLowerCase() || '';
if (!extension) return;
if (documentFileType.includes(extension)) {
return {
type: ChatFileTypeEnum.file,
name: filename,
url
};
}
if (imageFileType.includes(extension)) {
return {
type: ChatFileTypeEnum.image,
name: filename,
url
};
}
};

View File

@@ -30,7 +30,8 @@ export const getChatTitleFromChatMessage = (message?: ChatItemType, defaultValue
// Keep the first n and last n characters
export const getHistoryPreview = (
completeMessages: ChatItemType[],
size = 100
size = 100,
useVision = false
): {
obj: `${ChatRoleEnum}`;
value: string;
@@ -48,7 +49,8 @@ export const getHistoryPreview = (
item.value
?.map((item) => {
if (item?.text?.content) return item?.text?.content;
if (item.file?.type === 'image') return 'Input an image';
if (item.file?.type === 'image' && useVision)
return `![Input an image](${item.file.url.slice(0, 100)}...)`;
return '';
})
.filter(Boolean)

View File

@@ -27,7 +27,9 @@ export enum FlowNodeInputTypeEnum { // render ui
settingDatasetQuotePrompt = 'settingDatasetQuotePrompt',
hidden = 'hidden',
custom = 'custom'
custom = 'custom',
fileSelect = 'fileSelect'
}
export const FlowNodeInputMap: Record<
FlowNodeInputTypeEnum,
@@ -85,6 +87,9 @@ export const FlowNodeInputMap: Record<
},
[FlowNodeInputTypeEnum.textarea]: {
icon: 'core/workflow/inputType/textarea'
},
[FlowNodeInputTypeEnum.fileSelect]: {
icon: 'core/workflow/inputType/file'
}
};
@@ -137,43 +142,43 @@ export enum FlowNodeTypeEnum {
// node IO value type
export const FlowValueTypeMap = {
[WorkflowIOValueTypeEnum.string]: {
label: 'string',
label: 'String',
value: WorkflowIOValueTypeEnum.string
},
[WorkflowIOValueTypeEnum.number]: {
label: 'number',
label: 'Number',
value: WorkflowIOValueTypeEnum.number
},
[WorkflowIOValueTypeEnum.boolean]: {
label: 'boolean',
label: 'Boolean',
value: WorkflowIOValueTypeEnum.boolean
},
[WorkflowIOValueTypeEnum.object]: {
label: 'object',
label: 'Object',
value: WorkflowIOValueTypeEnum.object
},
[WorkflowIOValueTypeEnum.arrayString]: {
label: 'array<string>',
label: 'Array<string>',
value: WorkflowIOValueTypeEnum.arrayString
},
[WorkflowIOValueTypeEnum.arrayNumber]: {
label: 'array<number>',
label: 'Array<number>',
value: WorkflowIOValueTypeEnum.arrayNumber
},
[WorkflowIOValueTypeEnum.arrayBoolean]: {
label: 'array<boolean>',
label: 'Array<boolean>',
value: WorkflowIOValueTypeEnum.arrayBoolean
},
[WorkflowIOValueTypeEnum.arrayObject]: {
label: 'array<object>',
label: 'Array<object>',
value: WorkflowIOValueTypeEnum.arrayObject
},
[WorkflowIOValueTypeEnum.arrayAny]: {
label: 'array',
label: 'Array',
value: WorkflowIOValueTypeEnum.arrayAny
},
[WorkflowIOValueTypeEnum.any]: {
label: 'any',
label: 'Any',
value: WorkflowIOValueTypeEnum.any
},
[WorkflowIOValueTypeEnum.chatHistory]: {

View File

@@ -216,5 +216,7 @@ export type AIChatNodeProps = {
[NodeInputKeyEnum.aiChatQuoteTemplate]?: string;
[NodeInputKeyEnum.aiChatQuotePrompt]?: string;
[NodeInputKeyEnum.aiChatVision]?: boolean;
[NodeInputKeyEnum.stringQuoteText]?: string;
[NodeInputKeyEnum.fileUrlList]?: string[];
};

View File

@@ -75,10 +75,17 @@ export const Input_Template_Text_Quote: FlowNodeInputItemType = {
description: i18nT('app:document_quote_tip'),
valueType: WorkflowIOValueTypeEnum.string
};
export const Input_Template_File_Link_Prompt: FlowNodeInputItemType = {
key: NodeInputKeyEnum.fileUrlList,
renderTypeList: [FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.input],
label: i18nT('app:file_quote_link'),
debugLabel: i18nT('app:file_quote_link'),
valueType: WorkflowIOValueTypeEnum.arrayString
};
export const Input_Template_File_Link: FlowNodeInputItemType = {
key: NodeInputKeyEnum.fileUrlList,
renderTypeList: [FlowNodeInputTypeEnum.reference],
required: true,
label: i18nT('app:workflow.user_file_input'),
debugLabel: i18nT('app:workflow.user_file_input'),
description: i18nT('app:workflow.user_file_input_desc'),

View File

@@ -17,7 +17,8 @@ import {
Input_Template_History,
Input_Template_System_Prompt,
Input_Template_UserChatInput,
Input_Template_Text_Quote
Input_Template_Text_Quote,
Input_Template_File_Link_Prompt
} from '../../input';
import { chatNodeSystemPromptTip, systemPromptTip } from '../../tip';
import { getHandleConfig } from '../../utils';
@@ -55,7 +56,7 @@ export const AiChatModule: FlowNodeTemplateType = {
showStatus: true,
isTool: true,
courseUrl: '/docs/workflow/modules/ai_chat/',
version: '481',
version: '4813',
inputs: [
Input_Template_SettingAiModel,
// --- settings modal
@@ -100,7 +101,7 @@ export const AiChatModule: FlowNodeTemplateType = {
},
Input_Template_History,
Input_Template_Dataset_Quote,
Input_Template_Text_Quote,
Input_Template_File_Link_Prompt,
{ ...Input_Template_UserChatInput, toolDescription: i18nT('workflow:user_question') }
],

View File

@@ -23,7 +23,7 @@ export const ReadFilesNode: FlowNodeTemplateType = {
name: i18nT('app:workflow.read_files'),
intro: i18nT('app:workflow.read_files_tip'),
showStatus: true,
version: '489',
version: '4812',
isTool: true,
inputs: [
{

View File

@@ -20,6 +20,7 @@ import { chatNodeSystemPromptTip, systemPromptTip } from '../tip';
import { LLMModelTypeEnum } from '../../../ai/constants';
import { getHandleConfig } from '../utils';
import { i18nT } from '../../../../../web/i18n/utils';
import { Input_Template_File_Link_Prompt } from '../input';
export const ToolModule: FlowNodeTemplateType = {
id: FlowNodeTypeEnum.tools,
@@ -32,7 +33,7 @@ export const ToolModule: FlowNodeTemplateType = {
intro: i18nT('workflow:template.tool_call_intro'),
showStatus: true,
courseUrl: '/docs/workflow/modules/tool/',
version: '481',
version: '4813',
inputs: [
{
...Input_Template_SettingAiModel,
@@ -57,7 +58,7 @@ export const ToolModule: FlowNodeTemplateType = {
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.boolean,
value: true
value: false
},
{
@@ -67,6 +68,7 @@ export const ToolModule: FlowNodeTemplateType = {
placeholder: chatNodeSystemPromptTip
},
Input_Template_History,
Input_Template_File_Link_Prompt,
Input_Template_UserChatInput
],
outputs: [

View File

@@ -56,6 +56,11 @@ export type FlowNodeInputItemType = InputComponentPropsType & {
canEdit?: boolean; // dynamic inputs
isPro?: boolean; // Pro version field
isToolOutput?: boolean;
// file
canSelectFile?: boolean;
canSelectImg?: boolean;
maxFiles?: number;
};
export type FlowNodeOutputItemType = {

View File

@@ -32,6 +32,7 @@ import { IfElseResultEnum } from './template/system/ifElse/constant';
import { RuntimeNodeItemType } from './runtime/type';
import { getReferenceVariableValue } from './runtime/utils';
import {
Input_Template_File_Link,
Input_Template_History,
Input_Template_Stream_MODE,
Input_Template_UserChatInput
@@ -261,8 +262,10 @@ export const appData2FlowNodeIO = ({
inputs: [
Input_Template_Stream_MODE,
Input_Template_History,
...(chatConfig?.fileSelectConfig?.canSelectFile || chatConfig?.fileSelectConfig?.canSelectImg
? [Input_Template_File_Link]
: []),
Input_Template_UserChatInput,
// ...(showFileLink ? [Input_Template_File_Link] : []),
...variableInput
],
outputs: [

View File

@@ -109,7 +109,7 @@ export const loadRequestMessages = async ({
}
return Promise.all(
messages.map(async (item) => {
if (item.type === 'image_url') {
if (item.type === 'image_url' && process.env.MULTIPLE_DATA_TO_BASE64 === 'true') {
// Remove url origin
const imgUrl = (() => {
if (origin && item.image_url.url.startsWith(origin)) {
@@ -149,7 +149,7 @@ export const loadRequestMessages = async ({
};
// Split question text and image
const parseStringWithImages = (input: string): ChatCompletionContentPart[] => {
if (!useVision) {
if (!useVision || input.length > 500) {
return [{ type: 'text', text: input || '' }];
}
@@ -170,8 +170,8 @@ export const loadRequestMessages = async ({
});
});
// Too many images or too long text, return text
if (httpsImages.length > 4 || input.length > 1000) {
// Too many images return text
if (httpsImages.length > 4) {
return [{ type: 'text', text: input || '' }];
}
@@ -179,7 +179,7 @@ export const loadRequestMessages = async ({
result.push({ type: 'text', text: input });
return result;
};
// Parse user content(text and img)
// Parse user content(text and img) Store history => api messages
const parseUserContent = async (content: string | ChatCompletionContentPart[]) => {
if (typeof content === 'string') {
return loadImageToBase64(parseStringWithImages(content));

View File

@@ -25,45 +25,16 @@ import { replaceVariable } from '@fastgpt/global/common/string/tools';
import { getMultiplePrompt, Prompt_Tool_Call } from './constants';
import { filterToolResponseToPreview } from './utils';
import { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import { getFileContentFromLinks, getHistoryFileLinks } from '../../tools/readFiles';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { Prompt_DocumentQuote } from '@fastgpt/global/core/ai/prompt/AIChat';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.answerText]: string;
[DispatchNodeResponseKeyEnum.interactive]?: InteractiveNodeResponseType;
}>;
/*
Tool call auth add file prompt to question。
Guide the LLM to call tool.
*/
export const toolCallMessagesAdapt = ({
userInput
}: {
userInput: UserChatItemValueItemType[];
}) => {
const files = userInput.filter((item) => item.type === 'file');
if (files.length > 0) {
return userInput.map((item) => {
if (item.type === 'text') {
const filesCount = files.filter((file) => file.file?.type === 'file').length;
const imgCount = files.filter((file) => file.file?.type === 'image').length;
const text = item.text?.content || '';
return {
...item,
text: {
content: getMultiplePrompt({ fileCount: filesCount, imgCount, question: text })
}
};
}
return item;
});
}
return userInput;
};
export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
const {
node: { nodeId, name, isEntry },
@@ -71,11 +42,21 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
runtimeEdges,
histories,
query,
params: { model, systemPrompt, userChatInput, history = 6 }
requestOrigin,
chatConfig,
runningAppInfo: { teamId },
params: {
model,
systemPrompt,
userChatInput,
history = 6,
fileUrlList: fileLinks,
aiChatVision
}
} = props;
const toolModel = getLLMModel(model);
const useVision = aiChatVision && toolModel.vision;
const chatHistories = getHistories(history, histories);
const toolNodeIds = filterToolNodeIdByEdges({ nodeId, edges: runtimeEdges });
@@ -109,18 +90,43 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
}
})();
props.node.isEntry = false;
const hasReadFilesTool = toolNodes.some(
(item) => item.flowNodeType === FlowNodeTypeEnum.readFiles
);
const globalFiles = chatValue2RuntimePrompt(query).files;
const { documentQuoteText, userFiles } = await getMultiInput({
histories,
requestOrigin,
maxFiles: chatConfig?.fileSelectConfig?.maxFiles || 20,
teamId,
fileLinks,
inputFiles: globalFiles
});
const concatenateSystemPrompt = [
toolModel.defaultSystemChatPrompt,
systemPrompt,
documentQuoteText
? replaceVariable(Prompt_DocumentQuote, {
quote: documentQuoteText
})
: ''
]
.filter(Boolean)
.join('\n\n===---===---===\n\n');
const messages: ChatItemType[] = (() => {
const value: ChatItemType[] = [
...getSystemPrompt_ChatItemType(toolModel.defaultSystemChatPrompt),
...getSystemPrompt_ChatItemType(systemPrompt),
...getSystemPrompt_ChatItemType(concatenateSystemPrompt),
// Add file input prompt to histories
...chatHistories.map((item) => {
if (item.obj === ChatRoleEnum.Human) {
return {
...item,
value: toolCallMessagesAdapt({
userInput: item.value
userInput: item.value,
skip: !hasReadFilesTool
})
};
}
@@ -129,9 +135,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
{
obj: ChatRoleEnum.Human,
value: toolCallMessagesAdapt({
skip: !hasReadFilesTool,
userInput: runtimePrompt2ChatsValue({
text: userChatInput,
files: chatValue2RuntimePrompt(query).files
files: userFiles
})
})
}
@@ -237,7 +244,11 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
childTotalPoints: flatUsages.reduce((sum, item) => sum + item.totalPoints, 0),
model: modelName,
query: userChatInput,
historyPreview: getHistoryPreview(GPTMessages2Chats(completeMessages, false), 10000),
historyPreview: getHistoryPreview(
GPTMessages2Chats(completeMessages, false),
10000,
useVision
),
toolDetail: childToolResponse,
mergeSignId: nodeId
},
@@ -253,3 +264,88 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
[DispatchNodeResponseKeyEnum.interactive]: toolWorkflowInteractiveResponse
};
};
const getMultiInput = async ({
histories,
fileLinks,
requestOrigin,
maxFiles,
teamId,
inputFiles
}: {
histories: ChatItemType[];
fileLinks?: string[];
requestOrigin?: string;
maxFiles: number;
teamId: string;
inputFiles: UserChatItemValueItemType['file'][];
}) => {
// Not file quote
if (!fileLinks) {
return {
documentQuoteText: '',
userFiles: inputFiles
};
}
const filesFromHistories = getHistoryFileLinks(histories);
const urls = [...fileLinks, ...filesFromHistories];
if (urls.length === 0) {
return {
documentQuoteText: '',
userFiles: []
};
}
// Get files from histories
const { text } = await getFileContentFromLinks({
// Concat fileUrlList and filesFromHistories; remove not supported files
urls,
requestOrigin,
maxFiles,
teamId
});
return {
documentQuoteText: text,
userFiles: fileLinks.map((url) => parseUrlToFileType(url))
};
};
/*
Tool call auth add file prompt to question。
Guide the LLM to call tool.
*/
const toolCallMessagesAdapt = ({
userInput,
skip
}: {
userInput: UserChatItemValueItemType[];
skip?: boolean;
}) => {
if (skip) return userInput;
const files = userInput.filter((item) => item.type === 'file');
if (files.length > 0) {
return userInput.map((item) => {
if (item.type === 'text') {
const filesCount = files.filter((file) => file.file?.type === 'file').length;
const imgCount = files.filter((file) => file.file?.type === 'image').length;
const text = item.text?.content || '';
return {
...item,
text: {
content: getMultiplePrompt({ fileCount: filesCount, imgCount, question: text })
}
};
}
return item;
});
}
return userInput;
};

View File

@@ -21,6 +21,7 @@ export type DispatchToolModuleProps = ModuleDispatchProps<{
[NodeInputKeyEnum.aiChatTemperature]: number;
[NodeInputKeyEnum.aiChatMaxToken]: number;
[NodeInputKeyEnum.aiChatVision]?: boolean;
[NodeInputKeyEnum.fileUrlList]?: string[];
}> & {
messages: ChatCompletionMessageParam[];
toolNodes: ToolNodeItemType[];

View File

@@ -46,6 +46,8 @@ import { WorkflowResponseType } from '../type';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import { AiChatQuoteRoleType } from '@fastgpt/global/core/workflow/template/system/aiChat/type';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getFileContentFromLinks, getHistoryFileLinks } from '../tools/readFiles';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
export type ChatProps = ModuleDispatchProps<
AIChatNodeProps & {
@@ -69,7 +71,9 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
histories,
node: { name },
query,
runningAppInfo: { teamId },
workflowStreamResponse,
chatConfig,
params: {
model,
temperature = 0,
@@ -83,10 +87,11 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
quoteTemplate,
quotePrompt,
aiChatVision,
stringQuoteText
fileUrlList: fileLinks, // node quote file links
stringQuoteText //abandon
}
} = props;
const { files: inputFiles } = chatValue2RuntimePrompt(query);
const { files: inputFiles } = chatValue2RuntimePrompt(query); // Chat box input files
if (!userChatInput && inputFiles.length === 0) {
return Promise.reject('Question is empty');
@@ -100,11 +105,22 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
return Promise.reject('The chat model is undefined, you need to select a chat model.');
}
const { datasetQuoteText } = await filterDatasetQuote({
quoteQA,
model: modelConstantsData,
quoteTemplate
});
const [{ datasetQuoteText }, { documentQuoteText, userFiles }] = await Promise.all([
filterDatasetQuote({
quoteQA,
model: modelConstantsData,
quoteTemplate
}),
getMultiInput({
histories,
inputFiles,
fileLinks,
stringQuoteText,
requestOrigin,
maxFiles: chatConfig?.fileSelectConfig?.maxFiles || 20,
teamId
})
]);
const [{ filterMessages }] = await Promise.all([
getChatMessages({
@@ -115,9 +131,9 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
aiChatQuoteRole,
datasetQuotePrompt: quotePrompt,
userChatInput,
inputFiles,
systemPrompt,
stringQuoteText
userFiles,
documentQuoteText
}),
(() => {
// censor model and system key
@@ -132,22 +148,9 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
})()
]);
// Get the request messages
const concatMessages = [
...(modelConstantsData.defaultSystemChatPrompt
? [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: modelConstantsData.defaultSystemChatPrompt
}
]
: []),
...filterMessages
] as ChatCompletionMessageParam[];
const [requestMessages, max_tokens] = await Promise.all([
loadRequestMessages({
messages: concatMessages,
messages: filterMessages,
useVision: modelConstantsData.vision && aiChatVision,
origin: requestOrigin
}),
@@ -242,7 +245,11 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
tokens,
query: `${userChatInput}`,
maxToken: max_tokens,
historyPreview: getHistoryPreview(chatCompleteMessages, 10000),
historyPreview: getHistoryPreview(
chatCompleteMessages,
10000,
modelConstantsData.vision && aiChatVision
),
contextTotalLen: completeMessages.length
},
[DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [
@@ -302,7 +309,70 @@ async function filterDatasetQuote({
datasetQuoteText
};
}
async function getMultiInput({
histories,
inputFiles,
fileLinks,
stringQuoteText,
requestOrigin,
maxFiles,
teamId
}: {
histories: ChatItemType[];
inputFiles: UserChatItemValueItemType['file'][];
fileLinks?: string[];
stringQuoteText?: string; // file quote
requestOrigin?: string;
maxFiles: number;
teamId: string;
}) {
// 旧版本适配====>
if (stringQuoteText) {
return {
documentQuoteText: stringQuoteText,
userFiles: inputFiles
};
}
// 没有引用文件参考,但是可能用了图片识别
if (!fileLinks) {
return {
documentQuoteText: '',
userFiles: inputFiles
};
}
// 旧版本适配<====
// If fileLinks params is not empty, it means it is a new version, not get the global file.
// Get files from histories
const filesFromHistories = getHistoryFileLinks(histories);
const urls = [...fileLinks, ...filesFromHistories];
if (urls.length === 0) {
return {
documentQuoteText: '',
userFiles: []
};
}
const { text } = await getFileContentFromLinks({
// Concat fileUrlList and filesFromHistories; remove not supported files
urls,
requestOrigin,
maxFiles,
teamId
});
return {
documentQuoteText: text,
userFiles: fileLinks.map((url) => parseUrlToFileType(url))
};
}
async function getChatMessages({
model,
aiChatQuoteRole,
datasetQuotePrompt = '',
datasetQuoteText,
@@ -310,10 +380,10 @@ async function getChatMessages({
histories = [],
systemPrompt,
userChatInput,
inputFiles,
model,
stringQuoteText
userFiles,
documentQuoteText
}: {
model: LLMModelItemType;
// dataset quote
aiChatQuoteRole: AiChatQuoteRoleType; // user: replace user prompt; system: replace system prompt
datasetQuotePrompt?: string;
@@ -323,10 +393,11 @@ async function getChatMessages({
histories: ChatItemType[];
systemPrompt: string;
userChatInput: string;
inputFiles: UserChatItemValueItemType['file'][];
model: LLMModelItemType;
stringQuoteText?: string; // file quote
userFiles: UserChatItemValueItemType['file'][];
documentQuoteText?: string; // document quote
}) {
// Dataset prompt ====>
// User role or prompt include question
const quoteRole =
aiChatQuoteRole === 'user' || datasetQuotePrompt.includes('{{question}}') ? 'user' : 'system';
@@ -337,6 +408,7 @@ async function getChatMessages({
? Prompt_userQuotePromptList[0].value
: Prompt_systemQuotePromptList[0].value;
// Reset user input, add dataset quote to user input
const replaceInputValue =
useDatasetQuote && quoteRole === 'user'
? replaceVariable(datasetQuotePromptTemplate, {
@@ -344,31 +416,33 @@ async function getChatMessages({
question: userChatInput
})
: userChatInput;
// Dataset prompt <====
const replaceSystemPrompt =
// Concat system prompt
const concatenateSystemPrompt = [
model.defaultSystemChatPrompt,
systemPrompt,
useDatasetQuote && quoteRole === 'system'
? `${systemPrompt ? systemPrompt + '\n\n------\n\n' : ''}${replaceVariable(
datasetQuotePromptTemplate,
{
quote: datasetQuoteText
}
)}`
: systemPrompt;
? replaceVariable(datasetQuotePromptTemplate, {
quote: datasetQuoteText
})
: '',
documentQuoteText
? replaceVariable(Prompt_DocumentQuote, {
quote: documentQuoteText
})
: ''
]
.filter(Boolean)
.join('\n\n===---===---===\n\n');
const messages: ChatItemType[] = [
...getSystemPrompt_ChatItemType(replaceSystemPrompt),
...(stringQuoteText // file quote
? getSystemPrompt_ChatItemType(
replaceVariable(Prompt_DocumentQuote, {
quote: stringQuoteText
})
)
: []),
...getSystemPrompt_ChatItemType(concatenateSystemPrompt),
...histories,
{
obj: ChatRoleEnum.Human,
value: runtimePrompt2ChatsValue({
files: inputFiles,
files: userFiles,
text: replaceInputValue
})
}

View File

@@ -17,12 +17,14 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty
import { authAppByTmbId } from '../../../../support/permission/app/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { getAppVersionById } from '../../../app/version/controller';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.userChatInput]: string;
[NodeInputKeyEnum.history]?: ChatItemType[] | number;
[NodeInputKeyEnum.fileUrlList]?: string[];
[NodeInputKeyEnum.forbidStream]?: boolean;
[NodeInputKeyEnum.fileUrlList]?: string[];
}>;
type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.answerText]: string;
@@ -40,8 +42,24 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
variables
} = props;
const { system_forbid_stream = false, userChatInput, history, ...childrenAppVariables } = params;
if (!userChatInput) {
const {
system_forbid_stream = false,
userChatInput,
history,
fileUrlList,
...childrenAppVariables
} = params;
const { files } = chatValue2RuntimePrompt(query);
const userInputFiles = (() => {
if (fileUrlList) {
return fileUrlList.map((url) => parseUrlToFileType(url));
}
// Adapt version 4.8.13 upgrade
return files;
})();
if (!userChatInput && !userInputFiles) {
return Promise.reject('Input is empty');
}
if (!appId) {
@@ -72,7 +90,6 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
}
const chatHistories = getHistories(history, histories);
const { files } = chatValue2RuntimePrompt(query);
// Rewrite children app variables
const systemVariables = filterSystemVariables(variables);
@@ -102,7 +119,7 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
histories: chatHistories,
variables: childrenRunVariables,
query: runtimePrompt2ChatsValue({
files,
files: userInputFiles,
text: userChatInput
}),
chatConfig

View File

@@ -1,4 +1,5 @@
import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
import { ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
@@ -11,6 +12,24 @@ export const dispatchPluginInput = (props: PluginInputProps) => {
const { params, query } = props;
const { files } = chatValue2RuntimePrompt(query);
/*
对 params 中文件类型数据进行处理
* 插件单独运行时,这里会是一个特殊的数组
* 插件调用的话,这个参数是一个 string[] 不会进行处理
* 硬性要求API 单独调用插件时,要避免这种特殊类型冲突
*/
for (const key in params) {
const val = params[key];
if (
Array.isArray(val) &&
val.every(
(item) => item.type === ChatFileTypeEnum.file || item.type === ChatFileTypeEnum.image
)
) {
params[key] = val.map((item) => item.url);
}
}
return {
...params,
[DispatchNodeResponseKeyEnum.nodeResponse]: {},

View File

@@ -2,16 +2,15 @@ import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runti
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import { documentFileType } from '@fastgpt/global/common/file/constants';
import axios from 'axios';
import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
import { MongoRawTextBuffer } from '../../../../common/buffer/rawText/schema';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
import { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
type Props = ModuleDispatchProps<{
@@ -48,12 +47,41 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
runningAppInfo: { teamId },
histories,
chatConfig,
node: { version },
params: { fileUrlList = [] }
} = props;
const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 20;
// Get files from histories
const filesFromHistories = histories
const filesFromHistories = version !== '489' ? [] : getHistoryFileLinks(histories);
const { text, readFilesResult } = await getFileContentFromLinks({
// Concat fileUrlList and filesFromHistories; remove not supported files
urls: [...fileUrlList, ...filesFromHistories],
requestOrigin,
maxFiles,
teamId
});
return {
[NodeOutputKeyEnum.text]: text,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
readFiles: readFilesResult.map((item) => ({
name: item?.filename || '',
url: item?.url || ''
})),
readFilesResult: readFilesResult
.map((item) => item?.nodeResponsePreviewText ?? '')
.join('\n******\n')
},
[DispatchNodeResponseKeyEnum.toolResponses]: {
fileContent: text
}
};
};
export const getHistoryFileLinks = (histories: ChatItemType[]) => {
return histories
.filter((item) => {
if (item.obj === ChatRoleEnum.Human) {
return item.value.filter((value) => value.type === 'file');
@@ -70,26 +98,27 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
return files;
})
.flat();
};
// Concat fileUrlList and filesFromHistories; remove not supported files
const parseUrlList = [...fileUrlList, ...filesFromHistories]
export const getFileContentFromLinks = async ({
urls,
requestOrigin,
maxFiles,
teamId
}: {
urls: string[];
requestOrigin?: string;
maxFiles: number;
teamId: string;
}) => {
const parseUrlList = urls
.map((url) => {
try {
// Avoid "/api/xxx" file error.
const origin = requestOrigin ?? 'http://localhost:3000';
// Check is system upload file
if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
// Parse url, get filename query. Keep only documents that can be parsed
const parseUrl = new URL(url, origin);
const filenameQuery = parseUrl.searchParams.get('filename');
// Not document
if (filenameQuery) {
const extensionQuery = filenameQuery.split('.').pop()?.toLowerCase() || '';
if (!documentFileType.includes(extensionQuery)) {
return '';
}
if (parseUrlToFileType(url)?.type !== 'file') {
return '';
}
// Remove the origin(Make intranet requests directly)
@@ -197,18 +226,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
const text = readFilesResult.map((item) => item?.text ?? '').join('\n******\n');
return {
[NodeOutputKeyEnum.text]: text,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
readFiles: readFilesResult.map((item) => ({
name: item?.filename || '',
url: item?.url || ''
})),
readFilesResult: readFilesResult
.map((item) => item?.nodeResponsePreviewText ?? '')
.join('\n******\n')
},
[DispatchNodeResponseKeyEnum.toolResponses]: {
fileContent: text
}
text,
readFilesResult
};
};

View File

@@ -1,14 +1,5 @@
import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { countPromptTokens } from '../../common/string/tiktoken/index';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import {
getPluginInputsFromStoreNodes,
getPluginRunContent
} from '@fastgpt/global/core/app/plugin/utils';
import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
import { RuntimeUserPromptType, UserChatItemType } from '@fastgpt/global/core/chat/type';
import { runtimePrompt2ChatsValue } from '@fastgpt/global/core/chat/adapt';
/* filter search result */
export const filterSearchResultsByMaxChars = async (

View File

@@ -40,6 +40,7 @@
"export_config_successful": "Configuration copied, some sensitive information automatically filtered. Please check for any remaining sensitive data.",
"export_configs": "Export Configurations",
"feedback_count": "User Feedback",
"file_quote_link": "Files",
"file_recover": "File will overwrite current content",
"file_upload": "File Upload",
"file_upload_tip": "Once enabled, documents/images can be uploaded. Documents are retained for 7 days, images for 15 days. Using this feature may incur additional costs. To ensure a good experience, please choose an AI model with a larger context length when using this feature.",
@@ -47,7 +48,7 @@
"go_to_chat": "Go to Conversation",
"go_to_run": "Go to Execution",
"image_upload": "Image Upload",
"image_upload_tip": "Please ensure to select a vision model that can process images.",
"image_upload_tip": "How to activate model image recognition capabilities",
"import_configs": "Import Configurations",
"import_configs_failed": "Import configuration failed, please ensure the configuration is correct!",
"import_configs_success": "Import Successful",
@@ -61,7 +62,7 @@
"intro": "A comprehensive model application orchestration system that offers out-of-the-box data processing and model invocation capabilities. It allows for rapid Dataset construction and workflow orchestration through Flow visualization, enabling complex Dataset scenarios!",
"llm_not_support_vision": "This model does not support image recognition",
"llm_use_vision": "Enable Image Recognition",
"llm_use_vision_tip": "Once image recognition is enabled, this model will automatically receive images uploaded from the 'dialog box' and image links in 'user questions'.",
"llm_use_vision_tip": "After clicking on the model selection, you can see whether the model supports image recognition and the ability to control whether to start image recognition. \nAfter starting image recognition, the model will read the image content in the file link, and if the user question is less than 500 words, it will automatically parse the image in the user question.",
"logs_chat_user": "user",
"logs_empty": "No logs yet~",
"logs_message_total": "Total Messages",
@@ -74,6 +75,7 @@
"month.unit": "Day",
"move_app": "Move Application",
"not_json_file": "Please select a JSON file",
"open_vision_function_tip": "Models with icon switches have image recognition capabilities. \nAfter being turned on, the model will parse the pictures in the file link and automatically parse the pictures in the user's question (user question ≤ 500 words).",
"or_drag_JSON": "or drag in JSON file",
"paste_config": "Paste Configuration",
"permission.des.manage": "Based on write permissions, you can configure publishing channels, view conversation logs, and assign permissions to the application.",
@@ -135,7 +137,7 @@
"version_back": "Revert to Original State",
"version_copy": "Duplicate",
"version_initial_copy": "Duplicate - Original State",
"vision_model_title": "Enable Image Recognition",
"vision_model_title": "Image recognition ability",
"week.Friday": "Friday",
"week.Monday": "Monday",
"week.Saturday": "Saturday",

View File

@@ -29,6 +29,7 @@
"multiple_AI_conversations": "Multiple AI Conversations",
"new_input_guide_lexicon": "New Lexicon",
"no_workflow_response": "No workflow data",
"not_select_file": "No file selected",
"plugins_output": "Plugin Output",
"question_tip": "From top to bottom, the response order of each module",
"response.node_inputs": "Node Inputs",
@@ -40,4 +41,4 @@
"upload": "Upload",
"view_citations": "View References",
"web_site_sync": "Web Site Sync"
}
}

View File

@@ -122,6 +122,8 @@
"pass_returned_object_as_output_to_next_nodes": "Pass the object returned in the code as output to the next nodes. The variable name needs to correspond to the return key.",
"plugin.Instruction_Tip": "You can configure an instruction to explain the purpose of the plugin. This instruction will be displayed each time the plugin is used. Supports standard Markdown syntax.",
"plugin.Instructions": "Instructions",
"plugin.global_file_input": "File links (deprecated)",
"plugin_file_abandon_tip": "Plugin global file upload has been deprecated, please adjust it as soon as possible. \nRelated functions can be achieved through plug-in input and adding image type input.",
"plugin_input": "Plugin Input",
"plugin_output_tool": "When the plug-in is executed as a tool, whether this field responds as a result of the tool",
"question_classification": "Question Classification",
@@ -190,4 +192,4 @@
"workflow.Switch_success": "Switch Successful",
"workflow.Team cloud": "Team Cloud",
"workflow.exit_tips": "Your changes have not been saved. 'Exit directly' will not save your edits."
}
}

View File

@@ -40,6 +40,7 @@
"export_config_successful": "已复制配置,自动过滤部分敏感信息,请注意检查是否仍有敏感数据",
"export_configs": "导出配置",
"feedback_count": "用户反馈",
"file_quote_link": "文件链接",
"file_recover": "文件将覆盖当前内容",
"file_upload": "文件上传",
"file_upload_tip": "开启后,可以上传文档/图片。文档保留7天图片保留15天。使用该功能可能产生较多额外费用。为保证使用体验使用该功能时请选择上下文长度较大的AI模型。",
@@ -47,7 +48,7 @@
"go_to_chat": "去对话",
"go_to_run": "去运行",
"image_upload": "图片上传",
"image_upload_tip": "请确保选择可处理图片的视觉模型",
"image_upload_tip": "如何启动模型图片识别能力",
"import_configs": "导入配置",
"import_configs_failed": "导入配置失败,请确保配置正常!",
"import_configs_success": "导入成功",
@@ -60,8 +61,8 @@
"interval.per_hour": "每小时",
"intro": "是一个大模型应用编排系统,提供开箱即用的数据处理、模型调用等能力,可以快速的构建知识库并通过 Flow 可视化进行工作流编排,实现复杂的知识库场景!",
"llm_not_support_vision": "该模型不支持图片识别",
"llm_use_vision": "启用图片识别",
"llm_use_vision_tip": "启用图片识别后,模型会自动接收来自“对话框上传”的图片,以及“用户问题中的图片链接。",
"llm_use_vision": "图片识别",
"llm_use_vision_tip": "点击模型选择后,可以看到模型是否支持图片识别以及控制是否启动图片识别的能力。启动图片识别后,模型会读取文件链接里图片内容,并且如果用户问题少于 500 字,会自动解析用户问题中的图片。",
"logs_chat_user": "使用者",
"logs_empty": "还没有日志噢~",
"logs_message_total": "消息总数",
@@ -72,9 +73,10 @@
"module.type": "\"{{type}}\"类型\n{{description}}",
"modules.Title is required": "模块名不能为空",
"month.unit": "号",
"move_app": "移动应用",
"move.hint": "移动后,所选应用/文件夹将继承新文件夹的权限设置,原先的权限设置失效。",
"move_app": "移动应用",
"not_json_file": "请选择JSON文件",
"open_vision_function_tip": "有图示开关的模型即拥有图片识别能力。若开启模型会解析文件链接里的图片并自动解析用户问题中的图片用户问题≤500字时生效。",
"or_drag_JSON": "或拖入JSON文件",
"paste_config": "粘贴配置",
"permission.des.manage": "写权限基础上,可配置发布渠道、查看对话日志、分配该应用权限",
@@ -136,7 +138,7 @@
"version_back": "回到初始状态",
"version_copy": "副本",
"version_initial_copy": "副本-初始状态",
"vision_model_title": "启用图片识别",
"vision_model_title": "图片识别能力",
"week.Friday": "星期五",
"week.Monday": "星期一",
"week.Saturday": "星期六",

View File

@@ -29,6 +29,7 @@
"multiple_AI_conversations": "多组 AI 对话",
"new_input_guide_lexicon": "新词库",
"no_workflow_response": "没有运行数据",
"not_select_file": "未选择文件",
"plugins_output": "插件输出",
"question_tip": "从上到下,为各个模块的响应顺序",
"response.child total points": "子工作流积分消耗",
@@ -41,4 +42,4 @@
"upload": "上传",
"view_citations": "查看引用",
"web_site_sync": "Web站点同步"
}
}

View File

@@ -123,6 +123,8 @@
"pass_returned_object_as_output_to_next_nodes": "将代码中 return 的对象作为输出,传递给后续的节点。变量名需要对应 return 的 key",
"plugin.Instruction_Tip": "可以配置一段说明,以解释该插件的用途。每次使用插件前,会显示该段说明。支持标准 Markdown 语法。",
"plugin.Instructions": "使用说明",
"plugin.global_file_input": "文件链接(弃用)",
"plugin_file_abandon_tip": "插件全局文件上传已弃用,请尽快调整。可以通过插件输入,添加图片类型输入来实现相关功能。",
"plugin_input": "插件输入",
"plugin_output_tool": "插件作为工具执行时,该字段是否作为工具响应结果",
"question_classification": "问题分类",
@@ -192,4 +194,4 @@
"workflow.Switch_success": "切换成功",
"workflow.Team cloud": "团队云端",
"workflow.exit_tips": "您的更改尚未保存,「直接退出」将不会保存您的编辑记录。"
}
}