update research mode, add search web and fetch url content

src/utils/modes.ts

@@ -78,6 +78,15 @@ export function getToolsForMode(groups: readonly GroupEntry[]): string[] {
 
 // Main modes configuration as an ordered array
 export const modes: readonly ModeConfig[] = [
+	{
+		slug: "research",
+		name: "Research",
+		roleDefinition:
+			"You are Infio, an advanced research assistant specialized in comprehensive investigation and analytical thinking. You excel at breaking down complex questions, exploring multiple perspectives, and synthesizing information to provide well-reasoned conclusions.",
+		groups: ["research"],
+		customInstructions:
+			"You can conduct thorough research by analyzing available information, connecting related concepts, and applying structured reasoning methods. Help users explore topics in depth by considering multiple angles, identifying relevant evidence, and evaluating the reliability of sources. Use step-by-step analysis when tackling complex problems, explaining your thought process clearly. Create visual representations like Mermaid diagrams when they help clarify relationships between ideas. Use Markdown tables to present statistical data or comparative information when appropriate. Present balanced viewpoints while highlighting the strength of evidence behind different conclusions.",
+	},
 	{
 		slug: "write",
 		name: "Write",

src/utils/parse-infio-block.ts

@@ -60,6 +60,14 @@ export type ParsedMsgBlock =
 		path: string
 		query: string
 		finish: boolean
+	} | {
+		type: 'search_web'
+		query: string
+		finish: boolean
+	} | {
+		type: 'fetch_urls_content'
+		urls: string[]
+		finish: boolean
 	}
 
 export function parseMsgBlocks(
@@ -416,6 +424,66 @@ export function parseMsgBlocks(
 				})
 				lastEndOffset = endOffset
 			}
+			else if (node.nodeName === 'search_web') {
+				if (!node.sourceCodeLocation) {
+					throw new Error('sourceCodeLocation is undefined')
+				}
+				const startOffset = node.sourceCodeLocation.startOffset
+				const endOffset = node.sourceCodeLocation.endOffset
+				if (startOffset > lastEndOffset) {
+					parsedResult.push({
+						type: 'string',
+						content: input.slice(lastEndOffset, startOffset),
+					})
+				}
+				let query: string | undefined
+				for (const childNode of node.childNodes) {
+					if (childNode.nodeName === 'query' && childNode.childNodes.length > 0) {
+						query = childNode.childNodes[0].value
+					}
+				}
+				parsedResult.push({
+					type: 'search_web',
+					query: query || '',
+					finish: node.sourceCodeLocation.endTag !== undefined
+				})
+				lastEndOffset = endOffset
+			} else if (node.nodeName === 'fetch_urls_content') {
+				if (!node.sourceCodeLocation) {
+					throw new Error('sourceCodeLocation is undefined')
+				}
+				const startOffset = node.sourceCodeLocation.startOffset
+				const endOffset = node.sourceCodeLocation.endOffset
+				if (startOffset > lastEndOffset) {
+					parsedResult.push({
+						type: 'string',
+						content: input.slice(lastEndOffset, startOffset),
+					})
+				}
+				
+				let urls: string[] = []
+				
+				for (const childNode of node.childNodes) {
+					if (childNode.nodeName === 'urls' && childNode.childNodes.length > 0) {
+						try {
+							const urlsJson = childNode.childNodes[0].value
+							const parsedUrls = JSON5.parse(urlsJson)
+							if (Array.isArray(parsedUrls)) {
+								urls = parsedUrls
+							}
+						} catch (error) {
+							console.error('Failed to parse URLs JSON', error)
+						}
+					}
+				}
+				
+				parsedResult.push({
+					type: 'fetch_urls_content',
+					urls,
+					finish: node.sourceCodeLocation.endTag !== undefined
+				})
+				lastEndOffset = endOffset
+			}
 		}
 
 		// handle the last part of the input

src/utils/prompt-generator.ts

@@ -18,7 +18,6 @@ import {
 import { InfioSettings } from '../types/settings'
 import { defaultModeSlug, getFullModeDetails } from "../utils/modes"
 
-import { listFilesAndFolders } from './glob-utils'
 import {
 	readTFileContent
 } from './obsidian'

src/utils/tool-groups.ts

@@ -30,6 +30,9 @@ export const TOOL_GROUPS: Record<string, ToolGroupConfig> = {
 	edit: {
 		tools: ["apply_diff", "write_to_file", "insert_content", "search_and_replace"],
 	},
+	research: {
+		tools: ["search_web", "fetch_urls_content"],
+	},
 	// browser: {
 	// 	tools: ["browser_action"],
 	// },
@@ -52,7 +55,6 @@ export const ALWAYS_AVAILABLE_TOOLS = [
 	"ask_followup_question",
 	"attempt_completion",
 	"switch_mode",
-	"new_task",
 ] as const
 
 // Tool name types for type safety
@@ -71,6 +73,7 @@ export function getToolOptions(toolConfig: string | readonly [ToolName, ...any[]
 export const GROUP_DISPLAY_NAMES: Record<ToolGroup, string> = {
 	read: "Read Files",
 	edit: "Edit Files",
+	research: "Research",
 	browser: "Use Browser",
 	command: "Run Commands",
 	mcp: "Use MCP",

src/utils/web-search.ts

@@ -0,0 +1,151 @@
+import https from 'https';
+
+import { htmlToMarkdown, requestUrl } from 'obsidian';
+
+import { YoutubeTranscript, isYoutubeUrl } from './youtube-transcript';
+
+const SERPER_API_KEY = 'a6fd4dc4b79f10b1e5008b688c81bacef0d24b4d5cd4e52071afa8329a67497c'
+
+interface SearchResult {
+	title: string;
+	link: string;
+	snippet: string;
+}
+
+interface SearchResponse {
+	organic_results?: SearchResult[];
+}
+
+export async function webSearch(query: string): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const url = `https://serpapi.com/search?q=${encodeURIComponent(query)}&engine=google&api_key=${SERPER_API_KEY}&num=20`;
+
+		console.log(url)
+
+		https.get(url, (res: any) => {
+			let data = '';
+
+			res.on('data', (chunk: Buffer) => {
+				data += chunk.toString();
+			});
+
+			res.on('end', () => {
+				try {
+					console.log(data)
+					let parsedData: SearchResponse;
+					try {
+						parsedData = JSON.parse(data);
+					} catch {
+						parsedData = { organic_results: undefined };
+					}
+					const results = parsedData?.organic_results;
+
+					if (!results) {
+						resolve('');
+						return;
+					}
+
+					const formattedResults = results.map((item: SearchResult) => {
+						return `title: ${item.title}\nurl: ${item.link}\nsnippet: ${item.snippet}\n`;
+					}).join('\n\n');
+
+					resolve(formattedResults);
+				} catch (error) {
+					reject(error);
+				}
+			});
+		}).on('error', (error: Error) => {
+			reject(error);
+		});
+	});
+}
+
+async function getWebsiteContent(url: string): Promise<string> {
+	if (isYoutubeUrl(url)) {
+		// TODO: pass language based on user preferences
+		const { title, transcript } =
+			await YoutubeTranscript.fetchTranscriptAndMetadata(url)
+
+		return `Title: ${title}
+Video Transcript:
+${transcript.map((t) => `${t.offset}: ${t.text}`).join('\n')}`
+	}
+
+	const response = await requestUrl({ url })
+
+	return htmlToMarkdown(response.text)
+}
+
+const USE_JINA = true
+
+export async function fetchUrlsContent(urls: string[]): Promise<string> {
+	return new Promise((resolve) => {
+		const results = urls.map(async (url) => {
+			try {
+				const content = USE_JINA ? await fetchJina(url) : await getWebsiteContent(url);
+				return `<url_content url="${url}">\n${content}\n</url_content>`;
+			} catch (error) {
+				console.error(`获取URL内容失败: ${url}`, error);
+				return `<url_content url="${url}">\n获取内容失败: ${error}\n</url_content>`;
+			}
+		});
+		
+		console.log('fetchUrlsContent', results);
+		
+		Promise.all(results).then((texts) => {
+			resolve(texts.join('\n\n'));
+		}).catch((error) => {
+			console.error('获取URLs内容时出错', error);
+			resolve('fetch urls content error'); // 即使出错也返回一些内容
+		});
+	});
+}
+
+function fetchJina(url: string): Promise<string> {
+	return new Promise((resolve) => {
+		const jinaUrl = `https://r.jina.ai/${url}`;
+
+		const jinaHeaders = {
+			'Authorization': 'Bearer jina_1d721eb8c4814a938b4351ae0c3a0f117FlTTAz1GOmpOsIDN7HvIyLbiOCe',
+			'X-No-Cache': 'true',
+		};
+
+		const jinaOptions: https.RequestOptions = {
+			method: 'GET',
+			headers: jinaHeaders,
+		};
+
+		const req = https.request(jinaUrl, jinaOptions, (res) => {
+			let data = '';
+
+			res.on('data', (chunk) => {
+				data += chunk;
+			});
+
+			res.on('end', () => {
+				console.log(data);
+				
+				try {
+					// 检查是否有错误响应
+					const response = JSON.parse(data);
+					if (response.code && response.message) {
+						console.error(`JINA API 错误: ${response.message}`);
+						resolve(`无法获取内容: ${response.message}`);
+						return;
+					}
+					resolve(data);
+				} catch (e) {
+					// 如果不是JSON格式，可能是正常的内容
+					resolve(data);
+				}
+			});
+		});
+
+		req.on('error', (e) => {
+			console.error(`Error: ${e.message}`);
+			resolve(`fetch jina error: ${e.message}`);
+		});
+
+		req.end();
+	});
+}