init

src/utils/apply.ts

@@ -0,0 +1,41 @@
+import { TFile } from 'obsidian'
+
+// 替换指定行范围的内容
+const replaceLines = (content: string, startLine: number, endLine: number, newContent: string): string => {
+	const lines = content.split('\n')
+	const beforeLines = lines.slice(0, startLine - 1)
+	const afterLines = lines.slice(endLine) // 这里不需要 +1 因为 endLine 指向的是要替换的最后一行
+	return [...beforeLines, newContent, ...afterLines].join('\n')
+}
+
+export const manualApplyChangesToFile = async (
+	content: string,
+	currentFile: TFile,
+	currentFileContent: string,
+	startLine?: number,
+	endLine?: number,
+): Promise<string | null> => {
+	console.log('Manual apply changes to file:', currentFile.path)
+	console.log('Start line:', startLine)
+	console.log('End line:', endLine)
+	console.log('Content:', content)
+	try {
+		if (!startLine || !endLine) {
+			console.error('Missing line numbers for edit')
+			return null
+		}
+
+		// 直接替换指定行范围的内容
+		const newContent = replaceLines(
+			currentFileContent,
+			startLine,
+			endLine,
+			content
+		)
+
+		return newContent
+	} catch (error) {
+		console.error('Error applying changes:', error)
+		return null
+	}
+}

src/utils/auto-complete.ts

@@ -0,0 +1,226 @@
+import { cloneDeep, each, get, has, isArray, isEqual, isNumber, isObject, isString, set, unset } from "lodash";
+import * as mm from "micromatch";
+import { err, ok, Result } from "neverthrow";
+import { z, ZodError, ZodIssueCode, ZodType } from 'zod';
+
+import { DEFAULT_SETTINGS, PluginData, Settings, settingsSchema } from "../settings/versions";
+import { isSettingsV0, isSettingsV1, migrateFromV0ToV1 } from "../settings/versions/migration";
+import { InfioSettings } from '../types/settings';
+
+type JSONObject = Record<string, any>;
+
+export function checkForErrors(settings: InfioSettings) {
+	const errors = new Map<string, string>();
+	const parsingResult = parseWithSchema(settingsSchema, settings);
+
+	if (parsingResult.isOk()) {
+		return errors;
+	}
+
+	if (parsingResult.error instanceof ZodError) {
+		for (const issue of parsingResult.error.issues) {
+			errors.set(issue.path.join('.'), issue.message);
+		}
+	} else {
+		throw parsingResult.error;
+	}
+
+	return errors;
+}
+
+
+export function fixStructureAndValueErrors<T extends ZodType>(
+	schema: T,
+	value: any | null | undefined,
+	defaultValue: z.infer<T>,
+): Result<ReturnType<T["parse"]>, Error> {
+	if (value === null || value === undefined) {
+		value = {};
+	}
+	let result = parseWithSchema(schema, value);
+
+	if (result.isErr()) {
+		value = addMissingKeys(value, result.error, defaultValue);
+		value = removeUnrecognizedKeys(value, result.error);
+		result = parseWithSchema(schema, value);
+	}
+
+	if (result.isErr() && value !== null && value !== undefined) {
+		value = replaceValuesWithErrorsByDefaultValue(value, result.error, defaultValue);
+		result = parseWithSchema(schema, value);
+	}
+
+	return result;
+}
+
+export function parseWithSchema<T extends ZodType>(
+	schema: T,
+	value: JSONObject | null | undefined
+): Result<ReturnType<T["parse"]>, ZodError> {
+	const parsingResult = schema.safeParse(value);
+	return parsingResult.success ? ok(parsingResult.data) : err(parsingResult.error);
+}
+
+function addMissingKeys<T extends object>(value: JSONObject, error: ZodError, defaultValue: T): JSONObject {
+	const invalidTypeIssues = error.issues.filter(issue => issue.code === ZodIssueCode.invalid_type);
+	const errorPaths = invalidTypeIssues
+		.map(issue => issue.path)
+		.map(path => reduceArrayPathToFirstObjectPath(path))
+		.map(path => path.join('.'));
+
+	return replaceValueWithDefaultValue(value, errorPaths, defaultValue);
+}
+
+function replaceValueWithDefaultValue<V, T>(
+	value: any,
+	paths: string[],
+	defaultValue: T,
+): V {
+	const result = cloneDeep(value) as any;
+	paths.forEach(path => {
+		const originalValue = has(defaultValue, path) ? get(defaultValue, path) : undefined;
+		set(result, path, originalValue);
+	});
+
+	return result;
+}
+
+
+function removeUnrecognizedKeys(value: JSONObject | null | undefined, error: ZodError): JSONObject {
+	if (typeof value !== 'object' || value === null || value === undefined) {
+		return {};
+	}
+	// Zod unrecognized_keys issues consist of two parts:
+	// - path to the nested object where the unrecognized key was found
+	// - the key itself which is unrecognized
+	const unrecognizedPaths = error.issues
+
+		.filter(issue => issue.code === ZodIssueCode.unrecognized_keys)
+		// Array path will be handled separately by the value replacement function
+		.filter(issue => !isAnArrayPath(issue.path))
+		.flatMap(issue => {
+			// @ts-ignore
+			const keys = issue.keys;
+			return keys.map(key => [...issue.path, key].join('.'));
+		});
+
+	unrecognizedPaths.forEach(path => {
+		unset(value, path);
+	});
+	return value;
+}
+
+
+function replaceValuesWithErrorsByDefaultValue<T>(
+	value: JSONObject,
+	error: ZodError,
+	defaultValue: T
+): T {
+	const errorPaths = error.issues
+		.map(issue => issue.path)
+		.map(path => reduceArrayPathToFirstObjectPath(path))
+		.map(path => path.join('.'));
+
+	return replaceValueWithDefaultValue(value, errorPaths, defaultValue);
+}
+
+function reduceArrayPathToFirstObjectPath(path: (string | number)[]): (string | number)[] {
+	const result: (string | number)[] = [];
+	for (const key of path) {
+		if (typeof key === 'number') {
+			break;
+		}
+		result.push(key);
+	}
+
+	return result;
+}
+
+function isAnArrayPath(path: (string | number)[]): boolean {
+	return path.some(key => typeof key === 'number');
+}
+
+export function serializeSettings(settings: Settings): PluginData {
+	return { settings: settings };
+}
+
+export function deserializeSettings(data: JSONObject | null | undefined): Result<Settings, Error> {
+	let settings: any;
+	if (data === null || data === undefined || !data.hasOwnProperty("settings")) {
+		settings = {};
+	} else {
+		settings = data.settings;
+	}
+
+	if (isSettingsV0(settings)) {
+		console.log("Migrating settings from v0 to v1");
+		settings = migrateFromV0ToV1(settings);
+	}
+	if (!isSettingsV1(settings)) {
+		console.log("Fixing settings structure and value errors");
+		return fixStructureAndValueErrors(settingsSchema, settings, DEFAULT_SETTINGS);
+	}
+
+	return parseWithSchema(settingsSchema, settings);
+}
+
+export function isRegexValid(value: string): boolean {
+	try {
+		const regex = new RegExp(value);
+		regex.test("");
+		return true;
+	} catch (e) {
+		return false;
+	}
+}
+
+export function isValidIgnorePattern(value: string): boolean {
+	try {
+		mm.isMatch("", value);
+		return true;
+	} catch (e) {
+		return false;
+	}
+}
+
+export function findEqualPaths(obj1: any, obj2: any, basePath = ''): string[] {
+	let paths: string[] = [];
+
+	if (
+		basePath === ''
+		&& (
+			!isObject(obj1)
+			|| !isObject(obj2)
+			|| isArray(obj1)
+			|| isArray(obj2)
+			|| isNumber(obj1)
+			|| isNumber(obj2)
+			|| isString(obj1)
+			|| isString(obj2)
+		)
+	) {
+		return [];
+	}
+
+	// Function to iterate over keys and compare values
+	function iterateKeys(value: any, key: string | number): void {
+		const path = basePath ? `${basePath}.${key}` : `${key}`;
+		if (isObject(value) && isObject(get(obj2, key))) {
+			// Recursively find paths for nested objects
+			paths = paths.concat(findEqualPaths(value, get(obj2, key), path));
+		} else if (isEqual(value, get(obj2, key))) {
+			// Add path to array if values are equal
+			paths.push(path);
+		}
+	}
+
+	// If both are arrays, iterate using each index
+	if (isArray(obj1) && isArray(obj2)) {
+		each(obj1, (value, index) => iterateKeys(value, `[${index}]`));
+	} else {
+		// Iterate over keys of the first object
+		each(obj1, iterateKeys);
+	}
+
+	return paths;
+}

src/utils/content-filter.ts

@@ -0,0 +1,9 @@
+/**
+ * Removes AI code block tags from content
+ * @param content The content to filter
+ * @returns The filtered content without AI code block tags
+ */
+export const removeAITags = (content: string): string => {
+	// Remove ```infioedit\n and ``` tags
+	return content.replace(/```infioedit\n```\n/g, '');
+}

src/utils/fuzzy-search.ts

@@ -0,0 +1,207 @@
+import fuzzysort from 'fuzzysort'
+import { App, TFile, TFolder } from 'obsidian'
+
+import {
+	MentionableFile,
+	MentionableFolder,
+	MentionableVault,
+} from '../types/mentionable'
+
+import { calculateFileDistance, getOpenFiles } from './obsidian'
+
+export type SearchableMentionable =
+	| MentionableFile
+	| MentionableFolder
+	| MentionableVault
+
+type VaultSearchItem = {
+	type: 'vault'
+	path: string
+}
+
+type FileWithMetadata = {
+	type: 'file'
+	path: string
+	name: string
+	file: TFile
+	opened: boolean
+	distance: number | null
+	daysSinceLastModified: number
+}
+
+type FolderWithMetadata = {
+	type: 'folder'
+	path: string
+	name: string
+	folder: TFolder
+	distance: number | null
+}
+
+type SearchItem = FolderWithMetadata | FileWithMetadata | VaultSearchItem
+
+function scoreFnWithBoost({
+	searchItem,
+	pathScore,
+	nameScore,
+}: {
+	searchItem: SearchItem
+	pathScore: number
+	nameScore: number
+}): number {
+	const score = Math.max(pathScore, nameScore)
+
+	let boost = 1
+	switch (searchItem.type) {
+		case 'file': {
+			const { opened, distance, daysSinceLastModified } = searchItem
+
+			// Boost for open files
+			if (opened) boost = Math.max(boost, 3)
+
+			// Boost for recently modified files
+			if (daysSinceLastModified < 30) {
+				const recentBoost = 1 + 2 / (daysSinceLastModified + 2)
+				boost = Math.max(boost, recentBoost)
+			}
+
+			// Boost for nearby files
+			if (distance !== null && distance > 0 && distance <= 5) {
+				const nearbyBoost = 1 + 0.5 / Math.max(distance - 1, 1)
+				boost = Math.max(boost, nearbyBoost)
+			}
+
+			break
+		}
+		case 'folder': {
+			const { distance } = searchItem
+
+			// Boost for nearby folders
+			if (distance !== null && distance > 0 && distance <= 5) {
+				const nearbyBoost = 1 + 0.5 / Math.max(distance - 1, 1)
+				boost = Math.max(boost, nearbyBoost)
+			}
+
+			break
+		}
+		case 'vault': {
+			if (score === 1) {
+				boost = 3
+			}
+
+			break
+		}
+	}
+
+	// Normalize the boost
+	const normalizedScore =
+		boost > 1 ? Math.log(boost * score + 1) / Math.log(boost + 1) : score
+	return normalizedScore
+}
+
+function getEmptyQueryResult(
+	searchItems: SearchItem[],
+	limit: number,
+): SearchableMentionable[] {
+	// Sort files based on a custom scoring function
+	const sortedFiles = searchItems.sort((a, b) => {
+		const scoreA = scoreFnWithBoost({
+			searchItem: a,
+			pathScore: 0.5, // Use 0.5 as a base score
+			nameScore: 0.5,
+		})
+		const scoreB = scoreFnWithBoost({
+			searchItem: b,
+			pathScore: 0.5,
+			nameScore: 0.5,
+		})
+		return scoreB - scoreA // Sort in descending order
+	})
+
+	// Return only the top 'limit' files
+	return sortedFiles
+		.slice(0, limit)
+		.map((item) => searchItemToMentionable(item))
+}
+
+export function fuzzySearch(app: App, query: string): SearchableMentionable[] {
+	const currentFile = app.workspace.getActiveFile()
+	const openFiles = getOpenFiles(app)
+
+	const allSupportedFiles = app.vault.getFiles().filter((file) => {
+		const extension = file.extension
+		return extension === 'md'
+	})
+
+	const allFilesWithMetadata: SearchItem[] = allSupportedFiles.map((file) => ({
+		type: 'file',
+		path: file.path,
+		name: file.name,
+		file,
+		opened: openFiles.some((f) => f.path === file.path),
+		distance: currentFile
+			? currentFile === file
+				? null
+				: calculateFileDistance(currentFile, file)
+			: null,
+		daysSinceLastModified:
+			(Date.now() - file.stat.mtime) / (1000 * 60 * 60 * 24),
+	}))
+
+	const allFolders = app.vault.getAllFolders()
+	const allFoldersWithMetadata: SearchItem[] = allFolders.map((folder) => ({
+		type: 'folder',
+		path: folder.path,
+		name: folder.name,
+		folder,
+		distance: currentFile ? calculateFileDistance(currentFile, folder) : null,
+	}))
+
+	const vaultItem: VaultSearchItem = {
+		type: 'vault',
+		path: 'vault',
+	}
+
+	const searchItems: SearchItem[] = [
+		...allFilesWithMetadata,
+		...allFoldersWithMetadata,
+		vaultItem,
+	]
+
+	if (!query) {
+		return getEmptyQueryResult(searchItems, 20)
+	}
+
+	const results = fuzzysort.go(query, searchItems, {
+		keys: ['path', 'name'],
+		threshold: 0.2,
+		limit: 20,
+		all: true,
+		scoreFn: (result) =>
+			scoreFnWithBoost({
+				searchItem: result.obj,
+				pathScore: result[0].score,
+				nameScore: result[1].score,
+			}),
+	})
+
+	return results.map((result) => searchItemToMentionable(result.obj))
+}
+
+function searchItemToMentionable(item: SearchItem): SearchableMentionable {
+	switch (item.type) {
+		case 'file':
+			return {
+				type: 'file',
+				file: item.file,
+			}
+		case 'folder':
+			return {
+				type: 'folder',
+				folder: item.folder,
+			}
+		case 'vault':
+			return {
+				type: 'vault',
+			}
+	}
+}

src/utils/glob-utils.ts

@@ -0,0 +1,12 @@
+import { minimatch } from 'minimatch'
+import { Vault } from 'obsidian'
+
+export const findFilesMatchingPatterns = async (
+	patterns: string[],
+	vault: Vault,
+) => {
+	const files = vault.getMarkdownFiles()
+	return files.filter((file) => {
+		return patterns.some((pattern) => minimatch(file.path, pattern))
+	})
+}

src/utils/image.ts

@@ -0,0 +1,34 @@
+import { MentionableImage } from '../types/mentionable'
+
+export function parseImageDataUrl(dataUrl: string): {
+	mimeType: string
+	base64Data: string
+} {
+	const matches = dataUrl.match(/^data:([^;]+);base64,(.+)/)
+	if (!matches) {
+		throw new Error('Invalid image data URL format')
+	}
+	const [, mimeType, base64Data] = matches
+	return { mimeType, base64Data }
+}
+
+export async function fileToMentionableImage(
+	file: File,
+): Promise<MentionableImage> {
+	const base64Data = await fileToBase64(file)
+	return {
+		type: 'image',
+		name: file.name,
+		mimeType: file.type,
+		data: base64Data,
+	}
+}
+
+function fileToBase64(file: File): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const reader = new FileReader()
+		reader.readAsDataURL(file)
+		reader.onload = () => resolve(reader.result as string)
+		reader.onerror = () => reject(new Error('Failed to read file'))
+	})
+}

src/utils/mentionable.ts

@@ -0,0 +1,164 @@
+import { App } from 'obsidian'
+
+import { Mentionable, SerializedMentionable } from '../types/mentionable'
+
+export const serializeMentionable = (
+	mentionable: Mentionable,
+): SerializedMentionable => {
+	switch (mentionable.type) {
+		case 'file':
+			return {
+				type: 'file',
+				file: mentionable.file.path,
+			}
+		case 'folder':
+			return {
+				type: 'folder',
+				folder: mentionable.folder.path,
+			}
+		case 'vault':
+			return {
+				type: 'vault',
+			}
+		case 'current-file':
+			return {
+				type: 'current-file',
+				file: mentionable.file?.path ?? null,
+			}
+		case 'block':
+			return {
+				type: 'block',
+				content: mentionable.content,
+				file: mentionable.file.path,
+				startLine: mentionable.startLine,
+				endLine: mentionable.endLine,
+			}
+		case 'url':
+			return {
+				type: 'url',
+				url: mentionable.url,
+			}
+		case 'image':
+			return {
+				type: 'image',
+				name: mentionable.name,
+				mimeType: mentionable.mimeType,
+				data: mentionable.data,
+			}
+	}
+}
+
+export const deserializeMentionable = (
+	mentionable: SerializedMentionable,
+	app: App,
+): Mentionable | null => {
+	try {
+		switch (mentionable.type) {
+			case 'file': {
+				const file = app.vault.getFileByPath(mentionable.file)
+				if (!file) {
+					return null
+				}
+				return {
+					type: 'file',
+					file: file,
+				}
+			}
+			case 'folder': {
+				const folder = app.vault.getFolderByPath(mentionable.folder)
+				if (!folder) {
+					return null
+				}
+				return {
+					type: 'folder',
+					folder: folder,
+				}
+			}
+			case 'vault':
+				return {
+					type: 'vault',
+				}
+			case 'current-file': {
+				if (!mentionable.file) {
+					return {
+						type: 'current-file',
+						file: null,
+					}
+				}
+				const file = app.vault.getFileByPath(mentionable.file)
+				return {
+					type: 'current-file',
+					file: file,
+				}
+			}
+			case 'block': {
+				const file = app.vault.getFileByPath(mentionable.file)
+				if (!file) {
+					return null
+				}
+				return {
+					type: 'block',
+					content: mentionable.content,
+					file: file,
+					startLine: mentionable.startLine,
+					endLine: mentionable.endLine,
+				}
+			}
+			case 'url': {
+				return {
+					type: 'url',
+					url: mentionable.url,
+				}
+			}
+			case 'image': {
+				return {
+					type: 'image',
+					name: mentionable.name,
+					mimeType: mentionable.mimeType,
+					data: mentionable.data,
+				}
+			}
+		}
+	} catch (e) {
+		console.error('Error deserializing mentionable', e)
+		return null
+	}
+}
+
+export function getMentionableKey(mentionable: SerializedMentionable): string {
+	switch (mentionable.type) {
+		case 'file':
+			return `file:${mentionable.file}`
+		case 'folder':
+			return `folder:${mentionable.folder}`
+		case 'vault':
+			return 'vault'
+		case 'current-file':
+			return `current-file:${mentionable.file ?? 'current'}`
+		case 'block':
+			return `block:${mentionable.file}:${mentionable.startLine}:${mentionable.endLine}:${mentionable.content}`
+		case 'url':
+			return `url:${mentionable.url}`
+		case 'image':
+			return `image:${mentionable.name}:${mentionable.data.length}:${mentionable.data.slice(-32)}`
+	}
+}
+
+export function getMentionableName(mentionable: Mentionable): string {
+	switch (mentionable.type) {
+		case 'file':
+			return mentionable.file.name
+		case 'folder':
+			return mentionable.folder.name
+		case 'vault':
+			return 'Vault'
+		case 'current-file':
+			return mentionable.file?.name ?? 'Current File'
+		case 'block':
+			return `${mentionable.file.name} (${mentionable.startLine}:${mentionable.endLine})`
+		case 'url':
+			return mentionable.url
+		case 'image':
+			return mentionable.name
+	}
+}

src/utils/obsidian.test.ts

@@ -0,0 +1,60 @@
+import { TFile, TFolder } from 'obsidian'
+
+import { calculateFileDistance } from './obsidian'
+
+describe('calculateFileDistance', () => {
+	// Mock TFile class
+	class MockTFile {
+		path: string
+		constructor(path: string) {
+			this.path = path
+		}
+	}
+
+	it('should calculate the correct distance between files in the same folder', () => {
+		const file1 = new MockTFile('folder/file1.md') as TFile
+		const file2 = new MockTFile('folder/file2.md') as TFile
+
+		const result = calculateFileDistance(file1, file2)
+		expect(result).toBe(2)
+	})
+
+	it('should calculate the correct distance between files in different subfolders', () => {
+		const file1 = new MockTFile('folder1/folder2/file1.md') as TFile
+		const file2 = new MockTFile('folder1/folder3/file2.md') as TFile
+
+		const result = calculateFileDistance(file1, file2)
+		expect(result).toBe(4)
+	})
+
+	it('should return null for files in different top-level folders', () => {
+		const file1 = new MockTFile('folder1/file1.md') as TFile
+		const file2 = new MockTFile('folder2/file2.md') as TFile
+
+		const result = calculateFileDistance(file1, file2)
+		expect(result).toBeNull()
+	})
+
+	it('should handle files at different depths', () => {
+		const file1 = new MockTFile('folder1/folder2/subfolder/file1.md') as TFile
+		const file2 = new MockTFile('folder1/folder3/file2.md') as TFile
+
+		const result = calculateFileDistance(file1, file2)
+		expect(result).toBe(5)
+	})
+
+	it('should return 0 for the same file', () => {
+		const file = new MockTFile('folder/file.md') as TFile
+
+		const result = calculateFileDistance(file, file)
+		expect(result).toBe(0)
+	})
+
+	it('should calculate the correct distance between a folder and a file', () => {
+		const file = new MockTFile('folder1/folder2/file1.md') as TFile
+		const folder = new MockTFile('folder1/folder2') as TFolder
+
+		const result = calculateFileDistance(file, folder)
+		expect(result).toBe(1)
+	})
+})

src/utils/obsidian.ts

@@ -0,0 +1,124 @@
+import { App, Editor, MarkdownView, TFile, TFolder, Vault } from 'obsidian'
+
+import { MentionableBlockData } from '../types/mentionable'
+
+export async function readTFileContent(
+	file: TFile,
+	vault: Vault,
+): Promise<string> {
+	return await vault.cachedRead(file)
+}
+
+export async function readMultipleTFiles(
+	files: TFile[],
+	vault: Vault,
+): Promise<string[]> {
+	// Read files in parallel
+	const readPromises = files.map((file) => readTFileContent(file, vault))
+	return await Promise.all(readPromises)
+}
+
+export function getNestedFiles(folder: TFolder, vault: Vault): TFile[] {
+	const files: TFile[] = []
+	for (const child of folder.children) {
+		if (child instanceof TFile) {
+			files.push(child)
+		} else if (child instanceof TFolder) {
+			files.push(...getNestedFiles(child, vault))
+		}
+	}
+	return files
+}
+
+export async function getMentionableBlockData(
+	editor: Editor,
+	view: MarkdownView,
+): Promise<MentionableBlockData | null> {
+	const file = view.file
+	if (!file) return null
+
+	const selection = editor.getSelection()
+	if (!selection) return null
+
+	const startLine = editor.getCursor('from').line
+	const endLine = editor.getCursor('to').line
+	const selectionContent = editor
+		.getValue()
+		.split('\n')
+		.slice(startLine, endLine + 1)
+		.join('\n')
+
+	return {
+		content: selectionContent,
+		file,
+		startLine: startLine + 1, // +1 because startLine is 0-indexed
+		endLine: endLine + 1, // +1 because startLine is 0-indexed
+	}
+}
+
+export function getOpenFiles(app: App): TFile[] {
+	try {
+		const leaves = app.workspace.getLeavesOfType('markdown')
+
+		return leaves.map((v) => (v.view as MarkdownView).file).filter((v) => !!v)
+	} catch (e) {
+		return []
+	}
+}
+
+export function calculateFileDistance(
+	file1: TFile | TFolder,
+	file2: TFile | TFolder,
+): number | null {
+	const path1 = file1.path.split('/')
+	const path2 = file2.path.split('/')
+
+	// Check if files are in different top-level folders
+	if (path1[0] !== path2[0]) {
+		return null
+	}
+
+	let distance = 0
+	let i = 0
+
+	// Find the common ancestor
+	while (i < path1.length && i < path2.length && path1[i] === path2[i]) {
+		i++
+	}
+
+	// Calculate distance from common ancestor to each file
+	distance += path1.length - i
+	distance += path2.length - i
+
+	return distance
+}
+
+export function openMarkdownFile(
+	app: App,
+	filePath: string,
+	startLine?: number,
+) {
+	const file = app.vault.getFileByPath(filePath)
+	if (!file) return
+
+	const existingLeaf = app.workspace
+		.getLeavesOfType('markdown')
+		.find(
+			(leaf) =>
+				leaf.view instanceof MarkdownView && leaf.view.file?.path === file.path,
+		)
+
+	if (existingLeaf) {
+		app.workspace.setActiveLeaf(existingLeaf, { focus: true })
+
+		if (startLine) {
+			const view = existingLeaf.view as MarkdownView
+			view.setEphemeralState({ line: startLine - 1 }) // -1 because line is 0-indexed
+		}
+	} else {
+		const leaf = app.workspace.getLeaf('tab')
+		leaf.openFile(file, {
+			eState: startLine ? { line: startLine - 1 } : undefined, // -1 because line is 0-indexed
+		})
+	}
+}

src/utils/ollama.ts

@@ -0,0 +1,12 @@
+import { requestUrl } from 'obsidian'
+
+export async function getOllamaModels(ollamaUrl: string) {
+	try {
+		const response = (await requestUrl(`${ollamaUrl}/api/tags`)).json as {
+			models: { name: string }[]
+		}
+		return response.models.map((model) => model.name)
+	} catch (error) {
+		return []
+	}
+}

src/utils/open-settings-modal.ts

@@ -0,0 +1,12 @@
+import { App } from 'obsidian'
+
+import { OpenSettingsModal } from '../open-settings-modal'
+
+export function openSettingsModalWithError(app: App, errorMessage: string) {
+	new OpenSettingsModal(app, errorMessage, () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const setting = (app as any).setting
+		setting.open()
+		setting.openTabById('infio-copilot')
+	}).open()
+}

src/utils/parse-infio-block.test.ts

@@ -0,0 +1,174 @@
+import { ParsedinfioBlock, parseinfioBlocks } from './parse-infio-block'
+
+describe('parseinfioBlocks', () => {
+	it('should parse a string with infio_block elements', () => {
+		const input = `Some text before
+<infio_block language="markdown" filename="example.md">
+# Example Markdown
+
+This is a sample markdown content for testing purposes.
+
+## Features
+
+- Lists
+- **Bold text**
+- *Italic text*
+- [Links](https://example.com)
+
+### Code Block
+\`\`\`python
+print("Hello, world!")
+\`\`\`
+</infio_block>
+Some text after`
+
+		const expected: ParsedinfioBlock[] = [
+			{ type: 'string', content: 'Some text before\n' },
+			{
+				type: 'infio_block',
+				content: `
+# Example Markdown
+
+This is a sample markdown content for testing purposes.
+
+## Features
+
+- Lists
+- **Bold text**
+- *Italic text*
+- [Links](https://example.com)
+
+### Code Block
+\`\`\`python
+print("Hello, world!")
+\`\`\`
+`,
+				language: 'markdown',
+				filename: 'example.md',
+			},
+			{ type: 'string', content: '\nSome text after' },
+		]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+
+	it('should handle empty infio_block elements', () => {
+		const input = `
+      <infio_block language="python"></infio_block>
+    `
+
+		const expected: ParsedinfioBlock[] = [
+			{ type: 'string', content: '\n      ' },
+			{
+				type: 'infio_block',
+				content: '',
+				language: 'python',
+				filename: undefined,
+			},
+			{ type: 'string', content: '\n    ' },
+		]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+
+	it('should handle input without infio_block elements', () => {
+		const input = 'Just a regular string without any infio_block elements.'
+
+		const expected: ParsedinfioBlock[] = [{ type: 'string', content: input }]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+
+	it('should handle multiple infio_block elements', () => {
+		const input = `Start
+<infio_block language="python" filename="script.py">
+def greet(name):
+    print(f"Hello, {name}!")
+</infio_block>
+Middle
+<infio_block language="markdown" filename="example.md">
+# Using tildes for code blocks
+
+Did you know that you can use tildes for code blocks?
+
+~~~python
+print("Hello, world!")
+~~~
+</infio_block>
+End`
+
+		const expected: ParsedinfioBlock[] = [
+			{ type: 'string', content: 'Start\n' },
+			{
+				type: 'infio_block',
+				content: `
+def greet(name):
+    print(f"Hello, {name}!")
+`,
+				language: 'python',
+				filename: 'script.py',
+			},
+			{ type: 'string', content: '\nMiddle\n' },
+			{
+				type: 'infio_block',
+				content: `
+# Using tildes for code blocks
+
+Did you know that you can use tildes for code blocks?
+
+~~~python
+print("Hello, world!")
+~~~
+`,
+				language: 'markdown',
+				filename: 'example.md',
+			},
+			{ type: 'string', content: '\nEnd' },
+		]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+
+	it('should handle unfinished infio_block with only opening tag', () => {
+		const input = `Start
+<infio_block language="markdown">
+# Unfinished infio_block
+
+Some text after without closing tag`
+		const expected: ParsedinfioBlock[] = [
+			{ type: 'string', content: 'Start\n' },
+			{
+				type: 'infio_block',
+				content: `
+# Unfinished infio_block
+
+Some text after without closing tag`,
+				language: 'markdown',
+				filename: undefined,
+			},
+		]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+
+	it('should handle infio_block with startline and endline attributes', () => {
+		const input = `<infio_block language="markdown" startline="2" endline="5"></infio_block>`
+		const expected: ParsedinfioBlock[] = [
+			{
+				type: 'infio_block',
+				content: '',
+				language: 'markdown',
+				startLine: 2,
+				endLine: 5,
+			},
+		]
+
+		const result = parseinfioBlocks(input)
+		expect(result).toEqual(expected)
+	})
+})

src/utils/parse-infio-block.ts

@@ -0,0 +1,80 @@
+import { parseFragment } from 'parse5'
+
+export type ParsedinfioBlock =
+	| { type: 'string'; content: string }
+	| {
+		type: 'infio_block'
+		content: string
+		language?: string
+		filename?: string
+		startLine?: number
+		endLine?: number
+		action?: 'edit' | 'new' | 'reference'
+	}
+
+export function parseinfioBlocks(input: string): ParsedinfioBlock[] {
+	const parsedResult: ParsedinfioBlock[] = []
+	const fragment = parseFragment(input, {
+		sourceCodeLocationInfo: true,
+	})
+	let lastEndOffset = 0
+	for (const node of fragment.childNodes) {
+		if (node.nodeName === 'infio_block') {
+			if (!node.sourceCodeLocation) {
+				throw new Error('sourceCodeLocation is undefined')
+			}
+			const startOffset = node.sourceCodeLocation.startOffset
+			const endOffset = node.sourceCodeLocation.endOffset
+			if (startOffset > lastEndOffset) {
+				parsedResult.push({
+					type: 'string',
+					content: input.slice(lastEndOffset, startOffset),
+				})
+			}
+
+			const language = node.attrs.find((attr) => attr.name === 'language')?.value
+			const filename = node.attrs.find((attr) => attr.name === 'filename')?.value
+			const startLine = node.attrs.find((attr) => attr.name === 'startline')?.value
+			const endLine = node.attrs.find((attr) => attr.name === 'endline')?.value
+			const action = node.attrs.find((attr) => attr.name === 'type')?.value as 'edit' | 'new' | 'reference'
+
+			const children = node.childNodes
+			if (children.length === 0) {
+				parsedResult.push({
+					type: 'infio_block',
+					content: '',
+					language,
+					filename,
+					startLine: startLine ? parseInt(startLine) : undefined,
+					endLine: endLine ? parseInt(endLine) : undefined,
+					action: action,
+				})
+			} else {
+				const innerContentStartOffset =
+					children[0].sourceCodeLocation?.startOffset
+				const innerContentEndOffset =
+					children[children.length - 1].sourceCodeLocation?.endOffset
+				if (!innerContentStartOffset || !innerContentEndOffset) {
+					throw new Error('sourceCodeLocation is undefined')
+				}
+				parsedResult.push({
+					type: 'infio_block',
+					content: input.slice(innerContentStartOffset, innerContentEndOffset),
+					language,
+					filename,
+					startLine: startLine ? parseInt(startLine) : undefined,
+					endLine: endLine ? parseInt(endLine) : undefined,
+					action: action,
+				})
+			}
+			lastEndOffset = endOffset
+		}
+	}
+	if (lastEndOffset < input.length) {
+		parsedResult.push({
+			type: 'string',
+			content: input.slice(lastEndOffset),
+		})
+	}
+	return parsedResult
+}

src/utils/price-calculator.ts

@@ -0,0 +1,58 @@
+import {
+	ANTHROPIC_PRICES,
+	GEMINI_PRICES,
+	GROQ_PRICES,
+	OPENAI_PRICES,
+} from '../constants'
+import { CustomLLMModel } from '../types/llm/model'
+import { ResponseUsage } from '../types/llm/response'
+
+// Returns the cost in dollars. Returns null if the model is not supported.
+export const calculateLLMCost = ({
+	model,
+	usage,
+}: {
+	model: CustomLLMModel
+	usage: ResponseUsage
+}): number | null => {
+	switch (model.provider) {
+		case 'openai': {
+			const modelPricing = OPENAI_PRICES[model.name]
+			if (!modelPricing) return null
+			return (
+				(usage.prompt_tokens * modelPricing.input +
+					usage.completion_tokens * modelPricing.output) /
+				1_000_000
+			)
+		}
+		case 'anthropic': {
+			const modelPricing = ANTHROPIC_PRICES[model.name]
+			if (!modelPricing) return null
+			return (
+				(usage.prompt_tokens * modelPricing.input +
+					usage.completion_tokens * modelPricing.output) /
+				1_000_000
+			)
+		}
+		case 'gemini': {
+			const modelPricing = GEMINI_PRICES[model.name]
+			if (!modelPricing) return null
+			return (
+				(usage.prompt_tokens * modelPricing.input +
+					usage.completion_tokens * modelPricing.output) /
+				1_000_000
+			)
+		}
+		case 'groq': {
+			const modelPricing = GROQ_PRICES[model.name]
+			if (!modelPricing) return null
+			return (
+				(usage.prompt_tokens * modelPricing.input +
+					usage.completion_tokens * modelPricing.output) /
+				1_000_000
+			)
+		}
+		default:
+			return null
+	}
+}

src/utils/prompt-generator.ts

@@ -0,0 +1,470 @@
+import { App, TFile, htmlToMarkdown, requestUrl } from 'obsidian'
+
+import { editorStateToPlainText } from '../components/chat-view/chat-input/utils/editor-state-to-plain-text'
+import { QueryProgressState } from '../components/chat-view/QueryProgress'
+import { RAGEngine } from '../core/rag/rag-engine'
+import { SelectVector } from '../database/schema'
+import { ChatMessage, ChatUserMessage } from '../types/chat'
+import { ContentPart, RequestMessage } from '../types/llm/request'
+import {
+	MentionableBlock,
+	MentionableFile,
+	MentionableFolder,
+	MentionableImage,
+	MentionableUrl,
+	MentionableVault,
+} from '../types/mentionable'
+import { InfioSettings } from '../types/settings'
+
+import {
+	getNestedFiles,
+	readMultipleTFiles,
+	readTFileContent,
+} from './obsidian'
+import { tokenCount } from './token'
+import { YoutubeTranscript, isYoutubeUrl } from './youtube-transcript'
+
+export class PromptGenerator {
+	private getRagEngine: () => Promise<RAGEngine>
+	private app: App
+	private settings: InfioSettings
+
+	constructor(
+		getRagEngine: () => Promise<RAGEngine>,
+		app: App,
+		settings: InfioSettings,
+	) {
+		this.getRagEngine = getRagEngine
+		this.app = app
+		this.settings = settings
+	}
+
+	public async generateRequestMessages({
+		messages,
+		useVaultSearch,
+		onQueryProgressChange,
+		type,
+	}: {
+		messages: ChatMessage[]
+		useVaultSearch?: boolean
+		onQueryProgressChange?: (queryProgress: QueryProgressState) => void
+		type?: string
+	}): Promise<{
+		requestMessages: RequestMessage[]
+		compiledMessages: ChatMessage[]
+	}> {
+		if (messages.length === 0) {
+			throw new Error('No messages provided')
+		}
+		const lastUserMessage = messages[messages.length - 1]
+		if (lastUserMessage.role !== 'user') {
+			throw new Error('Last message is not a user message')
+		}
+
+		const { promptContent, shouldUseRAG, similaritySearchResults } =
+			await this.compileUserMessagePrompt({
+				message: lastUserMessage,
+				useVaultSearch,
+				onQueryProgressChange,
+			})
+		let compiledMessages = [
+			...messages.slice(0, -1),
+			{
+				...lastUserMessage,
+				promptContent,
+				similaritySearchResults,
+			},
+		]
+
+		// Safeguard: ensure all user messages have parsed content
+		compiledMessages = await Promise.all(
+			compiledMessages.map(async (message) => {
+				if (message.role === 'user' && !message.promptContent) {
+					const { promptContent, similaritySearchResults } =
+						await this.compileUserMessagePrompt({
+							message,
+						})
+					return {
+						...message,
+						promptContent,
+						similaritySearchResults,
+					}
+				}
+				return message
+			}),
+		)
+
+		const systemMessage = this.getSystemMessage(shouldUseRAG, type)
+
+		const customInstructionMessage = this.getCustomInstructionMessage()
+
+		const currentFile = lastUserMessage.mentionables.find(
+			(m) => m.type === 'current-file',
+		)?.file
+		const currentFileMessage = currentFile
+			? await this.getCurrentFileMessage(currentFile)
+			: undefined
+
+		const requestMessages: RequestMessage[] = [
+			systemMessage,
+			...(customInstructionMessage ? [customInstructionMessage] : []),
+			...(currentFileMessage ? [currentFileMessage] : []),
+			...compiledMessages.slice(-20).map((message): RequestMessage => {
+				if (message.role === 'user') {
+					return {
+						role: 'user',
+						content: message.promptContent ?? '',
+					}
+				} else {
+					return {
+						role: 'assistant',
+						content: message.content,
+					}
+				}
+			}),
+			...(shouldUseRAG ? [this.getRagInstructionMessage()] : []),
+		]
+
+		return {
+			requestMessages,
+			compiledMessages,
+		}
+	}
+
+	private async compileUserMessagePrompt({
+		message,
+		useVaultSearch,
+		onQueryProgressChange,
+	}: {
+		message: ChatUserMessage
+		useVaultSearch?: boolean
+		onQueryProgressChange?: (queryProgress: QueryProgressState) => void
+	}): Promise<{
+		promptContent: ChatUserMessage['promptContent']
+		shouldUseRAG: boolean
+		similaritySearchResults?: (Omit<SelectVector, 'embedding'> & {
+			similarity: number
+		})[]
+	}> {
+		if (!message.content) {
+			return {
+				promptContent: '',
+				shouldUseRAG: false,
+			}
+		}
+		const query = editorStateToPlainText(message.content)
+		let similaritySearchResults = undefined
+
+		useVaultSearch =
+			// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+			useVaultSearch ||
+			message.mentionables.some(
+				(m): m is MentionableVault => m.type === 'vault',
+			)
+
+		onQueryProgressChange?.({
+			type: 'reading-mentionables',
+		})
+		const files = message.mentionables
+			.filter((m): m is MentionableFile => m.type === 'file')
+			.map((m) => m.file)
+		const folders = message.mentionables
+			.filter((m): m is MentionableFolder => m.type === 'folder')
+			.map((m) => m.folder)
+		const nestedFiles = folders.flatMap((folder) =>
+			getNestedFiles(folder, this.app.vault),
+		)
+		const allFiles = [...files, ...nestedFiles]
+		const fileContents = await readMultipleTFiles(allFiles, this.app.vault)
+
+		// Count tokens incrementally to avoid long processing times on large content sets
+		const exceedsTokenThreshold = async () => {
+			let accTokenCount = 0
+			for (const content of fileContents) {
+				const count = await tokenCount(content)
+				accTokenCount += count
+				if (accTokenCount > this.settings.ragOptions.thresholdTokens) {
+					return true
+				}
+			}
+			return false
+		}
+		const shouldUseRAG = useVaultSearch || (await exceedsTokenThreshold())
+
+		let filePrompt: string
+		if (shouldUseRAG) {
+			similaritySearchResults = useVaultSearch
+				? await (
+					await this.getRagEngine()
+				).processQuery({
+					query,
+					onQueryProgressChange: onQueryProgressChange,
+				}) // TODO: Add similarity boosting for mentioned files or folders
+				: await (
+					await this.getRagEngine()
+				).processQuery({
+					query,
+					scope: {
+						files: files.map((f) => f.path),
+						folders: folders.map((f) => f.path),
+					},
+					onQueryProgressChange: onQueryProgressChange,
+				})
+			filePrompt = `## Potentially Relevant Snippets from the current vault
+${similaritySearchResults
+					.map(({ path, content, metadata }) => {
+						const contentWithLineNumbers = this.addLineNumbersToContent({
+							content,
+							startLine: metadata.startLine,
+						})
+						return `\`\`\`${path}\n${contentWithLineNumbers}\n\`\`\`\n`
+					})
+					.join('')}\n`
+		} else {
+			filePrompt = allFiles
+				.map((file, index) => {
+					return `\`\`\`${file.path}\n${fileContents[index]}\n\`\`\`\n`
+				})
+				.join('')
+		}
+
+		const blocks = message.mentionables.filter(
+			(m): m is MentionableBlock => m.type === 'block',
+		)
+		const blockPrompt = blocks
+			.map(({ file, content, startLine, endLine }) => {
+				return `\`\`\`${file.path}#L${startLine}-${endLine}\n${content}\n\`\`\`\n`
+			})
+			.join('')
+
+		const urls = message.mentionables.filter(
+			(m): m is MentionableUrl => m.type === 'url',
+		)
+
+		const urlPrompt =
+			urls.length > 0
+				? `## Potentially Relevant Websearch Results
+${(
+					await Promise.all(
+						urls.map(
+							async ({ url }) => `\`\`\`
+Website URL: ${url}
+Website Content:
+${await this.getWebsiteContent(url)}
+\`\`\``,
+						),
+					)
+				).join('\n')}
+`
+				: ''
+
+		const imageDataUrls = message.mentionables
+			.filter((m): m is MentionableImage => m.type === 'image')
+			.map(({ data }) => data)
+
+		return {
+			promptContent: [
+				...imageDataUrls.map(
+					(data): ContentPart => ({
+						type: 'image_url',
+						image_url: {
+							url: data,
+						},
+					}),
+				),
+				{
+					type: 'text',
+					text: `${filePrompt}${blockPrompt}${urlPrompt}\n\n${query}\n\n`,
+				},
+			],
+			shouldUseRAG,
+			similaritySearchResults: similaritySearchResults,
+		}
+	}
+
+	private getSystemMessage(shouldUseRAG: boolean, type?: string): RequestMessage {
+		const systemPromptEdit = `You are an intelligent assistant to help edit text content based on user instructions. You will be given the current text content and the user's instruction for how to modify it.
+
+1. Your response should contain the modified text content wrapped in <infio_block> tags with appropriate attributes:
+   <infio_block filename="path/to/file.md" language="markdown" startLine="10" endLine="20" type="edit">
+   [modified content here]
+   </infio_block>
+
+2. Preserve the original formatting, indentation and line breaks unless specifically instructed otherwise.
+
+3. Make minimal changes necessary to fulfill the user's instruction. Do not modify parts of the text that don't need to change.
+
+4. If the instruction is unclear or cannot be fulfilled, respond with "ERROR: " followed by a brief explanation.`
+
+		const systemPrompt = `You are an intelligent assistant to help answer any questions that the user has, particularly about editing and organizing markdown files in Obsidian.
+
+1. Please keep your response as concise as possible. Avoid being verbose.
+
+2. When the user is asking for edits to their markdown, please provide a simplified version of the markdown block emphasizing only the changes. Use comments to show where unchanged content has been skipped. Wrap the markdown block with <infio_block> tags. Add filename, language, startLine, endLine and type attributes to the <infio_block> tags. If the user provides line numbers in the file path (e.g. file.md#L10-20), use those line numbers in the startLine and endLine attributes. For example:
+<infio_block filename="path/to/file.md" language="markdown" startLine="10" endLine="20" type="edit">
+<!-- ... existing content ... -->
+{{ edit_1 }}
+<!-- ... existing content ... -->
+{{ edit_2 }}
+<!-- ... existing content ... -->
+</infio_block>
+The user has full access to the file, so they prefer seeing only the changes in the markdown. Often this will mean that the start/end of the file will be skipped, but that's okay! Rewrite the entire file only if specifically requested. Always provide a brief explanation of the updates, except when the user specifically asks for just the content.
+
+3. Do not lie or make up facts.
+
+4. Respond in the same language as the user's message.
+
+5. Format your response in markdown.
+
+6. When writing out new markdown blocks, also wrap them with <infio_block> tags. For example:
+<infio_block language="markdown" type="new">
+{{ content }}
+</infio_block>
+
+7. When providing markdown blocks for an existing file, add the filename and language attributes to the <infio_block> tags. Restate the relevant section or heading, so the user knows which part of the file you are editing. For example:
+<infio_block filename="path/to/file.md" language="markdown" type="reference">
+## Section Title
+...
+{{ content }}
+...
+</infio_block>`
+
+		const systemPromptRAG = `You are an intelligent assistant to help answer any questions that the user has, particularly about editing and organizing markdown files in Obsidian. You will be given your conversation history with them and potentially relevant blocks of markdown content from the current vault.
+      
+1. Do not lie or make up facts.
+
+2. Respond in the same language as the user's message.
+
+3. Format your response in markdown.
+
+4. When referencing markdown blocks in your answer, keep the following guidelines in mind:
+
+  a. Never include line numbers in the output markdown.
+
+  b. Wrap the markdown block with <infio_block> tags. Include language attribute and type. For example:
+  <infio_block language="markdown" type="new">
+  {{ content }}
+  </infio_block>
+
+  c. When providing markdown blocks for an existing file, also include the filename attribute to the <infio_block> tags. For example:
+  <infio_block filename="path/to/file.md" language="markdown" type="reference">
+  {{ content }}
+  </infio_block>
+
+  d. When referencing a markdown block the user gives you, add the startLine and endLine attributes to the <infio_block> tags. Write related content outside of the <infio_block> tags. The content inside the <infio_block> tags will be ignored and replaced with the actual content of the markdown block. For example:
+  <infio_block filename="path/to/file.md" language="markdown" startLine="2" endLine="30" type="reference"></infio_block>`
+
+		if (type === 'edit') {
+			return {
+				role: 'system',
+				content: systemPromptEdit,
+			}
+		}
+
+		return {
+			role: 'system',
+			content: shouldUseRAG ? systemPromptRAG : systemPrompt,
+		}
+	}
+
+	private getCustomInstructionMessage(): RequestMessage | null {
+		const customInstruction = this.settings.systemPrompt.trim()
+		if (!customInstruction) {
+			return null
+		}
+		return {
+			role: 'user',
+			content: `Here are additional instructions to follow in your responses when relevant. There's no need to explicitly acknowledge them:
+<custom_instructions>
+${customInstruction}
+</custom_instructions>`,
+		}
+	}
+
+	private async getCurrentFileMessage(
+		currentFile: TFile,
+	): Promise<RequestMessage> {
+		const fileContent = await readTFileContent(currentFile, this.app.vault)
+		return {
+			role: 'user',
+			content: `# Inputs
+## Current File
+Here is the file I'm looking at.
+\`\`\`${currentFile.path}
+${fileContent}
+\`\`\`\n\n`,
+		}
+	}
+
+	public async generateEditMessages({
+		currentFile,
+		selectedContent,
+		instruction,
+		startLine,
+		endLine,
+	}: {
+		currentFile: TFile
+		selectedContent: string
+		instruction: string
+		startLine: number
+		endLine: number
+	}): Promise<RequestMessage[]> {
+		const systemMessage = this.getSystemMessage(false, 'edit')
+		const currentFileMessage = await this.getCurrentFileMessage(currentFile)
+		const userMessage: RequestMessage = {
+			role: 'user',
+			content: `Selected text (lines ${startLine}-${endLine}):\n${selectedContent}\n\nInstruction:\n${instruction}`,
+		}
+
+		return [systemMessage, currentFileMessage, userMessage]
+	}
+
+	private getRagInstructionMessage(): RequestMessage {
+		return {
+			role: 'user',
+			content: `If you need to reference any of the markdown blocks I gave you, add the startLine and endLine attributes to the <infio_block> tags without any content inside. For example:
+<infio_block filename="path/to/file.md" language="markdown" startLine="200" endLine="310" type="reference"></infio_block>
+
+When writing out new markdown blocks, remember not to include "line_number|" at the beginning of each line.`,
+		}
+	}
+
+	private addLineNumbersToContent({
+		content,
+		startLine,
+	}: {
+		content: string
+		startLine: number
+	}): string {
+		const lines = content.split('\n')
+		const linesWithNumbers = lines.map((line, index) => {
+			return `${startLine + index}|${line}`
+		})
+		return linesWithNumbers.join('\n')
+	}
+
+	/**
+	 * TODO: Improve markdown conversion logic
+	 * - filter visually hidden elements
+	 * ...
+	 */
+	private async getWebsiteContent(url: string): Promise<string> {
+		if (isYoutubeUrl(url)) {
+			try {
+				// TODO: pass language based on user preferences
+				const { title, transcript } =
+					await YoutubeTranscript.fetchTranscriptAndMetadata(url)
+
+				return `Title: ${title}
+Video Transcript:
+${transcript.map((t) => `${t.offset}: ${t.text}`).join('\n')}`
+			} catch (error) {
+				console.error('Error fetching YouTube transcript', error)
+			}
+		}
+
+		const response = await requestUrl({ url })
+
+		return htmlToMarkdown(response.text)
+	}
+}

src/utils/token.ts

@@ -0,0 +1,11 @@
+import { getEncoding } from 'js-tiktoken'
+
+// TODO: Replace js-tiktoken with tiktoken library for better performance
+// Note: tiktoken uses WebAssembly, requiring esbuild configuration
+
+// Caution: tokenCount is computationally expensive for large inputs.
+// Frequent use, especially on large files, may significantly impact performance.
+export async function tokenCount(text: string): Promise<number> {
+  const encoder = getEncoding('cl100k_base')
+  return encoder.encode(text).length
+}

src/utils/youtube-transcript.ts

@@ -0,0 +1,198 @@
+/**
+ * This source code is licensed under the MIT license.
+ * Original source: https://github.com/Kakulukian/youtube-transcript
+ *
+ * Modified from the original code
+ */
+
+import { requestUrl } from 'obsidian'
+
+const RE_YOUTUBE =
+	/(?:youtube\.com\/(?:[^/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?/\s]{11})/i
+const USER_AGENT =
+	'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)'
+const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g
+
+export function isYoutubeUrl(url: string) {
+	return RE_YOUTUBE.test(url)
+}
+
+export class YoutubeTranscriptError extends Error {
+	constructor(message: string) {
+		super(`[YoutubeTranscript] 🚨 ${message}`)
+	}
+}
+
+export class YoutubeTranscriptTooManyRequestError extends YoutubeTranscriptError {
+	constructor() {
+		super(
+			'YouTube is receiving too many requests from this IP and now requires solving a captcha to continue',
+		)
+	}
+}
+
+export class YoutubeTranscriptVideoUnavailableError extends YoutubeTranscriptError {
+	constructor(videoId: string) {
+		super(`The video is no longer available (${videoId})`)
+	}
+}
+
+export class YoutubeTranscriptDisabledError extends YoutubeTranscriptError {
+	constructor(videoId: string) {
+		super(`Transcript is disabled on this video (${videoId})`)
+	}
+}
+
+export class YoutubeTranscriptNotAvailableError extends YoutubeTranscriptError {
+	constructor(videoId: string) {
+		super(`No transcripts are available for this video (${videoId})`)
+	}
+}
+
+export class YoutubeTranscriptNotAvailableLanguageError extends YoutubeTranscriptError {
+	constructor(lang: string, availableLangs: string[], videoId: string) {
+		super(
+			`No transcripts are available in ${lang} this video (${videoId}). Available languages: ${availableLangs.join(
+				', ',
+			)}`,
+		)
+	}
+}
+
+export type TranscriptConfig = {
+	lang?: string
+}
+export type Transcript = {
+	text: string
+	duration: number
+	offset: number
+	lang?: string
+}
+
+export type TranscriptAndMetadataResponse = {
+	title: string
+	transcript: Transcript[]
+}
+
+/**
+ * Class to retrieve transcript if exist
+ */
+export class YoutubeTranscript {
+	/**
+	 * Fetch transcript from YTB Video
+	 * @param videoId Video url or video identifier
+	 * @param config Get transcript in a specific language ISO
+	 */
+	public static async fetchTranscriptAndMetadata(
+		videoId: string,
+		config?: TranscriptConfig,
+	): Promise<TranscriptAndMetadataResponse> {
+		const identifier = this.retrieveVideoId(videoId)
+		const videoPageResponse = await requestUrl({
+			url: `https://www.youtube.com/watch?v=${identifier}`,
+			headers: {
+				...(config?.lang && { 'Accept-Language': config.lang }),
+				'User-Agent': USER_AGENT,
+			},
+		})
+		const videoPageBody = videoPageResponse.text
+
+		// Extract title using regex from <title> tags
+		const titleMatch = videoPageBody.match(/<title>(.*?)<\/title>/)
+		const title = titleMatch
+			? titleMatch[1].replace(' - YouTube', '').trim()
+			: ''
+
+		const splittedHTML = videoPageBody.split('"captions":')
+
+		if (splittedHTML.length <= 1) {
+			if (videoPageBody.includes('class="g-recaptcha"')) {
+				throw new YoutubeTranscriptTooManyRequestError()
+			}
+			if (!videoPageBody.includes('"playabilityStatus":')) {
+				throw new YoutubeTranscriptVideoUnavailableError(videoId)
+			}
+			throw new YoutubeTranscriptDisabledError(videoId)
+		}
+
+		const captions = (() => {
+			try {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-return
+				return JSON.parse(
+					splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''),
+				)
+			} catch (e) {
+				return undefined
+			}
+		})()?.playerCaptionsTracklistRenderer
+
+		if (!captions) {
+			throw new YoutubeTranscriptDisabledError(videoId)
+		}
+
+		if (!('captionTracks' in captions)) {
+			throw new YoutubeTranscriptNotAvailableError(videoId)
+		}
+
+		if (
+			config?.lang &&
+			!captions.captionTracks.some(
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any
+				(track: any) => track.languageCode === config?.lang,
+			)
+		) {
+			throw new YoutubeTranscriptNotAvailableLanguageError(
+				config?.lang,
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return
+				captions.captionTracks.map((track: any) => track.languageCode),
+				videoId,
+			)
+		}
+
+		const transcriptURL: string = (
+			config?.lang
+				? captions.captionTracks.find(
+					// eslint-disable-next-line @typescript-eslint/no-explicit-any
+					(track: any) => track.languageCode === config?.lang,
+				)
+				: captions.captionTracks[0]
+		).baseUrl
+
+		const transcriptResponse = await requestUrl({
+			url: transcriptURL,
+			headers: {
+				...(config?.lang && { 'Accept-Language': config.lang }),
+				'User-Agent': USER_AGENT,
+			},
+		})
+		if (transcriptResponse.status !== 200) {
+			throw new YoutubeTranscriptNotAvailableError(videoId)
+		}
+		const transcriptBody = transcriptResponse.text
+		const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)]
+		return {
+			title,
+			transcript: results.map((result) => ({
+				text: result[3],
+				duration: parseFloat(result[2]),
+				offset: parseFloat(result[1]),
+				lang: config?.lang ?? captions.captionTracks[0].languageCode,
+			})),
+		}
+	}
+
+	/**
+	 * Retrieve video id from url or string
+	 * @param videoId video url or video id
+	 */
+	private static retrieveVideoId(videoId: string) {
+		if (videoId.length === 11) {
+			return videoId
+		}
+		const matchId = videoId.match(RE_YOUTUBE)
+		if (matchId?.length) {
+			return matchId[1]
+		}
+		throw new YoutubeTranscriptError('Impossible to retrieve Youtube video ID.')
+	}
+}