update files search methods && semantic search mehtod
This commit is contained in:
@@ -17,7 +17,7 @@ import { GroqProvider } from './groq'
|
||||
import { InfioProvider } from './infio'
|
||||
import { OllamaProvider } from './ollama'
|
||||
import { OpenAIAuthenticatedProvider } from './openai'
|
||||
import { OpenAICompatibleProvider } from './openai-compatible-provider'
|
||||
import { OpenAICompatibleProvider } from './openai-compatible'
|
||||
|
||||
|
||||
export type LLMManagerInterface = {
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import https from 'https'
|
||||
import { URL } from 'url'
|
||||
|
||||
import { GoogleGenerativeAI } from '@google/generative-ai'
|
||||
import { OpenAI } from 'openai'
|
||||
|
||||
@@ -176,6 +179,42 @@ export const getEmbeddingModel = (
|
||||
},
|
||||
}
|
||||
}
|
||||
case ApiProvider.OpenAICompatible: {
|
||||
const openai = new OpenAI({
|
||||
apiKey: settings.openaicompatibleProvider.apiKey,
|
||||
baseURL: settings.openaicompatibleProvider.baseUrl,
|
||||
dangerouslyAllowBrowser: true,
|
||||
});
|
||||
return {
|
||||
id: settings.embeddingModelId,
|
||||
dimension: 0,
|
||||
getEmbedding: async (text: string) => {
|
||||
try {
|
||||
if (!openai.apiKey) {
|
||||
throw new LLMAPIKeyNotSetException(
|
||||
'OpenAI Compatible API key is missing. Please set it in settings menu.',
|
||||
)
|
||||
}
|
||||
const embedding = await openai.embeddings.create({
|
||||
model: settings.embeddingModelId,
|
||||
input: text,
|
||||
encoding_format: "float",
|
||||
})
|
||||
return embedding.data[0].embedding
|
||||
} catch (error) {
|
||||
if (
|
||||
error.status === 429 &&
|
||||
error.message.toLowerCase().includes('rate limit')
|
||||
) {
|
||||
throw new LLMRateLimitExceededException(
|
||||
'OpenAI Compatible API rate limit exceeded. Please try again later.',
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
default:
|
||||
throw new Error('Invalid embedding model')
|
||||
}
|
||||
|
||||
@@ -34,7 +34,8 @@ export class RAGEngine {
|
||||
}
|
||||
|
||||
async initializeDimension(): Promise<void> {
|
||||
if (this.embeddingModel.dimension === 0 && this.settings.embeddingModelProvider === ApiProvider.Ollama) {
|
||||
if (this.embeddingModel.dimension === 0 &&
|
||||
(this.settings.embeddingModelProvider === ApiProvider.Ollama || this.settings.embeddingModelProvider === ApiProvider.OpenAICompatible)) {
|
||||
this.embeddingModel.dimension = (await this.embeddingModel.getEmbedding("hello world")).length
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,8 +30,8 @@ export function truncateLine(line: string, maxLength: number = MAX_LINE_LENGTH):
|
||||
return line.length > maxLength ? line.substring(0, maxLength) + " [truncated...]" : line
|
||||
}
|
||||
|
||||
async function getBinPath(): Promise<string | undefined> {
|
||||
const binPath = path.join("/opt/homebrew/bin/", binName)
|
||||
async function getBinPath(ripgrepPath: string): Promise<string | undefined> {
|
||||
const binPath = path.join(ripgrepPath, binName)
|
||||
return (await pathExists(binPath)) ? binPath : undefined
|
||||
}
|
||||
|
||||
@@ -86,20 +86,21 @@ async function execRipgrep(bin: string, args: string[]): Promise<string> {
|
||||
export async function regexSearchFiles(
|
||||
directoryPath: string,
|
||||
regex: string,
|
||||
ripgrepPath: string,
|
||||
): Promise<string> {
|
||||
const rgPath = await getBinPath()
|
||||
const rgPath = await getBinPath(ripgrepPath)
|
||||
|
||||
if (!rgPath) {
|
||||
throw new Error("Could not find ripgrep binary")
|
||||
}
|
||||
|
||||
// 使用--glob参数排除.obsidian目录
|
||||
// use --glob param to exclude .obsidian directory
|
||||
const args = [
|
||||
"--json",
|
||||
"-e",
|
||||
regex,
|
||||
"--glob",
|
||||
"!.obsidian/**", // 排除.obsidian目录及其所有子目录
|
||||
"!.obsidian/**", // exclude .obsidian directory and all its subdirectories
|
||||
"--glob",
|
||||
"!.git/**",
|
||||
"--context",
|
||||
Reference in New Issue
Block a user