gitea/infio-copilot
1
0
Fork 0
mirror of https://github.com/EthanMarti/infio-copilot.git synced 2026-05-09 08:30:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

update save trans to database

Browse Source
This commit is contained in:
duanfuxiang
2025-06-30 11:26:24 +08:00
parent f3a0252ab6
commit 4f5b3f5d04
14 changed files with 869 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/database/modules/insight/insight-manager.ts
View File

@@ -1,8 +1,8 @@
import { App, TFile } from 'obsidian'
import { InsertSourceInsight, SelectSourceInsight } from '../../schema'
import { EmbeddingModel } from '../../../types/embedding'
import { DBManager } from '../../database-manager'
import { InsertSourceInsight, SelectSourceInsight } from '../../schema'
import { InsightRepository } from './insight-repository'
@@ -51,6 +51,7 @@ export class InsightManager {
insight: string
sourceType: 'document' | 'tag' | 'folder'
sourcePath: string
sourceMtime: number
embedding: number[]
},
embeddingModel: EmbeddingModel,
@@ -60,6 +61,7 @@ export class InsightManager {
insight: insightData.insight,
source_type: insightData.sourceType,
source_path: insightData.sourcePath,
source_mtime: insightData.sourceMtime,
embedding: insightData.embedding,
}
@@ -75,6 +77,7 @@ export class InsightManager {
insight: string
sourceType: 'document' | 'tag' | 'folder'
sourcePath: string
sourceMtime: number
embedding: number[]
}>,
embeddingModel: EmbeddingModel,
@@ -84,6 +87,7 @@ export class InsightManager {
insight: data.insight,
source_type: data.sourceType,
source_path: data.sourcePath,
source_mtime: data.sourceMtime,
embedding: data.embedding,
}))
@@ -100,6 +104,7 @@ export class InsightManager {
insight?: string
sourceType?: 'document' | 'tag' | 'folder'
sourcePath?: string
sourceMtime?: number
embedding?: number[]
},
embeddingModel: EmbeddingModel,
@@ -118,6 +123,9 @@ export class InsightManager {
if (updates.sourcePath !== undefined) {
updateData.source_path = updates.sourcePath
}
if (updates.sourceMtime !== undefined) {
updateData.source_mtime = updates.sourceMtime
}
if (updates.embedding !== undefined) {
updateData.embedding = updates.embedding
}
@@ -318,4 +326,26 @@ export class InsightManager {
return filteredInsights
}
// /**
// * 根据源文件修改时间范围获取洞察
// */
// async getInsightsByMtimeRange(
// minMtime: number,
// maxMtime: number,
// embeddingModel: EmbeddingModel,
// ): Promise<SelectSourceInsight[]> {
// return await this.repository.getInsightsByMtimeRange(minMtime, maxMtime, embeddingModel)
// }
// /**
// * 根据源文件修改时间获取需要更新的洞察
// */
// async getOutdatedInsights(
// sourcePath: string,
// currentMtime: number,
// embeddingModel: EmbeddingModel,
// ): Promise<SelectSourceInsight[]> {
// return await this.repository.getOutdatedInsights(sourcePath, currentMtime, embeddingModel)
// }
}

49
src/database/modules/insight/insight-repository.ts
View File

@@ -139,8 +139,8 @@ export class InsightRepository {
// 构建批量插入的 SQL
const values = data.map((insight, index) => {
const offset = index * 6
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6})`
const offset = index * 7
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6}, $${offset + 7})`
}).join(',')
const params = data.flatMap(insight => [
@@ -148,12 +148,13 @@ export class InsightRepository {
insight.insight.replace(/\0/g, ''), // 清理null字节
insight.source_type,
insight.source_path,
insight.source_mtime,
`[${insight.embedding.join(',')}]`, // 转换为PostgreSQL vector格式
new Date() // updated_at
])
await this.db.query(
`INSERT INTO "${tableName}" (insight_type, insight, source_type, source_path, embedding, updated_at)
`INSERT INTO "${tableName}" (insight_type, insight, source_type, source_path, source_mtime, embedding, updated_at)
VALUES ${values}`,
params
)
@@ -197,6 +198,12 @@ export class InsightRepository {
paramIndex++
}
if (data.source_mtime !== undefined) {
fields.push(`source_mtime = $${paramIndex}`)
params.push(data.source_mtime)
paramIndex++
}
if (data.embedding !== undefined) {
fields.push(`embedding = $${paramIndex}`)
params.push(`[${data.embedding.join(',')}]`)
@@ -235,7 +242,7 @@ export class InsightRepository {
}
const tableName = this.getTableName(embeddingModel)
let whereConditions = ['1 - (embedding <=> $1::vector) > $2']
const whereConditions: string[] = ['1 - (embedding <=> $1::vector) > $2']
const params: unknown[] = [`[${queryVector.join(',')}]`, options.minSimilarity, options.limit]
let paramIndex = 4
@@ -259,7 +266,7 @@ export class InsightRepository {
const query = `
SELECT
id, insight_type, insight, source_type, source_path, created_at, updated_at,
id, insight_type, insight, source_type, source_path, source_mtime, created_at, updated_at,
1 - (embedding <=> $1::vector) as similarity
FROM "${tableName}"
WHERE ${whereConditions.join(' AND ')}
@@ -271,4 +278,36 @@ export class InsightRepository {
const result = await this.db.query<SearchResult>(query, params)
return result.rows
}
// async getInsightsByMtimeRange(
// minMtime: number,
// maxMtime: number,
// embeddingModel: EmbeddingModel,
// ): Promise<SelectSourceInsight[]> {
// if (!this.db) {
// throw new DatabaseNotInitializedException()
// }
// const tableName = this.getTableName(embeddingModel)
// const result = await this.db.query<SelectSourceInsight>(
// `SELECT * FROM "${tableName}" WHERE source_mtime >= $1 AND source_mtime <= $2 ORDER BY created_at DESC`,
// [minMtime, maxMtime]
// )
// return result.rows
// }
// async getOutdatedInsights(
// sourcePath: string,
// currentMtime: number,
// embeddingModel: EmbeddingModel,
// ): Promise<SelectSourceInsight[]> {
// if (!this.db) {
// throw new DatabaseNotInitializedException()
// }
// const tableName = this.getTableName(embeddingModel)
// const result = await this.db.query<SelectSourceInsight>(
// `SELECT * FROM "${tableName}" WHERE source_path = $1 AND source_mtime < $2 ORDER BY created_at DESC`,
// [sourcePath, currentMtime]
// )
// return result.rows
// }
}

3
src/database/schema.ts
View File

@@ -1,7 +1,6 @@
import { SerializedLexicalNode } from 'lexical'
import { SUPPORT_EMBEDDING_SIMENTION } from '../constants'
import { ApplyStatus } from '../types/apply'
// import { EmbeddingModelId } from '../types/embedding'
// PostgreSQL column types
@@ -184,6 +183,7 @@ export type SourceInsightRecord = {
insight: string
source_type: 'document' | 'tag' | 'folder'
source_path: string
source_mtime: number
embedding: number[]
created_at: Date
updated_at: Date
@@ -203,6 +203,7 @@ const createSourceInsightTable = (dimension: number): TableDefinition => {
insight: { type: 'TEXT', notNull: true },
source_type: { type: 'TEXT', notNull: true },
source_path: { type: 'TEXT', notNull: true },
source_mtime: { type: 'BIGINT', notNull: true },
embedding: { type: 'VECTOR', dimensions: dimension },
created_at: { type: 'TIMESTAMP', notNull: true, defaultNow: true },
updated_at: { type: 'TIMESTAMP', notNull: true, defaultNow: true }

16
src/database/sql.ts
View File

@@ -104,6 +104,7 @@ export const migrations: Record<string, SqlMigration> = {
"insight" text NOT NULL,
"source_type" text NOT NULL,
"source_path" text NOT NULL,
"source_mtime" bigint NOT NULL,
"embedding" vector(1536),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
@@ -115,6 +116,7 @@ export const migrations: Record<string, SqlMigration> = {
"insight" text NOT NULL,
"source_type" text NOT NULL,
"source_path" text NOT NULL,
"source_mtime" bigint NOT NULL,
"embedding" vector(1024),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
@@ -126,6 +128,7 @@ export const migrations: Record<string, SqlMigration> = {
"insight" text NOT NULL,
"source_type" text NOT NULL,
"source_path" text NOT NULL,
"source_mtime" bigint NOT NULL,
"embedding" vector(768),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
@@ -137,6 +140,7 @@ export const migrations: Record<string, SqlMigration> = {
"insight" text NOT NULL,
"source_type" text NOT NULL,
"source_path" text NOT NULL,
"source_mtime" bigint NOT NULL,
"embedding" vector(512),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
@@ -148,6 +152,7 @@ export const migrations: Record<string, SqlMigration> = {
"insight" text NOT NULL,
"source_type" text NOT NULL,
"source_path" text NOT NULL,
"source_mtime" bigint NOT NULL,
"embedding" vector(384),
"created_at" timestamp DEFAULT now() NOT NULL,
"updated_at" timestamp DEFAULT now() NOT NULL
@@ -245,5 +250,16 @@ export const migrations: Record<string, SqlMigration> = {
"created_at" timestamp DEFAULT now() NOT NULL
);
`
},
add_source_mtime: {
description: "Adds missing source_mtime column to existing source insight tables",
sql: `
-- Add source_mtime column to existing source insight tables if it doesn't exist
ALTER TABLE "source_insight_1536" ADD COLUMN IF NOT EXISTS "source_mtime" bigint NOT NULL DEFAULT 0;
ALTER TABLE "source_insight_1024" ADD COLUMN IF NOT EXISTS "source_mtime" bigint NOT NULL DEFAULT 0;
ALTER TABLE "source_insight_768" ADD COLUMN IF NOT EXISTS "source_mtime" bigint NOT NULL DEFAULT 0;
ALTER TABLE "source_insight_512" ADD COLUMN IF NOT EXISTS "source_mtime" bigint NOT NULL DEFAULT 0;
ALTER TABLE "source_insight_384" ADD COLUMN IF NOT EXISTS "source_mtime" bigint NOT NULL DEFAULT 0;
`
}
};