update, use json database replace pglite, for sync

This commit is contained in:
duanfuxiang
2025-04-24 16:08:44 +08:00
parent 10970a8803
commit 96b9fcef3b
20 changed files with 863 additions and 229 deletions

View File

@@ -2,27 +2,21 @@
import { type PGliteWithLive } from '@electric-sql/pglite/live'
import { App } from 'obsidian'
// import { PGLITE_DB_PATH } from '../constants'
import { createAndInitDb } from '../pgworker'
import { CommandManager } from './modules/command/command-manager'
import { ConversationManager } from './modules/conversation/conversation-manager'
import { CommandManager as CommandManager } from './modules/command/command-manager'
import { VectorManager } from './modules/vector/vector-manager'
// import { pgliteResources } from './pglite-resources'
// import { migrations } from './sql'
export class DBManager {
private app: App
// private dbPath: string
private db: PGliteWithLive | null = null
// private db: PgliteDatabase | null = null
private vectorManager: VectorManager
private CommandManager: CommandManager
private conversationManager: ConversationManager
constructor(app: App) {
this.app = app
// this.dbPath = dbPath
}
static async create(app: App): Promise<DBManager> {
@@ -36,123 +30,24 @@ export class DBManager {
return dbManager
}
getPgClient() {
getPgClient(): PGliteWithLive | null {
return this.db
}
getVectorManager() {
getVectorManager(): VectorManager {
return this.vectorManager
}
getCommandManager() {
getCommandManager(): CommandManager {
return this.CommandManager
}
getConversationManager() {
getConversationManager(): ConversationManager {
return this.conversationManager
}
// private async createNewDatabase() {
// const { fsBundle, wasmModule, vectorExtensionBundlePath } =
// await this.loadPGliteResources()
// this.db = await PGlite.create({
// fsBundle: fsBundle,
// wasmModule: wasmModule,
// extensions: {
// vector: vectorExtensionBundlePath,
// live,
// },
// })
// }
// private async loadExistingDatabase() {
// try {
// const databaseFileExists = await this.app.vault.adapter.exists(
// this.dbPath,
// )
// if (!databaseFileExists) {
// return null
// }
// const fileBuffer = await this.app.vault.adapter.readBinary(this.dbPath)
// const fileBlob = new Blob([fileBuffer], { type: 'application/x-gzip' })
// const { fsBundle, wasmModule, vectorExtensionBundlePath } =
// await this.loadPGliteResources()
// this.db = await PGlite.create({
// loadDataDir: fileBlob,
// fsBundle: fsBundle,
// wasmModule: wasmModule,
// extensions: {
// vector: vectorExtensionBundlePath,
// live
// },
// })
// // return drizzle(this.pgClient)
// } catch (error) {
// console.error('Error loading database:', error)
// return null
// }
// }
// private async migrateDatabase(): Promise<void> {
// if (!this.db) {
// throw new Error('Database client not initialized');
// }
// try {
// // Execute SQL migrations
// for (const [_key, migration] of Object.entries(migrations)) {
// // Split SQL into individual commands and execute them one by one
// const commands = migration.sql.split('\n\n').filter(cmd => cmd.trim());
// for (const command of commands) {
// await this.db.query(command);
// }
// }
// } catch (error) {
// console.error('Error executing SQL migrations:', error);
// throw error;
// }
// }
async save(): Promise<void> {
console.warn("need remove")
}
async cleanup() {
this.db?.close()
this.db = null
}
// private async loadPGliteResources(): Promise<{
// fsBundle: Blob
// wasmModule: WebAssembly.Module
// vectorExtensionBundlePath: URL
// }> {
// try {
// // Convert base64 to binary data
// const wasmBinary = Buffer.from(pgliteResources.wasmBase64, 'base64')
// const dataBinary = Buffer.from(pgliteResources.dataBase64, 'base64')
// const vectorBinary = Buffer.from(pgliteResources.vectorBase64, 'base64')
// // Create blobs from binary data
// const fsBundle = new Blob([dataBinary], {
// type: 'application/octet-stream',
// })
// const wasmModule = await WebAssembly.compile(wasmBinary)
// // Create a blob URL for the vector extension
// const vectorBlob = new Blob([vectorBinary], {
// type: 'application/gzip',
// })
// const vectorExtensionBundlePath = URL.createObjectURL(vectorBlob)
// return {
// fsBundle,
// wasmModule,
// vectorExtensionBundlePath: new URL(vectorExtensionBundlePath),
// }
// } catch (error) {
// console.error('Error loading PGlite resources:', error)
// throw error
// }
// }
}

85
src/database/json/base.ts Normal file
View File

@@ -0,0 +1,85 @@
import * as path from 'path'
import { App, normalizePath } from 'obsidian'
export abstract class AbstractJsonRepository<T, M> {
protected dataDir: string
protected app: App
constructor(app: App, dataDir: string) {
this.app = app
this.dataDir = normalizePath(dataDir)
this.ensureDirectory()
}
private async ensureDirectory(): Promise<void> {
if (!(await this.app.vault.adapter.exists(this.dataDir))) {
await this.app.vault.adapter.mkdir(this.dataDir)
}
}
// Each subclass implements how to generate a file name from a data row.
protected abstract generateFileName(row: T): string
// Each subclass implements how to parse a file name into metadata.
protected abstract parseFileName(fileName: string): M | null
public async create(row: T): Promise<void> {
const fileName = this.generateFileName(row)
const filePath = normalizePath(path.join(this.dataDir, fileName))
const content = JSON.stringify(row, null, 2)
if (await this.app.vault.adapter.exists(filePath)) {
throw new Error(`File already exists: ${filePath}`)
}
await this.app.vault.adapter.write(filePath, content)
}
public async update(oldRow: T, newRow: T): Promise<void> {
const oldFileName = this.generateFileName(oldRow)
const newFileName = this.generateFileName(newRow)
const content = JSON.stringify(newRow, null, 2)
if (oldFileName === newFileName) {
// Simple update - filename hasn't changed
const filePath = normalizePath(path.join(this.dataDir, oldFileName))
await this.app.vault.adapter.write(filePath, content)
} else {
// Filename has changed - create new file and delete old one
const newFilePath = normalizePath(path.join(this.dataDir, newFileName))
await this.app.vault.adapter.write(newFilePath, content)
await this.delete(oldFileName)
}
}
// List metadata for all records by parsing file names.
public async listMetadata(): Promise<(M & { fileName: string })[]> {
const files = await this.app.vault.adapter.list(this.dataDir)
return files.files
.map((filePath) => path.basename(filePath))
.filter((fileName) => fileName.endsWith('.json'))
.map((fileName) => {
const metadata = this.parseFileName(fileName)
return metadata ? { ...metadata, fileName } : null
})
.filter(
(metadata): metadata is M & { fileName: string } => metadata !== null,
)
}
public async read(fileName: string): Promise<T | null> {
const filePath = normalizePath(path.join(this.dataDir, fileName))
if (!(await this.app.vault.adapter.exists(filePath))) return null
const content = await this.app.vault.adapter.read(filePath)
return JSON.parse(content) as T
}
public async delete(fileName: string): Promise<void> {
const filePath = normalizePath(path.join(this.dataDir, fileName))
if (await this.app.vault.adapter.exists(filePath)) {
await this.app.vault.adapter.remove(filePath)
}
}
}

View File

@@ -0,0 +1,115 @@
import { App } from 'obsidian'
import { v4 as uuidv4 } from 'uuid'
import { AbstractJsonRepository } from '../base'
import { CHAT_DIR, ROOT_DIR } from '../constants'
import { EmptyChatTitleException } from '../exception'
import {
CHAT_SCHEMA_VERSION,
ChatConversation,
ChatConversationMetadata,
} from './types'
export class ChatManager extends AbstractJsonRepository<
ChatConversation,
ChatConversationMetadata
> {
constructor(app: App) {
super(app, `${ROOT_DIR}/${CHAT_DIR}`)
}
protected generateFileName(chat: ChatConversation): string {
// Format: v{schemaVersion}_{title}_{updatedAt}_{id}.json
const encodedTitle = encodeURIComponent(chat.title)
return `v${chat.schemaVersion}_${encodedTitle}_${chat.updatedAt}_${chat.id}.json`
}
protected parseFileName(fileName: string): ChatConversationMetadata | null {
// Parse: v{schemaVersion}_{title}_{updatedAt}_{id}.json
const regex = new RegExp(
`^v${CHAT_SCHEMA_VERSION}_(.+)_(\\d+)_([0-9a-f-]+)\\.json$`,
)
const match = fileName.match(regex)
if (!match) return null
const title = decodeURIComponent(match[1])
const updatedAt = parseInt(match[2], 10)
const id = match[3]
return {
id,
schemaVersion: CHAT_SCHEMA_VERSION,
title,
updatedAt,
}
}
public async createChat(
initialData: Partial<ChatConversation>,
): Promise<ChatConversation> {
if (initialData.title && initialData.title.length === 0) {
throw new EmptyChatTitleException()
}
const now = Date.now()
const newChat: ChatConversation = {
id: uuidv4(),
title: 'New chat',
messages: [],
createdAt: now,
updatedAt: now,
schemaVersion: CHAT_SCHEMA_VERSION,
...initialData,
}
await this.create(newChat)
return newChat
}
public async findById(id: string): Promise<ChatConversation | null> {
const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.id === id)
if (!targetMetadata) return null
return this.read(targetMetadata.fileName)
}
public async updateChat(
id: string,
updates: Partial<
Omit<ChatConversation, 'id' | 'createdAt' | 'updatedAt' | 'schemaVersion'>
>,
): Promise<ChatConversation | null> {
const chat = await this.findById(id)
if (!chat) return null
if (updates.title !== undefined && updates.title.length === 0) {
throw new EmptyChatTitleException()
}
const updatedChat: ChatConversation = {
...chat,
...updates,
updatedAt: Date.now(),
}
await this.update(chat, updatedChat)
return updatedChat
}
public async deleteChat(id: string): Promise<boolean> {
const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.id === id)
if (!targetMetadata) return false
await this.delete(targetMetadata.fileName)
return true
}
public async listChats(): Promise<ChatConversationMetadata[]> {
const metadata = await this.listMetadata()
return metadata.sort((a, b) => b.updatedAt - a.updatedAt)
}
}

View File

@@ -0,0 +1,19 @@
import { SerializedChatMessage } from '../../../types/chat'
export const CHAT_SCHEMA_VERSION = 1
export type ChatConversation = {
id: string
title: string
messages: SerializedChatMessage[]
createdAt: number
updatedAt: number
schemaVersion: number
}
export type ChatConversationMetadata = {
id: string
title: string
updatedAt: number
schemaVersion: number
}

View File

@@ -0,0 +1,148 @@
import fuzzysort from 'fuzzysort'
import { App } from 'obsidian'
import { v4 as uuidv4 } from 'uuid'
import { AbstractJsonRepository } from '../base'
import { ROOT_DIR, TEMPLATE_DIR } from '../constants'
import {
DuplicateTemplateException,
EmptyTemplateNameException,
} from '../exception'
import { TEMPLATE_SCHEMA_VERSION, Template, TemplateMetadata } from './types'
export class TemplateManager extends AbstractJsonRepository<
Template,
TemplateMetadata
> {
constructor(app: App) {
super(app, `${ROOT_DIR}/${TEMPLATE_DIR}`)
}
protected generateFileName(template: Template): string {
// Format: v{schemaVersion}_name_id.json (with name encoded)
const encodedName = encodeURIComponent(template.name)
return `v${TEMPLATE_SCHEMA_VERSION}_${encodedName}_${template.id}.json`
}
protected parseFileName(fileName: string): TemplateMetadata | null {
const match = fileName.match(
new RegExp(`^v${TEMPLATE_SCHEMA_VERSION}_(.+)_([0-9a-f-]+)\\.json$`),
)
if (!match) return null
const encodedName = match[1]
const id = match[2]
const name = decodeURIComponent(encodedName)
return { id, name, schemaVersion: TEMPLATE_SCHEMA_VERSION }
}
public async createTemplate(
template: Omit<
Template,
'id' | 'createdAt' | 'updatedAt' | 'schemaVersion'
>,
): Promise<Template> {
if (template.name !== undefined && template.name.length === 0) {
throw new EmptyTemplateNameException()
}
const existingTemplate = await this.findByName(template.name)
if (existingTemplate) {
throw new DuplicateTemplateException(template.name)
}
const newTemplate: Template = {
id: uuidv4(),
...template,
createdAt: Date.now(),
updatedAt: Date.now(),
schemaVersion: TEMPLATE_SCHEMA_VERSION,
}
await this.create(newTemplate)
return newTemplate
}
public async ListTemplates(): Promise<Template[]> {
const allMetadata = await this.listMetadata()
const allTemplates = await Promise.all(allMetadata.map(async (meta) => this.read(meta.fileName)))
return allTemplates.sort((a, b) => b.updatedAt - a.updatedAt)
}
public async findById(id: string): Promise<Template | null> {
const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.id === id)
if (!targetMetadata) return null
return this.read(targetMetadata.fileName)
}
public async findByName(name: string): Promise<Template | null> {
const allMetadata = await this.listMetadata()
const targetMetadata = allMetadata.find((meta) => meta.name === name)
if (!targetMetadata) return null
return this.read(targetMetadata.fileName)
}
public async updateTemplate(
id: string,
updates: Partial<
Omit<Template, 'id' | 'createdAt' | 'updatedAt' | 'schemaVersion'>
>,
): Promise<Template | null> {
if (updates.name !== undefined && updates.name.length === 0) {
throw new EmptyTemplateNameException()
}
const template = await this.findById(id)
if (!template) return null
if (updates.name && updates.name !== template.name) {
const existingTemplate = await this.findByName(updates.name)
if (existingTemplate) {
throw new DuplicateTemplateException(updates.name)
}
}
const updatedTemplate: Template = {
...template,
...updates,
updatedAt: Date.now(),
}
await this.update(template, updatedTemplate)
return updatedTemplate
}
public async deleteTemplate(id: string): Promise<boolean> {
const template = await this.findById(id)
if (!template) return false
const fileName = this.generateFileName(template)
await this.delete(fileName)
return true
}
public async searchTemplates(query: string): Promise<Template[]> {
const allMetadata = await this.listMetadata()
const results = fuzzysort.go(query, allMetadata, {
keys: ['name'],
threshold: 0.2,
limit: 20,
all: true,
})
const templates = (
await Promise.all(
results.map(async (result) => this.read(result.obj.fileName)),
)
).filter((template): template is Template => template !== null)
return templates
}
}

View File

@@ -0,0 +1,18 @@
import { SerializedLexicalNode } from 'lexical'
export const TEMPLATE_SCHEMA_VERSION = 1
export type Template = {
id: string
name: string
content: { nodes: SerializedLexicalNode[] }
createdAt: number
updatedAt: number
schemaVersion: number
}
export type TemplateMetadata = {
id: string
name: string
schemaVersion: number
}

View File

@@ -0,0 +1,4 @@
export const ROOT_DIR = '.infio_json_db'
export const TEMPLATE_DIR = 'templates'
export const CHAT_DIR = 'chats'
export const INITIAL_MIGRATION_MARKER = '.initial_migration_completed'

View File

@@ -0,0 +1,20 @@
export class DuplicateTemplateException extends Error {
constructor(templateName: string) {
super(`Template with name "${templateName}" already exists`)
this.name = 'DuplicateTemplateException'
}
}
export class EmptyTemplateNameException extends Error {
constructor() {
super('Template name cannot be empty')
this.name = 'EmptyTemplateNameException'
}
}
export class EmptyChatTitleException extends Error {
constructor() {
super('Chat title cannot be empty')
this.name = 'EmptyChatTitleException'
}
}

View File

@@ -0,0 +1,118 @@
import { App, normalizePath } from 'obsidian'
import { DBManager } from '../database-manager'
import { DuplicateTemplateException } from '../exception'
import { ConversationManager } from '../modules/conversation/conversation-manager'
import { ChatManager } from './chat/ChatManager'
import { INITIAL_MIGRATION_MARKER, ROOT_DIR } from './constants'
import { TemplateManager } from './command/TemplateManager'
import { serializeChatMessage } from './utils'
async function hasMigrationCompleted(app: App): Promise<boolean> {
const markerPath = normalizePath(`${ROOT_DIR}/${INITIAL_MIGRATION_MARKER}`)
return await app.vault.adapter.exists(markerPath)
}
async function markMigrationCompleted(app: App): Promise<void> {
const markerPath = normalizePath(`${ROOT_DIR}/${INITIAL_MIGRATION_MARKER}`)
await app.vault.adapter.write(
markerPath,
`Migration completed on ${new Date().toISOString()}`,
)
}
async function transferChatHistory(app: App, dbManager: DBManager): Promise<void> {
const oldChatManager = new ConversationManager(app, dbManager)
const newChatManager = new ChatManager(app)
const chatList = await oldChatManager.conversations()
for (const chatMeta of chatList) {
try {
const existingChat = await newChatManager.findById(chatMeta.id)
if (existingChat) {
continue
}
const oldChatMessageList = await oldChatManager.findConversation(chatMeta.id)
if (!oldChatMessageList) {
continue
}
await newChatManager.createChat({
id: chatMeta.id,
title: chatMeta.title,
messages: oldChatMessageList.map(msg => serializeChatMessage(msg)),
createdAt: chatMeta.created_at instanceof Date ? chatMeta.created_at.getTime() : chatMeta.created_at,
updatedAt: chatMeta.updated_at instanceof Date ? chatMeta.updated_at.getTime() : chatMeta.updated_at,
})
const verifyChat = await newChatManager.findById(chatMeta.id)
if (!verifyChat) {
throw new Error(`Failed to verify migration of chat ${chatMeta.id}`)
}
await oldChatManager.deleteConversation(chatMeta.id)
} catch (error) {
console.error(`Error migrating chat ${chatMeta.id}:`, error)
}
}
console.log('Chat history migration to JSON database completed')
}
async function transferTemplates(
app: App,
dbManager: DBManager,
): Promise<void> {
const jsonTemplateManager = new TemplateManager(app)
const templateManager = dbManager.getCommandManager()
const templates = await templateManager.findAllCommands()
for (const template of templates) {
try {
if (await jsonTemplateManager.findByName(template.name)) {
// Template already exists, skip
continue
}
await jsonTemplateManager.createTemplate({
name: template.name,
content: template.content,
})
const verifyTemplate = await jsonTemplateManager.findByName(template.name)
if (!verifyTemplate) {
throw new Error(
`Failed to verify migration of template ${template.name}`,
)
}
await templateManager.deleteCommand(template.id)
} catch (error) {
if (error instanceof DuplicateTemplateException) {
console.log(`Duplicate template found: ${template.name}. Skipping...`)
} else {
console.error(`Error migrating template ${template.name}:`, error)
}
}
}
console.log('Templates migration to JSON database completed')
}
export async function migrateToJsonDatabase(
app: App,
dbManager: DBManager,
onMigrationComplete?: () => void,
): Promise<void> {
if (await hasMigrationCompleted(app)) {
return
}
await transferChatHistory(app, dbManager)
await transferTemplates(app, dbManager)
await markMigrationCompleted(app)
onMigrationComplete?.()
}

View File

@@ -0,0 +1,63 @@
import { App } from 'obsidian'
import { ChatMessage, SerializedChatMessage } from '../../types/chat'
import { Mentionable } from '../../types/mentionable'
import {
deserializeMentionable,
serializeMentionable,
} from '../../utils/mentionable'
export const serializeChatMessage = (message: ChatMessage): SerializedChatMessage => {
switch (message.role) {
case 'user':
return {
role: 'user',
applyStatus: message.applyStatus,
content: message.content,
promptContent: message.promptContent,
id: message.id,
mentionables: message.mentionables.map(serializeMentionable),
similaritySearchResults: message.similaritySearchResults,
}
case 'assistant':
return {
role: 'assistant',
applyStatus: message.applyStatus,
content: message.content,
reasoningContent: message.reasoningContent,
id: message.id,
metadata: message.metadata,
}
}
}
export const deserializeChatMessage = (
message: SerializedChatMessage,
app: App,
): ChatMessage => {
switch (message.role) {
case 'user': {
return {
role: 'user',
applyStatus: message.applyStatus,
content: message.content,
promptContent: message.promptContent,
id: message.id,
mentionables: message.mentionables
.map((m) => deserializeMentionable(m, app))
.filter((m): m is Mentionable => m !== null),
similaritySearchResults: message.similaritySearchResults,
}
}
case 'assistant':
return {
role: 'assistant',
applyStatus: message.applyStatus,
content: message.content,
reasoningContent: message.reasoningContent,
id: message.id,
metadata: message.metadata,
}
}
}

View File

@@ -32,7 +32,7 @@ export class CommandRepository {
throw new DatabaseNotInitializedException()
}
const result = await this.db.query<SelectTemplate>(
`SELECT * FROM "template"`
`SELECT * FROM "template" ORDER BY created_at DESC`
)
return result.rows
}

View File

@@ -1,5 +1,5 @@
import { App } from 'obsidian'
import { Transaction } from '@electric-sql/pglite'
import { App } from 'obsidian'
import { editorStateToPlainText } from '../../../components/chat-view/chat-input/utils/editor-state-to-plain-text'
import { ChatAssistantMessage, ChatConversationMeta, ChatMessage, ChatUserMessage } from '../../../types/chat'
@@ -77,6 +77,10 @@ export class ConversationManager {
await this.repository.delete(id)
}
async conversations(): Promise<SelectConversation[]>{
return this.repository.findAll()
}
getAllConversations(callback: (conversations: ChatConversationMeta[]) => void): void {
const db = this.dbManager.getPgClient()
db?.live.query('SELECT * FROM conversations ORDER BY updated_at DESC', [], (results: { rows: Array<SelectConversation> }) => {