feat: use sqlite instead of lowdb

This commit is contained in:
ikechan8370 2025-04-09 16:27:21 +08:00
parent 9c41251164
commit fd197abb33
22 changed files with 3519 additions and 39 deletions

View file

@ -40,7 +40,7 @@ export class bym extends plugin {
let recall = false let recall = false
let presetId = ChatGPTConfig.bym.defaultPreset let presetId = ChatGPTConfig.bym.defaultPreset
if (ChatGPTConfig.bym.presetMap && ChatGPTConfig.bym.presetMap.length > 0) { if (ChatGPTConfig.bym.presetMap && ChatGPTConfig.bym.presetMap.length > 0) {
const option = ChatGPTConfig.bym.presetMap.sort((a, b) => a.priority - b.priority) const option = ChatGPTConfig.bym.presetMap.sort((a, b) => b.priority - a.priority)
.find(item => item.keywords.find(keyword => e.msg?.includes(keyword))) .find(item => item.keywords.find(keyword => e.msg?.includes(keyword)))
if (option) { if (option) {
presetId = option.presetId presetId = option.presetId

View file

@ -1,7 +1,7 @@
import Config from '../config/config.js' import Config from '../config/config.js'
import { Chaite, SendMessageOption } from 'chaite' import { Chaite, SendMessageOption } from 'chaite'
import { getPreset, intoUserMessage, toYunzai } from '../utils/message.js' import { getPreset, intoUserMessage, toYunzai } from '../utils/message.js'
import { YunzaiUserState } from '../models/chaite/user_state_storage.js' import { YunzaiUserState } from '../models/chaite/storage/lowdb/user_state_storage.js'
import { getGroupContextPrompt, getGroupHistory } from '../utils/group.js' import { getGroupContextPrompt, getGroupHistory } from '../utils/group.js'
import * as crypto from 'node:crypto' import * as crypto from 'node:crypto'

View file

@ -166,15 +166,17 @@ class ChatGPTConfig {
// 工具目录,相对于插件目录下 // 工具目录,相对于插件目录下
toolsDirPath: 'utils/tools', toolsDirPath: 'utils/tools',
// 云端API url // 云端API url
cloudBaseUrl: '', cloudBaseUrl: 'https://api.chaite.cloud',
// 云端API Key // 云端API Key
cloudApiKey: '', cloudApiKey: '',
// jwt key非必要勿修改修改需重启 // jwt key非必要勿修改修改需重启
authKey: '', authKey: '',
// 管理面板监听地址 // 管理面板监听地址
host: '', host: '0.0.0.0',
// 管理面板监听端口 // 管理面板监听端口
port: 48370 port: 48370,
// 存储实现 sqlite lowdb
storage: 'sqlite'
} }
constructor () { constructor () {
@ -212,26 +214,59 @@ class ChatGPTConfig {
} }
}) })
const createDeepProxy = (obj, handler) => { const createDeepProxy = (obj, handler, seen = new WeakMap()) => {
// 基本类型或非对象直接返回
if (obj === null || typeof obj !== 'object') return obj if (obj === null || typeof obj !== 'object') return obj
// 检查循环引用
if (seen.has(obj)) {
return seen.get(obj)
}
// 创建代理对象
const proxy = new Proxy(obj, handler)
// 记录已创建的代理,避免循环引用
seen.set(obj, proxy)
// 处理子对象
for (let key of Object.keys(obj)) { for (let key of Object.keys(obj)) {
if (typeof obj[key] === 'object' && obj[key] !== null) { if (typeof obj[key] === 'object' && obj[key] !== null) {
obj[key] = createDeepProxy(obj[key], handler) obj[key] = createDeepProxy(obj[key], handler, seen)
} }
} }
return new Proxy(obj, handler) return proxy
} }
// 创建处理器
const handler = { const handler = {
set: (target, prop, value) => { set: (target, prop, value) => {
if (prop !== 'watcher' && prop !== 'configPath') { if (prop !== 'watcher' && prop !== 'configPath') {
target[prop] = typeof value === 'object' && value !== null // 避免递归创建代理
? createDeepProxy(value, handler) if (typeof value === 'object' && value !== null) {
: value // 检查 value 是否已经是代理
this.saveToFile() if (!value.__isProxy) {
const newProxy = createDeepProxy(value, handler)
// 标记为代理对象
Object.defineProperty(newProxy, '__isProxy', {
value: true,
enumerable: false,
configurable: false
})
target[prop] = newProxy
} else {
target[prop] = value
}
} else {
target[prop] = value
}
// 避免在代理对象保存时再次触发
if (!target.__isSaving) {
target.__isSaving = true
this.saveToFile()
target.__isSaving = false
}
} }
return true return true
} }

View file

@ -11,19 +11,25 @@ import {
ToolsGroupManager ToolsGroupManager
} from 'chaite' } from 'chaite'
import ChatGPTConfig from '../../config/config.js' import ChatGPTConfig from '../../config/config.js'
import { LowDBChannelStorage } from './channel_storage.js' import { LowDBChannelStorage } from './storage/lowdb/channel_storage.js'
import { LowDBChatPresetsStorage } from './chat_preset_storage.js' import { LowDBChatPresetsStorage } from './storage/lowdb/chat_preset_storage.js'
import { LowDBToolsStorage } from './tools_storage.js' import { LowDBToolsStorage } from './storage/lowdb/tools_storage.js'
import { LowDBProcessorsStorage } from './processors_storage.js' import { LowDBProcessorsStorage } from './storage/lowdb/processors_storage.js'
import { ChatGPTUserModeSelector } from './user_mode_selector.js' import { ChatGPTUserModeSelector } from './user_mode_selector.js'
import { LowDBUserStateStorage } from './user_state_storage.js' import { LowDBUserStateStorage } from './storage/lowdb/user_state_storage.js'
import { LowDBHistoryManager } from './history_manager.js' import { LowDBHistoryManager } from './storage/lowdb/history_manager.js'
import { VectraVectorDatabase } from './vector_database.js' import { VectraVectorDatabase } from './vector_database.js'
import ChatGPTStorage, { ChatGPTHistoryStorage } from '../storage.js'
import path from 'path' import path from 'path'
import fs from 'fs' import fs from 'fs'
import { migrateDatabase } from '../../utils/initDB.js' import { migrateDatabase } from '../../utils/initDB.js'
import { LowDBToolsGroupDTOsStorage } from './tool_groups_storage.js' import { SQLiteChannelStorage } from './storage/sqlite/channel_storage.js'
import { dataDir } from '../../utils/common.js'
import { SQLiteChatPresetStorage } from './storage/sqlite/chat_preset_storage.js'
import { SQLiteToolsStorage } from './storage/sqlite/tools_storage.js'
import { SQLiteProcessorsStorage } from './storage/sqlite/processors_storage.js'
import { SQLiteUserStateStorage } from './storage/sqlite/user_state_storage.js'
import { SQLiteToolsGroupStorage } from './storage/sqlite/tool_groups_storage.js'
import { checkMigrate } from './storage/sqlite/migrate.js'
/** /**
* 认证以便共享上传 * 认证以便共享上传
@ -35,7 +41,7 @@ export async function authCloud (apiKey = ChatGPTConfig.chaite.cloudApiKey) {
await Chaite.getInstance().auth(apiKey) await Chaite.getInstance().auth(apiKey)
return Chaite.getInstance().getToolsManager().cloudService.getUser() return Chaite.getInstance().getToolsManager().cloudService.getUser()
} catch (err) { } catch (err) {
logger.error(err)
} }
} }
@ -121,26 +127,56 @@ export async function initRagManager (model, dimensions) {
} }
export async function initChaite () { export async function initChaite () {
await ChatGPTStorage.init() const storage = ChatGPTConfig.chaite.storage
const channelsManager = await ChannelsManager.init(new LowDBChannelStorage(ChatGPTStorage), new DefaultChannelLoadBalancer()) let channelsStorage, chatPresetsStorage, toolsStorage, processorsStorage, userStateStorage, historyStorage, toolsGroupStorage
switch (storage) {
case 'sqlite': {
const dbPath = path.join(dataDir, 'data.db')
channelsStorage = new SQLiteChannelStorage(dbPath)
await channelsStorage.initialize()
chatPresetsStorage = new SQLiteChatPresetStorage(dbPath)
await chatPresetsStorage.initialize()
toolsStorage = new SQLiteToolsStorage(dbPath)
await toolsStorage.initialize()
processorsStorage = new SQLiteProcessorsStorage(dbPath)
await processorsStorage.initialize()
userStateStorage = new SQLiteUserStateStorage(dbPath)
await userStateStorage.initialize()
toolsGroupStorage = new SQLiteToolsGroupStorage(dbPath)
await toolsGroupStorage.initialize()
await checkMigrate()
break
}
case 'lowdb': {
const ChatGPTStorage = (await import('storage/lowdb/storage.js')).default
await ChatGPTStorage.init()
channelsStorage = new LowDBChannelStorage(ChatGPTStorage)
chatPresetsStorage = new LowDBChatPresetsStorage(ChatGPTStorage)
toolsStorage = new LowDBToolsStorage(ChatGPTStorage)
processorsStorage = new LowDBProcessorsStorage(ChatGPTStorage)
userStateStorage = new LowDBUserStateStorage(ChatGPTStorage)
const ChatGPTHistoryStorage = (await import('storage/lowdb/storage.js')).ChatGPTHistoryStorage
await ChatGPTHistoryStorage.init()
historyStorage = new LowDBHistoryManager(ChatGPTHistoryStorage)
break
}
}
const channelsManager = await ChannelsManager.init(channelsStorage, new DefaultChannelLoadBalancer())
const toolsDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.toolsDirPath) const toolsDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.toolsDirPath)
if (!fs.existsSync(toolsDir)) { if (!fs.existsSync(toolsDir)) {
fs.mkdirSync(toolsDir, { recursive: true }) fs.mkdirSync(toolsDir, { recursive: true })
} }
const toolsManager = await ToolManager.init(toolsDir, new LowDBToolsStorage(ChatGPTStorage)) const toolsManager = await ToolManager.init(toolsDir, toolsStorage)
const processorsDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.processorsDirPath) const processorsDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.processorsDirPath)
if (!fs.existsSync(processorsDir)) { if (!fs.existsSync(processorsDir)) {
fs.mkdirSync(processorsDir, { recursive: true }) fs.mkdirSync(processorsDir, { recursive: true })
} }
const processorsManager = await ProcessorsManager.init(processorsDir, new LowDBProcessorsStorage(ChatGPTStorage)) const processorsManager = await ProcessorsManager.init(processorsDir, processorsStorage)
const chatPresetManager = await ChatPresetManager.init(new LowDBChatPresetsStorage(ChatGPTStorage)) const chatPresetManager = await ChatPresetManager.init(chatPresetsStorage)
const toolsGroupManager = await ToolsGroupManager.init(new LowDBToolsGroupDTOsStorage(ChatGPTStorage)) const toolsGroupManager = await ToolsGroupManager.init(toolsGroupStorage)
const userModeSelector = new ChatGPTUserModeSelector() const userModeSelector = new ChatGPTUserModeSelector()
const userStateStorage = new LowDBUserStateStorage(ChatGPTStorage)
await ChatGPTHistoryStorage.init()
const historyManager = new LowDBHistoryManager(ChatGPTHistoryStorage)
let chaite = Chaite.init(channelsManager, toolsManager, processorsManager, chatPresetManager, toolsGroupManager, let chaite = Chaite.init(channelsManager, toolsManager, processorsManager, chatPresetManager, toolsGroupManager,
userModeSelector, userStateStorage, historyManager, logger) userModeSelector, userStateStorage, historyStorage, logger)
logger.info('Chaite 初始化完成') logger.info('Chaite 初始化完成')
chaite.setCloudService(ChatGPTConfig.chaite.cloudBaseUrl) chaite.setCloudService(ChatGPTConfig.chaite.cloudBaseUrl)
logger.info('Chaite.Cloud 初始化完成') logger.info('Chaite.Cloud 初始化完成')

View file

@ -3,7 +3,7 @@ import { Low } from 'lowdb'
import { JSONFile } from 'lowdb/node' import { JSONFile } from 'lowdb/node'
import path from 'path' import path from 'path'
import fs from 'fs' import fs from 'fs'
import ChatGPTConfig from '../config/config.js' import { dataDir } from '../../../../utils/common.js'
/** /**
* 基于 LowDB 的简单存储类提供 CRUD 和条件查询功能 * 基于 LowDB 的简单存储类提供 CRUD 和条件查询功能
@ -348,10 +348,6 @@ export class LowDBCollection {
} }
} }
export const dataDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.dataDir)
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true })
}
const storageLocation = path.resolve(dataDir, 'storage.json') const storageLocation = path.resolve(dataDir, 'storage.json')
if (!fs.existsSync(storageLocation)) { if (!fs.existsSync(storageLocation)) {
fs.writeFileSync(storageLocation, JSON.stringify({ collections: {} })) fs.writeFileSync(storageLocation, JSON.stringify({ collections: {} }))

View file

@ -4,6 +4,10 @@ import { ChaiteStorage, ToolDTO } from 'chaite'
* @extends {ChaiteStorage<import('chaite').ToolDTO>} * @extends {ChaiteStorage<import('chaite').ToolDTO>}
*/ */
export class LowDBToolsStorage extends ChaiteStorage { export class LowDBToolsStorage extends ChaiteStorage {
getName () {
return 'LowDBToolsStorage'
}
/** /**
* *
* @param { LowDBStorage } storage * @param { LowDBStorage } storage

View file

@ -0,0 +1,518 @@
import { ChaiteStorage, Channel } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
/**
* @extends {ChaiteStorage<import('chaite').Channel>}
*/
export class SQLiteChannelStorage extends ChaiteStorage {
getName () {
return 'SQLiteChannelStorage'
}
/**
*
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'channels'
}
/**
* 初始化数据库连接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保目录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建Channel表将主要属性分列存储
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
adapterType TEXT NOT NULL,
type TEXT NOT NULL,
weight INTEGER DEFAULT 1,
priority INTEGER DEFAULT 0,
status TEXT DEFAULT 'enabled',
disabledReason TEXT,
models TEXT,
options TEXT,
statistics TEXT,
uploader TEXT,
cloudId INTEGER,
createdAt TEXT,
updatedAt TEXT,
md5 TEXT,
embedded INTEGER DEFAULT 0,
extra TEXT -- 存储其他额外数据的JSON
)`, (err) => {
if (err) {
return reject(err)
}
// 创建索引提高查询性能
const promises = [
// 按类型和状态索引
new Promise((resolve, reject) => {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_type ON ${this.tableName} (type)`, err => {
if (err) reject(err)
else resolve()
})
}),
new Promise((resolve, reject) => {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_status ON ${this.tableName} (status)`, err => {
if (err) reject(err)
else resolve()
})
})
]
Promise.all(promises)
.then(() => {
this.initialized = true
resolve()
})
.catch(reject)
})
})
})
}
/**
* 确保数据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* Channel 对象转换为数据库记录
* @param {import('chaite').Channel} channel
* @returns {Object} 数据库记录
*/
_channelToRecord (channel) {
// 提取主要字段
const {
id, name, description, adapterType, type, weight, priority,
status, disabledReason, models, options, statistics,
uploader, cloudId, createdAt, updatedAt, md5, embedded, ...rest
} = channel
return {
id: id || '',
name: name || '',
description: description || '',
adapterType: adapterType || type || '',
type: type || '',
weight: weight || 1,
priority: priority || 0,
status: status || 'enabled',
disabledReason: disabledReason || null,
models: Array.isArray(models) ? JSON.stringify(models) : '[]',
options: options ? JSON.stringify(options) : null,
statistics: statistics ? JSON.stringify(statistics) : null,
uploader: uploader ? JSON.stringify(uploader) : null,
cloudId: cloudId || null,
createdAt: createdAt || '',
updatedAt: updatedAt || '',
md5: md5 || '',
embedded: embedded ? 1 : 0,
extra: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null
}
}
/**
* 将数据库记录转换为 Channel 对象
* @param {Object} record 数据库记录
* @returns {import('chaite').Channel} Channel 对象
*/
_recordToChannel (record) {
if (!record) return null
// 解析JSON字段
let models = []
try {
if (record.models) {
models = JSON.parse(record.models)
}
} catch (e) {
// 解析错误,使用空数组
}
let options = {}
try {
if (record.options) {
options = JSON.parse(record.options)
}
} catch (e) {
// 解析错误,使用空对象
}
let statistics = {}
try {
if (record.statistics) {
statistics = JSON.parse(record.statistics)
}
} catch (e) {
// 解析错误,使用空对象
}
let uploader = null
try {
if (record.uploader) {
uploader = JSON.parse(record.uploader)
}
} catch (e) {
// 解析错误使用null
}
let extra = {}
try {
if (record.extra) {
extra = JSON.parse(record.extra)
}
} catch (e) {
// 解析错误,使用空对象
}
// 构造Channel对象
const channelData = {
id: record.id,
name: record.name,
description: record.description,
adapterType: record.adapterType,
type: record.type,
weight: Number(record.weight),
priority: Number(record.priority),
status: record.status,
disabledReason: record.disabledReason,
models,
options,
statistics,
uploader,
cloudId: record.cloudId,
createdAt: record.createdAt,
updatedAt: record.updatedAt,
md5: record.md5,
embedded: Boolean(record.embedded),
...extra
}
return new Channel(channelData)
}
/**
* 获取单个渠道
* @param {string} key 渠道ID
* @returns {Promise<import('chaite').Channel>}
*/
async getItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [key], (err, row) => {
if (err) {
return reject(err)
}
const channel = this._recordToChannel(row)
resolve(channel)
})
})
}
/**
* 保存渠道
* @param {string} id 渠道ID
* @param {import('chaite').Channel} channel 渠道对象
* @returns {Promise<string>}
*/
async setItem (id, channel) {
await this.ensureInitialized()
// 转换为数据库记录
const record = this._channelToRecord(channel)
record.id = id // 确保ID是指定的ID
// 构建插入或更新SQL
const fields = Object.keys(record)
const placeholders = fields.map(() => '?').join(', ')
const updates = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => record[field])
const duplicateValues = [...values] // 用于ON CONFLICT时的更新
return new Promise((resolve, reject) => {
this.db.run(
`INSERT INTO ${this.tableName} (${fields.join(', ')})
VALUES (${placeholders})
ON CONFLICT(id) DO UPDATE SET ${updates}`,
[...values, ...duplicateValues],
function (err) {
if (err) {
return reject(err)
}
resolve(id)
}
)
})
}
/**
* 删除渠道
* @param {string} key 渠道ID
* @returns {Promise<void>}
*/
async removeItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE id = ?`, [key], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 查询所有渠道
* @returns {Promise<import('chaite').Channel[]>}
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const channels = rows.map(row => this._recordToChannel(row)).filter(Boolean)
resolve(channels)
})
})
}
/**
* 根据条件筛选渠道
* @param {Record<string, unknown>} filter 筛选条件
* @returns {Promise<import('chaite').Channel[]>}
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
// 如果没有筛选条件,返回所有
if (!filter || Object.keys(filter).length === 0) {
return this.listItems()
}
// 尝试使用SQL字段直接过滤
const directFields = ['id', 'name', 'description', 'adapterType', 'type', 'status', 'cloudId']
const numericFields = ['weight', 'priority']
const sqlFilters = []
const sqlParams = []
const extraFilters = {}
let hasExtraFilters = false
// 区分数据库字段和额外字段
for (const key in filter) {
const value = filter[key]
// 如果是直接支持的字段构建SQL条件
if (directFields.includes(key)) {
sqlFilters.push(`${key} = ?`)
sqlParams.push(value)
} else if (numericFields.includes(key)) {
// 数值型字段
sqlFilters.push(`${key} = ?`)
sqlParams.push(Number(value))
} else if (key === 'embedded') {
// embedded 字段需要特殊处理为 0/1
sqlFilters.push('embedded = ?')
sqlParams.push(value ? 1 : 0)
} else if (key === 'models' && typeof value === 'string') {
// models字段需要特殊处理判断是否包含某模型
// 注意:这种方式仅适用于单个模型的查询,不适用于完全匹配数组
sqlFilters.push('models LIKE ?')
sqlParams.push(`%${value}%`)
} else {
// 其他字段需要在结果中进一步过滤
extraFilters[key] = value
hasExtraFilters = true
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let channels = rows.map(row => this._recordToChannel(row)).filter(Boolean)
// 如果有需要在内存中过滤的额外<E9A29D><E5A496><EFBFBD>
if (hasExtraFilters) {
channels = channels.filter(channel => {
for (const key in extraFilters) {
if (channel[key] !== extraFilters[key]) {
return false
}
}
return true
})
}
resolve(channels)
})
})
}
/**
* 根据IN条件筛选渠道
* @param {Array<{ field: string; values: unknown[]; }>} query
* @returns {Promise<import('chaite').Channel[]>}
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
// 如果没有查询条件,返回所有
if (!query || query.length === 0) {
return this.listItems()
}
// 尝试使用SQL IN子句来优化查询
const directFields = ['id', 'name', 'description', 'adapterType', 'type', 'status', 'cloudId']
const numericFields = ['weight', 'priority']
const sqlFilters = []
const sqlParams = []
const extraQueries = []
// 处理每个查询条件
for (const { field, values } of query) {
if (values.length === 0) continue
// 如果是直接支持的字段使用SQL IN子句
if (directFields.includes(field)) {
const placeholders = values.map(() => '?').join(', ')
sqlFilters.push(`${field} IN (${placeholders})`)
sqlParams.push(...values)
} else if (numericFields.includes(field)) {
// 数值型字段
const placeholders = values.map(() => '?').join(', ')
sqlFilters.push(`${field} IN (${placeholders})`)
sqlParams.push(...values.map(v => Number(v)))
} else if (field === 'embedded') {
// embedded 字段需要特殊处理
const boolValues = values.map(v => v ? 1 : 0)
const placeholders = boolValues.map(() => '?').join(', ')
sqlFilters.push(`embedded IN (${placeholders})`)
sqlParams.push(...boolValues)
} else if (field === 'models') {
// models字段需要特殊处理判断是否包含某模型
// 由于无法直接使用IN查询JSON字段这里使用OR和LIKE的组合
const modelFilters = values.map(() => 'models LIKE ?').join(' OR ')
sqlFilters.push(`(${modelFilters})`)
values.forEach(value => {
sqlParams.push(`%${value}%`)
})
} else {
// 其他字段在内存中过滤
extraQueries.push({ field, values })
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let channels = rows.map(row => this._recordToChannel(row)).filter(Boolean)
// 如果有需要在内存中过滤的条件
if (extraQueries.length > 0) {
channels = channels.filter(channel => {
for (const { field, values } of extraQueries) {
if (!values.includes(channel[field])) {
return false
}
}
return true
})
}
resolve(channels)
})
})
}
/**
* 清空表中所有数据
* @returns {Promise<void>}
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,513 @@
import { ChaiteStorage, ChatPreset } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
/**
* @extends {ChaiteStorage<import('chaite').ChatPreset>}
*/
export class SQLiteChatPresetStorage extends ChaiteStorage {
getName () {
return 'SQLiteChatPresetStorage'
}
/**
*
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'chat_presets'
}
/**
* 初始化数据库连接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保目录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建 ChatPreset 表,将主要属性分列存储
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
prefix TEXT NOT NULL,
local INTEGER DEFAULT 1,
namespace TEXT,
sendMessageOption TEXT NOT NULL,
cloudId INTEGER,
createdAt TEXT,
updatedAt TEXT,
md5 TEXT,
embedded INTEGER DEFAULT 0,
uploader TEXT,
extraData TEXT
)`, (err) => {
if (err) {
return reject(err)
}
// 创建索引提高查询性能
const promises = [
new Promise((resolve, reject) => {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_prefix ON ${this.tableName} (prefix)`, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
}),
new Promise((resolve, reject) => {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_name ON ${this.tableName} (name)`, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
]
Promise.all(promises)
.then(() => {
this.initialized = true
resolve()
})
.catch(reject)
})
})
})
}
/**
* 确保<EFBFBD><EFBFBD><EFBFBD>据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* ChatPreset 对象转换为数据库记录
* @param {import('chaite').ChatPreset} preset
* @returns {Object} 数据库记录
*/
_presetToRecord (preset) {
// 提取主要字段
const {
id, name, description, prefix, local, namespace,
sendMessageOption, cloudId, createdAt, updatedAt, md5,
embedded, uploader, ...rest
} = preset
return {
id: id || '',
name: name || '',
description: description || '',
prefix: prefix || '',
local: local === false ? 0 : 1,
namespace: namespace || null,
sendMessageOption: JSON.stringify(sendMessageOption || {}),
cloudId: cloudId || null,
createdAt: createdAt || '',
updatedAt: updatedAt || '',
md5: md5 || '',
embedded: embedded ? 1 : 0,
uploader: uploader ? JSON.stringify(uploader) : null,
extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null
}
}
/**
* 将数<EFBFBD><EFBFBD><EFBFBD>库记录转换为 ChatPreset 对象
* @param {Object} record 数据库记录
* @returns {import('chaite').ChatPreset} ChatPreset 对象
*/
_recordToPreset (record) {
if (!record) return null
// 解析 JSON 字<><E5AD97>
let sendMessageOption = {}
try {
if (record.sendMessageOption) {
sendMessageOption = JSON.parse(record.sendMessageOption)
}
} catch (e) {
// 解析错误,使用空对象
}
let uploader = null
try {
if (record.uploader) {
uploader = JSON.parse(record.uploader)
}
} catch (e) {
// 解析错误,使用 null
}
let extraData = {}
try {
if (record.extraData) {
extraData = JSON.parse(record.extraData)
}
} catch (e) {
// 解析错误,使用空对象
}
// 构造 ChatPreset 对象
const presetData = {
id: record.id,
name: record.name,
description: record.description,
prefix: record.prefix,
local: Boolean(record.local),
namespace: record.namespace,
sendMessageOption,
cloudId: record.cloudId,
createdAt: record.createdAt,
updatedAt: record.updatedAt,
md5: record.md5,
embedded: Boolean(record.embedded),
uploader,
...extraData
}
return new ChatPreset(presetData)
}
/**
* 获取单个聊天预设
* @param {string} key 预设ID
* @returns {Promise<import('chaite').ChatPreset>}
*/
async getItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [key], (err, row) => {
if (err) {
return reject(err)
}
const preset = this._recordToPreset(row)
resolve(preset)
})
})
}
/**
* 保存聊天预设
* @param {string} id 预设ID
* @param {import('chaite').ChatPreset} preset 预设对象
* @returns {Promise<string>}
*/
async setItem (id, preset) {
await this.ensureInitialized()
// 转换为数据库记录
const record = this._presetToRecord(preset)
record.id = id // 确保ID是指定的ID
// 构建插入或更新SQL
const fields = Object.keys(record)
const placeholders = fields.map(() => '?').join(', ')
const updates = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => record[field])
const duplicateValues = [...values] // 用于ON CONFLICT时的更新
return new Promise((resolve, reject) => {
this.db.run(
`INSERT INTO ${this.tableName} (${fields.join(', ')})
VALUES (${placeholders})
ON CONFLICT(id) DO UPDATE SET ${updates}`,
[...values, ...duplicateValues],
function (err) {
if (err) {
return reject(err)
}
resolve(id)
}
)
})
}
/**
* 删除聊天预设
* @param {string} key 预设ID
* @returns {Promise<void>}
*/
async removeItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE id = ?`, [key], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 查询所有聊天预设
* @returns {Promise<import('chaite').ChatPreset[]>}
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const presets = rows.map(row => this._recordToPreset(row)).filter(Boolean)
resolve(presets)
})
})
}
/**
* 根据条件筛选聊天预设
* @param {Record<string, unknown>} filter 筛选条件
* @returns {Promise<import('chaite').ChatPreset[]>}
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
// 如果没有筛选条件,返回所有
if (!filter || Object.keys(filter).length === 0) {
return this.listItems()
}
// 尝试使用SQL字段直接过滤
const directFields = ['id', 'name', 'description', 'prefix', 'namespace', 'cloudId']
const sqlFilters = []
const sqlParams = []
const extraFilters = {}
let hasExtraFilters = false
// 区分数据库字段和额外字段
for (const key in filter) {
const value = filter[key]
// 如果是直接支持的字段构建SQL条件
if (directFields.includes(key)) {
sqlFilters.push(`${key} = ?`)
sqlParams.push(value)
} else if (key === 'local') {
// local 字段需要特殊处理为 0/1
sqlFilters.push('local = ?')
sqlParams.push(value ? 1 : 0)
} else if (key === 'embedded') {
// embedded 字段需要特殊处理为 0/1
sqlFilters.push('embedded = ?')
sqlParams.push(value ? 1 : 0)
} else {
// 其他字段需要在结果中进一步过滤
extraFilters[key] = value
hasExtraFilters = true
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let presets = rows.map(row => this._recordToPreset(row)).filter(Boolean)
// 如果有需要在内存中过滤的额外字段
if (hasExtraFilters) {
presets = presets.filter(preset => {
for (const key in extraFilters) {
const filterValue = extraFilters[key]
// 处理 sendMessageOption 字段的深层过滤
if (key.startsWith('sendMessageOption.')) {
const optionKey = key.split('.')[1]
if (preset.sendMessageOption && preset.sendMessageOption[optionKey] !== filterValue) {
return false
}
} else if (preset[key] !== filterValue) {
// 其他字段直接比较
return false
}
}
return true
})
}
resolve(presets)
})
})
}
/**
* 根据IN条件筛选聊天预设
* @param {Array<{ field: string; values: unknown[]; }>} query
* @returns {Promise<import('chaite').ChatPreset[]>}
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
// 如果没有查询条件,返回所有
if (!query || query.length === 0) {
return this.listItems()
}
// 尝试使用SQL IN子句来优化查询
const directFields = ['id', 'name', 'description', 'prefix', 'namespace', 'cloudId']
const sqlFilters = []
const sqlParams = []
const extraQueries = []
// 处理每个查询条件
for (const { field, values } of query) {
if (values.length === 0) continue
// 如果是直接支持的字段使用SQL IN子句
if (directFields.includes(field)) {
const placeholders = values.map(() => '?').join(', ')
sqlFilters.push(`${field} IN (${placeholders})`)
sqlParams.push(...values)
} else if (field === 'local') {
// local 字段需要特殊处理
const boolValues = values.map(v => v ? 1 : 0)
const placeholders = boolValues.map(() => '?').join(', ')
sqlFilters.push(`local IN (${placeholders})`)
sqlParams.push(...boolValues)
} else if (field === 'embedded') {
// embedded 字段需要特殊处理
const boolValues = values.map(v => v ? 1 : 0)
const placeholders = boolValues.map(() => '?').join(', ')
sqlFilters.push(`embedded IN (${placeholders})`)
sqlParams.push(...boolValues)
} else {
// 其他字段在内存中过滤
extraQueries.push({ field, values })
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let presets = rows.map(row => this._recordToPreset(row)).filter(Boolean)
// 如果有需要在内存中过滤的条件
if (extraQueries.length > 0) {
presets = presets.filter(preset => {
for (const { field, values } of extraQueries) {
// 处<><E5A484><EFBFBD> sendMessageOption 字段的深层过滤
if (field.startsWith('sendMessageOption.')) {
const optionKey = field.split('.')[1]
const presetValue = preset.sendMessageOption?.[optionKey]
if (!values.includes(presetValue)) {
return false
}
} else if (!values.includes(preset[field])) {
// 其他字段直接比较
return false
}
}
return true
})
}
resolve(presets)
})
})
}
/**
* 根据前缀获取聊天预设
* @param {string} prefix 前缀
* @returns {Promise<import('chaite').ChatPreset | null>}
*/
async getPresetByPrefix (prefix) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE prefix = ?`, [prefix], (err, row) => {
if (err) {
return reject(err)
}
const preset = this._recordToPreset(row)
resolve(preset)
})
})
}
/**
* 清空表中所有数据
* @returns {Promise<void>}
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,596 @@
import { AbstractHistoryManager } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
import crypto from 'crypto'
export class SQLiteHistoryManager extends AbstractHistoryManager {
/**
*
* @param {string} dbPath 数据库文件路径
* @param {string} imagesDir 图片存储目录默认为数据库同级的 images 目录
*/
constructor (dbPath, imagesDir) {
super()
this.dbPath = dbPath
this.imagesDir = imagesDir || path.join(path.dirname(dbPath), 'images')
this.db = null
this.initialized = false
this.tableName = 'history'
}
/**
* 初始化数据库连接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保目录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
// 确保图片目录存在
if (!fs.existsSync(this.imagesDir)) {
fs.mkdirSync(this.imagesDir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建 history 表
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
parentId TEXT,
conversationId TEXT,
role TEXT,
messageData TEXT,
createdAt TEXT
)`, (err) => {
if (err) {
return reject(err)
}
// 创建索引,加速查询
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_conversation ON ${this.tableName} (conversationId)`, (err) => {
if (err) {
return reject(err)
}
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_parent ON ${this.tableName} (parentId)`, (err) => {
if (err) {
return reject(err)
}
this.initialized = true
resolve()
})
})
})
})
})
}
/**
* 确保数据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* 计算文本的md5值
* @param {string} text
* @returns {string}
*/
_getMd5 (text) {
return crypto.createHash('md5').update(text).digest('hex')
}
/**
* 是否为base64编码的图片
* @param {string} str
* @returns {boolean}
*/
_isBase64Image (str) {
if (!str || typeof str !== 'string') {
return false
}
// 处理带前缀的 base64 格式
if (str.startsWith('data:image/')) {
return true
}
// 处理纯 base64 字符串
// base64 编码只会包含字母、数字、+、/,以及末尾可能有 = 或 == 用于填充
return /^[A-Za-z0-9+/]+={0,2}$/.test(str)
}
/**
* 从base64提取图片的mime类型或使用默认类型
* @param {string} base64
* @param {string} defaultMimeType 默认 MIME 类型
* @returns {string}
*/
_getMimeTypeFromBase64 (base64, defaultMimeType = 'image/jpeg') {
if (base64 && base64.startsWith('data:image/')) {
const match = base64.match(/^data:(image\/[a-zA-Z+]+);base64,/)
if (match) {
return match[1]
}
}
return defaultMimeType // 对于纯 base64 字符串,使用默认类型
}
/**
* 获取图片扩展名
* @param {string} mimeType
* @returns {string}
*/
_getExtensionFromMimeType (mimeType) {
const map = {
'image/jpeg': '.jpg',
'image/png': '.png',
'image/gif': '.gif',
'image/webp': '.webp',
'image/svg+xml': '.svg'
}
return map[mimeType] || '.png'
}
/**
* 处理消息中的图片内容将base64图片保存到本地文件
* @param {object} message
* @returns {object} 处理后的消息对象
*/
_processMessageImages (message) {
if (!message.content || !Array.isArray(message.content)) {
return message
}
// 深拷贝避免修改原对象
const processedMessage = JSON.parse(JSON.stringify(message))
processedMessage.content = processedMessage.content.map(item => {
if (item.type === 'image' && item.image) {
// 检查是否是base64图片数据
if (this._isBase64Image(item.image)) {
let base64Data = item.image
let mimeType = item.mimeType || 'image/jpeg' // 使用项目指定的 MIME 类型或默认值
// 如果是data:image格式提取纯base64部分
if (base64Data.startsWith('data:')) {
const parts = base64Data.split(',')
if (parts.length > 1) {
base64Data = parts[1]
// 更新 MIME 类型
mimeType = this._getMimeTypeFromBase64(item.image, mimeType)
}
}
try {
// 计算MD5
const md5 = this._getMd5(base64Data)
const ext = this._getExtensionFromMimeType(mimeType)
const filePath = path.join(this.imagesDir, `${md5}${ext}`)
// 如果文件不存在,则保存
if (!fs.existsSync(filePath)) {
fs.writeFileSync(filePath, Buffer.from(base64Data, 'base64'))
}
// 替换为引用格式: $image:md5:ext
item.image = `$image:${md5}:${ext}`
item._type = mimeType // 保存原始类型
} catch (error) {
console.error('保存图片失败:', error)
}
}
}
return item
})
return processedMessage
}
/**
* 恢复消息中的图片引用转换回base64
* @param {object} message
* @returns {object} 处理后的消息对象
*/
_restoreMessageImages (message) {
if (!message || !message.content || !Array.isArray(message.content)) {
return message
}
// 深拷贝避免修改原对象
const restoredMessage = JSON.parse(JSON.stringify(message))
// 标记是否需要添加[图片]文本
let needImageText = true
let hasRemovedImage = false
restoredMessage.content = restoredMessage.content.filter((item, index) => {
if (item.type === 'image' && item.image && typeof item.image === 'string') {
// 检查是否是图片引用格式
const match = item.image.match(/^\$image:([a-f0-9]+):(\.[a-z]+)$/)
if (match) {
// eslint-disable-next-line no-unused-vars
const [_, md5, ext] = match
const filePath = path.join(this.imagesDir, `${md5}${ext}`)
// 检查文件是否存在
if (fs.existsSync(filePath)) {
try {
// 读取文件并转换为base64
const imageBuffer = fs.readFileSync(filePath)
item.image = imageBuffer.toString('base64')
return true
} catch (error) {
console.error('读取图片文件失败:', filePath, error)
hasRemovedImage = true
return false
}
} else {
// 文件不存在删除这个image元素
hasRemovedImage = true
return false
}
}
}
if (item.type === 'text') {
needImageText = false
}
return true
})
// 如果移除了图片且没有文本内容,添加[图片]提示
if (hasRemovedImage) {
if (restoredMessage.content.length === 0) {
restoredMessage.content.push({
type: 'text',
text: '[图片]'
})
} else if (needImageText) {
// 查找第一个文本元素
const textIndex = restoredMessage.content.findIndex(item => item.type === 'text')
if (textIndex !== -1) {
restoredMessage.content[textIndex].text = `[图片] ${restoredMessage.content[textIndex].text}`
} else {
// 如果没有文本元素,添加一个
restoredMessage.content.unshift({
type: 'text',
text: '[图片]'
})
}
}
}
return restoredMessage
}
/**
* 将消息对象转换为数据库记录
* @param {import('chaite').HistoryMessage} message
* @param {string} conversationId
* @returns {Object} 数据库记录
*/
_messageToRecord (message, conversationId) {
// 处理图片将base64图片保存到本地文件
const processedMessage = this._processMessageImages(message)
// 将 content 和 toolCalls 等转为 JSON
const { id, parentId, role } = processedMessage
const messageData = JSON.stringify(processedMessage)
return {
id: id || '',
parentId: parentId || null,
conversationId: conversationId || '',
role: role || '',
messageData,
createdAt: new Date().toISOString()
}
}
/**
* 将数据库记录转换为消息对象
* @param {Object} record 数据库记录
* @returns {import('chaite').HistoryMessage} 消息对象
*/
_recordToMessage (record) {
if (!record) return null
try {
// 解析存储的消息数据
const message = JSON.parse(record.messageData)
// 恢复图片引用为base64
return this._restoreMessageImages(message)
} catch (e) {
// 解析失败,尝试构造最小结构
return {
id: record.id,
parentId: record.parentId,
role: record.role,
conversationId: record.conversationId,
content: []
}
}
}
/**
* 保存历史消息
* @param {import('chaite').HistoryMessage} message 消息对象
* @param {string} conversationId 会话ID
* @returns {Promise<void>}
*/
async saveHistory (message, conversationId) {
await this.ensureInitialized()
const record = this._messageToRecord(message, conversationId)
return new Promise((resolve, reject) => {
// 检查消息是否已存在
if (message.id) {
this.db.get(`SELECT id FROM ${this.tableName} WHERE id = ?`, [message.id], (err, row) => {
if (err) {
return reject(err)
}
if (row) {
// 消息已存在,更新
const fields = Object.keys(record)
const updates = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => record[field])
this.db.run(`UPDATE ${this.tableName} SET ${updates} WHERE id = ?`, [...values, message.id], (err) => {
if (err) {
return reject(err)
}
resolve()
})
} else {
// 消息不存在,插入
this._insertMessage(record, resolve, reject)
}
})
} else {
// 没有ID直接插入
this._insertMessage(record, resolve, reject)
}
})
}
/**
* 内部方法插入消息记录
* @private
*/
_insertMessage (record, resolve, reject) {
const fields = Object.keys(record)
const placeholders = fields.map(() => '?').join(', ')
const values = fields.map(field => record[field])
this.db.run(
`INSERT INTO ${this.tableName} (${fields.join(', ')}) VALUES (${placeholders})`,
values,
function (err) {
if (err) {
return reject(err)
}
resolve()
}
)
}
/**
* 获取历史消息
* @param {string} messageId 消息ID
* @param {string} conversationId 会话ID
* @returns {Promise<import('chaite').HistoryMessage[]>}
*/
async getHistory (messageId, conversationId) {
await this.ensureInitialized()
if (messageId) {
return this._getMessageChain(messageId)
} else if (conversationId) {
return this._getConversationMessages(conversationId)
}
return []
}
/**
* 获取消息链从指定消息追溯到根消息
* @private
*/
async _getMessageChain (messageId) {
return new Promise((resolve, reject) => {
const messages = []
const getMessageById = (id) => {
if (!id) {
resolve(messages)
return
}
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [id], (err, row) => {
if (err) {
return reject(err)
}
if (!row) {
resolve(messages)
return
}
const message = this._recordToMessage(row)
messages.unshift(message) // 将消息添加到数组开头
getMessageById(row.parentId) // 递归获取父消息
})
}
getMessageById(messageId)
})
}
/**
* 获取会话中的所有消息
* @private
*/
async _getConversationMessages (conversationId) {
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName} WHERE conversationId = ? ORDER BY createdAt`, [conversationId], (err, rows) => {
if (err) {
return reject(err)
}
const messages = rows.map(row => this._recordToMessage(row)).filter(Boolean)
resolve(messages)
})
})
}
/**
* 删除会话
* @param {string} conversationId 会话ID
* @returns {Promise<void>}
*/
async deleteConversation (conversationId) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE conversationId = ?`, [conversationId], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 获取单条历史消息
* @param {string} messageId 消息ID
* @param {string} conversationId 会话ID
* @returns {Promise<import('chaite').HistoryMessage | null>}
*/
async getOneHistory (messageId, conversationId) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
const conditions = []
const params = []
if (messageId) {
conditions.push('id = ?')
params.push(messageId)
}
if (conversationId) {
conditions.push('conversationId = ?')
params.push(conversationId)
}
if (conditions.length === 0) {
return resolve(null)
}
const whereClause = conditions.join(' AND ')
this.db.get(`SELECT * FROM ${this.tableName} WHERE ${whereClause} LIMIT 1`, params, (err, row) => {
if (err) {
return reject(err)
}
resolve(this._recordToMessage(row))
})
})
}
/**
* 清理未引用的图片文件
* @returns {Promise<{deleted: number, total: number}>}
*/
async cleanupUnusedImages () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
// 获取所有消息数据
this.db.all(`SELECT messageData FROM ${this.tableName}`, async (err, rows) => {
if (err) {
return reject(err)
}
try {
// 从数据库中提取所有图片引用
const usedImageRefs = new Set()
rows.forEach(row => {
try {
const message = JSON.parse(row.messageData)
if (message.content && Array.isArray(message.content)) {
message.content.forEach(item => {
if (item.type === 'image' && typeof item.image === 'string') {
const match = item.image.match(/^\$image:([a-f0-9]+):(\.[a-z]+)$/)
if (match) {
usedImageRefs.add(`${match[1]}${match[2]}`)
}
}
})
}
} catch (e) {
// 忽略解析错误
}
})
// 获取图片目录中的所有文件
const files = fs.readdirSync(this.imagesDir)
// 删除未引用的图片
let deletedCount = 0
for (const file of files) {
if (!usedImageRefs.has(file)) {
fs.unlinkSync(path.join(this.imagesDir, file))
deletedCount++
}
}
resolve({
deleted: deletedCount,
total: files.length
})
} catch (error) {
reject(error)
}
})
})
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,153 @@
import path from 'path'
import { dataDir } from '../../../../utils/common.js'
import { SQLiteChannelStorage } from './channel_storage.js'
import { LowDBChannelStorage } from '../lowdb/channel_storage.js'
import { SQLiteChatPresetStorage } from './chat_preset_storage.js'
import { LowDBChatPresetsStorage } from '../lowdb/chat_preset_storage.js'
import { SQLiteToolsStorage } from './tools_storage.js'
import { LowDBToolsStorage } from '../lowdb/tools_storage.js'
import { SQLiteProcessorsStorage } from './processors_storage.js'
import { LowDBProcessorsStorage } from '../lowdb/processors_storage.js'
import { SQLiteUserStateStorage } from './user_state_storage.js'
import { LowDBUserStateStorage } from '../lowdb/user_state_storage.js'
import fs from 'fs'
export async function checkMigrate () {
logger.debug('检查是否需要从 LowDB 迁移数据到 SQLite...')
try {
// 导入所需的模块
const { default: ChatGPTStorage } = await import('../lowdb/storage.js')
await ChatGPTStorage.init()
const { ChatGPTHistoryStorage } = await import('../lowdb/storage.js')
await ChatGPTHistoryStorage.init()
const dbPath = path.join(dataDir, 'data.db')
// 定义要检查的存储对
const storagePairs = [
{
name: '渠道',
lowdbStorageClass: LowDBChannelStorage,
sqliteStorageClass: SQLiteChannelStorage,
collection: 'channel'
},
{
name: '预设',
lowdbStorageClass: LowDBChatPresetsStorage,
sqliteStorageClass: SQLiteChatPresetStorage,
collection: 'chat_presets'
},
{
name: '工具',
lowdbStorageClass: LowDBToolsStorage,
sqliteStorageClass: SQLiteToolsStorage,
collection: 'tools'
},
{
name: '处理器',
lowdbStorageClass: LowDBProcessorsStorage,
sqliteStorageClass: SQLiteProcessorsStorage,
collection: 'processors'
},
{
name: '用户状态',
lowdbStorageClass: LowDBUserStateStorage,
sqliteStorageClass: SQLiteUserStateStorage,
collection: 'userState',
isSpecial: true
}
]
// 检查是否有任何数据需要迁移
const needMigrate = await Promise.all(storagePairs.map(async pair => {
if (pair.isSpecial) {
// 用户状态特殊处理
const collection = ChatGPTStorage.collection(pair.collection)
const items = await collection.findAll()
return items.length > 0
} else {
// 标准集合处理
const collection = ChatGPTStorage.collection(pair.collection)
const items = await collection.findAll()
return items.length > 0
}
})).then(results => results.some(result => result))
if (!needMigrate) {
logger.debug('LowDB 存储为空,无需迁移')
return
}
// 检查 SQLite 中是否已有数据
const testStorage = new SQLiteChannelStorage(dbPath)
await testStorage.initialize()
const channels = await testStorage.listItems()
if (channels.length > 0) {
logger.debug('SQLite 存储已有数据,跳过迁移')
await testStorage.close()
return
}
await testStorage.close()
logger.info('开始从 LowDB 迁移数据到 SQLite...')
// 迁移每种数据
for (const pair of storagePairs) {
const collection = ChatGPTStorage.collection(pair.collection)
const items = await collection.findAll()
if (items.length > 0) {
logger.info(`迁移${pair.name}数据...`)
// eslint-disable-next-line new-cap
const sqliteStorage = new pair.sqliteStorageClass(dbPath)
await sqliteStorage.initialize()
for (const item of items) {
await sqliteStorage.setItem(item.id, item)
}
logger.info(`迁移了 ${items.length}${pair.name}`)
await sqliteStorage.close()
}
}
// 迁移完成后,备份并清空 LowDB 数据
const backupDir = path.join(dataDir, 'backup')
if (!fs.existsSync(backupDir)) {
fs.mkdirSync(backupDir, { recursive: true })
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
// 备份并清空<E6B885><E7A9BA>数据
if (fs.existsSync(ChatGPTStorage.filePath)) {
fs.copyFileSync(
ChatGPTStorage.filePath,
path.join(backupDir, `storage-backup-${timestamp}.json`)
)
// 清空数据但保留文件结构
for (const pair of storagePairs) {
if (!pair.collection) continue
await ChatGPTStorage.collection(pair.collection).deleteAll()
}
}
// 备份并清空历史数据
if (fs.existsSync(ChatGPTHistoryStorage.filePath)) {
fs.copyFileSync(
ChatGPTHistoryStorage.filePath,
path.join(backupDir, `history-backup-${timestamp}.json`)
)
// 清空历史数据
for (const collectionName of ChatGPTHistoryStorage.listCollections()) {
await ChatGPTHistoryStorage.collection(collectionName).deleteAll()
}
}
logger.debug(`迁移完成,原数据已备份至 ${backupDir} 目录`)
} catch (error) {
logger.error('数据迁移过程中发生错误:', error)
}
}

View file

@ -0,0 +1,430 @@
import { ChaiteStorage, ProcessorDTO } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
/**
* @extends {ChaiteStorage<import('chaite').ProcessorDTO>}
*/
export class SQLiteProcessorsStorage extends ChaiteStorage {
getName () {
return 'SQLiteProcessorsStorage'
}
/**
*
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'processors'
}
/**
* 初始化数据库连接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保<E7A1AE><E4BF9D>录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建处理器表,将主要属性分列存储
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
type TEXT NOT NULL,
code TEXT,
cloudId INTEGER,
createdAt TEXT,
updatedAt TEXT,
md5 TEXT,
embedded INTEGER DEFAULT 0,
uploader TEXT,
extraData TEXT
)`, (err) => {
if (err) {
return reject(err)
}
// 创建索引
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_type ON ${this.tableName} (type)`, (err) => {
if (err) {
return reject(err)
}
this.initialized = true
resolve()
})
})
})
})
}
/**
* 确保数据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* ProcessorDTO 对象转换为数据库记录
* @param {import('chaite').ProcessorDTO} processor
* @returns {Object} 数据库记录
*/
_processorToRecord (processor) {
// 提取主要字段
const {
id, name, description, type, code, cloudId,
createdAt, updatedAt, md5, embedded, uploader, ...rest
} = processor
return {
id: id || '',
name: name || '',
description: description || '',
type: type || '', // 'pre' 或 'post'
code: code || '',
cloudId: cloudId || null,
createdAt: createdAt || '',
updatedAt: updatedAt || '',
md5: md5 || '',
embedded: embedded ? 1 : 0,
uploader: uploader ? JSON.stringify(uploader) : null,
extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null
}
}
/**
* 将数据库记录转换为 ProcessorDTO 对象
* @param {Object} record 数据库记录
* @returns {import('chaite').ProcessorDTO} ProcessorDTO 对象
*/
_recordToProcessor (record) {
if (!record) return null
// 解析 JSON 字段
let uploader = null
try {
if (record.uploader) {
uploader = JSON.parse(record.uploader)
}
} catch (e) {
// 解析错误,使用 null
}
let extraData = {}
try {
if (record.extraData) {
extraData = JSON.parse(record.extraData)
}
} catch (e) {
// 解析错误,使用空对象
}
// 构造 ProcessorDTO 对象
const processorData = {
id: record.id,
name: record.name,
description: record.description,
type: record.type, // 'pre' 或 'post'
code: record.code,
cloudId: record.cloudId,
createdAt: record.createdAt,
updatedAt: record.updatedAt,
md5: record.md5,
embedded: Boolean(record.embedded),
uploader,
...extraData
}
return new ProcessorDTO(processorData)
}
/**
* 获取单个处理器
* @param {string} key 处理器ID
* @returns {Promise<import('chaite').ProcessorDTO>}
*/
async getItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [key], (err, row) => {
if (err) {
return reject(err)
}
const processor = this._recordToProcessor(row)
resolve(processor)
})
})
}
/**
* 保存处理器
* @param {string} id 处理器ID
* @param {import('chaite').ProcessorDTO} processor 处理器对象
* @returns {Promise<string>}
*/
async setItem (id, processor) {
await this.ensureInitialized()
// 转换为数据库记录
const record = this._processorToRecord(processor)
record.id = id // 确保ID是指定的ID
// 构建插入或更新SQL
const fields = Object.keys(record)
const placeholders = fields.map(() => '?').join(', ')
const updates = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => record[field])
const duplicateValues = [...values] // 用于ON CONFLICT时的更新
return new Promise((resolve, reject) => {
this.db.run(
`INSERT INTO ${this.tableName} (${fields.join(', ')})
VALUES (${placeholders})
ON CONFLICT(id) DO UPDATE SET ${updates}`,
[...values, ...duplicateValues],
function (err) {
if (err) {
return reject(err)
}
resolve(id)
}
)
})
}
/**
* 删除处<EFBFBD><EFBFBD><EFBFBD>
* @param {string} key 处理器ID
* @returns {Promise<void>}
*/
async removeItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE id = ?`, [key], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 查询所有处理器
* @returns {Promise<import('chaite').ProcessorDTO[]>}
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const processors = rows.map(row => this._recordToProcessor(row)).filter(Boolean)
resolve(processors)
})
})
}
/**
* 根据条件筛选处理器
* @param {Record<string, unknown>} filter 筛选条件
* @returns {Promise<import('chaite').ProcessorDTO[]>}
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
// 如果没有筛选条件,返回所有
if (!filter || Object.keys(filter).length === 0) {
return this.listItems()
}
// 尝试使用SQL字段直接过滤
const directFields = ['id', 'name', 'description', 'type', 'cloudId']
const sqlFilters = []
const sqlParams = []
const extraFilters = {}
let hasExtraFilters = false
// 区分数据库字段和额外字段
for (const key in filter) {
const value = filter[key]
// 如果是直接支持的字段构建SQL条件
if (directFields.includes(key)) {
sqlFilters.push(`${key} = ?`)
sqlParams.push(value)
} else if (key === 'embedded') {
// embedded 字段需要特殊处理为 0/1
sqlFilters.push('embedded = ?')
sqlParams.push(value ? 1 : 0)
} else {
// 其他字段需要在结果中进一步过滤
extraFilters[key] = value
hasExtraFilters = true
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let processors = rows.map(row => this._recordToProcessor(row)).filter(Boolean)
// 如果有需要在内存中过滤的额外字段
if (hasExtraFilters) {
processors = processors.filter(processor => {
for (const key in extraFilters) {
if (processor[key] !== extraFilters[key]) {
return false
}
}
return true
})
}
resolve(processors)
})
})
}
/**
* 根据IN条<EFBFBD><EFBFBD>筛选处理器
* @param {Array<{ field: string; values: unknown[]; }>} query
* @returns {Promise<import('chaite').ProcessorDTO[]>}
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
// 如果没有查询条件,返回所有
if (!query || query.length === 0) {
return this.listItems()
}
// 尝试使用SQL IN子句来优化查询
const directFields = ['id', 'name', 'description', 'type', 'cloudId']
const sqlFilters = []
const sqlParams = []
const extraQueries = []
// 处理每个查询条件
for (const { field, values } of query) {
if (values.length === 0) continue
// 如果是直接支持的字段使用SQL IN子句
if (directFields.includes(field)) {
const placeholders = values.map(() => '?').join(', ')
sqlFilters.push(`${field} IN (${placeholders})`)
sqlParams.push(...values)
} else if (field === 'embedded') {
// embedded 字段需要特殊处理
const boolValues = values.map(v => v ? 1 : 0)
const placeholders = boolValues.map(() => '?').join(', ')
sqlFilters.push(`embedded IN (${placeholders})`)
sqlParams.push(...boolValues)
} else {
// 其他字段在内存中过滤
extraQueries.push({ field, values })
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let processors = rows.map(row => this._recordToProcessor(row)).filter(Boolean)
// 如果有需要在内存中过滤的条件
if (extraQueries.length > 0) {
processors = processors.filter(processor => {
for (const { field, values } of extraQueries) {
if (!values.includes(processor[field])) {
return false
}
}
return true
})
}
resolve(processors)
})
})
}
/**
* 清空表中所有数据
* @returns {Promise<void>}
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,347 @@
import { ChaiteStorage } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
/**
* @extends {ChaiteStorage<import('chaite').ToolsGroupDTO>}
*/
export class SQLiteToolsGroupStorage extends ChaiteStorage {
getName () {
return 'SQLiteToolsGroupStorage'
}
/**
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'tools_groups'
}
/**
* 初始化数据库连接和表结构
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) return reject(err)
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
tools TEXT NOT NULL,
createdAt TEXT,
updatedAt TEXT
)`, (err) => {
if (err) return reject(err)
this.db.run(`CREATE INDEX IF NOT EXISTS idx_tools_groups_name ON ${this.tableName} (name)`, (err) => {
if (err) return reject(err)
this.initialized = true
resolve()
})
})
})
})
}
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* 获取工具组
* @param {string} key 工具组ID
*/
async getItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [key], (err, row) => {
if (err) return reject(err)
if (!row) return resolve(null)
try {
const toolsGroup = {
...row,
tools: JSON.parse(row.tools)
}
resolve(toolsGroup)
} catch (e) {
console.error(`解析工具组数据错误: ${key}`, e)
resolve({
...row,
tools: []
})
}
})
})
}
/**
* 保存工具组
* @param {string} id 工具组ID
* @param {Object} data 工具组数据
*/
async setItem (id, data) {
await this.ensureInitialized()
// 提取工具组数据
const { name, description, tools } = data
const updatedAt = Date.now()
// 将工具列表序列化为JSON字符串
const toolsJson = JSON.stringify(tools || [])
return new Promise((resolve, reject) => {
// 检查工具组是否已存在
this.db.get(`SELECT id FROM ${this.tableName} WHERE id = ?`, [id], (err, row) => {
if (err) {
return reject(err)
}
if (row) {
// 更新现有工具组
this.db.run(
`UPDATE ${this.tableName} SET name = ?, description = ?, tools = ?, updatedAt = ? WHERE id = ?`,
[name, description, toolsJson, updatedAt, id],
(err) => {
if (err) {
return reject(err)
}
resolve(id)
}
)
} else {
// 插入新工具组
this.db.run(
`INSERT INTO ${this.tableName} (id, name, description, tools, updatedAt) VALUES (?, ?, ?, ?, ?)`,
[id, name, description, toolsJson, updatedAt],
(err) => {
if (err) {
return reject(err)
}
resolve(id)
}
)
}
})
})
}
/**
* 删除工具组
* @param {string} key 工具组ID
*/
async removeItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE id = ?`, [key], function (err) {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 获取所有工具组
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const toolsGroups = rows.map(row => {
try {
return {
...row,
tools: JSON.parse(row.tools)
}
} catch (e) {
console.error(`解析工具组数据错误: ${row.id}`, e)
return {
...row,
tools: []
}
}
})
resolve(toolsGroups)
})
})
}
/**
* 根据条件筛选工具组
* @param {Record<string, unknown>} filter 筛选条件
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
if (!filter || Object.keys(filter).length === 0) {
return this.listItems()
}
const directFields = ['id', 'name', 'description']
const conditions = []
const params = []
for (const key in filter) {
if (directFields.includes(key)) {
conditions.push(`${key} = ?`)
params.push(filter[key])
}
}
const sql = conditions.length > 0
? `SELECT * FROM ${this.tableName} WHERE ${conditions.join(' AND ')}`
: `SELECT * FROM ${this.tableName}`
return new Promise((resolve, reject) => {
this.db.all(sql, params, (err, rows) => {
if (err) return reject(err)
const toolsGroups = rows.map(row => {
try {
const group = {
...row,
tools: JSON.parse(row.tools || '[]')
}
// 过滤非直接字段
for (const key in filter) {
if (!directFields.includes(key) && group[key] !== filter[key]) {
return null
}
}
return group
} catch (e) {
console.error(`解析工具组数据错误: ${row.id}`, e)
return null
}
}).filter(Boolean)
resolve(toolsGroups)
})
})
}
/**
* 根据IN条件筛选工具组
* @param {Array<{field: string, values: unknown[]}>} query IN查询条件
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
if (!query || query.length === 0) {
return this.listItems()
}
const directFields = ['id', 'name', 'description']
const conditions = []
const params = []
const memoryQueries = []
for (const item of query) {
if (directFields.includes(item.field) && Array.isArray(item.values) && item.values.length > 0) {
const placeholders = item.values.map(() => '?').join(',')
conditions.push(`${item.field} IN (${placeholders})`)
params.push(...item.values)
} else if (item.values.length > 0) {
memoryQueries.push(item)
}
}
const sql = conditions.length > 0
? `SELECT * FROM ${this.tableName} WHERE ${conditions.join(' AND ')}`
: `SELECT * FROM ${this.tableName}`
return new Promise((resolve, reject) => {
this.db.all(sql, params, (err, rows) => {
if (err) return reject(err)
let toolsGroups = rows.map(row => {
try {
return {
...row,
tools: JSON.parse(row.tools || '[]')
}
} catch (e) {
console.error(`解析工具组数据错误: ${row.id}`, e)
return null
}
}).filter(Boolean)
// 内存中过滤其它字段
if (memoryQueries.length > 0) {
toolsGroups = toolsGroups.filter(group => {
for (const { field, values } of memoryQueries) {
if (!values.includes(group[field])) {
return false
}
}
return true
})
}
resolve(toolsGroups)
})
})
}
/**
* 清空所有工具组
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) return reject(err)
resolve()
})
})
}
/**
* 关闭数据库连接
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,455 @@
import { ChaiteStorage, ToolDTO } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
/**
* @extends {ChaiteStorage<import('chaite').ToolDTO>}
*/
export class SQLiteToolsStorage extends ChaiteStorage {
getName () {
return 'SQLiteToolsStorage'
}
/**
*
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'tools'
}
/**
* 初始化数据库连接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保目录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建工具表,将主要属性分列存储
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
modelType TEXT,
code TEXT,
cloudId INTEGER,
embedded INTEGER,
uploader TEXT,
createdAt TEXT,
updatedAt TEXT,
md5 TEXT,
status TEXT,
permission TEXT,
extraData TEXT -- 存储其他额外数据的JSON
)`, (err) => {
if (err) {
reject(err)
} else {
// 创建索引以提高查询性能
this.db.run(`CREATE INDEX IF NOT EXISTS idx_tools_name ON ${this.tableName} (name)`, (err) => {
if (err) {
reject(err)
} else {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_tools_status ON ${this.tableName} (status)`, (err) => {
if (err) {
reject(err)
} else {
this.db.run(`CREATE INDEX IF NOT EXISTS idx_tools_permission ON ${this.tableName} (permission)`, (err) => {
if (err) {
reject(err)
} else {
this.initialized = true
resolve()
}
})
}
})
}
})
}
})
})
})
}
/**
* 确保数据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* ToolDTO 对象转换为数据库记录
* @param {import('chaite').ToolDTO} tool
* @returns {Object} 数据库记录
*/
_toolToRecord (tool) {
// 提取主要字段剩余的放入extraData
const {
id, name, description, modelType, code, cloudId,
embedded, uploader, createdAt, updatedAt, md5,
status, permission, ...rest
} = tool
// 序列化上传者对象
const uploaderStr = uploader ? JSON.stringify(uploader) : null
return {
id: id || '',
name: name || '',
description: description || '',
modelType: modelType || '',
code: code || null,
cloudId: cloudId || null,
embedded: embedded ? 1 : 0,
uploader: uploaderStr,
createdAt: createdAt || '',
updatedAt: updatedAt || '',
md5: md5 || '',
status: status || 'enabled',
permission: permission || 'public',
extraData: Object.keys(rest).length > 0 ? JSON.stringify(rest) : null
}
}
/**
* 将数据库记录转换为 ToolDTO 对象
* @param {Object} record 数据库记录
* @returns {import('chaite').ToolDTO} ToolDTO对象
*/
_recordToTool (record) {
// 若记录不存在则返回null
if (!record) return null
// 解析上传者
let uploader = null
try {
if (record.uploader) {
uploader = JSON.parse(record.uploader)
}
} catch (e) {
// 解析错误使用null
}
// 解析额外数据
let extraData = {}
try {
if (record.extraData) {
extraData = JSON.parse(record.extraData)
}
} catch (e) {
// 解析错误,使用空对象
}
// 构造基本对象
const toolData = {
id: record.id,
name: record.name,
description: record.description,
modelType: record.modelType,
code: record.code,
cloudId: record.cloudId,
embedded: Boolean(record.embedded),
uploader,
createdAt: record.createdAt,
updatedAt: record.updatedAt,
md5: record.md5,
status: record.status,
permission: record.permission,
...extraData
}
return new ToolDTO(toolData)
}
/**
* 获取单个工具
* @param {string} key 工具ID
* @returns {Promise<import('chaite').ToolDTO>}
*/
async getItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE id = ?`, [key], (err, row) => {
if (err) {
return reject(err)
}
const tool = this._recordToTool(row)
resolve(tool)
})
})
}
/**
* 保存工具
* @param {string} id 工具ID
* @param {import('chaite').ToolDTO} tool 工具对象
* @returns {Promise<string>}
*/
async setItem (id, tool) {
await this.ensureInitialized()
// 转换为数据库记录
const record = this._toolToRecord(tool)
record.id = id // 确保ID是指定的ID
// 构建插入或更新SQL
const fields = Object.keys(record)
const placeholders = fields.map(() => '?').join(', ')
const updates = fields.map(field => `${field} = ?`).join(', ')
const values = fields.map(field => record[field])
const duplicateValues = [...values] // 用于ON CONFLICT时的更新
return new Promise((resolve, reject) => {
this.db.run(
`INSERT INTO ${this.tableName} (${fields.join(', ')})
VALUES (${placeholders})
ON CONFLICT(id) DO UPDATE SET ${updates}`,
[...values, ...duplicateValues],
function (err) {
if (err) {
return reject(err)
}
resolve(id)
}
)
})
}
/**
* 删除工具
* @param {string} key 工具ID
* @returns {Promise<void>}
*/
async removeItem (key) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE id = ?`, [key], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 查询所有工具
* @returns {Promise<import('chaite').ToolDTO[]>}
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const tools = rows.map(row => this._recordToTool(row)).filter(Boolean)
resolve(tools)
})
})
}
/**
* 根据条件筛选工具直接使用SQL查询避免全表扫描
* @param {Record<string, unknown>} filter 筛选条件
* @returns {Promise<import('chaite').ToolDTO[]>}
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
// 如果没有筛选条件,返回所有
if (!filter || Object.keys(filter).length === 0) {
return this.listItems()
}
// 尝试使用SQL字段直接过滤
const directFields = ['id', 'name', 'description', 'modelType', 'cloudId', 'status', 'permission']
const sqlFilters = []
const sqlParams = []
const extraFilters = {}
let hasExtraFilters = false
// 区分数据库字段和额外字段
for (const key in filter) {
const value = filter[key]
// 如果是直接支持的字段构建SQL条件
if (directFields.includes(key)) {
sqlFilters.push(`${key} = ?`)
sqlParams.push(value)
} else if (key === 'embedded') {
// embedded 字段需要特殊处理为 0/1
sqlFilters.push('embedded = ?')
sqlParams.push(value ? 1 : 0)
} else {
// 其他字段需要在结果中进一步过滤
extraFilters[key] = value
hasExtraFilters = true
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let tools = rows.map(row => this._recordToTool(row)).filter(Boolean)
// 如果有需要在内存中过滤的额外字段
if (hasExtraFilters) {
tools = tools.filter(tool => {
for (const key in extraFilters) {
if (tool[key] !== extraFilters[key]) {
return false
}
}
return true
})
}
resolve(tools)
})
})
}
/**
* 根据IN条件筛选工具
* @param {Array<{ field: string; values: unknown[]; }>} query
* @returns {Promise<import('chaite').ToolDTO[]>}
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
// 如果没有查询条件,返回所有
if (!query || query.length === 0) {
return this.listItems()
}
// 尝试使用SQL IN子句来优化查询
const directFields = ['id', 'name', 'description', 'modelType', 'cloudId', 'status', 'permission']
const sqlFilters = []
const sqlParams = []
const extraQueries = []
// 处理每个查询条件
for (const { field, values } of query) {
if (values.length === 0) continue
// 如果是直接支持的字段使用SQL IN子句
if (directFields.includes(field)) {
const placeholders = values.map(() => '?').join(', ')
sqlFilters.push(`${field} IN (${placeholders})`)
sqlParams.push(...values)
// embedded 字段需要特殊处理
} else if (field === 'embedded') {
const boolValues = values.map(v => v ? 1 : 0)
const placeholders = boolValues.map(() => '?').join(', ')
sqlFilters.push(`embedded IN (${placeholders})`)
sqlParams.push(...boolValues)
} else {
// 其他字段在内存中过滤
extraQueries.push({ field, values })
}
}
// 构建SQL查询
let sql = `SELECT * FROM ${this.tableName}`
if (sqlFilters.length > 0) {
sql += ` WHERE ${sqlFilters.join(' AND ')}`
}
return new Promise((resolve, reject) => {
this.db.all(sql, sqlParams, (err, rows) => {
if (err) {
return reject(err)
}
let tools = rows.map(row => this._recordToTool(row)).filter(Boolean)
// 如果有需要在内存中过滤的条件
if (extraQueries.length > 0) {
tools = tools.filter(tool => {
for (const { field, values } of extraQueries) {
if (!values.includes(tool[field])) {
return false
}
}
return true
})
}
resolve(tools)
})
})
}
/**
* 清空表中所有数据
* @returns {Promise<void>}
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) return Promise.resolve()
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -0,0 +1,388 @@
import { ChaiteStorage } from 'chaite'
import sqlite3 from 'sqlite3'
import path from 'path'
import fs from 'fs'
import crypto from 'node:crypto'
/**
* 基于SQLite的用户状态存储实现
* @extends {ChaiteStorage<import('chaite').UserState>}
*/
export class SQLiteUserStateStorage extends ChaiteStorage {
/**
* 构造函数
* @param {string} dbPath 数据库文件路径
*/
constructor (dbPath) {
super()
this.dbPath = dbPath
this.db = null
this.initialized = false
this.tableName = 'user_states'
}
/**
* 初始化数据库<EFBFBD><EFBFBD>接和表结构
* @returns {Promise<void>}
*/
async initialize () {
if (this.initialized) return
return new Promise((resolve, reject) => {
// 确保目录存在
const dir = path.dirname(this.dbPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
this.db = new sqlite3.Database(this.dbPath, async (err) => {
if (err) {
return reject(err)
}
// 创建用户状态表
this.db.run(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
id TEXT PRIMARY KEY,
userId TEXT NOT NULL,
nickname TEXT,
card TEXT,
conversations TEXT NOT NULL,
settings TEXT NOT NULL,
current TEXT NOT NULL,
updatedAt INTEGER
)`, (err) => {
if (err) {
return reject(err)
}
// 创建索引以加快查询
this.db.run(`CREATE INDEX IF NOT EXISTS idx_${this.tableName}_userId ON ${this.tableName} (userId)`, (err) => {
if (err) {
return reject(err)
}
this.initialized = true
resolve()
})
})
})
})
}
/**
* 确保数据库已初始化
*/
async ensureInitialized () {
if (!this.initialized) {
await this.initialize()
}
}
/**
* 获取用户状态
* @param {string} userId 用户ID
* @returns {Promise<import('chaite').UserState|null>}
*/
async getItem (userId) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.get(`SELECT * FROM ${this.tableName} WHERE userId = ?`, [userId], (err, row) => {
if (err) {
return reject(err)
}
if (!row) {
return resolve(null)
}
try {
const userState = {
userId: row.userId,
nickname: row.nickname,
card: row.card,
conversations: JSON.parse(row.conversations),
settings: JSON.parse(row.settings),
current: JSON.parse(row.current)
}
resolve(userState)
} catch (e) {
console.error(`解析用户状态数据错误: ${userId}`, e)
resolve(null)
}
})
})
}
/**
* 保存用户状态
* @param {string} userId 用户ID
* @param {import('chaite').UserState} userState 用户状态数据
* @returns {Promise<string>} 返回用户ID
*/
async setItem (userId, userState) {
await this.ensureInitialized()
// 提取用户状态数据
const { nickname, card, conversations, settings, current } = userState
const updatedAt = Date.now()
// 序列化数据
const conversationsJson = JSON.stringify(conversations || [])
const settingsJson = JSON.stringify(settings || {})
const currentJson = JSON.stringify(current || {})
return new Promise((resolve, reject) => {
// 检查用户状态是否已存在
this.db.get(`SELECT userId FROM ${this.tableName} WHERE userId = ?`, [userId], (err, row) => {
if (err) {
return reject(err)
}
if (row) {
// 更新现有用户状态
this.db.run(
`UPDATE ${this.tableName} SET
nickname = ?,
card = ?,
conversations = ?,
settings = ?,
current = ?,
updatedAt = ?
WHERE userId = ?`,
[nickname, card, conversationsJson, settingsJson, currentJson, updatedAt, userId],
(err) => {
if (err) {
return reject(err)
}
resolve(userId)
}
)
} else {
// 插入新用户状态
this.db.run(
`INSERT INTO ${this.tableName}
(id, userId, nickname, card, conversations, settings, current, updatedAt)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[crypto.randomUUID(), userId, nickname, card, conversationsJson, settingsJson, currentJson, updatedAt],
(err) => {
if (err) {
return reject(err)
}
resolve(userId)
}
)
}
})
})
}
/**
* 删除用户状态
* @param {string} userId 用户ID
* @returns {Promise<void>}
*/
async removeItem (userId) {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName} WHERE userId = ?`, [userId], (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 获取所有用户状态
* @returns {Promise<import('chaite').UserState[]>}
*/
async listItems () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName}`, (err, rows) => {
if (err) {
return reject(err)
}
const userStates = rows.map(row => {
try {
return {
userId: row.userId,
nickname: row.nickname,
card: row.card,
conversations: JSON.parse(row.conversations),
settings: JSON.parse(row.settings),
current: JSON.parse(row.current)
}
} catch (e) {
console.error(`解析用户状态数据错误: ${row.userId}`, e)
return null
}
}).filter(Boolean)
resolve(userStates)
})
})
}
/**
* 根据过滤条件查询用户状态
* @param {Record<string, unknown>} filter 过滤条件
* @returns {Promise<import('chaite').UserState[]>}
*/
async listItemsByEqFilter (filter) {
await this.ensureInitialized()
// 只支持userId、nickname、card的过滤
const supportedFilters = ['userId', 'nickname', 'card']
const conditions = []
const params = []
for (const key of supportedFilters) {
if (filter[key] !== undefined) {
conditions.push(`${key} = ?`)
params.push(filter[key])
}
}
if (conditions.length === 0) {
return this.listItems()
}
const whereClause = conditions.join(' AND ')
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName} WHERE ${whereClause}`, params, (err, rows) => {
if (err) {
return reject(err)
}
const userStates = rows.map(row => {
try {
return {
userId: row.userId,
nickname: row.nickname,
card: row.card,
conversations: JSON.parse(row.conversations),
settings: JSON.parse(row.settings),
current: JSON.parse(row.current)
}
} catch (e) {
console.error(`解析用户状态数据错误: ${row.userId}`, e)
return null
}
}).filter(Boolean)
resolve(userStates)
})
})
}
/**
* 根据IN查询条件查询用户状<EFBFBD><EFBFBD>
* @param {Array<{field: string, values: unknown[]}>} query IN查询条件
* @returns {Promise<import('chaite').UserState[]>}
*/
async listItemsByInQuery (query) {
await this.ensureInitialized()
if (!query || !query.length) {
return this.listItems()
}
// 只支持userId、nickname、card的过滤
const supportedFields = ['userId', 'nickname', 'card']
const conditions = []
const params = []
for (const item of query) {
if (supportedFields.includes(item.field) && Array.isArray(item.values) && item.values.length > 0) {
const placeholders = item.values.map(() => '?').join(',')
conditions.push(`${item.field} IN (${placeholders})`)
params.push(...item.values)
}
}
if (conditions.length === 0) {
return this.listItems()
}
const whereClause = conditions.join(' AND ')
return new Promise((resolve, reject) => {
this.db.all(`SELECT * FROM ${this.tableName} WHERE ${whereClause}`, params, (err, rows) => {
if (err) {
return reject(err)
}
const userStates = rows.map(row => {
try {
return {
userId: row.userId,
nickname: row.nickname,
card: row.card,
conversations: JSON.parse(row.conversations),
settings: JSON.parse(row.settings),
current: JSON.parse(row.current)
}
} catch (e) {
console.error(`解析用户状态数据错误: ${row.userId}`, e)
return null
}
}).filter(Boolean)
resolve(userStates)
})
})
}
/**
* 清空所有用户状态
* @returns {Promise<void>}
*/
async clear () {
await this.ensureInitialized()
return new Promise((resolve, reject) => {
this.db.run(`DELETE FROM ${this.tableName}`, (err) => {
if (err) {
return reject(err)
}
resolve()
})
})
}
/**
* 获取存储名称
* @returns {string}
*/
getName () {
return 'SQLiteUserStateStorage'
}
/**
* 关闭数据库连接
* @returns {Promise<void>}
*/
async close () {
if (!this.db) {
return Promise.resolve()
}
return new Promise((resolve, reject) => {
this.db.close(err => {
if (err) {
reject(err)
} else {
this.initialized = false
this.db = null
resolve()
}
})
})
}
}

View file

@ -1,6 +1,6 @@
{ {
"name": "chatgpt-plugin", "name": "chatgpt-plugin",
"version": "3.0.0-alpha.2", "version": "3.0.0-beta.1",
"type": "module", "type": "module",
"author": "ikechan8370", "author": "ikechan8370",
"dependencies": { "dependencies": {
@ -9,7 +9,8 @@
"keyv": "^5.3.1", "keyv": "^5.3.1",
"keyv-file": "^5.1.2", "keyv-file": "^5.1.2",
"lowdb": "^7.0.1", "lowdb": "^7.0.1",
"vectra": "^0.9.0" "vectra": "^0.9.0",
"sqlite3": "^5.1.6"
}, },
"pnpm": {} "pnpm": {}
} }

View file

@ -1,4 +1,7 @@
import * as crypto from 'node:crypto' import * as crypto from 'node:crypto'
import path from 'path'
import ChatGPTConfig from '../config/config.js'
import fs from 'fs'
export function md5 (str) { export function md5 (str) {
return crypto.createHash('md5').update(str).digest('hex') return crypto.createHash('md5').update(str).digest('hex')
} }
@ -63,3 +66,8 @@ function formatDate (date, format) {
function padZero (num) { function padZero (num) {
return num < 10 ? '0' + num : num.toString() return num < 10 ? '0' + num : num.toString()
} }
export const dataDir = path.resolve('./plugins/chatgpt-plugin', ChatGPTConfig.chaite.dataDir)
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true })
}