feat: v0.6.1 — AriaEngine 可选自研 KVStore 后端(storageBackend: 'kv')+ KVStore APPEND 日志类型 + 10 个 aria+kv 集成测试 + 文档全量同步
This commit is contained in:
+2
-2
@@ -10,8 +10,8 @@
|
||||
/** 存储模式 */
|
||||
export type StorageMode = 'memory' | 'disk' | 'hybrid' | 'aria';
|
||||
|
||||
/** 磁盘引擎类型(v0.6.0: IndexedDB 已移除,'memory' 供 aria 内存后端) */
|
||||
export type DiskEngine = 'opfs' | 'memory';
|
||||
/** 磁盘引擎类型(v0.6.0: IndexedDB 已移除;'memory' 供 aria 内存后端;'kv' 供 aria 自研 KVStore 后端) */
|
||||
export type DiskEngine = 'opfs' | 'memory' | 'kv';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 字段类型
|
||||
|
||||
+2
-1
@@ -557,8 +557,9 @@ export class MetonaSqlark {
|
||||
return new KVStoreEngine();
|
||||
case 'aria':
|
||||
// v0.4.5: 透传 AriaEngine 专属配置(walSyncMode/checkpointInterval/encryption/pageStorage 等)
|
||||
// v0.6.1: diskEngine 'kv' → 自研 KVStore 后端
|
||||
return new AriaEngine({
|
||||
storageBackend: diskEngine === 'memory' ? 'memory' : 'opfs',
|
||||
storageBackend: diskEngine === 'memory' ? 'memory' : diskEngine === 'kv' ? 'kv' : 'opfs',
|
||||
...(this.config.aria ?? {}),
|
||||
});
|
||||
case 'hybrid':
|
||||
|
||||
@@ -22,6 +22,7 @@ import { DatabaseLock } from './locks';
|
||||
import { WALRecordType, type WALRecord } from './types';
|
||||
import { CheckpointManager } from './wal/checkpoint';
|
||||
import { MemoryBackend, type IStorageBackend } from './store/backend';
|
||||
import { KVStoreBackend } from './store/kvstore_backend';
|
||||
import { OPFSBackend } from './store/opfs_backend';
|
||||
import { EncryptedBackend } from './store/encrypted_backend';
|
||||
import { PageSSTableStore } from './store/page_sstable_store';
|
||||
@@ -117,6 +118,9 @@ export class AriaEngine implements IStorageEngine {
|
||||
let baseBackend: IStorageBackend;
|
||||
if (this.config.storageBackend === 'opfs') {
|
||||
baseBackend = new OPFSBackend();
|
||||
} else if (this.config.storageBackend === 'kv') {
|
||||
// v0.6.1: 自研 KVStore 后端(aria 完全跑在自研存储栈上,不依赖浏览器 OPFS)
|
||||
baseBackend = new KVStoreBackend();
|
||||
} else {
|
||||
baseBackend = new MemoryBackend();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* KVStoreBackend — 基于自研 KVStore 的 AriaEngine 存储后端
|
||||
* @module engine/aria/store/kvstore_backend
|
||||
*
|
||||
* v0.6.1: AriaEngine 可选后端(storageBackend: 'kv'),完全跑在自研 KVStore 上:
|
||||
* - write/read/delete/listKeys/exists/clear → KVStore 直接映射
|
||||
* - append → KVStore.appendValue(日志 APPEND 类型,O(chunk) 高效,aria WAL 分片用)
|
||||
* - writeMany/deleteMany → KVStore.putMany/deleteMany(单日志记录真原子,
|
||||
* 此前 OPFS 逐文件写靠空洞检测兜底,KV 后端原生原子)
|
||||
* - 崩溃恢复:KVStore 快照+日志恢复 → aria 打开重放自己的 WAL(双层恢复)
|
||||
*
|
||||
* 介质:浏览器 OPFS(KVStore 默认)或 Node SharedMemory——aria 不再依赖浏览器 OPFS API。
|
||||
*/
|
||||
|
||||
import type { IStorageBackend } from './backend';
|
||||
import { KVStore } from '../../kvstore/index';
|
||||
|
||||
export class KVStoreBackend implements IStorageBackend {
|
||||
private kv: KVStore;
|
||||
|
||||
constructor(medium?: IStorageBackend, checkpointThreshold?: number) {
|
||||
this.kv = new KVStore(medium, checkpointThreshold);
|
||||
}
|
||||
|
||||
/** 底层 KVStore(测试/诊断用) */
|
||||
getKV(): KVStore {
|
||||
return this.kv;
|
||||
}
|
||||
|
||||
async open(name: string): Promise<void> {
|
||||
await this.kv.open(name);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.kv.close();
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.kv.isOpen();
|
||||
}
|
||||
|
||||
async read(key: string): Promise<ArrayBuffer | null> {
|
||||
return this.kv.get(key);
|
||||
}
|
||||
|
||||
async write(key: string, data: ArrayBuffer): Promise<void> {
|
||||
await this.kv.put(key, data);
|
||||
}
|
||||
|
||||
/** 追加写入(KVStore APPEND 日志,O(chunk)) */
|
||||
async append(key: string, data: ArrayBuffer): Promise<void> {
|
||||
await this.kv.appendValue(key, data);
|
||||
}
|
||||
|
||||
/** 多 key 原子写入(单日志记录) */
|
||||
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
|
||||
await this.kv.putMany(entries);
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
await this.kv.delete(key);
|
||||
}
|
||||
|
||||
/** 多 key 原子删除(单日志记录) */
|
||||
async deleteMany(keys: string[]): Promise<void> {
|
||||
await this.kv.deleteMany(keys);
|
||||
}
|
||||
|
||||
async listKeys(): Promise<string[]> {
|
||||
return this.kv.listKeys();
|
||||
}
|
||||
|
||||
async exists(key: string): Promise<boolean> {
|
||||
return this.kv.exists(key);
|
||||
}
|
||||
|
||||
async clear(): Promise<void> {
|
||||
await this.kv.clear();
|
||||
}
|
||||
}
|
||||
@@ -207,7 +207,7 @@ export interface AriaEngineConfig {
|
||||
/** 是否启用页面压缩(默认 false) */
|
||||
compression?: boolean;
|
||||
/** 存储后端 */
|
||||
storageBackend?: 'opfs' | 'memory';
|
||||
storageBackend?: 'opfs' | 'memory' | 'kv';
|
||||
/** WAL 大小阈值(字节,超过则强制 checkpoint,默认 16MB) */
|
||||
walSizeThreshold?: number;
|
||||
/** 最大内存预算(MB,默认 64) */
|
||||
|
||||
@@ -223,6 +223,18 @@ export class KVStore {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.6.1: 追加写入(value 拼接语义)— aria WAL 分片等追加型数据用。
|
||||
* 日志记录 APPEND 类型(O(chunk) 高效),恢复时按 seq 顺序拼接,
|
||||
* checkpoint 后快照含最终值。崩溃时该次追加全有或全无(单记录原子)。
|
||||
*/
|
||||
async appendValue(key: string, chunk: ArrayBuffer): Promise<void> {
|
||||
if (chunk.byteLength === 0) return;
|
||||
await this.enqueue(async () => {
|
||||
await this.appendRecord({}, [], { [key]: chunk });
|
||||
});
|
||||
}
|
||||
|
||||
// =======================================================================
|
||||
// 维护
|
||||
// =======================================================================
|
||||
@@ -301,9 +313,13 @@ export class KVStore {
|
||||
}
|
||||
|
||||
/** 追加一条日志记录并更新内存索引(队列内调用,无并发) */
|
||||
private async appendRecord(puts: Record<string, ArrayBuffer>, deletes: string[]): Promise<void> {
|
||||
private async appendRecord(
|
||||
puts: Record<string, ArrayBuffer>,
|
||||
deletes: string[],
|
||||
appends: Record<string, ArrayBuffer> = {},
|
||||
): Promise<void> {
|
||||
this.seq++;
|
||||
const record = encodeLogRecord(this.seq, puts, deletes);
|
||||
const record = encodeLogRecord(this.seq, puts, deletes, appends);
|
||||
try {
|
||||
// 日志追加:介质 append(真追加)或回退读+拼+写
|
||||
const data = record.buffer.slice(record.byteOffset, record.byteOffset + record.byteLength) as ArrayBuffer;
|
||||
@@ -334,6 +350,17 @@ export class KVStore {
|
||||
for (const key of deletes) {
|
||||
this.index.delete(key);
|
||||
}
|
||||
for (const [key, chunk] of Object.entries(appends)) {
|
||||
const existing = this.index.get(key);
|
||||
if (existing) {
|
||||
const combined = new Uint8Array(existing.byteLength + chunk.byteLength);
|
||||
combined.set(new Uint8Array(existing), 0);
|
||||
combined.set(new Uint8Array(chunk), existing.byteLength);
|
||||
this.index.set(key, combined.buffer as ArrayBuffer);
|
||||
} else {
|
||||
this.index.set(key, chunk);
|
||||
}
|
||||
}
|
||||
this.logBytes += record.byteLength;
|
||||
|
||||
// 自动 checkpoint(日志超阈值)
|
||||
@@ -358,6 +385,16 @@ export class KVStore {
|
||||
for (const e of entries) {
|
||||
if (e.op === KVLogOp.PUT) {
|
||||
this.index.set(e.key, e.value);
|
||||
} else if (e.op === KVLogOp.APPEND) {
|
||||
const existing = this.index.get(e.key);
|
||||
if (existing) {
|
||||
const combined = new Uint8Array(existing.byteLength + e.value.byteLength);
|
||||
combined.set(new Uint8Array(existing), 0);
|
||||
combined.set(new Uint8Array(e.value), existing.byteLength);
|
||||
this.index.set(e.key, combined.buffer as ArrayBuffer);
|
||||
} else {
|
||||
this.index.set(e.key, e.value);
|
||||
}
|
||||
} else {
|
||||
this.index.delete(e.key);
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import { crc32 } from '../aria/crc32';
|
||||
export const enum KVLogOp {
|
||||
PUT = 1,
|
||||
DELETE = 2,
|
||||
/** v0.6.1: 追加写入(value 拼接语义,恢复时按序 concat;aria WAL 分片用) */
|
||||
APPEND = 3,
|
||||
}
|
||||
|
||||
/** 解析后的日志记录 */
|
||||
@@ -40,11 +42,13 @@ export interface KVLogRecord {
|
||||
* @param seq 日志序号
|
||||
* @param puts key → value 写入条目
|
||||
* @param deletes 删除 key 列表
|
||||
* @param appends key → 追加块列表(APPEND 语义,恢复时拼接)
|
||||
*/
|
||||
export function encodeLogRecord(
|
||||
seq: number,
|
||||
puts: Record<string, ArrayBuffer>,
|
||||
deletes: string[] = [],
|
||||
appends: Record<string, ArrayBuffer> = {},
|
||||
): Uint8Array {
|
||||
const encoder = new TextEncoder();
|
||||
const entries: { op: KVLogOp; key: string; value: ArrayBuffer }[] = [];
|
||||
@@ -54,6 +58,9 @@ export function encodeLogRecord(
|
||||
for (const key of deletes) {
|
||||
entries.push({ op: KVLogOp.DELETE, key, value: new ArrayBuffer(0) });
|
||||
}
|
||||
for (const [key, value] of Object.entries(appends)) {
|
||||
entries.push({ op: KVLogOp.APPEND, key, value });
|
||||
}
|
||||
|
||||
// 预编码 key 字节,计算总长度
|
||||
const entryBytes: { op: KVLogOp; key: Uint8Array; value: Uint8Array }[] = [];
|
||||
|
||||
Reference in New Issue
Block a user