feat: v0.6.1 — AriaEngine 可选自研 KVStore 后端(storageBackend: 'kv')+ KVStore APPEND 日志类型 + 10 个 aria+kv 集成测试 + 文档全量同步
CI / test (20.x) (push) Successful in 10m55s
CI / test (22.x) (push) Successful in 10m49s
CI / e2e (push) Successful in 9m54s
CI / test (18.x) (push) Successful in 10m58s
CI / test (24.x) (push) Successful in 10m41s

This commit is contained in:
thzxx
2026-08-10 14:52:38 +08:00
parent a56a496148
commit 25bab0f9aa
21 changed files with 895 additions and 42 deletions
+4
View File
@@ -22,6 +22,7 @@ import { DatabaseLock } from './locks';
import { WALRecordType, type WALRecord } from './types';
import { CheckpointManager } from './wal/checkpoint';
import { MemoryBackend, type IStorageBackend } from './store/backend';
import { KVStoreBackend } from './store/kvstore_backend';
import { OPFSBackend } from './store/opfs_backend';
import { EncryptedBackend } from './store/encrypted_backend';
import { PageSSTableStore } from './store/page_sstable_store';
@@ -117,6 +118,9 @@ export class AriaEngine implements IStorageEngine {
let baseBackend: IStorageBackend;
if (this.config.storageBackend === 'opfs') {
baseBackend = new OPFSBackend();
} else if (this.config.storageBackend === 'kv') {
// v0.6.1: 自研 KVStore 后端(aria 完全跑在自研存储栈上,不依赖浏览器 OPFS)
baseBackend = new KVStoreBackend();
} else {
baseBackend = new MemoryBackend();
}
+80
View File
@@ -0,0 +1,80 @@
/**
* KVStoreBackend — 基于自研 KVStore 的 AriaEngine 存储后端
* @module engine/aria/store/kvstore_backend
*
* v0.6.1: AriaEngine 可选后端(storageBackend: 'kv'),完全跑在自研 KVStore 上:
* - write/read/delete/listKeys/exists/clear → KVStore 直接映射
* - append → KVStore.appendValue(日志 APPEND 类型,O(chunk) 高效,aria WAL 分片用)
* - writeMany/deleteMany → KVStore.putMany/deleteMany(单日志记录真原子,
* 此前 OPFS 逐文件写靠空洞检测兜底,KV 后端原生原子)
* - 崩溃恢复:KVStore 快照+日志恢复 → aria 打开重放自己的 WAL(双层恢复)
*
* 介质:浏览器 OPFSKVStore 默认)或 Node SharedMemory——aria 不再依赖浏览器 OPFS API。
*/
import type { IStorageBackend } from './backend';
import { KVStore } from '../../kvstore/index';
export class KVStoreBackend implements IStorageBackend {
private kv: KVStore;
constructor(medium?: IStorageBackend, checkpointThreshold?: number) {
this.kv = new KVStore(medium, checkpointThreshold);
}
/** 底层 KVStore(测试/诊断用) */
getKV(): KVStore {
return this.kv;
}
async open(name: string): Promise<void> {
await this.kv.open(name);
}
async close(): Promise<void> {
await this.kv.close();
}
isOpen(): boolean {
return this.kv.isOpen();
}
async read(key: string): Promise<ArrayBuffer | null> {
return this.kv.get(key);
}
async write(key: string, data: ArrayBuffer): Promise<void> {
await this.kv.put(key, data);
}
/** 追加写入(KVStore APPEND 日志,O(chunk) */
async append(key: string, data: ArrayBuffer): Promise<void> {
await this.kv.appendValue(key, data);
}
/** 多 key 原子写入(单日志记录) */
async writeMany(entries: Record<string, ArrayBuffer>): Promise<void> {
await this.kv.putMany(entries);
}
async delete(key: string): Promise<void> {
await this.kv.delete(key);
}
/** 多 key 原子删除(单日志记录) */
async deleteMany(keys: string[]): Promise<void> {
await this.kv.deleteMany(keys);
}
async listKeys(): Promise<string[]> {
return this.kv.listKeys();
}
async exists(key: string): Promise<boolean> {
return this.kv.exists(key);
}
async clear(): Promise<void> {
await this.kv.clear();
}
}
+1 -1
View File
@@ -207,7 +207,7 @@ export interface AriaEngineConfig {
/** 是否启用页面压缩(默认 false) */
compression?: boolean;
/** 存储后端 */
storageBackend?: 'opfs' | 'memory';
storageBackend?: 'opfs' | 'memory' | 'kv';
/** WAL 大小阈值(字节,超过则强制 checkpoint,默认 16MB */
walSizeThreshold?: number;
/** 最大内存预算(MB,默认 64) */
+39 -2
View File
@@ -223,6 +223,18 @@ export class KVStore {
});
}
/**
* v0.6.1: 追加写入(value 拼接语义)— aria WAL 分片等追加型数据用。
* 日志记录 APPEND 类型(O(chunk) 高效),恢复时按 seq 顺序拼接,
* checkpoint 后快照含最终值。崩溃时该次追加全有或全无(单记录原子)。
*/
async appendValue(key: string, chunk: ArrayBuffer): Promise<void> {
if (chunk.byteLength === 0) return;
await this.enqueue(async () => {
await this.appendRecord({}, [], { [key]: chunk });
});
}
// =======================================================================
// 维护
// =======================================================================
@@ -301,9 +313,13 @@ export class KVStore {
}
/** 追加一条日志记录并更新内存索引(队列内调用,无并发) */
private async appendRecord(puts: Record<string, ArrayBuffer>, deletes: string[]): Promise<void> {
private async appendRecord(
puts: Record<string, ArrayBuffer>,
deletes: string[],
appends: Record<string, ArrayBuffer> = {},
): Promise<void> {
this.seq++;
const record = encodeLogRecord(this.seq, puts, deletes);
const record = encodeLogRecord(this.seq, puts, deletes, appends);
try {
// 日志追加:介质 append(真追加)或回退读+拼+写
const data = record.buffer.slice(record.byteOffset, record.byteOffset + record.byteLength) as ArrayBuffer;
@@ -334,6 +350,17 @@ export class KVStore {
for (const key of deletes) {
this.index.delete(key);
}
for (const [key, chunk] of Object.entries(appends)) {
const existing = this.index.get(key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + chunk.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(chunk), existing.byteLength);
this.index.set(key, combined.buffer as ArrayBuffer);
} else {
this.index.set(key, chunk);
}
}
this.logBytes += record.byteLength;
// 自动 checkpoint(日志超阈值)
@@ -358,6 +385,16 @@ export class KVStore {
for (const e of entries) {
if (e.op === KVLogOp.PUT) {
this.index.set(e.key, e.value);
} else if (e.op === KVLogOp.APPEND) {
const existing = this.index.get(e.key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + e.value.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(e.value), existing.byteLength);
this.index.set(e.key, combined.buffer as ArrayBuffer);
} else {
this.index.set(e.key, e.value);
}
} else {
this.index.delete(e.key);
}
+7
View File
@@ -23,6 +23,8 @@ import { crc32 } from '../aria/crc32';
export const enum KVLogOp {
PUT = 1,
DELETE = 2,
/** v0.6.1: 追加写入(value 拼接语义,恢复时按序 concataria WAL 分片用) */
APPEND = 3,
}
/** 解析后的日志记录 */
@@ -40,11 +42,13 @@ export interface KVLogRecord {
* @param seq 日志序号
* @param puts key → value 写入条目
* @param deletes 删除 key 列表
* @param appends key → 追加块列表(APPEND 语义,恢复时拼接)
*/
export function encodeLogRecord(
seq: number,
puts: Record<string, ArrayBuffer>,
deletes: string[] = [],
appends: Record<string, ArrayBuffer> = {},
): Uint8Array {
const encoder = new TextEncoder();
const entries: { op: KVLogOp; key: string; value: ArrayBuffer }[] = [];
@@ -54,6 +58,9 @@ export function encodeLogRecord(
for (const key of deletes) {
entries.push({ op: KVLogOp.DELETE, key, value: new ArrayBuffer(0) });
}
for (const [key, value] of Object.entries(appends)) {
entries.push({ op: KVLogOp.APPEND, key, value });
}
// 预编码 key 字节,计算总长度
const entryBytes: { op: KVLogOp; key: Uint8Array; value: Uint8Array }[] = [];