feat: v0.6.1 — AriaEngine 可选自研 KVStore 后端(storageBackend: 'kv')+ KVStore APPEND 日志类型 + 10 个 aria+kv 集成测试 + 文档全量同步
CI / test (20.x) (push) Successful in 10m55s
CI / test (22.x) (push) Successful in 10m49s
CI / e2e (push) Successful in 9m54s
CI / test (18.x) (push) Successful in 10m58s
CI / test (24.x) (push) Successful in 10m41s

This commit is contained in:
thzxx
2026-08-10 14:52:38 +08:00
parent a56a496148
commit 25bab0f9aa
21 changed files with 895 additions and 42 deletions
+113 -4
View File
@@ -1120,14 +1120,17 @@ var KVLogOp;
(function (KVLogOp) {
KVLogOp[KVLogOp["PUT"] = 1] = "PUT";
KVLogOp[KVLogOp["DELETE"] = 2] = "DELETE";
/** v0.6.1: 追加写入(value 拼接语义,恢复时按序 concataria WAL 分片用) */
KVLogOp[KVLogOp["APPEND"] = 3] = "APPEND";
})(KVLogOp || (KVLogOp = {}));
/**
* 编码一条日志记录
* @param seq 日志序号
* @param puts key value 写入条目
* @param deletes 删除 key 列表
* @param appends key 追加块列表APPEND 语义恢复时拼接
*/
function encodeLogRecord(seq, puts, deletes = []) {
function encodeLogRecord(seq, puts, deletes = [], appends = {}) {
const encoder = new TextEncoder();
const entries = [];
for (const [key, value] of Object.entries(puts)) {
@@ -1136,6 +1139,9 @@ function encodeLogRecord(seq, puts, deletes = []) {
for (const key of deletes) {
entries.push({ op: KVLogOp.DELETE, key, value: new ArrayBuffer(0) });
}
for (const [key, value] of Object.entries(appends)) {
entries.push({ op: KVLogOp.APPEND, key, value });
}
// 预编码 key 字节,计算总长度
const entryBytes = [];
let total = 4 + 4 + 4; // recordLen + seq + entryCount
@@ -1541,6 +1547,18 @@ class KVStore {
await this.appendRecord({}, keys);
});
}
/**
* v0.6.1: 追加写入value 拼接语义 aria WAL 分片等追加型数据用
* 日志记录 APPEND 类型O(chunk) 高效恢复时按 seq 顺序拼接
* checkpoint 后快照含最终值崩溃时该次追加全有或全无单记录原子
*/
async appendValue(key, chunk) {
if (chunk.byteLength === 0)
return;
await this.enqueue(async () => {
await this.appendRecord({}, [], { [key]: chunk });
});
}
// =======================================================================
// 维护
// =======================================================================
@@ -1613,9 +1631,9 @@ class KVStore {
return run;
}
/** 追加一条日志记录并更新内存索引(队列内调用,无并发) */
async appendRecord(puts, deletes) {
async appendRecord(puts, deletes, appends = {}) {
this.seq++;
const record = encodeLogRecord(this.seq, puts, deletes);
const record = encodeLogRecord(this.seq, puts, deletes, appends);
try {
// 日志追加:介质 append(真追加)或回退读+拼+写
const data = record.buffer.slice(record.byteOffset, record.byteOffset + record.byteLength);
@@ -1648,6 +1666,18 @@ class KVStore {
for (const key of deletes) {
this.index.delete(key);
}
for (const [key, chunk] of Object.entries(appends)) {
const existing = this.index.get(key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + chunk.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(chunk), existing.byteLength);
this.index.set(key, combined.buffer);
}
else {
this.index.set(key, chunk);
}
}
this.logBytes += record.byteLength;
// 自动 checkpoint(日志超阈值)
if (this.checkpointThreshold > 0 && this.logBytes >= this.checkpointThreshold) {
@@ -1671,6 +1701,18 @@ class KVStore {
if (e.op === KVLogOp.PUT) {
this.index.set(e.key, e.value);
}
else if (e.op === KVLogOp.APPEND) {
const existing = this.index.get(e.key);
if (existing) {
const combined = new Uint8Array(existing.byteLength + e.value.byteLength);
combined.set(new Uint8Array(existing), 0);
combined.set(new Uint8Array(e.value), existing.byteLength);
this.index.set(e.key, combined.buffer);
}
else {
this.index.set(e.key, e.value);
}
}
else {
this.index.delete(e.key);
}
@@ -4818,6 +4860,68 @@ class MemoryBackend {
}
}
/**
* KVStoreBackend 基于自研 KVStore AriaEngine 存储后端
* @module engine/aria/store/kvstore_backend
*
* v0.6.1: AriaEngine 可选后端storageBackend: 'kv'完全跑在自研 KVStore
* - write/read/delete/listKeys/exists/clear KVStore
* - append KVStore.appendValue日志 APPEND 类型O(chunk) 高效aria WAL 分片用
* - writeMany/deleteMany KVStore.putMany/deleteMany单日志记录真原子
* 此前 OPFS 逐文件写靠空洞检测兜底KV 后端原生原子
* - 崩溃恢复KVStore 快照+日志恢复 aria 打开重放自己的 WAL双层恢复
*
* 介质浏览器 OPFSKVStore 默认 Node SharedMemoryaria 不再依赖浏览器 OPFS API
*/
class KVStoreBackend {
constructor(medium, checkpointThreshold) {
this.kv = new KVStore(medium, checkpointThreshold);
}
/** 底层 KVStore(测试/诊断用) */
getKV() {
return this.kv;
}
async open(name) {
await this.kv.open(name);
}
async close() {
await this.kv.close();
}
isOpen() {
return this.kv.isOpen();
}
async read(key) {
return this.kv.get(key);
}
async write(key, data) {
await this.kv.put(key, data);
}
/** 追加写入(KVStore APPEND 日志,O(chunk) */
async append(key, data) {
await this.kv.appendValue(key, data);
}
/** 多 key 原子写入(单日志记录) */
async writeMany(entries) {
await this.kv.putMany(entries);
}
async delete(key) {
await this.kv.delete(key);
}
/** 多 key 原子删除(单日志记录) */
async deleteMany(keys) {
await this.kv.deleteMany(keys);
}
async listKeys() {
return this.kv.listKeys();
}
async exists(key) {
return this.kv.exists(key);
}
async clear() {
await this.kv.clear();
}
}
/**
* AriaEngine Crypto 页面级 AES-GCM 加密
* @module engine/aria/crypto
@@ -5953,6 +6057,10 @@ class AriaEngine {
if (this.config.storageBackend === 'opfs') {
baseBackend = new OPFSBackend();
}
else if (this.config.storageBackend === 'kv') {
// v0.6.1: 自研 KVStore 后端(aria 完全跑在自研存储栈上,不依赖浏览器 OPFS)
baseBackend = new KVStoreBackend();
}
else {
baseBackend = new MemoryBackend();
}
@@ -11590,8 +11698,9 @@ class MetonaSqlark {
return new KVStoreEngine();
case 'aria':
// v0.4.5: 透传 AriaEngine 专属配置(walSyncMode/checkpointInterval/encryption/pageStorage 等)
// v0.6.1: diskEngine 'kv' → 自研 KVStore 后端
return new AriaEngine({
storageBackend: diskEngine === 'memory' ? 'memory' : 'opfs',
storageBackend: diskEngine === 'memory' ? 'memory' : diskEngine === 'kv' ? 'kv' : 'opfs',
...(this.config.aria ?? {}),
});
case 'hybrid':