release: v0.5.1 — 存储后端生产级硬化(CRC-32/全库加密/WAL分片/页面化存储/多标签页锁/e2e)+ 深度审查修复(假实现接线/死代码清理)
CI / test (18.x) (push) Successful in 10m10s
CI / test (20.x) (push) Successful in 10m10s
CI / test (22.x) (push) Successful in 10m6s
CI / e2e (push) Successful in 9m51s
CI / test (24.x) (push) Successful in 10m28s

This commit is contained in:
thzxx
2026-08-10 12:07:00 +08:00
parent cff98b0903
commit 334067d89e
88 changed files with 15713 additions and 10626 deletions
+250 -218
View File
@@ -1,218 +1,250 @@
/**
* AriaEngine LZ4 压缩 + LSM Merge Iterator 单元测试
* 注:LZ4 为简化演示实现(默认 compression:false),测试聚焦于「不卡死」
*/
import { compressLZ4, decompressLZ4 } from '../../src/engine/aria/compression/lz4';
import { MergeIterator, ArrayEntrySource } from '../../src/engine/aria/index/merge_iterator';
// ===================================================================
// LZ4 压缩 — 安全烟雾测试(不卡死)
// ===================================================================
describe('AriaEngine — LZ4 Compression', () => {
it('短于 4 字节时压缩为纯字面量 token 且往返一致', () => {
const input = new Uint8Array([1, 2]);
const compressed = compressLZ4(input);
// token(lo=0) + 2 字节字面量
expect(compressed.byteLength).toBe(3);
expect(compressed[0]).toBe(0x20); // litLen=2, matchField=0
const restored = decompressLZ4(compressed, 2);
expect(Array.from(restored)).toEqual([1, 2]);
});
it('简单文本压缩不抛出异常且产生输出', () => {
const input = new TextEncoder().encode('hello world hello world hello world');
const compressed = compressLZ4(input);
expect(compressed).toBeInstanceOf(Uint8Array);
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('重复数据有压缩效果', () => {
const pattern = 'ABCD';
const repeated = pattern.repeat(100);
const input = new TextEncoder().encode(repeated);
const compressed = compressLZ4(input);
expect(compressed.byteLength).toBeLessThan(input.byteLength);
});
it('随机不可压缩数据不卡死', () => {
const input = new Uint8Array(256);
for (let i = 0; i < 256; i++) input[i] = Math.floor(Math.random() * 256);
const compressed = compressLZ4(input);
expect(compressed).toBeInstanceOf(Uint8Array);
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('长文本压缩不卡死', () => {
const input = new TextEncoder().encode('The quick brown fox jumps over the lazy dog. '.repeat(10));
const compressed = compressLZ4(input);
expect(compressed).toBeTruthy();
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('多种长度输入均不卡死', () => {
for (const size of [10, 50, 100, 200, 500]) {
const input = new Uint8Array(size);
for (let i = 0; i < size; i++) input[i] = i % 256;
const compressed = compressLZ4(input);
expect(compressed.byteLength).toBeLessThanOrEqual(input.byteLength + 16);
}
});
it('解压不抛出异常', () => {
const input = new TextEncoder().encode('test data for decompression smoke test');
const compressed = compressLZ4(input);
expect(() => decompressLZ4(compressed, input.byteLength)).not.toThrow();
});
// ---- v0.2.6 补强:压缩 → 解压 往返一致性 ----
it('往返一致:重复模式数据', () => {
const input = new TextEncoder().encode('ABCD'.repeat(100));
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:自然文本数据', () => {
const input = new TextEncoder().encode(
'The quick brown fox jumps over the lazy dog. '.repeat(10),
);
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:随机不可压缩数据', () => {
const input = new Uint8Array(512);
for (let i = 0; i < 512; i++) input[i] = Math.floor(Math.random() * 256);
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:多种长度与字节模式', () => {
for (const size of [4, 5, 15, 16, 17, 50, 100, 300, 1000]) {
const input = new Uint8Array(size);
for (let i = 0; i < size; i++) input[i] = i % 7 === 0 ? i % 256 : 0x41;
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
}
});
it('往返一致:恰好 15 字节字面量边界', () => {
// 字面量长度恰好 15(token 上限)时不应丢字节
const input = new Uint8Array(15);
for (let i = 0; i < 15; i++) input[i] = i;
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:超过 15 字节的连续匹配', () => {
const input = new TextEncoder().encode('X'.repeat(200) + 'Y' + 'X'.repeat(60));
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
});
// ===================================================================
// MergeIterator — 归并迭代器单元测试
// ===================================================================
describe('AriaEngine — MergeIterator', () => {
it('单数据源归并', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([
['a', { v: 1 }],
['b', { v: 2 }],
['c', { v: 3 }],
]));
const result = mi.drain();
expect(result).toHaveLength(3);
expect(result.map(([k]) => k)).toEqual(['a', 'b', 'c']);
});
it('多数据源归并去重(保留最新)', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 'new' }], ['c', { v: 3 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 'old' }], ['b', { v: 2 }]]));
const result = mi.drain();
expect(result).toHaveLength(3);
expect(result[0][0]).toBe('a');
expect(result[0][1].v).toBe('new');
expect(result[1][0]).toBe('b');
expect(result[2][0]).toBe('c');
});
// ---- v0.2.6 回归:同 key 多来源时保留 sourceIndex 最小(最新)的条目 ----
it('回归:同 key 出现在多个来源时返回 sourceIndex 最小(最新来源)的值', () => {
const mi = new MergeIterator();
// 语义:sourceIndex 越小越新(memtable=0 < immutable=1 < sstable=2+
mi.addSource(new ArrayEntrySource([['a', { v: 'source0' }]])); // 最新来源
mi.addSource(new ArrayEntrySource([['a', { v: 'source1' }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 'source2' }]])); // 最旧来源
const result = mi.drain();
expect(result).toHaveLength(1);
// 取的是 sourceIndex 最小(最新来源)的条目,而非堆序决定的任意条目
expect(result[0][1].v).toBe('source0');
});
it('回归:最新来源的值位于中间 sourceIndex 时仍取 sourceIndex 最小者', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 'middle' }]])); // index 0 = 最新来源
mi.addSource(new ArrayEntrySource([['a', { v: 'newest' }]])); // index 1
mi.addSource(new ArrayEntrySource([['a', { v: 'oldest' }]])); // index 2
const result = mi.drain();
expect(result).toHaveLength(1);
expect(result[0][1].v).toBe('middle'); // index 0 的条目胜出
});
it('回归:多个同 key 来源 + 其他独立 key 混合', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 1 }], ['b', { v: 10 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 2 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 3 }], ['c', { v: 30 }]]));
const result = mi.drain();
expect(result.map(([k]) => k)).toEqual(['a', 'b', 'c']);
expect(result[0][1].v).toBe(1); // sourceIndex 0 = 最新
expect(result[1][1].v).toBe(10);
expect(result[2][1].v).toBe(30);
});
it('空数据源归并', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([]));
const result = mi.drain();
expect(result).toHaveLength(0);
});
it('大数量归并', () => {
const mi = new MergeIterator();
for (let s = 0; s < 5; s++) {
const entries: [string, Record<string, unknown>][] = [];
for (let i = 0; i < 100; i++) {
entries.push([`src${s}-key-${String(i).padStart(3, '0')}`, { src: s, idx: i }]);
}
mi.addSource(new ArrayEntrySource(entries));
}
const result = mi.drain();
// 5 sources × 100 unique keys = 500 total (keys are unique per source)
expect(result).toHaveLength(500);
});
it('ArrayEntrySource — 迭代器用完返回 null', () => {
const src = new ArrayEntrySource([['k', { v: 1 }]]);
expect(src.next()).not.toBeNull();
expect(src.next()).toBeNull();
expect(src.next()).toBeNull();
});
it('ArrayEntrySource — reset 重置', () => {
const src = new ArrayEntrySource([['k1', { v: 1 }], ['k2', { v: 2 }]]);
src.next();
src.reset();
const val = src.next();
expect(val![0]).toBe('k1');
});
});
/**
* AriaEngine LZ4 压缩 + LSM Merge Iterator 单元测试
* 注:LZ4 为简化演示实现(默认 compression:false),测试聚焦于「不卡死」
*/
import { compressLZ4, decompressLZ4 } from '../../src/engine/aria/compression/lz4';
import { MergeIterator, ArrayEntrySource } from '../../src/engine/aria/index/merge_iterator';
// ===================================================================
// LZ4 压缩 — 安全烟雾测试(不卡死)
// ===================================================================
describe('AriaEngine — LZ4 Compression', () => {
it('短于 4 字节时压缩为纯字面量 token 且往返一致(v0.4.5 带 4 字节原始大小头)', () => {
const input = new Uint8Array([1, 2]);
const compressed = compressLZ4(input);
// HEADER(4) + token(lo=0) + 2 字节字面量
expect(compressed.byteLength).toBe(7);
// 头部 = 原始大小 2LE
expect(new DataView(compressed.buffer).getUint32(0, true)).toBe(2);
expect(compressed[4]).toBe(0x20); // litLen=2, matchField=0
const restored = decompressLZ4(compressed, 2);
expect(Array.from(restored)).toEqual([1, 2]);
});
it('简单文本压缩不抛出异常且产生输出', () => {
const input = new TextEncoder().encode('hello world hello world hello world');
const compressed = compressLZ4(input);
expect(compressed).toBeInstanceOf(Uint8Array);
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('重复数据有压缩效果', () => {
const pattern = 'ABCD';
const repeated = pattern.repeat(100);
const input = new TextEncoder().encode(repeated);
const compressed = compressLZ4(input);
expect(compressed.byteLength).toBeLessThan(input.byteLength);
});
it('随机不可压缩数据不卡死', () => {
const input = new Uint8Array(256);
for (let i = 0; i < 256; i++) input[i] = Math.floor(Math.random() * 256);
const compressed = compressLZ4(input);
expect(compressed).toBeInstanceOf(Uint8Array);
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('长文本压缩不卡死', () => {
const input = new TextEncoder().encode('The quick brown fox jumps over the lazy dog. '.repeat(10));
const compressed = compressLZ4(input);
expect(compressed).toBeTruthy();
expect(compressed.byteLength).toBeGreaterThan(0);
});
it('多种长度输入均不卡死', () => {
for (const size of [10, 50, 100, 200, 500]) {
const input = new Uint8Array(size);
for (let i = 0; i < size; i++) input[i] = i % 256;
const compressed = compressLZ4(input);
// 头部 +4 字节开销
expect(compressed.byteLength).toBeLessThanOrEqual(input.byteLength + 4 + 16);
}
});
it('解压不抛出异常', () => {
const input = new TextEncoder().encode('test data for decompression smoke test');
const compressed = compressLZ4(input);
expect(() => decompressLZ4(compressed, input.byteLength)).not.toThrow();
});
// ---- v0.2.6 补强:压缩 → 解压 往返一致性 ----
it('往返一致:重复模式数据', () => {
const input = new TextEncoder().encode('ABCD'.repeat(100));
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:自然文本数据', () => {
const input = new TextEncoder().encode(
'The quick brown fox jumps over the lazy dog. '.repeat(10),
);
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:随机不可压缩数据', () => {
const input = new Uint8Array(512);
for (let i = 0; i < 512; i++) input[i] = Math.floor(Math.random() * 256);
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:多种长度与字节模式', () => {
for (const size of [4, 5, 15, 16, 17, 50, 100, 300, 1000]) {
const input = new Uint8Array(size);
for (let i = 0; i < size; i++) input[i] = i % 7 === 0 ? i % 256 : 0x41;
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
}
});
it('往返一致:恰好 15 字节字面量边界', () => {
// 字面量长度恰好 15(token 上限)时不应丢字节
const input = new Uint8Array(15);
for (let i = 0; i < 15; i++) input[i] = i;
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:超过 15 字节的连续匹配', () => {
const input = new TextEncoder().encode('X'.repeat(200) + 'Y' + 'X'.repeat(60));
const compressed = compressLZ4(input);
const restored = decompressLZ4(compressed, input.byteLength);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('往返一致:空输入', () => {
const compressed = compressLZ4(new Uint8Array(0));
expect(compressed.byteLength).toBe(4); // 仅头部
expect(new DataView(compressed.buffer).getUint32(0, true)).toBe(0);
const restored = decompressLZ4(compressed);
expect(restored.byteLength).toBe(0);
});
it('往返一致:头部自描述,无需外部原始大小(高压缩率回归)', () => {
// 中文重复内容 → 极高压缩率(此前 buf.length*2 估算不足导致解压截断)
const text = '这是需要加密的敏感内容'.repeat(500);
const input = new TextEncoder().encode(text);
const compressed = compressLZ4(input);
// 压缩率验证:远小于原始
expect(compressed.byteLength).toBeLessThan(input.byteLength / 2);
// 不传 originalSize:从头部恢复
const restored = decompressLZ4(compressed);
expect(Array.from(restored)).toEqual(Array.from(input));
});
it('损坏头(原始大小非法)→ 抛错而非静默截断', () => {
const bad = new Uint8Array([0xff, 0xff, 0xff, 0xff, 0x20, 0x01, 0x02]);
expect(() => decompressLZ4(bad)).toThrow('Invalid LZ4 header');
});
it('过短输入(无头)→ 抛错', () => {
expect(() => decompressLZ4(new Uint8Array([1, 2, 3]))).toThrow('missing header');
});
});
// ===================================================================
// MergeIterator — 归并迭代器单元测试
// ===================================================================
describe('AriaEngine — MergeIterator', () => {
it('单数据源归并', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([
['a', { v: 1 }],
['b', { v: 2 }],
['c', { v: 3 }],
]));
const result = mi.drain();
expect(result).toHaveLength(3);
expect(result.map(([k]) => k)).toEqual(['a', 'b', 'c']);
});
it('多数据源归并去重(保留最新)', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 'new' }], ['c', { v: 3 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 'old' }], ['b', { v: 2 }]]));
const result = mi.drain();
expect(result).toHaveLength(3);
expect(result[0][0]).toBe('a');
expect(result[0][1].v).toBe('new');
expect(result[1][0]).toBe('b');
expect(result[2][0]).toBe('c');
});
// ---- v0.2.6 回归:同 key 多来源时保留 sourceIndex 最小(最新)的条目 ----
it('回归:同 key 出现在多个来源时返回 sourceIndex 最小(最新来源)的值', () => {
const mi = new MergeIterator();
// 语义:sourceIndex 越小越新(memtable=0 < immutable=1 < sstable=2+
mi.addSource(new ArrayEntrySource([['a', { v: 'source0' }]])); // 最新来源
mi.addSource(new ArrayEntrySource([['a', { v: 'source1' }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 'source2' }]])); // 最旧来源
const result = mi.drain();
expect(result).toHaveLength(1);
// 取的是 sourceIndex 最小(最新来源)的条目,而非堆序决定的任意条目
expect(result[0][1].v).toBe('source0');
});
it('回归:最新来源的值位于中间 sourceIndex 时仍取 sourceIndex 最小者', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 'middle' }]])); // index 0 = 最新来源
mi.addSource(new ArrayEntrySource([['a', { v: 'newest' }]])); // index 1
mi.addSource(new ArrayEntrySource([['a', { v: 'oldest' }]])); // index 2
const result = mi.drain();
expect(result).toHaveLength(1);
expect(result[0][1].v).toBe('middle'); // index 0 的条目胜出
});
it('回归:多个同 key 来源 + 其他独立 key 混合', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([['a', { v: 1 }], ['b', { v: 10 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 2 }]]));
mi.addSource(new ArrayEntrySource([['a', { v: 3 }], ['c', { v: 30 }]]));
const result = mi.drain();
expect(result.map(([k]) => k)).toEqual(['a', 'b', 'c']);
expect(result[0][1].v).toBe(1); // sourceIndex 0 = 最新
expect(result[1][1].v).toBe(10);
expect(result[2][1].v).toBe(30);
});
it('空数据源归并', () => {
const mi = new MergeIterator();
mi.addSource(new ArrayEntrySource([]));
const result = mi.drain();
expect(result).toHaveLength(0);
});
it('大数量归并', () => {
const mi = new MergeIterator();
for (let s = 0; s < 5; s++) {
const entries: [string, Record<string, unknown>][] = [];
for (let i = 0; i < 100; i++) {
entries.push([`src${s}-key-${String(i).padStart(3, '0')}`, { src: s, idx: i }]);
}
mi.addSource(new ArrayEntrySource(entries));
}
const result = mi.drain();
// 5 sources × 100 unique keys = 500 total (keys are unique per source)
expect(result).toHaveLength(500);
});
it('ArrayEntrySource — 迭代器用完返回 null', () => {
const src = new ArrayEntrySource([['k', { v: 1 }]]);
expect(src.next()).not.toBeNull();
expect(src.next()).toBeNull();
expect(src.next()).toBeNull();
});
it('ArrayEntrySource — reset 重置', () => {
const src = new ArrayEntrySource([['k1', { v: 1 }], ['k2', { v: 2 }]]);
src.next();
src.reset();
const val = src.next();
expect(val![0]).toBe('k1');
});
});