release: v0.2.3 AriaEngine 引擎加固 — RB-Tree fixDelete/LSM SSTable缓存预热/LZ4格式修复
This commit is contained in:
thzxx
@@ -1,168 +1,91 @@
|
||||
/**
|
||||
* AriaEngine LZ4 Compression — 简易 LZ4 压缩
|
||||
* AriaEngine LZ4 Compression — 简化 LZ4 压缩/解压
|
||||
* @module engine/aria/compression/lz4
|
||||
*
|
||||
* LZ4 是一种极快的压缩算法，适合页面级数据压缩。
|
||||
* 此处实现一个简化版，用于演示概念。
|
||||
* Token 格式（1 字节）:
|
||||
* hi 4bit = litLen (0-15)
|
||||
* lo 4bit = matchField (0-15, 实际匹配 = field+4)
|
||||
*
|
||||
* 压缩格式:
|
||||
* LITERAL_RUN: [token: 1B] [literals: N bytes]
|
||||
* MATCH: [offset: 2B LE] [matchLength: N]
|
||||
*
|
||||
* 实际生产环境建议使用 lz4 或 snappy 库。
|
||||
* 字面量-匹配序列: [token] [litLen bytes] [2B LE offset]
|
||||
* 末尾纯字面量: [token with lo=0] [litLen bytes] ← 仅在流末尾出现
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 常量
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MIN_MATCH = 4;
|
||||
const MAX_LITERAL_LENGTH = 15;
|
||||
const MAX_MATCH_LENGTH = 18;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 压缩
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 压缩数据。如果压缩后比原始大，返回原始数据（标记未压缩）。
|
||||
*/
|
||||
export function compressLZ4(input: Uint8Array): Uint8Array {
|
||||
if (input.byteLength < MIN_MATCH) {
|
||||
// 太小不值得压缩
|
||||
return input;
|
||||
}
|
||||
if (input.byteLength < MIN_MATCH) return input;
|
||||
|
||||
const maxOutputSize = input.byteLength + (input.byteLength / 255) + 16;
|
||||
const output = new Uint8Array(maxOutputSize);
|
||||
let srcIdx = 0;
|
||||
let dstIdx = 0;
|
||||
const maxOut = input.byteLength + (input.byteLength >> 8) + 32;
|
||||
const out = new Uint8Array(maxOut);
|
||||
let si = 0, di = 0;
|
||||
let litStart = 0;
|
||||
|
||||
while (srcIdx < input.byteLength) {
|
||||
// 查找最长匹配
|
||||
let bestMatchLen = 0;
|
||||
let bestMatchOffset = 0;
|
||||
const searchStart = Math.max(0, srcIdx - 65535);
|
||||
const searchEnd = srcIdx;
|
||||
|
||||
for (let i = searchStart; i < searchEnd; i++) {
|
||||
let matchLen = 0;
|
||||
while (
|
||||
srcIdx + matchLen < input.byteLength &&
|
||||
i + matchLen < srcIdx &&
|
||||
input[i + matchLen] === input[srcIdx + matchLen] &&
|
||||
matchLen < 255
|
||||
) {
|
||||
matchLen++;
|
||||
}
|
||||
if (matchLen > bestMatchLen && matchLen >= MIN_MATCH) {
|
||||
bestMatchLen = matchLen;
|
||||
bestMatchOffset = srcIdx - i;
|
||||
}
|
||||
while (si < input.byteLength) {
|
||||
// 搜索最长 backward match
|
||||
let bestLen = 0, bestOff = 0;
|
||||
const searchStart = Math.max(0, si - 65535);
|
||||
for (let p = searchStart; p < si; p++) {
|
||||
let ml = 0;
|
||||
while (si + ml < input.byteLength && p + ml < si &&
|
||||
input[p + ml] === input[si + ml] && ml < 255) ml++;
|
||||
if (ml >= MIN_MATCH && ml > bestLen) { bestLen = ml; bestOff = si - p; }
|
||||
}
|
||||
|
||||
if (bestMatchLen >= MIN_MATCH) {
|
||||
// 写入匹配
|
||||
const literalLen = 0;
|
||||
const matchLen = Math.min(bestMatchLen - MIN_MATCH, MAX_MATCH_LENGTH);
|
||||
|
||||
output[dstIdx++] = ((literalLen & 0x0F) << 4) | (matchLen & 0x0F);
|
||||
output[dstIdx++] = bestMatchOffset & 0xFF;
|
||||
output[dstIdx++] = (bestMatchOffset >> 8) & 0xFF;
|
||||
srcIdx += matchLen + MIN_MATCH;
|
||||
if (bestLen >= MIN_MATCH && (si - litStart) <= 15) {
|
||||
// 有匹配 → 输出组合 token（字面量+匹配）
|
||||
const litLen = si - litStart;
|
||||
const matchField = Math.min(bestLen - MIN_MATCH, 15);
|
||||
out[di++] = ((litLen & 0x0F) << 4) | (matchField & 0x0F);
|
||||
for (let j = 0; j < litLen; j++) out[di++] = input[litStart + j];
|
||||
out[di++] = bestOff & 0xFF;
|
||||
out[di++] = (bestOff >> 8) & 0xFF;
|
||||
si += bestLen;
|
||||
litStart = si;
|
||||
} else {
|
||||
// 写入字面量：收集连续无匹配的字节，直到遇到可匹配序列或末尾
|
||||
let litStart = srcIdx;
|
||||
while (srcIdx < input.byteLength) {
|
||||
const remaining = input.byteLength - srcIdx;
|
||||
if (remaining < MIN_MATCH) {
|
||||
srcIdx += remaining;
|
||||
break;
|
||||
}
|
||||
|
||||
// 检查当前位置开始是否有 >= MIN_MATCH 长度的匹配
|
||||
let hasMatch = false;
|
||||
for (let i = Math.max(0, srcIdx - 65535); i < srcIdx && !hasMatch; i++) {
|
||||
let ml = 0;
|
||||
while (srcIdx + ml < input.byteLength && i + ml < srcIdx && input[i + ml] === input[srcIdx + ml] && ml < MIN_MATCH) {
|
||||
ml++;
|
||||
}
|
||||
if (ml >= MIN_MATCH) hasMatch = true;
|
||||
}
|
||||
|
||||
if (hasMatch) {
|
||||
// 当前位置开始可匹配，停止字面量收集（不输出当前字节，交给下一轮匹配处理）
|
||||
break;
|
||||
}
|
||||
|
||||
// 无匹配，将此字节纳入字面量
|
||||
srcIdx++;
|
||||
}
|
||||
|
||||
let litLen = srcIdx - litStart;
|
||||
while (litLen > 0) {
|
||||
const chunk = Math.min(litLen, MAX_LITERAL_LENGTH);
|
||||
output[dstIdx++] = ((chunk & 0x0F) << 4);
|
||||
for (let j = 0; j < chunk; j++) {
|
||||
output[dstIdx++] = input[litStart + j];
|
||||
}
|
||||
litLen -= chunk;
|
||||
litStart += chunk;
|
||||
}
|
||||
// 无匹配或字面量已满 15 → 继续累积（不单独输出，等下个匹配合并）
|
||||
si++;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果压缩后更大，返回原始
|
||||
if (dstIdx >= input.byteLength) {
|
||||
return input;
|
||||
// 输出末尾纯字面量（matchField=0，无 offset）
|
||||
let remaining = si - litStart;
|
||||
while (remaining > 0) {
|
||||
const chunk = Math.min(remaining, 15);
|
||||
out[di++] = (chunk & 0x0F) << 4; // lo=0 表示无匹配/无 offset
|
||||
for (let j = 0; j < chunk; j++) out[di++] = input[litStart + j];
|
||||
remaining -= chunk;
|
||||
litStart += chunk;
|
||||
}
|
||||
|
||||
return output.slice(0, dstIdx);
|
||||
return di >= input.byteLength ? input : out.slice(0, di);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 解压
|
||||
// ---------------------------------------------------------------------------
|
||||
export function decompressLZ4(input: Uint8Array, originalSize: number): Uint8Array {
|
||||
const out = new Uint8Array(originalSize);
|
||||
let si = 0, di = 0;
|
||||
|
||||
/**
|
||||
* 解压 LZ4 数据。
|
||||
*
|
||||
* 与 compressLZ4 配套的简化格式：
|
||||
* 每个 token 要么是字面量（低 4 位 = 0），要么是匹配（高 4 位 = 0）。
|
||||
* 字面量 token: [hi4 = litLen] | 0x00 后跟 [litLen bytes]
|
||||
* 匹配 token: 0x00 | [lo4 = matchLen] 后跟 [offset: 2B LE]
|
||||
*/
|
||||
export function decompressLZ4(
|
||||
input: Uint8Array,
|
||||
originalSize: number,
|
||||
): Uint8Array {
|
||||
const output = new Uint8Array(originalSize);
|
||||
let srcIdx = 0;
|
||||
let dstIdx = 0;
|
||||
|
||||
while (srcIdx < input.byteLength && dstIdx < originalSize) {
|
||||
const token = input[srcIdx++];
|
||||
while (si < input.byteLength && di < originalSize) {
|
||||
const token = input[si++];
|
||||
const litLen = (token >> 4) & 0x0F;
|
||||
const matchLenField = token & 0x0F;
|
||||
const matchField = token & 0x0F;
|
||||
|
||||
// 复制字面量
|
||||
for (let i = 0; i < litLen && srcIdx < input.byteLength && dstIdx < originalSize; i++) {
|
||||
output[dstIdx++] = input[srcIdx++];
|
||||
for (let i = 0; i < litLen && si < input.byteLength && di < originalSize; i++) {
|
||||
out[di++] = input[si++];
|
||||
}
|
||||
|
||||
if (srcIdx >= input.byteLength || dstIdx >= originalSize) break;
|
||||
if (di >= originalSize || si >= input.byteLength) break;
|
||||
|
||||
if (matchLenField > 0) {
|
||||
// 读取偏移量并复制匹配
|
||||
const offset = input[srcIdx++] | (input[srcIdx++] << 8);
|
||||
const matchLen = matchLenField + MIN_MATCH;
|
||||
|
||||
for (let i = 0; i < matchLen && dstIdx < originalSize; i++) {
|
||||
output[dstIdx] = output[dstIdx - offset];
|
||||
dstIdx++;
|
||||
// 非末尾 → 必有 offset + 匹配（即使 matchField==0 也复制 MIN_MATCH 字节）
|
||||
if (si + 1 < input.byteLength) {
|
||||
const offset = input[si++] | (input[si++] << 8);
|
||||
const matchLen = matchField + MIN_MATCH;
|
||||
for (let i = 0; i < matchLen && di < originalSize; i++) {
|
||||
out[di] = out[di - offset];
|
||||
di++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -105,6 +105,15 @@ export class LSM {
|
||||
this.nextSSTableId = Math.max(...metas.map((m) => m.id)) + 1;
|
||||
}
|
||||
|
||||
// 预加载所有 SSTable 数据到缓存（避免后续 cache miss 返回 null）
|
||||
for (const meta of metas) {
|
||||
try {
|
||||
await this.preloadSSTable(meta.id);
|
||||
} catch {
|
||||
// 单个文件加载失败不影响整体启动
|
||||
}
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -230,9 +230,89 @@ class RedBlackTree<K, V> {
|
||||
if (this.root) this.root.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private fixDelete(_x: RBNode<K, V> | null, _parent: RBNode<K, V> | null): void {
|
||||
// 简化：在实际生产环境中需要完整的删除修复
|
||||
// 这里使用简化版，仅处理常见情况
|
||||
private fixDelete(x: RBNode<K, V> | null, parent: RBNode<K, V> | null): void {
|
||||
// 标准 RB-Tree 删除修复（修复"双黑"问题）
|
||||
let node = x;
|
||||
let nodeParent = parent;
|
||||
|
||||
while ((!node || node.color === Color.BLACK) && node !== this.root) {
|
||||
if (!nodeParent) break;
|
||||
|
||||
if (node === nodeParent.left) {
|
||||
let sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
|
||||
// Case 1: 兄弟是红色
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateLeft(nodeParent);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
// Case 2: 兄弟的两个子节点都是黑色
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
// Case 3: 兄弟右子黑色（左子红色）
|
||||
if (!sibRight || sibRight.color === Color.BLACK) {
|
||||
if (sibLeft) sibLeft.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateRight(sibling);
|
||||
sibling = nodeParent.right;
|
||||
if (!sibling) break;
|
||||
}
|
||||
// Case 4: 兄弟右子红色
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.right) sibling.right.color = Color.BLACK;
|
||||
this.rotateLeft(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
} else {
|
||||
// 镜像：node 是父节点的右子
|
||||
let sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
|
||||
if (sibling.color === Color.RED) {
|
||||
sibling.color = Color.BLACK;
|
||||
nodeParent.color = Color.RED;
|
||||
this.rotateRight(nodeParent);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
|
||||
const sibLeft = sibling.left;
|
||||
const sibRight = sibling.right;
|
||||
if ((!sibLeft || sibLeft.color === Color.BLACK) &&
|
||||
(!sibRight || sibRight.color === Color.BLACK)) {
|
||||
sibling.color = Color.RED;
|
||||
node = nodeParent;
|
||||
nodeParent = node.parent;
|
||||
} else {
|
||||
if (!sibLeft || sibLeft.color === Color.BLACK) {
|
||||
if (sibRight) sibRight.color = Color.BLACK;
|
||||
sibling.color = Color.RED;
|
||||
this.rotateLeft(sibling);
|
||||
sibling = nodeParent.left;
|
||||
if (!sibling) break;
|
||||
}
|
||||
sibling.color = nodeParent.color;
|
||||
nodeParent.color = Color.BLACK;
|
||||
if (sibling.left) sibling.left.color = Color.BLACK;
|
||||
this.rotateRight(nodeParent);
|
||||
node = this.root;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (node) node.color = Color.BLACK;
|
||||
}
|
||||
|
||||
private rotateLeft(x: RBNode<K, V>): void {
|
||||
|
||||
Reference in New Issue
Block a user