MetonaTeam/MetonaSqlark
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: v0.4.4 — SSTable 编码修复（UTF-8 字节估算 + u32 长度字段 + v1/v2 双格式兼容），大内容不再崩溃
CI / test (18.x) (push) Successful in 10m3s
Details
CI / test (20.x) (push) Successful in 10m4s
Details
CI / test (22.x) (push) Successful in 9m58s
Details
CI / test (24.x) (push) Successful in 9m56s
Details

Browse Source
This commit is contained in:
thzxx
2026-08-09 21:34:28 +08:00
parent d269bdfb75
commit cff98b0903
21 changed files with 1166 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine/aria/index/sstable_builder.ts
+244 -247
View File
@@ -1,247 +1,244 @@
/**
* AriaEngine SSTable Builder — 构建有序字符串表
* @module engine/aria/index/sstable_builder
*
* 将排序后的 key-value 数据写入 SSTable 格式。
*
* SSTable 文件布局:
* ┌──────────────────────────────────────────────┐
* │ Data Block 0 │
* │ Data Block 1 │
* │ ... │
* │ Index Block (block offset → key range) │
* │ Bloom Filter │
* │ Footer (32 bytes) │
* │ - index_offset (u32) │
* │ - index_size (u32) │
* │ - bloom_offset (u32) │
* │ - bloom_size (u32) │
* │ - bloom_hash_count (u32) │
* │ - entry_count (u32) │
* │ - magic_number (u32, 0x53535442 ="SSTB")│
* │ - checksum (u32) │
* └──────────────────────────────────────────────┘
*/
import { BloomFilter } from './bloom';
import type { IndexEntry } from '../types';
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
const SSTABLE_FOOTER_SIZE = 32;
// ---------------------------------------------------------------------------
// SSTableBuilder
// ---------------------------------------------------------------------------
export class SSTableBuilder {
private entries: [string, Record<string, unknown>][] = [];
private currentBlock: [string, Record<string, unknown>][] = [];
private currentBlockStartKey = '';
private blockSizeLimit: number;
constructor(blockSizeLimit: number = 4096) {
this.blockSizeLimit = blockSizeLimit;
}
/** 添加一个 key-value 条目（必须按键排序添加） */
add(key: string, value: Record<string, unknown>): void {
if (this.currentBlock.length === 0) {
this.currentBlockStartKey = key;
}
this.currentBlock.push([key, value]);
this.entries.push([key, value]);
// 如果当前 Block 达到大小限制，切割
const estimated = this.estimateBlockSize();
if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) {
// 当前 block 结束（不在这里切割，在 build 时统一处理）
}
}
/**
* 构建 SSTable 文件的二进制数据。
* 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
*/
build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
const blocks = this.splitIntoBlocks();
const bloomFilter = new BloomFilter(this.entries.length);
// 预计算总大小
let totalSize = 0;
const blockOffsets: number[] = [];
for (const block of blocks) {
blockOffsets.push(totalSize);
const blockSize = this.computeBlockSize(block);
totalSize += blockSize;
}
// 索引块
const indexEntries: IndexEntry[] = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
const lastKey = block[block.length - 1][0];
const blockSize = this.computeBlockSize(block);
indexEntries.push({
key: lastKey,
blockOffset: blockOffsets[i],
blockSize,
});
}
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
// 序列化 bloom filter 以获取其大小
const bloomData = bloomFilter.serialize();
const bloomSize = bloomData.byteLength;
// 写入到 buffer（包含 bloom block）
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
const buf = new ArrayBuffer(finalSize);
const view = new DataView(buf);
let offset = 0;
// ---- Data Blocks ----
for (const block of blocks) {
offset = this.writeDataBlock(view, offset, block, bloomFilter);
}
// ---- Index Block ----
const indexOffset = offset;
offset = this.writeIndexBlock(view, offset, indexEntries);
// ---- Bloom Filter Block ----
const bloomOffset = offset;
new Uint8Array(view.buffer).set(bloomData, offset);
offset += bloomSize;
// ---- Footer ----
const footerOffset = offset;
view.setUint32(footerOffset, indexOffset, false); // index_offset
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
view.setUint32(footerOffset + 20, this.entries.length, false);
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
return {
sstableData: new Uint8Array(buf),
indexEntries,
};
}
/** 获取条目数 */
getEntryCount(): number {
return this.entries.length;
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
private splitIntoBlocks(): [string, Record<string, unknown>][][] {
const blocks: [string, Record<string, unknown>][][] = [];
let current: [string, Record<string, unknown>][] = [];
for (const entry of this.entries) {
current.push(entry);
if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
blocks.push(current.slice(0, -1));
current = [entry];
}
}
if (current.length > 0) blocks.push(current);
return blocks;
}
private estimateBlockSize(): number {
return this.estimateBlockSizeFromEntries(this.currentBlock);
}
private estimateBlockSizeFromEntries(entries: [string, unknown][]): number {
let size = 0;
for (const [key, value] of entries) {
size += 4 + key.length + JSON.stringify(value).length;
}
return size;
}
private computeBlockSize(block: [string, unknown][]): number {
// entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
let size = 4;
for (const [key, value] of block) {
const json = JSON.stringify(value);
size += 2 + key.length + 2 + json.length;
}
return size;
}
private writeDataBlock(
view: DataView,
offset: number,
block: [string, Record<string, unknown>][],
bloomFilter: BloomFilter,
): number {
// entry count
view.setUint32(offset, block.length, false);
offset += 4;
for (const [key, value] of block) {
const encoder = new TextEncoder();
const keyBytes = encoder.encode(key);
const valueBytes = encoder.encode(JSON.stringify(value));
// key length
view.setUint16(offset, keyBytes.length, false);
offset += 2;
// key
new Uint8Array(view.buffer).set(keyBytes, offset);
offset += keyBytes.length;
// value length
view.setUint16(offset, valueBytes.length, false);
offset += 2;
// value
new Uint8Array(view.buffer).set(valueBytes, offset);
offset += valueBytes.length;
// 插入 bloom filter
bloomFilter.insert(key);
}
return offset;
}
private estimateIndexBlockSize(entries: IndexEntry[]): number {
// entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
let size = 4;
for (const entry of entries) {
size += 2 + entry.key.length + 8;
}
return size;
}
private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
view.setUint32(offset, entries.length, false);
offset += 4;
for (const entry of entries) {
const encoder = new TextEncoder();
const keyBytes = encoder.encode(entry.key);
view.setUint16(offset, keyBytes.length, false);
offset += 2;
new Uint8Array(view.buffer).set(keyBytes, offset);
offset += keyBytes.length;
view.setUint32(offset, entry.blockOffset, false);
offset += 4;
view.setUint32(offset, entry.blockSize, false);
offset += 4;
}
return offset;
}
}
/**
* AriaEngine SSTable Builder — 构建有序字符串表
* @module engine/aria/index/sstable_builder
*
* 将排序后的 key-value 数据写入 SSTable 格式。
*
* v0.4.4 格式 v2（magic "SSTC"）修复：
* - 块大小估算改用 UTF-8 字节长度（TextEncoder 预编码），
* 此前用字符串 .length（UTF-16 码元）估算而实际写入 UTF-8 字节，
* 中文内容（1 字 3 字节）导致缓冲区低估 → 写入越界崩溃
* - keyLen/valueLen 从 u16 升级为 u32（此前 >64KB 的 value 长度被截断，
* 线性格式整体错乱）
* - 大 value 单条独立成块（切分逻辑基于字节估算）
*
* SSTable 文件布局 (v2):
* ┌──────────────────────────────────────────────┐
* │ Data Block 0 │
* │ Data Block 1 │
* │ ... │
* │ Index Block (block offset → key range) │
* │ Bloom Filter │
* │ Footer (32 bytes) │
* │ - index_offset (u32) │
* │ - index_size (u32) │
* │ - bloom_offset (u32) │
* │ - bloom_size (u32) │
* │ - bloom_hash_count (u32) │
* │ - entry_count (u32) │
* │ - magic_number (u32, 0x53535443 ="SSTC")│
* │ - checksum (u32) │
* └──────────────────────────────────────────────┘
* 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value]
* 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)]
*/
import { BloomFilter } from './bloom';
import type { IndexEntry } from '../types';
/** v1 格式魔数（"SSTB"，u16 长度字段，兼容旧文件读取） */
export const SSTABLE_MAGIC_V1 = 0x53535442;
/** v2 格式魔数（"SSTC"，u32 长度字段 + 字节精确估算，v0.4.4） */
export const SSTABLE_MAGIC_V2 = 0x53535443;
const SSTABLE_FOOTER_SIZE = 32;
// ---------------------------------------------------------------------------
// SSTableBuilder
// ---------------------------------------------------------------------------
interface EncodedEntry {
key: string;
keyBytes: Uint8Array;
valueBytes: Uint8Array;
}
export class SSTableBuilder {
private entries: [string, Record<string, unknown>][] = [];
private blockSizeLimit: number;
constructor(blockSizeLimit: number = 4096) {
this.blockSizeLimit = blockSizeLimit;
}
/** 添加一个 key-value 条目（必须按键排序添加） */
add(key: string, value: Record<string, unknown>): void {
this.entries.push([key, value]);
}
/**
* 构建 SSTable 文件的二进制数据（v2 格式）。
* 返回 { data: Uint8Array, indexEntries: IndexEntry[] }
*/
build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
// v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度，
// 字符串 .length 是 UTF-16 码元（中文 1 字 3 字节 vs 1 码元）→ 缓冲区低估越界
const encoder = new TextEncoder();
const encoded: EncodedEntry[] = this.entries.map(([key, value]) => ({
key,
keyBytes: encoder.encode(key),
valueBytes: encoder.encode(JSON.stringify(value)),
}));
const blocks = this.splitIntoBlocks(encoded);
const bloomFilter = new BloomFilter(this.entries.length);
// 预计算总大小（字节）
let totalSize = 0;
const blockOffsets: number[] = [];
for (const block of blocks) {
blockOffsets.push(totalSize);
totalSize += this.computeBlockSize(block);
}
// 索引块（块内最后一个 key 作为索引键）
const indexEntries: IndexEntry[] = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
indexEntries.push({
key: block[block.length - 1].key,
blockOffset: blockOffsets[i],
blockSize: this.computeBlockSize(block),
});
}
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
// 序列化 bloom filter 以获取其大小
const bloomData = bloomFilter.serialize();
const bloomSize = bloomData.byteLength;
// 写入到 buffer（包含 bloom block）
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
const buf = new ArrayBuffer(finalSize);
const view = new DataView(buf);
let offset = 0;
// ---- Data Blocks ----
for (const block of blocks) {
offset = this.writeDataBlock(view, offset, block, bloomFilter);
}
// ---- Index Block ----
const indexOffset = offset;
offset = this.writeIndexBlock(view, offset, indexEntries);
// ---- Bloom Filter Block ----
const bloomOffset = offset;
new Uint8Array(view.buffer).set(bloomData, offset);
offset += bloomSize;
// ---- Footer ----
const footerOffset = offset;
view.setUint32(footerOffset, indexOffset, false); // index_offset
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
view.setUint32(footerOffset + 20, this.entries.length, false);
view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
return {
sstableData: new Uint8Array(buf),
indexEntries,
};
}
/** 获取条目数 */
getEntryCount(): number {
return this.entries.length;
}
// -----------------------------------------------------------------------
// 内部
// -----------------------------------------------------------------------
/** 按 UTF-8 字节大小切分数据块；大 value 单条独立成块 */
private splitIntoBlocks(encoded: EncodedEntry[]): EncodedEntry[][] {
const blocks: EncodedEntry[][] = [];
let current: EncodedEntry[] = [];
for (const entry of encoded) {
current.push(entry);
// v0.4.4-fix: 基于字节估算；单条超大条目（length===1）独立成块不强行切分
if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) {
blocks.push(current.slice(0, -1));
current = [entry];
}
}
if (current.length > 0) blocks.push(current);
return blocks;
}
/** 块字节大小：entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */
private computeBlockSize(block: EncodedEntry[]): number {
let size = 4;
for (const e of block) {
size += 4 + e.keyBytes.length + 4 + e.valueBytes.length;
}
return size;
}
private writeDataBlock(
view: DataView,
offset: number,
block: EncodedEntry[],
bloomFilter: BloomFilter,
): number {
// entry count
view.setUint32(offset, block.length, false);
offset += 4;
for (const e of block) {
// v0.4.4-fix: 长度字段 u32（此前 u16 截断 >64KB 的 value）
if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) {
throw new Error('SSTable entry too large (exceeds u32 length field)');
}
view.setUint32(offset, e.keyBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(e.keyBytes, offset);
offset += e.keyBytes.length;
view.setUint32(offset, e.valueBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(e.valueBytes, offset);
offset += e.valueBytes.length;
// 插入 bloom filter
bloomFilter.insert(e.key);
}
return offset;
}
private estimateIndexBlockSize(entries: IndexEntry[]): number {
// entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32)
let size = 4;
const encoder = new TextEncoder();
for (const entry of entries) {
size += 4 + encoder.encode(entry.key).byteLength + 8;
}
return size;
}
private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
view.setUint32(offset, entries.length, false);
offset += 4;
for (const entry of entries) {
const encoder = new TextEncoder();
const keyBytes = encoder.encode(entry.key);
view.setUint32(offset, keyBytes.length, false);
offset += 4;
new Uint8Array(view.buffer).set(keyBytes, offset);
offset += keyBytes.length;
view.setUint32(offset, entry.blockOffset, false);
offset += 4;
view.setUint32(offset, entry.blockSize, false);
offset += 4;
}
return offset;
}
}