release: v0.3.2 — 质量加固 + SQL扩展 + 表达式 + 并发同步
v0.2.6 质量加固: - 修复 AriaEngine 二级索引 SSTable 互相覆盖(命名空间隔离) - 修复 LSM 多版本读取顺序错误 + MergeIterator 取最新来源 - 重写 LZ4 压缩器(往返一致性 + 缓冲区溢出) - sstableCache LRU 上限 + 预加载兜底(BufferPool 配置生效) - 修复 React/Vue 集成 import type 运行时 bug + exports 子路径 - 新增 38 个测试(LZ4往返/Crypto/集成), 删除伪测试 v0.3.0 SQL 功能扩展: - 多语句 parseAll + 事务语句 BEGIN/COMMIT/ROLLBACK - INSERT INTO ... SELECT + UNION/UNION ALL + EXISTS 关联子查询 - CREATE/DROP INDEX 五引擎实现 + 别名 WHERE 修复 - benchmark 页面 + 36 个新测试 v0.3.1 表达式与性能: - CASE WHEN 表达式(SELECT 列/WHERE/聚合) - JOIN + 关联子查询逐行绑定 - WAL 批量组提交(写放大 O(N)→O(1)) - 修复 pending frozen 可见性 + flush 缓存竞争 v0.3.2 并发: - CASE WHEN 用于 WHERE/聚合 + JOIN 哈希连接 - 多标签页同步(multiTabSync + BroadcastChannel) - IndexedDB schema 持久化(reopen 后表结构恢复) - 修复 where-matcher 顶层 $not - 修复 CJS 产物 .js 被 ESM 解析(exports 空) — .cjs 后缀 + exports 修正 - 836 测试 / 44 套件 / 81.0% 覆盖率
This commit is contained in:
thzxx
@@ -1,249 +1,247 @@
|
||||
/**
|
||||
* AriaEngine SSTable Builder — 构建有序字符串表
|
||||
* @module engine/aria/index/sstable_builder
|
||||
*
|
||||
* 将排序后的 key-value 数据写入 SSTable 格式。
|
||||
*
|
||||
* SSTable 文件布局:
|
||||
* ┌──────────────────────────────────────────────┐
|
||||
* │ Data Block 0 │
|
||||
* │ Data Block 1 │
|
||||
* │ ... │
|
||||
* │ Index Block (block offset → key range) │
|
||||
* │ Bloom Filter │
|
||||
* │ Footer (32 bytes) │
|
||||
* │ - index_offset (u32) │
|
||||
* │ - index_size (u32) │
|
||||
* │ - bloom_offset (u32) │
|
||||
* │ - bloom_size (u32) │
|
||||
* │ - bloom_hash_count (u32) │
|
||||
* │ - entry_count (u32) │
|
||||
* │ - magic_number (u32, 0x53535442 ="SSTB")│
|
||||
* │ - checksum (u32) │
|
||||
* └──────────────────────────────────────────────┘
|
||||
*/
|
||||
|
||||
import { BloomFilter } from './bloom';
|
||||
import type { IndexEntry, DataBlock } from '../types';
|
||||
|
||||
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
||||
const SSTABLE_FOOTER_SIZE = 32;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSTableBuilder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class SSTableBuilder {
|
||||
private entries: [string, Record<string, unknown>][] = [];
|
||||
private currentBlock: [string, Record<string, unknown>][] = [];
|
||||
private currentBlockStartKey = '';
|
||||
private blockSizeLimit: number;
|
||||
|
||||
constructor(blockSizeLimit: number = 4096) {
|
||||
this.blockSizeLimit = blockSizeLimit;
|
||||
}
|
||||
|
||||
/** 添加一个 key-value 条目（必须按键排序添加） */
|
||||
add(key: string, value: Record<string, unknown>): void {
|
||||
if (this.currentBlock.length === 0) {
|
||||
this.currentBlockStartKey = key;
|
||||
}
|
||||
|
||||
this.currentBlock.push([key, value]);
|
||||
this.entries.push([key, value]);
|
||||
|
||||
// 如果当前 Block 达到大小限制，切割
|
||||
const estimated = this.estimateBlockSize();
|
||||
if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) {
|
||||
// 当前 block 结束（不在这里切割，在 build 时统一处理）
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 SSTable 文件的二进制数据。
|
||||
* 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
|
||||
*/
|
||||
build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
|
||||
const blocks = this.splitIntoBlocks();
|
||||
const bloomFilter = new BloomFilter(this.entries.length);
|
||||
|
||||
// 预计算总大小
|
||||
let totalSize = 0;
|
||||
const blockOffsets: number[] = [];
|
||||
|
||||
for (const block of blocks) {
|
||||
blockOffsets.push(totalSize);
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
totalSize += blockSize;
|
||||
}
|
||||
|
||||
// 索引块
|
||||
const indexEntries: IndexEntry[] = [];
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
const lastKey = block[block.length - 1][0];
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
indexEntries.push({
|
||||
key: lastKey,
|
||||
blockOffset: blockOffsets[i],
|
||||
blockSize,
|
||||
});
|
||||
}
|
||||
|
||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||
|
||||
// 序列化 bloom filter 以获取其大小
|
||||
const bloomData = bloomFilter.serialize();
|
||||
const bloomSize = bloomData.byteLength;
|
||||
|
||||
// 写入到 buffer（包含 bloom block）
|
||||
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||
const buf = new ArrayBuffer(finalSize);
|
||||
const view = new DataView(buf);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// ---- Data Blocks ----
|
||||
for (const block of blocks) {
|
||||
offset = this.writeDataBlock(view, offset, block, bloomFilter);
|
||||
}
|
||||
|
||||
// ---- Index Block ----
|
||||
const indexOffset = offset;
|
||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||
|
||||
// ---- Bloom Filter Block ----
|
||||
const bloomOffset = offset;
|
||||
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||
offset += bloomSize;
|
||||
|
||||
// ---- Footer ----
|
||||
const footerOffset = offset;
|
||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
|
||||
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
|
||||
|
||||
return {
|
||||
sstableData: new Uint8Array(buf),
|
||||
indexEntries,
|
||||
};
|
||||
}
|
||||
|
||||
/** 获取条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.entries.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private splitIntoBlocks(): [string, Record<string, unknown>][][] {
|
||||
const blocks: [string, Record<string, unknown>][][] = [];
|
||||
let current: [string, Record<string, unknown>][] = [];
|
||||
|
||||
for (const entry of this.entries) {
|
||||
current.push(entry);
|
||||
if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
|
||||
blocks.push(current.slice(0, -1));
|
||||
current = [entry];
|
||||
}
|
||||
}
|
||||
if (current.length > 0) blocks.push(current);
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private estimateBlockSize(): number {
|
||||
return this.estimateBlockSizeFromEntries(this.currentBlock);
|
||||
}
|
||||
|
||||
private estimateBlockSizeFromEntries(entries: [string, unknown][]): number {
|
||||
let size = 0;
|
||||
for (const [key, value] of entries) {
|
||||
size += 4 + key.length + JSON.stringify(value).length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private computeBlockSize(block: [string, unknown][]): number {
|
||||
// entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
|
||||
let size = 4;
|
||||
for (const [key, value] of block) {
|
||||
const json = JSON.stringify(value);
|
||||
size += 2 + key.length + 2 + json.length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeDataBlock(
|
||||
view: DataView,
|
||||
offset: number,
|
||||
block: [string, Record<string, unknown>][],
|
||||
bloomFilter: BloomFilter,
|
||||
): number {
|
||||
const start = offset;
|
||||
|
||||
// entry count
|
||||
view.setUint32(offset, block.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const [key, value] of block) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(key);
|
||||
const valueBytes = encoder.encode(JSON.stringify(value));
|
||||
|
||||
// key length
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
// key
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
// value length
|
||||
view.setUint16(offset, valueBytes.length, false);
|
||||
offset += 2;
|
||||
// value
|
||||
new Uint8Array(view.buffer).set(valueBytes, offset);
|
||||
offset += valueBytes.length;
|
||||
|
||||
// 插入 bloom filter
|
||||
bloomFilter.insert(key);
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
private estimateIndexBlockSize(entries: IndexEntry[]): number {
|
||||
// entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
|
||||
let size = 4;
|
||||
for (const entry of entries) {
|
||||
size += 2 + entry.key.length + 8;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
|
||||
view.setUint32(offset, entries.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const entry of entries) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(entry.key);
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
view.setUint32(offset, entry.blockOffset, false);
|
||||
offset += 4;
|
||||
view.setUint32(offset, entry.blockSize, false);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AriaEngine SSTable Builder — 构建有序字符串表
|
||||
* @module engine/aria/index/sstable_builder
|
||||
*
|
||||
* 将排序后的 key-value 数据写入 SSTable 格式。
|
||||
*
|
||||
* SSTable 文件布局:
|
||||
* ┌──────────────────────────────────────────────┐
|
||||
* │ Data Block 0 │
|
||||
* │ Data Block 1 │
|
||||
* │ ... │
|
||||
* │ Index Block (block offset → key range) │
|
||||
* │ Bloom Filter │
|
||||
* │ Footer (32 bytes) │
|
||||
* │ - index_offset (u32) │
|
||||
* │ - index_size (u32) │
|
||||
* │ - bloom_offset (u32) │
|
||||
* │ - bloom_size (u32) │
|
||||
* │ - bloom_hash_count (u32) │
|
||||
* │ - entry_count (u32) │
|
||||
* │ - magic_number (u32, 0x53535442 ="SSTB")│
|
||||
* │ - checksum (u32) │
|
||||
* └──────────────────────────────────────────────┘
|
||||
*/
|
||||
|
||||
import { BloomFilter } from './bloom';
|
||||
import type { IndexEntry } from '../types';
|
||||
|
||||
const SSTABLE_MAGIC = 0x53535442; // "SSTB"
|
||||
const SSTABLE_FOOTER_SIZE = 32;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SSTableBuilder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export class SSTableBuilder {
|
||||
private entries: [string, Record<string, unknown>][] = [];
|
||||
private currentBlock: [string, Record<string, unknown>][] = [];
|
||||
private currentBlockStartKey = '';
|
||||
private blockSizeLimit: number;
|
||||
|
||||
constructor(blockSizeLimit: number = 4096) {
|
||||
this.blockSizeLimit = blockSizeLimit;
|
||||
}
|
||||
|
||||
/** 添加一个 key-value 条目（必须按键排序添加） */
|
||||
add(key: string, value: Record<string, unknown>): void {
|
||||
if (this.currentBlock.length === 0) {
|
||||
this.currentBlockStartKey = key;
|
||||
}
|
||||
|
||||
this.currentBlock.push([key, value]);
|
||||
this.entries.push([key, value]);
|
||||
|
||||
// 如果当前 Block 达到大小限制，切割
|
||||
const estimated = this.estimateBlockSize();
|
||||
if (estimated >= this.blockSizeLimit && this.currentBlock.length > 1) {
|
||||
// 当前 block 结束（不在这里切割，在 build 时统一处理）
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 SSTable 文件的二进制数据。
|
||||
* 返回 { data: Uint8Array, indexEntries: IndexEntry[], bloomFilter: BloomFilter }
|
||||
*/
|
||||
build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
|
||||
const blocks = this.splitIntoBlocks();
|
||||
const bloomFilter = new BloomFilter(this.entries.length);
|
||||
|
||||
// 预计算总大小
|
||||
let totalSize = 0;
|
||||
const blockOffsets: number[] = [];
|
||||
|
||||
for (const block of blocks) {
|
||||
blockOffsets.push(totalSize);
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
totalSize += blockSize;
|
||||
}
|
||||
|
||||
// 索引块
|
||||
const indexEntries: IndexEntry[] = [];
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
const lastKey = block[block.length - 1][0];
|
||||
const blockSize = this.computeBlockSize(block);
|
||||
indexEntries.push({
|
||||
key: lastKey,
|
||||
blockOffset: blockOffsets[i],
|
||||
blockSize,
|
||||
});
|
||||
}
|
||||
|
||||
const indexBlockSize = this.estimateIndexBlockSize(indexEntries);
|
||||
|
||||
// 序列化 bloom filter 以获取其大小
|
||||
const bloomData = bloomFilter.serialize();
|
||||
const bloomSize = bloomData.byteLength;
|
||||
|
||||
// 写入到 buffer（包含 bloom block）
|
||||
const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
|
||||
const buf = new ArrayBuffer(finalSize);
|
||||
const view = new DataView(buf);
|
||||
|
||||
let offset = 0;
|
||||
|
||||
// ---- Data Blocks ----
|
||||
for (const block of blocks) {
|
||||
offset = this.writeDataBlock(view, offset, block, bloomFilter);
|
||||
}
|
||||
|
||||
// ---- Index Block ----
|
||||
const indexOffset = offset;
|
||||
offset = this.writeIndexBlock(view, offset, indexEntries);
|
||||
|
||||
// ---- Bloom Filter Block ----
|
||||
const bloomOffset = offset;
|
||||
new Uint8Array(view.buffer).set(bloomData, offset);
|
||||
offset += bloomSize;
|
||||
|
||||
// ---- Footer ----
|
||||
const footerOffset = offset;
|
||||
view.setUint32(footerOffset, indexOffset, false); // index_offset
|
||||
view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
|
||||
view.setUint32(footerOffset + 8, bloomOffset, false); // bloom_offset
|
||||
view.setUint32(footerOffset + 12, bloomSize, false); // bloom_size
|
||||
view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
|
||||
view.setUint32(footerOffset + 20, this.entries.length, false);
|
||||
view.setUint32(footerOffset + 24, SSTABLE_MAGIC, false);
|
||||
view.setUint32(footerOffset + 28, 0, false); // checksum (simplified: 0)
|
||||
|
||||
return {
|
||||
sstableData: new Uint8Array(buf),
|
||||
indexEntries,
|
||||
};
|
||||
}
|
||||
|
||||
/** 获取条目数 */
|
||||
getEntryCount(): number {
|
||||
return this.entries.length;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// 内部
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private splitIntoBlocks(): [string, Record<string, unknown>][][] {
|
||||
const blocks: [string, Record<string, unknown>][][] = [];
|
||||
let current: [string, Record<string, unknown>][] = [];
|
||||
|
||||
for (const entry of this.entries) {
|
||||
current.push(entry);
|
||||
if (this.estimateBlockSizeFromEntries(current) >= this.blockSizeLimit && current.length > 1) {
|
||||
blocks.push(current.slice(0, -1));
|
||||
current = [entry];
|
||||
}
|
||||
}
|
||||
if (current.length > 0) blocks.push(current);
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private estimateBlockSize(): number {
|
||||
return this.estimateBlockSizeFromEntries(this.currentBlock);
|
||||
}
|
||||
|
||||
private estimateBlockSizeFromEntries(entries: [string, unknown][]): number {
|
||||
let size = 0;
|
||||
for (const [key, value] of entries) {
|
||||
size += 4 + key.length + JSON.stringify(value).length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private computeBlockSize(block: [string, unknown][]): number {
|
||||
// entryCount (u32) + 每对: keyLen(u16) + key + valueLen(u16) + value json
|
||||
let size = 4;
|
||||
for (const [key, value] of block) {
|
||||
const json = JSON.stringify(value);
|
||||
size += 2 + key.length + 2 + json.length;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeDataBlock(
|
||||
view: DataView,
|
||||
offset: number,
|
||||
block: [string, Record<string, unknown>][],
|
||||
bloomFilter: BloomFilter,
|
||||
): number {
|
||||
// entry count
|
||||
view.setUint32(offset, block.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const [key, value] of block) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(key);
|
||||
const valueBytes = encoder.encode(JSON.stringify(value));
|
||||
|
||||
// key length
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
// key
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
// value length
|
||||
view.setUint16(offset, valueBytes.length, false);
|
||||
offset += 2;
|
||||
// value
|
||||
new Uint8Array(view.buffer).set(valueBytes, offset);
|
||||
offset += valueBytes.length;
|
||||
|
||||
// 插入 bloom filter
|
||||
bloomFilter.insert(key);
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
private estimateIndexBlockSize(entries: IndexEntry[]): number {
|
||||
// entryCount(u32) + each: keyLen(u16)+key+blockOffset(u32)+blockSize(u32)
|
||||
let size = 4;
|
||||
for (const entry of entries) {
|
||||
size += 2 + entry.key.length + 8;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
|
||||
view.setUint32(offset, entries.length, false);
|
||||
offset += 4;
|
||||
|
||||
for (const entry of entries) {
|
||||
const encoder = new TextEncoder();
|
||||
const keyBytes = encoder.encode(entry.key);
|
||||
view.setUint16(offset, keyBytes.length, false);
|
||||
offset += 2;
|
||||
new Uint8Array(view.buffer).set(keyBytes, offset);
|
||||
offset += keyBytes.length;
|
||||
view.setUint32(offset, entry.blockOffset, false);
|
||||
offset += 4;
|
||||
view.setUint32(offset, entry.blockSize, false);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user