MetonaSqlark/src/engine/aria/index/sstable_builder.ts

/**
 * AriaEngine SSTable Builder — 构建有序字符串表
 * @module engine/aria/index/sstable_builder
 *
 * 将排序后的 key-value 数据写入 SSTable 格式。
 *
 * v0.4.4 格式 v2（magic "SSTC"）修复：
 * - 块大小估算改用 UTF-8 字节长度（TextEncoder 预编码），
 *   此前用字符串 .length（UTF-16 码元）估算而实际写入 UTF-8 字节，
 *   中文内容（1 字 3 字节）导致缓冲区低估 → 写入越界崩溃
 * - keyLen/valueLen 从 u16 升级为 u32（此前 >64KB 的 value 长度被截断，
 *   线性格式整体错乱）
 * - 大 value 单条独立成块（切分逻辑基于字节估算）
 *
 * SSTable 文件布局 (v2):
 * ┌──────────────────────────────────────────────┐
 * │  Data Block 0                               │
 * │  Data Block 1                               │
 * │  ...                                        │
 * │  Index Block (block offset → key range)     │
 * │  Bloom Filter                               │
 * │  Footer (32 bytes)                          │
 * │    - index_offset  (u32)                    │
 * │    - index_size    (u32)                    │
 * │    - bloom_offset  (u32)                    │
 * │    - bloom_size    (u32)                    │
 * │    - bloom_hash_count (u32)                 │
 * │    - entry_count   (u32)                    │
 * │    - magic_number  (u32, 0x53535443 ="SSTC")│
 * │    - checksum      (u32)                    │
 * └──────────────────────────────────────────────┘
 * 数据块条目: entryCount(u32) + [keyLen(u32) + key + valueLen(u32) + value]
 * 索引块条目: [keyLen(u32) + key + blockOffset(u32) + blockSize(u32)]
 */

import { BloomFilter } from './bloom';
import { DEFAULT_BLOOM_BITS_PER_KEY } from '../types';
import { crc32 } from '../crc32';
import type { IndexEntry } from '../types';

/** v1 格式魔数（"SSTB"，u16 长度字段，兼容旧文件读取） */
export const SSTABLE_MAGIC_V1 = 0x53535442;
/** v2 格式魔数（"SSTC"，u32 长度字段 + 字节精确估算，v0.4.4） */
export const SSTABLE_MAGIC_V2 = 0x53535443;
const SSTABLE_FOOTER_SIZE = 32;

// ---------------------------------------------------------------------------
//  SSTableBuilder
// ---------------------------------------------------------------------------

interface EncodedEntry {
  key: string;
  keyBytes: Uint8Array;
  valueBytes: Uint8Array;
}

export class SSTableBuilder {
  /** Bloom Filter 每 key 位数（由配置透传） */
  private bloomBitsPerKey: number;
  private entries: [string, Record<string, unknown>][] = [];
  private blockSizeLimit: number;

  /**
   * @param blockSizeLimit 块大小上限（字节）
   * @param bloomBitsPerKey Bloom Filter 每 key 位数（v0.8.0 审查修复：此前
   *        无论 `bloomFilterBitsPerKey` 配成多少，这里都写死用默认值 ——
   *        配置项被接受却完全不起作用）
   */
  constructor(
    blockSizeLimit: number = 4096,
    bloomBitsPerKey: number = DEFAULT_BLOOM_BITS_PER_KEY,
  ) {
    this.blockSizeLimit = blockSizeLimit;
    this.bloomBitsPerKey = Number.isFinite(bloomBitsPerKey) && bloomBitsPerKey > 0
      ? Math.floor(bloomBitsPerKey)
      : DEFAULT_BLOOM_BITS_PER_KEY;
  }

  /** 添加一个 key-value 条目（必须按键排序添加） */
  add(key: string, value: Record<string, unknown>): void {
    this.entries.push([key, value]);
  }

  /**
   * 构建 SSTable 文件的二进制数据（v2 格式）。
   * 返回 { data: Uint8Array, indexEntries: IndexEntry[] }
   */
  build(): { sstableData: Uint8Array; indexEntries: IndexEntry[] } {
    // v0.4.4-fix: 预编码全部条目 — 块大小估算必须基于 UTF-8 字节长度，
    // 字符串 .length 是 UTF-16 码元（中文 1 字 3 字节 vs 1 码元）→ 缓冲区低估越界
    const encoder = new TextEncoder();
    const encoded: EncodedEntry[] = this.entries.map(([key, value]) => ({
      key,
      keyBytes: encoder.encode(key),
      valueBytes: encoder.encode(JSON.stringify(value)),
    }));

    const blocks = this.splitIntoBlocks(encoded);
    const bloomFilter = new BloomFilter(this.entries.length, this.bloomBitsPerKey);

    // 预计算总大小（字节）
    let totalSize = 0;
    const blockOffsets: number[] = [];
    for (const block of blocks) {
      blockOffsets.push(totalSize);
      totalSize += this.computeBlockSize(block);
    }

    // 索引块（块内最后一个 key 作为索引键）
    const indexEntries: IndexEntry[] = [];
    for (let i = 0; i < blocks.length; i++) {
      const block = blocks[i];
      indexEntries.push({
        key: block[block.length - 1].key,
        blockOffset: blockOffsets[i],
        blockSize: this.computeBlockSize(block),
      });
    }

    const indexBlockSize = this.estimateIndexBlockSize(indexEntries);

    // 序列化 bloom filter 以获取其大小
    const bloomData = bloomFilter.serialize();
    const bloomSize = bloomData.byteLength;

    // 写入到 buffer（包含 bloom block）
    const finalSize = totalSize + indexBlockSize + bloomSize + SSTABLE_FOOTER_SIZE;
    const buf = new ArrayBuffer(finalSize);
    const view = new DataView(buf);

    let offset = 0;

    // ---- Data Blocks ----
    for (const block of blocks) {
      offset = this.writeDataBlock(view, offset, block, bloomFilter);
    }

    // ---- Index Block ----
    const indexOffset = offset;
    offset = this.writeIndexBlock(view, offset, indexEntries);

    // ---- Bloom Filter Block ----
    const bloomOffset = offset;
    new Uint8Array(view.buffer).set(bloomData, offset);
    offset += bloomSize;

    // ---- Footer ----
    const footerOffset = offset;
    view.setUint32(footerOffset, indexOffset, false);        // index_offset
    view.setUint32(footerOffset + 4, indexBlockSize, false); // index_size
    view.setUint32(footerOffset + 8, bloomOffset, false);    // bloom_offset
    view.setUint32(footerOffset + 12, bloomSize, false);     // bloom_size
    view.setUint32(footerOffset + 16, bloomFilter.getHashCount(), false);
    view.setUint32(footerOffset + 20, this.entries.length, false);
    view.setUint32(footerOffset + 24, SSTABLE_MAGIC_V2, false);
    // checksum 字段先写 0，全部字节就绪后计算整文件 CRC32 再回填
    view.setUint32(footerOffset + 28, 0, false);

    // v0.4.5: 真实 CRC-32 校验和 — 覆盖除自身（最后 4 字节）外的全部内容。
    // checksum 永远非 0（计算结果为 0 时用 1 代替），读取端以 0 识别旧版无校验文件
    const all = new Uint8Array(buf);
    let checksum = crc32(all.subarray(0, all.byteLength - 4));
    if (checksum === 0) checksum = 1;
    view.setUint32(footerOffset + 28, checksum, false);

    return {
      sstableData: new Uint8Array(buf),
      indexEntries,
    };
  }

  /** 获取条目数 */
  getEntryCount(): number {
    return this.entries.length;
  }

  // -----------------------------------------------------------------------
  //  内部
  // -----------------------------------------------------------------------

  /** 按 UTF-8 字节大小切分数据块；大 value 单条独立成块 */
  private splitIntoBlocks(encoded: EncodedEntry[]): EncodedEntry[][] {
    const blocks: EncodedEntry[][] = [];
    let current: EncodedEntry[] = [];

    for (const entry of encoded) {
      current.push(entry);
      // v0.4.4-fix: 基于字节估算；单条超大条目（length===1）独立成块不强行切分
      if (this.computeBlockSize(current) >= this.blockSizeLimit && current.length > 1) {
        blocks.push(current.slice(0, -1));
        current = [entry];
      }
    }
    if (current.length > 0) blocks.push(current);

    return blocks;
  }

  /** 块字节大小：entryCount(u32) + 每对 [keyLen(u32) + key + valueLen(u32) + value] */
  private computeBlockSize(block: EncodedEntry[]): number {
    let size = 4;
    for (const e of block) {
      size += 4 + e.keyBytes.length + 4 + e.valueBytes.length;
    }
    return size;
  }

  private writeDataBlock(
    view: DataView,
    offset: number,
    block: EncodedEntry[],
    bloomFilter: BloomFilter,
  ): number {
    // entry count
    view.setUint32(offset, block.length, false);
    offset += 4;

    for (const e of block) {
      // v0.4.4-fix: 长度字段 u32（此前 u16 截断 >64KB 的 value）
      if (e.keyBytes.length > 0xFFFFFFFF || e.valueBytes.length > 0xFFFFFFFF) {
        throw new Error('SSTable entry too large (exceeds u32 length field)');
      }
      view.setUint32(offset, e.keyBytes.length, false);
      offset += 4;
      new Uint8Array(view.buffer).set(e.keyBytes, offset);
      offset += e.keyBytes.length;
      view.setUint32(offset, e.valueBytes.length, false);
      offset += 4;
      new Uint8Array(view.buffer).set(e.valueBytes, offset);
      offset += e.valueBytes.length;

      // 插入 bloom filter
      bloomFilter.insert(e.key);
    }

    return offset;
  }

  private estimateIndexBlockSize(entries: IndexEntry[]): number {
    // entryCount(u32) + each: keyLen(u32)+key+blockOffset(u32)+blockSize(u32)
    let size = 4;
    const encoder = new TextEncoder();
    for (const entry of entries) {
      size += 4 + encoder.encode(entry.key).byteLength + 8;
    }
    return size;
  }

  private writeIndexBlock(view: DataView, offset: number, entries: IndexEntry[]): number {
    view.setUint32(offset, entries.length, false);
    offset += 4;

    for (const entry of entries) {
      const encoder = new TextEncoder();
      const keyBytes = encoder.encode(entry.key);
      view.setUint32(offset, keyBytes.length, false);
      offset += 4;
      new Uint8Array(view.buffer).set(keyBytes, offset);
      offset += keyBytes.length;
      view.setUint32(offset, entry.blockOffset, false);
      offset += 4;
      view.setUint32(offset, entry.blockSize, false);
      offset += 4;
    }

    return offset;
  }
}