feat(B-1): 统一行校验 choke point —— 消除三份分叉的校验实现(A12/A17)
背景(PLAN-v0.7.5.md 根因 1):
修复前有**三份**行校验实现,覆盖面各不相同:
位置 类型 required PK非空 maxLength min/max 未知列
engine/memory.ts(disk/hybrid 共用) ✓ ✓ ✓ ✗ ✗ 静默丢弃
engine/aria/index.ts → checkFieldType ✓ ✓ ✓ ✓ ✓ 静默丢弃
table/schema.ts ✓ ✓ ✓ ✓ ✓ 静默丢弃
后果一(A12):同一份 schema、同一条 INSERT 是否报约束错误取决于引擎选择 ——
`CREATE TABLE t (name STRING(3))` + 插入 'abcdef' 在 Aria 抛错,在
memory/disk/hybrid 静默写入超长值。
后果二(A17):四个引擎对未知列一律静默丢弃。`INSERT INTO t (id, nope) VALUES
('1',2)` 报成功,随后 `SELECT nope` 报 COLUMN_NOT_FOUND —— 同一列名在写路径与
读路径得到**相反结论**。TABLE API 直通路径尤其明显(executor 按 schema 列序
构造行,nope 那个位置根本没有值,所以连"校验 stmt.columns"都拦不住)。
根治方式:
1. 新增 src/table/validation.ts —— 唯一校验定义 `compileValidator(schema)`,
约束覆盖面取三者并集,并把**规范化**(default 填充、undefined 跳过、
__proto__ 防污染)与校验放在同一处。
三种载荷形态刻意分成三个显式入口,不合成带 options 的函数:
- validateRow(row, knownColumns?) INSERT 语义(default 生效、缺列合法)
- validatePartial(row) UPDATE 语义(只校验出现的列)
- assertNoUnknownColumns 独立可复用的列名存在性检查
混成一个函数会让"required 是否生效"取决于调用方参数,重新引入跨路径差异。
2. MemoryEngine / AriaEngine 的私有 validateRow 改为委托;schema.ts 的公开
validateRow 同样委托(API 不变,实现只剩一份)。
3. 四个引擎新增 validatePayload(table, rows, mode)(IStorageEngine 契约),
Executor 在**任何副作用之前**调用:多行批量整体判定,错误消息一次列出全部
未知列与已知列清单。
4. executeInsert 显式校验 stmt.columns 全部存在(A17)。
5. UPDATE 的外键级联写入(applyUpdateCascade)从"直接赋值"改为过
validatePartial —— 此前 CASCADE 把新主键写进引用列时绕过 maxLength/min/max,
与 A12 属同一类"校验只在部分写入路径生效"。
连带修正(测试夹具本身不忠实,B-1 使其暴露):
- tests/engine/aria-cache.test.ts 的 makeRows 无条件返回 {id,name,age},
部分用例的表只有 {id,name} —— 多余列被静默丢弃所以"通过"。新增 rowsFor()
按 schema 裁剪,让夹具忠实反映表结构(而不是放宽校验)。
- tests/v073-fixes.test.ts "schema 外列不持久化" 改为断言写路径即拒绝,
并保留"合法行落盘后不含额外列"的检查。
验证:
- 新增 tests/v080-unified-validation.test.ts:8 项 × 4 引擎 + 9 项校验器
单元契约,共 41 断言;
- 全量 84 套件 / 1499 测试通过;typecheck(src+tests) 与 lint 零错误。
This commit is contained in:
@@ -33,6 +33,27 @@ function makeRows(count: number): Record<string, unknown>[] {
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 schema 列裁剪行(v0.8.0 B-1 连带修正)。
|
||||
*
|
||||
* 此前 `makeRows` 无条件返回 `{id, name, age}`,而部分用例的表只有 `{id, name}` ——
|
||||
* 多余列被引擎**静默丢弃**,测试因此"通过"。B-1 把未知列变成
|
||||
* COLUMN_NOT_FOUND 后这些看起来无关的用例暴露出来。
|
||||
*
|
||||
* 这里的修法是让夹具**忠实反映表结构**(而不是放宽校验):测试本就不该依赖
|
||||
* "写了不存在的列也不报错"这一行为。
|
||||
*/
|
||||
function rowsFor(schema: ReturnType<typeof createSchema>, count: number): Record<string, unknown>[] {
|
||||
const allowed = Object.keys(schema.columns);
|
||||
return makeRows(count).map((row) => {
|
||||
const picked: Record<string, unknown> = {};
|
||||
for (const key of allowed) {
|
||||
if (key in row) picked[key] = row[key];
|
||||
}
|
||||
return picked;
|
||||
});
|
||||
}
|
||||
|
||||
describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('缓存大小受 cacheLimitBytes 约束', async () => {
|
||||
const engine = createSmallCacheEngine(2); // 2 * 4096 = 8KB 上限
|
||||
@@ -82,11 +103,13 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('超大 SSTable 常驻缓存(驱逐会导致读取静默跳过整个文件)', async () => {
|
||||
const engine = createSmallCacheEngine(1); // 4KB 上限,单个 SSTable 必然超过
|
||||
await engine.open('cache-oversized-pin', 1);
|
||||
await engine.createTable(createSchema('users', {
|
||||
const schema = createSchema('users', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string' },
|
||||
}));
|
||||
await engine.insert('users', makeRows(300));
|
||||
});
|
||||
await engine.createTable(schema);
|
||||
// v0.8.0(B-1):夹具按 schema 裁剪(此前 makeRows 多带的 age 列被静默丢弃)
|
||||
await engine.insert('users', rowsFor(schema, 300));
|
||||
const lsm = (engine as any).lsm as {
|
||||
flush(): Promise<void>;
|
||||
getCacheSize(): number;
|
||||
@@ -109,12 +132,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('缓存驱逐后全表扫描仍返回完整数据(prefetch 兜底)', async () => {
|
||||
const engine = createSmallCacheEngine(1); // 4KB 上限,必然触发驱逐
|
||||
await engine.open('cache-evict-fullscan', 1);
|
||||
await engine.createTable(createSchema('users', {
|
||||
const schema = createSchema('users', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string' },
|
||||
}));
|
||||
});
|
||||
await engine.createTable(schema);
|
||||
|
||||
const rows = makeRows(300);
|
||||
// v0.8.0(B-1):夹具按 schema 裁剪(此前 makeRows 多带的 age 列被静默丢弃)
|
||||
const rows = rowsFor(schema, 300);
|
||||
await engine.insert('users', rows);
|
||||
|
||||
// v0.8.0: 这是最关键的数据完整性断言 —— 缓存上限极小(4KB)而 SSTable 更大时,
|
||||
@@ -128,12 +153,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('缓存驱逐后 PK 等值查询仍正确(prefetchKeys 兜底)', async () => {
|
||||
const engine = createSmallCacheEngine(1);
|
||||
await engine.open('cache-evict-pk', 1);
|
||||
await engine.createTable(createSchema('users', {
|
||||
const schema = createSchema('users', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string' },
|
||||
}));
|
||||
});
|
||||
await engine.createTable(schema);
|
||||
|
||||
const rows = makeRows(300);
|
||||
// v0.8.0(B-1):夹具按 schema 裁剪(此前 makeRows 多带的 age 列被静默丢弃)
|
||||
const rows = rowsFor(schema, 300);
|
||||
await engine.insert('users', rows);
|
||||
|
||||
// 分散查询多个 PK,每次都会经历 驱逐+重新加载
|
||||
@@ -203,14 +230,16 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('写入路径不突破缓存上限(flush 后立即裁剪)', async () => {
|
||||
const engine = createSmallCacheEngine(2);
|
||||
await engine.open('cache-write-bound', 1);
|
||||
await engine.createTable(createSchema('users', {
|
||||
const schema = createSchema('users', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string' },
|
||||
}));
|
||||
});
|
||||
await engine.createTable(schema);
|
||||
|
||||
// 分批写入,每批都触发多次 flush
|
||||
for (let batch = 0; batch < 10; batch++) {
|
||||
await engine.insert('users', makeRows(30).map((r, i) => ({ ...r, id: `b${batch}_u${i}` })));
|
||||
// v0.8.0(B-1):夹具按 schema 裁剪(此前 makeRows 的 age 列被静默丢弃)
|
||||
await engine.insert('users', rowsFor(schema, 30).map((r, i) => ({ ...r, id: `b${batch}_u${i}` })));
|
||||
const lsm = (engine as any).lsm;
|
||||
expect(lsm.getCacheSize()).toBeLessThanOrEqual(lsm.getCacheLimit());
|
||||
}
|
||||
@@ -278,12 +307,14 @@ describe('AriaEngine SSTable 缓存内存上限', () => {
|
||||
test('回归:删除后 tombstone 跨 flush 仍生效(不残留旧数据)', async () => {
|
||||
const engine = createSmallCacheEngine(4);
|
||||
await engine.open('regression-tombstone', 1);
|
||||
await engine.createTable(createSchema('users', {
|
||||
const schema = createSchema('users', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
age: { type: 'number', index: true },
|
||||
}));
|
||||
});
|
||||
await engine.createTable(schema);
|
||||
|
||||
await engine.insert('users', makeRows(120));
|
||||
// v0.8.0(B-1):夹具按 schema 裁剪(此前 makeRows 的 name 列被静默丢弃)
|
||||
await engine.insert('users', rowsFor(schema, 120));
|
||||
|
||||
// 分批删除,触发多次 flush
|
||||
for (let batch = 0; batch < 4; batch++) {
|
||||
|
||||
@@ -447,10 +447,19 @@ describe('v0.7.3: KVStore insert 持久化 validated 行', () => {
|
||||
await db2.close();
|
||||
});
|
||||
|
||||
test('schema 外列不持久化', async () => {
|
||||
test('schema 外列在写路径即被拒绝(v0.8.0 B-1 由静默丢弃改为报错)', async () => {
|
||||
const db = await MetonaSqlark.create({ name: 'v073-kv-extra-col', mode: 'disk', diskEngine: 'memory' });
|
||||
await db.defineTable('users', { id: { type: 'string', primaryKey: true } });
|
||||
await db.table('users').insert({ id: '1', junk: 'x' } as never);
|
||||
// v0.8.0(B-1 / 缺陷 A17):此断言此前写作"junk 被静默丢弃、插入成功"。
|
||||
// 静默丢弃是错的:INSERT 报成功、`SELECT junk` 又报 COLUMN_NOT_FOUND ——
|
||||
// 同一列名在写/读路径得到相反结论,用户以为写进去了。
|
||||
// 现在写路径直接抛 COLUMN_NOT_FOUND(与读路径同一错误码),
|
||||
// 因此"不持久化"这一目的以更强的形式成立(连内存表都不会有脏列)。
|
||||
await expect(db.table('users').insert({ id: '1', junk: 'x' } as never)).rejects.toMatchObject({
|
||||
code: 'COLUMN_NOT_FOUND',
|
||||
});
|
||||
// 合法的行仍可写入,且落盘后不含任何额外列
|
||||
await db.table('users').insert({ id: '1' } as never);
|
||||
await db.close();
|
||||
const db2 = await MetonaSqlark.create({ name: 'v073-kv-extra-col', mode: 'disk', diskEngine: 'memory' });
|
||||
const rows = rowsOf<Record<string, unknown>>(await db2.query('SELECT * FROM users'));
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* v0.8.0 回归套件 —— B-1 统一行校验(工作流 B 第 1 项)
|
||||
* ============================================================================
|
||||
* 背景(PLAN-v0.7.5.md 根因 1 + 缺陷 A12/A17):
|
||||
* 修复前存在**三份**行校验实现,覆盖面各不相同 —— Memory/KVStore/Hybrid 共用
|
||||
* 一份"只查类型"的实现(**没有** maxLength/min/max),Aria 走 checkFieldType
|
||||
* (有约束),schema.ts 是第三份。于是:
|
||||
*
|
||||
* A12: `CREATE TABLE t (name STRING(3))` + `INSERT ... 'abcdef'`
|
||||
* → Aria 抛 VALIDATION_ERROR,memory/disk/hybrid **静默写入超长值**。
|
||||
* 用户的约束是否生效取决于选了哪个引擎。
|
||||
*
|
||||
* A17: `INSERT INTO t (id, nope) VALUES ('1', 2)`
|
||||
* → 四个引擎全部**静默丢弃 nope**、INSERT 报成功;随后 `SELECT nope`
|
||||
* 又抛 COLUMN_NOT_FOUND —— 同一列名在写路径与读路径得到相反结论。
|
||||
*
|
||||
* 本套件对四引擎逐一断言相同结论,锁定"校验只有一份实现"。
|
||||
*/
|
||||
import { MetonaSqlark } from '../src/core';
|
||||
import { compileValidator } from '../src/table/validation';
|
||||
import { createSchema } from '../src/table/schema';
|
||||
import { rows as rowsOf, expectCode } from './helpers/assertions';
|
||||
import type { DatabaseConfig } from '../src/constants';
|
||||
|
||||
const ENGINES: Array<[string, DatabaseConfig['mode'], Partial<DatabaseConfig>]> = [
|
||||
['memory', 'memory', {}],
|
||||
['disk', 'disk', {}],
|
||||
['hybrid', 'hybrid', {}],
|
||||
['aria', 'aria', { diskEngine: 'memory' }],
|
||||
];
|
||||
|
||||
describe('[v0.8.0] B-1 统一行校验 —— 四引擎行为必须一致', () => {
|
||||
describe.each(ENGINES)('%s 引擎', (label, mode, extra) => {
|
||||
let db: MetonaSqlark;
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await MetonaSqlark.create({ name: `b1-${label}-${Math.random().toString(36).slice(2)}`, mode, ...extra });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await db.close();
|
||||
});
|
||||
|
||||
// ---- A12: maxLength / min / max ----
|
||||
|
||||
it('maxLength 超长 → VALIDATION_ERROR(A12)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string', maxLength: 3 },
|
||||
});
|
||||
await expectCode(db.query("INSERT INTO t VALUES ('1', 'abcd')"), 'VALIDATION_ERROR');
|
||||
// 边界值恰好等于上限必须通过
|
||||
await db.query("INSERT INTO t VALUES ('2', 'abc')");
|
||||
expect(rowsOf(await db.query('SELECT id FROM t'))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('min / max 越界 → VALIDATION_ERROR(A12)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
n: { type: 'number', min: 1, max: 10 },
|
||||
});
|
||||
await expectCode(db.query("INSERT INTO t VALUES ('1', 0)"), 'VALIDATION_ERROR');
|
||||
await expectCode(db.query("INSERT INTO t VALUES ('2', 11)"), 'VALIDATION_ERROR');
|
||||
await db.query("INSERT INTO t VALUES ('3', 1), ('4', 10)");
|
||||
expect(rowsOf(await db.query('SELECT id FROM t'))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('UPDATE 同样受 maxLength / min / max 约束(A12)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string', maxLength: 3 },
|
||||
n: { type: 'number', min: 0 },
|
||||
});
|
||||
await db.query("INSERT INTO t VALUES ('1', 'abc', 5)");
|
||||
await expectCode(db.query("UPDATE t SET name = 'abcd' WHERE id = '1'"), 'VALIDATION_ERROR');
|
||||
await expectCode(db.query('UPDATE t SET n = -1 WHERE id = \'1\''), 'VALIDATION_ERROR');
|
||||
// 失败的更新不得留下任何部分修改
|
||||
const after = rowsOf<{ name: string; n: number }>(await db.query('SELECT name, n FROM t'));
|
||||
expect(after).toEqual([{ name: 'abc', n: 5 }]);
|
||||
});
|
||||
|
||||
// ---- A17: 未知列 ----
|
||||
|
||||
it('INSERT 未知列 → COLUMN_NOT_FOUND(A17)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
n: { type: 'number' },
|
||||
});
|
||||
await expectCode(db.query("INSERT INTO t (id, nope) VALUES ('1', 2)"), 'COLUMN_NOT_FOUND');
|
||||
// 关键:不能"部分插入"——报错后表必须仍是空的
|
||||
expect(rowsOf(await db.query('SELECT id FROM t'))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('INSERT 未知列的错误消息列出已知列(便于自查)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
n: { type: 'number' },
|
||||
});
|
||||
await expect(db.query("INSERT INTO t (id, nope) VALUES ('1', 2)")).rejects.toThrow(
|
||||
/Unknown column "nope" in table "t"\. Known columns: id, n/,
|
||||
);
|
||||
});
|
||||
|
||||
it('TABLE API 直通路径同样拒绝未知列(A17 的真实形态)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
n: { type: 'number' },
|
||||
});
|
||||
// db.table().insert() 不经 Executor —— 校验必须在引擎边界生效,
|
||||
// 否则这条路径仍然静默丢列
|
||||
await expectCode(
|
||||
db.table('t').insert({ id: '1', nope: 2 } as never),
|
||||
'COLUMN_NOT_FOUND',
|
||||
);
|
||||
expect(rowsOf(await db.query('SELECT id FROM t'))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('UPDATE 未知列 → COLUMN_NOT_FOUND(原有行为,回归护栏)', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
n: { type: 'number' },
|
||||
});
|
||||
await db.query("INSERT INTO t VALUES ('1', 2)");
|
||||
await expectCode(db.query('UPDATE t SET nope = 3 WHERE id = \'1\''), 'COLUMN_NOT_FOUND');
|
||||
});
|
||||
|
||||
// ---- 批量原子性 ----
|
||||
|
||||
it('多行 INSERT 中任一行违反约束 → 整条语句不写入', async () => {
|
||||
await db.defineTable('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string', maxLength: 3 },
|
||||
});
|
||||
await expectCode(
|
||||
db.query("INSERT INTO t VALUES ('1', 'ok'), ('2', 'toolong')"),
|
||||
'VALIDATION_ERROR',
|
||||
);
|
||||
// 第 1 行合法,但语句级原子性要求整批拒绝
|
||||
expect(rowsOf(await db.query('SELECT id FROM t'))).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('[v0.8.0] B-1 校验器单元契约(compileValidator)', () => {
|
||||
const schema = createSchema('t', {
|
||||
id: { type: 'string', primaryKey: true },
|
||||
name: { type: 'string', maxLength: 3 },
|
||||
n: { type: 'number', min: 0, max: 10 },
|
||||
tag: { type: 'string', required: true },
|
||||
flag: { type: 'boolean', default: false },
|
||||
});
|
||||
|
||||
it('validateRow 填充 default、跳过 undefined、只保留 schema 列', () => {
|
||||
const validator = compileValidator(schema);
|
||||
const row = validator.validateRow({ id: '1', tag: 'a' });
|
||||
expect(row).toEqual({ id: '1', tag: 'a', flag: false });
|
||||
expect('name' in row).toBe(false);
|
||||
});
|
||||
|
||||
it('validateRow 拒绝未知列并一次列出全部', () => {
|
||||
const validator = compileValidator(schema);
|
||||
expect(() => validator.validateRow({ id: '1', tag: 'a', x: 1, y: 2 })).toThrow(
|
||||
/Unknown columns "x", "y"/,
|
||||
);
|
||||
});
|
||||
|
||||
it('validateRow 的 knownColumns 收窄允许集合(INSERT 显式列清单)', () => {
|
||||
const validator = compileValidator(schema);
|
||||
// 只允许 id/tag 出现 → name 虽在 schema 里也算未知列
|
||||
expect(() => validator.validateRow({ id: '1', tag: 'a', name: 'x' }, ['id', 'tag'])).toThrow(
|
||||
/Unknown column "name"/,
|
||||
);
|
||||
expect(validator.validateRow({ id: '1', tag: 'a' }, ['id', 'tag'])).toEqual({
|
||||
id: '1',
|
||||
tag: 'a',
|
||||
flag: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('validatePartial 只校验出现的列(undefined 不触发 required)', () => {
|
||||
const validator = compileValidator(schema);
|
||||
const { values, unknown } = validator.validatePartial({ n: 5, ghost: 1 });
|
||||
expect(values).toEqual({ n: 5 });
|
||||
expect(unknown).toEqual(['ghost']);
|
||||
// required 列未出现 → 不报错(UPDATE 语义)
|
||||
expect(validator.validatePartial({ name: 'ab' }).values).toEqual({ name: 'ab' });
|
||||
// 但出现且越界 → 报错
|
||||
expect(() => validator.validatePartial({ name: 'abcd' })).toThrow('exceeds max length');
|
||||
});
|
||||
|
||||
it('validatePartial 拒绝 __proto__ 列名(原型污染防护)', () => {
|
||||
const validator = compileValidator(schema);
|
||||
expect(() => validator.validatePartial(JSON.parse('{"__proto__": {"polluted": 1}}'))).toThrow(
|
||||
/__proto__/,
|
||||
);
|
||||
});
|
||||
|
||||
it('null 通过 required 之外的约束(NULL 不是类型错误)', () => {
|
||||
const validator = compileValidator(schema);
|
||||
expect(validator.validateRow({ id: '1', tag: 'a', name: null, n: null })).toEqual({
|
||||
id: '1',
|
||||
tag: 'a',
|
||||
name: null,
|
||||
n: null,
|
||||
flag: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('primaryKey 隐含 NOT NULL', () => {
|
||||
const validator = compileValidator(schema);
|
||||
expect(() => validator.validateRow({ id: null, tag: 'a' })).toThrow(/Primary key column "id"/);
|
||||
expect(() => validator.validateRow({ tag: 'a' })).toThrow(/Primary key column "id"/);
|
||||
});
|
||||
|
||||
it('required 与 primaryKey 各自给出稳定错误消息', () => {
|
||||
const validator = compileValidator(schema);
|
||||
expect(() => validator.validateRow({ id: '1', tag: null })).toThrow('Column "tag" is required in table "t"');
|
||||
});
|
||||
|
||||
it('类型错误使用 TYPE_ERROR 码(可被调用方分类处理)', () => {
|
||||
const validator = compileValidator(schema);
|
||||
try {
|
||||
validator.validateRow({ id: 1, tag: 'a' });
|
||||
throw new Error('should have thrown');
|
||||
} catch (error) {
|
||||
expect((error as { code?: string }).code).toBe('TYPE_ERROR');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user