MetonaSqlark/tests/sql/lexer.test.ts

/**
 * Lexer 边缘场景测试
 */

import { Lexer, tokenize } from '../../src/sql/lexer';
import { TokenType } from '../../src/sql/tokens';

describe('Lexer 边缘场景', () => {
  it('空字符串返回 EOF', () => {
    const tokens = tokenize('');
    expect(tokens).toHaveLength(1);
    expect(tokens[0].type).toBe(TokenType.EOF);
  });

  it('只包含空格', () => {
    const tokens = tokenize('   \t\n  ');
    expect(tokens).toHaveLength(1);
    expect(tokens[0].type).toBe(TokenType.EOF);
  });

  it('非法字符返回 ILLEGAL', () => {
    const tokens = tokenize('@');
    expect(tokens[0].type).toBe(TokenType.ILLEGAL);
  });

  it('完整的 CREATE TABLE 语句', () => {
    const tokens = tokenize('CREATE TABLE users (id STRING PRIMARY KEY, name STRING NOT NULL, age NUMBER DEFAULT 0)');
    const types = tokens.map(t => t.type);
    expect(types).toEqual([
      TokenType.CREATE, TokenType.TABLE, TokenType.IDENTIFIER, TokenType.LPAREN,
      TokenType.IDENTIFIER, TokenType.IDENTIFIER, TokenType.PRIMARY, TokenType.KEY,
      TokenType.COMMA,
      TokenType.IDENTIFIER, TokenType.IDENTIFIER, TokenType.NOT, TokenType.NULL,
      TokenType.COMMA,
      TokenType.IDENTIFIER, TokenType.IDENTIFIER, TokenType.DEFAULT, TokenType.NUMBER,
      TokenType.RPAREN, TokenType.EOF,
    ]);
  });

  it('带转义单引号的字符串', () => {
    // SQL 中两个单引号 '' 表示转义的单引号
    const tokens = tokenize("SELECT 'hello world'");
    expect(tokens[1].type).toBe(TokenType.STRING);
    expect(tokens[1].value).toBe('hello world');
  });

  // v0.8.0 更正：双引号在 SQL 标准（及 SQLite/PostgreSQL）中是**分隔标识符**，
  // 不是字符串字面量。此前本用例把 `"hello world"` 钉死为 STRING，正是
  // `SELECT "name" FROM t` 静默产出常量列（行数正确、值全错）的原因。
  it('双引号是分隔标识符（非字符串字面量）', () => {
    const tokens = tokenize('SELECT "hello world"');
    expect(tokens[1].type).toBe(TokenType.QUOTED_IDENTIFIER);
    expect(tokens[1].value).toBe('hello world');
    // 单引号才是字符串
    const strTokens = tokenize("SELECT 'hello world'");
    expect(strTokens[1].type).toBe(TokenType.STRING);
  });

  it('分隔标识符内的 "" 表示一个双引号', () => {
    const tokens = tokenize('SELECT "a""b"');
    expect(tokens[1].type).toBe(TokenType.QUOTED_IDENTIFIER);
    expect(tokens[1].value).toBe('a"b');
  });

  it('未闭合的分隔标识符报 PARSE_ERROR', () => {
    expect(() => tokenize('SELECT "abc')).toThrow(/Unterminated quoted identifier/);
  });

  it('未闭合的块注释报 PARSE_ERROR（此前静默吞掉剩余 SQL）', () => {
    expect(() => tokenize('SELECT 1 /* oops')).toThrow(/Unterminated block comment/);
  });

  it('反斜杠是普通字符（与参数绑定器保持一致）', () => {
    const tokens = tokenize("SELECT 'C:\\'");
    expect(tokens[1].type).toBe(TokenType.STRING);
    expect(tokens[1].value).toBe('C:\\');
  });

  it('所有关键字', () => {
    const sql = 'SELECT FROM WHERE INSERT INTO VALUES UPDATE SET DELETE CREATE TABLE DROP ORDER BY ASC DESC LIMIT OFFSET AND OR NOT LIKE IN PRIMARY KEY UNIQUE DEFAULT NULL TRUE FALSE';
    const tokens = tokenize(sql);
    const nonEof = tokens.filter(t => t.type !== TokenType.EOF);
    expect(nonEof).toHaveLength(30);
    // 验证都是关键字，不是 IDENTIFIER
    for (const t of nonEof) {
      expect(t.type).not.toBe(TokenType.IDENTIFIER);
    }
  });

  it('下划线标识符', () => {
    const tokens = tokenize('SELECT my_col_1 FROM user_table');
    expect(tokens[1].type).toBe(TokenType.IDENTIFIER);
    expect(tokens[1].value).toBe('my_col_1');
    expect(tokens[3].type).toBe(TokenType.IDENTIFIER);
    expect(tokens[3].value).toBe('user_table');
  });

  it('运算符分隔符', () => {
    const tokens = tokenize('= != > >= < <= <> ( ) , ; *');
    const types = tokens.map(t => t.type);
    expect(types).toEqual([
      TokenType.EQ, TokenType.NEQ, TokenType.GT, TokenType.GTE,
      TokenType.LT, TokenType.LTE, TokenType.NEQ,
      TokenType.LPAREN, TokenType.RPAREN, TokenType.COMMA,
      TokenType.SEMICOLON, TokenType.STAR, TokenType.EOF,
    ]);
  });

  it('数字: 整数和浮点数和负数', () => {
    const tokens = tokenize('42 3.14 -7 -2.5');
    expect(tokens[0].value).toBe('42');
    expect(tokens[1].value).toBe('3.14');
    expect(tokens[2].value).toBe('-7');
    expect(tokens[3].value).toBe('-2.5');
  });

  it('Token position 正确', () => {
    const lexer = new Lexer('SELECT * FROM users');
    const tok = lexer.nextToken();
    expect(tok.position).toBe(0); // SELECT 从位置 0 开始
  });
});