[#222] test: 파서 property 기반 테스트 도입 (+ CR 공백 처리 1줄)

Cargo.toml

@@ -44,6 +44,9 @@ memmap2 = "0.9.5"
 [target.'cfg(windows)'.dependencies]
 winreg = "0.10.1"
 
+[dev-dependencies]
+proptest = "1"
+
 [lib]
 name = "rrdb"
 path = "./src/lib.rs"

proptest-regressions/engine/parser/test/property.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 8ffe95073da5485fe68a17fbe91960cee2c6d1183f21b1a6a3c67cf5676587e5 # shrinks to spaces = "\r"

src/engine/lexer/test/eof_lookahead.rs

@@ -0,0 +1,81 @@
+#![cfg(test)]
+//! Regression tests for lookahead at end-of-input (found by the property
+//! tests added for issue #222).
+
+use crate::engine::lexer::predule::{OperatorToken, Token, Tokenizer};
+
+/// Operators whose tokenizer branch peeks at the next character to decide
+/// between a one- and two-character token (`--`, `//`, `<=`, `>=`, `!=`).
+/// When such an operator is the final character of the input, the lookahead
+/// hits EOF, and the paired `unread_char` used to rewind over a character
+/// that was never consumed — so the tokenizer emitted the same operator
+/// forever and never reached EOF.
+#[test]
+fn trailing_lookahead_operator_terminates() {
+    let cases = [
+        ("-", OperatorToken::Minus),
+        ("/", OperatorToken::Slash),
+        ("<", OperatorToken::Lt),
+        (">", OperatorToken::Gt),
+        ("!", OperatorToken::Not),
+    ];
+
+    for (input, expected) in cases {
+        let mut tokenizer = Tokenizer::new(input.to_owned());
+
+        assert_eq!(
+            tokenizer.get_token().unwrap(),
+            Token::Operator(expected),
+            "first token for {input:?}"
+        );
+        assert_eq!(
+            tokenizer.get_token().unwrap(),
+            Token::EOF,
+            "{input:?} must reach EOF instead of repeating the operator"
+        );
+    }
+}
+
+/// The same operators must still combine with a following character.
+#[test]
+fn lookahead_operator_still_combines() {
+    let cases = [
+        ("<=", Token::Operator(OperatorToken::Lte)),
+        (">=", Token::Operator(OperatorToken::Gte)),
+        ("!=", Token::Operator(OperatorToken::Neq)),
+    ];
+
+    for (input, expected) in cases {
+        let mut tokenizer = Tokenizer::new(input.to_owned());
+        assert_eq!(tokenizer.get_token().unwrap(), expected, "input {input:?}");
+        assert_eq!(tokenizer.get_token().unwrap(), Token::EOF);
+    }
+}
+
+/// `\r` is whitespace. Without this, any statement carrying CRLF line endings
+/// — a script saved on Windows, or a client that sends CRLF — failed to
+/// tokenize with `unexpected character: '\r'`.
+#[test]
+fn carriage_return_is_whitespace() {
+    let mut tokenizer = Tokenizer::new("\r\n".to_owned());
+    assert_eq!(tokenizer.get_token().unwrap(), Token::EOF);
+
+    let tokens = Tokenizer::string_to_tokens("SELECT\r\n1".to_owned()).unwrap();
+    assert!(
+        tokens.iter().any(|token| matches!(token, Token::Select)),
+        "CRLF-separated SQL should tokenize, got {tokens:?}"
+    );
+}
+
+/// A quoted identifier with no closing quote used to spin forever: the loop
+/// scanning for the closing `"` had no EOF guard, so it read past the end of
+/// the buffer indefinitely. It must terminate with an error instead.
+#[test]
+fn unterminated_quoted_identifier_errors() {
+    assert!(Tokenizer::string_to_tokens("\"".to_owned()).is_err());
+    assert!(Tokenizer::string_to_tokens("SELECT \"col".to_owned()).is_err());
+
+    // A properly closed quoted identifier still works.
+    let tokens = Tokenizer::string_to_tokens("\"col\"".to_owned()).unwrap();
+    assert_eq!(tokens, vec![Token::Identifier("col".to_owned())]);
+}

src/engine/lexer/tokenizer.rs

@@ -24,7 +24,10 @@ impl Tokenizer {
     }
 
     pub fn is_whitespace(&self) -> bool {
-        self.last_char == ' ' || self.last_char == '\n' || self.last_char == '\t'
+        self.last_char == ' '
+            || self.last_char == '\n'
+            || self.last_char == '\t'
+            || self.last_char == '\r'
     }
 
     pub fn is_digit(&self) -> bool {

src/engine/parser/test/mod.rs

@@ -23,3 +23,4 @@ pub(crate) mod ddl;
 
 pub(crate) mod coverage;
 pub(crate) mod parser;
+pub(crate) mod property;

src/engine/parser/test/property.rs

@@ -0,0 +1,272 @@
+#![cfg(test)]
+//! Property-based tests for the lexer/parser (issue #222, phase 1-2).
+//!
+//! Example-based tests can only cover inputs the author thought of. These
+//! generate thousands of inputs per run and assert invariants that must hold
+//! for *every* input, which is where malformed-SQL handling tends to break.
+
+use proptest::prelude::*;
+
+use crate::engine::lexer::predule::Tokenizer;
+use crate::engine::parser::context::ParserContext;
+use crate::engine::parser::predule::Parser;
+
+/// Parse a statement the way the engine does: tokenize, then parse.
+fn try_parse(sql: &str) -> crate::errors::Result<Vec<crate::engine::ast::SQLStatement>> {
+    let tokens = Tokenizer::string_to_tokens(sql.to_owned())?;
+    Parser::new(tokens).parse(ParserContext::default())
+}
+
+/// Truly arbitrary text: `proptest::char::any()` covers control characters
+/// (NUL, lone `\r`), quotes and unbalanced delimiters that the previous
+/// ASCII-only regex silently skipped. Every character is adversarial on
+/// purpose -- the invariant under test is that none of them can make the
+/// tokenizer or parser panic.
+fn arbitrary_sql_text() -> impl Strategy<Value = String> {
+    proptest::collection::vec(proptest::char::any(), 0..80).prop_map(String::from_iter)
+}
+
+/// Fragments drawn from real SQL keywords, so the generator spends more of its
+/// budget on inputs that reach deeper into the parser instead of being
+/// rejected by the tokenizer immediately.
+///
+/// The leading keyword is chosen separately from the rest. A uniform shuffle
+/// of keywords almost never starts with one that opens a statement, so the
+/// parser returns an empty statement list and the deeper code is never
+/// entered: measured over 1024 generated inputs, exactly 0-1 produced a
+/// statement. Pinning the first token keeps the mutations where they are
+/// useful -- inside a statement the parser has actually committed to.
+fn sql_like_fragment() -> impl Strategy<Value = String> {
+    let leading = prop::sample::select(vec![
+        "SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER",
+    ]);
+    let keyword = prop::sample::select(vec![
+        "SELECT", "FROM", "WHERE", "INSERT", "INTO", "VALUES", "UPDATE", "SET", "DELETE", "CREATE",
+        "TABLE", "DATABASE", "INDEX", "DROP", "ALTER", "AND", "OR", "NOT", "NULL", "PRIMARY",
+        "KEY", "ORDER", "BY", "GROUP", "JOIN", "ON", "AS", "(", ")", ",", ";", "*", "=", "'x'",
+        "1", "foo",
+    ]);
+
+    (leading, prop::collection::vec(keyword, 0..12)).prop_map(|(head, rest)| {
+        let mut parts = Vec::with_capacity(rest.len() + 1);
+        parts.push(head);
+        parts.extend(rest);
+        parts.join(" ")
+    })
+}
+
+/// Well-formed statements with one mutation applied. The fragment generators
+/// above explore malformed input, which is where panics hide, but they almost
+/// never produce something the parser accepts -- so on their own they never
+/// exercise the code that runs *after* a statement is recognised. These start
+/// from a valid statement and perturb it, so the parser commits to a statement
+/// kind first and then meets the unexpected token.
+fn mutated_statement() -> impl Strategy<Value = String> {
+    let base = prop::sample::select(vec![
+        "SELECT 1",
+        "SELECT * FROM foo",
+        "SELECT foo, bar FROM baz WHERE foo = 1",
+        "INSERT INTO foo (a, b) VALUES (1, 'x')",
+        "UPDATE foo SET a = 1 WHERE b = 'x'",
+        "DELETE FROM foo WHERE a = 1",
+        "CREATE TABLE foo (a INTEGER PRIMARY KEY)",
+        "DROP TABLE foo",
+    ]);
+    let injection = prop::sample::select(vec![
+        "", " ", ",", "(", ")", "'", ";", "*", "=", "NULL", "SELECT", "WHERE", "\n", "\t",
+    ]);
+
+    (base, injection, 0usize..40).prop_map(|(base, injection, at)| {
+        let at = at.min(base.len());
+        let mut out = String::with_capacity(base.len() + injection.len());
+        out.push_str(&base[..base.floor_char_boundary(at)]);
+        out.push_str(injection);
+        out.push_str(&base[base.floor_char_boundary(at)..]);
+        out
+    })
+}
+
+/// Statements that are valid by construction. Needed because every other
+/// generator here explores *malformed* input: they prove the parser does not
+/// panic, but nothing so far asserts that well-formed SQL actually parses. A
+/// parser that rejected everything would satisfy all the panic and determinism
+/// properties below.
+fn valid_statement() -> impl Strategy<Value = String> {
+    let table = prop::sample::select(vec!["foo", "bar", "baz_1"]);
+    let column = prop::sample::select(vec!["a", "b", "id"]);
+    let literal = prop::sample::select(vec!["1", "42", "'x'"]);
+    let terminator = prop::sample::select(vec!["", ";"]);
+
+    (table, column, literal, terminator).prop_flat_map(|(table, column, literal, terminator)| {
+        let shapes = vec![
+            format!("SELECT {} FROM {}", column, table),
+            format!("SELECT * FROM {}", table),
+            format!(
+                "SELECT {} FROM {} WHERE {} = {}",
+                column, table, column, literal
+            ),
+            format!("INSERT INTO {} ({}) VALUES ({})", table, column, literal),
+            format!("UPDATE {} SET {} = {}", table, column, literal),
+            format!("DELETE FROM {} WHERE {} = {}", table, column, literal),
+            format!("DROP TABLE {}", table),
+        ];
+        prop::sample::select(shapes).prop_map(move |shape| format!("{}{}", shape, terminator))
+    })
+}
+
+proptest! {
+    #![proptest_config(ProptestConfig::with_cases(1024))]
+
+    /// The parser must never panic, however malformed the input. Any input it
+    /// cannot handle has to come back as an `Err`, not an abort — a panic here
+    /// would take down the connection task that is parsing the statement.
+    #[test]
+    fn parser_never_panics_on_arbitrary_text(sql in arbitrary_sql_text()) {
+        let _ = try_parse(&sql);
+    }
+
+    /// Same invariant, but with inputs built from real SQL keywords so the
+    /// generator reaches the statement parsers rather than failing at the
+    /// tokenizer.
+    #[test]
+    fn parser_never_panics_on_sql_like_input(sql in sql_like_fragment()) {
+        let _ = try_parse(&sql);
+    }
+
+    /// The same invariant on inputs the parser actually accepts far more often:
+    /// a valid statement with one token injected. Measured over 1024 generated
+    /// inputs, `sql_like_fragment` yields 0-1 parsed statements while this
+    /// yields ~480, so this is the generator that reaches the code paths
+    /// running after a statement kind has been recognised.
+    #[test]
+    fn parser_never_panics_on_mutated_statement(sql in mutated_statement()) {
+        let _ = try_parse(&sql);
+    }
+
+    /// Parsing a mutated statement is deterministic too. Kept separate from
+    /// the fragment version because this one exercises the statement bodies.
+    #[test]
+    fn parsing_a_mutated_statement_is_deterministic(sql in mutated_statement()) {
+        let first = try_parse(&sql);
+        let second = try_parse(&sql);
+
+        match (first, second) {
+            (Ok(first), Ok(second)) => {
+                prop_assert_eq!(format!("{:?}", first), format!("{:?}", second));
+            }
+            (Err(first), Err(second)) => {
+                // An Err/Err outcome is only deterministic if it is the same
+                // error, not just any error.
+                prop_assert_eq!(first.kind, second.kind);
+            }
+            (first, second) => {
+                prop_assert!(
+                    false,
+                    "nondeterministic parse of {:?}: {:?} vs {:?}",
+                    sql,
+                    first.map(|_| ()),
+                    second.map(|_| ())
+                );
+            }
+        }
+    }
+
+    /// Parsing is a pure function of its input: the same text must always
+    /// produce the same outcome. This guards against parser state leaking
+    /// across runs through shared or cached state.
+    #[test]
+    fn parsing_is_deterministic(sql in sql_like_fragment()) {
+        let first = try_parse(&sql);
+        let second = try_parse(&sql);
+
+        match (first, second) {
+            (Ok(first), Ok(second)) => {
+                prop_assert_eq!(format!("{:?}", first), format!("{:?}", second));
+            }
+            (Err(first), Err(second)) => {
+                prop_assert_eq!(first.kind, second.kind);
+            }
+            (first, second) => {
+                prop_assert!(
+                    false,
+                    "nondeterministic parse of {:?}: {:?} vs {:?}",
+                    sql,
+                    first.map(|_| ()),
+                    second.map(|_| ())
+                );
+            }
+        }
+    }
+
+    /// Leading and trailing whitespace must not change the parse result.
+    #[test]
+    fn surrounding_whitespace_is_insignificant(sql in sql_like_fragment()) {
+        let bare = try_parse(&sql);
+        let padded = try_parse(&format!("  \t{sql}\n "));
+
+        match (bare, padded) {
+            (Ok(bare), Ok(padded)) => {
+                prop_assert_eq!(format!("{:?}", bare), format!("{:?}", padded));
+            }
+            (Err(bare), Err(padded)) => {
+                prop_assert_eq!(bare.kind, padded.kind);
+            }
+            (bare, padded) => {
+                prop_assert!(
+                    false,
+                    "whitespace changed the parse result of {:?}: {:?} vs {:?}",
+                    sql,
+                    bare.map(|_| ()),
+                    padded.map(|_| ())
+                );
+            }
+        }
+    }
+
+    /// The tokenizer must terminate and must not invent trailing tokens for
+    /// input that is only whitespace.
+    /// The success half of the contract: well-formed SQL must parse, and must
+    /// produce exactly one statement. Without this, "the parser never panics"
+    /// would also hold for a parser that rejected every input.
+    #[test]
+    fn valid_statements_parse_into_exactly_one_statement(sql in valid_statement()) {
+        let parsed = try_parse(&sql);
+        prop_assert!(
+            parsed.is_ok(),
+            "valid SQL should parse: {:?} -> {:?}",
+            sql,
+            parsed.as_ref().err()
+        );
+        prop_assert_eq!(
+            parsed.unwrap().len(),
+            1,
+            "valid SQL should yield one statement: {:?}",
+            sql
+        );
+    }
+
+    /// The failure half: input that cannot be a statement must come back as an
+    /// error, not as a silently empty parse. A statement keyword with nothing
+    /// after it is unambiguously incomplete.
+    #[test]
+    fn a_bare_statement_keyword_is_an_error(
+        keyword in prop::sample::select(vec!["SELECT", "INSERT", "UPDATE", "DELETE", "DROP"])
+    ) {
+        let parsed = try_parse(keyword);
+        prop_assert!(
+            parsed.is_err(),
+            "{:?} alone is incomplete and must be rejected, got {:?}",
+            keyword,
+            parsed.ok()
+        );
+    }
+
+    #[test]
+    fn whitespace_only_input_yields_no_statements(
+        spaces in proptest::string::string_regex(r"[ \t\r\n]{0,40}").expect("valid regex")
+    ) {
+        let parsed = try_parse(&spaces);
+        prop_assert!(parsed.is_ok(), "whitespace should parse, got {:?}", parsed.err());
+        prop_assert!(parsed.unwrap().is_empty());
+    }
+}