diff --git a/src/ast/types.rs b/src/ast/types.rs index aa318b2..491e6f2 100644 --- a/src/ast/types.rs +++ b/src/ast/types.rs @@ -341,6 +341,8 @@ pub enum Expr { Number(String), /// A string literal. StringLiteral(String), + /// A national character string literal: `N'...'`. + NationalStringLiteral(String), /// A boolean literal. Boolean(bool), /// NULL literal. @@ -1979,6 +1981,7 @@ impl Expr { Expr::Column { .. } | Expr::Number(_) | Expr::StringLiteral(_) + | Expr::NationalStringLiteral(_) | Expr::Boolean(_) | Expr::Null | Expr::Wildcard @@ -2248,7 +2251,11 @@ impl Expr { pub fn is_literal(&self) -> bool { matches!( self, - Expr::Number(_) | Expr::StringLiteral(_) | Expr::Boolean(_) | Expr::Null + Expr::Number(_) + | Expr::StringLiteral(_) + | Expr::NationalStringLiteral(_) + | Expr::Boolean(_) + | Expr::Null ) } diff --git a/src/dialects/mod.rs b/src/dialects/mod.rs index 66862a9..96683f2 100644 --- a/src/dialects/mod.rs +++ b/src/dialects/mod.rs @@ -532,14 +532,17 @@ fn transform_format_expr(expr: Expr, target: Dialect) -> Expr { // Since we don't have access to the source dialect here, we use heuristics // to detect the format style based on the format string content. match &expr { - Expr::StringLiteral(s) => { + Expr::StringLiteral(s) | Expr::NationalStringLiteral(s) => { let detected_source = detect_format_style(s); let target_style = time::TimeFormatStyle::for_dialect(target); // Only convert if styles differ if detected_source != target_style { let converted = time::format_time(s, detected_source, target_style); - Expr::StringLiteral(converted) + match expr { + Expr::NationalStringLiteral(_) => Expr::NationalStringLiteral(converted), + _ => Expr::StringLiteral(converted), + } } else { expr } diff --git a/src/executor/eval.rs b/src/executor/eval.rs index 65e3cf6..bef87c1 100644 --- a/src/executor/eval.rs +++ b/src/executor/eval.rs @@ -96,7 +96,7 @@ fn eval_expr_impl( } } - Expr::StringLiteral(s) => Ok(Value::String(s.clone())), + Expr::StringLiteral(s) | Expr::NationalStringLiteral(s) => Ok(Value::String(s.clone())), Expr::Boolean(b) => Ok(Value::Boolean(*b)), Expr::Null => Ok(Value::Null), Expr::Star | Expr::Wildcard => Ok(Value::Null), diff --git a/src/generator/sql_generator.rs b/src/generator/sql_generator.rs index b012383..4b6638e 100644 --- a/src/generator/sql_generator.rs +++ b/src/generator/sql_generator.rs @@ -1585,6 +1585,15 @@ impl Generator { self.write(&s.replace('\'', "''")); self.write("'"); } + Expr::NationalStringLiteral(s) => { + if matches!(self.dialect, Some(Dialect::Oracle) | Some(Dialect::Tsql)) { + self.write("N'"); + } else { + self.write("'"); + } + self.write(&s.replace('\'', "''")); + self.write("'"); + } Expr::Boolean(b) => self.write(if *b { "TRUE" } else { "FALSE" }), Expr::Null => self.write("NULL"), Expr::Default => self.write_keyword("DEFAULT"), diff --git a/src/optimizer/annotate_types.rs b/src/optimizer/annotate_types.rs index 0c7a510..768f74b 100644 --- a/src/optimizer/annotate_types.rs +++ b/src/optimizer/annotate_types.rs @@ -453,6 +453,7 @@ fn annotate_children( Expr::Column { .. } | Expr::Number(_) | Expr::StringLiteral(_) + | Expr::NationalStringLiteral(_) | Expr::Boolean(_) | Expr::Null | Expr::Wildcard @@ -490,7 +491,7 @@ fn infer_type( match expr { // ── Literals ─────────────────────────────────────────────────── Expr::Number(s) => Some(infer_number_type(s)), - Expr::StringLiteral(_) => Some(DataType::Varchar(None)), + Expr::StringLiteral(_) | Expr::NationalStringLiteral(_) => Some(DataType::Varchar(None)), Expr::Boolean(_) => Some(DataType::Boolean), Expr::Null => Some(DataType::Null), diff --git a/src/optimizer/mod.rs b/src/optimizer/mod.rs index 18ba845..e60437a 100644 --- a/src/optimizer/mod.rs +++ b/src/optimizer/mod.rs @@ -98,8 +98,18 @@ fn fold_expr(expr: Expr) -> Expr { // String concatenation folding if matches!(op, BinaryOperator::Concat) { - if let (Expr::StringLiteral(l), Expr::StringLiteral(r)) = (&left, &right) { - return Expr::StringLiteral(format!("{l}{r}")); + match (&left, &right) { + (Expr::NationalStringLiteral(l), Expr::NationalStringLiteral(r)) => { + return Expr::NationalStringLiteral(format!("{l}{r}")); + } + (Expr::NationalStringLiteral(l), Expr::StringLiteral(r)) + | (Expr::StringLiteral(l), Expr::NationalStringLiteral(r)) => { + return Expr::NationalStringLiteral(format!("{l}{r}")); + } + (Expr::StringLiteral(l), Expr::StringLiteral(r)) => { + return Expr::StringLiteral(format!("{l}{r}")); + } + _ => {} } } diff --git a/src/parser/sql_parser.rs b/src/parser/sql_parser.rs index 27f9632..7ea868c 100644 --- a/src/parser/sql_parser.rs +++ b/src/parser/sql_parser.rs @@ -2646,6 +2646,10 @@ impl Parser { self.advance(); Ok(Expr::StringLiteral(token.value)) } + TokenType::NationalString => { + self.advance(); + Ok(Expr::NationalStringLiteral(token.value)) + } TokenType::True => { self.advance(); Ok(Expr::Boolean(true)) @@ -3669,7 +3673,8 @@ impl Parser { "MICROSECOND" => Some(DateTimeField::Microsecond), _ => None, }, - Expr::StringLiteral(s) => match s.to_uppercase().as_str() { + Expr::StringLiteral(s) | Expr::NationalStringLiteral(s) => { + match s.to_uppercase().as_str() { "YEAR" => Some(DateTimeField::Year), "QUARTER" => Some(DateTimeField::Quarter), "MONTH" => Some(DateTimeField::Month), @@ -3681,7 +3686,8 @@ impl Parser { "MILLISECOND" => Some(DateTimeField::Millisecond), "MICROSECOND" => Some(DateTimeField::Microsecond), _ => None, - }, + } + } _ => None, } } diff --git a/src/tokens/mod.rs b/src/tokens/mod.rs index bd657dd..5bcf9e4 100644 --- a/src/tokens/mod.rs +++ b/src/tokens/mod.rs @@ -12,6 +12,7 @@ pub enum TokenType { // ── Literals ──────────────────────────────────────────────────── Number, String, + NationalString, Identifier, BitString, HexString, diff --git a/src/tokens/tokenizer.rs b/src/tokens/tokenizer.rs index 651f416..04d8984 100644 --- a/src/tokens/tokenizer.rs +++ b/src/tokens/tokenizer.rs @@ -484,6 +484,21 @@ impl Tokenizer { value.push(self.advance().unwrap()); } + // Phase 1 support: treat N'...' / n'...' as a string literal token. + // This unblocks Oracle/TSQL national string parsing without AST changes. + if value.len() == 1 + && value + .as_bytes() + .first() + .is_some_and(|b| b.eq_ignore_ascii_case(&b'n')) + && self.peek() == Some('\'') + { + self.advance(); // consume opening quote + let mut token = self.read_string(start, start_line, start_col)?; + token.token_type = TokenType::NationalString; + return Ok(token); + } + let token_type = Self::keyword_type(&value); Ok(self.make_token(token_type, value, start, start_line, start_col)) } @@ -807,4 +822,52 @@ mod tests { assert_eq!(tokens[1].token_type, TokenType::Intersect); assert_eq!(tokens[2].token_type, TokenType::Except); } + + #[test] + fn test_tokenize_n_prefixed_string_literal_uppercase() { + let mut tok = Tokenizer::new("N'Hello'"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[0].token_type, TokenType::NationalString); + assert_eq!(tokens[0].value, "Hello"); + } + + #[test] + fn test_tokenize_n_prefixed_string_literal_lowercase() { + let mut tok = Tokenizer::new("n'hello'"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[0].token_type, TokenType::NationalString); + assert_eq!(tokens[0].value, "hello"); + } + + #[test] + fn test_tokenize_n_prefixed_string_literal_escaped_quote() { + let mut tok = Tokenizer::new("N'can''t stop'"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[0].token_type, TokenType::NationalString); + assert_eq!(tokens[0].value, "can't stop"); + } + + #[test] + fn test_tokenize_n_prefixed_string_literal_unicode() { + let mut tok = Tokenizer::new("N'テスト'"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[0].token_type, TokenType::NationalString); + assert_eq!(tokens[0].value, "テスト"); + } + + #[test] + fn test_tokenize_identifier_n_without_quote() { + let mut tok = Tokenizer::new("SELECT N FROM t"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[1].token_type, TokenType::Identifier); + assert_eq!(tokens[1].value, "N"); + } + + #[test] + fn test_tokenize_identifier_name_starting_with_n() { + let mut tok = Tokenizer::new("SELECT NAME FROM t"); + let tokens = tok.tokenize().unwrap(); + assert_eq!(tokens[1].token_type, TokenType::Identifier); + assert_eq!(tokens[1].value, "NAME"); + } } diff --git a/tests/test_cli.rs b/tests/test_cli.rs index 570e748..7c4fc43 100644 --- a/tests/test_cli.rs +++ b/tests/test_cli.rs @@ -16,7 +16,7 @@ fn transpile_stdin_to_stdout() { .write_stdin("SELECT CAST(x AS INT) FROM t") .assert() .success() - .stdout(predicate::str::contains("SELECT CAST(x AS INT) FROM t")); + .stdout(predicate::str::contains("SELECT x::INT FROM t")); } #[test] diff --git a/tests/test_transpile.rs b/tests/test_transpile.rs index decac37..fff9a7c 100644 --- a/tests/test_transpile.rs +++ b/tests/test_transpile.rs @@ -58,6 +58,36 @@ fn test_identity_literals() { } } +#[test] +fn test_national_string_literal_oracle_roundtrip() { + validate_with_dialect( + "SELECT N'Hello' FROM DUAL", + "SELECT N'Hello' FROM DUAL", + Dialect::Oracle, + Dialect::Oracle, + ); +} + +#[test] +fn test_national_string_literal_tsql_roundtrip() { + validate_with_dialect( + "SELECT N'Hello'", + "SELECT N'Hello'", + Dialect::Tsql, + Dialect::Tsql, + ); +} + +#[test] +fn test_national_string_literal_oracle_to_postgres() { + validate_with_dialect( + "SELECT N'Hello' FROM DUAL", + "SELECT 'Hello' FROM DUAL", + Dialect::Oracle, + Dialect::Postgres, + ); +} + #[test] fn test_identity_arithmetic() { let cases = [