chore: sync the wip-cst branch with the main branch

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+  - package-ecosystem: cargo
+    directory: "/sqlparser_bench"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10

.github/workflows/rust.yml

@@ -1,30 +1,78 @@
 name: Rust
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
+
+  codestyle:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+        with:
+          components: rustfmt
+          # Note that `nightly` is required for `license_template_path`, as
+          # it's an unstable feature.
+          rust-version: nightly
+      - uses: actions/checkout@v2
+      - run: cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+        with:
+          components: clippy
+      - uses: actions/checkout@v2
+      - run: cargo clippy --all-targets --all-features -- -D warnings
+
+  compile:
     runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+      - uses: actions/checkout@master
+      - run: cargo check --all-targets --all-features
 
+  test:
+    strategy:
+      matrix:
+        rust: [stable, beta, nightly]
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
     - name: Setup Rust
-      run: |
-        rustup toolchain install nightly --profile default
-        rustup toolchain install stable
-        rustup override set stable
-    # Clippy must be run first, as its lints are only triggered during
-    # compilation. Put another way: after a successful `cargo build`, `cargo
-    # clippy` is guaranteed to produce no results. This bug is known upstream:
-    # https://github.com/rust-lang/rust-clippy/issues/2604.
-#    - name: Clippy
-#      run: cargo clippy -- --all-targets --all-features -- -D warnings
-    - name: Check formatting
-      run: |
-        cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
-    - name: Build
-      run: cargo build --verbose
-    - name: Run tests
-      run: cargo test --verbose
-    - name: Run tests for all features
-      run: cargo test --verbose -- all-features
+      uses: hecrj/setup-rust-action@v1
+      with:
+        rust-version: ${{ matrix.rust }}
+    - name: Install Tarpaulin
+      uses: actions-rs/[email protected]
+      with:
+        crate: cargo-tarpaulin
+        version: 0.14.2
+        use-tool-cache: true
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Test
+      run: cargo test --all-features
+    - name: Coverage
+      if: matrix.rust == 'stable'
+      run: cargo tarpaulin -o Lcov --output-dir ./coverage
+    - name: Coveralls
+      if: matrix.rust == 'stable'
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  publish-crate:
+    if: startsWith(github.ref, 'refs/tags/v0')
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+      - name: Set up Rust
+        uses: hecrj/setup-rust-action@v1
+      - uses: actions/checkout@v2
+      - name: Publish
+        shell: bash
+        run: |
+          cargo publish --token ${{ secrets.CRATES_TOKEN }}

.gitignore

@@ -1,6 +1,7 @@
 # Generated by Cargo
 # will have compiled files and executables
 /target/
+/sqlparser_bench/target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock

CHANGELOG.md

@@ -6,11 +6,74 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 Given that the parser produces a typed AST, any changes to the AST will technically be breaking and thus will result in a `0.(N+1)` version. We document changes that break via addition as "Added".
 
 ## [Unreleased]
-Check https://github.com/andygrove/sqlparser-rs/commits/master for undocumented changes.
+Check https://github.com/ballista-compute/sqlparser-rs/commits/main for undocumented changes.
+
+
+## [0.8.0] 2020-02-20
+
+### Added
+* Introduce Hive QL dialect `HiveDialect` and syntax (#235) - Thanks @hntd187!
+* Add `SUBSTRING(col [FROM <expr>] [FOR <expr>])` syntax (#293)
+* Support parsing floats without leading digits `.01` (#294)
+* Support parsing multiple show variables (#290) - Thanks @francis-du!
+* Support SQLite `INSERT OR [..]` syntax (#281) - Thanks @zhangli-pear!
+
+## [0.7.0] 2020-12-28
 
 ### Changed
+- Change the MySQL dialect to support `` `identifiers` `` quoted with backticks instead of the standard `"double-quoted"` identifiers (#247) - thanks @mashuai!
+- Update bigdecimal requirement from 0.1 to 0.2 (#268)
+
+### Added
+- Enable dialect-specific behaviours in the parser (`dialect_of!()`) (#254) - thanks @eyalleshem!
+- Support named arguments in function invocations (`ARG_NAME => val`) (#250) - thanks @eyalleshem!
+- Support `TABLE()` functions in `FROM` (#253) - thanks @eyalleshem!
+- Support Snowflake's single-line comments starting with '#' or '//' (#264) - thanks @eyalleshem!
+- Support PostgreSQL `PREPARE`, `EXECUTE`, and `DEALLOCATE` (#243) - thanks @silathdiir!
+- Support PostgreSQL math operators (#267) - thanks @alex-dukhno!
+- Add SQLite dialect (#248) - thanks @mashuai!
+- Add Snowflake dialect (#259) - thanks @eyalleshem!
+- Support for Recursive CTEs - thanks @rhanqtl!
+- Support `FROM (table_name) alias` syntax - thanks @eyalleshem!
+- Support for `EXPLAIN [ANALYZE] VERBOSE` - thanks @ovr!
+- Support `ANALYZE TABLE`
+- DDL:
+    - Support `OR REPLACE` in `CREATE VIEW`/`TABLE` (#239)  - thanks @Dandandan!
+    - Support specifying `ASC`/`DESC` in index columns (#249) - thanks @mashuai!
+    - Support SQLite `AUTOINCREMENT` and MySQL `AUTO_INCREMENT` column option in `CREATE TABLE` (#234) - thanks @mashuai!
+    - Support PostgreSQL `IF NOT EXISTS` for `CREATE SCHEMA` (#276) - thanks @alex-dukhno!
+
+### Fixed
+- Fix a typo in `JSONFILE` serialization, introduced in 0.3.1 (#237)
+- Change `CREATE INDEX` serialization to not end with a semicolon, introduced in 0.5.1 (#245)
+- Don't fail parsing `ALTER TABLE ADD COLUMN` ending with a semicolon, introduced in 0.5.1 (#246) - thanks @mashuai
+
+## [0.6.1] - 2020-07-20
+
+### Added
+- Support BigQuery `ASSERT` statement (#226)
+
+## [0.6.0] - 2020-07-20
+
+### Added
+- Support SQLite's `CREATE TABLE (...) WITHOUT ROWID` (#208) - thanks @mashuai!
+- Support SQLite's `CREATE VIRTUAL TABLE` (#209) - thanks @mashuai!
+
+## [0.5.1] - 2020-06-26
+This release should have been called `0.6`, as it introduces multiple incompatible changes to the API. If you don't want to upgrade yet, you can revert to the previous version by changing your `Cargo.toml` to:
+
+    sqlparser = "= 0.5.0"
+
+
+### Changed
+- **`Parser::parse_sql` now accepts a `&str` instead of `String` (#182)** - thanks @Dandandan!
 - Change `Ident` (previously a simple `String`) to store the parsed (unquoted) `value` of the identifier and the `quote_style` separately (#143) - thanks @apparebit!
 - Support Snowflake's `FROM (table_name)` (#155) - thanks @eyalleshem!
+- Add line and column number to TokenizerError (#194) - thanks @Dandandan!
+- Use Token::EOF instead of Option<Token> (#195)
+- Make the units keyword following `INTERVAL '...'` optional (#184) - thanks @maxcountryman!
+- Generalize `DATE`/`TIME`/`TIMESTAMP` literals representation in the AST (`TypedString { data_type, value }`) and allow `DATE` and other keywords to be used as identifiers when not followed by a string (#187) - thanks @maxcountryman!
+- Output DataType capitalized (`fmt::Display`) (#202) - thanks @Dandandan!
 
 ### Added
 - Support MSSQL `TOP (<N>) [ PERCENT ] [ WITH TIES ]` (#150) - thanks @alexkyllo!
@@ -21,9 +84,16 @@ Check https://github.com/andygrove/sqlparser-rs/commits/master for undocumented
 - Support basic forms of `CREATE SCHEMA` and `DROP SCHEMA` (#173) - thanks @alex-dukhno!
 - Support `NULLS FIRST`/`LAST` in `ORDER BY` expressions (#176) - thanks @houqp!
 - Support `LISTAGG()` (#174) - thanks @maxcountryman!
+- Support the string concatentation operator `||` (#178) - thanks @Dandandan!
+- Support bitwise AND (`&`), OR (`|`), XOR (`^`) (#181) - thanks @Dandandan!
+- Add serde support to AST structs and enums (#196) - thanks @panarch!
+- Support `ALTER TABLE ADD COLUMN`, `RENAME COLUMN`, and `RENAME TO` (#203) - thanks @mashuai!
+- Support `ALTER TABLE DROP COLUMN` (#148) - thanks @ivanceras!
+- Support `CREATE TABLE ... AS ...` (#206) - thanks @Dandandan!
 
 ### Fixed
 - Report an error for unterminated string literals (#165)
+- Make file format (`STORED AS`) case insensitive (#200) and don't allow quoting it (#201) - thanks @Dandandan!
 
 ## [0.5.0] - 2019-10-10
 
@@ -138,3 +208,4 @@ We don't have a changelog for the changes made in 2018, but thanks to @crw5996,
 
 ## [0.1.0] - 2018-09-03
 Initial release
+

Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "sqlparser"
 description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011"
-version = "0.5.1-alpha-0"
+version = "0.8.1-alpha.0"
 authors = ["Andy Grove <[email protected]>"]
-homepage = "https://github.com/andygrove/sqlparser-rs"
+homepage = "https://github.com/ballista-compute/sqlparser-rs"
 documentation = "https://docs.rs/sqlparser/"
 keywords = [ "ansi", "sql", "lexer", "parser" ]
-repository = "https://github.com/andygrove/sqlparser-rs"
+repository = "https://github.com/ballista-compute/sqlparser-rs"
 license = "Apache-2.0"
 include = [
     "src/**/*.rs",
@@ -19,15 +19,25 @@ name = "sqlparser"
 path = "src/lib.rs"
 
 [features]
-cst = ["rowan"]  # Retain a concrete synatax tree, available as `parser.syntax()`
+cst = ["rowan"] # Retain a concrete synatax tree, available as `parser.syntax()`
+json_example = ["serde_json", "serde"] # Enable JSON output in the `cli` example:
 
 [dependencies]
-bigdecimal = { version = "0.1.0", optional = true, features = ["serde"] }
-log = "0.4.5"
+bigdecimal = { version = "0.2", features = ["serde"], optional = true }
+log = "0.4"
 rowan = { version = "0.10.0", optional = true, features = ["serde1"] }
-serde = { version = "1.0.106", features = ["derive"] }
-serde_json = "1.0.52"
+serde = { version = "1.0", features = ["derive"], optional = true }
+# serde_json is only used in examples/cli, but we have to put it outside
+# of dev-dependencies because of
+# https://github.com/rust-lang/cargo/issues/1596
+serde_json = { version = "1.0", optional = true }
 
 [dev-dependencies]
-simple_logger = "1.0.1"
+simple_logger = "1.9"
 matches = "0.1"
+
+[package.metadata.release]
+# Instruct `cargo release` to not run `cargo publish` locally:
+# https://github.com/sunng87/cargo-release/blob/master/docs/reference.md#config-fields
+# See docs/releasing.md for details.
+disable-publish = true

README.md

@@ -185,17 +185,17 @@ Remarks
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Version](https://img.shields.io/crates/v/sqlparser.svg)](https://crates.io/crates/sqlparser)
-[![Build Status](https://travis-ci.org/andygrove/sqlparser-rs.svg?branch=master)](https://travis-ci.org/andygrove/sqlparser-rs)
-[![Coverage Status](https://coveralls.io/repos/github/andygrove/sqlparser-rs/badge.svg?branch=master)](https://coveralls.io/github/andygrove/sqlparser-rs?branch=master)
+[![Build Status](https://github.com/ballista-compute/sqlparser-rs/workflows/Rust/badge.svg?branch=main)](https://github.com/ballista-compute/sqlparser-rs/actions?query=workflow%3ARust+branch%3Amain)
+[![Coverage Status](https://coveralls.io/repos/github/ballista-compute/sqlparser-rs/badge.svg?branch=main)](https://coveralls.io/github/ballista-compute/sqlparser-rs?branch=main)
 [![Gitter Chat](https://badges.gitter.im/sqlparser-rs/community.svg)](https://gitter.im/sqlparser-rs/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 The goal of this project is to build a SQL lexer and parser capable of parsing
 SQL that conforms with the [ANSI/ISO SQL standard][sql-standard] while also
 making it easy to support custom dialects so that this crate can be used as a
 foundation for vendor-specific parsers.
 
-This parser is currently being used by the [DataFusion] query engine and
-[LocustDB].
+This parser is currently being used by the [DataFusion] query engine,
+[LocustDB], and [Ballista].
 
 ## Example
 
@@ -223,6 +223,12 @@ This outputs
 AST: [Query(Query { ctes: [], body: Select(Select { distinct: false, projection: [UnnamedExpr(Identifier("a")), UnnamedExpr(Identifier("b")), UnnamedExpr(Value(Long(123))), UnnamedExpr(Function(Function { name: ObjectName(["myfunc"]), args: [Identifier("b")], over: None, distinct: false }))], from: [TableWithJoins { relation: Table { name: ObjectName(["table_1"]), alias: None, args: [], with_hints: [] }, joins: [] }], selection: Some(BinaryOp { left: BinaryOp { left: Identifier("a"), op: Gt, right: Identifier("b") }, op: And, right: BinaryOp { left: Identifier("b"), op: Lt, right: Value(Long(100)) } }), group_by: [], having: None }), order_by: [OrderByExpr { expr: Identifier("a"), asc: Some(false) }, OrderByExpr { expr: Identifier("b"), asc: None }], limit: None, offset: None, fetch: None })]
 ```
 
+## Command line
+To parse a file and dump the results as JSON:
+```
+$ cargo run --features json_example --example cli FILENAME.sql [--dialectname]
+```
+
 ## SQL compliance
 
 SQL was first standardized in 1987, and revisions of the standard have been
@@ -300,9 +306,10 @@ resources.
 
 [tdop-tutorial]: https://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing
 [`cargo fmt`]: https://github.com/rust-lang/rustfmt#on-the-stable-toolchain
-[current issues]: https://github.com/andygrove/sqlparser-rs/issues
+[current issues]: https://github.com/ballista-compute/sqlparser-rs/issues
 [DataFusion]: https://github.com/apache/arrow/tree/master/rust/datafusion
 [LocustDB]: https://github.com/cswinter/LocustDB
+[Ballista]: https://github.com/ballista-compute/ballista
 [Pratt Parser]: https://tdop.github.io/
 [sql-2016-grammar]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html
 [sql-standard]: https://en.wikipedia.org/wiki/ISO/IEC_9075

docs/benchmarking.md

@@ -0,0 +1,6 @@
+# Benchmarking
+
+Run `cargo bench` in the project `sqlparser_bench` execute the queries.
+It will report results using the `criterion` library to perform the benchmarking.
+
+The bench project lives in another crate, to avoid the negative impact on building the `sqlparser` crate.

docs/releasing.md

@@ -0,0 +1,58 @@
+# Releasing
+
+## Prerequisites
+Publishing to crates.io has been automated via GitHub Actions, so you will only
+need push access to the [ballista-compute GitHub repository](https://github.com/ballista-compute/sqlparser-rs)
+in order to publish a release.
+
+We use the [`cargo release`](https://github.com/sunng87/cargo-release)
+subcommand to ensure correct versioning. Install via:
+
+```
+$ cargo install cargo-release
+```
+
+## Process
+
+1. **Before releasing** ensure `CHANGELOG.md` is updated appropriately and that
+    you have a clean checkout of the `main` branch of the sqlparser repository:
+    ```
+    $ git fetch && git status
+    On branch main
+    Your branch is up to date with 'upstream/main'.
+
+    nothing to commit, working tree clean
+    ```
+    * If you have the time, check that the examples in the README are up to date.
+
+2. Using `cargo-release` we can publish a new release like so:
+
+    ```
+    $ cargo release minor --push-remote upstream
+    ```
+
+    You can add `--dry-run` to see what the command is going to do,
+    or `--skip-push` to stop before actually publishing the release.
+
+    `cargo release` will then:
+
+    * Bump the minor part of the version in `Cargo.toml` (e.g. `0.7.1-alpha.0`
+       -> `0.8.0`. You can use `patch` instead of `minor`, as appropriate).
+    * Create a new tag (e.g. `v0.8.0`) locally
+    * Push the new tag to the specified remote (`upstream` in the above
+      example), which will trigger a publishing process to crates.io as part of
+      the [corresponding GitHub Action](https://github.com/ballista-compute/sqlparser-rs/blob/main/.github/workflows/rust.yml).
+
+      Note that credentials for authoring in this way are securely stored in
+      the (GitHub) repo secrets as `CRATE_TOKEN`.
+    * Bump the crate version again (to something like `0.8.1-alpha.0`) to
+      indicate the start of new development cycle.
+
+3. Push the updates to the `main` branch upstream:
+    ```
+    $ git push upstream
+    ```
+
+4. Check that the new version of the crate is available on crates.io:
+    https://crates.io/crates/sqlparser
+