diff --git a/.github/workflows/publish.dispatch.yml b/.github/workflows/publish.dispatch.yml index c48b585c..0f1b3d5f 100644 --- a/.github/workflows/publish.dispatch.yml +++ b/.github/workflows/publish.dispatch.yml @@ -39,7 +39,7 @@ jobs: return; } - if (releases.length < 100) { + if (releases.length < 100) { exhausted = true; } else if (page >= 10) { throw new Error("We iterated over 10 pages. Does the script work?"); @@ -55,8 +55,8 @@ jobs: - name: Abort if: steps.validate-release.outputs.has-release != 'true' run: | - { - echo "Tag ${{ github.event.inputs.release_tag }} not found." + { + echo "Tag ${{ github.event.inputs.release-tag }} not found." exit 1 } diff --git a/.github/workflows/publish.reusable.yml b/.github/workflows/publish.reusable.yml index fb0ff8fa..3d85b3c1 100644 --- a/.github/workflows/publish.reusable.yml +++ b/.github/workflows/publish.reusable.yml @@ -9,7 +9,6 @@ on: is-prerelease: type: string required: true - default: "false" jobs: publish: @@ -52,8 +51,8 @@ jobs: - name: Publish npm packages as nightly if: inputs.is-prerelease == 'true' run: | - for package in packages/@pglt/*; do - npm publish $package --tag nightly --access public --provenance + for package in packages/@pglt/*; do + npm publish "$package" --tag nightly --access public --provenance done env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} # @@ -61,8 +60,8 @@ jobs: - name: Publish npm packages as latest if: inputs.is-prerelease != 'true' run: | - for package in packages/@pglt/*; do - npm publish $package --tag latest --access public --provenance + for package in packages/@pglt/*; do + npm publish "$package" --tag latest --access public --provenance done env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/crates/pglt_lexer/src/lib.rs b/crates/pglt_lexer/src/lib.rs index 63abe365..3c4ed2af 100644 --- a/crates/pglt_lexer/src/lib.rs +++ b/crates/pglt_lexer/src/lib.rs @@ -61,8 +61,18 @@ pub static WHITESPACE_TOKENS: &[SyntaxKind] = &[ SyntaxKind::SqlComment, ]; -static PATTERN_LEXER: LazyLock = - LazyLock::new(|| Regex::new(r"(?P +)|(?P\r?\n+)|(?P\t+)").unwrap()); +static PATTERN_LEXER: LazyLock = LazyLock::new(|| { + #[cfg(windows)] + { + // On Windows, treat \r\n as a single newline token + Regex::new(r"(?P +)|(?P(\r\n|\n)+)|(?P\t+)").unwrap() + } + #[cfg(not(windows))] + { + // On other platforms, just check for \n + Regex::new(r"(?P +)|(?P\n+)|(?P\t+)").unwrap() + } +}); fn whitespace_tokens(input: &str) -> VecDeque { let mut tokens = VecDeque::new(); @@ -202,6 +212,22 @@ mod tests { assert_eq!(tokens[1].kind, SyntaxKind::Newline); } + #[test] + fn test_consecutive_newlines() { + // Test with multiple consecutive newlines + #[cfg(windows)] + let input = "select\r\n\r\n1"; + #[cfg(not(windows))] + let input = "select\n\n1"; + + let tokens = lex(input).unwrap(); + + // Check that we have exactly one newline token between "select" and "1" + assert_eq!(tokens[0].kind, SyntaxKind::Select); + assert_eq!(tokens[1].kind, SyntaxKind::Newline); + assert_eq!(tokens[2].kind, SyntaxKind::Iconst); + } + #[test] fn test_whitespace_tokens() { let input = "select 1"; diff --git a/crates/pglt_statement_splitter/src/lib.rs b/crates/pglt_statement_splitter/src/lib.rs index 24ca4c6a..7881ce27 100644 --- a/crates/pglt_statement_splitter/src/lib.rs +++ b/crates/pglt_statement_splitter/src/lib.rs @@ -114,6 +114,12 @@ mod tests { ]); } + #[test] + fn single_newlines() { + Tester::from("select 1\nfrom contact\n\nselect 3") + .expect_statements(vec!["select 1\nfrom contact", "select 3"]); + } + #[test] fn alter_column() { Tester::from("alter table users alter column email drop not null;") diff --git a/crates/pglt_statement_splitter/src/parser.rs b/crates/pglt_statement_splitter/src/parser.rs index 915c9ad2..f89f28dd 100644 --- a/crates/pglt_statement_splitter/src/parser.rs +++ b/crates/pglt_statement_splitter/src/parser.rs @@ -183,6 +183,17 @@ impl Parser { } } +#[cfg(windows)] +/// Returns true if the token is relevant for the paring process +/// +/// On windows, a newline is represented by `\r\n` which is two characters. +fn is_irrelevant_token(t: &Token) -> bool { + WHITESPACE_TOKENS.contains(&t.kind) + && (t.kind != SyntaxKind::Newline || t.text == "\r\n" || t.text.chars().count() == 1) +} + +#[cfg(not(windows))] +/// Returns true if the token is relevant for the paring process fn is_irrelevant_token(t: &Token) -> bool { WHITESPACE_TOKENS.contains(&t.kind) && (t.kind != SyntaxKind::Newline || t.text.chars().count() == 1) diff --git a/crates/pglt_statement_splitter/tests/data/with_comments__4.sql b/crates/pglt_statement_splitter/tests/data/with_comments__4.sql new file mode 100644 index 00000000..652185cd --- /dev/null +++ b/crates/pglt_statement_splitter/tests/data/with_comments__4.sql @@ -0,0 +1,13 @@ +-- test +select id, name, test1231234123, unknown from co; + +-- in between two statements + +select 14433313331333 -- after a statement + +alter table --within a statement +test drop column id; + +select lower('test'); +--after a statement +