From e778f0cbf75b2c2358e432fa6decf6f1c48dec0b Mon Sep 17 00:00:00 2001 From: Saransh Kasliwal Date: Sat, 22 Nov 2025 11:16:30 +0530 Subject: [PATCH 1/6] Handle extra semicolons after import statements Enhanced the parser to detect and consume extra semicolons following import statements, storing them in the whitespace for accurate source representation. Added a debug test to verify import parsing and whitespace handling. --- .../ReloadableJava21ParserVisitor.java | 31 ++++++- .../org/openrewrite/java/MyParserDebug.java | 80 +++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java diff --git a/rewrite-java-21/src/main/java/org/openrewrite/java/isolated/ReloadableJava21ParserVisitor.java b/rewrite-java-21/src/main/java/org/openrewrite/java/isolated/ReloadableJava21ParserVisitor.java index 0d5012f6f5..6a2ee43f08 100644 --- a/rewrite-java-21/src/main/java/org/openrewrite/java/isolated/ReloadableJava21ParserVisitor.java +++ b/rewrite-java-21/src/main/java/org/openrewrite/java/isolated/ReloadableJava21ParserVisitor.java @@ -1984,7 +1984,36 @@ private Space statementDelim(@Nullable Tree t) { case ASSERT: case ASSIGNMENT: case DO_WHILE_LOOP: - case IMPORT: + case IMPORT:{ + System.err.println("IMPORT: cursor before=" + cursor + ", char='" + + (cursor < source.length() ? source.charAt(cursor) : "EOF") + "'"); + + // Only consume the required semicolon + Space result = sourceBefore(";"); + + // Check for extra semicolons - consume them and handle specially + if (cursor < source.length() && source.charAt(cursor) == ';') { + // Count and consume extra semicolons + int extraSemicolons = 0; + while (cursor < source.length() && source.charAt(cursor) == ';') { + extraSemicolons++; + cursor++; + } + + // Store extra semicolons in the whitespace with a special format + // Prepend the semicolons to the existing whitespace + String currentWhitespace = result.getWhitespace(); + String extraSemiStr = ";".repeat(extraSemicolons); + result = result.withWhitespace(extraSemiStr + currentWhitespace); + + System.err.println("IMPORT: consumed " + extraSemicolons + " extra semicolon(s), cursor now at " + cursor); + } + + + System.err.println("IMPORT: cursor after=" + cursor + ", char='" + + (cursor < source.length() ? source.charAt(cursor) : "EOF") + "'"); + return result; + } case METHOD_INVOCATION: case NEW_CLASS: case THROW: diff --git a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java new file mode 100644 index 0000000000..3d4a52fedd --- /dev/null +++ b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java @@ -0,0 +1,80 @@ +package org.openrewrite.java; + +import org.junit.jupiter.api.Test; +import org.openrewrite.SourceFile; +import org.openrewrite.java.tree.J; +import org.openrewrite.tree.ParseError; +import java.util.List; +import java.util.stream.Collectors; +import org.openrewrite.java.TreeVisitingPrinter; + + +class MyParserDebug { + + @Test + void debugDirectly() { + JavaParser parser = JavaParser.fromJavaVersion().build(); + + String sourceCode = """ + import java.util.*;; + import java.io.*; + + public class Main { + public static void main(String[] args) {} + } + """; + + List sourceFiles = parser.parse(sourceCode) + .collect(Collectors.toList()); + + SourceFile result = sourceFiles.get(0); + + // --- CHECK FOR PARSE ERROR --- + if (result instanceof ParseError) { + ParseError error = (ParseError) result; + System.err.println("\n\n========================================"); + System.err.println("CRITICAL: PARSER FAILED"); + System.err.println("The parser could not understand the code."); + System.err.println("Error Message: " + error.getText()); // This usually holds the source + System.err.println("To String: " + error.toString()); + System.err.println("========================================\n\n"); + + // Test should fail if parsing fails + throw new RuntimeException("Parser failed: " + error.toString()); + } + else if (result instanceof J.CompilationUnit) { + // This part runs if parsing SUCCEEDS (what we were trying before) + J.CompilationUnit cu = (J.CompilationUnit) result; + var imports = cu.getImports(); + + System.err.println("========================================"); + System.err.println("SUCCESS: Parser succeeded!"); + System.err.println("Number of imports: " + imports.size()); + + if (imports.size() > 1) { + String prefix = imports.get(1).getPrefix().getWhitespace(); + System.err.println("Second import prefix hex: " + stringToHex(prefix)); + System.err.println("Second import prefix text: '" + prefix.replace("\n", "\\n").replace("\r", "\\r") + "'"); + } + + String printedSource = cu.print(); + System.err.println("Original source length: " + sourceCode.length()); + System.err.println("Printed source length: " + printedSource.length()); + System.err.println("Sources match: " + sourceCode.equals(printedSource)); + + System.err.println("LST Structure:"); + System.err.println(TreeVisitingPrinter.printTree((cu))); + + System.err.println("========================================"); + } + } + + private String stringToHex(String input) { + if (input == null) return "null"; + StringBuilder sb = new StringBuilder(); + for (char c : input.toCharArray()) { + sb.append(String.format("\\u%04x ", (int) c)); + } + return sb.toString(); + } +} \ No newline at end of file From c6f951bef5ef56cfbdec43733fc789cd64157c29 Mon Sep 17 00:00:00 2001 From: saranshflip Date: Mon, 24 Nov 2025 10:04:55 +0530 Subject: [PATCH 2/6] Update comment Refine comment for clarity in parsing success case. --- .../src/test/java/org/openrewrite/java/MyParserDebug.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java index 3d4a52fedd..a8d0a9ad60 100644 --- a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java +++ b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java @@ -43,7 +43,7 @@ public static void main(String[] args) {} throw new RuntimeException("Parser failed: " + error.toString()); } else if (result instanceof J.CompilationUnit) { - // This part runs if parsing SUCCEEDS (what we were trying before) + // This part runs if parsing SUCCEEDS J.CompilationUnit cu = (J.CompilationUnit) result; var imports = cu.getImports(); @@ -77,4 +77,4 @@ private String stringToHex(String input) { } return sb.toString(); } -} \ No newline at end of file +} From 09d3d15c13110b970051e810d93866c0c26597de Mon Sep 17 00:00:00 2001 From: Tim te Beek Date: Thu, 4 Dec 2025 12:53:31 +0100 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../org/openrewrite/java/MyParserDebug.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java index a8d0a9ad60..6837efcd3a 100644 --- a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java +++ b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java @@ -1,3 +1,18 @@ +/* + * Copyright 2025 the original author or authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * https://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openrewrite.java; import org.junit.jupiter.api.Test; @@ -5,7 +20,8 @@ import org.openrewrite.java.tree.J; import org.openrewrite.tree.ParseError; import java.util.List; -import java.util.stream.Collectors; +import static java.util.stream.Collectors.toList; + import org.openrewrite.java.TreeVisitingPrinter; @@ -25,7 +41,7 @@ public static void main(String[] args) {} """; List sourceFiles = parser.parse(sourceCode) - .collect(Collectors.toList()); + .collect(toList()); SourceFile result = sourceFiles.get(0); From 497c25196825e6d1a3541c1a044dc9951afcae62 Mon Sep 17 00:00:00 2001 From: Saransh Kasliwal Date: Tue, 9 Dec 2025 10:40:13 +0530 Subject: [PATCH 4/6] Changes made in MyParserDebug file to use RewriteTest --- .../src/test/java/org/openrewrite/java/MyParserDebug.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java index 6837efcd3a..efee8acb66 100644 --- a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java +++ b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java @@ -15,9 +15,11 @@ */ package org.openrewrite.java; +import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.Test; import org.openrewrite.SourceFile; import org.openrewrite.java.tree.J; +import org.openrewrite.test.RewriteTest; import org.openrewrite.tree.ParseError; import java.util.List; import static java.util.stream.Collectors.toList; @@ -25,12 +27,13 @@ import org.openrewrite.java.TreeVisitingPrinter; -class MyParserDebug { +class MyParserDebug implements RewriteTest { @Test void debugDirectly() { JavaParser parser = JavaParser.fromJavaVersion().build(); + @Language("java") String sourceCode = """ import java.util.*;; import java.io.*; From 21a4794b6a610cdc20ba1c61ecc856b391041d9c Mon Sep 17 00:00:00 2001 From: Tim te Beek Date: Tue, 9 Dec 2025 12:16:32 +0100 Subject: [PATCH 5/6] Expand and enable `ImportTest.semicolonAfterPackage` --- .../java/org/openrewrite/java/tree/ImportTest.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rewrite-java-tck/src/main/java/org/openrewrite/java/tree/ImportTest.java b/rewrite-java-tck/src/main/java/org/openrewrite/java/tree/ImportTest.java index 8771747a89..666b7beb3e 100644 --- a/rewrite-java-tck/src/main/java/org/openrewrite/java/tree/ImportTest.java +++ b/rewrite-java-tck/src/main/java/org/openrewrite/java/tree/ImportTest.java @@ -188,20 +188,25 @@ void spaceBeforeSemiColon() { ); } - @Disabled("Parser does not support semicolon after package declaration yet") @Issue("https://github.com/openrewrite/rewrite-migrate-java/issues/396") @Test void semicolonAfterPackage() { - //language=java rewriteRun( spec -> spec - .typeValidationOptions(TypeValidation.all().allowNonWhitespaceInWhitespace(true)), + .typeValidationOptions(TypeValidation.all() + .allowNonWhitespaceInWhitespace(true) + .parseAndPrintEquality(false)), java( + //language=java """ package p;; import java.util.List; class AfterPackage { } - """ + """, + spec -> spec.beforeRecipe(cu -> { + System.out.println(cu.printAll()); + assertThat(cu.getImports().getFirst().getQualid()).hasToString("java.util.List"); + }) ) ); } From 3842e428c9ab63ce9c5d263e6560ef65034173a7 Mon Sep 17 00:00:00 2001 From: Tim te Beek Date: Tue, 9 Dec 2025 12:17:20 +0100 Subject: [PATCH 6/6] Delete MyParserDebug, as the same can be achieved in `ImportTest` --- .../org/openrewrite/java/MyParserDebug.java | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100644 rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java diff --git a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java b/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java deleted file mode 100644 index efee8acb66..0000000000 --- a/rewrite-java/src/test/java/org/openrewrite/java/MyParserDebug.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2025 the original author or authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * https://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.openrewrite.java; - -import org.intellij.lang.annotations.Language; -import org.junit.jupiter.api.Test; -import org.openrewrite.SourceFile; -import org.openrewrite.java.tree.J; -import org.openrewrite.test.RewriteTest; -import org.openrewrite.tree.ParseError; -import java.util.List; -import static java.util.stream.Collectors.toList; - -import org.openrewrite.java.TreeVisitingPrinter; - - -class MyParserDebug implements RewriteTest { - - @Test - void debugDirectly() { - JavaParser parser = JavaParser.fromJavaVersion().build(); - - @Language("java") - String sourceCode = """ - import java.util.*;; - import java.io.*; - - public class Main { - public static void main(String[] args) {} - } - """; - - List sourceFiles = parser.parse(sourceCode) - .collect(toList()); - - SourceFile result = sourceFiles.get(0); - - // --- CHECK FOR PARSE ERROR --- - if (result instanceof ParseError) { - ParseError error = (ParseError) result; - System.err.println("\n\n========================================"); - System.err.println("CRITICAL: PARSER FAILED"); - System.err.println("The parser could not understand the code."); - System.err.println("Error Message: " + error.getText()); // This usually holds the source - System.err.println("To String: " + error.toString()); - System.err.println("========================================\n\n"); - - // Test should fail if parsing fails - throw new RuntimeException("Parser failed: " + error.toString()); - } - else if (result instanceof J.CompilationUnit) { - // This part runs if parsing SUCCEEDS - J.CompilationUnit cu = (J.CompilationUnit) result; - var imports = cu.getImports(); - - System.err.println("========================================"); - System.err.println("SUCCESS: Parser succeeded!"); - System.err.println("Number of imports: " + imports.size()); - - if (imports.size() > 1) { - String prefix = imports.get(1).getPrefix().getWhitespace(); - System.err.println("Second import prefix hex: " + stringToHex(prefix)); - System.err.println("Second import prefix text: '" + prefix.replace("\n", "\\n").replace("\r", "\\r") + "'"); - } - - String printedSource = cu.print(); - System.err.println("Original source length: " + sourceCode.length()); - System.err.println("Printed source length: " + printedSource.length()); - System.err.println("Sources match: " + sourceCode.equals(printedSource)); - - System.err.println("LST Structure:"); - System.err.println(TreeVisitingPrinter.printTree((cu))); - - System.err.println("========================================"); - } - } - - private String stringToHex(String input) { - if (input == null) return "null"; - StringBuilder sb = new StringBuilder(); - for (char c : input.toCharArray()) { - sb.append(String.format("\\u%04x ", (int) c)); - } - return sb.toString(); - } -}