Tokenizer/PHP: fix tokens missing/file content being removed by attribute retokenization

src/Tokenizers/PHP.php

@@ -1446,15 +1446,11 @@ protected function tokenize($string)
                 && strpos($token[1], '#[') === 0
             ) {
                 $subTokens = $this->parsePhpAttribute($tokens, $stackPtr);
-                if ($subTokens !== null) {
-                    array_splice($tokens, $stackPtr, 1, $subTokens);
-                    $numTokens = count($tokens);
+                array_splice($tokens, $stackPtr, 1, $subTokens);
+                $numTokens = count($tokens);
 
-                    $tokenIsArray = true;
-                    $token        = $tokens[$stackPtr];
-                } else {
-                    $token[0] = T_ATTRIBUTE;
-                }
+                $tokenIsArray = true;
+                $token        = $tokens[$stackPtr];
             }
 
             if ($tokenIsArray === true
@@ -4105,11 +4101,10 @@ private function findCloser(array &$tokens, $start, $openerTokens, $closerChar)
      * @param array $tokens   The original array of tokens (as returned by token_get_all).
      * @param int   $stackPtr The current position in token array.
      *
-     * @return array|null The array of parsed attribute tokens
+     * @return array The array of parsed attribute tokens
      */
     private function parsePhpAttribute(array &$tokens, $stackPtr)
     {
-
         $token = $tokens[$stackPtr];
 
         $commentBody = substr($token[1], 2);
@@ -4121,18 +4116,24 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)
                 && strpos($subToken[1], '#[') === 0
             ) {
                 $reparsed = $this->parsePhpAttribute($subTokens, $i);
-                if ($reparsed !== null) {
-                    array_splice($subTokens, $i, 1, $reparsed);
-                } else {
-                    $subToken[0] = T_ATTRIBUTE;
-                }
+                array_splice($subTokens, $i, 1, $reparsed);
             }
         }
 
         array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
 
         // Go looking for the close bracket.
         $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+
+        /*
+         * No closer bracket found, this might be a multi-line attribute,
+         * but it could also be an unfinished attribute (parse error).
+         *
+         * If it is a multi-line attribute, we need to grab a larger part of the code.
+         * If it is a parse error, we need to stick with only handling the line
+         * containing the attribute opener.
+         */
+
         if (PHP_VERSION_ID < 80000 && $bracketCloser === null) {
             foreach (array_slice($tokens, ($stackPtr + 1)) as $token) {
                 if (is_array($token) === true) {
@@ -4142,20 +4143,17 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)
                 }
             }
 
-            $subTokens = @token_get_all('<?php '.$commentBody);
-            array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
+            $newSubTokens = @token_get_all('<?php '.$commentBody);
+            array_splice($newSubTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
 
-            $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+            $bracketCloser = $this->findCloser($newSubTokens, 1, '[', ']');
             if ($bracketCloser !== null) {
-                array_splice($tokens, ($stackPtr + 1), count($tokens), array_slice($subTokens, ($bracketCloser + 1)));
-                $subTokens = array_slice($subTokens, 0, ($bracketCloser + 1));
+                // We found the closer, overwrite the original $subTokens array.
+                array_splice($tokens, ($stackPtr + 1), count($tokens), array_slice($newSubTokens, ($bracketCloser + 1)));
+                $subTokens = array_slice($newSubTokens, 0, ($bracketCloser + 1));
             }
         }
 
-        if ($bracketCloser === null) {
-            return null;
-        }
-
         return $subTokens;
 
     }//end parsePhpAttribute()

tests/Core/Tokenizers/PHP/AttributesParseError1Test.inc

@@ -0,0 +1,8 @@
+<?php
+
+// Intentional parse error.
+// This must be the only test in the file.
+
+/* testInvalidAttribute */
+#[ThisIsNotAnAttribute
+function invalid_attribute_test() {}

tests/Core/Tokenizers/PHP/AttributesParseError1Test.php

@@ -0,0 +1,66 @@
+<?php
+/**
+ * Tests the support of PHP 8 attributes
+ *
+ * @author    Alessandro Chitolina <[email protected]>
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2019-2023 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @copyright 2023 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/HEAD/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+final class AttributesParseError1Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that invalid attribute (or comment starting with #[ and without ]) are parsed correctly
+     * and that tokens "within" the attribute are not removed.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
+     *
+     * @return void
+     */
+    public function testInvalidAttribute()
+    {
+        $tokens = $this->phpcsFile->getTokens();
+
+        $attribute = $this->getTargetToken('/* testInvalidAttribute */', T_ATTRIBUTE);
+
+        $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
+        $this->assertNull($tokens[$attribute]['attribute_closer']);
+
+        $expectedTokenCodes = [
+            'T_ATTRIBUTE',
+            'T_STRING',
+            'T_WHITESPACE',
+            'T_FUNCTION',
+        ];
+        $length = count($expectedTokenCodes);
+
+        $map = array_map(
+            function ($token) {
+                if ($token['code'] === T_ATTRIBUTE) {
+                    $this->assertArrayHasKey('attribute_closer', $token);
+                    $this->assertNull($token['attribute_closer']);
+                } else {
+                    $this->assertArrayNotHasKey('attribute_closer', $token);
+                }
+
+                return $token['type'];
+            },
+            array_slice($tokens, $attribute, $length)
+        );
+
+        $this->assertSame($expectedTokenCodes, $map);
+
+    }//end testInvalidAttribute()
+
+
+}//end class

tests/Core/Tokenizers/PHP/AttributesParseError2Test.inc

@@ -0,0 +1,8 @@
+<?php
+
+// Intentional parse error.
+// This must be the only test in the file.
+
+/* testLiveCoding */
+#[AttributeName(10)
+function hasUnfinishedAttribute() {}

tests/Core/Tokenizers/PHP/AttributesParseError2Test.php

@@ -0,0 +1,67 @@
+<?php
+/**
+ * Tests the support of PHP 8 attributes
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/HEAD/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+final class AttributesParseError2Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that invalid attribute (or comment starting with #[ and without ]) are parsed correctly
+     * and that tokens "within" the attribute are not removed.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
+     *
+     * @return void
+     */
+    public function testInvalidAttribute()
+    {
+        $tokens = $this->phpcsFile->getTokens();
+
+        $attribute = $this->getTargetToken('/* testLiveCoding */', T_ATTRIBUTE);
+
+        $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
+        $this->assertNull($tokens[$attribute]['attribute_closer']);
+
+        $expectedTokenCodes = [
+            'T_ATTRIBUTE',
+            'T_STRING',
+            'T_OPEN_PARENTHESIS',
+            'T_LNUMBER',
+            'T_CLOSE_PARENTHESIS',
+            'T_WHITESPACE',
+            'T_FUNCTION',
+        ];
+
+        $length = count($expectedTokenCodes);
+
+        $map = array_map(
+            function ($token) {
+                if ($token['code'] === T_ATTRIBUTE) {
+                    $this->assertArrayHasKey('attribute_closer', $token);
+                    $this->assertNull($token['attribute_closer']);
+                } else {
+                    $this->assertArrayNotHasKey('attribute_closer', $token);
+                }
+
+                return $token['type'];
+            },
+            array_slice($tokens, $attribute, $length)
+        );
+
+        $this->assertSame($expectedTokenCodes, $map);
+
+    }//end testInvalidAttribute()
+
+
+}//end class

tests/Core/Tokenizers/PHP/AttributesParseError3Test.inc

@@ -0,0 +1,10 @@
+<?php
+
+// Intentional parse error.
+// This must be the only test in the file.
+
+class LiveCoding {
+    /* testLiveCoding */
+    #[AttributeName(10), SecondAttribute(
+    public final function hasUnfinishedAttribute() {}
+}

tests/Core/Tokenizers/PHP/AttributesParseError3Test.php

@@ -0,0 +1,76 @@
+<?php
+/**
+ * Tests the support of PHP 8 attributes
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/HEAD/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+final class AttributesParseError3Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that invalid attribute (or comment starting with #[ and without ]) are parsed correctly
+     * and that tokens "within" the attribute are not removed.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::findCloser
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
+     *
+     * @return void
+     */
+    public function testInvalidAttribute()
+    {
+        $tokens = $this->phpcsFile->getTokens();
+
+        $attribute = $this->getTargetToken('/* testLiveCoding */', T_ATTRIBUTE);
+
+        $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
+        $this->assertNull($tokens[$attribute]['attribute_closer']);
+
+        $expectedTokenCodes = [
+            'T_ATTRIBUTE',
+            'T_STRING',
+            'T_OPEN_PARENTHESIS',
+            'T_LNUMBER',
+            'T_CLOSE_PARENTHESIS',
+            'T_COMMA',
+            'T_WHITESPACE',
+            'T_STRING',
+            'T_OPEN_PARENTHESIS',
+            'T_WHITESPACE',
+            'T_WHITESPACE',
+            'T_PUBLIC',
+            'T_WHITESPACE',
+            'T_FINAL',
+            'T_WHITESPACE',
+            'T_FUNCTION',
+        ];
+
+        $length = count($expectedTokenCodes);
+
+        $map = array_map(
+            function ($token) {
+                if ($token['code'] === T_ATTRIBUTE) {
+                    $this->assertArrayHasKey('attribute_closer', $token);
+                    $this->assertNull($token['attribute_closer']);
+                } else {
+                    $this->assertArrayNotHasKey('attribute_closer', $token);
+                }
+
+                return $token['type'];
+            },
+            array_slice($tokens, $attribute, $length)
+        );
+
+        $this->assertSame($expectedTokenCodes, $map);
+
+    }//end testInvalidAttribute()
+
+
+}//end class

tests/Core/Tokenizers/PHP/AttributesParseError4Test.inc

@@ -0,0 +1,8 @@
+<?php
+
+// Intentional parse error.
+// This must be the only test in the file.
+
+/* testLiveCoding */
+#[ClosedAttribute] #[UnfinishedAttribute
+function hasUnfinishedAttribute() {}