GH-2412: Support for indexing and analyzing phars

.github/workflows/ci.yml

@@ -126,7 +126,7 @@ jobs:
                     composer-options: "--no-scripts"
             -
                 name: "Run PHPUnit"
-                run: "php -dzend.assertions=1 vendor/bin/phpunit"
+                run: "php -dphar.readonly=0 -dzend.assertions=1 vendor/bin/phpunit"
     phpstan:
         name: "PHPStan (${{ matrix.php-version }})"
 

CHANGELOG.md

@@ -3,6 +3,10 @@ Changelog
 
 ## master
 
+Features:
+
+  - PHAR Indexing #2412 @dantleech
+
 Improvements:
 
   - Basic support for `array_reduce` stub #2576

doc/reference/configuration.rst

@@ -1614,7 +1614,7 @@ Type: array
 Glob patterns to include while indexing
 
 
-**Default**: ``["\/**\/*.php"]``
+**Default**: ``["\/**\/*.php","\/**\/*.phar"]``
 
 
 .. _param_indexer.exclude_patterns:
@@ -1758,7 +1758,7 @@ Type: array
 File extensions (e.g. `php`) for files that should be indexed
 
 
-**Default**: ``["php"]``
+**Default**: ``["php","phar"]``
 
 
 .. _ObjectRendererExtension:

lib/Indexer/Adapter/Filesystem/FilesystemFileListProvider.php

@@ -20,7 +20,7 @@ public function __construct(
         private Filesystem $filesystem,
         private array $includePatterns = [],
         private array $excludePatterns = [],
-        private array $supportedExtensions = ['php'],
+        private array $supportedExtensions = ['php', 'phar'],
     ) {
     }
 

lib/Indexer/Adapter/Php/Serialized/FileRepository.php

@@ -17,7 +17,7 @@ class FileRepository
     /**
      * Increment this number each time there is a B/C break in the index.
      */
-    private const VERSION = 1;
+    private const VERSION = 2;
 
     /**
      * Flush to the filesystem after BATCH_SIZE updates

lib/Indexer/Adapter/Tolerant/Indexer/AbstractClassLikeIndexer.php

@@ -68,7 +68,13 @@ protected function getClassLikeRecord(string $type, Node $node, Index $index, Te
         if (empty($name)) {
             throw new CannotIndexNode(sprintf(
                 'Name is empty for file "%s"',
-                $document->uri()->path()
+                $document->uri()?->__toString() ?? 'unknown',
+            ));
+        }
+        if (!$document->uri()) {
+            throw new CannotIndexNode(sprintf(
+                'Document has no URI for class "%s"',
+                $name
             ));
         }
 
@@ -77,7 +83,7 @@ protected function getClassLikeRecord(string $type, Node $node, Index $index, Te
         /** @var ClassDeclaration|InterfaceDeclaration|EnumDeclaration|TraitDeclaration $node */
         $record->setStart(ByteOffset::fromInt($node->name->getStartPosition()));
         $record->setEnd(ByteOffset::fromInt($node->name->getEndPosition()));
-        $record->setFilePath($document->uri()->path());
+        $record->setFilePath($document->uriOrThrow());
         $record->setType($type);
 
         return $record;

lib/Indexer/Adapter/Tolerant/Indexer/ClassDeclarationIndexer.php

@@ -25,7 +25,7 @@ public function index(Index $index, TextDocument $document, Node $node): void
         if ($node->name instanceof MissingToken) {
             throw new CannotIndexNode(sprintf(
                 'Class name is missing (maybe a reserved word) in: %s',
-                $document->uri()?->path() ?? '?',
+                $document->uri()?->__toString() ?? '?',
             ));
         }
         $record = $this->getClassLikeRecord(ClassRecord::TYPE_CLASS, $node, $index, $document);

lib/Indexer/Adapter/Tolerant/Indexer/ClassLikeReferenceIndexer.php

@@ -31,7 +31,7 @@ public function canIndex(Node $node): bool
 
     public function beforeParse(Index $index, TextDocument $document): void
     {
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
 
         foreach ($fileRecord->references() as $outgoingReference) {
@@ -67,11 +67,11 @@ public function index(Index $index, TextDocument $document, Node $node): void
 
         $targetRecord = $index->get(ClassRecord::fromName($name));
         assert($targetRecord instanceof ClassRecord);
-        $targetRecord->addReference($document->uri()->path());
+        $targetRecord->addReference($document->uriOrThrow()->__toString());
 
         $index->write($targetRecord);
 
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
         $reference = new RecordReference(
             ClassRecord::RECORD_TYPE,

lib/Indexer/Adapter/Tolerant/Indexer/ConstantDeclarationIndexer.php

@@ -69,7 +69,7 @@ private function fromConstDeclaration(Node $node, Index $index, TextDocument $do
             assert($record instanceof ConstantRecord);
             $record->setStart(ByteOffset::fromInt($node->getStartPosition()));
             $record->setEnd(ByteOffset::fromInt($node->getEndPosition()));
-            $record->setFilePath($document->uri()->path());
+            $record->setFilePath($document->uriOrThrow());
             $index->write($record);
         }
     }
@@ -95,7 +95,7 @@ private function fromDefine(CallExpression $node, Index $index, TextDocument $do
             assert($record instanceof ConstantRecord);
             $record->setStart(ByteOffset::fromInt($node->getStartPosition()));
             $record->setEnd(ByteOffset::fromInt($node->getEndPosition()));
-            $record->setFilePath($document->uri()->path());
+            $record->setFilePath($document->uriOrThrow());
             $index->write($record);
 
             // Return after the first argument, because we only need the name of the constant.

lib/Indexer/Adapter/Tolerant/Indexer/EnumDeclarationIndexer.php

@@ -23,7 +23,7 @@ public function index(Index $index, TextDocument $document, Node $node): void
         if ($node->name instanceof MissingToken) {
             throw new CannotIndexNode(sprintf(
                 'Class name is missing (maybe a reserved word) in: %s',
-                $document->uri()?->path() ?? '?',
+                $document->uri()?->__toString() ?? '?',
             ));
         }
         $record = $this->getClassLikeRecord(ClassRecord::TYPE_ENUM, $node, $index, $document);

lib/Indexer/Adapter/Tolerant/Indexer/FunctionDeclarationIndexer.php

@@ -24,7 +24,7 @@ public function index(Index $index, TextDocument $document, Node $node): void
         assert($record instanceof FunctionRecord);
         $record->setStart(ByteOffset::fromInt($node->getStartPosition()));
         $record->setEnd(ByteOffset::fromInt($node->getEndPosition()));
-        $record->setFilePath($document->uri()->path());
+        $record->setFilePath($document->uriOrThrow());
         $index->write($record);
     }
 

lib/Indexer/Adapter/Tolerant/Indexer/FunctionReferenceIndexer.php

@@ -20,7 +20,7 @@ public function canIndex(Node $node): bool
 
     public function beforeParse(Index $index, TextDocument $document): void
     {
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
 
         foreach ($fileRecord->references() as $outgoingReference) {
@@ -48,10 +48,10 @@ public function index(Index $index, TextDocument $document, Node $node): void
 
         $targetRecord = $index->get(FunctionRecord::fromName($name));
         assert($targetRecord instanceof FunctionRecord);
-        $targetRecord->addReference($document->uri()->path());
+        $targetRecord->addReference($document->uriOrThrow());
         $index->write($targetRecord);
 
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
 
         $fileRecord->addReference(

lib/Indexer/Adapter/Tolerant/Indexer/InterfaceDeclarationIndexer.php

@@ -23,7 +23,7 @@ public function index(Index $index, TextDocument $document, Node $node): void
         if ($node->name instanceof MissingToken) {
             throw new CannotIndexNode(sprintf(
                 'Class name is missing (maybe a reserved word) in: %s',
-                $document->uri()?->path() ?? '?',
+                $document->uri()?->__toString() ?? '?',
             ));
         }
         $record = $this->getClassLikeRecord(ClassRecord::TYPE_INTERFACE, $node, $index, $document);

lib/Indexer/Adapter/Tolerant/Indexer/MemberIndexer.php

@@ -27,7 +27,7 @@ public function canIndex(Node $node): bool
 
     public function beforeParse(Index $index, TextDocument $document): void
     {
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
 
         foreach ($fileRecord->references() as $outgoingReference) {
@@ -181,10 +181,10 @@ private function writeIndex(
     ): void {
         $record = $index->get(MemberRecord::fromMemberReference(MemberReference::create($memberType, $containerFqn, $memberName)));
         assert($record instanceof MemberRecord);
-        $record->addReference($document->uri()->path());
+        $record->addReference($document->uriOrThrow()->__toString());
         $index->write($record);
 
-        $fileRecord = $index->get(FileRecord::fromPath($document->uri()->path()));
+        $fileRecord = $index->get(FileRecord::fromPath($document->uriOrThrow()->__toString()));
         assert($fileRecord instanceof FileRecord);
         $fileRecord->addReference(
             RecordReference::fromRecordAndOffsetAndContainerType($record, $offsetStart, $offsetEnd, $containerFqn)

lib/Indexer/Adapter/Tolerant/Indexer/TraitDeclarationIndexer.php

@@ -23,7 +23,7 @@ public function index(Index $index, TextDocument $document, Node $node): void
         if ($node->name instanceof MissingToken) {
             throw new CannotIndexNode(sprintf(
                 'Class name is missing (maybe a reserved word) in: %s',
-                $document->uri()?->path() ?? '?',
+                $document->uri()?->__toString() ?? '?',
             ));
         }
         $record = $this->getClassLikeRecord(ClassRecord::TYPE_TRAIT, $node, $index, $document);

lib/Indexer/Adapter/Tolerant/TolerantIndexBuilder.php

@@ -63,7 +63,7 @@ public function index(TextDocument $document): void
             $indexer->beforeParse($this->index, $document);
         }
 
-        $node = $this->parser->parseSourceFile($document->__toString(), $document->uri()->path());
+        $node = $this->parser->parseSourceFile($document->__toString(), $document->uri()->__toString());
         $this->indexNode($document, $node);
     }
 
@@ -83,7 +83,7 @@ private function indexNode(TextDocument $document, Node $node): void
                 $this->logger->warning(sprintf(
                     'Cannot index node of class "%s" in file "%s": %s',
                     get_class($node),
-                    $document->uri()->__toString(),
+                    $document->uri()?->__toString() ?? 'unknown',
                     $cannotIndexNode->getMessage()
                 ));
             }

lib/Indexer/Extension/IndexerExtension.php

@@ -66,7 +66,7 @@ class IndexerExtension implements Extension
     public const PARAM_IMPLEMENTATIONS_DEEP_REFERENCES = 'indexer.implementation_finder.deep';
     public const PARAM_STUB_PATHS = 'indexer.stub_paths';
     public const PARAM_SUPPORTED_EXTENSIONS = 'indexer.supported_extensions';
-    const TAG_WATCHER = 'indexer.watcher';
+    public const TAG_WATCHER = 'indexer.watcher';
     private const SERVICE_INDEXER_EXCLUDE_PATTERNS = 'indexer.exclude_patterns';
     private const SERVICE_INDEXER_INCLUDE_PATTERNS = 'indexer.include_patterns';
     private const PARAM_PROJECT_ROOT = 'indexer.project_root';
@@ -79,6 +79,7 @@ public function configure(Resolver $schema): void
             self::PARAM_INDEX_PATH => '%cache%/index/%project_id%',
             self::PARAM_INCLUDE_PATTERNS => [
                 '/**/*.php',
+                '/**/*.phar',
             ],
             self::PARAM_EXCLUDE_PATTERNS => [
                 '/vendor/**/Tests/**/*',
@@ -92,7 +93,7 @@ public function configure(Resolver $schema): void
             self::PARAM_PROJECT_ROOT => '%project_root%',
             self::PARAM_REFERENCES_DEEP_REFERENCES => true,
             self::PARAM_IMPLEMENTATIONS_DEEP_REFERENCES => true,
-            self::PARAM_SUPPORTED_EXTENSIONS => ['php'],
+            self::PARAM_SUPPORTED_EXTENSIONS => ['php', 'phar'],
         ]);
         $schema->setDescriptions([
             self::PARAM_ENABLED_WATCHERS => 'List of allowed watchers. The first watcher that supports the current system will be used',

lib/Indexer/IndexAgentBuilder.php

@@ -45,6 +45,7 @@ final class IndexAgentBuilder
      */
     private array $includePatterns = [
         '/**/*.php',
+        '/**/*.phar',
     ];
 
     /**
@@ -68,7 +69,7 @@ final class IndexAgentBuilder
     /**
      * @var list<string>
      */
-    private array $supportedExtensions = ['php'];
+    private array $supportedExtensions = ['php', 'phar'];
 
     private LoggerInterface $logger;
 

lib/Indexer/Model/IndexJob.php

@@ -2,8 +2,12 @@
 
 namespace Phpactor\Indexer\Model;
 
+use FilesystemIterator;
 use Generator;
+use Phar;
+use PharFileInfo;
 use Phpactor\TextDocument\TextDocumentBuilder;
+use RecursiveIteratorIterator;
 use SplFileInfo;
 
 class IndexJob
@@ -17,8 +21,17 @@ public function __construct(private IndexBuilder $indexBuilder, private FileList
      */
     public function generator(): Generator
     {
+
         foreach ($this->fileList as $fileInfo) {
             assert($fileInfo instanceof SplFileInfo);
+
+            // TODO: could refactor this to iterate the PHAR in the file list provider.
+            if ($fileInfo->getExtension() === 'phar') {
+                $phar = new Phar($fileInfo->getPathname(), FilesystemIterator::CURRENT_AS_FILEINFO | FilesystemIterator::KEY_AS_FILENAME);
+                yield from $this->indexPharFile($phar);
+                continue;
+            }
+
             $contents = @file_get_contents($fileInfo->getPathname());
 
             if (false === $contents) {
@@ -42,4 +55,25 @@ public function size(): int
     {
         return $this->fileList->count();
     }
+    /**
+     * @return Generator<string>
+     */
+    private function indexPharFile(Phar $phar): Generator
+    {
+        $iterator = new RecursiveIteratorIterator($phar);
+        /** @var PharFileInfo $file */
+        foreach ($iterator as $file) {
+            if (!$file->isFile()) {
+                continue;
+            }
+            if ($file->getExtension() !== 'php') {
+                continue;
+            }
+
+            $this->indexBuilder->index(
+                TextDocumentBuilder::fromUri($file->getPathname())->build()
+            );
+            yield $file->getPathname();
+        }
+    }
 }

lib/Indexer/Model/Query/Criteria/FileAbsolutePathBeginsWith.php

@@ -18,7 +18,14 @@ public function isSatisfiedBy(Record $record): bool
         if (!$record instanceof HasPath) {
             return false;
         }
+        $path = $record->filePath();
+        if (!$path) {
+            return false;
+        }
+        if ($pos = strpos($path, ':///')) {
+            $path = substr($path, $pos + 3);
+        }
 
-        return str_starts_with($record->filePath() ?? '', $this->prefix);
+        return str_starts_with($path, $this->prefix);
     }
 }

lib/Indexer/Model/Record/FileRecord.php

@@ -18,9 +18,9 @@ class FileRecord implements HasPath, Record
      */
     private array $references = [];
 
-    public function __construct(string $filePath)
+    private function __construct(string $filePath)
     {
-        $this->setFilePath($filePath);
+        $this->filePath = $filePath;
     }
 
     public function __wakeup(): void

lib/Indexer/Model/Record/HasPath.php

@@ -2,12 +2,17 @@
 
 namespace Phpactor\Indexer\Model\Record;
 
+use Phpactor\TextDocument\TextDocumentUri;
+
 interface HasPath
 {
     /**
      * @return $this
      */
-    public function setFilePath(string $filePath);
+    public function setFilePath(TextDocumentUri $filePath);
 
+    /**
+     * Rename to URI
+     */
     public function filePath(): ?string;
 }

lib/Indexer/Model/Record/HasPathTrait.php

@@ -2,13 +2,15 @@
 
 namespace Phpactor\Indexer\Model\Record;
 
+use Phpactor\TextDocument\TextDocumentUri;
+
 trait HasPathTrait
 {
     protected ?string $filePath = null;
 
-    public function setFilePath(string $filePath): self
+    public function setFilePath(TextDocumentUri $uri): self
     {
-        $this->filePath = $filePath;
+        $this->filePath = $uri->__toString();
         return $this;
     }