feat: update phpdocumentor to 4.0 (#4040)

* feat: update phpdocumentor to 4.0

* revert changes to AcceleratorTypes::list_

.kokoro/presubmit/run-tests.sh

@@ -45,10 +45,14 @@ echo "Running Snippet Test Suite"
 vendor/bin/phpunit -c phpunit-snippets.xml.dist --verbose --log-junit \
                    ${SNIPPETS_LOG_FILENAME}
 
-echo "Running Doc Generator"
+# Run docs gen on PHP 7.3 only
+if [ "7.3" == ${PHP_VERSION:0:3} ]; then
+    echo "Running Doc Generator"
 
-# Exclude "cloud-compute" so docs are not generated for cloud-compute (PHP 7.0 only)
-# Exclude the directory "Compute" so the google-cloud component does not generate docs for Compute
-php -d 'memory_limit=-1' dev/google-cloud doc --exclude cloud-compute --common-excludes Compute
+    # Require phpdocumentor:4 for docs generation
+    composer require --dev --with-dependencies phpdocumentor/reflection:^4.0
+
+    php -d 'memory_limit=-1' dev/google-cloud doc
+fi
 
 popd

Core/src/LongRunning/LROTrait.php

@@ -96,7 +96,7 @@ public function resumeOperation($operationName, array $info = [])
      *     @type string $pageToken A previously-returned page token used to
      *           resume the loading of results from a specific point.
      * }
-     * @return ItemIterator<InstanceConfiguration>
+     * @return ItemIterator<LongRunningOperation>
      */
     public function longRunningOperations(array $options = [])
     {

Core/src/ServiceBuilder.php

@@ -30,7 +30,7 @@
 use Google\Cloud\Speech\SpeechClient;
 use Google\Cloud\Storage\StorageClient;
 use Google\Cloud\Trace\TraceClient;
-use Google\Cloud\Translate\TranslateClient;
+use Google\Cloud\Translate\V2\TranslateClient;
 use Google\Cloud\Vision\VisionClient;
 use Psr\Cache\CacheItemPoolInterface;
 

PubSub/src/Topic.php

@@ -361,7 +361,7 @@ public function exists(array $options = [])
      * may find that Topic::exists() is a better fit for a true/false check.
      *
      * This method will use the previously cached result, if available. To force
-     * a refresh from the API, use {@see Google\Cloud\Pubsub\Topic::reload()}.
+     * a refresh from the API, use {@see Google\Cloud\PubSub\Topic::reload()}.
      *
      * Example:
      * ```
@@ -395,7 +395,7 @@ public function info(array $options = [])
      * may find that Topic::exists() is a better fit for a true/false check.
      *
      * This method will retrieve a new result from the API. To use a previously
-     * cached result, if one exists, use {@see Google\Cloud\Pubsub\Topic::info()}.
+     * cached result, if one exists, use {@see Google\Cloud\PubSub\Topic::info()}.
      *
      * Example:
      * ```

dev/src/DocGenerator/DocBlock/DescriptionFactory.php

@@ -0,0 +1,182 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * This file has been copied from PHP Documentor in order to override
+ * some of the behavior which was resulting in unwanted behavior.
+ * @see https://github.com/phpDocumentor/ReflectionDocBlock/issues/274
+ *
+ * @link      http://phpdoc.org
+ */
+
+namespace Google\Cloud\Dev\DocGenerator\DocBlock;
+
+use phpDocumentor\Reflection\DocBlock\DescriptionFactory as BaseDescriptionFactory;
+use phpDocumentor\Reflection\Types\Context as TypeContext;
+use phpDocumentor\Reflection\Utils;
+use phpDocumentor\Reflection\DocBlock\Description;
+use phpDocumentor\Reflection\DocBlock\TagFactory;
+use function count;
+use function explode;
+use function implode;
+use function ltrim;
+use function min;
+use function str_replace;
+use function strlen;
+use function strpos;
+use function substr;
+use function trim;
+use const PREG_SPLIT_DELIM_CAPTURE;
+
+/**
+ * Creates a new Description object given a body of text.
+ *
+ * Descriptions in phpDocumentor are somewhat complex entities as they can contain one or more tags inside their
+ * body that can be replaced with a readable output. The replacing is done by passing a Formatter object to the
+ * Description object's `render` method.
+ *
+ * In addition to the above does a Description support two types of escape sequences:
+ *
+ * 1. `{@}` to escape the `@` character to prevent it from being interpreted as part of a tag, i.e. `{{@}link}`
+ * 2. `{}` to escape the `}` character, this can be used if you want to use the `}` character in the description
+ *    of an inline tag.
+ *
+ * If a body consists of multiple lines then this factory will also remove any superfluous whitespace at the beginning
+ * of each line while maintaining any indentation that is used. This will prevent formatting parsers from tripping
+ * over unexpected spaces as can be observed with tag descriptions.
+ */
+class DescriptionFactory extends BaseDescriptionFactory
+{
+    /** @var TagFactory */
+    private $tagFactory;
+
+    /**
+     * Initializes this factory with the means to construct (inline) tags.
+     */
+    public function __construct(TagFactory $tagFactory)
+    {
+        $this->tagFactory = $tagFactory;
+    }
+
+    /**
+     * Returns the parsed text of this description.
+     */
+    public function create(string $contents, ?TypeContext $context = null) : Description
+    {
+        $tokens   = $this->lex($contents);
+        $count    = count($tokens);
+        $tagCount = 0;
+        $tags     = [];
+
+        for ($i = 1; $i < $count; $i += 2) {
+            $tags[]     = $this->tagFactory->create($tokens[$i], $context);
+            $tokens[$i] = '%' . ++$tagCount . '$s';
+        }
+
+        //In order to allow "literal" inline tags, the otherwise invalid
+        //sequence "{@}" is changed to "@", and "{}" is changed to "}".
+        //"%" is escaped to "%%" because of vsprintf.
+        //See unit tests for examples.
+        for ($i = 0; $i < $count; $i += 2) {
+            // @TODO: Modified the following line so that "{}" is not replaced
+            // with "}". So far we have not seen any adverse effects
+            // $tokens[$i] = str_replace(['{@}', '{}', '%'], ['@', '}', '%%'], $tokens[$i]);
+            $tokens[$i] = str_replace(['{@}', '%'], ['@', '%%'], $tokens[$i]);
+        }
+
+        return new Description(implode('', $tokens), $tags);
+    }
+
+    /**
+     * Strips the contents from superfluous whitespace and splits the description into a series of tokens.
+     *
+     * @return string[] A series of tokens of which the description text is composed.
+     */
+    private function lex(string $contents) : array
+    {
+        $contents = $this->removeSuperfluousStartingWhitespace($contents);
+
+        // performance optimalization; if there is no inline tag, don't bother splitting it up.
+        if (strpos($contents, '{@') === false) {
+            return [$contents];
+        }
+
+        return Utils::pregSplit(
+            '/\{
+                # "{@}" is not a valid inline tag. This ensures that we do not treat it as one, but treat it literally.
+                (?!@\})
+                # We want to capture the whole tag line, but without the inline tag delimiters.
+                (\@
+                    # Match everything up to the next delimiter.
+                    [^{}]*
+                    # Nested inline tag content should not be captured, or it will appear in the result separately.
+                    (?:
+                        # Match nested inline tags.
+                        (?:
+                            # Because we did not catch the tag delimiters earlier, we must be explicit with them here.
+                            # Notice that this also matches "{}", as a way to later introduce it as an escape sequence.
+                            \{(?1)?\}
+                            |
+                            # Make sure we match hanging "{".
+                            \{
+                        )
+                        # Match content after the nested inline tag.
+                        [^{}]*
+                    )* # If there are more inline tags, match them as well. We use "*" since there may not be any
+                       # nested inline tags.
+                )
+            \}/Sux',
+            $contents,
+            0,
+            PREG_SPLIT_DELIM_CAPTURE
+        );
+    }
+
+    /**
+     * Removes the superfluous from a multi-line description.
+     *
+     * When a description has more than one line then it can happen that the second and subsequent lines have an
+     * additional indentation. This is commonly in use with tags like this:
+     *
+     *     {@}since 1.1.0 This is an example
+     *         description where we have an
+     *         indentation in the second and
+     *         subsequent lines.
+     *
+     * If we do not normalize the indentation then we have superfluous whitespace on the second and subsequent
+     * lines and this may cause rendering issues when, for example, using a Markdown converter.
+     */
+    private function removeSuperfluousStartingWhitespace(string $contents) : string
+    {
+        $lines = explode("\n", $contents);
+
+        // if there is only one line then we don't have lines with superfluous whitespace and
+        // can use the contents as-is
+        if (count($lines) <= 1) {
+            return $contents;
+        }
+
+        // determine how many whitespace characters need to be stripped
+        $startingSpaceCount = 9999999;
+        for ($i = 1, $iMax = count($lines); $i < $iMax; ++$i) {
+            // lines with a no length do not count as they are not indented at all
+            if (trim($lines[$i]) === '') {
+                continue;
+            }
+
+            // determine the number of prefixing spaces by checking the difference in line length before and after
+            // an ltrim
+            $startingSpaceCount = min($startingSpaceCount, strlen($lines[$i]) - strlen(ltrim($lines[$i])));
+        }
+
+        // strip the number of spaces from each line
+        if ($startingSpaceCount > 0) {
+            for ($i = 1, $iMax = count($lines); $i < $iMax; ++$i) {
+                $lines[$i] = substr($lines[$i], $startingSpaceCount);
+            }
+        }
+
+        return implode("\n", $lines);
+    }
+}

dev/src/DocGenerator/DocGenerator.php

@@ -20,7 +20,16 @@
 use Google\Cloud\Dev\DocGenerator\Parser\CodeParser;
 use Google\Cloud\Dev\DocGenerator\Parser\MarkdownParser;
 use Symfony\Component\Console\Output\OutputInterface;
-use phpDocumentor\Reflection\FileReflector;
+use phpDocumentor\Reflection\File\LocalFile;
+use phpDocumentor\Reflection\Php\ProjectFactory;
+use phpDocumentor\Reflection\Php\Factory;
+use phpDocumentor\Reflection\Php\NodesFactory;
+use phpDocumentor\Reflection\DocBlock\DescriptionFactory;
+use phpDocumentor\Reflection\DocBlock\StandardTagFactory;
+use phpDocumentor\Reflection\FqsenResolver;
+use phpDocumentor\Reflection\DocBlockFactory;
+use phpDocumentor\Reflection\TypeResolver;
+use PhpParser\PrettyPrinter\Standard as PrettyPrinter;
 
 /**
  * Parses given files and builds documentation for our common docs site.
@@ -77,23 +86,30 @@ public function __construct(
      */
     public function generate($basePath, $pretty)
     {
-        $fileReflectorRegister = new ReflectorRegister();
+        $localFiles = [];
+        foreach ($this->files as $fileName) {
+            $localFiles[] = new LocalFile($fileName);
+        }
+        list($projectFactory, $descriptionFactory) = $this->createFactories();
+        $project = $projectFactory->create($this->componentId, $localFiles);
+        $fileRegister = new ReflectorRegister($project);
 
         $rootPath = $this->executionPath;
-        foreach ($this->files as $file) {
+        foreach ($project->getFiles() as $file) {
+            $filePath = $file->getPath();
             $currentFileArr = $this->isComponent
-                ? explode("/$basePath/", $file)
-                : explode("$rootPath", $file);
+                ? explode("/$basePath/", $filePath)
+                : explode("$rootPath", $filePath);
 
             if (isset($currentFileArr[1])) {
                 $currentFile = str_replace('src/', '', $currentFileArr[1]);
             } else {
                 throw new \Exception(
-                    sprintf('Failed to determine currentFile: %s', $file)
+                    sprintf('Failed to determine currentFile: %s', $filePath)
                 );
             }
 
-            $isPhp = strrpos($file, '.php') == strlen($file) - strlen('.php');
+            $isPhp = strrpos($filePath, '.php') == strlen($filePath) - strlen('.php');
             $pathInfo = pathinfo($currentFile);
             $servicePath = $pathInfo['dirname'] === '.'
                 ? strtolower($pathInfo['filename'])
@@ -105,7 +121,8 @@ public function generate($basePath, $pretty)
             if ($isPhp) {
                 $parser = new CodeParser(
                     $file,
-                    $fileReflectorRegister,
+                    $fileRegister,
+                    $descriptionFactory,
                     $rootPath,
                     $this->componentId,
                     $this->manifestPath,
@@ -115,7 +132,7 @@ public function generate($basePath, $pretty)
                     $this->isComponent
                 );
             } else {
-                $content = file_get_contents($file);
+                $content = file_get_contents($filePath);
                 $parser = new MarkdownParser($currentFile, $content, $id);
             }
 
@@ -132,4 +149,36 @@ public function generate($basePath, $pretty)
             }
         }
     }
+
+    private function createFactories()
+    {
+        $fqsenResolver      = new FqsenResolver();
+        $tagFactory         = new StandardTagFactory($fqsenResolver);
+
+        $descriptionFactory = new DocBlock\DescriptionFactory($tagFactory);
+
+        $tagFactory->addService($descriptionFactory, DescriptionFactory::class);
+        $tagFactory->addService(new TypeResolver($fqsenResolver));
+
+        $docBlockFactory = new DocBlockFactory($descriptionFactory, $tagFactory);
+
+        $projectFactory = new ProjectFactory(
+            [
+                new Factory\Argument(new PrettyPrinter()),
+                new Factory\Class_(),
+                new Factory\Define(new PrettyPrinter()),
+                new Factory\GlobalConstant(new PrettyPrinter()),
+                new Factory\ClassConstant(new PrettyPrinter()),
+                new Factory\DocBlock($docBlockFactory),
+                new Factory\File(NodesFactory::createInstance()),
+                new Factory\Function_(),
+                new Factory\Interface_(),
+                new Factory\Method(),
+                new Factory\Property(new PrettyPrinter()),
+                new Factory\Trait_(),
+            ]
+        );
+
+        return [$projectFactory, $descriptionFactory];
+    }
 }