NEXT-11081 - Improve performance of RepositoryIterator and remove a possible next request

changelog/_unreleased/2020-09-07-repository-iterator-prevent-predictable-last-empty-request.md

@@ -0,0 +1,11 @@
+---
+title: Prevent predictable empty request in RepositoryIterator and improve developer experience
+issue: NEXT-11081
+author: Joshua Behrens
+author_email: code@joshua-behrens.de
+author_github: @JoshuaBehrens
+---
+# Core
+* Added `\Shopware\Core\Framework\DataAbstractionLayer\Dbal\Common\RepositoryIterator::reset` to either remove `increment` filter if the iterated entity has an auto increment column or otherwise removes the offset. This helps to implement iterator resets without knowing how the iteration is implemented
+* Added internal state into `\Shopware\Core\Framework\DataAbstractionLayer\Dbal\Common\RepositoryIterator`, whether the previous `fetch` or `fetchIds` result had less items than expected and therefore the next search will be empty anyways 
+* Added methods `iterateIds` and `iterateEntities` to class `\Shopware\Core\Framework\DataAbstractionLayer\Dbal\Common\RepositoryIterator` to automatically perform iteration on respectively `fetchIds` and `fetch`

src/Core/Checkout/Cart/RuleLoader.php

@@ -50,9 +50,6 @@ public function load(Context $context): RuleCollection
                     $rules->add($rule);
                 }
             }
-            if ($result->count() < 500) {
-                break;
-            }
         }
 
         return $rules;

src/Core/Framework/DataAbstractionLayer/Dbal/Common/RepositoryIterator.php

@@ -3,8 +3,10 @@
 namespace Shopware\Core\Framework\DataAbstractionLayer\Dbal\Common;
 
 use Shopware\Core\Framework\Context;
+use Shopware\Core\Framework\DataAbstractionLayer\Entity;
 use Shopware\Core\Framework\DataAbstractionLayer\EntityCollection;
 use Shopware\Core\Framework\DataAbstractionLayer\EntityRepository;
+use Shopware\Core\Framework\DataAbstractionLayer\PartialEntity;
 use Shopware\Core\Framework\DataAbstractionLayer\Search\Criteria;
 use Shopware\Core\Framework\DataAbstractionLayer\Search\EntitySearchResult;
 use Shopware\Core\Framework\DataAbstractionLayer\Search\Filter\RangeFilter;
@@ -28,6 +30,8 @@ class RepositoryIterator
 
     private bool $autoIncrement = false;
 
+    private bool $endReached = false;
+
     /**
      * @param EntityRepository<TEntityCollection> $repository
      */
@@ -56,6 +60,23 @@ public function __construct(
         $this->context = clone $context;
     }
 
+    public function reset(): void
+    {
+        if ($this->autoIncrement) {
+            $filters = $this->criteria->getFilters();
+            $this->criteria->resetFilters();
+            unset($filters['increment']);
+
+            foreach ($filters as $filterKey => $filter) {
+                $this->criteria->setFilter($filterKey, $filter);
+            }
+        } else {
+            $this->criteria->setOffset(0);
+        }
+
+        $this->endReached = false;
+    }
+
     public function getTotal(): int
     {
         $criteria = clone $this->criteria;
@@ -71,19 +92,29 @@ public function getTotal(): int
      */
     public function fetchIds(): ?array
     {
+        if ($this->endReached) {
+            return null;
+        }
+
         $this->criteria->setTotalCountMode(Criteria::TOTAL_COUNT_MODE_NONE);
 
         $ids = $this->repository->searchIds($this->criteria, $this->context);
 
         $values = $ids->getIds();
 
         if (empty($values)) {
+            $this->endReached = true;
+
             return null;
         }
 
         if (!$this->autoIncrement) {
             $this->criteria->setOffset($this->criteria->getOffset() + $this->criteria->getLimit());
 
+            if (count($values) < $this->criteria->getLimit()) {
+                $this->endReached = true;
+            }
+
             return $values;
         }
 
@@ -95,6 +126,10 @@ public function fetchIds(): ?array
         $increment = $ids->getDataFieldOfId($last, 'autoIncrement') ?? 0;
         $this->criteria->setFilter('increment', new RangeFilter('autoIncrement', [RangeFilter::GT => $increment]));
 
+        if (count($values) < $this->criteria->getLimit()) {
+            $this->endReached = true;
+        }
+
         return $values;
     }
 
@@ -103,6 +138,10 @@ public function fetchIds(): ?array
      */
     public function fetch(): ?EntitySearchResult
     {
+        if ($this->endReached) {
+            return null;
+        }
+
         $this->criteria->setTotalCountMode(Criteria::TOTAL_COUNT_MODE_NONE);
 
         $result = $this->repository->search(clone $this->criteria, $this->context);
@@ -111,9 +150,47 @@ public function fetch(): ?EntitySearchResult
         $this->criteria->setOffset($this->criteria->getOffset() + $this->criteria->getLimit());
 
         if (empty($result->getIds())) {
+            $this->endReached = true;
+
             return null;
         }
 
+        if ($result->count() < $this->criteria->getLimit()) {
+            $this->endReached = true;
+        }
+
         return $result;
     }
+
+    /**
+     * @return iterable<Entity|PartialEntity>
+     */
+    public function iterateEntities(): iterable
+    {
+        try {
+            while (($entityResult = $this->fetch()) instanceof EntitySearchResult) {
+                // yield from is okay, as getElements keys by unique key
+                yield from $entityResult->getElements();
+            }
+        } finally {
+            $this->reset();
+        }
+    }
+
+    /**
+     * @return iterable<string>|iterable<array<string, string>>
+     */
+    public function iterateIds(): iterable
+    {
+        try {
+            while (\is_array($ids = $this->fetchIds())) {
+                // do not use yield from to ensure re-keying
+                foreach ($ids as $id) {
+                    yield $id;
+                }
+            }
+        } finally {
+            $this->reset();
+        }
+    }
 }