Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Replace CatchUpHooks with ContentRepositoryHooks #4995

Draft
wants to merge 13 commits into
base: feature/4746-rework-catchup-mechanism
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 3 additions & 5 deletions .github/workflows/build.yml
Expand Up @@ -180,11 +180,9 @@ jobs:
# We enable the race condition tracker
presets:
'default':
projections:
'Neos.ContentRepository:ContentGraph':
catchUpHooks:
'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerCatchUpHookFactory
hooks:
'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerContentRepositoryHookFactory
ContentRepository:
BehavioralTests:
raceConditionTracker:
Expand Down
Expand Up @@ -17,7 +17,7 @@
use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry;
use Neos\Flow\Cli\CommandController;
use Neos\Neos\Fusion\Cache\GraphProjectorCatchUpHookForCacheFlushing;
use Neos\Neos\Fusion\Cache\ContentRepositoryHookForCacheFlushing;

final class PerformanceMeasurementCommandController extends CommandController
{
Expand Down Expand Up @@ -46,7 +46,7 @@ public function preparePerformanceTestCommand(int $nodesPerLevel, int $levels):
{
$this->performanceMeasurementService->removeEverything();
$this->outputLine("All removed. Starting to fill.");
GraphProjectorCatchUpHookForCacheFlushing::disabled(
ContentRepositoryHookForCacheFlushing::disabled(
fn() => $this->performanceMeasurementService->createNodesForPerformanceTest($nodesPerLevel, $levels)
);
}
Expand Down
Expand Up @@ -32,6 +32,7 @@
use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated;
use Neos\ContentRepository\Core\Feature\WorkspaceCreation\Command\CreateRootWorkspace;
use Neos\ContentRepository\Core\NodeType\NodeTypeName;
use Neos\ContentRepository\Core\Projection\Projections;
use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification;
use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId;
Expand All @@ -52,7 +53,8 @@ public function __construct(
private readonly EventPersister $eventPersister,
private readonly ContentRepository $contentRepository,
private readonly Connection $dbal,
private readonly ContentRepositoryId $contentRepositoryId
private readonly ContentRepositoryId $contentRepositoryId,
private readonly Projections $projections,
) {
$this->contentStreamId = contentStreamId::fromString('cs-identifier');
$this->dimensionSpacePoints = new DimensionSpacePointSet([
Expand All @@ -72,7 +74,7 @@ public function removeEverything(): void
{
$eventTableName = DoctrineEventStoreFactory::databaseTableName($this->contentRepositoryId);
$this->dbal->executeStatement('TRUNCATE ' . $this->dbal->quoteIdentifier($eventTableName));
$this->contentRepository->resetProjectionStates();
$this->projections->resetAll();
}

public function createNodesForPerformanceTest(int $nodesPerLevel, int $levels): void
Expand Down
Expand Up @@ -36,7 +36,8 @@ public function build(
$serviceFactoryDependencies->eventPersister,
$serviceFactoryDependencies->contentRepository,
$this->dbal,
$serviceFactoryDependencies->contentRepositoryId
$serviceFactoryDependencies->contentRepositoryId,
$serviceFactoryDependencies->projections,
);
}
}
Expand Up @@ -23,7 +23,7 @@
use Symfony\Component\Console\Helper\Table;

/**
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
*/
Expand Down
Expand Up @@ -19,7 +19,7 @@
/**
* Value object for a list of {@see TraceEntry} objects, as stored in-order in the Redis stream
*
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
* @implements \ArrayAccess<int,TraceEntry>
Expand Down
Expand Up @@ -16,14 +16,14 @@

namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto;

use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerCatchUpHook;
use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerContentRepositoryHook;
use Symfony\Component\Console\Helper\TableCell;
use Symfony\Component\Console\Helper\TableCellStyle;

/**
* Value object for a single trace entry, as stored in Redis.
*
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
*/
Expand Down
Expand Up @@ -17,7 +17,7 @@
namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto;

/**
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
*/
Expand Down
Expand Up @@ -17,7 +17,7 @@
use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntries;
use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntryType;
use Neos\ContentRepository\Core\EventStore\EventInterface;
use Neos\ContentRepository\Core\Projection\CatchUpHookInterface;
use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryHookInterface;
use Neos\EventStore\Model\EventEnvelope;
use Neos\Flow\Annotations as Flow;

Expand Down Expand Up @@ -68,7 +68,7 @@
*
* ## Implementation Idea: Race Detector with Redis
*
* We implement a custom CatchUpHook (this class {@see RaceTrackerCatchUpHook}) which is notified during
* We implement a custom ContentRepositoryHook (this class {@see RaceTrackerContentRepositoryHook}) which is notified during
* the projection run.
*
* When {@see onBeforeEvent} is called, we know that we are inside applyEvent() in the diagram above,
Expand Down Expand Up @@ -98,7 +98,7 @@
*
* @internal
*/
final class RaceTrackerCatchUpHook implements CatchUpHookInterface
final class RaceTrackerContentRepositoryHook implements ContentRepositoryHookInterface
{
/**
* @Flow\InjectConfiguration("raceConditionTracker")
Expand All @@ -107,12 +107,12 @@ final class RaceTrackerCatchUpHook implements CatchUpHookInterface
protected $configuration;
private bool $inCriticalSection = false;

public function onBeforeCatchUp(): void
public function onBeforeEvents(): void
{
RedisInterleavingLogger::connect($this->configuration['redis']['host'], $this->configuration['redis']['port']);
}

public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void
public function onBeforeEvent(EventInterface $event, EventEnvelope $eventEnvelope): void
{
$this->inCriticalSection = true;
RedisInterleavingLogger::trace(TraceEntryType::InCriticalSection, [
Expand All @@ -122,20 +122,16 @@ public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $even
]);
}

public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void
public function onAfterEvent(EventInterface $event, EventEnvelope $eventEnvelope): void
{
}

public function onBeforeBatchCompleted(): void
public function onAfterEvents(): void
{
// we only want to track relevant lock release calls (i.e. if we were in the event processing loop before)
if ($this->inCriticalSection) {
$this->inCriticalSection = false;
RedisInterleavingLogger::trace(TraceEntryType::LockWillBeReleasedIfItWasAcquiredBefore);
}
}

public function onAfterCatchUp(): void
{
}
}
Expand Up @@ -15,18 +15,17 @@
namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester;

use Neos\ContentRepository\Core\ContentRepository;
use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface;
use Neos\ContentRepository\Core\Projection\CatchUpHookInterface;
use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryHookFactoryInterface;

/**
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
*/
final class RaceTrackerCatchUpHookFactory implements CatchUpHookFactoryInterface
final class RaceTrackerContentRepositoryHookFactory implements ContentRepositoryHookFactoryInterface
{
public function build(ContentRepository $contentRepository): CatchUpHookInterface
public function build(ContentRepository $contentRepository, array $options): RaceTrackerContentRepositoryHook
{
return new RaceTrackerCatchUpHook();
return new RaceTrackerContentRepositoryHook();
}
}
Expand Up @@ -21,7 +21,7 @@
use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntryType;

/**
* For full docs and context, see {@see RaceTrackerCatchUpHook}
* For full docs and context, see {@see RaceTrackerContentRepositoryHook}
*
* @internal
*/
Expand Down
Expand Up @@ -18,6 +18,8 @@
use Behat\Gherkin\Node\TableNode;
use Doctrine\DBAL\Connection;
use Neos\ContentRepository\Core\ContentRepository;
use Neos\ContentRepository\Core\Service\ProjectionService;
use Neos\ContentRepository\Core\Service\ProjectionServiceFactory;
use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\GherkinTableNodeBasedContentDimensionSource;
use Neos\EventStore\EventStoreInterface;
Expand Down Expand Up @@ -163,7 +165,7 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository
*
* This was an actual bug which bit us and made our tests unstable :D :D
*
* How did we find this? By the virtue of our Race Tracker (Docs: see {@see RaceTrackerCatchUpHook}), which
* How did we find this? By the virtue of our Race Tracker (Docs: see {@see RaceTrackerContentRepositoryHook}), which
* checks for events being applied multiple times to a projection.
* ... and additionally by using {@see logToRaceConditionTracker()} to find the interleavings between the
* Catch Up process and the testcase reset.
Expand All @@ -179,7 +181,9 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository
$databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore);
$eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value);
$databaseConnection->executeStatement('TRUNCATE ' . $eventTableName);
$contentRepository->resetProjectionStates();
/** @var ProjectionService $projectionService */
$projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->getObject(ProjectionServiceFactory::class));
$projectionService->resetAllProjections();

return $contentRepository;
}
Expand Down
Expand Up @@ -5,11 +5,9 @@ Neos:
#ContentRepositoryRegistry:
# presets:
# 'default':
# projections:
# 'Neos.ContentRepository:ContentGraph':
# catchUpHooks:
# 'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
# factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerCatchUpHookFactory
# hooks:
# 'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
# factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerContentRepositoryHookFactory

ContentRepository:
BehavioralTests:
Expand Down
Expand Up @@ -25,14 +25,12 @@ Neos:
password: 'db'
charset: 'UTF8'

# We enable the race condition tracker. For details on how this works, see RaceTrackerCatchUpHook.php
# We enable the race condition tracker. For details on how this works, see RaceTrackerContentRepositoryHook.php
presets:
'default':
projections:
'Neos.ContentRepository:ContentGraph':
catchUpHooks:
'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerCatchUpHookFactory
hooks:
'Neos.ContentRepository.BehavioralTests:RaceConditionTracker':
factoryObjectName: Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\RaceTrackerContentRepositoryHookFactory
ContentRepository:
BehavioralTests:
raceConditionTracker:
Expand Down
Expand Up @@ -13,7 +13,7 @@
# FLOW_CONTEXT=Testing/Behat ../../../../../flow raceConditionTracker:analyzeTrace
# docker compose --project-directory . --file Packages/Neos/Neos.ContentRepository.BehavioralTests/docker-compose-full.yml down -v
#
# For details on how the race condition tracker works, see RaceTrackerCatchUpHook.php
# For details on how the race condition tracker works, see RaceTrackerContentRepositoryHook.php
#
# Note: the context of this dockerfile is the ROOT directory of the Flow Distribution (FLOW_PATH_ROOT)
version: '3.5'
Expand Down Expand Up @@ -66,7 +66,7 @@ services:
- 15432:5432

# for running race-condition tests, we use Redis as a single-threaded time store.
# For details on how this works, see RaceTrackerCatchUpHook.php
# For details on how this works, see RaceTrackerContentRepositoryHook.php
redis:
image: redis:7.0.4
ports:
Expand Down