Skip to content

Commit

Permalink
making it so transformers know what sources matter
Browse files Browse the repository at this point in the history
  • Loading branch information
@alnutile
alnutile committed Jun 17, 2023
1 parent 6a79786 commit baaaece
Show file tree
Hide file tree
Showing 10 changed files with 37 additions and 53 deletions.
10 changes: 5 additions & 5 deletions app/Jobs/ProcessSourceTransformers.php
View file
Expand Up @@ -28,24 +28,24 @@ public function __construct(public Document $document)
*/
public function handle(): void
{
logger("Job ProcessSourceTransformers");
logger('Job ProcessSourceTransformers');

/** @var Transformer $transformerModel */
foreach($this->document->source->project->transformers as $transformerModel) {
foreach ($this->document->source->project->transformers as $transformerModel) {
$transformerType = $transformerModel->type->value;
$sourceType = $this->document->source->type->value;
$requires = config('larachain.transformers.'.$transformerType.'.requires.sources', []);
$needsToRun = in_array($sourceType, $requires);
if(!$needsToRun) {
if (! $needsToRun) {
$needsToRun = config('larachain.transformers.'.$transformerType.'.global', false);
}

$transformer = config('larachain.transformers.'.$transformerType);

if($needsToRun) {
if ($needsToRun) {
/** @var BaseTransformer $transformer */
$transformer = app()->make($transformer['class'], [
'document' => $this->document
'document' => $this->document,
]);
$transformer->handle($transformerModel);
}
Expand Down
2 changes: 0 additions & 2 deletions app/Listeners/RunTransformersListener.php
View file
Expand Up @@ -24,7 +24,5 @@ public function handle(SourceRunCompleteEvent $event): void
* @TODO
* Not sure I need this
*/


}
}
1 change: 1 addition & 0 deletions app/Models/Source.php
View file
Expand Up @@ -51,6 +51,7 @@ public function run(array $payload = []): Document
{
try {
$sourceType = $this->getSourceTypeClass();

return $sourceType->setPayload($payload)->handle();
} catch (\Exception $e) {
logger($e);
Expand Down
10 changes: 4 additions & 6 deletions app/Source/Types/WebHook.php
View file
Expand Up @@ -3,22 +3,20 @@
namespace App\Source\Types;

use App\Ingress\StatusEnum;
use App\Jobs\ProcessSourceTransformers;
use App\Models\Document;
use Ramsey\Uuid\Uuid;

class WebHook extends BaseSourceType
{
public function handle(): Document
{
$content = json_encode($this->payload);
$contentId = md5($content);
$contentId = $contentId . ".json";
$contentId = $contentId.'.json';

$document = Document::query()
->where("guid", $contentId)
->where("source_id", $this->source->id)
->firstOr(function() use ($content, $contentId) {
->where('guid', $contentId)
->where('source_id', $this->source->id)
->firstOr(function () use ($content, $contentId) {
$document = Document::create(
[
'guid' => $contentId,
Expand Down
4 changes: 2 additions & 2 deletions app/Transformer/Types/JsonTransformer.php
View file
Expand Up @@ -17,8 +17,8 @@ public function handle(Transformer $transformer): Document
$fileContents = $this->document->content;
$guid = md5($fileContents);

logger("Running JsonTransformer");
if (!DocumentChunk::query()
logger('Running JsonTransformer');
if (! DocumentChunk::query()
->where('document_id', $this->document->id)
->where('guid', $guid)
->exists()) {
Expand Down
12 changes: 6 additions & 6 deletions config/larachain.php
View file
Expand Up @@ -68,7 +68,7 @@
'web_file',
'file_upload_source',
's3_directory',
]
],
],
],
'embed_transformer' => [
Expand All @@ -77,7 +77,7 @@
'icon' => 'ArrowsRightLeftIcon',
'class' => 'App\\Transformer\\Types\\EmbedTransformer',
'active' => 1,
"global" => true,
'global' => true,
'background' => 'bg-red-700',
],
'html2text' => [
Expand All @@ -86,8 +86,8 @@
'class' => 'App\\Transformer\\Types\\Html2Text',
'requires' => [
'sources' => [
'scrape_web_page'
]
'scrape_web_page',
],
],
'active' => 1,
'icon' => 'GlobeAltIcon',
Expand All @@ -104,7 +104,7 @@
'web_file',
'file_upload_source',
's3_directory',
]
],
],
'active' => 1,
],
Expand All @@ -120,7 +120,7 @@
'web_file',
'file_upload_source',
's3_directory',
]
],
],
'active' => 1,
],
Expand Down
1 change: 0 additions & 1 deletion database/factories/TransformerFactory.php
View file
Expand Up @@ -26,7 +26,6 @@ public function definition(): array
];
}


public function embed()
{
return $this->state(function (array $attributes) {
Expand Down
10 changes: 3 additions & 7 deletions tests/Feature/Listeners/RunTransformersListenerTest.php
View file
Expand Up @@ -2,16 +2,12 @@

namespace Tests\Feature\Listeners;

use App\Events\SourceRunCompleteEvent;
use App\Listeners\RunTransformersListener;
use App\Models\Document;
use App\Models\Source;
use Tests\TestCase;

class RunTransformersListenerTest extends TestCase
{

public function test_coming_soon() {
$this->markTestSkipped("@TODO got ahead of myself");
public function test_coming_soon()
{
$this->markTestSkipped('@TODO got ahead of myself');
}
}
38 changes: 15 additions & 23 deletions tests/Feature/ProcessSourceTransformersTest.php
View file
Expand Up @@ -2,23 +2,17 @@

namespace Tests\Feature;

use App\Jobs\EmbedJob;
use App\Jobs\ProcessSourceTransformers;
use App\Models\Document;
use App\Models\Project;
use App\Models\Source;
use App\Models\Transformer;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Foundation\Testing\WithFaker;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;

class ProcessSourceTransformersTest extends TestCase
{


public function test_runs_webhook_transformers()
{
/**
Expand All @@ -28,27 +22,25 @@ public function test_runs_webhook_transformers()
$project = Project::factory()->create();

$source = Source::factory()->webHook()->create([
'project_id' => $project->id
'project_id' => $project->id,
]);

Transformer::factory()->json()->create([
'project_id' => $project->id
'project_id' => $project->id,
]);

$this->assertDatabaseCount("document_chunks", 0);

$this->assertDatabaseCount('document_chunks', 0);

$document = Document::factory()->create([
'guid' => "foo.json",
'guid' => 'foo.json',
'source_id' => $source->id,
'content' => json_encode(['foo' => "bar"])
'content' => json_encode(['foo' => 'bar']),
]);
$job = new ProcessSourceTransformers($document);
$job->handle();
$this->assertDatabaseCount("document_chunks", 1);
$this->assertDatabaseCount('document_chunks', 1);
}


public function test_does_not_run_transformer()
{
/**
Expand All @@ -58,21 +50,21 @@ public function test_does_not_run_transformer()
$project = Project::factory()->create();

$source = Source::factory()->fileUpload()->create([
'project_id' => $project->id
'project_id' => $project->id,
]);

Transformer::factory()->json()->create([
'project_id' => $project->id
]);
'project_id' => $project->id,
]);

$document = Document::factory()->create([
'source_id' => $source->id,
'content' => json_encode(['foo' => "bar"])
'content' => json_encode(['foo' => 'bar']),
]);
$this->assertDatabaseCount("document_chunks", 0);
$this->assertDatabaseCount('document_chunks', 0);
$job = new ProcessSourceTransformers($document);
$job->handle();
$this->assertDatabaseCount("document_chunks", 0);
$this->assertDatabaseCount('document_chunks', 0);
}

public function test_embed()
Expand All @@ -85,16 +77,16 @@ public function test_embed()
$project = Project::factory()->create();

$source = Source::factory()->fileUpload()->create([
'project_id' => $project->id
'project_id' => $project->id,
]);

Transformer::factory()->embed()->create([
'project_id' => $project->id
'project_id' => $project->id,
]);

$document = Document::factory()->create([
'source_id' => $source->id,
'content' => json_encode(['foo' => "bar"])
'content' => json_encode(['foo' => 'bar']),
]);

$job = new ProcessSourceTransformers($document);
Expand Down
2 changes: 1 addition & 1 deletion tests/Feature/WebHookTest.php
View file
Expand Up @@ -29,7 +29,7 @@ public function test_makes_document()

$document = $webFileSourceType->handle();
$this->assertDatabaseCount('documents', 1);
$this->assertStringContainsString(".json", $document->guid);
$this->assertStringContainsString('.json', $document->guid);

Queue::assertPushed(ProcessSourceTransformers::class);

Expand Down

0 comments on commit baaaece

Please sign in to comment.