Skip to content

Commit

Permalink
[dataquery] Use batch insert instead of prepared statement (#9205)
Browse files Browse the repository at this point in the history
Use a batch insert instead of a prepared statement loop to
populate temporary table in new DQT for SQLQueryEngine.

This makes the SQLQueryEngine use the same logic as
modules/dataquery/php/query.class.inc, which is much more
efficient for large data sets.
  • Loading branch information
driusan committed Apr 18, 2024
1 parent 07c9ca4 commit 9c590b5
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1780,7 +1780,6 @@ function testGetCandidateDataMemory()
}

$memory10dataAfter = memory_get_usage();
$memory10peak = memory_get_peak_usage();

$iterator10usage = $memory10dataAfter - $memory10data;

Expand All @@ -1803,11 +1802,9 @@ function testGetCandidateDataMemory()
$memoryBigDataAfter = memory_get_usage();
$iteratorBigUsage = $memoryBigDataAfter - $memoryBigDataBefore;

$memoryBigPeak = memory_get_peak_usage();
// We tested 20,000 candidates. Give 2k buffer room for variation in
// memory usage.
$this->assertTrue($iteratorBigUsage <= ($iterator10usage + (1024*2)));
$this->assertTrue($memoryBigPeak <= ($memory10peak + (1024*2)));
$this->DB->run("DROP TEMPORARY TABLE IF EXISTS candidate");
}

Expand Down
9 changes: 4 additions & 5 deletions src/Data/Query/SQLQueryEngine.php
Original file line number Diff line number Diff line change
Expand Up @@ -538,12 +538,11 @@ protected function createTemporaryCandIDTable(\Database $DB, string $tablename,
);"
);

$insertstmt = "INSERT INTO $tablename VALUES (:CandID)";

$q = $DB->prepare($insertstmt);
foreach ($candidates as $candidate) {
$q->execute(['CandID' => $candidate]);
}
$insertstmt = "INSERT INTO $tablename VALUES"
. " (" . join('),(', $candidates) . ')';
$q = $DB->prepare($insertstmt);
$q->execute([]);
}

protected $useBufferedQuery = true;
Expand Down

0 comments on commit 9c590b5

Please sign in to comment.