Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Correctly set X-Partial-Results #2348

Open
wants to merge 1 commit into
base: integration
Choose a base branch
from
Open

Correctly set X-Partial-Results #2348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 8 additions & 11 deletions web-services/query/src/main/java/datawave/webservice/query/runner/RunningQuery.java
View file
Open in desktop
Expand Up @@ -408,14 +408,15 @@ public ResultsPage next() throws Exception {
long pageStartTime = System.currentTimeMillis();
this.logic.setPageProcessingStartTime(pageStartTime);
List<Object> resultList = new ArrayList<>();
boolean hitPageByteTrigger = false;
boolean hitPageTimeTrigger = false;
boolean hitIntermediateResult = false;
boolean hitShortCircuitForLongRunningQuery = false;

int currentPageCount = 0;
long currentPageBytes = 0;
int maxPageSize = Math.min(this.settings.getPagesize(), this.logic.getMaxPageSize());

try {
addNDC();
int currentPageCount = 0;
long currentPageBytes = 0;

// test for any exceptions prior to loop as hasNext() would likely be false;
testForUncaughtException(resultList.size());
Expand Down Expand Up @@ -447,7 +448,6 @@ public ResultsPage next() throws Exception {
// if the logic had a page byte trigger, and we have reached that, then break out
if (this.logic.getPageByteTrigger() > 0 && currentPageBytes >= this.logic.getPageByteTrigger()) {
log.info("Query logic max page byte trigger has been reached, aborting query.next call");
hitPageByteTrigger = true;
break;
}
// if the logic had a max num results (across all pages) and we have reached that (or the maxResultsOverride if set), then break out
Expand All @@ -473,11 +473,9 @@ public ResultsPage next() throws Exception {
// this page.
long pageTimeInCall = (System.currentTimeMillis() - pageStartTime);

int maxPageSize = Math.min(this.settings.getPagesize(), this.logic.getMaxPageSize());
if (timing != null && currentPageCount > 0 && timing.shouldReturnPartialResults(currentPageCount, maxPageSize, pageTimeInCall)) {
log.info("Query logic max expire before page is full, returning existing results " + currentPageCount + " " + maxPageSize + " "
+ pageTimeInCall + " " + timing);
hitPageTimeTrigger = true;
break;
}

Expand Down Expand Up @@ -557,10 +555,9 @@ public ResultsPage next() throws Exception {
if (!resultList.isEmpty()) {
log.info("Returning page of results");
// we have results!
return new ResultsPage(resultList,
((hitPageByteTrigger || hitPageTimeTrigger || hitIntermediateResult || hitShortCircuitForLongRunningQuery)
? ResultsPage.Status.PARTIAL
: ResultsPage.Status.COMPLETE));
// we also indicate whether we returned less than the requested page size in the response
return new ResultsPage(resultList, ((iter.hasNext() && numResults < this.maxResults && currentPageCount < maxPageSize) || hitIntermediateResult
|| hitShortCircuitForLongRunningQuery) ? ResultsPage.Status.PARTIAL : ResultsPage.Status.COMPLETE);
} else {
// we have no results. Let us determine whether we are done or not.

Expand Down
8 changes: 7 additions & 1 deletion ...rvices/query/src/test/java/datawave/webservice/query/runner/ExtendedRunningQueryTest.java
View file
Open in desktop
Expand Up @@ -173,6 +173,7 @@ public void testNext_HappyPathUsingDeprecatedConstructor() throws Exception {
expect(this.transformIterator.next()).andReturn(iterator.next());
expect(this.transformIterator.getTransformer()).andReturn(transformer);
}
expect(this.transformIterator.hasNext()).andReturn(iterator.hasNext());
expect(this.query.getPagesize()).andReturn(pageSize).anyTimes();
expect(this.queryLogic.getMaxPageSize()).andReturn(maxPageSize).anyTimes();
expect(this.queryLogic.getPageByteTrigger()).andReturn(pageByteTrigger).anyTimes();
Expand Down Expand Up @@ -265,6 +266,7 @@ public void testNextMaxResults_HappyPathUsingDeprecatedConstructor() throws Exce
expect(this.transformIterator.next()).andReturn(iterator.next());
count++;
}
expect(this.transformIterator.hasNext()).andReturn(iterator.hasNext());
expect(this.transformIterator.getTransformer()).andReturn(transformer).times(count);

expect(this.query.getPagesize()).andReturn(pageSize).anyTimes();
Expand Down Expand Up @@ -316,6 +318,8 @@ public void testNext_NoResultsAfterCancellationUsingDeprecatedConstructor() thro
DatawaveUser user = new DatawaveUser(SubjectIssuerDNPair.of("userDN", "issuerDN"), UserType.USER, Collections.singleton(methodAuths), null, null, 0L);
DatawavePrincipal principal = new DatawavePrincipal(Collections.singletonList(user));
long maxResults = 100L;
int pageSize = 5;
int maxPageSize = 5;

// Set expectations
expect(this.queryLogic.getCollectQueryMetrics()).andReturn(true);
Expand All @@ -342,6 +346,8 @@ public void testNext_NoResultsAfterCancellationUsingDeprecatedConstructor() thro
expect(this.queryLogic.isLongRunningQuery()).andReturn(false);
expect(this.queryLogic.getResultLimit(eq(this.query))).andReturn(maxResults);
expect(this.queryLogic.getMaxResults()).andReturn(maxResults);
expect(this.query.getPagesize()).andReturn(pageSize).anyTimes();
expect(this.queryLogic.getMaxPageSize()).andReturn(maxPageSize).anyTimes();
this.queryLogic.preInitialize(this.query, WSAuthorizationsUtil.buildAuthorizations(Collections.singleton(Collections.singleton("AUTH_1"))));
expect(this.queryLogic.getUserOperations()).andReturn(null);
this.queryLogic.setPageProcessingStartTime(anyLong());
Expand Down Expand Up @@ -475,7 +481,7 @@ public void testNextWithDnResultLimit_HappyPathUsingDeprecatedConstructor() thro
expect(this.transformIterator.next()).andReturn(iterator.next());
count++;
}

expect(this.transformIterator.hasNext()).andReturn(iterator.hasNext());
// now that the results thread is separate from the running query thread, we could continue getting stuff
expect(this.transformIterator.getTransformer()).andReturn(transformer).anyTimes();
expect(this.transformIterator.hasNext()).andReturn(iterator.hasNext()).anyTimes();
Expand Down