Skip to content

Commit

Permalink
rgw/s3select: limit operator bug fix
Browse files Browse the repository at this point in the history
fix for json parser processing more chunks than specified in the limit

Signed-off-by: Albin Antony <albinantony20@gmail.com>
  • Loading branch information
albin-antony committed Feb 20, 2024
1 parent 1ae458f commit 71e4031
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 1 deletion.
6 changes: 5 additions & 1 deletion example/s3select_example.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -434,11 +434,15 @@ int process_json_query(const char* input_query,const char* fname)
}
if(m_s3_json_object.is_sql_limit_reached())
{
std::cout << "json processing reached limit " << std::endl;
break;
}
read_sz = input_file_stream.read(buff.data(),BUFFER_SIZE).gcount();
}
if(m_s3_json_object.is_sql_limit_reached())
{
std::cout << "json processing reached limit " << std::endl;
return 0;
}
try{
result.clear();
m_s3_json_object.run_s3select_on_stream(result, 0, 0, object_sz, json.output_json_format);
Expand Down
5 changes: 5 additions & 0 deletions include/s3select.h
Original file line number Diff line number Diff line change
Expand Up @@ -3193,6 +3193,7 @@ class json_object : public base_s3object
if(is_sql_limit_reached())
{
status = JSON_PROCESSING_LIMIT_REACHED;//returning number since sql_execution_on_row_cb is a callback; the caller can not access the object
m_sql_processing_status = s3selectEngine::base_s3object::Status::LIMIT_REACHED;
}

m_sa->clear_data();
Expand All @@ -3201,6 +3202,10 @@ class json_object : public base_s3object
std::string end_of_row;
end_of_row = "#=== " + std::to_string(m_row_count++) + " ===#\n";
m_s3select_result->append(end_of_row);

if(m_is_limit_on && m_row_count == m_limit) {
m_sql_processing_status = s3selectEngine::base_s3object::Status::LIMIT_REACHED;
}
}
return status;
}
Expand Down

0 comments on commit 71e4031

Please sign in to comment.