Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

replace chunkalloc with std::pmr, resolve big-rows issue in case of s… #96

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

replace chunkalloc with std::pmr, resolve big-rows issue in case of s… #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Open in desktop
Expand Up @@ -9,7 +9,7 @@ if(Arrow_FOUND)
add_definitions(-D_ARROW_EXIST)
endif()

set(CMAKE_CXX_FLAGS "-std=gnu++17 -ggdb -Wnon-virtual-dtor -Wreorder -Wunused-variable -Wtype-limits -Wsign-compare -Wmaybe-uninitialized")
set(CMAKE_CXX_FLAGS "-std=gnu++2a -ggdb -Wnon-virtual-dtor -Wreorder -Wunused-variable -Wtype-limits -Wsign-compare -Wmaybe-uninitialized")

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
Expand Down
28 changes: 13 additions & 15 deletions include/s3select_oper.h
View file
Open in desktop
Expand Up @@ -9,6 +9,7 @@
#include <algorithm>
#include <cstring>
#include <cmath>
#include <memory_resource>

#include <boost/lexical_cast.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
Expand Down Expand Up @@ -104,7 +105,8 @@ class base_statement;
//typedef std::vector<base_statement *> bs_stmt_vec_t; //without specific allocator

//ChunkAllocator, prevent allocation from heap.
typedef std::vector<base_statement*, ChunkAllocator<base_statement*, 256> > bs_stmt_vec_t;
//typedef std::vector<base_statement*, ChunkAllocator<base_statement*, 256> > bs_stmt_vec_t;
typedef std::pmr::vector<base_statement*> bs_stmt_vec_t;

class base_s3select_exception
{
Expand Down Expand Up @@ -416,10 +418,10 @@ class value

private:
value_t __val;
//std::string m_to_string;
std::basic_string<char,std::char_traits<char>,ChunkAllocator<char,256>> m_to_string;
//std::string m_str_value;
std::basic_string<char,std::char_traits<char>,ChunkAllocator<char,256>> m_str_value;
std::string m_to_string;
//std::basic_string<char,std::char_traits<char>,ChunkAllocator<char,256>> m_to_string;
std::string m_str_value;
//std::basic_string<char,std::char_traits<char>,ChunkAllocator<char,256>> m_str_value;

public:
enum class value_En_t
Expand Down Expand Up @@ -1003,7 +1005,7 @@ class scratch_area

scratch_area():m_upper_bound(-1),parquet_type(false),buff_loc(0)
{
m_schema_values = new std::vector<value>(128,value(""));
m_schema_values = new std::vector<value>(128);
}

~scratch_area()
Expand Down Expand Up @@ -1364,9 +1366,8 @@ class variable : public base_statement
std::string _name;
int column_pos;
value var_value;
std::string m_star_op_result;
char m_star_op_result_charc[4096]; //TODO cause larger allocations for other objects containing variable (dynamic is one solution)
value star_operation_values[16];//TODO cause larger allocations for other objects containing variable (dynamic is one solution)
std::vector<value> star_operation_values;

const int undefined_column_pos = -1;
const int column_alias = -2;
Expand Down Expand Up @@ -1516,11 +1517,7 @@ class variable : public base_statement
size_t num_of_columns = m_scratch->get_num_of_columns();
var_value.multiple_values.clear(); //TODO var_value.clear()??

if(sizeof(star_operation_values)/sizeof(value) < num_of_columns)
{
throw base_s3select_exception(std::string("not enough memory for star-operation"), base_s3select_exception::s3select_exp_en_t::FATAL);
}

star_operation_values.reserve(num_of_columns);
for(size_t i=0; i<num_of_columns; i++)
{
size_t len = m_scratch->get_column_value(i).size();
Expand All @@ -1532,8 +1529,9 @@ class variable : public base_statement
memcpy(&m_star_op_result_charc[pos], m_scratch->get_column_value(i).data(), len);//TODO using string_view will avoid copy
m_star_op_result_charc[ pos + len ] = 0;

star_operation_values[i] = &m_star_op_result_charc[pos];//set string value
var_value.multiple_values.push_value( &star_operation_values[i] );
value v1(m_star_op_result_charc+pos);
star_operation_values.push_back(v1);
var_value.multiple_values.push_value( &star_operation_values.back() );

pos += len;
pos ++;
Expand Down