Revert to C++17 build

cpp/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.21) # TARGET_RUNTIME_DLLS
 # Make the `project` command handle the version of the project.
 cmake_policy(SET CMP0048 NEW)
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # We do not need any compilers' extensions, so we disable them.
 set(CMAKE_CXX_EXTENSIONS OFF)

cpp/arcticdb/CMakeLists.txt

@@ -347,7 +347,6 @@ set(arcticdb_srcs
         util/memory_mapped_file.hpp
         util/name_validation.hpp
         util/offset_string.hpp
-        util/offset_string.hpp
         util/optional_defaults.hpp
         util/pb_util.hpp
         util/preconditions.hpp
@@ -473,7 +472,6 @@ set(arcticdb_srcs
         util/memory_mapped_file.hpp
         util/name_validation.cpp
         util/offset_string.cpp
-        util/offset_string.cpp
         util/sparse_utils.cpp
         util/string_utils.cpp
         util/trace.cpp
@@ -488,7 +486,7 @@ set(arcticdb_srcs
         version/symbol_list.cpp
         version/version_map_batch_methods.cpp
         storage/s3/ec2_utils.cpp
-)
+        storage/lmdb/lmdb.hpp)
 
 if(${ARCTICDB_INCLUDE_ROCKSDB})
     list (APPEND arcticdb_srcs
@@ -680,6 +678,7 @@ target_include_directories(arcticdb_python PRIVATE
         ${LIBMONGOCXX_STATIC_INCLUDE_DIRS}
         ${LIBBSONCXX_STATIC_INCLUDE_DIRS}
         ${BITMAGIC_INCLUDE_DIRS}
+        ${LMDB_LIBRARIES}
         )
 
 

cpp/arcticdb/column_store/column.hpp

@@ -677,7 +677,7 @@ class Column {
     template <
             typename input_tdt,
             typename functor>
-    requires std::is_invocable_r_v<void, functor, typename input_tdt::DataTypeTag::raw_type>
+    // requires std::is_invocable_r_v<void, functor, typename input_tdt::DataTypeTag::raw_type> //TODO reinstate with C++20
     static void for_each(const Column& input_column, functor&& f) {
         auto input_data = input_column.data();
         std::for_each(input_data.cbegin<input_tdt>(), input_data.cend<input_tdt>(), std::forward<functor>(f));
@@ -686,7 +686,7 @@ class Column {
     template <
             typename input_tdt,
             typename functor>
-    requires std::is_invocable_r_v<void, functor, typename ColumnData::Enumeration<typename input_tdt::DataTypeTag::raw_type>>
+    //requires std::is_invocable_r_v<void, functor, typename ColumnData::Enumeration<typename input_tdt::DataTypeTag::raw_type>>
     static void for_each_enumerated(const Column& input_column, functor&& f) {
         auto input_data = input_column.data();
         if (input_column.is_sparse()) {
@@ -702,7 +702,7 @@ class Column {
         typename input_tdt,
         typename output_tdt,
         typename functor>
-    requires std::is_invocable_r_v<typename output_tdt::DataTypeTag::raw_type, functor, typename input_tdt::DataTypeTag::raw_type>
+    //  requires std::is_invocable_r_v<typename output_tdt::DataTypeTag::raw_type, functor, typename input_tdt::DataTypeTag::raw_type>
     static void transform(const Column& input_column, Column& output_column, functor&& f) {
         auto input_data = input_column.data();
         initialise_output_column(input_column, output_column);
@@ -720,11 +720,11 @@ class Column {
             typename right_input_tdt,
             typename output_tdt,
             typename functor>
-    requires std::is_invocable_r_v<
+    /*requires std::is_invocable_r_v<
             typename output_tdt::DataTypeTag::raw_type,
             functor,
             typename left_input_tdt::DataTypeTag::raw_type,
-            typename right_input_tdt::DataTypeTag::raw_type>
+            typename right_input_tdt::DataTypeTag::raw_type>*/
     static void transform(const Column& left_input_column,
                           const Column& right_input_column,
                           Column& output_column,
@@ -783,7 +783,8 @@ class Column {
 
     template <
             typename input_tdt,
-            std::predicate<typename input_tdt::DataTypeTag::raw_type> functor>
+            typename functor>
+    // std::predicate<typename input_tdt::DataTypeTag::raw_type> functor>
     static void transform(const Column& input_column,
                           util::BitSet& output_bitset,
                           bool sparse_missing_value_output,
@@ -806,7 +807,8 @@ class Column {
     template <
             typename left_input_tdt,
             typename right_input_tdt,
-            std::relation<typename left_input_tdt::DataTypeTag::raw_type, typename right_input_tdt::DataTypeTag::raw_type> functor>
+            // std::relation<typename left_input_tdt::DataTypeTag::raw_type, typename right_input_tdt::DataTypeTag::raw_type> functor>
+            typename functor>
     static void transform(const Column& left_input_column,
                           const Column& right_input_column,
                           util::BitSet& output_bitset,

cpp/arcticdb/column_store/column_data.hpp

@@ -258,7 +258,7 @@ struct ColumnData {
             return data_.ptr_ == other.data_.ptr_;
         }
 
-        base_type::reference dereference() const {
+        typename base_type::reference dereference() const {
             if constexpr (iterator_type == IteratorType::ENUMERATED) {
                 return data_;
             } else {
@@ -270,7 +270,7 @@ struct ColumnData {
         ColumnData* parent_{nullptr};
         std::optional<TypedBlockData<TDT>> opt_block_{std::nullopt};
         std::size_t remaining_values_in_block_{0};
-        base_type::value_type data_;
+        typename base_type::value_type data_;
     };
 
     ColumnData(

cpp/arcticdb/log/log.cpp

@@ -72,7 +72,7 @@ struct Loggers::Impl
 
 
 constexpr auto get_default_log_level() {
-    return spdlog::level::info;
+    return spdlog::level::trace;
 }
 
 spdlog::logger &storage() {

cpp/arcticdb/pipeline/index_writer.hpp

@@ -17,7 +17,7 @@
 
 namespace arcticdb::pipelines::index {
 // TODO: change the name - something like KeysSegmentWriter or KeyAggragator or  better
-template<ValidIndex Index>
+template<typename Index>
 class IndexWriter {
     // All index segments are row-count indexed in the sense that the keys are
     // already ordered - they don't need an additional index

cpp/arcticdb/pipeline/input_tensor_frame.hpp

@@ -18,18 +18,6 @@ namespace arcticdb::pipelines {
 
 using namespace arcticdb::entity;
 
-/// @TODO Move to a separate "util" header
-template <typename T, typename... U>
-concept is_any_of = (std::same_as<T, U> || ...);
-
-template <typename IndexT>
-concept ValidIndex = is_any_of<
-    std::remove_cvref_t<std::remove_pointer_t<std::decay_t<IndexT>>>,
-    stream::TimeseriesIndex,
-    stream::RowCountIndex,
-    stream::TableIndex,
-    stream::EmptyIndex>;
-
 
 struct InputTensorFrame {
     InputTensorFrame() :

cpp/arcticdb/processing/clause.hpp

@@ -351,6 +351,13 @@ inline StreamDescriptor empty_descriptor(arcticdb::proto::descriptors::IndexDesc
 }
 
 struct NamedAggregator {
+    NamedAggregator(const std::string& s, const std::string& t, const std::string& v) :
+            aggregation_operator_(s),
+            input_column_name_(t),
+            output_column_name_(v){
+
+    }
+
     std::string aggregation_operator_;
     std::string input_column_name_;
     std::string output_column_name_;

cpp/arcticdb/processing/test/benchmark_clause.cpp

@@ -89,7 +89,6 @@ static void BM_merge_ordered(benchmark::State& state){
 }
 
 template <typename integer>
-requires std::integral<integer>
 void BM_hash_grouping_int(benchmark::State& state) {
     auto num_rows = state.range(0);
     auto num_unique_values = state.range(1);

cpp/arcticdb/storage/lmdb/lmdb.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20.
+// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068
+// See: https://en.cppreference.com/w/cpp/types/is_pod
+// This suppresses the warning.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#ifdef ARCTICDB_USING_CONDA
+#include <lmdb++.h>
+#else
+#include <third_party/lmdbcxx/lmdb++.h>
+#endif
+#pragma GCC diagnostic pop

cpp/arcticdb/storage/lmdb/lmdb_client_wrapper.hpp

@@ -12,18 +12,10 @@
 #include <arcticdb/entity/variant_key.hpp>
 #include <arcticdb/storage/storage_utils.hpp>
 
-// LMDB++ is using `std::is_pod` in `lmdb++.h`, which is deprecated as of C++20.
-// See: https://github.com/drycpp/lmdbxx/blob/0b43ca87d8cfabba392dfe884eb1edb83874de02/lmdb%2B%2B.h#L1068
-// See: https://en.cppreference.com/w/cpp/types/is_pod
-// This suppresses the warning.
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#ifdef ARCTICDB_USING_CONDA
-#include <lmdb++.h>
-#else
-#include <third_party/lmdbxx/lmdb++.h>
-#endif
-#pragma GCC diagnostic pop
+namespace lmdb {
+class txn;
+class dbi;
+}
 
 
 namespace arcticdb::storage::lmdb {

cpp/arcticdb/storage/lmdb/lmdb_mock_client.cpp

@@ -11,6 +11,7 @@
 #include <arcticdb/entity/atom_key.hpp>
 #include <arcticdb/entity/serialized_key.hpp>
 #include <arcticdb/util/string_utils.hpp>
+#include <arcticdb/storage/lmdb/lmdb.hpp>
 
 
 namespace arcticdb::storage::lmdb {

cpp/arcticdb/storage/lmdb/lmdb_real_client.cpp

@@ -11,7 +11,7 @@
 #include <arcticdb/entity/variant_key.hpp>
 #include <arcticdb/storage/lmdb/lmdb_real_client.hpp>
 #include <arcticdb/storage/storage_utils.hpp>
-
+#include <arcticdb/storage/lmdb/lmdb.hpp>
 
 namespace arcticdb::storage::lmdb {
 

cpp/arcticdb/storage/lmdb/lmdb_storage.cpp

@@ -18,14 +18,13 @@
 #include <arcticdb/storage/open_mode.hpp>
 #include <arcticdb/util/format_bytes.hpp>
 #include <arcticdb/util/preconditions.hpp>
-#include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/entity/serialized_key.hpp>
 #include <arcticdb/storage/storage.hpp>
 #include <arcticdb/storage/storage_options.hpp>
-#include <arcticdb/storage/storage_utils.hpp>
 
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <folly/gen/Base.h>
+#include <arcticdb/storage/lmdb/lmdb.hpp>
 
 namespace arcticdb::storage::lmdb {
 
@@ -49,14 +48,23 @@ void raise_lmdb_exception(const ::lmdb::error& e) {
     raise<ErrorCode::E_UNEXPECTED_LMDB_ERROR>(fmt::format("Unexpected LMDB Error: LMDBError#{}: {}", error_code, e.what()));
 }
 
+::lmdb::env& LmdbStorage::env() {
+    if (!env_) {
+        raise<ErrorCode::E_UNEXPECTED_LMDB_ERROR>("Unexpected LMDB Error: Invalid operation: LMDB environment has been removed. "
+                                                  "Possibly because the library has been deleted");
+    }
+    return *env_;
+}
+
+
 void LmdbStorage::do_write_internal(Composite<KeySegmentPair>&& kvs, ::lmdb::txn& txn) {
     auto fmt_db = [](auto &&kv) { return kv.key_type(); };
 
     (fg::from(kvs.as_range()) | fg::move | fg::groupBy(fmt_db)).foreach([&](auto &&group) {
         auto db_name = fmt::format("{}", group.key());
 
         ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0)
-        ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name);
+        ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name);
 
         ARCTICDB_SUBSAMPLE(LmdbStorageWriteValues, 0)
         for (auto &kv : group.values()) {
@@ -117,7 +125,7 @@ void LmdbStorage::do_read(Composite<VariantKey>&& ks, const ReadVisitor& visitor
     (fg::from(ks.as_range()) | fg::move | fg::groupBy(fmt_db)).foreach([&](auto &&group) {
         auto db_name = fmt::format("{}", group.key());
         ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0)
-        ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name);
+        ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name);
         for (auto &k : group.values()) {
             auto stored_key = to_serialized_key(k);
             try {
@@ -155,7 +163,7 @@ bool LmdbStorage::do_key_exists(const VariantKey&key) {
     ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0)
 
     try {
-        ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name);
+        ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name);
         auto stored_key = to_serialized_key(key);
         return lmdb_client_->exists(db_name, stored_key, txn, dbi);
     } catch ([[maybe_unused]] const ::lmdb::not_found_error &ex) {
@@ -176,7 +184,7 @@ std::vector<VariantKey> LmdbStorage::do_remove_internal(Composite<VariantKey>&&
         ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0)
         try {
             // If no key of this type has been written before, this can fail
-            ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name);
+            ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name);
             for (auto &k : group.values()) {
                 auto stored_key = to_serialized_key(k);
 
@@ -230,7 +238,7 @@ bool LmdbStorage::do_fast_delete() {
         auto db_name = fmt::format("{}", key_type);
         ARCTICDB_SUBSAMPLE(LmdbStorageOpenDb, 0)
         ARCTICDB_DEBUG(log::storage(), "dropping {}", db_name);
-        ::lmdb::dbi& dbi = dbi_by_key_type_.at(db_name);
+        ::lmdb::dbi& dbi = *dbi_by_key_type_.at(db_name);
         try {
             ::lmdb::dbi_drop(dtxn, dbi);
         } catch (const ::lmdb::error& ex) {
@@ -246,7 +254,7 @@ void LmdbStorage::do_iterate_type(KeyType key_type, const IterateTypeVisitor& vi
     ARCTICDB_SAMPLE(LmdbStorageItType, 0);
     auto txn = ::lmdb::txn::begin(env(), nullptr, MDB_RDONLY); // scoped abort on
     std::string type_db = fmt::format("{}", key_type);
-    ::lmdb::dbi& dbi = dbi_by_key_type_.at(type_db);
+    ::lmdb::dbi& dbi = *dbi_by_key_type_.at(type_db);
 
     try {
         auto keys = lmdb_client_->list(type_db, prefix, txn, dbi, key_type);
@@ -332,7 +340,7 @@ LmdbStorage::LmdbStorage(const LibraryPath &library_path, OpenMode mode, const C
     }
     write_mutex_ = std::make_unique<std::mutex>();
     env_ = std::make_unique<::lmdb::env>(::lmdb::env::create(conf.flags()));
-    dbi_by_key_type_ = std::unordered_map<std::string, ::lmdb::dbi>{};
+    dbi_by_key_type_ = std::unordered_map<std::string, std::unique_ptr<::lmdb::dbi>>{};
 
     fs::path root_path = conf.path().c_str();
     auto lib_path_str = library_path.to_delim_path(fs::path::preferred_separator);
@@ -381,7 +389,7 @@ LmdbStorage::LmdbStorage(const LibraryPath &library_path, OpenMode mode, const C
         arcticdb::entity::foreach_key_type([&txn, this](KeyType &&key_type) {
             std::string db_name = fmt::format("{}", key_type);
             ::lmdb::dbi dbi = ::lmdb::dbi::open(txn, db_name.data(), MDB_CREATE);
-            dbi_by_key_type_.insert(std::make_pair(std::move(db_name), std::move(dbi)));
+            dbi_by_key_type_.insert(std::make_pair(std::move(db_name), std::make_unique<::lmdb::dbi>(std::move(dbi))));
         });
     } catch (const ::lmdb::error& ex) {
         raise_lmdb_exception(ex);

cpp/arcticdb/storage/lmdb/lmdb_storage.hpp

@@ -20,6 +20,11 @@
 
 namespace fs = std::filesystem;
 
+namespace lmdb {
+class env;
+class dbi;
+}
+
 namespace arcticdb::storage::lmdb {
 
 class LmdbStorage final : public Storage {
@@ -54,13 +59,7 @@ class LmdbStorage final : public Storage {
 
     bool do_is_path_valid(const std::string_view path) const final;
 
-    ::lmdb::env& env() {
-        if (!env_) {
-            raise<ErrorCode::E_UNEXPECTED_LMDB_ERROR>("Unexpected LMDB Error: Invalid operation: LMDB environment has been removed. "
-                                                      "Possibly because the library has been deleted");
-        }
-        return *env_;
-    }
+    ::lmdb::env& env();
 
     std::string do_key_path(const VariantKey&) const final { return {}; };
 
@@ -72,7 +71,7 @@ class LmdbStorage final : public Storage {
     std::unique_ptr<std::mutex> write_mutex_;
     std::unique_ptr<::lmdb::env> env_;
 
-    std::unordered_map<std::string, ::lmdb::dbi> dbi_by_key_type_;
+    std::unordered_map<std::string, std::unique_ptr<::lmdb::dbi>> dbi_by_key_type_;
 
     std::filesystem::path lib_dir_;