New Concurrency Model

.gitignore

@@ -23,7 +23,10 @@ dask-worker-space/
 ###################
 *.com
 *.class
+*.exp
+*.lib
 *.dll
+*.pdb
 *.exe
 *.o
 *.so
@@ -218,3 +221,6 @@ benchmarks/torch_data
 # API docs
 api_docs/
 
+/cpp/cmake-build-debug/
+/cpp/external/
+_deeplake/**/*.pyi

cpp/CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_minimum_required(VERSION 3.16)
+project(deeplake)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24:
+if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
+	cmake_policy(SET CMP0135 NEW)
+endif()
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+option(PYTHON_EXECUTABLE "Path to python executable")
+
+if(APPLE)
+    set (CMAKE_OSX_DEPLOYMENT_TARGET 10.15)
+else()
+    #skip multi architecture build for linux
+    set (CMAKE_OSX_ARCHITECTURES)
+endif()
+
+set(DEFAULT_PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(PYTHON_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/../)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindStduuid.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindJson.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindPybind11.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindGoogleTest.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindSpdlog.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindBackward.cmake)
+
+add_subdirectory(storage)
+add_subdirectory(deeplog)
+add_subdirectory(tests)
+add_subdirectory(py_api)
+

cpp/cmake/FindBackward.cmake

@@ -0,0 +1,13 @@
+include(FetchContent)
+
+set(backward_URL https://github.com/bombela/backward-cpp/archive/refs/tags/v1.6.tar.gz)
+set(backward_URL_HASH c654d0923d43f1cea23d086729673498e4741fb2457e806cfaeaea7b20c97c10)
+set(backward_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/backward)
+
+FetchContent_Declare(
+    backward
+    URL ${backward_URL}
+    URL_HASH SHA256=${backward_URL_HASH}
+    SOURCE_DIR ${backward_SOURCE_DIR}
+)
+FetchContent_MakeAvailable(backward)

cpp/cmake/FindGoogleTest.cmake

@@ -0,0 +1,15 @@
+include(FetchContent)
+
+set(googletest_URL https://github.com/google/googletest.git)
+set(googletest_TAG v1.12.0)
+set(googletest_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/googletest)
+set(googletest_INCLUDE_DIRS ${SOURCE_DIR}/googletest/include ${SOURCE_DIR}/googlemock/include)
+
+FetchContent_Declare(googletest
+    GIT_REPOSITORY ${googletest_URL}
+    GIT_TAG ${googletest_TAG}
+    SOURCE_DIR ${googletest_SOURCE_DIR}
+)
+
+FetchContent_MakeAvailable(googletest)
+include_directories(${googletest_INCLUDE_DIR})

cpp/cmake/FindJson.cmake

@@ -0,0 +1,13 @@
+include(FetchContent)
+
+set(json_URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz)
+set(json_URL_HASH 8c4b26bf4b422252e13f332bc5e388ec0ab5c3443d24399acb675e68278d341f)
+set(json_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/json)
+
+FetchContent_Declare(
+    json
+    URL ${json_URL}
+    URL_HASH SHA256=${json_URL_HASH}
+    SOURCE_DIR ${json_SOURCE_DIR}
+)
+FetchContent_MakeAvailable(json)

cpp/cmake/FindPybind11.cmake

@@ -0,0 +1,13 @@
+include(FetchContent)
+
+set(pybind11_URL https://github.com/pybind/pybind11.git)
+set(pybind11_TAG v2.11.1)
+set(pybind11_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/pybind11)
+
+FetchContent_Declare(pybind11
+    GIT_REPOSITORY ${pybind11_URL}
+    GIT_TAG ${pybind11_TAG}
+    SOURCE_DIR ${pybind11_SOURCE_DIR}
+)
+
+FetchContent_MakeAvailable(pybind11)

cpp/cmake/FindSpdlog.cmake

@@ -0,0 +1,13 @@
+include(FetchContent)
+
+set(spdlog_URL https://github.com/gabime/spdlog/archive/refs/tags/v1.12.0.tar.gz)
+set(spdlog_URL_HASH 4dccf2d10f410c1e2feaff89966bfc49a1abb29ef6f08246335b110e001e09a9)
+set(spdlog_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/spdlog)
+
+FetchContent_Declare(
+    spdlog
+    URL ${spdlog_URL}
+    URL_HASH SHA256=${spdlog_URL_HASH}
+    SOURCE_DIR ${spdlog_SOURCE_DIR}
+)
+FetchContent_MakeAvailable(spdlog)

cpp/cmake/FindStduuid.cmake

@@ -0,0 +1,13 @@
+include(FetchContent)
+
+set(stduuid_URL https://github.com/mariusbancila/stduuid/archive/refs/tags/v1.2.3.zip)
+set(stduuid_URL_HASH 0f867768ce55f2d8fa361be82f87f0ea5e51438bc47ca30cd92c9fd8b014e84e)
+set(stduuid_SOURCE_DIR ${DEFAULT_PARENT_DIR}/external/stduuid)
+
+FetchContent_Declare(
+    stduuid
+    URL ${stduuid_URL}
+    URL_HASH SHA256=${stduuid_URL_HASH}
+    SOURCE_DIR ${stduuid_SOURCE_DIR}
+)
+FetchContent_MakeAvailable(stduuid)

cpp/deeplog/CMakeLists.txt

@@ -0,0 +1,18 @@
+project(deeplog)
+
+include(FetchContent)
+
+find_package(Arrow REQUIRED)
+find_package(Parquet REQUIRED)
+find_package(ArrowDataset REQUIRED)
+
+file(GLOB_RECURSE SOURCES "*.cpp")
+
+add_library(deeplog ${SOURCES})
+
+target_link_libraries(deeplog PUBLIC storage)
+target_link_libraries(deeplog PUBLIC stduuid nlohmann_json::nlohmann_json)
+target_link_libraries(deeplog PUBLIC "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Arrow::arrow_static,Arrow::arrow_shared>")
+target_link_libraries(deeplog PUBLIC "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Parquet::parquet_static,Parquet::parquet_shared>")
+target_link_libraries(deeplog PUBLIC "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowDataset::arrow_dataset_static,ArrowDataset::arrow_dataset_shared>")
+target_link_libraries(deeplog PUBLIC spdlog::spdlog)

cpp/deeplog/actions/action.cpp

@@ -0,0 +1,6 @@
+#include "action.hpp"
+
+namespace deeplog {
+
+
+};

cpp/deeplog/actions/action.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <arrow/api.h>
+#include "deeplog_serializable.hpp"
+
+namespace deeplog {
+
+    class action : public deeplog_serializable {
+
+    public:
+        virtual nlohmann::json to_json() = 0;
+
+        virtual std::string action_name() = 0;
+
+        virtual std::shared_ptr<arrow::StructType> action_type() = 0;
+    };
+}

cpp/deeplog/actions/add_file_action.cpp

@@ -0,0 +1,46 @@
+#include "add_file_action.hpp"
+
+namespace deeplog {
+
+    std::shared_ptr<arrow::StructType> add_file_action::arrow_type = std::dynamic_pointer_cast<arrow::StructType>(
+            arrow::struct_({
+                                   arrow::field("path", arrow::utf8(), true),
+                                   arrow::field("type", arrow::utf8(), true),
+                                   arrow::field("size", arrow::uint64(), true),
+                                   arrow::field("modificationTime", arrow::uint64(), true),
+                                   arrow::field("dataChange", arrow::boolean(), true),
+                                   arrow::field("numSamples", arrow::uint64(), true),
+                           }));
+
+    add_file_action::add_file_action(std::string path, std::string type, const long &size, const long &modification_time, const bool &data_change, const long &num_samples) :
+            path(std::move(path)), type(std::move(type)), size(size), modification_time(modification_time), data_change(data_change), num_samples(num_samples) {}
+
+    add_file_action::add_file_action(const std::shared_ptr<arrow::StructScalar> &value) {
+        path = from_struct<std::string>("path", value).value();
+        type = from_struct<std::string>("type", value).value();
+        size = from_struct<unsigned long>("size", value).value();
+        modification_time = from_struct<long>("modificationTime", value).value();
+        data_change = from_struct<bool>("dataChange", value).value();
+        num_samples = from_struct<unsigned long>("numSamples", value).value();
+    }
+
+    std::string add_file_action::action_name() {
+        return "add";
+    }
+
+    std::shared_ptr<arrow::StructType> add_file_action::action_type() {
+        return arrow_type;
+    }
+
+    nlohmann::json add_file_action::to_json() {
+        nlohmann::json json;
+        json["path"] = path;
+        json["type"] = type;
+        json["size"] = size;
+        json["modificationTime"] = modification_time;
+        json["dataChange"] = data_change;
+        json["numSamples"] = num_samples;
+
+        return json;
+    }
+}

cpp/deeplog/actions/add_file_action.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "action.hpp"
+
+namespace deeplog {
+    class add_file_action : public action {
+
+    public:
+        std::string path;
+        std::string type;
+        unsigned long size;
+        long modification_time;
+        bool data_change;
+        unsigned long num_samples;
+
+    public:
+        static std::shared_ptr<arrow::StructType> arrow_type;
+
+        add_file_action(std::string path, std::string type, const long &size, const long &modification_time, const bool &data_change, const long &num_samples);
+
+        explicit add_file_action(const std::shared_ptr<arrow::StructScalar> &struct_scalar);
+
+        nlohmann::json to_json() override;
+
+        std::string action_name() override;
+
+        std::shared_ptr<arrow::StructType> action_type() override;
+    };
+}

cpp/deeplog/actions/create_branch_action.cpp

@@ -0,0 +1,42 @@
+#include "create_branch_action.hpp"
+
+#include <utility>
+
+namespace deeplog {
+    std::shared_ptr<arrow::StructType> create_branch_action::arrow_type = std::dynamic_pointer_cast<arrow::StructType>(
+            arrow::struct_({
+                                   arrow::field("id", arrow::utf8(), true),
+                                   arrow::field("name", arrow::utf8(), true),
+                                   arrow::field("fromId", arrow::utf8(), true),
+                                   arrow::field("fromVersion", arrow::uint64(), true),
+                           }));
+
+
+    create_branch_action::create_branch_action(std::string id, std::string name, std::optional<std::string> from_id, const std::optional<unsigned long> &from_version) :
+            id(std::move(id)), name(std::move(name)), from_id(std::move(from_id)), from_version(from_version) {}
+
+    create_branch_action::create_branch_action(const std::shared_ptr<arrow::StructScalar> &value) {
+        id = from_struct<std::string>("id", value).value();
+        name = from_struct<std::string>("name", value).value();
+        from_id = from_struct<std::string>("fromId", value);
+        from_version = from_struct<long>("fromVersion", value);
+    }
+
+    std::string create_branch_action::action_name() {
+        return "branch";
+    }
+
+    std::shared_ptr<arrow::StructType> create_branch_action::action_type() {
+        return arrow_type;
+    }
+
+    nlohmann::json create_branch_action::to_json() {
+        nlohmann::json json;
+        json["id"] = id;
+        json["name"] = name;
+        json["fromId"] = to_json_value(from_id);
+        json["fromVersion"] = to_json_value(from_version);
+
+        return json;
+    }
+}

cpp/deeplog/actions/create_branch_action.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "action.hpp"
+
+namespace deeplog {
+    class create_branch_action : public action {
+
+    public:
+        std::string id;
+        std::string name;
+        std::optional<std::string> from_id;
+        std::optional<unsigned long> from_version;
+
+    public:
+        static std::shared_ptr<arrow::StructType> arrow_type;
+
+        create_branch_action(std::string id, std::string name, std::optional<std::string> from_id, const std::optional<unsigned long> &from_version);
+
+        explicit create_branch_action(const std::shared_ptr<arrow::StructScalar> &struct_scalar);
+
+        nlohmann::json to_json() override;
+
+        std::string action_name() override;
+
+        std::shared_ptr<arrow::StructType> action_type() override;
+    };
+}

cpp/deeplog/actions/create_commit_action.cpp

@@ -0,0 +1,44 @@
+#include "create_commit_action.hpp"
+
+namespace deeplog {
+
+    std::shared_ptr<arrow::StructType> create_commit_action::arrow_type = std::dynamic_pointer_cast<arrow::StructType>(
+            arrow::struct_({
+                                   arrow::field("id", arrow::utf8(), true),
+                                   arrow::field("branchId", arrow::utf8(), true),
+                                   arrow::field("branchVersion", arrow::uint64(), true),
+                                   arrow::field("message", arrow::utf8(), true),
+                                   arrow::field("commitTime", arrow::uint64(), true),
+                           }));
+
+    create_commit_action::create_commit_action(std::string id, std::string branch_id, const unsigned long &branch_version, const std::optional<std::string> &message, const long &commit_time) :
+            id(std::move(id)), branch_id(std::move(branch_id)), branch_version(branch_version), message(std::move(message)), commit_time(commit_time) {}
+
+    create_commit_action::create_commit_action(const std::shared_ptr<arrow::StructScalar> &value) {
+        id = from_struct<std::string>("id", value).value();
+        branch_id = from_struct<std::string>("branchId", value).value();
+        branch_version = from_struct<long>("branchVersion", value).value();
+        message = from_struct<std::string>("message", value);
+        commit_time = from_struct<long>("commitTime", value).value();
+    }
+
+    std::string create_commit_action::action_name() {
+        return "commit";
+    }
+
+    std::shared_ptr<arrow::StructType> create_commit_action::action_type() {
+        return arrow_type;
+    }
+
+    nlohmann::json create_commit_action::to_json() {
+        nlohmann::json json;
+
+        json["id"] = id;
+        json["branchId"] = branch_id;
+        json["branchVersion"] = branch_version;
+        json["message"] = to_json_value<std::string>(message);
+        json["commitTime"] = commit_time;
+
+        return json;
+    }
+}