Add explore operator and table format (TUI)

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "libtenzir/aux/caf"]
 	path = libtenzir/aux/caf
 	url = https://github.com/tenzir/actor-framework
+[submodule "plugins/tui/aux/ftxui"]
+	path = plugins/explore/aux/ftxui
+	url = https://github.com/ArthurSonzogni/FTXUI.git

libtenzir/builtins/formats/pcap.cpp

@@ -501,12 +501,6 @@ class pcap_printer final : public plugin_printer {
 class plugin final : public virtual parser_plugin<pcap_parser>,
                      public virtual printer_plugin<pcap_printer> {
 public:
-  auto initialize(const record& config, const record& /* global_config */)
-    -> caf::error override {
-    config_ = config;
-    return caf::none;
-  }
-
   auto parse_parser(parser_interface& p) const
     -> std::unique_ptr<plugin_parser> override {
     auto parser = argument_parser{
@@ -531,9 +525,6 @@ class plugin final : public virtual parser_plugin<pcap_parser>,
   auto name() const -> std::string override {
     return "pcap";
   }
-
-private:
-  record config_;
 };
 
 } // namespace

libtenzir/include/tenzir/type.hpp

@@ -1231,12 +1231,15 @@ class record_type final : public stateful_type_base {
   /// Returns an iterable view over the leaf fields of a record type.
   [[nodiscard]] generator<leaf_view> leaves() const noexcept;
 
-  /// Returns the numnber of fields in a record.
+  /// Returns the number of fields in a record.
   [[nodiscard]] size_t num_fields() const noexcept;
 
   /// Returns the number of leaf fields in a record.
   [[nodiscard]] size_t num_leaves() const noexcept;
 
+  /// Returns the number of nested sub-records.
+  [[nodiscard]] size_t depth() const noexcept;
+
   /// Resolves a flat index into an offset.
   [[nodiscard]] offset resolve_flat_index(size_t flat_index) const noexcept;
 

libtenzir/src/type.cpp

@@ -2656,6 +2656,64 @@ size_t record_type::num_leaves() const noexcept {
   return num_leaves;
 }
 
+// TODO: the implementation is copied from above. We may want to factor the two
+// functions.
+size_t record_type::depth() const noexcept {
+  auto index = offset{0};
+  auto record_depth = index.size();
+  auto history = detail::stack_vector<const fbs::type::RecordType*, 64>{
+    table().type_as_record_type()};
+  while (!index.empty()) {
+    const auto* record = history.back();
+    TENZIR_ASSERT(record);
+    const auto* fields = record->fields();
+    TENZIR_ASSERT(fields);
+    // This is our exit condition: If we arrived at the end of a record, we need
+    // to step out one layer. We must also reset the target key at this point.
+    if (index.back() >= fields->size()) {
+      history.pop_back();
+      index.pop_back();
+      if (!index.empty())
+        ++index.back();
+      continue;
+    }
+    const auto* field = record->fields()->Get(index.back());
+    TENZIR_ASSERT(field);
+    const auto* field_type = resolve_transparent(field->type_nested_root());
+    TENZIR_ASSERT(field_type);
+    switch (field_type->type_type()) {
+      case fbs::type::Type::pattern_type:
+        __builtin_unreachable();
+      case fbs::type::Type::NONE:
+      case fbs::type::Type::bool_type:
+      case fbs::type::Type::int64_type:
+      case fbs::type::Type::uint64_type:
+      case fbs::type::Type::double_type:
+      case fbs::type::Type::duration_type:
+      case fbs::type::Type::time_type:
+      case fbs::type::Type::string_type:
+      case fbs::type::Type::ip_type:
+      case fbs::type::Type::subnet_type:
+      case fbs::type::Type::enumeration_type:
+      case fbs::type::Type::list_type:
+      case fbs::type::Type::map_type: {
+        ++index.back();
+        break;
+      }
+      case fbs::type::Type::record_type: {
+        history.emplace_back(field_type->type_as_record_type());
+        index.push_back(0);
+        record_depth = std::max(record_depth, index.size());
+        break;
+      }
+      case fbs::type::Type::enriched_type:
+        __builtin_unreachable();
+        break;
+    }
+  }
+  return record_depth;
+}
+
 offset record_type::resolve_flat_index(size_t flat_index) const noexcept {
   size_t current_flat_index = 0;
   auto index = offset{0};

nix/overlay.nix

@@ -39,6 +39,17 @@ in {
   # spdlog in nixpkgs uses `fmt_8` directly, but we want version 9, so we use a
   # little hack here.
   fmt_8 = prev.fmt;
+  ftxui =
+    if !isStatic
+    then prev.ftxui
+    else
+      prev.ftxui.overrideAttrs (orig: {
+        cmakeFlags = [
+          "-DFTXUI_BUILD_EXAMPLES=OFF"
+          "-DFTXUI_BUILD_DOCS=OFF"
+          "-DFTXUI_BUILD_TESTS=OFF"
+        ];
+      });
   # We need boost 1.8.1 at minimum for URL.
   boost = prev.boost18x;
   rapidjson = prev.rapidjson.overrideAttrs (_: {

nix/tenzir/default.nix

@@ -18,6 +18,7 @@
     arrow-cpp,
     fast_float,
     flatbuffers,
+    ftxui,
     spdlog,
     libyamlcpp,
     simdjson,
@@ -57,6 +58,7 @@
     extraPlugins' = map (x: "extra-plugins/${baseNameOf x}") extraPlugins;
     bundledPlugins =
       [
+        "plugins/explore"
         "plugins/kafka"
         "plugins/nic"
         "plugins/parquet"
@@ -90,6 +92,7 @@
         buildInputs = [
           boost
           fast_float
+          ftxui
           libpcap
           libunwind
           libyamlcpp

plugins/explore/CMakeLists.txt

@@ -0,0 +1,48 @@
+cmake_minimum_required(VERSION 3.19...3.24 FATAL_ERROR)
+
+project(
+  explore
+  DESCRIPTION "explore plugin for Tenzir"
+  LANGUAGES CXX)
+
+include(CTest)
+
+find_package(Tenzir REQUIRED)
+
+file(GLOB_RECURSE explore_sources CONFIGURE_DEPENDS
+     "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp")
+
+TenzirRegisterPlugin(
+  TARGET explore
+  ENTRYPOINT src/plugin.cpp
+  SOURCES ${explore_sources}
+  INCLUDE_DIRECTORIES include)
+
+option(TENZIR_ENABLE_BUNDLED_FTXUI "Always use the bundled ftxui" OFF)
+add_feature_info("TENZIR_ENABLE_BUNDLED_FTXUI" TENZIR_ENABLE_BUNDLED_FTXUI
+                 "always use the bundled ftxui.")
+  if (NOT TENZIR_ENABLE_BUNDLED_FTXUI)
+  find_package(ftxui QUIET)
+  if (ftxui_FOUND)
+    string(APPEND TENZIR_FIND_DEPENDENCY_LIST
+           "\nfind_package(ftxui REQUIRED)")
+    dependency_summary("ftxui" ftxui::screen "Dependencies")
+  endif ()
+endif ()
+if (NOT ftxui_FOUND)
+  if (NOT TENZIR_ENABLE_BUNDLED_FTXUI)
+    message(
+      STATUS
+        "Cannot find installed ftxui; falling back to bundled version")
+  endif ()
+  unset(COMPILE_WARNING_AS_ERROR)
+  set(CMAKE_CXX_STANDARD 20)
+  add_subdirectory(aux/ftxui)
+  set(COMPILE_WARNING_AS_ERROR)
+  unset(CMAKE_CXX_STANDARD)
+  dependency_summary("ftxui" "${CMAKE_CURRENT_SOURCE_DIR}/aux/ftxui"
+                     "Dependencies")
+endif ()
+
+target_link_libraries(explore PUBLIC tenzir::libtenzir tenzir::internal)
+target_link_libraries(explore PUBLIC ftxui::screen ftxui::dom ftxui::component)

plugins/explore/include/explore/components.hpp

@@ -0,0 +1,94 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2023 The Tenzir Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include "explore/ui_state.hpp"
+
+#include <tenzir/fwd.hpp>
+#include <tenzir/view.hpp>
+
+#include <ftxui/component/component.hpp>
+#include <ftxui/dom/elements.hpp>
+
+namespace tenzir::plugins::explore {
+
+/// Lifts an FTXUI element into a component.
+auto lift(ftxui::Element e) -> ftxui::Component;
+
+/// Makes a component a vertically scrollable in a frame.
+auto enframe(const ftxui::Component& component) -> ftxui::Component;
+
+// We are adding a "deep" event catching helper here becase we are facing the
+// same issue of a parent component masking the events from its children as
+// reported in https://github.com/ArthurSonzogni/FTXUI/discussions/428.
+
+enum class catch_policy {
+  child,
+  parent,
+};
+
+template <catch_policy Policy>
+class CatchBase : public ftxui::ComponentBase {
+public:
+  // Constructor.
+  explicit CatchBase(std::function<bool(ftxui::Event)> on_event)
+    : on_event_{std::move(on_event)} {
+  }
+
+  auto OnEvent(ftxui::Event event) -> bool override {
+    if constexpr (Policy == catch_policy::child)
+      return ComponentBase::OnEvent(event) || on_event_(event);
+    else
+      return on_event_(event) || ComponentBase::OnEvent(event);
+  }
+
+protected:
+  std::function<bool(ftxui::Event)> on_event_;
+};
+
+template <catch_policy Policy>
+auto Catch(ftxui::Component child,
+           std::function<bool(ftxui::Event event)> on_event)
+  -> ftxui::Component {
+  auto out = Make<CatchBase<Policy>>(std::move(on_event));
+  out->Add(std::move(child));
+  return out;
+}
+
+template <catch_policy Policy>
+auto Catch(std::function<bool(ftxui::Event)> on_event)
+  -> ftxui::ComponentDecorator {
+  return [on_event = std::move(on_event)](ftxui::Component child) {
+    return Catch<Policy>(std::move(child),
+                         [on_event = on_event](ftxui::Event event) {
+                           return on_event(std::move(event));
+                         });
+  };
+}
+
+/// A major UI component in a focusable, bordered frame.
+auto Pane(ui_state* state, ftxui::Component component) -> ftxui::Component;
+
+/// A data frame.
+auto DataFrame(ui_state* state, const type& schema) -> ftxui::Component;
+
+/// The schema navigator.
+auto Navigator(ui_state* state, int* index, int* width) -> ftxui::Component;
+
+/// The Explorer.
+auto Explorer(ui_state* state) -> ftxui::Component;
+
+/// The top-level component of the application.
+/// @param screen The screen to hook for UI events.
+/// @param state The UI state.
+/// @returns The FTXUI component.
+auto MainWindow(ftxui::ScreenInteractive* screen, ui_state* state)
+  -> ftxui::Component;
+
+} // namespace tenzir::plugins::explore

plugins/explore/include/explore/elements.hpp

@@ -0,0 +1,18 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2023 The Tenzir Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <ftxui/dom/elements.hpp>
+
+namespace tenzir::plugins::explore {
+
+/// Renders the Tenzir logo.
+auto logo() -> ftxui::Element;
+
+} // namespace tenzir::plugins::explore

plugins/explore/include/explore/operator_args.hpp

@@ -0,0 +1,38 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2023 The Tenzir Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <tenzir/location.hpp>
+
+#include <optional>
+#include <string>
+
+namespace tenzir::plugins::explore {
+
+/// The plugin configuration.
+struct operator_args {
+  std::optional<located<int>> width;
+  std::optional<located<int>> height;
+  std::optional<location> fullscreen;
+  std::optional<location> hide_types;
+  std::optional<located<std::string>> navigator_position;
+  std::optional<location> navigator_auto_hide;
+
+  friend auto inspect(auto& f, operator_args& x) -> bool {
+    return f.object(x)
+      .pretty_name("operator_args")
+      .fields(f.field("width", x.width), f.field("height", x.height),
+              f.field("fullscreen", x.fullscreen),
+              f.field("hide_types", x.hide_types),
+              f.field("navigator_position", x.navigator_position),
+              f.field("navigator_auto_hide", x.navigator_auto_hide));
+  }
+};
+
+} // namespace tenzir::plugins::explore

plugins/explore/include/explore/printer_args.hpp

@@ -0,0 +1,34 @@
+//    _   _____   __________
+//   | | / / _ | / __/_  __/     Visibility
+//   | |/ / __ |_\ \  / /          Across
+//   |___/_/ |_/___/ /_/       Space and Time
+//
+// SPDX-FileCopyrightText: (c) 2023 The Tenzir Contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <tenzir/location.hpp>
+
+#include <optional>
+#include <string>
+
+namespace tenzir::plugins::explore {
+
+/// The operator configuration.
+struct printer_args {
+  std::optional<located<int>> width;
+  std::optional<located<int>> height;
+  bool real_time;
+  bool hide_types;
+
+  friend auto inspect(auto& f, printer_args& x) -> bool {
+    return f.object(x)
+      .pretty_name("printer_args")
+      .fields(f.field("width", x.width), f.field("height", x.height),
+              f.field("real-time", x.real_time),
+              f.field("hide_types", x.hide_types));
+  }
+};
+
+} // namespace tenzir::plugins::explore