Add support for Float Scaling, WebP and Dictionary encoding filters (#…

…240)

* Add support for Float Scaling, WebP and Dictionary encoding filters

This adds a few new and missing filter types from the discovery and
create table paths.

* Bump version

docker/Dockerfile

@@ -55,7 +55,7 @@ WORKDIR /tmp
 
 ENV MARIADB_VERSION="mariadb-10.5.13"
 
-ARG MYTILE_VERSION="0.22.0"
+ARG MYTILE_VERSION="0.22.1"
 
 ARG TILEDB_VERSION="2.15.0"
 ARG TILEDB_VERSION_SHORT_SHA="1fb59c4"

docker/Dockerfile-R

@@ -72,7 +72,7 @@ WORKDIR /tmp
 
 ENV MARIADB_VERSION="mariadb-10.5.13"
 
-ARG MYTILE_VERSION="0.22.0"
+ARG MYTILE_VERSION="0.22.1"
 
 ARG TILEDB_VERSION="2.15.0"
 ARG TILEDB_VERSION_SHORT_SHA="1fb59c4"

docker/Dockerfile-min

@@ -53,7 +53,7 @@ ENV MTR_MEM=/tmp \
  MARIADB_VERSION="mariadb-10.5.13"
 WORKDIR /tmp
 
-ARG MYTILE_VERSION="0.22.0"
+ARG MYTILE_VERSION="0.22.1"
 
 ARG TILEDB_VERSION="2.15.0"
 ARG TILEDB_VERSION_SHORT_SHA="1fb59c4"

docker/Dockerfile-server

@@ -54,7 +54,7 @@ WORKDIR /tmp
 
 ENV MARIADB_VERSION="mariadb-10.5.13"
 
-ARG MYTILE_VERSION="0.22.0"
+ARG MYTILE_VERSION="0.22.1"
 
 ARG TILEDB_VERSION="2.15.0"
 ARG TILEDB_VERSION_SHORT_SHA="1fb59c4"

mysql-test/mytile/r/filters.result

@@ -79,3 +79,35 @@ t1	CREATE TABLE `t1` (
   `attr1` bigint(20) DEFAULT NULL `filters`='BYTESHUFFLE'
 ) ENGINE=MyTile DEFAULT CHARSET=latin1 `coordinate_filters`='LZ4=-1' `offset_filters`='BZIP2=3'
 DROP TABLE t1;
+# Float Scaling
+CREATE TABLE t1 (
+dim1 bigint dimension=1 lower_bound="0" upper_bound="100" tile_extent="10",
+attr1 float filters='SCALE_FLOAT=(8-1.0-0.0)'
+) ENGINE=mytile coordinate_filters='LZ4=-1' offset_filters='BZIP2=3';
+INSERT INTO t1 VALUES (1,1.1);
+SELECT * FROM t1;
+dim1	attr1
+1	0
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `dim1` bigint(20) DEFAULT NULL `dimension`=1 `lower_bound`='0' `upper_bound`='100' `tile_extent`='10',
+  `attr1` float DEFAULT NULL `filters`='SCALE_FLOAT=(8-1.0-0.0)'
+) ENGINE=MyTile DEFAULT CHARSET=latin1 `coordinate_filters`='LZ4=-1' `offset_filters`='BZIP2=3'
+DROP TABLE t1;
+# Dictionary
+CREATE TABLE t1 (
+dim1 bigint dimension=1 lower_bound="0" upper_bound="100" tile_extent="10",
+attr1 varchar(255) filters='DICTIONARY_ENCODING'
+) ENGINE=mytile coordinate_filters='LZ4=-1' offset_filters='BZIP2=3';
+INSERT INTO t1 VALUES (1, "abc");
+SELECT * FROM t1;
+dim1	attr1
+1	abc
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `dim1` bigint(20) DEFAULT NULL `dimension`=1 `lower_bound`='0' `upper_bound`='100' `tile_extent`='10',
+  `attr1` varchar(255) DEFAULT NULL `filters`='DICTIONARY_ENCODING'
+) ENGINE=MyTile DEFAULT CHARSET=latin1 `coordinate_filters`='LZ4=-1' `offset_filters`='BZIP2=3'
+DROP TABLE t1;

mysql-test/mytile/t/filters.test

@@ -50,3 +50,25 @@ SELECT * FROM t1;
 
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
+
+--echo # Float Scaling
+CREATE TABLE t1 (
+  dim1 bigint dimension=1 lower_bound="0" upper_bound="100" tile_extent="10",
+  attr1 float filters='SCALE_FLOAT=(8-1.0-0.0)'
+) ENGINE=mytile coordinate_filters='LZ4=-1' offset_filters='BZIP2=3';
+INSERT INTO t1 VALUES (1,1.1);
+SELECT * FROM t1;
+
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+--echo # Dictionary
+CREATE TABLE t1 (
+  dim1 bigint dimension=1 lower_bound="0" upper_bound="100" tile_extent="10",
+  attr1 varchar(255) filters='DICTIONARY_ENCODING'
+) ENGINE=mytile coordinate_filters='LZ4=-1' offset_filters='BZIP2=3';
+INSERT INTO t1 VALUES (1, "abc");
+SELECT * FROM t1;
+
+SHOW CREATE TABLE t1;
+DROP TABLE t1;

mytile/ha_mytile.cc

@@ -1303,9 +1303,9 @@ const COND *tile::mytile::cond_push_cond(Item_cond *cond_item) {
   std::shared_ptr<tiledb::QueryCondition> operatorCondition;
 
   // Depending on the condition type (OR, AND), we create a combination of
-  // conditions. Then we combine this combined Query Condition with the "primary"
-  // Query Condition with an AND. In the end, the "primary" Query Condition
-  // contains all the sub conditions.
+  // conditions. Then we combine this combined Query Condition with the
+  // "primary" Query Condition with an AND. In the end, the "primary" Query
+  // Condition contains all the sub conditions.
   for (uint32_t i = 0; i < arglist->elements; i++) {
     if ((subitem = li++)) {
       // COND_ITEMs
@@ -2458,33 +2458,37 @@ void tile::mytile::open_array_for_reads(THD *thd) {
       this->ctx = build_context(this->config);
     }
     if (this->table->s->option_struct->open_at != UINT64_MAX) {
-      #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
-        this->array = std::make_shared<tiledb::Array>(
-            this->ctx, this->uri, TILEDB_READ,
-            tiledb::TemporalPolicy(tiledb::TimeTravel, this->table->s->option_struct->open_at),
-            tiledb::EncryptionAlgorithm(encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-            this->table->s->option_struct->encryption_key));
-      #else
-        this->array = std::make_shared<tiledb::Array>(
-            this->ctx, this->uri, TILEDB_READ,
-            encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-            encryption_key, this->table->s->option_struct->open_at);
-      #endif
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
+      this->array = std::make_shared<tiledb::Array>(
+          this->ctx, this->uri, TILEDB_READ,
+          tiledb::TemporalPolicy(tiledb::TimeTravel,
+                                 this->table->s->option_struct->open_at),
+          tiledb::EncryptionAlgorithm(
+              encryption_key.empty() ? TILEDB_NO_ENCRYPTION
+                                     : TILEDB_AES_256_GCM,
+              this->table->s->option_struct->encryption_key));
+#else
+      this->array = std::make_shared<tiledb::Array>(
+          this->ctx, this->uri, TILEDB_READ,
+          encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
+          encryption_key, this->table->s->option_struct->open_at);
+#endif
 
     } else {
 
-      #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
-        this->array = std::make_shared<tiledb::Array>(
-            this->ctx, this->uri, TILEDB_READ,
-            tiledb::TemporalPolicy(),
-            tiledb::EncryptionAlgorithm(encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-            this->table->s->option_struct->encryption_key));
-      #else
-        this->array = std::make_shared<tiledb::Array>(
-            this->ctx, this->uri, TILEDB_READ,
-            encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-            encryption_key);
-      #endif
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
+      this->array = std::make_shared<tiledb::Array>(
+          this->ctx, this->uri, TILEDB_READ, tiledb::TemporalPolicy(),
+          tiledb::EncryptionAlgorithm(
+              encryption_key.empty() ? TILEDB_NO_ENCRYPTION
+                                     : TILEDB_AES_256_GCM,
+              this->table->s->option_struct->encryption_key));
+#else
+      this->array = std::make_shared<tiledb::Array>(
+          this->ctx, this->uri, TILEDB_READ,
+          encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
+          encryption_key);
+#endif
     }
     this->query =
         std::make_unique<tiledb::Query>(this->ctx, *this->array, TILEDB_READ);
@@ -2552,20 +2556,20 @@ void tile::mytile::open_array_for_writes(THD *thd) {
       this->config = cfg;
       this->ctx = build_context(this->config);
     }
-    
-    #if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
-      this->array = std::make_shared<tiledb::Array>(
-          this->ctx, this->uri, TILEDB_WRITE,
-          tiledb::TemporalPolicy(),
-          tiledb::EncryptionAlgorithm(encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-          this->table->s->option_struct->encryption_key));
-    #else
-      this->array = std::make_shared<tiledb::Array>(
-          this->ctx, this->uri, TILEDB_WRITE,
-          encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
-          encryption_key);
-    #endif
-      this->query =
+
+#if TILEDB_VERSION_MAJOR >= 2 && TILEDB_VERSION_MINOR >= 15
+    this->array = std::make_shared<tiledb::Array>(
+        this->ctx, this->uri, TILEDB_WRITE, tiledb::TemporalPolicy(),
+        tiledb::EncryptionAlgorithm(
+            encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
+            this->table->s->option_struct->encryption_key));
+#else
+    this->array = std::make_shared<tiledb::Array>(
+        this->ctx, this->uri, TILEDB_WRITE,
+        encryption_key.empty() ? TILEDB_NO_ENCRYPTION : TILEDB_AES_256_GCM,
+        encryption_key);
+#endif
+    this->query =
         std::make_unique<tiledb::Query>(this->ctx, *this->array, TILEDB_WRITE);
     // Else lets try to open reopen and use existing contexts
   } else {
@@ -3357,7 +3361,7 @@ mysql_declare_plugin(mytile){
     PLUGIN_LICENSE_PROPRIETARY, /* the plugin license (PLUGIN_LICENSE_XXX) */
     mytile_init_func,           /* Plugin Init */
     NULL,                       /* Plugin Deinit */
-    0x0220,                     /* version number (0.22.0) */
+    0x0221,                     /* version number (0.22.1) */
     tile::statusvars::mytile_status_variables, /* status variables */
     tile::sysvars::mytile_system_variables,    /* system variables */
     NULL,                                      /* config options */
@@ -3374,9 +3378,9 @@ maria_declare_plugin(mytile){
     PLUGIN_LICENSE_PROPRIETARY, /* the plugin license (PLUGIN_LICENSE_XXX) */
     mytile_init_func,           /* Plugin Init */
     NULL,                       /* Plugin Deinit */
-    0x0220,                     /* version number (0.22.0) */
+    0x0221,                     /* version number (0.22.1) */
     tile::statusvars::mytile_status_variables, /* status variables */
     tile::sysvars::mytile_system_variables,    /* system variables */
-    "0.22.0",                                  /* string version */
+    "0.22.1",                                  /* string version */
     MariaDB_PLUGIN_MATURITY_BETA               /* maturity */
 } maria_declare_plugin_end;

mytile/mytile.cc

@@ -1283,6 +1283,56 @@ tiledb::FilterList tile::parse_filter_list(tiledb::Context &ctx,
         filter.set_option(TILEDB_POSITIVE_DELTA_MAX_WINDOW, value);
         break;
       }
+      case TILEDB_FILTER_SCALE_FLOAT: {
+        sql_print_information("TILEDB_SCALE_FLOAT Data=%s", f[1].c_str());
+        size_t start = 0;
+        size_t end = 0;
+        if (f[1][0] == '(') {
+          start = 1;
+        }
+        if (f[1][f[1].size() - 1] == ')') {
+          end = f[1].size() - 1;
+        }
+        std::vector<std::string> values = split(f[1].substr(start, end), '-');
+
+        auto value_bytewidth = parse_value<uint64_t>(values[0]);
+        sql_print_information("TILEDB_SCALE_FLOAT_BYTEWIDTH=%d",
+                              value_bytewidth);
+        filter.set_option(TILEDB_SCALE_FLOAT_BYTEWIDTH, value_bytewidth);
+
+        auto value_factor = parse_value<double>(values[0]);
+        sql_print_information("TILEDB_SCALE_FLOAT_FACTOR=%d", value_factor);
+        filter.set_option(TILEDB_SCALE_FLOAT_FACTOR, value_factor);
+
+        auto value_offset = parse_value<double>(values[0]);
+        sql_print_information("TILEDB_SCALE_FLOAT_OFFSET=%d", value_offset);
+        filter.set_option(TILEDB_SCALE_FLOAT_OFFSET, value_offset);
+        break;
+      }
+      case TILEDB_FILTER_WEBP: {
+        size_t start = 0;
+        size_t end = 0;
+        if (f[1][0] == '(') {
+          start = 1;
+        }
+        if (f[1][f[1].size() - 1] == ')') {
+          end = f[1].size() - 1;
+        }
+        std::vector<std::string> values = split(f[1].substr(start, end), '-');
+
+        auto value_format = parse_value<uint8_t>(values[0]);
+        sql_print_information("TILEDB_WEBP_INPUT_FORMAT=%d", value_format);
+        filter.set_option(TILEDB_WEBP_INPUT_FORMAT, value_format);
+
+        auto value_lossless = parse_value<uint8_t>(values[0]);
+        sql_print_information("TILEDB_WEBP_LOSSLESS=%d", value_lossless);
+        filter.set_option(TILEDB_WEBP_LOSSLESS, value_lossless);
+
+        auto value_quality = parse_value<float>(values[0]);
+        sql_print_information("TILEDB_WEBP_QUALITY=%f", value_quality);
+        filter.set_option(TILEDB_WEBP_QUALITY, value_quality);
+        break;
+      }
       // The following have no filter options
       case TILEDB_FILTER_NONE:
       case TILEDB_FILTER_RLE:
@@ -1291,6 +1341,7 @@ tiledb::FilterList tile::parse_filter_list(tiledb::Context &ctx,
       case TILEDB_FILTER_DOUBLE_DELTA:
       case TILEDB_FILTER_CHECKSUM_MD5:
       case TILEDB_FILTER_CHECKSUM_SHA256:
+      case TILEDB_FILTER_DICTIONARY:
         break;
       // Handle all compressions with default
       default: {
@@ -1325,6 +1376,35 @@ std::string tile::filter_list_to_str(const tiledb::FilterList &filter_list) {
       str << "=" << value << ",";
       break;
     }
+    case TILEDB_FILTER_SCALE_FLOAT: {
+      uint64_t value_bytewidth;
+      filter.get_option<uint64_t>(TILEDB_SCALE_FLOAT_BYTEWIDTH,
+                                  &value_bytewidth);
+      str << "=(" << value_bytewidth << "-";
+
+      double value_factor;
+      filter.get_option<double>(TILEDB_SCALE_FLOAT_FACTOR, &value_factor);
+      str << value_factor << "-";
+
+      double value_offset;
+      filter.get_option<double>(TILEDB_SCALE_FLOAT_OFFSET, &value_offset);
+      str << value_offset << "),";
+      break;
+    }
+    case TILEDB_FILTER_WEBP: {
+      uint8_t value_format;
+      filter.get_option<uint8_t>(TILEDB_WEBP_INPUT_FORMAT, &value_format);
+      str << "=(" << value_format << "-";
+
+      uint8_t value_lossless;
+      filter.get_option<uint8_t>(TILEDB_WEBP_LOSSLESS, &value_lossless);
+      str << value_lossless << "-";
+
+      float value_quality;
+      filter.get_option<float>(TILEDB_WEBP_QUALITY, &value_quality);
+      str << value_quality << "),";
+      break;
+    }
     // The following have no filter options
     case TILEDB_FILTER_NONE:
     case TILEDB_FILTER_RLE:
@@ -1333,6 +1413,7 @@ std::string tile::filter_list_to_str(const tiledb::FilterList &filter_list) {
     case TILEDB_FILTER_DOUBLE_DELTA:
     case TILEDB_FILTER_CHECKSUM_MD5:
     case TILEDB_FILTER_CHECKSUM_SHA256:
+    case TILEDB_FILTER_DICTIONARY:
       str << ",";
       break;
     // Handle all compressions with default