feat: add format ReQL command

.github/actions/driver-tests/action.yml

@@ -14,12 +14,9 @@ inputs:
   commit_hash:
     description: The commit hash to use at checkout.
     required: false
-  # TODO: This commit is pointing after driver extraction, hence it should be the baseline.
-  #  When all tests are passing using this commit hash, update the hash to the latest and
-  #  fix the newly raised issues.
   default_python_driver_commit_hash:
     description: The commit hash to use at checkout for the default Python driver.
-    default: f0bf3b7e5279d23b72ffc8d8d3ae7e26dd5384e4
+    default: abbefd7e06b1e1449804e5a9df2af87b368cbed4
     required: false
   install_command:
     description: The test runner command to execute.
@@ -50,6 +47,10 @@ inputs:
 runs:
   using: composite
   steps:
+    - uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+
     - name: download ${{ inputs.driver_name }} driver
       if: ${{ inputs.driver_name != 'python' }}
       shell: bash
@@ -68,9 +69,9 @@ runs:
         git clone https://github.com/rethinkdb/rethinkdb-python /tmp/python-driver
         cd /tmp/python-driver
         git checkout ${{ inputs.default_python_driver_commit_hash }}
-        sudo apt install -y rsync
-        pip install -r requirements.txt
-        make prepare
+        pip install poetry
+        poetry install
+        make ql2.proto
         cd -
 
     - name: setup ${{ inputs.driver_name }} driver

.github/actions/tests/action.yml

@@ -14,10 +14,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: actions/setup-python@v3
-      with:
-        python-version: '2.7'
-
     - name: install prerequisites
       shell: bash
       run: |

.github/workflows/build.yml

@@ -22,6 +22,10 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '2.7'
+
       - name: cpplint
         uses: ./.github/actions/tests
         with:
@@ -38,7 +42,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
-          python-version: '2.7'
+          python-version: '3.8'
 
       - name: apt install
         run: |
@@ -81,6 +85,10 @@ jobs:
     needs: build
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '2.7'
+
       - name: unit tests
         uses: ./.github/actions/tests
         with:
@@ -109,6 +117,10 @@ jobs:
     needs: unit-tests
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
       - name: integration tests
         uses: ./.github/actions/driver-tests
         with:
@@ -137,6 +149,10 @@ jobs:
     needs: unit-tests
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
       - name: long running tests
         uses: ./.github/actions/driver-tests
         with:
@@ -177,7 +193,7 @@ jobs:
         with:
           driver_name: python
           driver_dist_dir: /tmp/python-driver/rethinkdb
-          interpreter: py2.7
+          interpreter: py3.8
           test_target: polyglot
 
       - name: compress test artifacts for upload
@@ -204,21 +220,18 @@ jobs:
         run: |
           curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
           source $HOME/.nvm/nvm.sh
-          nvm install 5.12
+          nvm install 16.14
 
       - name: polyglot tests for javascript
         uses: ./.github/actions/driver-tests
         with:
           driver_name: javascript
           driver_dist_dir: /tmp/driver/dist
           repo_url: https://github.com/rethinkdb/rethinkdb-javascript.git
-          # TODO: This commit is pointing after driver extraction, hence it should be the baseline.
-          #  When all tests are passing using this commit hash, update the hash to the latest and
-          #  fix the newly raised issues.
-          commit_hash: c717cb9e2bdab77b55b7a31a5d780ba293c5fadf
-          interpreter: js5
+          commit_hash: 9e3aa0a229d2f5759a5215844317dae4e7d58d10
+          interpreter: js16
           test_target: polyglot
-          env_activate: source $HOME/.nvm/nvm.sh && nvm use 5.12
+          env_activate: source $HOME/.nvm/nvm.sh && nvm use 16.14
           install_command: npm install && npm run build
 
       - name: compress test artifacts for upload
@@ -256,7 +269,7 @@ jobs:
           driver_name: ruby
           driver_dist_dir: /tmp/driver/lib
           repo_url: https://github.com/rethinkdb/rethinkdb-ruby.git
-          commit_hash: 25781763f1af4e85116c80fd0cc988927e9c6829
+          commit_hash: 002f5476b760b696ed83213eba9f5d2e565ca488
           interpreter: rb2.5
           test_target: polyglot
           env_activate: source /etc/profile.d/rvm.sh && rvm use 2.5

scripts/visualize_btree.py

@@ -371,7 +371,7 @@ def btree_to_html(btree, filename):
 
     superblock, values = btree
 
-    with file(filename, "w") as f:
+    with open(filename, "w") as f:
         sys.stdout = f
         try:
             print """

scripts/visualize_log_serializer.py

@@ -660,15 +660,14 @@ def print_html(self):
 
 def file_to_database(filename):
 
-    with file(filename) as f:
+    with open(filename) as f:
         text = f.read()
 
     return Database(text)
 
 def database_to_html(db, filename):
 
-    with file(filename, "w") as f:
-
+    with open(filename, "w") as f:
         sys.stdout = f
 
         try:

src/pprint/js_pprint.cc

@@ -903,6 +903,7 @@ static void pprint_update_reminder() {
     case Term::FLOOR:
     case Term::CEIL:
     case Term::ROUND:
+    case Term::FORMAT:
         break;
     }
     ql::datum_t::type_t d = ql::datum_t::type_t::R_NULL;

src/rdb_protocol/ql2.proto

@@ -798,6 +798,9 @@ message Term {
         BIT_NOT = 194;
         BIT_SAL = 195;
         BIT_SAR = 196;
+
+        // String formatting
+        FORMAT = 197;
     }
     optional TermType type = 1;
 

src/rdb_protocol/term.cc

@@ -215,6 +215,7 @@ counted_t<const term_t> compile_on_current_stack(
     case Term::FLOOR:              return make_floor_term(env, t);
     case Term::CEIL:               return make_ceil_term(env, t);
     case Term::ROUND:              return make_round_term(env, t);
+    case Term::FORMAT:             return make_format_term(env, t);
     default: unreachable();
     }
     unreachable();

src/rdb_protocol/term_walker.cc

@@ -392,6 +392,7 @@ bool term_type_is_valid(Term::TermType type) {
     case Term::FLOOR:
     case Term::CEIL:
     case Term::ROUND:
+    case Term::FORMAT:
         return true;
     default:
         return false;
@@ -589,6 +590,7 @@ bool term_is_write_or_meta(Term::TermType type) {
     case Term::FLOOR:
     case Term::CEIL:
     case Term::ROUND:
+    case Term::FORMAT:
         return false;
     default: unreachable();
     }
@@ -784,6 +786,7 @@ bool term_forbids_writes(Term::TermType type) {
     case Term::FLOOR:
     case Term::CEIL:
     case Term::ROUND:
+    case Term::FORMAT:
         return false;
     default: unreachable();
     }

src/rdb_protocol/terms/string.cc

@@ -1,12 +1,14 @@
-// Copyright 2010-2015 RethinkDB, all rights reserved.
+// Copyright 2010-2022 RethinkDB, all rights reserved.
 #include "rdb_protocol/terms/terms.hpp"
 
 #include <re2/re2.h>
 
 #include <algorithm>
 
 #include "parsing/utf8.hpp"
+#include "rdb_protocol/datum_string.hpp"
 #include "rdb_protocol/error.hpp"
+#include "rdb_protocol/minidriver.hpp"
 #include "rdb_protocol/op.hpp"
 
 namespace ql {
@@ -125,6 +127,75 @@ class match_term_t : public op_term_t {
     virtual const char *name() const { return "match"; }
 };
 
+class format_term_t : public op_term_t {
+public:
+    format_term_t(compile_env_t *env, const raw_term_t &term)
+            : op_term_t{env, term, argspec_t{2}}, compile_env{env} {}
+private:
+    virtual scoped_ptr_t <val_t> eval_impl(scope_env_t *env, args_t *args, eval_flags_t) const {
+        const std::string templ = args->arg(env, 0)->as_str().to_std();
+        const datum_t datum = args->arg(env, 1)->as_datum();
+
+        const size_t template_length = templ.size();
+        std::string formatted{};
+
+        for (size_t pos = 0; pos < template_length; ++pos) {
+            rcheck(templ[pos] != '}',
+                   base_exc_t::LOGIC,
+                   "No parameter tags to close");
+
+            if (templ[pos] == '{') {
+                size_t param_end_pos {1};
+                std::string param_name {};
+
+                while (pos + param_end_pos < template_length && templ[pos + param_end_pos] != '}') {
+                    char next_char{templ[pos + param_end_pos]};
+
+                    rcheck(next_char != '{',
+                           base_exc_t::LOGIC,
+                           "Nested template parameters are not allowed");
+
+                    // Reserve ":" for later use to implement formatting, "\\" for escaping.
+                    rcheck(next_char != ':',
+                           base_exc_t::LOGIC,
+                           strprintf("Formatting separator `%c` is not allowed in parameter name", next_char));
+                    rcheck(next_char != '\\',
+                           base_exc_t::LOGIC,
+                           strprintf("Reserved escape character `%c` is not allowed in parameter name", next_char));
+
+
+                    param_name.push_back(next_char);
+                    ++param_end_pos;
+                }
+
+                rcheck(pos + param_end_pos != template_length,
+                       base_exc_t::LOGIC,
+                       "Parameter tag must be closed");
+
+                datum_t field{datum.get_field(datum_string_t{param_name})};
+
+                minidriver_t r{backtrace()};
+                counted_t<const term_t> term = compile_term(
+                        compile_env,
+                        r.expr(std::move(field)).coerce_to("STRING").root_term());
+
+                const datum_string_t result{term->eval(env)->as_str()};
+
+                formatted.append(result.data(), result.size());
+                pos += param_end_pos;
+            } else {
+                formatted += templ[pos];
+            }
+        }
+
+        return new_val(datum_t(formatted));
+    }
+
+    virtual const char *name() const { return "format"; }
+
+    compile_env_t *compile_env;
+};
+
 template <typename It>
 It find_utf8_pred(It start, It end, std::function<bool(char32_t)> &&fn) {
     It pos(start);
@@ -304,5 +375,9 @@ counted_t<term_t> make_split_term(
         compile_env_t *env, const raw_term_t &term) {
     return make_counted<split_term_t>(env, term);
 }
+counted_t<term_t> make_format_term(
+        compile_env_t *env, const raw_term_t &term) {
+    return make_counted<format_term_t>(env, term);
+}
 
 }  // namespace ql

src/rdb_protocol/terms/terms.hpp

@@ -184,11 +184,13 @@ counted_t<term_t> make_json_term(
 counted_t<term_t> make_to_json_string_term(
     compile_env_t *env, const raw_term_t &term);
 
-// match.cc
+// string.cc
 counted_t<term_t> make_match_term(
     compile_env_t *env, const raw_term_t &term);
 counted_t<term_t> make_split_term(
     compile_env_t *env, const raw_term_t &term);
+counted_t<term_t> make_format_term(
+    compile_env_t *env, const raw_term_t &term);
 
 // case.cc
 counted_t<term_t> make_upcase_term(

test/common/http_support/jinja2/loaders.py

@@ -56,7 +56,7 @@ def get_source(self, environment, template):
                 if not exists(path):
                     raise TemplateNotFound(template)
                 mtime = getmtime(path)
-                with file(path) as f:
+                with open(path) as f:
                     source = f.read().decode('utf-8')
                 return source, path, lambda: mtime == getmtime(path)
     """

test/common/test_framework.py

@@ -465,7 +465,7 @@ def update_status(self):
         self.show_status()
 
     def clear_status(self):
-        self.buffer += self.clear_line
+        self.buffer += self.clear_line.decode()
 
     def show_status(self):
         if self.running_list:
@@ -756,7 +756,8 @@ def parse(self, args, groups, group=None):
         return filter
 
     def combine(self, type, other):
-        for name in set(self.tree.keys() + other.tree.keys()):
+        combined_keys = set(list(self.tree.keys()) + list(other.tree.keys()))
+        for name in combined_keys:
             self.zoom(name, create=True).combine(type, other.zoom(name))
         if other.default == self.INCLUDE:
             self.default = type
@@ -961,7 +962,7 @@ def killed(self):
         return os.path.exists(join(self.dir, "killed"))
 
     def dump_file(self, name):
-        with file(join(self.dir, name)) as f:
+        with open(join(self.dir, name)) as f:
             for line in f:
                 print(line, end=' ')