Support between/not between operator

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java

@@ -52,7 +52,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.window.aggregation.AggregateWindowFunction;
-import com.amazon.opendistroforelasticsearch.sql.expression.window.ranking.RankingWindowFunction;
 import com.google.common.collect.ImmutableSet;
 import java.util.ArrayList;
 import java.util.Arrays;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java

@@ -461,6 +461,14 @@ public FunctionExpression notLike(Expression... expressions) {
     return function(BuiltinFunctionName.NOT_LIKE, expressions);
   }
 
+  public FunctionExpression between(Expression... expressions) {
+    return function(BuiltinFunctionName.BETWEEN, expressions);
+  }
+
+  public FunctionExpression not_between(Expression... expressions) {
+    return not(between(expressions));
+  }
+
   public Aggregator avg(Expression... expressions) {
     return aggregate(BuiltinFunctionName.AVG, expressions);
   }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java

@@ -92,7 +92,7 @@ public enum BuiltinFunctionName {
   MODULES(FunctionName.of("%")),
 
   /**
-   * Boolean Operators.
+   * Comparison Operators.
    */
   AND(FunctionName.of("and")),
   OR(FunctionName.of("or")),
@@ -106,6 +106,7 @@ public enum BuiltinFunctionName {
   GTE(FunctionName.of(">=")),
   LIKE(FunctionName.of("like")),
   NOT_LIKE(FunctionName.of("not like")),
+  BETWEEN(FunctionName.of("between")),
 
   /**
    * Aggregation Function.

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java

@@ -20,8 +20,13 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL;
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
@@ -63,6 +68,7 @@ public static void register(BuiltinFunctionRepository repository) {
     repository.register(like());
     repository.register(notLike());
     repository.register(regexp());
+    repository.register(between());
   }
 
   /**
@@ -262,6 +268,22 @@ private static FunctionResolver notLike() {
             STRING));
   }
 
+  private static FunctionResolver between() {
+    return FunctionDSL.define(BuiltinFunctionName.BETWEEN.getName(),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, DOUBLE, DOUBLE, DOUBLE),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, STRING, STRING, STRING),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, DATE, DATE, DATE),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, DATETIME, DATETIME, DATETIME),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, TIME, TIME, TIME),
+        FunctionDSL.impl(FunctionDSL.nullMissingHandling(OperatorUtils::between),
+            BOOLEAN, TIMESTAMP, TIMESTAMP, TIMESTAMP));
+  }
+
   private static ExprValue lookupTableFunction(ExprValue arg1, ExprValue arg2,
                                                Table<ExprValue, ExprValue, ExprValue> table) {
     if (table.contains(arg1, arg2)) {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/OperatorUtils.java

@@ -15,9 +15,12 @@
 
 package com.amazon.opendistroforelasticsearch.sql.utils;
 
+import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprNumberValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import java.sql.Timestamp;
 import java.util.regex.Pattern;
 import lombok.experimental.UtilityClass;
 
@@ -99,4 +102,26 @@ private static String patternToRegex(String patternString) {
     regex.append('$');
     return regex.toString();
   }
+
+  /**
+   * BETWEEN ... AND ... operator util.
+   * Expression { expr BETWEEN min AND max } is to judge if min <= expr <= max.
+   */
+  public static ExprBooleanValue between(ExprValue expr, ExprValue min, ExprValue max) {
+    return ExprBooleanValue.of(isBetween(expr, min, max));
+  }
+
+  private static boolean isBetween(ExprValue expr, ExprValue min, ExprValue max) {
+    if (expr instanceof AbstractExprNumberValue) {
+      return  ((AbstractExprNumberValue) expr).compare(min) >= 0
+          && ((AbstractExprNumberValue) expr).compare(max) <= 0;
+    } else if (expr instanceof ExprStringValue) {
+      return ((ExprStringValue) expr).compare(min) >= 0
+          && ((ExprStringValue) expr).compare(max) <= 0;
+    } else {
+      return expr.compareTo(min) >= 0 && expr.compareTo(max) <= 0;
+    }
+  }
 }
+
+

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java

@@ -22,6 +22,7 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
 import static java.util.Collections.emptyList;

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java

@@ -27,14 +27,18 @@
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue;
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.fromObjectValue;
-import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.missingValue;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME;
+import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
 import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare;
 import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
@@ -49,14 +53,15 @@
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils;
+import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException;
 import com.amazon.opendistroforelasticsearch.sql.expression.DSL;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase;
 import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
+import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
-import com.sun.org.apache.xpath.internal.Arg;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.ObjectInputStream;
@@ -72,7 +77,6 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.mockito.Mock;
 
 class BinaryPredicateOperatorTest extends ExpressionTestBase {
 
@@ -166,6 +170,45 @@ private static Stream<Arguments> testLikeArguments() {
     return builder.build();
   }
 
+  private static Stream<Arguments> testBetweenArguments() {
+    List<List> arguments = Arrays.asList(
+        Arrays.asList(1, 0, 2), Arrays.asList(1, 2, 0),
+        Arrays.asList(1L, 1L, 2L), Arrays.asList(2L, 1L, 2L),
+        Arrays.asList(3F, 1F, 2F), Arrays.asList(0F, 1F, 2F),
+        Arrays.asList(1D, 1D, 1D), Arrays.asList(1D, 2D, 2D),
+        Arrays.asList("b", "a", "c"), Arrays.asList("b", "c", "a"),
+        Arrays.asList("a", "a", "b"), Arrays.asList("b", "a", "b"),
+        Arrays.asList("c", "a", "b"), Arrays.asList("a", "b", "c"),
+        Arrays.asList("a", "a", "a"), Arrays.asList("b", "a", "a"));
+    Stream.Builder<Arguments> builder = Stream.builder();
+    for (List<Object> argGroup: arguments) {
+      builder.add(Arguments.of(fromObjectValue(argGroup.get(0)), fromObjectValue(argGroup.get(1)),
+          fromObjectValue(argGroup.get(2))));
+    }
+    builder
+        .add(Arguments.of(fromObjectValue("2021-01-02", DATE),
+            fromObjectValue("2021-01-01", DATE), fromObjectValue("2021-01-03", DATE)))
+        .add(Arguments.of(fromObjectValue("2021-01-02", DATE),
+            fromObjectValue("2021-01-03", DATE), fromObjectValue("2021-01-01", DATE)))
+        .add(Arguments.of(fromObjectValue("01:00:00", TIME),
+            fromObjectValue("01:00:00", TIME), fromObjectValue("02:00:00", TIME)))
+        .add(Arguments.of(fromObjectValue("02:00:00", TIME),
+            fromObjectValue("01:00:00", TIME), fromObjectValue("02:00:00", TIME)))
+        .add(Arguments.of(fromObjectValue("2021-01-01 03:00:00", DATETIME),
+            fromObjectValue("2021-01-01 01:00:00", DATETIME),
+            fromObjectValue("2021-01-01 02:00:00", DATETIME)))
+        .add(Arguments.of(fromObjectValue("2021-01-01 00:00:00", DATETIME),
+            fromObjectValue("2021-01-01 01:00:00", DATETIME),
+            fromObjectValue("2021-01-01 02:00:00", DATETIME)))
+        .add(Arguments.of(fromObjectValue("2021-01-01 01:00:00", TIMESTAMP),
+            fromObjectValue("2021-01-01 01:00:00", TIMESTAMP),
+            fromObjectValue("2021-01-01 01:00:00", TIMESTAMP)))
+        .add(Arguments.of(fromObjectValue("2021-01-01 00:00:00", TIMESTAMP),
+            fromObjectValue("2021-01-01 01:00:00", TIMESTAMP),
+            fromObjectValue("2021-01-01 01:00:00", TIMESTAMP)));
+    return builder.build();
+  }
+
   @ParameterizedTest(name = "and({0}, {1})")
   @MethodSource("binaryPredicateArguments")
   public void test_and(Boolean v1, Boolean v2) {
@@ -832,4 +875,44 @@ public void compare_int_long() {
     FunctionExpression equal = dsl.equal(DSL.literal(1), DSL.literal(1L));
     assertTrue(equal.valueOf(valueEnv()).booleanValue());
   }
+
+  @ParameterizedTest(name = "between({0}, {1}, {2})")
+  @MethodSource("testBetweenArguments")
+  public void between(ExprValue value, ExprValue minValue, ExprValue maxValue) {
+    FunctionExpression between = dsl.between(
+        DSL.literal(value), DSL.literal(minValue), DSL.literal(maxValue));
+    assertEquals(BOOLEAN, between.type());
+    assertEquals(OperatorUtils.between(value, minValue, maxValue), between.valueOf(valueEnv()));
+  }
+
+  @ParameterizedTest(name = "not between({0}, {1}, {2})")
+  @MethodSource("testBetweenArguments")
+  public void not_between(ExprValue value, ExprValue minValue, ExprValue maxValue) {
+    FunctionExpression notBetween = dsl.not_between(
+        DSL.literal(value), DSL.literal(minValue), DSL.literal(maxValue));
+    assertEquals(BOOLEAN, notBetween.type());
+    assertEquals(!OperatorUtils.between(value, minValue, maxValue).booleanValue(),
+        notBetween.valueOf(valueEnv()).booleanValue());
+  }
+
+  @Test
+  public void between_different_types() {
+    assertThrows(ExpressionEvaluationException.class, () ->
+        dsl.between(DSL.literal(1), DSL.literal(1), DSL.literal("1")));
+  }
+
+  @Test
+  public void between_null_missing() {
+    FunctionExpression between = dsl.between(
+        DSL.literal(1), DSL.literal(0), DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER));
+    assertTrue(between.valueOf(valueEnv()).isNull());
+
+    between = dsl.between(
+        DSL.literal(1), DSL.literal(0), DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER));
+    assertTrue(between.valueOf(valueEnv()).isMissing());
+
+    between = dsl.between(DSL.literal(1), DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER),
+        DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER));
+    assertTrue(between.valueOf(valueEnv()).isMissing());
+  }
 }

docs/user/dql/expressions.rst

@@ -130,6 +130,8 @@ Operators
 +----------------+----------------------------------------+
 | REGEXP         | String matches regular expression test |
 +----------------+----------------------------------------+
+| BETWEEN AND    | In the range of two values             |
++----------------+----------------------------------------+
 
 
 Basic Comparison Operator
@@ -185,6 +187,21 @@ expr REGEXP pattern. The expr is string value, pattern is supports regular expre
     | 1                      | 0                |
     +------------------------+------------------+
 
+
+BETWEEN AND
+-----------
+
+expr BETWEEN min AND max. This operator is to judge if expr is in the range from min to max (min <= expr <= max), and returns 1 for true, 0 for false. expr NOT BETWEEN min AND max is the equivalent to NOT expr BETWEEN min AND max. The three expressions expr, min and max should be consistent in their types for value comparisons, or expression evaluation exception would be thrown. The supported types in this operator include number, string, and date and time related types. Implicit casting is not supported yet, so you would need to explicitly specifies the types of compared values. Here follow some examples::
+
+    od> SELECT 1 BETWEEN 0 AND 2 AS res1, '1' BETWEEN '2' AND '0' AS res2, date('2021-03-05') BETWEEN date('2021-03-05') AND date('2021-03-05') AS res3;
+    fetched rows / total rows = 1/1
+    +--------+--------+--------+
+    | res1   | res2   | res3   |
+    |--------+--------+--------|
+    | True   | False  | True   |
+    +--------+--------+--------+
+
+
 Function Call
 =============
 

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/FilterQueryBuilder.java

@@ -62,6 +62,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor<QueryBuilder, Obje
           .put(BuiltinFunctionName.GREATER.getName(), new RangeQuery(Comparison.GT))
           .put(BuiltinFunctionName.LTE.getName(), new RangeQuery(Comparison.LTE))
           .put(BuiltinFunctionName.GTE.getName(), new RangeQuery(Comparison.GTE))
+          .put(BuiltinFunctionName.BETWEEN.getName(), new RangeQuery(Comparison.BETWEEN))
           .put(BuiltinFunctionName.LIKE.getName(), new WildcardQuery())
           .build();
 

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/LuceneQuery.java

@@ -23,6 +23,8 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
 import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression;
 import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
+import java.util.List;
+import java.util.stream.Collectors;
 import org.elasticsearch.index.query.QueryBuilder;
 
 /**
@@ -40,7 +42,7 @@ public abstract class LuceneQuery {
    * @return        return true if supported, otherwise false.
    */
   public boolean canSupport(FunctionExpression func) {
-    return (func.getArguments().size() == 2)
+    return (func.getArguments().size() >= 2)
         && (func.getArguments().get(0) instanceof ReferenceExpression)
         && (func.getArguments().get(1) instanceof LiteralExpression);
   }
@@ -53,8 +55,14 @@ public boolean canSupport(FunctionExpression func) {
    */
   public QueryBuilder build(FunctionExpression func) {
     ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0);
-    LiteralExpression literal = (LiteralExpression) func.getArguments().get(1);
-    return doBuild(ref.getAttr(), ref.type(), literal.valueOf(null));
+    if (func.getArguments().size() > 2) {
+      List<ExprValue> literalList = func.getArguments().stream().skip(1)
+          .map(v -> v.valueOf(null)).collect(Collectors.toList());
+      return doBuild(ref.getAttr(), ref.type(), literalList);
+    } else {
+      LiteralExpression literal = (LiteralExpression) func.getArguments().get(1);
+      return doBuild(ref.getAttr(), ref.type(), literal.valueOf(null));
+    }
   }
 
   /**
@@ -71,6 +79,11 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l
         "Subclass doesn't implement this and build method either");
   }
 
+  protected QueryBuilder doBuild(String fieldName, ExprType fieldType, List<ExprValue> literals) {
+    throw new UnsupportedOperationException(
+        "Subclass doesn't implement this and build method either");
+  }
+
   /**
    * Convert multi-field text field name to its inner keyword field. The limitation and assumption
    * is that the keyword field name is always "keyword" which is true by default.

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/script/filter/lucene/RangeQuery.java

@@ -18,6 +18,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -53,7 +54,24 @@ protected QueryBuilder doBuild(String fieldName, ExprType fieldType, ExprValue l
       case GTE:
         return query.gte(value);
       default:
-        throw new IllegalStateException("Comparison is supported by range query: " + comparison);
+        throw new IllegalStateException(
+            "Comparison is not supported by range query or improper number of arguments for "
+                + comparison);
+    }
+  }
+
+  @Override
+  public QueryBuilder doBuild(String fieldName, ExprType fieldType, List<ExprValue> literals) {
+    Object minValue = literals.get(0).value();
+    Object maxValue = literals.get(1).value();
+    RangeQueryBuilder query = QueryBuilders.rangeQuery(fieldName);
+    switch (comparison) {
+      case BETWEEN:
+        return query.gte(minValue).lte(maxValue);
+      default:
+        throw new IllegalStateException(
+            "Comparison is not supported by range query or improper number of arguments for "
+                + comparison);
     }
   }