Merge pull request #138 from Flipkart/0.4

0.4 release
Flipkart · Apr 25, 2016 · d9f7fe4 · d9f7fe4
2 parents 0f67c27 + 197d95c
commit d9f7fe4
Show file tree

Hide file tree

Showing 152 changed files with 5,080 additions and 3,147 deletions.
diff --git a/.gitignore b/.gitignore
@@ -36,3 +36,5 @@
 */dependency-reduced-pom.xml
 
 .DS_Store
+target/*
+config/test.yml
diff --git a/.travis.yml b/.travis.yml
@@ -1,3 +1,4 @@
+sudo: false
 language: java
 jdk:
-  - oraclejdk7
+  - oraclejdk8
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,25 @@
+FROM ubuntu:14.04
+MAINTAINER Swapnil Marghade <s.g.marghade [at] gmail.com>
+
+
+RUN \
+  apt-get install -y --no-install-recommends software-properties-common \
+  && add-apt-repository ppa:webupd8team/java \
+  && gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
+  && apt-get update \
+  && echo debconf shared/accepted-oracle-license-v1-1 select true |  debconf-set-selections \
+  && echo debconf shared/accepted-oracle-license-v1-1 seen true |  debconf-set-selections \
+  && apt-get install -y --no-install-recommends oracle-java8-installer \
+  && apt-get install -y --no-install-recommends curl
+
+EXPOSE 17000
+EXPOSE 17001
+
+VOLUME /var/log/foxtrot-server
+
+ADD config/docker.yml docker.yml
+ADD foxtrot-server/target/foxtrot*.jar server.jar
+ADD scripts/local_es_setup.sh local_es_setup.sh
+
+CMD sh -c "sleep 15 ; java -jar server.jar initialize docker.yml || true ;  java -Dfile.encoding=utf-8 -jar server.jar server docker.yml"
+
diff --git a/config/docker.yml b/config/docker.yml
@@ -0,0 +1,29 @@
+http:
+  port: 17000
+  adminPort: 17001
+  connectorType: nonblocking
+
+elasticsearch:
+  hosts:
+  - elasticsearch
+  cluster: elasticsearch
+  tableNamePrefix: foxtrot 
+
+hbase:
+  secure : false
+  tableName: foxtrot
+  hbaseZookeeperQuorum: hbase
+  hbaseZookeeperClientPort: 2181
+
+cluster:
+  name: foxtrot
+  disableMulticast: true
+  members: ["localhost:5701", "localhost:5702"]
+
+logging:
+  level: INFO
+
+deletionconfig:
+  active: true
+  interval: 86400
+  initialdelay: 60
diff --git a/config/local.yml b/config/local.yml
@@ -7,10 +7,13 @@ elasticsearch:
   hosts:
   - localhost
   cluster: foxtrot
+  tableNamePrefix: foxtrot
 
 hbase:
   secure : false
   tableName: foxtrot
+  hbaseZookeeperQuorum: localhost
+  hbaseZookeeperClientPort: 2181
 
 cluster:
   name: foxtrot

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,34 @@
+elasticsearch:
+  image: library/elasticsearch:1.7.3
+  hostname: elasticsearch
+  command: elasticsearch -Des.http.cors.enabled=true
+  ports:
+   - "9200:9200"
+   - "9300:9300"
+
+# hbase compose
+hbase:
+  image: banno/hbase-standalone
+  hostname: hbase
+  ports:
+   - "2181:2181"
+   - "60000:60000"
+   - "60010:60010"
+   - "60020:60020"
+   - "60030:60030"
+
+#enitity store compose
+foxtrot-server:
+  links:
+   - elasticsearch
+   - hbase
+  container_name: foxtrot_server
+  build: .
+  ports:
+   - "17000:17000"
+   - "17001:17001"
+  volumes:
+   - /var/log/foxtrot-server
+   # enviroment valiables
+#  environment:
+#   - ZK_CONNECTION_STRING=hbase:2181
diff --git a/foxtrot-common/pom.xml b/foxtrot-common/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>foxtrot</artifactId>
         <groupId>com.flipkart.foxtrot</groupId>
-        <version>0.3</version>
+        <version>0.4.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/ActionRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/ActionRequest.java
@@ -36,4 +36,5 @@ public List<Filter> getFilters() {
     public void setFilters(List<Filter> filters) {
         this.filters = filters;
     }
+
 }
diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Document.java
@@ -36,6 +36,8 @@ public class Document implements Serializable {
     @JsonProperty
     private long timestamp;
 
+    private DocumentMetadata metadata;
+
     @NotNull
     @JsonProperty
     private JsonNode data;
@@ -50,6 +52,13 @@ public Document(String id, long timestamp, JsonNode data) {
         this.data = data;
     }
 
+    public Document(String id, long timestamp, DocumentMetadata metadata, JsonNode data) {
+        this.id = id;
+        this.timestamp = timestamp;
+        this.metadata = metadata;
+        this.data = data;
+    }
+
     public String getId() {
         return id;
     }
@@ -73,4 +82,22 @@ public long getTimestamp() {
     public void setTimestamp(long timestamp) {
         this.timestamp = timestamp;
     }
+
+    public DocumentMetadata getMetadata() {
+        return metadata;
+    }
+
+    public void setMetadata(DocumentMetadata metadata) {
+        this.metadata = metadata;
+    }
+
+    @Override
+    public String toString() {
+        return "Document{" +
+                "id='" + id + '\'' +
+                ", timestamp=" + timestamp +
+                ", metadata=" + metadata +
+                ", data=" + data +
+                '}';
+    }
 }
diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/DocumentMetadata.java
@@ -0,0 +1,41 @@
+package com.flipkart.foxtrot.common;
+
+/**
+ * Metadata for a document
+ */
+public class DocumentMetadata {
+    private String id;
+    private String rawStorageId;
+
+    public DocumentMetadata() {
+    }
+
+    public DocumentMetadata(String id, String rawStorageId) {
+        this.id = id;
+        this.rawStorageId = rawStorageId;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getRawStorageId() {
+        return rawStorageId;
+    }
+
+    public void setRawStorageId(String rawStorageId) {
+        this.rawStorageId = rawStorageId;
+    }
+
+    @Override
+    public String toString() {
+        return "DocumentMetadata{" +
+                "id='" + id + '\'' +
+                ", rawStorageId='" + rawStorageId + '\'' +
+                '}';
+    }
+}
diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/count/CountRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/count/CountRequest.java
@@ -1,17 +1,17 @@
 package com.flipkart.foxtrot.common.count;
 
 import com.flipkart.foxtrot.common.ActionRequest;
-import org.hibernate.validator.constraints.NotEmpty;
+import com.flipkart.foxtrot.common.util.CollectionUtils;
+import org.apache.commons.lang3.builder.ToStringBuilder;
 
-import javax.validation.constraints.NotNull;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * Created by rishabh.goyal on 02/11/14.
  */
 public class CountRequest extends ActionRequest {
 
-    @NotNull
-    @NotEmpty
     private String table;
 
     private String field;
@@ -48,11 +48,11 @@ public void setDistinct(boolean isDistinct) {
 
     @Override
     public String toString() {
-        return "CountRequest{" +
-                "table='" + table + '\'' +
-                ", field='" + field + '\'' +
-                ", isDistinct=" + isDistinct +
-                ", filters=" + getFilters() +
-                '}';
+        return new ToStringBuilder(this)
+                .appendSuper(super.toString())
+                .append("table", table)
+                .append("field", field)
+                .append("isDistinct", isDistinct)
+                .toString();
     }
 }
diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/distinct/DistinctRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/distinct/DistinctRequest.java
@@ -18,9 +18,7 @@
 import com.flipkart.foxtrot.common.ActionRequest;
 import com.flipkart.foxtrot.common.query.ResultSort;
 import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.hibernate.validator.constraints.NotEmpty;
 
-import javax.validation.constraints.NotNull;
 import java.util.List;
 
 /**
@@ -29,12 +27,9 @@
  * Time: 4:52 PM
  */
 public class DistinctRequest extends ActionRequest {
-    @NotNull
-    @NotEmpty
+
     private String table;
 
-    @NotNull
-    @NotEmpty
     private List<ResultSort> nesting;
 
     public DistinctRequest() {
@@ -48,7 +43,6 @@ public void setTable(String table) {
         this.table = table;
     }
 
-
     public List<ResultSort> getNesting() {
         return nesting;
     }

diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java
@@ -17,9 +17,7 @@
 
 import com.flipkart.foxtrot.common.ActionRequest;
 import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.hibernate.validator.constraints.NotEmpty;
 
-import javax.validation.constraints.NotNull;
 import java.util.List;
 
 /**
@@ -28,13 +26,9 @@
  * Time: 4:52 PM
  */
 public class GroupRequest extends ActionRequest {
-    @NotNull
-    @NotEmpty
-    private String table;
 
+    private String table;
 
-    @NotNull
-    @NotEmpty
     private List<String> nesting;
 
     public GroupRequest() {

diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/histogram/HistogramRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/histogram/HistogramRequest.java
@@ -18,23 +18,16 @@
 import com.flipkart.foxtrot.common.ActionRequest;
 import com.flipkart.foxtrot.common.Period;
 import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.hibernate.validator.constraints.NotEmpty;
-
-import javax.validation.constraints.NotNull;
 
 /**
  * User: Santanu Sinha (santanu.sinha@flipkart.com)
  * Date: 21/03/14
  * Time: 12:06 AM
  */
 public class HistogramRequest extends ActionRequest {
-    @NotNull
-    @NotEmpty
-    private String table;
 
+    private String table;
 
-    @NotNull
-    @NotEmpty
     private String field = "_timestamp";
 
     private Period period;
@@ -77,5 +70,4 @@ public String toString() {
                 .append("period", period)
                 .toString();
     }
-
 }
diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/Filter.java
@@ -22,9 +22,11 @@
 import com.flipkart.foxtrot.common.query.general.*;
 import com.flipkart.foxtrot.common.query.numeric.*;
 import com.flipkart.foxtrot.common.query.string.ContainsFilter;
+import com.flipkart.foxtrot.common.util.CollectionUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 
-import javax.validation.constraints.NotNull;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * User: Santanu Sinha (santanu.sinha@flipkart.com)
@@ -46,6 +48,7 @@
         @JsonSubTypes.Type(value = NotEqualsFilter.class, name = FilterOperator.not_equals),
         @JsonSubTypes.Type(value = AnyFilter.class, name = FilterOperator.any),
         @JsonSubTypes.Type(value = ExistsFilter.class, name = FilterOperator.exists),
+        @JsonSubTypes.Type(value = MissingFilter.class, name = FilterOperator.missing),
 
         //String
         @JsonSubTypes.Type(value = ContainsFilter.class, name = FilterOperator.contains),
@@ -58,7 +61,6 @@ public abstract class Filter {
     @JsonIgnore
     private final String operator;
 
-    @NotNull
     private String field;
 
     protected Filter(String operator) {
@@ -117,4 +119,12 @@ public boolean isTemporal() {
         return false;
     }
 
+    public Set<String> validate() {
+        Set<String> validationErrors = new HashSet<>();
+        if (CollectionUtils.isNullOrEmpty(field)) {
+            validationErrors.add("filter field cannot be null or empty");
+        }
+        return validationErrors;
+    }
+
 }