Proto3 & expr.Expr json

expr/node.proto

@@ -1,118 +1,112 @@
-syntax = "proto2";
+syntax = "proto3";
 package expr;
 
+// protoc --go_out=. *.proto
 
-//  protoc --proto_path=$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. --gofast_out=. node.proto
 
-import "github.com/gogo/protobuf/gogoproto/gogo.proto";
-
-option (gogoproto.marshaler_all) = true;
-option (gogoproto.sizer_all) = true;
-option (gogoproto.unmarshaler_all) = true;
-option (gogoproto.goproto_getters_all) = false;
-
-// The generic Expr
-message ExprPb {
-  optional int32 op = 1 [(gogoproto.nullable) = true];
-  repeated ExprPb args = 2 [(gogoproto.nullable) = true];
+// Expr an S-Expression https://en.wikipedia.org/wiki/S-expression representation
+// of the AST tree of expression.
+// EITHER (op,args) OR (one of ident, val) will be present but not both.
+message Expr {
+  string op = 1;
+  int32 opType = 2;
+  repeated Expr args = 3;
 
-  optional string ident = 4 [(gogoproto.nullable) = true];
-  optional string val = 5 [(gogoproto.nullable) = true];
-  optional int64 ival = 6 [(gogoproto.nullable) = true];
-  optional bool bval = 7 [(gogoproto.nullable) = true];
-  optional double fval = 8 [(gogoproto.nullable) = true];
+  string ident = 4;
+  string val = 5;
+  int32 valType = 6;
 }
 
-// The generic Node, must be exactly one of these types
+// Node expression must be exactly one of these types
 message NodePb {
-  optional BinaryNodePb bn = 1 [(gogoproto.nullable) = true];
-  optional BooleanNodePb booln = 2 [(gogoproto.nullable) = true];
-  optional UnaryNodePb un = 3 [(gogoproto.nullable) = true];
-  optional FuncNodePb fn = 4 [(gogoproto.nullable) = true];
-  optional TriNodePb tn = 5 [(gogoproto.nullable) = true];
-  optional ArrayNodePb an = 6 [(gogoproto.nullable) = true];
-  optional NumberNodePb nn = 10 [(gogoproto.nullable) = true];
-  optional ValueNodePb vn = 11 [(gogoproto.nullable) = true];
-  optional IdentityNodePb in = 12 [(gogoproto.nullable) = true];
-  optional StringNodePb sn = 13 [(gogoproto.nullable) = true];
-  optional IncludeNodePb incn = 14 [(gogoproto.nullable) = true];
-  optional NullNodePb niln = 15 [(gogoproto.nullable) = true];
+  BinaryNodePb bn = 1;
+  BooleanNodePb booln = 2;
+  UnaryNodePb un = 3;
+  FuncNodePb fn = 4;
+  TriNodePb tn = 5;
+  ArrayNodePb an = 6;
+  NumberNodePb nn = 10;
+  ValueNodePb vn = 11;
+  IdentityNodePb in = 12;
+  StringNodePb sn = 13;
+  IncludeNodePb incn = 14;
+  NullNodePb niln = 15;
 }
 
-// Binary Node, two child args
+// BinaryNodePb two child args and operation
 message BinaryNodePb {
-	required int32 op = 1 [(gogoproto.nullable) = false];
-	optional bool paren = 2 [(gogoproto.nullable) = false];
-	repeated NodePb args = 3 [(gogoproto.nullable) = false];
+	int32 op = 1;
+	bool paren = 2;
+	repeated NodePb args = 3;
 }
 
 // Boolean Node, n child args
 message BooleanNodePb {
-	required int32 op = 1 [(gogoproto.nullable) = false];
-	repeated NodePb args = 2 [(gogoproto.nullable) = false];
+	int32 op = 1;
+	repeated NodePb args = 2;
 }
 
 // Include Node, two child args
 message IncludeNodePb {
-	required int32 op = 1 [(gogoproto.nullable) = false];
-	required bool negated = 2 [(gogoproto.nullable) = false];
-	required IdentityNodePb identity = 3 [(gogoproto.nullable) = false];
+	int32 op = 1;
+	bool negated = 2;
+	IdentityNodePb identity = 3;
 }
 
 // Unary Node, one child
 message UnaryNodePb {
-	required int32 op = 1 [(gogoproto.nullable) = false];
-	optional bool paren = 2 [(gogoproto.nullable) = false];
-	required NodePb arg = 3 [(gogoproto.nullable) = false];
+	int32 op = 1;
+	bool paren = 2;
+	NodePb arg = 3;
 }
 
 // Func Node, args are children
 message FuncNodePb {
-	required string name = 1 [(gogoproto.nullable) = false];
-	repeated NodePb args = 2 [(gogoproto.nullable) = false];
+	string name = 1;
+	repeated NodePb args = 2;
 }
 
-// Tri Node, may hve children
+// TriNodePb, may have children
 message TriNodePb {
-	required int32 op = 1 [(gogoproto.nullable) = false];
-	repeated NodePb args = 2 [(gogoproto.nullable) = false];
+	int32 op = 1;
+	repeated NodePb args = 2;
 }
 
 // Array Node
 message ArrayNodePb {
-	required int32 wrap = 1 [(gogoproto.nullable) = true];
-	repeated NodePb args = 3 [(gogoproto.nullable) = false];
+	int32 wrap = 1;
+	repeated NodePb args = 3;
 }
 
 // String literal, no children
 message StringNodePb {
-	optional bool noquote = 1 [(gogoproto.nullable) = true];
-	optional int32 quote = 2 [(gogoproto.nullable) = true];
-	optional string text = 3 [(gogoproto.nullable) = false];
+	bool noquote = 1;
+	int32 quote = 2;
+	string text = 3;
 }
 
 // Identity
 message IdentityNodePb {
-	optional int32 quote = 1 [(gogoproto.nullable) = true];
-	optional string text = 3 [(gogoproto.nullable) = false];
+	int32 quote = 1;
+	string text = 3;
 }
 
 // Number Node
 message NumberNodePb {
-	optional bool isint = 1 [(gogoproto.nullable) = false];
-	optional bool isfloat = 2 [(gogoproto.nullable) = false];
-	required int64 iv = 3 [(gogoproto.nullable) = false];
-	required double fv = 4 [(gogoproto.nullable) = false];
-	required string text = 5 [(gogoproto.nullable) = false];
+	bool isint = 1;
+	bool isfloat = 2;
+	int64 iv = 3;
+	double fv = 4;
+	string text = 5;
 }
 
 // Value Node
 message ValueNodePb {
-	required int32 valuetype = 1 [(gogoproto.nullable) = false];
-	required bytes value = 2;
+	int32 valuetype = 1;
+	bytes value = 2;
 }
 
 // NullNode
 message NullNodePb {
-	optional int32 niltype = 1 [(gogoproto.nullable) = false];
+	int32 niltype = 1;
 }

expr/node_test.go

@@ -30,7 +30,7 @@ func TestNodePb(t *testing.T) {
 		exp, err := expr.ParseExpression(exprText)
 		assert.Equal(t, err, nil, "Should not error parse expr but got ", err, "for ", exprText)
 		pb := exp.NodePb()
-		assert.True(t, pb != nil, "was nil PB: %#v", exp)
+		assert.NotEqual(t, nil, pb, "was nil PB: %#v", exp)
 		pbBytes, err := proto.Marshal(pb)
 		assert.True(t, err == nil, "Should not error on proto.Marshal but got [%v] for %s pb:%#v", err, exprText, pb)
 		n2, err := expr.NodeFromPb(pbBytes)
@@ -40,6 +40,39 @@ func TestNodePb(t *testing.T) {
 	}
 }
 
+func TestNodePb2(t *testing.T) {
+	t.Parallel()
+	for _, et := range exprTests {
+		exprText := et.qlText
+		if et.ok {
+			n, err := expr.ParseExpression(exprText)
+			assert.Equal(t, err, nil, "Should not error parse expr but got ", err, "for ", exprText)
+			exp := n.Expr()
+			assert.NotEqual(t, nil, exp)
+			pbBytes, err := proto.Marshal(exp)
+			assert.Equal(t, nil, err, "Should not error on proto.Marshal but got [%v] for %s pb:%#v", err, exprText, exp)
+			exp2 := &expr.Expr{}
+			err = proto.Unmarshal(pbBytes, exp2)
+			assert.Equal(t, nil, err, exprText)
+			n2, err := expr.NodeFromExpr(exp2)
+			assert.Equal(t, nil, err, "Should not error but got %v for %v", err, exprText)
+
+			if !n.Equal(n2) {
+				by, err := json.MarshalIndent(exp, "", "  ")
+				assert.Equal(t, err, nil)
+				u.Debugf("%T %s", n, string(by))
+				by, err = json.MarshalIndent(exp2, "", "  ")
+				assert.Equal(t, err, nil)
+				u.Debugf("%T %s", n2, string(by))
+				assert.True(t, n.Equal(n2), "Expected Equal but got\npre\t%v\npost\t%v", n, n2)
+				return
+			}
+			assert.True(t, n.Equal(n2), "Expected Equal but got\n\t%v\n\t%v", n, n2)
+			u.Infof("pre/post: \n\t%s\n\t%s", n, n2)
+		}
+	}
+}
+
 func TestExprRoundTrip(t *testing.T) {
 	t.Parallel()
 	for _, et := range exprTests {

lex/dialect_sql_parse_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	u "github.com/araddon/gou"
+	"github.com/gogo/protobuf/proto"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/araddon/qlbridge/rel"
@@ -18,7 +19,7 @@ func parseSqlTest(t *testing.T, sql string) {
 		_, err2 := rel.ParseSqlSelect(sql)
 		assert.Equal(t, nil, err2)
 		pb := ss.ToPbStatement()
-		pbb, err := pb.Marshal()
+		pbb, err := proto.Marshal(pb)
 		assert.Equal(t, nil, err)
 		ss2, err := rel.SqlFromPb(pbb)
 		assert.Equal(t, nil, err)

plan/plan.go

@@ -434,24 +434,13 @@ func (m *Select) Marshal() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return m.pbplan.Marshal()
-}
-func (m *Select) MarshalTo(data []byte) (int, error) {
-	err := m.serializeToPb()
-	if err != nil {
-		return 0, err
-	}
-	return m.pbplan.MarshalTo(data)
-}
-func (m *Select) Size() (n int) {
-	m.serializeToPb()
-	return m.pbplan.Size()
+	return proto.Marshal(m.pbplan)
 }
 func (m *Select) Unmarshal(data []byte) error {
 	if m.pbplan == nil {
 		m.pbplan = &PlanPb{Select: &SelectPb{}}
 	}
-	return m.pbplan.Unmarshal(data)
+	return proto.Unmarshal(data, m.pbplan)
 }
 func (m *Select) serializeToPb() error {
 	if m.pbplan == nil {
@@ -706,21 +695,13 @@ func (m *Source) Equal(t Task) bool {
 }
 func (m *Source) Marshal() ([]byte, error) {
 	m.serializeToPb()
-	return m.SourcePb.Marshal()
-}
-func (m *Source) MarshalTo(data []byte) (n int, err error) {
-	m.serializeToPb()
-	return m.SourcePb.MarshalTo(data)
-}
-func (m *Source) Size() (n int) {
-	m.serializeToPb()
-	return m.SourcePb.Size()
+	return proto.Marshal(m.SourcePb)
 }
 func (m *Source) Unmarshal(data []byte) error {
 	if m.SourcePb == nil {
 		m.SourcePb = &SourcePb{}
 	}
-	return m.SourcePb.Unmarshal(data)
+	return proto.Unmarshal(data, m.SourcePb)
 }
 func (m *Source) serializeToPb() error {
 	if m.pbplan == nil {