From e88f9dd114275cf06eeb1196577bb6a9d554f2a8 Mon Sep 17 00:00:00 2001 From: David Symonds Date: Wed, 7 Oct 2020 21:44:02 +1100 Subject: [PATCH] test(spanner/spannertest): migrate most test code to integration_test.go (#2977) This makes it much easier to execute the same tests against both spannertest and a real Spanner database, which has surfaced a few divergences (noted with TODOs). This also exercises more of the Spanner client library's interaction with spannertest, ensuring better coverage of inmem.go too. --- spanner/go.mod | 1 + spanner/spannertest/db_test.go | 592 +----------------------- spanner/spannertest/integration_test.go | 588 ++++++++++++++++++++++- 3 files changed, 595 insertions(+), 586 deletions(-) diff --git a/spanner/go.mod b/spanner/go.mod index 761df9dce9c..c4ceacfcc0b 100644 --- a/spanner/go.mod +++ b/spanner/go.mod @@ -13,4 +13,5 @@ require ( google.golang.org/api v0.32.0 google.golang.org/genproto v0.0.0-20201006033701-bcad7cf615f2 google.golang.org/grpc v1.32.0 + google.golang.org/protobuf v1.25.0 ) diff --git a/spanner/spannertest/db_test.go b/spanner/spannertest/db_test.go index c59e6fedc16..bce01655cd7 100644 --- a/spanner/spannertest/db_test.go +++ b/spanner/spannertest/db_test.go @@ -16,6 +16,8 @@ limitations under the License. package spannertest +// TODO: More of this test should be moved into integration_test.go. + import ( "fmt" "io" @@ -31,19 +33,19 @@ import ( "cloud.google.com/go/spanner/spansql" ) -var stdTestTable = &spansql.CreateTable{ - Name: "Staff", - Columns: []spansql.ColumnDef{ - {Name: "Tenure", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "ID", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "Name", Type: spansql.Type{Base: spansql.String}}, - {Name: "Cool", Type: spansql.Type{Base: spansql.Bool}}, - {Name: "Height", Type: spansql.Type{Base: spansql.Float64}}, - }, - PrimaryKey: []spansql.KeyPart{{Column: "Name"}, {Column: "ID"}}, -} - func TestTableCreation(t *testing.T) { + stdTestTable := &spansql.CreateTable{ + Name: "Staff", + Columns: []spansql.ColumnDef{ + {Name: "Tenure", Type: spansql.Type{Base: spansql.Int64}}, + {Name: "ID", Type: spansql.Type{Base: spansql.Int64}}, + {Name: "Name", Type: spansql.Type{Base: spansql.String}}, + {Name: "Cool", Type: spansql.Type{Base: spansql.Bool}}, + {Name: "Height", Type: spansql.Type{Base: spansql.Float64}}, + }, + PrimaryKey: []spansql.KeyPart{{Column: "Name"}, {Column: "ID"}}, + } + var db database st := db.ApplyDDL(stdTestTable) if st.Code() != codes.OK { @@ -79,572 +81,6 @@ func TestTableCreation(t *testing.T) { } } -func TestTableData(t *testing.T) { - var db database - st := db.ApplyDDL(stdTestTable) - if st.Code() != codes.OK { - t.Fatalf("Creating table: %v", st.Err()) - } - - // Insert a subset of columns. - tx := db.NewTransaction() - tx.Start() - err := db.Insert(tx, "Staff", []spansql.ID{"ID", "Name", "Tenure", "Height"}, []*structpb.ListValue{ - // int64 arrives as a decimal string. - listV(stringV("1"), stringV("Jack"), stringV("10"), floatV(1.85)), - listV(stringV("2"), stringV("Daniel"), stringV("11"), floatV(1.83)), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - // Insert a different set of columns. - err = db.Insert(tx, "Staff", []spansql.ID{"Name", "ID", "Cool", "Tenure", "Height"}, []*structpb.ListValue{ - listV(stringV("Sam"), stringV("3"), boolV(false), stringV("9"), floatV(1.75)), - listV(stringV("Teal'c"), stringV("4"), boolV(true), stringV("8"), floatV(1.91)), - listV(stringV("George"), stringV("5"), nullV(), stringV("6"), floatV(1.73)), - listV(stringV("Harry"), stringV("6"), boolV(true), nullV(), nullV()), - }) - if err != nil { - t.Fatalf("Inserting more data: %v", err) - } - // Delete that last one. - err = db.Delete(tx, "Staff", []*structpb.ListValue{listV(stringV("Harry"), stringV("6"))}, nil, false) - if err != nil { - t.Fatalf("Deleting a row: %v", err) - } - // Turns out this guy isn't cool after all. - err = db.Update(tx, "Staff", []spansql.ID{"Name", "ID", "Cool"}, []*structpb.ListValue{ - // Missing columns should be left alone. - listV(stringV("Daniel"), stringV("2"), boolV(false)), - }) - if err != nil { - t.Fatalf("Updating a row: %v", err) - } - if _, err := tx.Commit(); err != nil { - t.Fatalf("Committing changes: %v", err) - } - - // Read some specific keys. - ri, err := db.Read("Staff", []spansql.ID{"Name", "Tenure"}, []*structpb.ListValue{ - listV(stringV("George"), stringV("5")), - listV(stringV("Harry"), stringV("6")), // Missing key should be silently ignored. - listV(stringV("Sam"), stringV("3")), - listV(stringV("George"), stringV("5")), // Duplicate key should be silently ignored. - }, nil, 0) - if err != nil { - t.Fatalf("Reading keys: %v", err) - } - all := slurp(t, ri) - wantAll := [][]interface{}{ - {"George", int64(6)}, - {"Sam", int64(9)}, - } - if !reflect.DeepEqual(all, wantAll) { - t.Errorf("Read data by keys wrong.\n got %v\nwant %v", all, wantAll) - } - // Read the same, but by key range. - ri, err = db.Read("Staff", []spansql.ID{"Name", "Tenure"}, nil, keyRangeList{ - {start: listV(stringV("Gabriel")), end: listV(stringV("Harpo"))}, // open/open - { - // closed/open - start: listV(stringV("Sam"), stringV("3")), - startClosed: true, - end: listV(stringV("Teal'c"), - stringV("4")), - }, - }, 0) - if err != nil { - t.Fatalf("Reading key ranges: %v", err) - } - all = slurp(t, ri) - if !reflect.DeepEqual(all, wantAll) { - t.Errorf("Read data by key ranges wrong.\n got %v\nwant %v", all, wantAll) - } - - // Read a subset of all rows, with a limit. - ri, err = db.ReadAll("Staff", []spansql.ID{"Tenure", "Name", "Height"}, 4) - if err != nil { - t.Fatalf("ReadAll: %v", err) - } - wantCols := []colInfo{ - {Name: "Tenure", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "Name", Type: spansql.Type{Base: spansql.String}}, - {Name: "Height", Type: spansql.Type{Base: spansql.Float64}}, - } - if !reflect.DeepEqual(ri.Cols(), wantCols) { - t.Errorf("ReadAll cols wrong.\n got %v\nwant %v", ri.Cols(), wantCols) - } - all = slurp(t, ri) - wantAll = [][]interface{}{ - // Primary key is (Name, ID), so results should come back sorted by Name then ID. - {int64(11), "Daniel", 1.83}, - {int64(6), "George", 1.73}, - {int64(10), "Jack", 1.85}, - {int64(9), "Sam", 1.75}, - } - if !reflect.DeepEqual(all, wantAll) { - t.Errorf("ReadAll data wrong.\n got %v\nwant %v", all, wantAll) - } - - // Add DATE and TIMESTAMP columns, and populate them with some data. - st = db.ApplyDDL(&spansql.AlterTable{ - Name: "Staff", - Alteration: spansql.AddColumn{Def: spansql.ColumnDef{ - Name: "FirstSeen", - Type: spansql.Type{Base: spansql.Date}, - }}, - }) - if st.Code() != codes.OK { - t.Fatalf("Adding column: %v", st.Err()) - } - st = db.ApplyDDL(&spansql.AlterTable{ - Name: "Staff", - Alteration: spansql.AddColumn{Def: spansql.ColumnDef{ - Name: "To", // keyword; will need quoting in queries - Type: spansql.Type{Base: spansql.Timestamp}, - }}, - }) - if st.Code() != codes.OK { - t.Fatalf("Adding column: %v", st.Err()) - } - tx = db.NewTransaction() - tx.Start() - err = db.Update(tx, "Staff", []spansql.ID{"Name", "ID", "FirstSeen", "To"}, []*structpb.ListValue{ - listV(stringV("Jack"), stringV("1"), stringV("1994-10-28"), nullV()), - listV(stringV("Daniel"), stringV("2"), stringV("1994-10-28"), nullV()), - listV(stringV("George"), stringV("5"), stringV("1997-07-27"), stringV("2008-07-29T11:22:43Z")), - }) - if err != nil { - t.Fatalf("Updating rows: %v", err) - } - if _, err := tx.Commit(); err != nil { - t.Fatalf("Committing changes: %v", err) - } - - // Add some more data, then delete it with a KeyRange. - // The queries below ensure that this was all deleted. - tx = db.NewTransaction() - tx.Start() - err = db.Insert(tx, "Staff", []spansql.ID{"Name", "ID"}, []*structpb.ListValue{ - listV(stringV("01"), stringV("1")), - listV(stringV("03"), stringV("3")), - listV(stringV("06"), stringV("6")), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - err = db.Delete(tx, "Staff", nil, keyRangeList{{ - start: listV(stringV("01"), stringV("1")), - startClosed: true, - end: listV(stringV("9")), - }}, false) - if err != nil { - t.Fatalf("Deleting key range: %v", err) - } - if _, err := tx.Commit(); err != nil { - t.Fatalf("Committing changes: %v", err) - } - // Re-add the data and delete with DML. - err = db.Insert(tx, "Staff", []spansql.ID{"Name", "ID"}, []*structpb.ListValue{ - listV(stringV("01"), stringV("1")), - listV(stringV("03"), stringV("3")), - listV(stringV("06"), stringV("6")), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - n, err := db.Execute(&spansql.Delete{ - Table: "Staff", - Where: spansql.LogicalOp{ - LHS: spansql.ComparisonOp{ - LHS: spansql.ID("Name"), - Op: spansql.Ge, - RHS: spansql.Param("min"), - }, - Op: spansql.And, - RHS: spansql.ComparisonOp{ - LHS: spansql.ID("Name"), - Op: spansql.Lt, - RHS: spansql.Param("max"), - }, - }, - }, queryParams{ - "min": stringParam("01"), - "max": stringParam("07"), - }) - if err != nil { - t.Fatalf("Deleting with DML: %v", err) - } - if n != 3 { - t.Errorf("Deleting with DML affected %d rows, want 3", n) - } - - // Add a BYTES column, and populate it with some data. - st = db.ApplyDDL(&spansql.AlterTable{ - Name: "Staff", - Alteration: spansql.AddColumn{Def: spansql.ColumnDef{ - Name: "RawBytes", - Type: spansql.Type{Base: spansql.Bytes, Len: spansql.MaxLen}, - }}, - }) - if st.Code() != codes.OK { - t.Fatalf("Adding column: %v", st.Err()) - } - tx = db.NewTransaction() - tx.Start() - err = db.Update(tx, "Staff", []spansql.ID{"Name", "ID", "RawBytes"}, []*structpb.ListValue{ - // bytes {0x01 0x00 0x01} encode as base-64 AQAB. - listV(stringV("Jack"), stringV("1"), stringV("AQAB")), - }) - if err != nil { - t.Fatalf("Updating rows: %v", err) - } - if _, err := tx.Commit(); err != nil { - t.Fatalf("Committing changes: %v", err) - } - - // Prepare the sample tables from the Cloud Spanner docs. - // https://cloud.google.com/spanner/docs/query-syntax#appendix-a-examples-with-sample-data - for _, ct := range []*spansql.CreateTable{ - // TODO: Roster, TeamMascot when we implement JOINs. - { - Name: "PlayerStats", - Columns: []spansql.ColumnDef{ - {Name: "LastName", Type: spansql.Type{Base: spansql.String}}, - {Name: "OpponentID", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "PointsScored", Type: spansql.Type{Base: spansql.Int64}}, - }, - PrimaryKey: []spansql.KeyPart{{Column: "LastName"}, {Column: "OpponentID"}}, // TODO: is this right? - }, - // JoinA and JoinB are "A" and "B" from https://cloud.google.com/spanner/docs/query-syntax#join_types. - { - Name: "JoinA", - Columns: []spansql.ColumnDef{ - {Name: "w", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "x", Type: spansql.Type{Base: spansql.String}}, - }, - PrimaryKey: []spansql.KeyPart{{Column: "w"}, {Column: "x"}}, - }, - { - Name: "JoinB", - Columns: []spansql.ColumnDef{ - {Name: "y", Type: spansql.Type{Base: spansql.Int64}}, - {Name: "z", Type: spansql.Type{Base: spansql.String}}, - }, - PrimaryKey: []spansql.KeyPart{{Column: "y"}, {Column: "z"}}, - }, - } { - st := db.ApplyDDL(ct) - if st.Code() != codes.OK { - t.Fatalf("Creating table: %v", st.Err()) - } - } - tx = db.NewTransaction() - tx.Start() - err = db.Insert(tx, "PlayerStats", []spansql.ID{"LastName", "OpponentID", "PointsScored"}, []*structpb.ListValue{ - listV(stringV("Adams"), stringV("51"), stringV("3")), - listV(stringV("Buchanan"), stringV("77"), stringV("0")), - listV(stringV("Coolidge"), stringV("77"), stringV("1")), - listV(stringV("Adams"), stringV("52"), stringV("4")), - listV(stringV("Buchanan"), stringV("50"), stringV("13")), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - err = db.Insert(tx, "JoinA", []spansql.ID{"w", "x"}, []*structpb.ListValue{ - listV(stringV("1"), stringV("a")), - listV(stringV("2"), stringV("b")), - listV(stringV("3"), stringV("c")), - listV(stringV("3"), stringV("d")), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - err = db.Insert(tx, "JoinB", []spansql.ID{"y", "z"}, []*structpb.ListValue{ - listV(stringV("2"), stringV("k")), - listV(stringV("3"), stringV("m")), - listV(stringV("3"), stringV("n")), - listV(stringV("4"), stringV("p")), - }) - if err != nil { - t.Fatalf("Inserting data: %v", err) - } - if _, err := tx.Commit(); err != nil { - t.Fatalf("Commiting changes: %v", err) - } - - // Do some complex queries. - tests := []struct { - q string - params queryParams - want [][]interface{} - }{ - { - `SELECT 17, "sweet", TRUE AND FALSE, NULL, B"hello"`, - nil, - [][]interface{}{{int64(17), "sweet", false, nil, []byte("hello")}}, - }, - // Check handling of NULL values for the IS operator. - // There was a bug that returned errors for some of these cases. - { - `SELECT @x IS TRUE, @x IS NOT TRUE, @x IS FALSE, @x IS NOT FALSE, @x IS NULL, @x IS NOT NULL`, - queryParams{"x": nullParam()}, - [][]interface{}{ - {false, true, false, true, true, false}, - }, - }, - { - `SELECT Name FROM Staff WHERE Cool`, - nil, - [][]interface{}{{"Teal'c"}}, - }, - { - `SELECT ID FROM Staff WHERE Cool IS NOT NULL ORDER BY ID DESC`, - nil, - [][]interface{}{{int64(4)}, {int64(3)}, {int64(2)}}, - }, - { - `SELECT Name, Tenure FROM Staff WHERE Cool IS NULL OR Cool ORDER BY Name LIMIT 2`, - nil, - [][]interface{}{ - {"George", int64(6)}, - {"Jack", int64(10)}, - }, - }, - { - `SELECT Name, ID + 100 FROM Staff WHERE @min <= Tenure AND Tenure < @lim ORDER BY Cool, Name DESC LIMIT @numResults`, - queryParams{"min": intParam(9), "lim": intParam(11), "numResults": intParam(100)}, - [][]interface{}{ - {"Jack", int64(101)}, - {"Sam", int64(103)}, - }, - }, - { - // Expression in SELECT list. - `SELECT Name, Cool IS NOT NULL FROM Staff WHERE Tenure/2 > 4 ORDER BY NOT Cool, Name`, - nil, - [][]interface{}{ - {"Daniel", true}, // Daniel has Cool==true - {"Jack", false}, // Jack has NULL Cool - {"Sam", true}, // Sam has Cool==false - }, - }, - { - `SELECT Name, Height FROM Staff ORDER BY Height DESC LIMIT 2`, - nil, - [][]interface{}{ - {"Teal'c", 1.91}, - {"Jack", 1.85}, - }, - }, - { - `SELECT Name FROM Staff WHERE Name LIKE "J%k" OR Name LIKE "_am"`, - nil, - [][]interface{}{ - {"Jack"}, - {"Sam"}, - }, - }, - { - `SELECT Name, Height FROM Staff WHERE Height BETWEEN @min AND @max ORDER BY Height DESC`, - queryParams{"min": floatParam(1.75), "max": floatParam(1.85)}, - [][]interface{}{ - {"Jack", 1.85}, - {"Daniel", 1.83}, - {"Sam", 1.75}, - }, - }, - { - `SELECT COUNT(*) FROM Staff WHERE Name < "T"`, - nil, - [][]interface{}{ - {int64(4)}, - }, - }, - { - // Check that aggregation still works for the empty set. - `SELECT COUNT(*) FROM Staff WHERE Name = "Nobody"`, - nil, - [][]interface{}{ - {int64(0)}, - }, - }, - { - `SELECT * FROM Staff WHERE Name LIKE "S%"`, - nil, - [][]interface{}{ - // These are returned in table column order. - // Note that the primary key columns get sorted first. - {"Sam", int64(3), int64(9), false, 1.75, nil, nil, nil}, - }, - }, - { - // Exactly the same as the previous, except with a redundant ORDER BY clause. - `SELECT * FROM Staff WHERE Name LIKE "S%" ORDER BY Name`, - nil, - [][]interface{}{ - {"Sam", int64(3), int64(9), false, 1.75, nil, nil, nil}, - }, - }, - { - `SELECT Name FROM Staff WHERE FirstSeen >= @min`, - queryParams{"min": dateParam("1996-01-01")}, - [][]interface{}{ - {"George"}, - }, - }, - { - `SELECT RawBytes FROM Staff WHERE RawBytes IS NOT NULL`, - nil, - [][]interface{}{ - {[]byte("\x01\x00\x01")}, - }, - }, - { - // The keyword "To" needs quoting in queries. - // Check coercion of comparison operator literal args too. - "SELECT COUNT(*) FROM Staff WHERE `To` > '2000-01-01T00:00:00Z'", - nil, - [][]interface{}{ - {int64(1)}, - }, - }, - { - `SELECT DISTINCT Cool, Tenure > 8 FROM Staff`, - nil, - [][]interface{}{ - // The non-distinct results are be - // [[false true] [ false] [ true] [false true] [true false]] - {false, true}, - {nil, false}, - {nil, true}, - {true, false}, - }, - }, - { - `SELECT Name FROM Staff WHERE ID IN UNNEST(@ids)`, - queryParams{"ids": queryParam{ - Value: []interface{}{int64(3), int64(1)}, - Type: spansql.Type{Base: spansql.Int64, Array: true}, - }}, - [][]interface{}{ - {"Jack"}, - {"Sam"}, - }, - }, - // From https://cloud.google.com/spanner/docs/query-syntax#group-by-clause_1: - { - // TODO: Ordering matters? Our implementation sorts by the GROUP BY key, - // but nothing documented seems to guarantee that. - `SELECT LastName, SUM(PointsScored) FROM PlayerStats GROUP BY LastName`, - nil, - [][]interface{}{ - {"Adams", int64(7)}, - {"Buchanan", int64(13)}, - {"Coolidge", int64(1)}, - }, - }, - { - // Another GROUP BY, but referring to an alias. - // Group by ID oddness, SUM over Tenure. - `SELECT ID&0x01 AS odd, SUM(Tenure) FROM Staff GROUP BY odd`, - nil, - [][]interface{}{ - {int64(0), int64(19)}, // Daniel(ID=2, Tenure=11), Teal'c(ID=4, Tenure=8) - {int64(1), int64(25)}, // Jack(ID=1, Tenure=10), Sam(ID=3, Tenure=9), George(ID=5, Tenure=6) - }, - }, - { - `SELECT MAX(Name) FROM Staff WHERE Name < @lim`, - queryParams{"lim": stringParam("Teal'c")}, - [][]interface{}{ - {"Sam"}, - }, - }, - { - `SELECT MAX(Name) FROM Staff WHERE Cool = @p1 LIMIT 1`, - queryParams{"p1": boolParam(true)}, - [][]interface{}{ - {"Teal'c"}, - }, - }, - { - `SELECT MIN(Name) FROM Staff`, - nil, - [][]interface{}{ - {"Daniel"}, - }, - }, - { - `SELECT ARRAY_AGG(Cool) FROM Staff ORDER BY Name`, - nil, - [][]interface{}{ - // Daniel, George (NULL), Jack (NULL), Sam, Teal'c - {[]interface{}{false, nil, nil, false, true}}, - }, - }, - // SELECT with aliases. - { - `SELECT s.Name FROM Staff AS s WHERE s.ID = 3 ORDER BY s.Tenure`, - nil, - [][]interface{}{ - {"Sam"}, - }, - }, - // Joins. - { - `SELECT * FROM JoinA LEFT OUTER JOIN JoinB AS B ON JoinA.w = B.y`, - nil, - [][]interface{}{ - {int64(1), "a", nil, nil}, - {int64(2), "b", int64(2), "k"}, - {int64(3), "c", int64(3), "m"}, - {int64(3), "c", int64(3), "n"}, - {int64(3), "d", int64(3), "m"}, - {int64(3), "d", int64(3), "n"}, - }, - }, - // Regression test for aggregating no rows; it used to return an empty row. - // https://github.com/googleapis/google-cloud-go/issues/2793 - { - `SELECT Cool, ARRAY_AGG(Name) FROM Staff WHERE Name > "zzz" GROUP BY Cool`, - nil, - nil, - }, - // Regression test for evaluating `IN` incorrectly using ==. - // https://github.com/googleapis/google-cloud-go/issues/2458 - { - `SELECT COUNT(*) FROM Staff WHERE RawBytes IN UNNEST(@arg)`, - queryParams{"arg": queryParam{ - Type: spansql.Type{Array: true, Base: spansql.Bytes}, - Value: []interface{}{ - []byte{0x02}, - []byte{0x01, 0x00, 0x01}, // only one present - }, - }}, - [][]interface{}{ - {int64(1)}, - }, - }, - } - for _, test := range tests { - q, err := spansql.ParseQuery(test.q) - if err != nil { - t.Errorf("ParseQuery(%q): %v", test.q, err) - continue - } - ri, err := db.Query(q, test.params) - if err != nil { - t.Errorf("Query(%q, %v): %v", test.q, test.params, err) - continue - } - all := slurp(t, ri) - if !reflect.DeepEqual(all, test.want) { - t.Errorf("Results from Query(%q, %v) are wrong.\n got %v\nwant %v", test.q, test.params, all, test.want) - } - } -} - func TestTableDescendingKey(t *testing.T) { var descTestTable = &spansql.CreateTable{ Name: "Timeseries", diff --git a/spanner/spannertest/integration_test.go b/spanner/spannertest/integration_test.go index ed40f6d8aac..609f7d3124d 100644 --- a/spanner/spannertest/integration_test.go +++ b/spanner/spannertest/integration_test.go @@ -32,6 +32,7 @@ import ( "testing" "time" + "cloud.google.com/go/civil" "cloud.google.com/go/spanner" dbadmin "cloud.google.com/go/spanner/admin/database/apiv1" "google.golang.org/api/iterator" @@ -41,6 +42,8 @@ import ( "google.golang.org/grpc/status" dbadminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" + spannerpb "google.golang.org/genproto/googleapis/spanner/v1" + structpb "google.golang.org/protobuf/types/known/structpb" ) var testDBFlag = flag.String("test_db", "", "Fully-qualified database name to test against; empty means use an in-memory fake.") @@ -143,14 +146,7 @@ func TestIntegration_SpannerBasics(t *testing.T) { if err != nil { t.Fatalf("Dropping old index: %v", err) } - err = updateDDL(t, adminClient, "DROP TABLE "+tableName) - // NotFound is an acceptable failure mode here. - if st, _ := status.FromError(err); st.Code() == codes.NotFound { - err = nil - } - if err != nil { - t.Fatalf("Dropping old table: %v", err) - } + dropTable(t, adminClient, tableName) err = updateDDL(t, adminClient, `CREATE TABLE `+tableName+` ( FirstName STRING(20) NOT NULL, @@ -400,6 +396,513 @@ func TestIntegration_SpannerBasics(t *testing.T) { } } +func TestIntegration_ReadsAndQueries(t *testing.T) { + client, adminClient, cleanup := makeClient(t) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Drop any old tables. + allTables := []string{ + "Staff", + "PlayerStats", + "JoinA", + "JoinB", + } + for _, table := range allTables { + dropTable(t, adminClient, table) + } + + err := updateDDL(t, adminClient, + `CREATE TABLE Staff ( + Tenure INT64, + ID INT64, + Name STRING(MAX), + Cool BOOL, + Height FLOAT64, + ) PRIMARY KEY (Name, ID)`) + if err != nil { + t.Fatal(err) + } + + // Insert a subset of columns. + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Insert("Staff", []string{"ID", "Name", "Tenure", "Height"}, []interface{}{1, "Jack", 10, 1.85}), + spanner.Insert("Staff", []string{"ID", "Name", "Tenure", "Height"}, []interface{}{2, "Daniel", 11, 1.83}), + }) + if err != nil { + t.Fatalf("Inserting data: %v", err) + } + // Insert a different set of columns. + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Sam", 3, false, 9, 1.75}), + spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Teal'c", 4, true, 8, 1.91}), + spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"George", 5, nil, 6, 1.73}), + spanner.Insert("Staff", []string{"Name", "ID", "Cool", "Tenure", "Height"}, []interface{}{"Harry", 6, true, nil, nil}), + }) + if err != nil { + t.Fatalf("Inserting more data: %v", err) + } + // Delete that last one. + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Delete("Staff", spanner.Key{"Harry", 6}), + }) + if err != nil { + t.Fatalf("Deleting a row: %v", err) + } + // Turns out this guy isn't cool after all. + _, err = client.Apply(ctx, []*spanner.Mutation{ + // Missing columns should be left alone. + spanner.Update("Staff", []string{"Name", "ID", "Cool"}, []interface{}{"Daniel", 2, false}), + }) + if err != nil { + t.Fatalf("Updating a row: %v", err) + } + + // Read some specific keys. + ri := client.Single().Read(ctx, "Staff", spanner.KeySets( + spanner.Key{"George", 5}, + spanner.Key{"Harry", 6}, // Missing key should be silently ignored. + spanner.Key{"Sam", 3}, + spanner.Key{"George", 5}, // Duplicate key should be silently ignored. + ), []string{"Name", "Tenure"}) + if err != nil { + t.Fatalf("Reading keys: %v", err) + } + all := mustSlurpRows(t, ri) + wantAll := [][]interface{}{ + {"George", int64(6)}, + {"Sam", int64(9)}, + } + if !reflect.DeepEqual(all, wantAll) { + t.Errorf("Read data by keys wrong.\n got %v\nwant %v", all, wantAll) + } + // Read the same, but by key range. + ri = client.Single().Read(ctx, "Staff", spanner.KeySets( + spanner.KeyRange{Start: spanner.Key{"Gabriel"}, End: spanner.Key{"Harpo"}, Kind: spanner.OpenOpen}, + spanner.KeyRange{Start: spanner.Key{"Sam", 3}, End: spanner.Key{"Teal'c", 4}, Kind: spanner.ClosedOpen}, + ), []string{"Name", "Tenure"}) + all = mustSlurpRows(t, ri) + if !reflect.DeepEqual(all, wantAll) { + t.Errorf("Read data by key ranges wrong.\n got %v\nwant %v", all, wantAll) + } + // Read a subset of all rows, with a limit. + ri = client.Single().ReadWithOptions(ctx, "Staff", spanner.AllKeys(), []string{"Tenure", "Name", "Height"}, + &spanner.ReadOptions{Limit: 4}) + all = mustSlurpRows(t, ri) + wantAll = [][]interface{}{ + // Primary key is (Name, ID), so results should come back sorted by Name then ID. + {int64(11), "Daniel", 1.83}, + {int64(6), "George", 1.73}, + {int64(10), "Jack", 1.85}, + {int64(9), "Sam", 1.75}, + } + if !reflect.DeepEqual(all, wantAll) { + t.Errorf("ReadAll data wrong.\n got %v\nwant %v", all, wantAll) + } + + // Add DATE and TIMESTAMP columns, and populate them with some data. + err = updateDDL(t, adminClient, + `ALTER TABLE Staff ADD COLUMN FirstSeen DATE`, + "ALTER TABLE Staff ADD COLUMN `To` TIMESTAMP", // "TO" is a keyword; needs quoting + ) + if err != nil { + t.Fatalf("Adding columns: %v", err) + } + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"Jack", 1, "1994-10-28", nil}), + spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"Daniel", 2, "1994-10-28", nil}), + spanner.Update("Staff", []string{"Name", "ID", "FirstSeen", "To"}, []interface{}{"George", 5, "1997-07-27", "2008-07-29T11:22:43Z"}), + }) + if err != nil { + t.Fatalf("Updating rows: %v", err) + } + + // Add some more data, then delete it with a KeyRange. + // The queries below ensure that this was all deleted. + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"01", 1}), + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"03", 3}), + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"06", 6}), + }) + if err != nil { + t.Fatalf("Inserting data: %v", err) + } + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Delete("Staff", spanner.KeyRange{ + /* This should work: + Start: spanner.Key{"01", 1}, + End: spanner.Key{"9"}, + However, that is unimplemented in the production Cloud Spanner, which rejects + that: ""For delete ranges, start and limit keys may only differ in the final key part" + */ + Start: spanner.Key{"01"}, + End: spanner.Key{"9"}, + Kind: spanner.ClosedOpen, + }), + }) + if err != nil { + t.Fatalf("Deleting key range: %v", err) + } + // Re-add the data and delete with DML. + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"01", 1}), + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"03", 3}), + spanner.Insert("Staff", []string{"Name", "ID"}, []interface{}{"06", 6}), + }) + if err != nil { + t.Fatalf("Inserting data: %v", err) + } + var n int64 + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *spanner.ReadWriteTransaction) error { + stmt := spanner.NewStatement("DELETE FROM Staff WHERE Name >= @min AND Name < @max") + stmt.Params["min"] = "01" + stmt.Params["max"] = "07" + n, err = tx.Update(ctx, stmt) + return err + }) + if err != nil { + t.Fatalf("Deleting with DML: %v", err) + } + if n != 3 { + t.Errorf("Deleting with DML affected %d rows, want 3", n) + } + + // Add a BYTES column, and populate it with some data. + err = updateDDL(t, adminClient, `ALTER TABLE Staff ADD COLUMN RawBytes BYTES(MAX)`) + if err != nil { + t.Fatalf("Adding column: %v", err) + } + _, err = client.Apply(ctx, []*spanner.Mutation{ + // bytes {0x01 0x00 0x01} encode as base-64 AQAB. + spanner.Update("Staff", []string{"Name", "ID", "RawBytes"}, []interface{}{"Jack", 1, []byte{0x01, 0x00, 0x01}}), + }) + if err != nil { + t.Fatalf("Updating rows: %v", err) + } + + // Prepare the sample tables from the Cloud Spanner docs. + // https://cloud.google.com/spanner/docs/query-syntax#appendix-a-examples-with-sample-data + err = updateDDL(t, adminClient, + // TODO: Roster, TeamMascot when we implement JOINs. + `CREATE TABLE PlayerStats ( + LastName STRING(MAX), + OpponentID INT64, + PointsScored INT64, + ) PRIMARY KEY (LastName, OpponentID)`, // TODO: is this right? + // JoinA and JoinB are "A" and "B" from https://cloud.google.com/spanner/docs/query-syntax#join_types. + `CREATE TABLE JoinA ( w INT64, x STRING(MAX) ) PRIMARY KEY (w, x)`, + `CREATE TABLE JoinB ( y INT64, z STRING(MAX) ) PRIMARY KEY (y, z)`, + ) + if err != nil { + t.Fatalf("Creating sample tables: %v", err) + } + _, err = client.Apply(ctx, []*spanner.Mutation{ + spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Adams", 51, 3}), + spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Buchanan", 77, 0}), + spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Coolidge", 77, 1}), + spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Adams", 52, 4}), + spanner.Insert("PlayerStats", []string{"LastName", "OpponentID", "PointsScored"}, []interface{}{"Buchanan", 50, 13}), + + spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{1, "a"}), + spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{2, "b"}), + spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{3, "c"}), + spanner.Insert("JoinA", []string{"w", "x"}, []interface{}{3, "d"}), + + spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{2, "k"}), + spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{3, "m"}), + spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{3, "n"}), + spanner.Insert("JoinB", []string{"y", "z"}, []interface{}{4, "p"}), + }) + if err != nil { + t.Fatalf("Inserting sample data: %v", err) + } + + // Do some complex queries. + tests := []struct { + q string + params map[string]interface{} + want [][]interface{} + }{ + { + `SELECT 17, "sweet", TRUE AND FALSE, NULL, B"hello"`, + nil, + [][]interface{}{{int64(17), "sweet", false, nil, []byte("hello")}}, + }, + // Check handling of NULL values for the IS operator. + // There was a bug that returned errors for some of these cases. + { + `SELECT @x IS TRUE, @x IS NOT TRUE, @x IS FALSE, @x IS NOT FALSE, @x IS NULL, @x IS NOT NULL`, + map[string]interface{}{"x": (*bool)(nil)}, + [][]interface{}{ + {false, true, false, true, true, false}, + }, + }, + { + `SELECT Name FROM Staff WHERE Cool`, + nil, + [][]interface{}{{"Teal'c"}}, + }, + { + `SELECT ID FROM Staff WHERE Cool IS NOT NULL ORDER BY ID DESC`, + nil, + [][]interface{}{{int64(4)}, {int64(3)}, {int64(2)}}, + }, + { + `SELECT Name, Tenure FROM Staff WHERE Cool IS NULL OR Cool ORDER BY Name LIMIT 2`, + nil, + [][]interface{}{ + {"George", int64(6)}, + {"Jack", int64(10)}, + }, + }, + { + `SELECT Name, ID + 100 FROM Staff WHERE @min <= Tenure AND Tenure < @lim ORDER BY Cool, Name DESC LIMIT @numResults`, + map[string]interface{}{"min": 9, "lim": 11, "numResults": 100}, + [][]interface{}{ + {"Jack", int64(101)}, + {"Sam", int64(103)}, + }, + }, + { + // Expression in SELECT list. + // TODO: This is broken against production; seems like BOOL ordering is wrong. + `SELECT Name, Cool IS NOT NULL FROM Staff WHERE Tenure/2 > 4 ORDER BY NOT Cool, Name`, + nil, + [][]interface{}{ + {"Daniel", true}, // Daniel has Cool==true + {"Jack", false}, // Jack has NULL Cool + {"Sam", true}, // Sam has Cool==false + }, + }, + { + `SELECT Name, Height FROM Staff ORDER BY Height DESC LIMIT 2`, + nil, + [][]interface{}{ + {"Teal'c", 1.91}, + {"Jack", 1.85}, + }, + }, + { + `SELECT Name FROM Staff WHERE Name LIKE "J%k" OR Name LIKE "_am"`, + nil, + [][]interface{}{ + {"Jack"}, + {"Sam"}, + }, + }, + { + `SELECT Name, Height FROM Staff WHERE Height BETWEEN @min AND @max ORDER BY Height DESC`, + map[string]interface{}{"min": 1.75, "max": 1.85}, + [][]interface{}{ + {"Jack", 1.85}, + {"Daniel", 1.83}, + {"Sam", 1.75}, + }, + }, + { + `SELECT COUNT(*) FROM Staff WHERE Name < "T"`, + nil, + [][]interface{}{ + {int64(4)}, + }, + }, + { + // Check that aggregation still works for the empty set. + `SELECT COUNT(*) FROM Staff WHERE Name = "Nobody"`, + nil, + [][]interface{}{ + {int64(0)}, + }, + }, + { + // TODO: This is broken against production; column order for * is wrong. + `SELECT * FROM Staff WHERE Name LIKE "S%"`, + nil, + [][]interface{}{ + // These are returned in table column order. + // Note that the primary key columns get sorted first. + {"Sam", int64(3), int64(9), false, 1.75, nil, nil, nil}, + }, + }, + { + // Exactly the same as the previous, except with a redundant ORDER BY clause. + // TODO: This is broken against production; column order for * is wrong. + `SELECT * FROM Staff WHERE Name LIKE "S%" ORDER BY Name`, + nil, + [][]interface{}{ + {"Sam", int64(3), int64(9), false, 1.75, nil, nil, nil}, + }, + }, + { + `SELECT Name FROM Staff WHERE FirstSeen >= @min`, + map[string]interface{}{"min": civil.Date{Year: 1996, Month: 1, Day: 1}}, + [][]interface{}{ + {"George"}, + }, + }, + { + `SELECT RawBytes FROM Staff WHERE RawBytes IS NOT NULL`, + nil, + [][]interface{}{ + {[]byte("\x01\x00\x01")}, + }, + }, + { + // The keyword "To" needs quoting in queries. + // Check coercion of comparison operator literal args too. + "SELECT COUNT(*) FROM Staff WHERE `To` > '2000-01-01T00:00:00Z'", + nil, + [][]interface{}{ + {int64(1)}, + }, + }, + { + // TODO: This is broken against production; ordering of result rows is incorrect + `SELECT DISTINCT Cool, Tenure > 8 FROM Staff`, + nil, + [][]interface{}{ + // The non-distinct results are be + // [[false true] [ false] [ true] [false true] [true false]] + {false, true}, + {nil, false}, + {nil, true}, + {true, false}, + }, + }, + { + `SELECT Name FROM Staff WHERE ID IN UNNEST(@ids)`, + map[string]interface{}{"ids": []int64{3, 1}}, + [][]interface{}{ + {"Jack"}, + {"Sam"}, + }, + }, + // From https://cloud.google.com/spanner/docs/query-syntax#group-by-clause_1: + { + // TODO: Ordering matters? Our implementation sorts by the GROUP BY key, + // but nothing documented seems to guarantee that. + `SELECT LastName, SUM(PointsScored) FROM PlayerStats GROUP BY LastName`, + nil, + [][]interface{}{ + {"Adams", int64(7)}, + {"Buchanan", int64(13)}, + {"Coolidge", int64(1)}, + }, + }, + { + // Another GROUP BY, but referring to an alias. + // Group by ID oddness, SUM over Tenure. + `SELECT ID&0x01 AS odd, SUM(Tenure) FROM Staff GROUP BY odd`, + nil, + [][]interface{}{ + {int64(0), int64(19)}, // Daniel(ID=2, Tenure=11), Teal'c(ID=4, Tenure=8) + {int64(1), int64(25)}, // Jack(ID=1, Tenure=10), Sam(ID=3, Tenure=9), George(ID=5, Tenure=6) + }, + }, + { + `SELECT MAX(Name) FROM Staff WHERE Name < @lim`, + map[string]interface{}{"lim": "Teal'c"}, + [][]interface{}{ + {"Sam"}, + }, + }, + { + `SELECT MAX(Name) FROM Staff WHERE Cool = @p1 LIMIT 1`, + map[string]interface{}{"p1": true}, + [][]interface{}{ + {"Teal'c"}, + }, + }, + { + `SELECT MIN(Name) FROM Staff`, + nil, + [][]interface{}{ + {"Daniel"}, + }, + }, + { + // TODO: This is broken against production; does not permit ORDER BY on something not grouped/aggregated. + `SELECT ARRAY_AGG(Cool) FROM Staff ORDER BY Name`, + nil, + [][]interface{}{ + // Daniel, George (NULL), Jack (NULL), Sam, Teal'c + {[]interface{}{false, nil, nil, false, true}}, + }, + }, + // SELECT with aliases. + { + `SELECT s.Name FROM Staff AS s WHERE s.ID = 3 ORDER BY s.Tenure`, + nil, + [][]interface{}{ + {"Sam"}, + }, + }, + // Joins. + { + // TODO: This is broken against production; row ordering is wrong. + `SELECT * FROM JoinA LEFT OUTER JOIN JoinB AS B ON JoinA.w = B.y`, + nil, + [][]interface{}{ + {int64(1), "a", nil, nil}, + {int64(2), "b", int64(2), "k"}, + {int64(3), "c", int64(3), "m"}, + {int64(3), "c", int64(3), "n"}, + {int64(3), "d", int64(3), "m"}, + {int64(3), "d", int64(3), "n"}, + }, + }, + // Regression test for aggregating no rows; it used to return an empty row. + // https://github.com/googleapis/google-cloud-go/issues/2793 + { + `SELECT Cool, ARRAY_AGG(Name) FROM Staff WHERE Name > "zzz" GROUP BY Cool`, + nil, + nil, + }, + // Regression test for evaluating `IN` incorrectly using ==. + // https://github.com/googleapis/google-cloud-go/issues/2458 + { + `SELECT COUNT(*) FROM Staff WHERE RawBytes IN UNNEST(@arg)`, + map[string]interface{}{ + "arg": [][]byte{ + {0x02}, + {0x01, 0x00, 0x01}, // only one present + }, + }, + [][]interface{}{ + {int64(1)}, + }, + }, + } + for _, test := range tests { + stmt := spanner.NewStatement(test.q) + stmt.Params = test.params + + ri = client.Single().Query(ctx, stmt) + all, err := slurpRows(t, ri) + if err != nil { + t.Errorf("Query(%q, %v): %v", test.q, test.params, err) + continue + } + if !reflect.DeepEqual(all, test.want) { + t.Errorf("Results from Query(%q, %v) are wrong.\n got %v\nwant %v", test.q, test.params, all, test.want) + } + } +} + +func dropTable(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, table string) { + t.Helper() + err := updateDDL(t, adminClient, "DROP TABLE "+table) + // NotFound is an acceptable failure mode here. + if st, _ := status.FromError(err); st.Code() == codes.NotFound { + err = nil + } + if err != nil { + t.Fatalf("Dropping old table %q: %v", table, err) + } +} + func updateDDL(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, statements ...string) error { t.Helper() ctx := context.Background() @@ -413,3 +916,72 @@ func updateDDL(t *testing.T, adminClient *dbadmin.DatabaseAdminClient, statement } return op.Wait(ctx) } + +func mustSlurpRows(t *testing.T, ri *spanner.RowIterator) [][]interface{} { + t.Helper() + all, err := slurpRows(t, ri) + if err != nil { + t.Fatalf("Reading rows: %v", err) + } + return all +} + +func slurpRows(t *testing.T, ri *spanner.RowIterator) (all [][]interface{}, err error) { + t.Helper() + err = ri.Do(func(r *spanner.Row) error { + var data []interface{} + for i := 0; i < r.Size(); i++ { + var gcv spanner.GenericColumnValue + if err := r.Column(i, &gcv); err != nil { + return err + } + data = append(data, genericValue(t, gcv)) + } + all = append(all, data) + return nil + }) + return +} + +func genericValue(t *testing.T, gcv spanner.GenericColumnValue) interface{} { + t.Helper() + + if _, ok := gcv.Value.Kind.(*structpb.Value_NullValue); ok { + return nil + } + if gcv.Type.Code == spannerpb.TypeCode_ARRAY { + var arr []interface{} + for _, v := range gcv.Value.GetListValue().Values { + arr = append(arr, genericValue(t, spanner.GenericColumnValue{ + Type: &spannerpb.Type{Code: gcv.Type.ArrayElementType.Code}, + Value: v, + })) + } + return arr + } + + var dst interface{} + switch gcv.Type.Code { + case spannerpb.TypeCode_BOOL: + dst = new(bool) + case spannerpb.TypeCode_INT64: + dst = new(int64) + case spannerpb.TypeCode_FLOAT64: + dst = new(float64) + case spannerpb.TypeCode_TIMESTAMP: + dst = new(time.Time) // TODO: do we need to force to UTC? + case spannerpb.TypeCode_DATE: + dst = new(civil.Date) + case spannerpb.TypeCode_STRING: + dst = new(string) + case spannerpb.TypeCode_BYTES: + dst = new([]byte) + } + if dst == nil { + t.Fatalf("Can't decode Spanner generic column value: %v", gcv.Type) + } + if err := gcv.Decode(dst); err != nil { + t.Fatalf("Decoding %v into %T: %v", gcv, dst, err) + } + return reflect.ValueOf(dst).Elem().Interface() +}