From 5cb0a60ac610aaf7f003c72039c83b07a00f06b1 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 10 Aug 2021 22:43:42 +0000 Subject: [PATCH 01/11] feat(bigquery/storage/managedwriter): improve protobuf support After some ongoing discussions with the Storage API team, this PR improves support for proto2/proto3 syntax in protocol buffer code. * Updates testdata so that we have a proto2 and proto3 form of our SimpleMessage data. * Add reference schemas to testdata. * Updates proto conversion code in the adapt package so it's creating proto2 messages by default. The code paths for doing proto3 conversions are present, but not exported yet as the storage API doesn't properly handle proto3 expectations for conversion. Namely, conversion doesn't properly account for default values and use of wrapper types. * Adds benchmarks for dynamic schema generation and static serialization, to aid in some internal discussions. --- .../managedwriter/adapt/protoconversion.go | 50 +++-- .../adapt/protoconversion_test.go | 115 +++++++++++- .../storage/managedwriter/integration_test.go | 41 ++-- .../managedwriter/testdata/messages.pb.go | 177 ------------------ .../testdata/messages_proto2.pb.go | 177 ++++++++++++++++++ .../testdata/messages_proto2.proto | 26 +++ .../testdata/messages_proto3.pb.go | 177 ++++++++++++++++++ .../{messages.proto => messages_proto3.proto} | 2 +- .../storage/managedwriter/testdata/schemas.go | 64 +++++++ 9 files changed, 607 insertions(+), 222 deletions(-) delete mode 100644 bigquery/storage/managedwriter/testdata/messages.pb.go create mode 100644 bigquery/storage/managedwriter/testdata/messages_proto2.pb.go create mode 100644 bigquery/storage/managedwriter/testdata/messages_proto2.proto create mode 100644 bigquery/storage/managedwriter/testdata/messages_proto3.pb.go rename bigquery/storage/managedwriter/testdata/{messages.proto => messages_proto3.proto} (96%) create mode 100644 bigquery/storage/managedwriter/testdata/schemas.go diff --git a/bigquery/storage/managedwriter/adapt/protoconversion.go b/bigquery/storage/managedwriter/adapt/protoconversion.go index 7b69b43eb4a..7803df6b203 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion.go @@ -27,15 +27,25 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) -// bqModeToFieldLabelMap holds mapping from field schema mode to proto label. -// proto3 no longer allows use of REQUIRED labels, so we solve that elsewhere -// and simply use optional. -var bqModeToFieldLabelMap = map[storagepb.TableFieldSchema_Mode]descriptorpb.FieldDescriptorProto_Label{ +var bqModeToFieldLabelMapProto2 = map[storagepb.TableFieldSchema_Mode]descriptorpb.FieldDescriptorProto_Label{ + storagepb.TableFieldSchema_NULLABLE: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL, + storagepb.TableFieldSchema_REPEATED: descriptorpb.FieldDescriptorProto_LABEL_REPEATED, + storagepb.TableFieldSchema_REQUIRED: descriptorpb.FieldDescriptorProto_LABEL_REQUIRED, +} + +var bqModeToFieldLabelMapProto3 = map[storagepb.TableFieldSchema_Mode]descriptorpb.FieldDescriptorProto_Label{ storagepb.TableFieldSchema_NULLABLE: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL, storagepb.TableFieldSchema_REPEATED: descriptorpb.FieldDescriptorProto_LABEL_REPEATED, storagepb.TableFieldSchema_REQUIRED: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL, } +func convertModeToLabel(mode storagepb.TableFieldSchema_Mode, useProto3 bool) *descriptorpb.FieldDescriptorProto_Label { + if useProto3 { + return bqModeToFieldLabelMapProto3[mode].Enum() + } + return bqModeToFieldLabelMapProto2[mode].Enum() +} + // Allows conversion between BQ schema type and FieldDescriptorProto's type. var bqTypeToFieldTypeMap = map[storagepb.TableFieldSchema_Type]descriptorpb.FieldDescriptorProto_Type{ storagepb.TableFieldSchema_BIGNUMERIC: descriptorpb.FieldDescriptorProto_TYPE_BYTES, @@ -114,7 +124,7 @@ func StorageSchemaToDescriptor(inSchema *storagepb.TableSchema, scope string) (p } // internal implementation of the conversion code. -func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope string, cache *dependencyCache, allowWrapperTypes bool) (protoreflect.Descriptor, error) { +func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope string, cache *dependencyCache, useProto3 bool) (protoreflect.Descriptor, error) { if inSchema == nil { return nil, newConversionError(scope, fmt.Errorf("no input schema was provided")) } @@ -145,7 +155,7 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st deps = append(deps, foundDesc.ParentFile()) } // construct field descriptor for the message - fdp, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, string(foundDesc.FullName()), allowWrapperTypes) + fdp, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, string(foundDesc.FullName()), useProto3) if err != nil { return nil, newConversionError(scope, fmt.Errorf("couldn't convert field to FieldDescriptorProto: %v", err)) } @@ -155,7 +165,7 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st ts := &storagepb.TableSchema{ Fields: f.GetFields(), } - desc, err := storageSchemaToDescriptorInternal(ts, currentScope, cache, allowWrapperTypes) + desc, err := storageSchemaToDescriptorInternal(ts, currentScope, cache, useProto3) if err != nil { return nil, newConversionError(currentScope, fmt.Errorf("couldn't convert message: %v", err)) } @@ -166,14 +176,14 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st if err != nil { return nil, newConversionError(currentScope, fmt.Errorf("failed to add descriptor to dependency cache: %v", err)) } - fdp, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, currentScope, allowWrapperTypes) + fdp, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, currentScope, useProto3) if err != nil { return nil, newConversionError(currentScope, fmt.Errorf("couldn't compute field schema : %v", err)) } fields = append(fields, fdp) } } else { - fd, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, currentScope, allowWrapperTypes) + fd, err := tableFieldSchemaToFieldDescriptorProto(f, fNumber, currentScope, useProto3) if err != nil { return nil, newConversionError(currentScope, err) } @@ -201,6 +211,9 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st Syntax: proto.String("proto3"), Dependency: depNames, } + if !useProto3 { + fdp.Syntax = proto.String("proto2") + } // We'll need a FileDescriptorSet as we have a FileDescriptorProto for the current // descriptor we're building, but we need to include all the referenced dependencies. @@ -223,33 +236,32 @@ func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope st } // tableFieldSchemaToFieldDescriptorProto builds individual field descriptors for a proto message. -// We're using proto3 syntax, but BigQuery supports the notion of NULLs which conflicts with proto3 default value -// behavior. To enable it, we look for nullable fields in the schema that should be scalars, and use the -// well-known wrapper types. +// +// For proto3, in cases where the mode is nullable we use the well known wrapper types. +// For proto2, we propagate the mode->label annotation as expected. // // Messages are always nullable, and repeated fields are as well. -func tableFieldSchemaToFieldDescriptorProto(field *storagepb.TableFieldSchema, idx int32, scope string, allowWrapperTypes bool) (*descriptorpb.FieldDescriptorProto, error) { +func tableFieldSchemaToFieldDescriptorProto(field *storagepb.TableFieldSchema, idx int32, scope string, useProto3 bool) (*descriptorpb.FieldDescriptorProto, error) { name := strings.ToLower(field.GetName()) if field.GetType() == storagepb.TableFieldSchema_STRUCT { return &descriptorpb.FieldDescriptorProto{ Name: proto.String(name), Number: proto.Int32(idx), TypeName: proto.String(scope), - Label: bqModeToFieldLabelMap[field.GetMode()].Enum(), + Label: convertModeToLabel(field.GetMode(), useProto3), }, nil } - // For (REQUIRED||REPEATED) fields, we use the expected scalar types, but the proto is - // still marked OPTIONAL (proto3 semantics). - if field.GetMode() != storagepb.TableFieldSchema_NULLABLE || !allowWrapperTypes { + // For (REQUIRED||REPEATED) fields for proto3, or all cases for proto2, we can use the expected scalar types. + if field.GetMode() != storagepb.TableFieldSchema_NULLABLE || !useProto3 { return &descriptorpb.FieldDescriptorProto{ Name: proto.String(name), Number: proto.Int32(idx), Type: bqTypeToFieldTypeMap[field.GetType()].Enum(), - Label: bqModeToFieldLabelMap[field.GetMode()].Enum(), + Label: convertModeToLabel(field.GetMode(), useProto3), }, nil } - // For NULLABLE, optionally use wrapper types. + // For NULLABLE proto3 fields, use a wrapper type. return &descriptorpb.FieldDescriptorProto{ Name: proto.String(name), Number: proto.Int32(idx), diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index ba6f1190f26..14d879bea3b 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -16,9 +16,13 @@ package adapt import ( "encoding/json" + "fmt" "reflect" "testing" + "time" + "cloud.google.com/go/bigquery" + "cloud.google.com/go/bigquery/storage/managedwriter/testdata" "github.com/google/go-cmp/cmp" storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2" "google.golang.org/protobuf/encoding/protojson" @@ -52,7 +56,7 @@ func TestSchemaToProtoConversion(t *testing.T) { Number: proto.Int32(1), Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()}, - {Name: proto.String("bar"), Number: proto.Int32(2), Type: descriptorpb.FieldDescriptorProto_TYPE_INT64.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()}, + {Name: proto.String("bar"), Number: proto.Int32(2), Type: descriptorpb.FieldDescriptorProto_TYPE_INT64.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_REQUIRED.Enum()}, {Name: proto.String("baz"), Number: proto.Int32(3), Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum()}, }, }, @@ -229,7 +233,7 @@ func TestProtoJSONSerialization(t *testing.T) { Name: proto.String("key"), Number: proto.Int32(1), Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), - Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Label: descriptorpb.FieldDescriptorProto_LABEL_REQUIRED.Enum(), }, { Name: proto.String("value"), @@ -277,3 +281,110 @@ func TestProtoJSONSerialization(t *testing.T) { } } + +var benchDescriptor protoreflect.Descriptor + +func BenchmarkStorageSchemaToDescriptor(b *testing.B) { + syntaxLabels := []string{"proto2", "proto3"} + for _, bm := range []struct { + name string + in bigquery.Schema + }{ + { + name: "SingleField", + in: bigquery.Schema{ + {Name: "field", Type: bigquery.StringFieldType}, + }, + }, + { + name: "NestedRecord", + in: bigquery.Schema{ + {Name: "field1", Type: bigquery.StringFieldType}, + {Name: "field2", Type: bigquery.IntegerFieldType}, + {Name: "field3", Type: bigquery.BooleanFieldType}, + { + Name: "field4", + Type: bigquery.RecordFieldType, + Schema: bigquery.Schema{ + {Name: "recordfield1", Type: bigquery.GeographyFieldType}, + {Name: "recordfield2", Type: bigquery.TimestampFieldType}, + }, + }, + }, + }, + { + name: "SimpleMessage", + in: testdata.SimpleMessageSchema, + }, + { + name: "GithubArchiveSchema", + in: testdata.GithubArchiveSchema, + }, + } { + for _, s := range syntaxLabels { + b.Run(fmt.Sprintf("%s-%s", bm.name, s), func(b *testing.B) { + convSchema, err := BQSchemaToStorageTableSchema(bm.in) + if err != nil { + b.Errorf("%q: schema conversion fail: %v", bm.name, err) + } + for n := 0; n < b.N; n++ { + dc := make(dependencyCache) + benchDescriptor, err = storageSchemaToDescriptorInternal(convSchema, "root", &dc, s == "proto3") + if err != nil { + b.Errorf("failed to convert %q: %v", bm.name, err) + } + } + }) + } + } +} + +var staticBytes []byte + +func BenchmarkStaticProtoSerialization(b *testing.B) { + for _, bm := range []struct { + name string + in bigquery.Schema + syntax string + setterF func() protoreflect.ProtoMessage + }{ + { + name: "SimpleMessageProto2", + in: bigquery.Schema{ + {Name: "field", Type: bigquery.StringFieldType}, + }, + setterF: func() protoreflect.ProtoMessage { + return &testdata.SimpleMessageProto2{ + Name: proto.String(fmt.Sprintf("test-%d", time.Now().UnixNano())), + Value: proto.Int64(time.Now().UnixNano()), + } + }, + }, + { + name: "SimpleMessageProto3", + in: bigquery.Schema{ + {Name: "field", Type: bigquery.StringFieldType}, + }, + setterF: func() protoreflect.ProtoMessage { + return &testdata.SimpleMessageProto3{ + Name: fmt.Sprintf("test-%d", time.Now().UnixNano()), + Value: time.Now().UnixNano(), + } + }, + }, + } { + b.Run(bm.name, func(b *testing.B) { + var totalBytes int64 + for n := 0; n < b.N; n++ { + m := bm.setterF() + out, err := proto.Marshal(m) + if err != nil { + b.Errorf("%q %q: Marshal: %v", bm.name, bm.syntax, err) + } + totalBytes = totalBytes + int64(len(out)) + staticBytes = out + } + b.Logf("avg bytes/message: %d", totalBytes/int64(b.N)) + }) + } +} diff --git a/bigquery/storage/managedwriter/integration_test.go b/bigquery/storage/managedwriter/integration_test.go index b685e64440f..5c3bf7f49bf 100644 --- a/bigquery/storage/managedwriter/integration_test.go +++ b/bigquery/storage/managedwriter/integration_test.go @@ -42,18 +42,13 @@ var ( defaultTestTimeout = 30 * time.Second ) -var testSimpleSchema = bigquery.Schema{ - {Name: "name", Type: bigquery.StringFieldType, Required: true}, - {Name: "value", Type: bigquery.IntegerFieldType, Required: true}, -} - // our test data has cardinality 5 for names, 3 for values -var testSimpleData = []*testdata.SimpleMessage{ - {Name: "one", Value: 1}, - {Name: "two", Value: 2}, - {Name: "three", Value: 3}, - {Name: "four", Value: 1}, - {Name: "five", Value: 2}, +var testSimpleData = []*testdata.SimpleMessageProto2{ + {Name: proto.String("one"), Value: proto.Int64(1)}, + {Name: proto.String("two"), Value: proto.Int64(2)}, + {Name: proto.String("three"), Value: proto.Int64(3)}, + {Name: proto.String("four"), Value: proto.Int64(1)}, + {Name: proto.String("five"), Value: proto.Int64(2)}, } func getTestClients(ctx context.Context, t *testing.T, opts ...option.ClientOption) (*Client, *bigquery.Client) { @@ -157,12 +152,12 @@ func TestIntegration_ManagedWriter(t *testing.T) { func testDefaultStream(ctx context.Context, t *testing.T, mwClient *Client, bqClient *bigquery.Client, dataset *bigquery.Dataset) { testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %q: %v", testTable.FullyQualifiedName(), err) } // We'll use a precompiled test proto, but we need it's corresponding descriptorproto representation // to send as the stream's schema. - m := &testdata.SimpleMessage{} + m := &testdata.SimpleMessageProto2{} descriptorProto := protodesc.ToDescriptorProto(m.ProtoReflect().Descriptor()) // setup a new stream. @@ -221,11 +216,11 @@ func testDefaultStream(ctx context.Context, t *testing.T, mwClient *Client, bqCl func testDefaultStreamDynamicJSON(ctx context.Context, t *testing.T, mwClient *Client, bqClient *bigquery.Client, dataset *bigquery.Dataset) { testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %s: %v", testTable.FullyQualifiedName(), err) } - md, descriptorProto := setupDynamicDescriptors(t, testSimpleSchema) + md, descriptorProto := setupDynamicDescriptors(t, testdata.SimpleMessageSchema) ms, err := mwClient.NewManagedStream(ctx, WithDestinationTable(fmt.Sprintf("projects/%s/datasets/%s/tables/%s", testTable.ProjectID, testTable.DatasetID, testTable.TableID)), @@ -276,11 +271,11 @@ func testDefaultStreamDynamicJSON(ctx context.Context, t *testing.T, mwClient *C func testBufferedStream(ctx context.Context, t *testing.T, mwClient *Client, bqClient *bigquery.Client, dataset *bigquery.Dataset) { testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %s: %v", testTable.FullyQualifiedName(), err) } - m := &testdata.SimpleMessage{} + m := &testdata.SimpleMessageProto2{} descriptorProto := protodesc.ToDescriptorProto(m.ProtoReflect().Descriptor()) ms, err := mwClient.NewManagedStream(ctx, @@ -336,11 +331,11 @@ func testBufferedStream(ctx context.Context, t *testing.T, mwClient *Client, bqC func testCommittedStream(ctx context.Context, t *testing.T, mwClient *Client, bqClient *bigquery.Client, dataset *bigquery.Dataset) { testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %s: %v", testTable.FullyQualifiedName(), err) } - m := &testdata.SimpleMessage{} + m := &testdata.SimpleMessageProto2{} descriptorProto := protodesc.ToDescriptorProto(m.ProtoReflect().Descriptor()) // setup a new stream. @@ -375,11 +370,11 @@ func testCommittedStream(ctx context.Context, t *testing.T, mwClient *Client, bq func testPendingStream(ctx context.Context, t *testing.T, mwClient *Client, bqClient *bigquery.Client, dataset *bigquery.Dataset) { testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %s: %v", testTable.FullyQualifiedName(), err) } - m := &testdata.SimpleMessage{} + m := &testdata.SimpleMessageProto2{} descriptorProto := protodesc.ToDescriptorProto(m.ProtoReflect().Descriptor()) ms, err := mwClient.NewManagedStream(ctx, @@ -443,11 +438,11 @@ func testInstrumentation(ctx context.Context, t *testing.T, mwClient *Client, bq } testTable := dataset.Table(tableIDs.New()) - if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testSimpleSchema}); err != nil { + if err := testTable.Create(ctx, &bigquery.TableMetadata{Schema: testdata.SimpleMessageSchema}); err != nil { t.Fatalf("failed to create test table %q: %v", testTable.FullyQualifiedName(), err) } - m := &testdata.SimpleMessage{} + m := &testdata.SimpleMessageProto2{} descriptorProto := protodesc.ToDescriptorProto(m.ProtoReflect().Descriptor()) // setup a new stream. diff --git a/bigquery/storage/managedwriter/testdata/messages.pb.go b/bigquery/storage/managedwriter/testdata/messages.pb.go deleted file mode 100644 index 162b0d9c537..00000000000 --- a/bigquery/storage/managedwriter/testdata/messages.pb.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.25.0 -// protoc v3.10.1 -// source: messages.proto - -package testdata - -import ( - reflect "reflect" - sync "sync" - - proto "github.com/golang/protobuf/proto" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - -// SimpleMessage represents a simple message that transmits a string and int64 value. -type SimpleMessage struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // name is a simple scalar string. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // value is a simple int64 value. - Value int64 `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *SimpleMessage) Reset() { - *x = SimpleMessage{} - if protoimpl.UnsafeEnabled { - mi := &file_messages_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SimpleMessage) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SimpleMessage) ProtoMessage() {} - -func (x *SimpleMessage) ProtoReflect() protoreflect.Message { - mi := &file_messages_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SimpleMessage.ProtoReflect.Descriptor instead. -func (*SimpleMessage) Descriptor() ([]byte, []int) { - return file_messages_proto_rawDescGZIP(), []int{0} -} - -func (x *SimpleMessage) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *SimpleMessage) GetValue() int64 { - if x != nil { - return x.Value - } - return 0 -} - -var File_messages_proto protoreflect.FileDescriptor - -var file_messages_proto_rawDesc = []byte{ - 0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x12, 0x08, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x22, 0x39, 0x0a, 0x0d, 0x53, 0x69, - 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, - 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_messages_proto_rawDescOnce sync.Once - file_messages_proto_rawDescData = file_messages_proto_rawDesc -) - -func file_messages_proto_rawDescGZIP() []byte { - file_messages_proto_rawDescOnce.Do(func() { - file_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_messages_proto_rawDescData) - }) - return file_messages_proto_rawDescData -} - -var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_messages_proto_goTypes = []interface{}{ - (*SimpleMessage)(nil), // 0: testdata.SimpleMessage -} -var file_messages_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_messages_proto_init() } -func file_messages_proto_init() { - if File_messages_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SimpleMessage); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_messages_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_messages_proto_goTypes, - DependencyIndexes: file_messages_proto_depIdxs, - MessageInfos: file_messages_proto_msgTypes, - }.Build() - File_messages_proto = out.File - file_messages_proto_rawDesc = nil - file_messages_proto_goTypes = nil - file_messages_proto_depIdxs = nil -} diff --git a/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go new file mode 100644 index 00000000000..80ea6920756 --- /dev/null +++ b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go @@ -0,0 +1,177 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.10.1 +// source: messages_proto2.proto + +package testdata + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// SimpleMessage represents a simple message that transmits a string and int64 value. +type SimpleMessageProto2 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // name is a simple scalar string. + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // value is a simple int64 value. + Value *int64 `protobuf:"varint,2,opt,name=value" json:"value,omitempty"` +} + +func (x *SimpleMessageProto2) Reset() { + *x = SimpleMessageProto2{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleMessageProto2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleMessageProto2) ProtoMessage() {} + +func (x *SimpleMessageProto2) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto2_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleMessageProto2.ProtoReflect.Descriptor instead. +func (*SimpleMessageProto2) Descriptor() ([]byte, []int) { + return file_messages_proto2_proto_rawDescGZIP(), []int{0} +} + +func (x *SimpleMessageProto2) GetName() string { + if x != nil && x.Name != nil { + return *x.Name + } + return "" +} + +func (x *SimpleMessageProto2) GetValue() int64 { + if x != nil && x.Value != nil { + return *x.Value + } + return 0 +} + +var File_messages_proto2_proto protoreflect.FileDescriptor + +var file_messages_proto2_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, + 0x61, 0x22, 0x3f, 0x0a, 0x13, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, + 0x61, +} + +var ( + file_messages_proto2_proto_rawDescOnce sync.Once + file_messages_proto2_proto_rawDescData = file_messages_proto2_proto_rawDesc +) + +func file_messages_proto2_proto_rawDescGZIP() []byte { + file_messages_proto2_proto_rawDescOnce.Do(func() { + file_messages_proto2_proto_rawDescData = protoimpl.X.CompressGZIP(file_messages_proto2_proto_rawDescData) + }) + return file_messages_proto2_proto_rawDescData +} + +var file_messages_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_messages_proto2_proto_goTypes = []interface{}{ + (*SimpleMessageProto2)(nil), // 0: testdata.SimpleMessageProto2 +} +var file_messages_proto2_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_messages_proto2_proto_init() } +func file_messages_proto2_proto_init() { + if File_messages_proto2_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_messages_proto2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleMessageProto2); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_messages_proto2_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_messages_proto2_proto_goTypes, + DependencyIndexes: file_messages_proto2_proto_depIdxs, + MessageInfos: file_messages_proto2_proto_msgTypes, + }.Build() + File_messages_proto2_proto = out.File + file_messages_proto2_proto_rawDesc = nil + file_messages_proto2_proto_goTypes = nil + file_messages_proto2_proto_depIdxs = nil +} diff --git a/bigquery/storage/managedwriter/testdata/messages_proto2.proto b/bigquery/storage/managedwriter/testdata/messages_proto2.proto new file mode 100644 index 00000000000..ccf1151ddfa --- /dev/null +++ b/bigquery/storage/managedwriter/testdata/messages_proto2.proto @@ -0,0 +1,26 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; +package testdata; +option go_package = "cloud.google.com/go/bigquery/storage/managedwriter/testdata"; + + +// SimpleMessage represents a simple message that transmits a string and int64 value. +message SimpleMessageProto2 { + // name is a simple scalar string. + optional string name = 1; + // value is a simple int64 value. + optional int64 value = 2; +} diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go new file mode 100644 index 00000000000..0618f9a81d7 --- /dev/null +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go @@ -0,0 +1,177 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.10.1 +// source: messages_proto3.proto + +package testdata + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +// SimpleMessage represents a simple message that transmits a string and int64 value. +type SimpleMessageProto3 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // name is a simple scalar string. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // value is a simple int64 value. + Value int64 `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *SimpleMessageProto3) Reset() { + *x = SimpleMessageProto3{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto3_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleMessageProto3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleMessageProto3) ProtoMessage() {} + +func (x *SimpleMessageProto3) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto3_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleMessageProto3.ProtoReflect.Descriptor instead. +func (*SimpleMessageProto3) Descriptor() ([]byte, []int) { + return file_messages_proto3_proto_rawDescGZIP(), []int{0} +} + +func (x *SimpleMessageProto3) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SimpleMessageProto3) GetValue() int64 { + if x != nil { + return x.Value + } + return 0 +} + +var File_messages_proto3_proto protoreflect.FileDescriptor + +var file_messages_proto3_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, + 0x61, 0x22, 0x3f, 0x0a, 0x13, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_messages_proto3_proto_rawDescOnce sync.Once + file_messages_proto3_proto_rawDescData = file_messages_proto3_proto_rawDesc +) + +func file_messages_proto3_proto_rawDescGZIP() []byte { + file_messages_proto3_proto_rawDescOnce.Do(func() { + file_messages_proto3_proto_rawDescData = protoimpl.X.CompressGZIP(file_messages_proto3_proto_rawDescData) + }) + return file_messages_proto3_proto_rawDescData +} + +var file_messages_proto3_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_messages_proto3_proto_goTypes = []interface{}{ + (*SimpleMessageProto3)(nil), // 0: testdata.SimpleMessageProto3 +} +var file_messages_proto3_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_messages_proto3_proto_init() } +func file_messages_proto3_proto_init() { + if File_messages_proto3_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_messages_proto3_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleMessageProto3); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_messages_proto3_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_messages_proto3_proto_goTypes, + DependencyIndexes: file_messages_proto3_proto_depIdxs, + MessageInfos: file_messages_proto3_proto_msgTypes, + }.Build() + File_messages_proto3_proto = out.File + file_messages_proto3_proto_rawDesc = nil + file_messages_proto3_proto_goTypes = nil + file_messages_proto3_proto_depIdxs = nil +} diff --git a/bigquery/storage/managedwriter/testdata/messages.proto b/bigquery/storage/managedwriter/testdata/messages_proto3.proto similarity index 96% rename from bigquery/storage/managedwriter/testdata/messages.proto rename to bigquery/storage/managedwriter/testdata/messages_proto3.proto index 9b77e539ef5..b66cdd565d0 100644 --- a/bigquery/storage/managedwriter/testdata/messages.proto +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.proto @@ -18,7 +18,7 @@ option go_package = "cloud.google.com/go/bigquery/storage/managedwriter/testdata // SimpleMessage represents a simple message that transmits a string and int64 value. -message SimpleMessage { +message SimpleMessageProto3 { // name is a simple scalar string. string name = 1; // value is a simple int64 value. diff --git a/bigquery/storage/managedwriter/testdata/schemas.go b/bigquery/storage/managedwriter/testdata/schemas.go new file mode 100644 index 00000000000..321dedb79ed --- /dev/null +++ b/bigquery/storage/managedwriter/testdata/schemas.go @@ -0,0 +1,64 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +import "cloud.google.com/go/bigquery" + +var ( + SimpleMessageSchema bigquery.Schema = bigquery.Schema{ + {Name: "name", Type: bigquery.StringFieldType}, + {Name: "value", Type: bigquery.IntegerFieldType}, + } + + GithubArchiveSchema bigquery.Schema = bigquery.Schema{ + {Name: "type", Type: bigquery.StringFieldType}, + {Name: "public", Type: bigquery.BooleanFieldType}, + {Name: "payload", Type: bigquery.StringFieldType}, + { + Name: "repo", + Type: bigquery.RecordFieldType, + Schema: bigquery.Schema{ + {Name: "id", Type: bigquery.IntegerFieldType}, + {Name: "name", Type: bigquery.StringFieldType}, + {Name: "url", Type: bigquery.StringFieldType}, + }, + }, + { + Name: "actor", + Type: bigquery.RecordFieldType, + Schema: bigquery.Schema{ + {Name: "id", Type: bigquery.IntegerFieldType}, + {Name: "login", Type: bigquery.StringFieldType}, + {Name: "gravatar_id", Type: bigquery.StringFieldType}, + {Name: "avatar_url", Type: bigquery.StringFieldType}, + {Name: "url", Type: bigquery.StringFieldType}, + }, + }, + { + Name: "org", + Type: bigquery.RecordFieldType, + Schema: bigquery.Schema{ + {Name: "id", Type: bigquery.IntegerFieldType}, + {Name: "login", Type: bigquery.StringFieldType}, + {Name: "gravatar_id", Type: bigquery.StringFieldType}, + {Name: "avatar_url", Type: bigquery.StringFieldType}, + {Name: "url", Type: bigquery.StringFieldType}, + }, + }, + {Name: "created_at", Type: bigquery.TimestampFieldType}, + {Name: "id", Type: bigquery.StringFieldType}, + {Name: "other", Type: bigquery.StringFieldType}, + } +) From 7893ad70832d526c470e0d13d20eb794bed7d73a Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 10 Aug 2021 23:33:40 +0000 Subject: [PATCH 02/11] add githubarchive protos and benchmarking --- .../adapt/protoconversion_test.go | 72 +++- .../testdata/messages_proto2.pb.go | 357 ++++++++++++++++- .../testdata/messages_proto2.proto | 29 ++ .../testdata/messages_proto3.pb.go | 360 +++++++++++++++++- .../testdata/messages_proto3.proto | 29 +- 5 files changed, 813 insertions(+), 34 deletions(-) diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index 14d879bea3b..deec367e74e 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -350,9 +350,6 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { }{ { name: "SimpleMessageProto2", - in: bigquery.Schema{ - {Name: "field", Type: bigquery.StringFieldType}, - }, setterF: func() protoreflect.ProtoMessage { return &testdata.SimpleMessageProto2{ Name: proto.String(fmt.Sprintf("test-%d", time.Now().UnixNano())), @@ -362,9 +359,6 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { }, { name: "SimpleMessageProto3", - in: bigquery.Schema{ - {Name: "field", Type: bigquery.StringFieldType}, - }, setterF: func() protoreflect.ProtoMessage { return &testdata.SimpleMessageProto3{ Name: fmt.Sprintf("test-%d", time.Now().UnixNano()), @@ -372,6 +366,72 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { } }, }, + { + name: "GithubArchiveProto2", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto2{ + Type: proto.String("SomeEvent"), + Public: proto.Bool(nowNano%2 == 0), + Payload: proto.String(fmt.Sprintf("stuff %d", nowNano)), + Repo: &testdata.GithubArchiveRepoProto2{ + Id: proto.Int64(nowNano), + Name: proto.String("staticname"), + Url: proto.String(fmt.Sprintf("foo.com/%d", nowNano)), + }, + Actor: &testdata.GithubArchiveEntityProto2{ + Id: proto.Int64(nowNano % 1000), + Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), + GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), + AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + }, + Org: &testdata.GithubArchiveEntityProto2{ + Id: proto.Int64(nowNano % 1000), + Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), + GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), + AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + }, + CreatedAt: proto.Int64(nowNano), + Id: proto.String(fmt.Sprintf("id%d", nowNano)), + Other: proto.String("other"), + } + }, + }, + { + name: "GithubArchiveProto3", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto3{ + Type: "SomeEvent", + Public: nowNano%2 == 0, + Payload: fmt.Sprintf("stuff %d", nowNano), + Repo: &testdata.GithubArchiveRepoProto3{ + Id: nowNano, + Name: "staticname", + Url: fmt.Sprintf("foo.com/%d", nowNano), + }, + Actor: &testdata.GithubArchiveEntityProto3{ + Id: nowNano % 1000, + Login: fmt.Sprintf("login-%d", nowNano%1000), + GravatarId: fmt.Sprintf("grav-%d", nowNano%1000000), + AvatarUrl: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + Url: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + }, + Org: &testdata.GithubArchiveEntityProto3{ + Id: nowNano % 1000, + Login: fmt.Sprintf("login-%d", nowNano%1000), + GravatarId: fmt.Sprintf("grav-%d", nowNano%1000000), + AvatarUrl: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + Url: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + }, + CreatedAt: nowNano, + Id: fmt.Sprintf("id%d", nowNano), + Other: "other", + } + }, + }, } { b.Run(bm.name, func(b *testing.B) { var totalBytes int64 diff --git a/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go index 80ea6920756..ccd52a67250 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go @@ -97,6 +97,260 @@ func (x *SimpleMessageProto2) GetValue() int64 { return 0 } +type GithubArchiveEntityProto2 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + Login *string `protobuf:"bytes,2,opt,name=login" json:"login,omitempty"` + GravatarId *string `protobuf:"bytes,3,opt,name=gravatar_id,json=gravatarId" json:"gravatar_id,omitempty"` + AvatarUrl *string `protobuf:"bytes,4,opt,name=avatar_url,json=avatarUrl" json:"avatar_url,omitempty"` + Url *string `protobuf:"bytes,5,opt,name=url" json:"url,omitempty"` +} + +func (x *GithubArchiveEntityProto2) Reset() { + *x = GithubArchiveEntityProto2{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto2_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveEntityProto2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveEntityProto2) ProtoMessage() {} + +func (x *GithubArchiveEntityProto2) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto2_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveEntityProto2.ProtoReflect.Descriptor instead. +func (*GithubArchiveEntityProto2) Descriptor() ([]byte, []int) { + return file_messages_proto2_proto_rawDescGZIP(), []int{1} +} + +func (x *GithubArchiveEntityProto2) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +func (x *GithubArchiveEntityProto2) GetLogin() string { + if x != nil && x.Login != nil { + return *x.Login + } + return "" +} + +func (x *GithubArchiveEntityProto2) GetGravatarId() string { + if x != nil && x.GravatarId != nil { + return *x.GravatarId + } + return "" +} + +func (x *GithubArchiveEntityProto2) GetAvatarUrl() string { + if x != nil && x.AvatarUrl != nil { + return *x.AvatarUrl + } + return "" +} + +func (x *GithubArchiveEntityProto2) GetUrl() string { + if x != nil && x.Url != nil { + return *x.Url + } + return "" +} + +type GithubArchiveRepoProto2 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + Name *string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` + Url *string `protobuf:"bytes,3,opt,name=url" json:"url,omitempty"` +} + +func (x *GithubArchiveRepoProto2) Reset() { + *x = GithubArchiveRepoProto2{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto2_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveRepoProto2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveRepoProto2) ProtoMessage() {} + +func (x *GithubArchiveRepoProto2) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto2_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveRepoProto2.ProtoReflect.Descriptor instead. +func (*GithubArchiveRepoProto2) Descriptor() ([]byte, []int) { + return file_messages_proto2_proto_rawDescGZIP(), []int{2} +} + +func (x *GithubArchiveRepoProto2) GetId() int64 { + if x != nil && x.Id != nil { + return *x.Id + } + return 0 +} + +func (x *GithubArchiveRepoProto2) GetName() string { + if x != nil && x.Name != nil { + return *x.Name + } + return "" +} + +func (x *GithubArchiveRepoProto2) GetUrl() string { + if x != nil && x.Url != nil { + return *x.Url + } + return "" +} + +// GithubArchiveMessageProto2 is the proto2 version of github archive row. +type GithubArchiveMessageProto2 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type *string `protobuf:"bytes,1,opt,name=type" json:"type,omitempty"` + Public *bool `protobuf:"varint,2,opt,name=public" json:"public,omitempty"` + Payload *string `protobuf:"bytes,3,opt,name=payload" json:"payload,omitempty"` + Repo *GithubArchiveRepoProto2 `protobuf:"bytes,4,opt,name=repo" json:"repo,omitempty"` + Actor *GithubArchiveEntityProto2 `protobuf:"bytes,5,opt,name=actor" json:"actor,omitempty"` + Org *GithubArchiveEntityProto2 `protobuf:"bytes,6,opt,name=org" json:"org,omitempty"` + CreatedAt *int64 `protobuf:"varint,7,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` + Id *string `protobuf:"bytes,8,opt,name=id" json:"id,omitempty"` + Other *string `protobuf:"bytes,9,opt,name=other" json:"other,omitempty"` +} + +func (x *GithubArchiveMessageProto2) Reset() { + *x = GithubArchiveMessageProto2{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto2_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveMessageProto2) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveMessageProto2) ProtoMessage() {} + +func (x *GithubArchiveMessageProto2) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto2_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveMessageProto2.ProtoReflect.Descriptor instead. +func (*GithubArchiveMessageProto2) Descriptor() ([]byte, []int) { + return file_messages_proto2_proto_rawDescGZIP(), []int{3} +} + +func (x *GithubArchiveMessageProto2) GetType() string { + if x != nil && x.Type != nil { + return *x.Type + } + return "" +} + +func (x *GithubArchiveMessageProto2) GetPublic() bool { + if x != nil && x.Public != nil { + return *x.Public + } + return false +} + +func (x *GithubArchiveMessageProto2) GetPayload() string { + if x != nil && x.Payload != nil { + return *x.Payload + } + return "" +} + +func (x *GithubArchiveMessageProto2) GetRepo() *GithubArchiveRepoProto2 { + if x != nil { + return x.Repo + } + return nil +} + +func (x *GithubArchiveMessageProto2) GetActor() *GithubArchiveEntityProto2 { + if x != nil { + return x.Actor + } + return nil +} + +func (x *GithubArchiveMessageProto2) GetOrg() *GithubArchiveEntityProto2 { + if x != nil { + return x.Org + } + return nil +} + +func (x *GithubArchiveMessageProto2) GetCreatedAt() int64 { + if x != nil && x.CreatedAt != nil { + return *x.CreatedAt + } + return 0 +} + +func (x *GithubArchiveMessageProto2) GetId() string { + if x != nil && x.Id != nil { + return *x.Id + } + return "" +} + +func (x *GithubArchiveMessageProto2) GetOther() string { + if x != nil && x.Other != nil { + return *x.Other + } + return "" +} + var File_messages_proto2_proto protoreflect.FileDescriptor var file_messages_proto2_proto_rawDesc = []byte{ @@ -106,11 +360,46 @@ var file_messages_proto2_proto_rawDesc = []byte{ 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, - 0x61, + 0x75, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x19, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x72, 0x61, 0x76, 0x61, 0x74, + 0x61, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x72, 0x61, + 0x76, 0x61, 0x74, 0x61, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x76, 0x61, 0x74, 0x61, + 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x76, 0x61, + 0x74, 0x61, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x4f, 0x0a, 0x17, 0x47, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x32, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xd0, 0x02, 0x0a, 0x1a, 0x47, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x35, + 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, + 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x52, + 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x52, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, + 0x12, 0x35, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, + 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x32, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x42, 0x3d, 0x5a, 0x3b, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, } var ( @@ -125,16 +414,22 @@ func file_messages_proto2_proto_rawDescGZIP() []byte { return file_messages_proto2_proto_rawDescData } -var file_messages_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_messages_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_messages_proto2_proto_goTypes = []interface{}{ - (*SimpleMessageProto2)(nil), // 0: testdata.SimpleMessageProto2 + (*SimpleMessageProto2)(nil), // 0: testdata.SimpleMessageProto2 + (*GithubArchiveEntityProto2)(nil), // 1: testdata.GithubArchiveEntityProto2 + (*GithubArchiveRepoProto2)(nil), // 2: testdata.GithubArchiveRepoProto2 + (*GithubArchiveMessageProto2)(nil), // 3: testdata.GithubArchiveMessageProto2 } var file_messages_proto2_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: testdata.GithubArchiveMessageProto2.repo:type_name -> testdata.GithubArchiveRepoProto2 + 1, // 1: testdata.GithubArchiveMessageProto2.actor:type_name -> testdata.GithubArchiveEntityProto2 + 1, // 2: testdata.GithubArchiveMessageProto2.org:type_name -> testdata.GithubArchiveEntityProto2 + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_messages_proto2_proto_init() } @@ -155,6 +450,42 @@ func file_messages_proto2_proto_init() { return nil } } + file_messages_proto2_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveEntityProto2); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto2_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveRepoProto2); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto2_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveMessageProto2); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -162,7 +493,7 @@ func file_messages_proto2_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_messages_proto2_proto_rawDesc, NumEnums: 0, - NumMessages: 1, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/bigquery/storage/managedwriter/testdata/messages_proto2.proto b/bigquery/storage/managedwriter/testdata/messages_proto2.proto index ccf1151ddfa..fdb2fb55e47 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto2.proto +++ b/bigquery/storage/managedwriter/testdata/messages_proto2.proto @@ -24,3 +24,32 @@ message SimpleMessageProto2 { // value is a simple int64 value. optional int64 value = 2; } + + + +message GithubArchiveEntityProto2 { + optional int64 id = 1; + optional string login = 2; + optional string gravatar_id = 3; + optional string avatar_url = 4; + optional string url = 5; +} + +message GithubArchiveRepoProto2 { + optional int64 id = 1; + optional string name = 2; + optional string url = 3; +} + +// GithubArchiveMessageProto2 is the proto2 version of github archive row. +message GithubArchiveMessageProto2 { + optional string type = 1; + optional bool public = 2; + optional string payload = 3; + optional GithubArchiveRepoProto2 repo = 4; + optional GithubArchiveEntityProto2 actor = 5; + optional GithubArchiveEntityProto2 org = 6; + optional int64 created_at = 7; + optional string id = 8; + optional string other = 9; +} \ No newline at end of file diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go index 0618f9a81d7..e8b429210c3 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go @@ -39,7 +39,7 @@ const ( // of the legacy proto package is being used. const _ = proto.ProtoPackageIsVersion4 -// SimpleMessage represents a simple message that transmits a string and int64 value. +// SimpleMessageProto3 represents a simple message that transmits a string and int64 value. type SimpleMessageProto3 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -97,6 +97,260 @@ func (x *SimpleMessageProto3) GetValue() int64 { return 0 } +type GithubArchiveEntityProto3 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Login string `protobuf:"bytes,2,opt,name=login,proto3" json:"login,omitempty"` + GravatarId string `protobuf:"bytes,3,opt,name=gravatar_id,json=gravatarId,proto3" json:"gravatar_id,omitempty"` + AvatarUrl string `protobuf:"bytes,4,opt,name=avatar_url,json=avatarUrl,proto3" json:"avatar_url,omitempty"` + Url string `protobuf:"bytes,5,opt,name=url,proto3" json:"url,omitempty"` +} + +func (x *GithubArchiveEntityProto3) Reset() { + *x = GithubArchiveEntityProto3{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto3_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveEntityProto3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveEntityProto3) ProtoMessage() {} + +func (x *GithubArchiveEntityProto3) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto3_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveEntityProto3.ProtoReflect.Descriptor instead. +func (*GithubArchiveEntityProto3) Descriptor() ([]byte, []int) { + return file_messages_proto3_proto_rawDescGZIP(), []int{1} +} + +func (x *GithubArchiveEntityProto3) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *GithubArchiveEntityProto3) GetLogin() string { + if x != nil { + return x.Login + } + return "" +} + +func (x *GithubArchiveEntityProto3) GetGravatarId() string { + if x != nil { + return x.GravatarId + } + return "" +} + +func (x *GithubArchiveEntityProto3) GetAvatarUrl() string { + if x != nil { + return x.AvatarUrl + } + return "" +} + +func (x *GithubArchiveEntityProto3) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +type GithubArchiveRepoProto3 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Url string `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` +} + +func (x *GithubArchiveRepoProto3) Reset() { + *x = GithubArchiveRepoProto3{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto3_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveRepoProto3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveRepoProto3) ProtoMessage() {} + +func (x *GithubArchiveRepoProto3) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto3_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveRepoProto3.ProtoReflect.Descriptor instead. +func (*GithubArchiveRepoProto3) Descriptor() ([]byte, []int) { + return file_messages_proto3_proto_rawDescGZIP(), []int{2} +} + +func (x *GithubArchiveRepoProto3) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *GithubArchiveRepoProto3) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *GithubArchiveRepoProto3) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +// GithubArchiveMessageProto3 is the proto3 version of github archive row. +type GithubArchiveMessageProto3 struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Public bool `protobuf:"varint,2,opt,name=public,proto3" json:"public,omitempty"` + Payload string `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + Repo *GithubArchiveRepoProto3 `protobuf:"bytes,4,opt,name=repo,proto3" json:"repo,omitempty"` + Actor *GithubArchiveEntityProto3 `protobuf:"bytes,5,opt,name=actor,proto3" json:"actor,omitempty"` + Org *GithubArchiveEntityProto3 `protobuf:"bytes,6,opt,name=org,proto3" json:"org,omitempty"` + CreatedAt int64 `protobuf:"varint,7,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Id string `protobuf:"bytes,8,opt,name=id,proto3" json:"id,omitempty"` + Other string `protobuf:"bytes,9,opt,name=other,proto3" json:"other,omitempty"` +} + +func (x *GithubArchiveMessageProto3) Reset() { + *x = GithubArchiveMessageProto3{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto3_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GithubArchiveMessageProto3) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GithubArchiveMessageProto3) ProtoMessage() {} + +func (x *GithubArchiveMessageProto3) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto3_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GithubArchiveMessageProto3.ProtoReflect.Descriptor instead. +func (*GithubArchiveMessageProto3) Descriptor() ([]byte, []int) { + return file_messages_proto3_proto_rawDescGZIP(), []int{3} +} + +func (x *GithubArchiveMessageProto3) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *GithubArchiveMessageProto3) GetPublic() bool { + if x != nil { + return x.Public + } + return false +} + +func (x *GithubArchiveMessageProto3) GetPayload() string { + if x != nil { + return x.Payload + } + return "" +} + +func (x *GithubArchiveMessageProto3) GetRepo() *GithubArchiveRepoProto3 { + if x != nil { + return x.Repo + } + return nil +} + +func (x *GithubArchiveMessageProto3) GetActor() *GithubArchiveEntityProto3 { + if x != nil { + return x.Actor + } + return nil +} + +func (x *GithubArchiveMessageProto3) GetOrg() *GithubArchiveEntityProto3 { + if x != nil { + return x.Org + } + return nil +} + +func (x *GithubArchiveMessageProto3) GetCreatedAt() int64 { + if x != nil { + return x.CreatedAt + } + return 0 +} + +func (x *GithubArchiveMessageProto3) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *GithubArchiveMessageProto3) GetOther() string { + if x != nil { + return x.Other + } + return "" +} + var File_messages_proto3_proto protoreflect.FileDescriptor var file_messages_proto3_proto_rawDesc = []byte{ @@ -106,11 +360,47 @@ var file_messages_proto3_proto_rawDesc = []byte{ 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, - 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x19, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x72, 0x61, 0x76, 0x61, 0x74, + 0x61, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x72, 0x61, + 0x76, 0x61, 0x74, 0x61, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x76, 0x61, 0x74, 0x61, + 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x76, 0x61, + 0x74, 0x61, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x4f, 0x0a, 0x17, 0x47, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x33, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xd0, 0x02, 0x0a, 0x1a, 0x47, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x75, + 0x62, 0x6c, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x35, + 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, + 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, + 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, + 0x12, 0x35, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, + 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x33, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x42, 0x3d, 0x5a, 0x3b, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -125,16 +415,22 @@ func file_messages_proto3_proto_rawDescGZIP() []byte { return file_messages_proto3_proto_rawDescData } -var file_messages_proto3_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_messages_proto3_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_messages_proto3_proto_goTypes = []interface{}{ - (*SimpleMessageProto3)(nil), // 0: testdata.SimpleMessageProto3 + (*SimpleMessageProto3)(nil), // 0: testdata.SimpleMessageProto3 + (*GithubArchiveEntityProto3)(nil), // 1: testdata.GithubArchiveEntityProto3 + (*GithubArchiveRepoProto3)(nil), // 2: testdata.GithubArchiveRepoProto3 + (*GithubArchiveMessageProto3)(nil), // 3: testdata.GithubArchiveMessageProto3 } var file_messages_proto3_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: testdata.GithubArchiveMessageProto3.repo:type_name -> testdata.GithubArchiveRepoProto3 + 1, // 1: testdata.GithubArchiveMessageProto3.actor:type_name -> testdata.GithubArchiveEntityProto3 + 1, // 2: testdata.GithubArchiveMessageProto3.org:type_name -> testdata.GithubArchiveEntityProto3 + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_messages_proto3_proto_init() } @@ -155,6 +451,42 @@ func file_messages_proto3_proto_init() { return nil } } + file_messages_proto3_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveEntityProto3); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto3_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveRepoProto3); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto3_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GithubArchiveMessageProto3); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -162,7 +494,7 @@ func file_messages_proto3_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_messages_proto3_proto_rawDesc, NumEnums: 0, - NumMessages: 1, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.proto b/bigquery/storage/managedwriter/testdata/messages_proto3.proto index b66cdd565d0..048f675032b 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.proto +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.proto @@ -17,7 +17,7 @@ package testdata; option go_package = "cloud.google.com/go/bigquery/storage/managedwriter/testdata"; -// SimpleMessage represents a simple message that transmits a string and int64 value. +// SimpleMessageProto3 represents a simple message that transmits a string and int64 value. message SimpleMessageProto3 { // name is a simple scalar string. string name = 1; @@ -26,3 +26,30 @@ message SimpleMessageProto3 { } +message GithubArchiveEntityProto3 { + int64 id = 1; + string login = 2; + string gravatar_id = 3; + string avatar_url = 4; + string url = 5; +} + +message GithubArchiveRepoProto3 { + int64 id = 1; + string name = 2; + string url = 3; +} + +// GithubArchiveMessageProto3 is the proto3 version of github archive row. +message GithubArchiveMessageProto3 { + string type = 1; + bool public = 2; + string payload = 3; + GithubArchiveRepoProto3 repo = 4; + GithubArchiveEntityProto3 actor = 5; + GithubArchiveEntityProto3 org = 6; + int64 created_at = 7; + string id = 8; + string other = 9; +} + From d8c26a43860d8cab9ceea6b2dbb53c02296f4242 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 10 Aug 2021 23:37:37 +0000 Subject: [PATCH 03/11] goimports on proto outputs --- .../storage/managedwriter/testdata/messages_proto2.pb.go | 5 +++-- .../storage/managedwriter/testdata/messages_proto3.pb.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go index ccd52a67250..d57ac20be31 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto2.pb.go @@ -21,11 +21,12 @@ package testdata import ( + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go index e8b429210c3..67ecfe4712e 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go @@ -21,11 +21,12 @@ package testdata import ( + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( From 4f8716c47953b1e890bd2040f2d36af53109a698 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 10 Aug 2021 23:56:11 +0000 Subject: [PATCH 04/11] switch proto3 definitions to wrapperspb --- .../adapt/protoconversion_test.go | 41 +-- .../testdata/messages_proto3.pb.go | 245 +++++++++++------- .../testdata/messages_proto3.proto | 31 +-- .../storage/managedwriter/testdata/schemas.go | 2 +- 4 files changed, 184 insertions(+), 135 deletions(-) diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index deec367e74e..d330901f38e 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -32,6 +32,7 @@ import ( "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/dynamicpb" + "google.golang.org/protobuf/types/known/wrapperspb" ) func TestSchemaToProtoConversion(t *testing.T) { @@ -362,7 +363,7 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { setterF: func() protoreflect.ProtoMessage { return &testdata.SimpleMessageProto3{ Name: fmt.Sprintf("test-%d", time.Now().UnixNano()), - Value: time.Now().UnixNano(), + Value: &wrapperspb.Int64Value{Value: time.Now().UnixNano()}, } }, }, @@ -404,31 +405,31 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { setterF: func() protoreflect.ProtoMessage { nowNano := time.Now().UnixNano() return &testdata.GithubArchiveMessageProto3{ - Type: "SomeEvent", - Public: nowNano%2 == 0, - Payload: fmt.Sprintf("stuff %d", nowNano), + Type: &wrapperspb.StringValue{Value: "SomeEvent"}, + Public: &wrapperspb.BoolValue{Value: nowNano%2 == 0}, + Payload: &wrapperspb.StringValue{Value: fmt.Sprintf("stuff %d", nowNano)}, Repo: &testdata.GithubArchiveRepoProto3{ - Id: nowNano, - Name: "staticname", - Url: fmt.Sprintf("foo.com/%d", nowNano), + Id: &wrapperspb.Int64Value{Value: nowNano}, + Name: &wrapperspb.StringValue{Value: "staticname"}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("foo.com/%d", nowNano)}, }, Actor: &testdata.GithubArchiveEntityProto3{ - Id: nowNano % 1000, - Login: fmt.Sprintf("login-%d", nowNano%1000), - GravatarId: fmt.Sprintf("grav-%d", nowNano%1000000), - AvatarUrl: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), - Url: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, + Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, + GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, + AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, }, Org: &testdata.GithubArchiveEntityProto3{ - Id: nowNano % 1000, - Login: fmt.Sprintf("login-%d", nowNano%1000), - GravatarId: fmt.Sprintf("grav-%d", nowNano%1000000), - AvatarUrl: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), - Url: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000), + Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, + Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, + GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, + AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, }, - CreatedAt: nowNano, - Id: fmt.Sprintf("id%d", nowNano), - Other: "other", + CreatedAt: &wrapperspb.Int64Value{Value: nowNano}, + Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, + Other: &wrapperspb.StringValue{Value: "other"}, } }, }, diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go index 67ecfe4712e..e119a759161 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go @@ -21,12 +21,12 @@ package testdata import ( - reflect "reflect" - sync "sync" - proto "github.com/golang/protobuf/proto" + wrappers "github.com/golang/protobuf/ptypes/wrappers" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) const ( @@ -49,7 +49,7 @@ type SimpleMessageProto3 struct { // name is a simple scalar string. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // value is a simple int64 value. - Value int64 `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"` + Value *wrappers.Int64Value `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` } func (x *SimpleMessageProto3) Reset() { @@ -91,11 +91,11 @@ func (x *SimpleMessageProto3) GetName() string { return "" } -func (x *SimpleMessageProto3) GetValue() int64 { +func (x *SimpleMessageProto3) GetValue() *wrappers.Int64Value { if x != nil { return x.Value } - return 0 + return nil } type GithubArchiveEntityProto3 struct { @@ -103,11 +103,11 @@ type GithubArchiveEntityProto3 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Login string `protobuf:"bytes,2,opt,name=login,proto3" json:"login,omitempty"` - GravatarId string `protobuf:"bytes,3,opt,name=gravatar_id,json=gravatarId,proto3" json:"gravatar_id,omitempty"` - AvatarUrl string `protobuf:"bytes,4,opt,name=avatar_url,json=avatarUrl,proto3" json:"avatar_url,omitempty"` - Url string `protobuf:"bytes,5,opt,name=url,proto3" json:"url,omitempty"` + Id *wrappers.Int64Value `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Login *wrappers.StringValue `protobuf:"bytes,2,opt,name=login,proto3" json:"login,omitempty"` + GravatarId *wrappers.StringValue `protobuf:"bytes,3,opt,name=gravatar_id,json=gravatarId,proto3" json:"gravatar_id,omitempty"` + AvatarUrl *wrappers.StringValue `protobuf:"bytes,4,opt,name=avatar_url,json=avatarUrl,proto3" json:"avatar_url,omitempty"` + Url *wrappers.StringValue `protobuf:"bytes,5,opt,name=url,proto3" json:"url,omitempty"` } func (x *GithubArchiveEntityProto3) Reset() { @@ -142,39 +142,39 @@ func (*GithubArchiveEntityProto3) Descriptor() ([]byte, []int) { return file_messages_proto3_proto_rawDescGZIP(), []int{1} } -func (x *GithubArchiveEntityProto3) GetId() int64 { +func (x *GithubArchiveEntityProto3) GetId() *wrappers.Int64Value { if x != nil { return x.Id } - return 0 + return nil } -func (x *GithubArchiveEntityProto3) GetLogin() string { +func (x *GithubArchiveEntityProto3) GetLogin() *wrappers.StringValue { if x != nil { return x.Login } - return "" + return nil } -func (x *GithubArchiveEntityProto3) GetGravatarId() string { +func (x *GithubArchiveEntityProto3) GetGravatarId() *wrappers.StringValue { if x != nil { return x.GravatarId } - return "" + return nil } -func (x *GithubArchiveEntityProto3) GetAvatarUrl() string { +func (x *GithubArchiveEntityProto3) GetAvatarUrl() *wrappers.StringValue { if x != nil { return x.AvatarUrl } - return "" + return nil } -func (x *GithubArchiveEntityProto3) GetUrl() string { +func (x *GithubArchiveEntityProto3) GetUrl() *wrappers.StringValue { if x != nil { return x.Url } - return "" + return nil } type GithubArchiveRepoProto3 struct { @@ -182,9 +182,9 @@ type GithubArchiveRepoProto3 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Url string `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` + Id *wrappers.Int64Value `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name *wrappers.StringValue `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Url *wrappers.StringValue `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` } func (x *GithubArchiveRepoProto3) Reset() { @@ -219,25 +219,25 @@ func (*GithubArchiveRepoProto3) Descriptor() ([]byte, []int) { return file_messages_proto3_proto_rawDescGZIP(), []int{2} } -func (x *GithubArchiveRepoProto3) GetId() int64 { +func (x *GithubArchiveRepoProto3) GetId() *wrappers.Int64Value { if x != nil { return x.Id } - return 0 + return nil } -func (x *GithubArchiveRepoProto3) GetName() string { +func (x *GithubArchiveRepoProto3) GetName() *wrappers.StringValue { if x != nil { return x.Name } - return "" + return nil } -func (x *GithubArchiveRepoProto3) GetUrl() string { +func (x *GithubArchiveRepoProto3) GetUrl() *wrappers.StringValue { if x != nil { return x.Url } - return "" + return nil } // GithubArchiveMessageProto3 is the proto3 version of github archive row. @@ -246,15 +246,15 @@ type GithubArchiveMessageProto3 struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` - Public bool `protobuf:"varint,2,opt,name=public,proto3" json:"public,omitempty"` - Payload string `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + Type *wrappers.StringValue `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + Public *wrappers.BoolValue `protobuf:"bytes,2,opt,name=public,proto3" json:"public,omitempty"` + Payload *wrappers.StringValue `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` Repo *GithubArchiveRepoProto3 `protobuf:"bytes,4,opt,name=repo,proto3" json:"repo,omitempty"` Actor *GithubArchiveEntityProto3 `protobuf:"bytes,5,opt,name=actor,proto3" json:"actor,omitempty"` Org *GithubArchiveEntityProto3 `protobuf:"bytes,6,opt,name=org,proto3" json:"org,omitempty"` - CreatedAt int64 `protobuf:"varint,7,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` - Id string `protobuf:"bytes,8,opt,name=id,proto3" json:"id,omitempty"` - Other string `protobuf:"bytes,9,opt,name=other,proto3" json:"other,omitempty"` + CreatedAt *wrappers.Int64Value `protobuf:"bytes,7,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Id *wrappers.StringValue `protobuf:"bytes,8,opt,name=id,proto3" json:"id,omitempty"` + Other *wrappers.StringValue `protobuf:"bytes,9,opt,name=other,proto3" json:"other,omitempty"` } func (x *GithubArchiveMessageProto3) Reset() { @@ -289,25 +289,25 @@ func (*GithubArchiveMessageProto3) Descriptor() ([]byte, []int) { return file_messages_proto3_proto_rawDescGZIP(), []int{3} } -func (x *GithubArchiveMessageProto3) GetType() string { +func (x *GithubArchiveMessageProto3) GetType() *wrappers.StringValue { if x != nil { return x.Type } - return "" + return nil } -func (x *GithubArchiveMessageProto3) GetPublic() bool { +func (x *GithubArchiveMessageProto3) GetPublic() *wrappers.BoolValue { if x != nil { return x.Public } - return false + return nil } -func (x *GithubArchiveMessageProto3) GetPayload() string { +func (x *GithubArchiveMessageProto3) GetPayload() *wrappers.StringValue { if x != nil { return x.Payload } - return "" + return nil } func (x *GithubArchiveMessageProto3) GetRepo() *GithubArchiveRepoProto3 { @@ -331,25 +331,25 @@ func (x *GithubArchiveMessageProto3) GetOrg() *GithubArchiveEntityProto3 { return nil } -func (x *GithubArchiveMessageProto3) GetCreatedAt() int64 { +func (x *GithubArchiveMessageProto3) GetCreatedAt() *wrappers.Int64Value { if x != nil { return x.CreatedAt } - return 0 + return nil } -func (x *GithubArchiveMessageProto3) GetId() string { +func (x *GithubArchiveMessageProto3) GetId() *wrappers.StringValue { if x != nil { return x.Id } - return "" + return nil } -func (x *GithubArchiveMessageProto3) GetOther() string { +func (x *GithubArchiveMessageProto3) GetOther() *wrappers.StringValue { if x != nil { return x.Other } - return "" + return nil } var File_messages_proto3_proto protoreflect.FileDescriptor @@ -357,51 +357,80 @@ var File_messages_proto3_proto protoreflect.FileDescriptor var file_messages_proto3_proto_rawDesc = []byte{ 0x0a, 0x15, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, - 0x61, 0x22, 0x3f, 0x0a, 0x13, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x61, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x5c, 0x0a, 0x13, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x19, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, - 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x72, 0x61, 0x76, 0x61, 0x74, - 0x61, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x72, 0x61, - 0x76, 0x61, 0x74, 0x61, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x76, 0x61, 0x74, 0x61, - 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x76, 0x61, - 0x74, 0x61, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x4f, 0x0a, 0x17, 0x47, 0x69, 0x74, 0x68, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, + 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, + 0xa8, 0x02, 0x0a, 0x19, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x2b, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, + 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x05, 0x6c, 0x6f, + 0x67, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x3d, + 0x0a, 0x0b, 0x67, 0x72, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x52, 0x0a, 0x67, 0x72, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x49, 0x64, 0x12, 0x3b, 0x0a, + 0x0a, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x09, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x55, 0x72, 0x6c, 0x12, 0x2e, 0x0a, 0x03, 0x75, 0x72, + 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xa8, 0x01, 0x0a, 0x17, 0x47, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x2b, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x30, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x81, 0x04, 0x0a, 0x1a, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x30, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x06, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x12, 0x36, 0x0a, 0x07, 0x70, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x12, 0x35, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x21, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x33, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xd0, 0x02, 0x0a, 0x1a, 0x47, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x75, - 0x62, 0x6c, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x35, - 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, - 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, - 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, 0x05, 0x61, 0x63, 0x74, 0x6f, 0x72, - 0x12, 0x35, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, - 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x33, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x42, 0x3d, 0x5a, 0x3b, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x67, 0x6f, 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, - 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x74, 0x6f, 0x33, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x63, 0x74, + 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, 0x05, 0x61, + 0x63, 0x74, 0x6f, 0x72, 0x12, 0x35, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x52, 0x03, 0x6f, 0x72, 0x67, 0x12, 0x3a, 0x0a, 0x0a, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x09, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x2c, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x42, 0x3d, 0x5a, 0x3b, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, + 0x2f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x77, 0x72, 0x69, 0x74, 0x65, 0x72, 0x2f, + 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -422,16 +451,34 @@ var file_messages_proto3_proto_goTypes = []interface{}{ (*GithubArchiveEntityProto3)(nil), // 1: testdata.GithubArchiveEntityProto3 (*GithubArchiveRepoProto3)(nil), // 2: testdata.GithubArchiveRepoProto3 (*GithubArchiveMessageProto3)(nil), // 3: testdata.GithubArchiveMessageProto3 + (*wrappers.Int64Value)(nil), // 4: google.protobuf.Int64Value + (*wrappers.StringValue)(nil), // 5: google.protobuf.StringValue + (*wrappers.BoolValue)(nil), // 6: google.protobuf.BoolValue } var file_messages_proto3_proto_depIdxs = []int32{ - 2, // 0: testdata.GithubArchiveMessageProto3.repo:type_name -> testdata.GithubArchiveRepoProto3 - 1, // 1: testdata.GithubArchiveMessageProto3.actor:type_name -> testdata.GithubArchiveEntityProto3 - 1, // 2: testdata.GithubArchiveMessageProto3.org:type_name -> testdata.GithubArchiveEntityProto3 - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 4, // 0: testdata.SimpleMessageProto3.value:type_name -> google.protobuf.Int64Value + 4, // 1: testdata.GithubArchiveEntityProto3.id:type_name -> google.protobuf.Int64Value + 5, // 2: testdata.GithubArchiveEntityProto3.login:type_name -> google.protobuf.StringValue + 5, // 3: testdata.GithubArchiveEntityProto3.gravatar_id:type_name -> google.protobuf.StringValue + 5, // 4: testdata.GithubArchiveEntityProto3.avatar_url:type_name -> google.protobuf.StringValue + 5, // 5: testdata.GithubArchiveEntityProto3.url:type_name -> google.protobuf.StringValue + 4, // 6: testdata.GithubArchiveRepoProto3.id:type_name -> google.protobuf.Int64Value + 5, // 7: testdata.GithubArchiveRepoProto3.name:type_name -> google.protobuf.StringValue + 5, // 8: testdata.GithubArchiveRepoProto3.url:type_name -> google.protobuf.StringValue + 5, // 9: testdata.GithubArchiveMessageProto3.type:type_name -> google.protobuf.StringValue + 6, // 10: testdata.GithubArchiveMessageProto3.public:type_name -> google.protobuf.BoolValue + 5, // 11: testdata.GithubArchiveMessageProto3.payload:type_name -> google.protobuf.StringValue + 2, // 12: testdata.GithubArchiveMessageProto3.repo:type_name -> testdata.GithubArchiveRepoProto3 + 1, // 13: testdata.GithubArchiveMessageProto3.actor:type_name -> testdata.GithubArchiveEntityProto3 + 1, // 14: testdata.GithubArchiveMessageProto3.org:type_name -> testdata.GithubArchiveEntityProto3 + 4, // 15: testdata.GithubArchiveMessageProto3.created_at:type_name -> google.protobuf.Int64Value + 5, // 16: testdata.GithubArchiveMessageProto3.id:type_name -> google.protobuf.StringValue + 5, // 17: testdata.GithubArchiveMessageProto3.other:type_name -> google.protobuf.StringValue + 18, // [18:18] is the sub-list for method output_type + 18, // [18:18] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_messages_proto3_proto_init() } diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.proto b/bigquery/storage/managedwriter/testdata/messages_proto3.proto index 048f675032b..2d298362449 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.proto +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.proto @@ -16,40 +16,41 @@ syntax = "proto3"; package testdata; option go_package = "cloud.google.com/go/bigquery/storage/managedwriter/testdata"; +import "google/protobuf/wrappers.proto"; // SimpleMessageProto3 represents a simple message that transmits a string and int64 value. message SimpleMessageProto3 { // name is a simple scalar string. string name = 1; // value is a simple int64 value. - int64 value = 2; + google.protobuf.Int64Value value = 2; } message GithubArchiveEntityProto3 { - int64 id = 1; - string login = 2; - string gravatar_id = 3; - string avatar_url = 4; - string url = 5; + google.protobuf.Int64Value id = 1; + google.protobuf.StringValue login = 2; + google.protobuf.StringValue gravatar_id = 3; + google.protobuf.StringValue avatar_url = 4; + google.protobuf.StringValue url = 5; } message GithubArchiveRepoProto3 { - int64 id = 1; - string name = 2; - string url = 3; + google.protobuf.Int64Value id = 1; + google.protobuf.StringValue name = 2; + google.protobuf.StringValue url = 3; } // GithubArchiveMessageProto3 is the proto3 version of github archive row. message GithubArchiveMessageProto3 { - string type = 1; - bool public = 2; - string payload = 3; + google.protobuf.StringValue type = 1; + google.protobuf.BoolValue public = 2; + google.protobuf.StringValue payload = 3; GithubArchiveRepoProto3 repo = 4; GithubArchiveEntityProto3 actor = 5; GithubArchiveEntityProto3 org = 6; - int64 created_at = 7; - string id = 8; - string other = 9; + google.protobuf.Int64Value created_at = 7; + google.protobuf.StringValue id = 8; + google.protobuf.StringValue other = 9; } diff --git a/bigquery/storage/managedwriter/testdata/schemas.go b/bigquery/storage/managedwriter/testdata/schemas.go index 321dedb79ed..02010bab6f6 100644 --- a/bigquery/storage/managedwriter/testdata/schemas.go +++ b/bigquery/storage/managedwriter/testdata/schemas.go @@ -18,7 +18,7 @@ import "cloud.google.com/go/bigquery" var ( SimpleMessageSchema bigquery.Schema = bigquery.Schema{ - {Name: "name", Type: bigquery.StringFieldType}, + {Name: "name", Type: bigquery.StringFieldType, Required: true}, {Name: "value", Type: bigquery.IntegerFieldType}, } From e58a850ad04dc4739796d691e8ef88531651ebd6 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 11 Aug 2021 00:02:26 +0000 Subject: [PATCH 05/11] goimports again --- .../storage/managedwriter/testdata/messages_proto3.pb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go index e119a759161..1422ac357d5 100644 --- a/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go +++ b/bigquery/storage/managedwriter/testdata/messages_proto3.pb.go @@ -21,12 +21,13 @@ package testdata import ( + reflect "reflect" + sync "sync" + proto "github.com/golang/protobuf/proto" wrappers "github.com/golang/protobuf/ptypes/wrappers" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" ) const ( From 9d676f0b656fe43518decbf0c3245978d4b16f46 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 11 Aug 2021 16:58:34 +0000 Subject: [PATCH 06/11] add a sparse benchmark case --- .../adapt/protoconversion_test.go | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index d330901f38e..1ca84e9e08b 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -400,6 +400,16 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { } }, }, + { + // Only set a single top-level field in a larger message. + name: "GithubArchiveProto2_Sparse", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto2{ + Id: proto.String(fmt.Sprintf("id%d", nowNano)), + } + }, + }, { name: "GithubArchiveProto3", setterF: func() protoreflect.ProtoMessage { @@ -433,6 +443,16 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { } }, }, + { + // Only set a single field in a larger message. + name: "GithubArchiveProto3_Sparse", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto3{ + Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, + } + }, + }, } { b.Run(bm.name, func(b *testing.B) { var totalBytes int64 @@ -445,7 +465,7 @@ func BenchmarkStaticProtoSerialization(b *testing.B) { totalBytes = totalBytes + int64(len(out)) staticBytes = out } - b.Logf("avg bytes/message: %d", totalBytes/int64(b.N)) + b.Logf("N=%d, avg bytes/message: %d", b.N, totalBytes/int64(b.N)) }) } } From 2d87e1e7f05609496a48fa385864515b55fcabdf Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 12 Aug 2021 17:29:12 +0000 Subject: [PATCH 07/11] move benchmarks to sub package, augment proto tests --- .../managedwriter/adapt/protoconversion.go | 13 +- .../adapt/protoconversion_test.go | 95 +++++++- .../benchmarks/protobenchmarks_test.go | 219 ++++++++++++++++++ 3 files changed, 314 insertions(+), 13 deletions(-) create mode 100644 bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go diff --git a/bigquery/storage/managedwriter/adapt/protoconversion.go b/bigquery/storage/managedwriter/adapt/protoconversion.go index 7803df6b203..d46ac173297 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion.go @@ -116,13 +116,22 @@ func (dm dependencyCache) add(schema *storagepb.TableSchema, descriptor protoref return nil } -// StorageSchemaToDescriptor builds a protoreflect.Descriptor for a given table schema. -func StorageSchemaToDescriptor(inSchema *storagepb.TableSchema, scope string) (protoreflect.Descriptor, error) { +// StorageSchemaToProto2Descriptor builds a protoreflect.Descriptor for a given table schema using proto2 syntax. +func StorageSchemaToProto2Descriptor(inSchema *storagepb.TableSchema, scope string) (protoreflect.Descriptor, error) { dc := make(dependencyCache) // TODO: b/193064992 tracks support for wrapper types. In the interim, disable wrapper usage. return storageSchemaToDescriptorInternal(inSchema, scope, &dc, false) } +// StorageSchemaToProto3Descriptor builds a protoreflt.Descriptor for a given table schema using proto3 syntax. +// +// NOTE: Currently the write API doesn't yet support proto3 behaviors (default value, wrapper types, etc), but this is provided for +// completeness. +func StorageSchemaToProto3Descriptor(inSchema *storagepb.TableSchema, scope string) (protoreflect.Descriptor, error) { + dc := make(dependencyCache) + return storageSchemaToDescriptorInternal(inSchema, scope, &dc, true) +} + // internal implementation of the conversion code. func storageSchemaToDescriptorInternal(inSchema *storagepb.TableSchema, scope string, cache *dependencyCache, useProto3 bool) (protoreflect.Descriptor, error) { if inSchema == nil { diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index 1ca84e9e08b..5e47fed2e3b 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -39,7 +39,8 @@ func TestSchemaToProtoConversion(t *testing.T) { testCases := []struct { description string bq *storagepb.TableSchema - want *descriptorpb.DescriptorProto + wantProto2 *descriptorpb.DescriptorProto + wantProto3 *descriptorpb.DescriptorProto }{ { description: "basic", @@ -49,7 +50,7 @@ func TestSchemaToProtoConversion(t *testing.T) { {Name: "bar", Type: storagepb.TableFieldSchema_INT64, Mode: storagepb.TableFieldSchema_REQUIRED}, {Name: "baz", Type: storagepb.TableFieldSchema_BYTES, Mode: storagepb.TableFieldSchema_REPEATED}, }}, - want: &descriptorpb.DescriptorProto{ + wantProto2: &descriptorpb.DescriptorProto{ Name: proto.String("root"), Field: []*descriptorpb.FieldDescriptorProto{ { @@ -61,6 +62,19 @@ func TestSchemaToProtoConversion(t *testing.T) { {Name: proto.String("baz"), Number: proto.Int32(3), Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum()}, }, }, + wantProto3: &descriptorpb.DescriptorProto{ + Name: proto.String("root"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("foo"), + Number: proto.Int32(1), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".google.protobuf.StringValue"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()}, + {Name: proto.String("bar"), Number: proto.Int32(2), Type: descriptorpb.FieldDescriptorProto_TYPE_INT64.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()}, + {Name: proto.String("baz"), Number: proto.Int32(3), Type: descriptorpb.FieldDescriptorProto_TYPE_BYTES.Enum(), Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum()}, + }, + }, }, { // exercise construct of a submessage @@ -79,7 +93,7 @@ func TestSchemaToProtoConversion(t *testing.T) { }, }, }, - want: &descriptorpb.DescriptorProto{ + wantProto2: &descriptorpb.DescriptorProto{ Name: proto.String("root"), Field: []*descriptorpb.FieldDescriptorProto{ { @@ -97,6 +111,25 @@ func TestSchemaToProtoConversion(t *testing.T) { }, }, }, + wantProto3: &descriptorpb.DescriptorProto{ + Name: proto.String("root"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("curdate"), + Number: proto.Int32(1), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".google.protobuf.Int32Value"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + }, + { + Name: proto.String("rec"), + Number: proto.Int32(2), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".root__rec"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + }, + }, + }, }, { // We expect to re-use the submessage twice, as the schema contains two identical structs. @@ -124,7 +157,7 @@ func TestSchemaToProtoConversion(t *testing.T) { }, }, }, - want: &descriptorpb.DescriptorProto{ + wantProto2: &descriptorpb.DescriptorProto{ Name: proto.String("root"), Field: []*descriptorpb.FieldDescriptorProto{ { @@ -149,24 +182,64 @@ func TestSchemaToProtoConversion(t *testing.T) { }, }, }, + wantProto3: &descriptorpb.DescriptorProto{ + Name: proto.String("root"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("curdate"), + Number: proto.Int32(1), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".google.protobuf.Int32Value"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + }, + { + Name: proto.String("rec1"), + Number: proto.Int32(2), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".root__rec1"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + }, + { + Name: proto.String("rec2"), + Number: proto.Int32(3), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".root__rec1"), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + }, + }, + }, }, } for _, tc := range testCases { - d, err := StorageSchemaToDescriptor(tc.bq, "root") + p2d, err := StorageSchemaToProto2Descriptor(tc.bq, "root") if err != nil { - t.Fatalf("case (%s) failed conversion: %v", tc.description, err) + t.Errorf("case (%s) failed proto2 conversion: %v", tc.description, err) } // convert it to DP form - mDesc, ok := d.(protoreflect.MessageDescriptor) + mDesc, ok := p2d.(protoreflect.MessageDescriptor) if !ok { - t.Fatalf("%s: couldn't convert to messagedescriptor", tc.description) + t.Errorf("%s: couldn't convert proto2 to messagedescriptor", tc.description) } gotDP := protodesc.ToDescriptorProto(mDesc) + if diff := cmp.Diff(gotDP, tc.wantProto2, protocmp.Transform()); diff != "" { + t.Errorf("%s proto2: -got, +want:\n%s", tc.description, diff) + } - if diff := cmp.Diff(gotDP, tc.want, protocmp.Transform()); diff != "" { - t.Fatalf("%s: -got, +want:\n%s", tc.description, diff) + p3d, err := StorageSchemaToProto3Descriptor(tc.bq, "root") + if err != nil { + t.Fatalf("case (%s) failed proto3 conversion: %v", tc.description, err) + } + + mDesc, ok = p3d.(protoreflect.MessageDescriptor) + if !ok { + t.Errorf("%s: couldn't convert proto3 to messagedescriptor", tc.description) + } + gotDP = protodesc.ToDescriptorProto(mDesc) + if diff := cmp.Diff(gotDP, tc.wantProto3, protocmp.Transform()); diff != "" { + t.Errorf("%s proto3: -got, +want:\n%s", tc.description, diff) } + } } @@ -187,7 +260,7 @@ func TestProtoJSONSerialization(t *testing.T) { }, } - descriptor, err := StorageSchemaToDescriptor(sourceSchema, "root") + descriptor, err := StorageSchemaToProto2Descriptor(sourceSchema, "root") if err != nil { t.Fatalf("failed to construct descriptor") } diff --git a/bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go b/bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go new file mode 100644 index 00000000000..dc69adf4242 --- /dev/null +++ b/bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go @@ -0,0 +1,219 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package benchmarks + +import ( + "fmt" + "testing" + "time" + + "cloud.google.com/go/bigquery" + "cloud.google.com/go/bigquery/storage/managedwriter/adapt" + "cloud.google.com/go/bigquery/storage/managedwriter/testdata" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var benchDescriptor protoreflect.Descriptor + +func BenchmarkStorageSchemaToDescriptor(b *testing.B) { + syntaxLabels := []string{"proto2", "proto3"} + for _, bm := range []struct { + name string + in bigquery.Schema + }{ + { + name: "SingleField", + in: bigquery.Schema{ + {Name: "field", Type: bigquery.StringFieldType}, + }, + }, + { + name: "NestedRecord", + in: bigquery.Schema{ + {Name: "field1", Type: bigquery.StringFieldType}, + {Name: "field2", Type: bigquery.IntegerFieldType}, + {Name: "field3", Type: bigquery.BooleanFieldType}, + { + Name: "field4", + Type: bigquery.RecordFieldType, + Schema: bigquery.Schema{ + {Name: "recordfield1", Type: bigquery.GeographyFieldType}, + {Name: "recordfield2", Type: bigquery.TimestampFieldType}, + }, + }, + }, + }, + { + name: "SimpleMessage", + in: testdata.SimpleMessageSchema, + }, + { + name: "GithubArchiveSchema", + in: testdata.GithubArchiveSchema, + }, + } { + for _, s := range syntaxLabels { + b.Run(fmt.Sprintf("%s-%s", bm.name, s), func(b *testing.B) { + convSchema, err := adapt.BQSchemaToStorageTableSchema(bm.in) + if err != nil { + b.Errorf("%q: schema conversion fail: %v", bm.name, err) + } + for n := 0; n < b.N; n++ { + if s == "proto3" { + benchDescriptor, err = adapt.StorageSchemaToProto3Descriptor(convSchema, "root") + } else { + benchDescriptor, err = adapt.StorageSchemaToProto2Descriptor(convSchema, "root") + } + if err != nil { + b.Errorf("failed to convert %q: %v", bm.name, err) + } + } + }) + } + } +} + +var staticBytes []byte + +func BenchmarkStaticProtoSerialization(b *testing.B) { + for _, bm := range []struct { + name string + in bigquery.Schema + syntax string + setterF func() protoreflect.ProtoMessage + }{ + { + name: "SimpleMessageProto2", + setterF: func() protoreflect.ProtoMessage { + return &testdata.SimpleMessageProto2{ + Name: proto.String(fmt.Sprintf("test-%d", time.Now().UnixNano())), + Value: proto.Int64(time.Now().UnixNano()), + } + }, + }, + { + name: "SimpleMessageProto3", + setterF: func() protoreflect.ProtoMessage { + return &testdata.SimpleMessageProto3{ + Name: fmt.Sprintf("test-%d", time.Now().UnixNano()), + Value: &wrapperspb.Int64Value{Value: time.Now().UnixNano()}, + } + }, + }, + { + name: "GithubArchiveProto2", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto2{ + Type: proto.String("SomeEvent"), + Public: proto.Bool(nowNano%2 == 0), + Payload: proto.String(fmt.Sprintf("stuff %d", nowNano)), + Repo: &testdata.GithubArchiveRepoProto2{ + Id: proto.Int64(nowNano), + Name: proto.String("staticname"), + Url: proto.String(fmt.Sprintf("foo.com/%d", nowNano)), + }, + Actor: &testdata.GithubArchiveEntityProto2{ + Id: proto.Int64(nowNano % 1000), + Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), + GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), + AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + }, + Org: &testdata.GithubArchiveEntityProto2{ + Id: proto.Int64(nowNano % 1000), + Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), + GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), + AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), + }, + CreatedAt: proto.Int64(nowNano), + Id: proto.String(fmt.Sprintf("id%d", nowNano)), + Other: proto.String("other"), + } + }, + }, + { + // Only set a single top-level field in a larger message. + name: "GithubArchiveProto2_Sparse", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto2{ + Id: proto.String(fmt.Sprintf("id%d", nowNano)), + } + }, + }, + { + name: "GithubArchiveProto3", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto3{ + Type: &wrapperspb.StringValue{Value: "SomeEvent"}, + Public: &wrapperspb.BoolValue{Value: nowNano%2 == 0}, + Payload: &wrapperspb.StringValue{Value: fmt.Sprintf("stuff %d", nowNano)}, + Repo: &testdata.GithubArchiveRepoProto3{ + Id: &wrapperspb.Int64Value{Value: nowNano}, + Name: &wrapperspb.StringValue{Value: "staticname"}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("foo.com/%d", nowNano)}, + }, + Actor: &testdata.GithubArchiveEntityProto3{ + Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, + Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, + GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, + AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + }, + Org: &testdata.GithubArchiveEntityProto3{ + Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, + Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, + GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, + AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, + }, + CreatedAt: &wrapperspb.Int64Value{Value: nowNano}, + Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, + Other: &wrapperspb.StringValue{Value: "other"}, + } + }, + }, + { + // Only set a single field in a larger message. + name: "GithubArchiveProto3_Sparse", + setterF: func() protoreflect.ProtoMessage { + nowNano := time.Now().UnixNano() + return &testdata.GithubArchiveMessageProto3{ + Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, + } + }, + }, + } { + b.Run(bm.name, func(b *testing.B) { + var totalBytes int64 + for n := 0; n < b.N; n++ { + m := bm.setterF() + out, err := proto.Marshal(m) + if err != nil { + b.Errorf("%q %q: Marshal: %v", bm.name, bm.syntax, err) + } + totalBytes = totalBytes + int64(len(out)) + staticBytes = out + } + b.Logf("N=%d, avg bytes/message: %d", b.N, totalBytes/int64(b.N)) + }) + } +} From 33af8068ac3e1055860bd7f9b617e3729613dd75 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 12 Aug 2021 18:11:49 +0000 Subject: [PATCH 08/11] fix comment typo --- bigquery/storage/managedwriter/adapt/protoconversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/storage/managedwriter/adapt/protoconversion.go b/bigquery/storage/managedwriter/adapt/protoconversion.go index d46ac173297..d65b66a4cda 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion.go @@ -123,7 +123,7 @@ func StorageSchemaToProto2Descriptor(inSchema *storagepb.TableSchema, scope stri return storageSchemaToDescriptorInternal(inSchema, scope, &dc, false) } -// StorageSchemaToProto3Descriptor builds a protoreflt.Descriptor for a given table schema using proto3 syntax. +// StorageSchemaToProto3Descriptor builds a protoreflect.Descriptor for a given table schema using proto3 syntax. // // NOTE: Currently the write API doesn't yet support proto3 behaviors (default value, wrapper types, etc), but this is provided for // completeness. From db95a768fd35ef76ea9cb3dba72b4cb513743863 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 12 Aug 2021 20:05:07 +0000 Subject: [PATCH 09/11] update test --- bigquery/storage/managedwriter/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/storage/managedwriter/integration_test.go b/bigquery/storage/managedwriter/integration_test.go index 5c3bf7f49bf..243432ce715 100644 --- a/bigquery/storage/managedwriter/integration_test.go +++ b/bigquery/storage/managedwriter/integration_test.go @@ -96,7 +96,7 @@ func setupDynamicDescriptors(t *testing.T, schema bigquery.Schema) (protoreflect t.Fatalf("adapt.BQSchemaToStorageTableSchema: %v", err) } - descriptor, err := adapt.StorageSchemaToDescriptor(convertedSchema, "root") + descriptor, err := adapt.StorageSchemaToProto2Descriptor(convertedSchema, "root") if err != nil { t.Fatalf("adapt.StorageSchemaToDescriptor: %v", err) } From d2b70f65a90b6d291e69ab86f1e554575e801967 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 12 Aug 2021 20:25:20 +0000 Subject: [PATCH 10/11] remove benchmarks copy from adapt --- .../adapt/protoconversion_test.go | 192 ------------------ 1 file changed, 192 deletions(-) diff --git a/bigquery/storage/managedwriter/adapt/protoconversion_test.go b/bigquery/storage/managedwriter/adapt/protoconversion_test.go index 5e47fed2e3b..5334b100632 100644 --- a/bigquery/storage/managedwriter/adapt/protoconversion_test.go +++ b/bigquery/storage/managedwriter/adapt/protoconversion_test.go @@ -16,13 +16,9 @@ package adapt import ( "encoding/json" - "fmt" "reflect" "testing" - "time" - "cloud.google.com/go/bigquery" - "cloud.google.com/go/bigquery/storage/managedwriter/testdata" "github.com/google/go-cmp/cmp" storagepb "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2" "google.golang.org/protobuf/encoding/protojson" @@ -32,7 +28,6 @@ import ( "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/descriptorpb" "google.golang.org/protobuf/types/dynamicpb" - "google.golang.org/protobuf/types/known/wrapperspb" ) func TestSchemaToProtoConversion(t *testing.T) { @@ -355,190 +350,3 @@ func TestProtoJSONSerialization(t *testing.T) { } } - -var benchDescriptor protoreflect.Descriptor - -func BenchmarkStorageSchemaToDescriptor(b *testing.B) { - syntaxLabels := []string{"proto2", "proto3"} - for _, bm := range []struct { - name string - in bigquery.Schema - }{ - { - name: "SingleField", - in: bigquery.Schema{ - {Name: "field", Type: bigquery.StringFieldType}, - }, - }, - { - name: "NestedRecord", - in: bigquery.Schema{ - {Name: "field1", Type: bigquery.StringFieldType}, - {Name: "field2", Type: bigquery.IntegerFieldType}, - {Name: "field3", Type: bigquery.BooleanFieldType}, - { - Name: "field4", - Type: bigquery.RecordFieldType, - Schema: bigquery.Schema{ - {Name: "recordfield1", Type: bigquery.GeographyFieldType}, - {Name: "recordfield2", Type: bigquery.TimestampFieldType}, - }, - }, - }, - }, - { - name: "SimpleMessage", - in: testdata.SimpleMessageSchema, - }, - { - name: "GithubArchiveSchema", - in: testdata.GithubArchiveSchema, - }, - } { - for _, s := range syntaxLabels { - b.Run(fmt.Sprintf("%s-%s", bm.name, s), func(b *testing.B) { - convSchema, err := BQSchemaToStorageTableSchema(bm.in) - if err != nil { - b.Errorf("%q: schema conversion fail: %v", bm.name, err) - } - for n := 0; n < b.N; n++ { - dc := make(dependencyCache) - benchDescriptor, err = storageSchemaToDescriptorInternal(convSchema, "root", &dc, s == "proto3") - if err != nil { - b.Errorf("failed to convert %q: %v", bm.name, err) - } - } - }) - } - } -} - -var staticBytes []byte - -func BenchmarkStaticProtoSerialization(b *testing.B) { - for _, bm := range []struct { - name string - in bigquery.Schema - syntax string - setterF func() protoreflect.ProtoMessage - }{ - { - name: "SimpleMessageProto2", - setterF: func() protoreflect.ProtoMessage { - return &testdata.SimpleMessageProto2{ - Name: proto.String(fmt.Sprintf("test-%d", time.Now().UnixNano())), - Value: proto.Int64(time.Now().UnixNano()), - } - }, - }, - { - name: "SimpleMessageProto3", - setterF: func() protoreflect.ProtoMessage { - return &testdata.SimpleMessageProto3{ - Name: fmt.Sprintf("test-%d", time.Now().UnixNano()), - Value: &wrapperspb.Int64Value{Value: time.Now().UnixNano()}, - } - }, - }, - { - name: "GithubArchiveProto2", - setterF: func() protoreflect.ProtoMessage { - nowNano := time.Now().UnixNano() - return &testdata.GithubArchiveMessageProto2{ - Type: proto.String("SomeEvent"), - Public: proto.Bool(nowNano%2 == 0), - Payload: proto.String(fmt.Sprintf("stuff %d", nowNano)), - Repo: &testdata.GithubArchiveRepoProto2{ - Id: proto.Int64(nowNano), - Name: proto.String("staticname"), - Url: proto.String(fmt.Sprintf("foo.com/%d", nowNano)), - }, - Actor: &testdata.GithubArchiveEntityProto2{ - Id: proto.Int64(nowNano % 1000), - Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), - GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), - AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), - Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), - }, - Org: &testdata.GithubArchiveEntityProto2{ - Id: proto.Int64(nowNano % 1000), - Login: proto.String(fmt.Sprintf("login-%d", nowNano%1000)), - GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)), - AvatarUrl: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), - Url: proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)), - }, - CreatedAt: proto.Int64(nowNano), - Id: proto.String(fmt.Sprintf("id%d", nowNano)), - Other: proto.String("other"), - } - }, - }, - { - // Only set a single top-level field in a larger message. - name: "GithubArchiveProto2_Sparse", - setterF: func() protoreflect.ProtoMessage { - nowNano := time.Now().UnixNano() - return &testdata.GithubArchiveMessageProto2{ - Id: proto.String(fmt.Sprintf("id%d", nowNano)), - } - }, - }, - { - name: "GithubArchiveProto3", - setterF: func() protoreflect.ProtoMessage { - nowNano := time.Now().UnixNano() - return &testdata.GithubArchiveMessageProto3{ - Type: &wrapperspb.StringValue{Value: "SomeEvent"}, - Public: &wrapperspb.BoolValue{Value: nowNano%2 == 0}, - Payload: &wrapperspb.StringValue{Value: fmt.Sprintf("stuff %d", nowNano)}, - Repo: &testdata.GithubArchiveRepoProto3{ - Id: &wrapperspb.Int64Value{Value: nowNano}, - Name: &wrapperspb.StringValue{Value: "staticname"}, - Url: &wrapperspb.StringValue{Value: fmt.Sprintf("foo.com/%d", nowNano)}, - }, - Actor: &testdata.GithubArchiveEntityProto3{ - Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, - Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, - GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, - AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, - Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, - }, - Org: &testdata.GithubArchiveEntityProto3{ - Id: &wrapperspb.Int64Value{Value: nowNano % 1000}, - Login: &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)}, - GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)}, - AvatarUrl: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, - Url: &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)}, - }, - CreatedAt: &wrapperspb.Int64Value{Value: nowNano}, - Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, - Other: &wrapperspb.StringValue{Value: "other"}, - } - }, - }, - { - // Only set a single field in a larger message. - name: "GithubArchiveProto3_Sparse", - setterF: func() protoreflect.ProtoMessage { - nowNano := time.Now().UnixNano() - return &testdata.GithubArchiveMessageProto3{ - Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)}, - } - }, - }, - } { - b.Run(bm.name, func(b *testing.B) { - var totalBytes int64 - for n := 0; n < b.N; n++ { - m := bm.setterF() - out, err := proto.Marshal(m) - if err != nil { - b.Errorf("%q %q: Marshal: %v", bm.name, bm.syntax, err) - } - totalBytes = totalBytes + int64(len(out)) - staticBytes = out - } - b.Logf("N=%d, avg bytes/message: %d", b.N, totalBytes/int64(b.N)) - }) - } -} From a597089dca2d9b496a80666e2339ed25bc0b4d30 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Mon, 16 Aug 2021 17:52:02 +0000 Subject: [PATCH 11/11] relocate benchmarks back to adapt --- .../managedwriter/{benchmarks => adapt}/protobenchmarks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename bigquery/storage/managedwriter/{benchmarks => adapt}/protobenchmarks_test.go (99%) diff --git a/bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go b/bigquery/storage/managedwriter/adapt/protobenchmarks_test.go similarity index 99% rename from bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go rename to bigquery/storage/managedwriter/adapt/protobenchmarks_test.go index dc69adf4242..4f1dfd31505 100644 --- a/bigquery/storage/managedwriter/benchmarks/protobenchmarks_test.go +++ b/bigquery/storage/managedwriter/adapt/protobenchmarks_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package benchmarks +package adapt_test import ( "fmt"