diff --git a/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto index bc2e4eb1..74fe927b 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/avro.proto b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto index 109ec86a..495132ec 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/avro.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto b/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto index 741e7d11..11e851be 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ option java_multiple_files = true; option java_outer_classname = "ProtoBufProto"; option java_package = "com.google.cloud.bigquery.storage.v1beta2"; -// Protobuf schema is an API presentation the proto buffer schema. +// ProtoSchema describes the schema of the serialized protocol buffer data rows. message ProtoSchema { // Descriptor for input message. The descriptor has to be self contained, // including all the nested types, excepted for proto buffer well known types @@ -31,7 +31,6 @@ message ProtoSchema { google.protobuf.DescriptorProto proto_descriptor = 1; } -// Protobuf rows. message ProtoRows { // A sequence of rows serialized as a Protocol Buffer. // diff --git a/google/cloud/bigquery_storage_v1beta2/proto/storage.proto b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto index 5538e29f..8c25b846 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/storage.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -71,8 +71,7 @@ service BigQueryRead { post: "/v1beta2/{read_session.table=projects/*/datasets/*/tables/*}" body: "*" }; - option (google.api.method_signature) = - "parent,read_session,max_stream_count"; + option (google.api.method_signature) = "parent,read_session,max_stream_count"; } // Reads rows from the stream in the format prescribed by the ReadSession. @@ -101,8 +100,7 @@ service BigQueryRead { // original, primary, and residual, that original[0-j] = primary[0-j] and // original[j-n] = residual[0-m] once the streams have been read to // completion. - rpc SplitReadStream(SplitReadStreamRequest) - returns (SplitReadStreamResponse) { + rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { option (google.api.http) = { get: "/v1beta2/{name=projects/*/locations/*/sessions/*/streams/*}" }; @@ -171,8 +169,7 @@ service BigQueryWrite { // Finalize a write stream so that no new data can be appended to the // stream. Finalize is not supported on the '_default' stream. - rpc FinalizeWriteStream(FinalizeWriteStreamRequest) - returns (FinalizeWriteStreamResponse) { + rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) { option (google.api.http) = { post: "/v1beta2/{name=projects/*/datasets/*/tables/*/streams/*}" body: "*" @@ -185,8 +182,7 @@ service BigQueryWrite { // Streams must be finalized before commit and cannot be committed multiple // times. Once a stream is committed, data in the stream becomes available // for read operations. - rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) - returns (BatchCommitWriteStreamsResponse) { + rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) { option (google.api.http) = { get: "/v1beta2/{parent=projects/*/datasets/*/tables/*}" }; @@ -303,6 +299,19 @@ message ReadRowsResponse { // Throttling state. If unset, the latest response still describes // the current throttling status. ThrottleState throttle_state = 5; + + // The schema for the read. If read_options.selected_fields is set, the + // schema may be different from the table schema as it will only contain + // the selected fields. This schema is equivelant to the one returned by + // CreateSession. This field is only populated in the first ReadRowsResponse + // RPC. + oneof schema { + // Output only. Avro schema. + AvroSchema avro_schema = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Arrow schema. + ArrowSchema arrow_schema = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + } } // Request message for `SplitReadStream`. @@ -342,7 +351,9 @@ message CreateWriteStreamRequest { // of `projects/{project}/datasets/{dataset}/tables/{table}`. string parent = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" } + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } ]; // Required. Stream to be created. @@ -360,9 +371,9 @@ message AppendRowsRequest { ProtoRows rows = 2; } - // Required. The stream that is the target of the append operation. This value - // must be specified for the initial request. If subsequent requests specify - // the stream name, it must equal to the value provided in the first request. + // Required. The stream that is the target of the append operation. This value must be + // specified for the initial request. If subsequent requests specify the + // stream name, it must equal to the value provided in the first request. // To write to the _default stream, populate this field with a string in the // format `projects/{project}/datasets/{dataset}/tables/{table}/_default`. string write_stream = 1 [ @@ -394,7 +405,7 @@ message AppendRowsRequest { // Response message for `AppendRows`. message AppendRowsResponse { - // A success append result. + // AppendResult is returned for successful append requests. message AppendResult { // The row offset at which the last append occurred. The offset will not be // set if appending using default streams. @@ -405,25 +416,32 @@ message AppendRowsResponse { // Result if the append is successful. AppendResult append_result = 1; - // Error in case of request failed. If set, it means rows are not accepted - // into the system. Users can retry or continue with other requests within - // the same connection. - // ALREADY_EXISTS: happens when offset is specified, it means the entire - // request is already appended, it is safe to ignore this error. - // OUT_OF_RANGE: happens when offset is specified, it means the specified - // offset is beyond the end of the stream. - // INVALID_ARGUMENT: error caused by malformed request or data. - // RESOURCE_EXHAUSTED: request rejected due to throttling. Only happens when - // append without offset. - // ABORTED: request processing is aborted because of prior failures, request - // can be retried if previous failure is fixed. - // INTERNAL: server side errors that can be retried. + // Error returned when problems were encountered. If present, + // it indicates rows were not accepted into the system. + // Users can retry or continue with other append requests within the + // same connection. + // + // Additional information about error signalling: + // + // ALREADY_EXISTS: Happens when an append specified an offset, and the + // backend already has received data at this offset. Typically encountered + // in retry scenarios, and can be ignored. + // + // OUT_OF_RANGE: Returned when the specified offset in the stream is beyond + // the current end of the stream. + // + // INVALID_ARGUMENT: Indicates a malformed request or data. + // + // ABORTED: Request processing is aborted because of prior failures. The + // request can be retried if previous failure is addressed. + // + // INTERNAL: Indicates server side error(s) that can be retried. google.rpc.Status error = 2; } // If backend detects a schema update, pass it to user so that user can - // use it to input new type of message. It will be empty when there is no - // schema updates. + // use it to input new type of message. It will be empty when no schema + // updates have occurred. TableSchema updated_schema = 3; } @@ -441,9 +459,11 @@ message GetWriteStreamRequest { // Request message for `BatchCommitWriteStreams`. message BatchCommitWriteStreamsRequest { - // Required. Parent table that all the streams should belong to, in the form - // of `projects/{project}/datasets/{dataset}/tables/{table}`. - string parent = 1 [(google.api.field_behavior) = REQUIRED]; + // Required. Parent table that all the streams should belong to, in the form of + // `projects/{project}/datasets/{dataset}/tables/{table}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED + ]; // Required. The group of streams that will be committed atomically. repeated string write_streams = 2 [(google.api.field_behavior) = REQUIRED]; @@ -452,11 +472,15 @@ message BatchCommitWriteStreamsRequest { // Response message for `BatchCommitWriteStreams`. message BatchCommitWriteStreamsResponse { // The time at which streams were committed in microseconds granularity. - // This field will only exist when there is no stream errors. + // This field will only exist when there are no stream errors. + // **Note** if this field is not set, it means the commit was not successful. google.protobuf.Timestamp commit_time = 1; // Stream level error if commit failed. Only streams with error will be in // the list. + // If empty, there is no error and all streams are committed successfully. + // If non empty, certain streams have errors and ZERO stream is committed due + // to atomicity guarantee. repeated StorageError stream_errors = 2; } @@ -500,8 +524,9 @@ message FlushRowsResponse { } // Structured custom BigQuery Storage error message. The error can be attached -// as error details in the returned rpc Status. User can use the info to process -// errors in a structural way, rather than having to parse error messages. +// as error details in the returned rpc Status. In particular, the use of error +// codes allows more structured error handling, and reduces the need to evaluate +// unstructured error text strings. message StorageError { // Error code for `StorageError`. enum StorageErrorCode { @@ -522,9 +547,12 @@ message StorageError { INVALID_STREAM_TYPE = 4; // Invalid Stream state. - // For example, you try to commit a stream that is not fianlized or is + // For example, you try to commit a stream that is not finalized or is // garbaged. INVALID_STREAM_STATE = 5; + + // Stream is finalized. + STREAM_FINALIZED = 6; } // BigQuery Storage specific error code. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/stream.proto b/google/cloud/bigquery_storage_v1beta2/proto/stream.proto index 2b0a58c9..d166e987 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/stream.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/stream.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -74,6 +74,8 @@ message ReadSession { // "nullable_field is not NULL" // "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" // "numeric_field BETWEEN 1.0 AND 5.0" + // + // Restricted to a maximum length for 1 MB. string row_restriction = 2; // Optional. Options specific to the Apache Arrow output format. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/table.proto b/google/cloud/bigquery_storage_v1beta2/proto/table.proto index fd8a0a75..670a4a64 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/table.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/table.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -70,6 +70,12 @@ message TableFieldSchema { // Numeric value NUMERIC = 12; + + // BigNumeric value + BIGNUMERIC = 13; + + // Interval + INTERVAL = 14; } enum Mode { diff --git a/google/cloud/bigquery_storage_v1beta2/types/protobuf.py b/google/cloud/bigquery_storage_v1beta2/types/protobuf.py index 99c0543c..9b88202f 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/protobuf.py +++ b/google/cloud/bigquery_storage_v1beta2/types/protobuf.py @@ -28,8 +28,8 @@ class ProtoSchema(proto.Message): - r"""Protobuf schema is an API presentation the proto buffer - schema. + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. Attributes: proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): @@ -47,7 +47,7 @@ class ProtoSchema(proto.Message): class ProtoRows(proto.Message): - r"""Protobuf rows. + r""" Attributes: serialized_rows (Sequence[bytes]): diff --git a/google/cloud/bigquery_storage_v1beta2/types/storage.py b/google/cloud/bigquery_storage_v1beta2/types/storage.py index 85369388..ee6d5210 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/storage.py +++ b/google/cloud/bigquery_storage_v1beta2/types/storage.py @@ -170,6 +170,10 @@ class ReadRowsResponse(proto.Message): Throttling state. If unset, the latest response still describes the current throttling status. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. """ avro_rows = proto.Field( @@ -186,6 +190,14 @@ class ReadRowsResponse(proto.Message): throttle_state = proto.Field(proto.MESSAGE, number=5, message="ThrottleState",) + avro_schema = proto.Field( + proto.MESSAGE, number=7, oneof="schema", message=avro.AvroSchema, + ) + + arrow_schema = proto.Field( + proto.MESSAGE, number=8, oneof="schema", message=arrow.ArrowSchema, + ) + class SplitReadStreamRequest(proto.Message): r"""Request message for ``SplitReadStream``. @@ -307,28 +319,38 @@ class AppendRowsResponse(proto.Message): append_result (google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse.AppendResult): Result if the append is successful. error (google.rpc.status_pb2.Status): - Error in case of request failed. If set, it means rows are - not accepted into the system. Users can retry or continue - with other requests within the same connection. - ALREADY_EXISTS: happens when offset is specified, it means - the entire request is already appended, it is safe to ignore - this error. OUT_OF_RANGE: happens when offset is specified, - it means the specified offset is beyond the end of the - stream. INVALID_ARGUMENT: error caused by malformed request - or data. RESOURCE_EXHAUSTED: request rejected due to - throttling. Only happens when append without offset. - ABORTED: request processing is aborted because of prior - failures, request can be retried if previous failure is - fixed. INTERNAL: server side errors that can be retried. + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. updated_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): If backend detects a schema update, pass it to user so that user can use it to input new - type of message. It will be empty when there is - no schema updates. + type of message. It will be empty when no schema + updates have occurred. """ class AppendResult(proto.Message): - r"""A success append result. + r"""AppendResult is returned for successful append requests. Attributes: offset (google.protobuf.wrappers_pb2.Int64Value): @@ -385,12 +407,17 @@ class BatchCommitWriteStreamsResponse(proto.Message): Attributes: commit_time (google.protobuf.timestamp_pb2.Timestamp): - The time at which streams were committed in - microseconds granularity. This field will only - exist when there is no stream errors. + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. stream_errors (Sequence[google.cloud.bigquery_storage_v1beta2.types.StorageError]): Stream level error if commit failed. Only streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. """ commit_time = proto.Field(proto.MESSAGE, number=1, message=timestamp.Timestamp,) @@ -455,9 +482,10 @@ class FlushRowsResponse(proto.Message): class StorageError(proto.Message): r"""Structured custom BigQuery Storage error message. The error - can be attached as error details in the returned rpc Status. - User can use the info to process errors in a structural way, - rather than having to parse error messages. + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. Attributes: code (google.cloud.bigquery_storage_v1beta2.types.StorageError.StorageErrorCode): @@ -476,6 +504,7 @@ class StorageErrorCode(proto.Enum): STREAM_NOT_FOUND = 3 INVALID_STREAM_TYPE = 4 INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 code = proto.Field(proto.ENUM, number=1, enum=StorageErrorCode,) diff --git a/google/cloud/bigquery_storage_v1beta2/types/stream.py b/google/cloud/bigquery_storage_v1beta2/types/stream.py index 77fb444c..e635e5a6 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/stream.py +++ b/google/cloud/bigquery_storage_v1beta2/types/stream.py @@ -105,7 +105,9 @@ class TableReadOptions(proto.Message): Examples: "int_field > 5" "date_field = CAST('2014-9-27' as DATE)" "nullable_field is not NULL" "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 - AND 5.0". + AND 5.0" + + Restricted to a maximum length for 1 MB. arrow_serialization_options (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions): Optional. Options specific to the Apache Arrow output format. diff --git a/google/cloud/bigquery_storage_v1beta2/types/table.py b/google/cloud/bigquery_storage_v1beta2/types/table.py index f1a209e0..f91b07b4 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/table.py +++ b/google/cloud/bigquery_storage_v1beta2/types/table.py @@ -72,6 +72,8 @@ class Type(proto.Enum): DATETIME = 10 GEOGRAPHY = 11 NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 class Mode(proto.Enum): r"""""" diff --git a/synth.metadata b/synth.metadata index 0574a81a..1eec6cb9 100644 --- a/synth.metadata +++ b/synth.metadata @@ -11,8 +11,8 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "c539b9b08b3366ee00c0ec1950f4df711552a269", - "internalRef": "365759522" + "sha": "b1614aa0668564ec41d78b72cf776e0292ffc98c", + "internalRef": "366811078" } }, {