/
stream.proto
148 lines (124 loc) · 6.01 KB
/
stream.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.bigquery.storage.v1;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/bigquery/storage/v1/arrow.proto";
import "google/cloud/bigquery/storage/v1/avro.proto";
import "google/protobuf/timestamp.proto";
option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage";
option java_multiple_files = true;
option java_outer_classname = "StreamProto";
option java_package = "com.google.cloud.bigquery.storage.v1";
option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
// Data format for input or output data.
enum DataFormat {
DATA_FORMAT_UNSPECIFIED = 0;
// Avro is a standard open source row based file format.
// See https://avro.apache.org/ for more details.
AVRO = 1;
// Arrow is a standard open source column-based message format.
// See https://arrow.apache.org/ for more details.
ARROW = 2;
}
// Information about the ReadSession.
message ReadSession {
option (google.api.resource) = {
type: "bigquerystorage.googleapis.com/ReadSession"
pattern: "projects/{project}/locations/{location}/sessions/{session}"
};
// Additional attributes when reading a table.
message TableModifiers {
// The snapshot time of the table. If not set, interpreted as now.
google.protobuf.Timestamp snapshot_time = 1;
}
// Options dictating how we read a table.
message TableReadOptions {
// Names of the fields in the table that should be read. If empty, all
// fields will be read. If the specified field is a nested field, all
// the sub-fields in the field will be selected. The output field order is
// unrelated to the order of fields in selected_fields.
repeated string selected_fields = 1;
// SQL text filtering statement, similar to a WHERE clause in a query.
// Aggregates are not supported.
//
// Examples: "int_field > 5"
// "date_field = CAST('2014-9-27' as DATE)"
// "nullable_field is not NULL"
// "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))"
// "numeric_field BETWEEN 1.0 AND 5.0"
//
// Restricted to a maximum length for 1 MB.
string row_restriction = 2;
oneof output_format_serialization_options {
// Optional. Options specific to the Apache Arrow output format.
ArrowSerializationOptions arrow_serialization_options = 3 [(google.api.field_behavior) = OPTIONAL];
}
}
// Output only. Unique identifier for the session, in the form
// `projects/{project_id}/locations/{location}/sessions/{session_id}`.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Time at which the session becomes invalid. After this time, subsequent
// requests to read this Session will return errors. The expire_time is
// automatically assigned and currently cannot be specified or updated.
google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Immutable. Data format of the output data.
DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE];
// The schema for the read. If read_options.selected_fields is set, the
// schema may be different from the table schema as it will only contain
// the selected fields.
oneof schema {
// Output only. Avro schema.
AvroSchema avro_schema = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. Arrow schema.
ArrowSchema arrow_schema = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Immutable. Table that this ReadSession is reading from, in the form
// `projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`
string table = 6 [
(google.api.field_behavior) = IMMUTABLE,
(google.api.resource_reference) = {
type: "bigquery.googleapis.com/Table"
}
];
// Optional. Any modifiers which are applied when reading from the specified table.
TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL];
// Optional. Read options for this session (e.g. column selection, filters).
TableReadOptions read_options = 8 [(google.api.field_behavior) = OPTIONAL];
// Output only. A list of streams created with the session.
//
// At least one stream is created with the session. In the future, larger
// request_stream_count values *may* result in this list being unpopulated,
// in that case, the user will need to use a List method to get the streams
// instead, which is not yet available.
repeated ReadStream streams = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
// Output only. An estimate on the number of bytes this session will scan when
// all streams are completely consumed. This estimate is based on
// metadata from the table which might be incomplete or stale.
int64 estimated_total_bytes_scanned = 12 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// Information about a single stream that gets data out of the storage system.
// Most of the information about `ReadStream` instances is aggregated, making
// `ReadStream` lightweight.
message ReadStream {
option (google.api.resource) = {
type: "bigquerystorage.googleapis.com/ReadStream"
pattern: "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}"
};
// Output only. Name of the stream, in the form
// `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}