-
Notifications
You must be signed in to change notification settings - Fork 62
/
pipeline.yaml
203 lines (193 loc) · 9.19 KB
/
pipeline.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
resources:
- type: bigquery_table
table_id: covid19_vaccination_search_insights
description: |-
Terms of use
To download or use the data, you must agree to the Google Terms of Service: https://policies.google.com/terms
Description
The COVID-19 Vaccination Search Insights data shows aggregated, anonymized trends in searches related to COVID-19 vaccination. The dataset provides a weekly time series for each region showing the relative interest of Google searches related to COVID-19 vaccination, across several categories.
The data is intended to help public health officials design, target, and evaluate public education campaigns.
To explore and download the data, use our interactive dashboard: http://goo.gle/covid19vaccinationinsights
To learn more about the dataset, how we generate it and preserve privacy, read the data documentation:
https://storage.googleapis.com/gcs-public-datasets/COVID-19%20Vaccination%20Search%20Insights%20documentation.pdf
time_partitioning:
type: "DAY"
require_partition_filter: False
clustering:
- "sub_region_1_code"
- "sub_region_2_code"
- "sub_region_3_code"
- "place_id"
schema: |-
[
{
"name": "date",
"description": "The first day of the week (starting on Monday) on which the searches took place. For example, in the weekly data the row labeled 2021-04-19 represents the search activity for the week of April 19 to April 25, 2021, inclusive. Calendar days start and end at midnight Pacific Standard Time.",
"type": "DATE",
"mode": "NULLABLE"
},
{
"name": "country_region",
"description": "The name of the country in English. For example, United States.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "country_region_code",
"description": "The ISO 3166-1 code for the country. For example, US.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_1",
"description": "The name of a region in the country. For example, California.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_1_code",
"description": "A country-specific ISO 3166-2 code for the region. For example, US-CA.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_2",
"description": "The name (or type) of a region in the country. Typically a subdivision of sub_region_1. For example, Santa Clara County or municipal_borough.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_2_code",
"description": "In the US, the FIPS code for a US county (or equivalent). For example, 06085.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_3",
"description": "The name (or type) of a region in the country. Typically a subdivision of sub_region_2. For example, Downtown or postal_code.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sub_region_3_code",
"description": "In the US, the ZIP code. For example 94303.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "place_id",
"description": "The Google place ID for the most-specific subregion. Used in the Google Places API and on Google Maps. For example, ChIJd_Y0eVIvkIARuQyDN0F1LBA.",
"type": "STRING",
"mode": "NULLABLE"
},
{
"name": "sni_covid19_vaccination",
"description": "The scaled normalized interest related to all COVID-19 vaccination for the region and date. For example, 87.02. Empty when data isn't available.",
"type": "FLOAT",
"mode": "NULLABLE"
},
{
"name": "sni_vaccination_intent",
"description": "The scaled normalized interest related to vaccination intent for the region and date. For example, 22.69. Empty when data isn't available.",
"type": "FLOAT",
"mode": "NULLABLE"
},
{
"name": "sni_safety_side_effects",
"description": "The scaled normalized interest related to safety and side effects of the vaccines for the region and date. For example, 17.96. Empty when data isn't available.",
"type": "FLOAT",
"mode": "NULLABLE"
}
]
dag:
airflow_version: 1
initialize:
dag_id: covid19_vaccination_search_insights
default_args:
owner: "Google"
depends_on_past: False
start_date: '2021-06-28'
max_active_runs: 1
schedule_interval: "@hourly"
catchup: False
default_view: graph
tasks:
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load global vaccination search insights CSV file from the covid19-open-data bucket to BQ"
args:
task_id: "gcs_to_bq_vaccination_search_insights"
bucket: "{{ var.json.covid19_vaccination_search_insights.source_bucket }}"
source_objects: ["{{ var.json.covid19_vaccination_search_insights.source_prefix }}/Global_vaccination_search_insights.csv"]
source_format: "CSV"
destination_project_dataset_table: "covid19_vaccination_search_insights.covid19_vaccination_search_insights"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- name: date
description: "The first day of the week (starting on Monday) on which the searches took place. For example, in the weekly data the row labeled 2021-04-19 represents the search activity for the week of April 19 to April 25, 2021, inclusive. Calendar days start and end at midnight Pacific Standard Time."
type: "DATE"
mode: "NULLABLE"
- name: country_region
description: "The name of the country in English. For example, United States."
type: "STRING"
mode: "NULLABLE"
- name: country_region_code
description: "The ISO 3166-1 code for the country. For example, US."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_1
description: "The name of a region in the country. For example, California."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_1_code
description: "A country-specific ISO 3166-2 code for the region. For example, US-CA."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_2
description: "The name (or type) of a region in the country. Typically a subdivision of sub_region_1. For example, Santa Clara County or municipal_borough."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_2_code
description: "In the US, the FIPS code for a US county (or equivalent). For example, 06085."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_3
description: "The name (or type) of a region in the country. Typically a subdivision of sub_region_2. For example, Downtown or postal_code."
type: "STRING"
mode: "NULLABLE"
- name: sub_region_3_code
description: "In the US, the ZIP code. For example 94303."
type: "STRING"
mode: "NULLABLE"
- name: place_id
description: "The Google place ID for the most-specific subregion. Used in the Google Places API and on Google Maps. For example, ChIJd_Y0eVIvkIARuQyDN0F1LBA."
type: "STRING"
mode: "NULLABLE"
- name: sni_covid19_vaccination
description: "The scaled normalized interest related to all COVID-19 vaccination for the region and date. For example, 87.02. Empty when data isn't available."
type: "FLOAT"
mode: "NULLABLE"
- name: sni_vaccination_intent
description: "The scaled normalized interest related to vaccination intent for the region and date. For example, 22.69. Empty when data isn't available."
type: "FLOAT"
mode: "NULLABLE"
- name: sni_safety_side_effects
description: "The scaled normalized interest related to safety and side effects of the vaccines for the region and date. For example, 17.96. Empty when data isn't available."
type: "FLOAT"
mode: "NULLABLE"
graph_paths:
- "gcs_to_bq_vaccination_search_insights"