-
Notifications
You must be signed in to change notification settings - Fork 14
/
airflow_variables_dev.json
283 lines (283 loc) · 8.42 KB
/
airflow_variables_dev.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
{
"affinity": {},
"api_key_path": "/home/airflow/gcs/data/apiKey.json",
"bq_dataset": "test_crypto_stellar_internal",
"bq_project": "test-hubble-319619",
"bq_dataset_audit_log": "audit_log",
"date_for_resets": {
"date": ["2023-03-15", "2023-06-14", "2023-09-13", "2023-12-13"]
},
"cluster_fields": {
"accounts": ["account_id", "last_modified_ledger"],
"account_signers": ["account_id", "signer", "last_modified_ledger"],
"claimable_balances": ["asset_id", "last_modified_ledger"],
"history_assets": ["asset_code", "asset_issuer", "asset_type"],
"history_effects": ["address", "operation_id", "type"],
"history_ledgers": ["sequence", "closed_at"],
"history_operations": ["transaction_id", "source_account", "type"],
"history_trades": ["selling_asset_id", "buying_asset_id", "trade_type"],
"history_transactions": ["account", "ledger_sequence", "successful"],
"offers": ["selling_asset_id", "buying_asset_id", "last_modified_ledger"],
"liquidity_pools": [
"liquidity_pool_id",
"asset_a_id",
"asset_b_id",
"last_modified_ledger"
],
"trust_lines": [
"account_id",
"asset_id",
"liquidity_pool_id",
"last_modified_ledger"
],
"enriched_history_operations": [
"ledger_sequence",
"transaction_id",
"account",
"type"
],
"enriched_meaningful_history_operations": [
"ledger_sequence",
"transaction_id",
"account",
"type"
],
"contract_data": ["last_modified_ledger", "contract_id"],
"contract_code": ["last_modified_ledger", "contract_code_hash"],
"config_settings": ["last_modified_ledger"],
"ttl": ["last_modified_ledger", "key_hash"]
},
"dbt_dataset": "test",
"dbt_full_refresh_models": {
"partnership_assets__account_holders_activity_fact": false,
"partnership_assets__asset_activity_fact": false
},
"dbt_image_name": "stellar/stellar-dbt:b7e0010",
"dbt_job_execution_timeout_seconds": 300,
"dbt_job_retries": 1,
"dbt_mart_dataset": "test_crypto_stellar_dbt",
"dbt_maximum_bytes_billed": 250000000000,
"dbt_project": "test-hubble-319619",
"dbt_target": "test",
"dbt_threads": 12,
"gcs_exported_data_bucket_name": "us-central1-test-hubble-2-5f1f2dbf-bucket",
"gcs_exported_object_prefix": "dag-exported",
"image_name": "stellar/stellar-etl:2f8c44a",
"image_output_path": "/etl/exported_data/",
"image_pull_policy": "IfNotPresent",
"kube_config_location": "",
"kubernetes_sidecar_image": "alpine",
"k8s_namespace": "hubble-composer",
"k8s_service_account": "hubble-composer-service-account",
"local_output_path": "/home/airflow/etlData/",
"output_file_names": {
"accounts": "accounts.txt",
"assets": "assets.txt",
"changes": "changes_folder",
"claimable_balances": "claimable_balances.txt",
"dimAccounts": "dimAccounts.txt",
"dimMarkets": "dimMarkets.txt",
"dimOffers": "dimOffers.txt",
"effects": "effects.txt",
"factEvents": "factEvents.txt",
"ledgers": "ledgers.txt",
"liquidity_pools": "liquidity_pools.txt",
"offers": "offers.txt",
"operations": "operations.txt",
"orderbooks": "orderbook_folder",
"signers": "account_signers.txt",
"trades": "trades.txt",
"transactions": "transactions.txt",
"trustlines": "trustlines.txt",
"contract_data": "contract_data.txt",
"contract_code": "contract_code.txt",
"config_settings": "config_settings.txt",
"ttl": "ttl.txt",
"diagnostic_events": "diagnostic_events.txt"
},
"output_path": "/home/airflow/gcs/data/",
"owner": "SDF",
"partition_fields": {
"enriched_history_operations": {
"type": "MONTH",
"field": "closed_at"
},
"enriched_meaningful_history_operations": {
"type": "MONTH",
"field": "closed_at"
},
"history_assets": {
"type": "MONTH",
"field": "batch_run_date"
},
"accounts": {
"type": "MONTH",
"field": "batch_run_date"
},
"account_signers": {
"type": "MONTH",
"field": "batch_run_date"
},
"claimable_balances": {
"type": "MONTH",
"field": "batch_run_date"
},
"history_effects": {
"type": "MONTH",
"field": "batch_run_date"
},
"history_ledgers": {
"type": "MONTH",
"field": "closed_at"
},
"history_operations": {
"type": "MONTH",
"field": "batch_run_date"
},
"history_trades": {
"type": "MONTH",
"field": "ledger_closed_at"
},
"history_transactions": {
"type": "MONTH",
"field": "batch_run_date"
},
"offers": {
"type": "MONTH",
"field": "batch_run_date"
},
"liquidity_pools": {
"type": "MONTH",
"field": "batch_run_date"
},
"trust_lines": {
"type": "MONTH",
"field": "batch_run_date"
},
"mgi": {
"field": "tran_evnt_date",
"type": "MONTH"
},
"contract_data": {
"field": "closed_at",
"type": "MONTH"
},
"contract_code": {
"field": "closed_at",
"type": "MONTH"
},
"config_settings": {
"field": "closed_at",
"type": "MONTH"
},
"ttl": {
"field": "closed_at",
"type": "MONTH"
}
},
"partners_data": {
"mgi": {
"prefix_folder": "mgi",
"prefix_id": "stellar_transaction_extract",
"table": "raw_mgi_stellar_transactions"
}
},
"partners_bucket": "ext-partner-sftp",
"public_dataset": "test_crypto_stellar",
"public_project": "test-hubble-319619",
"resources": {
"default": {
"requests": {
"cpu": "0.3",
"memory": "900Mi",
"ephemeral-storage": "1Gi"
}
},
"cc": {
"requests": {
"cpu": "0.3",
"memory": "900Mi",
"ephemeral-storage": "1Gi"
}
},
"wocc": {
"requests": {
"cpu": "0.3",
"memory": "900Mi",
"ephemeral-storage": "1Gi"
}
},
"state": {
"requests": {
"cpu": "0.3",
"memory": "900Mi",
"ephemeral-storage": "1Gi"
}
}
},
"sentry_dsn": "https://9e0a056541c3445083329b072f2df690@o14203.ingest.us.sentry.io/6190849",
"sentry_environment": "development",
"schema_filepath": "/home/airflow/gcs/dags/schemas/",
"table_ids": {
"accounts": "accounts",
"assets": "history_assets",
"claimable_balances": "claimable_balances",
"effects": "history_effects",
"ledgers": "history_ledgers",
"liquidity_pools": "liquidity_pools",
"offers": "offers",
"operations": "history_operations",
"signers": "account_signers",
"trades": "history_trades",
"transactions": "history_transactions",
"trustlines": "trust_lines",
"enriched_history_operations": "enriched_history_operations",
"contract_data": "contract_data",
"contract_code": "contract_code",
"config_settings": "config_settings",
"ttl": "ttl"
},
"task_timeout": {
"build_batch_stats": 180,
"build_bq_insert_job": 180,
"build_copy_table": 180,
"build_dbt_task": 300,
"build_delete_data_task": 180,
"build_export_task": 180,
"build_gcs_to_bq_task": 300,
"build_time_task": 120
},
"dbt_tables": {
"signers_current": "account_signers_current",
"accounts_current": "accounts_current",
"config_settings_current": "config_settings_current",
"contract_code_current": "contract_code_current",
"contract_data_current": "contract_data_current",
"liquidity_pools_current": "liquidity_pools_current",
"offers_current": "offers_current",
"trustlines_current": "trust_lines_current",
"ttl_current": "ttl_current"
},
"use_testnet": "True",
"sandbox_dataset": "crypto_stellar_internal_sandbox",
"volume_config": {},
"volume_name": "etl-data",
"use_futurenet": "False",
"currency_ohlc": {
"currency": "euro_ohlc",
"table_name": "euro_usd_ohlc",
"endpoint": "https://api.coingecko.com/api/v3/coins/tether-eurt/ohlc?vs_currency=usd&days=1",
"columns_ohlc_currency": ["time", "open", "high", "low", "close"]
},
"currency_bucket": "currencies_ohlc",
"max_db_entry_age_in_days": 90,
"internal_source_db": "test-hubble-319619",
"internal_source_schema": "test_crypto_stellar_internal",
"public_source_db": "test-hubble-319619",
"public_source_schema": "test_crypto_stellar",
"slack_elementary_channel": "stellar-elementary-alerts",
"elementary_secret": "slack-token-elementary",
"dbt_elementary_dataset": "test_elementary",
"use_captive_core": "False",
"txmeta_datastore_path": "sdf-ledger-close-metas/ledgers"
}