adding dlthub (#152)

Makefile

@@ -9,6 +9,7 @@ build:
 	mkdir -p data/data_catalog/analysis
 
 run:
+	cd dlt && python3 nba_pipeline.py
 	cd transform && dbt build
 	cd evidence && npm run sources
 
@@ -35,4 +36,4 @@ docker-run-evidence:
 		--env MDS_LATEST_RATINGS=true \
 		--env MDS_ENABLE_EXPORT=true \
 		--env ENVIRONMENT=docker \
-		mdsbox make run serve
+		mdsbox make run serve

data/nba/nba_team_ratings.csv

@@ -1,31 +1,31 @@
-team,team_long,conf,elo_rating,win_total,tournament_group
-BOS,Boston Celtics,East,1710,54.7,east_c
-MIL,Milwaukee Bucks,East,1702,53.9,east_b
-MIA,Miami Heat,East,1624,45.3,east_b
-BRK,Brooklyn Nets,East,1563,37.3,east_c
-PHO,Phoenix Suns,West,1684,51.5,west_a
-LAC,Los Angeles Clippers,West,1646,46.5,west_b
-PHI,Philadelphia 76ers,East,1652,48.1,east_a
-DEN,Denver Nuggets,West,1694,52.5,west_b
-GSW,Golden State Warriors,West,1661,48.3,west_c
-DAL,Dallas Mavericks,West,1625,44.3,west_b
-MEM,Memphis Grizzlies,West,1638,45.9,west_a
-UTA,Utah Jazz,West,1552,35.5,west_a
-ATL,Atlanta Hawks,East,1603,42.5,east_a
-NOP,New Orleans Pelicans,West,1626,44.5,west_b
-TOR,Toronto Raptors,East,1554,36.5,east_c
-MIN,Minnesota Timberwolves,West,1629,44.7,west_c
-LAL,Los Angeles Lakers,West,1654,47.5,west_a
-POR,Portland Trail Blazers,West,1491,28.5,west_a
-CHI,Chicago Bulls,East,1563,37.5,east_c
-CHO,Charlotte Hornets,East,1506,30.9,east_b
-NYK,New York Knicks,East,1631,45.5,east_b
-IND,Indiana Pacers,East,1573,38.7,east_a
-CLE,Cleveland Cavaliers,East,1671,50.5,east_a
-SAC,Sacramento Kings,West,1631,44.5,west_c
-SAS,San Antonio Spurs,West,1498,29.1,west_c
-WAS,Washington Wizards,East,1445,24.5,east_b
-ORL,Orlando Magic,East,1555,36.7,east_c
-DET,Detroit Pistons,East,1475,27.5,east_a
-OKC,Oklahoma City Thunder,West,1626,44.5,west_c
-HOU,Houston Rockets,West,1516,31.5,west_b
+team,team_long,conf,elo_rating,win_total,tournament_group,alt_key
+BOS,Boston Celtics,East,1710,54.7,east_c,BOS
+MIL,Milwaukee Bucks,East,1702,53.9,east_b,MIL
+MIA,Miami Heat,East,1624,45.3,east_b,MIA
+BRK,Brooklyn Nets,East,1563,37.3,east_c,BKN
+PHO,Phoenix Suns,West,1684,51.5,west_a,PHX
+LAC,Los Angeles Clippers,West,1646,46.5,west_b,LAC
+PHI,Philadelphia 76ers,East,1652,48.1,east_a,PHI
+DEN,Denver Nuggets,West,1694,52.5,west_b,DEN
+GSW,Golden State Warriors,West,1661,48.3,west_c,GSW
+DAL,Dallas Mavericks,West,1625,44.3,west_b,DAL
+MEM,Memphis Grizzlies,West,1638,45.9,west_a,MEM
+UTA,Utah Jazz,West,1552,35.5,west_a,UTA
+ATL,Atlanta Hawks,East,1603,42.5,east_a,ATL
+NOP,New Orleans Pelicans,West,1626,44.5,west_b,NOP
+TOR,Toronto Raptors,East,1554,36.5,east_c,TOR
+MIN,Minnesota Timberwolves,West,1629,44.7,west_c,MIN
+LAL,Los Angeles Lakers,West,1654,47.5,west_a,LAL
+POR,Portland Trail Blazers,West,1491,28.5,west_a,POR
+CHI,Chicago Bulls,East,1563,37.5,east_c,CHI
+CHO,Charlotte Hornets,East,1506,30.9,east_b,CHA
+NYK,New York Knicks,East,1631,45.5,east_b,NYK
+IND,Indiana Pacers,East,1573,38.7,east_a,IND
+CLE,Cleveland Cavaliers,East,1671,50.5,east_a,CLE
+SAC,Sacramento Kings,West,1631,44.5,west_c,SAC
+SAS,San Antonio Spurs,West,1498,29.1,west_c,SAS
+WAS,Washington Wizards,East,1445,24.5,east_b,WAS
+ORL,Orlando Magic,East,1555,36.7,east_c,ORL
+DET,Detroit Pistons,East,1475,27.5,east_a,DET
+OKC,Oklahoma City Thunder,West,1626,44.5,west_c,OKC
+HOU,Houston Rockets,West,1516,31.5,west_b,HOU

dlt/.dlt/.sources

@@ -0,0 +1,2 @@
+engine_version: 1
+sources:

dlt/.dlt/config.toml

@@ -0,0 +1,6 @@
+# put your configuration values here
+
+[runtime]
+log_level="WARNING"  # the system log level of dlt
+# use the dlthub_telemetry setting to enable/disable anonymous usage data reporting, see https://dlthub.com/docs/telemetry
+dlthub_telemetry = true

dlt/.gitignore

@@ -0,0 +1,10 @@
+# ignore secrets, virtual environments and typical python compilation artifacts
+secrets.toml
+# ignore basic python artifacts
+.env
+**/__pycache__/
+**/*.py[cod]
+**/*$py.class
+# ignore duckdb
+*.duckdb
+*.wal

dlt/nba_pipeline.py

@@ -0,0 +1,22 @@
+import dlt
+from dlt.sources.helpers import requests
+
+# Specify the URL of the API endpoint
+url = "https://api.pbpstats.com/get-games/nba?Season=2023-24&SeasonType=Regular%20Season"
+# Make a request and check if it was successful
+response = requests.get(url)
+response.raise_for_status()
+
+pipeline = dlt.pipeline(
+    pipeline_name="nba_pipeline",
+    destination="duckdb",
+    dataset_name="nba_data",
+)
+# The response contains a list of issues
+load_info = pipeline.run(
+    response.json().get('results', []),
+    table_name="games", write_disposition="replace", 
+    destination=dlt.destinations.duckdb(credentials="../data/data_catalog/mdsbox.duckdb")
+    )
+
+print(load_info)