Create a Streaming Data Lake on Cloud Storage: Challenge Lab

export BUCKET_NAME=
export TOPIC_ID=
export PROJECT_ID=

gcloud config set compute/region "[REGION]"

gcloud services disable dataflow.googleapis.com

gcloud services enable dataflow.googleapis.com

PROJECT_ID=$(gcloud config get-value project)
REGION=us-central1
AE_REGION=us-central


// Task 1 //

gcloud pubsub topics create $TOPIC_ID


// Task 2 //

gcloud app create --region=$AE_REGION

gcloud scheduler jobs create pubsub publisher-job --schedule="* * * * *" \
    --topic=$TOPIC_ID --message-body="[MESSAGE]"

gcloud scheduler jobs run publisher-job


// Task 3 //

gsutil mb gs://$BUCKET_NAME


// Task 4 //

docker run -it -e DEVSHELL_PROJECT_ID=$DEVSHELL_PROJECT_ID python:3.7 /bin/bash

git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git

cd python-docs-samples/pubsub/streaming-analytics

pip install -U -r requirements.txt  # Install Apache Beam dependencies

python PubSubToGCS.py \
    --project=PROJECT_ID \
    --region=REGION \
    --input_topic=projects/PROJECT_ID/topics/TOPIC_ID \
    --output_path=gs://BUCKET_NAME/samples/output \
    --runner=DataflowRunner \
    --window_size=2 \
    --num_shards=2 \
    --temp_location=gs://BUCKET_NAME/temp