Update

Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Rename Add placeholder Move ECR to Makefile and split AWS_ACCOUNT_ID Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD Test CI/CD
kaboomshebang · Apr 19, 2023 · 89a60b9 · 89a60b9
1 parent c117eb6
commit 89a60b9
Show file tree

Hide file tree

Showing 7 changed files with 104 additions and 46 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -5,14 +5,14 @@ on:
   push:
     branches:
     # - main
-    - fs-117-ci-cd # test CI/CD config
+    - fs-120-ci-cd-backend # test CI/CD config
 
 jobs:
   deploy-frontend:
     runs-on: ubuntu-latest
     env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      SCW_ACCESS_KEY_ID: ${{ secrets.SCW_ACCESS_KEY_ID }}
+      SCW_SECRET_ACCESS_KEY: ${{ secrets.SCW_SECRET_ACCESS_KEY }}
       CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
       CLOUDFLARE_ZONE: ${{ secrets.CLOUDFLARE_ZONE }}
       RCLONE_SYNC_DIR: dist
@@ -29,12 +29,13 @@ jobs:
           node-version: '18'
       - name: Install `pnpm`
         run: npm install -g pnpm
+        working-directory: ./app
       - name: Install dependencies
         run: pnpm install
+        working-directory: ./app
       - name: Build frontend
         run: make build
-      - name: List dist folder
-        run: ls dist
+        working-directory: ./app
 
       # Upload to Scaleway
       - name: Install Rclone
@@ -51,12 +52,17 @@ jobs:
             echo "endpoint = s3.nl-ams.scw.cloud" >> rclone.conf
             echo "acl = private" >> rclone.conf
             echo "bucket_acl = private" >> rclone.conf
+        working-directory: ./app
       - name: Verify `rlcone.conf`
         run: cat rclone.conf
-      - name: Set bucket authentication
-        run: export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
-      - name: Copy `dist` folder to Scaleway bucket
-        run: rclone sync --auto-confirm --config="rclone.conf" ./"$RCLONE_SYNC_DIR"/ "$RCLONE_REMOTE":"$RCLONE_BUCKET_NAME"
+        working-directory: ./app
+      - name: Rclone copy `dist` folder to Scaleway bucket
+        run: |
+          # Set bucket authentication (Rclone expects AWS_)
+          export AWS_ACCESS_KEY_ID=$SCW_ACCESS_KEY_ID
+          export AWS_SECRET_ACCESS_KEY=$SCW_SECRET_ACCESS_KEY
+          rclone sync --auto-confirm --config="rclone.conf" ./"$RCLONE_SYNC_DIR"/ "$RCLONE_REMOTE":"$RCLONE_BUCKET_NAME"
+        working-directory: ./app
 
       # Cloudflare cache
       - name: Purge Cloudflare cache
@@ -65,12 +71,55 @@ jobs:
 
   deploy-backend:
     runs-on: ubuntu-latest
+    env:
+      # .aws/.env vars
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      # .env vars
+      CORS_DOMAIN: ${{ vars.CORS_DOMAIN }}
+      # make.docker vars
+      AWS_ARN: ${{ vars.AWS_ACCOUNT_ID }}
+      AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
+      AWS_REGION: ${{ vars.AWS_REGION }}
+      LAMBDA_FUNCTION_NAME: ${{ vars.LAMBDA_FUNCTION_NAME }}
+      ECR_DOCKER_REPO_NAME: ${{ vars.ECR_DOCKER_REPO_NAME }}
+      ECR_DOCKER_IMAGE_NAME: ${{ vars.ECR_DOCKER_IMAGE_NAME }}
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
       - name: Install Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-      - run: python --version
-      - run: make --version
+
+      - name: Create `.aws` credentials file
+        run: |
+          echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" > .env
+          echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env
+        working-directory: ./lambdas/todos/.aws
+
+      - name: Create `.env` file
+        run: |
+          echo "CORS_DOMAIN=$CORS_DOMAIN" > .env
+          # set the Airtable environment variables manually
+        working-directory: ./lambdas/todos
+
+      - name: Create `.make.docker` config file
+        run: |
+          echo "ECR_DOCKER_REPO_NAME=$ECR_DOCKER_REPO_NAME" > .make.docker
+          echo "ECR_DOCKER_IMAGE_NAME=$ECR_DOCKER_IMAGE_NAME" >> .make.docker
+          echo "LAMBDA_FUNCTION_NAME=$LAMBDA_FUNCTION_NAME" >> .make.docker
+          echo "AWS_REGION=$AWS_REGION" >> .make.docker
+          echo "AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID" >> .make.docker
+          echo "AWS_ARN=$AWS_ACCOUNT_ID" >> .make.docker
+        working-directory: ./lambdas/todos
+
+      - name: Login to ECR Docker registry
+        run: |
+          make auth
+        working-directory: ./lambdas/todos
+
+      - name: Push changes and update function
+        run: |
+          make update
+        working-directory: ./lambdas/todos
diff --git a/docs/README.md b/docs/README.md
@@ -326,5 +326,6 @@ Update CI/CD
 	- first follow all the above steps to create a manual deployment
 		- so that all the environment variables and policies are set
 	- edit `.github/workflows/deploy.yml`
-	- set all the correct environment variables
+		- set all the correct environment variables
+	- modify code
 	- push to/merge with `main`
diff --git a/lambdas/todos/.aws/.env.template b/lambdas/todos/.aws/.env.template
@@ -6,4 +6,7 @@ export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXX
 export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 export AWS_DEFAULT_REGION=us-central-1
 
+# get your AWS_ACCOUNT_ID
+# aws sts get-caller-identity
+
 # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
diff --git a/lambdas/todos/.env.example → lambdas/todos/.env.template b/lambdas/todos/.env.example → lambdas/todos/.env.template
diff --git a/lambdas/todos/.make.docker b/lambdas/todos/.make.docker
@@ -1,7 +1,7 @@
-# Docker Make config
-region=eu-central-1
-ecr=aws_account_id.dkr.ecr.region.amazonaws.com
-docker_repo=kbsb-demodash
-image_name=kbsb-demodash-todos
-function_name=todos
-arn=100000000000
+# Docker Make config variables
+ECR_DOCKER_REPO_NAME=kbsb-demodash
+ECR_DOCKER_IMAGE_NAME=kbsb-demodash-todos
+LAMBDA_FUNCTION_NAME=todos
+AWS_REGION=eu-central-1
+AWS_ACCOUNT_ID=100000000000
+AWS_ARN=100000000000
diff --git a/lambdas/todos/.make.pytest b/lambdas/todos/.make.pytest
@@ -1,4 +1,4 @@
-# Pytest Make config
-url_local=http://localhost:8000
-url_docker=http://localhost:9000/2015-03-31/functions/function/invocations
-url_lambda=https://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX.lambda-url.eu-central-1.on.aws
+# Pytest Make config variables
+PYTEST_URL_LOCAL=http://localhost:8000
+PYTEST_URL_DOCKER=http://localhost:9000/2015-03-31/functions/function/invocations
+PYTEST_URL_LAMBDA=https://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX.lambda-url.eu-central-1.on.aws
diff --git a/lambdas/todos/Makefile b/lambdas/todos/Makefile
@@ -6,35 +6,40 @@ include .make.pytest
 # all targets refer to tasks (instead of files)
 .PHONY: *
 
+# set Make variables
+ECR_DOMAIN := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
+
+
 ######## FASTAPI local development
 
 # app:app refers to app.py app function
 api:
 	export LOCAL=1 && uvicorn --host 0.0.0.0 app:app --reload
 
 pytest-local:
-	export TEST_URL="$(url_local)" && pytest -s -k test_endpoints
+	export TEST_URL="$(PYTEST_URL_LOCAL)" && pytest -s -k test_endpoints
 # `pytest -s` includes stdout
 
 pytest-docker:
-	export TEST_URL="$(url_docker)"; export TEST_DOCKER=1 && pytest -s -k test_endpoints
+	export TEST_URL="$(PYTEST_URL_DOCKER)"; export TEST_DOCKER=1 && pytest -s -k test_endpoints
 
 pytest-prod:
-	export TEST_URL="$(url_lambda)" && pytest -s -k test_endpoints
+	export TEST_URL="$(PYTEST_URL_LAMBDA)" && pytest -s -k test_endpoints
 
 ######## AWS Authentication
 
 login:
 	aws sso login
 
+# aws ecr get-login-password --region $(AWS_region) | docker login --username AWS --password-stdin .dkr.ecr.eu-central-1.amazonaws.com
 auth:
-	aws ecr get-login-password --region "$(region)" | docker login --username AWS --password-stdin "$(ecr)"
+	aws ecr get-login-password --region $(AWS_REGION) | docker login --username AWS --password-stdin $(ECR_DOMAIN)
 
 repo:
 	aws ecr create-repository \
-		--repository-name "$(docker_repo)" \
+		--repository-name "$(ECR_DOCKER_REPO_NAME)" \
 		--image-scanning-configuration scanOnPush=true \
-		--region "$(region)"
+		--region "$(AWS_REGION)"
 
 iamrole:
 	aws iam create-role --role-name lambda-ex --assume-role-policy-document '{"Version": "2012-10-17","Statement": [{ "Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"}]}'
@@ -44,59 +49,59 @@ iamrole:
 ######## DOCKER tasks
 
 build:
-	docker build -t "$(image_name)" .
+	docker build -t "$(ECR_DOCKER_IMAGE_NAME)" .
 
 run:
-	docker run --env-file .env -e LOG_LEVEL=DEBUG -e LOCAL=1 -p 9000:8080 "$(image_name)"
+	docker run --env-file .env -e LOG_LEVEL=DEBUG -e LOCAL=1 -p 9000:8080 "$(ECR_DOCKER_IMAGE_NAME)"
 
 shell:
-	docker run -it --entrypoint /bin/bash -p 9000:8080 "$(image_name)"
+	docker run -it --entrypoint /bin/bash -p 9000:8080 "$(ECR_DOCKER_IMAGE_NAME)"
 
 logs:
-	docker logs "$(image_name)"
+	docker logs "$(ECR_DOCKER_IMAGE_NAME)"
 
 tag:
-	docker tag "$(image_name)" "$(ecr)"/"$(docker_repo)"
+	docker tag "$(ECR_DOCKER_IMAGE_NAME)" "$(ECR_DOMAIN)"/"$(ECR_DOCKER_REPO_NAME)"
 
 push:
-	docker push "$(ecr)"/"$(docker_repo)"
+	docker push "$(ECR_DOMAIN)"/"$(ECR_DOCKER_REPO_NAME)"
 
 ######## AWS tasks
 
 lambda-create:
 	aws lambda create-function \
-	--region "$(region)" \
-	--function-name "$(function_name)" \
+	--region "$(AWS_REGION)" \
+	--function-name "$(LAMBDA_FUNCTION_NAME)" \
     --package-type Image \
-    --code ImageUri="$(ecr)"/"$(docker_repo)":latest \
-    --role arn:aws:iam::$(arn):role/lambda-ex
+    --code ImageUri="$(ECR_DOMAIN)"/"$(ECR_DOCKER_REPO_NAME)":latest \
+    --role arn:aws:iam::$(AWS_ARN):role/lambda-ex
 
 lambda-url:
 	aws lambda create-function-url-config \
-	--function-name "$(function_name)" \
+	--function-name "$(LAMBDA_FUNCTION_NAME)" \
 	--auth-type NONE
 
 lambda-public:
 	aws lambda add-permission \
-    --function-name "$(function_name)" \
+    --function-name "$(LAMBDA_FUNCTION_NAME)" \
     --action lambda:invokeFunctionUrl \
     --statement-id FunctionURLAllowPublicAccess \
     --principal "*" \
 	--cli-input-json '{"FunctionUrlAuthType":"NONE"}'
 
 lambda-update:
 	aws lambda update-function-code \
-	--region "$(region)" \
-	--function-name "$(function_name)" \
-    --image-uri "$(ecr)"/"$(docker_repo)":latest
+	--region "$(AWS_REGION)" \
+	--function-name "$(LAMBDA_FUNCTION_NAME)" \
+    --image-uri "$(ECR_DOMAIN)"/"$(ECR_DOCKER_REPO_NAME)":latest
 
 lambda-env:
 	aws lambda update-function-configuration \
-	--function-name "$(function_name)" \
+	--function-name "$(LAMBDA_FUNCTION_NAME)" \
 	--environment Variables={AIRTABLE_API_KEY="$(AIRTABLE_API_KEY)",AIRTABLE_BASE_ID="$(AIRTABLE_BASE_ID)",AIRTABLE_TABLE_NAME="$(AIRTABLE_TABLE_NAME)"}
 
 invoke:
-	aws lambda invoke --function-name "$(function_name)" output.json
+	aws lambda invoke --function-name "$(LAMBDA_FUNCTION_NAME)" output.json
 
 # run update to publish changes
 update: build tag push lambda-update