From 3bfe4c4507bc4cb20d0b3dc4c26f01674a56ad69 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:44:27 +0800 Subject: [PATCH 01/19] Update config.yml line 95 --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 037645ab2..bc9e9eecc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,7 +92,8 @@ jobs: - run: name: Build and run Dockerfile (see https://docs.railway.app/deploy/dockerfiles) command: | - cd section-08-deploying-with-containers && railway up --detach + cd section-08-deploying-with-containers + railway up --detach --verbose test_regression_model_py37: docker: From 21500dd7bec069805c9f15c5d21a42ef6cbb9773 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:13:31 +0800 Subject: [PATCH 02/19] Update config.yml line 95 --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc9e9eecc..037645ab2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,8 +92,7 @@ jobs: - run: name: Build and run Dockerfile (see https://docs.railway.app/deploy/dockerfiles) command: | - cd section-08-deploying-with-containers - railway up --detach --verbose + cd section-08-deploying-with-containers && railway up --detach test_regression_model_py37: docker: From 996292e463f492c2c37f840ac7e6dd76567e2215 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:13:54 +0800 Subject: [PATCH 03/19] Update workflows --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 037645ab2..c40cd6b2e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -274,7 +274,7 @@ workflows: filters: branches: only: - - master + - main - demo # upload after git tags are created - section_07_test_and_upload_regression_model: @@ -284,7 +284,7 @@ workflows: filters: branches: only: - - master + - main - demo # test-all: From d151d0b1d62950a7c551b024082ee74e1a1bc303 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:27:51 +0800 Subject: [PATCH 04/19] write comment on line 274 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c40cd6b2e..ced79025a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,11 +271,11 @@ workflows: - section_07_deploy_app_to_railway: requires: - section_07_test_app - filters: - branches: - only: - - main - - demo + #filters: + #branches: + # only: + # - main + # - demo # upload after git tags are created - section_07_test_and_upload_regression_model: <<: *tags_only From 096b48ee301e24faa2341f020d731fd549cf01f7 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:34:20 +0800 Subject: [PATCH 05/19] add pip extra index url --- .../house-prices-api/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/section-07-ci-and-publishing/house-prices-api/requirements.txt b/section-07-ci-and-publishing/house-prices-api/requirements.txt index c6f00d968..1a75fc090 100644 --- a/section-07-ci-and-publishing/house-prices-api/requirements.txt +++ b/section-07-ci-and-publishing/house-prices-api/requirements.txt @@ -1,3 +1,5 @@ +--extra-index-url=${PIP_EXTRA_INDEX_URL} + uvicorn>=0.20.0,<0.30.0 fastapi>=0.88.0,<1.0.0 python-multipart>=0.0.5,<0.1.0 @@ -6,4 +8,4 @@ typing_extensions>=4.2.0,<5.0.0 loguru>=0.5.3,<1.0.0 # We will explain this in the course tid-regression-model>=3.2.0 -feature-engine>=1.0.2,<1.6.0 # breaking change in v1.6.0 \ No newline at end of file +feature-engine>=1.0.2,<1.6.0 # breaking change in v1.6.0 From 19680327a6f1d398d7656fb36757fada424b08b0 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:57:37 +0800 Subject: [PATCH 06/19] Update railway project id --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ced79025a..c6288ca31 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | - cd section-07-ci-and-publishing/house-prices-api && railway up --detach + cd section-07-ci-and-publishing/house-prices-api && railway up --project c7e97216-7edf-43f8-86e3-9029b0fd7fd5 --detach section_07_test_and_upload_regression_model: <<: *defaults From 4f4acb01e5a053b0603d89e42ec4a8769903ab06 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:05:51 +0800 Subject: [PATCH 07/19] Update config.yml --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6288ca31..13f2947b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,8 @@ jobs: - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | - cd section-07-ci-and-publishing/house-prices-api && railway up --project c7e97216-7edf-43f8-86e3-9029b0fd7fd5 --detach + cd section-07-ci-and-publishing/house-prices-api railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 railway up --detach + section_07_test_and_upload_regression_model: <<: *defaults From b41ce8d834bf2cf5caefa7f7850cafdefa9e22bf Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:24:39 +0800 Subject: [PATCH 08/19] Update command railway --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13f2947b5..44dd3e908 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | - cd section-07-ci-and-publishing/house-prices-api railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 railway up --detach + cd section-07-ci-and-publishing/house-prices-api && railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 railway up --detach section_07_test_and_upload_regression_model: From db6117cbbda7fbdf49813ff51339787a8a7e0401 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:37:29 +0800 Subject: [PATCH 09/19] Update link --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44dd3e908..f6b9f9a5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,11 @@ jobs: node-version: '16.13' - run: node --version - run: npm i -g @railway/cli + - run: + name: Link Railway Project + command: | + cd section-07-ci-and-publishing/house-prices-api && + railway link https://railway.app/project/c7e97216-7edf-43f8-86e3-9029b0fd7fd5 - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | From 3b97bf4eabef15ca9f3e1c0a3b78df2891975eca Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:27:40 +0800 Subject: [PATCH 10/19] rewrite back the railway --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6b9f9a5d..3cbe2a477 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,8 +54,7 @@ jobs: - run: name: Link Railway Project command: | - cd section-07-ci-and-publishing/house-prices-api && - railway link https://railway.app/project/c7e97216-7edf-43f8-86e3-9029b0fd7fd5 + cd section-07-ci-and-publishing/house-prices-api && railway up --detach - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | From 44778a6991e9cb4a83b1bb8e4ec6cacc33b1c8f1 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:40:35 +0800 Subject: [PATCH 11/19] updated service name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cbe2a477..84d948acc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: - run: name: Link Railway Project command: | - cd section-07-ci-and-publishing/house-prices-api && railway up --detach + cd section-07-ci-and-publishing/house-prices-api && railway up --service deployment CircleCi --detach - run: name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) command: | From f3ba84ca9905acdd99090a22c3c68adbff9c133b Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:14:21 +0800 Subject: [PATCH 12/19] Update requirements --- section-07-ci-and-publishing/house-prices-api/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/section-07-ci-and-publishing/house-prices-api/requirements.txt b/section-07-ci-and-publishing/house-prices-api/requirements.txt index 1a75fc090..720f311d0 100644 --- a/section-07-ci-and-publishing/house-prices-api/requirements.txt +++ b/section-07-ci-and-publishing/house-prices-api/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url=${PIP_EXTRA_INDEX_URL} uvicorn>=0.20.0,<0.30.0 fastapi>=0.88.0,<1.0.0 From f530b02821688fcb646854c90dd2f5dbb253cc52 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:34:50 +0800 Subject: [PATCH 13/19] new section 7 railway --- .circleci/config.yml | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84d948acc..9db48fad4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,23 +42,28 @@ jobs: command: | tox - section_07_deploy_app_to_railway: - <<: *defaults - steps: - - checkout: - path: ~/project/ - - node/install: - node-version: '16.13' - - run: node --version - - run: npm i -g @railway/cli - - run: - name: Link Railway Project - command: | - cd section-07-ci-and-publishing/house-prices-api && railway up --service deployment CircleCi --detach - - run: - name: Deploy to Railway App (You must set RAILWAY_TOKEN env var) - command: | - cd section-07-ci-and-publishing/house-prices-api && railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 railway up --detach + section_07_deploy_app_to_railway: + <<: *defaults + steps: + - checkout: + path: ~/project/ + - node/install: + node-version: '16.13' + - run: node --version + - run: npm i -g @railway/cli + - run: + name: Verify Railway CLI Installation + command: railway --version + - run: + name: Link Railway Project + command: | + cd section-07-ci-and-publishing/house-prices-api + railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 + - run: + name: Deploy to Railway + command: | + cd section-07-ci-and-publishing/house-prices-api + railway up --detach --verbose section_07_test_and_upload_regression_model: From 816d69c3ee0c6503fdf08c50f09d06cd650cea95 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:35:42 +0800 Subject: [PATCH 14/19] Update config.yml --- .circleci/config.yml | 70 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9db48fad4..08941d8a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,49 +42,49 @@ jobs: command: | tox - section_07_deploy_app_to_railway: - <<: *defaults - steps: - - checkout: - path: ~/project/ - - node/install: - node-version: '16.13' - - run: node --version - - run: npm i -g @railway/cli - - run: - name: Verify Railway CLI Installation - command: railway --version - - run: - name: Link Railway Project - command: | - cd section-07-ci-and-publishing/house-prices-api - railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 - - run: - name: Deploy to Railway - command: | - cd section-07-ci-and-publishing/house-prices-api - railway up --detach --verbose - - - section_07_test_and_upload_regression_model: + section_07_deploy_app_to_railway: <<: *defaults - working_directory: ~/project/section-07-ci-and-publishing/model-package steps: - checkout: - path: ~/project - - *prepare_tox + path: ~/project/ + - node/install: + node-version: '16.13' + - run: node --version + - run: npm i -g @railway/cli - run: - name: Fetch the data - command: | - tox -e fetch_data + name: Verify Railway CLI Installation + command: railway --version - run: - name: Test the model + name: Link Railway Project command: | - tox + cd section-07-ci-and-publishing/house-prices-api + railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 - run: - name: Publish model to Gemfury + name: Deploy to Railway command: | - tox -e publish_model + cd section-07-ci-and-publishing/house-prices-api + railway up --detach --verbose + + + section_07_test_and_upload_regression_model: + <<: *defaults + working_directory: ~/project/section-07-ci-and-publishing/model-package + steps: + - checkout: + path: ~/project + - *prepare_tox + - run: + name: Fetch the data + command: | + tox -e fetch_data + - run: + name: Test the model + command: | + tox + - run: + name: Publish model to Gemfury + command: | + tox -e publish_model section_08_deploy_app_container_via_railway: From 41805106f14e386dc9c78e548c2c451eb75a0ea4 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:46:14 +0800 Subject: [PATCH 15/19] Update full tukar --- .circleci/config.yml | 131 +++++++++++++------------------------------ 1 file changed, 38 insertions(+), 93 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08941d8a3..345e22636 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,11 +38,11 @@ jobs: path: ~/project - *prepare_tox - run: - name: Runnning app tests + name: Running app tests command: | tox - section_07_deploy_app_to_railway: + section_07_deploy_app_to_railway: <<: *defaults steps: - checkout: @@ -65,33 +65,30 @@ jobs: cd section-07-ci-and-publishing/house-prices-api railway up --detach --verbose - - section_07_test_and_upload_regression_model: - <<: *defaults - working_directory: ~/project/section-07-ci-and-publishing/model-package - steps: - - checkout: - path: ~/project - - *prepare_tox - - run: - name: Fetch the data - command: | - tox -e fetch_data - - run: - name: Test the model - command: | - tox - - run: - name: Publish model to Gemfury - command: | - tox -e publish_model - + section_07_test_and_upload_regression_model: + <<: *defaults + working_directory: ~/project/section-07-ci-and-publishing/model-package + steps: + - checkout: + path: ~/project + - *prepare_tox + - run: + name: Fetch the data + command: | + tox -e fetch_data + - run: + name: Test the model + command: | + tox + - run: + name: Publish model to Gemfury + command: | + tox -e publish_model section_08_deploy_app_container_via_railway: <<: *defaults steps: - setup_remote_docker: - # Supported versions: https://circleci.com/docs/2.0/building-docker-images/#docker-version version: 20.10.18 - checkout: path: ~/project/ @@ -195,7 +192,7 @@ jobs: pip install -r packages/ml_api/diff_test_requirements.txt PYTHONPATH=./packages/ml_api python3 packages/ml_api/tests/capture_model_predictions.py - run: - name: Runnning differential tests + name: Running differential tests command: | . venv/bin/activate pip install -r packages/ml_api/requirements.txt @@ -228,12 +225,12 @@ jobs: - checkout - setup_remote_docker - run: - name: Publishing docker image to aws ECR + name: Publishing docker image to AWS ECR command: | sudo pip install awscli eval $(aws ecr get-login --no-include-email --region us-east-1) make build-ml-api-aws tag-ml-api push-ml-api-aws - aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition first-run-task-definition --force-new-deployment + aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition first-run-task-definition --force-new-deployment section_13_train_and_upload_neural_network_model: docker: @@ -265,7 +262,6 @@ jobs: chmod +x ./scripts/publish_model.sh ./scripts/publish_model.sh ./packages/neural_network_model/ - tags_only: &tags_only filters: branches: @@ -281,71 +277,20 @@ workflows: - section_07_deploy_app_to_railway: requires: - section_07_test_app - #filters: - #branches: - # only: - # - main - # - demo - # upload after git tags are created - section_07_test_and_upload_regression_model: <<: *tags_only - - section_08_deploy_app_container_via_railway: - filters: - branches: - only: - - main - - demo - -# test-all: -# jobs: -# - test_regression_model_py36 -# - test_regression_model_py37 -# - test_regression_model_py38 -# - test_ml_api_py36 -# - test_ml_api_py37 -# # - test_ml_api_py38 pending NN model update -# - section_9_differential_tests -# - train_and_upload_regression_model: -# requires: -# - test_regression_model_py36 -# - test_regression_model_py37 -# - test_regression_model_py38 -# - test_ml_api_py36 -# - test_ml_api_py37 -# - section_9_differential_tests -# filters: -# branches: -# only: -# - master -# - section_10_deploy_to_heroku: -# requires: -# - train_and_upload_regression_model -# filters: -# branches: -# only: -# - master -# - section_11_build_and_push_to_heroku_docker: -# requires: -# - train_and_upload_regression_model -# filters: -# branches: -# only: -# - master -# - section_12_publish_docker_image_to_aws: -# requires: -# - train_and_upload_regression_model -# filters: -# branches: -# only: -# - master -# - section_13_train_and_upload_neural_network_model: -# requires: -# - test_regression_model -# - test_ml_api -# - section_9_differential_tests -# - train_and_upload_regression_model -# filters: -# branches: -# only: -# - master + filters: + branches: + only: + - main + - section_11_build_and_push_to_heroku_docker: + filters: + branches: + only: + - main + - section_12_publish_docker_image_to_aws: + filters: + branches: + only: + - main From da22c6322f2ce7afca7990659c74e7853d75fbc7 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:04:50 +0800 Subject: [PATCH 16/19] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 345e22636..481610072 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ jobs: name: Link Railway Project command: | cd section-07-ci-and-publishing/house-prices-api - railway link c7e97216-7edf-43f8-86e3-9029b0fd7fd5 + railway link - run: name: Deploy to Railway command: | From 613da3df3527f0f464f15fca9be4c9a363c85ced Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:09:43 +0800 Subject: [PATCH 17/19] Updated config.yml --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 481610072..f1c1dc998 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,9 @@ jobs: - run: name: Verify Railway CLI Installation command: railway --version + - run: + name: Railway token link + command: RAILWAY_TOKEN= fc1f0cdb-a81d-402e-9677-049f531d04a0 railway run - run: name: Link Railway Project command: | @@ -63,7 +66,7 @@ jobs: name: Deploy to Railway command: | cd section-07-ci-and-publishing/house-prices-api - railway up --detach --verbose + railway up --detach section_07_test_and_upload_regression_model: <<: *defaults From 11ec71bbf84b7545f4176dd350d288d08ba4762e Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:15:22 +0800 Subject: [PATCH 18/19] Updated config.yml --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f1c1dc998..11c2591f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,11 +57,6 @@ jobs: - run: name: Railway token link command: RAILWAY_TOKEN= fc1f0cdb-a81d-402e-9677-049f531d04a0 railway run - - run: - name: Link Railway Project - command: | - cd section-07-ci-and-publishing/house-prices-api - railway link - run: name: Deploy to Railway command: | From 27b480c3065606400ec8132874a62ea6808940e9 Mon Sep 17 00:00:00 2001 From: Shahada <121425395+sitishahada@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:31:44 +0800 Subject: [PATCH 19/19] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11c2591f6..c82c9a4f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,7 +56,7 @@ jobs: command: railway --version - run: name: Railway token link - command: RAILWAY_TOKEN= fc1f0cdb-a81d-402e-9677-049f531d04a0 railway run + command: RAILWAY_TOKEN=fc1f0cdb-a81d-402e-9677-049f531d04a0 railway run - run: name: Deploy to Railway command: |