From dcddb59ba9158af78cf1c3c5e18c6e23cb0488f3 Mon Sep 17 00:00:00 2001 From: karl-richter Date: Thu, 12 Jan 2023 16:45:28 -0800 Subject: [PATCH 01/13] run metrics job from a fork --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 5510a13c7..707c2cd99 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -55,7 +55,7 @@ jobs: cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "\n" >> report.md # Post reports as comments in GitHub PRs - cml comment update --pr report.md # post to PR + cml comment update --target pr/${{github.event.issue.number}} --repo=${{ github.event.pull_request.head.repo.html_url }} report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 0f191c7cdee73a19fd3bbfee5245907d47025e87 Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:26:22 -0800 Subject: [PATCH 02/13] commented out pytest --- .github/workflows/metrics.yml | 56 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 707c2cd99..478fb7314 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -19,42 +19,46 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - pip3 install -e . - pip3 install pytest - pip3 install tabulate # required to md export - pip3 install kaleido # required for plotly export - - name: Train model - run: | - pytest tests/test_model_performance.py - - name: Download metrics from main - uses: dawidd6/action-download-artifact@v2 - with: - repo: ourownstory/neural_prophet - branch: main - name: metrics - path: tests/metrics-main/ - if_no_artifact_found: warn - - name: Compare performance - run: | - echo "## Model Benchmark" >> report.md - python tests/metrics/compareMetrics.py >> report.md + #- name: Install dependencies + # run: | + # python3 -m pip install --upgrade pip + # pip3 install -e . + # pip3 install pytest + # pip3 install tabulate # required to md export + # pip3 install kaleido # required for plotly export + #- name: Train model + # run: | + # pytest tests/test_model_performance.py + #- name: Download metrics from main + # uses: dawidd6/action-download-artifact@v2 + # with: + # repo: ourownstory/neural_prophet + # branch: main + # name: metrics + # path: tests/metrics-main/ + # if_no_artifact_found: warn + #- name: Compare performance + # run: | + # echo "## Model Benchmark" >> report.md + # python tests/metrics/compareMetrics.py >> report.md - name: Publish report env: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "
\nModel training plots\n" >> report.md + # echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md - cml asset publish tests/metrics/PeytonManning.svg --md >> report.md + # cml asset publish tests/metrics/PeytonManning.svg --md >> report.md echo "### YosemiteTemps" >> report.md - cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md + # cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md echo "### AirPassengers" >> report.md - cml asset publish tests/metrics/AirPassengers.svg --md >> report.md + # cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "\n
" >> report.md # Post reports as comments in GitHub PRs + echo + echo $GITHUB_SHA + echo $GITHUB_REPOSITORY_OWNER + echo ${{ github.event.pull_request.head.repo.html_url }} cml comment update --target pr/${{github.event.issue.number}} --repo=${{ github.event.pull_request.head.repo.html_url }} report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main From 2ce232e3cd5c0071e242f1bce54c80da2428c963 Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:32:14 -0800 Subject: [PATCH 03/13] Update metrics.yml --- .github/workflows/metrics.yml | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 478fb7314..133f17768 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -19,43 +19,20 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - #- name: Install dependencies - # run: | - # python3 -m pip install --upgrade pip - # pip3 install -e . - # pip3 install pytest - # pip3 install tabulate # required to md export - # pip3 install kaleido # required for plotly export - #- name: Train model - # run: | - # pytest tests/test_model_performance.py - #- name: Download metrics from main - # uses: dawidd6/action-download-artifact@v2 - # with: - # repo: ourownstory/neural_prophet - # branch: main - # name: metrics - # path: tests/metrics-main/ - # if_no_artifact_found: warn - #- name: Compare performance - # run: | - # echo "## Model Benchmark" >> report.md - # python tests/metrics/compareMetrics.py >> report.md + - name: Compare performance + run: | + echo "## Model Benchmark" >> report.md + python tests/metrics/compareMetrics.py >> report.md - name: Publish report env: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # echo "
\nModel training plots\n" >> report.md + echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md - # cml asset publish tests/metrics/PeytonManning.svg --md >> report.md echo "### YosemiteTemps" >> report.md - # cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md echo "### AirPassengers" >> report.md - # cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "\n
" >> report.md - # Post reports as comments in GitHub PRs - echo echo $GITHUB_SHA echo $GITHUB_REPOSITORY_OWNER echo ${{ github.event.pull_request.head.repo.html_url }} From c5fee6064367028a8f74f07be17c84acdc1af04b Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:34:56 -0800 Subject: [PATCH 04/13] Update metrics.yml --- .github/workflows/metrics.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 133f17768..09a66866d 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -19,10 +19,17 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Download metrics from main + uses: dawidd6/action-download-artifact@v2 + with: + repo: ourownstory/neural_prophet + branch: main + name: metrics + path: tests/metrics-main/ + if_no_artifact_found: warn - name: Compare performance - run: | - echo "## Model Benchmark" >> report.md - python tests/metrics/compareMetrics.py >> report.md + run: | + echo "## Model Benchmark" >> report.md - name: Publish report env: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -30,13 +37,14 @@ jobs: echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md + cml asset publish tests/metrics/PeytonManning.svg --md >> report.md echo "### YosemiteTemps" >> report.md + cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md echo "### AirPassengers" >> report.md + cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "\n
" >> report.md - echo $GITHUB_SHA - echo $GITHUB_REPOSITORY_OWNER - echo ${{ github.event.pull_request.head.repo.html_url }} - cml comment update --target pr/${{github.event.issue.number}} --repo=${{ github.event.pull_request.head.repo.html_url }} report.md # post to PR + # Post reports as comments in GitHub PRs + cml comment update --pr report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 3ccb1cfb3e5743b829af0ff3598a5a25125f0681 Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:39:51 -0800 Subject: [PATCH 05/13] Update metrics.yml --- .github/workflows/metrics.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 09a66866d..e03464ac8 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -37,14 +37,11 @@ jobs: echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md - cml asset publish tests/metrics/PeytonManning.svg --md >> report.md echo "### YosemiteTemps" >> report.md - cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md echo "### AirPassengers" >> report.md - cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "\n
" >> report.md # Post reports as comments in GitHub PRs - cml comment update --pr report.md # post to PR + cml comment update --target=pr/1113 report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 153634916717c8c6b1964114d934014ec73116a0 Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:48:12 -0800 Subject: [PATCH 06/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index e03464ac8..cf10e00a3 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -41,7 +41,7 @@ jobs: echo "### AirPassengers" >> report.md echo "\n
" >> report.md # Post reports as comments in GitHub PRs - cml comment update --target=pr/1113 report.md # post to PR + cml comment update --target=pr/1114 report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From ae550650a0ebdd802f87c5dbcaa36b0f9f51b81a Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 13:57:04 -0800 Subject: [PATCH 07/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index cf10e00a3..1a4be0d24 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -41,7 +41,7 @@ jobs: echo "### AirPassengers" >> report.md echo "\n
" >> report.md # Post reports as comments in GitHub PRs - cml comment update --target=pr/1114 report.md # post to PR + cml comment update --target=pr report.md # post to PR cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From b9ac2380b72c077c164fd09d9ce040c59779bc4a Mon Sep 17 00:00:00 2001 From: karl-richter Date: Fri, 20 Jan 2023 14:25:08 -0800 Subject: [PATCH 08/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 1a4be0d24..1c6f3f699 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -42,7 +42,7 @@ jobs: echo "\n" >> report.md # Post reports as comments in GitHub PRs cml comment update --target=pr report.md # post to PR - cml check create --title=ModelReport report.md # update status of check in PR + # cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 with: From de852e65e126b2f5997fa78f371940c426222ab9 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Fri, 23 Feb 2024 12:49:05 -0800 Subject: [PATCH 09/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 0fc5cb6b6..8037c87d8 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -56,7 +56,9 @@ jobs: echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md echo "### PeytonManning" >> report.md + cml asset publish tests/metrics/PeytonManning.svg --md >> report.md echo "### YosemiteTemps" >> report.md + cml asset publish tests/metrics/YosemiteTemps.svg --md >> report.md echo "### AirPassengers" >> report.md cml asset publish tests/metrics/AirPassengers.svg --md >> report.md echo "### EnergyPriceDaily" >> report.md From 19d09ef0c3746a33bd5f22368d871e5928d0a59e Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Fri, 23 Feb 2024 12:53:11 -0800 Subject: [PATCH 10/13] Update metrics.yml --- .github/workflows/metrics.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 8037c87d8..c1b78095b 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -18,7 +18,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.workflow_run.head.sha }} - name: Install Python 3.12 uses: actions/setup-python@v5 with: @@ -52,6 +52,7 @@ jobs: - name: Publish Report with CML env: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} run: | echo "
\nModel training plots\n" >> report.md echo "## Model Training" >> report.md @@ -64,7 +65,10 @@ jobs: echo "### EnergyPriceDaily" >> report.md cml asset publish tests/metrics/EnergyPriceDaily.svg --md >> report.md echo "\n
" >> report.md - cml comment update --target=pr report.md # Post reports as comments in GitHub PRs + echo + echo $GITHUB_SHA + echo $GITHUB_REPOSITORY_OWNER + cml comment update --target=pr --commit-sha=${{ github.event.pull_request.head.sha }} report.md # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 6d6a9a368bb228ea9ec53f1a1e1ca404b3ea5279 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Fri, 23 Feb 2024 12:58:33 -0800 Subject: [PATCH 11/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index c1b78095b..a9280f492 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -68,7 +68,7 @@ jobs: echo echo $GITHUB_SHA echo $GITHUB_REPOSITORY_OWNER - cml comment update --target=pr --commit-sha=${{ github.event.pull_request.head.sha }} report.md # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} + cml comment update --target=pr/${{ github.event.pull_request.head.sha }} report.md # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 1f7cb0f31bbf1d870735e02dd313af0a87aa8723 Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Fri, 23 Feb 2024 13:12:43 -0800 Subject: [PATCH 12/13] Update metrics.yml --- .github/workflows/metrics.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index a9280f492..e1f208bd6 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -49,7 +49,7 @@ jobs: run: echo "## Model Benchmark" >> report.md - name: Write Benchmark Report run: poetry run python tests/metrics/compareMetrics.py >> report.md - - name: Publish Report with CML + - name: Update report.md plots with CML env: REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} @@ -65,10 +65,16 @@ jobs: echo "### EnergyPriceDaily" >> report.md cml asset publish tests/metrics/EnergyPriceDaily.svg --md >> report.md echo "\n
" >> report.md + - name: Post comment to PR with CML + env: + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | echo - echo $GITHUB_SHA + echo $GITHUB_SHA + echo ${{ github.event.pull_request.head.sha }} echo $GITHUB_REPOSITORY_OWNER - cml comment update --target=pr/${{ github.event.pull_request.head.sha }} report.md # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} + cml comment update --target=pr/${{ github.event.pull_request.head.sha }} report.md --repo=${{ github.event.pull_request.head.repo.html_url }} # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3 From 652f6d29f35f442ae32dd4beb7caf00e0ae99a7d Mon Sep 17 00:00:00 2001 From: Oskar Triebe Date: Fri, 23 Feb 2024 13:28:55 -0800 Subject: [PATCH 13/13] Update metrics.yml --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index e1f208bd6..015fb0300 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -74,7 +74,7 @@ jobs: echo $GITHUB_SHA echo ${{ github.event.pull_request.head.sha }} echo $GITHUB_REPOSITORY_OWNER - cml comment update --target=pr/${{ github.event.pull_request.head.sha }} report.md --repo=${{ github.event.pull_request.head.repo.html_url }} # post to PR # --repo=${{ github.event.pull_request.head.repo.html_url }} + cml comment update --target=pr report.md # post to PR # --target=pr/${{ github.event.pull_request.head.sha }} --repo=${{ github.event.pull_request.head.repo.html_url }} cml check create --title=ModelReport report.md # update status of check in PR - name: Upload metrics if on main uses: actions/upload-artifact@v3