From 22725d080926f2fca5e7c6ee658beba664fb27b2 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Tue, 4 Aug 2026 14:15:03 -0400 Subject: [PATCH 01/11] build: add Woodpecker CI to test on more data sources --- .woodpecker/dataset-tests.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .woodpecker/dataset-tests.yml diff --git a/.woodpecker/dataset-tests.yml b/.woodpecker/dataset-tests.yml new file mode 100644 index 000000000..dedb92c66 --- /dev/null +++ b/.woodpecker/dataset-tests.yml @@ -0,0 +1,23 @@ +when: + - event: push + branch: main + - event: pull_request + +labels: + backend: docker + +steps: + - name: build and run data tests + image: codeberg.org/mdekstrand/woodpecker-mise:v1-f44-dev + pull: true + settings: + task: test --coverage -m realdata + mise-env: ci + + - name: upload code coverage + image: woodpeckerci/plugin-codecov + settings: + files: + - coverage.xml + token: + from_secret: codecov-token From a85d91d9459dfae42c69c96198dce34702d61693 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Tue, 4 Aug 2026 14:21:35 -0400 Subject: [PATCH 02/11] build: add CI preparation step & use in dataset tests --- .woodpecker/dataset-tests.yml | 2 +- mise/tasks/ci/prepare.sh | 23 +++++++++++++++++++++++ mise/tasks/test.sh | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100755 mise/tasks/ci/prepare.sh diff --git a/.woodpecker/dataset-tests.yml b/.woodpecker/dataset-tests.yml index dedb92c66..11237f59f 100644 --- a/.woodpecker/dataset-tests.yml +++ b/.woodpecker/dataset-tests.yml @@ -11,7 +11,7 @@ steps: image: codeberg.org/mdekstrand/woodpecker-mise:v1-f44-dev pull: true settings: - task: test --coverage -m realdata + task: "ci:prepare ::: test --coverage -m realdata" mise-env: ci - name: upload code coverage diff --git a/mise/tasks/ci/prepare.sh b/mise/tasks/ci/prepare.sh new file mode 100755 index 000000000..523aae702 --- /dev/null +++ b/mise/tasks/ci/prepare.sh @@ -0,0 +1,23 @@ +#!/bin/zsh +#MISE description="Prepare CI environment" +#USAGE flag "-v --verbose" help="Enable verbose logging." + +. "$MISE_PROJECT_ROOT/mise/task-functions.sh" +. "${UV_PROJECT_ENVIRONMENT:-$MISE_PROJECT_ROOT/.venv}/bin/activate" + +if [[ -z $CI ]]; then + msg "not in CI, skipping environment setup" +fi + +if [[ $CI_SYSTEM_NAME = woodpecker ]]; then + if [[ -d /datasets ]]; then + msg "linking data from /datasets" + for f in /datasets/*; do + ln -s $f data/${f#/datasets/} + done + else + msg -warn "running on Woodpecker but /datasets does not exist" + fi +else + msg "nothing to do for CI $CI_SYSTEM_NAME" +fi diff --git a/mise/tasks/test.sh b/mise/tasks/test.sh index add734d5f..eba8dac69 100755 --- a/mise/tasks/test.sh +++ b/mise/tasks/test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash #MISE description="Run tests" +#MISE wait_for=["ci:prepare"] #USAGE flag "--coverage" help="Run tests with coverage." #USAGE flag "--accel-coverage" help="Run tests with Rust accelerator coverage." #USAGE flag "--slow" help="Include slow tests." From 9cb0bd67ed7aa7070fefce722bc12dd489fb8c58 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Tue, 4 Aug 2026 14:29:01 -0400 Subject: [PATCH 03/11] build: skip hk in CI --- mise/config.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mise/config.toml b/mise/config.toml index fd420ee46..de15c0b9b 100644 --- a/mise/config.toml +++ b/mise/config.toml @@ -23,4 +23,8 @@ aqua.github_attestations = false HK_MISE = 1 [hooks] -postinstall = "hk install" +postinstall = ''' +if [ -z "$CI" ]; then + hk install +fi +''' From 3ecd3949883d205038742cb1def49038effbf5a3 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Tue, 4 Aug 2026 14:45:44 -0400 Subject: [PATCH 04/11] build: set up CI environment --- mise/config.ci.toml | 5 +++++ mise/config.toml | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 mise/config.ci.toml diff --git a/mise/config.ci.toml b/mise/config.ci.toml new file mode 100644 index 000000000..ec0d92c3e --- /dev/null +++ b/mise/config.ci.toml @@ -0,0 +1,5 @@ +[settings] +disable_tools = ["pkl", "perl"] + +[settings.python] +uv_venv_auto = "source" diff --git a/mise/config.toml b/mise/config.toml index de15c0b9b..09a083fa4 100644 --- a/mise/config.toml +++ b/mise/config.toml @@ -12,6 +12,7 @@ yamlfmt = "latest" ruff = "latest" [settings] +experimental = true idiomatic_version_file_enable_tools = ["rust"] github.github_attestations = false aqua.github_attestations = false @@ -19,6 +20,8 @@ aqua.github_attestations = false [settings.python] # uv_venv_auto = "source" +[deps.uv] + [env] HK_MISE = 1 From 9b68bbd244b5f422692207be9ef2c11955212bb6 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 11:00:14 -0400 Subject: [PATCH 05/11] docs: add notes on dataset to LensKit license --- LICENSE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/LICENSE.md b/LICENSE.md index bc0f03d53..2342ec1e4 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,3 +1,8 @@ +> [!NOTE] +> The following license applies to the LensKit source code. The data sets +> under `data/` have their own terms, described in their respective README +> files. + Copyright (c) 2018–2023 Boise State University Copyright (c) 2023-2026 Drexel University and contributors From 1e7c32bf158e689433492a54b2a04744a0a6bddf Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 11:59:10 -0400 Subject: [PATCH 06/11] build: make data tests verbose --- .woodpecker/dataset-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/dataset-tests.yml b/.woodpecker/dataset-tests.yml index 11237f59f..d75f39665 100644 --- a/.woodpecker/dataset-tests.yml +++ b/.woodpecker/dataset-tests.yml @@ -11,7 +11,7 @@ steps: image: codeberg.org/mdekstrand/woodpecker-mise:v1-f44-dev pull: true settings: - task: "ci:prepare ::: test --coverage -m realdata" + task: "ci:prepare ::: test -v --coverage -m realdata" mise-env: ci - name: upload code coverage From 76dfa1e2878394221400cf8a1055685805f2c454 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 12:04:25 -0400 Subject: [PATCH 07/11] build: run CLI data conversion tests --- .woodpecker/dataset-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/dataset-tests.yml b/.woodpecker/dataset-tests.yml index d75f39665..2969de2c0 100644 --- a/.woodpecker/dataset-tests.yml +++ b/.woodpecker/dataset-tests.yml @@ -11,7 +11,7 @@ steps: image: codeberg.org/mdekstrand/woodpecker-mise:v1-f44-dev pull: true settings: - task: "ci:prepare ::: test -v --coverage -m realdata" + task: "ci:prepare ::: test -v --coverage -m realdata ::: test-cli tests/cli/test-data-convert.sh" mise-env: ci - name: upload code coverage From 76586fb785b3f2f30cbfb5695236862b02af6b61 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 15:23:33 -0400 Subject: [PATCH 08/11] deps: re-add jq to mise --- mise/config.toml | 1 + mise/mise.lock | 41 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mise/config.toml b/mise/config.toml index 09a083fa4..5c447e532 100644 --- a/mise/config.toml +++ b/mise/config.toml @@ -2,6 +2,7 @@ rust = "stable" usage = "latest" uv = "latest" +jq = "latest" shfmt = "latest" tombi = "latest" perl = { version = "latest", os = ["linux", "macos"] } diff --git a/mise/mise.lock b/mise/mise.lock index 556f0d2cc..8319455d4 100644 --- a/mise/mise.lock +++ b/mise/mise.lock @@ -34,9 +34,44 @@ checksum = "sha256:a2e5c6131d01cfd6d62204872798f97fc933e7f142d718b3e191434e2d31e url = "https://github.com/jdx/hk/releases/download/v1.53.0/hk-x86_64-pc-windows-msvc.zip" url_api = "https://api.github.com/repos/jdx/hk/releases/assets/487429261" -[[tools.oxfmt]] -version = "0.60.0" -backend = "npm:oxfmt" +[[tools.jq]] +version = "1.8.2" +backend = "aqua:jqlang/jq" + +[tools.jq."platforms.linux-arm64"] +checksum = "sha256:8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-arm64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012756" + +[tools.jq."platforms.linux-arm64-musl"] +checksum = "sha256:8b85c817833814ddca00a144c33705546355afccf0cf39b188f3cdb48b852309" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-arm64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012756" + +[tools.jq."platforms.linux-x64"] +checksum = "sha256:b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012752" + +[tools.jq."platforms.linux-x64-musl"] +checksum = "sha256:b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012752" + +[tools.jq."platforms.macos-arm64"] +checksum = "sha256:2d75340ba57a4b4b4c8708a21c2dc8e958a48aaa8bba13b27f77f6e4c0eca07e" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-macos-arm64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012783" + +[tools.jq."platforms.macos-x64"] +checksum = "sha256:e94b266e3c26690550006abe63152b782280f4e14374accdf04cbde844f00bc0" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-macos-amd64" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012782" + +[tools.jq."platforms.windows-x64"] +checksum = "sha256:a6fc67fedaf9128a3309a1e2ebb8b986aeccf70122ee46d2cb4849e423f0c627" +url = "https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-windows-amd64.exe" +url_api = "https://api.github.com/repos/jqlang/jq/releases/assets/453012788" [[tools.perl]] version = "5.44.0.0" From 9ebb2b5a1cc375af87f3ec4c57b81d040ce4bb30 Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 15:24:31 -0400 Subject: [PATCH 09/11] build: make CLI tests wait for unit tests --- mise/tasks/test-cli.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/mise/tasks/test-cli.sh b/mise/tasks/test-cli.sh index 7a144d1ba..3422db94f 100755 --- a/mise/tasks/test-cli.sh +++ b/mise/tasks/test-cli.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash #MISE description="Run CLI tests" +#MISE wait_for=["test"] exec ./tests/cli/run.sh "$@" From 131068e1a27dc8bc9b04f444f32fed7a101551ad Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 16:42:23 -0400 Subject: [PATCH 10/11] build: remove a lot of downloads in the GitHub eval --- .github/workflows/eval.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index f92de2a61..c8bbf6e34 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -50,15 +50,7 @@ jobs: - name: Download datasets run: | - lenskit data fetch -D data --movielens ml-100k ml-1m ml-10m - lenskit data fetch -D data/az23 --amazon --core Video_Games - lenskit data fetch -D data/az18 --amazon --edition 2018 Video_Games - lenskit data fetch -D data/az14 --amazon --edition 2014 Video_Games - lenskit data fetch --ms-web - if [[ ! -f data/australian_users_items.json.gz ]]; then - curl -fsL -o data/australian_users_items.json.gz.tmp https://mcauleylab.ucsd.edu/public_datasets/data/steam/australian_users_items.json.gz - mv data/australian_users_items.json.gz.tmp data/australian_users_items.json.gz - fi + lenskit data fetch -D data --movielens ml-100k - name: Run Eval Tests run: | @@ -103,15 +95,7 @@ jobs: - name: Download datasets run: | - coverage run -a --source=src/lenskit -m lenskit data fetch -D data --movielens ml-100k ml-20m ml-1m ml-10m - coverage run -a --source=src/lenskit -m lenskit data fetch -D data/az23 --amazon --core Video_Games - coverage run -a --source=src/lenskit -m lenskit data fetch -D data/az18 --amazon --edition 2018 Video_Games - coverage run -a --source=src/lenskit -m lenskit data fetch -D data/az14 --amazon --edition 2014 Video_Games - coverage run -a --source=src/lenskit -m lenskit data fetch --ms-web - if [[ ! -f data/australian_users_items.json.gz ]]; then - curl -fsL -o data/australian_users_items.json.gz.tmp https://mcauleylab.ucsd.edu/public_datasets/data/steam/australian_users_items.json.gz - mv data/australian_users_items.json.gz.tmp data/australian_users_items.json.gz - fi + coverage run -a --source=src/lenskit -m lenskit data fetch -D data --movielens ml-100k - name: πŸ•ΊπŸ» Test LensKit CLI run: | From 1821af30c269aca8dcbc920d617539fbc5bb664e Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 5 Aug 2026 17:57:43 -0400 Subject: [PATCH 11/11] build: add codecov token to codecov.yml --- codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codecov.yml b/codecov.yml index c18d3d1fd..14a84db35 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,6 +2,7 @@ codecov: notify: after_n_builds: 10 wait_for_ci: true + token: ab58c9cf-25b8-4283-a485-0b6382dc9a61 ignore: - build*.py - build-tools/*