Release crates that don't yet exist

[brainstorm]

I'm trying to push a totally new set of crates to crates.io and I expected this action to push them forward even if they were never published before, but this action apparently assumes that they are already there? See a bit of the linked action log output below:

/cc @mmalenic

Run MarcoIeni/release-plz-action@main
/usr/bin/docker run --name dc62bd37cb5467bb7551575a1f54582_e24c03 --label 290506 --workdir /github/workspace --rm -e "GITHUB_TOKEN" -e "CARGO_REGISTRY_TOKEN" -e "INPUT_COMMAND" -e "INPUT_REGISTRY" -e "INPUT_UPDATE_DEPENDENCIES" -e "INPUT_NO_CHANGELOG" -e "INPUT_PROJECT_MANIFEST" -e "INPUT_CHANGELOG_CONFIG" -e "INPUT_ARGS" -e "HOME" -e "GITHUB_JOB" -e "GITHUB_REF" -e "GITHUB_SHA" -e "GITHUB_REPOSITORY" -e "GITHUB_REPOSITORY_OWNER" -e "GITHUB_RUN_ID" -e "GITHUB_RUN_NUMBER" -e "GITHUB_RETENTION_DAYS" -e "GITHUB_RUN_ATTEMPT" -e "GITHUB_ACTOR" -e "GITHUB_TRIGGERING_ACTOR" -e "GITHUB_WORKFLOW" -e "GITHUB_HEAD_REF" -e "GITHUB_BASE_REF" -e "GITHUB_EVENT_NAME" -e "GITHUB_SERVER_URL" -e "GITHUB_API_URL" -e "GITHUB_GRAPHQL_URL" -e "GITHUB_REF_NAME" -e "GITHUB_REF_PROTECTED" -e "GITHUB_REF_TYPE" -e "GITHUB_WORKSPACE" -e "GITHUB_ACTION" -e "GITHUB_EVENT_PATH" -e "GITHUB_ACTION_REPOSITORY" -e "GITHUB_ACTION_REF" -e "GITHUB_PATH" -e "GITHUB_ENV" -e "GITHUB_STEP_SUMMARY" -e "GITHUB_STATE" -e "GITHUB_OUTPUT" -e "RUNNER_OS" -e "RUNNER_ARCH" -e "RUNNER_NAME" -e "RUNNER_TOOL_CACHE" -e "RUNNER_TEMP" -e "RUNNER_WORKSPACE" -e "ACTIONS_RUNTIME_URL" -e "ACTIONS_RUNTIME_TOKEN" -e "ACTIONS_CACHE_URL" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" -v "/home/runner/work/htsget-rs/htsget-rs":"/github/workspace" 290506:6dc62bd37cb5467bb7551575a1f54582
  2023-01-05T04:41:18.473327Z  INFO  downloading packages from cargo registry
    in download_packages with packages=["htsget-config", "htsget-http-actix", "htsget-http-core", "htsget-search", "htsget-test-utils", "htsget-http-lambda"] directory="/tmp/.tmp8nkg27" registry=None
    in next_versions with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in update with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in release_pr with input=ReleasePrRequest { git: Github(GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }), update_request: UpdateRequest { local_manifest: "/github/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) } }

note: Cloning into "/tmp/.tmp8nkg27/htsget-config"
    Updating crates.io index
note: Cloning into "/tmp/.tmp8nkg27/htsget-http-actix"
note: Cloning into "/tmp/.tmp8nkg27/htsget-http-core"
note: Cloning into "/tmp/.tmp8nkg27/htsget-search"
note: Cloning into "/tmp/.tmp8nkg27/htsget-test-utils"
note: Cloning into "/tmp/.tmp8nkg27/htsget-http-lambda"
  2023-01-05T04:41:39.033526Z  WARN  can't download htsget-config: Package `htsget-config@*.*.*` not found
    in download_packages with packages=["htsget-config", "htsget-http-actix", "htsget-http-core", "htsget-search", "htsget-test-utils", "htsget-http-lambda"] directory="/tmp/.tmp8nkg27" registry=None
    in next_versions with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in update with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in release_pr with input=ReleasePrRequest { git: Github(GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }), update_request: UpdateRequest { local_manifest: "/github/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) } }

  2023-01-05T04:41:39.034398Z  WARN  can't download htsget-http-actix: Package `htsget-http-actix@*.*.*` not found
    in download_packages with packages=["htsget-config", "htsget-http-actix", "htsget-http-core", "htsget-search", "htsget-test-utils", "htsget-http-lambda"] directory="/tmp/.tmp8nkg27" registry=None
    in next_versions with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in update with input=UpdateRequest { local_manifest: "/tmp/.tmpgl608m/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) }
    in release_pr with input=ReleasePrRequest { git: Github(GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }), update_request: UpdateRequest { local_manifest: "/github/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) } }
  (...)

Would it be possible (reasonable?) to have the automatic publishing of non-existing crates as default behavior?

[brainstorm]

... well, it's actually a bit harder to solve than I thought because of rust-lang/cargo#1169 and defects on https://gitlab.com/torkleyy/cargo-publish-all

[marcoieni]

I was expecting the action to publish non existing crates, yes.
These should be only warnings from cargo, but release-plz should handle this case.
I will try looking into this issue today. 👍

[marcoieni]

The actual error is at the end of the logs:

Error: the working directory of this project has uncommitted changes. Please commit or stash these changes:
["output.json"]

With this change errors should be more visible 👍

The real problem is that the file output.json has uncommitted changes.

I think we have several options here:

• you revert this file before running release-plz
• you pass to release-plz release-pr the flag --allow-dirty. In this case, output.json will be part of the release PR. In this case, I will add the option to the GitHub action.
• Somehow, I add an option to ignore files to release-plz, so after I do git add . I remove these files from the git staging area.
• I change release-plz so that it runs git add . just inside each crate (not in the whole repository). In this case, it shouldn't be a problem if there are uncommitted changes in other directories. This only benefits repositories that don't have a crate in their top directory.

[brainstorm]

Thanks for taking such a fast peek at this! output.js is part of the cargo-criterion benchmark output and so I have no interest in adding/commiting/pushing that file since benchmark-action/github-action-benchmark#145 does it for me and pushes it to gh-pages branch for benchmarks run plotting.

In a nutshell I think that options 1 and 3 are reasonable. Perhaps both since you might find projects that leave stray temporary directories/files after a successful CI run and authors might not want to explicitly cleanup after on action.yml (CI envs are often treated as disposable).

So for now I'll go with removing output.js and even doing a git checkout . to make sure all is clean before release, but offering option 3 would be a nice plus for release-plz, IMHO.

[brainstorm]

I applied the option/fix nr 1 and also published the crates manually, but now there's an interesting error when fetching metadata for htsget-lambda:

note: Cloning into "/tmp/.tmpfPPFAu/htsget-config"
    Updating crates.io index
note: Downloading htsget-config 0.1.0
 Downloading crates ...
  Downloaded htsget-config v0.1.0
note: Cloning into "/tmp/.tmpfPPFAu/htsget-actix"
note: Downloading htsget-actix 0.1.0
 Downloading crates ...
  Downloaded htsget-actix v0.1.0
note: Cloning into "/tmp/.tmpfPPFAu/htsget-http"
note: Downloading htsget-http 0.1.0
 Downloading crates ...
  Downloaded htsget-http v0.1.0
note: Cloning into "/tmp/.tmpfPPFAu/htsget-search"
note: Downloading htsget-search 0.1.0
 Downloading crates ...
  Downloaded htsget-search v0.1.0
note: Cloning into "/tmp/.tmpfPPFAu/htsget-test"
note: Downloading htsget-test 0.1.0
 Downloading crates ...
  Downloaded htsget-test v0.1.0
note: Cloning into "/tmp/.tmpfPPFAu/htsget-lambda"
note: Downloading htsget-lambda 0.1.0
 Downloading crates ...
  Downloaded htsget-lambda v0.1.0
Error: cannot copy directory "/tmp/.tmp0eZwzT/workspace" to TempDir { path: "/tmp/.tmp198FOz" }

Caused by:
    Io error. Look inside err_kind for more details.

[marcoieni]

That's a shame, I'm sorry :/ I will release a new version where I print the err_kind, so we have more insights.

[marcoieni]

One thing. I noticed you are running the release-plz-action workflow on every branch. (because of on: [push].
I suggest running release-plz only when you push in the main branch.
Like this:

on:
  push:
    branches:
      - main

Otherwise, it is going to run on every pr. Right? 🤔
But maybe this is something you are already aware of, and you are just testing the workflow 👍

EDIT: here's the pr. I will deploy a new version soon.
EDIT 2: shipping the new version now. We have to wait for the CI to publish the new docker image. Then you can try another time. Again, sorry for the inconvenience, and thanks for your patience in helping me troubleshoot this :)

[marcoieni]

The new docker image is up!
Can you try again?
We should see the ErrorKind now.

[brainstorm]

Wow, that's a bit unexpected: https://github.com/umccr/htsget-rs/actions/runs/3859787713/jobs/6579637876

 Downloading crates ...
  Downloaded htsget-lambda v0.1.0
  2023-01-07T01:53:35.184733Z ERROR  cannot copy directory "/tmp/.tmpDNGznW/workspace" to TempDir { path: "/tmp/.tmpJr0BBG" }: Io error. Look inside err_kind for more details. Error kind: Io(Os { code: 28, kind: StorageFull, message: "No space left on device" })

[marcoieni]

Wow, that's something. 😂
Maybe you can move the release-plz workflow to a separate file, like I do in the release-plz project itself:
https://github.com/MarcoIeni/release-plz/blob/main/.github/workflows/release-plz.yml

(you don't necessarily need the first part of the workflow where I generate a token)

If this doesn't fix it, we can think of something else 🤔

[brainstorm]

Wow, that's something. 😂 Maybe you can move the release-plz workflow to a separate file, like I do in the release-plz project itself: https://github.com/MarcoIeni/release-plz/blob/main/.github/workflows/release-plz.yml

(you don't necessarily need the first part of the workflow where I generate a token)

If this doesn't fix it, we can think of something else 🤔

Yeah, thanks! I wanted to do this anyway (split the monolithic GHA), but there seems to be a hiccup still:

(...)
    Updating crates.io index
  2023-01-07T10:28:37.121549Z  INFO  opened pr: https://github.com/umccr/htsget-rs/pull/129
    in open_pr with self=GitHubClient { github: GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }, client: Octocrab { client: Client { accepts: Accepts, proxies: [Proxy(System({}), None)], referer: true, default_headers: {"accept": "*/*", "user-agent": "octocrab", "authorization": "***"} }, base_url: Url { scheme: "https", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("api.github.com")), port: None, path: "/", query: None, fragment: None }, auth_state: None } } default_branch="main"
    in open_pr with self=GitHub(GitHubClient { github: GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }, client: Octocrab { client: Client { accepts: Accepts, proxies: [Proxy(System({}), None)], referer: true, default_headers: {"accept": "*/*", "user-agent": "octocrab", "authorization": "***"} }, base_url: Url { scheme: "https", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("api.github.com")), port: None, path: "/", query: None, fragment: None }, auth_state: None } })
    in release_pr with input=ReleasePrRequest { git: Github(GitHub { owner: "umccr", repo: "htsget-rs", token: Secret([REDACTED alloc::string::String]), base_url: None }), update_request: UpdateRequest { local_manifest: "/github/workspace/Cargo.toml", registry_manifest: None, single_package: None, changelog_req: Some(ChangelogRequest { release_date: None, changelog_config: None }), registry: None, update_dependencies: false, allow_dirty: false, github_repo: Some(RepoUrl { scheme: "https", host: "github.com", port: None, owner: "umccr", name: "htsget-rs" }) } }

error: The following required arguments were not provided:
  --git-token <GIT_TOKEN>

Usage: release-plz release --git-token <GIT_TOKEN>

For more information try '--help'

I used the example (verbatim?) from the release-plz README and the GH token specified in the .yml has been used successfully in the past by the benchmarks action... the scopes/allowed actions are fairly broad but I might have missed a special one that your action requires? 🤷🏻

The release got generated though: umccr/htsget-rs#129.

Cheers!

[marcoieni]

This is a bug I will fix! That argument should be optional.
I will release a new version in a couple of hours 👍

[marcoieni]

I released a fix. When you run again the GitHub action, release-plz should publish the unpublished packages to crates.io.
In your case, nothing should happen since you said you already published them 👍