diff --git a/.github/workflows/build-renku-release.yml b/.github/workflows/build-renku-release.yml new file mode 100644 index 0000000000000..5a3b8833f097a --- /dev/null +++ b/.github/workflows/build-renku-release.yml @@ -0,0 +1,197 @@ +name: Build rclone release for Renku + +# Trigger the workflow on tag pushes or manually +on: + push: + tags: + - '**' + workflow_dispatch: + +jobs: + build: + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os'] + + include: + - job_name: linux + os: ubuntu-latest + go: '>=1.23.0-rc.1' + gotags: cmount + build_flags: '-include "^linux/"' + check: true + quicktest: true + racequicktest: true + librclonetest: true + deploy: true + + - job_name: linux_386 + os: ubuntu-latest + go: '>=1.23.0-rc.1' + goarch: 386 + gotags: cmount + quicktest: true + + - job_name: mac_amd64 + os: macos-latest + go: '>=1.23.0-rc.1' + gotags: 'cmount' + build_flags: '-include "^darwin/amd64" -cgo' + quicktest: true + racequicktest: true + deploy: true + + - job_name: mac_arm64 + os: macos-latest + go: '>=1.23.0-rc.1' + gotags: 'cmount' + build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' + deploy: true + + - job_name: windows + os: windows-latest + go: '>=1.23.0-rc.1' + gotags: cmount + cgo: '0' + build_flags: '-include "^windows/"' + build_args: '-buildmode exe' + quicktest: true + deploy: true + + - job_name: other_os + os: ubuntu-latest + go: '>=1.23.0-rc.1' + build_flags: '-exclude "^(windows/|darwin/|linux/)"' + compile_all: true + deploy: true + + name: ${{ matrix.job_name }} + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + + - name: Set environment variables + shell: bash + run: | + echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV + echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV + echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV + if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi + if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + + - name: Install Libraries on Linux + shell: bash + run: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get update + sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common + if: matrix.os == 'ubuntu-latest' + + - name: Install Libraries on macOS + shell: bash + run: | + # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788 + # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008 + unset HOMEBREW_NO_INSTALL_FROM_API + brew untap --force homebrew/core + brew untap --force homebrew/cask + brew update + brew install --cask macfuse + brew install git-annex git-annex-remote-rclone + if: matrix.os == 'macos-latest' + + - name: Install Libraries on Windows + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + if ($env:GOARCH -eq "386") { + choco install -y mingw --forcex86 --force + echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + if: matrix.os == 'windows-latest' + + - name: Print Go version and environment + shell: bash + run: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nRclone environment:\n\n" + make vars + printf "\n\nSystem environment:\n\n" + env + + - name: Build rclone + shell: bash + run: | + make + + - name: Rclone version + shell: bash + run: | + rclone version + + - name: Run tests + shell: bash + run: | + make quicktest + if: matrix.quicktest + + - name: Race test + shell: bash + run: | + make racequicktest + if: matrix.racequicktest + + - name: Run librclone tests + shell: bash + run: | + make -C librclone/ctest test + make -C librclone/ctest clean + librclone/python/test_rclone.py + if: matrix.librclonetest + + - name: Compile all architectures test + shell: bash + run: | + make + make compile_all + if: matrix.compile_all + + - name: Build binaries for release + shell: bash + run: | + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + make ci_gha + ls -al build/ + if: matrix.deploy + + - name: Upload artifacts to GitHub + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.job_name }} + path: build + retention-days: 7 + if: matrix.deploy diff --git a/.github/workflows/build-renku.yml b/.github/workflows/build-renku.yml new file mode 100644 index 0000000000000..66719c58661a0 --- /dev/null +++ b/.github/workflows/build-renku.yml @@ -0,0 +1,237 @@ +name: Build rclone for Renku + +# Trigger the workflow on push or pull request +on: + push: + branches: + - "**" + tags: + - "**" + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + job_name: ["linux"] + + include: + - job_name: linux + os: ubuntu-latest + go: ">=1.23.0-rc.1" + gotags: cmount + build_flags: '-include "linux/amd64|linux/arm64"' + deploy: true + + name: ${{ matrix.job_name }} + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + + - name: Set environment variables + shell: bash + run: | + echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV + echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV + echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV + if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi + if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + + - name: Install Libraries on Linux + shell: bash + run: | + sudo modprobe fuse + sudo chmod 666 /dev/fuse + sudo chown root:$USER /etc/fuse.conf + sudo apt-get update + sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common + if: matrix.os == 'ubuntu-latest' + + - name: Install Libraries on macOS + shell: bash + run: | + # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788 + # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008 + unset HOMEBREW_NO_INSTALL_FROM_API + brew untap --force homebrew/core + brew untap --force homebrew/cask + brew update + brew install --cask macfuse + brew install git-annex git-annex-remote-rclone + if: matrix.os == 'macos-latest' + + - name: Install Libraries on Windows + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + choco install -y winfsp zip + echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + if ($env:GOARCH -eq "386") { + choco install -y mingw --forcex86 --force + echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + } + # Copy mingw32-make.exe to make.exe so the same command line + # can be used on Windows as on macOS and Linux + $path = (get-command mingw32-make.exe).Path + Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') + if: matrix.os == 'windows-latest' + + - name: Print Go version and environment + shell: bash + run: | + printf "Using go at: $(which go)\n" + printf "Go version: $(go version)\n" + printf "\n\nGo environment:\n\n" + go env + printf "\n\nRclone environment:\n\n" + make vars + printf "\n\nSystem environment:\n\n" + env + + - name: Build rclone + shell: bash + run: | + make + + - name: Rclone version + shell: bash + run: | + rclone version + + - name: Run tests + shell: bash + run: | + make quicktest + if: matrix.quicktest + + - name: Race test + shell: bash + run: | + make racequicktest + if: matrix.racequicktest + + - name: Run librclone tests + shell: bash + run: | + make -C librclone/ctest test + make -C librclone/ctest clean + librclone/python/test_rclone.py + if: matrix.librclonetest + + - name: Compile all architectures test + shell: bash + run: | + make + make compile_all + if: matrix.compile_all + + - name: Build binaries for release + shell: bash + run: | + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + make ci_gha + ls -al build/ + if: matrix.deploy + + - name: Upload artifacts to GitHub + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.job_name }} + path: build + retention-days: 1 + if: matrix.deploy + + image: + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + timeout-minutes: 60 + runs-on: ubuntu-latest + needs: [build] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: build-linux + path: rclone + + - name: List downloaded files + run: ls -R rclone + + - name: Extract rclone binary - amd64 + run: | + unzip "rclone/*-amd64.zip" -d rclone-unzip + ls -R rclone-unzip + mkdir -p linux/amd64 + mv rclone-unzip/*/rclone linux/amd64/rclone + - name: Extract rclone binary - arm64 + run: | + unzip "rclone/*-arm64.zip" -d rclone-unzip + ls -R rclone-unzip + mkdir -p linux/arm64 + mv rclone-unzip/*/rclone linux/arm64/rclone + - name: Docker image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + + - name: Extract Docker image name + id: docker_image + env: + IMAGE_TAGS: ${{ steps.meta.outputs.tags }} + run: | + IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1) + IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1) + IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2) + echo "image=$IMAGE" >> "$GITHUB_OUTPUT" + echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + - name: Set up Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up Docker + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: .renku/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b251e7ec38cd4..8b876e1a2603a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,12 +21,12 @@ on: jobs: build: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) timeout-minutes: 60 strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22'] + job_name: ['linux', 'linux_386'] include: - job_name: linux @@ -47,31 +47,31 @@ jobs: gotags: cmount quicktest: true - - job_name: mac_amd64 - os: macos-latest - go: '>=1.23.0-rc.1' - gotags: 'cmount' - build_flags: '-include "^darwin/amd64" -cgo' - quicktest: true - racequicktest: true - deploy: true - - - job_name: mac_arm64 - os: macos-latest - go: '>=1.23.0-rc.1' - gotags: 'cmount' - build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' - deploy: true - - - job_name: windows - os: windows-latest - go: '>=1.23.0-rc.1' - gotags: cmount - cgo: '0' - build_flags: '-include "^windows/"' - build_args: '-buildmode exe' - quicktest: true - deploy: true + # - job_name: mac_amd64 + # os: macos-latest + # go: '>=1.23.0-rc.1' + # gotags: 'cmount' + # build_flags: '-include "^darwin/amd64" -cgo' + # quicktest: true + # racequicktest: true + # deploy: true + + # - job_name: mac_arm64 + # os: macos-latest + # go: '>=1.23.0-rc.1' + # gotags: 'cmount' + # build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' + # deploy: true + + # - job_name: windows + # os: windows-latest + # go: '>=1.23.0-rc.1' + # gotags: cmount + # cgo: '0' + # build_flags: '-include "^windows/"' + # build_args: '-buildmode exe' + # quicktest: true + # deploy: true - job_name: other_os os: ubuntu-latest @@ -80,18 +80,6 @@ jobs: compile_all: true deploy: true - - job_name: go1.21 - os: ubuntu-latest - go: '1.21' - quicktest: true - racequicktest: true - - - job_name: go1.22 - os: ubuntu-latest - go: '1.22' - quicktest: true - racequicktest: true - name: ${{ matrix.job_name }} runs-on: ${{ matrix.os }} @@ -205,19 +193,19 @@ jobs: make compile_all if: matrix.compile_all - - name: Deploy built binaries - shell: bash - run: | - if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi - make ci_beta - env: - RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} - # working-directory: '$(modulePath)' - # Deploy binaries if enabled in config && not a PR && not a fork - if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' + # - name: Deploy built binaries + # shell: bash + # run: | + # if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi + # make ci_beta + # env: + # RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} + # # working-directory: '$(modulePath)' + # # Deploy binaries if enabled in config && not a PR && not a fork + # if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' lint: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) timeout-minutes: 30 name: "lint" runs-on: ubuntu-latest diff --git a/.github/workflows/build_publish_beta_docker_image.yml b/.github/workflows/build_publish_beta_docker_image.yml deleted file mode 100644 index fa838c6ff0e4e..0000000000000 --- a/.github/workflows/build_publish_beta_docker_image.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Docker beta build - -on: - push: - branches: - - master -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - # This is the user that triggered the Workflow. In this case, it will - # either be the user whom created the Release or manually triggered - # the workflow_dispatch. - username: ${{ github.actor }} - # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by - # GitHub Actions at the start of a workflow run to identify the job. - # This is used to authenticate against GitHub Container Registry. - # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret - # for more detailed information. - password: ${{ secrets.GITHUB_TOKEN }} - - name: Show disk usage - shell: bash - run: | - df -h . - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - push: true # push the image to ghcr - tags: | - ghcr.io/rclone/rclone:beta - rclone/rclone:beta - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - cache-from: type=gha, scope=${{ github.workflow }} - cache-to: type=gha, mode=max, scope=${{ github.workflow }} - provenance: false - # Eventually cache will need to be cleared if builds more frequent than once a week - # https://github.com/docker/build-push-action/issues/252 - - name: Show disk usage - shell: bash - run: | - df -h . diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml new file mode 100644 index 0000000000000..abc017bc347ac --- /dev/null +++ b/.github/workflows/build_publish_docker_image.yml @@ -0,0 +1,294 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*- + +name: Build & Push Docker Images + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '**' + tags: + - '**' + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + build-image: + if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request') + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: ubuntu-24.04 + - platform: linux/386 + runs-on: ubuntu-24.04 + - platform: linux/arm64 + runs-on: ubuntu-24.04-arm + - platform: linux/arm/v7 + runs-on: ubuntu-24.04-arm + - platform: linux/arm/v6 + runs-on: ubuntu-24.04-arm + + name: Build Docker Image for ${{ matrix.platform }} + runs-on: ${{ matrix.runs-on }} + + steps: + - name: Free Space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Set PLATFORM Variable + run: | + platform=${{ matrix.platform }} + echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV + + - name: Set CACHE_NAME Variable + shell: python + run: | + import os, re + + def slugify(input_string, max_length=63): + slug = input_string.lower() + slug = re.sub(r'[^a-z0-9 -]', ' ', slug) + slug = slug.strip() + slug = re.sub(r'\s+', '-', slug) + slug = re.sub(r'-+', '-', slug) + slug = slug[:max_length] + slug = re.sub(r'[-]+$', '', slug) + return slug + + ref_name_slug = "cache" + + if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request": + ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME']) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"CACHE_NAME={ref_name_slug}\n") + + - name: Get ImageOS + # There's no way around this, because "ImageOS" is only available to + # processes, but the setup-go action uses it in its key. + id: imageos + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + return process.env.ImageOS + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages) + with: + images: | + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load Go Build Cache for Docker + id: go-cache + uses: actions/cache@v4 + with: + key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + # Cache only the go builds, the module download is cached via the docker layer caching + path: | + go-build-cache + + - name: Inject Go Build Cache into Docker + uses: reproducible-containers/buildkit-cache-dance@v3 + with: + cache-map: | + { + "go-build-cache": "/root/.cache/go-build" + } + skip-extraction: ${{ steps.go-cache.outputs.cache-hit }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image Digest + id: build + uses: docker/build-push-action@v6 + with: + file: Dockerfile + context: . + provenance: false + # don't specify 'tags' here (error "get can't push tagged ref by digest") + # tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + platforms: ${{ matrix.platform }} + outputs: | + type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + cache-to: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd + + - name: Export Image Digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload Image Digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM }} + path: /tmp/digests/* + retention-days: 1 + if-no-files-found: error + + merge-image: + name: Merge & Push Final Docker Image + runs-on: ubuntu-24.04 + needs: + - build-image + + steps: + - name: Download Image Digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + with: + images: | + ${{ env.REPO_NAME }} + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Extract Tags + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + tags = [f"--tag '{tag}'" for tag in metadata["tags"]] + tags_string = " ".join(tags) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"TAGS={tags_string}\n") + + - name: Extract Annotations + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]] + annotations_string = " ".join(annotations) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"ANNOTATIONS={annotations_string}\n") + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create & Push Manifest List + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + ${{ env.TAGS }} \ + ${{ env.ANNOTATIONS }} \ + $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *) + + - name: Inspect and Run Multi-Platform Image + run: | + docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version diff --git a/.github/workflows/build_publish_docker_plugin.yml b/.github/workflows/build_publish_docker_plugin.yml new file mode 100644 index 0000000000000..42af5265c7d14 --- /dev/null +++ b/.github/workflows/build_publish_docker_plugin.yml @@ -0,0 +1,49 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*- + +name: Release Build for Docker Plugin + +on: + release: + types: [published] + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + build_docker_volume_plugin: + if: inputs.manual || github.repository == 'rclone/rclone' + name: Build docker plugin job + runs-on: ubuntu-latest + steps: + - name: Free some space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + - name: Checkout master + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build and publish docker plugin + shell: bash + run: | + VER=${GITHUB_REF#refs/tags/} + PLUGIN_USER=rclone + docker login --username ${{ secrets.DOCKER_HUB_USER }} \ + --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" + for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do + export PLUGIN_USER PLUGIN_ARCH + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + done + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml deleted file mode 100644 index b69285a7e5bb8..0000000000000 --- a/.github/workflows/build_publish_release_docker_image.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Docker release build - -on: - release: - types: [published] - -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get actual patch version - id: actual_patch_version - run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g') - - name: Get actual minor version - id: actual_minor_version - run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2) - - name: Get actual major version - id: actual_major_version - run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1) - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - push: true - tags: | - rclone/rclone:latest - rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }} - rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }} - rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }} - - build_docker_volume_plugin: - if: github.repository == 'rclone/rclone' - needs: build - runs-on: ubuntu-latest - name: Build docker plugin job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build and publish docker plugin - shell: bash - run: | - VER=${GITHUB_REF#refs/tags/} - PLUGIN_USER=rclone - docker login --username ${{ secrets.DOCKER_HUB_USER }} \ - --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" - for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do - export PLUGIN_USER PLUGIN_ARCH - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} - done - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} diff --git a/.renku/Dockerfile b/.renku/Dockerfile new file mode 100644 index 0000000000000..a3e34b8007186 --- /dev/null +++ b/.renku/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +ARG TARGETPLATFORM +COPY ${TARGETPLATFORM}/rclone rclone diff --git a/Dockerfile b/Dockerfile index 82f16aad784a1..bb086159492c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,47 @@ FROM golang:alpine AS builder -COPY . /go/src/github.com/rclone/rclone/ +ARG CGO_ENABLED=0 + WORKDIR /go/src/github.com/rclone/rclone/ -RUN apk add --no-cache make bash gawk git -RUN \ - CGO_ENABLED=0 \ - make -RUN ./rclone version +RUN echo "**** Set Go Environment Variables ****" && \ + go env -w GOCACHE=/root/.cache/go-build + +RUN echo "**** Install Dependencies ****" && \ + apk add --no-cache \ + make \ + bash \ + gawk \ + git + +COPY go.mod . +COPY go.sum . + +RUN echo "**** Download Go Dependencies ****" && \ + go mod download -x + +RUN echo "**** Verify Go Dependencies ****" && \ + go mod verify + +COPY . . + +RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \ + echo "**** Build Binary ****" && \ + make + +RUN echo "**** Print Version Binary ****" && \ + ./rclone version # Begin final image FROM alpine:latest -RUN apk --no-cache add ca-certificates fuse3 tzdata && \ - echo "user_allow_other" >> /etc/fuse.conf +RUN echo "**** Install Dependencies ****" && \ + apk add --no-cache \ + ca-certificates \ + fuse3 \ + tzdata && \ + echo "Enable user_allow_other in fuse" && \ + echo "user_allow_other" >> /etc/fuse.conf COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/ diff --git a/MANUAL.html b/MANUAL.html index 29c5bfac1b4d4..471b22e475fbc 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -5,6 +5,7 @@ + rclone(1) User Manual