From 38e808ee5fa7fb4e9d4d2b724bb3af6d95de51be Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Wed, 19 Feb 2025 20:23:46 +0530 Subject: [PATCH] build: cache go build & test Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- .github/workflows/build.yml | 328 +++++++----------- .github/workflows/build_android.yml | 212 +++++++++++ .../workflows/build_publish_docker_image.yml | 75 ++-- .github/workflows/lint.yml | 104 ++++++ Makefile | 6 +- bin/go-test-cache/go.mod | 3 + bin/go-test-cache/main.go | 123 +++++++ 7 files changed, 626 insertions(+), 225 deletions(-) create mode 100644 .github/workflows/build_android.yml create mode 100644 .github/workflows/lint.yml create mode 100644 bin/go-test-cache/go.mod create mode 100644 bin/go-test-cache/main.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 615dd2231..d9aef7acd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,11 @@ # Github Actions build for rclone # -*- compile-command: "yamllint -f parsable build.yml" -*- -name: build +name: Build & Push Binary Builds + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true # Trigger the workflow on push or pull request on: @@ -30,6 +34,7 @@ jobs: include: - job_name: linux + platform: linux/amd64/go1.24 os: ubuntu-latest go: '>=1.24.0-rc.1' gotags: cmount @@ -41,6 +46,7 @@ jobs: deploy: true - job_name: linux_386 + platform: linux/386/go1.24 os: ubuntu-latest go: '>=1.24.0-rc.1' goarch: 386 @@ -48,6 +54,7 @@ jobs: quicktest: true - job_name: mac_amd64 + platform: darwin/amd64/go1.24 os: macos-latest go: '>=1.24.0-rc.1' gotags: 'cmount' @@ -57,6 +64,7 @@ jobs: deploy: true - job_name: mac_arm64 + platform: darwin/arm64/go1.24 os: macos-latest go: '>=1.24.0-rc.1' gotags: 'cmount' @@ -64,6 +72,7 @@ jobs: deploy: true - job_name: windows + platform: windows/amd64/go1.24 os: windows-latest go: '>=1.24.0-rc.1' gotags: cmount @@ -74,6 +83,7 @@ jobs: deploy: true - job_name: other_os + platform: linux/amd64/other/go1.24 os: ubuntu-latest go: '>=1.24.0-rc.1' build_flags: '-exclude "^(windows/|darwin/|linux/)"' @@ -81,17 +91,17 @@ jobs: deploy: true - job_name: go1.23 + platform: linux/amd64/go1.23 os: ubuntu-latest go: '1.23' quicktest: true racequicktest: true name: ${{ matrix.job_name }} - runs-on: ${{ matrix.os }} steps: - - name: Checkout + - name: Checkout Repository uses: actions/checkout@v4 with: fetch-depth: 0 @@ -101,17 +111,9 @@ jobs: with: go-version: ${{ matrix.go }} check-latest: true + cache: false - - name: Set environment variables - shell: bash - run: | - echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV - echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV - echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV - if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi - if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi - - - name: Install Libraries on Linux + - name: Install FUSE & Libraries on Linux shell: bash run: | sudo modprobe fuse @@ -121,7 +123,7 @@ jobs: sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common if: matrix.os == 'ubuntu-latest' - - name: Install Libraries on macOS + - name: Install FUSE & Libraries on macOS shell: bash run: | # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788 @@ -134,7 +136,7 @@ jobs: brew install git-annex git-annex-remote-rclone if: matrix.os == 'macos-latest' - - name: Install Libraries on Windows + - name: Install FUSE & Libraries on Windows shell: powershell run: | $ProgressPreference = 'SilentlyContinue' @@ -150,7 +152,55 @@ jobs: Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe') if: matrix.os == 'windows-latest' - - name: Print Go version and environment + - name: Set Environment Variables + shell: bash + run: | + echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV + echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV + echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV + if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi + if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + + - name: Set Go Environment Variables + shell: bash + run: | + echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV + echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_ENV + if: matrix.os != 'windows-latest' + + - name: Set Go Environment Variables + # This is necessary for Windows because the default location of C: is particularly slow for github runners: https://github.com/actions/setup-go/pull/515 + shell: bash + run: | + echo "GOCACHE=D:\gocache" >> $GITHUB_ENV + echo "GOMODCACHE=D:\gomodcache" >> $GITHUB_ENV + echo "GOTMPDIR=D:\gotmp" >> $GITHUB_ENV + mkdir D:/gocache D:/gomodcache D:/gotmp + if: matrix.os == 'windows-latest' + + - name: Set PLATFORM Variable + shell: bash + run: | + platform=${{ matrix.platform }} + echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV + + - name: Get ImageOS + # There's no way around this, because "ImageOS" is only available to + # processes, but the setup-go action uses it in its key. + id: imageos + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + return process.env.ImageOS + + - name: Set CACHE_PREFIX Variable + shell: bash + run: | + cache_prefix=${{ runner.os }}-${{ steps.imageos.outputs.result }}-${{ env.PLATFORM }} + echo "CACHE_PREFIX=${cache_prefix}" >> $GITHUB_ENV + + - name: Print Go Version and Environment shell: bash run: | printf "Using go at: $(which go)\n" @@ -162,44 +212,95 @@ jobs: printf "\n\nSystem environment:\n\n" env + - name: Load Go Module Cache + uses: actions/cache@v4 + with: + path: | + ${{ env.GOMODCACHE }} + key: ${{ env.CACHE_PREFIX }}-modcache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-modcache + + # Both load & update the cache when on default branch + - name: Load Go Build & Test Cache + id: go-cache + uses: actions/cache@v4 + if: github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' + with: + path: | + ${{ env.GOCACHE }} + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache + + # Only load the cache when not on default branch + - name: Load Go Build & Test Cache + id: go-cache-restore + uses: actions/cache/restore@v4 + if: github.ref_name != github.event.repository.default_branch || github.event_name == 'pull_request' + with: + path: | + ${{ env.GOCACHE }} + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache + - name: Build rclone shell: bash run: | make - - name: Rclone version + - name: Print rclone Version shell: bash run: | rclone version - - name: Run tests + - name: Update mtime + shell: bash + run: | + go run ./bin/go-test-cache/main.go + + - name: Run Tests shell: bash run: | make quicktest if: matrix.quicktest - - name: Race test + - name: Race Test shell: bash run: | make racequicktest - if: matrix.racequicktest + if: (success() || failure()) && matrix.racequicktest - - name: Run librclone tests + - name: Run librclone Tests shell: bash run: | make -C librclone/ctest test make -C librclone/ctest clean librclone/python/test_rclone.py - if: matrix.librclonetest + if: (success() || failure()) && matrix.librclonetest - - name: Compile all architectures test + - name: Compile All Architectures Test shell: bash run: | make make compile_all - if: matrix.compile_all + if: (success() || failure()) && matrix.compile_all - - name: Deploy built binaries + - name: Delete Existing Cache + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cache_ids=($(gh cache list --key "${{ env.CACHE_PREFIX }}-cache" --json id | jq '.[].id')) + for cache_id in "${cache_ids[@]}"; do + echo "Deleting Cache: $cache_id" + gh cache delete "$cache_id" + done + if: (success() || failure()) && github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' && steps.go-cache.outputs.cache-hit != 'true' + + - name: Deploy Built Binaries shell: bash run: | if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi @@ -209,180 +310,3 @@ jobs: # working-directory: '$(modulePath)' # Deploy binaries if enabled in config && not a PR && not a fork if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone' - - lint: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) - timeout-minutes: 30 - name: "lint" - runs-on: ubuntu-latest - - steps: - - name: Get runner parameters - id: get-runner-parameters - shell: bash - run: | - echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT - echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT - - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Go - id: setup-go - uses: actions/setup-go@v5 - with: - go-version: '>=1.23.0-rc.1' - check-latest: true - cache: false - - - name: Cache - uses: actions/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/.cache/golangci-lint - key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} - restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}- - - - name: Code quality test (Linux) - uses: golangci/golangci-lint-action@v6 - with: - version: latest - skip-cache: true - - - name: Code quality test (Windows) - uses: golangci/golangci-lint-action@v6 - env: - GOOS: "windows" - with: - version: latest - skip-cache: true - - - name: Code quality test (macOS) - uses: golangci/golangci-lint-action@v6 - env: - GOOS: "darwin" - with: - version: latest - skip-cache: true - - - name: Code quality test (FreeBSD) - uses: golangci/golangci-lint-action@v6 - env: - GOOS: "freebsd" - with: - version: latest - skip-cache: true - - - name: Code quality test (OpenBSD) - uses: golangci/golangci-lint-action@v6 - env: - GOOS: "openbsd" - with: - version: latest - skip-cache: true - - - name: Install govulncheck - run: go install golang.org/x/vuln/cmd/govulncheck@latest - - - name: Scan for vulnerabilities - run: govulncheck ./... - - android: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) - timeout-minutes: 30 - name: "android-all" - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Upgrade together with NDK version - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '>=1.24.0-rc.1' - - - name: Set global environment variables - shell: bash - run: | - echo "VERSION=$(make version)" >> $GITHUB_ENV - - - name: build native rclone - run: | - make - - - name: install gomobile - run: | - go install golang.org/x/mobile/cmd/gobind@latest - go install golang.org/x/mobile/cmd/gomobile@latest - env PATH=$PATH:~/go/bin gomobile init - echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV - - - name: arm-v7a gomobile build - run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile - - - name: arm-v7a Set environment variables - shell: bash - run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV - echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV - echo 'GOOS=android' >> $GITHUB_ENV - echo 'GOARCH=arm' >> $GITHUB_ENV - echo 'GOARM=7' >> $GITHUB_ENV - echo 'CGO_ENABLED=1' >> $GITHUB_ENV - echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - - - name: arm-v7a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a . - - - name: arm64-v8a Set environment variables - shell: bash - run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV - echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV - echo 'GOOS=android' >> $GITHUB_ENV - echo 'GOARCH=arm64' >> $GITHUB_ENV - echo 'CGO_ENABLED=1' >> $GITHUB_ENV - echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - - - name: arm64-v8a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a . - - - name: x86 Set environment variables - shell: bash - run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV - echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV - echo 'GOOS=android' >> $GITHUB_ENV - echo 'GOARCH=386' >> $GITHUB_ENV - echo 'CGO_ENABLED=1' >> $GITHUB_ENV - echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - - - name: x86 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 . - - - name: x64 Set environment variables - shell: bash - run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV - echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV - echo 'GOOS=android' >> $GITHUB_ENV - echo 'GOARCH=amd64' >> $GITHUB_ENV - echo 'CGO_ENABLED=1' >> $GITHUB_ENV - echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - - - name: x64 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 . - - - name: Upload artifacts - run: | - make ci_upload - env: - RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} - # Upload artifacts if not a PR && not a fork - if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone' diff --git a/.github/workflows/build_android.yml b/.github/workflows/build_android.yml new file mode 100644 index 000000000..895b54b9b --- /dev/null +++ b/.github/workflows/build_android.yml @@ -0,0 +1,212 @@ +--- +# Github Actions build for rclone +# -*- compile-command: "yamllint -f parsable build_android.yml" -*- + +name: Build & Push Android Builds + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '**' + tags: + - '**' + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + android: + if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - job_name: android-all + platform: linux/amd64/android/go1.24 + os: ubuntu-latest + go: '>=1.24.0-rc.1' + + name: ${{ matrix.job_name }} + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Upgrade together with NDK version + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + check-latest: true + cache: false + + - name: Set Environment Variables + shell: bash + run: | + echo "GOMODCACHE=$(go env GOMODCACHE)" >> $GITHUB_ENV + echo "GOCACHE=$(go env GOCACHE)" >> $GITHUB_ENV + echo "VERSION=$(make version)" >> $GITHUB_ENV + + - name: Set PLATFORM Variable + shell: bash + run: | + platform=${{ matrix.platform }} + echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV + + - name: Get ImageOS + # There's no way around this, because "ImageOS" is only available to + # processes, but the setup-go action uses it in its key. + id: imageos + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + return process.env.ImageOS + + - name: Set CACHE_PREFIX Variable + shell: bash + run: | + cache_prefix=${{ runner.os }}-${{ steps.imageos.outputs.result }}-${{ env.PLATFORM }} + echo "CACHE_PREFIX=${cache_prefix}" >> $GITHUB_ENV + + - name: Load Go Module Cache + uses: actions/cache@v4 + with: + path: | + ${{ env.GOMODCACHE }} + key: ${{ env.CACHE_PREFIX }}-modcache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-modcache + + # Both load & update the cache when on default branch + - name: Load Go Build & Test Cache + id: go-cache + uses: actions/cache@v4 + if: github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' + with: + path: | + ${{ env.GOCACHE }} + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache + + # Only load the cache when not on default branch + - name: Load Go Build & Test Cache + id: go-cache-restore + uses: actions/cache/restore@v4 + if: github.ref_name != github.event.repository.default_branch || github.event_name == 'pull_request' + with: + path: | + ${{ env.GOCACHE }} + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache + + - name: Build Native rclone + shell: bash + run: | + make + + - name: Install gomobile + shell: bash + run: | + go install golang.org/x/mobile/cmd/gobind@latest + go install golang.org/x/mobile/cmd/gomobile@latest + env PATH=$PATH:~/go/bin gomobile init + echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV + + - name: arm-v7a - gomobile build + shell: bash + run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile + + - name: arm-v7a - Set Environment Variables + shell: bash + run: | + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV + echo 'GOOS=android' >> $GITHUB_ENV + echo 'GOARCH=arm' >> $GITHUB_ENV + echo 'GOARM=7' >> $GITHUB_ENV + echo 'CGO_ENABLED=1' >> $GITHUB_ENV + echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV + + - name: arm-v7a - Build + shell: bash + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a . + + - name: arm64-v8a - Set Environment Variables + shell: bash + run: | + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV + echo 'GOOS=android' >> $GITHUB_ENV + echo 'GOARCH=arm64' >> $GITHUB_ENV + echo 'CGO_ENABLED=1' >> $GITHUB_ENV + echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV + + - name: arm64-v8a - Build + shell: bash + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a . + + - name: x86 - Set Environment Variables + shell: bash + run: | + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV + echo 'GOOS=android' >> $GITHUB_ENV + echo 'GOARCH=386' >> $GITHUB_ENV + echo 'CGO_ENABLED=1' >> $GITHUB_ENV + echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV + + - name: x86 - Build + shell: bash + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 . + + - name: x64 - Set Environment Variables + shell: bash + run: | + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV + echo 'GOOS=android' >> $GITHUB_ENV + echo 'GOARCH=amd64' >> $GITHUB_ENV + echo 'CGO_ENABLED=1' >> $GITHUB_ENV + echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV + + - name: x64 - Build + shell: bash + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 . + + - name: Delete Existing Cache + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cache_ids=($(gh cache list --key "${{ env.CACHE_PREFIX }}-cache" --json id | jq '.[].id')) + for cache_id in "${cache_ids[@]}"; do + echo "Deleting Cache: $cache_id" + gh cache delete "$cache_id" + done + if: github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' && steps.go-cache.outputs.cache-hit != 'true' + + - name: Deploy Built Binaries + shell: bash + run: | + make ci_upload + env: + RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }} + # Upload artifacts if not a PR && not a fork + if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone' diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml index abc017bc3..2b8de9849 100644 --- a/.github/workflows/build_publish_docker_image.yml +++ b/.github/workflows/build_publish_docker_image.yml @@ -4,6 +4,10 @@ name: Build & Push Docker Images +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + # Trigger the workflow on push or pull request on: push: @@ -41,32 +45,26 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - name: Free Space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout Repository uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set REPO_NAME Variable + shell: bash run: | echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} - name: Set PLATFORM Variable + shell: bash run: | platform=${{ matrix.platform }} echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV - name: Set CACHE_NAME Variable shell: python + env: + GITHUB_EVENT_REPOSITORY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | import os, re @@ -82,8 +80,11 @@ jobs: ref_name_slug = "cache" - if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request": - ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME']) + if os.environ.get("GITHUB_REF_NAME"): + if os.environ['GITHUB_EVENT_NAME'] == "pull_request": + ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME']) + elif os.environ['GITHUB_REF_NAME'] != os.environ['GITHUB_EVENT_REPOSITORY_DEFAULT_BRANCH']: + ref_name_slug += "-ref-" + slugify(os.environ['GITHUB_REF_NAME']) with open(os.environ['GITHUB_ENV'], 'a') as env: env.write(f"CACHE_NAME={ref_name_slug}\n") @@ -98,6 +99,12 @@ jobs: script: | return process.env.ImageOS + - name: Set CACHE_PREFIX Variable + shell: bash + run: | + cache_prefix=${{ runner.os }}-${{ steps.imageos.outputs.result }}-${{ env.PLATFORM }}-docker-go + echo "CACHE_PREFIX=${cache_prefix}" >> $GITHUB_ENV + - name: Extract Metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 @@ -130,22 +137,35 @@ jobs: - name: Load Go Build Cache for Docker id: go-cache uses: actions/cache@v4 + if: github.ref_name == github.event.repository.default_branch with: - key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} # Cache only the go builds, the module download is cached via the docker layer caching path: | - go-build-cache + /tmp/go-build-cache + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache + + - name: Load Go Build Cache for Docker + id: go-cache-restore + uses: actions/cache/restore@v4 + if: github.ref_name != github.event.repository.default_branch + with: + # Cache only the go builds, the module download is cached via the docker layer caching + path: | + /tmp/go-build-cache + key: ${{ env.CACHE_PREFIX }}-cache-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + restore-keys: | + ${{ env.CACHE_PREFIX }}-cache - name: Inject Go Build Cache into Docker uses: reproducible-containers/buildkit-cache-dance@v3 with: cache-map: | { - "go-build-cache": "/root/.cache/go-build" + "/tmp/go-build-cache": "/root/.cache/go-build" } - skip-extraction: ${{ steps.go-cache.outputs.cache-hit }} + skip-extraction: ${{ steps.go-cache.outputs.cache-hit || steps.go-cache-restore.outputs.cache-hit }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -172,9 +192,10 @@ jobs: outputs: | type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.PLATFORM }}-${{ env.CACHE_NAME }} + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.PLATFORM }}-cache cache-to: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.PLATFORM }}-${{ env.CACHE_NAME }},image-manifest=true,mode=max,compression=zstd - name: Export Image Digest run: | @@ -190,6 +211,19 @@ jobs: retention-days: 1 if-no-files-found: error + - name: Delete Existing Cache + if: github.ref_name == github.event.repository.default_branch && steps.go-cache.outputs.cache-hit != 'true' + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cache_ids=($(gh cache list --key "${{ env.CACHE_PREFIX }}-cache" --json id | jq '.[].id')) + for cache_id in "${cache_ids[@]}"; do + echo "Deleting Cache: $cache_id" + gh cache delete "$cache_id" + done + merge-image: name: Merge & Push Final Docker Image runs-on: ubuntu-24.04 @@ -205,6 +239,7 @@ jobs: merge-multiple: true - name: Set REPO_NAME Variable + shell: bash run: | echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..35060c06e --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,104 @@ +--- +# Github Actions build for rclone +# -*- compile-command: "yamllint -f parsable lint.yml" -*- + +name: Lint & Vulnerability Check + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '**' + tags: + - '**' + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + lint: + if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + timeout-minutes: 30 + name: "lint" + runs-on: ubuntu-latest + + steps: + - name: Get runner parameters + id: get-runner-parameters + shell: bash + run: | + echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT + echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Go + id: setup-go + uses: actions/setup-go@v5 + with: + go-version: '>=1.23.0-rc.1' + check-latest: true + cache: false + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/.cache/golangci-lint + key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }} + restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}- + + - name: Code quality test (Linux) + uses: golangci/golangci-lint-action@v6 + with: + version: latest + skip-cache: true + + - name: Code quality test (Windows) + uses: golangci/golangci-lint-action@v6 + env: + GOOS: "windows" + with: + version: latest + skip-cache: true + + - name: Code quality test (macOS) + uses: golangci/golangci-lint-action@v6 + env: + GOOS: "darwin" + with: + version: latest + skip-cache: true + + - name: Code quality test (FreeBSD) + uses: golangci/golangci-lint-action@v6 + env: + GOOS: "freebsd" + with: + version: latest + skip-cache: true + + - name: Code quality test (OpenBSD) + uses: golangci/golangci-lint-action@v6 + env: + GOOS: "openbsd" + with: + version: latest + skip-cache: true + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Scan for vulnerabilities + run: govulncheck ./... diff --git a/Makefile b/Makefile index 2e55c5a0c..d79c1ba76 100644 --- a/Makefile +++ b/Makefile @@ -88,13 +88,13 @@ test: rclone test_all # Quick test quicktest: - RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./... + RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./... racequicktest: - RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./... + RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./... compiletest: - RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./... + RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./... # Do source code quality checks check: rclone diff --git a/bin/go-test-cache/go.mod b/bin/go-test-cache/go.mod new file mode 100644 index 000000000..3cec67bb6 --- /dev/null +++ b/bin/go-test-cache/go.mod @@ -0,0 +1,3 @@ +module go-test-cache + +go 1.24 diff --git a/bin/go-test-cache/main.go b/bin/go-test-cache/main.go new file mode 100644 index 000000000..39bb9b1ab --- /dev/null +++ b/bin/go-test-cache/main.go @@ -0,0 +1,123 @@ +// This code was copied from: +// https://github.com/fastly/cli/blob/main/scripts/go-test-cache/main.go +// which in turn is based on the following script and was generated using AI. +// https://github.com/airplanedev/blog-examples/blob/main/go-test-caching/update_file_timestamps.py?ref=airplane.ghost.io +// +// REFERENCE ARTICLE: +// https://web.archive.org/web/20240308061717/https://www.airplane.dev/blog/caching-golang-tests-in-ci +// +// It updates the mtime of the files to a mtime dervived from the sha1 hash of their contents. +package main + +import ( + "crypto/sha1" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + bufSize = 65536 + baseDate = 1684178360 + timeFormat = "2006-01-02 15:04:05" +) + +func main() { + repoRoot := "." + allDirs := make([]string, 0) + + err := filepath.Walk(repoRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + dirPath := filepath.Join(repoRoot, path) + relPath, _ := filepath.Rel(repoRoot, dirPath) + + if strings.HasPrefix(relPath, ".") { + return nil + } + + allDirs = append(allDirs, dirPath) + } else { + filePath := filepath.Join(repoRoot, path) + relPath, _ := filepath.Rel(repoRoot, filePath) + + if strings.HasPrefix(relPath, ".") { + return nil + } + + sha1Hash, err := getFileSHA1(filePath) + if err != nil { + return err + } + + modTime := getModifiedTime(sha1Hash) + + log.Printf("Setting modified time of file %s to %s\n", relPath, modTime.Format(timeFormat)) + err = os.Chtimes(filePath, modTime, modTime) + if err != nil { + return err + } + } + + return nil + }) + if err != nil { + log.Fatal("Error:", err) + } + + sort.Slice(allDirs, func(i, j int) bool { + return len(allDirs[i]) > len(allDirs[j]) || (len(allDirs[i]) == len(allDirs[j]) && allDirs[i] < allDirs[j]) + }) + + for _, dirPath := range allDirs { + relPath, _ := filepath.Rel(repoRoot, dirPath) + + log.Printf("Setting modified time of directory %s to %s\n", relPath, time.Unix(baseDate, 0).Format(timeFormat)) + err := os.Chtimes(dirPath, time.Unix(baseDate, 0), time.Unix(baseDate, 0)) + if err != nil { + log.Fatal("Error:", err) + } + } + + log.Println("Done") +} + +func getFileSHA1(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + // G401: Use of weak cryptographic primitive + // Disabling as the hash is used not for security reasons. + // The hash is used as a cache key to improve test run times. + // #nosec + // nosemgrep: go.lang.security.audit.crypto.use_of_weak_crypto.use-of-sha1 + hash := sha1.New() + if _, err := io.CopyBuffer(hash, file, make([]byte, bufSize)); err != nil { + return "", err + } + + return string(hash.Sum(nil)), nil +} + +func getModifiedTime(sha1Hash string) time.Time { + hashBytes := []byte(sha1Hash) + lastFiveBytes := hashBytes[:5] + lastFiveValue := int64(0) + + for _, b := range lastFiveBytes { + lastFiveValue = (lastFiveValue << 8) + int64(b) + } + + modTime := baseDate - (lastFiveValue % 10000) + return time.Unix(modTime, 0) +}