Merge pull request #706 from crazy-max/workflow

Enhance workflow

.dockerignore

@@ -1,8 +1 @@
-/.idea
-/*.iml
-
-/.dev
 /bin
-/dist
-/site
-/coverage.txt

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 name: build
 
 concurrency:
-  group: build-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
@@ -15,10 +15,37 @@ on:
 env:
   DOCKERHUB_SLUG: crazymax/diun
   GHCR_SLUG: ghcr.io/crazy-max/diun
+  DESTDIR: ./bin
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      validate-targets: ${{ steps.validate-targets.outputs.matrix }}
+      artifact-platforms: ${{ steps.artifact-platforms.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Validate targets matrix
+        id: validate-targets
+        run: |
+          echo "matrix=$(docker buildx bake validate --print | jq -cr '.group.default.targets')" >> $GITHUB_OUTPUT
+      -
+        name: Artifact platforms matrix
+        id: artifact-platforms
+        run: |
+          echo "matrix=$(docker buildx bake artifact-all --print | jq -cr '.target."artifact-all".platforms')" >> $GITHUB_OUTPUT
+
   validate:
     runs-on: ubuntu-latest
+    needs:
+      - prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJson(needs.prepare.outputs.validate-targets) }}
     steps:
       -
         name: Checkout
@@ -30,8 +57,7 @@ jobs:
         name: Validate
         uses: docker/bake-action@v2
         with:
-          targets: validate
-          pull: true
+          targets: ${{ matrix.target }}
 
   test:
     runs-on: ubuntu-latest
@@ -54,12 +80,90 @@ jobs:
         name: Upload coverage
         uses: codecov/codecov-action@v3
         with:
-          file: ./coverage.txt
+          directory: ${{ env.DESTDIR }}/coverage
 
-  build:
+  artifact:
     runs-on: ubuntu-latest
     needs:
+      - prepare
       - validate
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.prepare.outputs.artifact-platforms) }}
+    steps:
+      -
+        name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build
+        uses: docker/bake-action@v2
+        with:
+          targets: artifact
+          pull: true
+          set: |
+            *.platform=${{ matrix.platform }}
+            *.cache-from=type=gha,scope=artifact-${{ env.PLATFORM_PAIR }}
+            *.cache-to=type=gha,scope=artifact-${{ env.PLATFORM_PAIR }},mode=max
+      -
+        name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: diun
+          path: ${{ env.DESTDIR }}
+          if-no-files-found: error
+
+  release:
+    runs-on: ubuntu-latest
+    needs:
+      - artifact
+      - test
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: diun
+          path: ${{ env.DESTDIR }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build
+        uses: docker/bake-action@v2
+        with:
+          targets: release
+      -
+        name: GitHub Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          draft: true
+          files: |
+            ${{ env.DESTDIR }}/release/*
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  image:
+    runs-on: ubuntu-latest
+    needs:
+      - artifact
       - test
     steps:
       -
@@ -67,6 +171,17 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      -
+        name: Prepare
+        run: |
+          cfroms=
+          while read -r platform; do
+            if [ -n "$cfroms" ]; then cfroms="${cfroms}\n"; fi
+            cfroms="${cfroms}*.cache-from=type=gha,scope=artifact-${platform//\//-}"
+          done < <(docker buildx bake artifact-all --print | jq -r '.target."artifact-all".platforms[]')
+          echo "CACHE_FROMS<<EOF" >> $GITHUB_ENV
+          echo -e "$cfroms" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
       -
         name: Docker meta
         id: meta
@@ -107,24 +222,7 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Build artifacts
-        uses: docker/bake-action@v2
-        with:
-          targets: artifact-all
-          pull: true
-      -
-        name: Move artifacts
-        run: |
-          mv ./dist/**/* ./dist/
-      -
-        name: Upload artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: diun
-          path: ./dist/*
-          if-no-files-found: error
-      -
-        name: Build image
+        name: Build
         uses: docker/bake-action@v2
         with:
           files: |
@@ -133,17 +231,8 @@ jobs:
           targets: image-all
           pull: true
           push: ${{ github.event_name != 'pull_request' }}
-      -
-        name: GitHub Release
-        uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
-        with:
-          draft: true
-          files: |
-            dist/*.tar.gz
-            dist/*.zip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          set: |
+            ${{ env.CACHE_FROMS }}
       -
         name: Check manifest
         if: github.event_name != 'pull_request'

.github/workflows/codeql.yml

@@ -1,7 +1,7 @@
 name: codeql
 
 concurrency:
-  group: codeql-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:

.github/workflows/docs.yml

@@ -1,7 +1,7 @@
 name: docs
 
 concurrency:
-  group: docs-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
@@ -11,6 +11,9 @@ on:
       - 'v*'
   pull_request:
 
+env:
+  DESTDIR: ./bin
+
 jobs:
   publish:
     runs-on: ubuntu-latest
@@ -40,7 +43,7 @@ jobs:
         uses: crazy-max/ghaction-github-pages@v3
         with:
           target_branch: gh-pages
-          build_dir: site
+          build_dir: ${{ env.DESTDIR }}/site
           jekyll: false
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/e2e.yml

@@ -1,7 +1,7 @@
 name: e2e
 
 concurrency:
-  group: e2e-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:

.github/workflows/labels.yml

@@ -1,7 +1,7 @@
 name: labels
 
 concurrency:
-  group: labels-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:

.gitignore

@@ -1,8 +1 @@
-/.idea
-/*.iml
-
-/.dev
 /bin
-/dist
-/site
-/coverage.txt

Dockerfile

@@ -1,59 +1,101 @@
 # syntax=docker/dockerfile:1
 
 ARG GO_VERSION="1.19"
-ARG GORELEASER_XX_VERSION="1.2.5"
+ARG ALPINE_VERSION="3.16"
+ARG XX_VERSION="1.1.2"
+
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 
-FROM --platform=$BUILDPLATFORM crazymax/goreleaser-xx:${GORELEASER_XX_VERSION} AS goreleaser-xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base
+COPY --from=xx / /
 ENV CGO_ENABLED=0
-COPY --from=goreleaser-xx / /
 RUN apk add --no-cache file git
 WORKDIR /src
 
+FROM base AS version
+RUN --mount=target=. \
+  echo $(git describe --match 'v[0-9]*' --dirty='.m' --always --tags) | tee /tmp/.version
+
 FROM base AS vendored
-RUN --mount=type=bind,source=.,target=/src,rw \
-  --mount=type=cache,target=/go/pkg/mod \
-  go mod tidy && go mod download
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+  go mod download
 
 FROM vendored AS test
 ENV CGO_ENABLED=1
 RUN apk add --no-cache gcc linux-headers musl-dev
 RUN --mount=type=bind,target=. \
-  --mount=type=cache,target=/go/pkg/mod \
-  --mount=type=cache,target=/root/.cache/go-build <<EOT
-go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic -race ./...
-go tool cover -func=/tmp/coverage.txt
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build <<EOT
+  set -ex
+  go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic -race ./...
+  go tool cover -func=/tmp/coverage.txt
 EOT
 
 FROM scratch AS test-coverage
 COPY --from=test /tmp/coverage.txt /coverage.txt
 
 FROM vendored AS build
-ARG GIT_REF
 ARG TARGETPLATFORM
 RUN --mount=type=bind,target=. \
-  --mount=type=cache,target=/root/.cache \
-  --mount=target=/go/pkg/mod,type=cache \
-  goreleaser-xx --debug \
-    --name "diun" \
-    --dist "/out" \
-    --main="./cmd" \
-    --flags="-trimpath" \
-    --ldflags="-s -w -X 'main.version={{.Version}}'" \
-    --files="CHANGELOG.md" \
-    --files="LICENSE" \
-    --files="README.md"
+    --mount=type=bind,from=version,source=/tmp/.version,target=/tmp/.version \
+    --mount=type=cache,target=/root/.cache \
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+  set -ex
+  xx-go build -trimpath -ldflags "-s -w -X main.version=$(cat /tmp/.version)" -o /usr/bin/diun ./cmd
+  xx-verify --static /usr/bin/diun
+EOT
+
+FROM scratch AS binary-unix
+COPY --link --from=build /usr/bin/diun /
+
+FROM scratch AS binary-windows
+COPY --link --from=build /usr/bin/diun /diun.exe
+
+FROM binary-unix AS binary-darwin
+FROM binary-unix AS binary-linux
+FROM binary-$TARGETOS AS binary
+
+FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS build-artifact
+RUN apk add --no-cache bash tar zip
+WORKDIR /work
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+RUN --mount=type=bind,target=/src \
+    --mount=type=bind,from=binary,target=/build \
+    --mount=type=bind,from=version,source=/tmp/.version,target=/tmp/.version <<EOT
+  set -ex
+  mkdir /out
+  version=$(cat /tmp/.version)
+  cp /build/* /src/CHANGELOG.md /src/LICENSE /src/README.md .
+  if [ "$TARGETOS" = "windows" ]; then
+    zip -r "/out/diun_${version#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.zip" .
+  else
+    tar -czvf "/out/diun_${version#v}_${TARGETOS}_${TARGETARCH}${TARGETVARIANT}.tar.tgz" .
+  fi
+EOT
 
 FROM scratch AS artifact
-COPY --from=build /out/*.tar.gz /
-COPY --from=build /out/*.zip /
+COPY --link --from=build-artifact /out /
 
-FROM scratch AS binary
-COPY --from=build /usr/local/bin/diun* /
+FROM scratch AS artifacts
+FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS releaser
+RUN apk add --no-cache bash coreutils
+WORKDIR /out
+RUN --mount=from=artifacts,source=.,target=/artifacts <<EOT
+  set -e
+  cp /artifacts/**/* /out/ 2>/dev/null || cp /artifacts/* /out/
+  sha256sum -b diun_* > ./checksums.txt
+  sha256sum -c --strict checksums.txt
+EOT
 
-FROM alpine:3.16
+FROM scratch AS release
+COPY --link --from=releaser /out /
+
+FROM alpine:${ALPINE_VERSION}
 RUN apk --update --no-cache add ca-certificates openssl
-COPY --from=build /usr/local/bin/diun /usr/local/bin/diun
+COPY --from=build /usr/bin/diun /usr/local/bin/diun
 ENV PROFILER_PATH="/profiler" \
   DIUN_DB_PATH="/data/diun.db"
 VOLUME [ "/data" ]

docker-bake.hcl

@@ -2,20 +2,17 @@ variable "GO_VERSION" {
   default = "1.19"
 }
 
-// GITHUB_REF is the actual ref that triggers the workflow
-// https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
-variable "GITHUB_REF" {
-  default = ""
+variable "DESTDIR" {
+  default = "./bin"
 }
 
 target "_common" {
   args = {
     GO_VERSION = GO_VERSION
-    GIT_REF = GITHUB_REF
   }
 }
 
-// Special target: https://github.com/docker/metadata-action#bake-definition
+# Special target: https://github.com/docker/metadata-action#bake-definition
 target "docker-metadata-action" {
   tags = ["diun:local"]
 }
@@ -27,13 +24,13 @@ group "default" {
 target "binary" {
   inherits = ["_common"]
   target = "binary"
-  output = ["./bin"]
+  output = ["${DESTDIR}/build"]
 }
 
 target "artifact" {
   inherits = ["_common"]
   target = "artifact"
-  output = ["./dist"]
+  output = ["${DESTDIR}/artifact"]
 }
 
 target "artifact-all" {
@@ -56,6 +53,14 @@ target "artifact-all" {
   ]
 }
 
+target "release" {
+  target = "release"
+  output = ["${DESTDIR}/release"]
+  contexts = {
+    artifacts = "${DESTDIR}/artifact"
+  }
+}
+
 target "image" {
   inherits = ["_common", "docker-metadata-action"]
 }
@@ -80,7 +85,7 @@ target "image-all" {
 target "test" {
   inherits = ["_common"]
   target = "test-coverage"
-  output = ["."]
+  output = ["${DESTDIR}/coverage"]
 }
 
 target "vendor" {
@@ -100,7 +105,7 @@ target "gen" {
 target "docs" {
   dockerfile = "./hack/docs.Dockerfile"
   target = "release"
-  output = ["./site"]
+  output = ["${DESTDIR}/site"]
 }
 
 target "gomod-outdated" {

hack/gen.Dockerfile

@@ -6,35 +6,41 @@ ARG GLIBC_VERSION="2.33-r0"
 
 FROM golang:${GO_VERSION}-alpine AS base
 ARG GLIBC_VERSION
-RUN apk add --no-cache curl file git unzip \
-  && curl -sSL "https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub" -o "/etc/apk/keys/sgerrand.rsa.pub" \
-  && curl -sSL "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk" -o "glibc.apk" \
-  && apk add glibc.apk \
-  && rm /etc/apk/keys/sgerrand.rsa.pub glibc.apk
+RUN apk add --no-cache curl file git unzip
+RUN <<EOT
+  set -e
+  curl -sSL "https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub" -o "/etc/apk/keys/sgerrand.rsa.pub"
+  curl -sSL "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk" -o "glibc.apk"
+  apk add glibc.apk
+  rm /etc/apk/keys/sgerrand.rsa.pub glibc.apk
+EOT
 ARG PROTOC_VERSION
-RUN curl -sSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" -o "protoc.zip" \
-  && unzip "protoc.zip" -d "/usr/local" \
-  && protoc --version \
-  && rm "protoc.zip"
+RUN <<EOT
+  set -e
+  curl -sSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" -o "protoc.zip"
+  unzip "protoc.zip" -d "/usr/local"
+  protoc --version
+  rm "protoc.zip"
+EOT
 WORKDIR /src
 
 FROM base AS vendored
-RUN --mount=type=bind,target=.,rw \
-  --mount=type=cache,target=/go/pkg/mod \
-  go mod tidy && go mod download
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+  go mod download
 
 FROM vendored AS tools
 RUN --mount=type=bind,target=.,rw \
-  --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/go/pkg/mod \
   go install -v $(sed -n -e 's|^\s*_\s*"\(.*\)".*$|\1| p' tools.go)
 
 FROM tools AS generate
 RUN --mount=type=bind,target=.,rw \
-  --mount=type=cache,target=/go/pkg/mod <<EOT
-set -e
-go generate ./...
-mkdir /out
-cp -Rf pb /out
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+  set -e
+  go generate ./...
+  mkdir /out
+  cp -Rf pb /out
 EOT
 
 FROM scratch AS update
@@ -42,13 +48,13 @@ COPY --from=generate /out /
 
 FROM generate AS validate
 RUN --mount=type=bind,target=.,rw <<EOT
-set -e
-git add -A
-cp -rf /out/* .
-diff=$(git status --porcelain -- pb)
-if [ -n "$diff" ]; then
-  echo >&2 'ERROR: Vendor result differs. Please vendor your package with "docker buildx bake gen"'
-  echo "$diff"
-  exit 1
-fi
+  set -e
+  git add -A
+  cp -rf /out/* .
+  diff=$(git status --porcelain -- pb)
+  if [ -n "$diff" ]; then
+    echo >&2 'ERROR: Vendor result differs. Please vendor your package with "docker buildx bake gen"'
+    echo "$diff"
+    exit 1
+  fi
 EOT

hack/lint.Dockerfile

@@ -11,6 +11,6 @@ WORKDIR /src
 FROM golangci/golangci-lint:${GOLANGCI_LINT_VERSION}-alpine AS golangci-lint
 FROM base AS lint
 RUN --mount=type=bind,target=. \
-  --mount=type=cache,target=/root/.cache \
-  --mount=from=golangci-lint,source=/usr/bin/golangci-lint,target=/usr/bin/golangci-lint \
+    --mount=type=cache,target=/root/.cache \
+    --mount=from=golangci-lint,source=/usr/bin/golangci-lint,target=/usr/bin/golangci-lint \
   golangci-lint run ./...

hack/vendor.Dockerfile

@@ -9,12 +9,12 @@ WORKDIR /src
 
 FROM base AS vendored
 RUN --mount=type=bind,target=.,rw \
-  --mount=type=cache,target=/go/pkg/mod <<EOT
-set -e
-go mod tidy
-go mod download
-mkdir /out
-cp go.mod go.sum /out
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+  set -e
+  go mod tidy
+  go mod download
+  mkdir /out
+  cp go.mod go.sum /out
 EOT
 
 FROM scratch AS update
@@ -22,20 +22,20 @@ COPY --from=vendored /out /
 
 FROM vendored AS validate
 RUN --mount=type=bind,target=.,rw <<EOT
-set -e
-git add -A
-cp -rf /out/* .
-diff=$(git status --porcelain -- go.mod go.sum)
-if [ -n "$diff" ]; then
-  echo >&2 'ERROR: Vendor result differs. Please vendor your package with "docker buildx bake vendor"'
-  echo "$diff"
-  exit 1
-fi
+  set -e
+  git add -A
+  cp -rf /out/* .
+  diff=$(git status --porcelain -- go.mod go.sum)
+  if [ -n "$diff" ]; then
+    echo >&2 'ERROR: Vendor result differs. Please vendor your package with "docker buildx bake vendor"'
+    echo "$diff"
+    exit 1
+  fi
 EOT
 
 FROM psampaz/go-mod-outdated:${GOMOD_OUTDATED_VERSION} AS go-mod-outdated
 FROM base AS outdated
 RUN --mount=type=bind,target=. \
-  --mount=type=cache,target=/go/pkg/mod \
-  --mount=from=go-mod-outdated,source=/home/go-mod-outdated,target=/usr/bin/go-mod-outdated \
-  go list -mod=readonly -u -m -json all | go-mod-outdated -update -direct
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=from=go-mod-outdated,source=/home/go-mod-outdated,target=/usr/bin/go-mod-outdated \
+    go list -mod=readonly -u -m -json all | go-mod-outdated -update -direct