From c0bf81ad7ceb00e711a18f90c72b5e2183238fd4 Mon Sep 17 00:00:00 2001 From: Julien Neuhart Date: Thu, 23 Jan 2025 17:04:28 +0100 Subject: [PATCH] ci(workflows): multiple jobs multiple architectures build and push --- .env | 14 ++ .github/workflows/continuous-delivery.yml | 58 ++++++ .github/workflows/continuous-integration.yml | 190 +++++++++++++++++++ .github/workflows/continuous_delivery.yml | 29 --- .github/workflows/continuous_integration.yml | 102 ---------- .github/workflows/gotenberg-build-push.yml | 51 +++++ .github/workflows/gotenberg-merge-clean.yml | 40 ++++ Makefile | 32 +--- scripts/release.sh | 84 -------- 9 files changed, 356 insertions(+), 244 deletions(-) create mode 100644 .env create mode 100644 .github/workflows/continuous-delivery.yml create mode 100644 .github/workflows/continuous-integration.yml delete mode 100644 .github/workflows/continuous_delivery.yml delete mode 100644 .github/workflows/continuous_integration.yml create mode 100644 .github/workflows/gotenberg-build-push.yml create mode 100644 .github/workflows/gotenberg-merge-clean.yml delete mode 100755 scripts/release.sh diff --git a/.env b/.env new file mode 100644 index 00000000..2c3e2685 --- /dev/null +++ b/.env @@ -0,0 +1,14 @@ +GOLANG_VERSION=1.23 +DOCKER_REGISTRY=gotenberg +DOCKER_REPOSITORY=gotenberg +GOTENBERG_VERSION=snapshot +GOTENBERG_USER_GID=1001 +GOTENBERG_USER_UID=1001 +NOTO_COLOR_EMOJI_VERSION=v2.047 # See https://github.com/googlefonts/noto-emoji/releases. +PDFTK_VERSION=v3.3.3 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package. +PDFCPU_VERSION=v0.8.1 # See https://github.com/pdfcpu/pdfcpu/releases. +GOLANGCI_LINT_VERSION=v1.63.4 # See https://github.com/golangci/golangci-lint/releases. +GOTENBERG_VERSION=snapshot +DOCKERFILE=build/Dockerfile +DOCKERFILE_CLOUDRUN=build/Dockerfile.cloudrun +DOCKER_BUILD_CONTEXT='.' \ No newline at end of file diff --git a/.github/workflows/continuous-delivery.yml b/.github/workflows/continuous-delivery.yml new file mode 100644 index 00000000..449008ac --- /dev/null +++ b/.github/workflows/continuous-delivery.yml @@ -0,0 +1,58 @@ +name: Continuous Delivery + +on: + release: + types: [ published ] + +permissions: + contents: read + +jobs: + release_amd64: + name: Release linux/amd64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/amd64' + gotenberg_version: ${{ github.event.release.tag_name }} + + release_386: + name: Release linux/386 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/386' + gotenberg_version: ${{ github.event.release.tag_name }} + + release_arm64: + name: Release linux/arm64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm64' + gotenberg_version: ${{ github.event.release.tag_name }} + + release_arm_v7: + name: Release linux/arm/v7 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm/v7' + gotenberg_version: ${{ github.event.release.tag_name }} + + merge_clean_release_tags: + needs: + - release_amd64 + - release_386 + - release_arm64 + - release_arm_v7 + name: Merge and clean release tags + uses: ./.github/workflows/gotenberg-merge-clean.yml + secrets: inherit + with: + tags: "${{ needs.release_amd64.outputs.tags }},${{ needs.release_386.outputs.tags }},${{ needs.release_arm64.outputs.tags }},${{ needs.release_arm_v7.outputs.tags }}" + alternate_registry: 'thecodingmachine' diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml new file mode 100644 index 00000000..e073bc6c --- /dev/null +++ b/.github/workflows/continuous-integration.yml @@ -0,0 +1,190 @@ +name: Continuous Integration + +on: + push: + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ (github.event_name == 'pull_request' && github.event.pull_request.number) || 'main' }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' + cache: false + + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Run linters + uses: golangci/golangci-lint-action@v6 + with: + version: v1.63.4 + + tests: + needs: + - lint + name: Tests + # TODO: once arm64 actions are available, also run the tests on this architecture. + # See: https://github.com/actions/virtual-environments/issues/2552#issuecomment-771478000. + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Build testing environment + run: make build build-tests + + - name: Run tests + run: make tests-once + + - name: Upload to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + + snapshot_amd64: + if: github.event_name == 'pull_request' + needs: + - tests + name: Snapshot linux/amd64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/amd64' + gotenberg_version: pr-${{ github.event.pull_request.number }} + alternate_repository: 'snapshot' + + snapshot_386: + if: github.event_name == 'pull_request' + needs: + - tests + name: Snapshot linux/386 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/386' + gotenberg_version: pr-${{ github.event.pull_request.number }} + alternate_repository: 'snapshot' + + snapshot_arm64: + if: github.event_name == 'pull_request' + needs: + - tests + name: Snapshot linux/arm64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm64' + gotenberg_version: pr-${{ github.event.pull_request.number }} + alternate_repository: 'snapshot' + + snapshot_arm_v7: + if: github.event_name == 'pull_request' + needs: + - tests + name: Snapshot linux/arm/v7 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm/v7' + gotenberg_version: pr-${{ github.event.pull_request.number }} + alternate_repository: 'snapshot' + + merge_clean_snapshot_tags: + needs: + - snapshot_amd64 + - snapshot_386 + - snapshot_arm64 + - snapshot_arm_v7 + name: Merge and clean snapshot tags + uses: ./.github/workflows/gotenberg-merge-clean.yml + secrets: inherit + with: + tags: "${{ needs.snapshot_amd64.outputs.tags }},${{ needs.snapshot_386.outputs.tags }},${{ needs.snapshot_arm64.outputs.tags }},${{ needs.snapshot_arm_v7.outputs.tags }}" + + edge_amd64: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - tests + name: Edge linux/amd64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/amd64' + gotenberg_version: 'edge' + + edge_386: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - tests + name: Edge linux/386 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-latest' + platform: 'linux/386' + gotenberg_version: 'edge' + + edge_arm64: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - tests + name: Edge linux/arm64 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm64' + gotenberg_version: 'edge' + + edge_arm_v7: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: + - tests + name: Edge linux/arm/v7 + uses: ./.github/workflows/gotenberg-build-push.yml + secrets: inherit + with: + runs_on: 'ubuntu-24.04-arm' + platform: 'linux/arm/v7' + gotenberg_version: 'edge' + + merge_clean_edge_tags: + needs: + - edge_amd64 + - edge_386 + - edge_arm64 + - edge_arm_v7 + name: Merge and clean edge tags + uses: ./.github/workflows/gotenberg-merge-clean.yml + secrets: inherit + with: + tags: "${{ needs.edge_amd64.outputs.tags }},${{ needs.edge_386.outputs.tags }},${{ needs.edge_arm64.outputs.tags }},${{ needs.edge_arm_v7.outputs.tags }}" + alternate_registry: 'thecodingmachine' diff --git a/.github/workflows/continuous_delivery.yml b/.github/workflows/continuous_delivery.yml deleted file mode 100644 index ca229fe0..00000000 --- a/.github/workflows/continuous_delivery.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Continuous Delivery - -on: - release: - types: [ published ] - -permissions: - contents: read - -jobs: - release: - name: Release Docker image - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Checkout source code - uses: actions/checkout@v4 - - name: Log in to Docker Hub Container Registry - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Docker image for release - run: | - make release GOTENBERG_VERSION=${{ github.event.release.tag_name }} - make release GOTENBERG_VERSION=${{ github.event.release.tag_name }} DOCKER_REGISTRY=thecodingmachine diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml deleted file mode 100644 index 51cd4eeb..00000000 --- a/.github/workflows/continuous_integration.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Continuous Integration - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ (github.event_name == 'pull_request' && github.event.pull_request.number) || 'main' }} - cancel-in-progress: true - -permissions: - contents: write - -jobs: - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23' - cache: false - - name: Checkout source code - uses: actions/checkout@v4 - - name: Run linters - uses: golangci/golangci-lint-action@v6 - with: - version: v1.63.4 - - tests: - needs: - - lint - name: Tests - # TODO: once arm64 actions are available, also run the tests on this architecture. - # See: https://github.com/actions/virtual-environments/issues/2552#issuecomment-771478000. - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Checkout source code - uses: actions/checkout@v4 - - name: Build testing environment - run: make build build-tests - - name: Run tests - run: make tests-once - - name: Upload to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - - snapshot_release: - if: github.event_name == 'pull_request' - needs: - - tests - name: Snapshot release - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Checkout source code - uses: actions/checkout@v4 - - name: Log in to Docker Hub Container Registry - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push snapshot Docker image (linux/amd64) - run: make release GOTENBERG_VERSION=${{ github.head_ref }} DOCKER_REPOSITORY=snapshot LINUX_AMD64_RELEASE=true - - edge_release: - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: - - tests - name: Edge release - runs-on: ubuntu-latest - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Checkout source code - uses: actions/checkout@v4 - - name: Log in to Docker Hub Container Registry - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push Docker image for main branch - run: | - make release GOTENBERG_VERSION=edge - make release GOTENBERG_VERSION=edge DOCKER_REGISTRY=thecodingmachine diff --git a/.github/workflows/gotenberg-build-push.yml b/.github/workflows/gotenberg-build-push.yml new file mode 100644 index 00000000..82384e8d --- /dev/null +++ b/.github/workflows/gotenberg-build-push.yml @@ -0,0 +1,51 @@ +on: + workflow_call: + inputs: + runs_on: + type: string + required: true + platform: + type: string + required: true + gotenberg_version: + type: string + required: true + alternate_repository: + type: string + default: '' + outputs: + tags: + value: ${{ jobs.gotenberg_build_push.outputs.tags }} + +permissions: + contents: read + +jobs: + gotenberg_build_push: + runs-on: ${{ inputs.runs_on }} + outputs: + tags: ${{ steps.build_push.outputs.tags }} + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push ${{ inputs.platform }} + id: build_push + uses: gotenberg/gotenberg-release-action@main + with: + step: build_push + version: ${{ inputs.gotenberg_version }} + platform: ${{ inputs.platform }} + alternate_repository: ${{ inputs.alternate_repository }} diff --git a/.github/workflows/gotenberg-merge-clean.yml b/.github/workflows/gotenberg-merge-clean.yml new file mode 100644 index 00000000..e2ddf51f --- /dev/null +++ b/.github/workflows/gotenberg-merge-clean.yml @@ -0,0 +1,40 @@ +on: + workflow_call: + inputs: + tags: + type: string + required: true + alternate_registry: + type: string + default: '' + +permissions: + contents: read + +jobs: + gotenberg_merge_clean: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Merge and clean + uses: gotenberg/gotenberg-release-action@main + with: + step: merge_clean + docker_hub_username: ${{ secrets.DOCKERHUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKERHUB_TOKEN }} + tags: ${{ inputs.tags }} + alternate_registry: ${{ inputs.alternate_registry }} diff --git a/Makefile b/Makefile index de2e0dbd..6066e4f7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +include .env + .PHONY: help help: ## Show the help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -5,17 +7,6 @@ help: ## Show the help .PHONY: it it: build build-tests ## Initialize the development environment -GOLANG_VERSION=1.23 -DOCKER_REGISTRY=gotenberg -DOCKER_REPOSITORY=gotenberg -GOTENBERG_VERSION=snapshot -GOTENBERG_USER_GID=1001 -GOTENBERG_USER_UID=1001 -NOTO_COLOR_EMOJI_VERSION=v2.047 # See https://github.com/googlefonts/noto-emoji/releases. -PDFTK_VERSION=v3.3.3 # See https://gitlab.com/pdftk-java/pdftk/-/releases - Binary package. -PDFCPU_VERSION=v0.8.1 # See https://github.com/pdfcpu/pdfcpu/releases. -GOLANGCI_LINT_VERSION=v1.63.4 # See https://github.com/golangci/golangci-lint/releases. - .PHONY: build build: ## Build the Gotenberg's Docker image docker build \ @@ -27,7 +18,7 @@ build: ## Build the Gotenberg's Docker image --build-arg PDFTK_VERSION=$(PDFTK_VERSION) \ --build-arg PDFCPU_VERSION=$(PDFCPU_VERSION) \ -t $(DOCKER_REGISTRY)/$(DOCKER_REPOSITORY):$(GOTENBERG_VERSION) \ - -f build/Dockerfile . + -f $(DOCKERFILE) $(DOCKER_BUILD_CONTEXT) GOTENBERG_GRACEFUL_SHUTDOWN_DURATION=30s API_PORT=3000 @@ -199,20 +190,3 @@ fmt: ## Format the code and "optimize" the dependencies godoc: ## Run a webserver with Gotenberg godoc $(info http://localhost:6060/pkg/github.com/gotenberg/gotenberg/v8) godoc -http=:6060 - -LINUX_AMD64_RELEASE=false - -.PHONY: release -release: ## Build the Gotenberg's Docker image and push it to a Docker repository - ./scripts/release.sh \ - $(GOLANG_VERSION) \ - $(GOTENBERG_VERSION) \ - $(GOTENBERG_USER_GID) \ - $(GOTENBERG_USER_UID) \ - $(NOTO_COLOR_EMOJI_VERSION) \ - $(PDFTK_VERSION) \ - $(PDFCPU_VERSION) \ - $(DOCKER_REGISTRY) \ - $(DOCKER_REPOSITORY) \ - $(LINUX_AMD64_RELEASE) - diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index f9c014ff..00000000 --- a/scripts/release.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -set -e - -# Args. -GOLANG_VERSION="$1" -GOTENBERG_VERSION="$2" -GOTENBERG_USER_GID="$3" -GOTENBERG_USER_UID="$4" -NOTO_COLOR_EMOJI_VERSION="$5" -PDFTK_VERSION="$6" -PDFCPU_VERSION="$7" -DOCKER_REGISTRY="$8" -DOCKER_REPOSITORY="$9" -LINUX_AMD64_RELEASE="${10}" - -# Find out if given version is "semver". -GOTENBERG_VERSION="${GOTENBERG_VERSION//v}" -IFS='.' read -ra SEMVER <<< "$GOTENBERG_VERSION" -VERSION_LENGTH=${#SEMVER[@]} -TAGS=() -TAGS_CLOUD_RUN=() - -if [ "$VERSION_LENGTH" -eq 3 ]; then - MAJOR="${SEMVER[0]}" - MINOR="${SEMVER[1]}" - PATCH="${SEMVER[2]}" - - TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:latest") - TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR") - TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR") - TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR.$PATCH") - - TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:latest-cloudrun") - TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR-cloudrun") - TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR-cloudrun") - TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$MAJOR.$MINOR.$PATCH-cloudrun") -else - # Normalizes version. - GOTENBERG_VERSION="${GOTENBERG_VERSION// /-}" - GOTENBERG_VERSION="$(echo "$GOTENBERG_VERSION" | tr -cd '[:alnum:]._\-')" - - if [[ "$GOTENBERG_VERSION" =~ ^[\.\-] ]]; then - GOTENBERG_VERSION="_${GOTENBERG_VERSION#?}" - fi - - if [ "${#GOTENBERG_VERSION}" -gt 128 ]; then - GOTENBERG_VERSION="${GOTENBERG_VERSION:0:128}" - fi - - TAGS+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION") - TAGS_CLOUD_RUN+=("-t" "$DOCKER_REGISTRY/$DOCKER_REPOSITORY:$GOTENBERG_VERSION-cloudrun") -fi - -# Multi-arch build takes a lot of time. -if [ "$LINUX_AMD64_RELEASE" = true ]; then - PLATFORM_FLAG="--platform linux/amd64" -else - PLATFORM_FLAG="--platform linux/amd64,linux/arm64,linux/386,linux/arm/v7" -fi - -docker buildx build \ - --build-arg GOLANG_VERSION="$GOLANG_VERSION" \ - --build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \ - --build-arg GOTENBERG_USER_GID="$GOTENBERG_USER_GID" \ - --build-arg GOTENBERG_USER_UID="$GOTENBERG_USER_UID" \ - --build-arg NOTO_COLOR_EMOJI_VERSION="$NOTO_COLOR_EMOJI_VERSION" \ - --build-arg PDFTK_VERSION="$PDFTK_VERSION" \ - --build-arg PDFCPU_VERSION="$PDFCPU_VERSION" \ - $PLATFORM_FLAG \ - "${TAGS[@]}" \ - --push \ - -f build/Dockerfile . - -# Cloud Run variant. -# Only linux/amd64! See https://github.com/gotenberg/gotenberg/issues/505#issuecomment-1264679278. -docker buildx build \ - --build-arg DOCKER_REGISTRY="$DOCKER_REGISTRY" \ - --build-arg DOCKER_REPOSITORY="$DOCKER_REPOSITORY" \ - --build-arg GOTENBERG_VERSION="$GOTENBERG_VERSION" \ - --platform linux/amd64 \ - "${TAGS_CLOUD_RUN[@]}" \ - --push \ - -f build/Dockerfile.cloudrun .