Refactor Dockerfile to build Go binary outside of Docker (#218)

This change moves building the Go binary out of the Dockerfile, and also
fixes some issues in the makefile. These things will speed up build
times and avoid rebuilding when not needed. Additionally it will make
getting version data easier to use as part of the user agent.

Signed-off-by: Philip Laine <philip.laine@gmail.com>

.dockerignore

@@ -1,3 +1,2 @@
-# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
-# Ignore build and test binaries.
-bin/
+**
+!bin/*/netbird-operator

.github/workflows/go.yaml

@@ -25,11 +25,7 @@ jobs:
         uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0
         with:
           go-version-file: go.mod
-      - name: Verify manifests are generated
-        run: |
-          make manifests
-          git diff --exit-code
-      - name: Verify code is generated
+      - name: Verify everything has been generated.
         run: |
           make generate
           git diff --exit-code

.github/workflows/helm.yaml

@@ -9,41 +9,3 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
       - name: Lint Helm Chart
         run: helm lint ./helm/kubernetes-operator
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Clone the code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
-      - name: Setup Go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0
-        with:
-          go-version-file: go.mod
-      - name: Install the latest version of kind
-        run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
-          chmod +x ./kind
-          sudo mv ./kind /usr/local/bin/kind
-      - name: Verify kind installation
-        run: kind version
-      - name: Create kind cluster
-        run: kind create cluster
-      - name: Prepare operator
-        run: |
-          make docker-build IMG=netbirdio/kubernetes-operator:debug
-          kind load docker-image netbirdio/kubernetes-operator:debug
-      - name: Install cert-manager via Helm
-        run: |
-          helm repo add jetstack https://charts.jetstack.io
-          helm repo update
-          helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --set installCRDs=true
-      - name: Wait for cert-manager to be ready
-        run: |
-          kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager
-          kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-cainjector
-          kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-webhook
-      - name: Install Helm chart for project
-        run: |
-          helm install test-chart --create-namespace --namespace netbird --set 'operator.image.tag=debug' ./helm/kubernetes-operator
-      - name: Check Helm release status
-        run: |
-          helm status test-chart --namespace netbird

.github/workflows/release.yaml

@@ -13,35 +13,29 @@ jobs:
       packages: write
       id-token: write
     steps:
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf #v6.0.0
+      - name: Clone the code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+      - name: Setup Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0
         with:
-          images: |
-            netbirdio/kubernetes-operator
-          tags: |
-            type=ref,event=pr
-            type=ref,event=branch
-            type=semver,pattern={{version}}
+          go-version-file: go.mod
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0
       - name: Login to Docker Hub
         uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #v4.1.0
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a #v4.0.0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0
-      - name: Build and push
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f #v7.1.0
-        with:
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: |
-            "org.opencontainers.image.created={{.Date}}"
-            "org.opencontainers.image.title={{.ProjectName}}"
-            "org.opencontainers.image.version={{.Version}}"
-            "org.opencontainers.image.revision={{.FullCommit}}"
-            "org.opencontainers.image.version={{.Version}}"
-            "maintainer=dev@netbird.io"
+      - name: Build image
+        id: build
+        run: |
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            IMG_TAG="${GITHUB_REF_NAME#v}"
+          else
+            IMG_TAG="${GITHUB_SHA::7}"
+          fi
+          IMG=$(make build-image-multiarch) || exit 1
+          echo "IMG=${IMG}" >> $GITHUB_OUTPUT
+      - name: Push image
+        run: |
+          docker push ${{ steps.build.outputs.IMG }}

.golangci.yml

@@ -6,7 +6,6 @@ linters:
     - dupl
     - errcheck
     - ginkgolinter
-    - goconst
     - gocyclo
     - govet
     - ineffassign

Dockerfile

@@ -1,36 +1,11 @@
-# This dockerfile is used for tests and local builds
-
-# Build the manager binary
-FROM docker.io/golang:1.26 AS builder
+FROM gcr.io/distroless/static:nonroot
 ARG TARGETOS
 ARG TARGETARCH
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY cmd/main.go cmd/main.go
-COPY api/ api/
-COPY internal/ internal/
-COPY pkg/ pkg/
-
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -ldflags="-w -s" -a -o manager cmd/main.go
-
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:nonroot
-WORKDIR /
-COPY --from=builder /workspace/manager .
+LABEL org.opencontainers.image.title="NetBird Operator" \
+      org.opencontainers.image.description="Kubernetes operator for NetBird." \
+      org.opencontainers.image.source="https://github.com/netbirdio/kubernetes-operator" \
+      org.opencontainers.image.vendor="NetBird" \
+      org.opencontainers.image.licenses="BSD-3-Clause"
+COPY bin/${TARGETOS}-${TARGETARCH}/netbird-operator .
 USER 65532:65532
-
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/netbird-operator"]

Dockerfile.release

@@ -1,8 +0,0 @@
-# This dockerfile is used for goreleaser
-
-FROM gcr.io/distroless/static:nonroot
-WORKDIR /
-COPY manager .
-USER 65532:65532
-
-ENTRYPOINT ["/manager"]

Makefile

@@ -1,40 +1,47 @@
-# Image URL to use all building/pushing image targets
-IMG ?= docker.io/netbirdio/kubernetes-operator:latest
-
-# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
-ifeq (,$(shell go env GOBIN))
-GOBIN=$(shell go env GOPATH)/bin
-else
-GOBIN=$(shell go env GOBIN)
-endif
-
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
-.PHONY: all
-all: build
+GOARCH = $(shell go env GOARCH)
+ifeq (,$(shell go env GOBIN))
+GOBIN = $(shell go env GOPATH)/bin
+else
+GOBIN = $(shell go env GOBIN)
+endif
 
-##@ Development
+IMG_REGISTRY ?= docker.io
+IMG_REPOSITORY ?= netbirdio/kubernetes-operator
+IMG_TAG ?= dev
+IMG := $(IMG_REGISTRY)/$(IMG_REPOSITORY):$(IMG_TAG)
 
-## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-.PHONY: manifests
-manifests:
-	go tool controller-gen crd paths="./..." output:crd:artifacts:config=helm/kubernetes-operator/crds
-	go tool crd-ref-docs --log-level error --output-path docs/api-reference.md --renderer markdown --source-path api/v1alpha1 --config docs/.crd-ref-docs.yaml
-
-## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 .PHONY: generate
-generate:
-	go tool controller-gen applyconfiguration:headerFile="hack/boilerplate.go.txt" object:headerFile="hack/boilerplate.go.txt" paths="./..."
+generate: api/v1/zz_generated.deepcopy.go api/v1alpha1/zz_generated.deepcopy.go pkg/applyconfigurations helm/kubernetes-operator/crds docs/api-reference.md
+
+api/v1/zz_generated.deepcopy.go api/v1alpha1/zz_generated.deepcopy.go: $(shell find api -not -name 'zz_generated*') hack/boilerplate.go.txt
+	@go tool controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+pkg/applyconfigurations: $(shell find api -not -name 'zz_generated*') hack/boilerplate.go.txt
+	@go tool controller-gen applyconfiguration:headerFile="hack/boilerplate.go.txt" object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	@touch pkg/applyconfigurations
+
+helm/kubernetes-operator/crds: $(shell find api)
+	@go tool controller-gen crd paths="./..." output:crd:artifacts:config=helm/kubernetes-operator/crds
+	@touch helm/kubernetes-operator/crds
+
+docs/api-reference.md: $(shell find api) docs/.crd-ref-docs.yaml
+	@go tool crd-ref-docs --log-level error --output-path docs/api-reference.md --renderer markdown --source-path api/v1alpha1 --config docs/.crd-ref-docs.yaml
+
+.PHONY: lint
+lint:
+	@golangci-lint run ./...
 
 .PHONY: test
-test: manifests setup-envtest
+test: setup-envtest
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 .PHONY: test-e2e
-test-e2e: manifests
+test-e2e: generate
 	@command -v kind >/dev/null 2>&1 || { \
 		echo "Kind is not installed. Please install Kind manually."; \
 		exit 1; \
@@ -45,27 +52,25 @@ test-e2e: manifests
 	}
 	go test ./test/e2e/ -v -ginkgo.v
 
-.PHONY: lint
-lint:
-	@golangci-lint run ./...
-
-##@ Build
-
 .PHONY: build
-build: manifests
-	go build -o bin/manager cmd/main.go
+build: generate bin/linux-$(GOARCH)/netbird-operator
 
-.PHONY: run
-run: manifests
-	go run ./cmd/main.go
+bin/linux-%/netbird-operator: $(shell find api cmd internal pkg) go.mod go.sum
+	@CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -ldflags="-w -s" -trimpath -o $@ cmd/main.go
 
-.PHONY: docker-build
-docker-build:
-	docker build -t ${IMG} .
+.PHONY: build-image
+build-image: build
+	@docker buildx build -t ${IMG} .
+	@echo ${IMG}
+
+.PHONY: build-image-multiarch
+build-image-multiarch: generate bin/linux-amd64/netbird-operator bin/linux-arm64/netbird-operator
+	@docker buildx build --platform linux/amd64,linux/arm64 -t ${IMG} .
+	@echo ${IMG}
 
 ## Generate a consolidated YAML with CRDs and deployment.
 .PHONY: build-installer
-build-installer: manifests
+build-installer: generate
 	mkdir -p manifests
 	helm template --include-crds kubernetes-operator helm/kubernetes-operator > manifests/install.yaml
 
@@ -77,17 +82,17 @@ endif
 
 ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 .PHONY: install
-install: manifests
+install: generate
 	kubectl apply --server-side -f helm/kubernetes-operator/crds
 
 ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 .PHONY: uninstall
-uninstall: manifests
+uninstall: generate
 	kubectl delete -f helm/kubernetes-operator/crds
 
 ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 .PHONY: deploy
-deploy: manifests
+deploy: genereate
 	helm install -n netbird --create-namespace kubernetes-operator --set operator.image.tag=$(word 2,$(subst :, ,${IMG})) helm/kubernetes-operator
 
 ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

examples/gateway-api/README.md

@@ -4,7 +4,7 @@ This example walks you through how to setup a Netbird Gateway API and expose Ngi
 
 Build image locally and load it into Kind.
 ```shell
-make docker-build IMG=docker.io/netbirdio/kubernetes-operator:dev
+make build-image
 kind load docker-image docker.io/netbirdio/kubernetes-operator:dev
 ```