BUILD/MINOR: docker: optimize image layers and build caching

Refactor Dockerfile and Dockerfile.dev to improve build cache utilization and reduce the final image size.

- Separate the base image setup (package installations, cleanup, and directory initialization) into a distinct step so it remains cached across source code changes.
- Consolidate the installation of build artifacts and permission changes into a single layer using `--mount` instead of multiple `COPY` instructions.
- Rename the golang builder stage to `builder-go` for better clarity.
- Output the controller binary directly to `fs/usr/local/sbin/hug` during the build stage to streamline the final assembly.

Author: oktalz

build/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.26-alpine AS builder
+FROM golang:1.26-alpine AS builder-go
 
 RUN apk --no-cache add git openssh
 
@@ -24,11 +24,15 @@ WORKDIR /src
 RUN go install -trimpath -ldflags="-s" github.com/haproxytech/gopherd@v1.0.2
 COPY / /src
 
-RUN mkdir -p /var/run/vars && \
-    cd /src && \
-    CGO_ENABLED=0 go build -buildvcs=true \
+RUN mkdir -p /var/run/vars
+
+# stage gopherd into the fs overlay tree
+RUN mkdir -p fs/usr/local/sbin && \
+    cp -a /go/bin/gopherd fs/usr/local/sbin/gopherd
+
+RUN CGO_ENABLED=0 go build -buildvcs=true \
         -tags="$BUILD_TAGS" \
-        -o fs/kubernetes-controller ./cmd/controller
+        -o fs/usr/local/sbin/hug ./cmd/controller
 
 FROM haproxytech/haproxy-alpine:3.2 AS builder-c
 RUN apk add --no-cache build-base gcc musl-dev
@@ -59,28 +63,29 @@ FROM haproxytech/haproxy-alpine:3.2
 
 ARG TARGETPLATFORM
 
-COPY /fs /
-COPY --from=builder /go/bin/gopherd /usr/local/sbin
-COPY --from=builder-c /src/libblock_secrets.so /usr/local/lib/libblock_secrets.so
-COPY --from=builder-c /src/haproxy_wrapper /usr/local/sbin/haproxy_wrapper
-
+# base image setup only: stays cached across source changes
 RUN apk --no-cache add socat openssl util-linux htop tzdata curl libcap && \
     rm -f /usr/local/bin/dataplaneapi /usr/bin/dataplaneapi /etc/haproxy/dataplaneapi.yml && \
     rm -f /usr/local/bin/dataplaneapi-v2 /usr/bin/dataplaneapi-v2 && \
     rm -f /etc/haproxy/haproxy.cfg && \
-    rm usr/local/hug/defaults.go && \
-    mkdir -p /usr/local/hug/aux && \
     mkdir -p /var/run/haproxy && \
+    chown -R "haproxy:haproxy" /usr/local/etc/haproxy /run /var && \
+    chmod -R ug+rwx /usr/local/etc/haproxy /run /var
+
+# all build artifacts installed and fixed up in a single layer
+RUN --mount=from=builder-go,source=/src/fs,target=/mnt/builder-go \
+    --mount=from=builder-c,source=/src,target=/mnt/builder-c \
+    cp -a /mnt/builder-go/. / && \
+    rm /usr/local/hug/defaults.go && \
+    cp -a /mnt/builder-c/libblock_secrets.so /usr/local/lib/libblock_secrets.so && \
+    cp -a /mnt/builder-c/haproxy_wrapper /usr/local/sbin/haproxy_wrapper && \
+    mkdir -p /usr/local/hug/aux && \
     chgrp -R haproxy /usr/local/hug && \
     chmod -R ug+rwx /usr/local/hug && \
-    chown -R "haproxy:haproxy" /usr/local/etc/haproxy /run /var && \
-    chmod -R ug+rwx /usr/local/etc/haproxy /run /var && \
     chmod 644 /etc/gopherd/gopherd.yml && \
     chmod u+rx /usr/local/sbin/haproxy_wrapper && \
     setcap 'cap_net_bind_service=+ep' /usr/local/sbin/haproxy_wrapper
 
-COPY --from=builder /src/fs/kubernetes-controller /usr/local/sbin/hug
-
 STOPSIGNAL SIGTERM
 
 ENTRYPOINT ["/usr/local/sbin/gopherd"]

build/Dockerfile.dev

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM golang:1.26-alpine AS builder
+FROM golang:1.26-alpine AS builder-go
 
 RUN apk --no-cache add git openssh
 WORKDIR /src
@@ -46,30 +46,32 @@ FROM haproxytech/haproxy-alpine:3.2
 
 ARG TARGETPLATFORM
 
-COPY /fs /
-COPY --from=builder /go/bin/gopherd /usr/local/sbin
-COPY --from=builder-c /src/libblock_secrets.so /usr/local/lib/libblock_secrets.so
-COPY --from=builder-c /src/haproxy_wrapper /usr/local/sbin/haproxy_wrapper
-
-RUN mkdir -p /usr/local/hug/aux && \
-    chgrp -R haproxy /usr/local/hug && \
-    chmod -R ug+rwx /usr/local/hug
-
+# base image setup only: stays cached across source changes
 RUN apk --no-cache add socat openssl util-linux htop tzdata curl libcap && \
     rm -f /usr/local/bin/dataplaneapi /usr/bin/dataplaneapi /etc/haproxy/dataplaneapi.yml && \
     rm -f /usr/local/bin/dataplaneapi-v2 /usr/bin/dataplaneapi-v2 && \
     rm -f /etc/haproxy/haproxy.cfg && \
-    rm usr/local/hug/defaults.go && \
-    mkdir -p /usr/local/hug/aux && \
     mkdir -p /var/run/haproxy && \
+    chown -R "haproxy:haproxy" /usr/local/etc/haproxy /run /var && \
+    chmod -R ug+rwx /usr/local/etc/haproxy /run /var
+
+# all build artifacts installed and fixed up in a single layer
+RUN --mount=type=bind,source=fs,target=/mnt/fs \
+    --mount=from=builder-go,source=/go/bin,target=/mnt/builder-go \
+    --mount=from=builder-c,source=/src,target=/mnt/builder-c \
+    cp -a /mnt/fs/. / && \
+    rm /usr/local/hug/defaults.go && \
+    cp -a /mnt/builder-go/gopherd /usr/local/sbin/gopherd && \
+    cp -a /mnt/builder-c/libblock_secrets.so /usr/local/lib/libblock_secrets.so && \
+    cp -a /mnt/builder-c/haproxy_wrapper /usr/local/sbin/haproxy_wrapper && \
+    mkdir -p /usr/local/hug/aux && \
     chgrp -R haproxy /usr/local/hug && \
     chmod -R ug+rwx /usr/local/hug && \
-    chown -R "haproxy:haproxy" /usr/local/etc/haproxy /run /var && \
-    chmod -R ug+rwx /usr/local/etc/haproxy /run /var && \
     chmod 644 /etc/gopherd/gopherd.yml && \
     chmod u+rx /usr/local/sbin/haproxy_wrapper && \
     setcap 'cap_net_bind_service=+ep' /usr/local/sbin/haproxy_wrapper
 
+# host-built binary last: only this layer rebuilds during dev iteration
 COPY build/kubernetes-controller /usr/local/sbin/hug
 
 STOPSIGNAL SIGTERM