From 8fee49bf0908dd215408296bdfc664b227bd716e Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Tue, 16 Jun 2026 01:50:50 +0200 Subject: [PATCH 1/3] cross compile busybox instead of emulation --- Dockerfile | 117 +++++++++++++++++++++++++++++++++++++++++++++++------ build.sh | 3 +- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4fe847a..6efe105 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,9 +12,22 @@ # it would need a glibc (Debian) base and produces a much larger image. See # README for details if you need it. # -# The Go builder cross-compiles, so it always runs NATIVELY on the build host -# ($BUILDPLATFORM) for speed; only the busybox stage and the final image run on -# the target platform. +# Both the Go (Tailscale) stage and the C (busybox) stage cross-compile: they +# always run NATIVELY on the build host ($BUILDPLATFORM) and produce binaries +# for $TARGETPLATFORM. This eliminates QEMU emulation entirely from the build, +# which is the main source of slowness in multi-arch builds. Only the final +# scratch stage pulls in the target-arch-specific layers (CA certs, busybox +# rootfs) which are just file copies with no emulated execution. +# +# Cross-compilation for C (busybox) is provided by tonistiigi/xx, which +# configures clang+lld as a cross-compiler and installs musl headers for the +# target arch via xx-apk. + +# ============================================================================= +# xx: Dockerfile cross-compilation helpers (provides xx-clang, xx-apk, etc.) +# ============================================================================= +# renovate: datasource=docker depName=tonistiigi/xx versioning=docker +FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707 AS xx # ============================================================================= # Stage 1: Build Tailscale combined binary (cross-compiled, runs natively) @@ -236,7 +249,7 @@ RUN printf '%s\n' \ chmod +x /out/usrlocalbin/entrypoint.sh # ============================================================================= -# Stage 2: Custom minimal busybox +# Stage 2: Custom minimal busybox (cross-compiled, runs natively on build host) # ============================================================================= # The official busybox:musl image ships all ~404 applets at ~1.24 MB. For a # debug shell on a flash-constrained router we only need ~100 applets, so we @@ -253,15 +266,56 @@ RUN printf '%s\n' \ # acceptable for an occasional debug shell. RouterOS stores the EXTRACTED # rootfs on disk (overlayfs), so the ~190 kB UPX saving is real on-disk space. # -# This stage runs on the TARGET platform (no --platform override): gcc then -# produces native target-arch binaries directly. Under buildx this is -# transparently emulated via binfmt/QEMU for non-native targets. -FROM alpine:3.24.0@sha256:a2d49ea686c2adfe3c992e47dc3b5e7fa6e6b5055609400dc2acaeb241c829f4 AS busybox +# This stage runs NATIVELY on the build host (--platform=$BUILDPLATFORM) and +# cross-compiles busybox for the target architecture using clang+lld via the +# tonistiigi/xx helpers. This eliminates QEMU emulation from the busybox build, +# which was the main source of slowness for arm64/arm/v7 targets. +# +# Cross-compilation setup: +# - xx-apk installs musl-dev and linux-headers for the TARGET arch under +# / (a secondary sysroot), while clang/lld/upx/make stay native. +# - xx-clang --setup-target-triple creates -clang / -cc +# aliases in PATH that busybox's Makefile picks up via CROSS_COMPILE. +# - Busybox make receives: +# CROSS_COMPILE=- → picks up -clang (from xx aliases) +# CC=clang → use clang (aliased target via CROSS_COMPILE) +# HOSTCC=gcc → compile host helper tools with native gcc +# - upx (native x86_64 binary) can compress target-arch binaries since UPX +# operates on the ELF file format regardless of the target ISA. +# +# Applet symlink probing: for native-arch builds the probe runs directly; +# for cross-compiled binaries we use QEMU user-mode emulation (from binfmt) +# only for this one lightweight probe step (busybox --help per applet), not +# for the compile itself. The probe can alternatively be skipped by using +# a pre-enumerated applet list, but the current approach is simpler. +FROM --platform=$BUILDPLATFORM alpine:3.24.0@sha256:a2d49ea686c2adfe3c992e47dc3b5e7fa6e6b5055609400dc2acaeb241c829f4 AS busybox + +# Copy xx cross-compilation helpers (xx-clang, xx-apk, xx-info, etc.) +COPY --from=xx / / # renovate: datasource=docker depName=busybox versioning=docker ARG BUSYBOX_VERSION=1.38.0 -RUN apk add --no-cache build-base linux-headers wget bzip2 perl upx +# Target platform ARGs (provided automatically by buildx). +ARG TARGETPLATFORM +ARG TARGETARCH +ARG TARGETVARIANT + +# Native build tools (clang/lld for cross-compiling; gcc/make/upx run natively). +# xx-apk installs the target-arch sysroot: musl-dev (C library headers + CRT), +# gcc (provides crtbeginS.o/crtendS.o and libgcc needed by clang on Alpine), +# and linux-headers (required by busybox for / ). +RUN apk add --no-cache \ + clang \ + lld \ + llvm \ + gcc \ + make \ + wget \ + bzip2 \ + perl \ + upx && \ + xx-apk add --no-cache musl-dev gcc linux-headers RUN wget -q https://busybox.net/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2 \ && tar xf busybox-${BUSYBOX_VERSION}.tar.bz2 @@ -269,7 +323,34 @@ WORKDIR /busybox-${BUSYBOX_VERSION} # allnoconfig = every feature OFF; then enable only the curated applet set. COPY busybox-applets.config /tmp/applets.config -RUN make allnoconfig && \ +# Set up xx cross-compiler aliases (-clang, -cc, etc.) and +# build busybox. +# +# Key make variables: +# ARCH — busybox ARCH; must match the cross-target, not the build +# host. busybox's Makefile would otherwise read SUBARCH from +# `uname -m` (the BUILD host's arch) which is wrong when +# cross-compiling. We map TARGETARCH to busybox's arch name. +# busybox uses -include arch/$(ARCH)/Makefile; missing arch +# dirs are silently ignored, so any value is safe. +# CC — busybox defaults to $(CROSS_COMPILE)gcc. We override CC to +# the full -clang path so it resolves to the xx alias +# (which sets --target and --sysroot for the cross-compiler). +# Setting CC= avoids needing a -gcc symlink. +# HOSTCC — native compiler for host-side build tools (scripts/kconfig, +# gen_build_files, etc.); must NOT be the cross-compiler. +# SKIP_STRIP — defer stripping to after symlink probing (we strip below +# with llvm-strip, which handles any target ELF arch). +RUN xx-clang --setup-target-triple && \ + CROSS=$(xx-info triple) && \ + # Map TARGETARCH to the busybox ARCH value. + case "${TARGETARCH}" in \ + amd64) BUSYBOX_ARCH=x86_64 ;; \ + arm64) BUSYBOX_ARCH=aarch64 ;; \ + arm) BUSYBOX_ARCH=arm ;; \ + *) BUSYBOX_ARCH=${TARGETARCH} ;; \ + esac && \ + make allnoconfig ARCH="${BUSYBOX_ARCH}" && \ while read -r sym; do \ case "$sym" in ''|\#*) continue ;; esac; \ if grep -q "^# CONFIG_${sym} is not set" .config; then \ @@ -278,9 +359,15 @@ RUN make allnoconfig && \ echo "CONFIG_${sym}=y" >> .config; \ fi; \ done < /tmp/applets.config && \ - yes "" | make oldconfig >/dev/null 2>&1 && \ - make -j"$(nproc)" >/dev/null 2>&1 && \ - strip busybox + yes "" | make oldconfig ARCH="${BUSYBOX_ARCH}" >/dev/null 2>&1 && \ + make -j"$(nproc)" \ + ARCH="${BUSYBOX_ARCH}" \ + CROSS_COMPILE="${CROSS}-" \ + CC="${CROSS}-clang" \ + HOSTCC=gcc \ + SKIP_STRIP=y \ + >/dev/null 2>&1 && \ + llvm-strip busybox # Lay out a minimal rootfs with busybox + an applet symlink per applet. # Symlinks (argv[0] dispatch) are how busybox selects an applet and make the @@ -290,6 +377,10 @@ RUN make allnoconfig && \ # for non-applet symbols like FEATURE_* / STATIC, which we filter out). # We generate symlinks from the UNCOMPRESSED binary (so the probe is reliable), # then UPX-compress the binary in place afterwards. +# +# Note: probing cross-compiled binaries requires binfmt/QEMU user-mode. This +# is only a lightweight per-applet help-flag check, not a full emulated build. +# If QEMU is unavailable in CI, replace the probe with a static applet list. RUN mkdir -p /rootfs/bin && \ grep '^CONFIG_.*=y' .config \ | sed -e 's/^CONFIG_//' -e 's/=y$//' \ diff --git a/build.sh b/build.sh index aa8363a..b225d4f 100755 --- a/build.sh +++ b/build.sh @@ -12,7 +12,8 @@ # # Requirements: # - docker with buildx -# - For non-native targets: binfmt/QEMU emulators registered, e.g.: +# - For non-native targets: binfmt/QEMU emulators registered for the applet +# symlink probe step (a minor step; the full C/Go compile is native): # docker run --privileged --rm tonistiigi/binfmt --install arm64,arm set -eu From 524b83d911311e04e57e21c30256465411b7ccdc Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Tue, 16 Jun 2026 01:57:20 +0200 Subject: [PATCH 2/3] Docker build caching --- .woodpecker/pr-build.yaml | 34 +++++++++++++++++++++++++++++++++- .woodpecker/release.yaml | 2 ++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.woodpecker/pr-build.yaml b/.woodpecker/pr-build.yaml index 6bf9606..37488d1 100644 --- a/.woodpecker/pr-build.yaml +++ b/.woodpecker/pr-build.yaml @@ -8,6 +8,10 @@ # # Reports pass/fail status back to Gitea, so it shows up as a required check on # the PR. +# +# Registry credentials are fetched from OpenBao (same AppRole as release.yaml) +# solely to read and write the build cache image. The build itself is still +# dry-run (nothing is published as a release image). # Changes that can't affect the image don't trigger the build: docs and the # RouterOS-side script (routeros/**: lives on the router, not in the image). @@ -29,12 +33,40 @@ when: exclude: *non_image_paths steps: + - name: Get registry creds from OpenBao + image: quay.io/openbao/openbao:2.5.4 + environment: + VAULT_ADDR: https://openbao.lumpiasty.xyz:8200 + ROLE_ID: + from_secret: renovate_role_id + SECRET_ID: + from_secret: renovate_secret_id + commands: + - bao write -field token auth/approle/login + role_id=$ROLE_ID + secret_id=$SECRET_ID > /woodpecker/.vault_id + - export VAULT_TOKEN=$(cat /woodpecker/.vault_id) + - 'printf "PLUGIN_USERNAME=%s\n" "$(bao kv get -mount secret -field REGISTRY_USERNAME container-registry)" > /woodpecker/registry.env' + - 'printf "PLUGIN_PASSWORD=%s\n" "$(bao kv get -mount secret -field REGISTRY_PASSWORD container-registry)" >> /woodpecker/registry.env' + - name: Build all arches (no push) image: woodpeckerci/plugin-docker-buildx:6.1.0 privileged: true settings: + registry: gitea.lumpiasty.xyz repo: mikrotik-tailscale platforms: linux/amd64,linux/arm64,linux/arm/v7 - dry-run: true + dry_run: true build_args: - OCI_VERSION=ci-${CI_COMMIT_SHA} + cache_images: + - gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale:buildcache + env_file: /woodpecker/registry.env + + - name: Invalidate OpenBao token + image: quay.io/openbao/openbao:2.5.4 + environment: + VAULT_ADDR: https://openbao.lumpiasty.xyz:8200 + commands: + - export VAULT_TOKEN=$(cat /woodpecker/.vault_id) + - bao write -f auth/token/revoke-self diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index 71b6e07..aba119e 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -54,6 +54,8 @@ steps: - stable build_args: - OCI_VERSION=${CI_COMMIT_TAG} + cache_images: + - gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale:buildcache # Credentials (PLUGIN_USERNAME / PLUGIN_PASSWORD) come from OpenBao. env_file: /woodpecker/registry.env - name: Invalidate OpenBao token From ff60452758b094352bc4ebd8e099dcd48d865cb8 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Tue, 16 Jun 2026 01:59:47 +0200 Subject: [PATCH 3/3] Empty commit to trigger CI