Merge pull request 'Speed up build pipeline' (#29) from feat/busybox-crosscompile into main

Reviewed-on: #29

.woodpecker/pr-build.yaml

@@ -8,19 +8,65 @@
 #
 # Reports pass/fail status back to Gitea, so it shows up as a required check on
 # the PR.
+#
+# Registry credentials are fetched from OpenBao (same AppRole as release.yaml)
+# solely to read and write the build cache image. The build itself is still
+# dry-run (nothing is published as a release image).
 
+# Changes that can't affect the image don't trigger the build: docs and the
+# RouterOS-side script (routeros/**: lives on the router, not in the image).
+# NOTE: if Gitea is ever configured to REQUIRE this check for merging, a
+# PR touching only excluded files will have no check at all — exempt such PRs
+# or merge manually. Renovate PRs always touch the Dockerfile or pipeline
+# files, so the automerge gate is unaffected by these exclusions.
 when:
   - event: pull_request
+    path:
+      exclude: &non_image_paths
+        - '**/*.md'
+        - 'docs/**'
+        - 'routeros/**'
+        - 'renovate.json'
   - event: push
     branch: main
+    path:
+      exclude: *non_image_paths
 
 steps:
+  - name: Get registry creds from OpenBao
+    image: quay.io/openbao/openbao:2.5.4
+    environment:
+      VAULT_ADDR: https://openbao.lumpiasty.xyz:8200
+      ROLE_ID:
+        from_secret: renovate_role_id
+      SECRET_ID:
+        from_secret: renovate_secret_id
+    commands:
+      - bao write -field token auth/approle/login
+          role_id=$ROLE_ID
+          secret_id=$SECRET_ID > /woodpecker/.vault_id
+      - export VAULT_TOKEN=$(cat /woodpecker/.vault_id)
+      - 'printf "PLUGIN_USERNAME=%s\n" "$(bao kv get -mount secret -field REGISTRY_USERNAME container-registry)" > /woodpecker/registry.env'
+      - 'printf "PLUGIN_PASSWORD=%s\n" "$(bao kv get -mount secret -field REGISTRY_PASSWORD container-registry)" >> /woodpecker/registry.env'
+
   - name: Build all arches (no push)
-    image: woodpeckerci/plugin-docker-buildx:5.2.2
+    image: woodpeckerci/plugin-docker-buildx:6.1.0
     privileged: true
     settings:
+      registry: gitea.lumpiasty.xyz
       repo: mikrotik-tailscale
       platforms: linux/amd64,linux/arm64,linux/arm/v7
-      dry-run: true
+      dry_run: true
       build_args:
         - OCI_VERSION=ci-${CI_COMMIT_SHA}
+      cache_images:
+        - gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale:buildcache
+      env_file: /woodpecker/registry.env
+
+  - name: Invalidate OpenBao token
+    image: quay.io/openbao/openbao:2.5.4
+    environment:
+      VAULT_ADDR: https://openbao.lumpiasty.xyz:8200
+    commands:
+      - export VAULT_TOKEN=$(cat /woodpecker/.vault_id)
+      - bao write -f auth/token/revoke-self

.woodpecker/release-tag.yaml

@@ -13,9 +13,19 @@
 # unchanged, so no tag is created and nothing is released — they ride along
 # with the next Tailscale bump or manual tag.
 
+# Skipped for pushes that can't introduce a new Tailscale version:
+# TAILSCALE_VERSION lives in the Dockerfile, so a push touching only docs or
+# the RouterOS-side script can never produce a new version to tag (the job
+# would just no-op after spinning up OpenBao + git containers).
 when:
   - event: push
     branch: main
+    path:
+      exclude:
+        - '**/*.md'
+        - 'docs/**'
+        - 'routeros/**'
+        - 'renovate.json'
 
 steps:
   - name: Get git token from OpenBao
@@ -34,7 +44,7 @@ steps:
       - bao kv get -mount secret -field RENOVATE_TOKEN renovate > /woodpecker/git_token
 
   - name: Auto-tag mt.1 on Tailscale bump
-    image: alpine/git:2.49.1
+    image: alpine/git:v2.52.0
     environment:
       CI_REPO_URL: https://gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale.git
     commands:

.woodpecker/release.yaml

@@ -43,7 +43,7 @@ steps:
       - 'printf "PLUGIN_PASSWORD=%s\n" "$(bao kv get -mount secret -field REGISTRY_PASSWORD container-registry)" >> /woodpecker/registry.env'
 
   - name: Build and push multi-arch image
-    image: woodpeckerci/plugin-docker-buildx:5.2.2
+    image: woodpeckerci/plugin-docker-buildx:6.1.0
     privileged: true
     settings:
       registry: gitea.lumpiasty.xyz
@@ -54,6 +54,8 @@ steps:
         - stable
       build_args:
         - OCI_VERSION=${CI_COMMIT_TAG}
+      cache_images:
+        - gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale:buildcache
       # Credentials (PLUGIN_USERNAME / PLUGIN_PASSWORD) come from OpenBao.
       env_file: /woodpecker/registry.env
   - name: Invalidate OpenBao token

.woodpecker/renovate.yaml

@@ -46,7 +46,7 @@ steps:
       - bao kv get -mount secret -field GITHUB_COM_TOKEN renovate > /woodpecker/github_com_token
   - name: renovate
     # Renovate's built-in "woodpecker" manager tracks this image automatically.
-    image: renovate/renovate:43.194.0
+    image: renovate/renovate:43.220.0
     environment:
       # --- platform / target ---
       RENOVATE_PLATFORM: gitea
@@ -58,8 +58,12 @@ steps:
       # Use the committed renovate.json; don't open an onboarding PR.
       RENOVATE_ONBOARDING: "false"
       RENOVATE_REQUIRE_CONFIG: "optional"
-      # Git identity for the branches/commits Renovate creates.
+      # Git identity for the branches/commits Renovate creates. MUST match the
-      RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate@localhost>"
+      # bot's Gitea account email: platform actions (automerge merge commits,
+      # "update branch") are attributed to the account email, and Renovate
+      # flags branches containing commits from unrecognized emails as
+      # "edited by someone else" and stops rebasing them.
+      RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate@lumpiasty.xyz>"
       # GitHub token (read-only, no repo access) lets Renovate fetch release
       # notes / changelogs and avoids GitHub API rate limits for the
       # github-releases datasource (tailscale). Optional but recommended.

Dockerfile

@@ -12,17 +12,30 @@
 # it would need a glibc (Debian) base and produces a much larger image. See
 # README for details if you need it.
 #
-# The Go builder cross-compiles, so it always runs NATIVELY on the build host
+# Both the Go (Tailscale) stage and the C (busybox) stage cross-compile: they
-# ($BUILDPLATFORM) for speed; only the busybox stage and the final image run on
+# always run NATIVELY on the build host ($BUILDPLATFORM) and produce binaries
-# the target platform.
+# for $TARGETPLATFORM. This eliminates QEMU emulation entirely from the build,
+# which is the main source of slowness in multi-arch builds. Only the final
+# scratch stage pulls in the target-arch-specific layers (CA certs, busybox
+# rootfs) which are just file copies with no emulated execution.
+#
+# Cross-compilation for C (busybox) is provided by tonistiigi/xx, which
+# configures clang+lld as a cross-compiler and installs musl headers for the
+# target arch via xx-apk.
+
+# =============================================================================
+# xx: Dockerfile cross-compilation helpers (provides xx-clang, xx-apk, etc.)
+# =============================================================================
+# renovate: datasource=docker depName=tonistiigi/xx versioning=docker
+FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.9.0@sha256:c64defb9ed5a91eacb37f96ccc3d4cd72521c4bd18d5442905b95e2226b0e707 AS xx
 
 # =============================================================================
 # Stage 1: Build Tailscale combined binary (cross-compiled, runs natively)
 # =============================================================================
-FROM --platform=$BUILDPLATFORM golang:1.26.3-alpine@sha256:91eda9776261207ea25fd06b5b7fed8d397dd2c0a283e77f2ab6e91bfa71079d AS builder
+FROM --platform=$BUILDPLATFORM golang:1.26.4-alpine@sha256:7a3e50096189ad57c9f9f865e7e4aa8585ed1585248513dc5cda498e2f41812c AS builder
 
-# renovate: datasource=github-releases depName=tailscale packageName=tailscale/tailscale
+# renovate: datasource=github-releases depName=tailscale packageName=tailscale/tailscale versioning=semver
-ARG TAILSCALE_VERSION=v1.98.3
+ARG TAILSCALE_VERSION=v1.98.5
 
 # Provided automatically by buildx for the target platform.
 ARG TARGETOS
@@ -40,6 +53,66 @@ RUN git clone --depth 1 --branch ${TAILSCALE_VERSION} \
 
 WORKDIR /src/tailscale
 
+# Inject a stderr verbosity filter into the tailscaled package.
+#
+# With logtail compiled out (ts_omit_logtail), tailscaled never installs
+# logpolicy (see `if buildfeatures.HasLogTail` in cmd/tailscaled/tailscaled.go),
+# so log output goes raw to stderr: the [v1]/[v2] verbosity tags embedded in
+# messages are neither parsed nor filtered, and --verbose has NO effect. The
+# result is constant log spam in the RouterOS container log (filter
+# "Accept: TCP" verdicts, "netcheck: [v1] report", "wg: [v2]" handshakes and
+# keepalives) — see tailscale/tailscale#12158 and #1548.
+#
+# The injected file (build-tagged ts_omit_logtail, so it's a no-op if logtail
+# is ever re-enabled) registers a log writer in init() that drops lines
+# carrying a [v1]+ tag, restoring the equivalent of logtail's StderrLevel=0
+# default. Setting TS_LOG_VERBOSITY=1 (or higher) in the container environment
+# disables the filter at runtime for debugging — no rebuild needed.
+COPY patches/stderr_verbosity_filter.go cmd/tailscaled/
+
+# Patch net/tstun/wrap.go: fix panic("unreachable") in invertGSOChecksum for
+# ts_omit_netstack builds.
+#
+# invertGSOChecksum is a gVisor/GSO helper that inverts a transport-layer
+# checksum before/after SNAT when gVisor hands us a segment with a partial
+# checksum (NeedsCsum=true). It is only meaningful when netstack (gVisor) is
+# compiled in (HasNetstack=true).
+#
+# The function correctly guards its body with:
+#   if !buildfeatures.HasNetstack { panic("unreachable") }
+#
+# When built with ts_omit_netstack, HasNetstack is a const false, so that guard
+# evaluates to `if true { panic(...) }` — the function always panics.
+#
+# The problem: invertGSOChecksum is called unconditionally from injectedRead()
+# (twice, around pc.snat()), even for the res.data path where res.packet==nil
+# and gso is a zero-value netstack_GSO (NeedsCsum=false). The HasNetstack
+# guard in the res.packet branch does NOT protect these calls.
+#
+# As a result, any code path that injects an outbound packet via InjectOutbound()
+# — which happens when enabling exit-node use (Tailscale sends TSMP messages
+# and synthesizes packets through the TUN injection path) — hits injectedRead
+# with res.data!=nil, calls invertGSOChecksum, and crashes with:
+#   panic: unreachable
+#   tailscale.com/net/tstun.invertGSOChecksum(...)
+#   tailscale.com/net/tstun.(*Wrapper).injectedRead(...)  wrap.go:1077
+#
+# Fix: replace the `panic("unreachable")` with a `return` in invertGSOChecksum.
+# When HasNetstack=false (ts_omit_netstack), a zero-value netstack_GSO always
+# has NeedsCsum=false, so the function is correctly a no-op anyway. This matches
+# what the function would do if the rest of its body ran: NeedsCsum=false → return.
+#
+# The sed expression targets the function precisely: it matches the three-line
+# sequence that opens invertGSOChecksum's HasNetstack guard, and replaces only
+# the panic line with return. The pattern is stable across minor reformats
+# because it anchors on the literal function comment and the specific panic string.
+#
+# See tailscale/tailscale issue for context (no upstream fix as of v1.98.5):
+# panic happens when using exit-node via a ts_omit_netstack build.
+RUN sed -i \
+    -e '/func invertGSOChecksum/,/^}/ s/\t\tpanic("unreachable")/\t\treturn/' \
+    net/tstun/wrap.go
+
 # Build a minimal combined binary (tailscale CLI + tailscaled daemon in one file).
 #
 # Tag strategy — ALLOWLIST, not blocklist:
@@ -69,6 +142,12 @@ WORKDIR /src/tailscale
 #                         trusted unix socket, so PermitRead/PermitWrite are
 #                         always false and EVERY CLI call (status, up, set, ...)
 #                         returns "access denied" (tailscale/tailscale#17873).
+#   ipnbus              — IPN bus watch. Without it, 'tailscale up' cannot wait
+#                         for completion: it fires config at the daemon and
+#                         returns immediately ("built with ts_omit_ipnbus; not
+#                         waiting for completion") WITHOUT printing the auth URL
+#                         or confirming success. Including it makes interactive
+#                         'up' behave normally (blocks, prints login URL).
 #
 # Everything else remains omitted, including (rationale):
 #   clientupdate  — DELIBERATELY removed. The built-in updater would download
@@ -111,6 +190,7 @@ RUN mkdir -p /out && \
           -e 's/ts_omit_health,\{0,1\}//g' \
           -e 's/ts_omit_iptables,\{0,1\}//g' \
           -e 's/ts_omit_unixsocketidentity,\{0,1\}//g' \
+          -e 's/ts_omit_ipnbus,\{0,1\}//g' \
           -e 's/,$//' \
     ) && \
     echo "Build tags: ${TAGS}" && \
@@ -150,8 +230,26 @@ RUN mkdir -p /out/usrlocalbin && \
     ln -s /usr/local/bin/tailscale.combined /out/usrlocalbin/tailscale && \
     ln -s /usr/local/bin/tailscale.combined /out/usrlocalbin/tailscaled
 
+# Entrypoint wrapper: enable IP forwarding inside the container's network
+# namespace, then exec tailscaled. tailscaled does NOT reliably enable IPv6
+# forwarding itself in a container netns ("IPv6 forwarding is disabled" warning),
+# which silently breaks advertised IPv6 subnet routes. The sysctls ARE writable
+# from inside a RouterOS container, so we set both here. Written in the builder
+# stage so it ships in the same single /usr/local/bin COPY layer (preserves the
+# overlayfs single-copy property). `exec` keeps tailscaled as PID 1.
+RUN printf '%s\n' \
+      '#!/bin/sh' \
+      '# Enable IPv4/IPv6 forwarding. Required for advertised subnet routes and' \
+      '# exit-node functionality.' \
+      'for f in /proc/sys/net/ipv4/ip_forward /proc/sys/net/ipv6/conf/all/forwarding; do' \
+      '  if [ -w "$f" ]; then echo 1 > "$f" 2>/dev/null || echo "warn: could not write $f"; fi' \
+      'done' \
+      'exec /usr/local/bin/tailscaled "$@"' \
+      > /out/usrlocalbin/entrypoint.sh && \
+    chmod +x /out/usrlocalbin/entrypoint.sh
+
 # =============================================================================
-# Stage 2: Custom minimal busybox
+# Stage 2: Custom minimal busybox (cross-compiled, runs natively on build host)
 # =============================================================================
 # The official busybox:musl image ships all ~404 applets at ~1.24 MB. For a
 # debug shell on a flash-constrained router we only need ~100 applets, so we
@@ -168,15 +266,56 @@ RUN mkdir -p /out/usrlocalbin && \
 # acceptable for an occasional debug shell. RouterOS stores the EXTRACTED
 # rootfs on disk (overlayfs), so the ~190 kB UPX saving is real on-disk space.
 #
-# This stage runs on the TARGET platform (no --platform override): gcc then
+# This stage runs NATIVELY on the build host (--platform=$BUILDPLATFORM) and
-# produces native target-arch binaries directly. Under buildx this is
+# cross-compiles busybox for the target architecture using clang+lld via the
-# transparently emulated via binfmt/QEMU for non-native targets.
+# tonistiigi/xx helpers. This eliminates QEMU emulation from the busybox build,
-FROM alpine:3.21.7@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d AS busybox
+# which was the main source of slowness for arm64/arm/v7 targets.
+#
+# Cross-compilation setup:
+#   - xx-apk installs musl-dev and linux-headers for the TARGET arch under
+#     /<triple> (a secondary sysroot), while clang/lld/upx/make stay native.
+#   - xx-clang --setup-target-triple creates <triple>-clang / <triple>-cc
+#     aliases in PATH that busybox's Makefile picks up via CROSS_COMPILE.
+#   - Busybox make receives:
+#       CROSS_COMPILE=<triple>-   → picks up <triple>-clang (from xx aliases)
+#       CC=clang                  → use clang (aliased target via CROSS_COMPILE)
+#       HOSTCC=gcc                → compile host helper tools with native gcc
+#   - upx (native x86_64 binary) can compress target-arch binaries since UPX
+#     operates on the ELF file format regardless of the target ISA.
+#
+# Applet symlink probing: for native-arch builds the probe runs directly;
+# for cross-compiled binaries we use QEMU user-mode emulation (from binfmt)
+# only for this one lightweight probe step (busybox --help per applet), not
+# for the compile itself. The probe can alternatively be skipped by using
+# a pre-enumerated applet list, but the current approach is simpler.
+FROM --platform=$BUILDPLATFORM alpine:3.24.0@sha256:a2d49ea686c2adfe3c992e47dc3b5e7fa6e6b5055609400dc2acaeb241c829f4 AS busybox
+
+# Copy xx cross-compilation helpers (xx-clang, xx-apk, xx-info, etc.)
+COPY --from=xx / /
 
 # renovate: datasource=docker depName=busybox versioning=docker
-ARG BUSYBOX_VERSION=1.37.0
+ARG BUSYBOX_VERSION=1.38.0
 
-RUN apk add --no-cache build-base linux-headers wget bzip2 perl upx
+# Target platform ARGs (provided automatically by buildx).
+ARG TARGETPLATFORM
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# Native build tools (clang/lld for cross-compiling; gcc/make/upx run natively).
+# xx-apk installs the target-arch sysroot: musl-dev (C library headers + CRT),
+# gcc (provides crtbeginS.o/crtendS.o and libgcc needed by clang on Alpine),
+# and linux-headers (required by busybox for <linux/*.h> / <net/*.h>).
+RUN apk add --no-cache \
+        clang \
+        lld \
+        llvm \
+        gcc \
+        make \
+        wget \
+        bzip2 \
+        perl \
+        upx && \
+    xx-apk add --no-cache musl-dev gcc linux-headers
 
 RUN wget -q https://busybox.net/downloads/busybox-${BUSYBOX_VERSION}.tar.bz2 \
  && tar xf busybox-${BUSYBOX_VERSION}.tar.bz2
@@ -184,7 +323,34 @@ WORKDIR /busybox-${BUSYBOX_VERSION}
 
 # allnoconfig = every feature OFF; then enable only the curated applet set.
 COPY busybox-applets.config /tmp/applets.config
-RUN make allnoconfig && \
+# Set up xx cross-compiler aliases (<triple>-clang, <triple>-cc, etc.) and
+# build busybox.
+#
+# Key make variables:
+#   ARCH           — busybox ARCH; must match the cross-target, not the build
+#                    host. busybox's Makefile would otherwise read SUBARCH from
+#                    `uname -m` (the BUILD host's arch) which is wrong when
+#                    cross-compiling. We map TARGETARCH to busybox's arch name.
+#                    busybox uses -include arch/$(ARCH)/Makefile; missing arch
+#                    dirs are silently ignored, so any value is safe.
+#   CC             — busybox defaults to $(CROSS_COMPILE)gcc. We override CC to
+#                    the full <triple>-clang path so it resolves to the xx alias
+#                    (which sets --target and --sysroot for the cross-compiler).
+#                    Setting CC= avoids needing a <triple>-gcc symlink.
+#   HOSTCC         — native compiler for host-side build tools (scripts/kconfig,
+#                    gen_build_files, etc.); must NOT be the cross-compiler.
+#   SKIP_STRIP     — defer stripping to after symlink probing (we strip below
+#                    with llvm-strip, which handles any target ELF arch).
+RUN xx-clang --setup-target-triple && \
+    CROSS=$(xx-info triple) && \
+    # Map TARGETARCH to the busybox ARCH value.
+    case "${TARGETARCH}" in \
+      amd64)  BUSYBOX_ARCH=x86_64 ;; \
+      arm64)  BUSYBOX_ARCH=aarch64 ;; \
+      arm)    BUSYBOX_ARCH=arm ;; \
+      *)      BUSYBOX_ARCH=${TARGETARCH} ;; \
+    esac && \
+    make allnoconfig ARCH="${BUSYBOX_ARCH}" && \
     while read -r sym; do \
       case "$sym" in ''|\#*) continue ;; esac; \
       if grep -q "^# CONFIG_${sym} is not set" .config; then \
@@ -193,9 +359,15 @@ RUN make allnoconfig && \
         echo "CONFIG_${sym}=y" >> .config; \
       fi; \
     done < /tmp/applets.config && \
-    yes "" | make oldconfig >/dev/null 2>&1 && \
+    yes "" | make oldconfig ARCH="${BUSYBOX_ARCH}" >/dev/null 2>&1 && \
-    make -j"$(nproc)" >/dev/null 2>&1 && \
+    make -j"$(nproc)" \
-    strip busybox
+        ARCH="${BUSYBOX_ARCH}" \
+        CROSS_COMPILE="${CROSS}-" \
+        CC="${CROSS}-clang" \
+        HOSTCC=gcc \
+        SKIP_STRIP=y \
+        >/dev/null 2>&1 && \
+    llvm-strip busybox
 
 # Lay out a minimal rootfs with busybox + an applet symlink per applet.
 # Symlinks (argv[0] dispatch) are how busybox selects an applet and make the
@@ -205,6 +377,10 @@ RUN make allnoconfig && \
 # for non-applet symbols like FEATURE_* / STATIC, which we filter out).
 # We generate symlinks from the UNCOMPRESSED binary (so the probe is reliable),
 # then UPX-compress the binary in place afterwards.
+#
+# Note: probing cross-compiled binaries requires binfmt/QEMU user-mode. This
+# is only a lightweight per-applet help-flag check, not a full emulated build.
+# If QEMU is unavailable in CI, replace the probe with a static applet list.
 RUN mkdir -p /rootfs/bin && \
     grep '^CONFIG_.*=y' .config \
       | sed -e 's/^CONFIG_//' -e 's/=y$//' \
@@ -267,7 +443,9 @@ ENV PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 # -----------------------------------------------------------------------------
 VOLUME ["/var/lib/tailscale"]
 
-ENTRYPOINT ["/usr/local/bin/tailscaled"]
+# entrypoint.sh enables IP forwarding (incl. IPv6) in the container netns, then
+# exec's tailscaled with the CMD flags below as its arguments.
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 
 # Default flags:
 #   --no-logs-no-support  disables logtail uploads (logtail binary code is

README.md

@@ -7,6 +7,8 @@ A minimal Tailscale Docker image built for MikroTik routers running
 16 MB internal flash. Built from source with only router-relevant features
 included.
 
+> Disclaimer: This project has been largely vibe-coded, but I stand behind design and implementation choices made.
+
 - **~4 MB** extracted rootfs (`FROM scratch` + UPX'd Tailscale binary + a custom
   static busybox debug shell).
 - **Multi-arch**: amd64, arm64, arm/v7 — one tag, RouterOS pulls the right one.
@@ -16,6 +18,23 @@ included.
 - **Flash-wear conscious**: minimal persistent state, no netmap disk-caching,
   tmpfs for scratch and runtime.
 
+## Motivation
+
+There is no built-in Tailscale integration in MikroTik, and other solutions
+feel underwhelming. I've used Fluent-networks' tailscale-mikrotik until now,
+but that basically forced me to connect external storage to my router
+just to use Tailscale. This approach, while works, is fragile, wasteful
+and overcomplicated, so I decided to do better one myself.
+
+| | **This project** | Fluent-networks/tailscale-mikrotik |
+|---|---|---|
+| Size | **~4 MB** | ~106 MB |
+| Size reduction technique | **Minimal container with custom Tailscale and Busybox builds, compressed by UPX** | Alpine Linux base, Tailscale binary compressed by UPX on build, but auto-update completely nullifies that on first launch |
+| Update mechanism | **Automatically released optimized container images with new Tailscale versions, scheduled script updating deployment on new version** | None, opt-in Tailscale built-in auto-update downloading official binaries |
+| Flash wear | **Write-heavy functionality compiled out, suitable for low-endurance flash chips** | High, constant netmap cache updates |
+| Stability | **Immutable container** | Tailscale app can update on its own |
+| Features | **Only router-useful Tailscale features compiled, Busybox providing shell and utils** | Full tailscale, OpenSSH server, Bash, IPTables |
+
 ## Documentation
 
 - **[Usage](docs/USAGE.md)** — deploy the published image on a MikroTik router
@@ -51,6 +70,7 @@ ARMv5 (hEX Refresh / hAP ax S) is **not** supported — see
 |---|---|
 | `Dockerfile` | Multi-stage, multi-arch build (cross-compiled Go + custom busybox) |
 | `busybox-applets.config` | Curated busybox applet set |
+| `patches/` | Source files injected into the Tailscale tree at build time (stderr verbosity filter) |
 | `build.sh` | Build all/one arch, optionally export per-arch tarballs |
 | `routeros/update-tailscale.rsc` | RouterOS auto-update script (digest compare + recreate) |
 | `.woodpecker/` | CI: Renovate cron, release tagging, multi-arch publish |

build.sh

@@ -12,7 +12,8 @@
 #
 # Requirements:
 #   - docker with buildx
-#   - For non-native targets: binfmt/QEMU emulators registered, e.g.:
+#   - For non-native targets: binfmt/QEMU emulators registered for the applet
+#     symlink probe step (a minor step; the full C/Go compile is native):
 #       docker run --privileged --rm tonistiigi/binfmt --install arm64,arm
 set -eu
 

docs/DESIGN.md

@@ -70,8 +70,21 @@ in a future release stays omitted until deliberately added to the Dockerfile.
   saves a real ~195 kB of flash (424 kB → 229 kB), not just transfer size.
 
 The final image is built `FROM scratch` — there is no base distro layer.
-It contains only the busybox binary + applet symlinks, the CA bundle, and
+It contains only the busybox binary + applet symlinks, the CA bundle, the
-the Tailscale binary.
+Tailscale binary, and a tiny `entrypoint.sh`.
+
+### Entrypoint: IP forwarding
+
+`ENTRYPOINT` is a small `entrypoint.sh` that enables IPv4 and IPv6 forwarding
+(`net.ipv4.ip_forward`, `net.ipv6.conf.all.forwarding`) in the container's
+network namespace, then `exec`s `tailscaled` (so the daemon stays PID 1). This
+is necessary because `tailscaled` does **not** reliably enable IPv6 forwarding
+itself inside a container netns — it logs "IPv6 forwarding is disabled" and
+advertised IPv6 subnet routes silently fail. The sysctls are writable from
+inside a RouterOS container, so the entrypoint sets them directly; no
+host-side or `/container` configuration is required. The script is created in
+the builder stage so it ships in the same single `/usr/local/bin` `COPY` layer
+(preserving the [single-copy property](#avoiding-overlayfs-layer-duplication)).
 
 ### Avoiding overlayfs layer duplication
 
@@ -143,6 +156,7 @@ that's a separate build, not just a `--platform` change.
 | iptables | Linux iptables support for routing rules |
 | osrouter | Configure kernel network stack and routing tables |
 | unixsocketidentity | **Required** — without it the localapi denies every CLI call with "access denied" ([tailscale#17873](https://github.com/tailscale/tailscale/issues/17873)) |
+| ipnbus | Lets `tailscale up` wait for completion and print the login URL; without it `up` returns immediately without confirming success |
 
 ## Features intentionally omitted
 
@@ -150,7 +164,7 @@ that's a separate build, not just a `--platform` change.
 |---|---|
 | `clientupdate` | **Deliberately removed** — see [Why the built-in updater is removed](#why-the-built-in-updater-is-removed) |
 | `cachenetmap` | **Deliberately removed** — see [Why netmap disk-caching is removed](#why-netmap-disk-caching-is-removed) |
-| `logtail` | Would attempt persistent log writes; wear flash |
+| `logtail` | Would attempt persistent log writes; wear flash. Removing it also removes stderr verbosity filtering — restored by an injected filter, see [Log verbosity filtering](#log-verbosity-filtering) |
 | `netlog` | Network flow logging; separate concern |
 | `netstack` + `gro` | Userspace/gVisor networking; router uses kernel TUN |
 | `ssh` | Access via MikroTik SSH + `tailscale` CLI instead |
@@ -212,6 +226,54 @@ the in-memory resilience (the common case) while eliminating per-netmap flash
 writes. Only `tailscaled.state` (written on auth / key rotation) ever touches
 flash.
 
+### Log verbosity filtering
+
+Upstream `tailscaled` embeds verbosity tags (`[v1]`, `[v2]`, …) inside its log
+messages and relies on the **logtail** subsystem to act on them: in a stock
+build, logtail's log policy intercepts everything written via the standard
+`log` package, parses the tag, and only writes a line to stderr when its level
+is within `--verbose` (default 0 — non-verbose messages only). The `--verbose`
+flag is literally wired into logtail (`pol.SetVerbosityLevel(args.verbose)` in
+`cmd/tailscaled/tailscaled.go`).
+
+This build omits logtail (`ts_omit_logtail`) to avoid log-upload code and
+flash writes — but that removed the stderr filtering along with it, as
+collateral damage. The result: every verbose line went **unfiltered** to
+stderr and into the RouterOS container log, with the literal `[v1]` tag still
+in the text. On an active node that means constant spam, several lines per
+minute:
+
+```
+tailscale: ... [v1] Accept: TCP{...:53256 > ...:50000} 391 tcp non-syn
+tailscale: ... netcheck: [v1] report: udp=true v6=true ... derp=22 ...
+tailscale: ... wg: [v2] [0GwzF] - Receiving keepalive packet
+```
+
+This is a [known](https://github.com/tailscale/tailscale/issues/12158)
+[long-standing](https://github.com/tailscale/tailscale/issues/1548) complaint
+even in full builds, and RouterOS logging offers no way to discard matching
+messages (no drop action, rules are all-match — a regex rule duplicates rather
+than diverts).
+
+The fix here: the build injects a ~20-line Go file
+(`patches/stderr_verbosity_filter.go`, copied into `cmd/tailscaled/` before
+`go build`) whose `init()` wraps the standard log output and silently drops
+any line carrying a `[v1]`/`[v2]`/`[v3]` tag. This restores the exact
+equivalent of logtail's default `StderrLevel=0` behavior without pulling in
+the upload machinery. Properties:
+
+- **No upstream sources modified** — it's a new file in the package, so it
+  survives Tailscale version bumps without rebasing (only relies on the
+  daemon using the stdlib `log` package, which is core behavior).
+- **Build-tagged `//go:build ts_omit_logtail`** — if logtail is ever
+  re-enabled, the file compiles out automatically and logtail's own filtering
+  takes over; the two can never conflict.
+- **Runtime escape hatch** — setting the `TS_LOG_VERBOSITY=1` environment
+  variable disables the filter (and, conveniently, the same knob is read by
+  upstream as the default `--verbose` level). Verbose logs are one
+  `/container/envs/add` away; no rebuild needed. See
+  [USAGE.md → Logging](USAGE.md#logging).
+
 ## Volume layout
 
 Two mount points, with different persistence requirements:
@@ -232,6 +294,22 @@ Only the small, rarely-written state file touches flash; the socket dir is
 tmpfs. The netmap is held in memory only — see
 [Why netmap disk-caching is removed](#why-netmap-disk-caching-is-removed).
 
+### What lives in the state dir
+
+| File | Purpose | Write frequency |
+|---|---|---|
+| `tailscaled.state` | Node identity, auth keys, prefs | On auth / key rotation / prefs change |
+| `derpmap.cached.json` | Cached DERP relay server list for **bootstrap DNS**: at cold start with broken/unavailable DNS, tailscaled asks DERP servers to resolve the control plane. The binary ships a static DERP list, but it goes stale; this cache keeps the current one. | Once at first auth, then **only when Tailscale's relay infrastructure changes** (a few times a year). `dnsfallback.UpdateCache` has a deep-equal guard and skips the write when the DERP map is unchanged — netmap churn never touches it. |
+
+`derpmap.cached.json` is intentionally **kept** despite the flash-wear policy:
+the policy targets *frequent* writes (netmap deltas, logs), not one-shot
+caches. On a router this cache is genuinely useful — after a power outage the
+device may boot with WAN up but upstream DNS broken, exactly the case where a
+fresh DERP list lets the node reach the control plane anyway. With
+`cachenetmap` omitted, this file and `tailscaled.state` are the only cold-start
+resilience the node has. (There is no `ts_omit_*` tag for it; it is written
+only because `--statedir` is set.)
+
 ## Flash wear protection
 
 Several measures are in place to avoid wearing out internal flash:

docs/USAGE.md

@@ -9,65 +9,61 @@ reasoning behind these choices, see [DESIGN.md](DESIGN.md).
 
 ## Deploy on MikroTik (RouterOS)
 
-Verified on RouterOS 7.21.2 (arm64, CRS418). Commands are grouped into
+Verified on RouterOS 7.23 (arm64, CRS418). Commands are grouped into
-copy-paste blocks; **only the values marked `CHANGE ME` need editing**.
+copy-paste blocks, defaults should fit most configurations.
 
 > Because the image has no built-in updater (the `clientupdate` feature is
 > [intentionally compiled out](DESIGN.md#why-the-built-in-updater-is-removed)),
-> updates are handled by a small script that only re-pulls when the published
+> updates are handled by a small script that recreates container when
-> image actually changed — see [step 7](#7-enable-automatic-updates).
+> the update is published — see [step 7](#7-enable-automatic-updates).
 
 ### 0. Prerequisites
 
-- RouterOS 7.x with the **container** package installed.
+- RouterOS >= 7.23 with the **container** package installed
-- Container mode enabled (needs physical access — press reset / cold-boot when
+  (7.23 is needed for container `restart-policy`; the deploy itself works on
-  prompted):
+  >= 7.13 if you drop the restart options).
+- Container mode enabled ([documentation](https://manual.mikrotik.com/docs/System%20Information%20and%20Utilities/device-mode/#changing-mode-of-device-mode)):
 
   ```
   /system/device-mode/update container=yes
   ```
 
-- A Tailscale **auth key** from the admin console
+### 1. Networking (veth + routing)
-  (**Settings → Keys**, reusable, optionally tagged). You'll use it in step 6.
 
-### 1. Networking (veth + bridge + NAT)
+Gives the container an internal IP and configures routing to the tailnet.
-
+Pick a subnet that doesn't clash with your LAN.
-Gives the container an internal IP and outbound internet via NAT. Pick a subnet
-that doesn't clash with your LAN.
 
 ```
 /interface/veth/add name=veth-tailscale address=172.20.0.2/24 gateway=172.20.0.1
 /interface/bridge/add name=containers
 /ip/address/add address=172.20.0.1/24 interface=containers
 /interface/bridge/port/add bridge=containers interface=veth-tailscale
-/ip/firewall/nat/add chain=srcnat action=masquerade src-address=172.20.0.0/24
+/ip/route/add dst-address=100.64.0.0/10 gateway=172.20.0.2 comment=Tailnet
 ```
 
+If you want the router to have access to subnets shared by other tailscale nodes,
+add route for each one.
+
+```
+/ip/route/add dst-address=[subnet CIDR] gateway=172.20.0.2 comment="Another network via tailscale"
+```
+
+If you want to share your LAN via tailscale, add it as an advertised route in
+[step 5](#5-authenticate). You may also need additional firewall configuration
+to accept connections to or from tailnet if you have one configured.
+You should not need any additional NAT rules.
+
 ### 2. Extraction scratch dir (tmpfs)
 
 Put the image extraction scratch dir on **tmpfs** (RAM) so the pull/extract
-never writes to flash:
+happen in RAM and doesn't fill up or wear out flash:
 
 ```
 /disk/add type=tmpfs tmpfs-max-size=256M slot=tmp
 /container/config/set tmpdir=tmp
 ```
 
-> **No `registry-url` change needed.** This guide puts the full registry host in
+### 3. Persistent state mount (the only thing on flash)
-> `remote-image` (step 5), and RouterOS pulls directly from that host — the
-> global `registry-url` is ignored when the image reference includes a host.
-> This is intentional: it leaves your existing `registry-url` untouched, so
-> other containers (e.g. ones pulling from Docker Hub or ghcr.io) keep working,
-> and multiple registries can be used side by side.
-
-### 3. Authentication note (no env needed)
-
-This image runs `tailscaled` directly and does **not** bundle Tailscale's
-`containerboot` wrapper, so the `TS_AUTHKEY` environment variable is **not**
-read automatically. You authenticate with `tailscale up --authkey=...` after the
-container starts (step 6) — this keeps the image minimal and needs no env list.
-
-### 4. Persistent state mount (the only thing on flash)
 
 Only the tiny `tailscaled.state` (node identity / key) needs to persist. Mount
 just that directory:
@@ -76,14 +72,7 @@ just that directory:
 /container/mounts/add list=tailscale_state src=tailscale/state dst=/var/lib/tailscale
 ```
 
-`src=tailscale/state` is on internal storage. This holds `tailscaled.state`
+### 4. Add and start the container
-(and `derpmap.cached.json`), written only on auth / key rotation / prefs
-change — **not** on every netmap update, because netmap disk-caching is omitted
-([why](DESIGN.md#why-netmap-disk-caching-is-removed)). Flash wear is therefore
-minimal. If you want *zero* persistent writes, point `src` at a tmpfs disk slot
-instead and accept re-authentication after a reboot.
-
-### 5. Add and start the container
 
 ```
 /container/add \
@@ -93,6 +82,8 @@ instead and accept re-authentication after a reboot.
     mountlists=tailscale_state \
     logging=yes \
     start-on-boot=yes \
+    restart-policy=on-failure \
+    restart-interval=10s \
     name=tailscale
 ```
 
@@ -106,16 +97,25 @@ Wait for the pull/extract to finish (`status=stopped`), then start it:
 
 The daemon is now running but **not yet authenticated**.
 
-### 6. Authenticate
+### 5. Authenticate
 
-Enter the container shell and bring Tailscale up with your auth key. You can set
+> This image runs `tailscaled` via a tiny entrypoint (which enables IP
-subnet routes / exit-node advertisement in the same command:
+forwarding, then `exec`s the daemon) and does **not** bundle Tailscale's
+`containerboot` wrapper, so the `TS_AUTHKEY` environment variable is **not**
+read automatically. You authenticate with `tailscale up --authkey=...` after the
+container starts.
+
+Enter the container shell and bring Tailscale up with your auth key.
+Use `tailscale up --help` to see list of commands, customize it to your needs,
+add subnets (eg. your LAN) or exit-node advertisements in command below.
 
 ```
 /container/shell [find where name=tailscale]
 # inside the container — CHANGE ME: your key (and adjust routes/subnet):
 tailscale up --authkey=tskey-auth-CHANGEME \
-  --advertise-routes=192.168.88.0/24 \
+  --accept-routes \
+  --snat-subnet-routes=false \
+  --advertise-routes=172.20.0.0/24 \
   --advertise-exit-node
 exit
 ```
@@ -124,7 +124,7 @@ The node now appears in your Tailscale admin console. Approve the advertised
 routes / exit node there. Because the auth state is written to the persisted
 `tailscaled.state`, you only do this once — it survives reboots and updates.
 
-### 7. Enable automatic updates
+### 6. Enable automatic updates
 
 First, edit the `CONFIG` block at the top of `routeros/update-tailscale.rsc` if
 you changed any names in the steps above. The defaults match this guide
@@ -136,8 +136,6 @@ create a **named script** from it and schedule it:
 
 ```
 # Create the named script from the uploaded file's contents.
-# (Do NOT use `/import` — that just runs the file once and does not create a
-#  reusable script for the scheduler to call.)
 /system/script/add name=update-tailscale source=[/file/get update-tailscale.rsc contents]
 
 # Run it daily.
@@ -179,14 +177,48 @@ queries to Tailscale's magic DNS resolver:
 add name="ts.net" type=FWD forward-to=100.100.100.100 match-subdomain=yes
 ```
 
-This avoids writing to `/etc/resolv.conf` inside the container (which would
+When this is configured, you can connect to other tailscale machines using
-happen if `--accept-dns` is passed to `tailscale up`). The container resolves
+`[device name].[tailnet name].ts.net`. You can see and change assigned
-Tailscale node names; the rest of the router uses its own DNS.
+Tailnet DNS name in Tailscale admin panel under DNS tab.
+
+## Logging
+
+The container logs to the RouterOS log (topic `container`) via `logging=yes`.
+
+Upstream `tailscaled` is notoriously chatty: by default it would emit a line
+for every accepted connection (`Accept: TCP{...}`), every netcheck report, and
+every WireGuard handshake/keepalive — several lines per minute on an active
+node ([tailscale#12158](https://github.com/tailscale/tailscale/issues/12158)).
+This image filters those verbose (`[v1]`/`[v2]`-tagged) messages out at the
+source, so only meaningful messages (startup, auth, route changes, warnings,
+errors) reach the RouterOS log. See
+[DESIGN.md → Log verbosity filtering](DESIGN.md#log-verbosity-filtering) for
+how and why.
+
+To temporarily get the verbose logs back for debugging (e.g. NAT-traversal
+issues), set the `TS_LOG_VERBOSITY` environment variable and recreate the
+container with the envlist attached:
+
+```
+/container/envs/add list=tailscale_envs name=TS_LOG_VERBOSITY value=1
+/container/set [find where name=tailscale] envlist=tailscale_envs
+/container/stop [find where name=tailscale]
+/container/start [find where name=tailscale]
+```
+
+Any value ≥ 1 disables the filter (and raises the daemon's own verbosity by
+the same amount). Remove the variable and restart to silence it again:
+
+```
+/container/envs/remove [find where name=TS_LOG_VERBOSITY]
+/container/stop [find where name=tailscale]
+/container/start [find where name=tailscale]
+```
 
 ## Updating
 
 You don't normally do anything: when a new release is published, the
-auto-update script ([step 7](#7-enable-automatic-updates)) detects the changed
+auto-update script ([step 6](#6-enable-automatic-updates)) detects the changed
 `:stable` image on its next scheduled run and recreates the container. Your
 node identity and settings persist across the update via the state mount.
 

patches/stderr_verbosity_filter.go

@@ -0,0 +1,45 @@
+// Copyright (c) mikrotik-tailscale build. Injected at image build time.
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build ts_omit_logtail
+
+package main
+
+// When logtail is compiled out (ts_omit_logtail), logpolicy is never
+// installed (see run() in tailscaled.go: `if buildfeatures.HasLogTail`),
+// so log.Printf output goes raw to stderr. Nothing parses the [v1]/[v2]
+// verbosity tags Tailscale embeds in log messages, which means every
+// verbose line (filter "Accept: TCP", "netcheck: [v1] report",
+// "wg: [v2]" handshakes/keepalives) is printed regardless of --verbose.
+//
+// This restores the equivalent of logtail's StderrLevel=0 behavior:
+// drop lines carrying a [v1]+ tag, unless TS_LOG_VERBOSITY is set to
+// 1 or higher (runtime escape hatch for debugging — no rebuild needed).
+
+import (
+	"bytes"
+	"log"
+	"os"
+)
+
+var verboseLogTags = [][]byte{[]byte("[v1] "), []byte("[v2] "), []byte("[v3] ")}
+
+type stderrVerbosityFilter struct{ w *os.File }
+
+func (f stderrVerbosityFilter) Write(p []byte) (int, error) {
+	for _, tag := range verboseLogTags {
+		if bytes.Contains(p, tag) {
+			// Claim success so the log package doesn't complain;
+			// the line is intentionally discarded.
+			return len(p), nil
+		}
+	}
+	return f.w.Write(p)
+}
+
+func init() {
+	if v := os.Getenv("TS_LOG_VERBOSITY"); v != "" && v != "0" {
+		return
+	}
+	log.SetOutput(stderrVerbosityFilter{os.Stderr})
+}

renovate.json

@@ -7,6 +7,17 @@
   ],
   "labels": ["dependencies"],
   "rebaseWhen": "behind-base-branch",
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "Update version ARGs annotated with a `# renovate:` comment (the dockerfile manager only handles FROM/image lines, not ARG values).",
+      "managerFilePatterns": ["/(^|/)Dockerfile$/"],
+      "matchStrings": [
+        "#\\s*renovate:\\s*datasource=(?<datasource>\\S+)\\s+depName=(?<depName>\\S+)(?:\\s+packageName=(?<packageName>\\S+))?(?:\\s+versioning=(?<versioning>\\S+))?\\s+ARG \\w+=(?<currentValue>\\S+)"
+      ],
+      "matchStringsStrategy": "any"
+    }
+  ],
   "packageRules": [
     {
       "matchManagers": ["dockerfile"],
@@ -16,9 +27,8 @@
     {
       "matchDatasources": ["github-releases"],
       "matchPackageNames": ["tailscale/tailscale"],
-      "description": "TAILSCALE_VERSION ARG: only stable releases. Tailscale uses EVEN minor versions for stable (v1.98.x); ODD minors (v1.99.x) are unstable, so filter to even minors and ignore pre-releases.",
+      "description": "TAILSCALE_VERSION ARG: only stable releases. Tailscale uses EVEN minor versions for stable (v1.98.x); ODD minors (v1.99.x) are unstable, so filter to even minors and ignore pre-releases. The `v` prefix is kept (no extractVersion) so the ARG value stays v-prefixed to match the git tags cloned in the Dockerfile.",
-      "extractVersion": "^v(?<version>\\d+\\.\\d+\\.\\d+)$",
+      "allowedVersions": "/^v\\d+\\.\\d*[02468]\\.\\d+$/",
-      "allowedVersions": "/^\\d+\\.\\d*[02468]\\.\\d+$/",
       "ignoreUnstable": true
     },
     {
@@ -30,8 +40,15 @@
     },
     {
       "matchManagers": ["dockerfile"],
-      "matchPackageNames": ["golang", "alpine", "busybox"],
+      "matchPackageNames": ["golang", "alpine"],
-      "description": "Automerge PATCH-only bumps of build components (Go/Alpine/busybox) once the PR build passes; review minor/major manually.",
+      "description": "Automerge PATCH-only bumps of build components (Go/Alpine) once the PR build passes; review minor/major manually.",
+      "matchUpdateTypes": ["patch"],
+      "automerge": true
+    },
+    {
+      "matchDatasources": ["docker"],
+      "matchPackageNames": ["busybox"],
+      "description": "busybox ARG (custom manager): automerge PATCH bumps once the PR build passes; review minor/major manually.",
       "matchUpdateTypes": ["patch"],
       "automerge": true
     },

routeros/update-tailscale.rsc

@@ -3,8 +3,9 @@
 # =============================================================================
 # Checks the Gitea registry for a new :stable image and, only if the published
 # image actually changed, recreates the container. Designed for RouterOS 7.x
-# (tested target: 7.21.2, arm64). Requires RouterOS >= 7.13 for the :deserialize
+# (tested target: 7.23, arm64). Requires RouterOS >= 7.23 for the container
-# command used to parse the registry token JSON.
+# restart-policy properties (and >= 7.13 for the :deserialize command used to
+# parse the registry token JSON).
 #
 # HOW IT DECIDES "something changed":
 #   It fetches the manifest digest of the :stable tag from the registry and
@@ -59,6 +60,12 @@
 :local cInterface "veth-tailscale"
 :local cLogging   yes
 :local cStartOnBoot yes
+# Restart the container automatically if tailscaled crashes (tailscaled is
+# PID 1; if it dies the container stops). on-failure restarts only on abnormal
+# exit (a manual /container/stop stays stopped); 10s is a gentle backoff.
+# Requires RouterOS >= 7.23.
+:local cRestartPolicy "on-failure"
+:local cRestartInterval "10s"
 # ----------------------------------------------------------------------------
 
 :log info "$scriptName: checking for image updates"
@@ -174,6 +181,8 @@
     mountlists=$cMountList \
     logging=$cLogging \
     start-on-boot=$cStartOnBoot \
+    restart-policy=$cRestartPolicy \
+    restart-interval=$cRestartInterval \
     name=$cName
 } do={
   :log error "$scriptName: container add failed: $e"