lower root kustomizations auto sync interval

Reviewed-on: #192

.gitignore

@@ -10,3 +10,4 @@ devenv.local.yaml
 
 # pre-commit
 .pre-commit-config.yaml
+.opencode

.vscode/extensions.json

@@ -2,6 +2,7 @@
   "recommendations": [
     "jnoortheen.nix-ide",
     "detachhead.basedpyright",
-    "mkhl.direnv"
+    "mkhl.direnv",
+    "mermaidchart.vscode-mermaid-chart"
   ]
 }

.woodpecker/flux-reconcile-source.yaml

@@ -0,0 +1,49 @@
+when:
+  - event: push
+    branch: fresh-start
+
+skip_clone: true
+
+steps:
+  - name: Get kubernetes access from OpenBao
+    image: quay.io/openbao/openbao:2.5.2
+    environment:
+      VAULT_ADDR: https://openbao.lumpiasty.xyz:8200
+      ROLE_ID:
+        from_secret: flux_reconcile_role_id
+      SECRET_ID:
+        from_secret: flux_reconcile_secret_id
+    commands:
+      - bao write -field token auth/approle/login
+          role_id=$ROLE_ID
+          secret_id=$SECRET_ID > /woodpecker/.vault_id
+      - export VAULT_TOKEN=$(cat /woodpecker/.vault_id)
+      - bao write -format json -f /kubernetes/creds/flux-reconcile > /woodpecker/kube_credentials
+  - name: Construct Kubeconfig
+    image: alpine/k8s:1.35.3
+    environment:
+      KUBECONFIG: /woodpecker/kubeconfig
+    commands:
+      - kubectl config set-cluster cluster
+          --server=https://$KUBERNETES_SERVICE_HOST
+          --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      - kubectl config set-credentials cluster
+          --token=$(jq -r .data.service_account_token /woodpecker/kube_credentials)
+      - kubectl config set-context cluster
+          --cluster cluster
+          --user cluster
+          --namespace flux-system
+      - kubectl config use-context cluster
+  - name: Reconcile git source
+    image: ghcr.io/fluxcd/flux-cli:v2.8.3
+    environment:
+      KUBECONFIG: /woodpecker/kubeconfig
+    commands:
+      - flux reconcile source git flux-system
+  - name: Invalidate OpenBao token
+    image: quay.io/openbao/openbao:2.5.2
+    environment:
+      VAULT_ADDR: https://openbao.lumpiasty.xyz:8200
+    commands:
+      - export VAULT_TOKEN=$(cat /woodpecker/.vault_id)
+      - bao write -f auth/token/revoke-self

Makefile

@@ -1,3 +1,7 @@
+SHELL := /usr/bin/env bash
+
+.PHONY: install-router gen-talos-config apply-talos-config get-kubeconfig
+
 install-router:
 	ansible-playbook ansible/playbook.yml -i ansible/hosts
 

README.md

@@ -2,6 +2,8 @@
 
 This repo contains configuration and documentation for my homelab setup, which is based on Talos OS for Kubernetes cluster and MikroTik router.
 
+[<img src="https://woodpecker.lumpiasty.xyz/api/badges/2/status.svg" alt="Pipeline status">](https://woodpecker.lumpiasty.xyz/repos/2)
+
 ## Architecture
 
 Physical setup consists of MikroTik router which connects to the internet and serves as a gateway for the cluster and other devices in the home network as shown in the diagram below.
@@ -141,7 +143,7 @@ Currently the k8s cluster consists of single node (hostname anapistula-delrosala
 
 ## Software stack
 
-The cluster itself is based on [Talos Linux](https://www.talos.dev/) (which is also a Kubernetes distribution) and uses [Cilium](https://cilium.io/) as CNI, IPAM, kube-proxy replacement, Load Balancer, and BGP control plane. Persistent volumes are managed by [OpenEBS LVM LocalPV](https://openebs.io/docs/user-guides/local-storage-user-guide/local-pv-lvm/lvm-overview). Applications are deployed using GitOps (this repo) and reconciled on cluster using [Flux](https://fluxcd.io/). Git repository is hosted on [Gitea](https://gitea.io/) running on a cluster itself. Secets are kept in [OpenBao](https://openbao.org/) (HashiCorp Vault fork) running on a cluster and synced to cluster objects using [Vault Secrets Operator](https://github.com/hashicorp/vault-secrets-operator). Deployments are kept up to date using self hosted [Renovate](https://www.mend.io/renovate/) bot updating manifests in the Git repository. Incoming HTTP traffic is routed to cluster using [Nginx Ingress Controller](https://kubernetes.github.io/ingress-nginx/) and certificates are issued by [cert-manager](https://cert-manager.io/) with [Let's Encrypt](https://letsencrypt.org/) ACME issuer with [cert-manager-webhook-ovh](https://github.com/aureq/cert-manager-webhook-ovh) resolving DNS-01 challanges. Cluster also runs [CloudNativePG](https://cloudnative-pg.io/) operator for managing PostgreSQL databases. Router is running [Mikrotik RouterOS](https://help.mikrotik.com/docs/spaces/ROS/pages/328059/RouterOS) and its configuration is managed via [Ansible](https://docs.ansible.com/) playbook in this repo. High level core cluster software architecture is shown on the diagram below.
+The cluster itself is based on [Talos Linux](https://www.talos.dev/) (which is also a Kubernetes distribution) and uses [Cilium](https://cilium.io/) as CNI, IPAM, kube-proxy replacement, Load Balancer, and BGP control plane. Persistent volumes are managed by [OpenEBS LVM LocalPV](https://openebs.io/docs/user-guides/local-storage-user-guide/local-pv-lvm/lvm-overview). Applications are deployed using GitOps (this repo) and reconciled on cluster using [Flux](https://fluxcd.io/). Git repository is hosted on [Gitea](https://gitea.io/) running on a cluster itself. Secets are kept in [OpenBao](https://openbao.org/) (HashiCorp Vault fork) running on a cluster and synced to cluster objects using [Vault Secrets Operator](https://github.com/hashicorp/vault-secrets-operator). Deployments are kept up to date using self hosted [Renovate](https://www.mend.io/renovate/) bot updating manifests in the Git repository. There is a [Woodpecker](https://woodpecker-ci.org/) instance watching repositories on Gitea and scheduling jobs on cluster. Incoming HTTP traffic is routed to cluster using [Nginx Ingress Controller](https://kubernetes.github.io/ingress-nginx/) and certificates are issued by [cert-manager](https://cert-manager.io/) with [Let's Encrypt](https://letsencrypt.org/) ACME issuer with [cert-manager-webhook-ovh](https://github.com/aureq/cert-manager-webhook-ovh) resolving DNS-01 challanges. Cluster also runs [CloudNativePG](https://cloudnative-pg.io/) operator for managing PostgreSQL databases. Router is running [Mikrotik RouterOS](https://help.mikrotik.com/docs/spaces/ROS/pages/328059/RouterOS) and its configuration is managed via [Ansible](https://docs.ansible.com/) playbook in this repo. High level core cluster software architecture is shown on the diagram below.
 
 > Talos Linux is an immutable Linux distribution purpose-built for running Kubernetes. The OS is distributed as an OCI (Docker) image and does not contain any package manager, shell, SSH, or any other tools for managing the system. Instead, all operations are performed using API, which can be accessed using `talosctl` CLI tool.
 
@@ -177,14 +179,23 @@ flowchart TD
       vault_operator -- "Retrieves secrets" --> vault[OpenBao] -- "Secret storage" --> lv
       vault -- "Auth method" --> kubeapi
 
+      gitea -- "Receives events" --> woodpecker[Woodpecker CI] -- "Schedules jobs" --> kubeapi
+
       gitea -- "Stores repositories" --> lv
 
-      gitea --> renovate[Renovate Bot] -- "Updates manifests" --> gitea
+      gitea--> renovate[Renovate Bot] -- "Updates manifests" --> gitea
-
 
    end
 ```
 
+### Reconcilation paths of each component
+
+- Kubernetes manifests are reconciled using Flux triggerred by Woodpecker CI on push
+- RouterOS configs are applied by Ansible <!-- ran by Gitea Action on push -->
+- Talos configs are applied using makefile <!-- switch to ansible and trigger on action push -->
+- Vault policies are applied by running `synchronize-vault.py` <!-- triggerred by Gitea action on push -->
+<!-- - Docker images are built and pushed to registry by Gitea Actions on push -->
+
 <!-- TODO: Backups, monitoring, logging, deployment with ansible etc -->
 
 ## Software
@@ -228,6 +239,7 @@ flowchart TD
 |------|------|-------------|
 | <img src="docs/assets/devenv.svg" alt="devenv" height="50" width="50"> | devenv | Tool for declarative managment of development environment using Nix |
 | <img src="docs/assets/renovate.svg" alt="Renovate" height="50" width="50"> | Renovate | Bot for keeping dependencies up to date |
+| <img src="docs/assets/woodpecker.svg" alt="Woodpecker" height="50" width="50"> | Woodpecker CI | Continous Integration system |
 
 ### AI infrastructure
 

apps/authentik/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - postgres-volume.yaml
+  - postgres-cluster.yaml
+  - secret.yaml
+  - release.yaml

apps/authentik/namespace.yaml

@@ -1,5 +1,4 @@
----
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: searxng
+  name: authentik

apps/authentik/postgres-cluster.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: authentik-postgresql-cluster-lvmhdd
+  namespace: authentik
+spec:
+  instances: 1
+
+  imageName: ghcr.io/cloudnative-pg/postgresql:17.4
+
+  bootstrap:
+    initdb:
+      database: authentik
+      owner: authentik
+
+  storage:
+    pvcTemplate:
+      storageClassName: hdd-lvmpv
+      resources:
+        requests:
+          storage: 10Gi
+      volumeName: authentik-postgresql-cluster-lvmhdd-1

apps/authentik/postgres-volume.yaml

@@ -1,13 +1,12 @@
----
 apiVersion: local.openebs.io/v1alpha1
 kind: LVMVolume
 metadata:
   labels:
     kubernetes.io/nodename: anapistula-delrosalae
-  name: garm-lvmhdd
+  name: authentik-postgresql-cluster-lvmhdd-1
   namespace: openebs
 spec:
-  capacity: 5Gi
+  capacity: 10Gi
   ownerNodeID: anapistula-delrosalae
   shared: "yes"
   thinProvision: "no"
@@ -17,10 +16,10 @@ spec:
 kind: PersistentVolume
 apiVersion: v1
 metadata:
-  name: garm-lvmhdd
+  name: authentik-postgresql-cluster-lvmhdd-1
 spec:
   capacity:
-    storage: 5Gi
+    storage: 10Gi
   accessModes:
     - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
@@ -29,18 +28,6 @@ spec:
   csi:
     driver: local.csi.openebs.io
     fsType: btrfs
-    volumeHandle: garm-lvmhdd
+    volumeHandle: authentik-postgresql-cluster-lvmhdd-1
 ---
-kind: PersistentVolumeClaim
+# PVCs are dynamically created by the Postgres operator
-apiVersion: v1
-metadata:
-  name: garm-lvmhdd
-  namespace: garm
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi
-  storageClassName: hdd-lvmpv
-  volumeName: garm-lvmhdd

apps/authentik/release.yaml

@@ -0,0 +1,61 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: authentik
+  namespace: authentik
+spec:
+  interval: 24h
+  url: https://charts.goauthentik.io
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: authentik
+  namespace: authentik
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: authentik
+      version: 2026.2.1
+      sourceRef:
+        kind: HelmRepository
+        name: authentik
+        namespace: authentik
+      interval: 12h
+  values:
+    authentik:
+      postgresql:
+        host: authentik-postgresql-cluster-lvmhdd-rw
+        name: authentik
+        user: authentik
+
+    global:
+      env:
+        - name: AUTHENTIK_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: authentik-secret
+              key: secret_key
+        - name: AUTHENTIK_POSTGRESQL__PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: authentik-postgresql-cluster-lvmhdd-app
+              key: password
+
+    postgresql:
+      enabled: false
+
+    server:
+      ingress:
+        enabled: true
+        ingressClassName: nginx-ingress
+        annotations:
+          cert-manager.io/cluster-issuer: letsencrypt
+        hosts:
+          - authentik.lumpiasty.xyz
+        tls:
+          - secretName: authentik-ingress
+            hosts:
+              - authentik.lumpiasty.xyz

apps/authentik/secret.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: authentik-secret
+  namespace: authentik
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: authentik
+  namespace: authentik
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: authentik
+    serviceAccount: authentik-secret
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: authentik-secret
+  namespace: authentik
+spec:
+  type: kv-v2
+
+  mount: secret
+  path: authentik
+
+  destination:
+    create: true
+    name: authentik-secret
+    type: Opaque
+    transformation:
+      excludeRaw: true
+
+  vaultAuthRef: authentik

apps/crawl4ai-proxy/deployment.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawl4ai-proxy
+  namespace: crawl4ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: crawl4ai-proxy
+  template:
+    metadata:
+      labels:
+        app: crawl4ai-proxy
+    spec:
+      containers:
+        - name: crawl4ai-proxy
+          image: gitea.lumpiasty.xyz/lumpiasty/crawl4ai-proxy-fit:latest
+          imagePullPolicy: Always
+          env:
+            - name: LISTEN_PORT
+              value: "8000"
+            - name: CRAWL4AI_ENDPOINT
+              value: http://crawl4ai.crawl4ai.svc.cluster.local:11235/crawl
+          ports:
+            - name: http
+              containerPort: 8000
+          readinessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 6
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 15
+            timeoutSeconds: 2
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 25m
+              memory: 32Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi

apps/crawl4ai-proxy/kustomization.yaml

@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - namespace.yaml
+  - deployment.yaml
-  - release.yaml
+  - service.yaml

apps/crawl4ai-proxy/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crawl4ai-proxy
+  namespace: crawl4ai
+spec:
+  type: ClusterIP
+  selector:
+    app: crawl4ai-proxy
+  ports:
+    - name: http
+      port: 8000
+      targetPort: 8000
+      protocol: TCP

apps/crawl4ai/deployment.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawl4ai
+  namespace: crawl4ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: crawl4ai
+  template:
+    metadata:
+      labels:
+        app: crawl4ai
+    spec:
+      containers:
+        - name: crawl4ai
+          image: unclecode/crawl4ai:latest
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: CRAWL4AI_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: crawl4ai-secret
+                  key: api_token
+                  optional: false
+            - name: MAX_CONCURRENT_TASKS
+              value: "5"
+          ports:
+            - name: http
+              containerPort: 11235
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 3
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          volumeMounts:
+            - name: dshm
+              mountPath: /dev/shm
+      volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Gi

apps/crawl4ai/kustomization.yaml

@@ -2,9 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  - pvc.yaml
-  - configmap.yaml
-  - service.yaml
-  - rbac.yaml
   - secret.yaml
   - deployment.yaml
+  - service.yaml

apps/crawl4ai/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: crawl4ai

apps/crawl4ai/secret.yaml

@@ -1,32 +1,38 @@
 ---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crawl4ai-secret
+  namespace: crawl4ai
+---
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultAuth
 metadata:
-  name: garm
+  name: crawl4ai
-  namespace: garm
+  namespace: crawl4ai
 spec:
   method: kubernetes
   mount: kubernetes
   kubernetes:
-    role: garm
+    role: crawl4ai
-    serviceAccount: garm
+    serviceAccount: crawl4ai-secret
 ---
 apiVersion: secrets.hashicorp.com/v1beta1
 kind: VaultStaticSecret
 metadata:
-  name: garm-config
+  name: crawl4ai-secret
-  namespace: garm
+  namespace: crawl4ai
 spec:
   type: kv-v2
 
   mount: secret
-  path: garm
+  path: crawl4ai
 
   destination:
     create: true
-    name: garm-config
+    name: crawl4ai-secret
     type: Opaque
     transformation:
       excludeRaw: true
 
-  vaultAuthRef: garm
+  vaultAuthRef: crawl4ai

apps/crawl4ai/service.yaml

@@ -1,14 +1,14 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: garm
+  name: crawl4ai
-  namespace: garm
+  namespace: crawl4ai
 spec:
   type: ClusterIP
   selector:
-    app: garm
+    app: crawl4ai
   ports:
     - name: http
-      port: 9997
+      port: 11235
-      targetPort: 9997
+      targetPort: 11235
       protocol: TCP

apps/garm/configmap.yaml

@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: garm-provider-k8s-config
-  namespace: garm
-data:
-  provider-config.yaml: |
-    kubeConfigPath: ""
-    runnerNamespace: "garm-runners"
-    podTemplate:
-      spec:
-        restartPolicy: Never
-    flavors:
-      default:
-        requests:
-          cpu: 100m
-          memory: 512Mi
-        limits:
-          memory: 2Gi

apps/garm/deployment.yaml

@@ -1,122 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: garm
-  namespace: garm
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: garm
-  template:
-    metadata:
-      labels:
-        app: garm
-    spec:
-      serviceAccountName: garm
-      initContainers:
-        - name: install-garm-provider-k8s
-          image: alpine:3.21
-          command:
-            - /bin/sh
-            - -ec
-            - |
-              wget -qO /tmp/garm-provider-k8s.tar.gz "https://github.com/mercedes-benz/garm-provider-k8s/releases/download/v0.3.2/garm-provider-k8s_Linux_x86_64.tar.gz"
-              tar -xzf /tmp/garm-provider-k8s.tar.gz -C /opt/garm/providers.d
-              chmod 0755 /opt/garm/providers.d/garm-provider-k8s
-          volumeMounts:
-            - name: provider-dir
-              mountPath: /opt/garm/providers.d
-        - name: render-garm-config
-          image: alpine:3.21
-          env:
-            - name: JWT_AUTH_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: garm-config
-                  key: jwt_auth_secret
-            - name: DATABASE_PASSPHRASE
-              valueFrom:
-                secretKeyRef:
-                  name: garm-config
-                  key: database_passphrase
-          command:
-            - /bin/sh
-            - -ec
-            - |
-              cat <<EOF > /etc/garm/config.toml
-              [default]
-              enable_webhook_management = true
-
-              [logging]
-              enable_log_streamer = true
-              log_format = "text"
-              log_level = "info"
-              log_source = false
-
-              [metrics]
-              enable = true
-              disable_auth = false
-
-              [jwt_auth]
-              secret = "${JWT_AUTH_SECRET}"
-              time_to_live = "8760h"
-
-              [apiserver]
-                bind = "0.0.0.0"
-                port = 9997
-                use_tls = false
-                [apiserver.webui]
-                  enable = true
-
-              [database]
-                backend = "sqlite3"
-                passphrase = "${DATABASE_PASSPHRASE}"
-                [database.sqlite3]
-                  db_file = "/data/garm.db"
-                  busy_timeout_seconds = 5
-
-              [[provider]]
-              name = "kubernetes_external"
-              description = "Kubernetes provider"
-              provider_type = "external"
-              [provider.external]
-              config_file = "/etc/garm/provider-config.yaml"
-              provider_executable = "/opt/garm/providers.d/garm-provider-k8s"
-              environment_variables = ["KUBERNETES_"]
-              EOF
-          volumeMounts:
-            - name: config-dir
-              mountPath: /etc/garm
-      containers:
-        - name: garm
-          image: ghcr.io/cloudbase/garm:v0.1.7
-          imagePullPolicy: IfNotPresent
-          command:
-            - /bin/garm
-            - --config
-            - /etc/garm/config.toml
-          ports:
-            - name: http
-              containerPort: 9997
-          volumeMounts:
-            - name: data
-              mountPath: /data
-            - name: config-dir
-              mountPath: /etc/garm
-            - name: provider-dir
-              mountPath: /opt/garm/providers.d
-            - name: provider-config
-              mountPath: /etc/garm/provider-config.yaml
-              subPath: provider-config.yaml
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: garm-lvmhdd
-        - name: config-dir
-          emptyDir: {}
-        - name: provider-dir
-          emptyDir: {}
-        - name: provider-config
-          configMap:
-            name: garm-provider-k8s-config

apps/garm/namespace.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: garm
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: garm-runners

apps/garm/rbac.yaml

@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: garm
-  namespace: garm
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: garm-provider-k8s
-  namespace: garm-runners
-rules:
-  - apiGroups: [""]
-    resources: ["pods", "pods/log", "configmaps", "secrets", "events"]
-    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: garm-provider-k8s
-  namespace: garm-runners
-subjects:
-  - kind: ServiceAccount
-    name: garm
-    namespace: garm
-roleRef:
-  kind: Role
-  name: garm-provider-k8s
-  apiGroup: rbac.authorization.k8s.io

apps/gitea/release.yaml

@@ -73,7 +73,7 @@ spec:
           ISSUE_INDEXER_TYPE: bleve
           REPO_INDEXER_ENABLED: true
         webhook:
-          ALLOWED_HOST_LIST: garm.garm.svc.cluster.local
+          ALLOWED_HOST_LIST: woodpecker.lumpiasty.xyz
       admin:
         username: GiteaAdmin
         email: gi@tea.com
@@ -90,6 +90,11 @@ spec:
         # Requirement for sharing ip with other service
         externalTrafficPolicy: Cluster
         ipFamilyPolicy: RequireDualStack
+      http:
+        type: ClusterIP
+        # We need the service to be at port 80 specifically
+        # to work around bug of Actions Runner
+        port: 80
 
     ingress:
       enabled: true
@@ -97,7 +102,7 @@ spec:
       annotations:
         cert-manager.io/cluster-issuer: letsencrypt
         acme.cert-manager.io/http01-edit-in-place: "true"
-        nginx.ingress.kubernetes.io/proxy-body-size: "100m"
+        nginx.ingress.kubernetes.io/proxy-body-size: "1g"
       hosts:
         - host: gitea.lumpiasty.xyz
           paths:

apps/immich/release.yaml

@@ -18,7 +18,7 @@ spec:
   chart:
     spec:
       chart: immich
-      version: 1.1.1
+      version: 1.2.2
       sourceRef:
         kind: HelmRepository
         name: secustor

apps/kustomization.yaml

@@ -1,14 +1,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - crawl4ai
+  - crawl4ai-proxy
+  - authentik
   - gitea
   - renovate
-  - librechat
   - frigate
   - llama
   - immich
   - nas
-  - searxng
   - ispeak3
   - openwebui
-  - garm
+  - woodpecker

apps/librechat/release.yaml

@@ -1,120 +0,0 @@
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: dynomite567-charts
-  namespace: librechat
-spec:
-  interval: 24h
-  url: https://dynomite567.github.io/helm-charts/
----
-# apiVersion: helm.toolkit.fluxcd.io/v2
-# kind: HelmRelease
-# metadata:
-#   name: librechat
-#   namespace: librechat
-# spec:
-#   interval: 30m
-#   chart:
-#     spec:
-#       chart: librechat
-#       version: 1.9.1
-#       sourceRef:
-#         kind: HelmRepository
-#         name: dynomite567-charts
-#   values:
-#     global:
-#       librechat:
-#         existingSecretName: librechat
-#     librechat:
-#       configEnv:
-#         PLUGIN_MODELS: null
-#         ALLOW_REGISTRATION: "false"
-#         TRUST_PROXY: "1"
-#         DOMAIN_CLIENT: https://librechat.lumpiasty.xyz
-#         SEARCH: "true"
-#       existingSecretName: librechat
-#       configYamlContent: |
-#         version: 1.0.3
-
-#         endpoints:
-#           custom:
-#             - name: "Llama.cpp"
-#               apiKey: "llama"
-#               baseURL: "http://llama.llama.svc.cluster.local:11434/v1"
-#               models:
-#                 default: [
-#                   "DeepSeek-R1-0528-Qwen3-8B-GGUF",
-#                   "Qwen3-8B-GGUF",
-#                   "Qwen3-8B-GGUF-no-thinking",
-#                   "gemma3n-e4b",
-#                   "gemma3-12b",
-#                   "gemma3-12b-q2",
-#                   "gemma3-12b-novision",
-#                   "gemma3-4b",
-#                   "gemma3-4b-novision",
-#                   "Qwen3-4B-Thinking-2507",
-#                   "Qwen3-4B-Thinking-2507-long-ctx",
-#                   "Qwen2.5-VL-7B-Instruct-GGUF",
-#                   "Qwen2.5-VL-32B-Instruct-GGUF-IQ1_S",
-#                   "Qwen2.5-VL-32B-Instruct-GGUF-Q2_K_L",
-#                   "Qwen3-VL-2B-Instruct-GGUF",
-#                   "Qwen3-VL-2B-Instruct-GGUF-unslothish",
-#                   "Qwen3-VL-2B-Thinking-GGUF",
-#                   "Qwen3-VL-4B-Instruct-GGUF",
-#                   "Qwen3-VL-4B-Instruct-GGUF-unslothish",
-#                   "Qwen3-VL-4B-Thinking-GGUF",
-#                   "Qwen3-VL-8B-Instruct-GGUF",
-#                   "Qwen3-VL-8B-Instruct-GGUF-unslothish",
-#                   "Qwen3-VL-8B-Thinking-GGUF",
-#                   "Huihui-Qwen3-VL-8B-Instruct-abliterated-GGUF",
-#                   "Huihui-Qwen3-VL-8B-Thinking-abliterated-GGUF"
-#                 ]
-#               titleConvo: true
-#               titleModel: "gemma3-4b-novision"
-#               summarize: false
-#               summaryModel: "gemma3-4b-novision"
-#               forcePrompt: false
-#               modelDisplayLabel: "Llama.cpp"
-
-#               # ✨ IMPORTANT: let llama-swap/llama-server own all these
-#               dropParams:
-#                 - "temperature"
-#                 - "top_p"
-#                 - "top_k"
-#                 - "presence_penalty"
-#                 - "frequency_penalty"
-#                 - "stop"
-#                 - "max_tokens"
-#       imageVolume:
-#         enabled: true
-#         size: 10G
-#         accessModes: ReadWriteOnce
-#         storageClassName: mayastor-single-hdd
-#     ingress:
-#       enabled: true
-#       className: nginx-ingress
-#       annotations:
-#         cert-manager.io/cluster-issuer: letsencrypt
-#         nginx.ingress.kubernetes.io/proxy-body-size: "0"
-#         nginx.ingress.kubernetes.io/proxy-buffering: "false"
-#         nginx.ingress.kubernetes.io/proxy-read-timeout: 30m
-#       hosts:
-#         - host: librechat.lumpiasty.xyz
-#           paths:
-#             - path: /
-#               pathType: ImplementationSpecific
-#       tls:
-#         - hosts:
-#             - librechat.lumpiasty.xyz
-#           secretName: librechat-ingress
-
-#     mongodb:
-#       persistence:
-#         storageClass: mayastor-single-hdd
-
-#     meilisearch:
-#       persistence:
-#         storageClass: mayastor-single-hdd
-#       auth:
-#         existingMasterKeySecret: librechat

apps/llama/configs/config.yaml

@@ -4,22 +4,19 @@ logToStdout: "both" # proxy and upstream
 
 macros:
   base_args: "--no-warmup --port ${PORT}"
-  common_args: "--fit-target 1536 --fit-ctx 65536 --no-warmup --port ${PORT}"
+  common_args: "--fit-target 1536 --no-warmup --port ${PORT}"
+  ctx_128k: "--ctx-size 131072"
+  ctx_256k: "--ctx-size 262144"
   gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95"
-  qwen35_sampling: "--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0"
+  qwen35_sampling: "--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q8_0 -ctv q8_0"
-  qwen35_35b_args: "--temp 1.0 --min-p 0.00 --top-p 0.95 --top-k 20 -ctk q4_0 -ctv q4_0"
+  qwen35_35b_args: "--temp 1.0 --min-p 0.00 --top-p 0.95 --top-k 20 -ctk q8_0 -ctv q8_0"
   qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf"
   qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf"
+  glm47_flash_args: "--temp 0.7 --top-p 1.0 --min-p 0.01 --repeat-penalty 1.0"
+  gemma4_sampling: "--temp 1.0 --top-p 0.95 --top-k 64"
   thinking_on: "--chat-template-kwargs '{\"enable_thinking\": true}'"
   thinking_off: "--chat-template-kwargs '{\"enable_thinking\": false}'"
 
-peers:
-  openrouter:
-    proxy: https://openrouter.ai/api
-    apiKey: ${env.OPENROUTER_API_KEY}
-    models:
-      - z-ai/glm-5
-
 hooks:
   on_startup:
     preload:
@@ -38,6 +35,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
+        ${ctx_128k}
         ${gemma_sampling}
         ${common_args}
 
@@ -45,6 +43,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
+        ${ctx_128k}
         ${gemma_sampling}
         --no-mmproj
         ${common_args}
@@ -53,6 +52,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
+        ${ctx_128k}
         ${gemma_sampling}
         ${common_args}
 
@@ -60,6 +60,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
+        ${ctx_128k}
         ${gemma_sampling}
         --no-mmproj
         ${common_args}
@@ -75,13 +76,14 @@ models:
         --top-p 0.95
         --top-k 40
         --repeat-penalty 1.0
-        -ctk q4_0 -ctv q4_0
+        -ctk q8_0 -ctv q8_0
         ${common_args}
 
   "Qwen3.5-35B-A3B-GGUF:Q4_K_M":
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_35b_args}
         ${common_args}
 
@@ -89,6 +91,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_35b_args}
         ${common_args}
         ${thinking_off}
@@ -100,6 +103,7 @@ models:
       /app/llama-server
         -hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
         ${qwen35_35b_heretic_mmproj}
+        ${ctx_256k}
         ${qwen35_35b_args}
         ${common_args}
 
@@ -108,6 +112,7 @@ models:
       /app/llama-server
         -hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
         ${qwen35_35b_heretic_mmproj}
+        ${ctx_256k}
         ${qwen35_35b_args}
         ${common_args}
         ${thinking_off}
@@ -116,6 +121,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-0.8B-GGUF:Q4_K_XL
+        ${ctx_256k}
         ${qwen35_sampling}
         ${base_args}
         ${thinking_on}
@@ -133,6 +139,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-2B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -141,6 +148,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-2B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
@@ -149,6 +157,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
+        ${ctx_128k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -157,6 +166,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
+        ${ctx_128k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
@@ -166,6 +176,7 @@ models:
       /app/llama-server
         -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
         ${qwen35_4b_heretic_mmproj}
+        ${ctx_128k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -175,6 +186,7 @@ models:
       /app/llama-server
         -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
         ${qwen35_4b_heretic_mmproj}
+        ${ctx_128k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
@@ -183,6 +195,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-9B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -191,6 +204,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-9B-GGUF:Q4_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
@@ -199,6 +213,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-9B-GGUF:Q3_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -207,6 +222,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-9B-GGUF:Q3_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
@@ -215,6 +231,7 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-27B-GGUF:Q3_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_on}
@@ -223,6 +240,46 @@ models:
     cmd: |
       /app/llama-server
         -hf unsloth/Qwen3.5-27B-GGUF:Q3_K_M
+        ${ctx_256k}
         ${qwen35_sampling}
         ${common_args}
         ${thinking_off}
+
+  "GLM-4.7-Flash-GGUF:Q4_K_M":
+    cmd: |
+      /app/llama-server
+        -hf unsloth/GLM-4.7-Flash-GGUF:Q4_K_M
+        ${glm47_flash_args}
+        ${common_args}
+
+  "gemma-4-26B-A4B-it:UD-Q4_K_XL":
+    cmd: |
+      /app/llama-server
+        -hf unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL \
+        ${ctx_256k}
+        ${gemma4_sampling}
+        ${common_args}
+
+  "gemma-4-26B-A4B-it:UD-Q2_K_XL":
+    cmd: |
+      /app/llama-server
+        -hf unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q2_K_XL \
+        ${ctx_256k}
+        ${gemma4_sampling}
+        ${common_args}
+
+  "unsloth/gemma-4-E4B-it-GGUF:UD-Q4_K_XL":
+    cmd: |
+      /app/llama-server
+        -hf unsloth/gemma-4-E4B-it-GGUF:UD-Q4_K_XL \
+        ${ctx_128k}
+        ${gemma4_sampling}
+        ${common_args}
+
+  "unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL":
+    cmd: |
+      /app/llama-server
+        -hf unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL \
+        ${ctx_128k}
+        ${gemma4_sampling}
+        ${common_args}

apps/llama/deployment.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: llama-swap
-          image: ghcr.io/mostlygeek/llama-swap:v197-vulkan-b8248
+          image: ghcr.io/mostlygeek/llama-swap:v199-vulkan-b8667
           imagePullPolicy: IfNotPresent
           command:
             - /app/llama-swap
@@ -29,12 +29,6 @@ spec:
             - containerPort: 8080
               name: http
               protocol: TCP
-          env:
-            - name: OPENROUTER_API_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: llama-openrouter
-                  key: OPENROUTER_API_KEY
           volumeMounts:
             - name: models
               mountPath: /root/.cache

apps/llama/pvc-ssd.yaml

@@ -7,7 +7,7 @@ metadata:
   name: llama-models-lvmssd
   namespace: openebs
 spec:
-  capacity: 200Gi
+  capacity: "322122547200"
   ownerNodeID: anapistula-delrosalae
   shared: "yes"
   thinProvision: "no"
@@ -20,7 +20,7 @@ metadata:
   name: llama-models-lvmssd
 spec:
   capacity:
-    storage: 200Gi
+    storage: 300Gi
   accessModes:
     - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
@@ -41,6 +41,6 @@ spec:
     - ReadWriteOnce
   resources:
     requests:
-      storage: 200Gi
+      storage: 300Gi
   storageClassName: ssd-lvmpv
   volumeName: llama-models-lvmssd

apps/llama/secret.yaml

@@ -36,26 +36,3 @@ spec:
       excludeRaw: true
 
   vaultAuthRef: llama
----
-apiVersion: secrets.hashicorp.com/v1beta1
-kind: VaultStaticSecret
-metadata:
-  name: llama-openrouter
-  namespace: llama
-spec:
-  type: kv-v2
-
-  mount: secret
-  path: openrouter
-
-  destination:
-    create: true
-    name: llama-openrouter
-    type: Opaque
-    transformation:
-      excludeRaw: true
-      templates:
-        OPENROUTER_API_KEY:
-          text: '{{ get .Secrets "API_KEY" }}'
-
-  vaultAuthRef: llama

apps/openwebui/kustomization.yaml

@@ -4,5 +4,6 @@ resources:
   - namespace.yaml
   - pvc.yaml
   - pvc-pipelines.yaml
+  - secret.yaml
   - release.yaml
   - ingress.yaml

apps/openwebui/release.yaml

@@ -18,7 +18,7 @@ spec:
   chart:
     spec:
       chart: open-webui
-      version: 12.10.0
+      version: 13.0.1
       sourceRef:
         kind: HelmRepository
         name: open-webui
@@ -44,3 +44,33 @@ spec:
       persistence:
         enabled: true
         existingClaim: openwebui-pipelines-lvmhdd
+
+    terminals:
+      enabled: true
+
+    # SSO with Authentik
+    extraEnvVars:
+      - name: WEBUI_URL
+        value: "https://openwebui.lumpiasty.xyz"
+      - name: OAUTH_CLIENT_ID
+        valueFrom:
+          secretKeyRef:
+            name: openwebui-authentik
+            key: client_id
+      - name: OAUTH_CLIENT_SECRET
+        valueFrom:
+          secretKeyRef:
+            name: openwebui-authentik
+            key: client_secret
+      - name: OAUTH_PROVIDER_NAME
+        value: "authentik"
+      - name: OPENID_PROVIDER_URL
+        value: "https://authentik.lumpiasty.xyz/application/o/open-web-ui/.well-known/openid-configuration"
+      - name: OPENID_REDIRECT_URI
+        value: "https://openwebui.lumpiasty.xyz/oauth/oidc/callback"
+      - name: ENABLE_OAUTH_SIGNUP
+        value: "true"
+      - name: ENABLE_LOGIN_FORM
+        value: "false"
+      - name: OAUTH_MERGE_ACCOUNTS_BY_EMAIL
+        value: "true"

apps/openwebui/secret.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: openwebui-secret
+  namespace: openwebui
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: openwebui
+  namespace: openwebui
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: openwebui
+    serviceAccount: openwebui-secret
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: openwebui-authentik
+  namespace: openwebui
+spec:
+  type: kv-v2
+
+  mount: secret
+  path: authentik/openwebui
+
+  destination:
+    create: true
+    name: openwebui-authentik
+    type: Opaque
+    transformation:
+      excludeRaw: true
+      templates:
+        client_id:
+          text: '{{ get .Secrets "client_id" }}'
+        client_secret:
+          text: '{{ get .Secrets "client_secret" }}'
+
+  vaultAuthRef: openwebui

apps/renovate/cronjob.yaml

@@ -15,7 +15,7 @@ spec:
             - name: renovate
               # Update this to the latest available and then enable Renovate on
               # the manifest
-              image: renovate/renovate:43.64.6-full
+              image: renovate/renovate:43.104.8-full
               envFrom:
                 - secretRef:
                     name: renovate-gitea-token

apps/searxng/configs/settings.yml

@@ -1 +0,0 @@
-use_default_settings: true

apps/searxng/deployment.yaml

@@ -1,42 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: searxng
-  namespace: searxng
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: searxng
-  template:
-    metadata:
-      labels:
-        app: searxng
-    spec:
-      containers:
-        - name: searxng
-          image: searxng/searxng:2025.8.12-6b1516d
-          ports:
-            - containerPort: 8080
-          env:
-            - name: SEARXNG_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: searxng-secret
-                  key: SEARXNG_SECRET
-                  optional: false
-          volumeMounts:
-            - name: config-volume
-              mountPath: /etc/searxng/settings.yml
-              subPath: settings.yml
-              readOnly: true
-            - name: searxng-persistent-data
-              mountPath: /var/cache/searxng
-      volumes:
-        - name: config-volume
-          configMap:
-            name: searxng-config
-        - name: searxng-persistent-data
-          persistentVolumeClaim:
-            claimName: searxng-persistent-data-lvmhdd

apps/searxng/ingress.yaml

@@ -1,25 +0,0 @@
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  namespace: searxng
-  name: searxng
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt
-spec:
-  ingressClassName: nginx-ingress
-  rules:
-    - host: searxng.lumpiasty.xyz
-      http:
-        paths:
-          - backend:
-              service:
-                name: searxng
-                port:
-                  number: 8080
-            path: /
-            pathType: Prefix
-  tls:
-    - hosts:
-        - searxng.lumpiasty.xyz
-      secretName: searxng-ingress

apps/searxng/kustomization.yaml

@@ -1,13 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - namespace.yaml
-  - pvc.yaml
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml
-configMapGenerator:
-  - name: searxng-config
-    namespace: searxng
-    files:
-      - settings.yml=configs/settings.yml

apps/searxng/pvc.yaml

@@ -1,46 +0,0 @@
----
-apiVersion: local.openebs.io/v1alpha1
-kind: LVMVolume
-metadata:
-  labels:
-    kubernetes.io/nodename: anapistula-delrosalae
-  name: searxng-persistent-data-lvmhdd
-  namespace: openebs
-spec:
-  capacity: 1Gi
-  ownerNodeID: anapistula-delrosalae
-  shared: "yes"
-  thinProvision: "no"
-  vgPattern: ^openebs-hdd$
-  volGroup: openebs-hdd
----
-kind: PersistentVolume
-apiVersion: v1
-metadata:
-  name: searxng-persistent-data-lvmhdd
-spec:
-  capacity:
-    storage: 1Gi
-  accessModes:
-    - ReadWriteOnce
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: hdd-lvmpv
-  volumeMode: Filesystem
-  csi:
-    driver: local.csi.openebs.io
-    fsType: btrfs
-    volumeHandle: searxng-persistent-data-lvmhdd
----
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: searxng-persistent-data-lvmhdd
-  namespace: searxng
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
-  storageClassName: hdd-lvmpv
-  volumeName: searxng-persistent-data-lvmhdd

apps/searxng/service.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: searxng
-  namespace: searxng
-spec:
-  selector:
-    app: searxng
-  ports:
-    - protocol: TCP
-      port: 8080
-      targetPort: 8080
-  type: ClusterIP

apps/woodpecker/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - postgres-volume.yaml
+  - postgres-cluster.yaml
+  - release.yaml
+  - secret.yaml

apps/woodpecker/namespace.yaml

@@ -2,4 +2,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: librechat
+  name: woodpecker

apps/woodpecker/postgres-cluster.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: woodpecker-postgresql-cluster
+  namespace: woodpecker
+spec:
+  instances: 1
+
+  imageName: ghcr.io/cloudnative-pg/postgresql:17.4
+
+  bootstrap:
+    initdb:
+      database: woodpecker
+      owner: woodpecker
+
+  storage:
+    pvcTemplate:
+      storageClassName: ssd-lvmpv
+      resources:
+        requests:
+          storage: 10Gi
+      volumeName: woodpecker-postgresql-cluster-lvmssd

apps/woodpecker/postgres-volume.yaml

@@ -0,0 +1,33 @@
+apiVersion: local.openebs.io/v1alpha1
+kind: LVMVolume
+metadata:
+  labels:
+    kubernetes.io/nodename: anapistula-delrosalae
+  name: woodpecker-postgresql-cluster-lvmssd
+  namespace: openebs
+spec:
+  capacity: 10Gi
+  ownerNodeID: anapistula-delrosalae
+  shared: "yes"
+  thinProvision: "no"
+  vgPattern: ^openebs-ssd$
+  volGroup: openebs-ssd
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: woodpecker-postgresql-cluster-lvmssd
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ssd-lvmpv
+  volumeMode: Filesystem
+  csi:
+    driver: local.csi.openebs.io
+    fsType: btrfs
+    volumeHandle: woodpecker-postgresql-cluster-lvmssd
+---
+# PVC is dynamically created by the Postgres operator

apps/woodpecker/release.yaml

@@ -0,0 +1,115 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: woodpecker
+  namespace: woodpecker
+spec:
+  interval: 24h
+  url: https://woodpecker-ci.org/
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: woodpecker
+  namespace: woodpecker
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: woodpecker
+      version: 3.5.1
+      sourceRef:
+        kind: HelmRepository
+        name: woodpecker
+        namespace: woodpecker
+      interval: 12h
+  values:
+    server:
+      enabled: true
+      statefulSet:
+        replicaCount: 1
+      
+      persistentVolume:
+        enabled: false # Using Postgresql database
+
+      env:
+        WOODPECKER_HOST: "https://woodpecker.lumpiasty.xyz"
+        # Gitea integration
+        WOODPECKER_GITEA: "true"
+        WOODPECKER_GITEA_URL: "https://gitea.lumpiasty.xyz"
+        # PostgreSQL database configuration
+        WOODPECKER_DATABASE_DRIVER: postgres
+        # Password is loaded from woodpecker-postgresql-cluster-app secret (created by CNPG)
+        WOODPECKER_DATABASE_DATASOURCE:
+          valueFrom:
+            secretKeyRef:
+              name: woodpecker-postgresql-cluster-app
+              key: fqdn-uri
+        # Allow logging in from all accounts on Gitea
+        WOODPECKER_OPEN: "true"
+        # Make lumpiasty admin
+        WOODPECKER_ADMIN: GiteaAdmin
+
+      createAgentSecret: true
+
+      extraSecretNamesForEnvFrom:
+        - woodpecker-secrets
+
+      ingress:
+        enabled: true
+        ingressClassName: nginx-ingress
+        annotations:
+          cert-manager.io/cluster-issuer: letsencrypt
+          acme.cert-manager.io/http01-edit-in-place: "true"
+        hosts:
+          - host: woodpecker.lumpiasty.xyz
+            paths:
+              - path: /
+                backend:
+                  serviceName: woodpecker-server
+                  servicePort: 80
+        tls:
+          - hosts:
+              - woodpecker.lumpiasty.xyz
+            secretName: woodpecker-ingress
+
+      resources:
+        requests:
+          cpu: 100m
+          memory: 256Mi
+
+      service:
+        type: ClusterIP
+        port: 80
+
+    agent:
+      enabled: true
+      replicaCount: 2
+
+      env:
+        WOODPECKER_SERVER: "woodpecker-server:9000"
+        WOODPECKER_BACKEND: kubernetes
+        WOODPECKER_BACKEND_K8S_NAMESPACE: woodpecker
+        WOODPECKER_BACKEND_K8S_STORAGE_CLASS: ssd-lvmpv
+        WOODPECKER_BACKEND_K8S_VOLUME_SIZE: 10G
+        WOODPECKER_BACKEND_K8S_STORAGE_RWX: false
+        WOODPECKER_CONNECT_RETRY_COUNT: "5"
+
+      mapAgentSecret: true
+
+      extraSecretNamesForEnvFrom:
+        - woodpecker-secrets
+
+      persistence:
+        enabled: false
+
+      serviceAccount:
+        create: true
+        rbac:
+          create: true
+
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi

apps/woodpecker/secret.yaml

@@ -0,0 +1,62 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: woodpecker-secret
+  namespace: woodpecker
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: woodpecker
+  namespace: woodpecker
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: woodpecker
+    serviceAccount: woodpecker-secret
+---
+# Main woodpecker secrets from Vault
+# Requires vault kv put secret/woodpecker \
+#   WOODPECKER_AGENT_SECRET="$(openssl rand -hex 32)" \
+#   WOODPECKER_GITEA_CLIENT="<gitea-oauth-client>" \
+#   WOODPECKER_GITEA_SECRET="<gitea-oauth-secret>"
+# Note: Database password comes from CNPG secret (woodpecker-postgresql-cluster-app)
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: woodpecker-secrets
+  namespace: woodpecker
+spec:
+  type: kv-v2
+  mount: secret
+  path: woodpecker
+  destination:
+    create: true
+    name: woodpecker-secrets
+    type: Opaque
+    transformation:
+      excludeRaw: true
+  vaultAuthRef: woodpecker
+---
+# Container registry credentials for Kaniko
+# Requires vault kv put secret/container-registry \
+#   REGISTRY_USERNAME="<username>" \
+#   REGISTRY_PASSWORD="<token>"
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: container-registry
+  namespace: woodpecker
+spec:
+  type: kv-v2
+  mount: secret
+  path: container-registry
+  destination:
+    create: true
+    name: container-registry
+    type: Opaque
+    transformation:
+      excludeRaw: true
+  vaultAuthRef: woodpecker

cluster/apps.yaml

@@ -4,7 +4,7 @@ metadata:
   name: apps
   namespace: flux-system
 spec:
-  interval: 10m0s
+  interval: 24h
   sourceRef:
     kind: GitRepository
     name: flux-system

cluster/flux-system/gotk-sync.yaml

@@ -6,7 +6,7 @@ metadata:
   name: flux-system
   namespace: flux-system
 spec:
-  interval: 1m0s
+  interval: 24h
   ref:
     branch: fresh-start
   secretRef:
@@ -19,7 +19,7 @@ metadata:
   name: flux-system
   namespace: flux-system
 spec:
-  interval: 10m0s
+  interval: 24h
   path: ./cluster
   prune: true
   sourceRef:

cluster/infra.yaml

@@ -4,7 +4,7 @@ metadata:
   name: infra
   namespace: flux-system
 spec:
-  interval: 10m0s
+  interval: 24h
   sourceRef:
     kind: GitRepository
     name: flux-system

devenv.lock

@@ -3,10 +3,11 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1769881431,
+        "lastModified": 1775201809,
+        "narHash": "sha256-WmpoCegCQ6Q2ZyxqO05zlz/7XXjt/l2iut4Nk5Nt+W4=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "72d5e66e2dd5112766ef4c9565872b51094b542d",
+        "rev": "42a5505d4700e791732e48a38b4cca05a755f94b",
         "type": "github"
       },
       "original": {
@@ -16,27 +17,13 @@
         "type": "github"
       }
     },
-    "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1767039857,
-        "owner": "NixOS",
-        "repo": "flake-compat",
-        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
       },
       "locked": {
         "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
         "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
@@ -48,47 +35,6 @@
         "type": "github"
       }
     },
-    "git-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1769069492,
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "rev": "a1ef738813b15cf8ec759bdff5761b027e3e1d23",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "type": "github"
-      }
-    },
-    "gitignore": {
-      "inputs": {
-        "nixpkgs": [
-          "git-hooks",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1762808025,
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "type": "github"
-      }
-    },
     "krew2nix": {
       "inputs": {
         "flake-utils": "flake-utils",
@@ -99,10 +45,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1769904483,
+        "lastModified": 1775175041,
+        "narHash": "sha256-lYCPSMIV26VazREzl/TIpbWhBXJ+vJ0EJ+308TrX/6w=",
         "owner": "a1994sc",
         "repo": "krew2nix",
-        "rev": "17d6ad3375899bd3f7d4d298481536155f3ec13c",
+        "rev": "15c594042f1ba80ce97ab190a9c684a44c613590",
         "type": "github"
       },
       "original": {
@@ -113,10 +60,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1769461804,
+        "lastModified": 1775036866,
+        "narHash": "sha256-ZojAnPuCdy657PbTq5V0Y+AHKhZAIwSIT2cb8UgAz/U=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bfc1b8a4574108ceef22f02bafcf6611380c100d",
+        "rev": "6201e203d09599479a3b3450ed24fa81537ebc4e",
         "type": "github"
       },
       "original": {
@@ -129,17 +77,14 @@
     "root": {
       "inputs": {
         "devenv": "devenv",
-        "git-hooks": "git-hooks",
         "krew2nix": "krew2nix",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs"
-        "pre-commit-hooks": [
-          "git-hooks"
-        ]
       }
     },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
         "owner": "nix-systems",
         "repo": "default",
         "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
@@ -154,6 +99,7 @@
     "systems_2": {
       "locked": {
         "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
         "owner": "nix-systems",
         "repo": "default",
         "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
@@ -173,10 +119,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1769691507,
+        "lastModified": 1773297127,
+        "narHash": "sha256-6E/yhXP7Oy/NbXtf1ktzmU8SdVqJQ09HC/48ebEGBpk=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "28b19c5844cc6e2257801d43f2772a4b4c050a1b",
+        "rev": "71b125cd05fbfd78cab3e070b73544abe24c5016",
         "type": "github"
       },
       "original": {
@@ -188,4 +135,4 @@
   },
   "root": "root",
   "version": 7
-}
+}

devenv.nix

@@ -41,6 +41,18 @@ in
     openbao
     pv-migrate
     mermaid-cli
+    (
+      # Wrapping opencode to set the OPENCODE_ENABLE_EXA environment variable
+      runCommand "opencode" {
+        buildInputs = [ makeWrapper ];
+      } ''
+        mkdir -p $out/bin
+        makeWrapper ${pkgs.opencode}/bin/opencode $out/bin/opencode \
+          --set OPENCODE_ENABLE_EXA "1"
+        ''
+    )
+    tea
+    woodpecker-cli
   ];
 
   # Scripts

docs/assets/woodpecker.svg

@@ -0,0 +1,10 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="284.538" height="253.96">
+  <style>
+    @media (prefers-color-scheme: dark) {
+      path {
+        fill: white;
+      }
+    }
+  </style>
+  <path d="M162.51 33.188c-26.77.411-54.004 6.885-71.494 3.745-1.313-.232-2.124 1.338-1.171 2.265 14.749 14.003 20.335 28.16 36.718 30.065l.476.103c-7.567 7.799-14.028 18.018-18.571 31.171-4.89 14.106-6.268 29.421-7.89 47.105-2.445 26.332-5.173 56.152-20.038 93.54a246.489 246.489 0 0 0-13.27 45.946h22.652a221.202 221.202 0 0 1 11.249-37.786c16.049-40.374 19.073-73.257 21.505-99.693 1.493-16.255 2.806-30.309 6.796-41.853 11.647-33.527 39.408-40.889 61.056-36.693 21.004 4.067 41.673 20.502 40.592 44.016-.772 15.985-7.76 23.166-12.87 28.43-2.793 2.883-5.47 5.611-6.731 9.498-3.037 9.19.101 19.434 8.494 27.568 22.24 20.734 34.338 59.717 33.681 106.513h22.176c.592-52.935-13.951-97.839-40.503-122.626-2.097-2.021-2.69-3.604-3.191-3.347 1.222-1.544 3.217-3.346 4.633-4.813 29.382-21.79 77.813-1.892 107.054 9.653 7.58 2.985 11.274-4.338 4.067-8.623-25.097-14.84-76.54-54.016-105.368-79.718-4.029-3.54-6.796-7.8-11.455-11.738-15.547-27.439-41.84-33.127-68.597-32.728Zm35.238 60.27a15.161 15.161 0 0 0-2.008.232 15.161 15.161 0 0 0-1.506 29.434 15.154 15.154 0 0 0 9.473-28.79 15.161 15.161 0 0 0-5.959-.876zm-44.286 147.17a2.033 2.033 0 0 0-1.133.374c-1.08.772-1.93 3.05-.772 5.701 5.38 12.394 9.1 25.445 12.536 40.413h22.484c-5.676-16.629-16.307-34.055-27.851-43.978-2.008-1.737-3.913-2.574-5.251-2.51z" style="stroke-width:12.8704" transform="translate(-67.27 -33.169)"/>
+</svg>

infra/configs/lvmpv-ssd-sc.yaml

@@ -3,6 +3,8 @@ apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   name: ssd-lvmpv
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
 parameters:
   storage: "lvm"
   volgroup: "openebs-ssd"

infra/configs/openbao-k8s-se-role.yaml

@@ -0,0 +1,32 @@
+# Roles with needed access for OpenBao's Kubernetes secret engine
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-full-secrets-abilities
+rules:
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["serviceaccounts", "serviceaccounts/token"]
+  verbs: ["create", "update", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings", "clusterrolebindings"]
+  verbs: ["create", "update", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "clusterroles"]
+  verbs: ["bind", "escalate", "create", "update", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: openbao-token-creator-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: k8s-full-secrets-abilities
+subjects:
+- kind: ServiceAccount
+  name: openbao
+  namespace: openbao

infra/controllers/cert-manager-webhook-ovh.yaml

@@ -18,7 +18,7 @@ spec:
   chart:
     spec:
       chart: cert-manager-webhook-ovh
-      version: 0.9.4
+      version: 0.9.6
       sourceRef:
         kind: HelmRepository
         name: cert-manager-webhook-ovh

infra/controllers/cert-manager.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: cert-manager
-      version: v1.20.0
+      version: v1.20.1
       sourceRef:
         kind: HelmRepository
         name: cert-manager

infra/controllers/cilium.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: cilium
-      version: 1.19.1
+      version: 1.19.2
       sourceRef:
         kind: HelmRepository
         name: cilium

infra/controllers/cloudnative-pg.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: cloudnative-pg
-      version: 0.27.1
+      version: 0.28.0
       sourceRef:
         kind: HelmRepository
         name: cnpg

infra/controllers/k8up.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: k8up
-      version: 4.8.6
+      version: 4.9.0
       sourceRef:
         kind: HelmRepository
         name: k8up-io

infra/controllers/nginx-ingress.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: ingress-nginx
-      version: 4.15.0
+      version: 4.15.1
       sourceRef:
         kind: HelmRepository
         name: ingress-nginx

infra/controllers/openbao.yaml

@@ -23,7 +23,7 @@ spec:
   chart:
     spec:
       chart: openbao
-      version: 0.25.7
+      version: 0.26.2
       sourceRef:
         kind: HelmRepository
         name: openbao

infra/kustomization.yaml

@@ -25,3 +25,4 @@ resources:
 
   - configs/openbao-volume.yaml
   - controllers/openbao.yaml
+  - configs/openbao-k8s-se-role.yaml

utils/synchronize-vault.py

@@ -2,6 +2,7 @@
 
 import argparse
 import os
+import pathlib
 from typing import Any, cast
 
 import hvac
@@ -42,7 +43,7 @@ def synchronize_auth_kubernetes_config(client: hvac.Client):
 def synchronize_kubernetes_roles(client: hvac.Client):
     kubernetes = Kubernetes(client.adapter)
 
-    policy_dir = os.path.join(os.path.dirname(__file__), '../vault/kubernetes-roles/')
+    policy_dir = os.path.join(os.path.dirname(__file__), '../vault/kubernetes-auth-roles/')
 
     roles: dict[str, Any] = {} # pyright:ignore[reportExplicitAny]
     for filename in os.listdir(policy_dir):
@@ -67,6 +68,69 @@ def synchronize_kubernetes_roles(client: hvac.Client):
         # Using write data instead of kubernetes.create_role, we can pass raw yaml
         _ = client.write_data(f'/auth/kubernetes/role/{role_name}', data=role_content) # pyright:ignore[reportAny]
 
+def synchronize_approle_auth(client: hvac.Client):
+    if client.sys.list_auth_methods().get('approle/') is None:
+        print('Enabling AppRole auth method')
+        client.sys.enable_auth_method('approle', 'AppRole authorization for CI')
+
+    roles_dir = pathlib.Path(__file__).parent.joinpath('../vault/approles/')
+    roles: dict[str, Any] = {}
+
+    for filename in roles_dir.iterdir():
+        with filename.open('r') as f:
+            role = yaml.safe_load(f.read())
+            assert type(role) is dict
+            roles[filename.stem] = role
+
+    roles_on_vault: list[str] = []
+    roles_response = client.list("auth/approle/roles")
+    if roles_response is not None:
+        roles_on_vault = roles_response['data']['keys']
+
+    for role in roles_on_vault:
+        if role not in roles:
+            print(f'Deleting role: {role}')
+            client.delete(f'auth/approle/role/{role}')
+
+    for role_name, role_content in roles.items():
+        print(f'Updating role: {role_name}')
+        client.write_data(f'auth/approle/role/{role_name}', data=role_content)
+
+def synchronize_kubernetes_secretengine(client: hvac.Client):
+    # Ensure kubernetes secret engine is enabled
+    if client.sys.list_mounted_secrets_engines().get('kubernetes/') is None:
+        print('Enabling kubernetes secret engine')
+        client.sys.enable_secrets_engine('kubernetes', 'kubernetes', 'Cluster access')
+
+    # Write empty config (all defaults, working on the same cluster)
+    client.write('kubernetes/config', None)
+
+    policy_dir = pathlib.Path(__file__).parent.joinpath('../vault/kubernetes-se-roles/')
+    roles: dict[str, Any] = {}
+
+    for filename in policy_dir.iterdir():
+        with filename.open('r') as f:
+            role = yaml.safe_load(f.read())
+            assert type(role) is dict
+            # generated_role_rules must be json or yaml formatted string, convert it
+            if 'generated_role_rules' in role and type(role['generated_role_rules']) is not str:
+                role['generated_role_rules'] = yaml.safe_dump(role['generated_role_rules'])
+            roles[filename.stem] = role
+
+    roles_on_vault: list[str] = []
+    roles_response = client.list("kubernetes/roles")
+    if roles_response is not None:
+        roles_on_vault = roles_response['data']['keys']
+
+    for role in roles_on_vault:
+        if role not in roles:
+            print(f'Deleting role: {role}')
+            client.delete(f'kubernetes/roles/{role}')
+
+    for role_name, role_content in roles.items():
+        print(f'Updating role: {role_name}')
+        client.write_data(f'kubernetes/roles/{role_name}', data=role_content)
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         prog="synchronizeVault",
@@ -82,5 +146,11 @@ if __name__ == '__main__':
     print('Synchronizing kubernetes config')
     synchronize_auth_kubernetes_config(client)
 
-    print('Synchronizing kubernetes roles')
+    print('Synchronizing kubernetes auth roles')
     synchronize_kubernetes_roles(client)
+
+    print('Synchronizing AppRole auth method')
+    synchronize_approle_auth(client)
+
+    print('Synchronizing kubernetes secret engine')
+    synchronize_kubernetes_secretengine(client)

vault/approles/ci-flux-reconcile.yaml

@@ -0,0 +1,4 @@
+token_ttl: 20m
+token_max_ttl: 20m
+policies:
+  - flux-reconcile

vault/kubernetes-auth-roles/authentik.yaml

@@ -0,0 +1,6 @@
+bound_service_account_names:
+  - authentik-secret
+bound_service_account_namespaces:
+  - authentik
+token_policies:
+  - authentik

vault/kubernetes-auth-roles/crawl4ai.yaml

@@ -1,6 +1,6 @@
 bound_service_account_names:
-  - garm
+  - crawl4ai-secret
 bound_service_account_namespaces:
-  - garm
+  - crawl4ai
 token_policies:
-  - garm
+  - crawl4ai

vault/kubernetes-auth-roles/openwebui.yaml

@@ -0,0 +1,6 @@
+bound_service_account_names:
+  - openwebui-secret
+bound_service_account_namespaces:
+  - openwebui
+token_policies:
+  - openwebui

vault/kubernetes-auth-roles/woodpecker.yaml

@@ -0,0 +1,6 @@
+bound_service_account_names:
+  - woodpecker-secret
+bound_service_account_namespaces:
+  - woodpecker
+token_policies:
+  - woodpecker

vault/kubernetes-se-roles/flux-reconcile.yaml

@@ -0,0 +1,6 @@
+allowed_kubernetes_namespaces: flux-system
+generated_role_rules:
+  rules:
+    - apiGroups: ["source.toolkit.fluxcd.io"]
+      resources: ["gitrepositories"]
+      verbs: ["get", "patch", "watch"]

vault/policy/authentik.hcl

@@ -0,0 +1,3 @@
+path "secret/data/authentik" {
+    capabilities = ["read"]
+}

vault/policy/crawl4ai.hcl

@@ -0,0 +1,3 @@
+path "secret/data/crawl4ai" {
+    capabilities = ["read"]
+}

vault/policy/flux-reconcile.hcl

@@ -0,0 +1,3 @@
+path "kubernetes/creds/flux-reconcile" {
+    capabilities = ["update"]
+}

vault/policy/garm.hcl

@@ -1,7 +0,0 @@
-path "secret/data/garm" {
-    capabilities = ["read"]
-}
-
-path "secret/data/backblaze" {
-    capabilities = ["read"]
-}

vault/policy/openwebui.hcl

@@ -0,0 +1,3 @@
+path "secret/data/authentik/openwebui" {
+    capabilities = ["read"]
+}

vault/policy/woodpecker.hcl

@@ -0,0 +1,7 @@
+path "secret/data/woodpecker" {
+    capabilities = ["read"]
+}
+
+path "secret/data/container-registry" {
+    capabilities = ["read"]
+}