Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Renovate
|
6bfe87e7a3 | ||
 Lumpiasty
|
9dfa780354
|
||
|
Lumpiasty
|
b1c616a20f
|
||
|
Lumpiasty
|
fa32fdfd28
|
||
|
Lumpiasty
|
1b66a8c230
|
@@ -21,7 +21,7 @@ steps:
|
||||
- bao kv get -mount secret -field RENOVATE_TOKEN renovate > /woodpecker/renovate_token
|
||||
- bao kv get -mount secret -field GITHUB_COM_TOKEN renovate > /woodpecker/github_com_token
|
||||
- name: Run Renovate
|
||||
image: renovate/renovate:43.207.4
|
||||
image: renovate/renovate:43.209.1
|
||||
environment:
|
||||
RENOVATE_AUTODISCOVER: "true"
|
||||
RENOVATE_ENDPOINT: https://gitea.lumpiasty.xyz/api/v1
|
||||
|
||||
@@ -15,6 +15,7 @@ gen-talos-config:
|
||||
--config-patch @talos/patches/ollama.patch \
|
||||
--config-patch @talos/patches/llama.patch \
|
||||
--config-patch @talos/patches/frigate.patch \
|
||||
--config-patch @talos/patches/woodpecker.patch \
|
||||
--config-patch @talos/patches/anapistula-delrosalae.patch \
|
||||
--output-types controlplane -o talos/generated/anapistula-delrosalae.yaml \
|
||||
homelab https://kube-api.homelab.lumpiasty.xyz:6443
|
||||
|
||||
@@ -248,6 +248,8 @@ flowchart TD
|
||||
| <img src="docs/assets/llama-cpp.svg" alt="LLaMA.cpp" height="50" width="50"> | LLaMA.cpp | LLM inference server running local models with GPU acceleration |
|
||||
| <img src="docs/assets/llama-swap.svg" alt="llama-swap" height="50" width="50"> | llama-swap | Model swapping for LLaMA.cpp |
|
||||
| <img src="docs/assets/meridian.svg" alt="meridian" height="50" width="50"> | Meridian | Proxy that bridges Anthropic's official SDK to third-party tools |
|
||||
| | whisper.cpp | High-performance Whisper Automatic Speech Recognition inference server |
|
||||
| | Kokoro-FastAPI | Kokoro-82M text-to-speech inference server |
|
||||
|
||||
### Applications/Services
|
||||
|
||||
@@ -258,6 +260,7 @@ flowchart TD
|
||||
| <img src="docs/assets/teamspeak.svg" alt="iSpeak3" height="50" width="50"> | iSpeak3.pl | [ts3server://ispeak3.pl](ts3server://ispeak3.pl) | Public TeamSpeak 3 voice communication server |
|
||||
| <img src="docs/assets/immich.svg" alt="Immich" height="50" width="50"> | Immich | https://immich.lumpiasty.xyz/ | Self-hosted photo and video backup and streaming service |
|
||||
| <img src="docs/assets/frigate.svg" alt="Frigate" height="50" width="50"> | Frigate | https://frigate.lumpiasty.xyz/ | NVR for camera system with AI object detection and classification |
|
||||
| <img src="docs/assets/kaneo.svg" alt="Kaneo" height="50" width="50"> | Kaneo | https://kaneo.lumpiasty.xyz | Project management software |
|
||||
|
||||
|
||||
## Development
|
||||
|
||||
@@ -39,7 +39,6 @@ Secret layout expected in OpenBao (KVv2, mount `secret`):
|
||||
|---|---|
|
||||
| `routeros_api` | `username`, `password` |
|
||||
| `wan_pppoe` | `username`, `password` |
|
||||
| `router_tailscale` | `container_password` |
|
||||
|
||||
## OpenWrt dlink AP
|
||||
|
||||
|
||||
@@ -39,15 +39,10 @@
|
||||
engine_mount_point=openbao_kv_mount
|
||||
).secret[openbao_fields.wan_pppoe.password_key]
|
||||
}}
|
||||
routeros_tailscale_container_password: >-
|
||||
{{
|
||||
lookup(
|
||||
'community.hashi_vault.vault_kv2_get',
|
||||
openbao_fields.routeros_tailscale_container.path,
|
||||
engine_mount_point=openbao_kv_mount
|
||||
).secret[openbao_fields.routeros_tailscale_container.container_password_key]
|
||||
}}
|
||||
|
||||
no_log: true
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
module_defaults:
|
||||
group/community.routeros.api:
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
community.routeros.api_modify:
|
||||
path: ip address
|
||||
data:
|
||||
- address: 172.17.0.1/16
|
||||
interface: dockers
|
||||
network: 172.17.0.0
|
||||
- address: 172.20.0.1/24
|
||||
interface: containers
|
||||
network: 172.20.0.0
|
||||
- address: 192.168.4.1/24
|
||||
interface: lo
|
||||
network: 192.168.4.0
|
||||
@@ -44,7 +44,7 @@
|
||||
from-pool: pool1
|
||||
interface: vlan2
|
||||
- address: 2001:470:61a3:500:ffff:ffff:ffff:ffff/64
|
||||
interface: dockers
|
||||
interface: containers
|
||||
- address: 2001:470:61a3:100::1/64
|
||||
advertise: false
|
||||
interface: vlan4
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
data:
|
||||
- name: bridge1
|
||||
vlan-filtering: true
|
||||
- name: dockers
|
||||
- name: containers
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
@@ -62,8 +62,8 @@
|
||||
community.routeros.api_modify:
|
||||
path: interface bridge port
|
||||
data:
|
||||
- bridge: dockers
|
||||
interface: veth1
|
||||
- bridge: containers
|
||||
interface: veth-tailscale
|
||||
comment: Tailscale container interface
|
||||
- bridge: bridge1
|
||||
interface: ether1
|
||||
@@ -190,6 +190,18 @@
|
||||
cache-size: 20480
|
||||
servers: 1.1.1.1,1.0.0.1,2606:4700:4700::1111,2606:4700:4700::1001
|
||||
|
||||
- name: Configure DNS static entries
|
||||
community.routeros.api_modify:
|
||||
path: ip dns static
|
||||
data:
|
||||
- name: ts.net
|
||||
type: FWD
|
||||
forward-to: 100.100.100.100
|
||||
match-subdomain: true
|
||||
comment: Tailscale MagicDNS
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
- name: Configure NAT-PMP global settings
|
||||
community.routeros.api_find_and_modify:
|
||||
ignore_dynamic: false
|
||||
@@ -202,7 +214,7 @@
|
||||
community.routeros.api_modify:
|
||||
path: ip nat-pmp interfaces
|
||||
data:
|
||||
- interface: dockers
|
||||
- interface: containers
|
||||
type: internal
|
||||
- interface: pppoe-gpon
|
||||
type: external
|
||||
@@ -223,7 +235,7 @@
|
||||
community.routeros.api_modify:
|
||||
path: ip upnp interfaces
|
||||
data:
|
||||
- interface: dockers
|
||||
- interface: containers
|
||||
type: internal
|
||||
- interface: pppoe-gpon
|
||||
type: external
|
||||
|
||||
@@ -5,28 +5,12 @@
|
||||
path: container config
|
||||
find: {}
|
||||
values:
|
||||
registry-url: https://ghcr.io
|
||||
tmpdir: /tmp1/pull
|
||||
tmpdir: tmp
|
||||
|
||||
- name: Configure container env lists
|
||||
community.routeros.api_modify:
|
||||
path: container envs
|
||||
data:
|
||||
- key: ADVERTISE_ROUTES
|
||||
list: tailscale
|
||||
value: 192.168.0.0/24,192.168.1.0/24,192.168.4.1/32,192.168.100.1/32,192.168.255.0/24,10.42.0.0/16,10.43.0.0/16,10.44.0.0/16,2001:470:61a3::/48
|
||||
- key: CONTAINER_GATEWAY
|
||||
list: tailscale
|
||||
value: 172.17.0.1
|
||||
- key: PASSWORD
|
||||
list: tailscale
|
||||
value: "{{ routeros_tailscale_container_password }}"
|
||||
- key: TAILSCALE_ARGS
|
||||
list: tailscale
|
||||
value: --accept-routes --advertise-exit-node --snat-subnet-routes=false
|
||||
- key: UPDATE_TAILSCALE
|
||||
list: tailscale
|
||||
value: y
|
||||
data: []
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
@@ -35,11 +19,8 @@
|
||||
path: container mounts
|
||||
data:
|
||||
- dst: /var/lib/tailscale
|
||||
list: tailscale
|
||||
src: /usb1/tailscale
|
||||
- dst: /root
|
||||
list: tailscale-root
|
||||
src: /tmp1/tailscale-root
|
||||
list: tailscale_state
|
||||
src: tailscale/state
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
@@ -48,16 +29,12 @@
|
||||
path: container
|
||||
data:
|
||||
- dns: 172.17.0.1
|
||||
envlists: tailscale
|
||||
hostname: mikrotik
|
||||
interface: veth1
|
||||
layer-dir: ""
|
||||
mountlists: tailscale
|
||||
name: tailscale-mikrotik:latest
|
||||
remote-image: fluent-networks/tailscale-mikrotik:latest
|
||||
root-dir: /usb1/containers/tailscale
|
||||
interface: veth-tailscale
|
||||
logging: true
|
||||
mountlists: tailscale_state
|
||||
name: tailscale
|
||||
remote-image: gitea.lumpiasty.xyz/lumpiasty/mikrotik-tailscale:stable
|
||||
root-dir: tailscale/root
|
||||
start-on-boot: true
|
||||
tmpfs: /tmp:67108864:01777
|
||||
workdir: /
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
@@ -65,8 +65,8 @@
|
||||
out-interface-list: wan
|
||||
- action: accept
|
||||
chain: forward
|
||||
comment: Allow from dockers to everywhere
|
||||
in-interface: dockers
|
||||
comment: Allow from containers to everywhere
|
||||
in-interface: containers
|
||||
- action: jump
|
||||
chain: forward
|
||||
comment: Allow port forwards
|
||||
@@ -137,14 +137,14 @@
|
||||
protocol: tcp
|
||||
- action: accept
|
||||
chain: input
|
||||
comment: Allow DNS from dockers
|
||||
comment: Allow DNS from containers
|
||||
dst-port: 53
|
||||
in-interface: dockers
|
||||
in-interface: containers
|
||||
protocol: udp
|
||||
- action: accept
|
||||
chain: input
|
||||
dst-port: 53
|
||||
in-interface: dockers
|
||||
in-interface: containers
|
||||
protocol: tcp
|
||||
- action: accept
|
||||
chain: input
|
||||
@@ -188,9 +188,9 @@
|
||||
protocol: udp
|
||||
- action: accept
|
||||
chain: input
|
||||
comment: NAT-PMP from dockers (for tailscale)
|
||||
comment: NAT-PMP from containers (for tailscale)
|
||||
dst-port: 5351
|
||||
in-interface: dockers
|
||||
in-interface: containers
|
||||
protocol: udp
|
||||
- action: reject
|
||||
chain: input
|
||||
@@ -229,8 +229,8 @@
|
||||
- action: accept
|
||||
chain: allow-ports
|
||||
comment: Allow anything udp to Tailscale
|
||||
dst-address: 172.17.0.2
|
||||
out-interface: dockers
|
||||
dst-address: 172.20.0.2
|
||||
out-interface: containers
|
||||
protocol: udp
|
||||
- action: accept
|
||||
chain: allow-ports
|
||||
@@ -419,14 +419,14 @@
|
||||
out-interface-list: wan
|
||||
- action: accept
|
||||
chain: forward
|
||||
comment: Allow from dockers to everywhere
|
||||
in-interface: dockers
|
||||
comment: Allow from containers to everywhere
|
||||
in-interface: containers
|
||||
- action: accept
|
||||
chain: forward
|
||||
comment: Allow from internet to dockers
|
||||
comment: Allow from internet to containers
|
||||
dst-address: 2001:470:61a3:500::/64
|
||||
in-interface-list: wan
|
||||
out-interface: dockers
|
||||
out-interface: containers
|
||||
- action: accept
|
||||
chain: forward
|
||||
comment: Allow tcp transmission port to LAN
|
||||
@@ -485,14 +485,14 @@
|
||||
protocol: tcp
|
||||
- action: accept
|
||||
chain: input
|
||||
comment: Allow DNS from dockers
|
||||
comment: Allow DNS from containers
|
||||
dst-port: 53
|
||||
in-interface: dockers
|
||||
in-interface: containers
|
||||
protocol: udp
|
||||
- action: accept
|
||||
chain: input
|
||||
dst-port: 53
|
||||
in-interface: dockers
|
||||
in-interface: containers
|
||||
protocol: tcp
|
||||
- action: accept
|
||||
chain: input
|
||||
|
||||
@@ -39,19 +39,43 @@
|
||||
loop_control:
|
||||
label: "{{ item.default_name }}"
|
||||
|
||||
- name: Configure temporary disk for containers
|
||||
community.routeros.api_modify:
|
||||
# community.routeros.api_modify can't remove hardware disks
|
||||
# but it tries to do so with handle_absent_entries: remove
|
||||
# Working around by manually deleting other ones
|
||||
|
||||
- name: Read current disk entries
|
||||
community.routeros.api_info:
|
||||
path: disk
|
||||
data:
|
||||
- slot: tmp1
|
||||
type: tmpfs
|
||||
# This is not ideal, there's no unique identifier for usb disk,
|
||||
# after reinstall it might be assigned to another slot
|
||||
# Just adding disk with slot usb1 and not specifying anything else
|
||||
# so ansible doesn't touch it
|
||||
- slot: usb1
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
register: routeros_disks
|
||||
check_mode: false
|
||||
|
||||
- name: Remove stale software-defined disk entries
|
||||
community.routeros.api:
|
||||
path: disk
|
||||
remove: "{{ item['.id'] }}"
|
||||
loop: >-
|
||||
{{
|
||||
routeros_disks.result
|
||||
| rejectattr('type', 'in', ['hardware', 'partition'])
|
||||
| rejectattr('slot', 'equalto', 'tmp')
|
||||
}}
|
||||
loop_control:
|
||||
label: "{{ item.slot }}"
|
||||
|
||||
- name: Create temporary disk for containers if absent
|
||||
community.routeros.api:
|
||||
path: disk
|
||||
add: "slot=tmp type=tmpfs"
|
||||
when: routeros_disks.result | selectattr('slot', 'equalto', 'tmp') | list | length == 0
|
||||
|
||||
- name: Configure temporary disk for containers
|
||||
community.routeros.api_find_and_modify:
|
||||
ignore_dynamic: false
|
||||
path: disk
|
||||
find:
|
||||
slot: tmp
|
||||
values:
|
||||
type: tmpfs
|
||||
|
||||
- name: Configure switch settings
|
||||
community.routeros.api_find_and_modify:
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
- name: Preflight checks
|
||||
ansible.builtin.import_tasks: preflight.yml
|
||||
|
||||
- name: Base network configuration
|
||||
ansible.builtin.import_tasks: base.yml
|
||||
|
||||
- name: WAN and tunnel interfaces
|
||||
ansible.builtin.import_tasks: wan.yml
|
||||
|
||||
- name: Base network configuration
|
||||
ansible.builtin.import_tasks: base.yml
|
||||
|
||||
- name: Hardware and platform tuning
|
||||
ansible.builtin.import_tasks: hardware.yml
|
||||
|
||||
|
||||
@@ -32,15 +32,4 @@
|
||||
fail_msg: "RouterOS device-mode does not report container as enabled. Payload: {{ routeros_device_mode | to_nice_json }}"
|
||||
success_msg: "RouterOS device-mode confirms container=yes"
|
||||
|
||||
- name: Read configured disks
|
||||
community.routeros.api_info:
|
||||
path: disk
|
||||
register: routeros_disks
|
||||
check_mode: false
|
||||
|
||||
- name: Assert usb1 disk is present
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- (routeros_disks.result | selectattr('slot', 'equalto', 'usb1') | list | length) > 0
|
||||
fail_msg: "Required disk slot usb1 is not present on router."
|
||||
success_msg: "Required disk usb1 is present"
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
disabled: false
|
||||
distance: 1
|
||||
dst-address: 100.64.0.0/10
|
||||
gateway: 172.17.0.2
|
||||
gateway: 172.20.0.2
|
||||
routing-table: main
|
||||
scope: 30
|
||||
suppress-hw-offload: false
|
||||
|
||||
@@ -19,6 +19,101 @@
|
||||
handle_absent_entries: ignore
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
# The RouterOS API can neither store multi-line script source (newlines
|
||||
# collapse into one line) nor evaluate the [/file/get ...] expression itself.
|
||||
# So we fetch the update logic as a .rsc file onto the router's flash, then run
|
||||
# a single-line bootstrap script (which the API CAN store) whose body RouterOS
|
||||
# evaluates natively: it builds the real, browsable, multi-line named script
|
||||
# from the file via [/file get ... contents]. The scheduler then runs that
|
||||
# named script by name (the upstream-intended design). The update logic stays
|
||||
# out of this repo entirely.
|
||||
- name: Download tailscale auto-update script to router
|
||||
community.routeros.api:
|
||||
path: tool
|
||||
cmd: >-
|
||||
fetch
|
||||
url=https://gitea.lumpiasty.xyz/Lumpiasty/mikrotik-tailscale/raw/branch/main/routeros/update-tailscale.rsc
|
||||
dst-path=update-tailscale.rsc
|
||||
mode=https
|
||||
changed_when: true
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Build the named auto-update script from the fetched file
|
||||
community.routeros.api:
|
||||
path: system script
|
||||
cmd: >-
|
||||
add name=update-tailscale-bootstrap
|
||||
source=":do { /system script remove update-tailscale } on-error={};
|
||||
/system script add name=update-tailscale
|
||||
comment=\"Check for mikrotik-tailscale image updates\"
|
||||
source=[/file get update-tailscale.rsc contents]"
|
||||
changed_when: true
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Find bootstrap script id
|
||||
community.routeros.api:
|
||||
path: system script
|
||||
extended_query:
|
||||
attributes: [.id, name]
|
||||
where:
|
||||
- attribute: name
|
||||
is: "=="
|
||||
value: update-tailscale-bootstrap
|
||||
register: routeros_bootstrap
|
||||
changed_when: false
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Run bootstrap to create the named auto-update script
|
||||
community.routeros.api:
|
||||
path: system script
|
||||
cmd: "run .id={{ routeros_bootstrap.msg[0]['.id'] }}"
|
||||
register: routeros_bootstrap_run
|
||||
failed_when:
|
||||
- routeros_bootstrap_run is failed
|
||||
- "'interrupted' not in (routeros_bootstrap_run.msg | string)"
|
||||
changed_when: true
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Verify named auto-update script exists
|
||||
community.routeros.api:
|
||||
path: system script
|
||||
extended_query:
|
||||
attributes: [.id, name]
|
||||
where:
|
||||
- attribute: name
|
||||
is: "=="
|
||||
value: update-tailscale
|
||||
register: routeros_named_script
|
||||
failed_when: (routeros_named_script.msg | length) == 0
|
||||
changed_when: false
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Remove bootstrap script
|
||||
community.routeros.api:
|
||||
path: system script
|
||||
remove: "{{ routeros_bootstrap.msg[0]['.id'] }}"
|
||||
changed_when: true
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Configure tailscale auto-update scheduler
|
||||
community.routeros.api_modify:
|
||||
path: system scheduler
|
||||
data:
|
||||
- name: update-tailscale
|
||||
interval: 1d
|
||||
on-event: /system script run update-tailscale
|
||||
comment: Check for mikrotik-tailscale image updates
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
tags:
|
||||
- tailscale-script
|
||||
|
||||
- name: Configure service ports and service enablement
|
||||
community.routeros.api_find_and_modify:
|
||||
ignore_dynamic: false
|
||||
|
||||
@@ -29,13 +29,13 @@
|
||||
community.routeros.api_modify:
|
||||
path: interface veth
|
||||
data:
|
||||
- address: 172.17.0.2/16,2001:470:61a3:500::1/64
|
||||
- address: 172.20.0.2/24,2001:470:61a3:500::1/64
|
||||
container-mac-address: 7E:7E:A1:B1:2A:7C
|
||||
dhcp: false
|
||||
gateway: 172.17.0.1
|
||||
gateway: 172.20.0.1
|
||||
gateway6: 2001:470:61a3:500:ffff:ffff:ffff:ffff
|
||||
mac-address: 7E:7E:A1:B1:2A:7B
|
||||
name: veth1
|
||||
name: veth-tailscale
|
||||
comment: Tailscale container
|
||||
handle_absent_entries: remove
|
||||
handle_entries_content: remove_as_much_as_possible
|
||||
|
||||
@@ -14,6 +14,4 @@ openbao_fields:
|
||||
path: wan_pppoe
|
||||
username_key: username
|
||||
password_key: password
|
||||
routeros_tailscale_container:
|
||||
path: router_tailscale
|
||||
container_password_key: container_password
|
||||
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
# App deployment guidelines
|
||||
|
||||
This document summarizes current guidelines, requirements, common patterns, and standards that newly deployed apps should meet.
|
||||
|
||||
## Structure
|
||||
|
||||
Each app on cluster should be contained in its own kustomization living in subdirectory under [apps](/apps) and imported from main [apps kustomization](/apps/kustomization.yaml). Apps that provide infrastructural services belong to [infra](/infra). Few examples:
|
||||
|
||||
- **Open WebUI**: Web app, belongs in [apps/openwebui](/apps/openwebui/) together with its direct and unique dependencies eg. database
|
||||
- **llama-swap** (llama.cpp + whisper + stablediffusion): Inference server, service used by other deployments on cluster but does not manages cluster, belongs in [apps/llama](/apps/llama/)
|
||||
- **kokoro**: Text to speech inference server, also service used by other deployments, I consider it closely related to llama-swap, so due to arbitrary decision, keeping it together with llama-swap under [apps/llama](/apps/llama/)
|
||||
- **crawl4ai**: Web scraper, another service used only by other apps, belongs in [apps/crawl4ai](/apps/crawl4ai/)
|
||||
- **Gitea**: Code forge, despite being essential for overall architecture (holding cluster's code) is not a core cluster software, belongs in [apps/gitea](/apps/gitea/)
|
||||
- **Woodpecker**: Continous Integration system, belongs in [apps/woodpecker](/apps/woodpecker/)
|
||||
- **Cilium**: Kubernetes CNI, core cluster functionality, belongs in [infra/controllers/cilium.yaml](/infra/controllers/cilium.yaml)
|
||||
- **Nginx Ingress Controller**: Provides ingress kubernetes functionality, belongs in [infra/controllers/nginx-ingress.yaml](/infra/controllers/nginx-ingress.yaml)
|
||||
- **CloudNativePG**: Kubernetes PostgreSQL operator, belongs in [infra/controllers/cloudnative-pg.yaml](/infra/controllers/cloudnative-pg.yaml)
|
||||
- **OpenBao** Secret storage and Kubernetes operator, belongs in [infra/controllers/openbao.yaml](/infra/controllers/openbao.yaml)
|
||||
|
||||
Kustomizations are reconciled on `git push` by flux running on cluster, triggered by [Woodpecker job](/.woodpecker/flux-reconcile-source.yaml). App Kustomization should import all resources related to app in `kustomization.yaml`:
|
||||
|
||||
```yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- pvc.yaml
|
||||
- release.yaml
|
||||
```
|
||||
|
||||
## Namespace
|
||||
|
||||
Each app kustomization should have its own kubernetes namespace to contain all resources related to app in `namespace.yaml`:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: immich
|
||||
```
|
||||
|
||||
## Helm charts
|
||||
|
||||
If app is distributed via Helm chart, you can deploy it using flux HelmRepository and HelmRelease resources like in following example:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: HelmRepository
|
||||
metadata:
|
||||
name: secustor
|
||||
namespace: immich
|
||||
spec:
|
||||
interval: 24h
|
||||
url: https://secustor.dev/helm-charts
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: immich
|
||||
namespace: immich
|
||||
spec:
|
||||
interval: 30m
|
||||
chart:
|
||||
spec:
|
||||
chart: immich
|
||||
version: 1.2.6
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: secustor
|
||||
values:
|
||||
<values>
|
||||
```
|
||||
|
||||
If the app does not have a helm repository, but helm chart is available in git repository directly in repository, you can make use of it using GitRepository flux source:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: GitRepository
|
||||
metadata:
|
||||
name: kaneo
|
||||
namespace: kaneo
|
||||
spec:
|
||||
interval: 24h
|
||||
url: https://github.com/usekaneo/kaneo.git
|
||||
ref:
|
||||
tag: v2.7.5
|
||||
ignore: |
|
||||
# exclude all
|
||||
/*
|
||||
# include charts directory
|
||||
!/charts/
|
||||
```
|
||||
|
||||
You can use third-party helm charts to deploy applications, consider this possibility if:
|
||||
|
||||
- There is no official helm chart for the application
|
||||
- The official helm chart is unmaintained
|
||||
- The official helm chart is using glaring bad practices
|
||||
- The official helm chart is missing configuration options for what we need
|
||||
|
||||
When deciding which helm chart to use, watch out for following things in particular:
|
||||
|
||||
- Development activity, stability, maturity
|
||||
- Whether the app deployed by chart is up to date - automated updates are large bonus
|
||||
- Unresolved / breaking issues
|
||||
- Configurability, can we configure things we need, disable undesired features
|
||||
|
||||
When configuring Helm chart, keep in mind:
|
||||
- Do not use bundled PVCs, bring our own one or at least configure chart to bind it to manually created `PersistentVolume` according to [Data / PVCs pattern](#data--pvcs-pattern)
|
||||
- Do not use bundled Postgres database unless the chart is using CloudNativePG's Cluster resource, bring our own one using [Postgres operator](#postgres-operator)
|
||||
- do not
|
||||
|
||||
## Bare Kubernetes deployments
|
||||
|
||||
If:
|
||||
|
||||
- the app is not packaged as a helm chart or
|
||||
- it would be simpler to deploy it without package (for example custom privileged pod with access to gpu) or
|
||||
- the app is so simple it doesn't make sense to make helm package it (for example, simple http proxy that alters headers or stateless single-binary app) or
|
||||
- for any other reason it would make more sense to skip helm
|
||||
|
||||
You can deploy app skipping helm chart and just create raw Kubernetes manifests like Deployment, StatefulSet and other supporting resources like ConfigMap, Service, Ingress directly.
|
||||
|
||||
## Data / PVCs pattern
|
||||
|
||||
Data are stored on local disk of node using OpenEBS LVM LocalPV. To create a persistent volume, use following example:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: local.openebs.io/v1alpha1
|
||||
kind: LVMVolume
|
||||
metadata:
|
||||
labels:
|
||||
kubernetes.io/nodename: anapistula-delrosalae
|
||||
name: immich-library-lvmhdd
|
||||
namespace: openebs
|
||||
spec:
|
||||
capacity: 150Gi
|
||||
ownerNodeID: anapistula-delrosalae
|
||||
shared: "yes"
|
||||
thinProvision: "no"
|
||||
vgPattern: ^openebs-hdd$
|
||||
volGroup: openebs-hdd
|
||||
---
|
||||
kind: PersistentVolume
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: immich-library-lvmhdd
|
||||
spec:
|
||||
capacity:
|
||||
storage: 150Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: hdd-lvmpv
|
||||
volumeMode: Filesystem
|
||||
csi:
|
||||
driver: local.csi.openebs.io
|
||||
fsType: btrfs
|
||||
volumeHandle: immich-library-lvmhdd
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: library-lvmhdd
|
||||
namespace: immich
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 150Gi
|
||||
storageClassName: hdd-lvmpv
|
||||
volumeName: immich-library-lvmhdd
|
||||
```
|
||||
|
||||
Create LVMVolume and PersistentVolume resources manually and **do not** rely on automatic scheduling of PVCs because we want created LVM LVs on disk to have deterministic names and be reused if already exist on disk, which scheduler does not give us. There are two LVM storage classes:
|
||||
|
||||
- **hdd-lvmpv**, volume group: openebs-hdd, use for bulk data, like media library
|
||||
- **ssd-lvmpv**, volume group: openebs-ssd, use for small datasets that benefit from quick storage access like databases, state data etc.
|
||||
|
||||
When deciding the size of the volume, make minimal prediction, starting with 1GiB if you do not predict app to use much disk space.
|
||||
|
||||
## Vault secrets
|
||||
|
||||
There is OpenBao installed on cluster that manages access to secrets. The KV2 secret engine is mounted at `secret`, use it to store static secrets like API keys to external services, passwords and other entries you do not want to keep in plaintext in git repository.
|
||||
|
||||
To access the KV secrets on cluster, use Vault Secrets Operator installed on cluster, which provides `VaultStaticSecret` custom resource that syncs a path from OpenBao to Kubernetes `Secret` object.
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: llama-proxy
|
||||
namespace: llama
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: llama
|
||||
namespace: llama
|
||||
spec:
|
||||
method: kubernetes
|
||||
mount: kubernetes
|
||||
kubernetes:
|
||||
role: llama-proxy
|
||||
serviceAccount: llama-proxy
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: llama-api-key
|
||||
namespace: llama
|
||||
spec:
|
||||
type: kv-v2
|
||||
|
||||
mount: secret
|
||||
path: ollama
|
||||
|
||||
destination:
|
||||
create: true
|
||||
name: llama-api-key
|
||||
type: Opaque
|
||||
transformation:
|
||||
excludeRaw: true
|
||||
|
||||
vaultAuthRef: llama
|
||||
```
|
||||
|
||||
To give access to specified secret for given k8s ServiceAccount, you need to create kubernetes auth role and policy. Create a kubernetes auth role named `llama-proxy`, by creating file `vault/kubernetes-auth-roles/llama-proxy.yaml`:
|
||||
|
||||
```yaml
|
||||
bound_service_account_names:
|
||||
- llama-proxy
|
||||
bound_service_account_namespaces:
|
||||
- llama
|
||||
token_policies:
|
||||
- ollama
|
||||
```
|
||||
|
||||
Create policy named `ollama` by creating file `vault/policy/ollama.hcl`:
|
||||
|
||||
```hcl
|
||||
path "secret/data/ollama" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
```
|
||||
|
||||
Once these files are created, ask operator to reconcile OpenBao configuration and create required secret.
|
||||
|
||||
## Postgres operator
|
||||
|
||||
There is CloudNativePG operator installed on cluster that manages databases of applications running on cluster. You can create Postgres database by creating `Cluster` resource:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: kaneo-db
|
||||
namespace: kaneo
|
||||
spec:
|
||||
instances: 1
|
||||
|
||||
storage:
|
||||
pvcTemplate:
|
||||
storageClassName: ssd-lvmpv
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
volumeName: kaneo-db-1
|
||||
|
||||
```
|
||||
|
||||
Create a `PersistentVolume` and `LVMVol` resources manually but **do not** create `PersistentVolumeClaim`, CloudNativePG will create one on its own referencing `PersistentVolume` specified in `volumeName`. Do not replicate the database, there is only one node in the cluster currently. The `Cluster` resource will automatically create secret, use it to configure app:
|
||||
|
||||
```
|
||||
Name: kaneo-db-app
|
||||
Namespace: kaneo
|
||||
Labels: app.kubernetes.io/managed-by=cloudnative-pg
|
||||
cnpg.io/cluster=kaneo-db
|
||||
cnpg.io/reload=true
|
||||
cnpg.io/userType=app
|
||||
Annotations: cnpg.io/operatorVersion: 1.29.1
|
||||
|
||||
Type: kubernetes.io/basic-auth
|
||||
|
||||
Data
|
||||
====
|
||||
dbname: 3 bytes
|
||||
fqdn-jdbc-uri: 145 bytes
|
||||
fqdn-uri: 126 bytes
|
||||
host: 11 bytes
|
||||
jdbc-uri: 127 bytes
|
||||
password: 64 bytes
|
||||
pgpass: 90 bytes
|
||||
port: 4 bytes
|
||||
uri: 108 bytes
|
||||
user: 3 bytes
|
||||
username: 3 bytes
|
||||
```
|
||||
|
||||
## LoadBalancers
|
||||
|
||||
You can expose installed app to the Internet using Cilium's LoadBalancer configured on cluster:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: teamspeak3
|
||||
namespace: ispeak3
|
||||
spec:
|
||||
selector:
|
||||
app: teamspeak3
|
||||
ports:
|
||||
- name: voice
|
||||
protocol: UDP
|
||||
port: 9987
|
||||
targetPort: 9987
|
||||
- name: filetransfer
|
||||
protocol: TCP
|
||||
port: 30033
|
||||
targetPort: 30033
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
ipFamilyPolicy: PreferDualStack
|
||||
```
|
||||
|
||||
IPv6 will be directly reachable from the internet by its assigned address, for IPv4 currently you need to configure port forward on router in `ansible/roles/routeros/firewall.yml`, that step is not yet automated. The assigned internal IP will be known after manifests are applied on cluster. For this reason, there is no ExternalDNS configured yet, if you need a DNS name, ask the operator to configure DNS name for LoadBalancer. Assign names from lumpiasty.xyz subdomains (eg. kaneo.lumpiasty.xyz) unless explicitly requested. Do not use LoadBalancer for exposing HTTP applications, use Ingress instead.
|
||||
|
||||
## Ingress
|
||||
|
||||
You can expose HTTP applications using NGINX Ingress Controller:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
namespace: llama
|
||||
name: llama
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt
|
||||
acme.cert-manager.io/http01-edit-in-place: "true"
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "false"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: 30m
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: 8m
|
||||
spec:
|
||||
ingressClassName: nginx-ingress
|
||||
rules:
|
||||
- host: llama.lumpiasty.xyz
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: llama-proxy
|
||||
port:
|
||||
number: 80
|
||||
path: /
|
||||
pathType: Prefix
|
||||
tls:
|
||||
- hosts:
|
||||
- llama.lumpiasty.xyz
|
||||
secretName: llama-ingress
|
||||
```
|
||||
|
||||
TLS certificates are automatically issued for subdomains of lumpiasty.xyz using cert-manager. DNS name assignment is not automatic yet, ask operator to create DNS name for ingress resources.
|
||||
|
||||
## Keeping app up to date
|
||||
|
||||
There is a Renovate job configured for this repository as [Woodpecker job](/.woodpecker/renovate.yaml) to keep applications up to date. Renovate automatically keeps track of:
|
||||
|
||||
- Docker images specified in Kubernetes manifests like Deployment, StatefulSet etc
|
||||
- HelmRelease versions
|
||||
- GitRepository tags
|
||||
|
||||
To make Renovate automatically update applications, always specify full versions of docker images or helm chart release. If you use ambigous tags, renovate will not have chance to update and the cluster will never download new image because this tag already existed on node. **Do not** use:
|
||||
|
||||
- latest (or its variants like stable, current, main, master current)
|
||||
- "Sliding" versions, like 1 or 1.2 that point at 1.2.1 currently and will change image it points at when version 1.2.2 is released
|
||||
|
||||
As a last resort if the application does not publish stable image tags, pin digest of image.
|
||||
|
||||
Renovate may require custom configuration if:
|
||||
|
||||
- App is using non-standard versioning schema
|
||||
|
||||
Example app versioned by date (unified-vulkan-2026-01-01), renovate.json:
|
||||
|
||||
```json
|
||||
{
|
||||
"matchDatasources": ["docker"],
|
||||
"matchPackageNames": ["ghcr.io/mostlygeek/llama-swap"],
|
||||
"versioning": "regex:^unified-vulkan-(?<major>\\d{4})-(?<minor>\\d{2})-(?<patch>\\d{2})$",
|
||||
"automerge": true,
|
||||
"automergeType": "pr",
|
||||
"platformAutomerge": true
|
||||
}
|
||||
```
|
||||
|
||||
- Docker image tag is specified in non-standard field that Renovate may not recognise automatically such as Helm values
|
||||
|
||||
Example app with non-standard image selected in helm values instead of image's default (which is latest in this chart):
|
||||
```yaml
|
||||
values:
|
||||
kaneo:
|
||||
image:
|
||||
tag: "2.7.3" # renovate: depName=ghcr.io/usekaneo/kaneo registryUrl=https://ghcr.io
|
||||
```
|
||||
|
||||
Renovate is configured so it automatically merges patch versions, other updates are created as pull requests to be manually reviewed and merged unless explicitly desired on per case basis.
|
||||
|
||||
## SSO / OIDC / Authentik
|
||||
|
||||
There is an Authentik running on cluster providing SSO for applications. Configure user-facing apps to utilize it correctly.
|
||||
|
||||
Authentik supports following protocols:
|
||||
|
||||
- OAuth2 / OpenID Connect
|
||||
- SAML
|
||||
- Radius
|
||||
- LDAP
|
||||
- SCIM
|
||||
|
||||
Currently, there is no Authentik configuration in code, ask operator to create application in the UI and save OAuth id and secret in OpenBao under `secret/authentik/<app>`. Authentik provides discovery URL for OAuth applications: `https://authentik.lumpiasty.xyz/application/o/<app slug>/.well-known/openid-configuration`.
|
||||
|
||||
Configure the app to disable guest access, built-in registration and automatically register unprivileged users with `user` role and privileged users with `admin` role as the app allows.
|
||||
|
||||
## Privileged apps
|
||||
|
||||
Some apps require direct access to devices, like GPU. There are no specific operators yet, apps that require access to GPU are simply launched as privileged pods, example:
|
||||
|
||||
```yaml
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llama-swap
|
||||
namespace: llama
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: llama-swap
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: llama-swap
|
||||
spec:
|
||||
containers:
|
||||
- name: llama-swap
|
||||
volumeMounts:
|
||||
- mountPath: /dev/kfd
|
||||
name: kfd
|
||||
- mountPath: /dev/dri
|
||||
name: dri
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumes:
|
||||
- name: kfd
|
||||
hostPath:
|
||||
path: /dev/kfd
|
||||
type: CharDevice
|
||||
- name: dri
|
||||
hostPath:
|
||||
path: /dev/dri
|
||||
type: Directory
|
||||
```
|
||||
|
||||
Creating of such pods is forbidden unless explicitly allowed in Talos config:
|
||||
|
||||
```yaml
|
||||
# CSI driver requirement
|
||||
cluster:
|
||||
apiServer:
|
||||
admissionControl:
|
||||
- name: PodSecurity
|
||||
configuration:
|
||||
apiVersion: pod-security.admission.config.k8s.io/v1beta1
|
||||
kind: PodSecurityConfiguration
|
||||
exemptions:
|
||||
namespaces:
|
||||
- llama
|
||||
```
|
||||
|
||||
Create the patch like this under `talos/patches/<app>.patch`, add it to `gen-talos-config` target in Makefile and ask operator to apply reconcile Talos config to allow privileged pods in specified namespace.
|
||||
@@ -0,0 +1,13 @@
|
||||
<svg width="136" height="136" viewBox="0 0 136 136" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_137_2)">
|
||||
<rect width="136" height="136" fill="#141414"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M62.6855 103.724C58.5716 104.595 56.0001 103.265 56 101.66L56 70.0264C56 69.8606 56.0032 69.686 56.0088 69.5069C56.0039 69.3848 56.001 69.249 56.001 69.0977L56.001 37.9444C56.015 36.6524 56.2588 35.7449 59.2588 35.1094L73.3145 32.2764C77.4285 31.405 80 32.7365 80 34.3408L80 65.9746C80 66.1409 79.9978 66.3155 79.9922 66.4951C79.997 66.6169 79.999 66.7526 79.999 66.9033L79.999 98.0567C79.9849 99.3483 79.7408 100.256 76.7412 100.892L62.6855 103.724Z" fill="#F5F5F5"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M30.6855 111.723C26.5716 112.594 24.0001 111.264 24 109.659L24 78.0244C24 77.8588 24.0032 77.6848 24.0088 77.5059C24.0039 77.3838 24.001 77.248 24.001 77.0967L24.001 45.9434C24.015 44.6514 24.2588 43.7439 27.2588 43.1084L41.3145 40.2754C45.4285 39.404 48 40.7355 48 42.3399L48 73.9737C48 74.1399 47.9978 74.3146 47.9922 74.4942C47.997 74.6159 47.999 74.7517 47.999 74.9024L47.999 106.056C47.9849 107.347 47.7408 108.255 44.7412 108.891L30.6855 111.723Z" fill="#F5F5F5"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M105.314 24.2754C109.428 23.404 112 24.7355 112 26.3398V37.1845L94.0576 60.5019L111.999 82.7802V90.0576C111.985 91.3492 111.741 92.2571 108.741 92.8925L94.6855 95.7246C90.5717 96.596 88.0002 95.2654 88 93.6611V62.0254C88 61.8598 88.0032 61.6856 88.0088 61.5068C88.0039 61.3848 88.001 61.2488 88.001 61.0976V29.9433C88.0151 28.6516 88.2591 27.7438 91.2588 27.1084L105.314 24.2754Z" fill="#F5F5F5"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_137_2">
|
||||
<rect width="136" height="136" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.7 KiB |
Reference in New Issue
Block a user