From f7fa525e1356ca8476c2666d9b8b9ab356eb87f4 Mon Sep 17 00:00:00 2001
From: Lumpiasty <arek.dzski@gmail.com>
Date: Mon, 16 Mar 2026 19:41:09 +0100
Subject: [PATCH] add crawl4ai deployment

---
 apps/crawl4ai/deployment.yaml        | 62 ++++++++++++++++++++++++++++
 apps/crawl4ai/kustomization.yaml     |  7 ++++
 apps/crawl4ai/namespace.yaml         |  4 ++
 apps/crawl4ai/secret.yaml            | 38 +++++++++++++++++
 apps/crawl4ai/service.yaml           | 14 +++++++
 vault/kubernetes-roles/crawl4ai.yaml |  6 +++
 vault/policy/crawl4ai.hcl            |  3 ++
 7 files changed, 134 insertions(+)
 create mode 100644 apps/crawl4ai/deployment.yaml
 create mode 100644 apps/crawl4ai/kustomization.yaml
 create mode 100644 apps/crawl4ai/namespace.yaml
 create mode 100644 apps/crawl4ai/secret.yaml
 create mode 100644 apps/crawl4ai/service.yaml
 create mode 100644 vault/kubernetes-roles/crawl4ai.yaml
 create mode 100644 vault/policy/crawl4ai.hcl

diff --git a/apps/crawl4ai/deployment.yaml b/apps/crawl4ai/deployment.yaml
new file mode 100644
index 0000000..3d1d610
--- /dev/null
+++ b/apps/crawl4ai/deployment.yaml
@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawl4ai
+  namespace: crawl4ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: crawl4ai
+  template:
+    metadata:
+      labels:
+        app: crawl4ai
+    spec:
+      containers:
+        - name: crawl4ai
+          image: unclecode/crawl4ai:latest
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: CRAWL4AI_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: crawl4ai-secret
+                  key: api_token
+                  optional: false
+            - name: MAX_CONCURRENT_TASKS
+              value: "5"
+          ports:
+            - name: http
+              containerPort: 11235
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 15
+            timeoutSeconds: 3
+            failureThreshold: 6
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+          volumeMounts:
+            - name: dshm
+              mountPath: /dev/shm
+      volumes:
+        - name: dshm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Gi
diff --git a/apps/crawl4ai/kustomization.yaml b/apps/crawl4ai/kustomization.yaml
new file mode 100644
index 0000000..52241e5
--- /dev/null
+++ b/apps/crawl4ai/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - secret.yaml
+  - deployment.yaml
+  - service.yaml
diff --git a/apps/crawl4ai/namespace.yaml b/apps/crawl4ai/namespace.yaml
new file mode 100644
index 0000000..699e660
--- /dev/null
+++ b/apps/crawl4ai/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: crawl4ai
diff --git a/apps/crawl4ai/secret.yaml b/apps/crawl4ai/secret.yaml
new file mode 100644
index 0000000..d7af094
--- /dev/null
+++ b/apps/crawl4ai/secret.yaml
@@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crawl4ai-secret
+  namespace: crawl4ai
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: crawl4ai
+  namespace: crawl4ai
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: crawl4ai
+    serviceAccount: crawl4ai-secret
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: crawl4ai-secret
+  namespace: crawl4ai
+spec:
+  type: kv-v2
+
+  mount: secret
+  path: crawl4ai
+
+  destination:
+    create: true
+    name: crawl4ai-secret
+    type: Opaque
+    transformation:
+      excludeRaw: true
+
+  vaultAuthRef: crawl4ai
diff --git a/apps/crawl4ai/service.yaml b/apps/crawl4ai/service.yaml
new file mode 100644
index 0000000..f644f4d
--- /dev/null
+++ b/apps/crawl4ai/service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crawl4ai
+  namespace: crawl4ai
+spec:
+  type: ClusterIP
+  selector:
+    app: crawl4ai
+  ports:
+    - name: http
+      port: 11235
+      targetPort: 11235
+      protocol: TCP
diff --git a/vault/kubernetes-roles/crawl4ai.yaml b/vault/kubernetes-roles/crawl4ai.yaml
new file mode 100644
index 0000000..c40264a
--- /dev/null
+++ b/vault/kubernetes-roles/crawl4ai.yaml
@@ -0,0 +1,6 @@
+bound_service_account_names:
+  - crawl4ai-secret
+bound_service_account_namespaces:
+  - crawl4ai
+token_policies:
+  - crawl4ai
diff --git a/vault/policy/crawl4ai.hcl b/vault/policy/crawl4ai.hcl
new file mode 100644
index 0000000..e23e974
--- /dev/null
+++ b/vault/policy/crawl4ai.hcl
@@ -0,0 +1,3 @@
+path "secret/data/crawl4ai" {
+    capabilities = ["read"]
+}