Update SHA

README.md

@@ -201,6 +201,25 @@ node -e 'console.log("admin:" + require("bcryptjs").hashSync("password", 10))'
 ```
 
 
+#### Restore Redis backup
+If the JuiceFS pods are crashing saying "Database not formatted."
+
+1. Download the backup Grab the latest JSON file from your pulumi-redis-backup bucket.
+2. Restore the Metadata You need to run the load command. You can do this from a temporary pod or any machine that can talk to the Redis service.
+Bash
+
+###### Spin up a temporary toolbox pod
+kubectl run -it --rm juicefs-restore --image=juicedata/mount:ce-v1.2.0 -- sh
+
+###### Inside the pod:
+1. Download your backup file (or copy/paste it if it's small)
+    (Assuming you copied the JSON content to a file named 'backup.json')
+2. Run the load command
+   Format: juicefs load redis://:PASS@HOST:PORT/DB backup.json
+juicefs load redis://:$(REDIS_PASS)@juicefs-redis.kube-system.svc.cluster.local:6379/1 backup.json
+3. Restart the JuiceFS Pods Once the load command finishes, the metadata is back. Delete the JuiceFS CSI pods (or the application pods using them) to force them to restart. They will connect, see the valid filesystem, and mount instantly.
+
+
 https://www.pulumi.com/registry/packages/docker-build/api-docs/image/
 https://www.pulumi.com/registry/packages/docker/api-docs/buildxbuilder/#create
 

deps.edn

@@ -4,7 +4,7 @@
         #_gigiaj/pulumicljs #_{:local/root "../pulumi-clojurescript"}
         gigiaj/pulumicljs
         {:git/url "https://github.com/GigiaJ/pulumi-clojurescript.git"
-         :git/sha "50c8098cbc6d0d07837afb1e8f763a818dd376d9"
+         :git/sha "5c85ae0ae5a4f99cd6b1eaa765b6a5d9854db08f"
          }
         funcool/promesa {:mvn/version "11.0.678"}
         }}

resources/backup-redis.yaml

@@ -0,0 +1,72 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: juicefs-metadata-backup
+  namespace: kube-system
+spec:
+  schedule: "0 3 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          volumes:
+            - name: backup-dir
+              emptyDir: {}
+          
+          initContainers:
+            - name: dumper
+              image: juicedata/mount:ce-v1.2.0
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  echo "Starting JuiceFS Metadata Dump..."
+                  juicefs dump redis://:$(REDIS_PASS)@juicefs-redis.kube-system.svc.cluster.local:6379/1 > /backups/juicefs-meta.json
+                  if [ -s /backups/juicefs-meta.json ]; then
+                    echo "Dump successful (Size: $(du -h /backups/juicefs-meta.json | cut -f1))"
+                  else
+                    echo "Dump failed: File is empty"
+                    exit 1
+                  fi
+              volumeMounts:
+                - name: backup-dir
+                  mountPath: /backups
+              env:
+                - name: REDIS_PASS
+                  valueFrom:
+                    secretKeyRef:
+                      name: juicefs-redis-secrets
+                      key: password
+
+          containers:
+            - name: uploader
+              image: minio/mc:latest
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  echo "Configuring S3 Client..."
+                  mc alias set s3storage $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY
+                  echo "Uploading backup..."
+                  FILENAME="juicefs-meta-$(date +%Y-%m-%d).json"
+                  mc cp /backups/juicefs-meta.json s3storage/$S3_BUCKET/backups/$FILENAME
+                  echo "Backup Complete!"
+              volumeMounts:
+                - name: backup-dir
+                  mountPath: /backups
+              env:
+                - name: S3_ENDPOINT
+                  value: "http://wasabi-proxy.wasabi-proxy.svc.cluster.local"
+                - name: S3_BUCKET
+                  value: "pulumi-redis-backup"
+                - name: S3_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: juicefs-csi-secrets
+                      key: access-key
+                - name: S3_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: juicefs-csi-secrets
+                      key: secret-key

src/main/infra/openbao.cljs

@@ -272,6 +272,7 @@
   {:stack [:k8s:namespace :k8s:chart :generic:execute]
    :app-namespace "vault"
    :app-name "openbao"
+   :no-namespace true
    :exec-fn  execute-fn
    :vault-load-yaml false
    :k8s:chart-opts {:repositoryOpts {:repo "https://openbao.github.io/openbao-helm"}

src/main/k8s/add_ons/cert_manager.cljs

@@ -7,9 +7,8 @@
    :is-prod? true
    :k8s:chart-opts {:repositoryOpts {:repo "https://charts.jetstack.io"}
                     :chart "cert-manager"
-                    :version "v1.15.0"
+                    :version "v1.19.1"
-                    :namespace "cert-manager"
+                    :namespace "cert-manager"}
-                    :values {:installCRDs true}}
    :k8s:secret-opts {:metadata {:name "api-token-secret"}
                      :stringData {:apiToken 'token}}
    :k8s:cluster-issuer-opts {:spec {:acme {:email 'email

src/main/k8s/add_ons/crd/cert_manager.cljs

@@ -3,5 +3,5 @@
 (def config
   {:stack [:k8s:config-file]
    :app-name "cert-manager"
-   :version "v1.15.0"
+   :version "v1.19.1"
    :k8s:config-file-opts {:file '(str "https://github.com/cert-manager/cert-manager/releases/download/" version "/cert-manager.crds.yaml")}})

src/main/k8s/add_ons/csi_driver/extra/redis.cljs

@@ -0,0 +1,42 @@
+(ns k8s.add-ons.csi-driver.extra.redis)
+
+(def config
+  {:stack [:vault:prepare :k8s:secret :k8s:pvc :k8s:deployment :k8s:service]
+   :app-name "juicefs-redis"
+   :app-namespace "kube-system"
+   :no-namespace true
+
+   :k8s:pvc-opts
+   {:metadata {:name "juicefs-redis-data"
+               :namespace "kube-system"}
+    :spec {:accessModes ["ReadWriteOnce"]
+           :storageClassName "hcloud-volumes"
+           :resources {:requests {:storage "10Gi"}}}}
+
+   :k8s:deployment-opts
+   {:metadata {:name "juicefs-redis" :namespace "kube-system"}
+    :spec {:replicas 1
+           :selector {:matchLabels {:app "juicefs-redis"}}
+           :template {:metadata {:labels {:app "juicefs-redis"}}
+                      :spec {:volumes [{:name "juicefs-redis-data"
+                                        :persistentVolumeClaim
+                                        {:claimName "juicefs-redis-data"}}]
+                             :containers
+                             [{:name "juicefs-redis"
+                               :image "redis:7-alpine"
+                               :args ["--requirepass" "$(REDIS_PASS)"
+                                      "--maxmemory-policy" "noeviction"
+                                      "--appendonly" "yes"]
+                               :env [{:name "REDIS_PASS"
+                                      :valueFrom {:secretKeyRef {:name "juicefs-redis-secrets"
+                                                                 :key "password"}}}]
+                               :ports [{:containerPort 6379}]
+                               :volumeMounts [{:name "juicefs-redis-data"
+                                               :mountPath "/data"}]
+                               }]}}}}
+
+   :k8s:service-opts
+   {:metadata {:name "juicefs-redis" :namespace "kube-system"}
+    :spec {:type "ClusterIP"
+           :selector {:app "juicefs-redis"}
+           :ports [{:port 6379 :targetPort 6379}]}}})

src/main/k8s/add_ons/csi_driver/hetzner.cljs

@@ -5,6 +5,7 @@
 (def config
   {:stack [:k8s:secret :k8s:chart]
    :app-namespace "kube-system"
+   :no-namespace true
    :app-name "hcloud-csi"
    :vault-load-yaml false
    :k8s:secret-opts {:metadata {:name "hcloud"

src/main/k8s/add_ons/csi_driver/juicefs.cljs

@@ -0,0 +1,29 @@
+(ns k8s.add-ons.csi-driver.juicefs)
+
+(def config
+  {:stack [:vault:prepare :k8s:secret :k8s:chart :k8s:csi-driver :k8s:storage-class]
+   :app-namespace "kube-system"
+   :no-namespace true
+   :app-name "juicefs-csi"
+   :k8s:csi-driver-opts
+   {:metadata {:name "csi.juicefs.com"}
+    :spec {:attachRequired false
+           :podInfoOnMount true
+           :volumeLifecycleModes ["Persistent"]}}
+
+   :k8s:chart-opts
+   {:repositoryOpts {:repo "https://juicedata.github.io/charts/"}
+    :chart "juicefs-csi-driver"
+    :version "0.30.3"
+    :namespace "kube-system"
+    :values {:kubeletDir "/var/lib/kubelet"}}
+   :k8s:storage-class-opts
+   {:metadata {:name "juicefs-sc"}
+    :provisioner "csi.juicefs.com"
+    :parameters {"csi.storage.k8s.io/provisioner-secret-name" "juicefs-csi-secrets"
+                 "csi.storage.k8s.io/provisioner-secret-namespace" "kube-system"
+                 "csi.storage.k8s.io/node-publish-secret-name" "juicefs-csi-secrets"
+                 "csi.storage.k8s.io/node-publish-secret-namespace" "kube-system"
+                 "csi.storage.k8s.io/controller-expand-secret-name" "juicefs-csi-secrets"
+                 "csi.storage.k8s.io/controller-expand-secret-namespace" "kube-system"
+                 "pathPattern" "${.pvc.namespace}/${.pvc.name}"}}})

src/main/k8s/add_ons/csi_driver/wasabi.cljs

@@ -10,17 +10,7 @@
                 :repositoryOpts {:repo "https://yandex-cloud.github.io/k8s-csi-s3/charts"}
                 :values {:controller {:enabled false
                                                         :existingSecret {:name "wasabi-csi-secrets"}
-                                                        :node {:existingSecret {:name "wasabi-csi-secrets"}}}}
+                                                        :node {:existingSecret {:name "wasabi-csi-secrets"}}}}}
-                                          
-                                          #_:storageClass #_{:create true
-                                                         :name "csi-s3-sc"
-                                                         :singleBucket "pulumi-harbor"
-                                                         :region "us-east-1"
-                                                         :accessKeyID "something"
-                                                         :secretAccessKey "something"
-                                                         ;;:bucket "pulumi-harbor"
-                                                         }}
    :k8s:secret-opts {:stringData {:accessKeyID (-> cfg :wasabiId)
                              :secretAccessKey (-> cfg :wasabiKey)
-                             :endpoint "http://wasabi-proxy.wasabi-proxy.svc.cluster.local"}}
+                             :endpoint "http://wasabi-proxy.wasabi-proxy.svc.cluster.local"}}})
-  :vault-load-yaml false})

src/main/k8s/services/foundryvtt/service.cljs

@@ -1,7 +1,7 @@
 (ns k8s.services.foundryvtt.service)
 
 (def config
-  {:stack [:vault:prepare :harbor:robot-account :docker:image [:k8s :deployment :service :httproute]]
+  {:stack [:vault:prepare :harbor:robot-account :docker:image [:k8s :pvc :deployment :service :httproute]]
    :image-port     30000
    :app-namespace "generic"
    :app-name      "foundry"
@@ -14,11 +14,23 @@
                        :tags ['(str registry-base "/" registry-namespace "/" app-name)]
                        :push true}
    :k8s:deployment-opts {:spec {:template {:spec {:imagePullSecrets [{:name "harbor-creds-secrets"}]
-                                                  :containers [{:name 'app-name :image '(str registry-base "/" registry-namespace "/" app-name ":latest")}]}}}}
+                                                  :volumes [{:name "data-vol"
+                                                            :persistentVolumeClaim {:claimName "vtt-assets"}}]
+                                                  :containers [{:name 'app-name :image '(str registry-base "/" registry-namespace "/" app-name ":latest")
+                                                                :volumeMounts [{:name "data-vol"
+                                                                                :mountPath "/root/.local/share"
+                                                                                :mountPropagation "HostToContainer"}]
+                                                                }]}}}}
    :harbor:robot-account-opts {:name 'app-name
                                :permissions [{:kind "project"
                                               :namespace 'registry-namespace
                                               :access [{:action "pull" :resource "repository"}
                                                        {:action "push" :resource "repository"}
                                                        {:action "list" :resource "repository"}]}]}
+   :k8s:pvc-opts
+   {:metadata {:name "vtt-assets"
+               :namespace "generic"}
+    :spec {:storageClassName "juicefs-sc"
+           :accessModes ["ReadWriteMany"] 
+           :resources {:requests {:storage "10Gi"}}}}
    :k8s:httproute-opts {:spec {::hostnames ['host]}}})

src/main/k8s/services/mesite/service.cljs

@@ -8,7 +8,7 @@
   {:stack [:vault:prepare
            :harbor:robot-account
            :docker:image
-           [:k8s :namespace :deployment :service :httproute]] 
+           [:k8s :deployment :service :httproute]] 
    :app-name "mesite"
    :app-namespace "generic"
    :docker:image-opts {:context {:location "https://codeberg.org/Gigia/mesite.git"}

src/main/stack_resource_definitions.cljs

@@ -11,7 +11,9 @@
    [k8s.add-ons.crd.cert-manager :as cert-manager-crd]
    [k8s.add-ons.crd.gateway-api :as gateway-api-crd]
    [k8s.add-ons.crd.traefik :as traefik-crds]
+   [k8s.add-ons.csi-driver.juicefs :as juicefs-csi]
    [k8s.add-ons.csi-driver.wasabi :as wasabi-csi]
+   [k8s.add-ons.csi-driver.extra.redis :as redis-juicefs]
    [k8s.add-ons.image-registry.harbor :as harbor]
    [k8s.add-ons.secret-replicator :as secret-replicator]
    [k8s.add-ons.proxy :as proxy]
@@ -20,6 +22,7 @@
    [k8s.services.gitea.service :as gitea-service]
    [k8s.services.act-runner.service :as act-runner-service]
    [k8s.services.foundryvtt.service :as foundryvtt-service]
+   [k8s.services.foundryvtt.service-2 :as girls-foundry-service]
    [k8s.services.productive.service :as productive-service]))
 
 (defn general-provider-output-refs []
@@ -51,13 +54,15 @@
 
 (def shared-resources-definition
   (create-resource-definition
-   [dns/config
+   [cert-manager-crd/config
-    cert-manager-crd/config
     gateway-api-crd/config
     traefik-crds/config
+    dns/config
+    wasabi-csi/config proxy/config secret-replicator/config
+    redis-juicefs/config
+    juicefs-csi/config
     cert-manager/config
     traefik/config
-    wasabi-csi/config proxy/config secret-replicator/config 
     harbor/config
     ]
    ["base" "init"]
@@ -72,7 +77,9 @@
 
 (def deployment-resources-definition
   (create-resource-definition
-   [#_nextcloud-service/config foundryvtt-service/config mesite-service/config productive-service/config gitea-service/config act-runner-service/config]
+   [#_nextcloud-service/config 
+    girls-foundry-service/config
+    foundryvtt-service/config mesite-service/config productive-service/config gitea-service/config act-runner-service/config]
    ["base" "init" "shared"]
    (general-provider-output-refs)))