From 27a6ed34984b19b5b6e8e446cc3f4ead5a8add4e Mon Sep 17 00:00:00 2001 From: GigiaJ Date: Sun, 23 Nov 2025 16:12:26 -0600 Subject: [PATCH] Move all files to root --- scripts/generate-crds.js | 114 ++++++++ src/main/utils/defaults.cljs | 12 + src/main/utils/docker.cljs | 25 ++ src/main/utils/general.clj | 82 ++++++ src/main/utils/general.cljs | 232 ++++++++++++++++ src/main/utils/harbor.cljs | 36 +++ src/main/utils/k8s.cljs | 198 ++++++++++++++ src/main/utils/providers.cljs | 135 +++++++++ src/main/utils/safe_fns.cljs | 48 ++++ src/main/utils/stack_processor.cljs | 406 ++++++++++++++++++++++++++++ src/main/utils/vault.cljs | 71 +++++ 11 files changed, 1359 insertions(+) create mode 100644 scripts/generate-crds.js create mode 100644 src/main/utils/defaults.cljs create mode 100644 src/main/utils/docker.cljs create mode 100644 src/main/utils/general.clj create mode 100644 src/main/utils/general.cljs create mode 100644 src/main/utils/harbor.cljs create mode 100644 src/main/utils/k8s.cljs create mode 100644 src/main/utils/providers.cljs create mode 100644 src/main/utils/safe_fns.cljs create mode 100644 src/main/utils/stack_processor.cljs create mode 100644 src/main/utils/vault.cljs diff --git a/scripts/generate-crds.js b/scripts/generate-crds.js new file mode 100644 index 0000000..08965b1 --- /dev/null +++ b/scripts/generate-crds.js @@ -0,0 +1,114 @@ +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const crypto = require('crypto'); +const { execSync } = require('child_process'); + +const PROJECT_ROOT = process.cwd(); + +const OUTPUT_DIR = path.join(PROJECT_ROOT, 'generated/crds'); +const TEMP_DIR = path.join(PROJECT_ROOT, 'temp_crds'); +const CHECKSUM_FILE = path.join(PROJECT_ROOT, '.crd2pulumi-checksum'); + +const GW_VERSION = 'v1.1.0'; +const GW_URL = `https://github.com/kubernetes-sigs/gateway-api/releases/download/${GW_VERSION}/experimental-install.yaml`; +const GW_FILE = 'gateway-api.yaml'; + +const CM_VERSION = 'v1.15.0'; +const CM_URL = `https://github.com/cert-manager/cert-manager/releases/download/${CM_VERSION}/cert-manager.crds.yaml`; +const CM_FILE = 'cert-manager.yaml'; + +const downloadFile = (url, filename) => { + return new Promise((resolve, reject) => { + const destPath = path.join(TEMP_DIR, filename); + const file = fs.createWriteStream(destPath); + + const request = (uri) => { + https.get(uri, (response) => { + if (response.statusCode === 301 || response.statusCode === 302) { + return request(response.headers.location); + } + if (response.statusCode !== 200) { + reject(new Error(`Failed to download ${uri}: ${response.statusCode}`)); + return; + } + console.log(`Downloading ${filename}...`); + response.pipe(file); + file.on('finish', () => { + file.close(); + resolve(destPath); + }); + }).on('error', (err) => { + fs.unlink(destPath, () => {}); + reject(err); + }); + }; + request(url); + }); +}; + +const computeHash = (filePaths) => { + const hash = crypto.createHash('sha256'); + filePaths.sort().forEach(fp => hash.update(fs.readFileSync(fp))); + return hash.digest('hex'); +}; + +async function main() { + if (fs.existsSync(TEMP_DIR)) fs.rmSync(TEMP_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEMP_DIR); + + try { + const gwPath = await downloadFile(GW_URL, GW_FILE); + const cmPath = await downloadFile(CM_URL, CM_FILE); + const allFiles = [gwPath, cmPath]; + const newHash = computeHash(allFiles); + + let oldHash = null; + if (fs.existsSync(CHECKSUM_FILE)) { + oldHash = fs.readFileSync(CHECKSUM_FILE, 'utf8').trim(); + } + + if (oldHash === newHash && fs.existsSync(OUTPUT_DIR)) { + console.log('CRDs unchanged. Skipping.'); + } else { + console.log('Regenerating CRDs...'); + if (fs.existsSync(OUTPUT_DIR)) fs.rmSync(OUTPUT_DIR, { recursive: true, force: true }); + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + execSync(`crd2pulumi --nodejsPath "${OUTPUT_DIR}" --force "${gwPath}" "${cmPath}"`, { stdio: 'inherit' }); + + const pkgPath = path.join(OUTPUT_DIR, 'package.json'); + if (fs.existsSync(pkgPath)) { + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')); + if (pkg.scripts) delete pkg.scripts; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2)); + } + + const tsconfig = { + compilerOptions: { + target: "es2020", + module: "commonjs", + moduleResolution: "node", + declaration: true, + skipLibCheck: true, + }, + include: ["**/*.ts"], + exclude: ["node_modules"] + }; + fs.writeFileSync(path.join(OUTPUT_DIR, 'tsconfig.json'), JSON.stringify(tsconfig, null, 2)); + + try { + execSync('npx tsc', { cwd: OUTPUT_DIR, stdio: 'inherit' }); + } catch (e) { console.warn("TSC warnings ignored."); } + + fs.writeFileSync(CHECKSUM_FILE, newHash); + console.log(`Success!`); + } + } catch (error) { + console.error(error); + process.exit(1); + } finally { + if (fs.existsSync(TEMP_DIR)) fs.rmSync(TEMP_DIR, { recursive: true, force: true }); + } +} + +main(); \ No newline at end of file diff --git a/src/main/utils/defaults.cljs b/src/main/utils/defaults.cljs new file mode 100644 index 0000000..1bf4e62 --- /dev/null +++ b/src/main/utils/defaults.cljs @@ -0,0 +1,12 @@ +(ns utils.defaults + (:require ["path" :as path] + [configs :refer [cfg]] + [utils.k8s :as k8s] + [utils.harbor :as harbor] + [utils.docker :as docker])) + + +(def defaults + {:k8s k8s/defaults + :harbor harbor/defaults + :docker docker/defaults}) diff --git a/src/main/utils/docker.cljs b/src/main/utils/docker.cljs new file mode 100644 index 0000000..1b58223 --- /dev/null +++ b/src/main/utils/docker.cljs @@ -0,0 +1,25 @@ +(ns utils.docker + (:require + [utils.general :refer [generic-transform deep-merge new-resource component-factory resource-factory deploy-stack-factory iterate-stack]] + ["@pulumi/docker-build" :as docker] + ["path" :as path] + [configs :refer [cfg]])) + +(defn image [env] + (let [{:keys [app-name docker:image-opts]} env + context-path (.. path (join "." (-> cfg :resource-path))) + dockerfile-path (.. path (join context-path (str app-name ".dockerfile"))) + base-args (if (:is-local docker:image-opts) + {:context {:location context-path} + :dockerfile {:location dockerfile-path} + :imageName (str (-> cfg :docker-repo) "/" app-name ":latest")} + {})] + base-args)) + +(def defaults + {:image image}) + +(def component-specs-defs + {:root-sym 'docker + :provider-key :harbor + :resources {:image {:path ['-Image]}}}) \ No newline at end of file diff --git a/src/main/utils/general.clj b/src/main/utils/general.clj new file mode 100644 index 0000000..7928f5a --- /dev/null +++ b/src/main/utils/general.clj @@ -0,0 +1,82 @@ +(ns utils.general + (:require + [clojure.walk])) + +(defmacro build-registry + "Generates a flat resource registry map from a nested provider definition map." + [definitions] + (let [map-entries + (reduce-kv + (fn [entries provider-group-kw provider-group-def] + (let [{:keys [root-sym provider-key resources]} provider-group-def + provider-ns (name provider-group-kw)] + (reduce-kv + (fn [entries resource-kw resource-def] + (let [{:keys [path defaults-fn defaults-name]} resource-def + resource-name (name resource-kw) + final-key (keyword provider-ns resource-name) + constructor-form (list* '.. root-sym path) + defaults-fn-form + (cond + defaults-fn defaults-fn + defaults-name (let [defaults-fn-sym (symbol "default" (name defaults-name))] + `(fn [env] (~defaults-fn-sym (:options env)))) + :else (let [defaults-fn-sym (symbol "default" resource-name)] + `(fn [env] (~defaults-fn-sym (:options env))))) + value-map `{:constructor ~constructor-form + :provider-key ~provider-key + :defaults-fn ~defaults-fn-form}] + (conj entries [final-key value-map]))) + entries + resources))) + [] + definitions)] + `~(into {} map-entries))) + + +(defn- p->-replace-percent + "Walks 'form' and replaces all instances of the symbol '%' + with the value of 'x'." + [form x] + (clojure.walk/postwalk + (fn [sub-form] + (if (= sub-form '%) x sub-form)) + form)) + +(defmacro pulet + "Sequential binding for Pulumi Outputs. + Looks like let*, but each binding is chained." + [bindings & body] + (if (empty? bindings) + `(do ~@body) + (let [[sym val & rest] bindings] + `(p-chain ~val + (fn [~sym] + (pulet [~@rest] ~@body)))))) + + + + +(defmacro p-> [x & forms] + (let [wrap (fn [acc form] + (cond + (map? form) + `(p-chain ~acc (fn [val#] ~form)) + + (keyword? form) + `(p-chain ~acc #(get % ~form)) + + (string? form) + `(p-chain ~acc #(aget % ~form)) + + (symbol? form) + `(p-chain ~acc #(~form %)) + + (list? form) + `(p-chain ~acc ~form) + + + :else + (throw (ex-info "Unsupported form in p->" {:form form}))))] + (reduce wrap x forms))) + diff --git a/src/main/utils/general.cljs b/src/main/utils/general.cljs new file mode 100644 index 0000000..50cbd76 --- /dev/null +++ b/src/main/utils/general.cljs @@ -0,0 +1,232 @@ +(ns utils.general (:require [clojure.walk :as walk])) + + +(defn new-resource [resource-type resource-name final-args provider dependencies] + (let [base-opts (if (or (some? provider) (seq dependencies)) + {:enableServerSideApply false} {}) + opts (cond-> base-opts + (some? provider) (assoc :provider provider) + (seq dependencies) (assoc :dependsOn dependencies))] + (if (seq opts) + (new resource-type resource-name (clj->js final-args) (clj->js opts)) + (new resource-type resource-name (clj->js final-args))))) + +(defn assoc-ins [m path-vals] + (reduce (fn [acc [path val]] (assoc-in acc path val)) m path-vals)) + +(declare deep-merge) + +(defn merge-by-name + "Merges two vectors of maps by :name key." + [a b] + (let [a-map (into {} (map #(vector (:name %) %) a)) + b-map (into {} (map #(vector (:name %) %) b)) + merged (merge-with deep-merge a-map b-map)] + (vec (vals merged)))) + +(defn deep-merge + "Recursively merges maps and intelligently merges vectors of maps by :name." + [a b] + (cond + (nil? b) a + (and (map? a) (map? b)) + (merge-with deep-merge a b) + + (and (vector? a) (vector? b) + (every? map? a) (every? map? b) + (some #(contains? % :name) (concat a b))) + (merge-by-name a b) + :else b)) + +(defn make-transformer + "Given f that takes {:app-name .. :secrets ..}, where :secrets is a plain map + (already unwrapped inside .apply), return a Helm transformer." + [f] + (fn [{:keys [base-values app-name secrets]}] + (.apply secrets + (fn [smap] + (let [m (js->clj smap :keywordize-keys true) + updates (f {:app-name app-name + :secrets m}) + after (clj->js (assoc-ins base-values updates))] + after))))) + +(defn make-paths [& path-groups] + (mapcat (fn [{:keys [paths backend]}] + (mapv (fn [p] + {:path p + :pathType "Prefix" + :backend {:service backend}}) + paths)) + path-groups)) + +(defn generic-make-transformer + "Returns a Pulumi-compatible transformer that unwraps Output values via .apply." + [f {:keys [secrets base-values]}] + (.apply secrets + (fn [smap] + (let [m (js->clj smap :keywordize-keys true) + updates (f {:function-keys m}) + result (clj->js (deep-merge base-values updates))] + result)))) + +(defn safe-parse-int [s] + (let [n (js/parseInt s 10)] + (if (js/isNaN n) nil n))) + +(defn string->int? [s] + (and (string? s) + (re-matches #"^-?\d+$" s))) + +(defn- coerce-value [v] + (if (string->int? v) + (safe-parse-int v) + v)) + +;; Whitelist functions for resolving templates. Intended to be extended. +(def ^:private safe-fns + {'str str + 'make-paths make-paths}) + +(defn resolve-template [template values secondary-values] + (walk/postwalk + (fn [x] + (cond + (and (list? x) (contains? safe-fns (first x))) + (apply (get safe-fns (first x)) (rest x)) + (symbol? x) + (if (contains? safe-fns x) + x + (let [kw (keyword x)] + (cond + (contains? values x) (coerce-value (get values x)) + (contains? values kw) (coerce-value (get values kw)) + (contains? secondary-values x) (coerce-value (get secondary-values x)) + (contains? secondary-values kw) (coerce-value (get secondary-values kw)) + :else x))) + :else x)) + template)) + +(defn generic-transform + "Takes a creator function and executes it with resolved arguments, + handling asynchronicity when secrets are present." + [creator-fn opts base-values secrets options] + (if (nil? secrets) + (let [final-args (clj->js (deep-merge base-values (resolve-template opts {} options)))] + (creator-fn final-args)) + (.apply secrets + (fn [smap] + (let [m (js->clj smap :keywordize-keys true) + final-args (clj->js (deep-merge base-values (resolve-template opts m options)))] + ;;(js/console.log final-args) + (creator-fn final-args)))))) + + +(defn resource-factory + [component-specs] + (fn [resource-type provider app-name dependencies opts] + (let [spec (get component-specs resource-type) + resource-class (:constructor spec)] + (if resource-class + (new-resource resource-class app-name opts provider dependencies) + (throw (js/Error. (str "Unknown resource type: " resource-type))))))) + + +(defn component-factory [create-resource] + (fn [requested-components + resource-type + provider + app-name + dependencies + component-opts + defaults + secrets + options] + + (when (requested-components resource-type) + (generic-transform + (fn [final-args] + (create-resource resource-type provider app-name dependencies final-args)) + component-opts + defaults + secrets + options)))) + +(defn deploy-stack-factory [func] + (fn [& args] + (let [[component-kws [options]] (split-with keyword? args) + requested-components (set component-kws)] + (func requested-components options)))) + +(defn iterate-stack + [provider vault-data options secrets requested-components create-component-fn component-specs lifecycle-hooks] + (let [base-components + (reduce + (fn [acc [k {:keys [deps-fn opts-key defaults-fn]}]] + (let [env {:provider provider + :options options + :secrets secrets + :components acc} + app-name (get options :resource-name) + deps (deps-fn env) + opts (get options opts-key) + defaults (defaults-fn env) + component (create-component-fn requested-components k provider app-name deps opts defaults secrets options)] + (assoc acc k component))) + {:vault-secrets vault-data} + (select-keys component-specs requested-components)) + + final-components + (if lifecycle-hooks + (reduce + (fn [acc k] + (if-let [hook (get lifecycle-hooks k)] + (assoc acc k (hook {:options options + :components acc + :secrets secrets})) + acc)) + base-components + requested-components) + base-components)] + final-components)) + +(defn flatten-resource-groups + "Transforms a nested resource map into a flat, qualified-keyword map. + Example: + (flatten-resource-groups {:k8s {:chart {} :ingress {}}}) + => {:k8s:chart {} :k8s:ingress {}}" + [config] + (into {} + (mapcat + (fn [[k v]] + (if (and (keyword? k) (map? v)) + (map (fn [[inner-k inner-v]] + [(keyword (name k) (name inner-k)) inner-v]) + v) + [[k v]])) + config))) + + +(defn- is-output? [x] (some? (and x (.-__pulumiOutput x)))) + +(defn p-apply-or-resolve + "Runtime helper. If 'v' is an Output, applies 'f' to it. + If 'v' is a plain value, calls 'f' with it." + [v f] + (if (is-output? v) + (.apply v f) + (f v))) + +(defn is-output? [x] + (some? (and x (.-__pulumiOutput x)))) + +(defn p-chain [v f] + (if (is-output? v) + (.apply v f) + (f v))) + +(defn p-map [v f] + (p-chain v #(f %))) + +(defn p-lift [v] + (if (is-output? v) v (js/Promise.resolve v))) diff --git a/src/main/utils/harbor.cljs b/src/main/utils/harbor.cljs new file mode 100644 index 0000000..0b2248e --- /dev/null +++ b/src/main/utils/harbor.cljs @@ -0,0 +1,36 @@ +(ns utils.harbor + (:require + ["@pulumiverse/harbor" :as harbor])) + +(defn project [{:keys [app-name]}] + {:name app-name + :public false}) + +(defn robot-account [{:keys [app-name]}] + {:name (str app-name "-robot") + :level "project" + :permissions [{:kind "project" + :namespace app-name + :access [{:action "push" :resource "repository"} + {:action "pull" :resource "repository"} + {:action "list" :resource "repository"}]}]}) + +(def defaults + {:project project + :robot-account robot-account}) + +(def provider-template + {:constructor (.. harbor -Provider) + :name "harbor-provider" + :config {:url 'url + :username 'username + :password 'password}}) + + +(def component-specs-defs + {:root-sym 'harbor + :provider-key :harbor + :resources + {:project {:path ['-Project]} + :robot-account {:path ['-RobotAccount] + :defaults-name 'robot}}}) \ No newline at end of file diff --git a/src/main/utils/k8s.cljs b/src/main/utils/k8s.cljs new file mode 100644 index 0000000..e2fa818 --- /dev/null +++ b/src/main/utils/k8s.cljs @@ -0,0 +1,198 @@ +(ns utils.k8s (:require ["@pulumi/kubernetes" :as k8s])) + + + +(defn cluster-issuer [{:keys [host is-prod?]}] + {:metadata {:name (if is-prod? "letsencrypt-prod" "letsencrypt-staging")} + :spec {:acme {:email "admin@example.com" + :server (if is-prod? "https://acme-v02.api.letsencrypt.org/directory" "https://acme-staging-v02.api.letsencrypt.org/directory") + :privateKeySecretRef {:name (if is-prod? "account-key-prod" "account-key-staging")} + :solvers [{:dns01 {:cloudflare {:apiTokenSecretRef + {:name "api-token-secret" + :key "apiToken"}}} + :selector {:dnsZones [(or host "example.com")]}}]}}}) + +(defn certificate + [{:keys [app-name app-namespace host is-prod?]}] + (let [secret-name (str app-name "-cert") + domain (or host (str app-name ".example.com")) + issuer-name (if is-prod? + "letsencrypt-prod" + "letsencrypt-staging")] + + {:apiVersion "cert-manager.io/v1" + :kind "Certificate" + :metadata {:name (str app-name "-cert") + :namespace app-namespace} + :spec {:secretName secret-name + :issuerRef {:name issuer-name + :kind "ClusterIssuer"} + :dnsNames [domain]}})) + + +(defn gateway + [{:keys [app-name]}] + {:apiVersion "gateway.networking.k8s.io/v1" + :kind "Gateway" + :metadata {:name "main-gateway" + :namespace "traefik"} + :spec {:gatewayClassName "traefik" + :listeners + [{:name "http" + :protocol "HTTP" + :port 80} + {:name "https" + :protocol "HTTPS" + :port 443 + :tls {:certificateRefs + [{:name (str app-name "-cert") + :kind "Secret"}]}}]}}) + + +(defn httproute [{:keys [app-name app-namespace host]}] + {:apiVersion "gateway.networking.k8s.io/v1" + :kind "HTTPRoute" + :metadata {:name (str app-name "-route") + :namespace app-namespace} + :spec {:parentRefs [{:name "main-gateway" + :namespace "traefik"}] + :hostnames [host] + :rules [{:matches [{:path {:type "PathPrefix" + :value "/"}}] + :backendRefs [{:name app-name + :port 80}]}]}}) + +(defn ingress [{:keys [app-name app-namespace host]}] + {:metadata {:name app-name + :namespace app-namespace} + :spec {:ingressClassName "caddy" + :rules [{:host host + :http {:paths [{:path "/" + :pathType "Prefix" + :backend {:service {:name app-name + :port {:number 80}}}}]}}]}}) + +(defn chart [{:keys [app-name app-namespace]}] + {:chart app-name + :namespace app-namespace + :transformations []}) + +(defn config-map [{:keys [app-name app-namespace]}] + {:metadata {:namespace app-namespace + :name app-name} + :data {}}) + +(defn service [{:keys [app-name app-namespace image-port]}] + {:metadata {:namespace app-namespace + :name app-name} + :spec {:selector {:app app-name} + :ports [{:port 80 :targetPort image-port}]}}) + +(defn deployment [{:keys [app-name app-namespace image image-port]}] + {:metadata {:namespace app-namespace + :name app-name} + :spec {:selector {:matchLabels {:app app-name}} + :replicas 1 + :template {:metadata {:labels {:app app-name}} + :spec {:containers + [{:name app-name + :image image + :ports [{:containerPort image-port}]}]}}}}) + + +(defn nspace [{:keys [app-namespace]}] + {:metadata {:name app-namespace}}) + +(defn secret [{:keys [app-name app-namespace]}] + {:metadata {:name (str app-name "-secrets") + :namespace app-namespace}}) + +(defn storage-class [{:keys [app-name]}] + {:metadata {:name app-name}}) + +(def defaults + {:ingress ingress + :gateway gateway + :httproute httproute + :certificate certificate + :cluster-issuer cluster-issuer + :chart chart + :config-map config-map + :service service + :deployment deployment + :namespace nspace + :secret secret + :storage-class storage-class}) + + +(def component-specs-defs + {:root-sym 'k8s + :provider-key :k8s + :resources + {:config-map {:path ['-core '-v1 '-ConfigMap]} + :storage-class {:path ['-core '-v1 '-StorageClass]} + :namespace {:path ['-core '-v1 '-Namespace]} + :secret {:path ['-core '-v1 '-Secret]} + :deployment {:path ['-apps '-v1 '-Deployment]} + :service {:path ['-core '-v1 '-Service]} + :ingress {:path ['-networking '-v1 '-Ingress]} + :chart {:path ['-helm '-v3 '-Chart] + :defaults-fn + '(fn [env] + (deep-merge (default/chart (:options env)) + (update-in (get-in (:options env) [:k8s:chart-opts]) [:values] + #(deep-merge % (or (:yaml-values (:options env)) {})))))}}}) + +#_(def component-specs + :k8s:namespace {:constructor (.. k8s -core -v1 -Namespace) + :provider-key :k8s + :defaults-fn (fn [env] (defaults/namespace (:options env)))} + + :k8s:secret {:constructor (.. k8s -core -v1 -Secret) + :provider-key :k8s + :defaults-fn (fn [env] (default/secret (:options env)))} + + :k8s:deployment {:constructor (.. k8s -apps -v1 -Deployment) + :provider-key :k8s + :defaults-fn (fn [env] (default/deployment (:options env)))} + + :k8s:service {:constructor (.. k8s -core -v1 -Service) + :provider-key :k8s + :defaults-fn (fn [env] (default/service (:options env)))} + + :k8s:ingress {:constructor (.. k8s -networking -v1 -Ingress) + :provider-key :k8s + :defaults-fn (fn [env] (default/ingress (:options env)))} + + :k8s:chart {:constructor (.. k8s -helm -v3 -Chart) + :provider-key :k8s + :defaults-fn (fn [env] + (deep-merge (default/chart (:options env)) + (update-in (get-in (:options env) [:k8s:chart-opts]) [:values] + #(deep-merge % (or (:yaml-values (:options env)) {})))))}) + +(def provider-template + {:constructor (.. k8s -Provider) + :name "k8s-provider" + :config {:kubeconfig 'kubeconfig}}) + + +(defn pre-deploy-rule + "k8s pre-deploy rule: scans the service registry and creates + all unique namespaces. Returns a map of created namespaces + keyed by their name." + [{:keys [resource-configs provider]}] + (let [namespaces (->> resource-configs + (remove #(contains? % :no-namespace)) + (map :app-namespace) + (remove nil?) + (set))] + (into {} + (for [ns-name namespaces] + (let [resource-name ns-name + ns-config {:metadata {:name resource-name + :namespace ns-name}} + ns-resource (new (.. k8s -core -v1 -Namespace) resource-name + (clj->js ns-config) + (clj->js {:provider provider}))] + [ns-name ns-resource]))))) \ No newline at end of file diff --git a/src/main/utils/providers.cljs b/src/main/utils/providers.cljs new file mode 100644 index 0000000..a12a367 --- /dev/null +++ b/src/main/utils/providers.cljs @@ -0,0 +1,135 @@ +(ns utils.providers + (:require + ["@pulumi/pulumi" :as pulumi] ["@pulumi/vault" :as vault] ["@pulumiverse/harbor" :as harbor] ["@pulumi/kubernetes" :as k8s] + [clojure.string :as str] [clojure.walk :as walk] + [utils.general :refer [resolve-template]] + [utils.k8s :as k8s-utils] + [utils.harbor :as harbor-utils] + [utils.docker :as docker-utils] [utils.vault :as vault-utils] + [utils.stack-processor :refer [deploy! component-specs]])) + +(defn resolve-provider-template [constructor name config] + {:constructor constructor + :name name + :config config}) + +(def provider-templates + (into {} (map (fn [[k v]] [k (apply resolve-provider-template (vals v))]) + {:vault vault-utils/provider-template + :harbor harbor-utils/provider-template + :k8s k8s-utils/provider-template}))) + +(defn get-provider-outputs-config [] + {:vault {:stack :init + :outputs ["vaultAddress" "vaultToken"]} + :harbor {:stack :shared + :outputs ["username" "password" "url"]} + :k8s {:stack :init + :outputs ["kubeconfig"]}}) + + +#_(defn get-stack-refs [] + {:init (new pulumi/StackReference "init") + :shared (new pulumi/StackReference "shared")}) + +(defn get-stack-refs [stack-ref-array] + (into {} + (map (fn [stack-name] + [(keyword stack-name) + (new pulumi/StackReference stack-name)]) + stack-ref-array))) + +(defn extract-expanded-keywords [stack] + (let [expand-chain + (fn [chain] + (when (and (sequential? chain) (keyword? (first chain))) + (let [ns (or (namespace (first chain)) (name (first chain)))] + (map #(keyword ns (name %)) (rest chain)))))] + + (mapcat (fn [item] + (cond + (and (sequential? item) (keyword? (first item))) + (expand-chain item) + (keyword? item) + [item] + :else + nil)) + stack))) + + + +(defn get-all-providers [resource-configs] + (->> resource-configs + (mapcat (comp extract-expanded-keywords :stack)) + + (map (fn [component-key] + (if-let [ns (namespace component-key)] + (keyword ns) + (let [k-name (name component-key) + parts (str/split k-name #":")] + (when (> (count parts) 1) + (keyword (first parts))))))) + (remove nil?) + (into #{}) + vec)) + +(def provider-rules + {:k8s k8s-utils/pre-deploy-rule}) + + +(defn provider-apply [stack-resources-definition pulumi-cfg] + (let [providers-needed (get-all-providers (:resource-configs stack-resources-definition)) + provider-outputs-config (:provider-external-inputs stack-resources-definition) + stack-refs (get-stack-refs (:stack-references stack-resources-definition)) + needed-output-configs (select-keys provider-outputs-config providers-needed) + ;; At some point we should add the ability for Providers to be passed Pulumi configs or our config map? + ;; Cloudflare and others may require or request a token. + outputs-to-fetch (reduce-kv + (fn [acc _provider-key data] + (let [stack-key (:stack data) + stack-ref (get stack-refs stack-key) + outputs (:outputs data)] + + (reduce + (fn [m output-name] + (assoc m (keyword output-name) (.getOutput stack-ref output-name))) + acc + outputs))) + {} + needed-output-configs) + + all-provider-inputs (pulumi/all (clj->js outputs-to-fetch))] + + (.apply all-provider-inputs + (fn [values] + (js/Promise. + (fn [resolve _reject] + (let [resolved-outputs (js->clj values :keywordize-keys true) + instantiated-providers + (reduce + (fn [acc provider-key] + (if-let [template (get provider-templates provider-key)] + (let [constructor (:constructor template) + provider-name (:name template) + resolved-config (resolve-template (:config template) {} resolved-outputs)] + + (assoc acc provider-key (new constructor provider-name (clj->js resolved-config)))) + acc)) + {} + providers-needed) + pre-deploy-results + (reduce-kv + (fn [acc provider-key provider-instance] + (if-let [rule-fn (get provider-rules provider-key)] + (let [rule-results (rule-fn {:resource-configs (:resource-configs stack-resources-definition) + :provider provider-instance})] + (assoc acc provider-key rule-results)) + acc)) + {} + instantiated-providers)] + (resolve + (deploy! + {:pulumi-cfg pulumi-cfg + :resource-configs (:resource-configs stack-resources-definition) + :all-providers instantiated-providers + :pre-deploy-deps pre-deploy-results}))))))))) \ No newline at end of file diff --git a/src/main/utils/safe_fns.cljs b/src/main/utils/safe_fns.cljs new file mode 100644 index 0000000..2849f9f --- /dev/null +++ b/src/main/utils/safe_fns.cljs @@ -0,0 +1,48 @@ +(ns utils.safe-fns) + +(defn make-paths [& path-groups] + (mapcat (fn [{:keys [paths backend]}] + (mapv (fn [p] + {:path p + :pathType "Prefix" + :backend {:service backend}}) + paths)) + path-groups)) + + +(defn make-listeners [domains-or-json] + (let [domains (if (string? domains-or-json) + (js->clj (js/JSON.parse domains-or-json)) + domains-or-json)] + (vec + (mapcat + (fn [domain] + (let [clean-name (clojure.string/replace domain #"\." "-") + secret-name (str clean-name "-tls")] + + [{:name (str "https-root-" clean-name) + :port 8443 + :protocol "HTTPS" + :hostname domain + :tls {:mode "Terminate" + :certificateRefs [{:name secret-name}]} + :allowedRoutes {:namespaces {:from "All"}} + } + + {:name (str "https-wild-" clean-name) + :port 8443 + :protocol "HTTPS" + :hostname (str "*." domain) + :tls {:mode "Terminate" + :certificateRefs [{:name secret-name}]} + :allowedRoutes {:namespaces {:from "All"}} + }])) + domains)))) + +(def ^:public safe-fns + {'str str + 'b64e (fn [s] (-> (.from js/Buffer s) (.toString "base64"))) + 'println #(js/console.log %) + 'make-paths make-paths + 'make-listeners make-listeners + 'parse #(js->clj (js/JSON.parse %))}) \ No newline at end of file diff --git a/src/main/utils/stack_processor.cljs b/src/main/utils/stack_processor.cljs new file mode 100644 index 0000000..fe24941 --- /dev/null +++ b/src/main/utils/stack_processor.cljs @@ -0,0 +1,406 @@ +(ns utils.stack-processor + (:require + ["@pulumi/kubernetes" :as k8s] + ["@local/crds/gateway" :as gateway-api] + ["@local/crds/cert_manager" :as cert-manager] + ["@pulumi/pulumi" :as pulumi] + ["@pulumi/vault" :as vault] + ["@pulumiverse/harbor" :as harbor] + [utils.defaults :as default] + [utils.vault :as vault-utils] + [utils.general :refer [deep-merge new-resource resource-factory deploy-stack-factory iterate-stack]] + ["@pulumi/docker" :as docker] + ["@pulumi/docker-build" :as docker-build] + [clojure.walk :as walk] + [clojure.string :as str] + ["path" :as path] + [configs :refer [cfg]] + [utils.k8s :as k8s-utils] + [utils.harbor :as harbor-utils] + [utils.docker :as docker-utils] + [utils.safe-fns :refer [safe-fns]]) + (:require-macros [utils.general :refer [p-> build-registry]])) + + +#_(def component-specs-defs + {:k8s k8s-utils/component-specs-defs + :harbor harbor-utils/component-specs-defs + :docker docker-utils/component-specs-defs}) + +#_(def component-specs (build-registry component-specs-defs)) + +(defn safe-parse-int [s] + (let [n (js/parseInt s 10)] + (if (js/isNaN n) nil n))) + +(defn string->int? [s] + (and (string? s) + (re-matches #"^-?\d+$" s))) + +(defn- coerce-value [v] + (if (string->int? v) + (safe-parse-int v) + v)) + + +(defn- is-output? [x] (some? (.-__pulumiOutput x))) + +(defn resolve-template + [template secrets-map options-map] + (let [data (merge (js->clj options-map) (js->clj secrets-map))] + (walk/postwalk + (fn [x] + (cond + (and (list? x) (contains? safe-fns (first x))) + (let [f (get safe-fns (first x)) + args (rest x)] + (if (some is-output? args) + (.apply (pulumi/all (clj->js args)) + (fn [resolved-args] + (apply f (js->clj resolved-args)))) + (apply f args))) + (and (list? x) (symbol? (first x))) + (cond + (= (first x) '->) + (let [[_ resource-key & steps] x + resource (get data resource-key)] + (if-not resource x + (reduce (fn [acc step] + (cond + (and (symbol? step) (str/starts-with? (name step) ".-")) + (let [prop-name (subs (name step) 2)] + (aget acc prop-name)) + (and (list? step) (= (first step) 'get)) + (get acc (second step) (nth step 2 nil)) + :else acc)) + resource + steps))) + + (= (first x) 'get) (get data (second x) (nth x 2 nil)) + (= (first x) 'get-in) (get-in data (second x) (nth x 2 nil)) + :else x) + (symbol? x) + (if (contains? safe-fns x) + x + (let [kw (keyword x)] + (cond + (contains? data x) (coerce-value (get data x)) + (contains? data kw) (coerce-value (get data kw)) + :else x))) + :else x)) + template))) + + +(defn generic-transform + "Takes a creator function and executes it with resolved arguments, + handling asynchronicity when secrets are present." + [creator-fn opts base-values secrets options] + (if (nil? secrets) + (let [final-args (clj->js (deep-merge base-values (resolve-template opts {} options)))] + (pulumi/output (creator-fn final-args))) + (.apply secrets + (fn [smap] + (let [m (js->clj smap :keywordize-keys true) + final-args (clj->js (deep-merge base-values (resolve-template opts m options)))] + (creator-fn final-args)))))) + +(def component-specs + {:vault {:provider-key :vault + :provider-deps [:k8s]} + ;; K8s Resources + :k8s:namespace {:constructor (.. k8s -core -v1 -Namespace) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :namespace]) (:options env)))} + + :k8s:secret {:constructor (.. k8s -core -v1 -Secret) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :secret]) (:options env)))} + + :k8s:config-map {:constructor (.. k8s -core -v1 -ConfigMap) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :config-map]) (:options env)))} + + :k8s:deployment {:constructor (.. k8s -apps -v1 -Deployment) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :deployment]) (:options env)))} + + :k8s:service {:constructor (.. k8s -core -v1 -Service) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :service]) (:options env)))} + + :k8s:ingress {:constructor (.. k8s -networking -v1 -Ingress) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :ingress]) (:options env)))} + + :k8s:chart {:constructor (.. k8s -helm -v3 -Chart) + :provider-key :k8s + :defaults-fn (fn [env] + (deep-merge ((get-in default/defaults [:k8s :chart]) (:options env)) + (update-in (get-in (:options env) [:k8s:chart-opts]) [:values] + #(deep-merge % (or (:yaml-values (:options env)) {})))))} + :k8s:storage-class {:constructor (.. k8s -storage -v1 -StorageClass) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :storage-class]) (:options env)))} + + :k8s:pvc {:constructor (.. k8s -storage -v1 -PVC) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :pvc]) (:options env)))} + + :k8s:gateway {:constructor (.. gateway-api -v1 -Gateway) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :gateway]) (:options env)))} + + :k8s:httproute {:constructor (.. gateway-api -v1 -HTTPRoute) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :httproute]) (:options env)))} + + :k8s:cluster-issuer {:constructor (.. cert-manager -v1 -ClusterIssuer) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :cluster-issuer]) (:options env)))} + + :k8s:certificates + {:constructor (.. cert-manager -v1 -Certificate) + :provider-key :k8s + :defaults-fn (fn [env] + (let [{:keys [app-namespace is-prod?]} (:options env)] + (p-> env :options :vault:prepare "stringData" .-domains + #(vec + (for [domain (js/JSON.parse %)] + (let [clean-name (clojure.string/replace domain #"\." "-")] + {:_suffix clean-name + :metadata {:namespace app-namespace} + :spec {:dnsNames [domain (str "*." domain)] + :secretName (str clean-name "-tls") + :issuerRef {:name (if is-prod? "letsencrypt-prod" "letsencrypt-staging") + :kind "ClusterIssuer"}}}))))))} + + :k8s:certificate + {:constructor (.. cert-manager -v1 -Certificate) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :certificate]) (:options env)))} + + ;; Docker Resources + :docker:image {:constructor (.. docker-build -Image) + :provider-key :docker + :defaults-fn (fn [env] ((get-in default/defaults [:docker :image]) (:options env)))} + + ;; Harbor Resources + :harbor:project {:constructor (.. harbor -Project) + :provider-key :harbor + :defaults-fn (fn [env] ((get-in default/defaults [:harbor :project]) (:options env)))} + + :harbor:robot-account {:constructor (.. harbor -RobotAccount) + :provider-key :harbor + :defaults-fn (fn [env] ((get-in default/defaults [:harbor :robot-account]) (:options env)))}}) + +(defmulti deploy-resource + "Generic resource deployment multimethod. + Dispatches on the fully-qualified resource keyword. + Returns a map of {:resource (the-pulumi-resource) :common-opts-update {map-of-new-state}}." + (fn [dispatch-key _config] dispatch-key)) + +(defmethod deploy-resource :default + [dispatch-key full-config] + + (if-let [spec (get component-specs dispatch-key)] + (let [app-name (:app-name full-config) + dependsOn (:dependsOn full-config) + provider-key (:provider-key spec) + provider (get full-config provider-key) + resource-class (:constructor spec) + opts-key (keyword (str (name dispatch-key) "-opts")) + component-opts (get full-config opts-key) + env {:options full-config :secrets (:secrets full-config) :component-opts component-opts} + raw-defaults (when-let [df (:defaults-fn spec)] (df env))] + + (if resource-class + (let [base-creator (fn [final-args suffix] + (let [final-name (if suffix + (str app-name "-" suffix) + app-name)] + (new-resource resource-class + final-name + final-args + provider + dependsOn)))] + + {:resource + (p-> raw-defaults + #(let [defaults-list (if (vector? %) + % + [%]) + is-multi? (vector? %) resources + (doall + (map-indexed + (fn [idx item] + (let [suffix (cond + (:_suffix item) (:_suffix item) + is-multi? (str idx) + :else nil) + clean-item (dissoc item :_suffix) + item-creator (fn [resolved-args] + (base-creator resolved-args suffix))] + + (generic-transform item-creator + component-opts + clean-item + (:secrets env) + full-config))) + defaults-list))] + (if is-multi? resources (first resources))))}) + + (throw (js/Error. (str "No :constructor found for spec: " dispatch-key))))) + + (throw (js/Error. (str "Unknown resource: " dispatch-key))))) + +(defmethod deploy-resource :vault:prepare + [_ config] + (let [prepare-opts (get config :vault:prepare-opts {}) + defaults {:provider (:k8s config) + :vault-provider (:vault config) + :app-name (:app-name config) + :app-namespace (:app-namespace config) + :load-yaml (get config :vault-load-yaml false)} + final-args (merge defaults prepare-opts) + + prepared-vault-data (try + (vault-utils/prepare final-args) + (catch js/Error e + (js/console.error "!!! Error in :vault:prepare :" e) + nil))] + {:common-opts-update prepared-vault-data + :resource (:bind-secrets prepared-vault-data)})) + +(defmethod deploy-resource :vault:retrieve + [_ config] + (let [retrieve-opts (get config :vault:retrieve-opts {}) + defaults {:vault-provider (:vault config) + :app-name (:app-name config) + :app-namespace (:app-namespace config)} + final-args (merge defaults retrieve-opts) + retrieved-data (try + (vault-utils/retrieve (:vault-provider final-args) + (:app-name final-args) + (:app-namespace final-args)) + (catch js/Error e + (js/console.error " Error in :vault:retrieve :" e) + nil))] + {:common-opts-update retrieved-data})) + +;; https://www.pulumi.com/docs/iac/concepts/resources/dynamic-providers/ +(defmethod deploy-resource :generic:execute + [_ full-config] + (let [app-name (:app-name full-config) + dependsOn (:dependsOn full-config) + component-opts (assoc (:execute-opts full-config) + :pulumi-cfg (:pulumi-cfg full-config) + :secrets (:secrets full-config) + ) + defaults {} + exec-fn (:exec-fn full-config) + resource-id (str app-name "-exec") + provider #js {:create (fn [inputs-js] + (js/Promise. + (fn [resolve _reject] + (resolve + #js {:id resource-id + :outs inputs-js})))) + :delete (fn [id old-inputs-js] + (js/Promise.resolve)) + :update (fn [id old-inputs-js new-inputs-js] + (js/Promise. + (fn [resolve _reject] + (resolve #js {:outs new-inputs-js}))))} + gen (generic-transform #(clj->js (exec-fn (js->clj % :keywordize-keys true))) component-opts defaults (:secrets full-config) full-config) + creator-fn (fn [inputs] + (pulumi/dynamic.Resource. + provider + resource-id + inputs + (clj->js {:dependsOn (vec dependsOn)}))) + resource (.apply gen #(creator-fn %))] + {:resource resource})) + + +(defn handle-keyword-item [last-resource item config common-opts] + (let [dispatch-key item + depends-on (when last-resource [last-resource]) + final-config (merge config common-opts {:dependsOn depends-on}) + result-map (deploy-resource dispatch-key final-config) + resource (:resource result-map) + opts-update (:common-opts-update result-map) + resource-update (when resource {dispatch-key resource})] + + [resource + resource-update + (merge common-opts opts-update resource-update)])) + +(defn handle-list-item [last-resource item config common-opts] + (let [provider-key (first item) + resource-keys (rest item) + + nested-result + (reduce + (fn [nested-acc resource-key] + (let [inner-last-resource (get nested-acc :last-resource) + inner-resources-map (get nested-acc :resources) + inner-common-opts (get nested-acc :common-opts) + dispatch-key (keyword (str (name provider-key) ":" (name resource-key))) + [new-resource new-resource-map new-common-opts] + (handle-keyword-item inner-last-resource dispatch-key config inner-common-opts)] + {:last-resource (or new-resource inner-last-resource) + :resources (merge inner-resources-map new-resource-map) + :common-opts new-common-opts})) + {:last-resource last-resource + :resources {} + :common-opts common-opts} + + resource-keys)] + + [(:last-resource nested-result) + (:resources nested-result) + (:common-opts nested-result)])) + +(defn process-stack + "Recursively processes a stack configuration, building a dependency chain. + Returns a map of all created resources keyed by their dispatch keyword." + [stack-items config initial-common-opts] + (let [result + (reduce + (fn [acc item] + (let [{:keys [last-resource common-opts]} acc + [new-resource new-resources new-common-opts] + (if (keyword? item) + (handle-keyword-item last-resource item config common-opts) + (handle-list-item last-resource item config common-opts))] + {:last-resource (or new-resource last-resource) + :resources-map (merge (:resources-map acc) new-resources) + :common-opts new-common-opts})) + {:last-resource nil + :resources-map {} + :common-opts initial-common-opts} + stack-items)] + (:resources-map result))) + +(defn deploy! [{:keys [pulumi-cfg resource-configs all-providers]}] + (let [ + + deployment-results + (into + {} + (for [config resource-configs] + (let [ + {:keys [stack app-name]} config + _ (when (nil? config) + (throw (js/Error. "Resource configs contain a nil value!"))) + + common-opts (merge + all-providers + (select-keys config [:app-name :app-namespace]) + {:pulumi-cfg pulumi-cfg}) + ] + + [app-name (process-stack stack config common-opts)]))) + ] + (clj->js deployment-results))) + diff --git a/src/main/utils/vault.cljs b/src/main/utils/vault.cljs new file mode 100644 index 0000000..7751261 --- /dev/null +++ b/src/main/utils/vault.cljs @@ -0,0 +1,71 @@ +(ns utils.vault + (:require + ["@pulumi/kubernetes" :as k8s] + ["@pulumi/pulumi" :as pulumi] + ["@pulumi/vault" :as vault] + ["fs" :as fs] + ["js-yaml" :as yaml] + ["path" :as path] + [configs :refer [cfg]])) + +(defn get-secret-val + "Extract a specific key from a Vault secret Output/Promise." + [secret-promise key] + (.then secret-promise #(aget (.-data %) key))) + +(defn initialize-mount [vault-provider vault-path service-name] + (let [service-secrets (into {} (get (-> cfg :secrets-json) (keyword service-name)))] + (new (.. vault -generic -Secret) + (str service-name "-secret") + (clj->js {:path (str vault-path) + :dataJson (js/JSON.stringify (clj->js service-secrets))}) + (clj->js {:provider vault-provider})))) + +(defn prepare + "Prepares common resources and values for a deployment from a single config map." + [config] + (let [{:keys [provider vault-provider app-name app-namespace load-yaml]} config + values-path (.join path js/__dirname ".." (-> cfg :resource-path) (str app-name ".yml"))] + + (let [yaml-values (when load-yaml + (js->clj (-> values-path + (fs/readFileSync "utf8") + (yaml/load)) + :keywordize-keys true)) + {:keys [secrets-data bind-secrets]} + (when vault-provider + (let [vault-path (str "secret/" app-name) + _ (initialize-mount vault-provider vault-path app-name) + secrets (pulumi/output (.getSecret (.-generic vault) + (clj->js {:path vault-path}) + (clj->js {:provider vault-provider}))) + secrets-data (.apply secrets #(.. % -data)) + bind-secrets (when (and provider app-namespace) + (new (.. k8s -core -v1 -Secret) (str app-name "-secrets") + (clj->js {:metadata {:name (str app-name "-secrets") + :namespace app-namespace} + :stringData secrets-data}) + (clj->js {:provider provider})))] + {:secrets-data secrets-data + :bind-secrets bind-secrets}))] + + {:secrets secrets-data + :yaml-path values-path + :yaml-values yaml-values + :bind-secrets bind-secrets}))) + + +(defn retrieve [vault-provider app-name] + (let [vault-path (str "secret/" app-name) + secrets (pulumi/output (.getSecret (.-generic vault) + (clj->js {:path vault-path}) + (clj->js {:provider vault-provider}))) + secrets-data (.apply secrets #(.. % -data))] + {:secrets secrets-data})) + + +(def provider-template + {:constructor (.. vault -Provider) + :name "vault-provider" + :config {:address 'vaultAddress + :token 'vaultToken}})