diff --git a/iac/src/main/utils/stack_processor.cljs b/iac/src/main/utils/stack_processor.cljs new file mode 100644 index 0000000..5489512 --- /dev/null +++ b/iac/src/main/utils/stack_processor.cljs @@ -0,0 +1,350 @@ +(ns utils.stack-processor + (:require + ["@pulumi/kubernetes" :as k8s] + ["@pulumi/pulumi" :as pulumi] + ["@pulumi/vault" :as vault] + ["@pulumiverse/harbor" :as harbor] + [utils.defaults :as default] + [utils.vault :as vault-utils] + [utils.general :refer [deep-merge new-resource component-factory resource-factory deploy-stack-factory iterate-stack]] + ["@pulumi/docker" :as docker] + ["@pulumi/docker-build" :as docker-build] + [clojure.walk :as walk] + [clojure.string :as str] + ["path" :as path] + [configs :refer [cfg]] + [utils.k8s :as k8s-utils] + [utils.harbor :as harbor-utils] + [utils.docker :as docker-utils]) + (:require-macros [utils.general :refer [build-registry]])) + + +#_(def component-specs-defs + {:k8s k8s-utils/component-specs-defs + :harbor harbor-utils/component-specs-defs + :docker docker-utils/component-specs-defs}) + +#_(def component-specs (build-registry component-specs-defs)) + +(defn make-paths [& path-groups] + (mapcat (fn [{:keys [paths backend]}] + (mapv (fn [p] + {:path p + :pathType "Prefix" + :backend {:service backend}}) + paths)) + path-groups)) + + +(defn safe-parse-int [s] + (let [n (js/parseInt s 10)] + (if (js/isNaN n) nil n))) + +(defn string->int? [s] + (and (string? s) + (re-matches #"^-?\d+$" s))) + +(defn- coerce-value [v] + (if (string->int? v) + (safe-parse-int v) + v)) + +;; Whitelist functions for resolving templates. Intended to be extended. +(def ^:private safe-fns + {'str str + 'b64e (fn [s] (-> (.from js/Buffer s) (.toString "base64"))) + 'println #(js/console.log %) + 'make-paths make-paths}) + +(defn- is-output? [x] (some? (.-__pulumiOutput x))) + +(defn resolve-template + [template secrets-map options-map] + (let [data (merge (js->clj options-map) (js->clj secrets-map))] + (walk/postwalk + (fn [x] + (cond + (and (list? x) (contains? safe-fns (first x))) + (let [f (get safe-fns (first x)) + args (rest x)] + (if (some is-output? args) + (.apply (pulumi/all (clj->js args)) + (fn [resolved-args] + (apply f (js->clj resolved-args)))) + (apply f args))) + (and (list? x) (symbol? (first x))) + (cond + (= (first x) '->) + (let [[_ resource-key & steps] x + resource (get data resource-key)] + (if-not resource x + (reduce (fn [acc step] + (cond + (and (symbol? step) (str/starts-with? (name step) ".-")) + (let [prop-name (subs (name step) 2)] + (aget acc prop-name)) + (and (list? step) (= (first step) 'get)) + (get acc (second step) (nth step 2 nil)) + :else acc)) + resource + steps))) + + (= (first x) 'get) (get data (second x) (nth x 2 nil)) + (= (first x) 'get-in) (get-in data (second x) (nth x 2 nil)) + :else x) + (symbol? x) + (if (contains? safe-fns x) + x + (let [kw (keyword x)] + (cond + (contains? data x) (coerce-value (get data x)) + (contains? data kw) (coerce-value (get data kw)) + :else x))) + :else x)) + template))) + + +(defn generic-transform + "Takes a creator function and executes it with resolved arguments, + handling asynchronicity when secrets are present." + [creator-fn opts base-values secrets options] + (if (nil? secrets) + (let [final-args (clj->js (deep-merge base-values (resolve-template opts {} options)))] + (pulumi/output (creator-fn final-args))) + (.apply secrets + (fn [smap] + (let [m (js->clj smap :keywordize-keys true) + final-args (clj->js (deep-merge base-values (resolve-template opts m options)))] + (creator-fn final-args)))))) + +(def component-specs + {:vault {:provider-key :vault + :provider-deps [:k8s]} + ;; K8s Resources + :k8s:namespace {:constructor (.. k8s -core -v1 -Namespace) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :namespace]) (:options env)))} + + :k8s:secret {:constructor (.. k8s -core -v1 -Secret) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :secret]) (:options env)))} + + :k8s:deployment {:constructor (.. k8s -apps -v1 -Deployment) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :deployment]) (:options env)))} + + :k8s:service {:constructor (.. k8s -core -v1 -Service) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :service]) (:options env)))} + + :k8s:ingress {:constructor (.. k8s -networking -v1 -Ingress) + :provider-key :k8s + :defaults-fn (fn [env] ((get-in default/defaults [:k8s :ingress]) (:options env)))} + + :k8s:chart {:constructor (.. k8s -helm -v3 -Chart) + :provider-key :k8s + :defaults-fn (fn [env] + (deep-merge ((get-in default/defaults [:k8s :chart]) (:options env)) + (update-in (get-in (:options env) [:k8s:chart-opts]) [:values] + #(deep-merge % (or (:yaml-values (:options env)) {})))))} + + ;; Docker Resources + :docker:image {:constructor (.. docker-build -Image) + :provider-key :docker + :defaults-fn (fn [env] ((get-in default/defaults [:docker :image]) (:options env)))} + + ;; Harbor Resources + :harbor:project {:constructor (.. harbor -Project) + :provider-key :harbor + :defaults-fn (fn [env] ((get-in default/defaults [:harbor :project]) (:options env)))} + :harbor:robot-account {:constructor (.. harbor -RobotAccount) + :provider-key :harbor + :defaults-fn (fn [env] ((get-in default/defaults [:harbor :robot-account]) (:options env)))}}) + +(defmulti deploy-resource + "Generic resource deployment multimethod. + Dispatches on the fully-qualified resource keyword. + Returns a map of {:resource (the-pulumi-resource) :common-opts-update {map-of-new-state}}." + (fn [dispatch-key _config] dispatch-key)) + +(defmethod deploy-resource :default + [dispatch-key + full-config] + + (if-let [spec (get component-specs dispatch-key)] + (let [app-name (:app-name full-config) + dependsOn (:dependsOn full-config) + provider-key (:provider-key spec) + provider (get full-config provider-key) + opts-key (keyword (str (name dispatch-key) "-opts")) + component-opts (get full-config opts-key) + env {:options full-config :secrets (:secrets full-config)} + defaults (when-let [defaults-fn (:defaults-fn spec)] + (defaults-fn env)) + + resource-class (:constructor spec)] + + (if resource-class + (let [creator-fn (fn [final-args] + (new-resource resource-class + app-name + final-args + provider + dependsOn)) + resource (generic-transform creator-fn component-opts defaults (:secrets env) full-config)] + + {:resource resource}) + (throw (js/Error. (str "No :constructor found for spec: " dispatch-key))))) + + (throw (js/Error. (str "Unknown resource: " dispatch-key))))) + +(defmethod deploy-resource :vault:prepare + [_ config] + (let [prepare-opts (get config :vault:prepare-opts {}) + defaults {:provider (:k8s config) + :vault-provider (:vault config) + :app-name (:app-name config) + :app-namespace (:app-namespace config) + :load-yaml (get config :vault-load-yaml false)} + final-args (merge defaults prepare-opts) + prepared-vault-data (try + (vault-utils/prepare final-args) + (catch js/Error e + (js/console.error "!!! Error in :vault:prepare :" e) + nil))] + {:common-opts-update prepared-vault-data})) + +(defmethod deploy-resource :vault:retrieve + [_ config] + (let [retrieve-opts (get config :vault:retrieve-opts {}) + defaults {:vault-provider (:vault config) + :app-name (:app-name config) + :app-namespace (:app-namespace config)} + final-args (merge defaults retrieve-opts) + retrieved-data (try + (vault-utils/retrieve (:vault-provider final-args) + (:app-name final-args) + (:app-namespace final-args)) + (catch js/Error e + (js/console.error " Error in :vault:retrieve :" e) + nil))] + {:common-opts-update retrieved-data})) + +;; https://www.pulumi.com/docs/iac/concepts/resources/dynamic-providers/ +(defmethod deploy-resource :generic:execute + [_ full-config] + (let [app-name (:app-name full-config) + dependsOn (:dependsOn full-config) + component-opts (assoc (:execute-opts full-config) + :pulumi-cfg (:pulumi-cfg full-config)) + env {:options full-config :secrets (:secrets full-config)} + defaults {} + exec-fn (:exec-fn full-config) + resource-id (str app-name "-exec") + provider #js {:create (fn [inputs-js] + (js/Promise. + (fn [resolve _reject] + (let [outs-js "cats"] + (resolve + #js {:id resource-id + :outs inputs-js}))))) + :delete (fn [id old-inputs-js] + (js/Promise.resolve)) + :update (fn [id old-inputs-js new-inputs-js] + (js/Promise. + (fn [resolve _reject] + (let [new-outs-js "cats"] + (resolve #js {:outs new-inputs-js})))))} + gen (generic-transform #(clj->js (exec-fn (js->clj % :keywordize-keys true))) component-opts defaults (:secrets env) full-config) + creator-fn (fn [inputs] + (pulumi/dynamic.Resource. + provider + resource-id + inputs + (clj->js {:dependsOn (vec dependsOn)}))) + resource (.apply gen #(creator-fn %))] + {:resource resource})) + + +(defn handle-keyword-item [last-resource item config common-opts] + (let [dispatch-key item + depends-on (when last-resource [last-resource]) + final-config (merge config common-opts {:dependsOn depends-on}) + result-map (deploy-resource dispatch-key final-config) + resource (:resource result-map) + opts-update (:common-opts-update result-map) + resource-update (when resource {dispatch-key resource})] + + [resource + resource-update + (merge common-opts opts-update resource-update)])) + +(defn handle-list-item [last-resource item config common-opts] + (let [provider-key (first item) + resource-keys (rest item) + + nested-result + (reduce + (fn [nested-acc resource-key] + (let [inner-last-resource (get nested-acc :last-resource) + inner-resources-map (get nested-acc :resources) + inner-common-opts (get nested-acc :common-opts) + dispatch-key (keyword (str (name provider-key) ":" (name resource-key))) + [new-resource new-resource-map new-common-opts] + (handle-keyword-item inner-last-resource dispatch-key config inner-common-opts)] + {:last-resource (or new-resource inner-last-resource) + :resources (merge inner-resources-map new-resource-map) + :common-opts new-common-opts})) + {:last-resource last-resource + :resources {} + :common-opts common-opts} + + resource-keys)] + + [(:last-resource nested-result) + (:resources nested-result) + (:common-opts nested-result)])) + +(defn process-stack + "Recursively processes a stack configuration, building a dependency chain. + Returns a map of all created resources keyed by their dispatch keyword." + [stack-items config initial-common-opts] + (let [result + (reduce + (fn [acc item] + (let [{:keys [last-resource common-opts]} acc + [new-resource new-resources new-common-opts] + (if (keyword? item) + (handle-keyword-item last-resource item config common-opts) + (handle-list-item last-resource item config common-opts))] + {:last-resource (or new-resource last-resource) + :resources-map (merge (:resources-map acc) new-resources) + :common-opts new-common-opts})) + {:last-resource nil + :resources-map {} + :common-opts initial-common-opts} + stack-items)] + (:resources-map result))) + +(defn deploy! [{:keys [pulumi-cfg service-registry all-providers]}] + (let [ + + deployment-results + (into + {} + (for [config service-registry] + (let [ + {:keys [stack app-name]} config + _ (when (nil? config) + (throw (js/Error. "Service registry contains a nil value!"))) + + common-opts (merge + all-providers + (select-keys config [:app-name :app-namespace]) + {:pulumi-cfg pulumi-cfg}) + ] + + [app-name (process-stack stack config common-opts)]))) + ] + (clj->js deployment-results))) +