Move all files to root

This commit is contained in:
2025-11-23 16:12:26 -06:00
commit 27a6ed3498
11 changed files with 1359 additions and 0 deletions

114
scripts/generate-crds.js Normal file
View File

@@ -0,0 +1,114 @@
const fs = require('fs');
const path = require('path');
const https = require('https');
const crypto = require('crypto');
const { execSync } = require('child_process');
const PROJECT_ROOT = process.cwd();
const OUTPUT_DIR = path.join(PROJECT_ROOT, 'generated/crds');
const TEMP_DIR = path.join(PROJECT_ROOT, 'temp_crds');
const CHECKSUM_FILE = path.join(PROJECT_ROOT, '.crd2pulumi-checksum');
const GW_VERSION = 'v1.1.0';
const GW_URL = `https://github.com/kubernetes-sigs/gateway-api/releases/download/${GW_VERSION}/experimental-install.yaml`;
const GW_FILE = 'gateway-api.yaml';
const CM_VERSION = 'v1.15.0';
const CM_URL = `https://github.com/cert-manager/cert-manager/releases/download/${CM_VERSION}/cert-manager.crds.yaml`;
const CM_FILE = 'cert-manager.yaml';
const downloadFile = (url, filename) => {
return new Promise((resolve, reject) => {
const destPath = path.join(TEMP_DIR, filename);
const file = fs.createWriteStream(destPath);
const request = (uri) => {
https.get(uri, (response) => {
if (response.statusCode === 301 || response.statusCode === 302) {
return request(response.headers.location);
}
if (response.statusCode !== 200) {
reject(new Error(`Failed to download ${uri}: ${response.statusCode}`));
return;
}
console.log(`Downloading ${filename}...`);
response.pipe(file);
file.on('finish', () => {
file.close();
resolve(destPath);
});
}).on('error', (err) => {
fs.unlink(destPath, () => {});
reject(err);
});
};
request(url);
});
};
const computeHash = (filePaths) => {
const hash = crypto.createHash('sha256');
filePaths.sort().forEach(fp => hash.update(fs.readFileSync(fp)));
return hash.digest('hex');
};
async function main() {
if (fs.existsSync(TEMP_DIR)) fs.rmSync(TEMP_DIR, { recursive: true, force: true });
fs.mkdirSync(TEMP_DIR);
try {
const gwPath = await downloadFile(GW_URL, GW_FILE);
const cmPath = await downloadFile(CM_URL, CM_FILE);
const allFiles = [gwPath, cmPath];
const newHash = computeHash(allFiles);
let oldHash = null;
if (fs.existsSync(CHECKSUM_FILE)) {
oldHash = fs.readFileSync(CHECKSUM_FILE, 'utf8').trim();
}
if (oldHash === newHash && fs.existsSync(OUTPUT_DIR)) {
console.log('CRDs unchanged. Skipping.');
} else {
console.log('Regenerating CRDs...');
if (fs.existsSync(OUTPUT_DIR)) fs.rmSync(OUTPUT_DIR, { recursive: true, force: true });
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
execSync(`crd2pulumi --nodejsPath "${OUTPUT_DIR}" --force "${gwPath}" "${cmPath}"`, { stdio: 'inherit' });
const pkgPath = path.join(OUTPUT_DIR, 'package.json');
if (fs.existsSync(pkgPath)) {
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
if (pkg.scripts) delete pkg.scripts;
fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2));
}
const tsconfig = {
compilerOptions: {
target: "es2020",
module: "commonjs",
moduleResolution: "node",
declaration: true,
skipLibCheck: true,
},
include: ["**/*.ts"],
exclude: ["node_modules"]
};
fs.writeFileSync(path.join(OUTPUT_DIR, 'tsconfig.json'), JSON.stringify(tsconfig, null, 2));
try {
execSync('npx tsc', { cwd: OUTPUT_DIR, stdio: 'inherit' });
} catch (e) { console.warn("TSC warnings ignored."); }
fs.writeFileSync(CHECKSUM_FILE, newHash);
console.log(`Success!`);
}
} catch (error) {
console.error(error);
process.exit(1);
} finally {
if (fs.existsSync(TEMP_DIR)) fs.rmSync(TEMP_DIR, { recursive: true, force: true });
}
}
main();

View File

@@ -0,0 +1,12 @@
(ns utils.defaults
(:require ["path" :as path]
[configs :refer [cfg]]
[utils.k8s :as k8s]
[utils.harbor :as harbor]
[utils.docker :as docker]))
(def defaults
{:k8s k8s/defaults
:harbor harbor/defaults
:docker docker/defaults})

View File

@@ -0,0 +1,25 @@
(ns utils.docker
(:require
[utils.general :refer [generic-transform deep-merge new-resource component-factory resource-factory deploy-stack-factory iterate-stack]]
["@pulumi/docker-build" :as docker]
["path" :as path]
[configs :refer [cfg]]))
(defn image [env]
(let [{:keys [app-name docker:image-opts]} env
context-path (.. path (join "." (-> cfg :resource-path)))
dockerfile-path (.. path (join context-path (str app-name ".dockerfile")))
base-args (if (:is-local docker:image-opts)
{:context {:location context-path}
:dockerfile {:location dockerfile-path}
:imageName (str (-> cfg :docker-repo) "/" app-name ":latest")}
{})]
base-args))
(def defaults
{:image image})
(def component-specs-defs
{:root-sym 'docker
:provider-key :harbor
:resources {:image {:path ['-Image]}}})

View File

@@ -0,0 +1,82 @@
(ns utils.general
(:require
[clojure.walk]))
(defmacro build-registry
"Generates a flat resource registry map from a nested provider definition map."
[definitions]
(let [map-entries
(reduce-kv
(fn [entries provider-group-kw provider-group-def]
(let [{:keys [root-sym provider-key resources]} provider-group-def
provider-ns (name provider-group-kw)]
(reduce-kv
(fn [entries resource-kw resource-def]
(let [{:keys [path defaults-fn defaults-name]} resource-def
resource-name (name resource-kw)
final-key (keyword provider-ns resource-name)
constructor-form (list* '.. root-sym path)
defaults-fn-form
(cond
defaults-fn defaults-fn
defaults-name (let [defaults-fn-sym (symbol "default" (name defaults-name))]
`(fn [env] (~defaults-fn-sym (:options env))))
:else (let [defaults-fn-sym (symbol "default" resource-name)]
`(fn [env] (~defaults-fn-sym (:options env)))))
value-map `{:constructor ~constructor-form
:provider-key ~provider-key
:defaults-fn ~defaults-fn-form}]
(conj entries [final-key value-map])))
entries
resources)))
[]
definitions)]
`~(into {} map-entries)))
(defn- p->-replace-percent
"Walks 'form' and replaces all instances of the symbol '%'
with the value of 'x'."
[form x]
(clojure.walk/postwalk
(fn [sub-form]
(if (= sub-form '%) x sub-form))
form))
(defmacro pulet
"Sequential binding for Pulumi Outputs.
Looks like let*, but each binding is chained."
[bindings & body]
(if (empty? bindings)
`(do ~@body)
(let [[sym val & rest] bindings]
`(p-chain ~val
(fn [~sym]
(pulet [~@rest] ~@body))))))
(defmacro p-> [x & forms]
(let [wrap (fn [acc form]
(cond
(map? form)
`(p-chain ~acc (fn [val#] ~form))
(keyword? form)
`(p-chain ~acc #(get % ~form))
(string? form)
`(p-chain ~acc #(aget % ~form))
(symbol? form)
`(p-chain ~acc #(~form %))
(list? form)
`(p-chain ~acc ~form)
:else
(throw (ex-info "Unsupported form in p->" {:form form}))))]
(reduce wrap x forms)))

232
src/main/utils/general.cljs Normal file
View File

@@ -0,0 +1,232 @@
(ns utils.general (:require [clojure.walk :as walk]))
(defn new-resource [resource-type resource-name final-args provider dependencies]
(let [base-opts (if (or (some? provider) (seq dependencies))
{:enableServerSideApply false} {})
opts (cond-> base-opts
(some? provider) (assoc :provider provider)
(seq dependencies) (assoc :dependsOn dependencies))]
(if (seq opts)
(new resource-type resource-name (clj->js final-args) (clj->js opts))
(new resource-type resource-name (clj->js final-args)))))
(defn assoc-ins [m path-vals]
(reduce (fn [acc [path val]] (assoc-in acc path val)) m path-vals))
(declare deep-merge)
(defn merge-by-name
"Merges two vectors of maps by :name key."
[a b]
(let [a-map (into {} (map #(vector (:name %) %) a))
b-map (into {} (map #(vector (:name %) %) b))
merged (merge-with deep-merge a-map b-map)]
(vec (vals merged))))
(defn deep-merge
"Recursively merges maps and intelligently merges vectors of maps by :name."
[a b]
(cond
(nil? b) a
(and (map? a) (map? b))
(merge-with deep-merge a b)
(and (vector? a) (vector? b)
(every? map? a) (every? map? b)
(some #(contains? % :name) (concat a b)))
(merge-by-name a b)
:else b))
(defn make-transformer
"Given f that takes {:app-name .. :secrets ..}, where :secrets is a plain map
(already unwrapped inside .apply), return a Helm transformer."
[f]
(fn [{:keys [base-values app-name secrets]}]
(.apply secrets
(fn [smap]
(let [m (js->clj smap :keywordize-keys true)
updates (f {:app-name app-name
:secrets m})
after (clj->js (assoc-ins base-values updates))]
after)))))
(defn make-paths [& path-groups]
(mapcat (fn [{:keys [paths backend]}]
(mapv (fn [p]
{:path p
:pathType "Prefix"
:backend {:service backend}})
paths))
path-groups))
(defn generic-make-transformer
"Returns a Pulumi-compatible transformer that unwraps Output values via .apply."
[f {:keys [secrets base-values]}]
(.apply secrets
(fn [smap]
(let [m (js->clj smap :keywordize-keys true)
updates (f {:function-keys m})
result (clj->js (deep-merge base-values updates))]
result))))
(defn safe-parse-int [s]
(let [n (js/parseInt s 10)]
(if (js/isNaN n) nil n)))
(defn string->int? [s]
(and (string? s)
(re-matches #"^-?\d+$" s)))
(defn- coerce-value [v]
(if (string->int? v)
(safe-parse-int v)
v))
;; Whitelist functions for resolving templates. Intended to be extended.
(def ^:private safe-fns
{'str str
'make-paths make-paths})
(defn resolve-template [template values secondary-values]
(walk/postwalk
(fn [x]
(cond
(and (list? x) (contains? safe-fns (first x)))
(apply (get safe-fns (first x)) (rest x))
(symbol? x)
(if (contains? safe-fns x)
x
(let [kw (keyword x)]
(cond
(contains? values x) (coerce-value (get values x))
(contains? values kw) (coerce-value (get values kw))
(contains? secondary-values x) (coerce-value (get secondary-values x))
(contains? secondary-values kw) (coerce-value (get secondary-values kw))
:else x)))
:else x))
template))
(defn generic-transform
"Takes a creator function and executes it with resolved arguments,
handling asynchronicity when secrets are present."
[creator-fn opts base-values secrets options]
(if (nil? secrets)
(let [final-args (clj->js (deep-merge base-values (resolve-template opts {} options)))]
(creator-fn final-args))
(.apply secrets
(fn [smap]
(let [m (js->clj smap :keywordize-keys true)
final-args (clj->js (deep-merge base-values (resolve-template opts m options)))]
;;(js/console.log final-args)
(creator-fn final-args))))))
(defn resource-factory
[component-specs]
(fn [resource-type provider app-name dependencies opts]
(let [spec (get component-specs resource-type)
resource-class (:constructor spec)]
(if resource-class
(new-resource resource-class app-name opts provider dependencies)
(throw (js/Error. (str "Unknown resource type: " resource-type)))))))
(defn component-factory [create-resource]
(fn [requested-components
resource-type
provider
app-name
dependencies
component-opts
defaults
secrets
options]
(when (requested-components resource-type)
(generic-transform
(fn [final-args]
(create-resource resource-type provider app-name dependencies final-args))
component-opts
defaults
secrets
options))))
(defn deploy-stack-factory [func]
(fn [& args]
(let [[component-kws [options]] (split-with keyword? args)
requested-components (set component-kws)]
(func requested-components options))))
(defn iterate-stack
[provider vault-data options secrets requested-components create-component-fn component-specs lifecycle-hooks]
(let [base-components
(reduce
(fn [acc [k {:keys [deps-fn opts-key defaults-fn]}]]
(let [env {:provider provider
:options options
:secrets secrets
:components acc}
app-name (get options :resource-name)
deps (deps-fn env)
opts (get options opts-key)
defaults (defaults-fn env)
component (create-component-fn requested-components k provider app-name deps opts defaults secrets options)]
(assoc acc k component)))
{:vault-secrets vault-data}
(select-keys component-specs requested-components))
final-components
(if lifecycle-hooks
(reduce
(fn [acc k]
(if-let [hook (get lifecycle-hooks k)]
(assoc acc k (hook {:options options
:components acc
:secrets secrets}))
acc))
base-components
requested-components)
base-components)]
final-components))
(defn flatten-resource-groups
"Transforms a nested resource map into a flat, qualified-keyword map.
Example:
(flatten-resource-groups {:k8s {:chart {} :ingress {}}})
=> {:k8s:chart {} :k8s:ingress {}}"
[config]
(into {}
(mapcat
(fn [[k v]]
(if (and (keyword? k) (map? v))
(map (fn [[inner-k inner-v]]
[(keyword (name k) (name inner-k)) inner-v])
v)
[[k v]]))
config)))
(defn- is-output? [x] (some? (and x (.-__pulumiOutput x))))
(defn p-apply-or-resolve
"Runtime helper. If 'v' is an Output, applies 'f' to it.
If 'v' is a plain value, calls 'f' with it."
[v f]
(if (is-output? v)
(.apply v f)
(f v)))
(defn is-output? [x]
(some? (and x (.-__pulumiOutput x))))
(defn p-chain [v f]
(if (is-output? v)
(.apply v f)
(f v)))
(defn p-map [v f]
(p-chain v #(f %)))
(defn p-lift [v]
(if (is-output? v) v (js/Promise.resolve v)))

View File

@@ -0,0 +1,36 @@
(ns utils.harbor
(:require
["@pulumiverse/harbor" :as harbor]))
(defn project [{:keys [app-name]}]
{:name app-name
:public false})
(defn robot-account [{:keys [app-name]}]
{:name (str app-name "-robot")
:level "project"
:permissions [{:kind "project"
:namespace app-name
:access [{:action "push" :resource "repository"}
{:action "pull" :resource "repository"}
{:action "list" :resource "repository"}]}]})
(def defaults
{:project project
:robot-account robot-account})
(def provider-template
{:constructor (.. harbor -Provider)
:name "harbor-provider"
:config {:url 'url
:username 'username
:password 'password}})
(def component-specs-defs
{:root-sym 'harbor
:provider-key :harbor
:resources
{:project {:path ['-Project]}
:robot-account {:path ['-RobotAccount]
:defaults-name 'robot}}})

198
src/main/utils/k8s.cljs Normal file
View File

@@ -0,0 +1,198 @@
(ns utils.k8s (:require ["@pulumi/kubernetes" :as k8s]))
(defn cluster-issuer [{:keys [host is-prod?]}]
{:metadata {:name (if is-prod? "letsencrypt-prod" "letsencrypt-staging")}
:spec {:acme {:email "admin@example.com"
:server (if is-prod? "https://acme-v02.api.letsencrypt.org/directory" "https://acme-staging-v02.api.letsencrypt.org/directory")
:privateKeySecretRef {:name (if is-prod? "account-key-prod" "account-key-staging")}
:solvers [{:dns01 {:cloudflare {:apiTokenSecretRef
{:name "api-token-secret"
:key "apiToken"}}}
:selector {:dnsZones [(or host "example.com")]}}]}}})
(defn certificate
[{:keys [app-name app-namespace host is-prod?]}]
(let [secret-name (str app-name "-cert")
domain (or host (str app-name ".example.com"))
issuer-name (if is-prod?
"letsencrypt-prod"
"letsencrypt-staging")]
{:apiVersion "cert-manager.io/v1"
:kind "Certificate"
:metadata {:name (str app-name "-cert")
:namespace app-namespace}
:spec {:secretName secret-name
:issuerRef {:name issuer-name
:kind "ClusterIssuer"}
:dnsNames [domain]}}))
(defn gateway
[{:keys [app-name]}]
{:apiVersion "gateway.networking.k8s.io/v1"
:kind "Gateway"
:metadata {:name "main-gateway"
:namespace "traefik"}
:spec {:gatewayClassName "traefik"
:listeners
[{:name "http"
:protocol "HTTP"
:port 80}
{:name "https"
:protocol "HTTPS"
:port 443
:tls {:certificateRefs
[{:name (str app-name "-cert")
:kind "Secret"}]}}]}})
(defn httproute [{:keys [app-name app-namespace host]}]
{:apiVersion "gateway.networking.k8s.io/v1"
:kind "HTTPRoute"
:metadata {:name (str app-name "-route")
:namespace app-namespace}
:spec {:parentRefs [{:name "main-gateway"
:namespace "traefik"}]
:hostnames [host]
:rules [{:matches [{:path {:type "PathPrefix"
:value "/"}}]
:backendRefs [{:name app-name
:port 80}]}]}})
(defn ingress [{:keys [app-name app-namespace host]}]
{:metadata {:name app-name
:namespace app-namespace}
:spec {:ingressClassName "caddy"
:rules [{:host host
:http {:paths [{:path "/"
:pathType "Prefix"
:backend {:service {:name app-name
:port {:number 80}}}}]}}]}})
(defn chart [{:keys [app-name app-namespace]}]
{:chart app-name
:namespace app-namespace
:transformations []})
(defn config-map [{:keys [app-name app-namespace]}]
{:metadata {:namespace app-namespace
:name app-name}
:data {}})
(defn service [{:keys [app-name app-namespace image-port]}]
{:metadata {:namespace app-namespace
:name app-name}
:spec {:selector {:app app-name}
:ports [{:port 80 :targetPort image-port}]}})
(defn deployment [{:keys [app-name app-namespace image image-port]}]
{:metadata {:namespace app-namespace
:name app-name}
:spec {:selector {:matchLabels {:app app-name}}
:replicas 1
:template {:metadata {:labels {:app app-name}}
:spec {:containers
[{:name app-name
:image image
:ports [{:containerPort image-port}]}]}}}})
(defn nspace [{:keys [app-namespace]}]
{:metadata {:name app-namespace}})
(defn secret [{:keys [app-name app-namespace]}]
{:metadata {:name (str app-name "-secrets")
:namespace app-namespace}})
(defn storage-class [{:keys [app-name]}]
{:metadata {:name app-name}})
(def defaults
{:ingress ingress
:gateway gateway
:httproute httproute
:certificate certificate
:cluster-issuer cluster-issuer
:chart chart
:config-map config-map
:service service
:deployment deployment
:namespace nspace
:secret secret
:storage-class storage-class})
(def component-specs-defs
{:root-sym 'k8s
:provider-key :k8s
:resources
{:config-map {:path ['-core '-v1 '-ConfigMap]}
:storage-class {:path ['-core '-v1 '-StorageClass]}
:namespace {:path ['-core '-v1 '-Namespace]}
:secret {:path ['-core '-v1 '-Secret]}
:deployment {:path ['-apps '-v1 '-Deployment]}
:service {:path ['-core '-v1 '-Service]}
:ingress {:path ['-networking '-v1 '-Ingress]}
:chart {:path ['-helm '-v3 '-Chart]
:defaults-fn
'(fn [env]
(deep-merge (default/chart (:options env))
(update-in (get-in (:options env) [:k8s:chart-opts]) [:values]
#(deep-merge % (or (:yaml-values (:options env)) {})))))}}})
#_(def component-specs
:k8s:namespace {:constructor (.. k8s -core -v1 -Namespace)
:provider-key :k8s
:defaults-fn (fn [env] (defaults/namespace (:options env)))}
:k8s:secret {:constructor (.. k8s -core -v1 -Secret)
:provider-key :k8s
:defaults-fn (fn [env] (default/secret (:options env)))}
:k8s:deployment {:constructor (.. k8s -apps -v1 -Deployment)
:provider-key :k8s
:defaults-fn (fn [env] (default/deployment (:options env)))}
:k8s:service {:constructor (.. k8s -core -v1 -Service)
:provider-key :k8s
:defaults-fn (fn [env] (default/service (:options env)))}
:k8s:ingress {:constructor (.. k8s -networking -v1 -Ingress)
:provider-key :k8s
:defaults-fn (fn [env] (default/ingress (:options env)))}
:k8s:chart {:constructor (.. k8s -helm -v3 -Chart)
:provider-key :k8s
:defaults-fn (fn [env]
(deep-merge (default/chart (:options env))
(update-in (get-in (:options env) [:k8s:chart-opts]) [:values]
#(deep-merge % (or (:yaml-values (:options env)) {})))))})
(def provider-template
{:constructor (.. k8s -Provider)
:name "k8s-provider"
:config {:kubeconfig 'kubeconfig}})
(defn pre-deploy-rule
"k8s pre-deploy rule: scans the service registry and creates
all unique namespaces. Returns a map of created namespaces
keyed by their name."
[{:keys [resource-configs provider]}]
(let [namespaces (->> resource-configs
(remove #(contains? % :no-namespace))
(map :app-namespace)
(remove nil?)
(set))]
(into {}
(for [ns-name namespaces]
(let [resource-name ns-name
ns-config {:metadata {:name resource-name
:namespace ns-name}}
ns-resource (new (.. k8s -core -v1 -Namespace) resource-name
(clj->js ns-config)
(clj->js {:provider provider}))]
[ns-name ns-resource])))))

View File

@@ -0,0 +1,135 @@
(ns utils.providers
(:require
["@pulumi/pulumi" :as pulumi] ["@pulumi/vault" :as vault] ["@pulumiverse/harbor" :as harbor] ["@pulumi/kubernetes" :as k8s]
[clojure.string :as str] [clojure.walk :as walk]
[utils.general :refer [resolve-template]]
[utils.k8s :as k8s-utils]
[utils.harbor :as harbor-utils]
[utils.docker :as docker-utils] [utils.vault :as vault-utils]
[utils.stack-processor :refer [deploy! component-specs]]))
(defn resolve-provider-template [constructor name config]
{:constructor constructor
:name name
:config config})
(def provider-templates
(into {} (map (fn [[k v]] [k (apply resolve-provider-template (vals v))])
{:vault vault-utils/provider-template
:harbor harbor-utils/provider-template
:k8s k8s-utils/provider-template})))
(defn get-provider-outputs-config []
{:vault {:stack :init
:outputs ["vaultAddress" "vaultToken"]}
:harbor {:stack :shared
:outputs ["username" "password" "url"]}
:k8s {:stack :init
:outputs ["kubeconfig"]}})
#_(defn get-stack-refs []
{:init (new pulumi/StackReference "init")
:shared (new pulumi/StackReference "shared")})
(defn get-stack-refs [stack-ref-array]
(into {}
(map (fn [stack-name]
[(keyword stack-name)
(new pulumi/StackReference stack-name)])
stack-ref-array)))
(defn extract-expanded-keywords [stack]
(let [expand-chain
(fn [chain]
(when (and (sequential? chain) (keyword? (first chain)))
(let [ns (or (namespace (first chain)) (name (first chain)))]
(map #(keyword ns (name %)) (rest chain)))))]
(mapcat (fn [item]
(cond
(and (sequential? item) (keyword? (first item)))
(expand-chain item)
(keyword? item)
[item]
:else
nil))
stack)))
(defn get-all-providers [resource-configs]
(->> resource-configs
(mapcat (comp extract-expanded-keywords :stack))
(map (fn [component-key]
(if-let [ns (namespace component-key)]
(keyword ns)
(let [k-name (name component-key)
parts (str/split k-name #":")]
(when (> (count parts) 1)
(keyword (first parts)))))))
(remove nil?)
(into #{})
vec))
(def provider-rules
{:k8s k8s-utils/pre-deploy-rule})
(defn provider-apply [stack-resources-definition pulumi-cfg]
(let [providers-needed (get-all-providers (:resource-configs stack-resources-definition))
provider-outputs-config (:provider-external-inputs stack-resources-definition)
stack-refs (get-stack-refs (:stack-references stack-resources-definition))
needed-output-configs (select-keys provider-outputs-config providers-needed)
;; At some point we should add the ability for Providers to be passed Pulumi configs or our config map?
;; Cloudflare and others may require or request a token.
outputs-to-fetch (reduce-kv
(fn [acc _provider-key data]
(let [stack-key (:stack data)
stack-ref (get stack-refs stack-key)
outputs (:outputs data)]
(reduce
(fn [m output-name]
(assoc m (keyword output-name) (.getOutput stack-ref output-name)))
acc
outputs)))
{}
needed-output-configs)
all-provider-inputs (pulumi/all (clj->js outputs-to-fetch))]
(.apply all-provider-inputs
(fn [values]
(js/Promise.
(fn [resolve _reject]
(let [resolved-outputs (js->clj values :keywordize-keys true)
instantiated-providers
(reduce
(fn [acc provider-key]
(if-let [template (get provider-templates provider-key)]
(let [constructor (:constructor template)
provider-name (:name template)
resolved-config (resolve-template (:config template) {} resolved-outputs)]
(assoc acc provider-key (new constructor provider-name (clj->js resolved-config))))
acc))
{}
providers-needed)
pre-deploy-results
(reduce-kv
(fn [acc provider-key provider-instance]
(if-let [rule-fn (get provider-rules provider-key)]
(let [rule-results (rule-fn {:resource-configs (:resource-configs stack-resources-definition)
:provider provider-instance})]
(assoc acc provider-key rule-results))
acc))
{}
instantiated-providers)]
(resolve
(deploy!
{:pulumi-cfg pulumi-cfg
:resource-configs (:resource-configs stack-resources-definition)
:all-providers instantiated-providers
:pre-deploy-deps pre-deploy-results})))))))))

View File

@@ -0,0 +1,48 @@
(ns utils.safe-fns)
(defn make-paths [& path-groups]
(mapcat (fn [{:keys [paths backend]}]
(mapv (fn [p]
{:path p
:pathType "Prefix"
:backend {:service backend}})
paths))
path-groups))
(defn make-listeners [domains-or-json]
(let [domains (if (string? domains-or-json)
(js->clj (js/JSON.parse domains-or-json))
domains-or-json)]
(vec
(mapcat
(fn [domain]
(let [clean-name (clojure.string/replace domain #"\." "-")
secret-name (str clean-name "-tls")]
[{:name (str "https-root-" clean-name)
:port 8443
:protocol "HTTPS"
:hostname domain
:tls {:mode "Terminate"
:certificateRefs [{:name secret-name}]}
:allowedRoutes {:namespaces {:from "All"}}
}
{:name (str "https-wild-" clean-name)
:port 8443
:protocol "HTTPS"
:hostname (str "*." domain)
:tls {:mode "Terminate"
:certificateRefs [{:name secret-name}]}
:allowedRoutes {:namespaces {:from "All"}}
}]))
domains))))
(def ^:public safe-fns
{'str str
'b64e (fn [s] (-> (.from js/Buffer s) (.toString "base64")))
'println #(js/console.log %)
'make-paths make-paths
'make-listeners make-listeners
'parse #(js->clj (js/JSON.parse %))})

View File

@@ -0,0 +1,406 @@
(ns utils.stack-processor
(:require
["@pulumi/kubernetes" :as k8s]
["@local/crds/gateway" :as gateway-api]
["@local/crds/cert_manager" :as cert-manager]
["@pulumi/pulumi" :as pulumi]
["@pulumi/vault" :as vault]
["@pulumiverse/harbor" :as harbor]
[utils.defaults :as default]
[utils.vault :as vault-utils]
[utils.general :refer [deep-merge new-resource resource-factory deploy-stack-factory iterate-stack]]
["@pulumi/docker" :as docker]
["@pulumi/docker-build" :as docker-build]
[clojure.walk :as walk]
[clojure.string :as str]
["path" :as path]
[configs :refer [cfg]]
[utils.k8s :as k8s-utils]
[utils.harbor :as harbor-utils]
[utils.docker :as docker-utils]
[utils.safe-fns :refer [safe-fns]])
(:require-macros [utils.general :refer [p-> build-registry]]))
#_(def component-specs-defs
{:k8s k8s-utils/component-specs-defs
:harbor harbor-utils/component-specs-defs
:docker docker-utils/component-specs-defs})
#_(def component-specs (build-registry component-specs-defs))
(defn safe-parse-int [s]
(let [n (js/parseInt s 10)]
(if (js/isNaN n) nil n)))
(defn string->int? [s]
(and (string? s)
(re-matches #"^-?\d+$" s)))
(defn- coerce-value [v]
(if (string->int? v)
(safe-parse-int v)
v))
(defn- is-output? [x] (some? (.-__pulumiOutput x)))
(defn resolve-template
[template secrets-map options-map]
(let [data (merge (js->clj options-map) (js->clj secrets-map))]
(walk/postwalk
(fn [x]
(cond
(and (list? x) (contains? safe-fns (first x)))
(let [f (get safe-fns (first x))
args (rest x)]
(if (some is-output? args)
(.apply (pulumi/all (clj->js args))
(fn [resolved-args]
(apply f (js->clj resolved-args))))
(apply f args)))
(and (list? x) (symbol? (first x)))
(cond
(= (first x) '->)
(let [[_ resource-key & steps] x
resource (get data resource-key)]
(if-not resource x
(reduce (fn [acc step]
(cond
(and (symbol? step) (str/starts-with? (name step) ".-"))
(let [prop-name (subs (name step) 2)]
(aget acc prop-name))
(and (list? step) (= (first step) 'get))
(get acc (second step) (nth step 2 nil))
:else acc))
resource
steps)))
(= (first x) 'get) (get data (second x) (nth x 2 nil))
(= (first x) 'get-in) (get-in data (second x) (nth x 2 nil))
:else x)
(symbol? x)
(if (contains? safe-fns x)
x
(let [kw (keyword x)]
(cond
(contains? data x) (coerce-value (get data x))
(contains? data kw) (coerce-value (get data kw))
:else x)))
:else x))
template)))
(defn generic-transform
"Takes a creator function and executes it with resolved arguments,
handling asynchronicity when secrets are present."
[creator-fn opts base-values secrets options]
(if (nil? secrets)
(let [final-args (clj->js (deep-merge base-values (resolve-template opts {} options)))]
(pulumi/output (creator-fn final-args)))
(.apply secrets
(fn [smap]
(let [m (js->clj smap :keywordize-keys true)
final-args (clj->js (deep-merge base-values (resolve-template opts m options)))]
(creator-fn final-args))))))
(def component-specs
{:vault {:provider-key :vault
:provider-deps [:k8s]}
;; K8s Resources
:k8s:namespace {:constructor (.. k8s -core -v1 -Namespace)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :namespace]) (:options env)))}
:k8s:secret {:constructor (.. k8s -core -v1 -Secret)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :secret]) (:options env)))}
:k8s:config-map {:constructor (.. k8s -core -v1 -ConfigMap)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :config-map]) (:options env)))}
:k8s:deployment {:constructor (.. k8s -apps -v1 -Deployment)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :deployment]) (:options env)))}
:k8s:service {:constructor (.. k8s -core -v1 -Service)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :service]) (:options env)))}
:k8s:ingress {:constructor (.. k8s -networking -v1 -Ingress)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :ingress]) (:options env)))}
:k8s:chart {:constructor (.. k8s -helm -v3 -Chart)
:provider-key :k8s
:defaults-fn (fn [env]
(deep-merge ((get-in default/defaults [:k8s :chart]) (:options env))
(update-in (get-in (:options env) [:k8s:chart-opts]) [:values]
#(deep-merge % (or (:yaml-values (:options env)) {})))))}
:k8s:storage-class {:constructor (.. k8s -storage -v1 -StorageClass)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :storage-class]) (:options env)))}
:k8s:pvc {:constructor (.. k8s -storage -v1 -PVC)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :pvc]) (:options env)))}
:k8s:gateway {:constructor (.. gateway-api -v1 -Gateway)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :gateway]) (:options env)))}
:k8s:httproute {:constructor (.. gateway-api -v1 -HTTPRoute)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :httproute]) (:options env)))}
:k8s:cluster-issuer {:constructor (.. cert-manager -v1 -ClusterIssuer)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :cluster-issuer]) (:options env)))}
:k8s:certificates
{:constructor (.. cert-manager -v1 -Certificate)
:provider-key :k8s
:defaults-fn (fn [env]
(let [{:keys [app-namespace is-prod?]} (:options env)]
(p-> env :options :vault:prepare "stringData" .-domains
#(vec
(for [domain (js/JSON.parse %)]
(let [clean-name (clojure.string/replace domain #"\." "-")]
{:_suffix clean-name
:metadata {:namespace app-namespace}
:spec {:dnsNames [domain (str "*." domain)]
:secretName (str clean-name "-tls")
:issuerRef {:name (if is-prod? "letsencrypt-prod" "letsencrypt-staging")
:kind "ClusterIssuer"}}}))))))}
:k8s:certificate
{:constructor (.. cert-manager -v1 -Certificate)
:provider-key :k8s
:defaults-fn (fn [env] ((get-in default/defaults [:k8s :certificate]) (:options env)))}
;; Docker Resources
:docker:image {:constructor (.. docker-build -Image)
:provider-key :docker
:defaults-fn (fn [env] ((get-in default/defaults [:docker :image]) (:options env)))}
;; Harbor Resources
:harbor:project {:constructor (.. harbor -Project)
:provider-key :harbor
:defaults-fn (fn [env] ((get-in default/defaults [:harbor :project]) (:options env)))}
:harbor:robot-account {:constructor (.. harbor -RobotAccount)
:provider-key :harbor
:defaults-fn (fn [env] ((get-in default/defaults [:harbor :robot-account]) (:options env)))}})
(defmulti deploy-resource
"Generic resource deployment multimethod.
Dispatches on the fully-qualified resource keyword.
Returns a map of {:resource (the-pulumi-resource) :common-opts-update {map-of-new-state}}."
(fn [dispatch-key _config] dispatch-key))
(defmethod deploy-resource :default
[dispatch-key full-config]
(if-let [spec (get component-specs dispatch-key)]
(let [app-name (:app-name full-config)
dependsOn (:dependsOn full-config)
provider-key (:provider-key spec)
provider (get full-config provider-key)
resource-class (:constructor spec)
opts-key (keyword (str (name dispatch-key) "-opts"))
component-opts (get full-config opts-key)
env {:options full-config :secrets (:secrets full-config) :component-opts component-opts}
raw-defaults (when-let [df (:defaults-fn spec)] (df env))]
(if resource-class
(let [base-creator (fn [final-args suffix]
(let [final-name (if suffix
(str app-name "-" suffix)
app-name)]
(new-resource resource-class
final-name
final-args
provider
dependsOn)))]
{:resource
(p-> raw-defaults
#(let [defaults-list (if (vector? %)
%
[%])
is-multi? (vector? %) resources
(doall
(map-indexed
(fn [idx item]
(let [suffix (cond
(:_suffix item) (:_suffix item)
is-multi? (str idx)
:else nil)
clean-item (dissoc item :_suffix)
item-creator (fn [resolved-args]
(base-creator resolved-args suffix))]
(generic-transform item-creator
component-opts
clean-item
(:secrets env)
full-config)))
defaults-list))]
(if is-multi? resources (first resources))))})
(throw (js/Error. (str "No :constructor found for spec: " dispatch-key)))))
(throw (js/Error. (str "Unknown resource: " dispatch-key)))))
(defmethod deploy-resource :vault:prepare
[_ config]
(let [prepare-opts (get config :vault:prepare-opts {})
defaults {:provider (:k8s config)
:vault-provider (:vault config)
:app-name (:app-name config)
:app-namespace (:app-namespace config)
:load-yaml (get config :vault-load-yaml false)}
final-args (merge defaults prepare-opts)
prepared-vault-data (try
(vault-utils/prepare final-args)
(catch js/Error e
(js/console.error "!!! Error in :vault:prepare :" e)
nil))]
{:common-opts-update prepared-vault-data
:resource (:bind-secrets prepared-vault-data)}))
(defmethod deploy-resource :vault:retrieve
[_ config]
(let [retrieve-opts (get config :vault:retrieve-opts {})
defaults {:vault-provider (:vault config)
:app-name (:app-name config)
:app-namespace (:app-namespace config)}
final-args (merge defaults retrieve-opts)
retrieved-data (try
(vault-utils/retrieve (:vault-provider final-args)
(:app-name final-args)
(:app-namespace final-args))
(catch js/Error e
(js/console.error " Error in :vault:retrieve :" e)
nil))]
{:common-opts-update retrieved-data}))
;; https://www.pulumi.com/docs/iac/concepts/resources/dynamic-providers/
(defmethod deploy-resource :generic:execute
[_ full-config]
(let [app-name (:app-name full-config)
dependsOn (:dependsOn full-config)
component-opts (assoc (:execute-opts full-config)
:pulumi-cfg (:pulumi-cfg full-config)
:secrets (:secrets full-config)
)
defaults {}
exec-fn (:exec-fn full-config)
resource-id (str app-name "-exec")
provider #js {:create (fn [inputs-js]
(js/Promise.
(fn [resolve _reject]
(resolve
#js {:id resource-id
:outs inputs-js}))))
:delete (fn [id old-inputs-js]
(js/Promise.resolve))
:update (fn [id old-inputs-js new-inputs-js]
(js/Promise.
(fn [resolve _reject]
(resolve #js {:outs new-inputs-js}))))}
gen (generic-transform #(clj->js (exec-fn (js->clj % :keywordize-keys true))) component-opts defaults (:secrets full-config) full-config)
creator-fn (fn [inputs]
(pulumi/dynamic.Resource.
provider
resource-id
inputs
(clj->js {:dependsOn (vec dependsOn)})))
resource (.apply gen #(creator-fn %))]
{:resource resource}))
(defn handle-keyword-item [last-resource item config common-opts]
(let [dispatch-key item
depends-on (when last-resource [last-resource])
final-config (merge config common-opts {:dependsOn depends-on})
result-map (deploy-resource dispatch-key final-config)
resource (:resource result-map)
opts-update (:common-opts-update result-map)
resource-update (when resource {dispatch-key resource})]
[resource
resource-update
(merge common-opts opts-update resource-update)]))
(defn handle-list-item [last-resource item config common-opts]
(let [provider-key (first item)
resource-keys (rest item)
nested-result
(reduce
(fn [nested-acc resource-key]
(let [inner-last-resource (get nested-acc :last-resource)
inner-resources-map (get nested-acc :resources)
inner-common-opts (get nested-acc :common-opts)
dispatch-key (keyword (str (name provider-key) ":" (name resource-key)))
[new-resource new-resource-map new-common-opts]
(handle-keyword-item inner-last-resource dispatch-key config inner-common-opts)]
{:last-resource (or new-resource inner-last-resource)
:resources (merge inner-resources-map new-resource-map)
:common-opts new-common-opts}))
{:last-resource last-resource
:resources {}
:common-opts common-opts}
resource-keys)]
[(:last-resource nested-result)
(:resources nested-result)
(:common-opts nested-result)]))
(defn process-stack
"Recursively processes a stack configuration, building a dependency chain.
Returns a map of all created resources keyed by their dispatch keyword."
[stack-items config initial-common-opts]
(let [result
(reduce
(fn [acc item]
(let [{:keys [last-resource common-opts]} acc
[new-resource new-resources new-common-opts]
(if (keyword? item)
(handle-keyword-item last-resource item config common-opts)
(handle-list-item last-resource item config common-opts))]
{:last-resource (or new-resource last-resource)
:resources-map (merge (:resources-map acc) new-resources)
:common-opts new-common-opts}))
{:last-resource nil
:resources-map {}
:common-opts initial-common-opts}
stack-items)]
(:resources-map result)))
(defn deploy! [{:keys [pulumi-cfg resource-configs all-providers]}]
(let [
deployment-results
(into
{}
(for [config resource-configs]
(let [
{:keys [stack app-name]} config
_ (when (nil? config)
(throw (js/Error. "Resource configs contain a nil value!")))
common-opts (merge
all-providers
(select-keys config [:app-name :app-namespace])
{:pulumi-cfg pulumi-cfg})
]
[app-name (process-stack stack config common-opts)])))
]
(clj->js deployment-results)))

71
src/main/utils/vault.cljs Normal file
View File

@@ -0,0 +1,71 @@
(ns utils.vault
(:require
["@pulumi/kubernetes" :as k8s]
["@pulumi/pulumi" :as pulumi]
["@pulumi/vault" :as vault]
["fs" :as fs]
["js-yaml" :as yaml]
["path" :as path]
[configs :refer [cfg]]))
(defn get-secret-val
"Extract a specific key from a Vault secret Output/Promise."
[secret-promise key]
(.then secret-promise #(aget (.-data %) key)))
(defn initialize-mount [vault-provider vault-path service-name]
(let [service-secrets (into {} (get (-> cfg :secrets-json) (keyword service-name)))]
(new (.. vault -generic -Secret)
(str service-name "-secret")
(clj->js {:path (str vault-path)
:dataJson (js/JSON.stringify (clj->js service-secrets))})
(clj->js {:provider vault-provider}))))
(defn prepare
"Prepares common resources and values for a deployment from a single config map."
[config]
(let [{:keys [provider vault-provider app-name app-namespace load-yaml]} config
values-path (.join path js/__dirname ".." (-> cfg :resource-path) (str app-name ".yml"))]
(let [yaml-values (when load-yaml
(js->clj (-> values-path
(fs/readFileSync "utf8")
(yaml/load))
:keywordize-keys true))
{:keys [secrets-data bind-secrets]}
(when vault-provider
(let [vault-path (str "secret/" app-name)
_ (initialize-mount vault-provider vault-path app-name)
secrets (pulumi/output (.getSecret (.-generic vault)
(clj->js {:path vault-path})
(clj->js {:provider vault-provider})))
secrets-data (.apply secrets #(.. % -data))
bind-secrets (when (and provider app-namespace)
(new (.. k8s -core -v1 -Secret) (str app-name "-secrets")
(clj->js {:metadata {:name (str app-name "-secrets")
:namespace app-namespace}
:stringData secrets-data})
(clj->js {:provider provider})))]
{:secrets-data secrets-data
:bind-secrets bind-secrets}))]
{:secrets secrets-data
:yaml-path values-path
:yaml-values yaml-values
:bind-secrets bind-secrets})))
(defn retrieve [vault-provider app-name]
(let [vault-path (str "secret/" app-name)
secrets (pulumi/output (.getSecret (.-generic vault)
(clj->js {:path vault-path})
(clj->js {:provider vault-provider})))
secrets-data (.apply secrets #(.. % -data))]
{:secrets secrets-data}))
(def provider-template
{:constructor (.. vault -Provider)
:name "vault-provider"
:config {:address 'vaultAddress
:token 'vaultToken}})