#!/usr/bin/env bash
#
# openshift-blockers/discover.sh: find what on this OpenShift cluster would refuse the Hopsworks
# install, or a part of it, before step 8 runs. It renders the chart with the install's own values
# files, as step 8 will, and compares the result with the cluster: what the namespace's quotas and
# limit ranges allow, which objects the chart creates already exist under another owner, which
# admission and network policies apply, and how many NodePorts and load balancer addresses the
# install needs. Run it as the account that will install Hopsworks, after install step 7.
#
# WHAT IT CHANGES ON THE CLUSTER
#   Nothing. Every call is "oc get", "oc auth can-i" or "oc version"; the chart is rendered
#   locally with "helm template", which does not contact the cluster.
#
# CREDENTIALS
#   It uses your current oc login. It reads no Secret's data: names, types and ownership only.
#
# WHAT IT CHECKS
#   B1  the installing account may create every cluster-scoped kind the chart creates
#   B2  NodePorts and LoadBalancer Services: how many the rendered chart creates (NodePorts must be
#       0 on a cluster without them), the Istio ingress gateway KServe's install job adds, the
#       namespace quota for both, and the load balancer address pools
#   B3  every ResourceQuota, ClusterResourceQuota and LimitRange that applies to the namespace,
#       against the rendered workloads: CPU and memory requests and limits, pods, Services, PVCs,
#       scoped quotas, and containers without limits (hook Jobs included) that a limits quota
#       refuses unless a LimitRange fills them in. Sidecar init containers (restartPolicy Always)
#       count for the whole life of the pod.
#   B4  Pod Security admission labels on the namespace
#   B5  admission webhooks and ValidatingAdmissionPolicies that can refuse pods (ACS, Gatekeeper,
#       Kyverno and others)
#   B6  cluster-scoped objects the chart creates, hooks included (CRDs, ClusterRoles, bindings,
#       webhooks, PriorityClasses, IngressClasses, APIServices, CSIDrivers, SCCs), that already
#       exist and belong to something else: Helm refuses the install on each one
#   B7  operators installed through OLM that bring the same components (KEDA, Strimzi,
#       cert-manager, Istio, Knative, KServe, Spark, Ray, Kueue, Velero)
#   B8  network policies in the namespace, cluster-wide admin network policies, egress firewalls
#   B9  image policies: allowed or blocked registries, signature verification
#   B10 Ingress and routes: the rendered hosts and IngressClasses, Routes or Ingresses elsewhere
#       on the same host, and whether any router admits a Route from the namespace
#   B11 the separately installed Consul (install step 5): the Secrets the chart mounts from it,
#       its server pods and its DNS Service
#
#   A read that fails (forbidden, timed out, an API that should exist) is reported as could not
#   determine, never as "nothing found". An API that does not exist on the cluster counts as
#   nothing found.
#
# USAGE
#   bash discover.sh "$SIZING_VALUES" values-openshift.yaml values-site.yaml values-images.yaml values-placement.yaml
#   The values files in the order step 8 passes them. Without them B2, B3, B6, B10 and B11 run
#   without the comparison with the chart.
#
# SETTINGS (environment variables, as in hopsworks-install.env)
#   NAMESPACE     the Hopsworks namespace (required)
#   RELEASE       the Helm release name (default hopsworks)
#   CHART         the chart archive (required with values files)
#   PROJECT_NS    optional: one project namespace from step 9, checked for B3, B4 and B8 too
#
# Needs: oc (logged in), jq (1.6 or later), awk; helm (4, or 3.20+) for the comparison with the chart.
#
# Exit codes: 0 nothing found that blocks, 1 at least one blocker, 2 could not determine.
# The output holds no secret and can be sent to Hopsworks as is.

set -uo pipefail

case "${1:-}" in
  -h|--help) awk 'NR == 1 { next } /^#/ { print; next } { exit }' "$0"; exit 0 ;;
esac

PASS=0; FAIL=0; UNKNOWN=0
step()    { printf '\n--- %s\n' "$*"; }
ok()      { printf '  [ok]      %s\n' "$*"; PASS=$(( PASS + 1 )); }
bad()     { printf '  [FAIL]    %s\n' "$*"; FAIL=$(( FAIL + 1 )); }
unknown() { printf '  [UNKNOWN] %s\n' "$*"; UNKNOWN=$(( UNKNOWN + 1 )); }
note()    { printf '            %s\n' "$*"; }
finish() {
  printf '\n%d pass, %d fail, %d unknown\n' "$PASS" "$FAIL" "$UNKNOWN"
  [ "$FAIL" -gt 0 ] && exit 1
  [ "$UNKNOWN" -gt 0 ] && exit 2
  exit 0
}

NAMESPACE="${NAMESPACE:-}"; RELEASE="${RELEASE:-hopsworks}"; PROJECT_NS="${PROJECT_NS:-}"
VALUES=("$@")
TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT

# oget <var> <oc args...>: 0 read into var; 1 absent (NotFound, or the API does not exist on this
# cluster); 2 not readable (forbidden, timeout, anything else). var is empty unless 0.
oget() {
  local __v=$1 __out __rc; shift
  __out=$(oc "$@" 2>"$TMP/oc.err"); __rc=$?
  if [ "$__rc" -eq 0 ]; then printf -v "$__v" '%s' "$__out"; return 0; fi
  printf -v "$__v" '%s' ""
  grep -qiE 'NotFound|not found|the server doesn.t have a resource type|no matches for kind' "$TMP/oc.err" && return 1
  return 2
}

printf '===== openshift-blockers discover =====\n'
printf 'date      : %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"

step "0. Tools, access and the rendered chart"
for c in oc jq; do command -v "$c" >/dev/null || { bad "$c not found"; finish; }; done
[ -n "$NAMESPACE" ] || { bad "NAMESPACE is not set"; finish; }
oc whoami >/dev/null 2>&1 || { bad "not logged in with oc"; finish; }
note "account $(oc whoami), namespace $NAMESPACE, release $RELEASE${PROJECT_NS:+, project namespace $PROJECT_NS}, $(jq --version 2>&1)"
oget nsjson get namespace "$NAMESPACE" -o json; rc=$?
if [ "$rc" = 1 ]; then unknown "namespace $NAMESPACE not found: create it (install step 2) and run again"; finish
elif [ "$rc" != 0 ]; then unknown "namespace $NAMESPACE not readable: $(head -1 "$TMP/oc.err")"; finish; fi

RENDERED=0
if [ "${#VALUES[@]}" -gt 0 ]; then
  if ! command -v helm >/dev/null; then
    unknown "helm not found: the comparison with the chart is skipped"
  elif [ -z "${CHART:-}" ] || [ ! -f "$CHART" ]; then
    unknown "CHART is not set or not a file: the comparison with the chart is skipped"
  else
    KV=$(oc version -o json 2>/dev/null | jq -r '.serverVersion.gitVersion // empty' | sed 's/+.*//')
    args=(); for v in "${VALUES[@]}"; do args+=(--values "$v"); done
    # helm template renders hook resources too, so pre-install Jobs and their RBAC are included.
    if helm template "$RELEASE" "$CHART" -n "$NAMESPACE" --include-crds ${KV:+--kube-version "$KV"} \
         --api-versions security.openshift.io/v1 --api-versions route.openshift.io/v1 "${args[@]}" \
         > "$TMP/render.yaml" 2> "$TMP/render.err" \
       && oc label --local -f "$TMP/render.yaml" -o json hopsworks-blockers-check=1 2>> "$TMP/render.err" \
          | jq -c 'if .kind == "List" then .items[] else . end' > "$TMP/render.jsonl" && [ -s "$TMP/render.jsonl" ]; then
      RENDERED=1
      ok "chart rendered with ${#VALUES[@]} values file(s): $(wc -l < "$TMP/render.jsonl" | tr -d ' ') objects"
    else
      bad "rendering the chart failed: $(tail -1 "$TMP/render.err")"
    fi
  fi
else
  note "no values files given: the comparison with the chart is skipped"
fi
# Every rendered object of the given kinds, one JSON object per line.
r() { [ "$RENDERED" = 1 ] && jq -c --arg k "$1" 'select(.kind | test("^(" + $k + ")$"))' "$TMP/render.jsonl"; }

step "B1. Cluster-scoped objects the installing account may create"
kinds=(customresourcedefinitions clusterroles clusterrolebindings mutatingwebhookconfigurations
       validatingwebhookconfigurations priorityclasses csidrivers apiservices ingressclasses)
missing=(); undecided=()
can() { local a; a=$(oc auth can-i "$@" 2>/dev/null); case "$a" in yes) return 0 ;; no) return 1 ;; *) return 2 ;; esac; }
for k in "${kinds[@]}"; do can create "$k"; case $? in 1) missing+=("create $k") ;; 2) undecided+=("create $k") ;; esac; done
for v in escalate bind; do can "$v" clusterroles; case $? in 1) missing+=("$v clusterroles") ;; 2) undecided+=("$v clusterroles") ;; esac; done
can use securitycontextconstraints/privileged; case $? in 1) missing+=("use scc/privileged") ;; 2) undecided+=("use scc/privileged") ;; esac
for k in rolebindings networkpolicies routes; do
  can create "$k" -n "$NAMESPACE"; case $? in 1) missing+=("create $k in $NAMESPACE") ;; 2) undecided+=("create $k in $NAMESPACE") ;; esac
done
[ "${#missing[@]}" -gt 0 ] && bad "not allowed: ${missing[*]}"
[ "${#undecided[@]}" -gt 0 ] && unknown "oc auth can-i gave no answer for: ${undecided[*]}"
[ "${#missing[@]}" -eq 0 ] && [ "${#undecided[@]}" -eq 0 ] && ok "every verb the install needs is allowed"

# All quotas that apply to a namespace, as {items: [{metadata.name, spec: {hard, scopes, scopeSelector}}]}:
# its ResourceQuotas and the ClusterResourceQuotas that select it. Returns 2 when unreadable.
quotas() {
  local ns=$1 rq crq rc
  oget rq get resourcequota -n "$ns" -o json; [ $? = 0 ] || return 2
  oget crq get clusterresourcequota -o json; rc=$?
  [ "$rc" = 2 ] && return 2
  [ "$rc" = 1 ] && crq='{"items":[]}'
  jq -n --argjson rq "$rq" --argjson crq "$crq" --arg ns "$ns" '{items: ($rq.items + [$crq.items[]
    | select(any(.status.namespaces[]?; .namespace == $ns))
    | {metadata: {name: ("clusterresourcequota/" + .metadata.name)},
       spec: {hard: (.spec.quota.hard // {}), scopes: .spec.quota.scopes, scopeSelector: .spec.quota.scopeSelector},
       shared: ([.status.namespaces[]?.namespace] | length)}])}'
}

step "B2. NodePorts and load balancer addresses"
if q=$(quotas "$NAMESPACE"); then
  qh() { jq -r --arg k "$1" '[.items[] | select(.spec.scopes == null and .spec.scopeSelector == null) | .spec.hard[$k] // empty]
    | if length == 0 then "none" else (map(tonumber) | min | tostring) end' <<<"$q"; }
  note "quota in $NAMESPACE: services.nodeports=$(qh services.nodeports), services.loadbalancers=$(qh services.loadbalancers)"
else
  unknown "quotas for $NAMESPACE not readable: $(head -1 "$TMP/oc.err")"
fi
if [ "$RENDERED" = 1 ]; then
  r Service | jq -r 'select(.spec.type == "NodePort" or .spec.type == "LoadBalancer") | "\(.spec.type)\t\(.metadata.name)\t\([.spec.ports[] | .port] | join(","))"' > "$TMP/svc.tsv"
  n_np=$(grep -c '^NodePort' "$TMP/svc.tsv"); n_lb=$(grep -c '^LoadBalancer' "$TMP/svc.tsv")
  # KServe's install job creates the Istio ingress gateway from this operator config, not from a
  # rendered Service: LoadBalancer or NodePort with 3 NodePorts in chart 5.2.3, ClusterIP when
  # kserve.istio.gateways.ingress.serviceType is ClusterIP (chart 5.2.3-trv.1, HWORKS-3717).
  # The ingress gateway is the first entry under ingressGateways, before cluster-local-gateway.
  gw=$(r ConfigMap | jq -r 'select(.data["istiooperator.yaml"]) | .data["istiooperator.yaml"]' \
       | awk '/- name: istio-ingressgateway/ {on=1} /- name: cluster-local-gateway/ {on=0} on')
  if [ -n "$gw" ]; then
    gw_type=$(awk '/^ *type: / {print $2; exit}' <<<"$gw"); gw_type=${gw_type:-LoadBalancer}
    gw_np=$(grep -c 'nodePort:' <<<"$gw")
    printf '%s\tistio-ingressgateway (created by the KServe install job)\t%s\n' "$gw_type" "$(awk '/^ *port: / {printf "%s%s", s, $2; s=","}' <<<"$gw")" >> "$TMP/svc.tsv"
    [ "$gw_type" = LoadBalancer ] && n_lb=$(( n_lb + 1 ))
    [ "$gw_type" = NodePort ] && n_np=$(( n_np + 1 ))
    [ "$gw_type" != ClusterIP ] && [ "$gw_np" -gt 0 ] && note "the Istio ingress gateway also takes $gw_np fixed NodePorts: $(grep -oE 'nodePort: *[0-9]+' <<<"$gw" | awk '{printf "%s%s", s, $2; s=", "}')"
  fi
  kl=$(r Kafka | jq -r '[.spec.kafka.listeners[]? | select(.type == "loadbalancer" or .type == "nodeport") | "\(.name):\(.type)"] | join(",")')
  [ -n "$kl" ] && note "Kafka listeners that Strimzi exposes itself: $kl (one Service per broker plus a bootstrap Service)"
  if [ -s "$TMP/svc.tsv" ]; then while IFS=$'\t' read -r t n p; do note "$t  $n  ports $p"; done < "$TMP/svc.tsv"; fi
  if [ "$n_np" -eq 0 ] && [ "$n_lb" -eq 0 ] && [ -z "$kl" ]; then
    ok "no NodePort or LoadBalancer Service: every Service is ClusterIP"
  else
    bad "the install creates $n_lb LoadBalancer and $n_np NodePort Service(s)${kl:+ plus the Kafka external listener}; each also allocates NodePorts unless allocateLoadBalancerNodePorts is false"
    note "for none: global._hopsworks.externalLoadBalancers.enabled false, rondb.rondb.meta.mysqld and"
    note "rdrs externalLoadBalancer.enabled false, a managed registry (managedDockerRegistery), and"
    note "kserve.istio.gateways.ingress.serviceType ClusterIP (patched chart) or KServe off"
  fi
else
  unknown "chart not rendered: cannot count the Services the install creates"
fi
oget pools get ipaddresspools.metallb.io -A -o json; rc=$?
[ "$rc" = 0 ] && jq -r '.items[] | "MetalLB pool \(.metadata.namespace)/\(.metadata.name): \(.spec.addresses | join(","))"' <<<"$pools" | while read -r l; do note "$l"; done
if oget svcs get svc -A -o json; then
  note "LoadBalancer Services on the cluster today: $(jq -r '[.items[] | select(.spec.type == "LoadBalancer")] | "\(length), \([.[] | select(.status.loadBalancer.ingress)] | length) with an address"' <<<"$svcs")"
fi

step "B3. Quotas and limit ranges"
check_quota() {  # ns label: prints the quotas and limit ranges; leaves them in $TMP/q.json and $TMP/lr.json
  local ns=$1 label=$2 lr
  if ! quotas "$ns" > "$TMP/q.json"; then unknown "$label: quotas not readable: $(head -1 "$TMP/oc.err")"; return 1; fi
  if ! oget lr get limitrange -n "$ns" -o json; then unknown "$label: limit ranges not readable: $(head -1 "$TMP/oc.err")"; return 1; fi
  printf '%s' "$lr" > "$TMP/lr.json"
  [ "$(jq '.items | length' "$TMP/q.json")" -eq 0 ] && ok "$label: no ResourceQuota or ClusterResourceQuota"
  jq -r '.items[] | "quota \(.metadata.name): \(.spec.hard | to_entries | map("\(.key)=\(.value)") | join(", "))\(if .shared then "  (shared by \(.shared) namespaces)" else "" end)\(if .spec.scopes or .spec.scopeSelector then "  scope: \(((.spec.scopes // []) + [(.spec.scopeSelector.matchExpressions // [])[] | "\(.scopeName) \(.operator) \(.values // [] | join(","))"]) | join("; "))" else "" end)"' "$TMP/q.json" | while read -r l; do note "$l"; done
  jq -r '.items[] | .metadata.name as $n | .spec.limits[] | "limitrange \($n) \(.type): default=\(.default // {} | tostring) defaultRequest=\(.defaultRequest // {} | tostring) max=\(.max // {} | tostring)"' "$TMP/lr.json" | while read -r l; do note "$l"; done
  return 0
}
if check_quota "$NAMESPACE" "$NAMESPACE" && [ "$RENDERED" = 1 ]; then
  workers=$(oc get nodes -l node-role.kubernetes.io/worker= -o json 2>/dev/null | jq '[.items[] | select(.spec.unschedulable != true)] | length')
  [[ "${workers:-}" =~ ^[0-9]+$ ]] && [ "$workers" -gt 0 ] || { unknown "worker nodes not readable: DaemonSets counted on 3 nodes"; workers=3; }
  # A container that sets only a limit requests that limit. A pod's request is the larger of its
  # running containers plus sidecar init containers (restartPolicy Always), and its largest
  # ordinary init container plus the sidecars. DaemonSets count one pod per schedulable worker.
  cat > "$TMP/sum.jq" <<'EOF'
def cpu: if . == null then 0 elif (tostring | test("m$")) then (tostring | rtrimstr("m") | tonumber) / 1000 else (tostring | tonumber) end;
def mem: if . == null then 0 else (tostring) as $s |
  if   ($s | test("Ki$")) then ($s | rtrimstr("Ki") | tonumber) / 1048576
  elif ($s | test("Mi$")) then ($s | rtrimstr("Mi") | tonumber) / 1024
  elif ($s | test("Gi$")) then ($s | rtrimstr("Gi") | tonumber)
  elif ($s | test("Ti$")) then ($s | rtrimstr("Ti") | tonumber) * 1024
  elif ($s | test("k$"))  then ($s | rtrimstr("k") | tonumber) / 1048576 * 1.024
  elif ($s | test("M$"))  then ($s | rtrimstr("M") | tonumber) * 0.000931
  elif ($s | test("G$"))  then ($s | rtrimstr("G") | tonumber) * 0.9313
  else ($s | tonumber) / 1073741824 end end;
def res(f; r): (.resources[f][r] // (if f == "requests" then .resources.limits[r] else null end));
def podspec: if .kind == "CronJob" then .spec.jobTemplate.spec.template.spec else .spec.template.spec end;
def podsum(f; r; conv): podspec as $p
  | ([$p.containers[] | res(f; r) | conv] | add // 0) as $c
  | ([($p.initContainers // [])[] | select(.restartPolicy == "Always") | res(f; r) | conv] | add // 0) as $s
  | ([($p.initContainers // [])[] | select(.restartPolicy != "Always") | res(f; r) | conv] | max // 0) as $i
  | [($c + $s), ($i + $s)] | max;
def nolimit: podspec | [(.containers + (.initContainers // []))[]
  | select(.resources.limits.cpu == null or .resources.limits.memory == null) | .name] | length;
[ inputs | select(.kind | test("^(Deployment|StatefulSet|DaemonSet|Job|CronJob)$")) ] as $all
| [ $all[] | select(.kind == "Deployment" or .kind == "StatefulSet" or .kind == "DaemonSet")
    | (if .kind == "DaemonSet" then $workers else (.spec.replicas // 1) end) as $n
    | { n: $n, prio: (podspec.priorityClassName // ""),
        rc: (podsum("requests"; "cpu"; cpu) * $n), rm: (podsum("requests"; "memory"; mem) * $n),
        lc: (podsum("limits"; "cpu"; cpu) * $n),   lm: (podsum("limits"; "memory"; mem) * $n) } ] as $w
| { pods: ($w | map(.n) | add), rc: ($w | map(.rc) | add), rm: ($w | map(.rm) | add),
    lc: ($w | map(.lc) | add), lm: ($w | map(.lm) | add),
    nolimit: ([$all[] | nolimit] | add // 0),
    nolimit_workloads: [$all[] | select(nolimit > 0) | "\(.kind)/\(.metadata.name)\(if .metadata.annotations["helm.sh/hook"] then " (hook \(.metadata.annotations["helm.sh/hook"]))" else "" end)"],
    prios: ([$all[] | podspec.priorityClassName // empty] | unique) }
EOF
  if ! jq -n --argjson workers "$workers" -f "$TMP/sum.jq" "$TMP/render.jsonl" > "$TMP/sum.json" 2> "$TMP/sum.err" \
     || ! jq -e '(.pods | type == "number") and (.rc | type == "number")' "$TMP/sum.json" >/dev/null 2>&1; then
    unknown "the resource sums could not be calculated: $(head -1 "$TMP/sum.err")"
  else
    read -r pods rc rm lc lm nolimit < <(jq -r '[.pods, (.rc*10|round/10), (.rm|round), (.lc*10|round/10), (.lm|round), .nolimit] | @tsv' "$TMP/sum.json")
    note "rendered workloads ($workers schedulable worker nodes for DaemonSets): $pods pods,"
    note "requests ${rc} cores / ${rm} GiB, limits ${lc} cores / ${lm} GiB; $nolimit containers (hook Jobs included) without a CPU or memory limit"
    note "not counted: pods the operators start later (Kafka brokers, shuffle service, Istio; about 1.5 cores /"
    note "10.5 GiB requests and 11 cores / 13 GiB limits on Hopsworks' test cluster), Jobs, workloads already in the"
    note "namespace (the separately installed Consul), and surge pods during upgrades"
    # Unscoped quotas only; a scoped one applies to a subset of pods.
    cmp_quota() { local key=$1 need=$2 unit=$3 hard h
      hard=$(jq -r --arg k "$key" '[.items[] | select(.spec.scopes == null and .spec.scopeSelector == null) | .spec.hard[$k] // empty] | first // empty' "$TMP/q.json")
      [ -z "$hard" ] && return
      h=$(jq -nr --arg v "$hard" --arg u "$unit" 'def cpu: if ($v|test("m$")) then ($v|rtrimstr("m")|tonumber)/1000 else ($v|tonumber) end;
        def mem: $v as $s | if ($s|test("Gi$")) then ($s|rtrimstr("Gi")|tonumber) elif ($s|test("Ti$")) then ($s|rtrimstr("Ti")|tonumber)*1024 elif ($s|test("Mi$")) then ($s|rtrimstr("Mi")|tonumber)/1024 else ($s|tonumber)/1073741824 end;
        if $u == "cores" then cpu elif $u == "GiB" then mem else ($v|tonumber) end' 2>/dev/null)
      if [ -z "$h" ]; then unknown "$key: cannot read the quota value $hard"
      elif jq -ne --argjson n "$need" --argjson h "$h" '$n <= $h' >/dev/null; then ok "$key: needs $need $unit of $hard"
      else bad "$key: needs $need $unit, quota allows $hard"; fi
    }
    cmp_quota requests.cpu "$rc" cores; cmp_quota requests.memory "$rm" GiB
    cmp_quota limits.cpu "$lc" cores;   cmp_quota limits.memory "$lm" GiB
    cmp_quota pods "$pods" pods
    cmp_quota services "$(r Service | wc -l | tr -d ' ')" Services
    cmp_quota persistentvolumeclaims "$(r PersistentVolumeClaim | wc -l | tr -d ' ')" "PVCs (StatefulSet claims not counted)"
    has_limit_quota=$(jq '[.items[] | .spec.hard | keys[] | select(test("^limits\\.(cpu|memory)$"))] | length' "$TMP/q.json")
    has_default=$(jq '[.items[].spec.limits[] | select(.type == "Container" and .default.cpu and .default.memory)] | length' "$TMP/lr.json")
    if [ "$has_limit_quota" -gt 0 ] && [ "$nolimit" -gt 0 ] && [ "$has_default" -eq 0 ]; then
      bad "a quota on limits.cpu or limits.memory and no LimitRange default: pods with these containers are refused"
      jq -r '.nolimit_workloads[]' "$TMP/sum.json" | head -60 | while read -r l; do note "$l"; done
    elif [ "$has_limit_quota" -gt 0 ] && [ "$nolimit" -gt 0 ]; then
      note "the LimitRange default is added to each of the $nolimit containers without limits, on top of the sums above"
    fi
    prios=$(jq -r '.prios | join(" ")' "$TMP/sum.json")
    for p in $prios; do
      jq -c --arg p "$p" '.items[] | select(any(.spec.scopeSelector.matchExpressions[]?; .scopeName == "PriorityClass"
        and ((.operator == "In" and ((.values // []) | index($p))) or (.operator == "NotIn" and (((.values // []) | index($p)) | not)) or .operator == "Exists")))
        | {name: .metadata.name, pods: (.spec.hard.pods // null)}' "$TMP/q.json" \
        | while read -r m; do
            if [ "$(jq -r '.pods' <<<"$m")" = 0 ]; then bad "priority class $p: scoped quota $(jq -r .name <<<"$m") allows 0 pods"
            else note "priority class $p: scoped quota $(jq -r .name <<<"$m")"; fi
          done
    done
    note "priority classes the chart uses: ${prios:-none}"
  fi
fi
if [ -n "$PROJECT_NS" ]; then
  if oget scratch get namespace "$PROJECT_NS" -o name; then check_quota "$PROJECT_NS" "$PROJECT_NS (project)"
  else unknown "$PROJECT_NS not found or not readable"; fi
fi

step "B4. Pod Security admission"
for ns in "$NAMESPACE" ${PROJECT_NS:+"$PROJECT_NS"}; do
  if ! oget j get namespace "$ns" -o json; then unknown "$ns not readable"; continue; fi
  lbl=$(jq -c '.metadata.labels // {} | with_entries(select(.key | test("pod-security.kubernetes.io/enforce$|podSecurityLabelSync")))' <<<"$j")
  enf=$(jq -r '.["pod-security.kubernetes.io/enforce"] // ""' <<<"$lbl")
  case "$enf" in
    restricted|baseline) if [ "$ns" = "$NAMESPACE" ]; then bad "$ns enforces $enf: filebeat and the HopsFS CSI plugin run privileged and are refused"
                         else note "$ns enforces $enf: user workloads must fit it"; fi ;;
    *) ok "$ns: enforce ${enf:-not set} $lbl" ;;
  esac
done

step "B5. Admission controllers that can refuse or change pods"
hooks_ok=1
if oget wh get validatingwebhookconfiguration,mutatingwebhookconfiguration -o json; then
  jq -r --arg ns "$NAMESPACE" '.items[] | select(all(.webhooks[]; .clientConfig.service.namespace != $ns))
    | select(.metadata.name | test("stackrox|acs|gatekeeper|kyverno|opa|policy|sigstore|sidecar|vault|falco|aqua|twistlock|prisma|neuvector"; "i"))
    | "\(.kind)/\(.metadata.name) failurePolicy=\([.webhooks[].failurePolicy] | unique | join(",")) service=\([.webhooks[].clientConfig.service | if . then "\(.namespace)/\(.name)" else "url" end] | unique | join(","))"' <<<"$wh" > "$TMP/hooks.txt"
else
  hooks_ok=0; unknown "admission webhooks not readable: $(head -1 "$TMP/oc.err")"; : > "$TMP/hooks.txt"
fi
oget vap get validatingadmissionpolicybinding -o json; rc=$?
if [ "$rc" = 0 ]; then
  jq -r '.items[] | select((.metadata.name + " " + .spec.policyName) | test("machine|mcn|network-annotation|bootimages|openshift|servicecidr|user-defined-network") | not)
    | "binding \(.metadata.name) policy=\(.spec.policyName) actions=\(.spec.validationActions | join(","))"' <<<"$vap" >> "$TMP/hooks.txt"
elif [ "$rc" = 2 ]; then hooks_ok=0; unknown "ValidatingAdmissionPolicyBindings not readable"; fi
if [ -s "$TMP/hooks.txt" ]; then
  unknown "policy engines found; which policies they enforce is set outside the cluster:"
  while read -r l; do note "$l"; done < "$TMP/hooks.txt"
  grep -qi stackrox "$TMP/hooks.txt" && note "ACS: ask which deploy-time policies enforce here; Hopsworks has privileged pods, root pods, and containers without limits"
elif [ "$hooks_ok" = 1 ]; then
  ok "no ACS, Gatekeeper, Kyverno or similar webhooks, no extra ValidatingAdmissionPolicies"
fi

step "B6. Cluster-scoped objects the chart creates that already exist"
resource_of() {
  case "$1" in
    CustomResourceDefinition) echo crd ;;
    MutatingWebhookConfiguration) echo mutatingwebhookconfiguration ;;
    ValidatingWebhookConfiguration) echo validatingwebhookconfiguration ;;
    SecurityContextConstraints) echo scc ;;
    *) echo "$1" | tr '[:upper:]' '[:lower:]' ;;
  esac
}
if [ "$RENDERED" = 1 ]; then
  r 'CustomResourceDefinition|ClusterRole|ClusterRoleBinding|MutatingWebhookConfiguration|ValidatingWebhookConfiguration|PriorityClass|IngressClass|APIService|CSIDriver|SecurityContextConstraints|StorageClass' \
    | jq -r '"\(.kind)\t\(.metadata.name)"' | sort -u > "$TMP/cluster.tsv"
  conflicts=0; unread=0
  # On an upgrade Helm does not apply crds/ again, so existing CRDs only matter for a first install.
  installed=0
  helm status "$RELEASE" -n "$NAMESPACE" >/dev/null 2>&1 && installed=1
  [ "$installed" = 1 ] && note "release $RELEASE is already installed in $NAMESPACE: CRDs are not checked (Helm applies crds/ on first install only)"
  while IFS=$'\t' read -r kind name; do
    res=$(resource_of "$kind")
    oget o get "$res" "$name" -o json --show-managed-fields; rc=$?
    [ "$rc" = 1 ] && continue
    [ "$rc" = 2 ] && { unread=$(( unread + 1 )); continue; }
    rel=$(jq -r '.metadata.annotations["meta.helm.sh/release-name"] // ""' <<<"$o")
    relns=$(jq -r '.metadata.annotations["meta.helm.sh/release-namespace"] // ""' <<<"$o")
    if [ "$kind" = CustomResourceDefinition ] && [ "$installed" = 1 ]; then
      continue
    elif [ "$kind" = CustomResourceDefinition ]; then
      # Helm does not track CRDs from crds/; Helm 4 applies them server-side and refuses fields
      # another field manager owns.
      mgr=$(jq -r '[.metadata.managedFields[]?.manager | select(. != "helm" and . != "kube-apiserver")] | unique | join(",")' <<<"$o")
      [ -n "$mgr" ] && { bad "CRD $name exists, written by $mgr"; conflicts=$(( conflicts + 1 )); }
    elif [ "$rel" != "$RELEASE" ] || [ "$relns" != "$NAMESPACE" ]; then
      bad "$kind $name exists, owned by ${rel:+Helm release $relns/$rel}${rel:-no Helm release}"; conflicts=$(( conflicts + 1 ))
    fi
  done < "$TMP/cluster.tsv"
  [ "$unread" -gt 0 ] && unknown "$unread of the cluster-scoped objects were not readable"
  [ "$conflicts" -eq 0 ] && [ "$unread" -eq 0 ] && ok "none of the $(wc -l < "$TMP/cluster.tsv" | tr -d ' ') cluster-scoped objects exists under another owner"
  note "existing CRDs are compared by owner, not by version: a CRD of another version owned by Helm is not reported"
else
  unknown "chart not rendered: cannot list the cluster-scoped objects it creates"
fi
for a in v1beta1.external.metrics.k8s.io v1beta1.custom.metrics.k8s.io; do
  if oget s get apiservice "$a" -o json; then
    note "APIService $a is served by $(jq -r '.spec.service | if . then "\(.namespace)/\(.name)" else "the API server" end' <<<"$s") (KEDA serves the first, the Prometheus adapter the second)"
  fi
done

step "B7. Operators installed through OLM that bring the same components"
if oget csvs get csv -A -o json; then
  jq -r '.items[] | select(.metadata.name | test("keda|custom-metrics|strimzi|streams|amq|cert-manager|trust|serverless|servicemesh|sail|istio|rhods|opendatahub|spark|ray|kueue|oadp|velero"; "i"))
    | "\(.metadata.name) \(.spec.version) \(.status.phase)"' <<<"$csvs" | sort -u > "$TMP/csv.txt"
  if [ -s "$TMP/csv.txt" ]; then
    unknown "operators that overlap with what Hopsworks installs; each needs a decision before the install:"
    while read -r l; do note "$l"; done < "$TMP/csv.txt"
  else
    ok "no overlapping operator"
  fi
else
  unknown "operators (ClusterServiceVersions) not readable: $(head -1 "$TMP/oc.err")"
fi

step "B8. Network policy"
for ns in "$NAMESPACE" ${PROJECT_NS:+"$PROJECT_NS"}; do
  if ! oget np get networkpolicy -n "$ns" -o json; then unknown "$ns: network policies not readable"; continue; fi
  own=$(jq -r '[.items[] | select(.metadata.annotations["meta.helm.sh/release-name"] == null and .metadata.ownerReferences == null) | .metadata.name] | join(", ")' <<<"$np")
  oget ef get egressfirewall -n "$ns" -o name; rc=$?
  [ "$rc" = 2 ] && { unknown "$ns: egress firewalls not readable"; continue; }
  ef=$(printf '%s' "$ef" | tr '\n' ' '); ef=${ef% }
  if [ -n "$own$ef" ]; then unknown "$ns has policies Hopsworks does not create: ${own}${ef:+ egressfirewall: $ef}"; note "project namespaces must reach the Hopsworks namespace, and both must reach S3, Quay and the package feeds"
  else ok "$ns: no network policy or egress firewall besides Hopsworks' own"; fi
done
anp=""; anp_ok=1
for k in adminnetworkpolicy baselineadminnetworkpolicy; do
  oget x get "$k" -o name; rc=$?
  [ "$rc" = 0 ] && anp="$anp $(tr '\n' ' ' <<<"$x")"
  [ "$rc" = 2 ] && anp_ok=0
done
if [ -n "${anp// /}" ]; then unknown "cluster-wide admin network policies:$anp"
elif [ "$anp_ok" = 1 ]; then ok "no AdminNetworkPolicy or BaselineAdminNetworkPolicy"
else unknown "admin network policies not readable"; fi
if oget tpl get project.config.openshift.io cluster -o jsonpath='{.spec.projectRequestTemplate.name}'; then
  [ -n "$tpl" ] && note "project request template $tpl: new project namespaces get what it contains"
else unknown "the cluster project configuration is not readable"; fi

step "B9. Image policies"
if oget ic get image.config.openshift.io cluster -o json; then
  rs=$(jq -c '.spec.registrySources // {} | {allowedRegistries, blockedRegistries}' <<<"$ic")
  if jq -e '.allowedRegistries or .blockedRegistries' <<<"$rs" >/dev/null; then unknown "registry restrictions: $rs; Quay must be allowed"
  else ok "no allowed or blocked registry list"; fi
else
  unknown "image.config.openshift.io/cluster not readable"
fi
sig=""; sig_ok=1
for k in clusterimagepolicy imagepolicy; do
  oget x get "$k" -A -o name; rc=$?
  [ "$rc" = 0 ] && sig="$sig $(tr '\n' ' ' <<<"$x")"
  [ "$rc" = 2 ] && sig_ok=0
done
if [ -n "${sig// /}" ]; then bad "signature verification policies:$sig; Hopsworks images are not signed"
elif [ "$sig_ok" = 1 ]; then ok "no ClusterImagePolicy or ImagePolicy"
else unknown "image signature policies not readable"; fi

step "B10. Ingress, routes and router shards"
if [ "$RENDERED" = 1 ]; then
  r Ingress | jq -r '"\(.metadata.name)\t\(.spec.ingressClassName // "-")\t\([.spec.rules[]?.host // "*"] | join(","))\t\(if .spec.tls then "tls" else "no tls" end)"' > "$TMP/ing.tsv"
  while IFS=$'\t' read -r n cls hosts tls; do
    note "Ingress $n: class $cls, host $hosts, $tls"
    if [ "$cls" != "-" ]; then
      oget scratch get ingressclass "$cls" -o name; rc=$?
      [ "$rc" = 1 ] && bad "IngressClass $cls of Ingress $n does not exist"
      [ "$rc" = 2 ] && unknown "IngressClass $cls not readable"
    fi
    case "$hosts" in *hopsworks.ai.local*) unknown "Ingress $n uses the chart's default host hopsworks.ai.local: set the hostname Trafikverket's router shard and DNS serve" ;; esac
  done < "$TMP/ing.tsv"
  if [ -s "$TMP/ing.tsv" ] && oget routes get route -A -o json; then
    for h in $(cut -f3 "$TMP/ing.tsv" | tr ',' ' '); do
      [ "$h" = "*" ] && continue
      other=$(jq -r --arg h "$h" --arg ns "$NAMESPACE" '[.items[] | select(.spec.host == $h and .metadata.namespace != $ns) | "\(.metadata.namespace)/\(.metadata.name)"] | join(", ")' <<<"$routes")
      [ -n "$other" ] && bad "host $h is already taken by Route $other"
    done
  fi
fi
nslabels=$(jq -c '.metadata.labels // {}' <<<"$nsjson")
if oget ics get ingresscontroller -n openshift-ingress-operator -o json; then
  jq -r --argjson l "$nslabels" '.items[]
    | (.spec.namespaceSelector // {}) as $s
    | (if ($s.matchExpressions // []) | length > 0 then "unknown"
       elif all(($s.matchLabels // {}) | to_entries[]; $l[.key] == .value) then "admits" else "skips" end) as $v
    | "\($v)\t\(.metadata.name)\tdomain=\(.status.domain)\tnamespaceSelector=\($s | tostring)\trouteSelector=\(.spec.routeSelector // {} | tostring)"' <<<"$ics" > "$TMP/routers.tsv"
  if grep -q '^admits' "$TMP/routers.tsv"; then ok "a router admits Routes from $NAMESPACE by its namespace selector"
  elif grep -q '^unknown' "$TMP/routers.tsv"; then unknown "router selectors use expressions: check which shard serves $NAMESPACE"
  else bad "no router admits Routes from $NAMESPACE: the UI would be unreachable"; fi
  while IFS=$'\t' read -r v rest; do note "$v $rest"; done < "$TMP/routers.tsv"
  note "a routeSelector, if set, must also match the labels on Hopsworks' Ingress"
else
  unknown "ingress controllers not readable: $(head -1 "$TMP/oc.err")"
fi

step "B11. The separately installed Consul (install step 5)"
if [ "$RENDERED" = 1 ]; then
  # Secrets the chart mounts or reads from the Consul release; the chart does not create them.
  r 'Deployment|StatefulSet|DaemonSet|Job|CronJob' | jq -r '(if .kind == "CronJob" then .spec.jobTemplate.spec.template.spec else .spec.template.spec end) as $p
    | ([$p.volumes[]?.secret.secretName] + [($p.containers + ($p.initContainers // []))[] | .env[]?.valueFrom.secretKeyRef.name, .envFrom[]?.secretRef.name])[]
    | select(. != null and test("consul"))' | sort -u > "$TMP/consul-secrets.txt"
  rendered_secrets=$(r Secret | jq -r '.metadata.name')
  missing=(); unread=0
  while read -r s; do
    [ -z "$s" ] && continue
    grep -qxF "$s" <<<"$rendered_secrets" && continue
    oget scratch get secret "$s" -n "$NAMESPACE" -o name; rc=$?
    [ "$rc" = 1 ] && missing+=("$s"); [ "$rc" = 2 ] && unread=$(( unread + 1 ))
  done < "$TMP/consul-secrets.txt"
  n=$(grep -c . "$TMP/consul-secrets.txt")
  if [ "${#missing[@]}" -gt 0 ]; then bad "Secrets the chart takes from Consul are missing: ${missing[*]} (install step 5, release hopsworks-consul)"
  elif [ "$unread" -gt 0 ]; then unknown "$unread of the Consul Secrets not readable"
  elif [ "$n" -gt 0 ]; then ok "the $n Secrets the chart takes from Consul exist"; fi
fi
if oget cp get pods -n "$NAMESPACE" -l app=consul,component=server -o json; then
  total=$(jq '.items | length' <<<"$cp"); ready=$(jq '[.items[] | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))] | length' <<<"$cp")
  if [ "$total" -eq 0 ]; then bad "no Consul server pod in $NAMESPACE: install step 5 first"
  elif [ "$ready" -lt "$total" ]; then bad "Consul servers ready: $ready of $total"
  else ok "Consul servers ready: $ready of $total"; fi
else
  unknown "Consul pods not readable"
fi
oget scratch get svc hopsworks-consul-dns -n "$NAMESPACE" -o name; rc=$?
[ "$rc" = 1 ] && bad "Service hopsworks-consul-dns not found: the .consul forward of install step 5 has nothing to point at"
[ "$rc" = 2 ] && unknown "Service hopsworks-consul-dns not readable"

finish