#!/bin/bash
# prom-scope.sh <namespace-regex>: limit Hopsworks' Prometheus to the namespaces matching the regex
# (the Hopsworks namespace and the project namespaces). Chart 5.2.4 discovers pods and endpoints in
# every namespace and keeps cAdvisor series for every container on every node, which on a large
# shared cluster outgrows the server's memory. This edits the live ConfigMap; the server's
# configmap-reload sidecar reloads it. A helm upgrade puts the chart's version back: run it again after.
#   NAMESPACE   the Hopsworks namespace (default tns-valkyria-platform)
#   DRY_RUN=1   write prometheus.scoped.yml and stop
set -euo pipefail
NSRE="${1:?usage: $0 '<namespace regex, e.g. tns-valkyria-.*>'}"
NAMESPACE="${NAMESPACE:-tns-valkyria-platform}"
CM=hopsworks-prometheus-server

oc -n "$NAMESPACE" get cm "$CM" -o jsonpath='{.data.prometheus\.yml}' > prometheus.orig.yml
grep -q 'job_name: kubernetes-nodes-cadvisor' prometheus.orig.yml || { echo "unexpected prometheus.yml: no cadvisor job"; exit 2; }
already=0; grep -qxF "    regex: ($NSRE)?" prometheus.orig.yml && already=1
# FORCE=1 on an already scoped config only hands the field back to Helm's field manager, for a
# ConfigMap an older version of this script wrote with "oc replace".
[ "$already" = 1 ] && [ "${FORCE:-0}" != 1 ] && { echo "already scoped to $NSRE (FORCE=1 rewrites it as Helm's)"; exit 0; }

# Pod and endpoint jobs: keep targets in matching namespaces only. cAdvisor: keep series of matching
# namespaces, and series without a namespace (node and system cgroups).
awk -v re="$NSRE" '
  /^alerting:/ { inscrape = 0 }
  /^scrape_configs:/ { inscrape = 1 }
  inscrape && /^- / { job = ""; sd = 0 }
  inscrape && /^(- |  )job_name: / { job = $NF }
  inscrape && /^  - role: (pod|endpoints)$/ { sd = 1 }
  inscrape && job == "kubernetes-nodes-cadvisor" && /^  relabel_configs:$/ {
    print "  metric_relabel_configs:"
    print "  - action: keep"
    print "    regex: (" re ")?"
    print "    source_labels:"
    print "    - namespace"
  }
  { print }
  inscrape && sd && /^  relabel_configs:$/ {
    print "  - action: keep"
    print "    regex: " re
    print "    source_labels:"
    print "    - __meta_kubernetes_namespace"
  }
' prometheus.orig.yml > prometheus.scoped.yml
[ "$already" = 1 ] && cp prometheus.orig.yml prometheus.scoped.yml

echo "scoped to $NSRE: $(grep -cxF "    regex: $NSRE" prometheus.scoped.yml) pod and endpoint jobs, $(grep -cxF "    regex: ($NSRE)?" prometheus.scoped.yml) cAdvisor job"
[ "${DRY_RUN:-0}" = 1 ] && { echo "dry run: prometheus.scoped.yml written"; exit 0; }

# Written as Helm's field manager: Helm 4 applies server-side, and a field owned by another manager
# (kubectl-replace, kubectl-edit) makes the next helm upgrade fail with "conflict occurred while applying
# object ... ConfigMap". As Helm's, the next upgrade simply puts the chart's version back.
jq -n --arg ns "$NAMESPACE" --arg cm "$CM" --rawfile f prometheus.scoped.yml \
  '{apiVersion: "v1", kind: "ConfigMap", metadata: {name: $cm, namespace: $ns}, data: {"prometheus.yml": $f}}' \
  | oc apply --server-side --field-manager=helm --force-conflicts -f -
echo "ConfigMap updated; the server reloads within a minute or two (kubelet ConfigMap sync)."