prometheus

modules/swarm/prometheus/cfg/prometheus.yml

@@ -0,0 +1,73 @@
+---
+global:
+  scrape_interval: 5s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: c3voc
+    scrape_interval: 30s
+    scheme: https
+    static_configs:
+      - targets: ["monitoring.c3voc.de:443"]
+
+  - job_name: vector
+    static_configs:
+      - targets: ["ax41-1.fsn.mon2.de:28668"]
+
+  - job_name: node
+    static_configs:
+      - targets:
+          - ingest-fsn.chaoswest.tv:9100
+          - ingest-nbg.chaoswest.tv:9100
+          - oldportier.chaoswest.tv:9100
+
+  - job_name: srs
+    static_configs:
+      - targets:
+          - ingest-fsn.chaoswest.tv:9185
+          - ingest-nbg.chaoswest.tv:9185
+
+  - job_name: ffmpeg
+    scrape_interval: 1s
+    static_configs:
+      - targets:
+          - oldportier.chaoswest.tv:2342
+
+  - job_name: "dockerswarm"
+    dockerswarm_sd_configs:
+      - host: tcp://docker:2375
+        role: tasks
+    relabel_configs:
+      # Only keep containers that should be running.
+      - source_labels: [__meta_dockerswarm_task_desired_state]
+        regex: running
+        action: keep
+      # Only keep containers that have a `prometheus-job` label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
+        regex: .+
+        action: keep
+      # Use the prometheus-job Swarm label as Prometheus job label.
+      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
+        target_label: job
+      # Only scrape within the metrics network
+      - source_labels: [__meta_dockerswarm_network_name]
+        regex: metrics
+        action: keep
+      # Fix address and use the label defined listening port
+      - source_labels:
+          [__address__, __meta_dockerswarm_service_label_prometheus_port]
+        separator: ":"
+        regex: "(.*):.*:(.*)"
+        target_label: __address__
+        replacement: "${1}:${2}"
+      # if this is a node exporter, use the node hostname as the instance label
+      - source_labels:
+          [__meta_dockerswarm_service_name, __meta_dockerswarm_node_hostname]
+        separator: ":"
+        regex: "prometheus-node-exporter:(.*)"
+        target_label: instance
+        replacement: "${1}"

modules/swarm/prometheus/docker-socket-proxy.tf

@@ -0,0 +1,54 @@
+# Prometheus has switched to running with 'nobody' user, which doesn't have
+# access to the docker socket. This service runs a proxy that allows prometheus
+# to access the docker socket.
+
+# Since it internally uses HAProxy, it can also restrict access to parts of the
+# docker API. By default, everything but the parts allowed in the envs below
+# is restricted.
+
+# Nice bonus: it also would allow us to run prometheus on a non-manager node.
+
+
+data "docker_registry_image" "docker_socket_proxy" {
+  name = "ghcr.io/tecnativa/docker-socket-proxy:0.1"
+}
+
+resource "docker_service" "docker_socket_proxy" {
+  name = "prometheus_docker_socket_proxy"
+
+  mode {
+    global = true
+  }
+
+  task_spec {
+    placement {
+      constraints = [
+        "node.role==manager",
+      ]
+    }
+
+    networks_advanced {
+      name = docker_network.docker_socket_proxy.id
+      aliases = [
+        "docker",
+      ]
+    }
+
+    container_spec {
+      image = "${data.docker_registry_image.docker_socket_proxy.name}@${data.docker_registry_image.docker_socket_proxy.sha256_digest}"
+
+      env = {
+        NODES    = "1"
+        NETWORKS = "1"
+        SERVICES = "1"
+        TASKS    = "1"
+      }
+
+      mounts {
+        target = "/var/run/docker.sock"
+        source = "/var/run/docker.sock"
+        type   = "bind"
+      }
+    }
+  }
+}

modules/swarm/prometheus/network.tf

@@ -0,0 +1,23 @@
+data "docker_network" "traefik" {
+  name = "traefik"
+}
+
+resource "docker_network" "metrics" {
+  name       = "metrics"
+  attachable = true
+  driver     = "overlay"
+
+  lifecycle {
+    ignore_changes = [labels]
+  }
+}
+
+resource "docker_network" "docker_socket_proxy" {
+  name       = "prometheus_docker_socket_proxy"
+  attachable = true
+  driver     = "overlay"
+
+  lifecycle {
+    ignore_changes = [labels]
+  }
+}

modules/swarm/prometheus/node-exporter.tf

@@ -0,0 +1,64 @@
+data "docker_registry_image" "node_exporter" {
+  name = "prom/node-exporter"
+}
+
+locals {
+  labels_node_exporter = {
+    "shepherd.auto-update" = "true",
+    "prometheus.job"       = "node",
+    "prometheus.port"      = "9100",
+  }
+}
+
+resource "docker_service" "node_exporter" {
+  name = "prometheus_node_exporter"
+
+  mode {
+    global = true
+  }
+
+  dynamic "labels" {
+    for_each = local.labels_node_exporter
+    content {
+      label = labels.key
+      value = labels.value
+    }
+  }
+
+  task_spec {
+    networks_advanced {
+      name = docker_network.metrics.id
+    }
+
+    container_spec {
+      image = "${data.docker_registry_image.node_exporter.name}@${data.docker_registry_image.node_exporter.sha256_digest}"
+
+      args = [
+        "--path.rootfs=/host/root",
+        "--path.procfs=/host/proc",
+        "--path.sysfs=/host/sys",
+        "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)"
+      ]
+      mounts {
+        target    = "/host/root"
+        source    = "/"
+        type      = "bind"
+        read_only = true
+      }
+
+      mounts {
+        target    = "/host/proc"
+        source    = "/proc"
+        type      = "bind"
+        read_only = true
+      }
+
+      mounts {
+        target    = "/host/sys"
+        source    = "/sys"
+        type      = "bind"
+        read_only = true
+      }
+    }
+  }
+}

modules/swarm/prometheus/prometheus.tf

@@ -0,0 +1,87 @@
+data "docker_registry_image" "prometheus" {
+  name = "prom/prometheus"
+}
+
+locals {
+  labels_prometheus = {
+    "shepherd.auto-update"                                      = "true",
+    "traefik.enable"                                            = "true"
+    "traefik.http.services.prometheus.loadbalancer.server.port" = "9090",
+    "traefik.http.routers.prometheus.rule"                      = "Host(`prometheus.chaoswest.tv`)",
+    "traefik.http.routers.prometheus.tls"                       = "true",
+    "traefik.http.routers.prometheus.tls.certresolver"          = "default",
+    "traefik.http.routers.prometheus.middlewares"               = "prometheus-auth",
+    "traefik.http.middlewares.prometheus-auth.basicauth.users"  = "prometheus:$2y$10$XK9vcKzVol9ZWJLiSbKruuFP2jBsVrFY8Vc4ANtm6JnhsXgbnfLYm"
+  }
+}
+
+resource "docker_config" "prometheus" {
+  name = "prometheus-yml-${replace(timestamp(), ":", ".")}"
+  data = base64encode(file("${path.module}/cfg/prometheus.yml"))
+  lifecycle {
+    ignore_changes        = [name]
+    create_before_destroy = true
+  }
+}
+
+resource "docker_service" "prometheus" {
+  name = "prometheus"
+
+  dynamic "labels" {
+    for_each = local.labels_prometheus
+    content {
+      label = labels.key
+      value = labels.value
+    }
+  }
+
+  task_spec {
+    networks_advanced {
+      name = data.docker_network.traefik.id
+    }
+
+    networks_advanced {
+      name = docker_network.metrics.id
+    }
+
+    networks_advanced {
+      name = docker_network.docker_socket_proxy.id
+    }
+
+    container_spec {
+      image = "${data.docker_registry_image.prometheus.name}@${data.docker_registry_image.prometheus.sha256_digest}"
+
+      configs {
+        config_id   = docker_config.prometheus.id
+        config_name = docker_config.prometheus.name
+        file_name   = "/etc/prometheus/prometheus.yml"
+        file_uid    = "0"
+        file_gid    = "0"
+        file_mode   = "0444"
+      }
+
+      mounts {
+        target = "/prometheus"
+        source = "/mnt/data/prometheus/"
+        type   = "bind"
+      }
+
+      mounts {
+        target = "/var/run/docker.sock"
+        source = "/var/run/docker.sock"
+        type   = "bind"
+      }
+    }
+  }
+}
+
+data "hetznerdns_zone" "primary" {
+  name = "chaoswest.tv"
+}
+
+resource "hetznerdns_record" "primary" {
+  zone_id = data.hetznerdns_zone.primary.id
+  name    = "prometheus"
+  value   = "ax41-1.fsn.mon2.de."
+  type    = "CNAME"
+}

modules/swarm/prometheus/version.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_version = "1.5.5"
+  required_providers {
+    hetznerdns = {
+      source  = "timohirt/hetznerdns"
+      version = "~>2.2"
+    }
+    docker = {
+      source  = "kreuzwerker/docker"
+      version = "~>3.0"
+    }
+  }
+}

stacks/ax41-1/.terraform.lock.hcl

@@ -20,6 +20,7 @@ provider "registry.terraform.io/kreuzwerker/docker" {
   version     = "3.0.2"
   constraints = "~> 3.0"
   hashes = [
+    "h1:cT2ccWOtlfKYBUE60/v2/4Q6Stk1KYTNnhxSck+VPlU=",
     "h1:tryCE8s9BiT6VyfnGgU1mUt9s0HcCKlRERdLd2fr010=",
     "zh:15b0a2b2b563d8d40f62f83057d91acb02cd0096f207488d8b4298a59203d64f",
     "zh:23d919de139f7cd5ebfd2ff1b94e6d9913f0977fcfc2ca02e1573be53e269f95",

stacks/ax41-1/README.md

@@ -23,6 +23,7 @@
 | <a name="module_grafana"></a> [grafana](#module\_grafana) | ../../modules/swarm/grafana | n/a |
 | <a name="module_hedgedoc"></a> [hedgedoc](#module\_hedgedoc) | ../../modules/swarm/hedgedoc | n/a |
 | <a name="module_jitsi"></a> [jitsi](#module\_jitsi) | ../../modules/swarm/jitsi | n/a |
+| <a name="module_prometheus"></a> [prometheus](#module\_prometheus) | ../../modules/swarm/prometheus | n/a |
 | <a name="module_shepherd"></a> [shepherd](#module\_shepherd) | ../../modules/swarm/shepherd | n/a |
 | <a name="module_shit"></a> [shit](#module\_shit) | ../../modules/swarm/shit | n/a |
 | <a name="module_spaceapi"></a> [spaceapi](#module\_spaceapi) | ../../modules/swarm/spaceapi | n/a |

stacks/ax41-1/main.tf

@@ -37,3 +37,7 @@ module "spaceapi" {
 module "forgejo" {
   source = "../../modules/swarm/forgejo"
 }
+
+module "prometheus" {
+  source = "../../modules/swarm/prometheus"
+}

stacks/ax41-1/secrets.enc.yaml

@@ -32,8 +32,8 @@ sops:
             WmlRUnowa2lMNWpDT0xEU0htV0w3U00K1f/SO/FBvC9lIBzveBEwhopj5ryMVCmD
             jw8AdxvmMwsCSfIROKkzMqiUs2zsj6FOMlYFI1Rb07mItSO2Yd7TsA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-01-25T17:33:50Z"
+    lastmodified: "2024-01-30T17:00:41Z"
-    mac: ENC[AES256_GCM,data:VUSGKproAmU286+mf9nM/IzddWcT18/6tk73guFUB6C3Turfv9DXDW1wSXu6vTxGlOinChxlcBCnsGfk7gfFisjAEZHJq9IJ0P2myYp+lKbybolm0fbTZ4jda7DUnvN3n4I0EqFoUa/vPN/DSkdt0hKj1Ayz5AdFIAvOtkphMPA=,iv:dHhMBT3T3bWPSUadDgo+h2KSf2qC32q+nM26eK9ivDo=,tag:30dNpRvB6vet5UoWy/DZtg==,type:str]
+    mac: ENC[AES256_GCM,data:GvVUAo5Qtp0Dcnffh42jGkpT3khDRYXf6ws6Q3n2dWk+q39+xDQ3oxCGKMsYwbxrQ5s1oSI7dENSfxQA1Rwk3+Z0wmrRry9fxlYEnDeYLiR2Jxp0+7zDWUcusfSnjC/ASmwCYSFBcQM4jhD4uyVmhluS0E5KrjOD223Z6vtjxck=,iv:5OFMGJatztWTUR2Xb49CYl0Z42UsieYPTR6YoBn9UmM=,tag:yibooLYgoAN+IeiHVyexKg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1