infra
/
tf
2
0
Fork 0
Code Pull Requests Activity

prometheus

This commit is contained in:
Jan Koppe 2024-01-30 23:59:54 +01:00
parent 607a39ce7e
commit 618cbeea59
Signed by: thunfisch
GPG Key ID: BE935B0735A2129B
13 changed files with 322 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
modules/swarm/prometheus/cfg/prometheus.yml Normal file
View File

@ -0,0 +1,73 @@
---
global:
scrape_interval: 5s
evaluation_interval: 15s
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: c3voc
scrape_interval: 30s
scheme: https
static_configs:
- targets: ["monitoring.c3voc.de:443"]
- job_name: vector
static_configs:
- targets: ["ax41-1.fsn.mon2.de:28668"]
- job_name: node
static_configs:
- targets:
- ingest-fsn.chaoswest.tv:9100
- ingest-nbg.chaoswest.tv:9100
- oldportier.chaoswest.tv:9100
- job_name: srs
static_configs:
- targets:
- ingest-fsn.chaoswest.tv:9185
- ingest-nbg.chaoswest.tv:9185
- job_name: ffmpeg
scrape_interval: 1s
static_configs:
- targets:
- oldportier.chaoswest.tv:2342
- job_name: "dockerswarm"
dockerswarm_sd_configs:
- host: tcp://docker:2375
role: tasks
relabel_configs:
# Only keep containers that should be running.
- source_labels: [__meta_dockerswarm_task_desired_state]
regex: running
action: keep
# Only keep containers that have a `prometheus-job` label.
- source_labels: [__meta_dockerswarm_service_label_prometheus_job]
regex: .+
action: keep
# Use the prometheus-job Swarm label as Prometheus job label.
- source_labels: [__meta_dockerswarm_service_label_prometheus_job]
target_label: job
# Only scrape within the metrics network
- source_labels: [__meta_dockerswarm_network_name]
regex: metrics
action: keep
# Fix address and use the label defined listening port
- source_labels:
[__address__, __meta_dockerswarm_service_label_prometheus_port]
separator: ":"
regex: "(.*):.*:(.*)"
target_label: __address__
replacement: "${1}:${2}"
# if this is a node exporter, use the node hostname as the instance label
- source_labels:
[__meta_dockerswarm_service_name, __meta_dockerswarm_node_hostname]
separator: ":"
regex: "prometheus-node-exporter:(.*)"
target_label: instance
replacement: "${1}"

54
modules/swarm/prometheus/docker-socket-proxy.tf Normal file
View File

@ -0,0 +1,54 @@
# Prometheus has switched to running with 'nobody' user, which doesn't have
# access to the docker socket. This service runs a proxy that allows prometheus
# to access the docker socket.
# Since it internally uses HAProxy, it can also restrict access to parts of the
# docker API. By default, everything but the parts allowed in the envs below
# is restricted.
# Nice bonus: it also would allow us to run prometheus on a non-manager node.
data "docker_registry_image" "docker_socket_proxy" {
name = "ghcr.io/tecnativa/docker-socket-proxy:0.1"
}
resource "docker_service" "docker_socket_proxy" {
name = "prometheus_docker_socket_proxy"
mode {
global = true
}
task_spec {
placement {
constraints = [
"node.role==manager",
]
}
networks_advanced {
name = docker_network.docker_socket_proxy.id
aliases = [
"docker",
]
}
container_spec {
image = "${data.docker_registry_image.docker_socket_proxy.name}@${data.docker_registry_image.docker_socket_proxy.sha256_digest}"
env = {
NODES = "1"
NETWORKS = "1"
SERVICES = "1"
TASKS = "1"
}
mounts {
target = "/var/run/docker.sock"
source = "/var/run/docker.sock"
type = "bind"
}
}
}
}

0
modules/swarm/prometheus/main.tf Normal file
View File

23
modules/swarm/prometheus/network.tf Normal file
View File

@ -0,0 +1,23 @@
data "docker_network" "traefik" {
name = "traefik"
}
resource "docker_network" "metrics" {
name = "metrics"
attachable = true
driver = "overlay"
lifecycle {
ignore_changes = [labels]
}
}
resource "docker_network" "docker_socket_proxy" {
name = "prometheus_docker_socket_proxy"
attachable = true
driver = "overlay"
lifecycle {
ignore_changes = [labels]
}
}

64
modules/swarm/prometheus/node-exporter.tf Normal file
View File

@ -0,0 +1,64 @@
data "docker_registry_image" "node_exporter" {
name = "prom/node-exporter"
}
locals {
labels_node_exporter = {
"shepherd.auto-update" = "true",
"prometheus.job" = "node",
"prometheus.port" = "9100",
}
}
resource "docker_service" "node_exporter" {
name = "prometheus_node_exporter"
mode {
global = true
}
dynamic "labels" {
for_each = local.labels_node_exporter
content {
label = labels.key
value = labels.value
}
}
task_spec {
networks_advanced {
name = docker_network.metrics.id
}
container_spec {
image = "${data.docker_registry_image.node_exporter.name}@${data.docker_registry_image.node_exporter.sha256_digest}"
args = [
"--path.rootfs=/host/root",
"--path.procfs=/host/proc",
"--path.sysfs=/host/sys",
"--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)"
]
mounts {
target = "/host/root"
source = "/"
type = "bind"
read_only = true
}
mounts {
target = "/host/proc"
source = "/proc"
type = "bind"
read_only = true
}
mounts {
target = "/host/sys"
source = "/sys"
type = "bind"
read_only = true
}
}
}
}

0
modules/swarm/prometheus/outputs.tf Normal file
View File

87
modules/swarm/prometheus/prometheus.tf Normal file
View File

@ -0,0 +1,87 @@
data "docker_registry_image" "prometheus" {
name = "prom/prometheus"
}
locals {
labels_prometheus = {
"shepherd.auto-update" = "true",
"traefik.enable" = "true"
"traefik.http.services.prometheus.loadbalancer.server.port" = "9090",
"traefik.http.routers.prometheus.rule" = "Host(`prometheus.chaoswest.tv`)",
"traefik.http.routers.prometheus.tls" = "true",
"traefik.http.routers.prometheus.tls.certresolver" = "default",
"traefik.http.routers.prometheus.middlewares" = "prometheus-auth",
"traefik.http.middlewares.prometheus-auth.basicauth.users" = "prometheus:$2y$10$XK9vcKzVol9ZWJLiSbKruuFP2jBsVrFY8Vc4ANtm6JnhsXgbnfLYm"
}
}
resource "docker_config" "prometheus" {
name = "prometheus-yml-${replace(timestamp(), ":", ".")}"
data = base64encode(file("${path.module}/cfg/prometheus.yml"))
lifecycle {
ignore_changes = [name]
create_before_destroy = true
}
}
resource "docker_service" "prometheus" {
name = "prometheus"
dynamic "labels" {
for_each = local.labels_prometheus
content {
label = labels.key
value = labels.value
}
}
task_spec {
networks_advanced {
name = data.docker_network.traefik.id
}
networks_advanced {
name = docker_network.metrics.id
}
networks_advanced {
name = docker_network.docker_socket_proxy.id
}
container_spec {
image = "${data.docker_registry_image.prometheus.name}@${data.docker_registry_image.prometheus.sha256_digest}"
configs {
config_id = docker_config.prometheus.id
config_name = docker_config.prometheus.name
file_name = "/etc/prometheus/prometheus.yml"
file_uid = "0"
file_gid = "0"
file_mode = "0444"
}
mounts {
target = "/prometheus"
source = "/mnt/data/prometheus/"
type = "bind"
}
mounts {
target = "/var/run/docker.sock"
source = "/var/run/docker.sock"
type = "bind"
}
}
}
}
data "hetznerdns_zone" "primary" {
name = "chaoswest.tv"
}
resource "hetznerdns_record" "primary" {
zone_id = data.hetznerdns_zone.primary.id
name = "prometheus"
value = "ax41-1.fsn.mon2.de."
type = "CNAME"
}

0
modules/swarm/prometheus/variables.tf Normal file
View File

13
modules/swarm/prometheus/version.tf Normal file
View File

@ -0,0 +1,13 @@
terraform {
required_version = "1.5.5"
required_providers {
hetznerdns = {
source = "timohirt/hetznerdns"
version = "~>2.2"
}
docker = {
source = "kreuzwerker/docker"
version = "~>3.0"
}
}
}

1
stacks/ax41-1/.terraform.lock.hcl
View File

@ -20,6 +20,7 @@ provider "registry.terraform.io/kreuzwerker/docker" {
version = "3.0.2"
constraints = "~> 3.0"
hashes = [
"h1:cT2ccWOtlfKYBUE60/v2/4Q6Stk1KYTNnhxSck+VPlU=",
"h1:tryCE8s9BiT6VyfnGgU1mUt9s0HcCKlRERdLd2fr010=",
"zh:15b0a2b2b563d8d40f62f83057d91acb02cd0096f207488d8b4298a59203d64f",
"zh:23d919de139f7cd5ebfd2ff1b94e6d9913f0977fcfc2ca02e1573be53e269f95",

1
stacks/ax41-1/README.md
View File

@ -23,6 +23,7 @@
| <a name="module_grafana"></a> [grafana](#module\_grafana) | ../../modules/swarm/grafana | n/a |
| <a name="module_hedgedoc"></a> [hedgedoc](#module\_hedgedoc) | ../../modules/swarm/hedgedoc | n/a |
| <a name="module_jitsi"></a> [jitsi](#module\_jitsi) | ../../modules/swarm/jitsi | n/a |
| <a name="module_prometheus"></a> [prometheus](#module\_prometheus) | ../../modules/swarm/prometheus | n/a |
| <a name="module_shepherd"></a> [shepherd](#module\_shepherd) | ../../modules/swarm/shepherd | n/a |
| <a name="module_shit"></a> [shit](#module\_shit) | ../../modules/swarm/shit | n/a |
| <a name="module_spaceapi"></a> [spaceapi](#module\_spaceapi) | ../../modules/swarm/spaceapi | n/a |

4
stacks/ax41-1/main.tf
View File

@ -37,3 +37,7 @@ module "spaceapi" {
module "forgejo" {
source = "../../modules/swarm/forgejo"
}
module "prometheus" {
source = "../../modules/swarm/prometheus"
}

4
stacks/ax41-1/secrets.enc.yaml
View File

@ -32,8 +32,8 @@ sops:
WmlRUnowa2lMNWpDT0xEU0htV0w3U00K1f/SO/FBvC9lIBzveBEwhopj5ryMVCmD
jw8AdxvmMwsCSfIROKkzMqiUs2zsj6FOMlYFI1Rb07mItSO2Yd7TsA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2024-01-25T17:33:50Z"
mac: ENC[AES256_GCM,data:VUSGKproAmU286+mf9nM/IzddWcT18/6tk73guFUB6C3Turfv9DXDW1wSXu6vTxGlOinChxlcBCnsGfk7gfFisjAEZHJq9IJ0P2myYp+lKbybolm0fbTZ4jda7DUnvN3n4I0EqFoUa/vPN/DSkdt0hKj1Ayz5AdFIAvOtkphMPA=,iv:dHhMBT3T3bWPSUadDgo+h2KSf2qC32q+nM26eK9ivDo=,tag:30dNpRvB6vet5UoWy/DZtg==,type:str]
lastmodified: "2024-01-30T17:00:41Z"
mac: ENC[AES256_GCM,data:GvVUAo5Qtp0Dcnffh42jGkpT3khDRYXf6ws6Q3n2dWk+q39+xDQ3oxCGKMsYwbxrQ5s1oSI7dENSfxQA1Rwk3+Z0wmrRry9fxlYEnDeYLiR2Jxp0+7zDWUcusfSnjC/ASmwCYSFBcQM4jhD4uyVmhluS0E5KrjOD223Z6vtjxck=,iv:5OFMGJatztWTUR2Xb49CYl0Z42UsieYPTR6YoBn9UmM=,tag:yibooLYgoAN+IeiHVyexKg==,type:str]
pgp: []
unencrypted_suffix: _unencrypted
version: 3.8.1