From 641feb1b847956faa4f012571cae53c055cbb612 Mon Sep 17 00:00:00 2001 From: goeranh Date: Fri, 20 Mar 2026 16:23:34 +0100 Subject: [PATCH 01/14] monitoring host# --- hosts/monitoring/default.nix | 205 +++++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 hosts/monitoring/default.nix diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix new file mode 100644 index 0000000..e92c246 --- /dev/null +++ b/hosts/monitoring/default.nix @@ -0,0 +1,205 @@ +{ + config, + lib, + pkgs, + modulesPath, + ... +}: +{ + imports = [ + "${modulesPath}/virtualisation/proxmox-lxc.nix" + ]; + + networking = { + hostName = "monitoring"; + fqdn = "monitoring.adm.htw.stura-dresden.de"; + interfaces.eth0.ipv4.addresses = [ + { + address = "141.56.51.20"; + prefixLength = 24; + } + ]; + defaultGateway = { + address = "141.56.51.254"; + interface = "eth0"; + }; + firewall = { + enable = true; + allowedTCPPorts = [ + 80 + 443 + ]; + }; + }; + + # Loki - Log aggregation system + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server = { + http_listen_port = 3100; + grpc_listen_port = 9096; + }; + common = { + path_prefix = "/var/lib/loki"; + storage.filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + replication_factor = 1; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + }; + schema_config = { + configs = [ + { + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + }; + }; + + # Mimir - Scalable metrics storage + services.mimir = { + enable = true; + configuration = { + multitenancy_enabled = false; + blocks_storage = { + backend = "filesystem"; + filesystem = { + dir = "/var/lib/mimir/data"; + }; + }; + compactor = { + data_dir = "/var/lib/mimir/compactor"; + }; + distributor = { + ring = { + kvstore = { + store = "memberlist"; + }; + }; + }; + ingester = { + ring = { + kvstore = { + store = "memberlist"; + }; + replication_factor = 1; + }; + }; + ruler_storage = { + backend = "filesystem"; + filesystem = { + dir = "/var/lib/mimir/rules"; + }; + }; + server = { + http_listen_port = 9009; + grpc_listen_port = 9095; + }; + store_gateway = { + sharding_ring = { + replication_factor = 1; + }; + }; + }; + }; + + # Grafana - Visualization and dashboarding + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3000; + domain = "monitoring.adm.htw.stura-dresden.de"; + root_url = "https://monitoring.adm.htw.stura-dresden.de"; + }; + security = { + admin_user = "admin"; + admin_password = "$__file{/var/lib/grafana/admin_password}"; + }; + }; + provision = { + enable = true; + datasources.settings.datasources = [ + { + name = "Mimir"; + type = "prometheus"; + url = "http://localhost:9009/prometheus"; + isDefault = true; + } + { + name = "Loki"; + type = "loki"; + url = "http://localhost:3100"; + } + ]; + }; + }; + + # Nginx reverse proxy with ACME certificates + services.nginx = { + enable = true; + + virtualHosts."logs.adm.htw.stura-dresden.de" = { + forceSSL = true; + enableACME = true; + locations."/" = { + proxyPass = "http://127.0.0.1:3100"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + + virtualHosts."metrics.adm.htw.stura-dresden.de" = { + forceSSL = true; + enableACME = true; + locations."/" = { + proxyPass = "http://127.0.0.1:9009"; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + + virtualHosts."monitoring.adm.htw.stura-dresden.de" = { + forceSSL = true; + enableACME = true; + locations."/" = { + proxyPass = "http://127.0.0.1:3000"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + }; + }; + + services.openssh.enable = true; + + system.stateVersion = "25.11"; +} From 9beef4e01368ea73ff794a292df5c6efa959f1f9 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 08:33:08 +0200 Subject: [PATCH 02/14] use recommended proxy settings --- hosts/monitoring/default.nix | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index e92c246..2dbfbf1 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -160,12 +160,7 @@ locations."/" = { proxyPass = "http://127.0.0.1:3100"; proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; + recommendedProxySettings = true; }; }; @@ -174,12 +169,7 @@ enableACME = true; locations."/" = { proxyPass = "http://127.0.0.1:9009"; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; + recommendedProxySettings = true; }; }; @@ -189,12 +179,7 @@ locations."/" = { proxyPass = "http://127.0.0.1:3000"; proxyWebsockets = true; - extraConfig = '' - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - ''; + recommendedProxySettings = true; }; }; }; From d435d8d48780a84629f2babf016b283eac77392a Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 11:52:26 +0200 Subject: [PATCH 03/14] use three letter domains --- hosts/monitoring/default.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index 2dbfbf1..a56cdb4 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -124,8 +124,8 @@ server = { http_addr = "127.0.0.1"; http_port = 3000; - domain = "monitoring.adm.htw.stura-dresden.de"; - root_url = "https://monitoring.adm.htw.stura-dresden.de"; + domain = "mon.adm.htw.stura-dresden.de"; + root_url = "https://mon.adm.htw.stura-dresden.de"; }; security = { admin_user = "admin"; @@ -154,7 +154,7 @@ services.nginx = { enable = true; - virtualHosts."logs.adm.htw.stura-dresden.de" = { + virtualHosts."log.adm.htw.stura-dresden.de" = { forceSSL = true; enableACME = true; locations."/" = { @@ -164,7 +164,7 @@ }; }; - virtualHosts."metrics.adm.htw.stura-dresden.de" = { + virtualHosts."met.adm.htw.stura-dresden.de" = { forceSSL = true; enableACME = true; locations."/" = { @@ -173,7 +173,7 @@ }; }; - virtualHosts."monitoring.adm.htw.stura-dresden.de" = { + virtualHosts."mon.adm.htw.stura-dresden.de" = { forceSSL = true; enableACME = true; locations."/" = { From 5ef710f8f270d70d388cbe38029ae5bb90919a27 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 11:52:40 +0200 Subject: [PATCH 04/14] dont generate domains, doesnt work for mon.adm.htw for example --- hosts/proxy/default.nix | 74 ++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/hosts/proxy/default.nix b/hosts/proxy/default.nix index a806790..9c09af9 100644 --- a/hosts/proxy/default.nix +++ b/hosts/proxy/default.nix @@ -295,43 +295,43 @@ "127.0.0.1" ]; listenOnIpv6 = [ ]; - zones = { - "htw.stura-dresden.de" = { - master = true; - file = pkgs.writeText "htw.stura-dresden.de.zone" '' - $TTL 3600 - @ IN SOA proxy.htw.stura-dresden.de. hostmaster.htw.stura-dresden.de. ( - 2026031301 ; Serial (YYYYMMDDNN) - 3600 ; Refresh (1 hour) - 1800 ; Retry (30 minutes) - 604800 ; Expire (1 week) - 86400 ) ; Minimum TTL (1 day) - - ; Name servers - @ IN NS proxy.htw.stura-dresden.de. - - ; Proxy host - main IPv4 gateway - proxy IN A 141.56.51.1 - proxy IN AAAA 2a01:4f8:1c19:96f8::1 - - ; Auto-generated CNAME records for all subdomains pointing to proxy - ${lib.foldlAttrs ( - prev: name: value: - let - zoneSuffix = ".htw.stura-dresden.de"; - # Check if this domain belongs to our zone - isInZone = lib.hasSuffix zoneSuffix value.domain; - # Extract subdomain by removing the zone suffix - subdomain = lib.removeSuffix zoneSuffix value.domain; - in - if isInZone && subdomain != "" && subdomain != "htw.stura-dresden.de" then - prev + "${subdomain}${" "}IN${" "}CNAME${" "}proxy.htw.stura-dresden.de.\n" - else - prev - ) "" forwards} - ''; - }; - }; + # zones = { + # "htw.stura-dresden.de" = { + # master = true; + # file = pkgs.writeText "htw.stura-dresden.de.zone" '' + # $TTL 3600 + # @ IN SOA proxy.htw.stura-dresden.de. hostmaster.htw.stura-dresden.de. ( + # 2026031301 ; Serial (YYYYMMDDNN) + # 3600 ; Refresh (1 hour) + # 1800 ; Retry (30 minutes) + # 604800 ; Expire (1 week) + # 86400 ) ; Minimum TTL (1 day) + # + # ; Name servers + # @ IN NS proxy.htw.stura-dresden.de. + # + # ; Proxy host - main IPv4 gateway + # proxy IN A 141.56.51.1 + # proxy IN AAAA 2a01:4f8:1c19:96f8::1 + # + # ; Auto-generated CNAME records for all subdomains pointing to proxy + # ${lib.foldlAttrs ( + # prev: name: value: + # let + # zoneSuffix = ".htw.stura-dresden.de"; + # # Check if this domain belongs to our zone + # isInZone = lib.hasSuffix zoneSuffix value.domain; + # # Extract subdomain by removing the zone suffix + # subdomain = lib.removeSuffix zoneSuffix value.domain; + # in + # if isInZone && subdomain != "" && subdomain != "htw.stura-dresden.de" then + # prev + "${subdomain}${" "}IN${" "}CNAME${" "}proxy.htw.stura-dresden.de.\n" + # else + # prev + # ) "" forwards} + # ''; + # }; + # }; }; # Chrony NTP server for the internal network From ef8607e38e0d203e791ac139f0224e424e499004 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 11:52:57 +0200 Subject: [PATCH 05/14] include monitoring module per default --- flake.nix | 1 + modules/monitoring.nix | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 modules/monitoring.nix diff --git a/flake.nix b/flake.nix index 2d1426d..f0f177e 100644 --- a/flake.nix +++ b/flake.nix @@ -193,6 +193,7 @@ [ ./hosts/${input} ./default.nix + ./modules/monitoring.nix disko.nixosModules.disko authentik.nixosModules.default mailserver.nixosModules.mailserver diff --git a/modules/monitoring.nix b/modules/monitoring.nix new file mode 100644 index 0000000..09a1bf7 --- /dev/null +++ b/modules/monitoring.nix @@ -0,0 +1,96 @@ +{ pkgs, lib, config, ... }: +let + cfg = config.stura.monitoring; +in { + options.stura.monitoring = { + extraLogFiles = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Additional log file paths for vector to scrape and forward to Loki."; + example = [ "/var/log/nginx/access.log" "/var/log/nginx/error.log" ]; + }; + + extraGroups = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Supplementary groups added to the vector systemd service to allow reading protected log files."; + example = [ "nginx" "postfix" ]; + }; + }; + + config = { + networking.hosts = { + "141.56.51.20" = [ + "mon.adm.htw.stura-dresden.de" + "log.adm.htw.stura-dresden.de" + "met.adm.htw.stura-dresden.de" + ]; + }; + services.vector = { + enable = true; + settings = { + sources = { + host_metrics = { + type = "host_metrics"; + collectors = [ "cpu" "disk" "filesystem" "load" "memory" "network" ]; + }; + + journald_logs = { + type = "journald"; + include_units = []; # empty = collect all units + }; + } // lib.optionalAttrs (cfg.extraLogFiles != [] || config.services.nginx.enable) { + extra_log_files = { + type = "file"; + include = lib.optional config.services.nginx.enable "/var/log/nginx/access.log" + ++ cfg.extraLogFiles; + }; + }; + + transforms = { + add_host_label_metrics = { + type = "remap"; + inputs = [ "host_metrics" ]; + source = '' + .tags.host = get_hostname!() + ''; + }; + + add_host_label_logs = { + type = "remap"; + inputs = [ "journald_logs" ] ++ lib.optional (cfg.extraLogFiles != [] || config.services.nginx.enable) "extra_log_files"; + source = '' + .host = get_hostname!() + ''; + }; + }; + + sinks = { + mimir = { + type = "prometheus_remote_write"; + inputs = [ "add_host_label_metrics" ]; + endpoint = "https://metrics.adm.htw.stura-dresden.de/api/v1/push"; + tls.verify_certificate = false; + }; + + loki = { + type = "loki"; + inputs = [ "add_host_label_logs" ]; + endpoint = "https://log.adm.htw.stura-dresden.de"; + labels = { + host = "{{ host }}"; + unit = "{{ _SYSTEMD_UNIT }}"; + }; + tls.verify_certificate = false; + encoding.codec = "json"; + }; + }; + }; + }; + + systemd.services.vector.serviceConfig.SupplementaryGroups = + [ "systemd-journal" ] + ++ lib.optional config.services.nginx.enable "nginx" + ++ cfg.extraGroups; + }; +} From e9fe620fa9ca3746c9de9aef51987fedac8e29d1 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 12:01:36 +0200 Subject: [PATCH 06/14] increase mimir and loki rate limits --- hosts/monitoring/default.nix | 11 +++++++++++ modules/monitoring.nix | 1 + 2 files changed, 12 insertions(+) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index a56cdb4..12f214c 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -53,6 +53,12 @@ kvstore.store = "inmemory"; }; }; + limits_config = { + ingestion_rate_mb = 32; + ingestion_burst_size_mb = 64; + per_stream_rate_limit = "32MB"; + per_stream_rate_limit_burst = "64MB"; + }; schema_config = { configs = [ { @@ -109,6 +115,11 @@ http_listen_port = 9009; grpc_listen_port = 9095; }; + limits = { + ingestion_rate = 100000; + ingestion_burst_size = 200000; + max_global_series_per_user = 0; + }; store_gateway = { sharding_ring = { replication_factor = 1; diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 09a1bf7..33834ae 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -20,6 +20,7 @@ in { config = { networking.hosts = { + # rewrite these host entries on each system, this does not go through proxy "141.56.51.20" = [ "mon.adm.htw.stura-dresden.de" "log.adm.htw.stura-dresden.de" From e7cba90a4594bcf014652330b3ee6bc0ff992dfd Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 12:10:18 +0200 Subject: [PATCH 07/14] fix vector settings --- modules/monitoring.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 33834ae..86f7573 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -62,6 +62,7 @@ in { inputs = [ "journald_logs" ] ++ lib.optional (cfg.extraLogFiles != [] || config.services.nginx.enable) "extra_log_files"; source = '' .host = get_hostname!() + .unit = string(."_SYSTEMD_UNIT") ?? "file" ''; }; }; @@ -70,7 +71,7 @@ in { mimir = { type = "prometheus_remote_write"; inputs = [ "add_host_label_metrics" ]; - endpoint = "https://metrics.adm.htw.stura-dresden.de/api/v1/push"; + endpoint = "https://met.adm.htw.stura-dresden.de/api/v1/push"; tls.verify_certificate = false; }; @@ -80,7 +81,7 @@ in { endpoint = "https://log.adm.htw.stura-dresden.de"; labels = { host = "{{ host }}"; - unit = "{{ _SYSTEMD_UNIT }}"; + unit = "{{ unit }}"; }; tls.verify_certificate = false; encoding.codec = "json"; From fd02a136efb40ecba78ef564644f0c2345dc6d87 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 13:06:04 +0200 Subject: [PATCH 08/14] scrape haproxy metrics --- hosts/proxy/default.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/hosts/proxy/default.nix b/hosts/proxy/default.nix index 9c09af9..83306c5 100644 --- a/hosts/proxy/default.nix +++ b/hosts/proxy/default.nix @@ -448,6 +448,7 @@ stats show-node stats show-modules stats admin if TRUE # Enable admin operations + http-request use-service prometheus-exporter if { path /metrics } frontend http-in bind *:80 @@ -547,8 +548,27 @@ ) "" forwards} ''; }; + vector.settings = { + sources.haproxy_metrics = { + type = "prometheus_scrape"; + endpoints = [ "http://127.0.0.1:8404/metrics" ]; + scrape_interval_secs = 15; + }; + transforms.add_host_label_haproxy = { + type = "remap"; + inputs = [ "haproxy_metrics" ]; + source = '' + .tags.host = get_hostname!() + ''; + }; + sinks.mimir.inputs = lib.mkForce [ + "add_host_label_metrics" + "add_host_label_haproxy" + ]; + }; }; + environment.systemPackages = with pkgs; [ ]; From dd2aa96e251a0fdd8ae0619938b2dd72ac8dd4ff Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 13:07:13 +0200 Subject: [PATCH 09/14] fix mimir --- hosts/monitoring/default.nix | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index 12f214c..bf2b501 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -81,6 +81,12 @@ enable = true; configuration = { multitenancy_enabled = false; + memberlist = { + bind_addr = [ "0.0.0.0" ]; + bind_port = 7946; + advertise_addr = "141.56.51.20"; + join_members = [ "141.56.51.20:7946" ]; + }; blocks_storage = { backend = "filesystem"; filesystem = { @@ -92,16 +98,12 @@ }; distributor = { ring = { - kvstore = { - store = "memberlist"; - }; + kvstore.store = "memberlist"; }; }; ingester = { ring = { - kvstore = { - store = "memberlist"; - }; + kvstore.store = "memberlist"; replication_factor = 1; }; }; @@ -123,6 +125,7 @@ store_gateway = { sharding_ring = { replication_factor = 1; + kvstore.store = "memberlist"; }; }; }; From 93e27dd3e5da7a886114c7db60606e9ed98c481f Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 13:07:21 +0200 Subject: [PATCH 10/14] receive proxmox logs --- hosts/monitoring/default.nix | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index bf2b501..2438b1f 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -28,6 +28,8 @@ allowedTCPPorts = [ 80 443 + 4317 # OTLP gRPC (Proxmox) + 4318 # OTLP HTTP (Proxmox) ]; }; }; @@ -198,6 +200,25 @@ }; }; + # Vector - receive OpenTelemetry data from Proxmox and forward to Loki/Mimir + services.vector.settings = { + sources.proxmox_otlp = { + type = "opentelemetry"; + grpc.address = "0.0.0.0:4317"; + http.address = "0.0.0.0:4318"; + }; + + sinks.mimir.inputs = lib.mkForce [ + "add_host_label_metrics" + "proxmox_otlp.metrics" + ]; + + sinks.loki.inputs = lib.mkForce [ + "add_host_label_logs" + "proxmox_otlp.logs" + ]; + }; + services.openssh.enable = true; system.stateVersion = "25.11"; From 53c592abd9e5e52f7d32a7ba3861ede6a4a7b35f Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 13:52:57 +0200 Subject: [PATCH 11/14] add extra inputs option for the loki and mimir sinks --- modules/monitoring.nix | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/modules/monitoring.nix b/modules/monitoring.nix index 86f7573..c46b660 100644 --- a/modules/monitoring.nix +++ b/modules/monitoring.nix @@ -16,6 +16,20 @@ in { description = "Supplementary groups added to the vector systemd service to allow reading protected log files."; example = [ "nginx" "postfix" ]; }; + + extraMetricInputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Additional vector component IDs to feed into the mimir sink alongside host_metrics."; + example = [ "proxmox_normalize_metrics" ]; + }; + + extraLogInputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = "Additional vector component IDs to feed into the loki sink alongside journald_logs."; + example = [ "proxmox_normalize_logs" ]; + }; }; config = { @@ -70,14 +84,15 @@ in { sinks = { mimir = { type = "prometheus_remote_write"; - inputs = [ "add_host_label_metrics" ]; + inputs = [ "add_host_label_metrics" ] ++ cfg.extraMetricInputs; endpoint = "https://met.adm.htw.stura-dresden.de/api/v1/push"; tls.verify_certificate = false; + healthcheck.enabled = false; }; loki = { type = "loki"; - inputs = [ "add_host_label_logs" ]; + inputs = [ "add_host_label_logs" ] ++ cfg.extraLogInputs; endpoint = "https://log.adm.htw.stura-dresden.de"; labels = { host = "{{ host }}"; From e1530c606f70b6946dfade296ff66d088b811d52 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 13:53:12 +0200 Subject: [PATCH 12/14] use extra input sink options --- hosts/monitoring/default.nix | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index 2438b1f..e0dbcaa 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -208,15 +208,28 @@ http.address = "0.0.0.0:4318"; }; - sinks.mimir.inputs = lib.mkForce [ - "add_host_label_metrics" - "proxmox_otlp.metrics" - ]; + transforms.proxmox_normalize_logs = { + type = "remap"; + inputs = [ "proxmox_otlp.logs" ]; + source = '' + .host = string(.resources."host.name") ?? "proxmox" + .unit = string(.attributes."service.name") ?? "proxmox" + ''; + }; - sinks.loki.inputs = lib.mkForce [ - "add_host_label_logs" - "proxmox_otlp.logs" - ]; + transforms.proxmox_normalize_metrics = { + type = "remap"; + inputs = [ "proxmox_otlp.metrics" ]; + source = '' + .tags.host = string(.resources."host.name") ?? "proxmox" + ''; + }; + + }; + + stura.monitoring = { + extraMetricInputs = [ "proxmox_normalize_metrics" ]; + extraLogInputs = [ "proxmox_normalize_logs" ]; }; services.openssh.enable = true; From 9a5716f2df8fcaf4ab184cca21c4ab5692f61b8d Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 14:10:33 +0200 Subject: [PATCH 13/14] increase max series for queries --- hosts/monitoring/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index e0dbcaa..f3c6829 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -60,6 +60,7 @@ ingestion_burst_size_mb = 64; per_stream_rate_limit = "32MB"; per_stream_rate_limit_burst = "64MB"; + max_query_series = 100000; }; schema_config = { configs = [ From c31063ca5632a0f3185fdbcbf185dacd49fd33c0 Mon Sep 17 00:00:00 2001 From: goeranh Date: Mon, 20 Apr 2026 14:28:13 +0200 Subject: [PATCH 14/14] purge logs and metrics after 4 weeks --- hosts/monitoring/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix index f3c6829..e30daba 100644 --- a/hosts/monitoring/default.nix +++ b/hosts/monitoring/default.nix @@ -61,6 +61,12 @@ per_stream_rate_limit = "32MB"; per_stream_rate_limit_burst = "64MB"; max_query_series = 100000; + retention_period = "672h"; # 28 days + }; + compactor = { + retention_enabled = true; + working_directory = "/var/lib/loki/compactor"; + delete_request_store = "filesystem"; }; schema_config = { configs = [ @@ -124,6 +130,7 @@ ingestion_rate = 100000; ingestion_burst_size = 200000; max_global_series_per_user = 0; + compactor_blocks_retention_period = "672h"; # 28 days }; store_gateway = { sharding_ring = {