stura-infra/hosts/monitoring/default.nix

{
  config,
  lib,
  pkgs,
  modulesPath,
  ...
}:
{
  imports = [
    "${modulesPath}/virtualisation/proxmox-lxc.nix"
  ];

  networking = {
    hostName = "monitoring";
    fqdn = "monitoring.adm.htw.stura-dresden.de";
    interfaces.eth0.ipv4.addresses = [
      {
        address = "141.56.51.20";
        prefixLength = 24;
      }
    ];
    defaultGateway = {
      address = "141.56.51.254";
      interface = "eth0";
    };
    firewall = {
      enable = true;
      allowedTCPPorts = [
        80
        443
        4317 # OTLP gRPC (Proxmox)
        4318 # OTLP HTTP (Proxmox)
      ];
    };
  };

  # Loki - Log aggregation system
  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
        grpc_listen_port = 9096;
      };
      common = {
        path_prefix = "/var/lib/loki";
        storage.filesystem = {
          chunks_directory = "/var/lib/loki/chunks";
          rules_directory = "/var/lib/loki/rules";
        };
        replication_factor = 1;
        ring = {
          instance_addr = "127.0.0.1";
          kvstore.store = "inmemory";
        };
      };
      limits_config = {
        ingestion_rate_mb = 32;
        ingestion_burst_size_mb = 64;
        per_stream_rate_limit = "32MB";
        per_stream_rate_limit_burst = "64MB";
      };
      schema_config = {
        configs = [
          {
            from = "2024-01-01";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };
    };
  };

  # Mimir - Scalable metrics storage
  services.mimir = {
    enable = true;
    configuration = {
      multitenancy_enabled = false;
      memberlist = {
        bind_addr = [ "0.0.0.0" ];
        bind_port = 7946;
        advertise_addr = "141.56.51.20";
        join_members = [ "141.56.51.20:7946" ];
      };
      blocks_storage = {
        backend = "filesystem";
        filesystem = {
          dir = "/var/lib/mimir/data";
        };
      };
      compactor = {
        data_dir = "/var/lib/mimir/compactor";
      };
      distributor = {
        ring = {
          kvstore.store = "memberlist";
        };
      };
      ingester = {
        ring = {
          kvstore.store = "memberlist";
          replication_factor = 1;
        };
      };
      ruler_storage = {
        backend = "filesystem";
        filesystem = {
          dir = "/var/lib/mimir/rules";
        };
      };
      server = {
        http_listen_port = 9009;
        grpc_listen_port = 9095;
      };
      limits = {
        ingestion_rate = 100000;
        ingestion_burst_size = 200000;
        max_global_series_per_user = 0;
      };
      store_gateway = {
        sharding_ring = {
          replication_factor = 1;
          kvstore.store = "memberlist";
        };
      };
    };
  };

  # Grafana - Visualization and dashboarding
  services.grafana = {
    enable = true;
    settings = {
      server = {
        http_addr = "127.0.0.1";
        http_port = 3000;
        domain = "mon.adm.htw.stura-dresden.de";
        root_url = "https://mon.adm.htw.stura-dresden.de";
      };
      security = {
        admin_user = "admin";
        admin_password = "$__file{/var/lib/grafana/admin_password}";
      };
    };
    provision = {
      enable = true;
      datasources.settings.datasources = [
        {
          name = "Mimir";
          type = "prometheus";
          url = "http://localhost:9009/prometheus";
          isDefault = true;
        }
        {
          name = "Loki";
          type = "loki";
          url = "http://localhost:3100";
        }
      ];
    };
  };

  # Nginx reverse proxy with ACME certificates
  services.nginx = {
    enable = true;

    virtualHosts."log.adm.htw.stura-dresden.de" = {
      forceSSL = true;
      enableACME = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:3100";
        proxyWebsockets = true;
        recommendedProxySettings = true;
      };
    };

    virtualHosts."met.adm.htw.stura-dresden.de" = {
      forceSSL = true;
      enableACME = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:9009";
        recommendedProxySettings = true;
      };
    };

    virtualHosts."mon.adm.htw.stura-dresden.de" = {
      forceSSL = true;
      enableACME = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:3000";
        proxyWebsockets = true;
        recommendedProxySettings = true;
      };
    };
  };

  # Vector - receive OpenTelemetry data from Proxmox and forward to Loki/Mimir
  services.vector.settings = {
    sources.proxmox_otlp = {
      type = "opentelemetry";
      grpc.address = "0.0.0.0:4317";
      http.address = "0.0.0.0:4318";
    };

    sinks.mimir.inputs = lib.mkForce [
      "add_host_label_metrics"
      "proxmox_otlp.metrics"
    ];

    sinks.loki.inputs = lib.mkForce [
      "add_host_label_logs"
      "proxmox_otlp.logs"
    ];
  };

  services.openssh.enable = true;

  system.stateVersion = "25.11";
}