Replace telegraf with node exporter

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
2025-04-19 17:57:34 +03:00
parent 3f1531fbd1
commit 1a445ab6fd
37 changed files with 3099 additions and 421 deletions

View File

@@ -74,7 +74,7 @@ in
jwks = [ { key = hmConfig.sops.placeholder."authelia/oidcKey"; } ];
authorization_policies = {
admin = {
admin_two_factor = {
default_policy = "deny";
rules = [
{
@@ -83,6 +83,16 @@ in
}
];
};
admin_one_factor = {
default_policy = "deny";
rules = [
{
policy = "one_factor";
subject = [ "group:admins" ];
}
];
};
};
};
@@ -105,6 +115,8 @@ in
};
theme = "auto";
telemetry.metrics.enabled = true;
}
);
@@ -125,13 +137,13 @@ in
networks.authelia.networkConfig.internal = true;
volumes = {
"authelia-redis" = { };
"authelia-postgresql" = { };
authelia-redis = { };
authelia-postgresql = { };
authelia = { };
};
containers = {
"authelia-init" = {
authelia-init = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-yq}";
volumes = [
@@ -140,7 +152,7 @@ in
];
exec = [
"eval-all"
". as $item ireduce ({}; . * $item)"
". as $item ireduce ({}; . *+ $item)"
"/etc/authelia/users.yaml"
"/etc/authelia/users.yaml.default"
"-i"
@@ -167,6 +179,7 @@ in
networks = [
networks.authelia.ref
networks.traefik.ref
networks.prometheus.ref
];
exec = [
"--config"
@@ -183,18 +196,18 @@ in
};
unitConfig.After = [
"${containers."authelia-init"._serviceName}.service"
"${containers."authelia-postgresql"._serviceName}.service"
"${containers."authelia-redis"._serviceName}.service"
"${containers.authelia-init._serviceName}.service"
"${containers.authelia-postgresql._serviceName}.service"
"${containers.authelia-redis._serviceName}.service"
"sops-nix.service"
];
};
"authelia-postgresql" = {
authelia-postgresql = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-postgresql}";
networks = [ networks.authelia.ref ];
volumes = [ "${volumes."authelia-postgresql".ref}:/var/lib/postgresql/data" ];
volumes = [ "${volumes.authelia-postgresql.ref}:/var/lib/postgresql/data" ];
environments = {
POSTGRES_DB = "authelia";
POSTGRES_USER = "authelia";
@@ -205,12 +218,37 @@ in
unitConfig.After = [ "sops-nix.service" ];
};
"authelia-redis".containerConfig = {
authelia-redis.containerConfig = {
image = "docker-archive:${selfPkgs.docker-redis}";
networks = [ networks.authelia.ref ];
volumes = [ "${volumes."authelia-redis".ref}:/var/lib/redis" ];
volumes = [ "${volumes.authelia-redis.ref}:/var/lib/redis" ];
exec = [ "--save 60 1" ];
};
prometheus-init.containerConfig.volumes =
let
autheliaConfig = (pkgs.formats.yaml { }).generate "authelia.yml" {
scrape_configs =
let
hostname = config.networking.hostName;
in
[
{
job_name = "${hostname}-authelia";
static_configs = [
{
targets = [ "authelia:9959" ];
labels = {
app = "authelia";
inherit user hostname;
};
}
];
}
];
};
in
[ "${autheliaConfig}:/etc/prometheus/conf.d/authelia.yml" ];
};
};

View File

@@ -11,6 +11,7 @@ in
(import ./authelia { inherit user home; })
(import ./grafana { inherit user home; })
(import ./ntfy { inherit user home; })
(import ./prometheus { inherit user home; })
(import ./traefik { inherit user home; })
(import ./whoami { inherit user home; })
];

File diff suppressed because it is too large Load Diff

View File

@@ -13,7 +13,7 @@
let
selfPkgs = inputs.self.packages.${system};
hmConfig = config.home-manager.users.${user};
inherit (hmConfig.virtualisation.quadlet) volumes containers networks;
inherit (hmConfig.virtualisation.quadlet) networks;
autheliaClientId = "4R5ofTZgOjO5Nrbcm9f6KqBLZXy8LwPS5s3E3BUfPS2mRy0wSV41XZGLrLgiR4Z0MblyGzW211AHL7GCCaJu5KonLUKyRjoyuiAr";
in
{
@@ -34,7 +34,7 @@ in
client_name = "Grafana";
client_secret = hmConfig.sops.placeholder."grafana/authelia/digest";
redirect_uris = [ "https://stats.karaolidis.com/login/generic_oauth" ];
authorization_policy = "admin";
authorization_policy = "admin_one_factor";
require_pkce = true;
pkce_challenge_method = "S256";
}
@@ -122,87 +122,9 @@ in
};
virtualisation.quadlet = {
networks = {
grafana.networkConfig.internal = true;
# Allow access to host telegraf via non-internal network
grafana-prometheus = { };
};
volumes = {
"grafana-prometheus-data" = { };
"grafana-prometheus-config" = { };
};
networks.grafana.networkConfig.internal = true;
containers = {
"grafana-prometheus-init" =
let
prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
global = {
scrape_interval = "10s";
evaluation_interval = "10s";
};
scrape_configs = [
{
job_name = "telegraf";
static_configs = [
{
targets = [ "host.containers.internal:9273" ];
labels.app = "telegraf";
}
{
targets = [
"host.containers.internal:${builtins.toString (9273 + config.users.users.${user}.uid)}"
];
labels.app = "telegraf-storm";
}
];
}
];
};
in
{
containerConfig = {
image = "docker-archive:${selfPkgs.docker-yq}";
volumes = [
"${volumes."grafana-prometheus-config".ref}:/etc/prometheus"
"${prometheusConfig}:/etc/prometheus/conf.d/prometheus.yml"
];
entrypoint = "/bin/bash";
exec = [
"-c"
"yq eval-all '. as $item ireduce ({}; . * $item)' /etc/prometheus/conf.d/*.yml > /etc/prometheus/prometheus.yml"
];
};
serviceConfig = {
Type = "oneshot";
Restart = "on-failure";
};
};
"grafana-prometheus" = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus}";
volumes = [
"${volumes."grafana-prometheus-config".ref}:/etc/prometheus"
"${volumes."grafana-prometheus-data".ref}:/var/lib/prometheus"
];
networks = [
networks.grafana.ref
networks.grafana-prometheus.ref
];
exec = [
"--config.file=/etc/prometheus/prometheus.yml"
"--storage.tsdb.path=/var/lib/prometheus"
"--storage.tsdb.retention.time=1y"
"--log.level=warn"
];
};
unitConfig.After = [ "${containers."grafana-prometheus-init"._serviceName}.service" ];
};
grafana = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-grafana}";
@@ -217,17 +139,12 @@ in
];
};
unitConfig.After = [
"${containers."grafana-prometheus"._serviceName}.service"
"${containers."grafana-image-renderer"._serviceName}.service"
];
unitConfig.After = [ "sops-nix.service" ];
};
"grafana-image-renderer" = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-grafana-image-renderer}";
networks = [ networks.grafana.ref ];
};
grafana-image-renderer.containerConfig = {
image = "docker-archive:${selfPkgs.docker-grafana-image-renderer}";
networks = [ networks.grafana.ref ];
};
authelia.containerConfig.volumes = [

View File

@@ -71,6 +71,8 @@ in
enable-signup = false;
enable-login = true;
enable-reservations = false;
metrics-listen-http = ":9090";
}
);
@@ -111,26 +113,54 @@ in
volumes.ntfy = { };
containers.ntfy = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-ntfy}";
networks = [
networks.ntfy.ref
networks.traefik.ref
];
volumes = [
"${volumes.ntfy.ref}:/var/lib/ntfy"
"${hmConfig.sops.templates."ntfy-server.yml".path}:/etc/ntfy/server.yml:ro"
"${hmConfig.sops.templates."ntfy-init.sh".path}:/entrypoint.sh:ro"
];
entrypoint = "/entrypoint.sh";
labels = [
"traefik.enable=true"
"traefik.http.routers.ntfy.rule=Host(`ntfy.karaolidis.com`)"
];
containers = {
ntfy = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-ntfy}";
networks = [
networks.ntfy.ref
networks.traefik.ref
networks.prometheus.ref
];
volumes = [
"${volumes.ntfy.ref}:/var/lib/ntfy"
"${hmConfig.sops.templates."ntfy-server.yml".path}:/etc/ntfy/server.yml:ro"
"${hmConfig.sops.templates."ntfy-init.sh".path}:/entrypoint.sh:ro"
];
entrypoint = "/entrypoint.sh";
labels = [
"traefik.enable=true"
"traefik.http.routers.ntfy.rule=Host(`ntfy.karaolidis.com`)"
];
};
unitConfig.After = [ "sops-nix.service" ];
};
unitConfig.After = [ "sops-nix.service" ];
prometheus-init.containerConfig.volumes =
let
ntfyConfig = (pkgs.formats.yaml { }).generate "ntfy.yml" {
scrape_configs =
let
hostname = config.networking.hostName;
in
[
{
job_name = "${hostname}-ntfy";
static_configs = [
{
targets = [ "ntfy:9090" ];
labels = {
app = "ntfy.sh";
inherit user hostname;
};
}
];
}
];
};
in
[ "${ntfyConfig}:/etc/prometheus/conf.d/ntfy.yml" ];
};
};
};

View File

@@ -0,0 +1,310 @@
{
user ? throw "user argument is required",
home ? throw "home argument is required",
}:
{
config,
inputs,
pkgs,
system,
lib,
...
}:
let
selfPkgs = inputs.self.packages.${system};
hmConfig = config.home-manager.users.${user};
jupiterVpsConfig = inputs.self.nixosConfigurations.jupiter-vps.config;
inherit (hmConfig.virtualisation.quadlet) volumes containers networks;
in
{
boot.kernelParams = [ "psi=1" ];
# The below containers all need to run as root to collect host metrics.
virtualisation.quadlet.containers = {
prometheus-node-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-node-exporter}";
# Allow collecting host metrics, port :9100 by default
networks = [ "host" ];
podmanArgs = [
"--pid"
"host"
];
volumes = [
"/:/host:ro,rslave"
"/run/udev:/run/udev:ro"
"/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket:ro"
"/etc/static/os-release:/host/etc/os-release:ro"
];
exec = [
"--log.level=warn"
"--path.rootfs=/host"
"--no-collector.arp"
"--no-collector.bonding"
"--no-collector.edac"
"--no-collector.fibrechannel"
"--no-collector.infiniband"
"--no-collector.ipvs"
"--no-collector.mdadm"
"--no-collector.nfs"
"--no-collector.nfsd"
"--no-collector.selinux"
"--no-collector.xfs"
"--no-collector.zfs"
"--collector.cpu_vulnerabilities"
"--collector.drm"
"--collector.ethtool"
"--collector.processes"
"--collector.systemd"
];
};
prometheus-podman-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-podman-exporter}";
publishPorts = [ "9882:9882" ];
volumes = [ "/run/podman/podman.sock:/run/podman/podman.sock:ro" ];
exec = [ "--collector.enable-all" ];
};
prometheus-fail2ban-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-fail2ban-exporter}";
publishPorts = [ "9191:9191" ];
volumes = [ "/run/fail2ban/fail2ban.sock:/var/run/fail2ban/fail2ban.sock:ro" ];
};
prometheus-smartctl-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-smartctl-exporter}";
publishPorts = [ "9633:9633" ];
podmanArgs = [ "--privileged" ];
};
};
home-manager.users.${user} = {
virtualisation.quadlet = {
networks = {
prometheus.networkConfig.internal = true;
prometheus-ext = { };
};
volumes = {
prometheus-data = { };
prometheus-config = { };
};
containers = {
prometheus-node-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-node-exporter}";
networks = [ networks.prometheus.ref ];
volumes =
let
uid = builtins.toString config.users.users.${user}.uid;
in
[ "/run/user/${uid}/bus:/var/run/dbus/system_bus_socket:ro" ];
exec = [
"--log.level=warn"
"--path.rootfs=/host"
"--collector.disable-defaults"
"--collector.systemd"
];
};
prometheus-podman-exporter.containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus-podman-exporter}";
networks = [ networks.prometheus.ref ];
volumes =
let
uid = builtins.toString config.users.users.${user}.uid;
in
[ "/run/user/${uid}/podman/podman.sock:/run/podman/podman.sock:ro" ];
exec = [ "--collector.enable-all" ];
};
prometheus-init =
let
prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
global.scrape_interval = "15s";
scrape_configs =
let
hostname = config.networking.hostName;
jupiterVpsHostname = jupiterVpsConfig.networking.hostName;
in
[
{
job_name = "${hostname}-node-exporter";
static_configs = [
{
targets = [ "host.containers.internal:9100" ];
labels = {
app = "node-exporter";
user = "root";
inherit hostname;
};
}
{
targets = [ "prometheus-node-exporter:9100" ];
labels = {
app = "node-exporter";
inherit user hostname;
};
}
];
}
{
job_name = "${hostname}-podman-exporter";
static_configs = [
{
targets = [ "host.containers.internal:9882" ];
labels = {
app = "podman-exporter";
user = "root";
inherit hostname;
};
}
{
targets = [ "prometheus-podman-exporter:9882" ];
labels = {
app = "podman-exporter";
inherit user hostname;
};
}
];
}
{
job_name = "${hostname}-fail2ban-exporter";
static_configs = [
{
targets = [ "host.containers.internal:9191" ];
labels = {
app = "fail2ban-exporter";
user = "root";
inherit hostname;
};
}
];
}
{
job_name = "${hostname}-smartctl-exporter";
static_configs = [
{
targets = [ "host.containers.internal:9633" ];
labels = {
app = "smartctl-exporter";
user = "root";
inherit hostname;
};
}
];
}
{
job_name = "${jupiterVpsHostname}-node-exporter";
static_configs = [
{
targets = [ "10.0.0.1:9100" ];
labels = {
app = "node-exporter";
user = "root";
hostname = jupiterVpsHostname;
};
}
];
}
{
job_name = "${jupiterVpsHostname}-podman-exporter";
static_configs = [
{
targets = [ "10.0.0.1:9882" ];
labels = {
app = "podman-exporter";
user = "root";
hostname = jupiterVpsHostname;
};
}
];
}
{
job_name = "${jupiterVpsHostname}-fail2ban-exporter";
static_configs = [
{
targets = [ "10.0.0.1:9191" ];
labels = {
app = "fail2ban-exporter";
user = "root";
hostname = jupiterVpsHostname;
};
}
];
}
];
};
in
{
containerConfig = {
image = "docker-archive:${selfPkgs.docker-yq}";
volumes = [
"${volumes.prometheus-config.ref}:/etc/prometheus"
"${prometheusConfig}:/etc/prometheus/conf.d/prometheus.yml"
];
entrypoint = "/bin/bash";
exec = [
"-c"
"yq eval-all '. as $item ireduce ({}; . *+ $item)' /etc/prometheus/conf.d/*.yml > /etc/prometheus/prometheus.yml"
];
};
serviceConfig = {
Type = "oneshot";
Restart = "on-failure";
};
};
prometheus = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-prometheus}";
volumes = [
"${volumes.prometheus-config.ref}:/etc/prometheus"
"${volumes.prometheus-data.ref}:/var/lib/prometheus"
];
networks = [
networks.grafana.ref
networks.prometheus.ref
# Access to root exporters
networks.prometheus-ext.ref
];
exec = [
"--log.level=debug"
"--config.file=/etc/prometheus/prometheus.yml"
"--storage.tsdb.path=/var/lib/prometheus"
"--storage.tsdb.retention.time=1y"
];
};
unitConfig.After = [ "${containers.prometheus-init._serviceName}.service" ];
};
grafana.containerConfig.volumes =
let
datasource = (pkgs.formats.yaml { }).generate "prometheus.yaml" {
apiVersion = 1;
datasources = [
{
name = "Prometheus";
type = "prometheus";
access = "proxy";
url = "http://prometheus:9090";
uid = "prometheus";
jsonData = {
httpMethod = "POST";
manageAlerts = true;
prometheusType = "Prometheus";
prometheusVersion = lib.strings.getVersion pkgs.prometheus;
};
}
];
};
in
[ "${datasource}:/etc/grafana/conf/provisioning/datasources/prometheus.yaml" ];
};
};
};
}

View File

@@ -43,7 +43,10 @@ in
traefik = {
containerConfig = {
image = "docker-archive:${selfPkgs.docker-traefik}";
networks = [ networks.traefik.ref ];
networks = [
networks.traefik.ref
networks.prometheus.ref
];
volumes =
let
uid = builtins.toString config.users.users.${user}.uid;
@@ -84,6 +87,8 @@ in
"--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=cloudflare"
"--certificatesresolvers.letsencrypt.acme.email=nick@karaolidis.com"
"--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
"--metrics.prometheus=true"
];
labels = [
"traefik.enable=true"
@@ -135,12 +140,37 @@ in
};
in
[ "${config}:/etc/authelia/conf.d/traefik.yaml:ro" ];
prometheus-init.containerConfig.volumes =
let
traefikConfig = (pkgs.formats.yaml { }).generate "traefik.yml" {
scrape_configs =
let
hostname = config.networking.hostName;
in
[
{
job_name = "${hostname}-traefik";
static_configs = [
{
targets = [ "traefik:8080" ];
labels = {
app = "traefik";
inherit user hostname;
};
}
];
}
];
};
in
[ "${traefikConfig}:/etc/prometheus/conf.d/traefik.yml" ];
};
};
# https://github.com/eriksjolund/podman-traefik-socket-activation
systemd.user.sockets = {
"traefik-http" = {
traefik-http = {
Socket = {
ListenStream = "0.0.0.0:80";
FileDescriptorName = "http";
@@ -152,7 +182,7 @@ in
};
};
"traefik-https" = {
traefik-https = {
Socket = {
ListenStream = "0.0.0.0:443";
FileDescriptorName = "https";