Add grafana

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

hosts/common/configs/system/sshd/default.nix

@@ -1,5 +1,13 @@
 { ... }:
 {
+  nixpkgs.overlays = [
+    (final: prev: {
+      fail2ban = prev.fail2ban.overrideAttrs (oldAttrs: {
+        patches = oldAttrs.patches or [ ] ++ [ ./remove-umask.patch ];
+      });
+    })
+  ];
+
   environment = {
     enableAllTerminfo = true;
     persistence."/persist/state"."/var/lib/fail2ban" = { };
@@ -24,4 +32,12 @@
       };
     };
   };
+
+  systemd.services.fail2ban.serviceConfig = {
+    User = "root";
+    Group = "fail2ban";
+    UMask = "0117";
+  };
+
+  users.groups.fail2ban = { };
 }

hosts/common/configs/system/sshd/remove-umask.patch

@@ -0,0 +1,15 @@
+diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
+index e438c4ca..aeee4075 100644
+--- a/fail2ban/server/server.py
++++ b/fail2ban/server/server.py
+@@ -108,9 +108,7 @@ class Server:
+ 		signal.signal(s, new)
+ 
+ 	def start(self, sock, pidfile, force=False, observer=True, conf={}):
+-		# First set the mask to only allow access to owner
+-		os.umask(0o077)
+-		# Second daemonize before logging etc, because it will close all handles:
++		# Daemonize before logging etc, because it will close all handles:
+ 		if self.__daemon: # pragma: no cover
+ 			logSys.info("Starting in daemon mode")
+ 			ret = self.__createDaemon()

hosts/common/configs/system/telegraf/default.nix

@@ -0,0 +1,117 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  security.polkit.extraConfig = ''
+    polkit.addRule(function(action, subject) {
+      if (
+        subject.user == "telegraf"
+        && action.id.indexOf("org.freedesktop.systemd1.") == 0
+        )
+          { return polkit.Result.YES; }
+    });
+  '';
+
+  services.telegraf = {
+    enable = true;
+
+    extraConfig = {
+      agent.quiet = true;
+
+      outputs.prometheus_client = [ { listen = ":9273"; } ];
+
+      inputs =
+        {
+          cpu = [ { report_active = true; } ];
+
+          disk = [
+            {
+              mount_points = lib.attrsets.mapAttrsToList (_: fs: fs.mountPoint) config.fileSystems;
+            }
+          ];
+
+          diskio = [ { skip_serial_number = false; } ];
+
+          kernel = [ { } ];
+
+          mem = [ { } ];
+
+          processes = [ { } ];
+
+          swap = [ { } ];
+
+          system = [ { } ];
+
+          internal = [ { } ];
+
+          # TODO: Enable
+          # linux_cpu = [ { } ];
+
+          net = [ { ignore_protocol_stats = true; } ];
+
+          # TODO: Enable
+          # sensors = [ { remove_numbers = false; } ];
+
+          smart = [ { } ];
+
+          # TODO: Enable
+          # amd_rocm_smi = [ { } ];
+
+          systemd_units = [ { } ];
+        }
+        // lib.attrsets.optionalAttrs config.virtualisation.podman.enable {
+          docker = [
+            {
+              endpoint = "unix:///var/run/podman/podman.sock";
+              perdevice = false;
+              perdevice_include = [
+                "cpu"
+                "blkio"
+                "network"
+              ];
+            }
+          ];
+        }
+        // lib.attrsets.optionalAttrs config.services.fail2ban.enable {
+          fail2ban = [ { } ];
+        }
+        // lib.attrsets.optionalAttrs (config.networking.wireguard.interfaces != { }) {
+          wireguard = [ { } ];
+        };
+    };
+  };
+
+  systemd.services.telegraf = {
+    path =
+      with pkgs;
+      [
+        dbus
+        smartmontools
+        # TODO: Enable
+        # lm_sensors
+        # rocmPackages.rocm-smi
+      ]
+      ++ lib.lists.optional config.services.fail2ban.enable fail2ban;
+
+    environment = {
+      DBUS_SYSTEM_BUS_ADDRESS = "unix:path=/var/run/dbus/system_bus_socket";
+    };
+
+    serviceConfig = {
+      AmbientCapabilities = [
+        "CAP_NET_RAW"
+        "CAP_SYS_RAWIO"
+      ] ++ lib.lists.optional (config.networking.wireguard.interfaces != { }) "CAP_NET_ADMIN";
+
+      SupplementaryGroups =
+        [
+          "disk"
+        ]
+        ++ lib.lists.optional config.virtualisation.podman.enable "podman"
+        ++ lib.lists.optional config.services.fail2ban.enable "fail2ban";
+    };
+  };
+}

hosts/common/configs/user/console/telegraf/default.nix

@@ -0,0 +1,58 @@
+{
+  user ? throw "user argument is required",
+  home ? throw "home argument is required",
+}:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  port = 9273 + config.users.users.${user}.uid;
+  hmConfig = config.home-manager.users.${user};
+in
+{
+  home-manager.users.${user}.systemd.user.services.telegraf =
+    let
+      telegrafConfig = (pkgs.formats.toml { }).generate "config.toml" {
+        agent.quiet = true;
+
+        outputs.prometheus_client = [ { listen = ":${builtins.toString port}"; } ];
+
+        inputs =
+          {
+            systemd_units = [
+              { scope = "user"; }
+            ];
+          }
+          // lib.attrsets.optionalAttrs hmConfig.services.podman.enable {
+            docker = [
+              {
+                endpoint =
+                  let
+                    uid = builtins.toString config.users.users.${user}.uid;
+                  in
+                  "unix:///var/run/user/${uid}/podman/podman.sock";
+                perdevice = false;
+                perdevice_include = [
+                  "cpu"
+                  "blkio"
+                  "network"
+                ];
+              }
+            ];
+          };
+      };
+    in
+    {
+      Unit.Description = "Telegraf Agent";
+
+      Install.WantedBy = [ "default.target" ];
+
+      Service = {
+        ExecStart = "${config.services.telegraf.package}/bin/telegraf -config ${telegrafConfig}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      };
+    };
+}

hosts/jupiter/default.nix

@@ -23,6 +23,7 @@
     ../common/configs/system/sshd
     ../common/configs/system/sudo
     ../common/configs/system/system
+    ../common/configs/system/telegraf
     ../common/configs/system/users
     ../common/configs/system/zsh
 

hosts/jupiter/users/storm/configs/console/podman/authelia/default.nix

@@ -132,7 +132,6 @@ in
         "authelia-init" = {
           containerConfig = {
             image = "docker-archive:${selfPkgs.docker-yq}";
-            networks = [ networks.authelia.ref ];
             volumes = [
               "${home}/.local/share/authelia/config:/etc/authelia"
               "${hmConfig.sops.templates."authelia-users.yaml".path}:/etc/authelia/users.yaml.default:ro"
@@ -167,7 +166,10 @@ in
               networks.authelia.ref
               networks.traefik.ref
             ];
-            exec = [ "--config /etc/authelia/conf.d/" ];
+            exec = [
+              "--config"
+              "/etc/authelia/conf.d/"
+            ];
             labels = [
               "traefik.enable=true"
               "traefik.http.routers.authelia.rule=Host(`id.karaolidis.com`)"

hosts/jupiter/users/storm/configs/console/podman/default.nix

@@ -9,6 +9,7 @@ in
 {
   imports = [
     (import ./authelia { inherit user home; })
+    (import ./grafana { inherit user home; })
     (import ./ntfy { inherit user home; })
     (import ./traefik { inherit user home; })
     (import ./whoami { inherit user home; })

hosts/jupiter/users/storm/configs/console/podman/grafana/default.nix

@@ -0,0 +1,239 @@
+{
+  user ? throw "user argument is required",
+  home ? throw "home argument is required",
+}:
+{
+  config,
+  inputs,
+  pkgs,
+  system,
+  lib,
+  ...
+}:
+let
+  selfPkgs = inputs.self.packages.${system};
+  hmConfig = config.home-manager.users.${user};
+  inherit (hmConfig.virtualisation.quadlet) volumes containers networks;
+  autheliaClientId = "4R5ofTZgOjO5Nrbcm9f6KqBLZXy8LwPS5s3E3BUfPS2mRy0wSV41XZGLrLgiR4Z0MblyGzW211AHL7GCCaJu5KonLUKyRjoyuiAr";
+in
+{
+  home-manager.users.${user} = {
+    sops = {
+      secrets = {
+        "grafana/authelia/password".sopsFile = ../../../../../../secrets/secrets.yaml;
+        "grafana/authelia/digest".sopsFile = ../../../../../../secrets/secrets.yaml;
+        "grafana/smtp".sopsFile = ../../../../../../secrets/secrets.yaml;
+      };
+
+      templates = {
+        "authelia-grafana.yaml".content = builtins.readFile (
+          (pkgs.formats.yaml { }).generate "grafana.yaml" {
+            identity_providers.oidc.clients = [
+              {
+                client_id = autheliaClientId;
+                client_name = "Grafana";
+                client_secret = hmConfig.sops.placeholder."grafana/authelia/digest";
+                redirect_uris = [ "https://stats.karaolidis.com/login/generic_oauth" ];
+                authorization_policy = "admin";
+                require_pkce = true;
+                pkce_challenge_method = "S256";
+              }
+            ];
+          }
+        );
+
+        "grafana.ini".content = builtins.readFile (
+          (pkgs.formats.ini { }).generate "grafana.ini" {
+            server.root_url = "https://stats.karaolidis.com";
+
+            analytics = {
+              reporting_enabled = false;
+              check_for_updates = false;
+              check_for_plugin_updates = false;
+            };
+
+            security.disable_initial_admin_creation = true;
+
+            dashboards = {
+              versions_to_keep = 100;
+              min_refresh_interval = "1s";
+            };
+
+            users.default_theme = "system";
+
+            auth.disable_login_form = true;
+
+            sso_settings.configurable_providers = lib.strings.concatStringsSep " " [ ];
+
+            "auth.generic_oauth" = {
+              name = "Authelia";
+              icon = "signin";
+              enabled = true;
+              auto_login = true;
+              client_id = autheliaClientId;
+              client_secret = hmConfig.sops.placeholder."grafana/authelia/password";
+              auth_url = "https://id.karaolidis.com/api/oidc/authorization";
+              token_url = "https://id.karaolidis.com/api/oidc/token";
+              api_url = "https://id.karaolidis.com/api/oidc/userinfo";
+              use_pkce = true;
+              scopes = lib.strings.concatStringsSep " " [
+                "openid"
+                "profile"
+                "email"
+                "groups"
+              ];
+              login_attribute_path = "preferred_username";
+              name_attribute_path = "name";
+              groups_attribute_path = "groups";
+              allow_assign_grafana_admin = true;
+              role_attribute_strict = true;
+              role_attribute_path = "contains(groups, 'admins') && 'GrafanaAdmin' || 'Viewer'";
+              org_attribute_path = "groups";
+              org_mapping = "admins:1:Admin *:1:Viewer";
+            };
+
+            smtp = {
+              enabled = true;
+              host = "smtp.protonmail.ch:587";
+              user = "jupiter@karaolidis.com";
+              password = hmConfig.sops.placeholder."grafana/smtp";
+              from_address = "jupiter@karaolidis.com";
+            };
+
+            unified_alerting.enabled = true;
+
+            "unified_alerting.screenshots".capture = true;
+
+            news.news_feed_enabled = false;
+
+            rendering = {
+              server_url = "http://grafana-image-renderer:8081/render";
+              callback_url = "http://grafana:3000";
+            };
+
+            plugins = {
+              plugin_admin_enabled = false;
+              preinstall = lib.strings.concatStringsSep " " [ ];
+              preinstall_async = false;
+            };
+          }
+        );
+      };
+    };
+
+    virtualisation.quadlet = {
+      networks = {
+        grafana.networkConfig.internal = true;
+        # Allow access to host telegraf via non-internal network
+        grafana-prometheus = { };
+      };
+
+      volumes = {
+        "grafana-prometheus-data" = { };
+        "grafana-prometheus-config" = { };
+      };
+
+      containers = {
+        "grafana-prometheus-init" =
+          let
+            prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
+              global = {
+                scrape_interval = "10s";
+                evaluation_interval = "10s";
+              };
+
+              scrape_configs = [
+                {
+                  job_name = "telegraf";
+                  static_configs = [
+                    {
+                      targets = [ "host.containers.internal:9273" ];
+                      labels.app = "telegraf";
+                    }
+                    {
+                      targets = [
+                        "host.containers.internal:${builtins.toString (9273 + config.users.users.${user}.uid)}"
+                      ];
+                      labels.app = "telegraf-storm";
+                    }
+                  ];
+                }
+              ];
+            };
+          in
+          {
+            containerConfig = {
+              image = "docker-archive:${selfPkgs.docker-yq}";
+              volumes = [
+                "${volumes."grafana-prometheus-config".ref}:/etc/prometheus"
+                "${prometheusConfig}:/etc/prometheus/conf.d/prometheus.yml"
+              ];
+              entrypoint = "/bin/bash";
+              exec = [
+                "-c"
+                "yq eval-all '. as $item ireduce ({}; . * $item)' /etc/prometheus/conf.d/*.yml > /etc/prometheus/prometheus.yml"
+              ];
+            };
+
+            serviceConfig = {
+              Type = "oneshot";
+              Restart = "on-failure";
+            };
+          };
+
+        "grafana-prometheus" = {
+          containerConfig = {
+            image = "docker-archive:${selfPkgs.docker-prometheus}";
+            volumes = [
+              "${volumes."grafana-prometheus-config".ref}:/etc/prometheus"
+              "${volumes."grafana-prometheus-data".ref}:/var/lib/prometheus"
+            ];
+            networks = [
+              networks.grafana.ref
+              networks.grafana-prometheus.ref
+            ];
+            exec = [
+              "--config.file=/etc/prometheus/prometheus.yml"
+              "--storage.tsdb.path=/var/lib/prometheus"
+              "--storage.tsdb.retention.time=1y"
+              "--log.level=warn"
+            ];
+          };
+
+          unitConfig.After = [ "${containers."grafana-prometheus-init"._serviceName}.service" ];
+        };
+
+        grafana = {
+          containerConfig = {
+            image = "docker-archive:${selfPkgs.docker-grafana}";
+            networks = [
+              networks.grafana.ref
+              networks.traefik.ref
+            ];
+            volumes = [ "${hmConfig.sops.templates."grafana.ini".path}:/etc/grafana/grafana.ini" ];
+            labels = [
+              "traefik.enable=true"
+              "traefik.http.routers.grafana.rule=Host(`stats.karaolidis.com`)"
+            ];
+          };
+
+          unitConfig.After = [
+            "${containers."grafana-prometheus"._serviceName}.service"
+            "${containers."grafana-image-renderer"._serviceName}.service"
+          ];
+        };
+
+        "grafana-image-renderer" = {
+          containerConfig = {
+            image = "docker-archive:${selfPkgs.docker-grafana-image-renderer}";
+            networks = [ networks.grafana.ref ];
+          };
+        };
+
+        authelia.containerConfig.volumes = [
+          "${hmConfig.sops.templates."authelia-grafana.yaml".path}:/etc/authelia/conf.d/grafana.yaml:ro"
+        ];
+      };
+    };
+  };
+}

hosts/jupiter/users/storm/configs/console/podman/traefik/default.nix

@@ -44,12 +44,14 @@ in
           containerConfig = {
             image = "docker-archive:${selfPkgs.docker-traefik}";
             networks = [ networks.traefik.ref ];
-            volumes = [
+            volumes =
-              "/run/user/${
+              let
-                builtins.toString config.users.users.${user}.uid
+                uid = builtins.toString config.users.users.${user}.uid;
-              }/podman/podman.sock:/var/run/docker.sock"
+              in
-              "${volumes.letsencrypt.ref}:/letsencrypt"
+              [
-            ];
+                "/run/user/${uid}/podman/podman.sock:/var/run/docker.sock"
+                "${volumes.letsencrypt.ref}:/letsencrypt"
+              ];
             exec = [
               "--api.dashboard=true"
               "--api.disabledashboardad=true"
@@ -90,7 +92,7 @@ in
               "traefik.http.routers.traefik-dashboard.service=dashboard@internal"
               "traefik.http.routers.traefik-dashboard.middlewares=authelia@docker"
 
-              "traefik.http.routers.traefik-api.rule='Host(`proxy.karaolidis.com`) && PathPrefix(`/api`)'"
+              "traefik.http.routers.traefik-api.rule=Host(`proxy.karaolidis.com`) && PathPrefix(`/api`)"
               "traefik.http.routers.traefik-api.service=api@internal"
               "traefik.http.routers.traefik-api.middlewares=authelia@docker"
 

hosts/jupiter/users/storm/default.nix

@@ -13,6 +13,7 @@ in
     (import ../../../common/configs/user/console/neovim { inherit user home; })
     (import ../../../common/configs/user/console/podman { inherit user home; })
     (import ../../../common/configs/user/console/sops { inherit user home; })
+    (import ../../../common/configs/user/console/telegraf { inherit user home; })
     (import ../../../common/configs/user/console/tmux { inherit user home; })
     (import ../../../common/configs/user/console/zsh { inherit user home; })
 

packages/default.nix

@@ -11,8 +11,11 @@
 
   docker-authelia = import ./docker/authelia { inherit pkgs; };
   docker-base = import ./docker/base { inherit pkgs; };
+  docker-grafana = import ./docker/grafana { inherit pkgs; };
+  docker-grafana-image-renderer = import ./docker/grafana-image-renderer { inherit pkgs; };
   docker-ntfy = import ./docker/ntfy { inherit pkgs; };
   docker-postgresql = import ./docker/postgresql { inherit pkgs; };
+  docker-prometheus = import ./docker/prometheus { inherit pkgs; };
   docker-redis = import ./docker/redis { inherit pkgs; };
   docker-traefik = import ./docker/traefik { inherit pkgs; };
   docker-whoami = import ./docker/whoami { inherit pkgs; };

packages/docker/grafana-image-renderer/default.nix

@@ -0,0 +1,19 @@
+{ pkgs, ... }:
+pkgs.dockerTools.buildImage {
+  name = "grafana-image-renderer";
+  fromImage = import ../base { inherit pkgs; };
+
+  copyToRoot = pkgs.buildEnv {
+    name = "root";
+    paths = with pkgs; [ grafana-image-renderer ];
+    pathsToLink = [ "/bin" ];
+  };
+
+  config = {
+    Entrypoint = [ "/bin/grafana-image-renderer" ];
+    Cmd = [ "server" ];
+    ExposedPorts = {
+      "8081/tcp" = { };
+    };
+  };
+}

packages/docker/grafana/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, ... }:
+pkgs.dockerTools.buildImage {
+  name = "grafana";
+  fromImage = import ../base { inherit pkgs; };
+
+  copyToRoot = pkgs.buildEnv {
+    name = "root";
+    paths = with pkgs; [ grafana ];
+    pathsToLink = [ "/bin" ];
+  };
+
+  runAsRoot = ''
+    ${pkgs.dockerTools.shadowSetup}
+    mkdir -p /etc/grafana/conf
+    cp -r ${pkgs.grafana}/share/grafana/conf/defaults.ini /etc/grafana/conf/defaults.ini
+    cp -r ${pkgs.grafana}/share/grafana/public /etc/grafana/public
+  '';
+
+  config = {
+    Entrypoint = [ "/bin/grafana" ];
+    Cmd = [
+      "server"
+      "--homepath"
+      "/etc/grafana"
+      "--config"
+      "/etc/grafana/grafana.ini"
+    ];
+    ExposedPorts = {
+      "3000/tcp" = { };
+    };
+  };
+}

packages/docker/prometheus/default.nix

@@ -0,0 +1,18 @@
+{ pkgs, ... }:
+pkgs.dockerTools.buildImage {
+  name = "prometheus";
+  fromImage = import ../base { inherit pkgs; };
+
+  copyToRoot = pkgs.buildEnv {
+    name = "root";
+    paths = with pkgs; [ prometheus ];
+    pathsToLink = [ "/bin" ];
+  };
+
+  config = {
+    Entrypoint = [ "/bin/prometheus" ];
+    ExposedPorts = {
+      "9090/tcp" = { };
+    };
+  };
+}