Add custom kubernetes module base

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

flake.lock

@@ -143,11 +143,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1738059769,
-        "narHash": "sha256-SBOwc5HSi0zThWoj3EfYh673X1d1dc78N2qCtcJmIvo=",
+        "lastModified": 1738150270,
+        "narHash": "sha256-GkH7I9LW0aFklGc3YxjaBW7TtJy5aWHE0rPBUuz35Hk=",
         "owner": "karaolidis",
         "repo": "nixpkgs",
-        "rev": "befe9d27e7e7be485aae35d541f135c8471bd508",
+        "rev": "e8e18ef6309d021fa600f5aa2665963d8cf76ab7",
         "type": "github"
       },
       "original": {

hosts/common/configs/system/kubernetes/addons/default.nix

@@ -1,7 +0,0 @@
-{ config, lib, ... }:
-{
-  services.kubernetes.addonManager.bootstrapAddons = lib.mkMerge [
-    (import ./bootstrap { inherit config; })
-    (import ./metrics-server { })
-  ];
-}

hosts/common/configs/system/kubernetes/default.nix

@@ -1,20 +1,11 @@
 {
   config,
-  lib,
   pkgs,
   ...
 }:
-let
-  adminKubeconfig = config.services.kubernetes.lib.mkKubeConfig "admin" {
-    caFile = config.sops.secrets."kubernetes/ca/crt".path;
-    keyFile = config.sops.secrets."kubernetes/accounts/admin/key".path;
-    certFile = config.sops.secrets."kubernetes/accounts/admin/crt".path;
-    server = config.services.kubernetes.apiserverAddress;
-  };
-in
 {
   imports = [
-    ./addons
+    ./options
     ./secrets
   ];
 
@@ -26,208 +17,36 @@ in
       "/var/lib/etcd" = { };
     };
 
-    etc."kubeconfig".source = adminKubeconfig;
+    etc."kubeconfig".source = config.services.kubernetes.kubeconfigs.admin;
     systemPackages = with pkgs; [ kubectl ];
   };
 
   services = {
     kubernetes = {
+      enable = true;
+
       roles = [
         "master"
         "node"
       ];
-
-      masterAddress = "localhost";
-      easyCerts = false;
-      caFile = config.sops.secrets."kubernetes/ca/crt".path;
-      addonManager.enable = true;
-
-      apiserver = {
-        allowPrivileged = true;
-
-        clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
-        kubeletClientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
-        tlsKeyFile = config.sops.secrets."kubernetes/apiserver/cert/key".path;
-        tlsCertFile = config.sops.secrets."kubernetes/apiserver/cert/crt".path;
-        kubeletClientKeyFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/key".path;
-        kubeletClientCertFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/crt".path;
-        proxyClientKeyFile = config.sops.secrets."kubernetes/front-proxy/client/key".path;
-        proxyClientCertFile = config.sops.secrets."kubernetes/front-proxy/client/crt".path;
-        serviceAccountSigningKeyFile = config.sops.secrets."kubernetes/sa/key".path;
-        serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/pub".path;
-
-        extraOpts = lib.strings.concatStringsSep " " [
-          "--enable-bootstrap-token-auth=true"
-          "--token-auth-file=${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/csv".path}"
-          "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
-          "--requestheader-allowed-names=front-proxy-client"
-          "--requestheader-extra-headers-prefix=X-Remote-Extra-"
-          "--requestheader-group-headers=X-Remote-Group"
-          "--requestheader-username-headers=X-Remote-User"
-        ];
-
-        etcd = {
-          servers = [ "https://etcd.local:2379" ];
-          caFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
-          keyFile = config.sops.secrets."kubernetes/apiserver/etcd-client/key".path;
-          certFile = config.sops.secrets."kubernetes/apiserver/etcd-client/crt".path;
-        };
-      };
-
-      controllerManager = {
-        rootCaFile = config.sops.secrets."kubernetes/ca/crt".path;
-        serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/key".path;
-
-        extraOpts = lib.strings.concatStringsSep " " [
-          "--client-ca-file=${config.sops.secrets."kubernetes/ca/crt".path}"
-          "--cluster-signing-cert-file=${config.sops.secrets."kubernetes/ca/crt".path}"
-          "--cluster-signing-key-file=${config.sops.secrets."kubernetes/ca/key".path}"
-          "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
-        ];
-
-        kubeconfig = {
-          caFile = config.sops.secrets."kubernetes/ca/crt".path;
-          keyFile = config.sops.secrets."kubernetes/accounts/controller-manager/key".path;
-          certFile = config.sops.secrets."kubernetes/accounts/controller-manager/crt".path;
-        };
-      };
-
-      kubelet = {
-        clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
-
-        extraOpts = lib.strings.concatStringsSep " " [
-          "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
-          "--kubeconfig=/var/lib/kubelet/kubeconfig"
-          "--cert-dir=/var/lib/kubelet"
-        ];
-
-        extraConfig = {
-          failSwapOn = false;
-          rotateCertificates = true;
-          serverTLSBootstrap = true;
-          memorySwap.swapBehavior = "LimitedSwap";
-        };
-
-        featureGates = {
-          RotateKubeletServerCertificate = true;
-          NodeSwap = true;
-        };
-      };
-
-      proxy.kubeconfig = {
-        caFile = config.sops.secrets."kubernetes/ca/crt".path;
-        keyFile = config.sops.secrets."kubernetes/accounts/proxy/key".path;
-        certFile = config.sops.secrets."kubernetes/accounts/proxy/crt".path;
-      };
-
-      scheduler.kubeconfig = {
-        caFile = config.sops.secrets."kubernetes/ca/crt".path;
-        keyFile = config.sops.secrets."kubernetes/accounts/scheduler/key".path;
-        certFile = config.sops.secrets."kubernetes/accounts/scheduler/crt".path;
-      };
     };
-
-    etcd = {
-      keyFile = config.sops.secrets."kubernetes/etcd/server/key".path;
-      certFile = config.sops.secrets."kubernetes/etcd/server/crt".path;
-      peerKeyFile = config.sops.secrets."kubernetes/etcd/peer/key".path;
-      peerCertFile = config.sops.secrets."kubernetes/etcd/peer/crt".path;
-      trustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
-      peerTrustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
-      listenClientUrls = [ "https://127.0.0.1:2379" ];
-      listenPeerUrls = [ "https://127.0.0.1:2380" ];
-      advertiseClientUrls = [ "https://etcd.local:2379" ];
-      initialCluster = [ "${config.services.kubernetes.masterAddress}=https://etcd.local:2380" ];
-      initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
-    };
-
-    flannel.kubeconfig = config.services.kubernetes.lib.mkKubeConfig "flannel" {
-      caFile = config.sops.secrets."kubernetes/ca/crt".path;
-      keyFile = config.sops.secrets."kubernetes/accounts/flannel/key".path;
-      certFile = config.sops.secrets."kubernetes/accounts/flannel/crt".path;
-      server = config.services.kubernetes.apiserverAddress;
-    };
-  };
-
-  networking = {
-    firewall.enable = false;
-    extraHosts = lib.strings.optionalString (config.services.etcd.enable) ''
-      127.0.0.1 etcd.${config.services.kubernetes.addons.dns.clusterDomain} etcd.local
-    '';
   };
 
   systemd.services = {
-    kube-addon-manager = {
-      after = [
-        "sops-nix.service"
-        config.environment.persistence."/persist"."/var/lib/kubernetes".mount
-      ];
+    kube-addon-manager.after = [
+      config.environment.persistence."/persist"."/var/lib/kubernetes".mount
+    ];
 
-      environment.KUBECONFIG = config.services.kubernetes.lib.mkKubeConfig "addon-manager" {
-        caFile = config.sops.secrets."kubernetes/ca/crt".path;
-        keyFile = config.sops.secrets."kubernetes/accounts/addon-manager/key".path;
-        certFile = config.sops.secrets."kubernetes/accounts/addon-manager/crt".path;
-        server = config.services.kubernetes.apiserverAddress;
-      };
-
-      serviceConfig.PermissionsStartOnly = true;
-
-      preStart = ''
-        export KUBECONFIG=${adminKubeconfig}
-        ${config.services.kubernetes.package}/bin/kubectl apply -f ${
-          lib.strings.concatStringsSep " \\\n -f " (
-            lib.attrsets.mapAttrsToList (
-              n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)
-            ) config.services.kubernetes.addonManager.bootstrapAddons
-          )
-        }
-      '';
-    };
-
-    kubelet = {
-      preStart = ''
-        mkdir -p /etc/kubernetes
-        cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
-        apiVersion: v1
-        kind: Config
-        clusters:
-        - cluster:
-            certificate-authority: ${config.sops.secrets."kubernetes/ca/crt".path}
-            server: ${config.services.kubernetes.apiserverAddress}
-          name: local
-        contexts:
-        - context:
-            cluster: local
-            user: kubelet-bootstrap
-          name: bootstrap
-        current-context: bootstrap
-        preferences: {}
-        users:
-        - name: kubelet-bootstrap
-          user:
-            token: $(<${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/token".path})
-        EOF
-      '';
-
-      after = [
-        "sops-nix.service"
-        config.environment.persistence."/persist"."/var/lib/kubelet".mount
-      ];
-    };
+    kubelet.after = [
+      config.environment.persistence."/persist"."/var/lib/kubelet".mount
+    ];
 
     kube-apiserver.after = [
-      "sops-nix.service"
       config.environment.persistence."/persist"."/var/lib/kubernetes".mount
     ];
 
     etcd.after = [
-      "sops-nix.service"
       config.environment.persistence."/persist"."/var/lib/etcd".mount
     ];
-
-    kube-controller-manager.after = [ "sops-nix.service" ];
-    kube-proxy.after = [ "sops-nix.service" ];
-    kube-scheduler.after = [ "sops-nix.service" ];
-    flannel.after = [ "sops-nix.service" ];
   };
 }

hosts/common/configs/system/kubernetes/options/addons/addon-manager/default.nix

@@ -0,0 +1,70 @@
+{ ... }:
+[
+  {
+    apiVersion = "rbac.authorization.k8s.io/v1";
+    kind = "Role";
+    metadata = {
+      name = "system:kube-addon-manager";
+      namespace = "kube-system";
+    };
+    rules = [
+      {
+        apiGroups = [ "*" ];
+        resources = [ "*" ];
+        verbs = [ "*" ];
+      }
+    ];
+  }
+  {
+    apiVersion = "rbac.authorization.k8s.io/v1";
+    kind = "RoleBinding";
+    metadata = {
+      name = "system:kube-addon-manager";
+      namespace = "kube-system";
+    };
+    roleRef = {
+      apiGroup = "rbac.authorization.k8s.io";
+      kind = "Role";
+      name = "system:kube-addon-manager";
+    };
+    subjects = [
+      {
+        apiGroup = "rbac.authorization.k8s.io";
+        kind = "User";
+        name = "system:kube-addon-manager";
+      }
+    ];
+  }
+  {
+    apiVersion = "rbac.authorization.k8s.io/v1";
+    kind = "ClusterRole";
+    metadata = {
+      name = "system:kube-addon-manager:cluster-lister";
+    };
+    rules = [
+      {
+        apiGroups = [ "*" ];
+        resources = [ "*" ];
+        verbs = [ "list" ];
+      }
+    ];
+  }
+  {
+    apiVersion = "rbac.authorization.k8s.io/v1";
+    kind = "ClusterRoleBinding";
+    metadata = {
+      name = "system:kube-addon-manager:cluster-lister";
+    };
+    roleRef = {
+      apiGroup = "rbac.authorization.k8s.io";
+      kind = "ClusterRole";
+      name = "system:kube-addon-manager:cluster-lister";
+    };
+    subjects = [
+      {
+        kind = "User";
+        name = "system:kube-addon-manager";
+      }
+    ];
+  }
+]

hosts/common/configs/system/kubernetes/options/addons/bootstrap/default.nix

@@ -1,6 +1,6 @@
 { config, ... }:
-{
-  bootstrap-node-bootstrapper-crb = {
+[
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -18,9 +18,8 @@
       name = "system:node-bootstrapper";
       apiGroup = "rbac.authorization.k8s.io";
     };
-  };
-
-  bootstrap-csr-nodeclient-crb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -38,9 +37,8 @@
       name = "system:certificates.k8s.io:certificatesigningrequests:nodeclient";
       apiGroup = "rbac.authorization.k8s.io";
     };
-  };
-
-  bootstrap-csr-selfnodeclient-crb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -58,9 +56,8 @@
       name = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient";
       apiGroup = "rbac.authorization.k8s.io";
     };
-  };
-
-  csr-approver-cr = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRole";
     metadata = {
@@ -102,9 +99,8 @@
         verbs = [ "create" ];
       }
     ];
-  };
-
-  csr-approver-crb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -123,18 +119,16 @@
         namespace = "kube-system";
       }
     ];
-  };
-
-  csr-approver-sa = {
+  }
+  {
     apiVersion = "v1";
     kind = "ServiceAccount";
     metadata = {
       name = "kubelet-csr-approver";
       namespace = "kube-system";
     };
-  };
-
-  csr-approver-d = {
+  }
+  {
     apiVersion = "apps/v1";
     kind = "Deployment";
     metadata = {
@@ -181,7 +175,7 @@
               env = [
                 {
                   name = "PROVIDER_REGEX";
-                  value = "^${config.services.kubernetes.kubelet.hostname}$";
+                  value = "^${config.networking.fqdnOrHostName}$";
                 }
                 {
                   name = "PROVIDER_IP_PREFIXES";
@@ -208,5 +202,5 @@
         };
       };
     };
-  };
-}
+  }
+]

hosts/common/configs/system/kubernetes/options/addons/kubelet-api-admin/default.nix

@@ -0,0 +1,21 @@
+{ ... }:
+[
+  {
+    apiVersion = "rbac.authorization.k8s.io/v1";
+    kind = "ClusterRoleBinding";
+    metadata = {
+      name = "system:kube-apiserver:kubelet-api-admin";
+    };
+    roleRef = {
+      apiGroup = "rbac.authorization.k8s.io";
+      kind = "ClusterRole";
+      name = "system:kubelet-api-admin";
+    };
+    subjects = [
+      {
+        kind = "User";
+        name = "system:kube-apiserver";
+      }
+    ];
+  }
+]

hosts/common/configs/system/kubernetes/options/addons/metrics-server/default.nix

@@ -1,6 +1,6 @@
 { ... }:
-{
-  metrics-server-sa = {
+[
+  {
     apiVersion = "v1";
     kind = "ServiceAccount";
     metadata = {
@@ -10,9 +10,8 @@
       name = "metrics-server";
       namespace = "kube-system";
     };
-  };
-
-  metrics-server-metrics-reader-cr = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRole";
     metadata = {
@@ -38,9 +37,8 @@
         ];
       }
     ];
-  };
-
-  metrics-server-cr = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRole";
     metadata = {
@@ -68,9 +66,8 @@
         ];
       }
     ];
-  };
-
-  metrics-server-rb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "RoleBinding";
     metadata = {
@@ -92,9 +89,8 @@
         namespace = "kube-system";
       }
     ];
-  };
-
-  metrics-server-auth-delegator-crb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -115,9 +111,8 @@
         namespace = "kube-system";
       }
     ];
-  };
-
-  metrics-server-crb = {
+  }
+  {
     apiVersion = "rbac.authorization.k8s.io/v1";
     kind = "ClusterRoleBinding";
     metadata = {
@@ -138,9 +133,8 @@
         namespace = "kube-system";
       }
     ];
-  };
-
-  metrics-server-s = {
+  }
+  {
     apiVersion = "v1";
     kind = "Service";
     metadata = {
@@ -163,9 +157,8 @@
         k8s-app = "metrics-server";
       };
     };
-  };
-
-  metrics-server-d = {
+  }
+  {
     apiVersion = "apps/v1";
     kind = "Deployment";
     metadata = {
@@ -271,9 +264,8 @@
         };
       };
     };
-  };
-
-  metrics-server-apis = {
+  }
+  {
     apiVersion = "apiregistration.k8s.io/v1";
     kind = "APIService";
     metadata = {
@@ -293,5 +285,5 @@
       version = "v1beta1";
       versionPriority = 100;
     };
-  };
-}
+  }
+]

hosts/common/configs/system/kubernetes/options/default.nix

@@ -0,0 +1,757 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.kubernetes;
+in
+{
+  options.services.kubernetes =
+    with lib;
+    with types;
+    let
+      mkCertOptions = name: {
+        key = mkOption {
+          description = "${name} key file.";
+          type = path;
+        };
+
+        crt = mkOption {
+          description = "${name} certificate file.";
+          type = path;
+        };
+      };
+    in
+    {
+      enable = mkEnableOption "kubernetes";
+
+      lib = mkOption {
+        description = "Kubernetes utility functions.";
+        type = raw;
+        readOnly = true;
+        default = {
+          mkKubeConfig =
+            name: ca: cert: key:
+            (pkgs.formats.json { }).generate "${name}-kubeconfig.json" {
+              apiVersion = "v1";
+              kind = "Config";
+              clusters = [
+                {
+                  name = "local";
+                  cluster = {
+                    server = cfg.apiserver._address;
+                    "certificate-authority" = ca;
+                  };
+                }
+              ];
+              users = [
+                {
+                  inherit name;
+                  user = {
+                    "client-certificate" = cert;
+                    "client-key" = key;
+                  };
+                }
+              ];
+              contexts = [
+                {
+                  name = "local";
+                  context = {
+                    cluster = "local";
+                    user = name;
+                  };
+                }
+              ];
+              current-context = "local";
+            };
+        };
+      };
+
+      roles = mkOption {
+        description = "Kubernetes role that this machine should take.";
+        type = listOf (enum [
+          "master"
+          "node"
+        ]);
+        default = [
+          "master"
+          "node"
+        ];
+      };
+
+      address = mkOption {
+        description = "Kubernetes master server address.";
+        type = str;
+        default = "localhost";
+      };
+
+      cidr = mkOption {
+        description = "Kubernetes cluster CIDR.";
+        type = str;
+        default = "10.0.0.0/24";
+      };
+
+      cas = {
+        kubernetes = mkCertOptions "Kubernetes CA";
+        frontProxy = mkCertOptions "Front Proxy CA";
+        etcd = mkCertOptions "ETCD CA";
+      };
+
+      certs = {
+        apiserver = {
+          server = mkCertOptions "Kubernetes API Server";
+          kubeletClient = mkCertOptions "Kubernetes API Server Kubelet Client";
+          etcdClient = mkCertOptions "Kubernetes API Server ETCD Client";
+        };
+
+        etcd = {
+          server = mkCertOptions "ETCD Server";
+          peer = mkCertOptions "ETCD Peer";
+        };
+
+        frontProxy = mkCertOptions "Front Proxy Client";
+
+        serviceAccount = {
+          public = mkOption {
+            description = "Service account public key file.";
+            type = path;
+          };
+
+          private = mkOption {
+            description = "Service account private key file.";
+            type = path;
+          };
+        };
+
+        accounts = {
+          scheduler = mkCertOptions "Kubernetes Scheduler";
+          controllerManager = mkCertOptions "Kubernetes Controller Manager";
+          addonManager = mkCertOptions "Kubernetes Addon Manager";
+          proxy = mkCertOptions "Kubernetes Proxy";
+          admin = mkCertOptions "Kubernetes Admin";
+        };
+      };
+
+      kubeconfigs = mkOption {
+        description = "Kubernetes kubeconfigs.";
+        type = attrsOf path;
+        default = { };
+      };
+
+      apiserver = {
+        _address = mkOption {
+          description = "Kubernetes API server address.";
+          internal = true;
+          type = str;
+        };
+
+        address = mkOption {
+          description = "Kubernetes API server listening address.";
+          type = str;
+          readOnly = true;
+          default = "0.0.0.0";
+        };
+
+        port = mkOption {
+          description = "Kubernetes API server listening port.";
+          type = port;
+          readOnly = true;
+          default = 6443;
+        };
+
+        bootstrapTokenFile = mkOption {
+          description = "Kubernetes API server bootstrap token file.";
+          type = path;
+        };
+      };
+
+      kubelet = {
+        address = mkOption {
+          description = "Kubernetes kubelet listening address.";
+          type = str;
+          readOnly = true;
+          default = "0.0.0.0";
+        };
+
+        port = mkOption {
+          description = "Kubernetes kubelet listening port.";
+          type = port;
+          readOnly = true;
+          default = 10250;
+        };
+
+        taints =
+          let
+            taintOptions =
+              { name, ... }:
+              {
+                key = mkOption {
+                  description = "Taint key.";
+                  type = str;
+                  default = name;
+                };
+
+                value = mkOption {
+                  description = "Taint value.";
+                  type = str;
+                };
+
+                effect = mkOption {
+                  description = "Taint effect.";
+                  type = enum [
+                    "NoSchedule"
+                    "PreferNoSchedule"
+                    "NoExecute"
+                  ];
+                };
+              };
+          in
+          mkOption {
+            description = "Taints to apply to the node.";
+            type = attrsOf (submodule taintOptions);
+            default = { };
+          };
+
+        bootstrapToken = mkOption {
+          description = "Kubelet bootstrap token file.";
+          type = path;
+        };
+
+        seedImages = mkOption {
+          description = "Container images to preload on the system.";
+          type = listOf package;
+          default = [ ];
+        };
+
+        cidr = mkOption {
+          description = "Kubernetes pod CIDR.";
+          type = str;
+          default = "10.1.0.0/16";
+        };
+      };
+
+      scheduler = {
+        address = mkOption {
+          description = "Kubernetes scheduler listening address.";
+          type = str;
+          readOnly = true;
+          default = "127.0.0.1";
+        };
+
+        port = mkOption {
+          description = "Kubernetes scheduler listening port.";
+          type = port;
+          readOnly = true;
+          default = 10251;
+        };
+      };
+
+      controllerManager = {
+        address = mkOption {
+          description = "Kubernetes controller manager listening address.";
+          type = str;
+          readOnly = true;
+          default = "127.0.0.1";
+        };
+
+        port = mkOption {
+          description = "Kubernetes controller manager listening port.";
+          type = port;
+          readOnly = true;
+          default = 10252;
+        };
+      };
+
+      proxy = {
+        address = mkOption {
+          description = "Kubernetes proxy listening address.";
+          type = str;
+          readOnly = true;
+          default = "0.0.0.0";
+        };
+      };
+
+      addonManager = {
+        addons = mkOption {
+          description = "Kubernetes addons.";
+          type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
+          default = { };
+        };
+
+        bootstrapAddons = mkOption {
+          description = "Kubernetes addons applied with cluster-admin permissions.";
+          type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
+          default = { };
+        };
+      };
+    };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      # master or node
+      {
+        services.kubernetes = {
+          apiserver._address = "https://${cfg.address}:${toString cfg.apiserver.port}";
+
+          kubeconfigs.admin =
+            cfg.lib.mkKubeConfig "admin" cfg.cas.kubernetes.crt cfg.certs.accounts.admin.crt
+              cfg.certs.accounts.admin.key;
+
+          addonManager.bootstrapAddons = {
+            addonManager = import ./addons/addon-manager { };
+            bootstrap = import ./addons/bootstrap { inherit config; };
+            kubeletApiAdmin = import ./addons/kubelet-api-admin { };
+            metricsServer = import ./addons/metrics-server { };
+          };
+        };
+
+        boot = {
+          kernel.sysctl = {
+            "net.bridge.bridge-nf-call-iptables" = 1;
+            "net.ipv4.ip_forward" = 1;
+            "net.bridge.bridge-nf-call-ip6tables" = 1;
+          };
+
+          kernelModules = [
+            "br_netfilter"
+            "overlay"
+          ];
+        };
+
+        users = {
+          users.kubernetes = {
+            uid = config.ids.uids.kubernetes;
+            group = "kubernetes";
+            home = "/var/lib/kubernetes";
+            homeMode = "755";
+            createHome = true;
+            description = "Kubernetes user";
+          };
+
+          groups.kubernetes.gid = config.ids.gids.kubernetes;
+        };
+
+        systemd = {
+          targets.kubernetes = {
+            description = "Kubernetes";
+            wantedBy = [ "multi-user.target" ];
+          };
+
+          tmpfiles.rules = [
+            "d /opt/cni/bin 0755 root root -"
+            "d /run/kubernetes 0755 kubernetes kubernetes -"
+          ];
+
+          services = {
+            kubelet =
+              let
+                kubeletConfig = (pkgs.formats.json { }).generate "config.json" ({
+                  apiVersion = "kubelet.config.k8s.io/v1beta1";
+                  kind = "KubeletConfiguration";
+                  address = cfg.kubelet.address;
+                  port = cfg.kubelet.port;
+                  authentication = {
+                    x509.clientCAFile = cfg.cas.kubernetes.crt;
+                    webhook = {
+                      enabled = true;
+                      cacheTTL = "10s";
+                    };
+                  };
+                  authorization.mode = "Webhook";
+                  cgroupDriver = "systemd";
+                  hairpinMode = "hairpin-veth";
+                  registerNode = true;
+                  containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock";
+                  failSwapOn = false;
+                  memorySwap.swapBehavior = "LimitedSwap";
+                  rotateCertificates = true;
+                  serverTLSBootstrap = true;
+                  featureGates = {
+                    RotateKubeletServerCertificate = true;
+                    NodeSwap = true;
+                  };
+                  healthzBindAddress = "127.0.0.1";
+                  healthzPort = 10248;
+                });
+
+                taints = lib.strings.concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
+                  lib.attrsets.mapAttrsToList (n: v: v) cfg.kubelet.taints
+                );
+
+                generateKubeletBootstrapKubeconfig = lib.meta.getExe (
+                  pkgs.writeShellApplication {
+                    name = "kubelet-bootstrap-kubeconfig";
+                    runtimeInputs = with pkgs; [ coreutils ];
+                    text = ''
+                      mkdir -p /etc/kubernetes
+                      cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
+                      apiVersion: v1
+                      kind: Config
+                      clusters:
+                      - cluster:
+                          certificate-authority: ${cfg.cas.kubernetes.crt}
+                          server: ${cfg.apiserver._address}
+                        name: local
+                      contexts:
+                      - context:
+                          cluster: local
+                          user: kubelet-bootstrap
+                        name: bootstrap
+                      current-context: bootstrap
+                      preferences: {}
+                      users:
+                      - name: kubelet-bootstrap
+                        user:
+                          token: $(<${cfg.kubelet.bootstrapToken})
+                      EOF
+                    '';
+                  }
+                );
+
+                seedContainerImages = lib.meta.getExe (
+                  pkgs.writeShellApplication {
+                    name = "seed-container-images";
+                    runtimeInputs = with pkgs; [
+                      gzip
+                      containerd
+                      coreutils
+                    ];
+                    text = ''
+                      ${lib.strings.concatMapStrings (img: ''
+                        echo "Seeding container image: ${img}"
+                        ${
+                          if (lib.hasSuffix "gz" img) then
+                            ''zcat "${img}" | ctr -n k8s.io image import -''
+                          else
+                            ''cat "${img}" | ctr -n k8s.io image import -''
+                        }
+                      '') cfg.kubelet.seedImages}
+                    '';
+                  }
+                );
+              in
+              {
+                description = "Kubernetes Kubelet";
+                wantedBy = [ "kubernetes.target" ];
+                after = [
+                  "network.target"
+                  "containerd.service"
+                  "kube-apisever.service"
+                ];
+                path = with pkgs; [
+                  kubernetes
+                  coreutils
+                  util-linux
+                  git
+                  openssh
+                  iproute2
+                  ethtool
+                  iptables
+                  socat
+                  thin-provisioning-tools
+                ];
+                preStart = ''
+                  ${generateKubeletBootstrapKubeconfig}
+                  ${seedContainerImages}
+                '';
+                script = lib.strings.concatStringsSep " " (
+                  [
+                    "kubelet"
+                    "--config=${kubeletConfig}"
+                    "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
+                    "--kubeconfig=/var/lib/kubelet/kubeconfig"
+                    "--cert-dir=/var/lib/kubelet/pki"
+                    "--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
+                    "--kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
+                    "--pod-infra-container-image=pause"
+                    "--root-dir=/var/lib/kubelet"
+                  ]
+                  ++ lib.lists.optional (taints != "") [
+                    "--register-with-taints=${taints}"
+                  ]
+                );
+                serviceConfig = {
+                  Slice = "kubernetes.slice";
+                  CPUAccounting = true;
+                  MemoryAccounting = true;
+                  Restart = "on-failure";
+                  RestartSec = "1000ms";
+                  WorkingDirectory = "/var/lib/kubelet";
+                };
+                unitConfig.StartLimitIntervalSec = 0;
+              };
+
+            kube-proxy = {
+              description = "Kubernetes Proxy";
+              wantedBy = [ "kubernetes.target" ];
+              after = [ "kube-apiserver.service" ];
+              path = with pkgs; [
+                kubernetes
+                iptables
+                conntrack-tools
+              ];
+              script = lib.strings.concatStringsSep " " [
+                "kube-proxy"
+                "--bind-address=${cfg.proxy.address}"
+                "--cluster-cidr=${cfg.kubelet.cidr}"
+                "--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
+                "--kubeconfig=${
+                  cfg.lib.mkKubeConfig "kube-proxy" cfg.cas.kubernetes.crt cfg.certs.accounts.proxy.crt
+                    cfg.certs.accounts.proxy.key
+                }"
+              ];
+              serviceConfig = {
+                Slice = "kubernetes.slice";
+                WorkingDirectory = "/var/lib/kubernetes";
+                Restart = "on-failure";
+                RestartSec = 5;
+              };
+              unitConfig.StartLimitIntervalSec = 0;
+            };
+          };
+        };
+
+        networking.firewall.enable = false;
+      }
+
+      # only master
+      (lib.mkIf (lib.all (m: m == "master") cfg.roles) {
+        services.kubernetes.kubelet.taints = {
+          unschedulable = {
+            value = "true";
+            effect = "NoSchedule";
+          };
+          "node-role.kubernetes.io/master" = {
+            value = "true";
+            effect = "NoSchedule";
+          };
+        };
+      })
+
+      # master
+      (lib.mkIf (lib.elem "master" cfg.roles) {
+        services = {
+          etcd = {
+            enable = true;
+            name = cfg.address;
+            keyFile = cfg.certs.etcd.server.key;
+            certFile = cfg.certs.etcd.server.crt;
+            trustedCaFile = cfg.cas.etcd.crt;
+            peerKeyFile = cfg.certs.etcd.peer.key;
+            peerCertFile = cfg.certs.etcd.peer.crt;
+            peerTrustedCaFile = cfg.cas.etcd.crt;
+            clientCertAuth = true;
+            peerClientCertAuth = true;
+            listenClientUrls = [ "https://0.0.0.0:2379" ];
+            listenPeerUrls = [ "https://0.0.0.0:2380" ];
+            advertiseClientUrls = [ "https://${cfg.address}:2379" ];
+            initialCluster = [ "${cfg.address}=https://${cfg.address}:2380" ];
+            initialAdvertisePeerUrls = [ "https://${cfg.address}:2380" ];
+          };
+        };
+
+        systemd.services = {
+          kube-apiserver = {
+            description = "Kubernetes API Server";
+            wantedBy = [ "kubernetes.target" ];
+            after = [ "network.target" ];
+            path = with pkgs; [ kubernetes ];
+            script = lib.strings.concatStringsSep " " [
+              "kube-apiserver"
+              "--allow-privileged=true"
+              "--authorization-mode=RBAC,Node"
+              "--bind-address=${cfg.apiserver.address}"
+              "--secure-port=${toString cfg.apiserver.port}"
+              "--client-ca-file=${cfg.cas.kubernetes.crt}"
+              "--tls-cert-file=${cfg.certs.apiserver.server.crt}"
+              "--tls-private-key-file=${cfg.certs.apiserver.server.key}"
+              "--enable-admission-plugins=${
+                lib.strings.concatStringsSep "," [
+                  "NamespaceLifecycle"
+                  "LimitRanger"
+                  "ServiceAccount"
+                  "ResourceQuota"
+                  "DefaultStorageClass"
+                  "DefaultTolerationSeconds"
+                  "NodeRestriction"
+                ]
+              }"
+              "--etcd-servers=${
+                lib.strings.concatStringsSep "," [
+                  "https://${cfg.address}:2379"
+                  "https://127.0.0.1:2379"
+                ]
+              }"
+              "--etcd-cafile=${cfg.cas.etcd.crt}"
+              "--etcd-certfile=${cfg.certs.apiserver.etcdClient.crt}"
+              "--etcd-keyfile=${cfg.certs.apiserver.etcdClient.key}"
+              "--kubelet-certificate-authority=${cfg.cas.kubernetes.crt}"
+              "--kubelet-client-certificate=${cfg.certs.apiserver.kubeletClient.crt}"
+              "--kubelet-client-key=${cfg.certs.apiserver.kubeletClient.key}"
+              "--proxy-client-cert-file=${cfg.certs.frontProxy.crt}"
+              "--proxy-client-key-file=${cfg.certs.frontProxy.key}"
+              "--runtime-config=authentication.k8s.io/v1beta1=true"
+              "--api-audiences=api,https://kubernetes.default.svc"
+              "--service-account-issuer=https://kubernetes.default.svc"
+              "--service-account-signing-key-file=${cfg.certs.serviceAccount.private}"
+              "--service-account-key-file=${cfg.certs.serviceAccount.public}"
+              "--service-cluster-ip-range=${cfg.cidr}"
+              "--storage-backend=etcd3"
+              "--enable-bootstrap-token-auth=true"
+              "--token-auth-file=${cfg.apiserver.bootstrapTokenFile}"
+              "--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
+              "--requestheader-allowed-names=front-proxy-client"
+              "--requestheader-extra-headers-prefix=X-Remote-Extra-"
+              "--requestheader-group-headers=X-Remote-Group"
+              "--requestheader-username-headers=X-Remote-User"
+            ];
+            serviceConfig = {
+              Slice = "kubernetes.slice";
+              WorkingDirectory = "/var/lib/kubernetes";
+              User = "kubernetes";
+              Group = "kubernetes";
+              AmbientCapabilities = "cap_net_bind_service";
+              Restart = "on-failure";
+              RestartSec = 5;
+            };
+
+            unitConfig.StartLimitIntervalSec = 0;
+          };
+
+          kube-scheduler = {
+            description = "Kubernetes Scheduler";
+            wantedBy = [ "kubernetes.target" ];
+            after = [ "kube-apiserver.service" ];
+            path = with pkgs; [ kubernetes ];
+            script = lib.strings.concatStringsSep " " [
+              "kube-scheduler"
+              "--bind-address=${cfg.scheduler.address}"
+              "--secure-port=${toString cfg.scheduler.port}"
+              "--leader-elect=true"
+              "--kubeconfig=${
+                cfg.lib.mkKubeConfig "kube-scheduler" cfg.cas.kubernetes.crt cfg.certs.accounts.scheduler.crt
+                  cfg.certs.accounts.scheduler.key
+              }"
+            ];
+            serviceConfig = {
+              Slice = "kubernetes.slice";
+              WorkingDirectory = "/var/lib/kubernetes";
+              User = "kubernetes";
+              Group = "kubernetes";
+              Restart = "on-failure";
+              RestartSec = 5;
+            };
+            unitConfig.StartLimitIntervalSec = 0;
+          };
+
+          kube-controller-manager = {
+            description = "Kubernetes Controller Manager";
+            wantedBy = [ "kubernetes.target" ];
+            after = [ "kube-apiserver.service" ];
+            path = with pkgs; [ kubernetes ];
+            script = lib.strings.concatStringsSep " " [
+              "kube-controller-manager"
+              "--allocate-node-cidrs=true"
+              "--bind-address=${cfg.controllerManager.address}"
+              "--secure-port=${toString cfg.controllerManager.port}"
+              "--cluster-cidr=${cfg.kubelet.cidr}"
+              "--kubeconfig=${
+                cfg.lib.mkKubeConfig "kube-controller-manager" cfg.cas.kubernetes.crt
+                  cfg.certs.accounts.controllerManager.crt
+                  cfg.certs.accounts.controllerManager.key
+              }"
+              "--leader-elect=true"
+              "--root-ca-file=${cfg.cas.kubernetes.crt}"
+              "--service-account-private-key-file=${cfg.certs.serviceAccount.private}"
+              "--use-service-account-credentials"
+              "--client-ca-file=${cfg.cas.kubernetes.crt}"
+              "--cluster-signing-cert-file=${cfg.cas.kubernetes.crt}"
+              "--cluster-signing-key-file=${cfg.cas.kubernetes.key}"
+              "--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
+            ];
+            serviceConfig = {
+              Slice = "kubernetes.slice";
+              Restart = "on-failure";
+              RestartSec = 30;
+              WorkingDirectory = "/var/lib/kubernetes";
+              User = "kubernetes";
+              Group = "kubernetes";
+            };
+            unitConfig.StartLimitIntervalSec = 0;
+          };
+
+          kube-addon-manager =
+            let
+              mkAddons =
+                addons:
+                lib.attrsets.mapAttrsToList (
+                  name: addon:
+                  (pkgs.formats.json { }).generate "${name}.json" {
+                    apiVersion = "v1";
+                    kind = "List";
+                    items = addon;
+                  }
+                ) addons;
+            in
+            {
+              description = "Kubernetes Addon Manager";
+              wantedBy = [ "kubernetes.target" ];
+              after = [ "kube-apiserver.service" ];
+              environment = {
+                ADDON_PATH = pkgs.runCommand "kube-addons" { } ''
+                  mkdir -p $out
+                  ${lib.strings.concatMapStringsSep "\n" (a: "ln -s ${a} $out/${baseNameOf a}") (
+                    mkAddons cfg.addonManager.addons
+                  )}
+                '';
+                KUBECONFIG =
+                  cfg.lib.mkKubeConfig "addon-manager" cfg.cas.kubernetes.crt cfg.certs.accounts.addonManager.crt
+                    cfg.certs.accounts.addonManager.key;
+              };
+              path = with pkgs; [
+                kubernetes
+                gawk
+              ];
+              preStart = ''
+                export KUBECONFIG=${cfg.kubeconfigs.admin}
+                kubectl apply -f ${lib.strings.concatStringsSep " \\\n -f " (mkAddons cfg.addonManager.bootstrapAddons)}
+              '';
+              script = "kube-addons";
+              serviceConfig = {
+                Slice = "kubernetes.slice";
+                PermissionsStartOnly = true;
+                WorkingDirectory = "/var/lib/kubernetes";
+                User = "kubernetes";
+                Group = "kubernetes";
+                Restart = "on-failure";
+                RestartSec = 10;
+              };
+              unitConfig.StartLimitIntervalSec = 0;
+            };
+        };
+      })
+
+      # node
+      (lib.mkIf (lib.elem "node" cfg.roles) {
+        virtualisation.containerd = {
+          enable = true;
+          settings = {
+            version = 2;
+            root = "/var/lib/containerd";
+            state = "/run/containerd";
+            oom_score = 0;
+            grpc.address = "/run/containerd/containerd.sock";
+            plugins."io.containerd.grpc.v1.cri" = {
+              containerd.runtimes.runc = {
+                runtime_type = "io.containerd.runc.v2";
+                options.SystemdCgroup = true;
+              };
+            };
+          };
+        };
+      })
+    ]
+  );
+}

hosts/common/configs/system/kubernetes/secrets/default.nix

@@ -1,204 +1,293 @@
-{ ... }:
+{ config, ... }:
 {
   sops.secrets = {
-    "kubernetes/ca/crt" = {
+    "kubernetes/ca/kubernetes/crt" = {
       owner = "kubernetes";
       group = "users";
       mode = "0440";
     };
 
-    "kubernetes/ca/key" = {
+    "kubernetes/ca/kubernetes/key" = {
       owner = "kubernetes";
       group = "users";
       mode = "0440";
     };
 
-    "kubernetes/front-proxy/ca/crt" = {
+    "kubernetes/ca/front-proxy/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/front-proxy/ca/key" = {
+    "kubernetes/ca/front-proxy/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/ca/crt" = {
+    "kubernetes/ca/etcd/crt" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/ca/key" = {
+    "kubernetes/ca/etcd/key" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/cert/crt" = {
+    "kubernetes/cert/apiserver/server/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/cert/key" = {
+    "kubernetes/cert/apiserver/server/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/kubelet-client/crt" = {
+    "kubernetes/cert/apiserver/etcd-client/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/kubelet-client/key" = {
+    "kubernetes/cert/apiserver/etcd-client/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/etcd-client/crt" = {
+    "kubernetes/cert/apiserver/kubelet-client/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/apiserver/etcd-client/key" = {
+    "kubernetes/cert/apiserver/kubelet-client/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/front-proxy/client/crt" = {
+    "kubernetes/cert/front-proxy/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/front-proxy/client/key" = {
+    "kubernetes/cert/front-proxy/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/server/crt" = {
+    "kubernetes/cert/etcd/server/crt" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/server/key" = {
+    "kubernetes/cert/etcd/server/key" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/peer/crt" = {
+    "kubernetes/cert/etcd/peer/crt" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/etcd/peer/key" = {
+    "kubernetes/cert/etcd/peer/key" = {
       owner = "etcd";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/sa/key" = {
+    "kubernetes/cert/sa/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/sa/pub" = {
+    "kubernetes/cert/sa/pub" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/admin/crt" = {
-      group = "kubernetes";
-    };
-
-    "kubernetes/accounts/admin/key" = {
-      group = "kubernetes";
-    };
-
-    "kubernetes/accounts/controller-manager/crt" = {
+    "kubernetes/cert/accounts/scheduler/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/controller-manager/key" = {
+    "kubernetes/cert/accounts/scheduler/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/addon-manager/crt" = {
+    "kubernetes/cert/accounts/controller-manager/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/addon-manager/key" = {
+    "kubernetes/cert/accounts/controller-manager/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/scheduler/crt" = {
+    "kubernetes/cert/accounts/addon-manager/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/scheduler/key" = {
+    "kubernetes/cert/accounts/addon-manager/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/proxy/crt" = {
+    "kubernetes/cert/accounts/proxy/crt" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/proxy/key" = {
+    "kubernetes/cert/accounts/proxy/key" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/flannel/crt" = {
+    "kubernetes/cert/accounts/admin/crt" = {
+      group = "kubernetes";
+    };
+
+    "kubernetes/cert/accounts/admin/key" = {
+      group = "kubernetes";
+    };
+
+    "kubernetes/token/kubelet-bootstrap/token" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
 
-    "kubernetes/accounts/flannel/key" = {
-      owner = "kubernetes";
-      group = "kubernetes";
-      mode = "0440";
-    };
-
-    "kubernetes/accounts/kubelet-bootstrap/token" = {
-      owner = "kubernetes";
-      group = "kubernetes";
-      mode = "0440";
-    };
-
-    "kubernetes/accounts/kubelet-bootstrap/csv" = {
+    "kubernetes/token/kubelet-bootstrap/csv" = {
       owner = "kubernetes";
       group = "kubernetes";
       mode = "0440";
     };
   };
+
+  services.kubernetes = {
+    cas = {
+      kubernetes = {
+        key = config.sops.secrets."kubernetes/ca/kubernetes/key".path;
+        crt = config.sops.secrets."kubernetes/ca/kubernetes/crt".path;
+      };
+
+      frontProxy = {
+        key = config.sops.secrets."kubernetes/ca/front-proxy/key".path;
+        crt = config.sops.secrets."kubernetes/ca/front-proxy/crt".path;
+      };
+
+      etcd = {
+        key = config.sops.secrets."kubernetes/ca/etcd/key".path;
+        crt = config.sops.secrets."kubernetes/ca/etcd/crt".path;
+      };
+    };
+
+    certs = {
+      apiserver = {
+        server = {
+          key = config.sops.secrets."kubernetes/cert/apiserver/server/key".path;
+          crt = config.sops.secrets."kubernetes/cert/apiserver/server/crt".path;
+        };
+
+        etcdClient = {
+          key = config.sops.secrets."kubernetes/cert/apiserver/etcd-client/key".path;
+          crt = config.sops.secrets."kubernetes/cert/apiserver/etcd-client/crt".path;
+        };
+
+        kubeletClient = {
+          key = config.sops.secrets."kubernetes/cert/apiserver/kubelet-client/key".path;
+          crt = config.sops.secrets."kubernetes/cert/apiserver/kubelet-client/crt".path;
+        };
+      };
+
+      etcd = {
+        server = {
+          key = config.sops.secrets."kubernetes/cert/etcd/server/key".path;
+          crt = config.sops.secrets."kubernetes/cert/etcd/server/crt".path;
+        };
+
+        peer = {
+          key = config.sops.secrets."kubernetes/cert/etcd/peer/key".path;
+          crt = config.sops.secrets."kubernetes/cert/etcd/peer/crt".path;
+        };
+      };
+
+      frontProxy = {
+        key = config.sops.secrets."kubernetes/cert/front-proxy/key".path;
+        crt = config.sops.secrets."kubernetes/cert/front-proxy/crt".path;
+      };
+
+      serviceAccount = {
+        private = config.sops.secrets."kubernetes/cert/sa/key".path;
+        public = config.sops.secrets."kubernetes/cert/sa/pub".path;
+      };
+
+      accounts = {
+        scheduler = {
+          key = config.sops.secrets."kubernetes/cert/accounts/scheduler/key".path;
+          crt = config.sops.secrets."kubernetes/cert/accounts/scheduler/crt".path;
+        };
+
+        controllerManager = {
+          key = config.sops.secrets."kubernetes/cert/accounts/controller-manager/key".path;
+          crt = config.sops.secrets."kubernetes/cert/accounts/controller-manager/crt".path;
+        };
+
+        addonManager = {
+          key = config.sops.secrets."kubernetes/cert/accounts/addon-manager/key".path;
+          crt = config.sops.secrets."kubernetes/cert/accounts/addon-manager/crt".path;
+        };
+
+        proxy = {
+          key = config.sops.secrets."kubernetes/cert/accounts/proxy/key".path;
+          crt = config.sops.secrets."kubernetes/cert/accounts/proxy/crt".path;
+        };
+
+        admin = {
+          key = config.sops.secrets."kubernetes/cert/accounts/admin/key".path;
+          crt = config.sops.secrets."kubernetes/cert/accounts/admin/crt".path;
+        };
+      };
+    };
+
+    kubelet.bootstrapToken = config.sops.secrets."kubernetes/token/kubelet-bootstrap/token".path;
+
+    apiserver.bootstrapTokenFile = config.sops.secrets."kubernetes/token/kubelet-bootstrap/csv".path;
+  };
+
+  systemd.services = {
+    kubelet.after = [ "sops-nix.service" ];
+    kube-apiserver.after = [ "sops-nix.service" ];
+    kube-controller-manager.after = [ "sops-nix.service" ];
+    kube-scheduler.after = [ "sops-nix.service" ];
+    kube-proxy.after = [ "sops-nix.service" ];
+    kube-addon-manager.after = [ "sops-nix.service" ];
+    etcd.after = [ "sops-nix.service" ];
+  };
 }

hosts/common/configs/system/kubernetes/secrets/generate-secrets.sh

@@ -138,28 +138,27 @@ if [ -z "${hostname}" ]; then
   exit 1
 fi
 
-generate_ca out ca ${DEFAULT_CA_DAYS} kubernetes-ca ""
-generate_ca out/front-proxy ca ${DEFAULT_CA_DAYS} kubernetes-front-proxy-ca ""
-generate_ca out/etcd ca ${DEFAULT_CA_DAYS} etcd-ca ""
+generate_ca out/ca kubernetes ${DEFAULT_CA_DAYS} kubernetes-ca ""
+generate_ca out/ca front-proxy ${DEFAULT_CA_DAYS} kubernetes-front-proxy-ca ""
+generate_ca out/ca etcd ${DEFAULT_CA_DAYS} etcd-ca ""
 
-generate_crt out/apiserver cert ${DEFAULT_CA_DAYS} kube-apiserver "" out/ca.key out/ca.crt "kubernetes" "kubernetes.default" "kubernetes.default.svc" "kubernetes.default.svc.cluster" "kubernetes.default.svc.cluster.local" "localhost" "10.0.0.1" "127.0.0.1"
-generate_crt out/apiserver kubelet-client ${DEFAULT_CA_DAYS} kube-apiserver-kubelet-client system:masters out/ca.key out/ca.crt ""
-generate_crt out/apiserver etcd-client ${DEFAULT_CA_DAYS} kube-apiserver-etcd-client "" out/etcd/ca.key out/etcd/ca.crt ""
-generate_crt out/front-proxy client ${DEFAULT_CA_DAYS} front-proxy-client "" out/front-proxy/ca.key out/front-proxy/ca.crt ""
-generate_crt out/etcd server ${DEFAULT_CA_DAYS} kube-etcd "" out/etcd/ca.key out/etcd/ca.crt "etcd.local" "etcd.cluster.local" "localhost" "127.0.0.1"
-generate_crt out/etcd peer ${DEFAULT_CA_DAYS} kube-etcd-peer "" out/etcd/ca.key out/etcd/ca.crt "etcd.local" "etcd.cluster.local" "localhost" "127.0.0.1"
+generate_crt out/cert/apiserver server ${DEFAULT_CA_DAYS} kube-apiserver "" out/ca/kubernetes.key out/ca/kubernetes.crt "kubernetes" "kubernetes.default" "kubernetes.default.svc" "kubernetes.default.svc.cluster" "kubernetes.default.svc.cluster.local" "localhost" "10.0.0.1" "127.0.0.1"
+generate_crt out/cert/apiserver etcd-client ${DEFAULT_CA_DAYS} kube-apiserver-etcd-client "" out/ca/etcd.key out/ca/etcd.crt ""
+generate_crt out/cert/apiserver kubelet-client ${DEFAULT_CA_DAYS} kube-apiserver-kubelet-client "" out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/etcd server ${DEFAULT_CA_DAYS} kube-etcd "" out/ca/etcd.key out/ca/etcd.crt "etcd.local" "etcd.cluster.local" "localhost" "127.0.0.1"
+generate_crt out/cert/etcd peer ${DEFAULT_CA_DAYS} kube-etcd-peer "" out/ca/etcd.key out/ca/etcd.crt "etcd.local" "etcd.cluster.local" "localhost" "127.0.0.1"
+generate_crt out/cert front-proxy ${DEFAULT_CA_DAYS} front-proxy-client "" out/ca/front-proxy.key out/ca/front-proxy.crt ""
 
-generate_key_pair out sa
+generate_key_pair out/cert sa
 
-generate_crt out/accounts admin ${DEFAULT_CA_DAYS} kubernetes-admin system:masters out/ca.key out/ca.crt ""
-generate_crt out/accounts users ${DEFAULT_CA_DAYS} kubernetes-users system:masters out/ca.key out/ca.crt ""
-generate_crt out/accounts controller-manager ${DEFAULT_CA_DAYS} system:kube-controller-manager "" out/ca.key out/ca.crt ""
-generate_crt out/accounts addon-manager ${DEFAULT_CA_DAYS} system:kube-addon-manager "" out/ca.key out/ca.crt ""
-generate_crt out/accounts scheduler ${DEFAULT_CA_DAYS} system:kube-scheduler "" out/ca.key out/ca.crt ""
-generate_crt out/accounts proxy ${DEFAULT_CA_DAYS} system:kube-proxy "" out/ca.key out/ca.crt ""
-generate_crt out/accounts flannel ${DEFAULT_CA_DAYS} flannel-client "" out/ca.key out/ca.crt ""
+generate_crt out/cert/accounts scheduler ${DEFAULT_CA_DAYS} system:kube-scheduler "" out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/accounts controller-manager ${DEFAULT_CA_DAYS} system:kube-controller-manager "" out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/accounts addon-manager ${DEFAULT_CA_DAYS} system:kube-addon-manager "" out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/accounts proxy ${DEFAULT_CA_DAYS} system:kube-proxy "" out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/accounts admin ${DEFAULT_CA_DAYS} kubernetes-admin system:masters out/ca/kubernetes.key out/ca/kubernetes.crt ""
+generate_crt out/cert/accounts users ${DEFAULT_CA_DAYS} kubernetes-users system:masters out/ca/kubernetes.key out/ca/kubernetes.crt ""
 
-generate_auth_token out/accounts kubelet-bootstrap "kubelet-bootstrap" 10001 "system:bootstrappers"
+generate_auth_token out/token kubelet-bootstrap "kubelet-bootstrap" 10001 "system:bootstrappers"
 
 sops_config="../../../../../$(hostname)/secrets/sops.yaml"
 secrets_file="../../../../../$(hostname)/secrets/secrets.yaml"
@@ -168,43 +167,41 @@ sops -d "${secrets_file}" > "${decrypted_secrets_file}"
 
 yq -i '
   del(.kubernetes) |
-  .kubernetes.ca.crt = load_str("out/ca.crt") |
-  .kubernetes.ca.key = load_str("out/ca.key") |
-  .kubernetes.front-proxy.ca.crt = load_str("out/front-proxy/ca.crt") |
-  .kubernetes.front-proxy.ca.key = load_str("out/front-proxy/ca.key") |
-  .kubernetes.etcd.ca.crt = load_str("out/etcd/ca.crt") |
-  .kubernetes.etcd.ca.key = load_str("out/etcd/ca.key") |
-  .kubernetes.apiserver.cert.crt = load_str("out/apiserver/cert.crt") |
-  .kubernetes.apiserver.cert.key = load_str("out/apiserver/cert.key") |
-  .kubernetes.apiserver.kubelet-client.crt = load_str("out/apiserver/kubelet-client.crt") |
-  .kubernetes.apiserver.kubelet-client.key = load_str("out/apiserver/kubelet-client.key") |
-  .kubernetes.apiserver.etcd-client.crt = load_str("out/apiserver/etcd-client.crt") |
-  .kubernetes.apiserver.etcd-client.key = load_str("out/apiserver/etcd-client.key") |
-  .kubernetes.front-proxy.client.crt = load_str("out/front-proxy/client.crt") |
-  .kubernetes.front-proxy.client.key = load_str("out/front-proxy/client.key") |
-  .kubernetes.etcd.server.crt = load_str("out/etcd/server.crt") |
-  .kubernetes.etcd.server.key = load_str("out/etcd/server.key") |
-  .kubernetes.etcd.peer.crt = load_str("out/etcd/peer.crt") |
-  .kubernetes.etcd.peer.key = load_str("out/etcd/peer.key") |
-  .kubernetes.sa.key = load_str("out/sa.key") |
-  .kubernetes.sa.pub = load_str("out/sa.pub") |
-  .kubernetes.accounts.admin.crt = load_str("out/accounts/admin.crt") |
-  .kubernetes.accounts.admin.key = load_str("out/accounts/admin.key") |
-  .kubernetes.accounts.users.crt = load_str("out/accounts/users.crt") |
-  .kubernetes.accounts.users.key = load_str("out/accounts/users.key") |
-  .kubernetes.accounts.controller-manager.crt = load_str("out/accounts/controller-manager.crt") |
-  .kubernetes.accounts.controller-manager.key = load_str("out/accounts/controller-manager.key") |
-  .kubernetes.accounts.addon-manager.crt = load_str("out/accounts/addon-manager.crt") |
-  .kubernetes.accounts.addon-manager.key = load_str("out/accounts/addon-manager.key") |
-  .kubernetes.accounts.scheduler.crt = load_str("out/accounts/scheduler.crt") |
-  .kubernetes.accounts.scheduler.key = load_str("out/accounts/scheduler.key") |
-  .kubernetes.accounts.proxy.crt = load_str("out/accounts/proxy.crt") |
-  .kubernetes.accounts.proxy.key = load_str("out/accounts/proxy.key") |
-  .kubernetes.accounts.flannel.crt = load_str("out/accounts/flannel.crt") |
-  .kubernetes.accounts.flannel.key = load_str("out/accounts/flannel.key") |
-  .kubernetes.accounts.kubelet-bootstrap.token = load_str("out/accounts/kubelet-bootstrap.token") |
-  .kubernetes.accounts.kubelet-bootstrap.csv = load_str("out/accounts/kubelet-bootstrap.csv")
+  .kubernetes.ca.kubernetes.crt = load_str("out/ca/kubernetes.crt") |
+  .kubernetes.ca.kubernetes.key = load_str("out/ca/kubernetes.key") |
+  .kubernetes.ca.front-proxy.crt = load_str("out/ca/front-proxy.crt") |
+  .kubernetes.ca.front-proxy.key = load_str("out/ca/front-proxy.key") |
+  .kubernetes.ca.etcd.crt = load_str("out/ca/etcd.crt") |
+  .kubernetes.ca.etcd.key = load_str("out/ca/etcd.key") |
+  .kubernetes.cert.apiserver.server.crt = load_str("out/cert/apiserver/server.crt") |
+  .kubernetes.cert.apiserver.server.key = load_str("out/cert/apiserver/server.key") |
+  .kubernetes.cert.apiserver.etcd-client.crt = load_str("out/cert/apiserver/etcd-client.crt") |
+  .kubernetes.cert.apiserver.etcd-client.key = load_str("out/cert/apiserver/etcd-client.key") |
+  .kubernetes.cert.apiserver.kubelet-client.crt = load_str("out/cert/apiserver/kubelet-client.crt") |
+  .kubernetes.cert.apiserver.kubelet-client.key = load_str("out/cert/apiserver/kubelet-client.key") |
+  .kubernetes.cert.etcd.server.crt = load_str("out/cert/etcd/server.crt") |
+  .kubernetes.cert.etcd.server.key = load_str("out/cert/etcd/server.key") |
+  .kubernetes.cert.etcd.peer.crt = load_str("out/cert/etcd/peer.crt") |
+  .kubernetes.cert.etcd.peer.key = load_str("out/cert/etcd/peer.key") |
+  .kubernetes.cert.front-proxy.crt = load_str("out/cert/front-proxy.crt") |
+  .kubernetes.cert.front-proxy.key = load_str("out/cert/front-proxy.key") |
+  .kubernetes.cert.sa.key = load_str("out/cert/sa.key") |
+  .kubernetes.cert.sa.pub = load_str("out/cert/sa.pub") |
+  .kubernetes.cert.accounts.scheduler.crt = load_str("out/cert/accounts/scheduler.crt") |
+  .kubernetes.cert.accounts.scheduler.key = load_str("out/cert/accounts/scheduler.key") |
+  .kubernetes.cert.accounts.controller-manager.crt = load_str("out/cert/accounts/controller-manager.crt") |
+  .kubernetes.cert.accounts.controller-manager.key = load_str("out/cert/accounts/controller-manager.key") |
+  .kubernetes.cert.accounts.addon-manager.crt = load_str("out/cert/accounts/addon-manager.crt") |
+  .kubernetes.cert.accounts.addon-manager.key = load_str("out/cert/accounts/addon-manager.key") |
+  .kubernetes.cert.accounts.proxy.crt = load_str("out/cert/accounts/proxy.crt") |
+  .kubernetes.cert.accounts.proxy.key = load_str("out/cert/accounts/proxy.key") |
+  .kubernetes.cert.accounts.admin.crt = load_str("out/cert/accounts/admin.crt") |
+  .kubernetes.cert.accounts.admin.key = load_str("out/cert/accounts/admin.key") |
+  .kubernetes.cert.accounts.users.crt = load_str("out/cert/accounts/users.crt") |
+  .kubernetes.cert.accounts.users.key = load_str("out/cert/accounts/users.key") |
+  .kubernetes.token.kubelet-bootstrap.token = load_str("out/token/kubelet-bootstrap.token") |
+  .kubernetes.token.kubelet-bootstrap.csv = load_str("out/token/kubelet-bootstrap.csv")
 ' "${decrypted_secrets_file}"
 
 sops --config "${sops_config}" -e "${decrypted_secrets_file}" > "${secrets_file}"
-rm -rf out
+rm -rf ${decrypted_secrets_file} out

hosts/common/configs/user/console/kubernetes/default.nix

@@ -25,19 +25,24 @@
   users.users.${user}.extraGroups = [ "kubernetes" ];
 
   sops.secrets = {
-    "kubernetes/accounts/${user}/crt" = {
-      key = "kubernetes/accounts/users/crt";
+    "kubernetes/cert/accounts/${user}/crt" = {
+      key = "kubernetes/cert/accounts/users/crt";
       group = "users";
       mode = "0440";
     };
 
-    "kubernetes/accounts/${user}/key" = {
-      key = "kubernetes/accounts/users/key";
+    "kubernetes/cert/accounts/${user}/key" = {
+      key = "kubernetes/cert/accounts/users/key";
       group = "users";
       mode = "0440";
     };
   };
 
+  services.kubernetes.kubeconfigs.${user} =
+    config.services.kubernetes.lib.mkKubeConfig user config.sops.secrets."kubernetes/ca/kubernetes/crt".path
+      config.sops.secrets."kubernetes/cert/accounts/${user}/crt".path
+      config.sops.secrets."kubernetes/cert/accounts/${user}/key".path;
+
   home-manager.users.${user} = {
     home = {
       packages = with pkgs; [
@@ -47,12 +52,7 @@
         kompose
       ];
 
-      file.".kube/local".source = config.services.kubernetes.lib.mkKubeConfig user {
-        caFile = config.sops.secrets."kubernetes/ca/crt".path;
-        certFile = config.sops.secrets."kubernetes/accounts/${user}/crt".path;
-        keyFile = config.sops.secrets."kubernetes/accounts/${user}/key".path;
-        server = config.services.kubernetes.apiserverAddress;
-      };
+      file.".kube/local".source = config.services.kubernetes.kubeconfigs.${user};
     };
 
     programs = {

hosts/common/configs/user/gui/obsidian/options.nix

@@ -37,154 +37,11 @@ let
     "workspaces"
     "zk-prefixer"
   ];
-
-  toCssName = path: lib.strings.removeSuffix ".css" (builtins.baseNameOf path);
 in
 {
   options.programs.obsidian =
     with lib;
     with types;
-    let
-      corePluginsOptions =
-        { config, ... }:
-        {
-          options = {
-            enable = mkOption {
-              type = bool;
-              default = true;
-              description = "Whether to enable the plugin.";
-            };
-
-            name = mkOption {
-              type = enum corePlugins;
-              description = "The plugin.";
-            };
-
-            options = mkOption {
-              type = attrsOf anything;
-              description = "Plugin options to include.";
-              default = { };
-            };
-          };
-        };
-
-      communityPluginsOptions =
-        { config, ... }:
-        {
-          options = {
-            enable = mkOption {
-              type = bool;
-              default = true;
-              description = "Whether to enable the plugin.";
-            };
-
-            pkg = mkOption {
-              type = package;
-              description = "The plugin package.";
-            };
-
-            options = mkOption {
-              type = attrsOf anything;
-              description = "Options to include in the plugin's `data.json`.";
-              default = { };
-            };
-          };
-        };
-
-      checkCssPath = path: lib.filesystem.pathIsRegularFile path && lib.strings.hasSuffix ".css" path;
-
-      cssSnippetsOptions =
-        { config, ... }:
-        {
-          options = {
-            enable = mkOption {
-              type = bool;
-              default = true;
-              description = "Whether to enable the snippet.";
-            };
-
-            name = mkOption {
-              type = str;
-              defaultText = literalExpression "lib.strings.removeSuffix \".css\" (builtins.baseNameOf source)";
-              description = "Name of the snippet.";
-            };
-
-            source = mkOption {
-              type = nullOr (addCheck path checkCssPath);
-              description = "Path of the source file.";
-              default = null;
-            };
-
-            text = mkOption {
-              type = nullOr str;
-              description = "Text of the file.";
-              default = null;
-            };
-          };
-
-          config.name = mkDefault (toCssName config.source);
-        };
-
-      themesOptions =
-        { config, ... }:
-        {
-          options = {
-            enable = mkOption {
-              type = bool;
-              default = true;
-              description = "Whether to set the theme as active.";
-            };
-
-            pkg = mkOption {
-              type = package;
-              description = "The theme package.";
-            };
-          };
-        };
-
-      hotkeysOptions =
-        { config, ... }:
-        {
-          options = {
-            modifiers = mkOption {
-              type = listOf str;
-              description = "The hotkey modifiers.";
-              default = [ ];
-            };
-
-            key = mkOption {
-              type = str;
-              description = "The hotkey.";
-            };
-          };
-        };
-
-      extraFilesOptions =
-        { name, config, ... }:
-        {
-          options = {
-            source = mkOption {
-              type = nullOr path;
-              description = "Path of the source file or directory.";
-              default = null;
-            };
-
-            text = mkOption {
-              type = nullOr str;
-              description = "Text of the file.";
-              default = null;
-            };
-
-            target = mkOption {
-              type = str;
-              defaultText = literalExpression "name";
-              description = "Path to target relative to the vault's directory.";
-            };
-          };
-
-          config.target = mkDefault name;
-        };
-    in
     {
       enable = mkEnableOption "obsidian";
       package = mkPackageOption pkgs "obsidian" { };
@@ -290,43 +147,196 @@ in
                     default = cfg.sharedSettings.appearance;
                   };
 
-                  corePlugins = mkOption {
-                    description = "Core plugins to activate.";
-                    type = listOf (coercedTo (enum corePlugins) (p: { name = p; }) (submodule corePluginsOptions));
-                    default = cfg.sharedSettings.corePlugins;
-                  };
+                  corePlugins =
+                    let
+                      corePluginsOptions =
+                        { config, ... }:
+                        {
+                          options = {
+                            enable = mkOption {
+                              type = bool;
+                              default = true;
+                              description = "Whether to enable the plugin.";
+                            };
 
-                  communityPlugins = mkOption {
-                    description = "Community plugins to install and activate.";
-                    type = listOf (coercedTo package (p: { pkg = p; }) (submodule communityPluginsOptions));
-                    default = cfg.sharedSettings.communityPlugins;
-                  };
+                            name = mkOption {
+                              type = enum corePlugins;
+                              description = "The plugin.";
+                            };
 
-                  cssSnippets = mkOption {
-                    description = "CSS snippets to install.";
-                    type = listOf (
-                      coercedTo (addCheck path checkCssPath) (p: { source = p; }) (submodule cssSnippetsOptions)
-                    );
-                    default = cfg.sharedSettings.cssSnippets;
-                  };
+                            options = mkOption {
+                              type = attrsOf anything;
+                              description = "Plugin options to include.";
+                              default = { };
+                            };
+                          };
+                        };
+                    in
+                    mkOption {
+                      description = "Core plugins to activate.";
+                      type = listOf (coercedTo (enum corePlugins) (p: { name = p; }) (submodule corePluginsOptions));
+                      default = cfg.sharedSettings.corePlugins;
+                    };
 
-                  themes = mkOption {
-                    description = "Themes to install.";
-                    type = listOf (coercedTo package (p: { pkg = p; }) (submodule themesOptions));
-                    default = cfg.sharedSettings.themes;
-                  };
+                  communityPlugins =
+                    let
+                      communityPluginsOptions =
+                        { config, ... }:
+                        {
+                          options = {
+                            enable = mkOption {
+                              type = bool;
+                              default = true;
+                              description = "Whether to enable the plugin.";
+                            };
 
-                  hotkeys = mkOption {
-                    description = "Hotkeys to configure.";
-                    type = attrsOf (listOf (submodule hotkeysOptions));
-                    default = cfg.sharedSettings.hotkeys;
-                  };
+                            pkg = mkOption {
+                              type = package;
+                              description = "The plugin package.";
+                            };
 
-                  extraFiles = mkOption {
-                    description = "Extra files to link to the vault directory.";
-                    type = attrsOf (submodule extraFilesOptions);
-                    default = cfg.sharedSettings.extraFiles;
-                  };
+                            options = mkOption {
+                              type = attrsOf anything;
+                              description = "Options to include in the plugin's `data.json`.";
+                              default = { };
+                            };
+                          };
+                        };
+                    in
+                    mkOption {
+                      description = "Community plugins to install and activate.";
+                      type = listOf (coercedTo package (p: { pkg = p; }) (submodule communityPluginsOptions));
+                      default = cfg.sharedSettings.communityPlugins;
+                    };
+
+                  cssSnippets =
+                    let
+                      checkCssPath = path: lib.filesystem.pathIsRegularFile path && lib.strings.hasSuffix ".css" path;
+                      toCssName = path: lib.strings.removeSuffix ".css" (builtins.baseNameOf path);
+                      cssSnippetsOptions =
+                        { config, ... }:
+                        {
+                          options = {
+                            enable = mkOption {
+                              type = bool;
+                              default = true;
+                              description = "Whether to enable the snippet.";
+                            };
+
+                            name = mkOption {
+                              type = str;
+                              defaultText = literalExpression "lib.strings.removeSuffix \".css\" (builtins.baseNameOf source)";
+                              description = "Name of the snippet.";
+                            };
+
+                            source = mkOption {
+                              type = nullOr (addCheck path checkCssPath);
+                              description = "Path of the source file.";
+                              default = null;
+                            };
+
+                            text = mkOption {
+                              type = nullOr str;
+                              description = "Text of the file.";
+                              default = null;
+                            };
+                          };
+
+                          config.name = mkDefault (toCssName config.source);
+                        };
+                    in
+                    mkOption {
+                      description = "CSS snippets to install.";
+                      type = listOf (
+                        coercedTo (addCheck path checkCssPath) (p: { source = p; }) (submodule cssSnippetsOptions)
+                      );
+                      default = cfg.sharedSettings.cssSnippets;
+                    };
+
+                  themes =
+                    let
+
+                      themesOptions =
+                        { config, ... }:
+                        {
+                          options = {
+                            enable = mkOption {
+                              type = bool;
+                              default = true;
+                              description = "Whether to set the theme as active.";
+                            };
+
+                            pkg = mkOption {
+                              type = package;
+                              description = "The theme package.";
+                            };
+                          };
+                        };
+                    in
+                    mkOption {
+                      description = "Themes to install.";
+                      type = listOf (coercedTo package (p: { pkg = p; }) (submodule themesOptions));
+                      default = cfg.sharedSettings.themes;
+                    };
+
+                  hotkeys =
+                    let
+                      hotkeysOptions =
+                        { config, ... }:
+                        {
+                          options = {
+                            modifiers = mkOption {
+                              type = listOf str;
+                              description = "The hotkey modifiers.";
+                              default = [ ];
+                            };
+
+                            key = mkOption {
+                              type = str;
+                              description = "The hotkey.";
+                            };
+                          };
+                        };
+                    in
+                    mkOption {
+                      description = "Hotkeys to configure.";
+                      type = attrsOf (listOf (submodule hotkeysOptions));
+                      default = cfg.sharedSettings.hotkeys;
+                    };
+
+                  extraFiles =
+                    let
+                      extraFilesOptions =
+                        { name, config, ... }:
+                        {
+                          options = {
+                            source = mkOption {
+                              type = nullOr path;
+                              description = "Path of the source file or directory.";
+                              default = null;
+                            };
+
+                            text = mkOption {
+                              type = nullOr str;
+                              description = "Text of the file.";
+                              default = null;
+                            };
+
+                            target = mkOption {
+                              type = str;
+                              defaultText = literalExpression "name";
+                              description = "Path to target relative to the vault's directory.";
+                            };
+                          };
+
+                          config.target = mkDefault name;
+                        };
+                    in
+                    mkOption {
+                      description = "Extra files to link to the vault directory.";
+                      type = attrsOf (submodule extraFilesOptions);
+                      default = cfg.sharedSettings.extraFiles;
+                    };
                 };
               };