{
  config,
  lib,
  pkgs,
  ...
}:
let
  cfg = config.services.kubernetes;
in
{
  options.services.kubernetes =
    with lib;
    with types;
    let
      mkCertOptions = name: {
        key = mkOption {
          description = "${name} key file.";
          type = path;
        };

        crt = mkOption {
          description = "${name} certificate file.";
          type = path;
        };
      };
    in
    {
      enable = mkEnableOption "kubernetes";

      lib = mkOption {
        description = "Kubernetes utility functions.";
        type = raw;
        readOnly = true;
        default = {
          mkKubeConfig =
            name: ca: cert: key:
            (pkgs.formats.json { }).generate "${name}-kubeconfig.json" {
              apiVersion = "v1";
              kind = "Config";
              clusters = [
                {
                  name = "local";
                  cluster = {
                    server = cfg.apiserver._address;
                    "certificate-authority" = ca;
                  };
                }
              ];
              users = [
                {
                  inherit name;
                  user = {
                    "client-certificate" = cert;
                    "client-key" = key;
                  };
                }
              ];
              contexts = [
                {
                  name = "local";
                  context = {
                    cluster = "local";
                    user = name;
                  };
                }
              ];
              current-context = "local";
            };
        };
      };

      roles = mkOption {
        description = "Kubernetes role that this machine should take.";
        type = listOf (enum [
          "master"
          "node"
        ]);
        default = [
          "master"
          "node"
        ];
      };

      address = mkOption {
        description = "Kubernetes master server address.";
        type = str;
        default = "localhost";
      };

      cidr = mkOption {
        description = "Kubernetes cluster CIDR.";
        type = str;
        default = "10.0.0.0/24";
      };

      cas = {
        kubernetes = mkCertOptions "Kubernetes CA";
        frontProxy = mkCertOptions "Front Proxy CA";
        etcd = mkCertOptions "ETCD CA";
      };

      certs = {
        apiserver = {
          server = mkCertOptions "Kubernetes API Server";
          kubeletClient = mkCertOptions "Kubernetes API Server Kubelet Client";
          etcdClient = mkCertOptions "Kubernetes API Server ETCD Client";
        };

        etcd = {
          server = mkCertOptions "ETCD Server";
          peer = mkCertOptions "ETCD Peer";
        };

        frontProxy = mkCertOptions "Front Proxy Client";

        serviceAccount = {
          public = mkOption {
            description = "Service account public key file.";
            type = path;
          };

          private = mkOption {
            description = "Service account private key file.";
            type = path;
          };
        };

        accounts = {
          scheduler = mkCertOptions "Kubernetes Scheduler";
          controllerManager = mkCertOptions "Kubernetes Controller Manager";
          addonManager = mkCertOptions "Kubernetes Addon Manager";
          proxy = mkCertOptions "Kubernetes Proxy";
          admin = mkCertOptions "Kubernetes Admin";
        };
      };

      kubeconfigs = mkOption {
        description = "Kubernetes kubeconfigs.";
        type = attrsOf path;
        default = { };
      };

      apiserver = {
        _address = mkOption {
          description = "Kubernetes API server address.";
          internal = true;
          type = str;
        };

        address = mkOption {
          description = "Kubernetes API server listening address.";
          type = str;
          readOnly = true;
          default = "0.0.0.0";
        };

        port = mkOption {
          description = "Kubernetes API server listening port.";
          type = port;
          readOnly = true;
          default = 6443;
        };

        bootstrapTokenFile = mkOption {
          description = "Kubernetes API server bootstrap token file.";
          type = path;
        };
      };

      kubelet = {
        address = mkOption {
          description = "Kubernetes kubelet listening address.";
          type = str;
          readOnly = true;
          default = "0.0.0.0";
        };

        port = mkOption {
          description = "Kubernetes kubelet listening port.";
          type = port;
          readOnly = true;
          default = 10250;
        };

        taints =
          let
            taintOptions =
              { name, ... }:
              {
                key = mkOption {
                  description = "Taint key.";
                  type = str;
                  default = name;
                };

                value = mkOption {
                  description = "Taint value.";
                  type = str;
                };

                effect = mkOption {
                  description = "Taint effect.";
                  type = enum [
                    "NoSchedule"
                    "PreferNoSchedule"
                    "NoExecute"
                  ];
                };
              };
          in
          mkOption {
            description = "Taints to apply to the node.";
            type = attrsOf (submodule taintOptions);
            default = { };
          };

        bootstrapToken = mkOption {
          description = "Kubelet bootstrap token file.";
          type = path;
        };

        seedImages = mkOption {
          description = "Container images to preload on the system.";
          type = listOf package;
          default = [ ];
        };

        cidr = mkOption {
          description = "Kubernetes pod CIDR.";
          type = str;
          default = "10.1.0.0/16";
        };
      };

      scheduler = {
        address = mkOption {
          description = "Kubernetes scheduler listening address.";
          type = str;
          readOnly = true;
          default = "127.0.0.1";
        };

        port = mkOption {
          description = "Kubernetes scheduler listening port.";
          type = port;
          readOnly = true;
          default = 10251;
        };
      };

      controllerManager = {
        address = mkOption {
          description = "Kubernetes controller manager listening address.";
          type = str;
          readOnly = true;
          default = "127.0.0.1";
        };

        port = mkOption {
          description = "Kubernetes controller manager listening port.";
          type = port;
          readOnly = true;
          default = 10252;
        };
      };

      proxy = {
        address = mkOption {
          description = "Kubernetes proxy listening address.";
          type = str;
          readOnly = true;
          default = "0.0.0.0";
        };
      };

      addonManager = {
        addons = mkOption {
          description = "Kubernetes addons.";
          type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
          default = { };
        };

        bootstrapAddons = mkOption {
          description = "Kubernetes addons applied with cluster-admin permissions.";
          type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
          default = { };
        };
      };
    };

  config = lib.mkIf cfg.enable (
    lib.mkMerge [
      # master or node
      {
        services.kubernetes = {
          apiserver._address = "https://${cfg.address}:${toString cfg.apiserver.port}";

          kubeconfigs.admin =
            cfg.lib.mkKubeConfig "admin" cfg.cas.kubernetes.crt cfg.certs.accounts.admin.crt
              cfg.certs.accounts.admin.key;

          addonManager.bootstrapAddons = {
            addonManager = import ./addons/addon-manager { };
            bootstrap = import ./addons/bootstrap { inherit config; };
            kubeletApiAdmin = import ./addons/kubelet-api-admin { };
            metricsServer = import ./addons/metrics-server { };
          };
        };

        boot = {
          kernel.sysctl = {
            "net.bridge.bridge-nf-call-iptables" = 1;
            "net.ipv4.ip_forward" = 1;
            "net.bridge.bridge-nf-call-ip6tables" = 1;
          };

          kernelModules = [
            "br_netfilter"
            "overlay"
          ];
        };

        users = {
          users.kubernetes = {
            uid = config.ids.uids.kubernetes;
            group = "kubernetes";
            home = "/var/lib/kubernetes";
            homeMode = "755";
            createHome = true;
            description = "Kubernetes user";
          };

          groups.kubernetes.gid = config.ids.gids.kubernetes;
        };

        systemd = {
          targets.kubernetes = {
            description = "Kubernetes";
            wantedBy = [ "multi-user.target" ];
          };

          tmpfiles.rules = [
            "d /opt/cni/bin 0755 root root -"
            "d /run/kubernetes 0755 kubernetes kubernetes -"
          ];

          services = {
            kubelet =
              let
                kubeletConfig = (pkgs.formats.json { }).generate "config.json" ({
                  apiVersion = "kubelet.config.k8s.io/v1beta1";
                  kind = "KubeletConfiguration";
                  address = cfg.kubelet.address;
                  port = cfg.kubelet.port;
                  authentication = {
                    x509.clientCAFile = cfg.cas.kubernetes.crt;
                    webhook = {
                      enabled = true;
                      cacheTTL = "10s";
                    };
                  };
                  authorization.mode = "Webhook";
                  cgroupDriver = "systemd";
                  hairpinMode = "hairpin-veth";
                  registerNode = true;
                  containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock";
                  failSwapOn = false;
                  memorySwap.swapBehavior = "LimitedSwap";
                  rotateCertificates = true;
                  serverTLSBootstrap = true;
                  featureGates = {
                    RotateKubeletServerCertificate = true;
                    NodeSwap = true;
                  };
                  healthzBindAddress = "127.0.0.1";
                  healthzPort = 10248;
                });

                taints = lib.strings.concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
                  lib.attrsets.mapAttrsToList (n: v: v) cfg.kubelet.taints
                );

                generateKubeletBootstrapKubeconfig = lib.meta.getExe (
                  pkgs.writeShellApplication {
                    name = "kubelet-bootstrap-kubeconfig";
                    runtimeInputs = with pkgs; [ coreutils ];
                    text = ''
                      mkdir -p /etc/kubernetes
                      cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
                      apiVersion: v1
                      kind: Config
                      clusters:
                      - cluster:
                          certificate-authority: ${cfg.cas.kubernetes.crt}
                          server: ${cfg.apiserver._address}
                        name: local
                      contexts:
                      - context:
                          cluster: local
                          user: kubelet-bootstrap
                        name: bootstrap
                      current-context: bootstrap
                      preferences: {}
                      users:
                      - name: kubelet-bootstrap
                        user:
                          token: $(<${cfg.kubelet.bootstrapToken})
                      EOF
                    '';
                  }
                );

                seedContainerImages = lib.meta.getExe (
                  pkgs.writeShellApplication {
                    name = "seed-container-images";
                    runtimeInputs = with pkgs; [
                      gzip
                      containerd
                      coreutils
                    ];
                    text = ''
                      ${lib.strings.concatMapStrings (img: ''
                        echo "Seeding container image: ${img}"
                        ${
                          if (lib.hasSuffix "gz" img) then
                            ''zcat "${img}" | ctr -n k8s.io image import -''
                          else
                            ''cat "${img}" | ctr -n k8s.io image import -''
                        }
                      '') cfg.kubelet.seedImages}
                    '';
                  }
                );
              in
              {
                description = "Kubernetes Kubelet";
                wantedBy = [ "kubernetes.target" ];
                after = [
                  "network.target"
                  "containerd.service"
                  "kube-apisever.service"
                ];
                path = with pkgs; [
                  kubernetes
                  coreutils
                  util-linux
                  git
                  openssh
                  iproute2
                  ethtool
                  iptables
                  socat
                  thin-provisioning-tools
                ];
                preStart = ''
                  ${generateKubeletBootstrapKubeconfig}
                  ${seedContainerImages}
                '';
                script = lib.strings.concatStringsSep " " (
                  [
                    "kubelet"
                    "--config=${kubeletConfig}"
                    "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
                    "--kubeconfig=/var/lib/kubelet/kubeconfig"
                    "--cert-dir=/var/lib/kubelet/pki"
                    "--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
                    "--kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
                    "--pod-infra-container-image=pause"
                    "--root-dir=/var/lib/kubelet"
                  ]
                  ++ lib.lists.optional (taints != "") [
                    "--register-with-taints=${taints}"
                  ]
                );
                serviceConfig = {
                  Slice = "kubernetes.slice";
                  CPUAccounting = true;
                  MemoryAccounting = true;
                  Restart = "on-failure";
                  RestartSec = "1000ms";
                  WorkingDirectory = "/var/lib/kubelet";
                };
                unitConfig.StartLimitIntervalSec = 0;
              };

            kube-proxy = {
              description = "Kubernetes Proxy";
              wantedBy = [ "kubernetes.target" ];
              after = [ "kube-apiserver.service" ];
              path = with pkgs; [
                kubernetes
                iptables
                conntrack-tools
              ];
              script = lib.strings.concatStringsSep " " [
                "kube-proxy"
                "--bind-address=${cfg.proxy.address}"
                "--cluster-cidr=${cfg.kubelet.cidr}"
                "--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
                "--kubeconfig=${
                  cfg.lib.mkKubeConfig "kube-proxy" cfg.cas.kubernetes.crt cfg.certs.accounts.proxy.crt
                    cfg.certs.accounts.proxy.key
                }"
              ];
              serviceConfig = {
                Slice = "kubernetes.slice";
                WorkingDirectory = "/var/lib/kubernetes";
                Restart = "on-failure";
                RestartSec = 5;
              };
              unitConfig.StartLimitIntervalSec = 0;
            };
          };
        };

        networking.firewall.enable = false;
      }

      # only master
      (lib.mkIf (lib.all (m: m == "master") cfg.roles) {
        services.kubernetes.kubelet.taints = {
          unschedulable = {
            value = "true";
            effect = "NoSchedule";
          };
          "node-role.kubernetes.io/master" = {
            value = "true";
            effect = "NoSchedule";
          };
        };
      })

      # master
      (lib.mkIf (lib.elem "master" cfg.roles) {
        services = {
          etcd = {
            enable = true;
            name = cfg.address;
            keyFile = cfg.certs.etcd.server.key;
            certFile = cfg.certs.etcd.server.crt;
            trustedCaFile = cfg.cas.etcd.crt;
            peerKeyFile = cfg.certs.etcd.peer.key;
            peerCertFile = cfg.certs.etcd.peer.crt;
            peerTrustedCaFile = cfg.cas.etcd.crt;
            clientCertAuth = true;
            peerClientCertAuth = true;
            listenClientUrls = [ "https://0.0.0.0:2379" ];
            listenPeerUrls = [ "https://0.0.0.0:2380" ];
            advertiseClientUrls = [ "https://${cfg.address}:2379" ];
            initialCluster = [ "${cfg.address}=https://${cfg.address}:2380" ];
            initialAdvertisePeerUrls = [ "https://${cfg.address}:2380" ];
          };
        };

        systemd.services = {
          kube-apiserver = {
            description = "Kubernetes API Server";
            wantedBy = [ "kubernetes.target" ];
            after = [ "network.target" ];
            path = with pkgs; [ kubernetes ];
            script = lib.strings.concatStringsSep " " [
              "kube-apiserver"
              "--allow-privileged=true"
              "--authorization-mode=RBAC,Node"
              "--bind-address=${cfg.apiserver.address}"
              "--secure-port=${toString cfg.apiserver.port}"
              "--client-ca-file=${cfg.cas.kubernetes.crt}"
              "--tls-cert-file=${cfg.certs.apiserver.server.crt}"
              "--tls-private-key-file=${cfg.certs.apiserver.server.key}"
              "--enable-admission-plugins=${
                lib.strings.concatStringsSep "," [
                  "NamespaceLifecycle"
                  "LimitRanger"
                  "ServiceAccount"
                  "ResourceQuota"
                  "DefaultStorageClass"
                  "DefaultTolerationSeconds"
                  "NodeRestriction"
                ]
              }"
              "--etcd-servers=${
                lib.strings.concatStringsSep "," [
                  "https://${cfg.address}:2379"
                  "https://127.0.0.1:2379"
                ]
              }"
              "--etcd-cafile=${cfg.cas.etcd.crt}"
              "--etcd-certfile=${cfg.certs.apiserver.etcdClient.crt}"
              "--etcd-keyfile=${cfg.certs.apiserver.etcdClient.key}"
              "--kubelet-certificate-authority=${cfg.cas.kubernetes.crt}"
              "--kubelet-client-certificate=${cfg.certs.apiserver.kubeletClient.crt}"
              "--kubelet-client-key=${cfg.certs.apiserver.kubeletClient.key}"
              "--proxy-client-cert-file=${cfg.certs.frontProxy.crt}"
              "--proxy-client-key-file=${cfg.certs.frontProxy.key}"
              "--runtime-config=authentication.k8s.io/v1beta1=true"
              "--api-audiences=api,https://kubernetes.default.svc"
              "--service-account-issuer=https://kubernetes.default.svc"
              "--service-account-signing-key-file=${cfg.certs.serviceAccount.private}"
              "--service-account-key-file=${cfg.certs.serviceAccount.public}"
              "--service-cluster-ip-range=${cfg.cidr}"
              "--storage-backend=etcd3"
              "--enable-bootstrap-token-auth=true"
              "--token-auth-file=${cfg.apiserver.bootstrapTokenFile}"
              "--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
              "--requestheader-allowed-names=front-proxy-client"
              "--requestheader-extra-headers-prefix=X-Remote-Extra-"
              "--requestheader-group-headers=X-Remote-Group"
              "--requestheader-username-headers=X-Remote-User"
            ];
            serviceConfig = {
              Slice = "kubernetes.slice";
              WorkingDirectory = "/var/lib/kubernetes";
              User = "kubernetes";
              Group = "kubernetes";
              AmbientCapabilities = "cap_net_bind_service";
              Restart = "on-failure";
              RestartSec = 5;
            };

            unitConfig.StartLimitIntervalSec = 0;
          };

          kube-scheduler = {
            description = "Kubernetes Scheduler";
            wantedBy = [ "kubernetes.target" ];
            after = [ "kube-apiserver.service" ];
            path = with pkgs; [ kubernetes ];
            script = lib.strings.concatStringsSep " " [
              "kube-scheduler"
              "--bind-address=${cfg.scheduler.address}"
              "--secure-port=${toString cfg.scheduler.port}"
              "--leader-elect=true"
              "--kubeconfig=${
                cfg.lib.mkKubeConfig "kube-scheduler" cfg.cas.kubernetes.crt cfg.certs.accounts.scheduler.crt
                  cfg.certs.accounts.scheduler.key
              }"
            ];
            serviceConfig = {
              Slice = "kubernetes.slice";
              WorkingDirectory = "/var/lib/kubernetes";
              User = "kubernetes";
              Group = "kubernetes";
              Restart = "on-failure";
              RestartSec = 5;
            };
            unitConfig.StartLimitIntervalSec = 0;
          };

          kube-controller-manager = {
            description = "Kubernetes Controller Manager";
            wantedBy = [ "kubernetes.target" ];
            after = [ "kube-apiserver.service" ];
            path = with pkgs; [ kubernetes ];
            script = lib.strings.concatStringsSep " " [
              "kube-controller-manager"
              "--allocate-node-cidrs=true"
              "--bind-address=${cfg.controllerManager.address}"
              "--secure-port=${toString cfg.controllerManager.port}"
              "--cluster-cidr=${cfg.kubelet.cidr}"
              "--kubeconfig=${
                cfg.lib.mkKubeConfig "kube-controller-manager" cfg.cas.kubernetes.crt
                  cfg.certs.accounts.controllerManager.crt
                  cfg.certs.accounts.controllerManager.key
              }"
              "--leader-elect=true"
              "--root-ca-file=${cfg.cas.kubernetes.crt}"
              "--service-account-private-key-file=${cfg.certs.serviceAccount.private}"
              "--use-service-account-credentials"
              "--client-ca-file=${cfg.cas.kubernetes.crt}"
              "--cluster-signing-cert-file=${cfg.cas.kubernetes.crt}"
              "--cluster-signing-key-file=${cfg.cas.kubernetes.key}"
              "--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
            ];
            serviceConfig = {
              Slice = "kubernetes.slice";
              Restart = "on-failure";
              RestartSec = 30;
              WorkingDirectory = "/var/lib/kubernetes";
              User = "kubernetes";
              Group = "kubernetes";
            };
            unitConfig.StartLimitIntervalSec = 0;
          };

          kube-addon-manager =
            let
              mkAddons =
                addons:
                lib.attrsets.mapAttrsToList (
                  name: addon:
                  (pkgs.formats.json { }).generate "${name}.json" {
                    apiVersion = "v1";
                    kind = "List";
                    items = addon;
                  }
                ) addons;
            in
            {
              description = "Kubernetes Addon Manager";
              wantedBy = [ "kubernetes.target" ];
              after = [ "kube-apiserver.service" ];
              environment = {
                ADDON_PATH = pkgs.runCommand "kube-addons" { } ''
                  mkdir -p $out
                  ${lib.strings.concatMapStringsSep "\n" (a: "ln -s ${a} $out/${baseNameOf a}") (
                    mkAddons cfg.addonManager.addons
                  )}
                '';
                KUBECONFIG =
                  cfg.lib.mkKubeConfig "addon-manager" cfg.cas.kubernetes.crt cfg.certs.accounts.addonManager.crt
                    cfg.certs.accounts.addonManager.key;
              };
              path = with pkgs; [
                kubernetes
                gawk
              ];
              preStart = ''
                export KUBECONFIG=${cfg.kubeconfigs.admin}
                kubectl apply -f ${lib.strings.concatStringsSep " \\\n -f " (mkAddons cfg.addonManager.bootstrapAddons)}
              '';
              script = "kube-addons";
              serviceConfig = {
                Slice = "kubernetes.slice";
                PermissionsStartOnly = true;
                WorkingDirectory = "/var/lib/kubernetes";
                User = "kubernetes";
                Group = "kubernetes";
                Restart = "on-failure";
                RestartSec = 10;
              };
              unitConfig.StartLimitIntervalSec = 0;
            };
        };
      })

      # node
      (lib.mkIf (lib.elem "node" cfg.roles) {
        virtualisation.containerd = {
          enable = true;
          settings = {
            version = 2;
            root = "/var/lib/containerd";
            state = "/run/containerd";
            oom_score = 0;
            grpc.address = "/run/containerd/containerd.sock";
            plugins."io.containerd.grpc.v1.cri" = {
              containerd.runtimes.runc = {
                runtime_type = "io.containerd.runc.v2";
                options.SystemdCgroup = true;
              };
            };
          };
        };
      })
    ]
  );
}