nix/hosts/common/configs/system/kubernetes/default.nix

{
  config,
  lib,
  pkgs,
  ...
}:
let
  adminKubeconfig = config.services.kubernetes.lib.mkKubeConfig "admin" {
    caFile = config.sops.secrets."kubernetes/ca/crt".path;
    keyFile = config.sops.secrets."kubernetes/accounts/admin/key".path;
    certFile = config.sops.secrets."kubernetes/accounts/admin/crt".path;
    server = config.services.kubernetes.apiserverAddress;
  };
in
{
  imports = [
    ./addons
    ./secrets
  ];

  environment = {
    persistence."/persist" = {
      "/var/lib/containerd" = { };
      "/var/lib/kubernetes" = { };
      "/var/lib/kubelet" = { };
      "/var/lib/etcd" = { };
    };

    etc."kubeconfig".source = adminKubeconfig;
    systemPackages = with pkgs; [ kubectl ];
  };

  services = {
    kubernetes = {
      roles = [
        "master"
        "node"
      ];

      masterAddress = "localhost";
      easyCerts = false;
      caFile = config.sops.secrets."kubernetes/ca/crt".path;
      addonManager.enable = true;

      apiserver = {
        allowPrivileged = true;

        clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
        kubeletClientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
        tlsKeyFile = config.sops.secrets."kubernetes/apiserver/cert/key".path;
        tlsCertFile = config.sops.secrets."kubernetes/apiserver/cert/crt".path;
        kubeletClientKeyFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/key".path;
        kubeletClientCertFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/crt".path;
        proxyClientKeyFile = config.sops.secrets."kubernetes/front-proxy/client/key".path;
        proxyClientCertFile = config.sops.secrets."kubernetes/front-proxy/client/crt".path;
        serviceAccountSigningKeyFile = config.sops.secrets."kubernetes/sa/key".path;
        serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/pub".path;

        extraOpts = lib.strings.concatStringsSep " " [
          "--enable-bootstrap-token-auth=true"
          "--token-auth-file=${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/csv".path}"
          "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
          "--requestheader-allowed-names=front-proxy-client"
          "--requestheader-extra-headers-prefix=X-Remote-Extra-"
          "--requestheader-group-headers=X-Remote-Group"
          "--requestheader-username-headers=X-Remote-User"
        ];

        etcd = {
          servers = [ "https://etcd.local:2379" ];
          caFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
          keyFile = config.sops.secrets."kubernetes/apiserver/etcd-client/key".path;
          certFile = config.sops.secrets."kubernetes/apiserver/etcd-client/crt".path;
        };
      };

      controllerManager = {
        rootCaFile = config.sops.secrets."kubernetes/ca/crt".path;
        serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/key".path;

        extraOpts = lib.strings.concatStringsSep " " [
          "--client-ca-file=${config.sops.secrets."kubernetes/ca/crt".path}"
          "--cluster-signing-cert-file=${config.sops.secrets."kubernetes/ca/crt".path}"
          "--cluster-signing-key-file=${config.sops.secrets."kubernetes/ca/key".path}"
          "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
        ];

        kubeconfig = {
          caFile = config.sops.secrets."kubernetes/ca/crt".path;
          keyFile = config.sops.secrets."kubernetes/accounts/controller-manager/key".path;
          certFile = config.sops.secrets."kubernetes/accounts/controller-manager/crt".path;
        };
      };

      kubelet = {
        clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;

        extraOpts = lib.strings.concatStringsSep " " [
          "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
          "--kubeconfig=/var/lib/kubelet/kubeconfig"
          "--cert-dir=/var/lib/kubelet"
        ];

        extraConfig = {
          failSwapOn = false;
          rotateCertificates = true;
          serverTLSBootstrap = true;
          memorySwap.swapBehavior = "LimitedSwap";
        };

        featureGates = {
          RotateKubeletServerCertificate = true;
          NodeSwap = true;
        };
      };

      proxy.kubeconfig = {
        caFile = config.sops.secrets."kubernetes/ca/crt".path;
        keyFile = config.sops.secrets."kubernetes/accounts/proxy/key".path;
        certFile = config.sops.secrets."kubernetes/accounts/proxy/crt".path;
      };

      scheduler.kubeconfig = {
        caFile = config.sops.secrets."kubernetes/ca/crt".path;
        keyFile = config.sops.secrets."kubernetes/accounts/scheduler/key".path;
        certFile = config.sops.secrets."kubernetes/accounts/scheduler/crt".path;
      };
    };

    etcd = {
      keyFile = config.sops.secrets."kubernetes/etcd/server/key".path;
      certFile = config.sops.secrets."kubernetes/etcd/server/crt".path;
      peerKeyFile = config.sops.secrets."kubernetes/etcd/peer/key".path;
      peerCertFile = config.sops.secrets."kubernetes/etcd/peer/crt".path;
      trustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
      peerTrustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
      listenClientUrls = [ "https://127.0.0.1:2379" ];
      listenPeerUrls = [ "https://127.0.0.1:2380" ];
      advertiseClientUrls = [ "https://etcd.local:2379" ];
      initialCluster = [ "${config.services.kubernetes.masterAddress}=https://etcd.local:2380" ];
      initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
    };

    flannel.kubeconfig = config.services.kubernetes.lib.mkKubeConfig "flannel" {
      caFile = config.sops.secrets."kubernetes/ca/crt".path;
      keyFile = config.sops.secrets."kubernetes/accounts/flannel/key".path;
      certFile = config.sops.secrets."kubernetes/accounts/flannel/crt".path;
      server = config.services.kubernetes.apiserverAddress;
    };
  };

  networking = {
    firewall.enable = false;
    extraHosts = lib.strings.optionalString (config.services.etcd.enable) ''
      127.0.0.1 etcd.${config.services.kubernetes.addons.dns.clusterDomain} etcd.local
    '';
  };

  systemd.services = {
    kube-addon-manager = {
      after = [
        "sops-nix.service"
        config.environment.persistence."/persist"."/var/lib/kubernetes".mount
      ];

      environment.KUBECONFIG = config.services.kubernetes.lib.mkKubeConfig "addon-manager" {
        caFile = config.sops.secrets."kubernetes/ca/crt".path;
        keyFile = config.sops.secrets."kubernetes/accounts/addon-manager/key".path;
        certFile = config.sops.secrets."kubernetes/accounts/addon-manager/crt".path;
        server = config.services.kubernetes.apiserverAddress;
      };

      serviceConfig.PermissionsStartOnly = true;

      preStart = ''
        export KUBECONFIG=${adminKubeconfig}
        ${config.services.kubernetes.package}/bin/kubectl apply -f ${
          lib.strings.concatStringsSep " \\\n -f " (
            lib.attrsets.mapAttrsToList (
              n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)
            ) config.services.kubernetes.addonManager.bootstrapAddons
          )
        }
      '';
    };

    kubelet = {
      preStart = ''
        mkdir -p /etc/kubernetes
        cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
        apiVersion: v1
        kind: Config
        clusters:
        - cluster:
            certificate-authority: ${config.sops.secrets."kubernetes/ca/crt".path}
            server: ${config.services.kubernetes.apiserverAddress}
          name: local
        contexts:
        - context:
            cluster: local
            user: kubelet-bootstrap
          name: bootstrap
        current-context: bootstrap
        preferences: {}
        users:
        - name: kubelet-bootstrap
          user:
            token: $(<${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/token".path})
        EOF
      '';

      after = [
        "sops-nix.service"
        config.environment.persistence."/persist"."/var/lib/kubelet".mount
      ];
    };

    kube-apiserver.after = [
      "sops-nix.service"
      config.environment.persistence."/persist"."/var/lib/kubernetes".mount
    ];

    etcd.after = [
      "sops-nix.service"
      config.environment.persistence."/persist"."/var/lib/etcd".mount
    ];

    kube-controller-manager.after = [ "sops-nix.service" ];
    kube-proxy.after = [ "sops-nix.service" ];
    kube-scheduler.after = [ "sops-nix.service" ];
    flannel.after = [ "sops-nix.service" ];
  };
}