{ config, lib, pkgs, ... }: let adminKubeconfig = config.services.kubernetes.lib.mkKubeConfig "admin" { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/admin/key".path; certFile = config.sops.secrets."kubernetes/accounts/admin/crt".path; server = config.services.kubernetes.apiserverAddress; }; in { imports = [ ./addons ./secrets ]; environment = { persistence."/persist" = { "/var/lib/containerd" = { }; "/var/lib/kubernetes" = { }; "/var/lib/kubelet" = { }; "/var/lib/etcd" = { }; }; etc."kubeconfig".source = adminKubeconfig; systemPackages = with pkgs; [ kubectl ]; }; services = { kubernetes = { roles = [ "master" "node" ]; masterAddress = "localhost"; easyCerts = false; caFile = config.sops.secrets."kubernetes/ca/crt".path; addonManager.enable = true; apiserver = { allowPrivileged = true; clientCaFile = config.sops.secrets."kubernetes/ca/crt".path; kubeletClientCaFile = config.sops.secrets."kubernetes/ca/crt".path; tlsKeyFile = config.sops.secrets."kubernetes/apiserver/cert/key".path; tlsCertFile = config.sops.secrets."kubernetes/apiserver/cert/crt".path; kubeletClientKeyFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/key".path; kubeletClientCertFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/crt".path; proxyClientKeyFile = config.sops.secrets."kubernetes/front-proxy/client/key".path; proxyClientCertFile = config.sops.secrets."kubernetes/front-proxy/client/crt".path; serviceAccountSigningKeyFile = config.sops.secrets."kubernetes/sa/key".path; serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/pub".path; extraOpts = lib.strings.concatStringsSep " " [ "--enable-bootstrap-token-auth=true" "--token-auth-file=${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/csv".path}" "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}" "--requestheader-allowed-names=front-proxy-client" "--requestheader-extra-headers-prefix=X-Remote-Extra-" "--requestheader-group-headers=X-Remote-Group" "--requestheader-username-headers=X-Remote-User" ]; etcd = { servers = [ "https://etcd.local:2379" ]; caFile = config.sops.secrets."kubernetes/etcd/ca/crt".path; keyFile = config.sops.secrets."kubernetes/apiserver/etcd-client/key".path; certFile = config.sops.secrets."kubernetes/apiserver/etcd-client/crt".path; }; }; controllerManager = { rootCaFile = config.sops.secrets."kubernetes/ca/crt".path; serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/key".path; extraOpts = lib.strings.concatStringsSep " " [ "--client-ca-file=${config.sops.secrets."kubernetes/ca/crt".path}" "--cluster-signing-cert-file=${config.sops.secrets."kubernetes/ca/crt".path}" "--cluster-signing-key-file=${config.sops.secrets."kubernetes/ca/key".path}" "--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}" ]; kubeconfig = { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/controller-manager/key".path; certFile = config.sops.secrets."kubernetes/accounts/controller-manager/crt".path; }; }; kubelet = { clientCaFile = config.sops.secrets."kubernetes/ca/crt".path; extraOpts = lib.strings.concatStringsSep " " [ "--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig" "--kubeconfig=/var/lib/kubelet/kubeconfig" "--cert-dir=/var/lib/kubelet" ]; extraConfig = { failSwapOn = false; rotateCertificates = true; serverTLSBootstrap = true; memorySwap.swapBehavior = "LimitedSwap"; }; featureGates = { RotateKubeletServerCertificate = true; NodeSwap = true; }; }; proxy.kubeconfig = { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/proxy/key".path; certFile = config.sops.secrets."kubernetes/accounts/proxy/crt".path; }; scheduler.kubeconfig = { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/scheduler/key".path; certFile = config.sops.secrets."kubernetes/accounts/scheduler/crt".path; }; }; etcd = { keyFile = config.sops.secrets."kubernetes/etcd/server/key".path; certFile = config.sops.secrets."kubernetes/etcd/server/crt".path; peerKeyFile = config.sops.secrets."kubernetes/etcd/peer/key".path; peerCertFile = config.sops.secrets."kubernetes/etcd/peer/crt".path; trustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path; peerTrustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path; listenClientUrls = [ "https://127.0.0.1:2379" ]; listenPeerUrls = [ "https://127.0.0.1:2380" ]; advertiseClientUrls = [ "https://etcd.local:2379" ]; initialCluster = [ "${config.services.kubernetes.masterAddress}=https://etcd.local:2380" ]; initialAdvertisePeerUrls = [ "https://etcd.local:2380" ]; }; flannel.kubeconfig = config.services.kubernetes.lib.mkKubeConfig "flannel" { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/flannel/key".path; certFile = config.sops.secrets."kubernetes/accounts/flannel/crt".path; server = config.services.kubernetes.apiserverAddress; }; }; networking = { firewall.enable = false; extraHosts = lib.strings.optionalString (config.services.etcd.enable) '' 127.0.0.1 etcd.${config.services.kubernetes.addons.dns.clusterDomain} etcd.local ''; }; systemd.services = { kube-addon-manager = { after = [ "sops-nix.service" config.environment.persistence."/persist"."/var/lib/kubernetes".mount ]; environment.KUBECONFIG = config.services.kubernetes.lib.mkKubeConfig "addon-manager" { caFile = config.sops.secrets."kubernetes/ca/crt".path; keyFile = config.sops.secrets."kubernetes/accounts/addon-manager/key".path; certFile = config.sops.secrets."kubernetes/accounts/addon-manager/crt".path; server = config.services.kubernetes.apiserverAddress; }; serviceConfig.PermissionsStartOnly = true; preStart = '' export KUBECONFIG=${adminKubeconfig} ${config.services.kubernetes.package}/bin/kubectl apply -f ${ lib.strings.concatStringsSep " \\\n -f " ( lib.attrsets.mapAttrsToList ( n: v: pkgs.writeText "${n}.json" (builtins.toJSON v) ) config.services.kubernetes.addonManager.bootstrapAddons ) } ''; }; kubelet = { preStart = '' mkdir -p /etc/kubernetes cat > /etc/kubernetes/bootstrap-kubeconfig <