Add kubernetes
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
233
hosts/common/configs/system/kubernetes/default.nix
Normal file
233
hosts/common/configs/system/kubernetes/default.nix
Normal file
@@ -0,0 +1,233 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
adminKubeconfig = config.services.kubernetes.lib.mkKubeConfig "admin" {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/admin/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/admin/crt".path;
|
||||
server = config.services.kubernetes.apiserverAddress;
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
./addons
|
||||
./secrets
|
||||
];
|
||||
|
||||
environment = {
|
||||
persistence."/persist" = {
|
||||
"/var/lib/containerd" = { };
|
||||
"/var/lib/kubernetes" = { };
|
||||
"/var/lib/kubelet" = { };
|
||||
"/var/lib/etcd" = { };
|
||||
};
|
||||
|
||||
etc."kubeconfig".source = adminKubeconfig;
|
||||
systemPackages = with pkgs; [ kubectl ];
|
||||
};
|
||||
|
||||
services = {
|
||||
kubernetes = {
|
||||
roles = [
|
||||
"master"
|
||||
"node"
|
||||
];
|
||||
|
||||
masterAddress = "localhost";
|
||||
easyCerts = false;
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
addonManager.enable = true;
|
||||
|
||||
apiserver = {
|
||||
allowPrivileged = true;
|
||||
|
||||
clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
kubeletClientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
tlsKeyFile = config.sops.secrets."kubernetes/apiserver/cert/key".path;
|
||||
tlsCertFile = config.sops.secrets."kubernetes/apiserver/cert/crt".path;
|
||||
kubeletClientKeyFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/key".path;
|
||||
kubeletClientCertFile = config.sops.secrets."kubernetes/apiserver/kubelet-client/crt".path;
|
||||
proxyClientKeyFile = config.sops.secrets."kubernetes/front-proxy/client/key".path;
|
||||
proxyClientCertFile = config.sops.secrets."kubernetes/front-proxy/client/crt".path;
|
||||
serviceAccountSigningKeyFile = config.sops.secrets."kubernetes/sa/key".path;
|
||||
serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/pub".path;
|
||||
|
||||
extraOpts = lib.strings.concatStringsSep " " [
|
||||
"--enable-bootstrap-token-auth=true"
|
||||
"--token-auth-file=${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/csv".path}"
|
||||
"--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
|
||||
"--requestheader-allowed-names=front-proxy-client"
|
||||
"--requestheader-extra-headers-prefix=X-Remote-Extra-"
|
||||
"--requestheader-group-headers=X-Remote-Group"
|
||||
"--requestheader-username-headers=X-Remote-User"
|
||||
];
|
||||
|
||||
etcd = {
|
||||
servers = [ "https://etcd.local:2379" ];
|
||||
caFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/apiserver/etcd-client/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/apiserver/etcd-client/crt".path;
|
||||
};
|
||||
};
|
||||
|
||||
controllerManager = {
|
||||
rootCaFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
serviceAccountKeyFile = config.sops.secrets."kubernetes/sa/key".path;
|
||||
|
||||
extraOpts = lib.strings.concatStringsSep " " [
|
||||
"--client-ca-file=${config.sops.secrets."kubernetes/ca/crt".path}"
|
||||
"--cluster-signing-cert-file=${config.sops.secrets."kubernetes/ca/crt".path}"
|
||||
"--cluster-signing-key-file=${config.sops.secrets."kubernetes/ca/key".path}"
|
||||
"--requestheader-client-ca-file=${config.sops.secrets."kubernetes/front-proxy/ca/crt".path}"
|
||||
];
|
||||
|
||||
kubeconfig = {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/controller-manager/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/controller-manager/crt".path;
|
||||
};
|
||||
};
|
||||
|
||||
kubelet = {
|
||||
clientCaFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
|
||||
extraOpts = lib.strings.concatStringsSep " " [
|
||||
"--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
|
||||
"--kubeconfig=/var/lib/kubelet/kubeconfig"
|
||||
"--cert-dir=/var/lib/kubelet"
|
||||
];
|
||||
|
||||
extraConfig = {
|
||||
failSwapOn = false;
|
||||
rotateCertificates = true;
|
||||
serverTLSBootstrap = true;
|
||||
memorySwap.swapBehavior = "LimitedSwap";
|
||||
};
|
||||
|
||||
featureGates = {
|
||||
RotateKubeletServerCertificate = true;
|
||||
NodeSwap = true;
|
||||
};
|
||||
};
|
||||
|
||||
proxy.kubeconfig = {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/proxy/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/proxy/crt".path;
|
||||
};
|
||||
|
||||
scheduler.kubeconfig = {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/scheduler/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/scheduler/crt".path;
|
||||
};
|
||||
};
|
||||
|
||||
etcd = {
|
||||
keyFile = config.sops.secrets."kubernetes/etcd/server/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/etcd/server/crt".path;
|
||||
peerKeyFile = config.sops.secrets."kubernetes/etcd/peer/key".path;
|
||||
peerCertFile = config.sops.secrets."kubernetes/etcd/peer/crt".path;
|
||||
trustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
|
||||
peerTrustedCaFile = config.sops.secrets."kubernetes/etcd/ca/crt".path;
|
||||
listenClientUrls = [ "https://127.0.0.1:2379" ];
|
||||
listenPeerUrls = [ "https://127.0.0.1:2380" ];
|
||||
advertiseClientUrls = [ "https://etcd.local:2379" ];
|
||||
initialCluster = [ "${config.services.kubernetes.masterAddress}=https://etcd.local:2380" ];
|
||||
initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
|
||||
};
|
||||
|
||||
flannel.kubeconfig = config.services.kubernetes.lib.mkKubeConfig "flannel" {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/flannel/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/flannel/crt".path;
|
||||
server = config.services.kubernetes.apiserverAddress;
|
||||
};
|
||||
};
|
||||
|
||||
networking = {
|
||||
firewall.enable = false;
|
||||
extraHosts = lib.strings.optionalString (config.services.etcd.enable) ''
|
||||
127.0.0.1 etcd.${config.services.kubernetes.addons.dns.clusterDomain} etcd.local
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
kube-addon-manager = {
|
||||
after = [
|
||||
"sops-nix.service"
|
||||
config.environment.persistence."/persist"."/var/lib/kubernetes".mount
|
||||
];
|
||||
|
||||
environment.KUBECONFIG = config.services.kubernetes.lib.mkKubeConfig "addon-manager" {
|
||||
caFile = config.sops.secrets."kubernetes/ca/crt".path;
|
||||
keyFile = config.sops.secrets."kubernetes/accounts/addon-manager/key".path;
|
||||
certFile = config.sops.secrets."kubernetes/accounts/addon-manager/crt".path;
|
||||
server = config.services.kubernetes.apiserverAddress;
|
||||
};
|
||||
|
||||
serviceConfig.PermissionsStartOnly = true;
|
||||
|
||||
preStart = ''
|
||||
export KUBECONFIG=${adminKubeconfig}
|
||||
${config.services.kubernetes.package}/bin/kubectl apply -f ${
|
||||
lib.strings.concatStringsSep " \\\n -f " (
|
||||
lib.attrsets.mapAttrsToList (
|
||||
n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)
|
||||
) config.services.kubernetes.addonManager.bootstrapAddons
|
||||
)
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
kubelet = {
|
||||
preStart = ''
|
||||
mkdir -p /etc/kubernetes
|
||||
cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority: ${config.sops.secrets."kubernetes/ca/crt".path}
|
||||
server: ${config.services.kubernetes.apiserverAddress}
|
||||
name: local
|
||||
contexts:
|
||||
- context:
|
||||
cluster: local
|
||||
user: kubelet-bootstrap
|
||||
name: bootstrap
|
||||
current-context: bootstrap
|
||||
preferences: {}
|
||||
users:
|
||||
- name: kubelet-bootstrap
|
||||
user:
|
||||
token: $(<${config.sops.secrets."kubernetes/accounts/kubelet-bootstrap/token".path})
|
||||
EOF
|
||||
'';
|
||||
|
||||
after = [
|
||||
"sops-nix.service"
|
||||
config.environment.persistence."/persist"."/var/lib/kubelet".mount
|
||||
];
|
||||
};
|
||||
|
||||
kube-apiserver.after = [
|
||||
"sops-nix.service"
|
||||
config.environment.persistence."/persist"."/var/lib/kubernetes".mount
|
||||
];
|
||||
|
||||
etcd.after = [
|
||||
"sops-nix.service"
|
||||
config.environment.persistence."/persist"."/var/lib/etcd".mount
|
||||
];
|
||||
|
||||
kube-controller-manager.after = [ "sops-nix.service" ];
|
||||
kube-proxy.after = [ "sops-nix.service" ];
|
||||
kube-scheduler.after = [ "sops-nix.service" ];
|
||||
flannel.after = [ "sops-nix.service" ];
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user