758 lines
25 KiB
Nix
758 lines
25 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.kubernetes;
|
|
in
|
|
{
|
|
options.services.kubernetes =
|
|
with lib;
|
|
with types;
|
|
let
|
|
mkCertOptions = name: {
|
|
key = mkOption {
|
|
description = "${name} key file.";
|
|
type = path;
|
|
};
|
|
|
|
crt = mkOption {
|
|
description = "${name} certificate file.";
|
|
type = path;
|
|
};
|
|
};
|
|
in
|
|
{
|
|
enable = mkEnableOption "kubernetes";
|
|
|
|
lib = mkOption {
|
|
description = "Kubernetes utility functions.";
|
|
type = raw;
|
|
readOnly = true;
|
|
default = {
|
|
mkKubeConfig =
|
|
name: ca: cert: key:
|
|
(pkgs.formats.json { }).generate "${name}-kubeconfig.json" {
|
|
apiVersion = "v1";
|
|
kind = "Config";
|
|
clusters = [
|
|
{
|
|
name = "local";
|
|
cluster = {
|
|
server = cfg.apiserver._address;
|
|
"certificate-authority" = ca;
|
|
};
|
|
}
|
|
];
|
|
users = [
|
|
{
|
|
inherit name;
|
|
user = {
|
|
"client-certificate" = cert;
|
|
"client-key" = key;
|
|
};
|
|
}
|
|
];
|
|
contexts = [
|
|
{
|
|
name = "local";
|
|
context = {
|
|
cluster = "local";
|
|
user = name;
|
|
};
|
|
}
|
|
];
|
|
current-context = "local";
|
|
};
|
|
};
|
|
};
|
|
|
|
roles = mkOption {
|
|
description = "Kubernetes role that this machine should take.";
|
|
type = listOf (enum [
|
|
"master"
|
|
"node"
|
|
]);
|
|
default = [
|
|
"master"
|
|
"node"
|
|
];
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes master server address.";
|
|
type = str;
|
|
default = "localhost";
|
|
};
|
|
|
|
cidr = mkOption {
|
|
description = "Kubernetes cluster CIDR.";
|
|
type = str;
|
|
default = "10.0.0.0/24";
|
|
};
|
|
|
|
cas = {
|
|
kubernetes = mkCertOptions "Kubernetes CA";
|
|
frontProxy = mkCertOptions "Front Proxy CA";
|
|
etcd = mkCertOptions "ETCD CA";
|
|
};
|
|
|
|
certs = {
|
|
apiserver = {
|
|
server = mkCertOptions "Kubernetes API Server";
|
|
kubeletClient = mkCertOptions "Kubernetes API Server Kubelet Client";
|
|
etcdClient = mkCertOptions "Kubernetes API Server ETCD Client";
|
|
};
|
|
|
|
etcd = {
|
|
server = mkCertOptions "ETCD Server";
|
|
peer = mkCertOptions "ETCD Peer";
|
|
};
|
|
|
|
frontProxy = mkCertOptions "Front Proxy Client";
|
|
|
|
serviceAccount = {
|
|
public = mkOption {
|
|
description = "Service account public key file.";
|
|
type = path;
|
|
};
|
|
|
|
private = mkOption {
|
|
description = "Service account private key file.";
|
|
type = path;
|
|
};
|
|
};
|
|
|
|
accounts = {
|
|
scheduler = mkCertOptions "Kubernetes Scheduler";
|
|
controllerManager = mkCertOptions "Kubernetes Controller Manager";
|
|
addonManager = mkCertOptions "Kubernetes Addon Manager";
|
|
proxy = mkCertOptions "Kubernetes Proxy";
|
|
admin = mkCertOptions "Kubernetes Admin";
|
|
};
|
|
};
|
|
|
|
kubeconfigs = mkOption {
|
|
description = "Kubernetes kubeconfigs.";
|
|
type = attrsOf path;
|
|
default = { };
|
|
};
|
|
|
|
apiserver = {
|
|
_address = mkOption {
|
|
description = "Kubernetes API server address.";
|
|
internal = true;
|
|
type = str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes API server listening address.";
|
|
type = str;
|
|
readOnly = true;
|
|
default = "0.0.0.0";
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes API server listening port.";
|
|
type = port;
|
|
readOnly = true;
|
|
default = 6443;
|
|
};
|
|
|
|
bootstrapTokenFile = mkOption {
|
|
description = "Kubernetes API server bootstrap token file.";
|
|
type = path;
|
|
};
|
|
};
|
|
|
|
kubelet = {
|
|
address = mkOption {
|
|
description = "Kubernetes kubelet listening address.";
|
|
type = str;
|
|
readOnly = true;
|
|
default = "0.0.0.0";
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes kubelet listening port.";
|
|
type = port;
|
|
readOnly = true;
|
|
default = 10250;
|
|
};
|
|
|
|
taints =
|
|
let
|
|
taintOptions =
|
|
{ name, ... }:
|
|
{
|
|
key = mkOption {
|
|
description = "Taint key.";
|
|
type = str;
|
|
default = name;
|
|
};
|
|
|
|
value = mkOption {
|
|
description = "Taint value.";
|
|
type = str;
|
|
};
|
|
|
|
effect = mkOption {
|
|
description = "Taint effect.";
|
|
type = enum [
|
|
"NoSchedule"
|
|
"PreferNoSchedule"
|
|
"NoExecute"
|
|
];
|
|
};
|
|
};
|
|
in
|
|
mkOption {
|
|
description = "Taints to apply to the node.";
|
|
type = attrsOf (submodule taintOptions);
|
|
default = { };
|
|
};
|
|
|
|
bootstrapToken = mkOption {
|
|
description = "Kubelet bootstrap token file.";
|
|
type = path;
|
|
};
|
|
|
|
seedImages = mkOption {
|
|
description = "Container images to preload on the system.";
|
|
type = listOf package;
|
|
default = [ ];
|
|
};
|
|
|
|
cidr = mkOption {
|
|
description = "Kubernetes pod CIDR.";
|
|
type = str;
|
|
default = "10.1.0.0/16";
|
|
};
|
|
};
|
|
|
|
scheduler = {
|
|
address = mkOption {
|
|
description = "Kubernetes scheduler listening address.";
|
|
type = str;
|
|
readOnly = true;
|
|
default = "127.0.0.1";
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes scheduler listening port.";
|
|
type = port;
|
|
readOnly = true;
|
|
default = 10251;
|
|
};
|
|
};
|
|
|
|
controllerManager = {
|
|
address = mkOption {
|
|
description = "Kubernetes controller manager listening address.";
|
|
type = str;
|
|
readOnly = true;
|
|
default = "127.0.0.1";
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes controller manager listening port.";
|
|
type = port;
|
|
readOnly = true;
|
|
default = 10252;
|
|
};
|
|
};
|
|
|
|
proxy = {
|
|
address = mkOption {
|
|
description = "Kubernetes proxy listening address.";
|
|
type = str;
|
|
readOnly = true;
|
|
default = "0.0.0.0";
|
|
};
|
|
};
|
|
|
|
addonManager = {
|
|
addons = mkOption {
|
|
description = "Kubernetes addons.";
|
|
type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
|
|
default = { };
|
|
};
|
|
|
|
bootstrapAddons = mkOption {
|
|
description = "Kubernetes addons applied with cluster-admin permissions.";
|
|
type = attrsOf (coercedTo (attrs) (a: [ a ]) (listOf attrs));
|
|
default = { };
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable (
|
|
lib.mkMerge [
|
|
# master or node
|
|
{
|
|
services.kubernetes = {
|
|
apiserver._address = "https://${cfg.address}:${toString cfg.apiserver.port}";
|
|
|
|
kubeconfigs.admin =
|
|
cfg.lib.mkKubeConfig "admin" cfg.cas.kubernetes.crt cfg.certs.accounts.admin.crt
|
|
cfg.certs.accounts.admin.key;
|
|
|
|
addonManager.bootstrapAddons = {
|
|
addonManager = import ./addons/addon-manager { };
|
|
bootstrap = import ./addons/bootstrap { inherit config; };
|
|
kubeletApiAdmin = import ./addons/kubelet-api-admin { };
|
|
metricsServer = import ./addons/metrics-server { };
|
|
};
|
|
};
|
|
|
|
boot = {
|
|
kernel.sysctl = {
|
|
"net.bridge.bridge-nf-call-iptables" = 1;
|
|
"net.ipv4.ip_forward" = 1;
|
|
"net.bridge.bridge-nf-call-ip6tables" = 1;
|
|
};
|
|
|
|
kernelModules = [
|
|
"br_netfilter"
|
|
"overlay"
|
|
];
|
|
};
|
|
|
|
users = {
|
|
users.kubernetes = {
|
|
uid = config.ids.uids.kubernetes;
|
|
group = "kubernetes";
|
|
home = "/var/lib/kubernetes";
|
|
homeMode = "755";
|
|
createHome = true;
|
|
description = "Kubernetes user";
|
|
};
|
|
|
|
groups.kubernetes.gid = config.ids.gids.kubernetes;
|
|
};
|
|
|
|
systemd = {
|
|
targets.kubernetes = {
|
|
description = "Kubernetes";
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
|
|
tmpfiles.rules = [
|
|
"d /opt/cni/bin 0755 root root -"
|
|
"d /run/kubernetes 0755 kubernetes kubernetes -"
|
|
];
|
|
|
|
services = {
|
|
kubelet =
|
|
let
|
|
kubeletConfig = (pkgs.formats.json { }).generate "config.json" ({
|
|
apiVersion = "kubelet.config.k8s.io/v1beta1";
|
|
kind = "KubeletConfiguration";
|
|
address = cfg.kubelet.address;
|
|
port = cfg.kubelet.port;
|
|
authentication = {
|
|
x509.clientCAFile = cfg.cas.kubernetes.crt;
|
|
webhook = {
|
|
enabled = true;
|
|
cacheTTL = "10s";
|
|
};
|
|
};
|
|
authorization.mode = "Webhook";
|
|
cgroupDriver = "systemd";
|
|
hairpinMode = "hairpin-veth";
|
|
registerNode = true;
|
|
containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock";
|
|
failSwapOn = false;
|
|
memorySwap.swapBehavior = "LimitedSwap";
|
|
rotateCertificates = true;
|
|
serverTLSBootstrap = true;
|
|
featureGates = {
|
|
RotateKubeletServerCertificate = true;
|
|
NodeSwap = true;
|
|
};
|
|
healthzBindAddress = "127.0.0.1";
|
|
healthzPort = 10248;
|
|
});
|
|
|
|
taints = lib.strings.concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
|
|
lib.attrsets.mapAttrsToList (n: v: v) cfg.kubelet.taints
|
|
);
|
|
|
|
generateKubeletBootstrapKubeconfig = lib.meta.getExe (
|
|
pkgs.writeShellApplication {
|
|
name = "kubelet-bootstrap-kubeconfig";
|
|
runtimeInputs = with pkgs; [ coreutils ];
|
|
text = ''
|
|
mkdir -p /etc/kubernetes
|
|
cat > /etc/kubernetes/bootstrap-kubeconfig <<EOF
|
|
apiVersion: v1
|
|
kind: Config
|
|
clusters:
|
|
- cluster:
|
|
certificate-authority: ${cfg.cas.kubernetes.crt}
|
|
server: ${cfg.apiserver._address}
|
|
name: local
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
user: kubelet-bootstrap
|
|
name: bootstrap
|
|
current-context: bootstrap
|
|
preferences: {}
|
|
users:
|
|
- name: kubelet-bootstrap
|
|
user:
|
|
token: $(<${cfg.kubelet.bootstrapToken})
|
|
EOF
|
|
'';
|
|
}
|
|
);
|
|
|
|
seedContainerImages = lib.meta.getExe (
|
|
pkgs.writeShellApplication {
|
|
name = "seed-container-images";
|
|
runtimeInputs = with pkgs; [
|
|
gzip
|
|
containerd
|
|
coreutils
|
|
];
|
|
text = ''
|
|
${lib.strings.concatMapStrings (img: ''
|
|
echo "Seeding container image: ${img}"
|
|
${
|
|
if (lib.hasSuffix "gz" img) then
|
|
''zcat "${img}" | ctr -n k8s.io image import -''
|
|
else
|
|
''cat "${img}" | ctr -n k8s.io image import -''
|
|
}
|
|
'') cfg.kubelet.seedImages}
|
|
'';
|
|
}
|
|
);
|
|
in
|
|
{
|
|
description = "Kubernetes Kubelet";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [
|
|
"network.target"
|
|
"containerd.service"
|
|
"kube-apisever.service"
|
|
];
|
|
path = with pkgs; [
|
|
kubernetes
|
|
coreutils
|
|
util-linux
|
|
git
|
|
openssh
|
|
iproute2
|
|
ethtool
|
|
iptables
|
|
socat
|
|
thin-provisioning-tools
|
|
];
|
|
preStart = ''
|
|
${generateKubeletBootstrapKubeconfig}
|
|
${seedContainerImages}
|
|
'';
|
|
script = lib.strings.concatStringsSep " " (
|
|
[
|
|
"kubelet"
|
|
"--config=${kubeletConfig}"
|
|
"--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
|
|
"--kubeconfig=/var/lib/kubelet/kubeconfig"
|
|
"--cert-dir=/var/lib/kubelet/pki"
|
|
"--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
|
|
"--kubeconfig=/etc/kubernetes/bootstrap-kubeconfig"
|
|
"--pod-infra-container-image=pause"
|
|
"--root-dir=/var/lib/kubelet"
|
|
]
|
|
++ lib.lists.optional (taints != "") [
|
|
"--register-with-taints=${taints}"
|
|
]
|
|
);
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
CPUAccounting = true;
|
|
MemoryAccounting = true;
|
|
Restart = "on-failure";
|
|
RestartSec = "1000ms";
|
|
WorkingDirectory = "/var/lib/kubelet";
|
|
};
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
|
|
kube-proxy = {
|
|
description = "Kubernetes Proxy";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
path = with pkgs; [
|
|
kubernetes
|
|
iptables
|
|
conntrack-tools
|
|
];
|
|
script = lib.strings.concatStringsSep " " [
|
|
"kube-proxy"
|
|
"--bind-address=${cfg.proxy.address}"
|
|
"--cluster-cidr=${cfg.kubelet.cidr}"
|
|
"--hostname-override=${lib.strings.toLower config.networking.fqdnOrHostName}"
|
|
"--kubeconfig=${
|
|
cfg.lib.mkKubeConfig "kube-proxy" cfg.cas.kubernetes.crt cfg.certs.accounts.proxy.crt
|
|
cfg.certs.accounts.proxy.key
|
|
}"
|
|
];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
WorkingDirectory = "/var/lib/kubernetes";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
|
|
networking.firewall.enable = false;
|
|
}
|
|
|
|
# only master
|
|
(lib.mkIf (lib.all (m: m == "master") cfg.roles) {
|
|
services.kubernetes.kubelet.taints = {
|
|
unschedulable = {
|
|
value = "true";
|
|
effect = "NoSchedule";
|
|
};
|
|
"node-role.kubernetes.io/master" = {
|
|
value = "true";
|
|
effect = "NoSchedule";
|
|
};
|
|
};
|
|
})
|
|
|
|
# master
|
|
(lib.mkIf (lib.elem "master" cfg.roles) {
|
|
services = {
|
|
etcd = {
|
|
enable = true;
|
|
name = cfg.address;
|
|
keyFile = cfg.certs.etcd.server.key;
|
|
certFile = cfg.certs.etcd.server.crt;
|
|
trustedCaFile = cfg.cas.etcd.crt;
|
|
peerKeyFile = cfg.certs.etcd.peer.key;
|
|
peerCertFile = cfg.certs.etcd.peer.crt;
|
|
peerTrustedCaFile = cfg.cas.etcd.crt;
|
|
clientCertAuth = true;
|
|
peerClientCertAuth = true;
|
|
listenClientUrls = [ "https://0.0.0.0:2379" ];
|
|
listenPeerUrls = [ "https://0.0.0.0:2380" ];
|
|
advertiseClientUrls = [ "https://${cfg.address}:2379" ];
|
|
initialCluster = [ "${cfg.address}=https://${cfg.address}:2380" ];
|
|
initialAdvertisePeerUrls = [ "https://${cfg.address}:2380" ];
|
|
};
|
|
};
|
|
|
|
systemd.services = {
|
|
kube-apiserver = {
|
|
description = "Kubernetes API Server";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "network.target" ];
|
|
path = with pkgs; [ kubernetes ];
|
|
script = lib.strings.concatStringsSep " " [
|
|
"kube-apiserver"
|
|
"--allow-privileged=true"
|
|
"--authorization-mode=RBAC,Node"
|
|
"--bind-address=${cfg.apiserver.address}"
|
|
"--secure-port=${toString cfg.apiserver.port}"
|
|
"--client-ca-file=${cfg.cas.kubernetes.crt}"
|
|
"--tls-cert-file=${cfg.certs.apiserver.server.crt}"
|
|
"--tls-private-key-file=${cfg.certs.apiserver.server.key}"
|
|
"--enable-admission-plugins=${
|
|
lib.strings.concatStringsSep "," [
|
|
"NamespaceLifecycle"
|
|
"LimitRanger"
|
|
"ServiceAccount"
|
|
"ResourceQuota"
|
|
"DefaultStorageClass"
|
|
"DefaultTolerationSeconds"
|
|
"NodeRestriction"
|
|
]
|
|
}"
|
|
"--etcd-servers=${
|
|
lib.strings.concatStringsSep "," [
|
|
"https://${cfg.address}:2379"
|
|
"https://127.0.0.1:2379"
|
|
]
|
|
}"
|
|
"--etcd-cafile=${cfg.cas.etcd.crt}"
|
|
"--etcd-certfile=${cfg.certs.apiserver.etcdClient.crt}"
|
|
"--etcd-keyfile=${cfg.certs.apiserver.etcdClient.key}"
|
|
"--kubelet-certificate-authority=${cfg.cas.kubernetes.crt}"
|
|
"--kubelet-client-certificate=${cfg.certs.apiserver.kubeletClient.crt}"
|
|
"--kubelet-client-key=${cfg.certs.apiserver.kubeletClient.key}"
|
|
"--proxy-client-cert-file=${cfg.certs.frontProxy.crt}"
|
|
"--proxy-client-key-file=${cfg.certs.frontProxy.key}"
|
|
"--runtime-config=authentication.k8s.io/v1beta1=true"
|
|
"--api-audiences=api,https://kubernetes.default.svc"
|
|
"--service-account-issuer=https://kubernetes.default.svc"
|
|
"--service-account-signing-key-file=${cfg.certs.serviceAccount.private}"
|
|
"--service-account-key-file=${cfg.certs.serviceAccount.public}"
|
|
"--service-cluster-ip-range=${cfg.cidr}"
|
|
"--storage-backend=etcd3"
|
|
"--enable-bootstrap-token-auth=true"
|
|
"--token-auth-file=${cfg.apiserver.bootstrapTokenFile}"
|
|
"--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
|
|
"--requestheader-allowed-names=front-proxy-client"
|
|
"--requestheader-extra-headers-prefix=X-Remote-Extra-"
|
|
"--requestheader-group-headers=X-Remote-Group"
|
|
"--requestheader-username-headers=X-Remote-User"
|
|
];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
WorkingDirectory = "/var/lib/kubernetes";
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
AmbientCapabilities = "cap_net_bind_service";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
|
|
kube-scheduler = {
|
|
description = "Kubernetes Scheduler";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
path = with pkgs; [ kubernetes ];
|
|
script = lib.strings.concatStringsSep " " [
|
|
"kube-scheduler"
|
|
"--bind-address=${cfg.scheduler.address}"
|
|
"--secure-port=${toString cfg.scheduler.port}"
|
|
"--leader-elect=true"
|
|
"--kubeconfig=${
|
|
cfg.lib.mkKubeConfig "kube-scheduler" cfg.cas.kubernetes.crt cfg.certs.accounts.scheduler.crt
|
|
cfg.certs.accounts.scheduler.key
|
|
}"
|
|
];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
WorkingDirectory = "/var/lib/kubernetes";
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
|
|
kube-controller-manager = {
|
|
description = "Kubernetes Controller Manager";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
path = with pkgs; [ kubernetes ];
|
|
script = lib.strings.concatStringsSep " " [
|
|
"kube-controller-manager"
|
|
"--allocate-node-cidrs=true"
|
|
"--bind-address=${cfg.controllerManager.address}"
|
|
"--secure-port=${toString cfg.controllerManager.port}"
|
|
"--cluster-cidr=${cfg.kubelet.cidr}"
|
|
"--kubeconfig=${
|
|
cfg.lib.mkKubeConfig "kube-controller-manager" cfg.cas.kubernetes.crt
|
|
cfg.certs.accounts.controllerManager.crt
|
|
cfg.certs.accounts.controllerManager.key
|
|
}"
|
|
"--leader-elect=true"
|
|
"--root-ca-file=${cfg.cas.kubernetes.crt}"
|
|
"--service-account-private-key-file=${cfg.certs.serviceAccount.private}"
|
|
"--use-service-account-credentials"
|
|
"--client-ca-file=${cfg.cas.kubernetes.crt}"
|
|
"--cluster-signing-cert-file=${cfg.cas.kubernetes.crt}"
|
|
"--cluster-signing-key-file=${cfg.cas.kubernetes.key}"
|
|
"--requestheader-client-ca-file=${cfg.cas.frontProxy.crt}"
|
|
];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
Restart = "on-failure";
|
|
RestartSec = 30;
|
|
WorkingDirectory = "/var/lib/kubernetes";
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
|
|
kube-addon-manager =
|
|
let
|
|
mkAddons =
|
|
addons:
|
|
lib.attrsets.mapAttrsToList (
|
|
name: addon:
|
|
(pkgs.formats.json { }).generate "${name}.json" {
|
|
apiVersion = "v1";
|
|
kind = "List";
|
|
items = addon;
|
|
}
|
|
) addons;
|
|
in
|
|
{
|
|
description = "Kubernetes Addon Manager";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
environment = {
|
|
ADDON_PATH = pkgs.runCommand "kube-addons" { } ''
|
|
mkdir -p $out
|
|
${lib.strings.concatMapStringsSep "\n" (a: "ln -s ${a} $out/${baseNameOf a}") (
|
|
mkAddons cfg.addonManager.addons
|
|
)}
|
|
'';
|
|
KUBECONFIG =
|
|
cfg.lib.mkKubeConfig "addon-manager" cfg.cas.kubernetes.crt cfg.certs.accounts.addonManager.crt
|
|
cfg.certs.accounts.addonManager.key;
|
|
};
|
|
path = with pkgs; [
|
|
kubernetes
|
|
gawk
|
|
];
|
|
preStart = ''
|
|
export KUBECONFIG=${cfg.kubeconfigs.admin}
|
|
kubectl apply -f ${lib.strings.concatStringsSep " \\\n -f " (mkAddons cfg.addonManager.bootstrapAddons)}
|
|
'';
|
|
script = "kube-addons";
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
PermissionsStartOnly = true;
|
|
WorkingDirectory = "/var/lib/kubernetes";
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
Restart = "on-failure";
|
|
RestartSec = 10;
|
|
};
|
|
unitConfig.StartLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
})
|
|
|
|
# node
|
|
(lib.mkIf (lib.elem "node" cfg.roles) {
|
|
virtualisation.containerd = {
|
|
enable = true;
|
|
settings = {
|
|
version = 2;
|
|
root = "/var/lib/containerd";
|
|
state = "/run/containerd";
|
|
oom_score = 0;
|
|
grpc.address = "/run/containerd/containerd.sock";
|
|
plugins."io.containerd.grpc.v1.cri" = {
|
|
containerd.runtimes.runc = {
|
|
runtime_type = "io.containerd.runc.v2";
|
|
options.SystemdCgroup = true;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
})
|
|
]
|
|
);
|
|
}
|