Implement a basic Kubeadm rust implementation

The deploying machine is able to choose to be deployed inside/outside of the cluster.

Signed-off-by: Ruoqing He <[email protected]>

Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "rk8s"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { version = "4.3.0", features = ["derive"] }
+regex = "1.8.3"
+serde = { version = "1.0.163", features = ["derive"] }
+serde_json = "1.0.96"
+serde_yaml = "0.9.21"
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"

src/config.rs

@@ -0,0 +1,203 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{Read, Write};
+
+#[allow(non_snake_case)]
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Config {
+    // The machine running this program.
+    pub instance_name: String,
+    // The ip address of running server.
+    pub instance_ip: String,
+    pub instance_hosts: HashMap<String, String>,
+
+    // Fields needed by `install cfssl` command.
+    pub cfssl_url: String,
+    pub cfssljson_url: String,
+    pub cfsslcertinfo_url: String,
+    // Fields needed by `install etcd` command.
+    pub etcd_url: String,
+    // Fields needed by `install docker` command.
+    pub docker_url: String,
+    // Fields needed by `install kubernetes` command.
+    pub kubernetes_url: String,
+
+    // Fields needed by `etcd` phase.
+    pub etcd_ca_CN: String,
+    pub etcd_CN: String,
+    pub etcd_key_algo: String,
+    pub etcd_key_size: i64,
+    pub etcd_expiry: String,
+    pub etcd_usages: Vec<String>,
+    pub etcd_names_C: String,
+    pub etcd_names_L: String,
+    pub etcd_names_ST: String,
+
+    // Fields needed by `kube_apiserver` phase.
+    pub kube_apiserver_CN: String,
+    pub kube_apiserver_key_algo: String,
+    pub kube_apiserver_key_size: i64,
+    pub kube_apiserver_expiry: String,
+    pub kube_apiserver_usages: Vec<String>,
+    pub kube_apiserver_names_C: String,
+    pub kube_apiserver_names_L: String,
+    pub kube_apiserver_names_ST: String,
+    pub kube_apiserver_names_O: String,
+    pub kube_apiserver_names_OU: String,
+
+    // Fields needed by `kube_controller_manager` phase.
+    pub kube_controller_manager_CN: String,
+    pub kube_controller_manager_key_algo: String,
+    pub kube_controller_manager_key_size: i64,
+    pub kube_controller_manager_names_C: String,
+    pub kube_controller_manager_names_L: String,
+    pub kube_controller_manager_names_ST: String,
+    pub kube_controller_manager_names_O: String,
+    pub kube_controller_manager_names_OU: String,
+
+    // Fields needed by `kube_scheduler` phase.
+    pub kube_scheduler_CN: String,
+    pub kube_scheduler_key_algo: String,
+    pub kube_scheduler_key_size: i64,
+    pub kube_scheduler_names_C: String,
+    pub kube_scheduler_names_L: String,
+    pub kube_scheduler_names_ST: String,
+    pub kube_scheduler_names_O: String,
+    pub kube_scheduler_names_OU: String,
+
+    // Fields needed by `kube_ctl` phase.
+    pub kube_ctl_CN: String,
+    pub kube_ctl_key_algo: String,
+    pub kube_ctl_key_size: i64,
+    pub kube_ctl_names_C: String,
+    pub kube_ctl_names_L: String,
+    pub kube_ctl_names_ST: String,
+    pub kube_ctl_names_O: String,
+    pub kube_ctl_names_OU: String,
+
+    // Fields needed by `kube_proxy` phase.
+    pub kube_proxy_CN: String,
+    pub kube_proxy_key_algo: String,
+    pub kube_proxy_key_size: i64,
+    pub kube_proxy_names_C: String,
+    pub kube_proxy_names_L: String,
+    pub kube_proxy_names_ST: String,
+    pub kube_proxy_names_O: String,
+    pub kube_proxy_names_OU: String,
+}
+
+impl Config {
+    pub fn init() -> Config {
+        tracing::info!("Reading config file...");
+        let mut file = File::open("cfg/config.yaml").expect("File `config.yaml` does not exist!");
+        let mut content = vec![];
+        file.read_to_end(&mut content)
+            .expect("Error happened when trying to read content of `config.yaml`");
+        let config = serde_yaml::from_slice(&content)
+            .expect("Something went wrong while parsing config.yaml");
+        tracing::info!("Config read");
+        config
+    }
+}
+
+pub fn generate_config_template() {
+    let config = Config {
+        instance_name: "master01".to_owned(),
+        instance_ip: "192.168.221.143".to_owned(),
+        instance_hosts: {
+            let mut map = HashMap::new();
+            map.insert(
+                "192.168.221.143".to_owned(),
+                "master01".to_owned(),
+            );
+            map.insert(
+                "192.168.221.147".to_owned(),
+                "worker01".to_owned(),
+            );
+            map.insert(
+                "192.168.221.148".to_owned(),
+                "worker02".to_owned(),
+            );
+            map
+        },
+
+        cfssl_url: "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64".to_owned(),
+        cfssljson_url: "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64".to_owned(),
+        cfsslcertinfo_url: "https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64".to_owned(),
+        etcd_url: "https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz".to_owned(),
+        docker_url: "https://download.docker.com/linux/static/stable/x86_64/docker-20.10.9.tgz".to_owned(),
+        kubernetes_url: "https://dl.k8s.io/v1.20.15/kubernetes-server-linux-amd64.tar.gz".to_owned(),
+
+        etcd_ca_CN: "etcd CA".to_owned(),
+        etcd_CN: "etcd".to_owned(),
+        etcd_key_algo: "rsa".to_owned(),
+        etcd_key_size: 2048,
+        etcd_expiry: "87600h".to_owned(),
+        etcd_usages: vec![
+            "signing".to_owned(),
+            "key encipherment".to_owned(),
+            "server auth".to_owned(),
+            "client auth".to_owned(),
+        ],
+        etcd_names_C: "CN".to_owned(),
+        etcd_names_L: "Beijing".to_owned(),
+        etcd_names_ST: "Beijing".to_owned(),
+
+        kube_apiserver_CN: "kubernetes".to_owned(),
+        kube_apiserver_key_algo: "rsa".to_owned(),
+        kube_apiserver_key_size: 2048,
+        kube_apiserver_expiry: "87600h".to_owned(),
+        kube_apiserver_usages: vec![
+            "signing".to_owned(),
+            "key encipherment".to_owned(),
+            "server auth".to_owned(),
+            "client auth".to_owned(),
+        ],
+        kube_apiserver_names_C: "CN".to_owned(),
+        kube_apiserver_names_L: "Beijing".to_owned(),
+        kube_apiserver_names_ST: "Beijing".to_owned(),
+        kube_apiserver_names_O: "k8s".to_owned(),
+        kube_apiserver_names_OU: "System".to_owned(),
+
+        kube_controller_manager_CN: "system:kube-controller-manager".to_owned(),
+        kube_controller_manager_key_algo: "rsa".to_owned(),
+        kube_controller_manager_key_size: 2048,
+        kube_controller_manager_names_C: "CN".to_owned(),
+        kube_controller_manager_names_L: "Beijing".to_owned(),
+        kube_controller_manager_names_ST: "Beijing".to_owned(),
+        kube_controller_manager_names_O: "system:masters".to_owned(),
+        kube_controller_manager_names_OU: "System".to_owned(),
+
+        kube_scheduler_CN: "system:kube-scheduler".to_owned(),
+        kube_scheduler_key_algo: "rsa".to_owned(),
+        kube_scheduler_key_size: 2048,
+        kube_scheduler_names_C: "CN".to_owned(),
+        kube_scheduler_names_L: "Beijing".to_owned(),
+        kube_scheduler_names_ST: "Beijing".to_owned(),
+        kube_scheduler_names_O: "system:masters".to_owned(),
+        kube_scheduler_names_OU: "System".to_owned(),
+
+        kube_ctl_CN: "admin".to_owned(),
+        kube_ctl_key_algo: "rsa".to_owned(),
+        kube_ctl_key_size: 2048,
+        kube_ctl_names_C: "CN".to_owned(),
+        kube_ctl_names_L: "Beijing".to_owned(),
+        kube_ctl_names_ST: "Beijing".to_owned(),
+        kube_ctl_names_O: "system:masters".to_owned(),
+        kube_ctl_names_OU: "System".to_owned(),
+
+        kube_proxy_CN: "system:kube-proxy".to_owned(),
+        kube_proxy_key_algo: "rsa".to_owned(),
+        kube_proxy_key_size: 2048,
+        kube_proxy_names_C: "CN".to_owned(),
+        kube_proxy_names_L: "Beijing".to_owned(),
+        kube_proxy_names_ST: "Beijing".to_owned(),
+        kube_proxy_names_O: "k8s".to_owned(),
+        kube_proxy_names_OU: "System".to_owned(),
+    };
+
+    let yaml = serde_yaml::to_string(&config).unwrap();
+    let mut file = File::create("cfg/config.yaml").unwrap();
+    file.write_all(yaml.as_bytes()).unwrap();
+}

src/deploy/docker.rs

@@ -0,0 +1,170 @@
+use std::process::Command;
+use std::path::{Path, PathBuf};
+use std::env;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+
+use crate::config::Config;
+
+struct DockerCfg;
+
+impl DockerCfg {
+    fn generate() {
+        // Send to /etc/docker/daemon.json
+        let mut docker_cfg = File::create("to_send/daemon.json")
+            .expect("Error happened when trying to create docker config file");
+        // Set youki (pre-built) as the default runtime.
+        let content = r#"{
+    "default-runtime": "youki",
+    "runtimes": {
+        "youki": {
+            "path": "/usr/bin/youki"
+        }
+    },
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "100m"
+    },
+    "storage-driver": "overlay2"
+}
+"#;
+        docker_cfg
+            .write_all(content.as_bytes())
+            .expect("Error happened when trying to write docker unit file");
+    }
+}
+
+struct DockerUnit;
+
+impl DockerUnit {
+    fn generate() {
+        let mut docker_unit = File::create("to_send/docker.service")
+            .expect("Error happened when trying to create docker unit file");
+        let content = r#"[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network-online.target firewalld.service
+Wants=network-online.target
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/dockerd
+ExecReload=/bin/kill -s HUP $MAINPID
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+TimeoutStartSec=0
+Delegate=yes
+KillMode=process
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target
+"#;
+        docker_unit
+            .write_all(content.as_bytes())
+            .expect("Error happened when trying to write docker unit file");
+    }
+}
+
+pub fn start(config: &Config) {
+    // Deploy docker to all hosts according to their name.
+    // Docker does not distinguish masters or workers.
+    tracing::info!("Preparing mutual .json, .service and docker binaries...");
+    tracing::info!("Change working directory into `docker`");
+    let prev_dir = Path::new("/rk8s");
+    let work_dir = Path::new("/rk8s/docker");
+    env::set_current_dir(&work_dir).expect("Error happened when trying to change into `etcd`");
+    tracing::info!("Changed to {}", env::current_dir().unwrap().display());
+
+    // Prepare directory to be sent.
+    let path = PathBuf::from("to_send/");
+    check_dir_exist_or_create(path);
+
+    tracing::info!("untaring docker binaries...");
+    Command::new("tar")
+        .arg("-zxf")
+        .arg("/rk8s/preparation/docker-20.10.9.tgz")
+        .status()
+        .expect("Error happened when trying to untar `docker` executable");
+
+    tracing::info!("Generating docker.service file to to_send/");
+    DockerUnit::generate();
+    tracing::info!("docker.service generated");
+
+    tracing::info!("Generating daemon.json to to_send/");
+    DockerCfg::generate();
+    tracing::info!("daemon.json generated");
+
+    for (ip, name) in &config.instance_hosts {
+        tracing::info!("Found instance {} on {},", name, ip);
+
+        Command::new("scp")
+            .arg("docker/containerd")
+            .arg("docker/containerd-shim")
+            .arg("docker/containerd-shim-runc-v2")
+            .arg("docker/ctr")
+            .arg("docker/docker")
+            .arg("docker/dockerd")
+            .arg("docker/docker-init")
+            .arg("docker/docker-proxy")
+            .arg("docker/runc")
+            // Send youki along with the process
+            .arg("/rk8s/preparation/youki")
+            .arg(format!("root@{}:/usr/bin", ip))
+            .status()
+            .expect("Error happened when trying to send files to other nodes");
+
+        Command::new("ssh")
+            .arg(format!("root@{}", ip))
+            .arg("mkdir /etc/docker")
+            .status()
+            .expect("Error happened when trying to create config directory on other nodes");
+
+        Command::new("scp")
+            .arg("to_send/daemon.json")
+            .arg(format!("root@{}:/etc/docker", ip))
+            .status()
+            .expect("Error happened when trying to send files to other nodes");
+
+        Command::new("scp")
+            .arg("to_send/docker.service")
+            .arg(format!("root@{}:/usr/lib/systemd/system/", ip))
+            .status()
+            .expect("Error happened when trying to send files to other nodes");
+
+        tracing::info!("Docker installed on {}, starting...", name);
+        Command::new("ssh")
+            .arg(format!("root@{}", ip))
+            .arg("systemctl daemon-reload")
+            .status()
+            .expect("Error happened when trying to reload systemd daemons");
+        Command::new("ssh")
+            .arg(format!("root@{}", ip))
+            .arg("systemctl start docker")
+            .status()
+            .expect("Error happened when trying to start docker");
+        Command::new("ssh")
+            .arg(format!("root@{}", ip))
+            .arg("systemctl enable docker")
+            .status()
+            .expect("Error happened when trying to enable docker");
+        tracing::info!("Docker started on {}", ip);
+    }
+
+    env::set_current_dir(&prev_dir).expect("Error happened when trying to change into `etcd`");
+    tracing::info!(
+        "Change working directory back to {}",
+        env::current_dir().unwrap().display()
+    );
+}
+
+fn check_dir_exist_or_create(path: PathBuf) {
+    if !path.is_dir() {
+        fs::create_dir_all(path).expect("Error happened when trying to create path");
+    }
+}