#!/usr/local/python3 # -*- encoding: utf-8 -*- """ @CreatedTime: 2019/12/16 @author: @File: ClusterDeployment @Editor: Pycharm @ProjectName: K8S auto installer @Version: 1.0 @License: Copyright © Huawei Technologies Co., Ltd. 2019-2024 """ import os import sys import paramiko from time import localtime, strftime class K8SInstaller: def __init__(self): self.phases = {"Init": 0, "DockerInstalled": 1, "RPMInstalled": 2, "ImagesLoaded": 3, "MasterInited": 4, "NodesJoined": 5} # temporary deploy parameters self.params = {"ExecuteIP": "170.41.152.6", "MasterList": ["170.41.152.5"], "NodeList": ["170.41.152.4"], "username": "root", "password": "huawei123", "workpath": "/root/", "packpath": "/root/", "docker": {"static": "docker-18.09.8.tgz", "script": "docker-install.sh"}} # connect to deploy machine self.execute = paramiko.SSHClient() self.execute.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # self.lwe_records = [] # command for join the node self.kubejoincmd = "" try: self.execute.connect(self.params["ExecuteIP"], username=self.params["username"], password=self.params['password']) except Exception as e: print(str(e)) self.error("Failed to init.") self.failure() self.teardown() '''deployment functions''' # some pre setting to target hosts def hostpresetting(self, targetIP: str): # open a temp ssh_client target = self.sshclient(targetIP) # close selinux commands = ["setenforce 0", "sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux", "systemctl stop firewalld", "systemctl disable firewalld", "sysctl -w net.bridge.bridge-nf-call-iptables=1", "echo \"net.bridge.bridge-nf-call-iptables=1\" > /etc/sysctl.d/k8s.conf", "swapoff -a", "cp -p /etc/fstab /etc/fstab.bak$(date '+%Y%m%d%H%M%S')", "sed -i \"s/\\/dev\\/mapper\\/centos-swap/\\#\\/dev\\/mapper\\/centos-swap/g\" /etc/fstab"] for cmd in commands: self.sendcommand(cmd, target, errIgnore = True) target.close() # transfer needed packs into target hosts def pactrans(self, targetIP: str): target = self.sshclient(targetIP) transcmd = "scp " + self.params["packpath"] + "remoteparts.zip " \ + self.params['username'] + '@' + targetIP \ + ':' + self.params['workpath'] self.sendcommand(transcmd, self.execute) unzipcmd = "unzip -o " + self.params['workpath'] + "remoteparts.zip" self.sendcommand(unzipcmd, target, errIgnore = True) target.close() # install docker def dockerinstall(self, targetIP: str): target = self.sshclient(targetIP) packpath = self.params["workpath"] + "remoteparts/docker/" + self.params["docker"]["static"] scriptpath = self.params["workpath"] + "remoteparts/docker/" + self.params["docker"]["script"] commands = ["chmod 777 " + scriptpath, "/usr/bin/bash " + scriptpath + " " + packpath] for cmd in commands: self.sendcommand(cmd, target, errIgnore = True) # checking status self.sendcommand("systemctl status docker", target, keyword_check = "active (running)") target.close() # install/uninstall rpm packages # op set to i as install, e as uninstall def rpminstall(self, targetIP: str, op = "i"): target = self.sshclient(targetIP) packpath = self.params["workpath"] + "remoteparts/K8Srpm/" stout = self.sendcommand("ls " + packpath, target) packlist = " ".join("".join(stout).split('\n')) if op == 'e': packlist = packlist.replace(".rpm", "") cmd = "cd " + packpath + " && rpm -" + op + " " + packlist + " --force --nodeps" self.sendcommand(cmd , target, errIgnore = True) # enable kubelet if op == 'i': self.sendcommand("systemctl enable kubelet", target, errIgnore = True) target.close() # load K8S fundamental docker images def imagesload(self, targetIP: str): target = self.sshclient(targetIP) imagespath = self.params["workpath"] + "remoteparts/K8Simages/" commands = ["ls " + imagespath + "*.rar > K8Simages.txt", "for i in `cat K8Simages.txt`; do docker load -i $i; done"] for cmd in commands: self.sendcommand(cmd, target) self.sendcommand("docker images", target, keyword_check = "k8s.gcr.io/pause") target.close() # init master def initmaster(self, masterIP: str): master = self.sshclient(masterIP) self.sendcommand("kubeadm reset -f ", master, errIgnore = True) #stdin, stdout, stderr = master.exec_command("kubeadm init --pod-network-cidr=10.244.0.0/16") #initresult = stdout.readlines() #initerr = stderr.readlines() initresult = self.sendcommand("kubeadm init --pod-network-cidr=10.244.0.0/16", master, errIgnore = True) #check result if "Your Kubernetes control-plane has initialized successfully!" in "".join(initresult): mastercmds = ["mkdir -p $HOME/.kube", "sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config", "sudo chown $(id -u):$(id -g) $HOME/.kube/config"] for cmd in mastercmds: self.sendcommand(cmd, master, errIgnore = True) for line in initresult: if "kubeadm join" in line and "--token" in line: x = line.replace('\\','').split()[-1] self.kubejoincmd = "kubeadm join " + masterIP + ":6443 --discovery-token-unsafe-skip-ca-verification --token " + x self.log("cluster join CMD:" + self.kubejoincmd ) # elif "/proc/sys/net/bridge/bridge-nf-call-iptables contents are not set to 1" in "".join(initresult): self.sendcommand("echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables", master) self.initmaster(masterIP) master.close() # join cluster def joincluster(self, nodeIP: str): node = self.sshclient(nodeIP) if self.kubejoincmd: self.sendcommand("kubeadm reset -f ", node, errIgnore = True) joinresult = self.sendcommand(self.kubejoincmd , node, errIgnore = True) # --discovery-token-unsafe-skip-ca-verification node.close() # get token def gettoken(self, masterIP:str): pass # show info def showinfo(self, masterIP:str): pass # networkProtocol def clusternetwork(self, masterIP: str): master = self.sshclient(masterIP) flannelPath = self.params["workpath"] + "remoteparts/kube-flannel.yml" self.sendcommand("kubectl apply -f " + flannelPath, master) master.close() '''integreted func''' # deploy def cluster_deployment(self): hosts = [] hosts.extend(self.params["MasterList"]) hosts.extend(self.params["NodeList"]) for host in hosts: self.pactrans(host) self.dockerinstall(host) self.hostpresetting(host) self.imagesload(host) self.rpminstall(host) for master in self.params["MasterList"]: self.initmaster(master) for node in self.params["NodeList"]: self.joincluster(node) for master in self.params["MasterList"]: self.clusternetwork(master) # debug def DEBUG_MODE(self): while True: cmd = input("TYPE TESTING CMD:").strip().split() hosts = [] hosts.extend(self.params["MasterList"]) hosts.extend(self.params["NodeList"]) if "EXIT" == cmd[0]: return elif "PRESET" == cmd[0]: for hostIP in hosts: self.hostpresetting(hostIP) elif "PACTRANS" == cmd[0]: for hostIP in hosts: self.pactrans(hostIP) elif "DOCKERINS" == cmd[0]: for hostIP in hosts: self.dockerinstall(hostIP) elif "IMGLOAD" == cmd[0]: for hostIP in hosts: self.imagesload(hostIP) elif "RPMINS" == cmd[0]: for hostIP in hosts: self.rpminstall(hostIP) elif "INIT" == cmd[0]: for masterIP in self.params["MasterList"]: self.initmaster(masterIP) elif "JOIN" == cmd[0]: for nodeIP in self.params["NodeList"]: self.joincluster(nodeIP) elif "NET" == cmd[0]: for masterIP in self.params["MasterList"]: self.clusternetwork(masterIP) elif "UINSRPM" == cmd[0]: for hostIP in hosts: self.rpminstall(hostIP, op='e') ''' support functions''' # ssh connect def sshclient(self, targetIP): self.log("Building a SSH connection to " + targetIP) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: ssh.connect(targetIP, username=self.params['username'], password=self.params['password']) self.log("Connected to" + targetIP) return ssh except Exception as e: self.error("Failed to connect to" + targetIP) self.error(str(e)) self.failure() self.teardown() # excute command # return command output list def sendcommand(self, cmd: str, ssh_client, keyword_check = "", errIgnore = False): def commandfailed(ssh): ssh.close() self.failure() self.teardown() self.log( "Executing: " + cmd) stdin, stout, stderr = ssh_client.exec_command(cmd) err = stderr.readlines() out = stout.readlines() if err: self.error("ErrorInfo From:" + cmd) self.error("".join(err)) if not errIgnore: commandfailed(ssh_client) return out else: self.log("command: " + cmd + " executed successfully.") self.log("".join(out)) if keyword_check != "": if keyword_check in "".join(out): self.log("key word " + keyword_check +" was found.") else: self.error("suppose to find \' " + keyword_check +"\' in [" + "".join(out) + "]. But there was found.") commandfailed(ssh_client) return out # log, warning, error def log(self, loginfo: str): print(strftime("%a, %d %b %Y %H:%M:%S +0000 ", localtime()) + loginfo) def warning(self, warninginfo: str): print(strftime("%a, %d %b %Y %H:%M:%S +0000 ", localtime()) + warninginfo) def error(self, errorinfo: str): print(strftime("%a, %d %b %Y %H:%M:%S +0000 ", localtime()) + errorinfo) # generate report def report(self): print("reporting") # when exception happens, call failure def failure(self): self.error("failure called") # finish the deployment def teardown(self): self.report() self.execute.close() sys.exit(-1) # Playground x = K8SInstaller() while True: cmd = input("choose your deploy mode: DEBUG for interactive, RUN for run through, EXIT for exit:").strip() if cmd == "DEBUG": x.DEBUG_MODE() break elif cmd == "RUN": x.cluster_deployment() break elif cmd == "EXIT": break else: print("Invalid command.")