diff --git a/.gitignore b/.gitignore index 28c0455..efd20a9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,5 @@ lib/__pycache__/ *.swp *.img *.etcd.token -kopsrox.k3stoken -kopsrox.kubeconfig -kopsrox_disk_import.log -kopsrox_imgpatch.log -kopsrox.etcd* +*.k3stoken +*.kubeconfig diff --git a/README.md b/README.md index 460f93a..ca31c42 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ -# :hamburger: kopsrox +# kopsrox -- kopsrox is a script to automate creation and management of simple ha k3s cluster on ProxmoxVE using cloud images :nerd_face: +- kopsrox is a script to create simple ha k3s cluster on ProxmoxVE +- use upstream cloud images - no iso's to mess around with - add more master/worker k3s nodes using a simple config file and cli interface :pray: -- kube-vip ( https://kube-vip.io/ ) built in providing full HA setup for the kube api :atom: +- kube-vip ( https://kube-vip.io/ ) built in providing full HA setup for the kube api and traefik :atom: - easy management of etcd S3 snapshot/restore operations - easily restore a cluster from s3! :floppy_disk: -- export the k3s token, your kubeconfig etc etc - its all automatic :nerd_face: +- export the k3s token, your kubeconfig etc etc - its all automatic :nerd_face: -# :book: docs + get it https://github.com/simonccc/kopsrox/releases + +# docs - [SETUP.md](docs/SETUP.md) - [GETSTARTED.md](docs/GETSTARTED.md) - [USAGE.md](docs/USAGE.md) - [FAQ.md](docs/FAQ.md) -# :boom: in progress - - Improving Docs - - Some code clean up - - Recent: bug fixes, machine type optimisations, kubevip improvements +# in progress + - Recent: add cluster restore option diff --git a/dev/helm.sh b/dev/helm.sh new file mode 100755 index 0000000..7cf89dd --- /dev/null +++ b/dev/helm.sh @@ -0,0 +1,2 @@ +c=`grep ^cluster_name kopsrox.ini | cut -d ' ' -f3` +helm --kubeconfig=${c}.kubeconfig ${@} diff --git a/dev/img-mirror/.gitignore b/dev/img-mirror/.gitignore index e56abd9..3fba00b 100644 --- a/dev/img-mirror/.gitignore +++ b/dev/img-mirror/.gitignore @@ -1,2 +1,4 @@ *.img *.qcow +log.txt +PID diff --git a/dev/img-mirror/get.sh b/dev/img-mirror/get.sh index 7db46fd..03a5301 100755 --- a/dev/img-mirror/get.sh +++ b/dev/img-mirror/get.sh @@ -1,15 +1,14 @@ # downloads image for use with serve.sh -ubr="oracular" -if [ ! -f "${ubr}-minimal-cloudimg-amd64.img" ] +if [ ! -f "noble-server-cloudimg-amd64.img" ] then -wget "https://cloud-images.ubuntu.com/minimal/daily/${ubr}/current/${ubr}-minimal-cloudimg-amd64.img" +wget "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" fi # untested -#if [ ! -f amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 ] -#then -#wget https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 -#fi +if [ ! -f amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 ] +then +wget https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 +fi #if [ ! -f Rocky-9-GenericCloud.latest.x86_64.qcow2 ] #then #wget https://mirrors.vinters.com/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 diff --git a/dev/img-mirror/serve.sh b/dev/img-mirror/serve.sh index 642106b..50868bc 100755 --- a/dev/img-mirror/serve.sh +++ b/dev/img-mirror/serve.sh @@ -1,2 +1,4 @@ #!/usr/bin/env bash -python3 -m http.server -b "::" +kill -9 $(cat PID) > /dev/null 2>&1 +python3 -m http.server -b "::" > log.txt 2>&1 & +echo $! > PID diff --git a/dev/rls_test.sh b/dev/rls_test.sh index 877511c..30075bb 100644 --- a/dev/rls_test.sh +++ b/dev/rls_test.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -e #set -x @@ -11,6 +13,7 @@ KCI="$KC info" KCC="$KC create" KCU="$KC update" KCD="$KC destroy" +KCR="$KC restore" KI="$K image" KID="$KI destroy" KIC="$KI create" @@ -30,69 +33,34 @@ kc() { # get pods get_pods="$KC kubectl get pods -A" -# 0 size cluster -kc workers 0 ; kc masters 1 +# 1 size cluster $KCD - -# create image -$KIC - -# create / update cluster -$KCC ; $KCU - -# take snapshot -$KES - -# destroy cluster +kc workers 0 ; kc masters 1 $KCD -# create / update cluster -$KCC ; $KCU - -# restore snapshot -$KERL +# ** 1 MASTER, SNAPSHOT RESTOR +# create image, create and update cluster +$KIC ; $KCC ; $KCU -# update cluster -$KCU +# take snapshot , destroy cluster, create, restore +$KES ; $KCD ; $KCC ; $KERL +# ** MULTIPLE MASTERS AND WORKERS TEST # add a worker and delete it kc workers 1 ; $KCU ; kc workers 0 ; $KCU # re add worker -kc workers 1 ; $KCU +kc workers 2 ; $KCU # add 3 masters and go back to 1 kc masters 3 ; $KCU ; kc masters 1 ; $KCU -# add 3 masters -kc masters 3 ; $KCU - -# take snapshot -$KES - -# destroy cluster -$KCD - -# create / update cluster -$KCC ; $KCU -# -# # restore snapshot -$KERL -# -# update cluster -$KCU - # change back to 1 node -kc masters 1 ; $KCU ; kc workers 0 ; $KCU - -# destroy cluster -$KCD - -# create / update cluster -$KCC ; $KCU -#restore snapshot -$KERL +kc masters 1 ; kc workers 0 +# ** TEST cluster restore +# destroy cluster +$KCD ; $KCR finish_time=$(date +%s) echo $((finish_time - start_time)) secs diff --git a/docs/FAQ.md b/docs/FAQ.md index e964254..11735da 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,4 +1,4 @@ -# :question: FAQ +# kopsrox FAQ :question: __can I migrate the kopsrox vms to other hosts in my proxmox cluster?__ diff --git a/docs/GETSTARTED.md b/docs/GETSTARTED.md index 17a74d8..c2e174b 100644 --- a/docs/GETSTARTED.md +++ b/docs/GETSTARTED.md @@ -1,6 +1,6 @@ -# :burger: get started +# get started -## 🇧🇸 setup kopsrox.ini +## setup a kopsrox.ini `./kopsrox.py` - a default kopsrox.ini will be created @@ -12,13 +12,13 @@ follow the guide in [SETUP.md](SETUP.md) `./kopsrox.py image create` -## 🥑 create a cluster +## create a cluster `./kopsrox.py cluster create` ## 🚑 add a worker -edit `kopsrox.ini` and set `workers = 1` in the `[cluster]` section +for example edit `kopsrox.ini` and set `workers = 1` in the `[cluster]` section `./kopsrox.py cluster update` diff --git a/docs/IMAGES.md b/docs/IMAGES.md new file mode 100644 index 0000000..8d70d4b --- /dev/null +++ b/docs/IMAGES.md @@ -0,0 +1,5 @@ +# cloud images + + +# amazon linux ( qcow ) +https://cdn.amazonlinux.com/al2023/os-images/2023.6.20250123.4/ diff --git a/docs/SETUP.md b/docs/SETUP.md index da66fc1..66a7955 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1,4 +1,4 @@ -# :hamburger: kopsrox setup +# kopsrox setup ## :hammer_and_wrench: requirements @@ -7,18 +7,12 @@ - a range of 10 free Proxmox qm/virtual machine id 'vmids' eg 600 to 610 - a range of 10 IP's on a network with internet access for kopsrox to work with eg 192.168.0.160 to 192.168.0.170 -## :bricks: install - ## :bricks: install -- clone the repo - or probably better one of the releases or stable branches - the 'main' branch can often be a bit broken -- sudo apt install libguestfs-tools python3-termcolor -y` +- get one of the releases or stable branches - the 'main' branch can often be a bit broken +- sudo apt install libguestfs-tools python3-termcolor python3-wget -y` - pip3 install --break-system-packages --user -r requirements.txt` -_installs the required pip packages vs using os packages_ - -( this will be improved in the future ) - ## :star: generate api key `sudo pvesh create /access/users/root@pam/token/kopsrox` @@ -29,9 +23,9 @@ _installs the required pip packages vs using os packages_ run `./kopsrox.py` and an example _kopsrox.ini_ will be generated - you will need to edit this for your setup -Most values should be obvious and commented accordingly - see below for more info +Most values should be hopefully obvious and commented accordingly... -# :mag_right: kopsrox.ini +# kopsrox.ini ## :computer: cluster_id @@ -66,18 +60,3 @@ would result in this: |9|629|192.168.0.179|worker 5|kopsrox-w5| The VIP IP ( here 192.168.0.170 ) is used by kube-vip to provide a highly available IP for the API when you have 3 master nodes - -## :pencil2: kopsrox - -### :rainbow: cloud_image_url - -`https://cloud-images.ubuntu.com/minimal/daily/mantic/current/mantic-minimal-cloudimg-amd64.img` - -url to the cloud image you want to use as the koprox base template. - -during `kopsrox.py image create` this is downloaded and patched via `virt-customise` to install `qemu-guest-agent` - -Tested images so far: - -https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 -https://mirrors.vinters.com/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 diff --git a/kopsrox.py b/kopsrox.py index 3fcad1d..dc47310 100755 --- a/kopsrox.py +++ b/kopsrox.py @@ -26,6 +26,7 @@ "create" : '', "update" : '', "destroy" : '', + 'restore' : '', }, "k3s": { "export-token" : '', @@ -48,9 +49,6 @@ "reboot" : 'hostname', "k3s-uninstall" : 'hostname', "rejoin-slave" : 'hostname', - }, - "kubevip": { - "reinstall": '', } } @@ -87,12 +85,12 @@ def cmds_help(verb): # if verb not found in cmds dict if not verb in verbs: - exit() + exit(0) # verb not found or passed except: verbs_help() - exit() + exit(0) # handle command try: @@ -105,7 +103,7 @@ def cmds_help(verb): if not cmd in list(cmds[verb]): exit() -# +# cmd not found except: cmds_help(verb) exit() diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py index 51f7f53..fac309d 100755 --- a/lib/kopsrox_config.py +++ b/lib/kopsrox_config.py @@ -1,29 +1,12 @@ #!/usr/bin/env python3 -# imports +# external imports import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -# proxmoxer api -from proxmoxer import ProxmoxAPI - -# prompt -kname = 'config-check' - -# look for strings in responses -import re - -# checks cmd line args - could be moved? -import sys - -# local os commands -import subprocess - -# used to encode ssh key import urllib.parse - -# datetime stuff for generating image date from datetime import datetime +from proxmoxer import ProxmoxAPI +import re,os,sys,subprocess,time,wget # kmsg from kopsrox_kmsg import kmsg @@ -75,28 +58,74 @@ def conf_check(section,value): # return string return(config_item) -# check config vars -# cluster name as required for error messages + +# cluster name cluster_name = conf_check('cluster', 'cluster_name') kname = cluster_name + '_config-check' -# get cluster id +# cluster id cluster_id = conf_check('cluster','cluster_id') +if cluster_id < 100: + kmsg(kname, f' cluster_id is too low - should be over 100', 'err') + exit(0) + +# assign master id +masterid = cluster_id + 1 + +# test connection to proxmox +try: -# proxmox -prox_endpoint = conf_check('proxmox','prox_endpoint') -port = conf_check('proxmox','port') -user = conf_check('proxmox','user') -token_name = conf_check('proxmox','token_name') -api_key = conf_check('proxmox','api_key') + # api connection + prox = ProxmoxAPI( + conf_check('proxmox','prox_endpoint'), + port=conf_check('proxmox','port'), + user=conf_check('proxmox','user'), + token_name=conf_check('proxmox','token_name'), + token_value=conf_check('proxmox','api_key'), + verify_ssl=False, + timeout=5) + + # check connection to cluster + prox.cluster.status.get() + +except: + kmsg(kname, f'API connection to Proxmox failed check [proxmox] settings', 'err') + exit(0) + +# map passed node name node = conf_check('proxmox','node') + +# try k8s ping +try: + k3s_ping = prox.nodes(node).qemu(masterid).agent.exec.post(command = '/usr/local/bin/k3s kubectl version') + #print(pingv) +except: + try: + qa_ping = prox.nodes(node).qemu(masterid).agent.ping.post() + kmsg(kname, f'k3s down but master up please investigate...', 'err') + exit(0) + except: + pass + +# proxmox cont +discovered_nodes = [node.get('node', None) for node in prox.nodes.get()] +if node not in discovered_nodes: + kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err') + exit(0) + +# storage storage = conf_check('proxmox','storage') -# kopsrox config checks +# kopsrox cloud_image_url = conf_check('kopsrox','cloud_image_url') vm_disk = conf_check('kopsrox','vm_disk') vm_cpu = conf_check('kopsrox','vm_cpu') + +# ram size and check vm_ram = conf_check('kopsrox','vm_ram') +if vm_ram < 2: + kmsg(kname, f'[kopsrox]/vm_ram - kopsrox vms need 2G RAM', 'err') + exit(0) # cloudinit cloudinituser = conf_check('kopsrox','cloudinituser') @@ -149,18 +178,6 @@ def conf_check(section,value): # dict of all config items - legacy support config = ({s:dict(kopsrox_config.items(s)) for s in kopsrox_config.sections()}) -# generated string to use in s3 commands -s3_string = \ -' --etcd-s3 ' + region_string + \ -' --etcd-s3-endpoint ' + s3endpoint + \ -' --etcd-s3-access-key ' + access_key + \ -' --etcd-s3-secret-key ' + access_secret + \ -' --etcd-s3-bucket ' + bucket + \ -' --etcd-s3-skip-ssl-verify ' - -# define masterid -masterid = cluster_id + 1 - # define vmnames vmnames = { (cluster_id): cluster_name +'-i0', @@ -175,29 +192,6 @@ def conf_check(section,value): (cluster_id + 9 ): cluster_name + '-w5', } -# proxmox api connection -try: - - # api connection - prox = ProxmoxAPI( - prox_endpoint, - port=port, - user=user, - token_name=token_name, - token_value=api_key, - verify_ssl=False, - timeout=5) - - # check connection to cluster - prox.cluster.status.get() - -except: - kmsg(kname, f'API connection to {prox_endpoint}:{port} failed check [proxmox] settings', 'err') - - # this could be improved - #kmsg(kname, prox.cluster.status.get(), 'sys') - exit() - # look up kopsrox_img name def kopsrox_img(): @@ -239,14 +233,6 @@ def list_kopsrox_vm(): def vm_info(vmid,node=node): return(prox.nodes(node).qemu(vmid).status.current.get()) -# get list of nodes -discovered_nodes = [node.get('node', None) for node in prox.nodes.get()] - -# if node not in list of nodes -if node not in discovered_nodes: - kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err') - exit() - # get list of storage in the cluster storage_list = prox.nodes(node).storage.get() @@ -304,7 +290,6 @@ def vm_info(vmid,node=node): # dummy cloud_image_vars overwritten below cloud_image_size = 0 cloud_image_desc = '' -cloud_image_created = '' # skip image check if image create is passed try: @@ -342,7 +327,6 @@ def vm_info(vmid,node=node): # get image created and desc from template template_data = prox.nodes(node).qemu(cluster_id).config.get() - cloud_image_created = str(datetime.fromtimestamp(int(template_data['meta'].split(',')[1].split('=')[1]))) cloud_image_desc = template_data['description'] except: @@ -378,25 +362,6 @@ def vmip(vmid: int): ip = f'{network_base}{(network_ip_prefix + (vmid - cluster_id))}' return(ip) -# cluster info -def cluster_info(): - kmsg(f'cluster_info', '', 'sys') - from kopsrox_k3s import kubectl, get_kube_vip_master - curr_master = get_kube_vip_master() - info_vms = list_kopsrox_vm() - - # for kopsrox vms - for vmid in info_vms: - if not cluster_id == vmid: - hostname = vmnames[vmid] - vmstatus = f'[{info_vms[vmid]}] {vmip(vmid)}/{network_mask}' - if hostname == curr_master: - vmstatus += f' vip {network_ip}/{network_mask}' - kmsg(f'{hostname}_{vmid}', f'{vmstatus}') - - # fix this - kmsg('kubectl_get-nodes', f'\n{kubectl("get nodes")}') - # run local os process def local_os_process(cmd): try: @@ -415,5 +380,13 @@ def image_info(): kname = f'image_' kmsg(f'{kname}desc', cloud_image_desc) kmsg(f'{kname}storage', f'{kopsrox_image_name} ({storage_type})') - kmsg(f'{kname}created', cloud_image_created) kmsg(f'{kname}size', f'{cloud_image_size}G') + +# tbc +def progress_bar(iteration, total, prefix='', suffix='', length=30, fill='^'): + percent = ("{0:.1f}").format(100 * (iteration / float(total))) + filled_length = int(length * iteration // total) + bar = fill * filled_length + '-' * (length - filled_length) + sys.stdout.write(f'\r{prefix} |{bar}| {percent}% {suffix}') + sys.stdout.flush() + diff --git a/lib/kopsrox_ini.py b/lib/kopsrox_ini.py index 587df38..6ea398e 100755 --- a/lib/kopsrox_ini.py +++ b/lib/kopsrox_ini.py @@ -14,11 +14,11 @@ def init_kopsrox_ini(): config.add_section('proxmox') # proxmox api endpoint - config.set('proxmox', '; domain name or IP to access proxmox') + config.set('proxmox', '; domain or IP to access proxmox') config.set('proxmox', 'prox_endpoint', '127.0.0.1') # proxmox api port - config.set('proxmox', '; port is usually 8006') + config.set('proxmox', '; api port ( usually 8006 ) ') config.set('proxmox', 'port', '8006') # username @@ -38,7 +38,7 @@ def init_kopsrox_ini(): config.set('proxmox', 'node', 'proxmox') # storage on node - config.set('proxmox', '; the proxmox storage to use for kopsrox - needs to be available on the proxmox host') + config.set('proxmox', '; the proxmox storage to use for kopsrox - needs to be available on the proxmox node') config.set('proxmox', 'storage', 'local-lvm') # kopsrox section @@ -46,10 +46,10 @@ def init_kopsrox_ini(): # upstream image config.set('kopsrox', '; the upstream cloud image used to create the kopsrox image') - config.set('kopsrox', 'cloud_image_url', '//cloud-images.ubuntu.com/minimal/daily/oracular/current/oracular-minimal-cloudimg-amd64.img') + config.set('kopsrox', 'cloud_image_url', 'https://cloud-images.ubuntu.com/minimal/daily/oracular/current/oracular-minimal-cloudimg-amd64.img') # disk size for kopsrox vms - config.set('kopsrox', '; size of kopsrox vm disk in Gib ') + config.set('kopsrox', '; size of vm disk in Gib ') config.set('kopsrox', 'vm_disk', '20') # number of cpu cores @@ -65,11 +65,11 @@ def init_kopsrox_ini(): config.set('kopsrox', 'cloudinituser', 'user') # cloud init user password - config.set('kopsrox', '; the password for the cloudinit user') + config.set('kopsrox', '; password for the cloudinit user') config.set('kopsrox', 'cloudinitpass', 'admin') # cloud init user ssh key - config.set('kopsrox', '; the ssh public key for the cloudinit user') + config.set('kopsrox', '; a ssh public key for the cloudinit user ( required ) ') config.set('kopsrox', 'cloudinitsshkey', 'ssh-rsa cioieocieo') # network bridge @@ -79,7 +79,6 @@ def init_kopsrox_ini(): # kopsrox network baseip config.set('kopsrox', '; first ip of the ip range used for this kopsrox cluster') - config.set('kopsrox', '; this ip is assigned to the template - the first master is this + 1') config.set('kopsrox', 'network_ip', '192.168.0.160') # netmask / subnet @@ -87,7 +86,7 @@ def init_kopsrox_ini(): config.set('kopsrox', 'network_mask', '24') # default gateway - config.set('kopsrox', '; default gateway for the kopsrox network ( needs to provide internet access ) ') + config.set('kopsrox', '; default gateway for the network ( needs to provide internet access ) ') config.set('kopsrox', 'network_gw', '192.168.0.1') # dns server @@ -96,17 +95,18 @@ def init_kopsrox_ini(): # network mtu config.set('kopsrox', '; interface mtu set on vms ') + config.set('kopsrox', '; set to 1450 if using sdn ') config.set('kopsrox', 'network_mtu', '1500') # cluster section config.add_section('cluster') # cluster vmid - config.set('cluster', '; id for the cluster vms.. eg from 620 - 630') + config.set('cluster', '; id for the cluster vm\'s eg from 620 - 630') config.set('cluster', 'cluster_id', '620') # cluster friendly name - config.set('cluster', '; name for the kopsrox cluster') + config.set('cluster', '; name of the cluster') config.set('cluster', 'cluster_name', 'mycluster') # number of masters diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py index c459ee9..f413f44 100755 --- a/lib/kopsrox_k3s.py +++ b/lib/kopsrox_k3s.py @@ -1,14 +1,8 @@ #!/usr/bin/env python3 # imports -from kopsrox_config import masterid, k3s_version, masters, workers, cluster_name, vmnames, vmip, cluster_info, list_kopsrox_vm, network_ip - -# standard imports -from kopsrox_proxmox import qaexec, prox_destroy, internet_check, clone -from kopsrox_kmsg import kmsg - -# standard imports -import re, time, os +from kopsrox_config import * +from kopsrox_proxmox import * # check for k3s status def k3s_check(vmid: int): @@ -33,7 +27,7 @@ def k3s_check(vmid: int): def k3s_init_node(vmid: int = masterid,nodetype = 'master'): # nodetype error check - if nodetype not in ['master', 'slave', 'worker']: + if nodetype not in ['master', 'slave', 'worker', 'restore']: kmsg('k3s_init-node', f'{nodetype} invalid nodetype', 'err') exit(0) @@ -45,10 +39,9 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'): exit(0) # defines - k3s_install_base = f'cat /k3s.sh | INSTALL_K3S_VERSION="{k3s_version}"' - k3s_install_flags = f' --disable servicelb --tls-san {network_ip}' - k3s_install_master = f'{k3s_install_base} sh -s - server --cluster-init {k3s_install_flags}' - k3s_install_worker = f'{k3s_install_base} K3S_URL="https://{network_ip}:6443" ' + k3s_install_master = f'cat /k3s.sh | sh -s - server --cluster-init' + k3s_install_worker = f'cat /k3s.sh | K3S_URL="https://{network_ip}:6443" ' + master_cmd = '' token = '' @@ -57,7 +50,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'): if not k3s_check(vmid): exit(0) except: - kmsg(f'k3s_{nodetype}-init', f'installing {k3s_version} on {vmnames[vmid]}') + kmsg(f'k3s_{nodetype}-init', f'configuring {k3s_version} on {vmnames[vmid]}') # get existing token if it exists token_fname = f'{cluster_name}.k3stoken' @@ -75,13 +68,29 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'): # slave if nodetype == 'slave': - init_cmd = f'{k3s_install_base}{k3s_token_cmd} sh -s - server --server https://{network_ip}:6443 {k3s_install_flags}' + init_cmd = f'cat /k3s.sh | sh -s - server --server https://{network_ip}:6443 {master_cmd}' # worker if nodetype == 'worker': - init_cmd = f'{k3s_install_worker}{k3s_token_cmd} sh -s' + init_cmd = f'rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s' + + # restore + if nodetype == 'restore': + kmsg(f'k3s_bootstrap', f'fetching latest snapshot') + + # get latest snapshot + bs_cmd = f'{k3s_install_master} {master_cmd} && /usr/local/bin/k3s etcd-snapshot ls 2>&1 && systemctl stop k3s && rm -rf /var/lib/rancher' + bs_cmd_out = qaexec(vmid,bs_cmd) - # stderr + # sort ls output so last is latest snapshot + for snap in sorted(bs_cmd_out.split('\n')): + if re.search(f'kopsrox-{cluster_name}', snap): + latest = snap.split()[0] + + kmsg(f'k3s_restore', f'restoring {latest}') + init_cmd = f'/usr/local/bin/k3s server --cluster-reset --cluster-reset-restore-path={latest} --token={token} 2>&1 && systemctl start k3s' + + # write log of install on node init_cmd = init_cmd + f' > /k3s_{nodetype}_install.log 2>&1' # run command @@ -106,21 +115,21 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'): time.sleep(1) # final steps for first master - kubevip, export kubeconfig and token - if nodetype == 'master': - install_kube_vip() + if nodetype in ['master', 'restore']: kubeconfig() export_k3s_token() # remove a node def k3s_remove_node(vmid: int): + + # get vmname vmname = vmnames[vmid] kmsg('k3s_remove-node', vmname) - # kubectl commands to remove node - # should add some error checking - kubectl('cordon ' + vmname) - kubectl('drain --timeout=10s --delete-emptydir-data --ignore-daemonsets --force ' + vmname) - kubectl('delete node ' + vmname) + if vmname != f'{cluster_name}-m1': + kubectl('cordon ' + vmname) + kubectl('drain --timeout=10s --delete-emptydir-data --ignore-daemonsets --force ' + vmname) + kubectl('delete node ' + vmname) # destroy vm prox_destroy(vmid) @@ -222,13 +231,14 @@ def k3s_update_cluster(): # kubeconfig def kubeconfig(): + + # define filename + kubeconfig_file = f'{cluster_name}.kubeconfig' # replace 127.0.0.1 with vip ip kconfig = qaexec(masterid, 'cat /etc/rancher/k3s/k3s.yaml').replace('127.0.0.1', network_ip) - - # write file out - with open(f'{cluster_name}.kubeconfig','w') as kfile: - kfile.write(kconfig) - kmsg('k3s_kubeconfig', ('saved ' + (cluster_name +'.kubeconfig'))) + with open(kubeconfig_file, 'w') as kubeconfig_file_handle: + kubeconfig_file_handle.write(kconfig) + kmsg('k3s_kubeconfig', f'saved {kubeconfig_file}') # kubectl def kubectl(cmd): @@ -248,7 +258,6 @@ def export_k3s_token(): # define token file name token_name = f'{cluster_name}.k3stoken' - # get masters token live_token = qaexec(masterid, 'cat /var/lib/rancher/k3s/server/token') @@ -279,28 +288,24 @@ def export_k3s_token(): token_file.write(live_token) kmsg('k3s_export-token', f'created: {token_name}') -# install kube vip -def install_kube_vip(): - - # read default kube vip manifest and replace with network_ip - kv_manifest = open('./lib/kubevip/kubevip.yaml', "r").read().replace('KOPSROX_IP', network_ip).strip() - - # create the manifest - qaexec(masterid, f'''cat < /tmp/kubevip.yaml -{kv_manifest} -EOF -''') - - # apply / replace the manifest - kubevip_install = kubectl('replace --force -f /tmp/kubevip.yaml') - - # check output - if not re.search('daemonset.apps/kubevip replaced', kubevip_install): - kmsg('k3s_kubevip', f'failed to install kube-vip\n{kubevip_install}', 'err') - exit(0) - - # install completed - kmsg('k3s_kubevip', f'{network_ip} vip active') +# cluster info +def cluster_info(): + + kmsg(f'cluster_info', '', 'sys') + curr_master = get_kube_vip_master() + info_vms = list_kopsrox_vm() + + # for kopsrox vms + for vmid in info_vms: + if not cluster_id == vmid: + hostname = vmnames[vmid] + vmstatus = f'[{info_vms[vmid]}] {vmip(vmid)}/{network_mask}' + if hostname == curr_master: + vmstatus += f' vip {network_ip}/{network_mask}' + kmsg(f'{hostname}_{vmid}', f'{vmstatus}') + + # fix this + kmsg('kubectl_get-nodes', f'\n{kubectl("get nodes")}') # return current vip master def get_kube_vip_master(): @@ -311,8 +316,3 @@ def get_kube_vip_master(): except: kubevip_m = '' return(kubevip_m) - -# reload kubevip -def kubevip_reload(): - reload = kubectl('rollout restart daemonset kubevip -n kube-system') - print(reload) diff --git a/lib/kopsrox_kmsg.py b/lib/kopsrox_kmsg.py index 6ea7fb3..a0fe1e0 100755 --- a/lib/kopsrox_kmsg.py +++ b/lib/kopsrox_kmsg.py @@ -11,7 +11,8 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'): # print cluster name cprint(knamea[0], "blue",attrs=["bold"], end='') - cprint(':', "cyan", end='') + cprint('-', "magenta",attrs=["bold"], end='') + cprint('<:', "cyan", end='') try: if knamea[1] and sev == 'info': @@ -21,9 +22,9 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'): if knamea[1] and sev == 'sys': cprint(knamea[1], "yellow", attrs=["bold"],end='') except: - cprint('parse error ', "magenta", attrs=["bold"], end='') - print(kname,msg) + cprint('parse error ', "magenta", attrs=["bold"], end='') + print(kname,msg) # final output - cprint(': ', "cyan", end='') + cprint(':> ', "cyan", end='') print(msg) diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py index e8d7326..e024c82 100755 --- a/lib/kopsrox_proxmox.py +++ b/lib/kopsrox_proxmox.py @@ -1,26 +1,21 @@ #!/usr/bin/env python3 -# imports -import time, re - # kopsrox -from kopsrox_config import prox, vmip, masterid -from kopsrox_config import node,network_bridge,cluster_id,vmnames,vm_cpu,vm_ram,vm_disk,network_mask,network_gw,network_dns,network_mtu, network_dns +from kopsrox_config import * from kopsrox_kmsg import kmsg # run a exec via qemu-agent -def qaexec(vmid: int = masterid,cmd = 'uptime'): +def qaexec(vmid: int = masterid,cmd = 'uptime', node: str = node): # define kname kname = 'proxmox_qaexec' - # get vmname - # fixme try? + # get vmname and node vmname = vmnames[vmid] - - # get node - # fixme try? - node = get_node(vmid) + try: + node = vms[vmid] + except: + pass # qagent no yet running check qagent_running = 'false' @@ -51,6 +46,10 @@ def qaexec(vmid: int = masterid,cmd = 'uptime'): # sleep 1 second then try again time.sleep(1) + #progress_bar(qagent_count, 30, prefix='', suffix='') + if qagent_count == 10: + kmsg(kname, f'no response for 10s {vmname} [{node}] cmd: {cmd}', 'sys') + # send command try: qa_exec = prox.nodes(node).qemu(vmid).agent.exec.post( @@ -106,52 +105,27 @@ def qaexec(vmid: int = masterid,cmd = 'uptime'): except: return('no output-' + cmd) -# return the node for a vmid -def get_node(vmid): - - # if it exists node is ok and we can return the configured node - if vmid == cluster_id: - return(node) - - # check for vm id in proxmox cluster - for vm in prox.cluster.resources.get(type = 'vm'): - - # matching id found - if vm.get('vmid') == vmid: - - # return node vm is running on - return(vm.get('node')) - - # error: node not found - if vmid == masterid: - kmsg('cluster_error', f'no cluster exists', 'err') - exit(0) - - # default error - kmsg('proxmox_get-node', f'node for {vmid} not found', 'err') - exit(0) - # stop and destroy vm -def prox_destroy(vmid): +def prox_destroy(vmid: int): kname = 'prox_destroy-vm' # get node and vmname vmname = vmnames[vmid] - node = get_node(vmid) + node = vms[vmid] # if destroying image if vmid == cluster_id: - task_status(prox.nodes(node).qemu(cluster_id).delete()) + prox_task(prox.nodes(node).qemu(cluster_id).delete()) return # power off and delete try: - task_status(prox.nodes(node).qemu(vmid).status.stop.post()) - task_status(prox.nodes(node).qemu(vmid).delete()) + prox_task(prox.nodes(node).qemu(vmid).status.stop.post(),node) + prox_task(prox.nodes(node).qemu(vmid).delete(),node) kmsg(kname, vmname) - except: - kmsg(kname, f'unable to destroy {vmid}', 'err') + except Exception as e: + kmsg(kname, f'unable to destroy {node}/{vmid}\n{e}', 'err') exit(0) # clone @@ -168,13 +142,13 @@ def clone(vmid): # hostname hostname = vmnames[vmid] - kmsg('proxmox_clone', f'{hostname} {ip} {vm_cpu}c/{vm_ram}G ram {vm_disk}G disk') + kmsg('proxmox_clone', f'{hostname} {ip} {vm_cpu}c/{vm_ram}G {vm_disk}G') # clone - task_status(prox.nodes(node).qemu(cluster_id).clone.post(newid = vmid)) + prox_task(prox.nodes(node).qemu(cluster_id).clone.post(newid = vmid)) # configure - task_status(prox.nodes(node).qemu(vmid).config.post( + prox_task(prox.nodes(node).qemu(vmid).config.post( name = hostname, onboot = 1, cores = vm_cpu, @@ -188,27 +162,30 @@ def clone(vmid): )) # resize disk - task_status(prox.nodes(node).qemu(vmid).resize.put( + prox_task(prox.nodes(node).qemu(vmid).resize.put( disk = 'scsi0', size = f'{vm_disk}G', )) # power on - task_status(prox.nodes(node).qemu(vmid).status.start.post()) + prox_task(prox.nodes(node).qemu(vmid).status.start.post()) # run uptime / wait for qagent to start internet_check(vmid) - kmsg(f'proxmox_{hostname}', 'ready') # proxmox task blocker -def task_status(task_id, node=node): +def prox_task(task_id, node=node): # define default status status = {"status": ""} # until task stopped - while (status["status"] != "stopped"): - status = prox.nodes(node).tasks(task_id).status.get() + try: + while (status["status"] != "stopped"): + status = prox.nodes(node).tasks(task_id).status.get() + except: + kmsg('proxmox_task-status', f'unable to get task {task_id} node: {node}', 'err') + exit(0) # if task not completed ok if not status["exitstatus"] == "OK": @@ -222,10 +199,17 @@ def task_log(task_id, node=node): logline = '' # for each value in list - for log in prox.nodes(node).tasks(task_id).log.get(): + # assuming task_id is valid + try: + for log in prox.nodes(node).tasks(task_id).log.get(): + + # append log to logline + logline += log['t'] + '\n' - # append log to logline - logline += log['t'] + '\n' + return(logline) + except: + kmsg('proxmox_task-log', f'failed to get log for task!', 'err') + exit(0) # return string return(logline) diff --git a/lib/verb_cluster.py b/lib/verb_cluster.py index ce6dc18..9d89d0d 100755 --- a/lib/verb_cluster.py +++ b/lib/verb_cluster.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # functions -from kopsrox_config import masterid,cluster_name,cluster_info,list_kopsrox_vm,cluster_id +from kopsrox_config import * from kopsrox_proxmox import clone,qaexec -from kopsrox_k3s import k3s_update_cluster,k3s_rm_cluster,k3s_init_node,export_k3s_token -from kopsrox_kmsg import kmsg - -# other imports -import sys +from kopsrox_k3s import * # passed command cmd = sys.argv[2] @@ -23,6 +19,16 @@ if cmd == 'update': k3s_update_cluster() +# restore from latest etcd snapshot +if cmd == 'restore': + k3s_rm_cluster() + kmsg(kname,f'id:{cluster_id} name:{cluster_name}', 'sys') + clone(masterid) + k3s_init_node(masterid, 'restore') + cluster_info() + kmsg(kname,f'restore completed') + k3s_update_cluster() + # create new cluster / master server if cmd == 'create': diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py index af451d1..b2c0f51 100755 --- a/lib/verb_etcd.py +++ b/lib/verb_etcd.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 -# standard imports -import sys, re, os - # kopsrox -from kopsrox_config import masterid, masters, workers, cluster_name, s3_string, bucket, s3endpoint -from kopsrox_proxmox import get_node, qaexec -from kopsrox_k3s import k3s_rm_cluster, kubectl, k3s_update_cluster, export_k3s_token, kubeconfig -from kopsrox_kmsg import kmsg +from kopsrox_config import * +from kopsrox_proxmox import * +from kopsrox_k3s import * # passed command cmd = sys.argv[2] @@ -17,9 +13,8 @@ token_fname = cluster_name + '.k3stoken' # check master is running / exists -# fails if node can't be found try: - get_node(masterid) + node = vms[masterid] except: kmsg(f'{kname}-check', 'cluster does not exist', 'err') exit(0) @@ -28,7 +23,7 @@ def s3_run(s3cmd): # run the command ( 2>&1 required ) - k3s_run = f'k3s etcd-snapshot {s3cmd} {s3_string} 2>&1' + k3s_run = f'k3s etcd-snapshot {s3cmd} 2>&1' cmd_out = qaexec(masterid, k3s_run) # look for fatal error in output @@ -61,6 +56,12 @@ def list_snapshots(): # test connection to s3 by getting list of snapshots snapshots = list_snapshots() +try: + node = vms[masterid] +except: + kmsg(f'{kname}-check', 'cluster does not exist', 'err') + exit(0) + # s3 prune if cmd == 'prune': kmsg(f'{kname}-prune', (f'{s3endpoint}/{bucket}\n' + s3_run('prune --name kopsrox')), 'sys') @@ -74,14 +75,13 @@ def list_snapshots(): export_k3s_token() # define snapshot command - snap_cmd = f'k3s etcd-snapshot save {s3_string} --name kopsrox --etcd-snapshot-compress 2>&1' - #print(snap_cmd) + snap_cmd = f'k3s etcd-snapshot save --name kopsrox 2>&1' snapout = qaexec(masterid,snap_cmd) # filter output snapout = snapout.split('\n') for line in snapout: - if re.search('upload complete', line): + if re.search('Snapshot', line): kmsg(kname, line) # print s3List @@ -135,7 +135,7 @@ def s3_list(): # define restore command restore_cmd = f'\ systemctl stop k3s && \ -k3s server --cluster-reset --cluster-reset-restore-path={snapshot} --token={token} {s3_string} 2>&1 ; \ +k3s server --cluster-reset --cluster-reset-restore-path={snapshot} --token={token} 2>&1 ; \ systemctl start k3s' # display some filtered restore contents diff --git a/lib/verb_image.py b/lib/verb_image.py index 0013223..6803962 100755 --- a/lib/verb_image.py +++ b/lib/verb_image.py @@ -1,19 +1,10 @@ #!/usr/bin/env python3 # functions -from kopsrox_config import prox, local_os_process, image_info, cloud_image_desc, kopsrox_img, k3s_version - -# variables -from kopsrox_config import node, storage, cluster_id, cloud_image_url, cluster_name, cloudinitsshkey, cloudinituser, cloudinitpass - -# general imports -import wget,sys,os +from kopsrox_config import * # proxmox functions -from kopsrox_proxmox import task_status, prox_destroy - -# kmsg -from kopsrox_kmsg import kmsg +from kopsrox_proxmox import prox_task, prox_destroy # define command cmd = sys.argv[2] @@ -28,8 +19,14 @@ # check if image already exists if os.path.isfile(cloud_image): - kmsg(f'{kname}check', f'Error! {cloud_image} already exists - please delete', 'err') - exit(0) + kmsg(f'image_check', f'{cloud_image} already exists - removing', 'warn') + try: + os.remove(cloud_image) + if os.path.isfile(cloud_image): + exit(0) + except: + kmsg(f'{kname}check', f'{cloud_image} cannot delete', 'err') + exit(0) # check img can be downloaded try: @@ -40,14 +37,35 @@ kmsg(f'{kname}check', f'unable to download {cloud_image_url}', 'err') exit(0) - # script to install disable selinux on Rocky + # kubevip + # open the generic kubevip deployment and patch it with our network_ip in memory + kv_manifest = open('./lib/kubevip/kubevip.yaml', 'r').read().replace('KOPSROX_IP', network_ip).strip() + + # define the name/location of the patched kubevip + kv_yaml = f'./lib/kubevip/{cluster_name}-kubevip.yaml' + + # open the new kubevip path + kv_write_out = open(kv_yaml, 'w') + + # write the patched in memory version to file + kv_write_out.write(kv_manifest) + + # close files + kv_write_out.close() + + # script to run in kopsrox image virtc_script = f'''\ curl -v https://get.k3s.io > /k3s.sh -cat /k3s.sh | \ -INSTALL_K3S_SKIP_ENABLE=true \ -INSTALL_K3S_SKIP_SELINUX_RPM=true \ -INSTALL_K3S_VERSION={k3s_version} \ -sh -s - server --cluster-init > /k3s-image-install.log 2>&1 + +if [ ! -f /usr/bin/qemu-ga ] +then + if [ -f /bin/yum ] + then + yum install -y qemu-guest-agent + else + apt update && apt install qemu-guest-agent -y + fi +fi if [ -f /etc/selinux/config ] then @@ -57,11 +75,48 @@ if [ -f /etc/sysconfig/qemu-ga ] then cp /dev/null /etc/sysconfig/qemu-ga -fi''' - # shouldn't really need root but run into permissions problems - virtc_cmd = f'sudo virt-customize --smp 2 -m 2048 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"' - +fi + +mkdir -p /var/lib/rancher/k3s/server/manifests/ +echo ' +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + service: + spec: + loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml + +mkdir -p /etc/rancher/k3s/config.yaml.d/ +echo -n ' +etcd-s3: true +etcd-snapshot-retention: 14 +etcd-s3-region: {region_string} +etcd-s3-endpoint: {s3endpoint} +etcd-s3-access-key: {access_key} +etcd-s3-secret-key: {access_secret} +etcd-s3-bucket: {bucket} +etcd-s3-skip-ssl-verify: true +etcd-snapshot-compress: true' > /etc/rancher/k3s/config.yaml.d/etcd-backup.yaml + +echo -n ' +disable: + - servicelb +disable-network-policy: true +disable-cloud-controller: true +disable-network-policy: true +flannel-backend: wireguard-native +tls-san: {network_ip}' > /etc/rancher/k3s/config.yaml +''' + # shouldn't really need root/sudo but run into permissions problems kmsg(f'{kname}virt-customize', 'configuring image') + virtc_cmd = f''' +sudo virt-customize -a {cloud_image} \ +--run-command "{virtc_script}" \ +--copy-in {kv_yaml}:/var/lib/rancher/k3s/server/manifests/''' local_os_process(virtc_cmd) # destroy template if it exists @@ -70,8 +125,21 @@ except: pass + # define image desc + img_ts = str(datetime.now()) + image_desc = f'''
+▗▖ ▗▖ ▗▄▖ ▗▄▄▖  ▗▄▄▖▗▄▄▖  ▗▄▖ ▗▖  ▗▖
+▐▌▗▞▘▐▌ ▐▌▐▌ ▐▌▐▌   ▐▌ ▐▌▐▌ ▐▌ ▝▚▞▘ 
+▐▛▚▖ ▐▌ ▐▌▐▛▀▘  ▝▀▚▖▐▛▀▚▖▐▌ ▐▌  ▐▌  
+▐▌ ▐▌▝▚▄▞▘▐▌   ▗▄▄▞▘▐▌ ▐▌▝▚▄▞▘▗▞▘▝▚▖
+
+cluster_name: {cluster_name}
+cloud_img: {cloud_image}
+k3s_version: {k3s_version}
+created: {img_ts}'''
+
   # create new server
-  task_status(prox.nodes(node).qemu.post(
+  prox_task(prox.nodes(node).qemu.post(
     vmid = cluster_id,
     cores = 1,
     memory = 1024,
@@ -88,7 +156,7 @@
     agent = ('enabled=true'),
     hotplug = 0,
     ciupgrade = 0,
-    description = f'
{cluster_name} image\nbased on: {cloud_image}\nk3s version: {k3s_version}',
+    description = image_desc,
     ciuser = cloudinituser, 
     cipassword = cloudinitpass,
     sshkeys = cloudinitsshkey,
@@ -97,15 +165,15 @@
   # shell to import disk
   # import-from requires the full path os.getcwd required here
   import_cmd = f'''
-sudo qm set {cluster_id} --scsi0 {storage}:0,import-from={os.getcwd()}/{cloud_image},iothread=true,aio=native
+sudo qm set {cluster_id} --scsi0 {storage}:0,import-from={os.getcwd()}/{cloud_image},iothread=true,aio=io_uring
 mv {cloud_image} {cloud_image}.patched'''
 
   # run shell command to import
   local_os_process(import_cmd)
 
   # convert to template via create base disk also vm config
-  task_status(prox.nodes(node).qemu(cluster_id).template.post())
-  task_status(prox.nodes(node).qemu(cluster_id).config.post(template = 1))
+  prox_task(prox.nodes(node).qemu(cluster_id).template.post())
+  prox_task(prox.nodes(node).qemu(cluster_id).config.post(template = 1))
   kmsg(f'{kname}qm-import', f'done')
 
 # image info
diff --git a/lib/verb_k3s.py b/lib/verb_k3s.py
index e7c1705..5fb2db3 100755
--- a/lib/verb_k3s.py
+++ b/lib/verb_k3s.py
@@ -1,11 +1,7 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_k3s import export_k3s_token, kubeconfig, k3s_check_config, kubectl
-from kopsrox_kmsg import kmsg
-
-# other imports
-import sys
+from kopsrox_k3s import * 
 
 # passed command
 cmd = sys.argv[2]
diff --git a/lib/verb_kubevip.py b/lib/verb_kubevip.py
deleted file mode 100755
index a03a6b1..0000000
--- a/lib/verb_kubevip.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-# functions
-from kopsrox_k3s import install_kube_vip
-
-# sys
-import sys
-
-# passed command
-cmd = sys.argv[2]
-
-# map arg if passed
-try:
-  arg = sys.argv[3]
-except:
-  pass
-
-# define kname
-kname = 'kubevip_'+cmd
-
-# create utility node
-if cmd == 'reinstall':
-    kmsg(kname, " reinstalling kubevip"
-    install_kube_vip()
diff --git a/lib/verb_node.py b/lib/verb_node.py
index e831dd3..e58b937 100755
--- a/lib/verb_node.py
+++ b/lib/verb_node.py
@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_config import vmnames,cluster_info, cluster_id, vms, vmip, cloudinituser, cloudinitpass
+from kopsrox_config import *
 from kopsrox_k3s import k3s_remove_node, k3s_init_node
 from kopsrox_proxmox import clone
-from kopsrox_kmsg import kmsg
-
-# other imports
-import sys,os,re
 
 # passed command
 cmd = sys.argv[2]
diff --git a/requirements.txt b/requirements.txt
index 6ef4ceb..f9b1040 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
-proxmoxer==2.0.1
-wget==3.2
+proxmoxer==2.1.0