From be5566e3f8ee5de7121ca788626cee507b73994d Mon Sep 17 00:00:00 2001
From: simonc <simonc@gmail.com>
Date: Sat, 31 Aug 2024 21:39:00 +0100
Subject: [PATCH 01/30] Update README.md

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 460f93a..51cc31b 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,9 @@
 - add more master/worker k3s nodes using a simple config file and cli interface :pray:
 - kube-vip ( https://kube-vip.io/ ) built in providing full HA setup for the kube api :atom:
 - easy management of etcd S3 snapshot/restore operations - easily restore a cluster from s3! :floppy_disk:
-- export the k3s token, your kubeconfig etc etc - its all automatic  :nerd_face:  
+- export the k3s token, your kubeconfig etc etc - its all automatic  :nerd_face:
+
+  get it https://github.com/simonccc/kopsrox/releases
 
 # :book: docs
  - [SETUP.md](docs/SETUP.md)

From 8084dbe0337f537a83de2f03fe84db452f8c27b7 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Mon, 2 Sep 2024 14:09:42 +0100
Subject: [PATCH 02/30] bug fix for delete on other node

---
 lib/kopsrox_proxmox.py | 43 ++++++++++++++++++++++++++----------------
 lib/verb_image.py      |  8 ++++----
 2 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index e8d7326..b766f2d 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -132,7 +132,7 @@ def get_node(vmid):
   exit(0)
 
 # stop and destroy vm
-def prox_destroy(vmid):
+def prox_destroy(vmid: int):
 
     kname = 'prox_destroy-vm'
 
@@ -142,16 +142,16 @@ def prox_destroy(vmid):
 
     # if destroying image
     if vmid == cluster_id:
-      task_status(prox.nodes(node).qemu(cluster_id).delete())
+      prox_task(prox.nodes(node).qemu(cluster_id).delete())
       return
 
     # power off and delete
     try:
-      task_status(prox.nodes(node).qemu(vmid).status.stop.post())
-      task_status(prox.nodes(node).qemu(vmid).delete())
+      prox_task(prox.nodes(node).qemu(vmid).status.stop.post(),node)
+      prox_task(prox.nodes(node).qemu(vmid).delete(),node)
       kmsg(kname, vmname)
-    except:
-      kmsg(kname, f'unable to destroy {vmid}', 'err')
+    except Exception as e:
+      kmsg(kname, f'unable to destroy {node}/{vmid}\n{e}', 'err')
       exit(0)
 
 # clone
@@ -171,10 +171,10 @@ def clone(vmid):
   kmsg('proxmox_clone', f'{hostname} {ip} {vm_cpu}c/{vm_ram}G ram {vm_disk}G disk')
 
   # clone
-  task_status(prox.nodes(node).qemu(cluster_id).clone.post(newid = vmid))
+  prox_task(prox.nodes(node).qemu(cluster_id).clone.post(newid = vmid))
 
   # configure
-  task_status(prox.nodes(node).qemu(vmid).config.post(
+  prox_task(prox.nodes(node).qemu(vmid).config.post(
     name = hostname,
     onboot = 1,
     cores = vm_cpu,
@@ -188,27 +188,31 @@ def clone(vmid):
   ))
 
   # resize disk
-  task_status(prox.nodes(node).qemu(vmid).resize.put(
+  prox_task(prox.nodes(node).qemu(vmid).resize.put(
     disk = 'scsi0',
     size = f'{vm_disk}G',
   ))
 
   # power on
-  task_status(prox.nodes(node).qemu(vmid).status.start.post())
+  prox_task(prox.nodes(node).qemu(vmid).status.start.post())
 
   # run uptime / wait for qagent to start
   internet_check(vmid)
   kmsg(f'proxmox_{hostname}', 'ready')
 
 # proxmox task blocker
-def task_status(task_id, node=node):
+def prox_task(task_id, node=node):
 
   # define default status
   status = {"status": ""}
 
   # until task stopped
-  while (status["status"] != "stopped"):
-    status = prox.nodes(node).tasks(task_id).status.get()
+  try:
+    while (status["status"] != "stopped"):
+      status = prox.nodes(node).tasks(task_id).status.get()
+  except:
+    kmsg('proxmox_task-status', f'unable to get task {task_id} node: {node}', 'err')
+    exit(0)
 
   # if task not completed ok
   if not status["exitstatus"] == "OK":
@@ -222,10 +226,17 @@ def task_log(task_id, node=node):
   logline = ''
 
   # for each value in list
-  for log in prox.nodes(node).tasks(task_id).log.get():
+  # assuming task_id is valid
+  try:
+    for log in prox.nodes(node).tasks(task_id).log.get():
+
+      # append log to logline
+      logline += log['t'] + '\n'
 
-    # append log to logline
-    logline += log['t'] + '\n'
+    return(logline)
+  except:
+    kmsg('proxmox_task-log', f'failed to get log for task!', 'err')
+    exit(0)
 
   # return string
   return(logline)
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 0013223..4e1c231 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -10,7 +10,7 @@
 import wget,sys,os
 
 # proxmox functions
-from kopsrox_proxmox import task_status, prox_destroy
+from kopsrox_proxmox import prox_task, prox_destroy
 
 # kmsg
 from kopsrox_kmsg import kmsg
@@ -71,7 +71,7 @@
     pass
 
   # create new server
-  task_status(prox.nodes(node).qemu.post(
+  prox_task(prox.nodes(node).qemu.post(
     vmid = cluster_id,
     cores = 1,
     memory = 1024,
@@ -104,8 +104,8 @@
   local_os_process(import_cmd)
 
   # convert to template via create base disk also vm config
-  task_status(prox.nodes(node).qemu(cluster_id).template.post())
-  task_status(prox.nodes(node).qemu(cluster_id).config.post(template = 1))
+  prox_task(prox.nodes(node).qemu(cluster_id).template.post())
+  prox_task(prox.nodes(node).qemu(cluster_id).config.post(template = 1))
   kmsg(f'{kname}qm-import', f'done')
 
 # image info

From e3cd577102804e61395d9674de3a4e9c48c0c4a6 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 26 Oct 2024 15:39:39 +0100
Subject: [PATCH 03/30] fix ini

---
 lib/kopsrox_ini.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/kopsrox_ini.py b/lib/kopsrox_ini.py
index 587df38..6ea398e 100755
--- a/lib/kopsrox_ini.py
+++ b/lib/kopsrox_ini.py
@@ -14,11 +14,11 @@ def init_kopsrox_ini():
   config.add_section('proxmox')
 
   # proxmox api endpoint
-  config.set('proxmox', '; domain name or IP to access proxmox')
+  config.set('proxmox', '; domain or IP to access proxmox')
   config.set('proxmox', 'prox_endpoint', '127.0.0.1')
 
   # proxmox api port
-  config.set('proxmox', '; port is usually 8006')
+  config.set('proxmox', '; api port ( usually 8006 ) ')
   config.set('proxmox', 'port', '8006')
 
   # username
@@ -38,7 +38,7 @@ def init_kopsrox_ini():
   config.set('proxmox', 'node', 'proxmox')
 
   # storage on node
-  config.set('proxmox', '; the proxmox storage to use for kopsrox - needs to be available on the proxmox host')
+  config.set('proxmox', '; the proxmox storage to use for kopsrox - needs to be available on the proxmox node')
   config.set('proxmox', 'storage', 'local-lvm')
 
   # kopsrox section
@@ -46,10 +46,10 @@ def init_kopsrox_ini():
 
   # upstream image
   config.set('kopsrox', '; the upstream cloud image used to create the kopsrox image')
-  config.set('kopsrox', 'cloud_image_url', '//cloud-images.ubuntu.com/minimal/daily/oracular/current/oracular-minimal-cloudimg-amd64.img')
+  config.set('kopsrox', 'cloud_image_url', 'https://cloud-images.ubuntu.com/minimal/daily/oracular/current/oracular-minimal-cloudimg-amd64.img')
 
   # disk size for kopsrox vms
-  config.set('kopsrox', '; size of kopsrox vm disk in Gib ')
+  config.set('kopsrox', '; size of vm disk in Gib ')
   config.set('kopsrox', 'vm_disk', '20')
 
   # number of cpu cores
@@ -65,11 +65,11 @@ def init_kopsrox_ini():
   config.set('kopsrox', 'cloudinituser', 'user')
 
   # cloud init user password
-  config.set('kopsrox', '; the password for the cloudinit user')
+  config.set('kopsrox', '; password for the cloudinit user')
   config.set('kopsrox', 'cloudinitpass', 'admin')
 
   # cloud init user ssh key
-  config.set('kopsrox', '; the ssh public key for the cloudinit user')
+  config.set('kopsrox', ';  a ssh public key for the cloudinit user ( required ) ')
   config.set('kopsrox', 'cloudinitsshkey', 'ssh-rsa cioieocieo')
 
   # network bridge
@@ -79,7 +79,6 @@ def init_kopsrox_ini():
 
   # kopsrox network baseip
   config.set('kopsrox', '; first ip of the ip range used for this kopsrox cluster')
-  config.set('kopsrox', '; this ip is assigned to the template - the first master is this + 1')
   config.set('kopsrox', 'network_ip', '192.168.0.160')
 
   # netmask / subnet
@@ -87,7 +86,7 @@ def init_kopsrox_ini():
   config.set('kopsrox', 'network_mask', '24')
 
   # default gateway
-  config.set('kopsrox', '; default gateway for the kopsrox network ( needs to provide internet access ) ')
+  config.set('kopsrox', '; default gateway for the network ( needs to provide internet access ) ')
   config.set('kopsrox', 'network_gw', '192.168.0.1')
 
   # dns server
@@ -96,17 +95,18 @@ def init_kopsrox_ini():
 
   # network mtu
   config.set('kopsrox', '; interface mtu set on vms ')
+  config.set('kopsrox', '; set to 1450 if using sdn ')
   config.set('kopsrox', 'network_mtu', '1500')
 
   # cluster section
   config.add_section('cluster')
 
   # cluster vmid
-  config.set('cluster', '; id for the cluster vms.. eg from 620 - 630')
+  config.set('cluster', '; id for the cluster vm\'s eg from 620 - 630')
   config.set('cluster', 'cluster_id', '620')
 
   # cluster friendly name
-  config.set('cluster', '; name for the kopsrox cluster')
+  config.set('cluster', '; name of the cluster')
   config.set('cluster', 'cluster_name', 'mycluster')
 
   # number of masters

From 2c56fa3aec31ffc7bf6318253e2fdd03f81720fa Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Fri, 8 Nov 2024 22:15:14 +0000
Subject: [PATCH 04/30] get date on image and other fixes

---
 docs/SETUP.md         | 10 ++--------
 lib/kopsrox_config.py |  3 ---
 lib/verb_image.py     | 26 ++++++++++++++++++++++----
 requirements.txt      |  3 +--
 4 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/docs/SETUP.md b/docs/SETUP.md
index da66fc1..d8df332 100644
--- a/docs/SETUP.md
+++ b/docs/SETUP.md
@@ -7,18 +7,12 @@
 - a range of 10 free Proxmox qm/virtual machine id 'vmids' eg 600 to 610
 - a range of 10 IP's on a network with internet access for kopsrox to work with eg 192.168.0.160 to 192.168.0.170
 
-## :bricks: install
-
 ## :bricks: install 
 
-- clone the repo - or probably better one of the releases or stable branches - the 'main' branch can often be a bit broken
-- sudo apt install libguestfs-tools python3-termcolor -y`
+- get one of the releases or stable branches - the 'main' branch can often be a bit broken
+- sudo apt install libguestfs-tools python3-termcolor python3-wget -y`
 - pip3 install --break-system-packages --user -r requirements.txt`
 
-_installs the required pip packages vs using os packages_
-
-( this will be improved in the future  ) 
-
 ## :star: generate api key
 
 `sudo pvesh create /access/users/root@pam/token/kopsrox`
diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 51f7f53..65f93ce 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -304,7 +304,6 @@ def vm_info(vmid,node=node):
 # dummy cloud_image_vars overwritten below
 cloud_image_size = 0
 cloud_image_desc = ''
-cloud_image_created = ''
 
 # skip image check if image create is passed
 try:
@@ -342,7 +341,6 @@ def vm_info(vmid,node=node):
 
     # get image created and desc from template
     template_data = prox.nodes(node).qemu(cluster_id).config.get()
-    cloud_image_created = str(datetime.fromtimestamp(int(template_data['meta'].split(',')[1].split('=')[1])))
     cloud_image_desc = template_data['description']
 
   except:
@@ -415,5 +413,4 @@ def image_info():
   kname = f'image_'
   kmsg(f'{kname}desc', cloud_image_desc)
   kmsg(f'{kname}storage', f'{kopsrox_image_name} ({storage_type})')
-  kmsg(f'{kname}created', cloud_image_created)
   kmsg(f'{kname}size', f'{cloud_image_size}G')
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 4e1c231..0a5a21c 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -15,6 +15,9 @@
 # kmsg
 from kopsrox_kmsg import kmsg
 
+# date
+from datetime import datetime
+
 # define command
 cmd = sys.argv[2]
 kname = 'image_'
@@ -28,8 +31,14 @@
 
   # check if image already exists
   if os.path.isfile(cloud_image):
-    kmsg(f'{kname}check', f'Error! {cloud_image} already exists - please delete', 'err')
-    exit(0)
+    kmsg(f'{kname}check', f'{cloud_image} already exists - removing', 'warn')
+    try:
+      os.remove(cloud_image)
+      if os.path.isfile(cloud_image):
+        exit(0)
+    except:
+      kmsg(f'{kname}check', f'{cloud_image} cannot delete', 'err')
+      exit(0)
 
   # check img can be downloaded
   try:
@@ -59,7 +68,7 @@
   cp /dev/null /etc/sysconfig/qemu-ga 
 fi'''
   # shouldn't really need root but run into permissions problems
-  virtc_cmd = f'sudo virt-customize --smp 2 -m 2048 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
+  virtc_cmd = f'sudo virt-customize --smp 1 -m 1024 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
 
   kmsg(f'{kname}virt-customize', 'configuring image')
   local_os_process(virtc_cmd)
@@ -70,6 +79,15 @@
   except:
     pass
 
+  # define image desc
+  img_ts = str(datetime.now())
+  img_chksum = abs(hash(cluster_name+cloud_image+k3s_version+img_ts))
+  image_desc = f'''<pre>{cluster_name}
+  image based on: {cloud_image}
+  k3s version: {k3s_version}
+  created: {img_ts}
+  checksum: {img_chksum}'''
+
   # create new server
   prox_task(prox.nodes(node).qemu.post(
     vmid = cluster_id,
@@ -88,7 +106,7 @@
     agent = ('enabled=true'),
     hotplug = 0,
     ciupgrade = 0,
-    description = f'<pre>{cluster_name} image\nbased on: {cloud_image}\nk3s version: {k3s_version}',
+    description = image_desc,
     ciuser = cloudinituser, 
     cipassword = cloudinitpass,
     sshkeys = cloudinitsshkey,
diff --git a/requirements.txt b/requirements.txt
index 6ef4ceb..f9b1040 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
-proxmoxer==2.0.1
-wget==3.2
+proxmoxer==2.1.0

From 25a2958f095bcf0a83a2d4aa6878ecbe6f4a6204 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 15:04:12 +0000
Subject: [PATCH 05/30] some image stuff

---
 lib/kopsrox_config.py |  3 +++
 lib/verb_image.py     | 33 ++++++++++++++-------------------
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 65f93ce..4db82e3 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -82,6 +82,9 @@ def conf_check(section,value):
 
 # get cluster id
 cluster_id = conf_check('cluster','cluster_id')
+if cluster_id < 100:
+  kmsg(kname, f' cluster_id is too low - should be over 100', 'err')
+  exit(0)
 
 # proxmox
 prox_endpoint = conf_check('proxmox','prox_endpoint')
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 0a5a21c..6ef71a6 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -1,23 +1,14 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_config import prox, local_os_process, image_info, cloud_image_desc, kopsrox_img, k3s_version
-
-# variables
-from kopsrox_config import node, storage, cluster_id, cloud_image_url, cluster_name, cloudinitsshkey, cloudinituser, cloudinitpass
+from kopsrox_config import * 
 
 # general imports
-import wget,sys,os
+import wget,os
 
 # proxmox functions
 from kopsrox_proxmox import prox_task, prox_destroy
 
-# kmsg
-from kopsrox_kmsg import kmsg
-
-# date
-from datetime import datetime
-
 # define command
 cmd = sys.argv[2]
 kname = 'image_'
@@ -68,7 +59,7 @@
   cp /dev/null /etc/sysconfig/qemu-ga 
 fi'''
   # shouldn't really need root but run into permissions problems
-  virtc_cmd = f'sudo virt-customize --smp 1 -m 1024 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
+  virtc_cmd = f'sudo virt-customize --smp 2 -m 2048 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
 
   kmsg(f'{kname}virt-customize', 'configuring image')
   local_os_process(virtc_cmd)
@@ -81,12 +72,16 @@
 
   # define image desc
   img_ts = str(datetime.now())
-  img_chksum = abs(hash(cluster_name+cloud_image+k3s_version+img_ts))
-  image_desc = f'''<pre>{cluster_name}
-  image based on: {cloud_image}
-  k3s version: {k3s_version}
-  created: {img_ts}
-  checksum: {img_chksum}'''
+  image_desc = f'''<pre>
+▗▖ ▗▖ ▗▄▖ ▗▄▄▖  ▗▄▄▖▗▄▄▖  ▗▄▖ ▗▖  ▗▖
+▐▌▗▞▘▐▌ ▐▌▐▌ ▐▌▐▌   ▐▌ ▐▌▐▌ ▐▌ ▝▚▞▘ 
+▐▛▚▖ ▐▌ ▐▌▐▛▀▘  ▝▀▚▖▐▛▀▚▖▐▌ ▐▌  ▐▌  
+▐▌ ▐▌▝▚▄▞▘▐▌   ▗▄▄▞▘▐▌ ▐▌▝▚▄▞▘▗▞▘▝▚▖
+
+cluster_name: {cluster_name}
+cloud_img: {cloud_image}
+k3s_version: {k3s_version}
+created: {img_ts}'''
 
   # create new server
   prox_task(prox.nodes(node).qemu.post(
@@ -115,7 +110,7 @@
   # shell to import disk
   # import-from requires the full path os.getcwd required here
   import_cmd = f'''
-sudo qm set {cluster_id} --scsi0 {storage}:0,import-from={os.getcwd()}/{cloud_image},iothread=true,aio=native
+sudo qm set {cluster_id} --scsi0 {storage}:0,import-from={os.getcwd()}/{cloud_image},iothread=true,aio=io_uring
 mv {cloud_image} {cloud_image}.patched'''
 
   # run shell command to import

From 0acfb1b43c02d2d8656b0a3404230b7d351a1c3d Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 17:42:59 +0000
Subject: [PATCH 06/30] add network_ip config to traefik helm chart values

---
 lib/kopsrox_config.py | 23 ++++++-----------------
 lib/kopsrox_k3s.py    |  8 +++++---
 lib/verb_image.py     | 22 ++++++++++++++++------
 lib/verb_node.py      |  6 +-----
 4 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 4db82e3..2dff9d5 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -1,29 +1,18 @@
 #!/usr/bin/env python3
 
-# imports
+# external imports
 import urllib3
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-# proxmoxer api
+import urllib.parse
+import wget
+from datetime import datetime
 from proxmoxer import ProxmoxAPI
 
-# prompt
-kname = 'config-check'
-
 # look for strings in responses
 import re
 
-# checks cmd line args - could be moved?
-import sys
-
-# local os commands
-import subprocess
-
-# used to encode ssh key
-import urllib.parse
-
-# datetime stuff for generating image date
-from datetime import datetime
+# checks cmd line args file ops and processes
+import os,sys,subprocess
 
 # kmsg
 from kopsrox_kmsg import kmsg
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index c459ee9..85cee29 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -1,14 +1,13 @@
 #!/usr/bin/env python3 
 
 # imports
-from kopsrox_config import masterid, k3s_version, masters, workers, cluster_name, vmnames, vmip, cluster_info, list_kopsrox_vm, network_ip
+from kopsrox_config import *
 
 # standard imports
 from kopsrox_proxmox import qaexec, prox_destroy, internet_check, clone
-from kopsrox_kmsg import kmsg
 
 # standard imports
-import re, time, os
+import time, os
 
 # check for k3s status
 def k3s_check(vmid: int):
@@ -302,6 +301,9 @@ def install_kube_vip():
   # install completed
   kmsg('k3s_kubevip', f'{network_ip} vip active')
 
+def patch_traefik():
+  return(kubectl(f'-n kube-system patch svc traefik -p \'{{"spec":{{"loadBalancerIP":"{network_ip}"}}}}\''))
+
 # return current vip master
 def get_kube_vip_master():
   kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}'
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 6ef71a6..4acf616 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -3,9 +3,6 @@
 # functions
 from kopsrox_config import * 
 
-# general imports
-import wget,os
-
 # proxmox functions
 from kopsrox_proxmox import prox_task, prox_destroy
 
@@ -22,7 +19,7 @@
 
   # check if image already exists
   if os.path.isfile(cloud_image):
-    kmsg(f'{kname}check', f'{cloud_image} already exists - removing', 'warn')
+    kmsg(f'image_check', f'{cloud_image} already exists - removing', 'warn')
     try:
       os.remove(cloud_image)
       if os.path.isfile(cloud_image):
@@ -47,7 +44,7 @@
 INSTALL_K3S_SKIP_ENABLE=true \
 INSTALL_K3S_SKIP_SELINUX_RPM=true \
 INSTALL_K3S_VERSION={k3s_version} \
-sh -s - server --cluster-init > /k3s-image-install.log 2>&1
+sh -s - server --cluster-init > /{cluster_name}-image-install.log 2>&1
 
 if [ -f /etc/selinux/config ] 
 then
@@ -57,7 +54,20 @@
 if [ -f /etc/sysconfig/qemu-ga ]
 then
   cp /dev/null /etc/sysconfig/qemu-ga 
-fi'''
+fi
+mkdir -p /var/lib/rancher/k3s/server/manifests/
+echo '
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    service:
+      spec:
+        loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+'''
   # shouldn't really need root but run into permissions problems
   virtc_cmd = f'sudo virt-customize --smp 2 -m 2048 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
 
diff --git a/lib/verb_node.py b/lib/verb_node.py
index e831dd3..e58b937 100755
--- a/lib/verb_node.py
+++ b/lib/verb_node.py
@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_config import vmnames,cluster_info, cluster_id, vms, vmip, cloudinituser, cloudinitpass
+from kopsrox_config import *
 from kopsrox_k3s import k3s_remove_node, k3s_init_node
 from kopsrox_proxmox import clone
-from kopsrox_kmsg import kmsg
-
-# other imports
-import sys,os,re
 
 # passed command
 cmd = sys.argv[2]

From 8f28b898767539d04c029efc148d0ad9cbb03dc9 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 20:14:06 +0000
Subject: [PATCH 07/30] move kubevip to autoinstall

---
 lib/kopsrox_k3s.py | 12 ------------
 lib/verb_image.py  | 14 +++++++++++---
 2 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 85cee29..49e6f27 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -106,7 +106,6 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # final steps for first master  - kubevip, export kubeconfig and token 
     if nodetype == 'master':
-      install_kube_vip()
       kubeconfig()
       export_k3s_token()
 
@@ -290,20 +289,9 @@ def install_kube_vip():
 EOF
 ''')
 
-  # apply / replace the manifest
-  kubevip_install = kubectl('replace --force -f /tmp/kubevip.yaml')
-
-  # check output 
-  if not re.search('daemonset.apps/kubevip replaced', kubevip_install):
-    kmsg('k3s_kubevip', f'failed to install kube-vip\n{kubevip_install}', 'err')
-    exit(0)
-
   # install completed
   kmsg('k3s_kubevip', f'{network_ip} vip active')
 
-def patch_traefik():
-  return(kubectl(f'-n kube-system patch svc traefik -p \'{{"spec":{{"loadBalancerIP":"{network_ip}"}}}}\''))
-
 # return current vip master
 def get_kube_vip_master():
   kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}'
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 4acf616..d1a4e35 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -37,7 +37,12 @@
     kmsg(f'{kname}check', f'unable to download {cloud_image_url}', 'err')
     exit(0)
 
-  # script to install disable selinux on Rocky
+  kv_manifest = open('./lib/kubevip/kubevip.yaml', 'r').read().replace('KOPSROX_IP', network_ip).strip()
+  kv_write_out = open('./kubevip-patched.yaml', 'w') 
+  kv_write_out.write(kv_manifest)
+  kv_write_out.close()
+
+  # script to in kopsrox image
   virtc_script = f'''\
 curl -v https://get.k3s.io > /k3s.sh 
 cat /k3s.sh | \
@@ -56,6 +61,7 @@
   cp /dev/null /etc/sysconfig/qemu-ga 
 fi
 mkdir -p /var/lib/rancher/k3s/server/manifests/
+
 echo '
 apiVersion: helm.cattle.io/v1
 kind: HelmChartConfig
@@ -69,9 +75,11 @@
         loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
 '''
   # shouldn't really need root but run into permissions problems
-  virtc_cmd = f'sudo virt-customize --smp 2 -m 2048 -a {cloud_image} --install qemu-guest-agent --run-command "{virtc_script}"'
-
   kmsg(f'{kname}virt-customize', 'configuring image')
+  virtc_cmd = f'''
+sudo virt-customize --smp 2 -m 2048 -a {cloud_image}  \
+--install qemu-guest-agent --run-command "{virtc_script}"  \
+--copy-in ./kubevip-patched.yaml:/var/lib/rancher/k3s/server/manifests/'''
   local_os_process(virtc_cmd)
 
   # destroy template if it exists

From d158f332b0b256e6afac4a297f92d01cbb33b19e Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 20:26:32 +0000
Subject: [PATCH 08/30] more kubevip clean up

---
 kopsrox.py          |  3 ---
 lib/kopsrox_k3s.py  | 20 --------------------
 lib/verb_kubevip.py | 24 ------------------------
 3 files changed, 47 deletions(-)
 delete mode 100755 lib/verb_kubevip.py

diff --git a/kopsrox.py b/kopsrox.py
index 3fcad1d..0ed64c5 100755
--- a/kopsrox.py
+++ b/kopsrox.py
@@ -48,9 +48,6 @@
     "reboot" : 'hostname',
     "k3s-uninstall" : 'hostname',
     "rejoin-slave" : 'hostname',
-  },
-  "kubevip": {
-    "reinstall": '',
   }
 }
 
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 49e6f27..ff0e942 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -277,21 +277,6 @@ def export_k3s_token():
       token_file.write(live_token)
     kmsg('k3s_export-token', f'created: {token_name}')
 
-# install kube vip
-def install_kube_vip():
-
-  # read default kube vip manifest and replace with network_ip
-  kv_manifest = open('./lib/kubevip/kubevip.yaml', "r").read().replace('KOPSROX_IP', network_ip).strip()
-
-  # create the manifest
-  qaexec(masterid, f'''cat <<EOF> /tmp/kubevip.yaml
-{kv_manifest}
-EOF
-''')
-
-  # install completed
-  kmsg('k3s_kubevip', f'{network_ip} vip active')
-
 # return current vip master
 def get_kube_vip_master():
   kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}'
@@ -301,8 +286,3 @@ def get_kube_vip_master():
   except:
     kubevip_m = ''
   return(kubevip_m)
-
-# reload kubevip
-def kubevip_reload():
-    reload = kubectl('rollout restart daemonset kubevip -n kube-system')
-    print(reload)
diff --git a/lib/verb_kubevip.py b/lib/verb_kubevip.py
deleted file mode 100755
index a03a6b1..0000000
--- a/lib/verb_kubevip.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-# functions
-from kopsrox_k3s import install_kube_vip
-
-# sys
-import sys
-
-# passed command
-cmd = sys.argv[2]
-
-# map arg if passed
-try:
-  arg = sys.argv[3]
-except:
-  pass
-
-# define kname
-kname = 'kubevip_'+cmd
-
-# create utility node
-if cmd == 'reinstall':
-    kmsg(kname, " reinstalling kubevip"
-    install_kube_vip()

From f651766a4ec470789cb6c6cf68ac250567b4b5ea Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 22:17:31 +0000
Subject: [PATCH 09/30] small fixes to kubevip install

---
 lib/kopsrox_proxmox.py |  1 -
 lib/verb_image.py      | 22 +++++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index b766f2d..b0c9f88 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -198,7 +198,6 @@ def clone(vmid):
 
   # run uptime / wait for qagent to start
   internet_check(vmid)
-  kmsg(f'proxmox_{hostname}', 'ready')
 
 # proxmox task blocker
 def prox_task(task_id, node=node):
diff --git a/lib/verb_image.py b/lib/verb_image.py
index d1a4e35..902b491 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -37,16 +37,28 @@
     kmsg(f'{kname}check', f'unable to download {cloud_image_url}', 'err')
     exit(0)
 
+  # kubevip
+  # open the generic kubevip deployment and patch it with our network_ip in memory
   kv_manifest = open('./lib/kubevip/kubevip.yaml', 'r').read().replace('KOPSROX_IP', network_ip).strip()
-  kv_write_out = open('./kubevip-patched.yaml', 'w') 
+
+  # define the name/location of the patched kubevip 
+  kv_yaml = f'./lib/kubevip/{cluster_name}-kubevip.yaml'
+
+  # open the new kubevip path
+  kv_write_out = open(kv_yaml, 'w') 
+
+  # write the patched in memory version to file
   kv_write_out.write(kv_manifest)
+
+  # close files
   kv_write_out.close()
 
-  # script to in kopsrox image
+  # script to run in kopsrox image
   virtc_script = f'''\
 curl -v https://get.k3s.io > /k3s.sh 
 cat /k3s.sh | \
 INSTALL_K3S_SKIP_ENABLE=true \
+INSTALL_K3S_SKIP_START=true \
 INSTALL_K3S_SKIP_SELINUX_RPM=true \
 INSTALL_K3S_VERSION={k3s_version} \
 sh -s - server --cluster-init > /{cluster_name}-image-install.log 2>&1
@@ -60,7 +72,6 @@
 then
   cp /dev/null /etc/sysconfig/qemu-ga 
 fi
-mkdir -p /var/lib/rancher/k3s/server/manifests/
 
 echo '
 apiVersion: helm.cattle.io/v1
@@ -74,12 +85,13 @@
       spec:
         loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
 '''
-  # shouldn't really need root but run into permissions problems
+  # shouldn't really need root/sudo but run into permissions problems
   kmsg(f'{kname}virt-customize', 'configuring image')
   virtc_cmd = f'''
 sudo virt-customize --smp 2 -m 2048 -a {cloud_image}  \
+--mkdir /var/lib/rancher/k3s/server/manifests/ \
 --install qemu-guest-agent --run-command "{virtc_script}"  \
---copy-in ./kubevip-patched.yaml:/var/lib/rancher/k3s/server/manifests/'''
+--copy-in {kv_yaml}:/var/lib/rancher/k3s/server/manifests/'''
   local_os_process(virtc_cmd)
 
   # destroy template if it exists

From c8ab3f99e7a881def5eab5b830e6cdaa34a5085a Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 9 Nov 2024 22:36:54 +0000
Subject: [PATCH 10/30] fixes for helm

---
 dev/helm.sh        |  2 ++
 lib/kopsrox_k3s.py | 13 +++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)
 create mode 100755 dev/helm.sh

diff --git a/dev/helm.sh b/dev/helm.sh
new file mode 100755
index 0000000..7cf89dd
--- /dev/null
+++ b/dev/helm.sh
@@ -0,0 +1,2 @@
+c=`grep ^cluster_name kopsrox.ini | cut -d ' ' -f3`
+helm --kubeconfig=${c}.kubeconfig ${@}
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index ff0e942..15fac19 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -220,13 +220,15 @@ def k3s_update_cluster():
 
 # kubeconfig
 def kubeconfig():
+
+  # define filename
+  kubeconfig_file = f'{cluster_name}.kubeconfig'
   # replace 127.0.0.1 with vip ip
   kconfig = qaexec(masterid, 'cat /etc/rancher/k3s/k3s.yaml').replace('127.0.0.1', network_ip)
-
-  # write file out
-  with open(f'{cluster_name}.kubeconfig','w') as kfile:
-    kfile.write(kconfig)
-  kmsg('k3s_kubeconfig', ('saved ' + (cluster_name +'.kubeconfig')))
+  # write file with 600 permissions
+  with os.fdopen(os.open(kubeconfig_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as handle:
+    handle.write(kconfig)
+  kmsg('k3s_kubeconfig', f'saved {kubeconfig_file}')
 
 # kubectl
 def kubectl(cmd):
@@ -246,7 +248,6 @@ def export_k3s_token():
 
   # define token file name
   token_name = f'{cluster_name}.k3stoken'
-
   # get masters token
   live_token = qaexec(masterid, 'cat /var/lib/rancher/k3s/server/token')
 

From f19e28909d342ed4f47d836f333d3290f627e747 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 30 Nov 2024 20:42:45 +0000
Subject: [PATCH 11/30] move s3 creds to k3s config file

---
 lib/kopsrox_config.py |  9 ---------
 lib/verb_etcd.py      |  8 ++++----
 lib/verb_image.py     | 16 ++++++++++++++--
 3 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 2dff9d5..e68d030 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -141,15 +141,6 @@ def conf_check(section,value):
 # dict of all config items - legacy support
 config = ({s:dict(kopsrox_config.items(s)) for s in kopsrox_config.sections()})
 
-# generated string to use in s3 commands
-s3_string = \
-' --etcd-s3 ' + region_string + \
-' --etcd-s3-endpoint ' + s3endpoint + \
-' --etcd-s3-access-key ' + access_key + \
-' --etcd-s3-secret-key ' + access_secret + \
-' --etcd-s3-bucket ' + bucket + \
-' --etcd-s3-skip-ssl-verify '
-
 # define masterid
 masterid = cluster_id + 1
 
diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py
index af451d1..c00a730 100755
--- a/lib/verb_etcd.py
+++ b/lib/verb_etcd.py
@@ -4,7 +4,7 @@
 import sys, re, os
 
 # kopsrox
-from kopsrox_config import masterid, masters, workers, cluster_name, s3_string, bucket, s3endpoint
+from kopsrox_config import masterid, masters, workers, cluster_name, bucket, s3endpoint
 from kopsrox_proxmox import get_node, qaexec
 from kopsrox_k3s import k3s_rm_cluster, kubectl, k3s_update_cluster, export_k3s_token, kubeconfig
 from kopsrox_kmsg import kmsg
@@ -28,7 +28,7 @@
 def s3_run(s3cmd):
 
   # run the command ( 2>&1 required )
-  k3s_run = f'k3s etcd-snapshot {s3cmd} {s3_string} 2>&1'
+  k3s_run = f'k3s etcd-snapshot {s3cmd} 2>&1'
   cmd_out = qaexec(masterid, k3s_run)
 
   # look for fatal error in output
@@ -74,7 +74,7 @@ def list_snapshots():
     export_k3s_token()
 
   # define snapshot command
-  snap_cmd = f'k3s etcd-snapshot save {s3_string} --name kopsrox --etcd-snapshot-compress 2>&1'
+  snap_cmd = f'k3s etcd-snapshot save -name kopsrox --etcd-snapshot-compress 2>&1'
   #print(snap_cmd)
   snapout = qaexec(masterid,snap_cmd)
 
@@ -135,7 +135,7 @@ def s3_list():
   # define restore command
   restore_cmd = f'\
 systemctl stop k3s &&  \
-k3s server --cluster-reset --cluster-reset-restore-path={snapshot} --token={token} {s3_string} 2>&1 ; \
+k3s server --cluster-reset --cluster-reset-restore-path={snapshot} --token={token} 2>&1 ; \
 systemctl start k3s'
 
   # display some filtered restore contents
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 902b491..dc5204a 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -73,6 +73,7 @@
   cp /dev/null /etc/sysconfig/qemu-ga 
 fi
 
+mkdir -p /var/lib/rancher/k3s/server/manifests/
 echo '
 apiVersion: helm.cattle.io/v1
 kind: HelmChartConfig
@@ -84,12 +85,23 @@
     service:
       spec:
         loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+
+# custom kopsrox config
+mkdir -p /etc/rancher/k3s/config.yaml.d/
+echo '
+etcd-s3: true
+etcd-s3-region: {region_string}
+etcd-s3-endpoint: {s3endpoint}
+etcd-s3-access-key: {access_key}
+etcd-s3-secret-key: {access_secret}
+etcd-s3-bucket: {bucket}
+etcd-s3-skip-ssl-verify: true
+etcd-snapshot-compress: true'  > /etc/rancher/k3s/config.yaml.d/etcd-backup.yaml
 '''
   # shouldn't really need root/sudo but run into permissions problems
   kmsg(f'{kname}virt-customize', 'configuring image')
   virtc_cmd = f'''
-sudo virt-customize --smp 2 -m 2048 -a {cloud_image}  \
---mkdir /var/lib/rancher/k3s/server/manifests/ \
+sudo virt-customize --smp 2 -m 2048 -a {cloud_image}   \
 --install qemu-guest-agent --run-command "{virtc_script}"  \
 --copy-in {kv_yaml}:/var/lib/rancher/k3s/server/manifests/'''
   local_os_process(virtc_cmd)

From 7c31e69082626bf675aeb9cdfceb43bc9adf86b6 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 30 Nov 2024 23:01:23 +0000
Subject: [PATCH 12/30] more startup config

---
 lib/kopsrox_k3s.py     |  3 +--
 lib/kopsrox_proxmox.py |  3 +--
 lib/verb_etcd.py       |  3 +--
 lib/verb_image.py      | 12 ++++++++++--
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 15fac19..df9683c 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -45,8 +45,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
   # defines
   k3s_install_base = f'cat /k3s.sh | INSTALL_K3S_VERSION="{k3s_version}"'
-  k3s_install_flags = f' --disable servicelb --tls-san {network_ip}'
-  k3s_install_master = f'{k3s_install_base} sh -s - server --cluster-init {k3s_install_flags}'
+  k3s_install_master = f'{k3s_install_base} sh -s - server --cluster-init'
   k3s_install_worker = f'{k3s_install_base} K3S_URL="https://{network_ip}:6443" '
   master_cmd = ''
   token = ''
diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index b0c9f88..aca242b 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -4,8 +4,7 @@
 import time, re
 
 # kopsrox
-from kopsrox_config import prox, vmip, masterid
-from kopsrox_config import node,network_bridge,cluster_id,vmnames,vm_cpu,vm_ram,vm_disk,network_mask,network_gw,network_dns,network_mtu, network_dns
+from kopsrox_config import *
 from kopsrox_kmsg import kmsg
 
 # run a exec via qemu-agent
diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py
index c00a730..e107214 100755
--- a/lib/verb_etcd.py
+++ b/lib/verb_etcd.py
@@ -74,8 +74,7 @@ def list_snapshots():
     export_k3s_token()
 
   # define snapshot command
-  snap_cmd = f'k3s etcd-snapshot save -name kopsrox --etcd-snapshot-compress 2>&1'
-  #print(snap_cmd)
+  snap_cmd = f'k3s etcd-snapshot save --name kopsrox 2>&1'
   snapout = qaexec(masterid,snap_cmd)
 
   # filter output
diff --git a/lib/verb_image.py b/lib/verb_image.py
index dc5204a..6f91e90 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -86,10 +86,10 @@
       spec:
         loadBalancerIP: "{network_ip}"' > /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
 
-# custom kopsrox config
 mkdir -p /etc/rancher/k3s/config.yaml.d/
-echo '
+echo -n '
 etcd-s3: true
+etcd-snapshot-retention: 14
 etcd-s3-region: {region_string}
 etcd-s3-endpoint: {s3endpoint}
 etcd-s3-access-key: {access_key}
@@ -97,6 +97,14 @@
 etcd-s3-bucket: {bucket}
 etcd-s3-skip-ssl-verify: true
 etcd-snapshot-compress: true'  > /etc/rancher/k3s/config.yaml.d/etcd-backup.yaml
+
+echo -n '
+disable:
+  - servicelb
+write-kubeconfig-mode: "0644"
+disable-cloud-controller: true
+disable-network-policy: true
+tls-san: {network_ip}' > /etc/rancher/k3s/config.yaml.d/kopsrox.yaml
 '''
   # shouldn't really need root/sudo but run into permissions problems
   kmsg(f'{kname}virt-customize', 'configuring image')

From 10b0f08197d9f10263d031dbec908e5159c1189d Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Fri, 6 Dec 2024 19:19:34 +0000
Subject: [PATCH 13/30] update git ignore

---
 .gitignore | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 28c0455..efd20a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,8 +4,5 @@ lib/__pycache__/
 *.swp
 *.img
 *.etcd.token
-kopsrox.k3stoken
-kopsrox.kubeconfig
-kopsrox_disk_import.log
-kopsrox_imgpatch.log
-kopsrox.etcd*
+*.k3stoken
+*.kubeconfig

From 8792d4189b3bf0f6a1771170c74d7712052ebcb6 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 7 Dec 2024 14:25:15 +0000
Subject: [PATCH 14/30] code clean

---
 lib/kopsrox_config.py  |  5 +----
 lib/kopsrox_proxmox.py | 43 +++++++-----------------------------------
 lib/verb_etcd.py       | 10 +++-------
 3 files changed, 11 insertions(+), 47 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index e68d030..99331c9 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -8,11 +8,8 @@
 from datetime import datetime
 from proxmoxer import ProxmoxAPI
 
-# look for strings in responses
-import re
-
 # checks cmd line args file ops and processes
-import os,sys,subprocess
+import re,os,sys,subprocess,time
 
 # kmsg
 from kopsrox_kmsg import kmsg
diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index aca242b..4fc2b35 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -1,25 +1,21 @@
 #!/usr/bin/env python3
 
-# imports
-import time, re
-
 # kopsrox
 from kopsrox_config import *
 from kopsrox_kmsg import kmsg
 
 # run a exec via qemu-agent
-def qaexec(vmid: int = masterid,cmd = 'uptime'):
+def qaexec(vmid: int = masterid,cmd = 'uptime', node: str = node):
 
   # define kname
   kname = 'proxmox_qaexec'
 
-  # get vmname
-  # fixme try?
+  # get vmname and node
   vmname = vmnames[vmid]
-
-  # get node
-  # fixme try?
-  node = get_node(vmid)
+  try: 
+    node = vms[vmid]
+  except:
+    pass
 
   # qagent no yet running check
   qagent_running = 'false'
@@ -105,31 +101,6 @@ def qaexec(vmid: int = masterid,cmd = 'uptime'):
     except:
       return('no output-' + cmd)
 
-# return the node for a vmid
-def get_node(vmid):
-
-  # if it exists node is ok and we can return the configured node
-  if vmid == cluster_id:
-    return(node)
-
-  # check for vm id in proxmox cluster
-  for vm in prox.cluster.resources.get(type = 'vm'):
-
-    # matching id found
-    if vm.get('vmid') == vmid:
-
-      # return node vm is running on
-      return(vm.get('node'))
-
-  # error: node not found
-  if vmid == masterid:
-    kmsg('cluster_error', f'no cluster exists', 'err')
-    exit(0)
-
-  # default error
-  kmsg('proxmox_get-node', f'node for {vmid} not found', 'err')
-  exit(0)
-
 # stop and destroy vm
 def prox_destroy(vmid: int):
 
@@ -137,7 +108,7 @@ def prox_destroy(vmid: int):
 
     # get node and vmname
     vmname = vmnames[vmid]
-    node = get_node(vmid)
+    node = vms[vmid]
 
     # if destroying image
     if vmid == cluster_id:
diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py
index e107214..c6c7210 100755
--- a/lib/verb_etcd.py
+++ b/lib/verb_etcd.py
@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 
-# standard imports
-import sys, re, os
-
 # kopsrox
-from kopsrox_config import masterid, masters, workers, cluster_name, bucket, s3endpoint
-from kopsrox_proxmox import get_node, qaexec
+from kopsrox_config import *
+from kopsrox_proxmox import qaexec
 from kopsrox_k3s import k3s_rm_cluster, kubectl, k3s_update_cluster, export_k3s_token, kubeconfig
-from kopsrox_kmsg import kmsg
 
 # passed command
 cmd = sys.argv[2]
@@ -19,7 +15,7 @@
 # check master is running / exists
 # fails if node can't be found
 try:
-  get_node(masterid)
+  node = vms[masterid]
 except:
   kmsg(f'{kname}-check', 'cluster does not exist', 'err')
   exit(0)

From 18b2f58d21c9c35194616866acd77003fe6d3974 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 4 Jan 2025 18:13:48 +0000
Subject: [PATCH 15/30] add nodes to cluster post k3s config files fix

---
 lib/kopsrox_k3s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index df9683c..2dd5aed 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -73,11 +73,11 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # slave
     if nodetype == 'slave':
-      init_cmd = f'{k3s_install_base}{k3s_token_cmd} sh -s - server --server https://{network_ip}:6443 {k3s_install_flags}'
+      init_cmd = f'{k3s_install_base}{k3s_token_cmd} sh -s - server --server https://{network_ip}:6443'
 
     # worker
     if nodetype == 'worker':
-      init_cmd = f'{k3s_install_worker}{k3s_token_cmd} sh -s'
+      init_cmd = f'rm -rf /etc/rancher/k3s/config.yaml.d/ && {k3s_install_worker}{k3s_token_cmd} sh -s'
 
     # stderr
     init_cmd = init_cmd + f' > /k3s_{nodetype}_install.log 2>&1'

From afaf193c79f035c370d99aab4d8e41e8b7ee4ee6 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 5 Jan 2025 10:37:07 +0000
Subject: [PATCH 16/30] some more config level tweaks

---
 lib/kopsrox_k3s.py | 5 ++---
 lib/verb_image.py  | 4 +++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 2dd5aed..9d0e99c 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -224,9 +224,8 @@ def kubeconfig():
   kubeconfig_file = f'{cluster_name}.kubeconfig'
   # replace 127.0.0.1 with vip ip
   kconfig = qaexec(masterid, 'cat /etc/rancher/k3s/k3s.yaml').replace('127.0.0.1', network_ip)
-  # write file with 600 permissions
-  with os.fdopen(os.open(kubeconfig_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as handle:
-    handle.write(kconfig)
+  with open(kubeconfig_file, 'w') as kubeconfig_file_handle:
+    kubeconfig_file_handle.write(kconfig)
   kmsg('k3s_kubeconfig', f'saved {kubeconfig_file}')
 
 # kubectl
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 6f91e90..b194eed 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -101,9 +101,11 @@
 echo -n '
 disable:
   - servicelb
-write-kubeconfig-mode: "0644"
+disable-network-policy: true
+write-kubeconfig-mode: "0600"
 disable-cloud-controller: true
 disable-network-policy: true
+flannel-backend: wireguard-native
 tls-san: {network_ip}' > /etc/rancher/k3s/config.yaml.d/kopsrox.yaml
 '''
   # shouldn't really need root/sudo but run into permissions problems

From 53c9aed785aa9dfc502f75fd1a253e8852bf4f0e Mon Sep 17 00:00:00 2001
From: simonc <simonc@gmail.com>
Date: Fri, 24 Jan 2025 19:00:18 +0000
Subject: [PATCH 17/30] Update SETUP.md

---
 docs/SETUP.md | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/docs/SETUP.md b/docs/SETUP.md
index d8df332..66a7955 100644
--- a/docs/SETUP.md
+++ b/docs/SETUP.md
@@ -1,4 +1,4 @@
-# :hamburger: kopsrox setup 
+# kopsrox setup 
 
 ## :hammer_and_wrench: requirements
 
@@ -23,9 +23,9 @@
 
 run `./kopsrox.py` and an example _kopsrox.ini_ will be generated - you will need to edit this for your setup
 
-Most values should be obvious and commented accordingly - see below for more info
+Most values should be hopefully obvious and commented accordingly...
 
-# :mag_right: kopsrox.ini 
+# kopsrox.ini 
 
 ## :computer: cluster_id 
 
@@ -60,18 +60,3 @@ would result in this:
 |9|629|192.168.0.179|worker 5|kopsrox-w5|
 
 The VIP IP ( here 192.168.0.170 ) is used by kube-vip to provide a highly available IP for the API when you have 3 master nodes
-
-## :pencil2: kopsrox
-
-### :rainbow: cloud_image_url 
-
-`https://cloud-images.ubuntu.com/minimal/daily/mantic/current/mantic-minimal-cloudimg-amd64.img` 
-
-url to the cloud image you want to use as the koprox base template. 
-
-during `kopsrox.py image create` this is downloaded and patched via `virt-customise` to install `qemu-guest-agent`
-
-Tested images so far: 
-
-https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2
-https://mirrors.vinters.com/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2

From 8b9ce08c43c5c96883d920a1ce1c6b5144e43ef5 Mon Sep 17 00:00:00 2001
From: simonc <simonc@gmail.com>
Date: Fri, 24 Jan 2025 19:00:51 +0000
Subject: [PATCH 18/30] Update FAQ.md

---
 docs/FAQ.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/FAQ.md b/docs/FAQ.md
index e964254..11735da 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -1,4 +1,4 @@
-# :question: FAQ
+# kopsrox FAQ
 
 :question:  __can I migrate the kopsrox vms to other hosts in my proxmox cluster?__
 

From 9398bf7cc85fa2228d786ce84b59ce7b7bc79589 Mon Sep 17 00:00:00 2001
From: simonc <simonc@gmail.com>
Date: Fri, 24 Jan 2025 19:01:53 +0000
Subject: [PATCH 19/30] Update GETSTARTED.md

---
 docs/GETSTARTED.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/GETSTARTED.md b/docs/GETSTARTED.md
index 17a74d8..c2e174b 100644
--- a/docs/GETSTARTED.md
+++ b/docs/GETSTARTED.md
@@ -1,6 +1,6 @@
-# :burger:  get started
+# get started
 
-## 🇧🇸 setup kopsrox.ini
+## setup a kopsrox.ini
 
 `./kopsrox.py` - a default kopsrox.ini will be created
 
@@ -12,13 +12,13 @@ follow the guide in [SETUP.md](SETUP.md)
 
 `./kopsrox.py image create`
 
-## 🥑 create a cluster
+## create a cluster
 
 `./kopsrox.py cluster create`
 
 ## 🚑 add a worker
 
-edit `kopsrox.ini` and set `workers = 1` in the `[cluster]` section
+for example edit `kopsrox.ini` and set `workers = 1` in the `[cluster]` section
 
 `./kopsrox.py cluster update`
 

From cf5e61ac7e2086adec847cfaa38c29b875d81dbe Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 1 Feb 2025 15:51:28 +0000
Subject: [PATCH 20/30] adds cluster restore option

---
 docs/IMAGES.md         |  5 ++++
 kopsrox.py             |  7 +++---
 lib/kopsrox_config.py  | 39 +++++++----------------------
 lib/kopsrox_k3s.py     | 56 ++++++++++++++++++++++++++++++++++++------
 lib/kopsrox_proxmox.py |  2 +-
 lib/verb_cluster.py    | 18 +++++++++-----
 lib/verb_etcd.py       | 11 ++++++---
 lib/verb_k3s.py        |  5 +---
 8 files changed, 88 insertions(+), 55 deletions(-)
 create mode 100644 docs/IMAGES.md

diff --git a/docs/IMAGES.md b/docs/IMAGES.md
new file mode 100644
index 0000000..8d70d4b
--- /dev/null
+++ b/docs/IMAGES.md
@@ -0,0 +1,5 @@
+# cloud images
+
+
+# amazon linux ( qcow ) 
+https://cdn.amazonlinux.com/al2023/os-images/2023.6.20250123.4/
diff --git a/kopsrox.py b/kopsrox.py
index 0ed64c5..dc47310 100755
--- a/kopsrox.py
+++ b/kopsrox.py
@@ -26,6 +26,7 @@
     "create" : '',
     "update" : '',
     "destroy" : '',
+    'restore' : '',
   },
   "k3s": {
     "export-token" : '',
@@ -84,12 +85,12 @@ def cmds_help(verb):
 
     # if verb not found in cmds dict
     if not verb in verbs:
-      exit()
+      exit(0)
 
 # verb not found or passed
 except:
   verbs_help()
-  exit()
+  exit(0)
 
 # handle command
 try:
@@ -102,7 +103,7 @@ def cmds_help(verb):
     if not cmd in list(cmds[verb]):
       exit()
 
-# 
+# cmd not found
 except:
   cmds_help(verb)
   exit()
diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 99331c9..97a64a7 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -4,12 +4,9 @@
 import urllib3
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 import urllib.parse
-import wget
 from datetime import datetime
 from proxmoxer import ProxmoxAPI
-
-# checks cmd line args file ops and processes
-import re,os,sys,subprocess,time
+import re,os,sys,subprocess,time,wget
 
 # kmsg
 from kopsrox_kmsg import kmsg
@@ -61,12 +58,11 @@ def conf_check(section,value):
     # return string
     return(config_item)
 
-# check config vars
-# cluster name as required for error messages
+# cluster name 
 cluster_name = conf_check('cluster', 'cluster_name')
 kname = cluster_name + '_config-check'
 
-# get cluster id
+# cluster id
 cluster_id = conf_check('cluster','cluster_id')
 if cluster_id < 100:
   kmsg(kname, f' cluster_id is too low - should be over 100', 'err')
@@ -81,12 +77,17 @@ def conf_check(section,value):
 node = conf_check('proxmox','node')
 storage = conf_check('proxmox','storage')
 
-# kopsrox config checks
+# kopsrox 
 cloud_image_url = conf_check('kopsrox','cloud_image_url')
 vm_disk = conf_check('kopsrox','vm_disk')
 vm_cpu = conf_check('kopsrox','vm_cpu')
 vm_ram = conf_check('kopsrox','vm_ram')
 
+# ram size check
+if vm_ram < 2:
+  kmsg(kname, f'[kopsrox]/vm_ram - kopsrox vms need 2G RAM', 'err')
+  exit(0)
+
 # cloudinit
 cloudinituser = conf_check('kopsrox','cloudinituser')
 cloudinitpass = conf_check('kopsrox','cloudinitpass')
@@ -173,9 +174,6 @@ def conf_check(section,value):
 
 except:
   kmsg(kname, f'API connection to {prox_endpoint}:{port} failed check [proxmox] settings', 'err')
-
-  # this could be improved
-  #kmsg(kname, prox.cluster.status.get(), 'sys')
   exit()
 
 # look up kopsrox_img name
@@ -356,25 +354,6 @@ def vmip(vmid: int):
   ip = f'{network_base}{(network_ip_prefix + (vmid - cluster_id))}'
   return(ip)
 
-# cluster info
-def cluster_info():
-  kmsg(f'cluster_info', '', 'sys')
-  from kopsrox_k3s import kubectl, get_kube_vip_master
-  curr_master = get_kube_vip_master()
-  info_vms = list_kopsrox_vm()
-
-  # for kopsrox vms
-  for vmid in info_vms:
-    if not cluster_id == vmid:
-      hostname = vmnames[vmid]
-      vmstatus = f'[{info_vms[vmid]}] {vmip(vmid)}/{network_mask}'
-      if hostname == curr_master:
-        vmstatus += f' vip {network_ip}/{network_mask}'
-      kmsg(f'{hostname}_{vmid}', f'{vmstatus}')
-
-  # fix this 
-  kmsg('kubectl_get-nodes', f'\n{kubectl("get nodes")}')
-
 # run local os process 
 def local_os_process(cmd):
   try:
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 9d0e99c..b09fdad 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -2,12 +2,7 @@
 
 # imports
 from kopsrox_config import *
-
-# standard imports
-from kopsrox_proxmox import qaexec, prox_destroy, internet_check, clone
-
-# standard imports
-import time, os
+from kopsrox_proxmox import * 
 
 # check for k3s status
 def k3s_check(vmid: int):
@@ -32,7 +27,7 @@ def k3s_check(vmid: int):
 def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
   # nodetype error check
-  if nodetype not in ['master', 'slave', 'worker']:
+  if nodetype not in ['master', 'slave', 'worker', 'restore']:
     kmsg('k3s_init-node', f'{nodetype} invalid nodetype', 'err')
     exit(0)
 
@@ -47,6 +42,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
   k3s_install_base = f'cat /k3s.sh | INSTALL_K3S_VERSION="{k3s_version}"'
   k3s_install_master = f'{k3s_install_base} sh -s - server --cluster-init'
   k3s_install_worker = f'{k3s_install_base} K3S_URL="https://{network_ip}:6443" '
+
   master_cmd = ''
   token = ''
  
@@ -55,7 +51,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
     if not k3s_check(vmid):
       exit(0)
   except:
-    kmsg(f'k3s_{nodetype}-init', f'installing {k3s_version} on {vmnames[vmid]}')
+    kmsg(f'k3s_{nodetype}-init', f'configuring {k3s_version} on {vmnames[vmid]}')
 
     # get existing token if it exists
     token_fname = f'{cluster_name}.k3stoken'
@@ -79,6 +75,25 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
     if nodetype == 'worker':
       init_cmd = f'rm -rf /etc/rancher/k3s/config.yaml.d/ && {k3s_install_worker}{k3s_token_cmd} sh -s'
 
+    # restore
+    if nodetype == 'restore':
+      kmsg(f'k3s_bootstrap', f'fetching latest snapshot')
+
+      # get latest snapshot
+      bs_cmd = f'systemctl start k3s && /usr/local/bin/k3s etcd-snapshot ls 2>&1 && systemctl stop k3s && rm -rf /var/lib/rancher'
+      bs_cmd_out = qaexec(vmid,bs_cmd)
+
+      # sort ls output so last is latest snapshot
+      for snap in sorted(bs_cmd_out.split('\n')):
+        if re.search(f'kopsrox-{cluster_name}', snap):
+            latest = snap
+
+      latest_snap = latest.split()[0]
+
+      kmsg(f'k3s_restore', f'restoring {latest_snap}')
+
+      init_cmd = f'/usr/local/bin/k3s server --cluster-reset --cluster-reset-restore-path={latest_snap} --token={token} 2>&1 && systemctl start k3s'
+
     # stderr
     init_cmd = init_cmd + f' > /k3s_{nodetype}_install.log 2>&1'
 
@@ -276,6 +291,31 @@ def export_k3s_token():
       token_file.write(live_token)
     kmsg('k3s_export-token', f'created: {token_name}')
 
+# cluster info
+def cluster_info():
+
+  try:
+    node = vms[masterid]
+  except:
+    kmsg(f'{kname}-check', 'cluster does not exist', 'err')
+    exit(0)
+
+  kmsg(f'cluster_info', '', 'sys')
+  curr_master = get_kube_vip_master()
+  info_vms = list_kopsrox_vm()
+
+  # for kopsrox vms
+  for vmid in info_vms:
+    if not cluster_id == vmid:
+      hostname = vmnames[vmid]
+      vmstatus = f'[{info_vms[vmid]}] {vmip(vmid)}/{network_mask}'
+      if hostname == curr_master:
+        vmstatus += f' vip {network_ip}/{network_mask}'
+      kmsg(f'{hostname}_{vmid}', f'{vmstatus}')
+
+  # fix this
+  kmsg('kubectl_get-nodes', f'\n{kubectl("get nodes")}')
+
 # return current vip master
 def get_kube_vip_master():
   kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}'
diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index 4fc2b35..b86ea05 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -138,7 +138,7 @@ def clone(vmid):
 
   # hostname
   hostname = vmnames[vmid]
-  kmsg('proxmox_clone', f'{hostname} {ip} {vm_cpu}c/{vm_ram}G ram {vm_disk}G disk')
+  kmsg('proxmox_clone', f'{hostname} {ip} {vm_cpu}c/{vm_ram}G {vm_disk}G')
 
   # clone
   prox_task(prox.nodes(node).qemu(cluster_id).clone.post(newid = vmid))
diff --git a/lib/verb_cluster.py b/lib/verb_cluster.py
index ce6dc18..9d89d0d 100755
--- a/lib/verb_cluster.py
+++ b/lib/verb_cluster.py
@@ -1,13 +1,9 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_config import masterid,cluster_name,cluster_info,list_kopsrox_vm,cluster_id
+from kopsrox_config import *
 from kopsrox_proxmox import clone,qaexec
-from kopsrox_k3s import k3s_update_cluster,k3s_rm_cluster,k3s_init_node,export_k3s_token
-from kopsrox_kmsg import kmsg
-
-# other imports
-import sys
+from kopsrox_k3s import * 
 
 # passed command
 cmd = sys.argv[2]
@@ -23,6 +19,16 @@
 if cmd == 'update':
   k3s_update_cluster()
 
+# restore from latest etcd snapshot
+if cmd == 'restore':
+  k3s_rm_cluster()
+  kmsg(kname,f'id:{cluster_id} name:{cluster_name}', 'sys')
+  clone(masterid)
+  k3s_init_node(masterid, 'restore')
+  cluster_info()
+  kmsg(kname,f'restore completed')
+  k3s_update_cluster()
+
 # create new cluster / master server
 if cmd == 'create':
 
diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py
index c6c7210..d00178a 100755
--- a/lib/verb_etcd.py
+++ b/lib/verb_etcd.py
@@ -2,8 +2,8 @@
 
 # kopsrox
 from kopsrox_config import *
-from kopsrox_proxmox import qaexec
-from kopsrox_k3s import k3s_rm_cluster, kubectl, k3s_update_cluster, export_k3s_token, kubeconfig
+from kopsrox_proxmox import *
+from kopsrox_k3s import *
 
 # passed command
 cmd = sys.argv[2]
@@ -13,7 +13,6 @@
 token_fname = cluster_name + '.k3stoken'
 
 # check master is running / exists
-# fails if node can't be found
 try:
   node = vms[masterid]
 except:
@@ -57,6 +56,12 @@ def list_snapshots():
 # test connection to s3 by getting list of snapshots
 snapshots = list_snapshots()
 
+try:
+  node = vms[masterid]
+except:
+  kmsg(f'{kname}-check', 'cluster does not exist', 'err')
+  exit(0)
+
 # s3 prune
 if cmd == 'prune':
   kmsg(f'{kname}-prune', (f'{s3endpoint}/{bucket}\n' + s3_run('prune --name kopsrox')), 'sys')
diff --git a/lib/verb_k3s.py b/lib/verb_k3s.py
index e7c1705..df1afcb 100755
--- a/lib/verb_k3s.py
+++ b/lib/verb_k3s.py
@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
 
 # functions
-from kopsrox_k3s import export_k3s_token, kubeconfig, k3s_check_config, kubectl
+from kopsrox_k3s import * 
 from kopsrox_kmsg import kmsg
 
-# other imports
-import sys
-
 # passed command
 cmd = sys.argv[2]
 

From 76889453218b09a2f2f43886b10f24c01783947c Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 1 Feb 2025 18:15:38 +0000
Subject: [PATCH 21/30] pre lenny

---
 lib/kopsrox_k3s.py  | 17 ++++++-----------
 lib/kopsrox_kmsg.py | 10 +++++-----
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index b09fdad..9713537 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -125,14 +125,15 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
 # remove a node
 def k3s_remove_node(vmid: int):
+  
+  # get vmname
   vmname = vmnames[vmid]
   kmsg('k3s_remove-node', vmname)
 
-  # kubectl commands to remove node
-  # should add some error checking
-  kubectl('cordon ' + vmname)
-  kubectl('drain --timeout=10s --delete-emptydir-data --ignore-daemonsets --force ' + vmname)
-  kubectl('delete node ' + vmname)
+  if vmname != f'{cluster_name}-m1':
+    kubectl('cordon ' + vmname)
+    kubectl('drain --timeout=10s --delete-emptydir-data --ignore-daemonsets --force ' + vmname)
+    kubectl('delete node ' + vmname)
 
   # destroy vm
   prox_destroy(vmid)
@@ -294,12 +295,6 @@ def export_k3s_token():
 # cluster info
 def cluster_info():
 
-  try:
-    node = vms[masterid]
-  except:
-    kmsg(f'{kname}-check', 'cluster does not exist', 'err')
-    exit(0)
-
   kmsg(f'cluster_info', '', 'sys')
   curr_master = get_kube_vip_master()
   info_vms = list_kopsrox_vm()
diff --git a/lib/kopsrox_kmsg.py b/lib/kopsrox_kmsg.py
index 6ea7fb3..9d93d56 100755
--- a/lib/kopsrox_kmsg.py
+++ b/lib/kopsrox_kmsg.py
@@ -11,7 +11,7 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'):
 
   # print cluster name
   cprint(knamea[0], "blue",attrs=["bold"], end='')
-  cprint(':', "cyan", end='')
+  cprint(':<:', "cyan", end='')
 
   try:
     if knamea[1] and sev == 'info':
@@ -21,9 +21,9 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'):
     if knamea[1] and sev == 'sys':
       cprint(knamea[1], "yellow", attrs=["bold"],end='')
   except:
-      cprint('parse error ', "magenta", attrs=["bold"], end='')
-      print(kname,msg)
+    cprint('parse error ', "magenta", attrs=["bold"], end='')
+    print(kname,msg)
 
   # final output
-  cprint(': ', "cyan", end='')
-  print(msg)
+  cprint(':>: ', "cyan", end='')
+  print(msg )

From 18b910f70a3a2f76701d7f40d0389fe9d735d8fa Mon Sep 17 00:00:00 2001
From: simonc <simonc@gmail.com>
Date: Sun, 2 Feb 2025 13:53:24 +0000
Subject: [PATCH 22/30] Update README.md

---
 README.md | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 51cc31b..ca31c42 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,19 @@
-# :hamburger: kopsrox
+# kopsrox
 
-- kopsrox is a script to automate creation and management of simple ha k3s cluster on ProxmoxVE using cloud images :nerd_face:
+- kopsrox is a script to create simple ha k3s cluster on ProxmoxVE
+- use upstream cloud images - no iso's to mess around with 
 - add more master/worker k3s nodes using a simple config file and cli interface :pray:
-- kube-vip ( https://kube-vip.io/ ) built in providing full HA setup for the kube api :atom:
+- kube-vip ( https://kube-vip.io/ ) built in providing full HA setup for the kube api and traefik :atom:
 - easy management of etcd S3 snapshot/restore operations - easily restore a cluster from s3! :floppy_disk:
 - export the k3s token, your kubeconfig etc etc - its all automatic  :nerd_face:
 
   get it https://github.com/simonccc/kopsrox/releases
 
-# :book: docs
+#  docs
  - [SETUP.md](docs/SETUP.md)
  - [GETSTARTED.md](docs/GETSTARTED.md)
  - [USAGE.md](docs/USAGE.md)
  - [FAQ.md](docs/FAQ.md)
 
-# :boom: in progress 
- - Improving Docs
- - Some code clean up
- - Recent: bug fixes, machine type optimisations, kubevip improvements
+#  in progress 
+ - Recent: add cluster restore option

From 58a5d855503ca536b2b3b9cc1b59d3052602e17e Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 2 Feb 2025 19:22:53 +0000
Subject: [PATCH 23/30] start to speed up config check

---
 lib/kopsrox_config.py  | 81 ++++++++++++++++++++++--------------------
 lib/kopsrox_proxmox.py |  4 +++
 2 files changed, 46 insertions(+), 39 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index 97a64a7..ceef3fa 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -58,6 +58,7 @@ def conf_check(section,value):
     # return string
     return(config_item)
 
+
 # cluster name 
 cluster_name = conf_check('cluster', 'cluster_name')
 kname = cluster_name + '_config-check'
@@ -68,22 +69,46 @@ def conf_check(section,value):
   kmsg(kname, f' cluster_id is too low - should be over 100', 'err')
   exit(0)
 
-# proxmox
-prox_endpoint = conf_check('proxmox','prox_endpoint')
-port = conf_check('proxmox','port')
-user = conf_check('proxmox','user')
-token_name = conf_check('proxmox','token_name')
-api_key = conf_check('proxmox','api_key')
+# assign master id
+masterid = cluster_id + 1
+
+# test connection to proxmox
+try:
+
+  # api connection
+  prox = ProxmoxAPI(
+    conf_check('proxmox','prox_endpoint'),
+    port=conf_check('proxmox','port'),
+    user=conf_check('proxmox','user'),
+    token_name=conf_check('proxmox','token_name'),
+    token_value=conf_check('proxmox','api_key'),
+    verify_ssl=False,
+    timeout=5)
+
+  # check connection to cluster
+  prox.cluster.status.get()
+
+except:
+  kmsg(kname, f'API connection to Proxmox failed check [proxmox] settings', 'err')
+  exit(0)
+
+# proxmox cont
 node = conf_check('proxmox','node')
+discovered_nodes = [node.get('node', None) for node in prox.nodes.get()]
+if node not in discovered_nodes:
+  kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err')
+  exit(0)
+
+# storage
 storage = conf_check('proxmox','storage')
 
 # kopsrox 
 cloud_image_url = conf_check('kopsrox','cloud_image_url')
 vm_disk = conf_check('kopsrox','vm_disk')
 vm_cpu = conf_check('kopsrox','vm_cpu')
-vm_ram = conf_check('kopsrox','vm_ram')
 
-# ram size check
+# ram size  and check
+vm_ram = conf_check('kopsrox','vm_ram')
 if vm_ram < 2:
   kmsg(kname, f'[kopsrox]/vm_ram - kopsrox vms need 2G RAM', 'err')
   exit(0)
@@ -139,9 +164,6 @@ def conf_check(section,value):
 # dict of all config items - legacy support
 config = ({s:dict(kopsrox_config.items(s)) for s in kopsrox_config.sections()})
 
-# define masterid
-masterid = cluster_id + 1
-
 # define vmnames
 vmnames = {
 (cluster_id): cluster_name +'-i0',
@@ -156,26 +178,6 @@ def conf_check(section,value):
 (cluster_id + 9 ): cluster_name + '-w5',
 }
 
-# proxmox api connection
-try: 
-
-  # api connection
-  prox = ProxmoxAPI(
-    prox_endpoint,
-    port=port,
-    user=user,
-    token_name=token_name,
-    token_value=api_key,
-    verify_ssl=False,
-    timeout=5)
-
-  # check connection to cluster
-  prox.cluster.status.get()
-
-except:
-  kmsg(kname, f'API connection to {prox_endpoint}:{port} failed check [proxmox] settings', 'err')
-  exit()
-
 # look up kopsrox_img name
 def kopsrox_img():
 
@@ -217,14 +219,6 @@ def list_kopsrox_vm():
 def vm_info(vmid,node=node):
   return(prox.nodes(node).qemu(vmid).status.current.get())
 
-# get list of nodes
-discovered_nodes = [node.get('node', None) for node in prox.nodes.get()]
-
-# if node not in list of nodes
-if node not in discovered_nodes:
-  kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err')
-  exit()
-
 # get list of storage in the cluster
 storage_list = prox.nodes(node).storage.get()
 
@@ -373,3 +367,12 @@ def image_info():
   kmsg(f'{kname}desc', cloud_image_desc)
   kmsg(f'{kname}storage', f'{kopsrox_image_name} ({storage_type})')
   kmsg(f'{kname}size', f'{cloud_image_size}G')
+
+# tbc
+def progress_bar(iteration, total, prefix='', suffix='', length=30, fill='^'):
+  percent = ("{0:.1f}").format(100 * (iteration / float(total)))
+  filled_length = int(length * iteration // total)
+  bar = fill * filled_length + '-' * (length - filled_length)
+  sys.stdout.write(f'\r{prefix} |{bar}| {percent}% {suffix}')
+  sys.stdout.flush()
+
diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py
index b86ea05..e024c82 100755
--- a/lib/kopsrox_proxmox.py
+++ b/lib/kopsrox_proxmox.py
@@ -46,6 +46,10 @@ def qaexec(vmid: int = masterid,cmd = 'uptime', node: str = node):
       # sleep 1 second then try again
       time.sleep(1)
 
+      #progress_bar(qagent_count, 30, prefix='', suffix='')
+      if qagent_count == 10:
+        kmsg(kname, f'no response for 10s {vmname} [{node}] cmd: {cmd}', 'sys')
+
   # send command
   try: 
     qa_exec = prox.nodes(node).qemu(vmid).agent.exec.post(

From 4a33dc0442aeca83fcddcb3a507ff563215cbbf3 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Fri, 7 Feb 2025 22:53:26 +0000
Subject: [PATCH 24/30] lastest fixes

---
 lib/kopsrox_config.py | 20 +++++++++++++++++---
 lib/kopsrox_k3s.py    | 15 ++++++---------
 lib/kopsrox_kmsg.py   |  7 ++++---
 lib/verb_etcd.py      |  2 +-
 lib/verb_image.py     | 16 +++++++++++++---
 5 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/lib/kopsrox_config.py b/lib/kopsrox_config.py
index ceef3fa..fac309d 100755
--- a/lib/kopsrox_config.py
+++ b/lib/kopsrox_config.py
@@ -92,12 +92,26 @@ def conf_check(section,value):
   kmsg(kname, f'API connection to Proxmox failed check [proxmox] settings', 'err')
   exit(0)
 
-# proxmox cont
+# map passed node name
 node = conf_check('proxmox','node')
+
+# try k8s ping
+try:
+  k3s_ping = prox.nodes(node).qemu(masterid).agent.exec.post(command = '/usr/local/bin/k3s kubectl version')
+  #print(pingv)
+except:
+  try:
+    qa_ping = prox.nodes(node).qemu(masterid).agent.ping.post()
+    kmsg(kname, f'k3s down but master up please investigate...', 'err')
+    exit(0)
+  except:
+    pass
+
+# proxmox cont
 discovered_nodes = [node.get('node', None) for node in prox.nodes.get()]
 if node not in discovered_nodes:
-  kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err')
-  exit(0)
+ kmsg(kname, f'"{node}" not found - discovered nodes: {discovered_nodes}', 'err')
+ exit(0)
 
 # storage
 storage = conf_check('proxmox','storage')
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 9713537..179fd85 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -86,15 +86,12 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
       # sort ls output so last is latest snapshot
       for snap in sorted(bs_cmd_out.split('\n')):
         if re.search(f'kopsrox-{cluster_name}', snap):
-            latest = snap
+            latest = snap.split()[0]
 
-      latest_snap = latest.split()[0]
+      kmsg(f'k3s_restore', f'restoring {latest}')
 
-      kmsg(f'k3s_restore', f'restoring {latest_snap}')
-
-      init_cmd = f'/usr/local/bin/k3s server --cluster-reset --cluster-reset-restore-path={latest_snap} --token={token} 2>&1 && systemctl start k3s'
-
-    # stderr
+      init_cmd = f'/usr/local/bin/k3s server --cluster-reset --cluster-reset-restore-path={latest} --token={token} 2>&1 && systemctl start k3s'
+    # write log of install on node
     init_cmd = init_cmd + f' > /k3s_{nodetype}_install.log 2>&1'
 
     # run command
@@ -119,7 +116,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
         time.sleep(1)
 
     # final steps for first master  - kubevip, export kubeconfig and token 
-    if nodetype == 'master':
+    if nodetype in [ 'master', 'restore']:
       kubeconfig()
       export_k3s_token()
 
@@ -294,7 +291,7 @@ def export_k3s_token():
 
 # cluster info
 def cluster_info():
-
+ 
   kmsg(f'cluster_info', '', 'sys')
   curr_master = get_kube_vip_master()
   info_vms = list_kopsrox_vm()
diff --git a/lib/kopsrox_kmsg.py b/lib/kopsrox_kmsg.py
index 9d93d56..a0fe1e0 100755
--- a/lib/kopsrox_kmsg.py
+++ b/lib/kopsrox_kmsg.py
@@ -11,7 +11,8 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'):
 
   # print cluster name
   cprint(knamea[0], "blue",attrs=["bold"], end='')
-  cprint(':<:', "cyan", end='')
+  cprint('-', "magenta",attrs=["bold"], end='')
+  cprint('<:', "cyan", end='')
 
   try:
     if knamea[1] and sev == 'info':
@@ -25,5 +26,5 @@ def kmsg(kname = 'kopsrox',msg = 'no msg', sev = 'info'):
     print(kname,msg)
 
   # final output
-  cprint(':>: ', "cyan", end='')
-  print(msg )
+  cprint(':> ', "cyan", end='')
+  print(msg)
diff --git a/lib/verb_etcd.py b/lib/verb_etcd.py
index d00178a..b2c0f51 100755
--- a/lib/verb_etcd.py
+++ b/lib/verb_etcd.py
@@ -81,7 +81,7 @@ def list_snapshots():
   # filter output
   snapout = snapout.split('\n')
   for line in snapout:
-    if re.search('upload complete', line):
+    if re.search('Snapshot', line):
       kmsg(kname, line)
 
 # print s3List
diff --git a/lib/verb_image.py b/lib/verb_image.py
index b194eed..610558e 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -63,6 +63,16 @@
 INSTALL_K3S_VERSION={k3s_version} \
 sh -s - server --cluster-init > /{cluster_name}-image-install.log 2>&1
 
+if [ ! -f /usr/bin/qemu-ga ] 
+then
+  if [ -f /bin/yum ]  
+  then 
+    yum install -y qemu-guest-agent
+  else
+    apt update && apt install qemu-guest-agent -y 
+  fi 
+fi
+
 if [ -f /etc/selinux/config ] 
 then
   sed -i s/enforcing/disabled/g /etc/selinux/config
@@ -106,13 +116,13 @@
 disable-cloud-controller: true
 disable-network-policy: true
 flannel-backend: wireguard-native
-tls-san: {network_ip}' > /etc/rancher/k3s/config.yaml.d/kopsrox.yaml
+tls-san: {network_ip}' > /etc/rancher/k3s/config.yaml
 '''
   # shouldn't really need root/sudo but run into permissions problems
   kmsg(f'{kname}virt-customize', 'configuring image')
   virtc_cmd = f'''
-sudo virt-customize --smp 2 -m 2048 -a {cloud_image}   \
---install qemu-guest-agent --run-command "{virtc_script}"  \
+sudo virt-customize -a {cloud_image}   \
+--run-command "{virtc_script}"  \
 --copy-in {kv_yaml}:/var/lib/rancher/k3s/server/manifests/'''
   local_os_process(virtc_cmd)
 

From 28c7ec33e1ef6a732011f9fa78d2dd2a3e6ad02b Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 8 Feb 2025 16:04:21 +0000
Subject: [PATCH 25/30] add worker fix

---
 lib/kopsrox_k3s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index 179fd85..e83953e 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -73,7 +73,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # worker
     if nodetype == 'worker':
-      init_cmd = f'rm -rf /etc/rancher/k3s/config.yaml.d/ && {k3s_install_worker}{k3s_token_cmd} sh -s'
+      init_cmd = f'rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s'
 
     # restore
     if nodetype == 'restore':
@@ -116,7 +116,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
         time.sleep(1)
 
     # final steps for first master  - kubevip, export kubeconfig and token 
-    if nodetype in [ 'master', 'restore']:
+    if nodetype in ['master', 'restore']:
       kubeconfig()
       export_k3s_token()
 

From 6764f1689af4432489ad3d5260246575eb60a91d Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 9 Feb 2025 20:02:37 +0000
Subject: [PATCH 26/30] dev helper script update

---
 dev/img-mirror/.gitignore |  2 ++
 dev/img-mirror/get.sh     | 13 ++++-----
 dev/img-mirror/serve.sh   |  4 ++-
 dev/rls_test.sh           | 61 +++++++++++----------------------------
 4 files changed, 28 insertions(+), 52 deletions(-)

diff --git a/dev/img-mirror/.gitignore b/dev/img-mirror/.gitignore
index e56abd9..3fba00b 100644
--- a/dev/img-mirror/.gitignore
+++ b/dev/img-mirror/.gitignore
@@ -1,2 +1,4 @@
 *.img
 *.qcow
+log.txt
+PID
diff --git a/dev/img-mirror/get.sh b/dev/img-mirror/get.sh
index 7db46fd..03a5301 100755
--- a/dev/img-mirror/get.sh
+++ b/dev/img-mirror/get.sh
@@ -1,15 +1,14 @@
 # downloads image for use with serve.sh
-ubr="oracular"
-if [ ! -f "${ubr}-minimal-cloudimg-amd64.img" ] 
+if [ ! -f "noble-server-cloudimg-amd64.img" ] 
 then
-wget "https://cloud-images.ubuntu.com/minimal/daily/${ubr}/current/${ubr}-minimal-cloudimg-amd64.img"
+wget "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
 fi 
 
 # untested
-#if [ ! -f amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 ] 
-#then
-#wget https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2
-#fi
+if [ ! -f amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2 ] 
+then
+wget https://cdn.amazonlinux.com/os-images/2.0.20240306.2/kvm/amzn2-kvm-2.0.20240306.2-x86_64.xfs.gpt.qcow2
+fi
 #if [ ! -f Rocky-9-GenericCloud.latest.x86_64.qcow2 ] 
 #then
 #wget https://mirrors.vinters.com/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2
diff --git a/dev/img-mirror/serve.sh b/dev/img-mirror/serve.sh
index 642106b..d4da3a4 100755
--- a/dev/img-mirror/serve.sh
+++ b/dev/img-mirror/serve.sh
@@ -1,2 +1,4 @@
 #!/usr/bin/env bash
-python3 -m http.server -b "::" 
+kill -9 $(cat PID)
+python3 -m http.server -b "::"  > log.txt 2>&1 &
+echo $! > PID
diff --git a/dev/rls_test.sh b/dev/rls_test.sh
index 877511c..e1d422c 100644
--- a/dev/rls_test.sh
+++ b/dev/rls_test.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 set -e
 #set -x
 
@@ -10,7 +12,7 @@ KC="$K cluster"
 KCI="$KC info"
 KCC="$KC create"
 KCU="$KC update"
-KCD="$KC destroy"
+KCR="$KC restore"
 KI="$K image"
 KID="$KI destroy"
 KIC="$KI create"
@@ -30,31 +32,19 @@ kc() {
 # get pods
 get_pods="$KC kubectl get pods -A"
 
-# 0 size cluster
-kc workers 0 ; kc masters 1
+# 1 size cluster
 $KCD
-
-# create image
-$KIC
-
-# create / update cluster
-$KCC ; $KCU
-
-# take snapshot
-$KES
-
-# destroy cluster
+kc workers 0 ; kc masters 1 
 $KCD
 
-# create / update cluster
-$KCC ; $KCU
-
-# restore snapshot
-$KERL
+# ** 1 MASTER, SNAPSHOT RESTOR
+# create image, create and update cluster
+$KIC ; $KCC ; $KCU
 
-# update cluster
-$KCU
+# take snapshot , destroy cluster, create, restore
+$KES ; $KCD ; $KCC ; $KERL
 
+# ** MULTIPLE MASTERS AND WORKERS TEST
 # add a worker and delete it
 kc workers 1 ; $KCU ; kc workers 0 ; $KCU
 
@@ -67,32 +57,15 @@ kc masters 3 ; $KCU ; kc masters 1  ; $KCU
 # add 3 masters 
 kc masters 3 ; $KCU 
 
-# take snapshot
-$KES
-
-# destroy cluster
-$KCD
-
-# create / update cluster
-$KCC ; $KCU
-#
-# # restore snapshot
-$KERL
-#
-# update cluster
-$KCU
+# take snapshot , destroy, create, restore
+$KES ; $KCD ; $KCC ; $KCU ; $KERL ; $KCD
 
 # change back to 1 node
-kc masters 1 ; $KCU ; kc workers 0  ; $KCU
-
-# destroy cluster
-$KCD
-
-# create / update cluster
-$KCC ; $KCU
-#restore snapshot
-$KERL
+kc masters 1 ; kc workers 0  
 
+# ** TEST cluster restore
+# destroy cluster 
+$KCR
 
 finish_time=$(date +%s) 
 echo  $((finish_time - start_time)) secs

From 0b82ed7161dd80d01593e746ccae40b67cf1b267 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 9 Feb 2025 21:04:25 +0000
Subject: [PATCH 27/30] fixes to node creation

---
 lib/kopsrox_k3s.py | 9 ++++-----
 lib/verb_image.py  | 3 ---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index e83953e..ea4244e 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -39,9 +39,8 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
     exit(0)
 
   # defines
-  k3s_install_base = f'cat /k3s.sh | INSTALL_K3S_VERSION="{k3s_version}"'
-  k3s_install_master = f'{k3s_install_base} sh -s - server --cluster-init'
-  k3s_install_worker = f'{k3s_install_base} K3S_URL="https://{network_ip}:6443" '
+  k3s_install_master = f'cat /k3s.sh | sh -s - server --cluster-init'
+  k3s_install_worker = f'cat /k3s.sh | K3S_URL="https://{network_ip}:6443" '
 
   master_cmd = ''
   token = ''
@@ -69,7 +68,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # slave
     if nodetype == 'slave':
-      init_cmd = f'{k3s_install_base}{k3s_token_cmd} sh -s - server --server https://{network_ip}:6443'
+      init_cmd = f'cat /k3s.sh | sh -s - server --server https://{network_ip}:6443 {master_cmd}'
 
     # worker
     if nodetype == 'worker':
@@ -89,8 +88,8 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
             latest = snap.split()[0]
 
       kmsg(f'k3s_restore', f'restoring {latest}')
-
       init_cmd = f'/usr/local/bin/k3s server --cluster-reset --cluster-reset-restore-path={latest} --token={token} 2>&1 && systemctl start k3s'
+
     # write log of install on node
     init_cmd = init_cmd + f' > /k3s_{nodetype}_install.log 2>&1'
 
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 610558e..97d98c7 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -57,8 +57,6 @@
   virtc_script = f'''\
 curl -v https://get.k3s.io > /k3s.sh 
 cat /k3s.sh | \
-INSTALL_K3S_SKIP_ENABLE=true \
-INSTALL_K3S_SKIP_START=true \
 INSTALL_K3S_SKIP_SELINUX_RPM=true \
 INSTALL_K3S_VERSION={k3s_version} \
 sh -s - server --cluster-init > /{cluster_name}-image-install.log 2>&1
@@ -112,7 +110,6 @@
 disable:
   - servicelb
 disable-network-policy: true
-write-kubeconfig-mode: "0600"
 disable-cloud-controller: true
 disable-network-policy: true
 flannel-backend: wireguard-native

From b747a2b2cb36eb15aa9cb019c07a3d5b0ed2891e Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 16 Feb 2025 00:26:12 +0000
Subject: [PATCH 28/30] scripts update

---
 dev/img-mirror/serve.sh |  2 +-
 dev/rls_test.sh         | 11 +++--------
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/dev/img-mirror/serve.sh b/dev/img-mirror/serve.sh
index d4da3a4..426277f 100755
--- a/dev/img-mirror/serve.sh
+++ b/dev/img-mirror/serve.sh
@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
-kill -9 $(cat PID)
+kill -9 $(cat PID) > /dev/null 2&1
 python3 -m http.server -b "::"  > log.txt 2>&1 &
 echo $! > PID
diff --git a/dev/rls_test.sh b/dev/rls_test.sh
index e1d422c..30075bb 100644
--- a/dev/rls_test.sh
+++ b/dev/rls_test.sh
@@ -12,6 +12,7 @@ KC="$K cluster"
 KCI="$KC info"
 KCC="$KC create"
 KCU="$KC update"
+KCD="$KC destroy"
 KCR="$KC restore"
 KI="$K image"
 KID="$KI destroy"
@@ -49,23 +50,17 @@ $KES ; $KCD ; $KCC ; $KERL
 kc workers 1 ; $KCU ; kc workers 0 ; $KCU
 
 # re add worker
-kc workers 1 ; $KCU 
+kc workers 2 ; $KCU 
 
 # add 3 masters and go back to 1
 kc masters 3 ; $KCU ; kc masters 1  ; $KCU
 
-# add 3 masters 
-kc masters 3 ; $KCU 
-
-# take snapshot , destroy, create, restore
-$KES ; $KCD ; $KCC ; $KCU ; $KERL ; $KCD
-
 # change back to 1 node
 kc masters 1 ; kc workers 0  
 
 # ** TEST cluster restore
 # destroy cluster 
-$KCR
+$KCD ; $KCR
 
 finish_time=$(date +%s) 
 echo  $((finish_time - start_time)) secs

From 0c838fc8ad11e63ec4e5e7efaad4fb2780d80894 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sun, 16 Feb 2025 00:42:32 +0000
Subject: [PATCH 29/30] worker fix

---
 lib/kopsrox_k3s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index ea4244e..e688e46 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -72,7 +72,7 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # worker
     if nodetype == 'worker':
-      init_cmd = f'rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s'
+      init_cmd = f'systemctl stop k3s && rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s'
 
     # restore
     if nodetype == 'restore':

From e3b8515d1d372ccbb07cecbc668ff72a60765f13 Mon Sep 17 00:00:00 2001
From: simonccc <simonc@gmail.com>
Date: Sat, 1 Mar 2025 21:59:54 +0000
Subject: [PATCH 30/30] latest ci fixes

---
 dev/img-mirror/serve.sh | 2 +-
 lib/kopsrox_k3s.py      | 4 ++--
 lib/verb_image.py       | 4 ----
 lib/verb_k3s.py         | 1 -
 4 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/dev/img-mirror/serve.sh b/dev/img-mirror/serve.sh
index 426277f..50868bc 100755
--- a/dev/img-mirror/serve.sh
+++ b/dev/img-mirror/serve.sh
@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
-kill -9 $(cat PID) > /dev/null 2&1
+kill -9 $(cat PID) > /dev/null 2>&1
 python3 -m http.server -b "::"  > log.txt 2>&1 &
 echo $! > PID
diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py
index e688e46..f413f44 100755
--- a/lib/kopsrox_k3s.py
+++ b/lib/kopsrox_k3s.py
@@ -72,14 +72,14 @@ def k3s_init_node(vmid: int = masterid,nodetype = 'master'):
 
     # worker
     if nodetype == 'worker':
-      init_cmd = f'systemctl stop k3s && rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s'
+      init_cmd = f'rm -rf /etc/rancher/k3s/* && {k3s_install_worker}{k3s_token_cmd} sh -s'
 
     # restore
     if nodetype == 'restore':
       kmsg(f'k3s_bootstrap', f'fetching latest snapshot')
 
       # get latest snapshot
-      bs_cmd = f'systemctl start k3s && /usr/local/bin/k3s etcd-snapshot ls 2>&1 && systemctl stop k3s && rm -rf /var/lib/rancher'
+      bs_cmd = f'{k3s_install_master} {master_cmd} && /usr/local/bin/k3s etcd-snapshot ls 2>&1 && systemctl stop k3s && rm -rf /var/lib/rancher'
       bs_cmd_out = qaexec(vmid,bs_cmd)
 
       # sort ls output so last is latest snapshot
diff --git a/lib/verb_image.py b/lib/verb_image.py
index 97d98c7..6803962 100755
--- a/lib/verb_image.py
+++ b/lib/verb_image.py
@@ -56,10 +56,6 @@
   # script to run in kopsrox image
   virtc_script = f'''\
 curl -v https://get.k3s.io > /k3s.sh 
-cat /k3s.sh | \
-INSTALL_K3S_SKIP_SELINUX_RPM=true \
-INSTALL_K3S_VERSION={k3s_version} \
-sh -s - server --cluster-init > /{cluster_name}-image-install.log 2>&1
 
 if [ ! -f /usr/bin/qemu-ga ] 
 then
diff --git a/lib/verb_k3s.py b/lib/verb_k3s.py
index df1afcb..5fb2db3 100755
--- a/lib/verb_k3s.py
+++ b/lib/verb_k3s.py
@@ -2,7 +2,6 @@
 
 # functions
 from kopsrox_k3s import * 
-from kopsrox_kmsg import kmsg
 
 # passed command
 cmd = sys.argv[2]