From a62f67f5b4c9ef53ae11819d806e30c5fca9a006 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Mon, 1 Jul 2024 01:41:23 +0200 Subject: [PATCH 01/91] feat: init base go api --- .gitignore | 2 +- go.mod | 7 +++++ go.sum | 6 ++++ main.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore index e2a3910..d86136d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -#ignore .env file +#ignore .env file .env diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..52ffd76 --- /dev/null +++ b/go.mod @@ -0,0 +1,7 @@ +module github.com/KittenConnect/rh-api + +go 1.22.4 + +require github.com/rabbitmq/amqp091-go v1.10.0 + +require github.com/joho/godotenv v1.5.1 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bac9bbf --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/main.go b/main.go new file mode 100644 index 0000000..b7d53df --- /dev/null +++ b/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "encoding/json" + "github.com/joho/godotenv" + "log" + "os" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +type Message struct { + Hostname string `json:"hostname"` + IpAddress string `json:"ipaddress"` + + Timestamp time.Time `json:"-"` +} + +func failOnError(err error, msg string) { + if err != nil { + log.Fatalf("%s: %s", msg, err) + } +} + +func main() { + err := godotenv.Load() + if err != nil { + log.Panicf("Error loading .env file : %s", err) + } + + conn, err := amqp.Dial(os.Getenv("RABBITMQ_URL")) + if err != nil { + log.Panicf("Failed to connect to broker : %s", err) + } + + defer conn.Close() + + ch, err := conn.Channel() + if err != nil { + log.Panicf("Failed to open a channel : %s", err) + } + + q, err := ch.QueueDeclare( + "onboarding", + true, + false, + false, + false, + nil, + ) + if err != nil { + log.Panicf("Failed to declare a queue : %s", err) + } + + // Consommation des messages + msgs, err := ch.Consume( + q.Name, // nom de la queue + "consumer", // consumer + true, // autoAck + false, // exclusive + false, // noLocal + false, // noWait + nil, // arguments + ) + failOnError(err, "Failed to register a consumer") + + // Canal pour signaler la fin du programme + forever := make(chan bool) + + go func() { + for d := range msgs { + msg := Message{Timestamp: d.Timestamp} + err := json.Unmarshal(d.Body, &msg) + if err != nil { + log.Printf("Error unmarshalling message : %s", err) + } + + //Make request to the rest of API + } + }() + + log.Printf(" [*] Waiting for messages. To exit press CTRL+C") + <-forever +} From 8ae142e8e2545f2021fc37bcd3ee917ad59e84ca Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 19:36:10 +0200 Subject: [PATCH 02/91] feat: extract message model --- main.go | 9 +-------- model/message.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 model/message.go diff --git a/main.go b/main.go index b7d53df..7fdae05 100644 --- a/main.go +++ b/main.go @@ -10,13 +10,6 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) -type Message struct { - Hostname string `json:"hostname"` - IpAddress string `json:"ipaddress"` - - Timestamp time.Time `json:"-"` -} - func failOnError(err error, msg string) { if err != nil { log.Fatalf("%s: %s", msg, err) @@ -70,7 +63,7 @@ func main() { go func() { for d := range msgs { - msg := Message{Timestamp: d.Timestamp} + msg := model.Message{Timestamp: d.Timestamp} err := json.Unmarshal(d.Body, &msg) if err != nil { log.Printf("Error unmarshalling message : %s", err) diff --git a/model/message.go b/model/message.go new file mode 100644 index 0000000..9f63d9d --- /dev/null +++ b/model/message.go @@ -0,0 +1,10 @@ +package model + +import "time" + +type Message struct { + Hostname string `json:"hostname"` + IpAddress string `json:"ipaddress"` + + Timestamp time.Time `json:"-"` +} From 3f5401f957317c476d6bc39fc94e999e8483a0b5 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 19:36:50 +0200 Subject: [PATCH 03/91] feat: do not continue if error when decoding json --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 7fdae05..f510961 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ func main() { err := json.Unmarshal(d.Body, &msg) if err != nil { log.Printf("Error unmarshalling message : %s", err) + continue } //Make request to the rest of API From b1aaf29ff02db5d243dd459d61996efdcf683af7 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 19:40:46 +0200 Subject: [PATCH 04/91] refactor: reorder imports --- main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/main.go b/main.go index f510961..b7c67ad 100644 --- a/main.go +++ b/main.go @@ -3,11 +3,9 @@ package main import ( "encoding/json" "github.com/joho/godotenv" + amqp "github.com/rabbitmq/amqp091-go" "log" "os" - "time" - - amqp "github.com/rabbitmq/amqp091-go" ) func failOnError(err error, msg string) { From b0ced71d098a17df446826e82e531a053ba9cd0b Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 19:41:31 +0200 Subject: [PATCH 05/91] feat: import netbox go library --- go.mod | 5 ++++- go.sum | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 52ffd76..126bd27 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,7 @@ go 1.22.4 require github.com/rabbitmq/amqp091-go v1.10.0 -require github.com/joho/godotenv v1.5.1 // indirect +require ( + github.com/joho/godotenv v1.5.1 + github.com/netbox-community/go-netbox/v4 v4.0.3-0 +) diff --git a/go.sum b/go.sum index bac9bbf..5d140b2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/netbox-community/go-netbox/v4 v4.0.3-0 h1:Q6NucmGe9ZmU3mjziuqQlKqHZmzC3FYbZn9r+Y507hs= +github.com/netbox-community/go-netbox/v4 v4.0.3-0/go.mod h1:eux5UXpY7T1BADzdFI1V7o+iIGxSfPLUuZvcjVWca/w= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From 85eccd3e6a7fc697748c233cb64c68b88372e8dc Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 21:37:15 +0200 Subject: [PATCH 06/91] feat: logging util --- util/log.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 util/log.go diff --git a/util/log.go b/util/log.go new file mode 100644 index 0000000..e63adaf --- /dev/null +++ b/util/log.go @@ -0,0 +1,28 @@ +package util + +import ( + "fmt" + "github.com/fatih/color" + "os" +) + +func Err(t string) { + red := color.New(color.FgRed).SprintFunc() + fmt.Printf("%s %s\n", red("[ERROR]"), red(t)) + os.Exit(-1) +} + +func Warn(t string) { + orange := color.New(color.FgYellow).SprintFunc() + fmt.Printf("%s %s\n", orange("[WARN]"), orange(t)) +} + +func Info(t string) { + blue := color.New(color.FgBlue).SprintFunc() + fmt.Printf("%s %s\n", blue("[INFO]"), blue(t)) +} + +func Success(t string) { + green := color.New(color.FgGreen).SprintFunc() + fmt.Printf("%s %s\n", green("[SUCCESS]"), green(t)) +} From 95b3ad23194b53f194f83110153acece56379d33 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 21:38:58 +0200 Subject: [PATCH 07/91] refactor: use existing function to show error --- main.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/main.go b/main.go index b7c67ad..f982823 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "fmt" + "github.com/KittenConnect/rh-api/util" "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" "log" @@ -10,27 +12,21 @@ import ( func failOnError(err error, msg string) { if err != nil { - log.Fatalf("%s: %s", msg, err) + util.Err(fmt.Sprintf("%s: %s", msg, err)) } } func main() { err := godotenv.Load() - if err != nil { - log.Panicf("Error loading .env file : %s", err) - } + failOnError(err, fmt.Sprintf("Error loading .env file : %s", err)) conn, err := amqp.Dial(os.Getenv("RABBITMQ_URL")) - if err != nil { - log.Panicf("Failed to connect to broker : %s", err) - } + failOnError(err, fmt.Sprintf("Failed to connect to broker : %s", err)) defer conn.Close() ch, err := conn.Channel() - if err != nil { - log.Panicf("Failed to open a channel : %s", err) - } + failOnError(err, fmt.Sprintf("Failed to open a channel : %s", err)) q, err := ch.QueueDeclare( "onboarding", @@ -40,9 +36,7 @@ func main() { false, nil, ) - if err != nil { - log.Panicf("Failed to declare a queue : %s", err) - } + failOnError(err, fmt.Sprintf("Failed to declare a queue : %s", err)) // Consommation des messages msgs, err := ch.Consume( @@ -64,7 +58,7 @@ func main() { msg := model.Message{Timestamp: d.Timestamp} err := json.Unmarshal(d.Body, &msg) if err != nil { - log.Printf("Error unmarshalling message : %s", err) + util.Warn(fmt.Sprintf("Error unmarshalling message : %s", err)) continue } @@ -72,6 +66,6 @@ func main() { } }() - log.Printf(" [*] Waiting for messages. To exit press CTRL+C") + util.Info(" [*] Waiting for messages. To exit press CTRL+C") <-forever } From e70623e067c469f45d98ad2983a98d577dbad40b Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 2 Jul 2024 21:40:35 +0200 Subject: [PATCH 08/91] feat: add color module --- go.mod | 7 +++++++ go.sum | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/go.mod b/go.mod index 126bd27..5b844d6 100644 --- a/go.mod +++ b/go.mod @@ -8,3 +8,10 @@ require ( github.com/joho/godotenv v1.5.1 github.com/netbox-community/go-netbox/v4 v4.0.3-0 ) + +require ( + github.com/fatih/color v1.17.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + golang.org/x/sys v0.18.0 // indirect +) diff --git a/go.sum b/go.sum index 5d140b2..11fae5a 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,19 @@ +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/netbox-community/go-netbox/v4 v4.0.3-0 h1:Q6NucmGe9ZmU3mjziuqQlKqHZmzC3FYbZn9r+Y507hs= github.com/netbox-community/go-netbox/v4 v4.0.3-0/go.mod h1:eux5UXpY7T1BADzdFI1V7o+iIGxSfPLUuZvcjVWca/w= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= From 927c58ed0c9fa1432fe4cf70564b85b0ba771ce6 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 4 Jul 2024 17:57:02 +0200 Subject: [PATCH 09/91] remove old code --- api/GenerateLoopback.py | 102 --------------------------------- api/util/netbox_utils.py | 109 ------------------------------------ api/util/wireguard_utils.py | 14 ----- requirements.txt | 2 - 4 files changed, 227 deletions(-) delete mode 100644 api/GenerateLoopback.py delete mode 100644 api/util/netbox_utils.py delete mode 100644 api/util/wireguard_utils.py delete mode 100644 requirements.txt diff --git a/api/GenerateLoopback.py b/api/GenerateLoopback.py deleted file mode 100644 index 89ef6c8..0000000 --- a/api/GenerateLoopback.py +++ /dev/null @@ -1,102 +0,0 @@ -import requests -import json -import random -import os -from dotenv import load_dotenv -import logging - -# Configuration de la journalisation -logging.basicConfig(level=logging.INFO) - -load_dotenv() -TOKEN = os.getenv("TOKEN") -NETBOX_API_URL = os.getenv("NETBOX_API_URL") -API_URL_IPAM = f"{NETBOX_API_URL}ipam/ip-addresses/" - -HEADERS = { - 'Authorization': f'Token {TOKEN}', - 'Content-Type': 'application/json', - 'Accept': 'application/json', -} - - -def generate_ipv6_suffix(prefix, byte_count): - """ - Génère un suffixe IPv6 aléatoire pour un préfixe donné. - :param prefix: Préfixe IPv6. - :param byte_count: Nombre de bytes à générer. - :return: Adresse IPv6 complète. - """ - suffix = "".join(random.choice("0123456789abcdef") for _ in range(byte_count)) - return f"{prefix[:-byte_count]}:{suffix}" - - -def make_netbox_request(url, method="get", data=None, params=None): - """ - Effectue une requête HTTP à NetBox. - :param url: URL de l'API NetBox. - :param method: Méthode HTTP ('get' ou 'post'). - :param data: Données pour la requête POST. - :param params: Paramètres pour la requête GET. - :return: Réponse de la requête. - """ - try: - if method == "get": - response = requests.get(url, headers=HEADERS, params=params) - else: - response = requests.post(url, headers=HEADERS, json=data) - - response.raise_for_status() - return response.json() - except requests.HTTPError as http_err: - logging.error(f"HTTP error occurred: {http_err}") - except Exception as err: - logging.error(f"An error occurred: {err}") - - return None - - -def check_ip_exist_in_netbox(ip_address): - """ - Vérifie si une adresse IP existe dans NetBox. - :param ip_address: Adresse IPv6 à vérifier. - :return: Booléen indiquant si l'adresse existe. - """ - params = {"address": ip_address} - response = make_netbox_request(API_URL_IPAM, params=params) - - if response and response["count"] > 0: - return True - return False - - -def create_ip_in_netbox(ip_address): - """ - Crée une adresse IP dans NetBox. - :param ip_address: Adresse IPv6 à créer. - """ - data = { - "address": ip_address, - "description": "Description de l'adresse IPv6", - "custom_fields": {"Pubkey": "Votre clé publique", "Userid": 5} - } - response = make_netbox_request(API_URL_IPAM, method="post", data=data) - - if response: - logging.info(f"L'adresse IPv6 {ip_address} a été créée avec succès dans NetBox.") - - -def create_loopback(): - """ - Crée une adresse loopback qui n'existe pas encore dans NetBox. - :return: Adresse loopback créée. - """ - while True: - temp_ip = generate_ipv6_suffix("2a13:79c0:ffff:fefe::/64", 4) - if not check_ip_exist_in_netbox(temp_ip): - create_ip_in_netbox(temp_ip) - return temp_ip - - -loopback = create_loopback() -print(f"{loopback}/128") diff --git a/api/util/netbox_utils.py b/api/util/netbox_utils.py deleted file mode 100644 index 556972c..0000000 --- a/api/util/netbox_utils.py +++ /dev/null @@ -1,109 +0,0 @@ -import requests -import random -import ipaddress - -NETBOX_API_URL = 'https://netbox.360p.kube.kittenconnect.net/api/' -HEADERS = { - 'Authorization': 'Token DANSTAMERELECODE ', - 'Content-Type': 'application/json', - 'Accept': 'application/json', -} - - -def GeneratePrefixIpV6(prefix, Byte): - # il faut le define prefix = "2a13:79c0:ffff:fefe::/64" - - # Générez un suffixe aléatoire de 4 caractères hexadécimaux pour /128 - suffix = "".join(random.choice("0123456789abcdef") for _ in range(4)) - - if Byte == 4: - ipv6_address = f"{prefix[:-Byte]}:{suffix}" - elif Byte == 5: - ipv6_address = f"{prefix[:-Byte]}:{suffix}:" - else: - raise NotImplementedError() - # Concaténez le préfixe et le suffixe pour former l'adresse IPv6 complète - - return ipv6_address - - -############ -# -# Fonction pour vérifier si l'adresse IPv6 existe déjà dans NetBox -# return True si l'adresse IPv6 existe déjà dans NetBox -# return False si l'adresse IPv6 n'existe pas dans NetBox -############ - -def CheckIpExistInNetbox(ip_address): - params = { - "address": ip_address, - } - apiurl = NETBOX_API_URL + "ipam/ip-addresses/" - try: - response = requests.get(apiurl, headers=HEADERS, params=params) - - if response.status_code == 200: - ip_data = response.json() - if ip_data["count"] > 0: - return True - else: - return False - else: - print("Erreur lors de la recherche de l'adresse IPv6 dans NetBox.") - print(response.text) - return False - except Exception as e: - print(f"Une erreur s'est produite : {str(e)}") - return False - - -############ - - -def generate_or_create_ipv6_address(): - while True: - ipv6_address = generate_short_ipv6_address() - if not checkIpinNetbox(ipv6_address): - CreateIpinNetbox(ipv6_address) - return ipv6_address - - -def get_available_address_prefix(prefix: str, length: int): - """ - Get all the IP addresses from Prefix IPv6 - :param prefix: - :param length: - :return: - """ - try: - ipv6_network = ipaddress.IPv6Network(f"{prefix}/{length}", strict=False) - network_address = ipv6_network.network_address - network_address_str = str(network_address) - - return network_address_str - except ValueError as e: - print(f"Error : {e}") - return None - - -test = GeneratePrefixIpV6("2a13:79c0:ffff:fefe::/64", 4) # generate loopback -test2 = GeneratePrefixIpV6("2a13:79c0:ffff:feff::/64", 4) + "/127" # generate prefix for wireguard -test3 = GeneratePrefixIpV6("2a13:79c0:ffff:feff:b00b::/80", 5) - -print(test + "/128") -print(test2) # + "/127") -print(test3 + "/96") -TAMMERE = ipaddress.IPv6Interface(test2).network - -test4fin = [*ipaddress.ip_network(TAMMERE).hosts()] -print(*test4fin) - - -# print(TAMERELALIST) -# 2a13:79c0:ffff:feff:b00b::/80 - - -## class NewConnection to allow Loopback IP address, and /127 subnet for wireguard and return public key -class NewConnection: - def __init__(): - return true diff --git a/api/util/wireguard_utils.py b/api/util/wireguard_utils.py deleted file mode 100644 index df7e342..0000000 --- a/api/util/wireguard_utils.py +++ /dev/null @@ -1,14 +0,0 @@ -class WireguardUtils: - - def __init__(self): - pass - - def get_tunnel_ip(self) -> list: - """ - Retourne la liste des ip qui vont constituer le tunnel - :return: - """ - pass - - def etc(self): - pass diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index da776d6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests==2.31.0 -dotenv==0.19.1 From 7fd5dc917974ff9f5943c9e115b0f8d7e2622f67 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 4 Jul 2024 18:00:29 +0200 Subject: [PATCH 10/91] feat: base netbox connector --- model/netbox.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 model/netbox.go diff --git a/model/netbox.go b/model/netbox.go new file mode 100644 index 0000000..5a123a2 --- /dev/null +++ b/model/netbox.go @@ -0,0 +1,102 @@ +package model + +import ( + "context" + "errors" + "github.com/netbox-community/go-netbox/v4" + "os" +) + +var MachinesSerials map[string]string + +var ( + NetboxVmSerialPrefix string = os.Getenv("NETBOX_VM_SERIAL_FIELD") +) + +// Netbox structure +// For internal use ONLY ! +// To get an instance, call NewNetbox method +type Netbox struct { + ctx context.Context + + Client *netbox.APIClient + + _isConnected bool +} + +// NewNetbox return a fresh Netbox object +func NewNetbox() Netbox { + nbx := Netbox{ + ctx: context.Background(), + Client: nil, + + _isConnected: false, + } + + return nbx +} + +func (n Netbox) IsConnected() bool { + return n._isConnected +} + +func (n Netbox) Connect() error { + if n._isConnected { + return nil + } + + n.Client = netbox.NewAPIClientFor(os.Getenv("NETBOX_API_URL"), os.Getenv("NETBOX_API_KEY")) + n._isConnected = true + + return nil +} + +func (n Netbox) CreateVM(serial string) error { + if !n._isConnected { + return errors.New("netbox is not connected") + } + + _ = n.Client.VirtualizationAPI.VirtualizationVirtualMachinesCreate(context.Background()) + + //TODO + return nil +} + +func (n Netbox) UpdateVM(serial string, conf string) error { + if !n._isConnected { + return errors.New("netbox is not connected") + } + + // Call netbox API with specific serial, then update his settings accordingly + _, exist := MachinesSerials[serial] + if exist { + return nil + } + + res, _, err := n.Client.VirtualizationAPI. + VirtualizationVirtualMachinesList(n.ctx). + //Name([]string{serial}). + //Limit(1). + Execute() + if err != nil { + return err + } + + var vmId int32 + var hasFoundVm bool = false + + for _, vm := range res.Results { + if vm.CustomFields[NetboxVmSerialPrefix] == serial { + vmId = vm.Id + hasFoundVm = true + break + } + } + + if !hasFoundVm { + //Create VM + print(vmId) //TODO + } + + return nil +} From de466ef4c17261bfab1342f5b57ddbd6bef97f13 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 4 Jul 2024 18:00:51 +0200 Subject: [PATCH 11/91] refactor: use our own logger --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index f982823..07ea54d 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "github.com/KittenConnect/rh-api/util" "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" - "log" "os" ) From 87300b7319ac8706f65610f3569aeb019c82efbd Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 4 Jul 2024 18:01:49 +0200 Subject: [PATCH 12/91] wip: init netbox connection --- main.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/main.go b/main.go index 07ea54d..e7da110 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "github.com/KittenConnect/rh-api/model" "github.com/KittenConnect/rh-api/util" "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" @@ -48,6 +49,16 @@ func main() { nil, // arguments ) failOnError(err, "Failed to register a consumer") + util.Info("Connected to message broker") + + netbox := model.NewNetbox() + err = netbox.Connect() + failOnError(err, "Failed to connect to netbox") + + if netbox.IsConnected() == false { + util.Err("Unable to connect to netbox") + os.Exit(-1) + } // Canal pour signaler la fin du programme forever := make(chan bool) From 18550fb7e80f01e03314418782590040ea6098f4 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 4 Jul 2024 18:02:17 +0200 Subject: [PATCH 13/91] feat: ignore ide files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d86136d..a32d148 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ #ignore .env file .env + +.idea/ +.vscode/ \ No newline at end of file From 51e676453c713391936bd14c92d675f026c2b537 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 17:46:42 +0200 Subject: [PATCH 14/91] feat: include machine serial inside message --- model/message.go | 1 + 1 file changed, 1 insertion(+) diff --git a/model/message.go b/model/message.go index 9f63d9d..ec2a105 100644 --- a/model/message.go +++ b/model/message.go @@ -5,6 +5,7 @@ import "time" type Message struct { Hostname string `json:"hostname"` IpAddress string `json:"ipaddress"` + Serial string `json:"serial"` Timestamp time.Time `json:"-"` } From 594822ab908755bb34262975c69c2c5d02b703e3 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:14:11 +0200 Subject: [PATCH 15/91] feat: add packages --- go.mod | 2 +- model/netbox.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5b844d6..f5bbf2b 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.22.4 require github.com/rabbitmq/amqp091-go v1.10.0 require ( + github.com/fatih/color v1.17.0 github.com/joho/godotenv v1.5.1 github.com/netbox-community/go-netbox/v4 v4.0.3-0 ) require ( - github.com/fatih/color v1.17.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/model/netbox.go b/model/netbox.go index 5a123a2..c040cf6 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -3,6 +3,8 @@ package model import ( "context" "errors" + "fmt" + "github.com/KittenConnect/rh-api/util" "github.com/netbox-community/go-netbox/v4" "os" ) From 8c0544d305fe190f5d1f0a84f6a28057b8933c3b Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:14:59 +0200 Subject: [PATCH 16/91] feat: show message when fail to handle vm creation/update --- main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main.go b/main.go index e7da110..a4060b5 100644 --- a/main.go +++ b/main.go @@ -73,6 +73,11 @@ func main() { } //Make request to the rest of API + err = netbox.CreateOrUpdateVM(msg) + if err != nil { + util.Warn(fmt.Sprintf("Error creating or updating VM : %s", err)) + return + } } }() From 0d1ba46defe839beb847ae104f2d786d65cade50 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:15:35 +0200 Subject: [PATCH 17/91] feat: function to update vm --- model/netbox.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index c040cf6..979d3ad 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -53,15 +53,30 @@ func (n Netbox) Connect() error { return nil } -func (n Netbox) CreateVM(serial string) error { +func (n Netbox) CreateVM(msg Message) (int32, error) { + var ( + id int32 + ) + if !n._isConnected { - return errors.New("netbox is not connected") + return id, errors.New("netbox is not connected") } - _ = n.Client.VirtualizationAPI.VirtualizationVirtualMachinesCreate(context.Background()) + vm := netbox.WritableVirtualMachineWithConfigContextRequest{ + Name: msg.Hostname, + CustomFields: map[string]interface{}{ + "machine_serial": msg.Serial, + }, + } - //TODO - return nil + _, result, err := n.Client.VirtualizationAPI.VirtualizationVirtualMachinesCreate(n.ctx).WritableVirtualMachineWithConfigContextRequest(vm).Execute() + if err != nil { + return id, err + } + + util.Info(fmt.Sprintf("Created machine ID : %s", result.Body)) + + return id, nil } func (n Netbox) UpdateVM(serial string, conf string) error { From 519bab485b7f54db2e510b2653d058e235c0db4c Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:16:05 +0200 Subject: [PATCH 18/91] feat: create vm if not existing in memory --- model/netbox.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/model/netbox.go b/model/netbox.go index 979d3ad..6df87dc 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -113,6 +113,11 @@ func (n Netbox) UpdateVM(serial string, conf string) error { if !hasFoundVm { //Create VM print(vmId) //TODO + vmId, err = n.CreateVM(msg) + + if err != nil { + return err + } } return nil From 1bd79e967fbdef41aa7db34623658256dad54177 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:31:50 +0200 Subject: [PATCH 19/91] refactor: use list for existing vm --- model/netbox.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index 6df87dc..f00ee4c 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -1,6 +1,7 @@ package model import ( + "container/list" "context" "errors" "fmt" @@ -9,7 +10,7 @@ import ( "os" ) -var MachinesSerials map[string]string +var MachinesSerials *list.List var ( NetboxVmSerialPrefix string = os.Getenv("NETBOX_VM_SERIAL_FIELD") @@ -26,6 +27,16 @@ type Netbox struct { _isConnected bool } +func contains(list *list.List, item string) bool { + for e := list.Front(); e != nil; e = e.Next() { + if e.Value == item { + return true + } + } + + return false +} + // NewNetbox return a fresh Netbox object func NewNetbox() Netbox { nbx := Netbox{ @@ -50,6 +61,8 @@ func (n Netbox) Connect() error { n.Client = netbox.NewAPIClientFor(os.Getenv("NETBOX_API_URL"), os.Getenv("NETBOX_API_KEY")) n._isConnected = true + MachinesSerials = list.New() + return nil } @@ -88,6 +101,7 @@ func (n Netbox) UpdateVM(serial string, conf string) error { _, exist := MachinesSerials[serial] if exist { return nil + exist := contains(MachinesSerials, msg.Serial) } res, _, err := n.Client.VirtualizationAPI. From 3edd92757c47de1a7fd604f3cafba407a6811950 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:32:04 +0200 Subject: [PATCH 20/91] feat: function to get vm boilerplate --- model/netbox.go | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index f00ee4c..bad9c97 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -66,6 +66,15 @@ func (n Netbox) Connect() error { return nil } +func getVm(name string, serial string) netbox.WritableVirtualMachineWithConfigContextRequest { + return netbox.WritableVirtualMachineWithConfigContextRequest{ + Name: name, + CustomFields: map[string]interface{}{ + "machine_serial": serial, + }, + } +} + func (n Netbox) CreateVM(msg Message) (int32, error) { var ( id int32 @@ -98,10 +107,35 @@ func (n Netbox) UpdateVM(serial string, conf string) error { } // Call netbox API with specific serial, then update his settings accordingly - _, exist := MachinesSerials[serial] - if exist { - return nil exist := contains(MachinesSerials, msg.Serial) + if !exist { + //If the vm don't exist in memory, fetch his details, if she exists in netbox + res, _, err := n.Client.VirtualizationAPI. + VirtualizationVirtualMachinesList(n.ctx). + Execute() + if err != nil { + return err + } + + for _, vm := range res.Results { + if vm.CustomFields[NetboxVmSerialPrefix] == msg.Serial { + vmId = vm.Id + hasFoundVm = true + break + } + } + + //Create VM if she doesn't exists in netbox + if !hasFoundVm { + vmId, err = n.CreateVM(msg) + + if err != nil { + return err + } + } + + // + MachinesSerials.PushBack(msg.Serial) } res, _, err := n.Client.VirtualizationAPI. From 9859149ff432a6536af4d01d549cc138881f2bd4 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:32:18 +0200 Subject: [PATCH 21/91] fix: creation of vm --- model/netbox.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index bad9c97..f0a3ba0 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -84,14 +84,12 @@ func (n Netbox) CreateVM(msg Message) (int32, error) { return id, errors.New("netbox is not connected") } - vm := netbox.WritableVirtualMachineWithConfigContextRequest{ - Name: msg.Hostname, - CustomFields: map[string]interface{}{ - "machine_serial": msg.Serial, - }, - } + vm := getVm(msg.Hostname, msg.Serial) - _, result, err := n.Client.VirtualizationAPI.VirtualizationVirtualMachinesCreate(n.ctx).WritableVirtualMachineWithConfigContextRequest(vm).Execute() + _, result, err := n.Client.VirtualizationAPI. + VirtualizationVirtualMachinesCreate(n.ctx). + WritableVirtualMachineWithConfigContextRequest(vm). + Execute() if err != nil { return id, err } From 968304f1162c9335be06da42b2a50df94f5128a4 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:32:28 +0200 Subject: [PATCH 22/91] feat: function to create vm --- model/netbox.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index f0a3ba0..790e286 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -99,7 +99,18 @@ func (n Netbox) CreateVM(msg Message) (int32, error) { return id, nil } -func (n Netbox) UpdateVM(serial string, conf string) error { +func (n Netbox) UpdateVM(id int32, msg Message) error { + vm := getVm(msg.Hostname, msg.Serial) + + _, _, err := n.Client.VirtualizationAPI. + VirtualizationVirtualMachinesUpdate(n.ctx, id). + WritableVirtualMachineWithConfigContextRequest(vm). + Execute() + + return err +} + +func (n Netbox) CreateOrUpdateVM(msg Message) error { if !n._isConnected { return errors.New("netbox is not connected") } From ce8d56c667635b52b037832ae3b25e53d35d0d3f Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:32:44 +0200 Subject: [PATCH 23/91] refactor: use extracted functions --- model/netbox.go | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 790e286..eaab00d 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -115,6 +115,10 @@ func (n Netbox) CreateOrUpdateVM(msg Message) error { return errors.New("netbox is not connected") } + var vmId int32 + var hasFoundVm bool = false + var err error + // Call netbox API with specific serial, then update his settings accordingly exist := contains(MachinesSerials, msg.Serial) if !exist { @@ -147,35 +151,11 @@ func (n Netbox) CreateOrUpdateVM(msg Message) error { MachinesSerials.PushBack(msg.Serial) } - res, _, err := n.Client.VirtualizationAPI. - VirtualizationVirtualMachinesList(n.ctx). - //Name([]string{serial}). - //Limit(1). - Execute() + //Sinon on update la vm + err = n.UpdateVM(vmId, msg) if err != nil { return err } - var vmId int32 - var hasFoundVm bool = false - - for _, vm := range res.Results { - if vm.CustomFields[NetboxVmSerialPrefix] == serial { - vmId = vm.Id - hasFoundVm = true - break - } - } - - if !hasFoundVm { - //Create VM - print(vmId) //TODO - vmId, err = n.CreateVM(msg) - - if err != nil { - return err - } - } - return nil } From 5b66f8db0487688016c104a0b16e4b6b88461219 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 18:34:24 +0200 Subject: [PATCH 24/91] refactor: use netbox vm custom field prefix --- model/netbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index eaab00d..04e17b3 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -70,7 +70,7 @@ func getVm(name string, serial string) netbox.WritableVirtualMachineWithConfigCo return netbox.WritableVirtualMachineWithConfigContextRequest{ Name: name, CustomFields: map[string]interface{}{ - "machine_serial": serial, + NetboxVmSerialPrefix: serial, }, } } From 9a5601c1c2ac98fc5d2de85f94c3775c97271dd4 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 20:58:23 +0200 Subject: [PATCH 25/91] refactor: do not update the vm if we created him just before --- model/netbox.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 04e17b3..2c5fd56 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -152,9 +152,11 @@ func (n Netbox) CreateOrUpdateVM(msg Message) error { } //Sinon on update la vm - err = n.UpdateVM(vmId, msg) - if err != nil { - return err + if !hasFoundVm { + err = n.UpdateVM(vmId, msg) + if err != nil { + return err + } } return nil From f6285a4e1cedcc9c5411a04538749e98e55de195 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 25 Jul 2024 21:00:31 +0200 Subject: [PATCH 26/91] refactor: simplify code --- model/netbox.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 2c5fd56..279ae63 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -149,10 +149,7 @@ func (n Netbox) CreateOrUpdateVM(msg Message) error { // MachinesSerials.PushBack(msg.Serial) - } - - //Sinon on update la vm - if !hasFoundVm { + } else { err = n.UpdateVM(vmId, msg) if err != nil { return err From 99cf6eaa60cd9651886904d2468a18b3bfe01bc5 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sat, 27 Jul 2024 22:10:20 +0200 Subject: [PATCH 27/91] feat: add docker compose file --- docker-compose.yml | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..cba98ca --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,88 @@ +services: + netbox: &netbox + image: docker.io/netboxcommunity/netbox:${VERSION-v3.7.5} + depends_on: + - postgres + - redis + - redis-cache + env_file: env/netbox.env + user: 'unit:root' + healthcheck: + start_period: 60s + timeout: 3s + interval: 15s + test: "curl -f http://localhost:8080/login/ || exit 1" + volumes: + - ./configuration:/etc/netbox/config:z,ro + - netbox-media-files:/opt/netbox/netbox/media:rw + - netbox-reports-files:/opt/netbox/netbox/reports:rw + - netbox-scripts-files:/opt/netbox/netbox/scripts:rw + ports: + - 8080:80 + netbox-worker: + <<: *netbox + depends_on: + netbox: + condition: service_healthy + command: + - /opt/netbox/venv/bin/python + - /opt/netbox/netbox/manage.py + - rqworker + healthcheck: + start_period: 20s + timeout: 3s + interval: 15s + test: "ps -aux | grep -v grep | grep -q rqworker || exit 1" + netbox-housekeeping: + <<: *netbox + depends_on: + netbox: + condition: service_healthy + command: + - /opt/netbox/housekeeping.sh + healthcheck: + start_period: 20s + timeout: 3s + interval: 15s + test: "ps -aux | grep -v grep | grep -q housekeeping || exit 1" + + # postgres + postgres: + image: docker.io/postgres:16-alpine + env_file: env/postgres.env + volumes: + - netbox-postgres-data:/var/lib/postgresql/data + + # redis + redis: + image: docker.io/redis:7-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis.env + volumes: + - netbox-redis-data:/data + redis-cache: + image: docker.io/redis:7-alpine + command: + - sh + - -c # this is to evaluate the $REDIS_PASSWORD from the env + - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose + env_file: env/redis-cache.env + volumes: + - netbox-redis-cache-data:/data + +volumes: + netbox-media-files: + driver: local + netbox-postgres-data: + driver: local + netbox-redis-cache-data: + driver: local + netbox-redis-data: + driver: local + netbox-reports-files: + driver: local + netbox-scripts-files: + driver: local \ No newline at end of file From 43c9acd21f67b6e7b6e1c7c4a1f43faabc362c20 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sat, 27 Jul 2024 22:10:38 +0200 Subject: [PATCH 28/91] feat: base docker conf --- configuration/configuration.py | 339 ++++++++++++++++++++++++++++++ configuration/extra.py | 49 +++++ configuration/ldap/extra.py | 28 +++ configuration/ldap/ldap_config.py | 111 ++++++++++ configuration/logging.py | 55 +++++ configuration/plugins.py | 13 ++ env/netbox.env | 38 ++++ env/postgres.env | 3 + env/redis-cache.env | 1 + env/redis.env | 1 + 10 files changed, 638 insertions(+) create mode 100644 configuration/configuration.py create mode 100644 configuration/extra.py create mode 100644 configuration/ldap/extra.py create mode 100644 configuration/ldap/ldap_config.py create mode 100644 configuration/logging.py create mode 100644 configuration/plugins.py create mode 100644 env/netbox.env create mode 100644 env/postgres.env create mode 100644 env/redis-cache.env create mode 100644 env/redis.env diff --git a/configuration/configuration.py b/configuration/configuration.py new file mode 100644 index 0000000..2145a25 --- /dev/null +++ b/configuration/configuration.py @@ -0,0 +1,339 @@ +#### +## We recommend to not edit this file. +## Create separate files to overwrite the settings. +## See `extra.py` as an example. +#### + +import re +from os import environ +from os.path import abspath, dirname, join +from typing import Any, Callable, Tuple + +# For reference see https://docs.netbox.dev/en/stable/configuration/ +# Based on https://github.com/netbox-community/netbox/blob/develop/netbox/netbox/configuration_example.py + +### +# NetBox-Docker Helper functions +### + +# Read secret from file +def _read_secret(secret_name: str, default: str | None = None) -> str | None: + try: + f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') + except EnvironmentError: + return default + else: + with f: + return f.readline().strip() + +# If the `map_fn` isn't defined, then the value that is read from the environment (or the default value if not found) is returned. +# If the `map_fn` is defined, then `map_fn` is invoked and the value (that was read from the environment or the default value if not found) +# is passed to it as a parameter. The value returned from `map_fn` is then the return value of this function. +# The `map_fn` is not invoked, if the value (that was read from the environment or the default value if not found) is None. +def _environ_get_and_map(variable_name: str, default: str | None = None, map_fn: Callable[[str], Any | None] = None) -> Any | None: + env_value = environ.get(variable_name, default) + + if env_value == None: + return env_value + + if not map_fn: + return env_value + + return map_fn(env_value) + +_AS_BOOL = lambda value : value.lower() == 'true' +_AS_INT = lambda value : int(value) +_AS_LIST = lambda value : list(filter(None, value.split(' '))) + +_BASE_DIR = dirname(dirname(abspath(__file__))) + +######################### +# # +# Required settings # +# # +######################### + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = environ.get('ALLOWED_HOSTS', '*').split(' ') +# ensure that '*' or 'localhost' is always in ALLOWED_HOSTS (needed for health checks) +if '*' not in ALLOWED_HOSTS and 'localhost' not in ALLOWED_HOSTS: + ALLOWED_HOSTS.append('localhost') + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + 'NAME': environ.get('DB_NAME', 'netbox'), # Database name + 'USER': environ.get('DB_USER', ''), # PostgreSQL username + 'PASSWORD': _read_secret('db_password', environ.get('DB_PASSWORD', '')), + # PostgreSQL password + 'HOST': environ.get('DB_HOST', 'localhost'), # Database server + 'PORT': environ.get('DB_PORT', ''), # Database port (leave blank for default) + 'OPTIONS': {'sslmode': environ.get('DB_SSLMODE', 'prefer')}, + # Database connection SSLMODE + 'CONN_MAX_AGE': _environ_get_and_map('DB_CONN_MAX_AGE', '300', _AS_INT), + # Max database connection age + 'DISABLE_SERVER_SIDE_CURSORS': _environ_get_and_map('DB_DISABLE_SERVER_SIDE_CURSORS', 'False', _AS_BOOL), + # Disable the use of server-side cursors transaction pooling +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': environ.get('REDIS_HOST', 'localhost'), + 'PORT': _environ_get_and_map('REDIS_PORT', 6379, _AS_INT), + 'SENTINELS': [tuple(uri.split(':')) for uri in _environ_get_and_map('REDIS_SENTINELS', '', _AS_LIST) if uri != ''], + 'SENTINEL_SERVICE': environ.get('REDIS_SENTINEL_SERVICE', 'default'), + 'SENTINEL_TIMEOUT': _environ_get_and_map('REDIS_SENTINEL_TIMEOUT', 10, _AS_INT), + 'USERNAME': environ.get('REDIS_USERNAME', ''), + 'PASSWORD': _read_secret('redis_password', environ.get('REDIS_PASSWORD', '')), + 'DATABASE': _environ_get_and_map('REDIS_DATABASE', 0, _AS_INT), + 'SSL': _environ_get_and_map('REDIS_SSL', 'False', _AS_BOOL), + 'INSECURE_SKIP_TLS_VERIFY': _environ_get_and_map('REDIS_INSECURE_SKIP_TLS_VERIFY', 'False', _AS_BOOL), + }, + 'caching': { + 'HOST': environ.get('REDIS_CACHE_HOST', environ.get('REDIS_HOST', 'localhost')), + 'PORT': _environ_get_and_map('REDIS_CACHE_PORT', environ.get('REDIS_PORT', '6379'), _AS_INT), + 'SENTINELS': [tuple(uri.split(':')) for uri in _environ_get_and_map('REDIS_CACHE_SENTINELS', '', _AS_LIST) if uri != ''], + 'SENTINEL_SERVICE': environ.get('REDIS_CACHE_SENTINEL_SERVICE', environ.get('REDIS_SENTINEL_SERVICE', 'default')), + 'USERNAME': environ.get('REDIS_CACHE_USERNAME', environ.get('REDIS_USERNAME', '')), + 'PASSWORD': _read_secret('redis_cache_password', environ.get('REDIS_CACHE_PASSWORD', environ.get('REDIS_PASSWORD', ''))), + 'DATABASE': _environ_get_and_map('REDIS_CACHE_DATABASE', '1', _AS_INT), + 'SSL': _environ_get_and_map('REDIS_CACHE_SSL', environ.get('REDIS_SSL', 'False'), _AS_BOOL), + 'INSECURE_SKIP_TLS_VERIFY': _environ_get_and_map('REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY', environ.get('REDIS_INSECURE_SKIP_TLS_VERIFY', 'False'), _AS_BOOL), + }, +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = _read_secret('secret_key', environ.get('SECRET_KEY', '')) + + +######################### +# # +# Optional settings # +# # +######################### + +# # Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# # application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + +if 'ALLOWED_URL_SCHEMES' in environ: + ALLOWED_URL_SCHEMES = _environ_get_and_map('ALLOWED_URL_SCHEMES', None, _AS_LIST) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +if 'BANNER_TOP' in environ: + BANNER_TOP = environ.get('BANNER_TOP', None) +if 'BANNER_BOTTOM' in environ: + BANNER_BOTTOM = environ.get('BANNER_BOTTOM', None) + +# Text to include on the login page above the login form. HTML is allowed. +if 'BANNER_LOGIN' in environ: + BANNER_LOGIN = environ.get('BANNER_LOGIN', None) + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +if 'CHANGELOG_RETENTION' in environ: + CHANGELOG_RETENTION = _environ_get_and_map('CHANGELOG_RETENTION', None, _AS_INT) + +# Maximum number of days to retain job results (scripts and reports). Set to 0 to retain job results in the database indefinitely. (Default: 90) +if 'JOB_RETENTION' in environ: + JOB_RETENTION = _environ_get_and_map('JOB_RETENTION', None, _AS_INT) +# JOBRESULT_RETENTION was renamed to JOB_RETENTION in the v3.5.0 release of NetBox. For backwards compatibility, map JOBRESULT_RETENTION to JOB_RETENTION +elif 'JOBRESULT_RETENTION' in environ: + JOB_RETENTION = _environ_get_and_map('JOBRESULT_RETENTION', None, _AS_INT) + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = _environ_get_and_map('CORS_ORIGIN_ALLOW_ALL', 'False', _AS_BOOL) +CORS_ORIGIN_WHITELIST = _environ_get_and_map('CORS_ORIGIN_WHITELIST', 'https://localhost', _AS_LIST) +CORS_ORIGIN_REGEX_WHITELIST = [re.compile(r) for r in _environ_get_and_map('CORS_ORIGIN_REGEX_WHITELIST', '', _AS_LIST)] + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. +# Never enable debugging on a production system. +DEBUG = _environ_get_and_map('DEBUG', 'False', _AS_BOOL) + +# This parameter serves as a safeguard to prevent some potentially dangerous behavior, +# such as generating new database schema migrations. +# Set this to True only if you are actively developing the NetBox code base. +DEVELOPER = _environ_get_and_map('DEVELOPER', 'False', _AS_BOOL) + +# Email settings +EMAIL = { + 'SERVER': environ.get('EMAIL_SERVER', 'localhost'), + 'PORT': _environ_get_and_map('EMAIL_PORT', 25, _AS_INT), + 'USERNAME': environ.get('EMAIL_USERNAME', ''), + 'PASSWORD': _read_secret('email_password', environ.get('EMAIL_PASSWORD', '')), + 'USE_SSL': _environ_get_and_map('EMAIL_USE_SSL', 'False', _AS_BOOL), + 'USE_TLS': _environ_get_and_map('EMAIL_USE_TLS', 'False', _AS_BOOL), + 'SSL_CERTFILE': environ.get('EMAIL_SSL_CERTFILE', ''), + 'SSL_KEYFILE': environ.get('EMAIL_SSL_KEYFILE', ''), + 'TIMEOUT': _environ_get_and_map('EMAIL_TIMEOUT', 10, _AS_INT), # seconds + 'FROM_EMAIL': environ.get('EMAIL_FROM', ''), +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +if 'ENFORCE_GLOBAL_UNIQUE' in environ: + ENFORCE_GLOBAL_UNIQUE = _environ_get_and_map('ENFORCE_GLOBAL_UNIQUE', None, _AS_BOOL) + +# By default, netbox sends census reporting data using a single HTTP request each time a worker starts. +# This data enables the project maintainers to estimate how many NetBox deployments exist and track the adoption of new versions over time. +# The only data reported by this function are the NetBox version, Python version, and a pseudorandom unique identifier. +# To opt out of census reporting, set CENSUS_REPORTING_ENABLED to False. +if 'CENSUS_REPORTING_ENABLED' in environ: + CENSUS_REPORTING_ENABLED = _environ_get_and_map('CENSUS_REPORTING_ENABLED', None, _AS_BOOL) + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form `.`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = _environ_get_and_map('EXEMPT_VIEW_PERMISSIONS', '', _AS_LIST) + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = _environ_get_and_map('INTERNAL_IPS', '127.0.0.1 ::1', _AS_LIST) + +# Enable GraphQL API. +if 'GRAPHQL_ENABLED' in environ: + GRAPHQL_ENABLED = _environ_get_and_map('GRAPHQL_ENABLED', None, _AS_BOOL) + +# # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# # https://docs.djangoproject.com/en/stable/topics/logging/ +# LOGGING = {} + +# Automatically reset the lifetime of a valid session upon each authenticated request. Enables users to remain +# authenticated to NetBox indefinitely. +LOGIN_PERSISTENCE = _environ_get_and_map('LOGIN_PERSISTENCE', 'False', _AS_BOOL) + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox (excluding secrets) but not make any changes. +LOGIN_REQUIRED = _environ_get_and_map('LOGIN_REQUIRED', 'False', _AS_BOOL) + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = _environ_get_and_map('LOGIN_TIMEOUT', 1209600, _AS_INT) + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +if 'MAINTENANCE_MODE' in environ: + MAINTENANCE_MODE = _environ_get_and_map('MAINTENANCE_MODE', None, _AS_BOOL) + +# Maps provider +if 'MAPS_URL' in environ: + MAPS_URL = environ.get('MAPS_URL', None) + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +if 'MAX_PAGE_SIZE' in environ: + MAX_PAGE_SIZE = _environ_get_and_map('MAX_PAGE_SIZE', None, _AS_INT) + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +MEDIA_ROOT = environ.get('MEDIA_ROOT', join(_BASE_DIR, 'media')) + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = _environ_get_and_map('METRICS_ENABLED', 'False', _AS_BOOL) + +# Determine how many objects to display per page within a list. (Default: 50) +if 'PAGINATE_COUNT' in environ: + PAGINATE_COUNT = _environ_get_and_map('PAGINATE_COUNT', None, _AS_INT) + +# # Enable installed plugins. Add the name of each plugin to the list. +# PLUGINS = [] + +# # Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# PLUGINS_CONFIG = { +# } + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +if 'PREFER_IPV4' in environ: + PREFER_IPV4 = _environ_get_and_map('PREFER_IPV4', None, _AS_BOOL) + +# The default value for the amperage field when creating new power feeds. +if 'POWERFEED_DEFAULT_AMPERAGE' in environ: + POWERFEED_DEFAULT_AMPERAGE = _environ_get_and_map('POWERFEED_DEFAULT_AMPERAGE', None, _AS_INT) + +# The default value (percentage) for the max_utilization field when creating new power feeds. +if 'POWERFEED_DEFAULT_MAX_UTILIZATION' in environ: + POWERFEED_DEFAULT_MAX_UTILIZATION = _environ_get_and_map('POWERFEED_DEFAULT_MAX_UTILIZATION', None, _AS_INT) + +# The default value for the voltage field when creating new power feeds. +if 'POWERFEED_DEFAULT_VOLTAGE' in environ: + POWERFEED_DEFAULT_VOLTAGE = _environ_get_and_map('POWERFEED_DEFAULT_VOLTAGE', None, _AS_INT) + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +if 'RACK_ELEVATION_DEFAULT_UNIT_HEIGHT' in environ: + RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = _environ_get_and_map('RACK_ELEVATION_DEFAULT_UNIT_HEIGHT', None, _AS_INT) +if 'RACK_ELEVATION_DEFAULT_UNIT_WIDTH' in environ: + RACK_ELEVATION_DEFAULT_UNIT_WIDTH = _environ_get_and_map('RACK_ELEVATION_DEFAULT_UNIT_WIDTH', None, _AS_INT) + +# Remote authentication support +REMOTE_AUTH_ENABLED = _environ_get_and_map('REMOTE_AUTH_ENABLED', 'False', _AS_BOOL) +REMOTE_AUTH_BACKEND = _environ_get_and_map('REMOTE_AUTH_BACKEND', 'netbox.authentication.RemoteUserBackend', _AS_LIST) +REMOTE_AUTH_HEADER = environ.get('REMOTE_AUTH_HEADER', 'HTTP_REMOTE_USER') +REMOTE_AUTH_AUTO_CREATE_USER = _environ_get_and_map('REMOTE_AUTH_AUTO_CREATE_USER', 'False', _AS_BOOL) +REMOTE_AUTH_DEFAULT_GROUPS = _environ_get_and_map('REMOTE_AUTH_DEFAULT_GROUPS', '', _AS_LIST) +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +RELEASE_CHECK_URL = environ.get('RELEASE_CHECK_URL', None) +# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +# Maximum execution time for background tasks, in seconds. +RQ_DEFAULT_TIMEOUT = _environ_get_and_map('RQ_DEFAULT_TIMEOUT', 300, _AS_INT) + +# The name to use for the csrf token cookie. +CSRF_COOKIE_NAME = environ.get('CSRF_COOKIE_NAME', 'csrftoken') + +# Cross-Site-Request-Forgery-Attack settings. If Netbox is sitting behind a reverse proxy, you might need to set the CSRF_TRUSTED_ORIGINS flag. +# Django 4.0 requires to specify the URL Scheme in this setting. An example environment variable could be specified like: +# CSRF_TRUSTED_ORIGINS=https://demo.netbox.dev http://demo.netbox.dev +CSRF_TRUSTED_ORIGINS = _environ_get_and_map('CSRF_TRUSTED_ORIGINS', '', _AS_LIST) + +# The name to use for the session cookie. +SESSION_COOKIE_NAME = environ.get('SESSION_COOKIE_NAME', 'sessionid') + +# If true, the `includeSubDomains` directive will be included in the HTTP Strict Transport Security (HSTS) header. +# This directive instructs the browser to apply the HSTS policy to all subdomains of the current domain. +SECURE_HSTS_INCLUDE_SUBDOMAINS = _environ_get_and_map('SECURE_HSTS_INCLUDE_SUBDOMAINS', 'False', _AS_BOOL) + +# If true, the `preload` directive will be included in the HTTP Strict Transport Security (HSTS) header. +# This directive instructs the browser to preload the site in HTTPS. Browsers that use the HSTS preload list will force the +# site to be accessed via HTTPS even if the user types HTTP in the address bar. +SECURE_HSTS_PRELOAD = _environ_get_and_map('SECURE_HSTS_PRELOAD', 'False', _AS_BOOL) + +# If set to a non-zero integer value, the SecurityMiddleware sets the HTTP Strict Transport Security (HSTS) header on all +# responses that do not already have it. This will instruct the browser that the website must be accessed via HTTPS, +# blocking any HTTP request. +SECURE_HSTS_SECONDS = _environ_get_and_map('SECURE_HSTS_SECONDS', 0, _AS_INT) + +# If true, all non-HTTPS requests will be automatically redirected to use HTTPS. +SECURE_SSL_REDIRECT = _environ_get_and_map('SECURE_SSL_REDIRECT', 'False', _AS_BOOL) + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = environ.get('SESSION_FILE_PATH', environ.get('SESSIONS_ROOT', None)) + +# Time zone (default: UTC) +TIME_ZONE = environ.get('TIME_ZONE', 'UTC') + diff --git a/configuration/extra.py b/configuration/extra.py new file mode 100644 index 0000000..8bd1337 --- /dev/null +++ b/configuration/extra.py @@ -0,0 +1,49 @@ +#### +## This file contains extra configuration options that can't be configured +## directly through environment variables. +#### + +## Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +## application errors (assuming correct email settings are provided). +# ADMINS = [ +# # ['John Doe', 'jdoe@example.com'], +# ] + + +## URL schemes that are allowed within links in NetBox +# ALLOWED_URL_SCHEMES = ( +# 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +# ) + +## Enable installed plugins. Add the name of each plugin to the list. +# from netbox.configuration.configuration import PLUGINS +# PLUGINS.append('my_plugin') + +## Plugins configuration settings. These settings are used by various plugins that the user may have installed. +## Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# from netbox.configuration.configuration import PLUGINS_CONFIG +# PLUGINS_CONFIG['my_plugin'] = { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } + + +## Remote authentication support +# REMOTE_AUTH_DEFAULT_PERMISSIONS = {} + + +## By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +## class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + + +## This file can contain arbitrary Python code, e.g.: +# from datetime import datetime +# now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") +# BANNER_TOP = f'This instance started on {now}.' diff --git a/configuration/ldap/extra.py b/configuration/ldap/extra.py new file mode 100644 index 0000000..4505197 --- /dev/null +++ b/configuration/ldap/extra.py @@ -0,0 +1,28 @@ +#### +## This file contains extra configuration options that can't be configured +## directly through environment variables. +## All vairables set here overwrite any existing found in ldap_config.py +#### + +# # This Python script inherits all the imports from ldap_config.py +# from django_auth_ldap.config import LDAPGroupQuery # Imported since not in ldap_config.py + +# # Sets a base requirement of membetship to netbox-user-ro, netbox-user-rw, or netbox-user-admin. +# AUTH_LDAP_REQUIRE_GROUP = ( +# LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com") +# | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com") +# | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com") +# ) + +# # Sets LDAP Flag groups variables with example. +# AUTH_LDAP_USER_FLAGS_BY_GROUP = { +# "is_staff": ( +# LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com") +# | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com") +# | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com") +# ), +# "is_superuser": "cn=netbox-user-admin,ou=groups,dc=example,dc=com", +# } + +# # Sets LDAP Mirror groups variables with example groups +# AUTH_LDAP_MIRROR_GROUPS = ["netbox-user-ro", "netbox-user-rw", "netbox-user-admin"] diff --git a/configuration/ldap/ldap_config.py b/configuration/ldap/ldap_config.py new file mode 100644 index 0000000..82fad72 --- /dev/null +++ b/configuration/ldap/ldap_config.py @@ -0,0 +1,111 @@ +from importlib import import_module +from os import environ + +import ldap +from django_auth_ldap.config import LDAPSearch + + +# Read secret from file +def _read_secret(secret_name, default=None): + try: + f = open('/run/secrets/' + secret_name, 'r', encoding='utf-8') + except EnvironmentError: + return default + else: + with f: + return f.readline().strip() + +# Import and return the group type based on string name +def _import_group_type(group_type_name): + mod = import_module('django_auth_ldap.config') + try: + return getattr(mod, group_type_name)() + except: + return None + +# Server URI +AUTH_LDAP_SERVER_URI = environ.get('AUTH_LDAP_SERVER_URI', '') + +# The following may be needed if you are binding to Active Directory. +AUTH_LDAP_CONNECTION_OPTIONS = { + ldap.OPT_REFERRALS: 0 +} + +AUTH_LDAP_BIND_AS_AUTHENTICATING_USER = environ.get('AUTH_LDAP_BIND_AS_AUTHENTICATING_USER', 'False').lower() == 'true' + +# Set the DN and password for the NetBox service account if needed. +if not AUTH_LDAP_BIND_AS_AUTHENTICATING_USER: + AUTH_LDAP_BIND_DN = environ.get('AUTH_LDAP_BIND_DN', '') + AUTH_LDAP_BIND_PASSWORD = _read_secret('auth_ldap_bind_password', environ.get('AUTH_LDAP_BIND_PASSWORD', '')) + +# Set a string template that describes any user’s distinguished name based on the username. +AUTH_LDAP_USER_DN_TEMPLATE = environ.get('AUTH_LDAP_USER_DN_TEMPLATE', None) + +# Enable STARTTLS for ldap authentication. +AUTH_LDAP_START_TLS = environ.get('AUTH_LDAP_START_TLS', 'False').lower() == 'true' + +# Include this setting if you want to ignore certificate errors. This might be needed to accept a self-signed cert. +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) +LDAP_IGNORE_CERT_ERRORS = environ.get('LDAP_IGNORE_CERT_ERRORS', 'False').lower() == 'true' + +# Include this setting if you want to validate the LDAP server certificates against a CA certificate directory on your server +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, LDAP_CA_CERT_DIR) +LDAP_CA_CERT_DIR = environ.get('LDAP_CA_CERT_DIR', None) + +# Include this setting if you want to validate the LDAP server certificates against your own CA. +# Note that this is a NetBox-specific setting which sets: +# ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, LDAP_CA_CERT_FILE) +LDAP_CA_CERT_FILE = environ.get('LDAP_CA_CERT_FILE', None) + +AUTH_LDAP_USER_SEARCH_BASEDN = environ.get('AUTH_LDAP_USER_SEARCH_BASEDN', '') +AUTH_LDAP_USER_SEARCH_ATTR = environ.get('AUTH_LDAP_USER_SEARCH_ATTR', 'sAMAccountName') +AUTH_LDAP_USER_SEARCH_FILTER: str = environ.get( + 'AUTH_LDAP_USER_SEARCH_FILTER', f'({AUTH_LDAP_USER_SEARCH_ATTR}=%(user)s)' +) + +AUTH_LDAP_USER_SEARCH = LDAPSearch( + AUTH_LDAP_USER_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, AUTH_LDAP_USER_SEARCH_FILTER +) + +# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group +# heirarchy. + +AUTH_LDAP_GROUP_SEARCH_BASEDN = environ.get('AUTH_LDAP_GROUP_SEARCH_BASEDN', '') +AUTH_LDAP_GROUP_SEARCH_CLASS = environ.get('AUTH_LDAP_GROUP_SEARCH_CLASS', 'group') + +AUTH_LDAP_GROUP_SEARCH_FILTER: str = environ.get( + 'AUTH_LDAP_GROUP_SEARCH_FILTER', f'(objectclass={AUTH_LDAP_GROUP_SEARCH_CLASS})' +) +AUTH_LDAP_GROUP_SEARCH = LDAPSearch( + AUTH_LDAP_GROUP_SEARCH_BASEDN, ldap.SCOPE_SUBTREE, AUTH_LDAP_GROUP_SEARCH_FILTER +) +AUTH_LDAP_GROUP_TYPE = _import_group_type(environ.get('AUTH_LDAP_GROUP_TYPE', 'GroupOfNamesType')) + +# Define a group required to login. +AUTH_LDAP_REQUIRE_GROUP = environ.get('AUTH_LDAP_REQUIRE_GROUP_DN') + +# Define special user types using groups. Exercise great caution when assigning superuser status. +AUTH_LDAP_USER_FLAGS_BY_GROUP = {} + +if AUTH_LDAP_REQUIRE_GROUP is not None: + AUTH_LDAP_USER_FLAGS_BY_GROUP = { + "is_active": environ.get('AUTH_LDAP_REQUIRE_GROUP_DN', ''), + "is_staff": environ.get('AUTH_LDAP_IS_ADMIN_DN', ''), + "is_superuser": environ.get('AUTH_LDAP_IS_SUPERUSER_DN', '') + } + +# For more granular permissions, we can map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = environ.get('AUTH_LDAP_FIND_GROUP_PERMS', 'True').lower() == 'true' +AUTH_LDAP_MIRROR_GROUPS = environ.get('AUTH_LDAP_MIRROR_GROUPS', '').lower() == 'true' + +# Cache groups for one hour to reduce LDAP traffic +AUTH_LDAP_CACHE_TIMEOUT = int(environ.get('AUTH_LDAP_CACHE_TIMEOUT', 3600)) + +# Populate the Django user from the LDAP directory. +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": environ.get('AUTH_LDAP_ATTR_FIRSTNAME', 'givenName'), + "last_name": environ.get('AUTH_LDAP_ATTR_LASTNAME', 'sn'), + "email": environ.get('AUTH_LDAP_ATTR_MAIL', 'mail') +} diff --git a/configuration/logging.py b/configuration/logging.py new file mode 100644 index 0000000..d786768 --- /dev/null +++ b/configuration/logging.py @@ -0,0 +1,55 @@ +# # Remove first comment(#) on each line to implement this working logging example. +# # Add LOGLEVEL environment variable to netbox if you use this example & want a different log level. +# from os import environ + +# # Set LOGLEVEL in netbox.env or docker-compose.overide.yml to override a logging level of INFO. +# LOGLEVEL = environ.get('LOGLEVEL', 'INFO') + +# LOGGING = { + +# 'version': 1, +# 'disable_existing_loggers': False, +# 'formatters': { +# 'verbose': { +# 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', +# 'style': '{', +# }, +# 'simple': { +# 'format': '{levelname} {message}', +# 'style': '{', +# }, +# }, +# 'filters': { +# 'require_debug_false': { +# '()': 'django.utils.log.RequireDebugFalse', +# }, +# }, +# 'handlers': { +# 'console': { +# 'level': LOGLEVEL, +# 'filters': ['require_debug_false'], +# 'class': 'logging.StreamHandler', +# 'formatter': 'simple' +# }, +# 'mail_admins': { +# 'level': 'ERROR', +# 'class': 'django.utils.log.AdminEmailHandler', +# 'filters': ['require_debug_false'] +# } +# }, +# 'loggers': { +# 'django': { +# 'handlers': ['console'], +# 'propagate': True, +# }, +# 'django.request': { +# 'handlers': ['mail_admins'], +# 'level': 'ERROR', +# 'propagate': False, +# }, +# 'django_auth_ldap': { +# 'handlers': ['console',], +# 'level': LOGLEVEL, +# } +# } +# } diff --git a/configuration/plugins.py b/configuration/plugins.py new file mode 100644 index 0000000..c0b1a1f --- /dev/null +++ b/configuration/plugins.py @@ -0,0 +1,13 @@ +# Add your plugins and plugin settings here. +# Of course uncomment this file out. + +# To learn how to build images with your required plugins +# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins + +# PLUGINS = ["netbox_bgp"] + +# PLUGINS_CONFIG = { +# "netbox_bgp": { +# ADD YOUR SETTINGS HERE +# } +# } diff --git a/env/netbox.env b/env/netbox.env new file mode 100644 index 0000000..9f6835a --- /dev/null +++ b/env/netbox.env @@ -0,0 +1,38 @@ +CORS_ORIGIN_ALLOW_ALL=True +DB_HOST=postgres +DB_NAME=netbox +DB_PASSWORD=J5brHrAXFLQSif0K +DB_USER=netbox + +EMAIL_FROM=netbox@bar.com +EMAIL_PASSWORD= +EMAIL_PORT=25 +EMAIL_SERVER=localhost +EMAIL_SSL_CERTFILE= +EMAIL_SSL_KEYFILE= +EMAIL_TIMEOUT=5 +EMAIL_USERNAME=netbox +# EMAIL_USE_SSL and EMAIL_USE_TLS are mutually exclusive, i.e. they can't both be `true`! +EMAIL_USE_SSL=false +EMAIL_USE_TLS=false + +GRAPHQL_ENABLED=true +HOUSEKEEPING_INTERVAL=86400 +MEDIA_ROOT=/opt/netbox/netbox/media +METRICS_ENABLED=false + +REDIS_CACHE_DATABASE=1 +REDIS_CACHE_HOST=redis-cache +REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false +REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36 +REDIS_CACHE_SSL=false +REDIS_DATABASE=0 +REDIS_HOST=redis +REDIS_INSECURE_SKIP_TLS_VERIFY=false +REDIS_PASSWORD=H733Kdjndks81 +REDIS_SSL=false + +RELEASE_CHECK_URL=https://api.github.com/repos/netbox-community/netbox/releases +SECRET_KEY='r(m)9nLGnz$(_q3N4z1k(EFsMCjjjzx08x9VhNVcfd%6RF#r!6DE@+V5Zk2X' +SKIP_SUPERUSER=true +WEBHOOKS_ENABLED=true diff --git a/env/postgres.env b/env/postgres.env new file mode 100644 index 0000000..bb7b53c --- /dev/null +++ b/env/postgres.env @@ -0,0 +1,3 @@ +POSTGRES_DB=netbox +POSTGRES_PASSWORD=J5brHrAXFLQSif0K +POSTGRES_USER=netbox diff --git a/env/redis-cache.env b/env/redis-cache.env new file mode 100644 index 0000000..6285c33 --- /dev/null +++ b/env/redis-cache.env @@ -0,0 +1 @@ +REDIS_PASSWORD=t4Ph722qJ5QHeQ1qfu36 diff --git a/env/redis.env b/env/redis.env new file mode 100644 index 0000000..44a1987 --- /dev/null +++ b/env/redis.env @@ -0,0 +1 @@ +REDIS_PASSWORD=H733Kdjndks81 From f87505bfefbf74d1a3bdaa0e7f1adf5bdc0e88f3 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sat, 27 Jul 2024 23:03:38 +0200 Subject: [PATCH 29/91] fix: attach functions to go object --- model/netbox.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 279ae63..c40cf3e 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -49,11 +49,11 @@ func NewNetbox() Netbox { return nbx } -func (n Netbox) IsConnected() bool { +func (n *Netbox) IsConnected() bool { return n._isConnected } -func (n Netbox) Connect() error { +func (n *Netbox) Connect() error { if n._isConnected { return nil } @@ -75,7 +75,7 @@ func getVm(name string, serial string) netbox.WritableVirtualMachineWithConfigCo } } -func (n Netbox) CreateVM(msg Message) (int32, error) { +func (n *Netbox) CreateVM(msg Message) (int32, error) { var ( id int32 ) @@ -99,7 +99,7 @@ func (n Netbox) CreateVM(msg Message) (int32, error) { return id, nil } -func (n Netbox) UpdateVM(id int32, msg Message) error { +func (n *Netbox) UpdateVM(id int32, msg Message) error { vm := getVm(msg.Hostname, msg.Serial) _, _, err := n.Client.VirtualizationAPI. @@ -110,7 +110,7 @@ func (n Netbox) UpdateVM(id int32, msg Message) error { return err } -func (n Netbox) CreateOrUpdateVM(msg Message) error { +func (n *Netbox) CreateOrUpdateVM(msg Message) error { if !n._isConnected { return errors.New("netbox is not connected") } From 1bce5d786fb1facbcd897d6459bbf5dfb863ee84 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:15:38 +0200 Subject: [PATCH 30/91] refactor: use standard error formatting --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index a4060b5..2464676 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ import ( func failOnError(err error, msg string) { if err != nil { - util.Err(fmt.Sprintf("%s: %s", msg, err)) + util.Err(fmt.Errorf("%s: %w", msg, err).Error()) } } From 8679d2eecc0c265be7db8b41cd5e379380f6f329 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:15:56 +0200 Subject: [PATCH 31/91] refactor: use queue name from env --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 2464676..0c750a9 100644 --- a/main.go +++ b/main.go @@ -28,8 +28,10 @@ func main() { ch, err := conn.Channel() failOnError(err, fmt.Sprintf("Failed to open a channel : %s", err)) + queueName := os.Getenv("QUEUE_NAME") + q, err := ch.QueueDeclare( - "onboarding", + queueName, true, false, false, From c3dd1b6e830c50ba318b0149141531893929833d Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:16:30 +0200 Subject: [PATCH 32/91] feat: add ttl to message --- model/message.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model/message.go b/model/message.go index ec2a105..149a24c 100644 --- a/model/message.go +++ b/model/message.go @@ -7,5 +7,8 @@ type Message struct { IpAddress string `json:"ipaddress"` Serial string `json:"serial"` + //Make following json field optional with default 0 + FailCount int `json:"failcount" binding:"optional"` + Timestamp time.Time `json:"-"` } From 6b2126cba169ba45dfd9ef291446c880d8067243 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:16:49 +0200 Subject: [PATCH 33/91] refactor: rework machine serial compute --- model/message.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/model/message.go b/model/message.go index 149a24c..3b8e804 100644 --- a/model/message.go +++ b/model/message.go @@ -1,14 +1,29 @@ package model -import "time" +import ( + "strings" + "time" +) type Message struct { Hostname string `json:"hostname"` IpAddress string `json:"ipaddress"` - Serial string `json:"serial"` + serial string `json:"serial" binding:"optional"` //Make following json field optional with default 0 FailCount int `json:"failcount" binding:"optional"` Timestamp time.Time `json:"-"` } + +func (m *Message) parseSerial() string { + return strings.Join(strings.Split(m.Hostname, "-")[1:], "-") +} + +func (m *Message) GetSerial() string { + if m.serial == "" { + m.serial = m.parseSerial() + } + + return m.serial +} From c5f77389ac25816bc81fbdb6af4edb58b11e8f94 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:17:34 +0200 Subject: [PATCH 34/91] refactor: remove old code --- model/netbox.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index c40cf3e..9c322d4 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -27,16 +27,6 @@ type Netbox struct { _isConnected bool } -func contains(list *list.List, item string) bool { - for e := list.Front(); e != nil; e = e.Next() { - if e.Value == item { - return true - } - } - - return false -} - // NewNetbox return a fresh Netbox object func NewNetbox() Netbox { nbx := Netbox{ From 8a6be400fd8f33e1a1599da0166cc4db132c3677 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:18:25 +0200 Subject: [PATCH 35/91] refactor: update code to use api v3 --- go.mod | 25 ++++++- go.sum | 193 +++++++++++++++++++++++++++++++++++++++++++++++- main.go | 68 +++++++++++++---- model/netbox.go | 177 ++++++++++++++++++++++++++++++++------------ 4 files changed, 399 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index f5bbf2b..417e650 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,34 @@ require github.com/rabbitmq/amqp091-go v1.10.0 require ( github.com/fatih/color v1.17.0 github.com/joho/godotenv v1.5.1 - github.com/netbox-community/go-netbox/v4 v4.0.3-0 + github.com/netbox-community/go-netbox v0.0.0-20230225105939-fe852c86b3d6 ) require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/go-openapi/analysis v0.21.2 // indirect + github.com/go-openapi/errors v0.20.2 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/loads v0.21.1 // indirect + github.com/go-openapi/runtime v0.23.3 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.21.2 // indirect + github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-openapi/validate v0.21.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + go.mongodb.org/mongo-driver v1.8.3 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.3.7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 11fae5a..ca1e632 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,208 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= +github.com/go-openapi/runtime v0.23.3 h1:/dxjx4KCOQI5ImBMz036F6v/DzZ2NUjSRvbLJs1rgoE= +github.com/go-openapi/runtime v0.23.3/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.21.2 h1:5NDNgadiX1Vhemth/TH4gCGopWSTdDjxl60H3B7f+os= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= +github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/netbox-community/go-netbox/v4 v4.0.3-0 h1:Q6NucmGe9ZmU3mjziuqQlKqHZmzC3FYbZn9r+Y507hs= -github.com/netbox-community/go-netbox/v4 v4.0.3-0/go.mod h1:eux5UXpY7T1BADzdFI1V7o+iIGxSfPLUuZvcjVWca/w= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/netbox-community/go-netbox v0.0.0-20230225105939-fe852c86b3d6 h1:vL8WkXAlu+rrmtWEbq+M9a8ZPaShDNpXu8nFrpek8IM= +github.com/netbox-community/go-netbox v0.0.0-20230225105939-fe852c86b3d6/go.mod h1:Tj90OQ0RNovdgAXEuVZtASdVhIGdZLvQtRd3p+upuuw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.3 h1:TDKlTkGDKm9kkJVUOAXDK5/fkqKHJVwYQSpoRfB43R4= +go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 0c750a9..ee9fe0a 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "github.com/KittenConnect/rh-api/model" @@ -8,6 +9,7 @@ import ( "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" "os" + "time" ) func failOnError(err error, msg string) { @@ -67,19 +69,59 @@ func main() { go func() { for d := range msgs { - msg := model.Message{Timestamp: d.Timestamp} - err := json.Unmarshal(d.Body, &msg) - if err != nil { - util.Warn(fmt.Sprintf("Error unmarshalling message : %s", err)) - continue - } - - //Make request to the rest of API - err = netbox.CreateOrUpdateVM(msg) - if err != nil { - util.Warn(fmt.Sprintf("Error creating or updating VM : %s", err)) - return - } + go func() { + msg := model.Message{Timestamp: d.Timestamp, FailCount: 20} + err := json.Unmarshal(d.Body, &msg) + if err != nil { + util.Warn(fmt.Sprintf("Error unmarshalling message : %s", err)) + return + } + + //Make request to the rest of API + err = netbox.CreateOrUpdateVM(msg) + if err != nil { + util.Warn(fmt.Errorf("error creating or updating VM : %s", err).Error()) + + dur, _ := time.ParseDuration("10s") + ctx, cancel := context.WithTimeout(context.Background(), dur) + defer cancel() + + newMsg := msg + newMsg.FailCount-- + + if newMsg.FailCount <= 0 { + return + } + + newMsgJson, _ := json.Marshal(newMsg) + + headers := amqp.Table{ + "x-delay": 60000, + } + + chErr := ch.PublishWithContext( + ctx, + "", + q.Name, + false, + false, + amqp.Publishing{ + ContentType: "application/json", + Body: newMsgJson, + Headers: headers, + }) + + if chErr != nil { + util.Warn(fmt.Sprintf("Error re-publishing message : %s", chErr)) + } else { + util.Warn(fmt.Sprintf("Re-sent message to RabbitMQ ®️ : %s", newMsgJson)) + } + + return + } + + util.Info("Connection successfully close from " + msg.GetSerial()) + }() } }() diff --git a/model/netbox.go b/model/netbox.go index 9c322d4..e3224a4 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -6,8 +6,14 @@ import ( "errors" "fmt" "github.com/KittenConnect/rh-api/util" - "github.com/netbox-community/go-netbox/v4" + "github.com/netbox-community/go-netbox/netbox" + "github.com/netbox-community/go-netbox/netbox/client" + "github.com/netbox-community/go-netbox/netbox/client/ipam" + "github.com/netbox-community/go-netbox/netbox/client/virtualization" + "github.com/netbox-community/go-netbox/netbox/models" "os" + "strconv" + "time" ) var MachinesSerials *list.List @@ -22,7 +28,7 @@ var ( type Netbox struct { ctx context.Context - Client *netbox.APIClient + Client *client.NetBoxAPI _isConnected bool } @@ -48,7 +54,7 @@ func (n *Netbox) Connect() error { return nil } - n.Client = netbox.NewAPIClientFor(os.Getenv("NETBOX_API_URL"), os.Getenv("NETBOX_API_KEY")) + n.Client = netbox.NewNetboxWithAPIKey(os.Getenv("NETBOX_API_URL"), os.Getenv("NETBOX_API_TOKEN")) n._isConnected = true MachinesSerials = list.New() @@ -56,46 +62,99 @@ func (n *Netbox) Connect() error { return nil } -func getVm(name string, serial string) netbox.WritableVirtualMachineWithConfigContextRequest { - return netbox.WritableVirtualMachineWithConfigContextRequest{ - Name: name, +func getVm(m Message) models.WritableVirtualMachineWithConfigContext { + var ( + status = "active" + cluster int64 = 1 + ) + + return models.WritableVirtualMachineWithConfigContext{ + Cluster: &cluster, + Name: &m.Hostname, + Status: status, + CustomFields: map[string]interface{}{ - NetboxVmSerialPrefix: serial, + "kc_serial_": m.GetSerial(), }, } } -func (n *Netbox) CreateVM(msg Message) (int32, error) { +func (n *Netbox) CreateVM(msg Message) (int64, error) { var ( - id int32 + id int64 ) if !n._isConnected { return id, errors.New("netbox is not connected") } - vm := getVm(msg.Hostname, msg.Serial) + vm := getVm(msg) + + params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&vm) + result, err := n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) + if err != nil { + if result != nil && result.Payload != nil { + return id, fmt.Errorf("error creating virtual machine: %w \n\t%s", err, result.Error()) + } + + return id, fmt.Errorf("error creating virtual machine: %w", err) + } + + util.Success(fmt.Sprintf("Created machine ID : %d", result.Payload.ID)) + + //Create management interface + var ( + mgmtInterfaceName = "mgmt" + ) + + ifParam := models.WritableVMInterface{ + Name: &mgmtInterfaceName, + Enabled: true, - _, result, err := n.Client.VirtualizationAPI. - VirtualizationVirtualMachinesCreate(n.ctx). - WritableVirtualMachineWithConfigContextRequest(vm). - Execute() + TaggedVlans: []int64{1}, + + VirtualMachine: &result.Payload.ID, + } + paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam) + res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { - return id, err + return id, fmt.Errorf("error creating virtual machine interface: %w", err) } + util.Success("\tSuccessfully created vm interface " + strconv.FormatInt(res.Payload.ID, 10)) + + var ( + ifId = res.Payload.ID + objectType = "virtualization.vminterface" + ) - util.Info(fmt.Sprintf("Created machine ID : %s", result.Body)) + //Set ip to the interface + ip := models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectID: &ifId, + AssignedObjectType: &objectType, + Status: models.IPAddressStatusValueActive, + } + ipParam := ipam.NewIpamIPAddressesCreateParams().WithData(&ip) + r, err := n.Client.Ipam.IpamIPAddressesCreate(ipParam, nil) + if err != nil { + return id, fmt.Errorf("error creating ip address: %w", err) + } + util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(r.Payload.ID, 10)) - return id, nil + return result.Payload.ID, nil } -func (n *Netbox) UpdateVM(id int32, msg Message) error { - vm := getVm(msg.Hostname, msg.Serial) +func (n *Netbox) UpdateVM(id int64, msg Message) error { + vm := getVm(msg) - _, _, err := n.Client.VirtualizationAPI. - VirtualizationVirtualMachinesUpdate(n.ctx, id). - WritableVirtualMachineWithConfigContextRequest(vm). - Execute() + //authContext + + updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ + Data: &vm, + ID: id, + } + + _, err := n.Client.Virtualization.VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(time.Duration(30)*time.Second), nil) return err } @@ -105,45 +164,67 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { return errors.New("netbox is not connected") } - var vmId int32 - var hasFoundVm bool = false + var vmId int64 + var hasFoundVm = false var err error // Call netbox API with specific serial, then update his settings accordingly - exist := contains(MachinesSerials, msg.Serial) - if !exist { - //If the vm don't exist in memory, fetch his details, if she exists in netbox - res, _, err := n.Client.VirtualizationAPI. - VirtualizationVirtualMachinesList(n.ctx). - Execute() - if err != nil { - return err + //exist := contains(MachinesSerials, msg.Hostname) //TODO + //if !exist { + //If the vm don't exist in memory, fetch his details, if she exists in netbox + req := virtualization. + NewVirtualizationVirtualMachinesListParams() + res, err := n.Client.Virtualization.VirtualizationVirtualMachinesList(req, nil) + if err != nil { + return fmt.Errorf("unable to get list of machines from netbox: %w", err) + } + + for _, vm := range res.Payload.Results { + if vm.Name == &msg.Hostname { + vmId = vm.ID + hasFoundVm = true + break } - for _, vm := range res.Results { - if vm.CustomFields[NetboxVmSerialPrefix] == msg.Serial { - vmId = vm.Id - hasFoundVm = true - break + var cf = vm.CustomFields.(map[string]interface{}) + var serial = "" + _ = serial + + for k, v := range cf { + switch c := v.(type) { + case string: + if k == "kc_serial_" { + serial = c + } + // fmt.Printf("Item %q is a string, containing %q\n", k, c) + //case float64: + // fmt.Printf("Looks like item %q is a number, specifically %f\n", k, c) + //default: + // fmt.Printf("Not sure what type item %q is, but I think it might be %T\n", k, c) } } - //Create VM if she doesn't exists in netbox - if !hasFoundVm { - vmId, err = n.CreateVM(msg) - - if err != nil { - return err - } + if serial == msg.GetSerial() { + vmId = vm.ID + hasFoundVm = true + break } + } - // - MachinesSerials.PushBack(msg.Serial) + //Create VM if she doesn't exists in netbox + if !hasFoundVm { + vmId, err = n.CreateVM(msg) + + if err != nil { + return fmt.Errorf("unable to create VM: %w", err) + } } else { err = n.UpdateVM(vmId, msg) if err != nil { - return err + return fmt.Errorf("unable to update VM: %w", err) } + + //util.Success("VM updated successfully") } return nil From 33b78feb0164400e0cf2618888de1206acb08697 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 17:19:58 +0200 Subject: [PATCH 36/91] refactor: remove old code --- model/netbox.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index e3224a4..dc4d289 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -1,7 +1,6 @@ package model import ( - "container/list" "context" "errors" "fmt" @@ -16,12 +15,6 @@ import ( "time" ) -var MachinesSerials *list.List - -var ( - NetboxVmSerialPrefix string = os.Getenv("NETBOX_VM_SERIAL_FIELD") -) - // Netbox structure // For internal use ONLY ! // To get an instance, call NewNetbox method @@ -57,8 +50,6 @@ func (n *Netbox) Connect() error { n.Client = netbox.NewNetboxWithAPIKey(os.Getenv("NETBOX_API_URL"), os.Getenv("NETBOX_API_TOKEN")) n._isConnected = true - MachinesSerials = list.New() - return nil } From 742dd4fec4b4898314c0fb9abea5f6020f9ca6ac Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 20:06:25 +0200 Subject: [PATCH 37/91] refactor: handle case of address already exists in netbox --- model/netbox.go | 118 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 20 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index dc4d289..80c7310 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -70,13 +70,30 @@ func getVm(m Message) models.WritableVirtualMachineWithConfigContext { } } -func (n *Netbox) CreateVM(msg Message) (int64, error) { - var ( - id int64 - ) +func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) error { + ip := models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectID: &ifId, + AssignedObjectType: &objectType, + Status: models.IPAddressStatusValueActive, + } + ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ + Data: &ip, + } + + _, err := n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(time.Duration(30)*time.Second), nil) + if err != nil { + return fmt.Errorf("error updating ip address: %w", err) + } + + util.Success("Update IP to VM interface") + return nil +} + +func (n *Netbox) CreateVM(msg Message) error { if !n._isConnected { - return id, errors.New("netbox is not connected") + return errors.New("netbox is not connected") } vm := getVm(msg) @@ -85,10 +102,10 @@ func (n *Netbox) CreateVM(msg Message) (int64, error) { result, err := n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) if err != nil { if result != nil && result.Payload != nil { - return id, fmt.Errorf("error creating virtual machine: %w \n\t%s", err, result.Error()) + return fmt.Errorf("error creating virtual machine: %w \n\t%s", err, result.Error()) } - return id, fmt.Errorf("error creating virtual machine: %w", err) + return fmt.Errorf("error creating virtual machine: %w", err) } util.Success(fmt.Sprintf("Created machine ID : %d", result.Payload.ID)) @@ -109,7 +126,7 @@ func (n *Netbox) CreateVM(msg Message) (int64, error) { paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam) res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { - return id, fmt.Errorf("error creating virtual machine interface: %w", err) + return fmt.Errorf("error creating virtual machine interface: %w", err) } util.Success("\tSuccessfully created vm interface " + strconv.FormatInt(res.Payload.ID, 10)) @@ -118,21 +135,82 @@ func (n *Netbox) CreateVM(msg Message) (int64, error) { objectType = "virtualization.vminterface" ) - //Set ip to the interface - ip := models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectID: &ifId, - AssignedObjectType: &objectType, - Status: models.IPAddressStatusValueActive, + //Verify if ip already exists + ipAlreadyExist := ipam.IpamIPAddressesListParams{ + Address: &msg.IpAddress, } - ipParam := ipam.NewIpamIPAddressesCreateParams().WithData(&ip) - r, err := n.Client.Ipam.IpamIPAddressesCreate(ipParam, nil) + req, err := n.Client.Ipam.IpamIPAddressesList(&ipAlreadyExist, nil) if err != nil { - return id, fmt.Errorf("error creating ip address: %w", err) + return fmt.Errorf("error checking ip addresses existance : %w", err) } - util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(r.Payload.ID, 10)) + var ( + zero = int64(0) + one = int64(1) + ) + + //We dont have that ip registered on netbox, so lets create him + if req.Payload.Count == &zero { + //Set ip to the interface + ip := models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectID: &ifId, + AssignedObjectType: &objectType, + Status: models.IPAddressStatusValueActive, + } + ipParam := ipam.NewIpamIPAddressesCreateParams().WithData(&ip) + r, err := n.Client.Ipam.IpamIPAddressesCreate(ipParam, nil) + if err != nil { + return fmt.Errorf("error creating ip address: %w", err) + } + util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(r.Payload.ID, 10)) + } else if req.Payload.Count == &one { + ip := req.Payload.Results[0] + + linkedInterfaceId := ip.AssignedObjectID + + //Si l'ip n'est pas liée à une interface + //On l'assigne à l'interface de la machine et zou + if linkedInterfaceId == nil { + return n.changeIPInterface(msg, ifId, objectType) + } + + //Sinon on vérifie sie la VM possède d'autres IP sur l'interface de management + interfaceId := *linkedInterfaceId + vmInterfaceParam := virtualization.NewVirtualizationInterfacesReadParams().WithID(interfaceId) + vmInterfaceResult, err := n.Client.Virtualization.VirtualizationInterfacesRead(vmInterfaceParam, nil) + if err != nil { + return fmt.Errorf("error reading virtual machine interface: %w", err) + } - return result.Payload.ID, nil + vmID := strconv.FormatInt(vmInterfaceResult.Payload.VirtualMachine.ID, 10) + + nestedVmParams := virtualization.VirtualizationInterfacesListParams{ + Name: &mgmtInterfaceName, + VirtualMachineID: &vmID, + } + nestedVmInterfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(&nestedVmParams, nil) + if err != nil { + return fmt.Errorf("error listing virtual machine interfaces: %w", err) + } + + mgmtInterface := nestedVmInterfaces.Payload.Results[0] + if mgmtInterface.CountIpaddresses > 1 { + //L'interface possède d'autres IPs + //Du coup, on prend l'ip en question + util.Info("Remove the link ...") + err := n.changeIPInterface(msg, ifId, objectType) + if err != nil { + return err + } + util.Success("IP changed of interface") + + return nil + } //Sinon on laisse l'ip sur la VM + + util.Warn("Trying to using existing IP on VM interface #" + strconv.FormatInt(mgmtInterface.ID, 10)) + } + + return nil } func (n *Netbox) UpdateVM(id int64, msg Message) error { @@ -204,7 +282,7 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { //Create VM if she doesn't exists in netbox if !hasFoundVm { - vmId, err = n.CreateVM(msg) + err = n.CreateVM(msg) if err != nil { return fmt.Errorf("unable to create VM: %w", err) From 7296efdca5e1ca4210a2e228fe09998c7daef19a Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 20:47:31 +0200 Subject: [PATCH 38/91] refactor: correct compare of request payload count --- model/netbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 80c7310..2f4c687 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -149,7 +149,7 @@ func (n *Netbox) CreateVM(msg Message) error { ) //We dont have that ip registered on netbox, so lets create him - if req.Payload.Count == &zero { + if *req.Payload.Count == zero { //Set ip to the interface ip := models.WritableIPAddress{ Address: &msg.IpAddress, @@ -163,7 +163,7 @@ func (n *Netbox) CreateVM(msg Message) error { return fmt.Errorf("error creating ip address: %w", err) } util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(r.Payload.ID, 10)) - } else if req.Payload.Count == &one { + } else if *req.Payload.Count == one { ip := req.Payload.Results[0] linkedInterfaceId := ip.AssignedObjectID From e2c9f4c24887ccc740754148fee0d6d0ec353ec1 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 28 Jul 2024 20:48:34 +0200 Subject: [PATCH 39/91] wip: handle update of vm management ip changes --- model/netbox.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 2f4c687..c42d9f2 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -216,16 +216,108 @@ func (n *Netbox) CreateVM(msg Message) error { func (n *Netbox) UpdateVM(id int64, msg Message) error { vm := getVm(msg) - //authContext - updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ Data: &vm, ID: id, } - _, err := n.Client.Virtualization.VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(time.Duration(30)*time.Second), nil) + //Update management IP + // 1. Get current interface IPs list + var ( + vmId = strconv.FormatInt(id, 10) + mgmtIfName = "mgmt" + ) + + ipIfParam := virtualization.VirtualizationInterfacesListParams{ + VirtualMachineID: &vmId, + Name: &mgmtIfName, + } + interfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(&ipIfParam, nil) + if err != nil { + return fmt.Errorf("error listing virtual machine interfaces: %w", err) + } + + // 2. If there is no interface, quit + var ( + ifCount = interfaces.Payload.Count + one = int64(1) + ) + if ifCount != &one { + //No virtual interface, create one + var ( + mgmtInterfaceName = "mgmt" + ) + + ifParam := models.WritableVMInterface{ + Name: &mgmtInterfaceName, + Enabled: true, + + TaggedVlans: []int64{1}, + + VirtualMachine: &id, + } + paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam) + _, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) + if err != nil { + return fmt.Errorf("error creating virtual machine interface: %w", err) + } + + _, err = n.Client.Virtualization.VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(time.Duration(30)*time.Second), nil) + if err != nil { + return fmt.Errorf("error updating virtual machine interface: %w", err) + } + + util.Info("Updated VM #" + strconv.FormatInt(id, 10) + " management interface with IP " + msg.IpAddress) + return nil + } - return err + // 3. Get the current management IP + mgmtInterface := interfaces.Payload.Results[0] + var mgmtInterfaceId = strconv.FormatInt(mgmtInterface.ID, 10) + params := ipam.NewIpamIPAddressesListParams() + params.SetInterfaceID(&mgmtInterfaceId) + + result, err := n.Client.Ipam.IpamIPAddressesList(params, nil) + if err != nil { + return fmt.Errorf("error listing ip addresses: %w", err) + } + + var ipCount = result.Payload.Count + util.Info("There are actually " + strconv.FormatInt(*ipCount, 10) + " IP(s) associated with the management interface") + + if *ipCount > one { + util.Warn("There are more than one management ip linked to the management interface") + return nil + } + + if *ipCount == one { + util.Warn("") + } + + ip := result.Payload.Results[0] + if *ip.Address == msg.IpAddress { + //Nothing to do + return nil + } + + // 4. The management IP changed, so : + // - unlink the old ip and interface + // - set the new ip to the interface + + oldIpUpdatePrams := models.WritableIPAddress{ + AssignedObjectType: nil, + AssignedObjectID: nil, + } + + paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams) + _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) + if err != nil { + return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) + } + + util.Success("Successfully updated management ip addresses of VM #" + vmId + " with new IP : " + msg.IpAddress) + + return nil } func (n *Netbox) CreateOrUpdateVM(msg Message) error { From 1efc75c1033e3dbb4a5aef0ef6c513caa2830a16 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 31 Jul 2024 17:48:08 +0200 Subject: [PATCH 40/91] feat: build go project on ci --- .github/workflows/go.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..a6346b0 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,23 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.22.4' + + - name: Build + run: go build -v ./... From 7859523249531346dd2317fd501e62e293732f29 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 31 Jul 2024 17:56:10 +0200 Subject: [PATCH 41/91] feat: upload build --- .github/workflows/go.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a6346b0..c863eb2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,3 +21,9 @@ jobs: - name: Build run: go build -v ./... + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: rh api + path: rh-api \ No newline at end of file From e8a5b76550bb2c357b15cf32353924f4c000f00c Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 31 Jul 2024 17:58:48 +0200 Subject: [PATCH 42/91] feat: set build output file --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c863eb2..07a42b9 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: go-version: '1.22.4' - name: Build - run: go build -v ./... + run: go build -v ./... -o rh-api - name: Upload artifact uses: actions/upload-artifact@v4 From a012f4d5d7e19738f57a1b4f871dc0f84d655c86 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 31 Jul 2024 18:02:16 +0200 Subject: [PATCH 43/91] Revert "feat: set build output file" This reverts commit e8a5b76550bb2c357b15cf32353924f4c000f00c. --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 07a42b9..c863eb2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: go-version: '1.22.4' - name: Build - run: go build -v ./... -o rh-api + run: go build -v ./... - name: Upload artifact uses: actions/upload-artifact@v4 From b2658298563adb7bfe967a8e640bd525ce5e9091 Mon Sep 17 00:00:00 2001 From: toinux <26522723+itzwam@users.noreply.github.com> Date: Sat, 10 Aug 2024 11:21:36 +0200 Subject: [PATCH 44/91] Update go.yml --- .github/workflows/go.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c863eb2..bfb22fe 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,10 +20,13 @@ jobs: go-version: '1.22.4' - name: Build - run: go build -v ./... + run: go build -o rh-api -v ./... + + - name: debug + run: ls -lah - name: Upload artifact uses: actions/upload-artifact@v4 with: name: rh api - path: rh-api \ No newline at end of file + path: rh-api From 242f674a4a47990aa650f86057d76d4ce8a8dd83 Mon Sep 17 00:00:00 2001 From: toinux <26522723+itzwam@users.noreply.github.com> Date: Sat, 10 Aug 2024 11:23:46 +0200 Subject: [PATCH 45/91] Update go.yml --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index bfb22fe..f542a53 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: go-version: '1.22.4' - name: Build - run: go build -o rh-api -v ./... + run: go build -o rh-api-linux -v . - name: debug run: ls -lah @@ -29,4 +29,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: rh api - path: rh-api + path: rh-api-linux From 741b8150b9236b5f2ce41caee9448de13510d716 Mon Sep 17 00:00:00 2001 From: Romain Date: Sat, 10 Aug 2024 12:57:25 +0200 Subject: [PATCH 46/91] refactor: add timeout --- model/netbox.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index c42d9f2..fe0d481 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -184,11 +184,11 @@ func (n *Netbox) CreateVM(msg Message) error { vmID := strconv.FormatInt(vmInterfaceResult.Payload.VirtualMachine.ID, 10) - nestedVmParams := virtualization.VirtualizationInterfacesListParams{ + nestedVmParams := &virtualization.VirtualizationInterfacesListParams{ Name: &mgmtInterfaceName, VirtualMachineID: &vmID, } - nestedVmInterfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(&nestedVmParams, nil) + nestedVmInterfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(nestedVmParams.WithTimeout(time.Duration(30)*time.Second), nil) if err != nil { return fmt.Errorf("error listing virtual machine interfaces: %w", err) } @@ -228,11 +228,11 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { mgmtIfName = "mgmt" ) - ipIfParam := virtualization.VirtualizationInterfacesListParams{ + ipIfParam := &virtualization.VirtualizationInterfacesListParams{ VirtualMachineID: &vmId, Name: &mgmtIfName, } - interfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(&ipIfParam, nil) + interfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(ipIfParam.WithTimeout(time.Duration(30)*time.Second), nil) if err != nil { return fmt.Errorf("error listing virtual machine interfaces: %w", err) } @@ -256,7 +256,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { VirtualMachine: &id, } - paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam) + paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam).WithTimeout(time.Duration(30) * time.Second) _, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) @@ -309,7 +309,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { AssignedObjectID: nil, } - paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams) + paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams).WithTimeout(time.Duration(30) * time.Second) _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) if err != nil { return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) From 23e76125172ecc9c6ddd5412fe0b749684e4a52d Mon Sep 17 00:00:00 2001 From: Romain Date: Sat, 10 Aug 2024 13:03:48 +0200 Subject: [PATCH 47/91] feat: add timeout when fetching machines list --- model/netbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index fe0d481..741e0b8 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -334,7 +334,7 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { //if !exist { //If the vm don't exist in memory, fetch his details, if she exists in netbox req := virtualization. - NewVirtualizationVirtualMachinesListParams() + NewVirtualizationVirtualMachinesListParams().WithTimeout(time.Duration(30) * time.Second) res, err := n.Client.Virtualization.VirtualizationVirtualMachinesList(req, nil) if err != nil { return fmt.Errorf("unable to get list of machines from netbox: %w", err) From 8b8797838351b7186716b537f49623091f9a1595 Mon Sep 17 00:00:00 2001 From: Romain Date: Sat, 10 Aug 2024 13:12:14 +0200 Subject: [PATCH 48/91] refactor: correct comparisons --- model/netbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 741e0b8..cc705d1 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -123,7 +123,7 @@ func (n *Netbox) CreateVM(msg Message) error { VirtualMachine: &result.Payload.ID, } - paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam) + paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam).WithTimeout(time.Duration(30) * time.Second) res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) @@ -242,7 +242,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { ifCount = interfaces.Payload.Count one = int64(1) ) - if ifCount != &one { + if *ifCount != one { //No virtual interface, create one var ( mgmtInterfaceName = "mgmt" From 1025a58252b6acd925100d8f2ee97191128595fc Mon Sep 17 00:00:00 2001 From: toinux <26522723+itzwam@users.noreply.github.com> Date: Sat, 10 Aug 2024 13:51:39 +0200 Subject: [PATCH 49/91] Update netbox.go --- model/netbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index cc705d1..e8529f0 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -119,7 +119,7 @@ func (n *Netbox) CreateVM(msg Message) error { Name: &mgmtInterfaceName, Enabled: true, - TaggedVlans: []int64{1}, + TaggedVlans: []int64{}, VirtualMachine: &result.Payload.ID, } @@ -252,7 +252,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { Name: &mgmtInterfaceName, Enabled: true, - TaggedVlans: []int64{1}, + TaggedVlans: []int64{}, VirtualMachine: &id, } From eb0725f3751891a42df26e52596b8be4feb040c4 Mon Sep 17 00:00:00 2001 From: Romain Date: Sat, 10 Aug 2024 14:01:55 +0200 Subject: [PATCH 50/91] wip: update interface ip --- model/netbox.go | 97 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 25 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index e8529f0..762403b 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -70,15 +70,20 @@ func getVm(m Message) models.WritableVirtualMachineWithConfigContext { } } -func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) error { - ip := models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectID: &ifId, - AssignedObjectType: &objectType, - Status: models.IPAddressStatusValueActive, +func (n *Netbox) getIpAddress(ip string) *models.WritableIPAddress { + return &models.WritableIPAddress{ + Address: &ip, + Status: models.IPAddressStatusValueActive, } +} + +func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) error { + ip := n.getIpAddress(msg.IpAddress) + ip.AssignedObjectID = &ifId + ip.AssignedObjectType = &objectType + ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ - Data: &ip, + Data: ip, } _, err := n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(time.Duration(30)*time.Second), nil) @@ -241,6 +246,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { var ( ifCount = interfaces.Payload.Count one = int64(1) + zero = int64(0) ) if *ifCount != one { //No virtual interface, create one @@ -286,37 +292,78 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { util.Info("There are actually " + strconv.FormatInt(*ipCount, 10) + " IP(s) associated with the management interface") if *ipCount > one { - util.Warn("There are more than one management ip linked to the management interface") - return nil + return errors.New("there are more than one management ip linked to the management interface") } if *ipCount == one { - util.Warn("") - } + ip := result.Payload.Results[0] + if *ip.Address == msg.IpAddress { + //Nothing to do + return nil + } + + // 4. The management IP changed, so : + // - unlink the old ip and interface + // - set the new ip to the interface + + oldIpUpdatePrams := models.WritableIPAddress{ + AssignedObjectType: nil, + AssignedObjectID: nil, + } + + paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams).WithTimeout(time.Duration(30) * time.Second) + _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) + if err != nil { + return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) + } - ip := result.Payload.Results[0] - if *ip.Address == msg.IpAddress { - //Nothing to do + util.Success("Successfully updated management ip addresses of VM #" + vmId + " with new IP : " + msg.IpAddress) return nil } - // 4. The management IP changed, so : - // - unlink the old ip and interface - // - set the new ip to the interface + // 5. No existing IP, but verify that she doesn't already exist in the netbox + ipSearchParams := ipam.NewIpamIPAddressesListParams() + result, err = n.Client.Ipam.IpamIPAddressesList(ipSearchParams, nil) + if err != nil { + return fmt.Errorf("error listing existing ip addresses: %w", err) + } + + existingIpCount := result.Payload.Count + newIpAddrId := int64(0) + if *existingIpCount == zero { + util.Info("There is no IP registered in the netbox. Create him.") + + newIp := &ipam.IpamIPAddressesCreateParams{ + Data: &models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectID: &mgmtInterface.ID, + }, + } + r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp, nil) + if err != nil { + return fmt.Errorf("error creating ip address: %w", err) + } - oldIpUpdatePrams := models.WritableIPAddress{ - AssignedObjectType: nil, - AssignedObjectID: nil, + newIpAddrId = r.Payload.ID + } else { + newIpAddrId = result.Payload.Results[0].ID } - paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams).WithTimeout(time.Duration(30) * time.Second) - _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) + var ipType = "virtualization.vminterface" + + ip := n.getIpAddress(msg.IpAddress) + ip.ID = newIpAddrId + ip.AssignedObjectID = &mgmtInterface.ID + ip.AssignedObjectType = &ipType + + ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ + Data: ip, + } + _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(time.Duration(30)*time.Second), nil) if err != nil { - return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) + return fmt.Errorf("error updating ip address: %w", err) } - util.Success("Successfully updated management ip addresses of VM #" + vmId + " with new IP : " + msg.IpAddress) - return nil } From a1d6d94a09c1a072fa7673a76a37ca20e1e6d72e Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 13:06:55 +0200 Subject: [PATCH 51/91] Fixes --- go.mod | 2 +- model/netbox.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 417e650..823b5df 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/KittenConnect/rh-api -go 1.22.4 +go 1.22 require github.com/rabbitmq/amqp091-go v1.10.0 diff --git a/model/netbox.go b/model/netbox.go index 762403b..71dc96d 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -86,7 +86,7 @@ func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) e Data: ip, } - _, err := n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(time.Duration(30)*time.Second), nil) + _, err := n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID).WithTimeout(time.Duration(30)*time.Second), nil) if err != nil { return fmt.Errorf("error updating ip address: %w", err) } @@ -153,6 +153,7 @@ func (n *Netbox) CreateVM(msg Message) error { one = int64(1) ) + util.Info(fmt.Sprintf("Found #%d IPs in %v", *req.Payload.Count, *req)) //We dont have that ip registered on netbox, so lets create him if *req.Payload.Count == zero { //Set ip to the interface @@ -199,7 +200,7 @@ func (n *Netbox) CreateVM(msg Message) error { } mgmtInterface := nestedVmInterfaces.Payload.Results[0] - if mgmtInterface.CountIpaddresses > 1 { + if mgmtInterface.CountIpaddresses == 1 { //L'interface possède d'autres IPs //Du coup, on prend l'ip en question util.Info("Remove the link ...") @@ -248,7 +249,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { one = int64(1) zero = int64(0) ) - if *ifCount != one { + if *ifCount < one { //No virtual interface, create one var ( mgmtInterfaceName = "mgmt" @@ -281,7 +282,8 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { mgmtInterface := interfaces.Payload.Results[0] var mgmtInterfaceId = strconv.FormatInt(mgmtInterface.ID, 10) params := ipam.NewIpamIPAddressesListParams() - params.SetInterfaceID(&mgmtInterfaceId) + params.SetVminterfaceID(&mgmtInterfaceId) + util.Info(fmt.Sprintf("Found MGMT Iface #%d -> %s", mgmtInterface.ID, mgmtInterfaceId)) result, err := n.Client.Ipam.IpamIPAddressesList(params, nil) if err != nil { @@ -323,6 +325,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { // 5. No existing IP, but verify that she doesn't already exist in the netbox ipSearchParams := ipam.NewIpamIPAddressesListParams() + ipSearchParams.Q = &msg.IpAddress result, err = n.Client.Ipam.IpamIPAddressesList(ipSearchParams, nil) if err != nil { return fmt.Errorf("error listing existing ip addresses: %w", err) @@ -332,14 +335,15 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { newIpAddrId := int64(0) if *existingIpCount == zero { util.Info("There is no IP registered in the netbox. Create him.") - + var ipType = "virtualization.vminterface" newIp := &ipam.IpamIPAddressesCreateParams{ Data: &models.WritableIPAddress{ Address: &msg.IpAddress, AssignedObjectID: &mgmtInterface.ID, + AssignedObjectType: &ipType, }, } - r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp, nil) + r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp.WithTimeout(time.Duration(30) * time.Second), nil) if err != nil { return fmt.Errorf("error creating ip address: %w", err) } From d943417a170fe795939cb9ce8353357d05991cc9 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 13:21:55 +0200 Subject: [PATCH 52/91] fix --- model/netbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 71dc96d..1213001 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -141,10 +141,10 @@ func (n *Netbox) CreateVM(msg Message) error { ) //Verify if ip already exists - ipAlreadyExist := ipam.IpamIPAddressesListParams{ + ipAlreadyExist := &ipam.IpamIPAddressesListParams{ Address: &msg.IpAddress, } - req, err := n.Client.Ipam.IpamIPAddressesList(&ipAlreadyExist, nil) + req, err := n.Client.Ipam.IpamIPAddressesList(ipAlreadyExist.WithTimeout(time.Duration(30) * time.Second), nil) if err != nil { return fmt.Errorf("error checking ip addresses existance : %w", err) } From 062ca180eb771e4eb1f0705ba056e937f465d5ca Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 13:26:23 +0200 Subject: [PATCH 53/91] Update Address --- model/netbox.go | 1 + 1 file changed, 1 insertion(+) diff --git a/model/netbox.go b/model/netbox.go index 1213001..79c7627 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -309,6 +309,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { // - set the new ip to the interface oldIpUpdatePrams := models.WritableIPAddress{ + Address: &msg.IpAddress, AssignedObjectType: nil, AssignedObjectID: nil, } From 4755c3f39c6ab0dd3287dabd1501f09751260dcc Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 13:31:27 +0200 Subject: [PATCH 54/91] refactor: extract default api call timeout duration --- model/netbox.go | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 79c7627..85f3e28 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -53,6 +53,10 @@ func (n *Netbox) Connect() error { return nil } +func (n *Netbox) GetDefaultTimeout() time.Duration { + return time.Duration(30) * time.Second +} + func getVm(m Message) models.WritableVirtualMachineWithConfigContext { var ( status = "active" @@ -86,7 +90,9 @@ func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) e Data: ip, } - _, err := n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID).WithTimeout(time.Duration(30)*time.Second), nil) + _, err := n.Client.Ipam. + IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID). + WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error updating ip address: %w", err) } @@ -128,7 +134,10 @@ func (n *Netbox) CreateVM(msg Message) error { VirtualMachine: &result.Payload.ID, } - paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam).WithTimeout(time.Duration(30) * time.Second) + paramInterface := virtualization. + NewVirtualizationInterfacesCreateParams(). + WithData(&ifParam). + WithTimeout(n.GetDefaultTimeout()) res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) @@ -144,7 +153,8 @@ func (n *Netbox) CreateVM(msg Message) error { ipAlreadyExist := &ipam.IpamIPAddressesListParams{ Address: &msg.IpAddress, } - req, err := n.Client.Ipam.IpamIPAddressesList(ipAlreadyExist.WithTimeout(time.Duration(30) * time.Second), nil) + req, err := n.Client.Ipam. + IpamIPAddressesList(ipAlreadyExist.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error checking ip addresses existance : %w", err) } @@ -182,7 +192,11 @@ func (n *Netbox) CreateVM(msg Message) error { //Sinon on vérifie sie la VM possède d'autres IP sur l'interface de management interfaceId := *linkedInterfaceId - vmInterfaceParam := virtualization.NewVirtualizationInterfacesReadParams().WithID(interfaceId) + vmInterfaceParam := virtualization. + NewVirtualizationInterfacesReadParams(). + WithID(interfaceId). + WithTimeout(n.GetDefaultTimeout()) + vmInterfaceResult, err := n.Client.Virtualization.VirtualizationInterfacesRead(vmInterfaceParam, nil) if err != nil { return fmt.Errorf("error reading virtual machine interface: %w", err) @@ -194,7 +208,8 @@ func (n *Netbox) CreateVM(msg Message) error { Name: &mgmtInterfaceName, VirtualMachineID: &vmID, } - nestedVmInterfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(nestedVmParams.WithTimeout(time.Duration(30)*time.Second), nil) + nestedVmInterfaces, err := n.Client.Virtualization. + VirtualizationInterfacesList(nestedVmParams.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error listing virtual machine interfaces: %w", err) } @@ -238,7 +253,8 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { VirtualMachineID: &vmId, Name: &mgmtIfName, } - interfaces, err := n.Client.Virtualization.VirtualizationInterfacesList(ipIfParam.WithTimeout(time.Duration(30)*time.Second), nil) + interfaces, err := n.Client.Virtualization. + VirtualizationInterfacesList(ipIfParam.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error listing virtual machine interfaces: %w", err) } @@ -263,13 +279,16 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { VirtualMachine: &id, } - paramInterface := virtualization.NewVirtualizationInterfacesCreateParams().WithData(&ifParam).WithTimeout(time.Duration(30) * time.Second) + paramInterface := virtualization. + NewVirtualizationInterfacesCreateParams(). + WithData(&ifParam).WithTimeout(n.GetDefaultTimeout()) _, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) } - _, err = n.Client.Virtualization.VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(time.Duration(30)*time.Second), nil) + _, err = n.Client.Virtualization. + VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error updating virtual machine interface: %w", err) } @@ -314,7 +333,11 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { AssignedObjectID: nil, } - paramUnlinkOldIp := ipam.NewIpamIPAddressesPartialUpdateParams().WithID(ip.ID).WithData(&oldIpUpdatePrams).WithTimeout(time.Duration(30) * time.Second) + paramUnlinkOldIp := ipam. + NewIpamIPAddressesPartialUpdateParams(). + WithID(ip.ID). + WithData(&oldIpUpdatePrams). + WithTimeout(n.GetDefaultTimeout()) _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) if err != nil { return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) @@ -344,7 +367,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { AssignedObjectType: &ipType, }, } - r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp.WithTimeout(time.Duration(30) * time.Second), nil) + r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error creating ip address: %w", err) } @@ -364,7 +387,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ Data: ip, } - _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(time.Duration(30)*time.Second), nil) + _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error updating ip address: %w", err) } @@ -386,7 +409,8 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { //if !exist { //If the vm don't exist in memory, fetch his details, if she exists in netbox req := virtualization. - NewVirtualizationVirtualMachinesListParams().WithTimeout(time.Duration(30) * time.Second) + NewVirtualizationVirtualMachinesListParams(). + WithTimeout(n.GetDefaultTimeout()) res, err := n.Client.Virtualization.VirtualizationVirtualMachinesList(req, nil) if err != nil { return fmt.Errorf("unable to get list of machines from netbox: %w", err) From 342411c962af66bdf96cea2f7fb11d02bbd1277b Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 13:32:27 +0200 Subject: [PATCH 55/91] refactor: extract creation of ip address --- model/netbox.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 85f3e28..aaab875 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -101,6 +101,29 @@ func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) e return nil } +func (n *Netbox) CreateIP(address string, status string, linkedObjectId int64, linkedObjectType string) (*ipam.IpamIPAddressesCreateCreated, error) { + ip := &models.WritableIPAddress{ + Address: &address, + Status: status, + } + + if linkedObjectId != -1 && linkedObjectType != "" { + ip.AssignedObjectID = &linkedObjectId + ip.AssignedObjectType = &linkedObjectType + } + + ipCreateParams := &ipam.IpamIPAddressesCreateParams{ + Data: ip, + } + + res, err := n.Client.Ipam.IpamIPAddressesCreate(ipCreateParams.WithTimeout(n.GetDefaultTimeout()), nil) + if err != nil { + return nil, fmt.Errorf("error creating ip address: %w", err) + } + + return res, nil +} + func (n *Netbox) CreateVM(msg Message) error { if !n._isConnected { @@ -167,18 +190,12 @@ func (n *Netbox) CreateVM(msg Message) error { //We dont have that ip registered on netbox, so lets create him if *req.Payload.Count == zero { //Set ip to the interface - ip := models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectID: &ifId, - AssignedObjectType: &objectType, - Status: models.IPAddressStatusValueActive, - } - ipParam := ipam.NewIpamIPAddressesCreateParams().WithData(&ip) - r, err := n.Client.Ipam.IpamIPAddressesCreate(ipParam, nil) + createdIP, err := n.CreateIP(msg.IpAddress, models.IPAddressStatusValueActive, ifId, objectType) if err != nil { - return fmt.Errorf("error creating ip address: %w", err) + return err } - util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(r.Payload.ID, 10)) + + util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(createdIP.Payload.ID, 10)) } else if *req.Payload.Count == one { ip := req.Payload.Results[0] From 7a914e2ddeaf2698be3849df031f3491fb52ea77 Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 13:33:16 +0200 Subject: [PATCH 56/91] refactor: only pass string --- model/netbox.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index aaab875..89f12bb 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -81,8 +81,8 @@ func (n *Netbox) getIpAddress(ip string) *models.WritableIPAddress { } } -func (n *Netbox) changeIPInterface(msg Message, ifId int64, objectType string) error { - ip := n.getIpAddress(msg.IpAddress) +func (n *Netbox) changeIPInterface(address string, ifId int64, objectType string) error { + ip := n.getIpAddress(address) ip.AssignedObjectID = &ifId ip.AssignedObjectType = &objectType @@ -204,7 +204,7 @@ func (n *Netbox) CreateVM(msg Message) error { //Si l'ip n'est pas liée à une interface //On l'assigne à l'interface de la machine et zou if linkedInterfaceId == nil { - return n.changeIPInterface(msg, ifId, objectType) + return n.changeIPInterface(msg.IpAddress, ifId, objectType) } //Sinon on vérifie sie la VM possède d'autres IP sur l'interface de management @@ -236,7 +236,7 @@ func (n *Netbox) CreateVM(msg Message) error { //L'interface possède d'autres IPs //Du coup, on prend l'ip en question util.Info("Remove the link ...") - err := n.changeIPInterface(msg, ifId, objectType) + err := n.changeIPInterface(msg.IpAddress, ifId, objectType) if err != nil { return err } From f687499fbab81f54fa7937eec02c5243032970c2 Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 13:33:53 +0200 Subject: [PATCH 57/91] refactor: code cleanup + more infos --- model/netbox.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 89f12bb..d05784e 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -125,7 +125,6 @@ func (n *Netbox) CreateIP(address string, status string, linkedObjectId int64, l } func (n *Netbox) CreateVM(msg Message) error { - if !n._isConnected { return errors.New("netbox is not connected") } @@ -186,8 +185,8 @@ func (n *Netbox) CreateVM(msg Message) error { one = int64(1) ) - util.Info(fmt.Sprintf("Found #%d IPs in %v", *req.Payload.Count, *req)) - //We dont have that ip registered on netbox, so lets create him + util.Info(fmt.Sprintf("Found #%d IPs in %v", *req.Payload.Count, *req)) + //We don't have that ip registered on netbox, so let's create him if *req.Payload.Count == zero { //Set ip to the interface createdIP, err := n.CreateIP(msg.IpAddress, models.IPAddressStatusValueActive, ifId, objectType) @@ -207,7 +206,7 @@ func (n *Netbox) CreateVM(msg Message) error { return n.changeIPInterface(msg.IpAddress, ifId, objectType) } - //Sinon on vérifie sie la VM possède d'autres IP sur l'interface de management + //Sinon on vérifie si la VM possède d'autres IP sur l'interface de management interfaceId := *linkedInterfaceId vmInterfaceParam := virtualization. NewVirtualizationInterfacesReadParams(). @@ -243,7 +242,10 @@ func (n *Netbox) CreateVM(msg Message) error { util.Success("IP changed of interface") return nil - } //Sinon on laisse l'ip sur la VM + } else { + //Sinon on laisse l'ip sur la VM + util.Info(fmt.Sprintf("L'IP %s reste sur l'interface n°%d", msg.IpAddress, mgmtInterface.ID)) + } util.Warn("Trying to using existing IP on VM interface #" + strconv.FormatInt(mgmtInterface.ID, 10)) } @@ -327,7 +329,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { } var ipCount = result.Payload.Count - util.Info("There are actually " + strconv.FormatInt(*ipCount, 10) + " IP(s) associated with the management interface") + util.Info(fmt.Sprintf("There are actually %d IP(s) associated with the management interface", ipCount)) if *ipCount > one { return errors.New("there are more than one management ip linked to the management interface") @@ -345,7 +347,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { // - set the new ip to the interface oldIpUpdatePrams := models.WritableIPAddress{ - Address: &msg.IpAddress, + Address: &msg.IpAddress, AssignedObjectType: nil, AssignedObjectID: nil, } @@ -379,8 +381,8 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { var ipType = "virtualization.vminterface" newIp := &ipam.IpamIPAddressesCreateParams{ Data: &models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectID: &mgmtInterface.ID, + Address: &msg.IpAddress, + AssignedObjectID: &mgmtInterface.ID, AssignedObjectType: &ipType, }, } From f494ec819022313c16ca23ed6f47d9638ae1e06a Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 14:06:30 +0200 Subject: [PATCH 58/91] refactor: use better name --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index ee9fe0a..8f89047 100644 --- a/main.go +++ b/main.go @@ -31,9 +31,10 @@ func main() { failOnError(err, fmt.Sprintf("Failed to open a channel : %s", err)) queueName := os.Getenv("QUEUE_NAME") + incomingQueue := os.Getenv("RABBITMQ_INCOMING_QUEUE") q, err := ch.QueueDeclare( - queueName, + incomingQueue, true, false, false, From 9595a296188085a697542ff635882176feefa2e3 Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 14:06:53 +0200 Subject: [PATCH 59/91] feat: declare exchange --- main.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 8f89047..154ed81 100644 --- a/main.go +++ b/main.go @@ -30,8 +30,8 @@ func main() { ch, err := conn.Channel() failOnError(err, fmt.Sprintf("Failed to open a channel : %s", err)) - queueName := os.Getenv("QUEUE_NAME") incomingQueue := os.Getenv("RABBITMQ_INCOMING_QUEUE") + outcomingQueue := os.Getenv("RABBITMQ_OUTGOING_QUEUE") q, err := ch.QueueDeclare( incomingQueue, @@ -43,6 +43,17 @@ func main() { ) failOnError(err, fmt.Sprintf("Failed to declare a queue : %s", err)) + err = ch.ExchangeDeclare( + incomingQueue, + "x-delayed-message", + true, + false, + false, + false, + nil, + ) + failOnError(err, fmt.Sprintf("Failed to declare an exchange : %s", err)) + // Consommation des messages msgs, err := ch.Consume( q.Name, // nom de la queue From ae4de93eef6b7c0ffe977ca153bbc834a2fc5d67 Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 14:07:32 +0200 Subject: [PATCH 60/91] wip: send successfully treated messages --- main.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 154ed81..104e6bc 100644 --- a/main.go +++ b/main.go @@ -91,8 +91,37 @@ func main() { //Make request to the rest of API err = netbox.CreateOrUpdateVM(msg) - if err != nil { - util.Warn(fmt.Errorf("error creating or updating VM : %s", err).Error()) + if err == nil { + util.Success(fmt.Sprintf("VM up to date %s", msg.Hostname)) + + dur, _ := time.ParseDuration("10s") + ctx, cancel := context.WithTimeout(context.Background(), dur) + defer cancel() + + newMsg := msg + + newMsgJson, _ := json.Marshal(newMsg) + + chErr := ch.PublishWithContext( + ctx, + "", + q.Name, + false, + false, + amqp.Publishing{ + ContentType: "application/json", + Body: newMsgJson, + }) + + if chErr != nil { + util.Warn(fmt.Sprintf("Error re-publishing message : %s", chErr)) + } else { + util.Warn(fmt.Sprintf("Re-sent message to RabbitMQ ®️ : %s", newMsgJson)) + } + + return + } else { + util.Warn(fmt.Errorf("error creating or updating VM : %w", err).Error()) dur, _ := time.ParseDuration("10s") ctx, cancel := context.WithTimeout(context.Background(), dur) @@ -131,8 +160,6 @@ func main() { return } - - util.Info("Connection successfully close from " + msg.GetSerial()) }() } }() From f2fda296db1a8a7a6e9e88950d2ee637b392de63 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 14:10:05 +0200 Subject: [PATCH 61/91] IP Addresses --- model/netbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index d05784e..5926270 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -329,7 +329,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { } var ipCount = result.Payload.Count - util.Info(fmt.Sprintf("There are actually %d IP(s) associated with the management interface", ipCount)) + util.Info(fmt.Sprintf("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10))) if *ipCount > one { return errors.New("there are more than one management ip linked to the management interface") From 5afe5a587fdd59ba8690f040ee897f5760ed48f0 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 14:13:22 +0200 Subject: [PATCH 62/91] Queue success + fail --- main.go | 60 +++++++++++++++++++++++++++---------------------------- shell.nix | 24 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 shell.nix diff --git a/main.go b/main.go index 104e6bc..0375ba6 100644 --- a/main.go +++ b/main.go @@ -91,36 +91,7 @@ func main() { //Make request to the rest of API err = netbox.CreateOrUpdateVM(msg) - if err == nil { - util.Success(fmt.Sprintf("VM up to date %s", msg.Hostname)) - - dur, _ := time.ParseDuration("10s") - ctx, cancel := context.WithTimeout(context.Background(), dur) - defer cancel() - - newMsg := msg - - newMsgJson, _ := json.Marshal(newMsg) - - chErr := ch.PublishWithContext( - ctx, - "", - q.Name, - false, - false, - amqp.Publishing{ - ContentType: "application/json", - Body: newMsgJson, - }) - - if chErr != nil { - util.Warn(fmt.Sprintf("Error re-publishing message : %s", chErr)) - } else { - util.Warn(fmt.Sprintf("Re-sent message to RabbitMQ ®️ : %s", newMsgJson)) - } - - return - } else { + if err != nil { util.Warn(fmt.Errorf("error creating or updating VM : %w", err).Error()) dur, _ := time.ParseDuration("10s") @@ -142,7 +113,7 @@ func main() { chErr := ch.PublishWithContext( ctx, - "", + incomingQueue, q.Name, false, false, @@ -160,6 +131,33 @@ func main() { return } + + util.Success(fmt.Sprintf("VM up to date %s", msg.Hostname)) + + dur, _ := time.ParseDuration("10s") + ctx, cancel := context.WithTimeout(context.Background(), dur) + defer cancel() + + newMsg := msg + + newMsgJson, _ := json.Marshal(newMsg) + + chErr := ch.PublishWithContext( + ctx, + "", + outcomingQueue, + false, + false, + amqp.Publishing{ + ContentType: "application/json", + Body: newMsgJson, + }) + + if chErr != nil { + util.Warn(fmt.Sprintf("Error publishing success message : %s", chErr)) + } else { + util.Warn(fmt.Sprintf("sent success message to RabbitMQ ®️ : %s", newMsgJson)) + } }() } }() diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..529b7f6 --- /dev/null +++ b/shell.nix @@ -0,0 +1,24 @@ +{ + pkgs ? import { }, + compile ? false, + ... +}: +let + inherit (pkgs) lib mkShell buildGoModule; + inherit (lib) fetchFromGitHub cleanSource; + + src = cleanSource ./.; + + package = buildGoModule { + name = "kittenMQ-consumer"; + inherit src; + # proxyVendor = true; + vendorHash = "sha256-XE5npxjcTRDmINM2IFS4C9NWfsAYiGs+h4sDIZX8AhU="; + + postInstall = '' + mv $out/bin/rh-api $out/bin/kittenMQ-consumer + ''; + + }; +in +mkShell { packages = (with pkgs; [ go ]) ++ lib.optional (compile) package; } From cb33c009e93063f410c2b70e2a09316fed0c494a Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 16:25:26 +0200 Subject: [PATCH 63/91] delay --- main.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 0375ba6..e40d43f 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "strconv" "fmt" "github.com/KittenConnect/rh-api/model" "github.com/KittenConnect/rh-api/util" @@ -18,6 +19,9 @@ func failOnError(err error, msg string) { } } +var RETRY_DELAY = 5 + + func main() { err := godotenv.Load() failOnError(err, fmt.Sprintf("Error loading .env file : %s", err)) @@ -33,6 +37,12 @@ func main() { incomingQueue := os.Getenv("RABBITMQ_INCOMING_QUEUE") outcomingQueue := os.Getenv("RABBITMQ_OUTGOING_QUEUE") + if value, ok := os.LookupEnv("RABBITMQ_RETRY_DELAY"); ok { + if i, err := strconv.Atoi(value); err == nil { + RETRY_DELAY = i + } + } + q, err := ch.QueueDeclare( incomingQueue, true, @@ -43,6 +53,10 @@ func main() { ) failOnError(err, fmt.Sprintf("Failed to declare a queue : %s", err)) + exchangeArgs := map[string]interface{}{ + "x-delayed-type": "direct", + } + err = ch.ExchangeDeclare( incomingQueue, "x-delayed-message", @@ -50,10 +64,18 @@ func main() { false, false, false, - nil, + exchangeArgs, ) failOnError(err, fmt.Sprintf("Failed to declare an exchange : %s", err)) + err = ch.QueueBind( + incomingQueue, // queue name + incomingQueue, // routing key + incomingQueue, // exchange + false, + nil) + failOnError(err, fmt.Sprintf("Failed to bind queue: %s", err)) + // Consommation des messages msgs, err := ch.Consume( q.Name, // nom de la queue @@ -108,7 +130,7 @@ func main() { newMsgJson, _ := json.Marshal(newMsg) headers := amqp.Table{ - "x-delay": 60000, + "x-delay": RETRY_DELAY * 1000, } chErr := ch.PublishWithContext( @@ -156,7 +178,7 @@ func main() { if chErr != nil { util.Warn(fmt.Sprintf("Error publishing success message : %s", chErr)) } else { - util.Warn(fmt.Sprintf("sent success message to RabbitMQ ®️ : %s", newMsgJson)) + util.Success(fmt.Sprintf("sent success message to RabbitMQ ®️ : %s", newMsgJson)) } }() } From 4b7b083e5671c8c0edba9505ea0136f45d948711 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 11 Aug 2024 19:18:05 +0200 Subject: [PATCH 64/91] Util.Err + failWithError --- main.go | 61 ++++++++++++++++++++++++++++++----------------------- util/log.go | 4 ++-- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/main.go b/main.go index e40d43f..fe99240 100644 --- a/main.go +++ b/main.go @@ -3,47 +3,46 @@ package main import ( "context" "encoding/json" - "strconv" "fmt" "github.com/KittenConnect/rh-api/model" "github.com/KittenConnect/rh-api/util" "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" "os" + "strconv" "time" ) -func failOnError(err error, msg string) { +func failWithError(err error, formatString string, args ...any) { if err != nil { - util.Err(fmt.Errorf("%s: %w", msg, err).Error()) + util.Err(fmt.Errorf(fmt.Sprintf("%s: %w",formatString), append(args, err)...).Error()) } } var RETRY_DELAY = 5 - func main() { err := godotenv.Load() - failOnError(err, fmt.Sprintf("Error loading .env file : %s", err)) + failWithError(err, "Error loading .env file") conn, err := amqp.Dial(os.Getenv("RABBITMQ_URL")) - failOnError(err, fmt.Sprintf("Failed to connect to broker : %s", err)) + failWithError(err, "Failed to connect to broker") defer conn.Close() ch, err := conn.Channel() - failOnError(err, fmt.Sprintf("Failed to open a channel : %s", err)) + failWithError(err, "Failed to open a channel") incomingQueue := os.Getenv("RABBITMQ_INCOMING_QUEUE") - outcomingQueue := os.Getenv("RABBITMQ_OUTGOING_QUEUE") + outgoingQueue := os.Getenv("RABBITMQ_OUTGOING_QUEUE") if value, ok := os.LookupEnv("RABBITMQ_RETRY_DELAY"); ok { if i, err := strconv.Atoi(value); err == nil { RETRY_DELAY = i } - } + } - q, err := ch.QueueDeclare( + inQ, err := ch.QueueDeclare( incomingQueue, true, false, @@ -51,11 +50,21 @@ func main() { false, nil, ) - failOnError(err, fmt.Sprintf("Failed to declare a queue : %s", err)) + failWithError(err, "Failed to declare queue %s", incomingQueue) - exchangeArgs := map[string]interface{}{ + outQ, err := ch.QueueDeclare( + outgoingQueue, + true, + false, + false, + false, + nil, + ) + failWithError(err, "Failed to declare queue %s", outgoingQueue) + + exchangeArgs := map[string]interface{}{ "x-delayed-type": "direct", - } + } err = ch.ExchangeDeclare( incomingQueue, @@ -66,19 +75,19 @@ func main() { false, exchangeArgs, ) - failOnError(err, fmt.Sprintf("Failed to declare an exchange : %s", err)) + failWithError(err, "Failed to declare exchange %s", incomingQueue) - err = ch.QueueBind( - incomingQueue, // queue name - incomingQueue, // routing key - incomingQueue, // exchange - false, - nil) - failOnError(err, fmt.Sprintf("Failed to bind queue: %s", err)) + err = ch.QueueBind( + incomingQueue, // queue name + incomingQueue, // routing key + incomingQueue, // exchange + false, + nil) + failWithError(err, "Failed to bind queue %s to exchange %s", incomingQueue, incomingQueue) // Consommation des messages msgs, err := ch.Consume( - q.Name, // nom de la queue + inQ.Name, // nom de la queue "consumer", // consumer true, // autoAck false, // exclusive @@ -86,12 +95,12 @@ func main() { false, // noWait nil, // arguments ) - failOnError(err, "Failed to register a consumer") + failWithError(err, "Failed to register %s consumer", inQ.Name) util.Info("Connected to message broker") netbox := model.NewNetbox() err = netbox.Connect() - failOnError(err, "Failed to connect to netbox") + failWithError(err, "Failed to connect to netbox") if netbox.IsConnected() == false { util.Err("Unable to connect to netbox") @@ -136,7 +145,7 @@ func main() { chErr := ch.PublishWithContext( ctx, incomingQueue, - q.Name, + inQ.Name, false, false, amqp.Publishing{ @@ -167,7 +176,7 @@ func main() { chErr := ch.PublishWithContext( ctx, "", - outcomingQueue, + outQ.Name, false, false, amqp.Publishing{ diff --git a/util/log.go b/util/log.go index e63adaf..530a4ae 100644 --- a/util/log.go +++ b/util/log.go @@ -6,9 +6,9 @@ import ( "os" ) -func Err(t string) { +func Err(s string, v ...any) { red := color.New(color.FgRed).SprintFunc() - fmt.Printf("%s %s\n", red("[ERROR]"), red(t)) + fmt.Printf(fmt.Sprintf(red("%s %s\n"), "[ERROR]", s), v...) os.Exit(-1) } From 945da46455ad112ab02b7ce129eef6b5b0ed579d Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 11 Aug 2024 19:34:05 +0200 Subject: [PATCH 65/91] refactor: correct initial vm status --- model/netbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/netbox.go b/model/netbox.go index 5926270..297f909 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -59,7 +59,7 @@ func (n *Netbox) GetDefaultTimeout() time.Duration { func getVm(m Message) models.WritableVirtualMachineWithConfigContext { var ( - status = "active" + status = "planned" cluster int64 = 1 ) From fb48a37bf84bcc099f22c41e5d1605893981bde4 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 18 Aug 2024 18:24:43 +0200 Subject: [PATCH 66/91] printf --- main.go | 14 +++++++------- model/netbox.go | 20 ++++++++++---------- util/log.go | 31 ++++++++++++++++++++----------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/main.go b/main.go index fe99240..c062b99 100644 --- a/main.go +++ b/main.go @@ -116,14 +116,14 @@ func main() { msg := model.Message{Timestamp: d.Timestamp, FailCount: 20} err := json.Unmarshal(d.Body, &msg) if err != nil { - util.Warn(fmt.Sprintf("Error unmarshalling message : %s", err)) + util.Warn("Error unmarshalling message : %w", err) return } //Make request to the rest of API err = netbox.CreateOrUpdateVM(msg) if err != nil { - util.Warn(fmt.Errorf("error creating or updating VM : %w", err).Error()) + util.Warn("error creating or updating VM : %w", err) dur, _ := time.ParseDuration("10s") ctx, cancel := context.WithTimeout(context.Background(), dur) @@ -155,15 +155,15 @@ func main() { }) if chErr != nil { - util.Warn(fmt.Sprintf("Error re-publishing message : %s", chErr)) + util.Warn("Error re-publishing message: %s", chErr) } else { - util.Warn(fmt.Sprintf("Re-sent message to RabbitMQ ®️ : %s", newMsgJson)) + util.Warn("Re-sent message to RabbitMQ®️: %s", newMsgJson) } return } - util.Success(fmt.Sprintf("VM up to date %s", msg.Hostname)) + util.Success("VM %s is up to date", msg.Hostname) dur, _ := time.ParseDuration("10s") ctx, cancel := context.WithTimeout(context.Background(), dur) @@ -185,9 +185,9 @@ func main() { }) if chErr != nil { - util.Warn(fmt.Sprintf("Error publishing success message : %s", chErr)) + util.Warn("Error publishing success message: %s", chErr) } else { - util.Success(fmt.Sprintf("sent success message to RabbitMQ ®️ : %s", newMsgJson)) + util.Success("sent success message to RabbitMQ®️: %s", newMsgJson) } }() } diff --git a/model/netbox.go b/model/netbox.go index 297f909..6ac26eb 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -141,7 +141,7 @@ func (n *Netbox) CreateVM(msg Message) error { return fmt.Errorf("error creating virtual machine: %w", err) } - util.Success(fmt.Sprintf("Created machine ID : %d", result.Payload.ID)) + util.Success("Created machine ID: %d", result.Payload.ID) //Create management interface var ( @@ -164,7 +164,7 @@ func (n *Netbox) CreateVM(msg Message) error { if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) } - util.Success("\tSuccessfully created vm interface " + strconv.FormatInt(res.Payload.ID, 10)) + util.Success("\tSuccessfully created vm interface %s", strconv.FormatInt(res.Payload.ID, 10)) var ( ifId = res.Payload.ID @@ -185,7 +185,7 @@ func (n *Netbox) CreateVM(msg Message) error { one = int64(1) ) - util.Info(fmt.Sprintf("Found #%d IPs in %v", *req.Payload.Count, *req)) + util.Info("Found #%d IPs in %v", *req.Payload.Count, *req) //We don't have that ip registered on netbox, so let's create him if *req.Payload.Count == zero { //Set ip to the interface @@ -194,7 +194,7 @@ func (n *Netbox) CreateVM(msg Message) error { return err } - util.Success("\tSuccessfully created vm management ip : " + strconv.FormatInt(createdIP.Payload.ID, 10)) + util.Success("\tSuccessfully created vm management ip: %s", strconv.FormatInt(createdIP.Payload.ID, 10)) } else if *req.Payload.Count == one { ip := req.Payload.Results[0] @@ -244,10 +244,10 @@ func (n *Netbox) CreateVM(msg Message) error { return nil } else { //Sinon on laisse l'ip sur la VM - util.Info(fmt.Sprintf("L'IP %s reste sur l'interface n°%d", msg.IpAddress, mgmtInterface.ID)) + util.Info("L'IP %s reste sur l'interface n°%d", msg.IpAddress, mgmtInterface.ID) } - util.Warn("Trying to using existing IP on VM interface #" + strconv.FormatInt(mgmtInterface.ID, 10)) + util.Warn("Trying to using existing IP on VM interface #%s", strconv.FormatInt(mgmtInterface.ID, 10)) } return nil @@ -321,7 +321,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { var mgmtInterfaceId = strconv.FormatInt(mgmtInterface.ID, 10) params := ipam.NewIpamIPAddressesListParams() params.SetVminterfaceID(&mgmtInterfaceId) - util.Info(fmt.Sprintf("Found MGMT Iface #%d -> %s", mgmtInterface.ID, mgmtInterfaceId)) + util.Info("Found MGMT Iface #%d -> %s", mgmtInterface.ID, mgmtInterfaceId) result, err := n.Client.Ipam.IpamIPAddressesList(params, nil) if err != nil { @@ -329,7 +329,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { } var ipCount = result.Payload.Count - util.Info(fmt.Sprintf("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10))) + util.Info("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10)) if *ipCount > one { return errors.New("there are more than one management ip linked to the management interface") @@ -359,10 +359,10 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { WithTimeout(n.GetDefaultTimeout()) _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) if err != nil { - return fmt.Errorf("error updating management ip addresses of VM #"+vmId+": %w", err) + return fmt.Errorf("error updating management ip addresses of VM #%d: %w", vmId, err) } - util.Success("Successfully updated management ip addresses of VM #" + vmId + " with new IP : " + msg.IpAddress) + util.Success("Successfully updated management ip addresses of VM #%d with new IP: %s", vmId, msg.IpAddress) return nil } diff --git a/util/log.go b/util/log.go index 530a4ae..359db43 100644 --- a/util/log.go +++ b/util/log.go @@ -6,23 +6,32 @@ import ( "os" ) +func Color(c color.Attribute, prefix string, s string, v ...any) { + colored := color.New(c).SprintFunc() + fmt.Printf(fmt.Sprintf(colored("[%s] %s\n"), prefix, s), v...) +} + func Err(s string, v ...any) { - red := color.New(color.FgRed).SprintFunc() - fmt.Printf(fmt.Sprintf(red("%s %s\n"), "[ERROR]", s), v...) + // red := color.New(color.FgRed).SprintFunc() + // fmt.Printf(fmt.Sprintf(red("%s %s\n"), "[ERROR]", s), v...) + Color(color.FgRed, "ERROR", s, v...) os.Exit(-1) } -func Warn(t string) { - orange := color.New(color.FgYellow).SprintFunc() - fmt.Printf("%s %s\n", orange("[WARN]"), orange(t)) +func Warn(s string, v ...any) { + // orange := color.New(color.FgYellow).SprintFunc() + // fmt.Printf("%s %s\n", orange("[WARN]"), orange(t)) + Color(color.FgYellow, "WARN", s, v...) } -func Info(t string) { - blue := color.New(color.FgBlue).SprintFunc() - fmt.Printf("%s %s\n", blue("[INFO]"), blue(t)) +func Info(s string, v ...any) { + // blue := color.New(color.FgBlue).SprintFunc() + // fmt.Printf("%s %s\n", blue("[INFO]"), blue(t)) + Color(color.FgBlue, "INFO", s, v...) } -func Success(t string) { - green := color.New(color.FgGreen).SprintFunc() - fmt.Printf("%s %s\n", green("[SUCCESS]"), green(t)) +func Success(s string, v ...any) { + // green := color.New(color.FgGreen).SprintFunc() + // fmt.Printf("%s %s\n", green("[SUCCESS]"), green(t)) + Color(color.FgGreen, "SUCCESS", s, v...) } From a522fe05b03fb3ea102fc5d8124720fe348269d4 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 21 Aug 2024 21:39:34 +0200 Subject: [PATCH 67/91] refactor: extract vm related functions: - list interfaces - creation of interface --- model/VirtualMachine.go | 76 +++++++++++++++++++++++++++++++++++++++ model/netbox.go | 79 +++++++++++------------------------------ 2 files changed, 96 insertions(+), 59 deletions(-) create mode 100644 model/VirtualMachine.go diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go new file mode 100644 index 0000000..a946e69 --- /dev/null +++ b/model/VirtualMachine.go @@ -0,0 +1,76 @@ +package model + +import ( + "fmt" + "github.com/KittenConnect/rh-api/util" + "github.com/netbox-community/go-netbox/netbox/client/virtualization" + "github.com/netbox-community/go-netbox/netbox/models" + "strconv" +) + +type VirtualMachine struct { + Id int64 `json:"id"` +} + +func NewVM() VirtualMachine { + vm := VirtualMachine{} + + return vm +} + +func (vm *VirtualMachine) Create(msg Message) models.WritableVirtualMachineWithConfigContext { + var ( + status = "planned" + cluster int64 = 1 + ) + + conf := models.WritableVirtualMachineWithConfigContext{ + Cluster: &cluster, + Name: &msg.Hostname, + Status: status, + + CustomFields: map[string]interface{}{ + "kc_serial_": msg.GetSerial(), + }, + } + + return conf +} + +func (vm *VirtualMachine) GetInterfaces(n *Netbox, name string) (*virtualization.VirtualizationInterfacesListOK, error) { + vmId := strconv.FormatInt(vm.Id, 10) + + ipIfParam := &virtualization.VirtualizationInterfacesListParams{ + VirtualMachineID: &vmId, + Name: &name, + } + interfaces, err := n.Client.Virtualization. + VirtualizationInterfacesList(ipIfParam.WithTimeout(n.GetDefaultTimeout()), nil) + if err != nil { + return nil, fmt.Errorf("error listing virtual machine interfaces: %w", err) + } + + return interfaces, nil +} + +func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualization.VirtualizationInterfacesCreateCreated, error) { + ifParam := models.WritableVMInterface{ + Name: &ifName, + Enabled: true, + + TaggedVlans: []int64{}, + + VirtualMachine: &vm.Id, + } + paramInterface := virtualization. + NewVirtualizationInterfacesCreateParams(). + WithData(&ifParam). + WithTimeout(n.GetDefaultTimeout()) + res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) + if err != nil { + return nil, fmt.Errorf("error creating virtual machine interface: %w", err) + } + util.Success("\tSuccessfully created vm interface %s", strconv.FormatInt(res.Payload.ID, 10)) + + return res, nil +} diff --git a/model/netbox.go b/model/netbox.go index 6ac26eb..982869d 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -57,23 +57,6 @@ func (n *Netbox) GetDefaultTimeout() time.Duration { return time.Duration(30) * time.Second } -func getVm(m Message) models.WritableVirtualMachineWithConfigContext { - var ( - status = "planned" - cluster int64 = 1 - ) - - return models.WritableVirtualMachineWithConfigContext{ - Cluster: &cluster, - Name: &m.Hostname, - Status: status, - - CustomFields: map[string]interface{}{ - "kc_serial_": m.GetSerial(), - }, - } -} - func (n *Netbox) getIpAddress(ip string) *models.WritableIPAddress { return &models.WritableIPAddress{ Address: &ip, @@ -129,9 +112,10 @@ func (n *Netbox) CreateVM(msg Message) error { return errors.New("netbox is not connected") } - vm := getVm(msg) + vm := NewVM() + vmCreateData := vm.Create(msg) - params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&vm) + params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&vmCreateData) result, err := n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) if err != nil { if result != nil && result.Payload != nil { @@ -142,29 +126,13 @@ func (n *Netbox) CreateVM(msg Message) error { } util.Success("Created machine ID: %d", result.Payload.ID) + vm.Id = result.Payload.ID //Create management interface - var ( - mgmtInterfaceName = "mgmt" - ) - - ifParam := models.WritableVMInterface{ - Name: &mgmtInterfaceName, - Enabled: true, - - TaggedVlans: []int64{}, - - VirtualMachine: &result.Payload.ID, - } - paramInterface := virtualization. - NewVirtualizationInterfacesCreateParams(). - WithData(&ifParam). - WithTimeout(n.GetDefaultTimeout()) - res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) + res, err := vm.CreateInterface(n, "mgmt") if err != nil { - return fmt.Errorf("error creating virtual machine interface: %w", err) + return err } - util.Success("\tSuccessfully created vm interface %s", strconv.FormatInt(res.Payload.ID, 10)) var ( ifId = res.Payload.ID @@ -180,14 +148,10 @@ func (n *Netbox) CreateVM(msg Message) error { if err != nil { return fmt.Errorf("error checking ip addresses existance : %w", err) } - var ( - zero = int64(0) - one = int64(1) - ) util.Info("Found #%d IPs in %v", *req.Payload.Count, *req) //We don't have that ip registered on netbox, so let's create him - if *req.Payload.Count == zero { + if *req.Payload.Count == 0 { //Set ip to the interface createdIP, err := n.CreateIP(msg.IpAddress, models.IPAddressStatusValueActive, ifId, objectType) if err != nil { @@ -195,7 +159,7 @@ func (n *Netbox) CreateVM(msg Message) error { } util.Success("\tSuccessfully created vm management ip: %s", strconv.FormatInt(createdIP.Payload.ID, 10)) - } else if *req.Payload.Count == one { + } else if *req.Payload.Count == 1 { ip := req.Payload.Results[0] linkedInterfaceId := ip.AssignedObjectID @@ -220,6 +184,7 @@ func (n *Netbox) CreateVM(msg Message) error { vmID := strconv.FormatInt(vmInterfaceResult.Payload.VirtualMachine.ID, 10) + mgmtInterfaceName := "mgmt" nestedVmParams := &virtualization.VirtualizationInterfacesListParams{ Name: &mgmtInterfaceName, VirtualMachineID: &vmID, @@ -254,10 +219,13 @@ func (n *Netbox) CreateVM(msg Message) error { } func (n *Netbox) UpdateVM(id int64, msg Message) error { - vm := getVm(msg) + vm := NewVM() + vm.Id = id + + vmConf := vm.Create(msg) updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ - Data: &vm, + Data: &vmConf, ID: id, } @@ -268,23 +236,16 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { mgmtIfName = "mgmt" ) - ipIfParam := &virtualization.VirtualizationInterfacesListParams{ - VirtualMachineID: &vmId, - Name: &mgmtIfName, - } - interfaces, err := n.Client.Virtualization. - VirtualizationInterfacesList(ipIfParam.WithTimeout(n.GetDefaultTimeout()), nil) + interfaces, err := vm.GetInterfaces(n, mgmtIfName) if err != nil { - return fmt.Errorf("error listing virtual machine interfaces: %w", err) + return err } // 2. If there is no interface, quit var ( ifCount = interfaces.Payload.Count - one = int64(1) - zero = int64(0) ) - if *ifCount < one { + if *ifCount < 1 { //No virtual interface, create one var ( mgmtInterfaceName = "mgmt" @@ -331,11 +292,11 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { var ipCount = result.Payload.Count util.Info("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10)) - if *ipCount > one { + if *ipCount > 1 { return errors.New("there are more than one management ip linked to the management interface") } - if *ipCount == one { + if *ipCount == 1 { ip := result.Payload.Results[0] if *ip.Address == msg.IpAddress { //Nothing to do @@ -376,7 +337,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { existingIpCount := result.Payload.Count newIpAddrId := int64(0) - if *existingIpCount == zero { + if *existingIpCount == 0 { util.Info("There is no IP registered in the netbox. Create him.") var ipType = "virtualization.vminterface" newIp := &ipam.IpamIPAddressesCreateParams{ From 8d93e62e4de6e3be8b9d90570353005bd0f2209f Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 21 Aug 2024 21:39:51 +0200 Subject: [PATCH 68/91] feat: add quotes to port mapping --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index cba98ca..966179c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: - netbox-reports-files:/opt/netbox/netbox/reports:rw - netbox-scripts-files:/opt/netbox/netbox/scripts:rw ports: - - 8080:80 + - "8888:8080" netbox-worker: <<: *netbox depends_on: From d764046bd71d3e8c6c25731b0148b415e512d853 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 21 Aug 2024 21:45:10 +0200 Subject: [PATCH 69/91] refactor: extract of create & change ip address related to vm interface --- model/VirtualMachine.go | 44 ++++++++++++++++++++++++++++++++++++ model/netbox.go | 49 +++-------------------------------------- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index a946e69..b2b31f6 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -3,6 +3,7 @@ package model import ( "fmt" "github.com/KittenConnect/rh-api/util" + "github.com/netbox-community/go-netbox/netbox/client/ipam" "github.com/netbox-community/go-netbox/netbox/client/virtualization" "github.com/netbox-community/go-netbox/netbox/models" "strconv" @@ -74,3 +75,46 @@ func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualiza return res, nil } + +func (vm *VirtualMachine) changeIPInterface(n *Netbox, address string, ifId int64, objectType string) error { + ip := n.getIpAddress(address) + ip.AssignedObjectID = &ifId + ip.AssignedObjectType = &objectType + + ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ + Data: ip, + } + + _, err := n.Client.Ipam. + IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID). + WithTimeout(n.GetDefaultTimeout()), nil) + if err != nil { + return fmt.Errorf("error updating ip address: %w", err) + } + + util.Success("Update IP to VM interface") + return nil +} + +func (vm *VirtualMachine) CreateIP(n *Netbox, address string, status string, linkedObjectId int64, linkedObjectType string) (*ipam.IpamIPAddressesCreateCreated, error) { + ip := &models.WritableIPAddress{ + Address: &address, + Status: status, + } + + if linkedObjectId != -1 && linkedObjectType != "" { + ip.AssignedObjectID = &linkedObjectId + ip.AssignedObjectType = &linkedObjectType + } + + ipCreateParams := &ipam.IpamIPAddressesCreateParams{ + Data: ip, + } + + res, err := n.Client.Ipam.IpamIPAddressesCreate(ipCreateParams.WithTimeout(n.GetDefaultTimeout()), nil) + if err != nil { + return nil, fmt.Errorf("error creating ip address: %w", err) + } + + return res, nil +} diff --git a/model/netbox.go b/model/netbox.go index 982869d..6dd3cd7 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -64,49 +64,6 @@ func (n *Netbox) getIpAddress(ip string) *models.WritableIPAddress { } } -func (n *Netbox) changeIPInterface(address string, ifId int64, objectType string) error { - ip := n.getIpAddress(address) - ip.AssignedObjectID = &ifId - ip.AssignedObjectType = &objectType - - ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ - Data: ip, - } - - _, err := n.Client.Ipam. - IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID). - WithTimeout(n.GetDefaultTimeout()), nil) - if err != nil { - return fmt.Errorf("error updating ip address: %w", err) - } - - util.Success("Update IP to VM interface") - return nil -} - -func (n *Netbox) CreateIP(address string, status string, linkedObjectId int64, linkedObjectType string) (*ipam.IpamIPAddressesCreateCreated, error) { - ip := &models.WritableIPAddress{ - Address: &address, - Status: status, - } - - if linkedObjectId != -1 && linkedObjectType != "" { - ip.AssignedObjectID = &linkedObjectId - ip.AssignedObjectType = &linkedObjectType - } - - ipCreateParams := &ipam.IpamIPAddressesCreateParams{ - Data: ip, - } - - res, err := n.Client.Ipam.IpamIPAddressesCreate(ipCreateParams.WithTimeout(n.GetDefaultTimeout()), nil) - if err != nil { - return nil, fmt.Errorf("error creating ip address: %w", err) - } - - return res, nil -} - func (n *Netbox) CreateVM(msg Message) error { if !n._isConnected { return errors.New("netbox is not connected") @@ -153,7 +110,7 @@ func (n *Netbox) CreateVM(msg Message) error { //We don't have that ip registered on netbox, so let's create him if *req.Payload.Count == 0 { //Set ip to the interface - createdIP, err := n.CreateIP(msg.IpAddress, models.IPAddressStatusValueActive, ifId, objectType) + createdIP, err := vm.CreateIP(n, msg.IpAddress, models.IPAddressStatusValueActive, ifId, objectType) if err != nil { return err } @@ -167,7 +124,7 @@ func (n *Netbox) CreateVM(msg Message) error { //Si l'ip n'est pas liée à une interface //On l'assigne à l'interface de la machine et zou if linkedInterfaceId == nil { - return n.changeIPInterface(msg.IpAddress, ifId, objectType) + return vm.changeIPInterface(n, msg.IpAddress, ifId, objectType) } //Sinon on vérifie si la VM possède d'autres IP sur l'interface de management @@ -200,7 +157,7 @@ func (n *Netbox) CreateVM(msg Message) error { //L'interface possède d'autres IPs //Du coup, on prend l'ip en question util.Info("Remove the link ...") - err := n.changeIPInterface(msg.IpAddress, ifId, objectType) + err := vm.changeIPInterface(n, msg.IpAddress, ifId, objectType) if err != nil { return err } From e85650d9204032f1579cee6882ed449bb9618e7e Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:47:40 +0200 Subject: [PATCH 70/91] feat: base cluster object --- model/Cluster.go | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 model/Cluster.go diff --git a/model/Cluster.go b/model/Cluster.go new file mode 100644 index 0000000..0ef9afa --- /dev/null +++ b/model/Cluster.go @@ -0,0 +1,5 @@ +package model + +type Cluster struct { + ID int64 `json:"id"` +} From 6b764bc33552fe64154cc6fdc313433147189799 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:48:12 +0200 Subject: [PATCH 71/91] refactor: add more fields to vm --- model/VirtualMachine.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index b2b31f6..9497116 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -6,11 +6,19 @@ import ( "github.com/netbox-community/go-netbox/netbox/client/ipam" "github.com/netbox-community/go-netbox/netbox/client/virtualization" "github.com/netbox-community/go-netbox/netbox/models" + "net" "strconv" ) type VirtualMachine struct { - Id int64 `json:"id"` + NetboxId int64 `json:"id"` + Cluster Cluster `json:"cluster"` + Name string `json:"name"` + Status string `json:"status"` + Serial string `json:"serial"` + + ManagementIP net.IP `json:"management_ip"` + n *Netbox } func NewVM() VirtualMachine { From 3295b2c193236c5495698065d0221b066c418e4b Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:50:01 +0200 Subject: [PATCH 72/91] refactor: change how vm object are initialized --- model/VirtualMachine.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 9497116..4921580 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -21,8 +21,13 @@ type VirtualMachine struct { n *Netbox } -func NewVM() VirtualMachine { - vm := VirtualMachine{} +func NewVM(n *Netbox, msg Message) *VirtualMachine { + vm := &VirtualMachine{ + n: n, + NetboxId: -1, + + Name: msg.Hostname, + } return vm } From 6ca97b7060b2e49fe6bf530daf21a6adada4c7fe Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:51:15 +0200 Subject: [PATCH 73/91] refactor: use new properties name --- model/VirtualMachine.go | 4 ++-- model/netbox.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 4921580..6fca38d 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -52,7 +52,7 @@ func (vm *VirtualMachine) Create(msg Message) models.WritableVirtualMachineWithC } func (vm *VirtualMachine) GetInterfaces(n *Netbox, name string) (*virtualization.VirtualizationInterfacesListOK, error) { - vmId := strconv.FormatInt(vm.Id, 10) + vmId := strconv.FormatInt(vm.NetboxId, 10) ipIfParam := &virtualization.VirtualizationInterfacesListParams{ VirtualMachineID: &vmId, @@ -74,7 +74,7 @@ func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualiza TaggedVlans: []int64{}, - VirtualMachine: &vm.Id, + VirtualMachine: &vm.NetboxId, } paramInterface := virtualization. NewVirtualizationInterfacesCreateParams(). diff --git a/model/netbox.go b/model/netbox.go index 6dd3cd7..f31cdc5 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -84,6 +84,7 @@ func (n *Netbox) CreateVM(msg Message) error { util.Success("Created machine ID: %d", result.Payload.ID) vm.Id = result.Payload.ID + vm.NetboxId = res.Payload.ID //Create management interface res, err := vm.CreateInterface(n, "mgmt") @@ -176,8 +177,8 @@ func (n *Netbox) CreateVM(msg Message) error { } func (n *Netbox) UpdateVM(id int64, msg Message) error { - vm := NewVM() - vm.Id = id + vm := NewVM(n, msg) + vm.NetboxId = id vmConf := vm.Create(msg) From d9be5292a0bc431dd3b488afa424893dddff513f Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:51:30 +0200 Subject: [PATCH 74/91] refactor: use new properties name --- model/netbox.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index f31cdc5..ab18322 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -82,8 +82,7 @@ func (n *Netbox) CreateVM(msg Message) error { return fmt.Errorf("error creating virtual machine: %w", err) } - util.Success("Created machine ID: %d", result.Payload.ID) - vm.Id = result.Payload.ID + util.Success("Created machine ID: %d", res.Payload.ID) vm.NetboxId = res.Payload.ID //Create management interface From eec2a3bb7df4549615e97b97aec52b997e9f2b40 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:51:47 +0200 Subject: [PATCH 75/91] feat: function to check if vm exist --- model/netbox.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/model/netbox.go b/model/netbox.go index ab18322..6b1cf0c 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -403,3 +403,39 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { return nil } + +func (n *Netbox) VmExists(hostname string, serial string) (bool, int64, error) { + //Check if the vm exist in netbox + req := virtualization. + NewVirtualizationVirtualMachinesListParams(). + WithTimeout(n.GetDefaultTimeout()) + res, err := n.Client.Virtualization.VirtualizationVirtualMachinesList(req, nil) + if err != nil { + return false, 0, fmt.Errorf("unable to get list of machines from netbox: %w", err) + } + + for _, v := range res.Payload.Results { + if *v.Name == hostname { + return true, v.ID, nil + } + + var cf = v.CustomFields.(map[string]interface{}) + var serial = "" + _ = serial + + for k, v := range cf { + switch c := v.(type) { + case string: + if k == "kc_serial_" { + serial = c + } + } + } + + if serial == serial { + return true, v.ID, nil + } + } + + return false, 0, nil +} From b3c7e2b47e28e0288561c91f7155536f036f2e39 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:52:53 +0200 Subject: [PATCH 76/91] refactor: use existing function --- model/netbox.go | 42 +++--------------------------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 6b1cf0c..0b0e8d9 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -338,55 +338,19 @@ func (n *Netbox) CreateOrUpdateVM(msg Message) error { } var vmId int64 - var hasFoundVm = false var err error // Call netbox API with specific serial, then update his settings accordingly //exist := contains(MachinesSerials, msg.Hostname) //TODO //if !exist { //If the vm don't exist in memory, fetch his details, if she exists in netbox - req := virtualization. - NewVirtualizationVirtualMachinesListParams(). - WithTimeout(n.GetDefaultTimeout()) - res, err := n.Client.Virtualization.VirtualizationVirtualMachinesList(req, nil) + exist, vmId, err := n.VmExists(msg.Hostname, msg.GetSerial()) if err != nil { - return fmt.Errorf("unable to get list of machines from netbox: %w", err) - } - - for _, vm := range res.Payload.Results { - if vm.Name == &msg.Hostname { - vmId = vm.ID - hasFoundVm = true - break - } - - var cf = vm.CustomFields.(map[string]interface{}) - var serial = "" - _ = serial - - for k, v := range cf { - switch c := v.(type) { - case string: - if k == "kc_serial_" { - serial = c - } - // fmt.Printf("Item %q is a string, containing %q\n", k, c) - //case float64: - // fmt.Printf("Looks like item %q is a number, specifically %f\n", k, c) - //default: - // fmt.Printf("Not sure what type item %q is, but I think it might be %T\n", k, c) - } - } - - if serial == msg.GetSerial() { - vmId = vm.ID - hasFoundVm = true - break - } + return fmt.Errorf("error checking if VM exists: %w", err) } //Create VM if she doesn't exists in netbox - if !hasFoundVm { + if !exist { err = n.CreateVM(msg) if err != nil { From d736653f674ec1f92d2cfc1fcebe13121cecd7a8 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 22 Aug 2024 23:53:43 +0200 Subject: [PATCH 77/91] refactor: extract vm creation --- model/netbox.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 0b0e8d9..5e8a848 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -69,14 +69,11 @@ func (n *Netbox) CreateVM(msg Message) error { return errors.New("netbox is not connected") } - vm := NewVM() - vmCreateData := vm.Create(msg) - - params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&vmCreateData) - result, err := n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) + vm := NewVM(n, msg) + res, err := vm.Create(msg) if err != nil { - if result != nil && result.Payload != nil { - return fmt.Errorf("error creating virtual machine: %w \n\t%s", err, result.Error()) + if res != nil && res.Payload != nil { + return fmt.Errorf("error creating virtual machine: %w \n\t%s", err, res.Error()) } return fmt.Errorf("error creating virtual machine: %w", err) From d38972111ab4de8b66dfe6da94529b4c6dd0d3cc Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Fri, 23 Aug 2024 00:03:49 +0200 Subject: [PATCH 78/91] refactor: use enw var --- model/netbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 5e8a848..2cbf8bb 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -83,13 +83,13 @@ func (n *Netbox) CreateVM(msg Message) error { vm.NetboxId = res.Payload.ID //Create management interface - res, err := vm.CreateInterface(n, "mgmt") + r, err := vm.CreateInterface(n, "mgmt") if err != nil { return err } var ( - ifId = res.Payload.ID + ifId = r.Payload.ID objectType = "virtualization.vminterface" ) From 0bc2ab8f2d231c52eb37e8042f16905d965ae5b7 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Fri, 23 Aug 2024 00:06:19 +0200 Subject: [PATCH 79/91] refactor: extract vm update function --- model/VirtualMachine.go | 45 ++++++++++++++++++++++++++++++++--------- model/netbox.go | 30 +++++---------------------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 6fca38d..db0ec17 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -32,23 +32,50 @@ func NewVM(n *Netbox, msg Message) *VirtualMachine { return vm } -func (vm *VirtualMachine) Create(msg Message) models.WritableVirtualMachineWithConfigContext { - var ( - status = "planned" - cluster int64 = 1 - ) +func (vm *VirtualMachine) Get() models.WritableVirtualMachineWithConfigContext { + // todo: implement netbox func + return models.WritableVirtualMachineWithConfigContext{ + Cluster: &vm.Cluster.ID, + Name: &vm.Name, + Status: vm.Status, + CustomFields: map[string]interface{}{ + "kc_serial_": vm.Serial, + }, + } +} + +func (vm *VirtualMachine) Create(msg Message) (*virtualization.VirtualizationVirtualMachinesCreateCreated, error) { conf := models.WritableVirtualMachineWithConfigContext{ - Cluster: &cluster, - Name: &msg.Hostname, - Status: status, + Cluster: &vm.Cluster.ID, + Name: &vm.Name, + Status: vm.Status, CustomFields: map[string]interface{}{ "kc_serial_": msg.GetSerial(), }, } - return conf + params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&conf) + return vm.n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) +} + +// Update vm infos to netbox +func (vm *VirtualMachine) Update() error { + data := vm.Get() + + updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ + Data: &data, + ID: vm.NetboxId, + } + + _, err := vm.n.Client.Virtualization. + VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(vm.n.GetDefaultTimeout()), nil) + if err != nil { + return fmt.Errorf("error updating virtual machine interface: %w", err) + } + + return nil } func (vm *VirtualMachine) GetInterfaces(n *Netbox, name string) (*virtualization.VirtualizationInterfacesListOK, error) { diff --git a/model/netbox.go b/model/netbox.go index 2cbf8bb..a8895a4 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -176,11 +176,11 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { vm := NewVM(n, msg) vm.NetboxId = id - vmConf := vm.Create(msg) + _, err := vm.Create(msg) - updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ - Data: &vmConf, - ID: id, + err = vm.Update() + if err != nil { + return err } //Update management IP @@ -201,32 +201,12 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { ) if *ifCount < 1 { //No virtual interface, create one - var ( - mgmtInterfaceName = "mgmt" - ) - - ifParam := models.WritableVMInterface{ - Name: &mgmtInterfaceName, - Enabled: true, - - TaggedVlans: []int64{}, - VirtualMachine: &id, - } - paramInterface := virtualization. - NewVirtualizationInterfacesCreateParams(). - WithData(&ifParam).WithTimeout(n.GetDefaultTimeout()) - _, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) + _, err := vm.CreateInterface(n, "mgmt") if err != nil { return fmt.Errorf("error creating virtual machine interface: %w", err) } - _, err = n.Client.Virtualization. - VirtualizationVirtualMachinesPartialUpdate(updateParams.WithTimeout(n.GetDefaultTimeout()), nil) - if err != nil { - return fmt.Errorf("error updating virtual machine interface: %w", err) - } - util.Info("Updated VM #" + strconv.FormatInt(id, 10) + " management interface with IP " + msg.IpAddress) return nil } From 04edad4f324456439643360764587a79b8d85ee7 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 25 Aug 2024 00:36:47 +0200 Subject: [PATCH 80/91] refactor: use existing netbox var in vm --- model/VirtualMachine.go | 19 ++++++++++--------- model/netbox.go | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index db0ec17..6ff8c01 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -78,15 +78,15 @@ func (vm *VirtualMachine) Update() error { return nil } -func (vm *VirtualMachine) GetInterfaces(n *Netbox, name string) (*virtualization.VirtualizationInterfacesListOK, error) { +func (vm *VirtualMachine) GetInterfaces(name string) (*virtualization.VirtualizationInterfacesListOK, error) { vmId := strconv.FormatInt(vm.NetboxId, 10) ipIfParam := &virtualization.VirtualizationInterfacesListParams{ VirtualMachineID: &vmId, Name: &name, } - interfaces, err := n.Client.Virtualization. - VirtualizationInterfacesList(ipIfParam.WithTimeout(n.GetDefaultTimeout()), nil) + interfaces, err := vm.n.Client.Virtualization. + VirtualizationInterfacesList(ipIfParam.WithTimeout(vm.n.GetDefaultTimeout()), nil) if err != nil { return nil, fmt.Errorf("error listing virtual machine interfaces: %w", err) } @@ -95,6 +95,7 @@ func (vm *VirtualMachine) GetInterfaces(n *Netbox, name string) (*virtualization } func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualization.VirtualizationInterfacesCreateCreated, error) { +func (vm *VirtualMachine) CreateInterface(ifName string) (*virtualization.VirtualizationInterfacesCreateCreated, error) { ifParam := models.WritableVMInterface{ Name: &ifName, Enabled: true, @@ -106,8 +107,8 @@ func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualiza paramInterface := virtualization. NewVirtualizationInterfacesCreateParams(). WithData(&ifParam). - WithTimeout(n.GetDefaultTimeout()) - res, err := n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) + WithTimeout(vm.n.GetDefaultTimeout()) + res, err := vm.n.Client.Virtualization.VirtualizationInterfacesCreate(paramInterface, nil) if err != nil { return nil, fmt.Errorf("error creating virtual machine interface: %w", err) } @@ -116,8 +117,8 @@ func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualiza return res, nil } -func (vm *VirtualMachine) changeIPInterface(n *Netbox, address string, ifId int64, objectType string) error { - ip := n.getIpAddress(address) +func (vm *VirtualMachine) UpdateInterfaceIP(address string, ifId int64, objectType string) error { + ip := vm.n.getIpAddress(address) ip.AssignedObjectID = &ifId ip.AssignedObjectType = &objectType @@ -125,9 +126,9 @@ func (vm *VirtualMachine) changeIPInterface(n *Netbox, address string, ifId int6 Data: ip, } - _, err := n.Client.Ipam. + _, err := vm.n.Client.Ipam. IpamIPAddressesPartialUpdate(ifUpdateParam.WithID(ip.ID). - WithTimeout(n.GetDefaultTimeout()), nil) + WithTimeout(vm.n.GetDefaultTimeout()), nil) if err != nil { return fmt.Errorf("error updating ip address: %w", err) } diff --git a/model/netbox.go b/model/netbox.go index a8895a4..f87e6bc 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -83,7 +83,7 @@ func (n *Netbox) CreateVM(msg Message) error { vm.NetboxId = res.Payload.ID //Create management interface - r, err := vm.CreateInterface(n, "mgmt") + r, err := vm.CreateInterface("mgmt") if err != nil { return err } @@ -121,7 +121,7 @@ func (n *Netbox) CreateVM(msg Message) error { //Si l'ip n'est pas liée à une interface //On l'assigne à l'interface de la machine et zou if linkedInterfaceId == nil { - return vm.changeIPInterface(n, msg.IpAddress, ifId, objectType) + return vm.UpdateInterfaceIP(msg.IpAddress, ifId, objectType) } //Sinon on vérifie si la VM possède d'autres IP sur l'interface de management @@ -154,7 +154,7 @@ func (n *Netbox) CreateVM(msg Message) error { //L'interface possède d'autres IPs //Du coup, on prend l'ip en question util.Info("Remove the link ...") - err := vm.changeIPInterface(n, msg.IpAddress, ifId, objectType) + err := vm.UpdateInterfaceIP(msg.IpAddress, ifId, objectType) if err != nil { return err } From 934b8a896c3836761d0132272629500fd653ebfa Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Sun, 25 Aug 2024 00:37:07 +0200 Subject: [PATCH 81/91] wip: extract update of vm management ip --- model/VirtualMachine.go | 174 +++++++++++++++++++++++++++++++++++++++- model/netbox.go | 124 +--------------------------- 2 files changed, 174 insertions(+), 124 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 6ff8c01..80ea0ce 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -21,6 +21,10 @@ type VirtualMachine struct { n *Netbox } +var ( + mgmtInterfaceName = "mgmt" +) + func NewVM(n *Netbox, msg Message) *VirtualMachine { vm := &VirtualMachine{ n: n, @@ -60,6 +64,10 @@ func (vm *VirtualMachine) Create(msg Message) (*virtualization.VirtualizationVir return vm.n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) } +func (vm *VirtualMachine) CreateOrUpdate(msg Message) { + // +} + // Update vm infos to netbox func (vm *VirtualMachine) Update() error { data := vm.Get() @@ -94,7 +102,53 @@ func (vm *VirtualMachine) GetInterfaces(name string) (*virtualization.Virtualiza return interfaces, nil } -func (vm *VirtualMachine) CreateInterface(n *Netbox, ifName string) (*virtualization.VirtualizationInterfacesCreateCreated, error) { +func (vm *VirtualMachine) GetInterfaceByID(id int64) (*models.VMInterface, error) { + vmId := strconv.FormatInt(vm.NetboxId, 10) + interfaceId := strconv.FormatInt(id, 10) + + ipIfParam := &virtualization.VirtualizationInterfacesListParams{ + VirtualMachineID: &vmId, + ID: &interfaceId, + } + i, err := vm.n.Client.Virtualization. + VirtualizationInterfacesList(ipIfParam.WithTimeout(vm.n.GetDefaultTimeout()), nil) + if err != nil { + return nil, fmt.Errorf("error listing virtual machine interfaces: %w", err) + } + + if *i.Payload.Count != 1 { + return nil, fmt.Errorf("error listing virtual machine interfaces: expected 1 item, got %d", i.Payload.Count) + } + + return i.Payload.Results[0], nil +} + +func (vm *VirtualMachine) GetManagementInterface() (*models.VMInterface, error) { + vmId := strconv.FormatInt(vm.NetboxId, 10) + + ipIfParam := &virtualization.VirtualizationInterfacesListParams{ + VirtualMachineID: &vmId, + Name: &mgmtInterfaceName, + } + in, err := vm.n.Client.Virtualization. + VirtualizationInterfacesList(ipIfParam.WithTimeout(vm.n.GetDefaultTimeout()), nil) + if err != nil { + return nil, fmt.Errorf("error listing virtual machine interfaces: %w", err) + } + + //If there are no management interface, create it + if *in.Payload.Count == 0 { + mgmtInterface, err := vm.CreateInterface("mgmt") + if err != nil { + return nil, fmt.Errorf("error creating virtual machine interface: %w", err) + } + + return mgmtInterface.Payload, nil + } + + return in.Payload.Results[0], nil +} + func (vm *VirtualMachine) CreateInterface(ifName string) (*virtualization.VirtualizationInterfacesCreateCreated, error) { ifParam := models.WritableVMInterface{ Name: &ifName, @@ -159,3 +213,121 @@ func (vm *VirtualMachine) CreateIP(n *Netbox, address string, status string, lin return res, nil } + +func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { + //Get vm management interface + itf, err := vm.GetManagementInterface() + if err != nil { + return fmt.Errorf("error getting interfaces: %w", err) + } + + objectType := "virtualization.vminterface" + + //Update management interface with latest IP + err = vm.UpdateInterfaceIP(msg.IpAddress, itf.ID, objectType) + if err != nil { + return err + } + + var mgmtInterfaceId = strconv.FormatInt(itf.ID, 10) + params := ipam.NewIpamIPAddressesListParams() + params.SetVminterfaceID(&mgmtInterfaceId) + + result, err := vm.n.Client.Ipam.IpamIPAddressesList(params, nil) + if err != nil { + return fmt.Errorf("error listing ip addresses: %w", err) + } + + var ipCount = result.Payload.Count + util.Info("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10)) + + if *ipCount > 1 { + return fmt.Errorf("there are more than one management ip linked to the management interface") + } + + if *ipCount == 1 { + ip := result.Payload.Results[0] + if *ip.Address == msg.IpAddress { + //Nothing to do + return nil + } + + // 4. The management IP changed, so : + // - unlink the old ip and interface + // - set the new ip to the interface + + oldIpUpdatePrams := models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectType: nil, + AssignedObjectID: nil, + } + + paramUnlinkOldIp := ipam. + NewIpamIPAddressesPartialUpdateParams(). + WithID(ip.ID). + WithData(&oldIpUpdatePrams). + WithTimeout(vm.n.GetDefaultTimeout()) + _, err = vm.n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) + if err != nil { + return fmt.Errorf("error updating management ip addresses of VM #%d: %w", vm.NetboxId, err) + } + + util.Success("Successfully updated management ip addresses of VM #%d with new IP: %s", vm.NetboxId, msg.IpAddress) + return nil + } + + // 5. No existing IP, but verify that she doesn't already exist in the netbox + ipSearchParams := ipam.NewIpamIPAddressesListParams() + ipSearchParams.Q = &msg.IpAddress + result, err = vm.n.Client.Ipam.IpamIPAddressesList(ipSearchParams, nil) + if err != nil { + return fmt.Errorf("error listing existing ip addresses: %w", err) + } + + existingIpCount := result.Payload.Count + newIpAddrId := int64(0) + if *existingIpCount == 0 { + util.Info("There is no IP registered in the netbox. Create him.") + var ipType = "virtualization.vminterface" + newIp := &ipam.IpamIPAddressesCreateParams{ + Data: &models.WritableIPAddress{ + Address: &msg.IpAddress, + AssignedObjectID: &itf.ID, + AssignedObjectType: &ipType, + }, + } + r, err := vm.n.Client.Ipam.IpamIPAddressesCreate(newIp.WithTimeout(vm.n.GetDefaultTimeout()), nil) + if err != nil { + return fmt.Errorf("error creating ip address: %w", err) + } + + newIpAddrId = r.Payload.ID + } else { + newIpAddrId = result.Payload.Results[0].ID + } + + var ipType = "virtualization.vminterface" + + ip := vm.n.getIpAddress(msg.IpAddress) + ip.ID = newIpAddrId + ip.AssignedObjectID = &itf.ID + ip.AssignedObjectType = &ipType + + ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ + Data: ip, + } + _, err = vm.n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(vm.n.GetDefaultTimeout()), nil) + if err != nil { + return fmt.Errorf("error updating ip address: %w", err) + } + + return nil +} + +func (vm *VirtualMachine) Exists(hostname string, serial string) (bool, int64, error) { + if vm.NetboxId <= 0 && vm.n == nil { + return false, 0, nil + } + + return vm.n.VmExists(hostname, serial) +} diff --git a/model/netbox.go b/model/netbox.go index f87e6bc..44ad961 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -184,129 +184,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { } //Update management IP - // 1. Get current interface IPs list - var ( - vmId = strconv.FormatInt(id, 10) - mgmtIfName = "mgmt" - ) - - interfaces, err := vm.GetInterfaces(n, mgmtIfName) - if err != nil { - return err - } - - // 2. If there is no interface, quit - var ( - ifCount = interfaces.Payload.Count - ) - if *ifCount < 1 { - //No virtual interface, create one - - _, err := vm.CreateInterface(n, "mgmt") - if err != nil { - return fmt.Errorf("error creating virtual machine interface: %w", err) - } - - util.Info("Updated VM #" + strconv.FormatInt(id, 10) + " management interface with IP " + msg.IpAddress) - return nil - } - - // 3. Get the current management IP - mgmtInterface := interfaces.Payload.Results[0] - var mgmtInterfaceId = strconv.FormatInt(mgmtInterface.ID, 10) - params := ipam.NewIpamIPAddressesListParams() - params.SetVminterfaceID(&mgmtInterfaceId) - util.Info("Found MGMT Iface #%d -> %s", mgmtInterface.ID, mgmtInterfaceId) - - result, err := n.Client.Ipam.IpamIPAddressesList(params, nil) - if err != nil { - return fmt.Errorf("error listing ip addresses: %w", err) - } - - var ipCount = result.Payload.Count - util.Info("There are actually %s IP(s) associated with the management interface", strconv.FormatInt(*ipCount, 10)) - - if *ipCount > 1 { - return errors.New("there are more than one management ip linked to the management interface") - } - - if *ipCount == 1 { - ip := result.Payload.Results[0] - if *ip.Address == msg.IpAddress { - //Nothing to do - return nil - } - - // 4. The management IP changed, so : - // - unlink the old ip and interface - // - set the new ip to the interface - - oldIpUpdatePrams := models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectType: nil, - AssignedObjectID: nil, - } - - paramUnlinkOldIp := ipam. - NewIpamIPAddressesPartialUpdateParams(). - WithID(ip.ID). - WithData(&oldIpUpdatePrams). - WithTimeout(n.GetDefaultTimeout()) - _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) - if err != nil { - return fmt.Errorf("error updating management ip addresses of VM #%d: %w", vmId, err) - } - - util.Success("Successfully updated management ip addresses of VM #%d with new IP: %s", vmId, msg.IpAddress) - return nil - } - - // 5. No existing IP, but verify that she doesn't already exist in the netbox - ipSearchParams := ipam.NewIpamIPAddressesListParams() - ipSearchParams.Q = &msg.IpAddress - result, err = n.Client.Ipam.IpamIPAddressesList(ipSearchParams, nil) - if err != nil { - return fmt.Errorf("error listing existing ip addresses: %w", err) - } - - existingIpCount := result.Payload.Count - newIpAddrId := int64(0) - if *existingIpCount == 0 { - util.Info("There is no IP registered in the netbox. Create him.") - var ipType = "virtualization.vminterface" - newIp := &ipam.IpamIPAddressesCreateParams{ - Data: &models.WritableIPAddress{ - Address: &msg.IpAddress, - AssignedObjectID: &mgmtInterface.ID, - AssignedObjectType: &ipType, - }, - } - r, err := n.Client.Ipam.IpamIPAddressesCreate(newIp.WithTimeout(n.GetDefaultTimeout()), nil) - if err != nil { - return fmt.Errorf("error creating ip address: %w", err) - } - - newIpAddrId = r.Payload.ID - } else { - newIpAddrId = result.Payload.Results[0].ID - } - - var ipType = "virtualization.vminterface" - - ip := n.getIpAddress(msg.IpAddress) - ip.ID = newIpAddrId - ip.AssignedObjectID = &mgmtInterface.ID - ip.AssignedObjectType = &ipType - - ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ - Data: ip, - } - _, err = n.Client.Ipam.IpamIPAddressesPartialUpdate(ifUpdateParam.WithTimeout(n.GetDefaultTimeout()), nil) - if err != nil { - return fmt.Errorf("error updating ip address: %w", err) - } - - return nil + return vm.UpdateManagementIP(msg) } func (n *Netbox) CreateOrUpdateVM(msg Message) error { From cbe4be51ac0642996f303075f058e8149774d86b Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 29 Aug 2024 21:24:42 +0200 Subject: [PATCH 82/91] refactor: correctly ink ip address to machine interface --- model/VirtualMachine.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 80ea0ce..555f934 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -256,7 +256,7 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { // - unlink the old ip and interface // - set the new ip to the interface - oldIpUpdatePrams := models.WritableIPAddress{ + oldIpUpdateParams := models.WritableIPAddress{ Address: &msg.IpAddress, AssignedObjectType: nil, AssignedObjectID: nil, @@ -265,11 +265,26 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { paramUnlinkOldIp := ipam. NewIpamIPAddressesPartialUpdateParams(). WithID(ip.ID). - WithData(&oldIpUpdatePrams). + WithData(&oldIpUpdateParams). WithTimeout(vm.n.GetDefaultTimeout()) _, err = vm.n.Client.Ipam.IpamIPAddressesPartialUpdate(paramUnlinkOldIp, nil) if err != nil { - return fmt.Errorf("error updating management ip addresses of VM #%d: %w", vm.NetboxId, err) + return fmt.Errorf("error unlinking management ip addresses of VM #%d: %w", vm.NetboxId, err) + } + + // Set the ip to the new machine + newIpUpdateParam := vm.n.getIpAddress(msg.IpAddress) + newIpUpdateParam.AssignedObjectID = &vm.NetboxId + newIpUpdateParam.AssignedObjectType = &objectType + + paramLinkNewIp := ipam. + NewIpamIPAddressesPartialUpdateParams(). + WithID(ip.ID). + WithData(newIpUpdateParam). + WithTimeout(vm.n.GetDefaultTimeout()) + _, err = vm.n.Client.Ipam.IpamIPAddressesPartialUpdate(paramLinkNewIp, nil) + if err != nil { + return fmt.Errorf("error linking ip with the new interface : %w", err) } util.Success("Successfully updated management ip addresses of VM #%d with new IP: %s", vm.NetboxId, msg.IpAddress) From 5f68f2b4a0231d354ec9cca260b5b64986ecab4f Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 5 Sep 2024 21:59:29 +0200 Subject: [PATCH 83/91] fix logic --- model/netbox.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/netbox.go b/model/netbox.go index 44ad961..5707eb8 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -240,7 +240,6 @@ func (n *Netbox) VmExists(hostname string, serial string) (bool, int64, error) { var cf = v.CustomFields.(map[string]interface{}) var serial = "" - _ = serial for k, v := range cf { switch c := v.(type) { @@ -251,7 +250,7 @@ func (n *Netbox) VmExists(hostname string, serial string) (bool, int64, error) { } } - if serial == serial { + if serial != "" { return true, v.ID, nil } } From d4ad0ecf1facc44aeef2397754d075982e94b158 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 29 Oct 2024 18:09:38 +0100 Subject: [PATCH 84/91] refactor: move docker conf --- docker-compose.yml | 10 +++++----- .../configuration}/configuration.py | 0 {configuration => docker/configuration}/extra.py | 0 {configuration => docker/configuration}/ldap/extra.py | 0 .../configuration}/ldap/ldap_config.py | 0 {configuration => docker/configuration}/logging.py | 0 {configuration => docker/configuration}/plugins.py | 0 {env => docker/env}/netbox.env | 0 {env => docker/env}/postgres.env | 0 {env => docker/env}/redis-cache.env | 0 {env => docker/env}/redis.env | 0 11 files changed, 5 insertions(+), 5 deletions(-) rename {configuration => docker/configuration}/configuration.py (100%) rename {configuration => docker/configuration}/extra.py (100%) rename {configuration => docker/configuration}/ldap/extra.py (100%) rename {configuration => docker/configuration}/ldap/ldap_config.py (100%) rename {configuration => docker/configuration}/logging.py (100%) rename {configuration => docker/configuration}/plugins.py (100%) rename {env => docker/env}/netbox.env (100%) rename {env => docker/env}/postgres.env (100%) rename {env => docker/env}/redis-cache.env (100%) rename {env => docker/env}/redis.env (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 966179c..2a3acf3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: - postgres - redis - redis-cache - env_file: env/netbox.env + env_file: docker/env/netbox.env user: 'unit:root' healthcheck: start_period: 60s @@ -13,7 +13,7 @@ services: interval: 15s test: "curl -f http://localhost:8080/login/ || exit 1" volumes: - - ./configuration:/etc/netbox/config:z,ro + - ./docker/configuration:/etc/netbox/config:z,ro - netbox-media-files:/opt/netbox/netbox/media:rw - netbox-reports-files:/opt/netbox/netbox/reports:rw - netbox-scripts-files:/opt/netbox/netbox/scripts:rw @@ -49,7 +49,7 @@ services: # postgres postgres: image: docker.io/postgres:16-alpine - env_file: env/postgres.env + env_file: docker/env/postgres.env volumes: - netbox-postgres-data:/var/lib/postgresql/data @@ -60,7 +60,7 @@ services: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis.env + env_file: docker/env/redis.env volumes: - netbox-redis-data:/data redis-cache: @@ -69,7 +69,7 @@ services: - sh - -c # this is to evaluate the $REDIS_PASSWORD from the env - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose - env_file: env/redis-cache.env + env_file: docker/env/redis-cache.env volumes: - netbox-redis-cache-data:/data diff --git a/configuration/configuration.py b/docker/configuration/configuration.py similarity index 100% rename from configuration/configuration.py rename to docker/configuration/configuration.py diff --git a/configuration/extra.py b/docker/configuration/extra.py similarity index 100% rename from configuration/extra.py rename to docker/configuration/extra.py diff --git a/configuration/ldap/extra.py b/docker/configuration/ldap/extra.py similarity index 100% rename from configuration/ldap/extra.py rename to docker/configuration/ldap/extra.py diff --git a/configuration/ldap/ldap_config.py b/docker/configuration/ldap/ldap_config.py similarity index 100% rename from configuration/ldap/ldap_config.py rename to docker/configuration/ldap/ldap_config.py diff --git a/configuration/logging.py b/docker/configuration/logging.py similarity index 100% rename from configuration/logging.py rename to docker/configuration/logging.py diff --git a/configuration/plugins.py b/docker/configuration/plugins.py similarity index 100% rename from configuration/plugins.py rename to docker/configuration/plugins.py diff --git a/env/netbox.env b/docker/env/netbox.env similarity index 100% rename from env/netbox.env rename to docker/env/netbox.env diff --git a/env/postgres.env b/docker/env/postgres.env similarity index 100% rename from env/postgres.env rename to docker/env/postgres.env diff --git a/env/redis-cache.env b/docker/env/redis-cache.env similarity index 100% rename from env/redis-cache.env rename to docker/env/redis-cache.env diff --git a/env/redis.env b/docker/env/redis.env similarity index 100% rename from env/redis.env rename to docker/env/redis.env From 60eda576142e54322596ca40d47bfa584e6d104b Mon Sep 17 00:00:00 2001 From: "Alice. B" <45518780+IDarKay@users.noreply.github.com> Date: Tue, 29 Oct 2024 23:48:08 +0100 Subject: [PATCH 85/91] refactor + fix (#6) * fix: catch ctrl+c + end of amqp queue in main message loop * refactor: Change useless ch.PublishWithContex to ch.Publish, refer to amqp manual for more informations * fix: chamge NewNetbox retunr pointer to a model.Netbox insted of copy of struct * refactor: pack all WritableVirtualMachineWiithConfigContext to VirtualMachine->Get() --- main.go | 39 +++++++++++++++++++++------------------ model/VirtualMachine.go | 22 ++++++---------------- model/netbox.go | 6 +++--- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/main.go b/main.go index c062b99..0d05fc7 100644 --- a/main.go +++ b/main.go @@ -4,18 +4,19 @@ import ( "context" "encoding/json" "fmt" + "os" + "os/signal" + "strconv" + "github.com/KittenConnect/rh-api/model" "github.com/KittenConnect/rh-api/util" "github.com/joho/godotenv" amqp "github.com/rabbitmq/amqp091-go" - "os" - "strconv" - "time" ) func failWithError(err error, formatString string, args ...any) { if err != nil { - util.Err(fmt.Errorf(fmt.Sprintf("%s: %w",formatString), append(args, err)...).Error()) + util.Err(fmt.Errorf(fmt.Sprintf("%s: %w", formatString), append(args, err)...).Error()) } } @@ -107,8 +108,18 @@ func main() { os.Exit(-1) } + // cancel context for whole conde + foreverCtx, foreverCancel := context.WithCancel(context.Background()) + // Canal pour signaler la fin du programme - forever := make(chan bool) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt) + // catch signal + go func() { + <-sigs + fmt.Printf("You pressed ctrl + C. User interrupted infinite loop.") + foreverCancel() + }() go func() { for d := range msgs { @@ -125,10 +136,6 @@ func main() { if err != nil { util.Warn("error creating or updating VM : %w", err) - dur, _ := time.ParseDuration("10s") - ctx, cancel := context.WithTimeout(context.Background(), dur) - defer cancel() - newMsg := msg newMsg.FailCount-- @@ -142,8 +149,7 @@ func main() { "x-delay": RETRY_DELAY * 1000, } - chErr := ch.PublishWithContext( - ctx, + chErr := ch.Publish( incomingQueue, inQ.Name, false, @@ -165,16 +171,11 @@ func main() { util.Success("VM %s is up to date", msg.Hostname) - dur, _ := time.ParseDuration("10s") - ctx, cancel := context.WithTimeout(context.Background(), dur) - defer cancel() - newMsg := msg newMsgJson, _ := json.Marshal(newMsg) - chErr := ch.PublishWithContext( - ctx, + chErr := ch.Publish( "", outQ.Name, false, @@ -191,8 +192,10 @@ func main() { } }() } + util.Info("End of queue reaches exit now !") + foreverCancel() }() util.Info(" [*] Waiting for messages. To exit press CTRL+C") - <-forever + <-foreverCtx.Done() } diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 555f934..2a027bc 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -36,9 +36,11 @@ func NewVM(n *Netbox, msg Message) *VirtualMachine { return vm } -func (vm *VirtualMachine) Get() models.WritableVirtualMachineWithConfigContext { +// dev chnage name ? +// Get: return tableVirtualMachineWithConfigContext for vm +func (vm *VirtualMachine) Get() *models.WritableVirtualMachineWithConfigContext { // todo: implement netbox func - return models.WritableVirtualMachineWithConfigContext{ + return &models.WritableVirtualMachineWithConfigContext{ Cluster: &vm.Cluster.ID, Name: &vm.Name, Status: vm.Status, @@ -50,17 +52,7 @@ func (vm *VirtualMachine) Get() models.WritableVirtualMachineWithConfigContext { } func (vm *VirtualMachine) Create(msg Message) (*virtualization.VirtualizationVirtualMachinesCreateCreated, error) { - conf := models.WritableVirtualMachineWithConfigContext{ - Cluster: &vm.Cluster.ID, - Name: &vm.Name, - Status: vm.Status, - - CustomFields: map[string]interface{}{ - "kc_serial_": msg.GetSerial(), - }, - } - - params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(&conf) + params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(vm.Get()) return vm.n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) } @@ -70,10 +62,8 @@ func (vm *VirtualMachine) CreateOrUpdate(msg Message) { // Update vm infos to netbox func (vm *VirtualMachine) Update() error { - data := vm.Get() - updateParams := &virtualization.VirtualizationVirtualMachinesPartialUpdateParams{ - Data: &data, + Data: vm.Get(), ID: vm.NetboxId, } diff --git a/model/netbox.go b/model/netbox.go index 5707eb8..bfe4f2f 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -27,7 +27,7 @@ type Netbox struct { } // NewNetbox return a fresh Netbox object -func NewNetbox() Netbox { +func NewNetbox() *Netbox { nbx := Netbox{ ctx: context.Background(), Client: nil, @@ -35,7 +35,7 @@ func NewNetbox() Netbox { _isConnected: false, } - return nbx + return &nbx } func (n *Netbox) IsConnected() bool { @@ -54,7 +54,7 @@ func (n *Netbox) Connect() error { } func (n *Netbox) GetDefaultTimeout() time.Duration { - return time.Duration(30) * time.Second + return 30 * time.Second } func (n *Netbox) getIpAddress(ip string) *models.WritableIPAddress { From 42a3b9cb4a1f92ec2c4e2603089857636bbc0e24 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 21 Oct 2025 19:11:17 +0200 Subject: [PATCH 86/91] refactor: use separate type for VM interfaces types --- model/VirtualMachine.go | 19 ++++++++++++------- model/netbox.go | 8 +++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 2a027bc..92fcc41 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -161,10 +161,12 @@ func (vm *VirtualMachine) CreateInterface(ifName string) (*virtualization.Virtua return res, nil } -func (vm *VirtualMachine) UpdateInterfaceIP(address string, ifId int64, objectType string) error { +func (vm *VirtualMachine) UpdateInterfaceIP(address string, ifId int64, objectType NetboxVmObjectType) error { + objType := string(objectType) + ip := vm.n.getIpAddress(address) ip.AssignedObjectID = &ifId - ip.AssignedObjectType = &objectType + ip.AssignedObjectType = &objType ifUpdateParam := &ipam.IpamIPAddressesPartialUpdateParams{ Data: ip, @@ -181,15 +183,17 @@ func (vm *VirtualMachine) UpdateInterfaceIP(address string, ifId int64, objectTy return nil } -func (vm *VirtualMachine) CreateIP(n *Netbox, address string, status string, linkedObjectId int64, linkedObjectType string) (*ipam.IpamIPAddressesCreateCreated, error) { +func (vm *VirtualMachine) CreateIP(n *Netbox, address string, status string, linkedObjectId int64, linkedObjectType NetboxVmObjectType) (*ipam.IpamIPAddressesCreateCreated, error) { ip := &models.WritableIPAddress{ Address: &address, Status: status, } if linkedObjectId != -1 && linkedObjectType != "" { + objectType := string(linkedObjectType) + ip.AssignedObjectID = &linkedObjectId - ip.AssignedObjectType = &linkedObjectType + ip.AssignedObjectType = &objectType } ipCreateParams := &ipam.IpamIPAddressesCreateParams{ @@ -211,7 +215,7 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { return fmt.Errorf("error getting interfaces: %w", err) } - objectType := "virtualization.vminterface" + objectType := VmInterfaceType //Update management interface with latest IP err = vm.UpdateInterfaceIP(msg.IpAddress, itf.ID, objectType) @@ -263,6 +267,7 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { } // Set the ip to the new machine + objectType := string(objectType) newIpUpdateParam := vm.n.getIpAddress(msg.IpAddress) newIpUpdateParam.AssignedObjectID = &vm.NetboxId newIpUpdateParam.AssignedObjectType = &objectType @@ -293,7 +298,7 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { newIpAddrId := int64(0) if *existingIpCount == 0 { util.Info("There is no IP registered in the netbox. Create him.") - var ipType = "virtualization.vminterface" + var ipType = string(VmInterfaceType) newIp := &ipam.IpamIPAddressesCreateParams{ Data: &models.WritableIPAddress{ Address: &msg.IpAddress, @@ -311,7 +316,7 @@ func (vm *VirtualMachine) UpdateManagementIP(msg Message) error { newIpAddrId = result.Payload.Results[0].ID } - var ipType = "virtualization.vminterface" + var ipType = string(VmInterfaceType) ip := vm.n.getIpAddress(msg.IpAddress) ip.ID = newIpAddrId diff --git a/model/netbox.go b/model/netbox.go index bfe4f2f..482432d 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -15,6 +15,12 @@ import ( "time" ) +type NetboxVmObjectType string + +const ( + VmInterfaceType NetboxVmObjectType = "virtualization.vminterface" +) + // Netbox structure // For internal use ONLY ! // To get an instance, call NewNetbox method @@ -90,7 +96,7 @@ func (n *Netbox) CreateVM(msg Message) error { var ( ifId = r.Payload.ID - objectType = "virtualization.vminterface" + objectType = VmInterfaceType ) //Verify if ip already exists From d56fbacb4d1656da00f85a029e7d32b82f6c9b5a Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Tue, 21 Oct 2025 19:26:07 +0200 Subject: [PATCH 87/91] fix: remove variable shadowing --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 0d05fc7..b7e9db1 100644 --- a/main.go +++ b/main.go @@ -125,7 +125,7 @@ func main() { for d := range msgs { go func() { msg := model.Message{Timestamp: d.Timestamp, FailCount: 20} - err := json.Unmarshal(d.Body, &msg) + err = json.Unmarshal(d.Body, &msg) if err != nil { util.Warn("Error unmarshalling message : %w", err) return From 331e396c581d9e89fe4409a0516369e3b1f639f8 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Wed, 22 Oct 2025 19:02:57 +0200 Subject: [PATCH 88/91] feat: correctly handling error return when trying to create vm --- model/netbox.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model/netbox.go b/model/netbox.go index 482432d..a43a54e 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -183,6 +183,9 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { vm.NetboxId = id _, err := vm.Create(msg) + if err != nil { + return err + } err = vm.Update() if err != nil { From b263e61ff461ffd8930cf069c23545cad1a8ff94 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 23 Oct 2025 17:32:13 +0200 Subject: [PATCH 89/91] refactor: remove unused parameter --- model/VirtualMachine.go | 2 +- model/netbox.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 92fcc41..ca42e0c 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -51,7 +51,7 @@ func (vm *VirtualMachine) Get() *models.WritableVirtualMachineWithConfigContext } } -func (vm *VirtualMachine) Create(msg Message) (*virtualization.VirtualizationVirtualMachinesCreateCreated, error) { +func (vm *VirtualMachine) Create() (*virtualization.VirtualizationVirtualMachinesCreateCreated, error) { params := virtualization.NewVirtualizationVirtualMachinesCreateParams().WithData(vm.Get()) return vm.n.Client.Virtualization.VirtualizationVirtualMachinesCreate(params, nil) } diff --git a/model/netbox.go b/model/netbox.go index a43a54e..63c606d 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -76,7 +76,7 @@ func (n *Netbox) CreateVM(msg Message) error { } vm := NewVM(n, msg) - res, err := vm.Create(msg) + res, err := vm.Create() if err != nil { if res != nil && res.Payload != nil { return fmt.Errorf("error creating virtual machine: %w \n\t%s", err, res.Error()) @@ -182,7 +182,7 @@ func (n *Netbox) UpdateVM(id int64, msg Message) error { vm := NewVM(n, msg) vm.NetboxId = id - _, err := vm.Create(msg) + _, err := vm.Create() if err != nil { return err } From 5e82570505acbe7c1daefcbe72e1a74d98e9ab06 Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 23 Oct 2025 17:33:12 +0200 Subject: [PATCH 90/91] fix: set missing vm serial --- model/VirtualMachine.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index ca42e0c..92d8ced 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -26,11 +26,14 @@ var ( ) func NewVM(n *Netbox, msg Message) *VirtualMachine { + serial := msg.parseSerial() + vm := &VirtualMachine{ n: n, NetboxId: -1, - Name: msg.Hostname, + Name: msg.Hostname, + Serial: serial, } return vm From aff87c28276d4b7fcdcdd3e9c3ca17bbfe9a153f Mon Sep 17 00:00:00 2001 From: Romain Neil Date: Thu, 23 Oct 2025 17:33:38 +0200 Subject: [PATCH 91/91] fix: correct imports sorting --- model/VirtualMachine.go | 5 +++-- model/netbox.go | 7 ++++--- util/log.go | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/model/VirtualMachine.go b/model/VirtualMachine.go index 92d8ced..be7dd7b 100644 --- a/model/VirtualMachine.go +++ b/model/VirtualMachine.go @@ -2,12 +2,13 @@ package model import ( "fmt" + "net" + "strconv" + "github.com/KittenConnect/rh-api/util" "github.com/netbox-community/go-netbox/netbox/client/ipam" "github.com/netbox-community/go-netbox/netbox/client/virtualization" "github.com/netbox-community/go-netbox/netbox/models" - "net" - "strconv" ) type VirtualMachine struct { diff --git a/model/netbox.go b/model/netbox.go index 63c606d..3526aa9 100644 --- a/model/netbox.go +++ b/model/netbox.go @@ -4,15 +4,16 @@ import ( "context" "errors" "fmt" + "os" + "strconv" + "time" + "github.com/KittenConnect/rh-api/util" "github.com/netbox-community/go-netbox/netbox" "github.com/netbox-community/go-netbox/netbox/client" "github.com/netbox-community/go-netbox/netbox/client/ipam" "github.com/netbox-community/go-netbox/netbox/client/virtualization" "github.com/netbox-community/go-netbox/netbox/models" - "os" - "strconv" - "time" ) type NetboxVmObjectType string diff --git a/util/log.go b/util/log.go index 359db43..6d11808 100644 --- a/util/log.go +++ b/util/log.go @@ -2,8 +2,9 @@ package util import ( "fmt" - "github.com/fatih/color" "os" + + "github.com/fatih/color" ) func Color(c color.Attribute, prefix string, s string, v ...any) {