From 25713944b116ccbf1adb908ccaf759dc7630d296 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Wed, 10 Dec 2025 10:00:29 +0100 Subject: [PATCH 1/6] Backup Kubernetes Manifests of Application This change provides basic code changes for deleting the Kubernetes manifests from the Kubernetes' namespace that hosts a PREvant application while preserving the deleted manifests in the database. --- .../20251210154013_Back_up_table.sql | 6 + api/src/apps/mod.rs | 53 +- api/src/apps/queue/mod.rs | 67 +- api/src/apps/queue/postgres.rs | 75 +- api/src/apps/queue/task.rs | 8 + api/src/apps/routes/mod.rs | 38 +- api/src/infrastructure/infrastructure.rs | 15 + .../kubernetes/deployment_unit.rs | 911 ++++++++++++++---- .../kubernetes/infrastructure.rs | 25 + 9 files changed, 938 insertions(+), 260 deletions(-) create mode 100644 api/migrations/20251210154013_Back_up_table.sql diff --git a/api/migrations/20251210154013_Back_up_table.sql b/api/migrations/20251210154013_Back_up_table.sql new file mode 100644 index 00000000..77fcd531 --- /dev/null +++ b/api/migrations/20251210154013_Back_up_table.sql @@ -0,0 +1,6 @@ +CREATE TABLE app_backup ( + app_name VARCHAR(512) PRIMARY KEY, + app JSONB NOT NULL, + infrastructure_payload JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); diff --git a/api/src/apps/mod.rs b/api/src/apps/mod.rs index 36c3cbd0..5eb95dea 100644 --- a/api/src/apps/mod.rs +++ b/api/src/apps/mod.rs @@ -74,10 +74,7 @@ impl Clone for Apps { } impl Apps { - pub fn new( - config: Config, - infrastructure: Box, - ) -> Result { + pub fn new(config: Config, infrastructure: Box) -> Result { Ok(Apps { config, infrastructure, @@ -121,6 +118,16 @@ impl Apps { Ok(self.infrastructure.fetch_apps().await?) } + pub async fn fetch_app_as_backup_based_infrastructure_payload( + &self, + app_name: &AppName, + ) -> Result>, AppsError> { + Ok(self + .infrastructure + .fetch_app_as_backup_based_infrastructure_payload(app_name) + .await?) + } + /// Provides a [`Receiver`](tokio::sync::watch::Receiver) that notifies about changes of the /// list of running [`apps`](AppsService::fetch_apps). pub async fn app_updates(&self) -> Receiver> { @@ -225,15 +232,13 @@ impl Apps { ) { (None, _) => None, (Some(validator), None) => Some( - UserDefinedParameters::new(serde_json::json!({}), &validator).map_err(|e| { - AppsError::InvalidUserDefinedParameters { err: e.to_string() } - })?, + UserDefinedParameters::new(serde_json::json!({}), &validator) + .map_err(|e| AppsError::InvalidUserDefinedParameters { err: e.to_string() })?, + ), + (Some(validator), Some(value)) => Some( + UserDefinedParameters::new(value, &validator) + .map_err(|e| AppsError::InvalidUserDefinedParameters { err: e.to_string() })?, ), - (Some(validator), Some(value)) => { - Some(UserDefinedParameters::new(value, &validator).map_err(|e| { - AppsError::InvalidUserDefinedParameters { err: e.to_string() } - })?) - } }; if let Some(app_limit) = self.config.app_limit() { @@ -381,6 +386,24 @@ impl Apps { } } + async fn delete_app_partially( + &self, + app_name: &AppName, + infrastructure_payload: &[serde_json::Value], + ) -> Result { + let Some(app) = self.infrastructure.fetch_app(app_name).await? else { + return Err(AppsError::AppNotFound { + app_name: app_name.clone(), + }); + }; + + self.infrastructure + .delete_infrastructure_objects_partially(app_name, infrastructure_payload) + .await?; + + Ok(app) + } + pub async fn stream_logs<'a>( &'a self, app_name: &'a AppName, @@ -1379,8 +1402,7 @@ Log msg 3 of service-a of app master } #[tokio::test] - async fn do_not_create_app_when_exceeding_application_number_limit( - ) -> Result<(), AppsError> { + async fn do_not_create_app_when_exceeding_application_number_limit() -> Result<(), AppsError> { let config = config_from_str!( r#" [applications] @@ -1418,8 +1440,7 @@ Log msg 3 of service-a of app master } #[tokio::test] - async fn do_update_app_when_exceeding_application_number_limit() -> Result<(), AppsError> - { + async fn do_update_app_when_exceeding_application_number_limit() -> Result<(), AppsError> { let config = config_from_str!( r#" [applications] diff --git a/api/src/apps/queue/mod.rs b/api/src/apps/queue/mod.rs index 80cda1ce..361ec594 100644 --- a/api/src/apps/queue/mod.rs +++ b/api/src/apps/queue/mod.rs @@ -136,6 +136,28 @@ impl AppTaskQueueProducer { Ok(status_id) } + pub async fn enqueue_backup_task( + &self, + app_name: AppName, + infrastructure_payload: Vec, + ) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name, + infrastructure_payload, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new backup task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + pub async fn try_wait_for_task( &self, status_id: &AppStatusChangeId, @@ -190,7 +212,6 @@ impl AppTaskQueueConsumer { let Some(task) = tasks.into_iter().reduce(|acc, e| acc.merge_with(e)) else { panic!("tasks must not be empty"); }; - let status_id = task.status_id().clone(); if log::log_enabled!(log::Level::Debug) { log::debug!( @@ -199,7 +220,23 @@ impl AppTaskQueueConsumer { task.app_name() ); } - match task { + match &task { + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + infrastructure_payload, + .. + } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!( + "Dropping infrastructure objects for {app_name} due to back up." + ); + } + + let result = apps + .delete_app_partially(app_name, infrastructure_payload) + .await; + (task, result) + } AppTask::CreateOrUpdate { app_name, replicate_from, @@ -211,23 +248,24 @@ impl AppTaskQueueConsumer { if log::log_enabled!(log::Level::Debug) { log::debug!("Creating or updating app {app_name}."); } - ( - status_id, - apps.create_or_update( + + let result = apps + .create_or_update( &app_name, - replicate_from, + replicate_from.clone(), &service_configs, - owners, - user_defined_parameters, + owners.clone(), + user_defined_parameters.clone(), ) - .await, - ) + .await; + (task, result) } AppTask::Delete { app_name, .. } => { if log::log_enabled!(log::Level::Debug) { log::debug!("Deleting app {app_name}."); } - (status_id, apps.delete_app(&app_name).await) + let result = apps.delete_app(&app_name).await; + (task, result) } } }) @@ -263,6 +301,7 @@ impl AppTaskQueueDB { pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { match self { AppTaskQueueDB::InMemory(mutex) => { + // TODO: fail for in memory backup let mut queue = mutex.lock().await; queue.push_back((task, AppTaskStatus::New)); } @@ -299,7 +338,7 @@ impl AppTaskQueueDB { async fn execute_tasks(&self, f: F) -> Result<()> where F: FnOnce(Vec) -> Fut, - Fut: Future)>, + Fut: Future)>, { match self { AppTaskQueueDB::InMemory(mutex) => { @@ -320,7 +359,7 @@ impl AppTaskQueueDB { }; let status_id = *task.status_id(); - let (task_id, result) = f(vec![task]).await; + let (task_worked_on, result) = f(vec![task]).await; let mut queue = mutex.lock().await; let Some(task) = queue @@ -330,7 +369,7 @@ impl AppTaskQueueDB { anyhow::bail!("Cannot update {status_id} in queue which should be present"); }; - assert!(task_id == *task.0.status_id()); + assert!(task_worked_on.status_id() == task.0.status_id()); task.1 = AppTaskStatus::Done((Utc::now(), result)); Ok(()) diff --git a/api/src/apps/queue/postgres.rs b/api/src/apps/queue/postgres.rs index 6bb129fd..cfea79d3 100644 --- a/api/src/apps/queue/postgres.rs +++ b/api/src/apps/queue/postgres.rs @@ -134,7 +134,7 @@ impl PostgresAppTaskQueueDB { pub async fn execute_tasks(&self, f: F) -> Result<()> where F: FnOnce(Vec) -> Fut, - Fut: Future)>, + Fut: Future)>, { let mut tx = self.pool.begin().await?; @@ -172,22 +172,14 @@ impl PostgresAppTaskQueueDB { return Ok(()); } - let (id, result) = f(tasks_to_work_on).await; + let (tasked_worked_on, result) = f(tasks_to_work_on).await; + let is_success = result.is_ok(); + let id = *tasked_worked_on.status_id(); - sqlx::query( - r#" - UPDATE app_task - SET status = 'done', result_success = $3, result_error = $2 - WHERE id = $1 - "#, - ) - .bind(id.as_uuid()) - .bind( - result - .as_ref() - .map_or_else(|e| serde_json::to_value(e).ok(), |_| None), - ) - .bind(result.map_or_else( + let failed_result = result + .as_ref() + .map_or_else(|e| serde_json::to_value(e).ok(), |_| None); + let success_result = result.map_or_else( |_| None, |app| { let (services, owner, user_defined_parameters) = @@ -210,10 +202,43 @@ impl PostgresAppTaskQueueDB { }; serde_json::to_value(raw).ok() }, - )) + ); + + sqlx::query( + r#" + UPDATE app_task + SET status = 'done', result_success = $3, result_error = $2 + WHERE id = $1 + "#, + ) + .bind(id.as_uuid()) + .bind(failed_result) + .bind(&success_result) .execute(&mut *tx) .await?; + if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + infrastructure_payload, + .. + } = tasked_worked_on + { + if is_success { + sqlx::query( + r#" + INSERT INTO app_backup (app_name, app, infrastructure_payload) + VALUES ($1, $2, $3); + "#, + ) + .bind(app_name.as_str()) + .bind(success_result) + .bind(serde_json::Value::Array(infrastructure_payload)) + .execute(&mut *tx) + .await?; + } + } + + // TODO: backup cannot be merged… and we should mark up to this task id. for (task_id_that_was_merged, _merged_task) in tasks.iter().filter(|task| task.0 != *id.as_uuid()) { @@ -301,9 +326,9 @@ mod tests { queue .execute_tasks(async |tasks| { - let id = tasks.last().unwrap().status_id().clone(); + let task = tasks.last().unwrap().clone(); ( - id, + task, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -375,9 +400,9 @@ mod tests { tokio::time::sleep(Duration::from_secs(4)).await; - let id = tasks.last().unwrap().status_id().clone(); + let task = tasks.last().unwrap().clone(); ( - id, + task, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -444,9 +469,9 @@ mod tests { let spawn_handle_1 = tokio::spawn(async move { spawned_queue .execute_tasks(async |tasks| { - let id = tasks.last().unwrap().status_id().clone(); + let task = tasks.last().unwrap().clone(); ( - id, + task, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -471,9 +496,9 @@ mod tests { let spawn_handle_2 = tokio::spawn(async move { spawned_queue .execute_tasks(async |tasks| { - let id = tasks.last().unwrap().status_id().clone(); + let task = tasks.last().unwrap().clone(); ( - id, + task, Ok(App::new( vec![Service { id: String::from("nginx-1234"), diff --git a/api/src/apps/queue/task.rs b/api/src/apps/queue/task.rs index f6bb5d76..918eec68 100644 --- a/api/src/apps/queue/task.rs +++ b/api/src/apps/queue/task.rs @@ -7,6 +7,11 @@ use std::collections::{HashMap, HashSet}; #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq)] #[serde(untagged)] pub(super) enum AppTask { + MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: AppStatusChangeId, + app_name: AppName, + infrastructure_payload: Vec, + }, CreateOrUpdate { app_name: AppName, status_id: AppStatusChangeId, @@ -26,12 +31,14 @@ impl AppTask { match self { AppTask::CreateOrUpdate { app_name, .. } => app_name, AppTask::Delete { app_name, .. } => app_name, + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { app_name, .. } => app_name, } } pub fn status_id(&self) -> &AppStatusChangeId { match self { AppTask::CreateOrUpdate { status_id, .. } => status_id, AppTask::Delete { status_id, .. } => status_id, + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { status_id, .. } => status_id, } } @@ -140,6 +147,7 @@ impl AppTask { status_id, app_name, }, + _ => unimplemented!(), } } } diff --git a/api/src/apps/routes/mod.rs b/api/src/apps/routes/mod.rs index 6eac9221..bc796970 100644 --- a/api/src/apps/routes/mod.rs +++ b/api/src/apps/routes/mod.rs @@ -63,6 +63,7 @@ pub fn apps_routes() -> Vec { delete_app_v2, create_app_v1, create_app_v2, + back_up_app, logs::logs, logs::stream_logs, change_status, @@ -308,6 +309,41 @@ pub async fn create_app_v2( .await } +#[rocket::put( + "//states", + format = "application/json", + // TODO payload data = "", +)] +pub async fn back_up_app( + app_name: Result, + apps: &State, + app_queue: &State, + user: Result, + options: WaitForQueueOptions, +) -> HttpResult>> { + // TODO: authorization hook to verify e.g. if a user is member of a GitLab group + let _user = user.map_err(HttpApiError::from)?; + + let app_name = app_name?; + let Some(infrastructure_payload) = apps + .fetch_app_as_backup_based_infrastructure_payload(&app_name) + .await? + else { + // TODO check if already backed up… + return Err(AppsError::AppNotFound { app_name }.into()); + }; + + let status_id = app_queue + .enqueue_backup_task(app_name.clone(), infrastructure_payload) + .await + .map_err(|e| { + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()) + })?; + + try_wait_for_task(app_queue, app_name, status_id, options, AppV2).await +} + #[rocket::put( "//states/", format = "application/json", @@ -611,7 +647,7 @@ mod tests { mod http_api_error { use super::super::*; use crate::{ - apps::{AppProcessingQueue, AppsError, Apps}, + apps::{AppProcessingQueue, Apps, AppsError}, infrastructure::Dummy, registry::RegistryError, }; diff --git a/api/src/infrastructure/infrastructure.rs b/api/src/infrastructure/infrastructure.rs index d1207613..8c98724e 100644 --- a/api/src/infrastructure/infrastructure.rs +++ b/api/src/infrastructure/infrastructure.rs @@ -42,6 +42,13 @@ pub trait Infrastructure: Send + Sync + DynClone { async fn fetch_app(&self, app_name: &AppName) -> Result>; + async fn fetch_app_as_backup_based_infrastructure_payload( + &self, + app_name: &AppName, + ) -> Result>> { + anyhow::bail!("Cannot back up {app_name}: not yet implemented for the configured backend") + } + async fn fetch_app_names(&self) -> Result> { Ok(self.fetch_apps().await?.into_keys().collect::>()) } @@ -66,6 +73,14 @@ pub trait Infrastructure: Send + Sync + DynClone { /// stopped. async fn stop_services(&self, app_name: &AppName) -> Result; + async fn delete_infrastructure_objects_partially( + &self, + app_name: &AppName, + infrastructure: &[serde_json::Value], + ) -> Result<()> { + anyhow::bail!("Cannot back up {app_name}: not yet implemented for the configured backend to delete {infrastructure:?}") + } + /// Streams the log lines with a the corresponding timestamps in it. async fn get_logs<'a>( &'a self, diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs index 8501039e..4ac8f3fd 100644 --- a/api/src/infrastructure/kubernetes/deployment_unit.rs +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -59,6 +59,255 @@ pub(super) struct K8sDeploymentUnit { traefik_middlewares: Vec, } +macro_rules! parse_from_dynamic_object { + ( + $roles:ident, + $role_bindings:ident, + $stateful_sets:ident, + $config_maps:ident, + $secrets:ident, + $pvcs:ident, + $services:ident, + $pods:ident, + $deployments:ident, + $jobs:ident, + $service_accounts:ident, + $ingresses:ident, + $policies:ident, + $traefik_ingresses:ident, + $traefik_middlewares:ident, + $api_version:ident, + $kind:ident, + $app_name:ident, + $dyn_obj:ident + ) => { + match ($api_version, $kind) { + (Role::API_VERSION, Role::KIND) => match $dyn_obj.clone().try_parse::() + { + Ok(role) => { + $roles.push(role); + } + Err(e) => { + error!("Cannot parse {:?} as Role: {e}", $dyn_obj.metadata.name); + } + }, + + (RoleBinding::API_VERSION, RoleBinding::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(role_binding) => { + $role_bindings.push(role_binding); + } + Err(e) => { + error!( + "Cannot parse {:?} as RoleBinding: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (StatefulSet::API_VERSION, StatefulSet::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(stateful_set) => { + $stateful_sets.push(stateful_set); + } + Err(e) => { + error!( + "Cannot parse {:?} as StatefulSet: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (ConfigMap::API_VERSION, ConfigMap::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(config_map) => { + $config_maps.push(config_map); + } + Err(e) => { + error!( + "Cannot parse {:?} as ConfigMap: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (Secret::API_VERSION, Secret::KIND) => { + if let serde_json::Value::Object(obj) = &mut $dyn_obj.data { + obj.entry("data").and_modify(|obj| { + if let serde_json::Value::Object(obj) = obj { + for (_k, v) in obj.iter_mut() { + if let serde_json::Value::String(str) = v { + // replacing new lines here because it is assumed + // that the data is base64 encoded and thus there + // must be no new lines + *v = str.replace('\n', "").into(); + } + } + } + }); + } + + match $dyn_obj.clone().try_parse::() { + Ok(secret) => { + $secrets.push(secret); + } + Err(e) => { + error!( + "Cannot parse {:?} as Secret: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (PersistentVolumeClaim::API_VERSION, PersistentVolumeClaim::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(pvc) => { + $pvcs.push(pvc); + } + Err(e) => { + error!( + "Cannot parse {:?} as PersistentVolumeClaim: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (Service::API_VERSION, Service::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(service) => { + $services.push(service); + } + Err(e) => { + error!( + "Cannot parse {:?} as Service: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (Deployment::API_VERSION, Deployment::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(mut deployment) => { + let service_name = deployment + .labels() + .get("app.kubernetes.io/component") + .cloned() + .unwrap_or_else(|| { + deployment.metadata.name.clone().unwrap_or_default() + }); + + deployment + .labels_mut() + .insert(SERVICE_NAME_LABEL.to_string(), service_name); + deployment.labels_mut().insert( + CONTAINER_TYPE_LABEL.to_string(), + ContainerType::ApplicationCompanion.to_string(), + ); + + $deployments.push(deployment); + } + Err(e) => { + error!( + "Cannot parse {:?} as Deployment: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (Pod::API_VERSION, Pod::KIND) => match $dyn_obj.clone().try_parse::() { + Ok(pod) => { + $pods.push(pod); + } + Err(e) => { + error!("Cannot parse {:?} as Pod: {e}", $dyn_obj.metadata.name); + } + }, + (Job::API_VERSION, Job::KIND) => match $dyn_obj.clone().try_parse::() { + Ok(job) => { + $jobs.push(job); + } + Err(e) => { + error!("Cannot parse {:?} as Job: {e}", $dyn_obj.metadata.name); + } + }, + (ServiceAccount::API_VERSION, ServiceAccount::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(service_account) => { + $service_accounts.push(service_account); + } + Err(e) => { + error!( + "Cannot parse {:?} as ServiceAccount: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (Ingress::API_VERSION, Ingress::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(ingress) => { + $ingresses.push(ingress); + } + Err(e) => { + error!( + "Cannot parse {:?} as Ingress: {e}", + $dyn_obj.metadata.name + ); + } + } + } + (NetworkPolicy::API_VERSION, NetworkPolicy::KIND) => { + match $dyn_obj.clone().try_parse::() { + Ok(policy) => { + $policies.push(policy); + } + Err(e) => { + error!( + "Cannot parse {:?} as NetworkPolicy: {e}", + $dyn_obj.metadata.name + ); + } + } + } + ("traefik.containo.us/v1alpha1", "Middleware") => { + match $dyn_obj.clone().try_parse::() { + Ok(middleware) => { + $traefik_middlewares.push(middleware); + } + Err(e) => { + error!( + "Cannot parse {:?} as Traefik middleware: {e}", + $dyn_obj.metadata.name + ); + } + } + } + ("traefik.containo.us/v1alpha1", "IngressRoute") => { + match $dyn_obj.clone().try_parse::() { + Ok(ingress) => { + $traefik_ingresses.push(ingress); + } + Err(e) => { + error!( + "Cannot parse {:?} as Traefik ingress route: {e}", + $dyn_obj.metadata.name + ); + } + } + } + _ => { + warn!( + "Cannot parse {name} ({api_version}, {kind}) for {app_name} because its kind is unknown", + api_version = $api_version, + kind = $kind, + app_name = $app_name, + name = $dyn_obj.metadata.name.unwrap_or_default() + ); + } + } + }; +} + impl K8sDeploymentUnit { async fn start_bootstrapping_pods( app_name: &AppName, @@ -273,6 +522,8 @@ impl K8sDeploymentUnit { let mut service_accounts = Vec::new(); let mut ingresses = Vec::new(); let mut policies = Vec::new(); + let mut traefik_ingresses = Vec::new(); + let mut traefik_middlewares = Vec::new(); for mut log_stream in log_streams.into_iter() { let mut stdout = String::new(); @@ -306,220 +557,42 @@ impl K8sDeploymentUnit { if log::log_enabled!(log::Level::Trace) { trace!( "Parsed {} ({api_version}, {kind}) for {app_name} as a bootstrap application element.", - dy.metadata - .name - .as_deref() - .unwrap_or_default(), + dy.metadata.name.as_deref().unwrap_or_default(), ); } - match (api_version, kind) { - (Role::API_VERSION, Role::KIND) => match dy.clone().try_parse::() - { - Ok(role) => { - roles.push(role); - } - Err(e) => { - error!("Cannot parse {:?} as Role: {e}", dy.metadata.name); - } - }, - - (RoleBinding::API_VERSION, RoleBinding::KIND) => { - match dy.clone().try_parse::() { - Ok(role_binding) => { - role_bindings.push(role_binding); - } - Err(e) => { - error!( - "Cannot parse {:?} as RoleBinding: {e}", - dy.metadata.name - ); - } - } - } - (StatefulSet::API_VERSION, StatefulSet::KIND) => { - match dy.clone().try_parse::() { - Ok(stateful_set) => { - stateful_sets.push(stateful_set); - } - Err(e) => { - error!( - "Cannot parse {:?} as StatefulSet: {e}", - dy.metadata.name - ); - } - } - } - (ConfigMap::API_VERSION, ConfigMap::KIND) => { - match dy.clone().try_parse::() { - Ok(config_map) => { - config_maps.push(config_map); - } - Err(e) => { - error!( - "Cannot parse {:?} as ConfigMap: {e}", - dy.metadata.name - ); - } - } - } - (Secret::API_VERSION, Secret::KIND) => { - if let serde_json::Value::Object(obj) = &mut dy.data { - obj.entry("data").and_modify(|obj| { - if let serde_json::Value::Object(obj) = obj { - for (_k, v) in obj.iter_mut() { - if let serde_json::Value::String(str) = v { - // replacing new lines here because it is assumed - // that the data is base64 encoded and thus there - // must be no new lines - *v = str.replace('\n', "").into(); - } - } - } - }); - } - - match dy.clone().try_parse::() { - Ok(secret) => { - secrets.push(secret); - } - Err(e) => { - error!( - "Cannot parse {:?} as Secret: {e}", - dy.metadata.name - ); - } - } - } - (PersistentVolumeClaim::API_VERSION, PersistentVolumeClaim::KIND) => { - match dy.clone().try_parse::() { - Ok(pvc) => { - pvcs.push(pvc); - } - Err(e) => { - error!( - "Cannot parse {:?} as PersistentVolumeClaim: {e}", - dy.metadata.name - ); - } - } - } - (Service::API_VERSION, Service::KIND) => { - match dy.clone().try_parse::() { - Ok(service) => { - services.push(service); - } - Err(e) => { - error!( - "Cannot parse {:?} as Service: {e}", - dy.metadata.name - ); - } - } - } - (Deployment::API_VERSION, Deployment::KIND) => { - match dy.clone().try_parse::() { - Ok(mut deployment) => { - let service_name = deployment - .labels() - .get("app.kubernetes.io/component") - .cloned() - .unwrap_or_else(|| { - deployment.metadata.name.clone().unwrap_or_default() - }); - - deployment - .labels_mut() - .insert(SERVICE_NAME_LABEL.to_string(), service_name); - deployment.labels_mut().insert( - CONTAINER_TYPE_LABEL.to_string(), - ContainerType::ApplicationCompanion.to_string(), - ); - - deployments.push(deployment); - } - Err(e) => { - error!( - "Cannot parse {:?} as Deployment: {e}", - dy.metadata.name - ); - } - } - } - (Pod::API_VERSION, Pod::KIND) => match dy.clone().try_parse::() { - Ok(pod) => { - pods.push(pod); - } - Err(e) => { - error!("Cannot parse {:?} as Pod: {e}", dy.metadata.name); - } - }, - (Job::API_VERSION, Job::KIND) => match dy.clone().try_parse::() { - Ok(job) => { - jobs.push(job); - } - Err(e) => { - error!("Cannot parse {:?} as Job: {e}", dy.metadata.name); - } - }, - (ServiceAccount::API_VERSION, ServiceAccount::KIND) => { - match dy.clone().try_parse::() { - Ok(service_account) => { - service_accounts.push(service_account); - } - Err(e) => { - error!( - "Cannot parse {:?} as ServiceAccount: {e}", - dy.metadata.name - ); - } - } - } - (Ingress::API_VERSION, Ingress::KIND) => { - match dy.clone().try_parse::() { - Ok(ingress) => { - ingresses.push(ingress); - } - Err(e) => { - error!( - "Cannot parse {:?} as Ingress: {e}", - dy.metadata.name - ); - } - } - } - (NetworkPolicy::API_VERSION, NetworkPolicy::KIND) => { - match dy.clone().try_parse::() { - Ok(policy) => { - policies.push(policy); - } - Err(e) => { - error!( - "Cannot parse {:?} as NetworkPolicy: {e}", - dy.metadata.name - ); - } - } - } - _ => { - warn!( - "Cannot parse {name} ({api_version}, {kind}) for {app_name} because its kind is unknown", - name=dy.metadata.name.unwrap_or_default() - ); - } - } + parse_from_dynamic_object!( + roles, + role_bindings, + stateful_sets, + config_maps, + secrets, + pvcs, + services, + pods, + deployments, + jobs, + service_accounts, + ingresses, + policies, + traefik_ingresses, + traefik_middlewares, + api_version, + kind, + app_name, + dy + ); } Err(err) => { - warn!("The output of a bootstrap container for {app_name} could not be parsed: {stdout}"); + warn!( + "The output of a bootstrap container for {app_name} could not be parsed: {stdout}" + ); return Err(err.into()); } } } } - let mut traefik_ingresses = Vec::new(); - let mut traefik_middlewares = Vec::new(); - for ingress in ingresses { let (route, middlewares) = match convert_k8s_ingress_to_traefik_ingress( ingress, @@ -528,7 +601,10 @@ impl K8sDeploymentUnit { ) { Ok((route, middlewares)) => (route, middlewares), Err((ingress, err)) => { - warn!("Cannot convert K8s ingress to Traefik ingress and middlewares for {app_name}: {err} ({})", serde_json::to_string(&ingress).unwrap()); + warn!( + "Cannot convert K8s ingress to Traefik ingress and middlewares for {app_name}: {err} ({})", + serde_json::to_string(&ingress).unwrap() + ); continue; } }; @@ -798,6 +874,101 @@ impl K8sDeploymentUnit { } } + pub(super) async fn fetch(client: Client, app_name: &AppName) -> Result { + let mut roles = Vec::new(); + let mut role_bindings = Vec::new(); + let mut stateful_sets = Vec::new(); + let mut config_maps = Vec::new(); + let mut secrets = Vec::new(); + let mut pvcs = Vec::new(); + let mut services = Vec::new(); + let mut pods = Vec::new(); + let mut deployments = Vec::new(); + let mut jobs = Vec::new(); + let mut service_accounts = Vec::new(); + let mut traefik_ingresses = Vec::new(); + let mut traefik_middlewares = Vec::new(); + let mut policies = Vec::new(); + + let namespace = app_name.to_rfc1123_namespace_id(); + + let api = Api::::namespaced(client, &namespace); + roles.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + role_bindings.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + stateful_sets.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + config_maps.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + secrets.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + pvcs.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + services.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + deployments.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + pods.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + jobs.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + service_accounts.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + traefik_ingresses.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + traefik_middlewares.extend(api.list(&Default::default()).await?.items); + + let api = Api::::namespaced(api.into_client(), &namespace); + policies.extend(api.list(&Default::default()).await?.items); + + Ok(Self { + roles, + role_bindings, + stateful_sets, + config_maps, + secrets, + pvcs, + services, + pods, + deployments, + jobs, + service_accounts, + policies, + traefik_ingresses, + traefik_middlewares, + }) + } + + pub fn is_empty(&self) -> bool { + self.roles.is_empty() + && self.role_bindings.is_empty() + && self.stateful_sets.is_empty() + && self.config_maps.is_empty() + && self.secrets.is_empty() + && self.pvcs.is_empty() + && self.services.is_empty() + && self.pods.is_empty() + && self.deployments.is_empty() + && self.jobs.is_empty() + && self.service_accounts.is_empty() + && self.policies.is_empty() + && self.traefik_ingresses.is_empty() + && self.traefik_middlewares.is_empty() + } + pub(super) async fn deploy( self, client: Client, @@ -851,6 +1022,338 @@ impl K8sDeploymentUnit { Ok(deployments) } + + pub(super) async fn delete(self, client: Client, app_name: &AppName) -> Result<()> { + let namespace = app_name.to_rfc1123_namespace_id(); + + let api = Api::::namespaced(client, &namespace); + for role in self.roles { + api.delete( + role.metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for role_binding in self.role_bindings { + api.delete( + role_binding + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for config_map in self.config_maps { + api.delete( + config_map + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for secret in self.secrets { + api.delete( + secret + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for pvc in self.pvcs { + api.delete( + pvc.metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for service in self.services { + api.delete( + service + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for service_account in self.service_accounts { + api.delete( + service_account + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for policy in self.policies { + api.delete( + policy + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for deployment in self.deployments { + api.delete( + deployment + .metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for job in self.jobs { + api.delete( + job.metadata() + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for stateful_set in self.stateful_sets { + api.delete( + stateful_set + .metadata + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for ingress in self.traefik_ingresses { + api.delete( + ingress + .metadata + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for middleware in self.traefik_middlewares { + api.delete( + middleware + .metadata + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + let api = Api::::namespaced(api.into_client(), &namespace); + for pod in self.pods { + api.delete( + pod.metadata + .name + .as_ref() + .map(|n| n.as_str()) + .unwrap_or_default(), + &Default::default(), + ) + .await?; + } + + Ok(()) + } + + pub fn prepare_for_back_up(mut self) -> Self { + self.pvcs.clear(); + self.secrets.clear(); + self.config_maps.clear(); + // TODO more??? + self + } + + pub fn parse_from_json<'a, I>(app_name: &AppName, payload: I) -> Result + where + I: IntoIterator, + ::Item: serde::de::Deserializer<'a>, + { + let mut roles = Vec::new(); + let mut role_bindings = Vec::new(); + let mut stateful_sets = Vec::new(); + let mut config_maps = Vec::new(); + let mut secrets = Vec::new(); + let mut pvcs = Vec::new(); + let mut services = Vec::new(); + let mut pods = Vec::new(); + let mut deployments = Vec::new(); + let mut jobs = Vec::new(); + let mut service_accounts = Vec::new(); + let mut ingresses = Vec::new(); + let mut policies = Vec::new(); + let mut traefik_middlewares = Vec::new(); + let mut traefik_ingresses = Vec::new(); + + for pa in payload.into_iter() { + match DynamicObject::deserialize(pa) { + Ok(mut dyn_obj) => { + let api_version = dyn_obj + .types + .as_ref() + .map(|t| t.api_version.as_str()) + .unwrap_or_default(); + let kind = dyn_obj + .types + .as_ref() + .map(|t| t.kind.as_str()) + .unwrap_or_default(); + + parse_from_dynamic_object!( + roles, + role_bindings, + stateful_sets, + config_maps, + secrets, + pvcs, + services, + pods, + deployments, + jobs, + service_accounts, + ingresses, + policies, + traefik_ingresses, + traefik_middlewares, + api_version, + kind, + app_name, + dyn_obj + ); + } + Err(_) => todo!(), + } + } + + Ok(Self { + roles, + role_bindings, + stateful_sets, + config_maps, + secrets, + pvcs, + services, + pods, + deployments, + jobs, + service_accounts, + policies, + traefik_ingresses, + traefik_middlewares, + }) + } + + pub fn to_json_vec(&self) -> Vec { + // TODO right capacity + let mut json = Vec::with_capacity(self.deployments.len()); + + for role in self.roles.iter() { + json.push(serde_json::to_value(role).unwrap()); + } + for config_map in self.config_maps.iter() { + json.push(serde_json::to_value(config_map).unwrap()); + } + for secret in self.secrets.iter() { + json.push(serde_json::to_value(secret).unwrap()); + } + for pvc in self.pvcs.iter() { + json.push(serde_json::to_value(pvc).unwrap()); + } + for service in self.services.iter() { + json.push(serde_json::to_value(service).unwrap()); + } + for service_account in self.service_accounts.iter() { + json.push(serde_json::to_value(service_account).unwrap()); + } + for policy in self.policies.iter() { + json.push(serde_json::to_value(policy).unwrap()); + } + for deployment in self.deployments.iter() { + json.push(serde_json::to_value(deployment).unwrap()); + } + for job in self.jobs.iter() { + json.push(serde_json::to_value(job).unwrap()); + } + for stateful_set in self.stateful_sets.iter() { + json.push(serde_json::to_value(stateful_set).unwrap()); + } + for ingress in self.traefik_ingresses.iter() { + json.push(serde_json::to_value(ingress).unwrap()); + } + for middleware in self.traefik_middlewares.iter() { + json.push(serde_json::to_value(middleware).unwrap()); + } + for pod in self.pods.iter() { + json.push(serde_json::to_value(pod).unwrap()); + } + + json + } } async fn create_or_patch(client: Client, app_name: &AppName, payload: T) -> Result diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index 26795229..24cf52a0 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -532,6 +532,20 @@ impl Infrastructure for KubernetesInfrastructure { Ok(Some(App::new(services, owners, udp))) } + async fn fetch_app_as_backup_based_infrastructure_payload( + &self, + app_name: &AppName, + ) -> Result>> { + let client = self.client().await?; + + let unit = K8sDeploymentUnit::fetch(client, app_name).await?; + if unit.is_empty() { + return Ok(None); + } + + Ok(Some(unit.prepare_for_back_up().to_json_vec())) + } + async fn fetch_app_names(&self) -> Result> { let client = self.client().await?; Ok(Api::::all(client) @@ -647,6 +661,17 @@ impl Infrastructure for KubernetesInfrastructure { Ok(services) } + async fn delete_infrastructure_objects_partially( + &self, + app_name: &AppName, + infrastructure_payload: &[serde_json::Value], + ) -> Result<()> { + let unit = K8sDeploymentUnit::parse_from_json(app_name, infrastructure_payload)?; + unit.prepare_for_back_up() + .delete(self.client().await?, app_name) + .await + } + async fn get_logs<'a>( &'a self, app_name: &'a AppName, From 301764a2a6cb177207b7db109382bb730e977a3d Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Tue, 23 Dec 2025 08:55:12 +0100 Subject: [PATCH 2/6] Restore Kubernetes Manifests of Application This changes restores the partial backup of the Kubernetes manifests. Additionally, his change provides some code improvements and tests for backing-up and restoring applications. --- api/src/apps/mod.rs | 13 + api/src/apps/queue/mod.rs | 44 +- api/src/apps/queue/postgres.rs | 28 +- api/src/apps/queue/task.rs | 9 +- api/src/apps/repository.rs | 60 + api/src/apps/routes/get_apps.rs | 4 +- api/src/apps/routes/mod.rs | 92 +- api/src/infrastructure/infrastructure.rs | 8 + .../kubernetes/deployment_unit.rs | 1093 +++++++++++++++-- .../kubernetes/infrastructure.rs | 10 + api/src/main.rs | 3 +- api/src/models/app.rs | 25 + api/src/models/mod.rs | 4 +- 13 files changed, 1251 insertions(+), 142 deletions(-) create mode 100644 api/src/apps/repository.rs diff --git a/api/src/apps/mod.rs b/api/src/apps/mod.rs index 5eb95dea..a273b4e3 100644 --- a/api/src/apps/mod.rs +++ b/api/src/apps/mod.rs @@ -26,6 +26,7 @@ mod fairing; mod host_meta_cache; mod queue; +mod repository; mod routes; use crate::config::ReplicateApplicationCondition; @@ -51,6 +52,7 @@ use log::error; use log::trace; pub use queue::AppProcessingQueue; pub use queue::AppTaskQueueProducer; +pub use repository::AppRepository; pub use routes::{apps_routes, delete_app_sync, AppV1}; use std::collections::{HashMap, HashSet}; use std::convert::From; @@ -373,6 +375,17 @@ impl Apps { Ok(apps) } + async fn restore_app_partially( + &self, + app_name: &AppName, + infrastructure_payload: &[serde_json::Value], + ) -> Result { + Ok(self + .infrastructure + .restore_infrastructure_objects_partially(app_name, infrastructure_payload) + .await?) + } + /// Deletes all services for the given `app_name`. async fn delete_app(&self, app_name: &AppName) -> Result { let app = self.infrastructure.stop_services(app_name).await?; diff --git a/api/src/apps/queue/mod.rs b/api/src/apps/queue/mod.rs index 361ec594..e8d241af 100644 --- a/api/src/apps/queue/mod.rs +++ b/api/src/apps/queue/mod.rs @@ -146,7 +146,7 @@ impl AppTaskQueueProducer { .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { status_id, app_name, - infrastructure_payload, + infrastructure_payload_to_back_up: infrastructure_payload, }) .await?; @@ -158,6 +158,28 @@ impl AppTaskQueueProducer { Ok(status_id) } + pub async fn enqueue_restore_task( + &self, + app_name: AppName, + infrastructure_payload: Vec, + ) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id, + app_name, + infrastructure_payload_to_restore: infrastructure_payload, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new restore task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + pub async fn try_wait_for_task( &self, status_id: &AppStatusChangeId, @@ -223,7 +245,7 @@ impl AppTaskQueueConsumer { match &task { AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { app_name, - infrastructure_payload, + infrastructure_payload_to_back_up, .. } => { if log::log_enabled!(log::Level::Debug) { @@ -233,10 +255,26 @@ impl AppTaskQueueConsumer { } let result = apps - .delete_app_partially(app_name, infrastructure_payload) + .delete_app_partially(app_name, infrastructure_payload_to_back_up) .await; (task, result) } + AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + app_name, + infrastructure_payload_to_restore, + .. + } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!( + "Restoring infrastructure objects for {app_name} due to restore task." + ); + } + + let result = apps + .restore_app_partially(app_name, infrastructure_payload_to_restore) + .await; + (task, result) + }, AppTask::CreateOrUpdate { app_name, replicate_from, diff --git a/api/src/apps/queue/postgres.rs b/api/src/apps/queue/postgres.rs index cfea79d3..26136eb6 100644 --- a/api/src/apps/queue/postgres.rs +++ b/api/src/apps/queue/postgres.rs @@ -217,13 +217,13 @@ impl PostgresAppTaskQueueDB { .execute(&mut *tx) .await?; - if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { - app_name, - infrastructure_payload, - .. - } = tasked_worked_on - { - if is_success { + if is_success { + if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + infrastructure_payload_to_back_up, + .. + } = tasked_worked_on + { sqlx::query( r#" INSERT INTO app_backup (app_name, app, infrastructure_payload) @@ -232,7 +232,19 @@ impl PostgresAppTaskQueueDB { ) .bind(app_name.as_str()) .bind(success_result) - .bind(serde_json::Value::Array(infrastructure_payload)) + .bind(serde_json::Value::Array(infrastructure_payload_to_back_up)) + .execute(&mut *tx) + .await?; + } else if let AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } = + tasked_worked_on + { + sqlx::query( + r#" + DELETE FROM app_backup + WHERE app_name = $1; + "#, + ) + .bind(app_name.as_str()) .execute(&mut *tx) .await?; } diff --git a/api/src/apps/queue/task.rs b/api/src/apps/queue/task.rs index 918eec68..a40c9536 100644 --- a/api/src/apps/queue/task.rs +++ b/api/src/apps/queue/task.rs @@ -10,7 +10,12 @@ pub(super) enum AppTask { MovePayloadToBackUpAndDeleteFromInfrastructure { status_id: AppStatusChangeId, app_name: AppName, - infrastructure_payload: Vec, + infrastructure_payload_to_back_up: Vec, + }, + RestoreOnInfrastructureAndDeleteFromBackup { + status_id: AppStatusChangeId, + app_name: AppName, + infrastructure_payload_to_restore: Vec, }, CreateOrUpdate { app_name: AppName, @@ -32,6 +37,7 @@ impl AppTask { AppTask::CreateOrUpdate { app_name, .. } => app_name, AppTask::Delete { app_name, .. } => app_name, AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { app_name, .. } => app_name, + AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } => app_name, } } pub fn status_id(&self) -> &AppStatusChangeId { @@ -39,6 +45,7 @@ impl AppTask { AppTask::CreateOrUpdate { status_id, .. } => status_id, AppTask::Delete { status_id, .. } => status_id, AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { status_id, .. } => status_id, + AppTask::RestoreOnInfrastructureAndDeleteFromBackup { status_id, .. } => status_id, } } diff --git a/api/src/apps/repository.rs b/api/src/apps/repository.rs new file mode 100644 index 00000000..b8a26d8d --- /dev/null +++ b/api/src/apps/repository.rs @@ -0,0 +1,60 @@ +use crate::models::AppName; +use anyhow::Result; +use rocket::{ + fairing::{Fairing, Info, Kind}, + Build, Rocket, +}; +use sqlx::PgPool; + +pub struct AppRepository {} + +impl AppRepository { + pub fn fairing() -> Self { + Self {} + } +} + +#[rocket::async_trait] +impl Fairing for AppRepository { + fn info(&self) -> Info { + Info { + name: "app-repository", + kind: Kind::Ignite | Kind::Liftoff, + } + } + + async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { + let repository = rocket + .state::() + .map(|pool| AppPostgresRepository { pool: pool.clone() }); + Ok(rocket.manage(repository)) + } +} + +pub struct AppPostgresRepository { + pool: PgPool, +} + +impl AppPostgresRepository { + pub async fn fetch_backup(&self, app_name: &AppName) -> Result>> { + let mut connection = self.pool.acquire().await?; + + let result = sqlx::query_as::<_, (sqlx::types::Json,)>( + r#" + SELECT infrastructure_payload + FROM app_backup + WHERE app_name = $1 + "#, + ) + .bind(app_name.as_str()) + .fetch_optional(&mut *connection) + .await?; + + Ok(result.map( + |mut infrastructure_payload| match infrastructure_payload.0.take() { + serde_json::Value::Array(values) => values, + v => vec![v], + }, + )) + } +} diff --git a/api/src/apps/routes/get_apps.rs b/api/src/apps/routes/get_apps.rs index 5d5ed72a..24ea1138 100644 --- a/api/src/apps/routes/get_apps.rs +++ b/api/src/apps/routes/get_apps.rs @@ -348,6 +348,7 @@ mod tests { mod url_rendering { use super::apps_v1; + use crate::apps::repository::AppPostgresRepository; use crate::apps::{AppProcessingQueue, Apps, HostMetaCache}; use crate::config::Config; use crate::infrastructure::Dummy; @@ -370,7 +371,7 @@ mod tests { .create_or_update( &AppName::master(), None, - &vec![sc!("service-a")], + &[sc!("service-a")], vec![], None, ) @@ -380,6 +381,7 @@ mod tests { .manage(host_meta_cache) .manage(apps) .manage(Config::default()) + .manage(None::) .manage(tokio::sync::watch::channel::>(HashMap::new()).1) .mount("/", rocket::routes![apps_v1]) .mount("/api/apps", crate::apps::apps_routes()) diff --git a/api/src/apps/routes/mod.rs b/api/src/apps/routes/mod.rs index bc796970..66d4eacc 100644 --- a/api/src/apps/routes/mod.rs +++ b/api/src/apps/routes/mod.rs @@ -25,14 +25,15 @@ */ use super::queue::AppTaskQueueProducer; +use crate::apps::repository::AppPostgresRepository; use crate::apps::{Apps, AppsError}; use crate::auth::UserValidatedByAccessMode; use crate::config::Config; use crate::deployment::hooks::Hooks; use crate::http_result::{HttpApiError, HttpResult}; use crate::models::{ - App, AppName, AppNameError, AppStatusChangeId, AppStatusChangeIdError, Owner, Service, - ServiceStatus, + App, AppName, AppNameError, AppStatus, AppStatusChangeId, AppStatusChangeIdError, Owner, + Service, ServiceStatus, }; use create_app_payload::CreateAppPayload; use http_api_problem::{HttpApiProblem, StatusCode}; @@ -55,21 +56,21 @@ mod static_openapi_spec; pub fn apps_routes() -> Vec { rocket::routes![ + change_app_status, + change_status, + create_app_v1, + create_app_v2, + delete_app_v1, + delete_app_v2, get_apps::apps_v1, get_apps::apps_v2, get_apps::stream_apps_v1, get_apps::stream_apps_v2, - delete_app_v1, - delete_app_v2, - create_app_v1, - create_app_v2, - back_up_app, logs::logs, logs::stream_logs, - change_status, + static_openapi_spec::static_open_api_spec, status_change_v1, status_change_v2, - static_openapi_spec::static_open_api_spec, ] } @@ -309,37 +310,70 @@ pub async fn create_app_v2( .await } -#[rocket::put( - "//states", - format = "application/json", - // TODO payload data = "", -)] -pub async fn back_up_app( +#[derive(Deserialize)] +pub struct AppStatesInput { + status: AppStatus, +} + +#[rocket::put("//states", format = "application/json", data = "")] +pub async fn change_app_status( app_name: Result, apps: &State, app_queue: &State, + app_repository: &State>, user: Result, options: WaitForQueueOptions, + payload: Json, ) -> HttpResult>> { // TODO: authorization hook to verify e.g. if a user is member of a GitLab group let _user = user.map_err(HttpApiError::from)?; - let app_name = app_name?; - let Some(infrastructure_payload) = apps - .fetch_app_as_backup_based_infrastructure_payload(&app_name) - .await? - else { - // TODO check if already backed up… - return Err(AppsError::AppNotFound { app_name }.into()); + let Some(app_repository) = &**app_repository else { + return Err( + HttpApiProblem::with_title_and_type(StatusCode::PRECONDITION_REQUIRED) + .detail("There is no database configured. This API is only available if there is a database configuration.") + .into(), + ); }; - let status_id = app_queue - .enqueue_backup_task(app_name.clone(), infrastructure_payload) - .await - .map_err(|e| { - HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) - .detail(e.to_string()) - })?; + let app_name = app_name?; + + let status_id = match payload.status { + AppStatus::Deployed => { + let Some(infrastructure_payload) = + app_repository.fetch_backup(&app_name).await.map_err(|e| { + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()) + })? + else { + return Err(AppsError::AppNotFound { app_name }.into()); + }; + + app_queue + .enqueue_restore_task(app_name.clone(), infrastructure_payload) + .await + .map_err(|e| { + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()) + })? + } + AppStatus::BackedUp => { + let Some(infrastructure_payload) = apps + .fetch_app_as_backup_based_infrastructure_payload(&app_name) + .await? + else { + return Err(AppsError::AppNotFound { app_name }.into()); + }; + + app_queue + .enqueue_backup_task(app_name.clone(), infrastructure_payload) + .await + .map_err(|e| { + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()) + })? + } + }; try_wait_for_task(app_queue, app_name, status_id, options, AppV2).await } diff --git a/api/src/infrastructure/infrastructure.rs b/api/src/infrastructure/infrastructure.rs index 8c98724e..96fd8b57 100644 --- a/api/src/infrastructure/infrastructure.rs +++ b/api/src/infrastructure/infrastructure.rs @@ -73,6 +73,14 @@ pub trait Infrastructure: Send + Sync + DynClone { /// stopped. async fn stop_services(&self, app_name: &AppName) -> Result; + async fn restore_infrastructure_objects_partially( + &self, + app_name: &AppName, + infrastructure: &[serde_json::Value], + ) -> Result { + anyhow::bail!("Cannot restore {app_name}: not yet implemented for the configured backend to restore {infrastructure:?}") + } + async fn delete_infrastructure_objects_partially( &self, app_name: &AppName, diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs index 4ac8f3fd..de0294a1 100644 --- a/api/src/infrastructure/kubernetes/deployment_unit.rs +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -196,13 +196,10 @@ macro_rules! parse_from_dynamic_object { deployment.metadata.name.clone().unwrap_or_default() }); - deployment - .labels_mut() - .insert(SERVICE_NAME_LABEL.to_string(), service_name); - deployment.labels_mut().insert( - CONTAINER_TYPE_LABEL.to_string(), - ContainerType::ApplicationCompanion.to_string(), - ); + deployment.labels_mut().entry(SERVICE_NAME_LABEL.to_string()) + .or_insert(service_name); + deployment.labels_mut().entry(CONTAINER_TYPE_LABEL.to_string()) + .or_insert(ContainerType::ApplicationCompanion.to_string()); $deployments.push(deployment); } @@ -308,6 +305,35 @@ macro_rules! parse_from_dynamic_object { }; } +macro_rules! empty_read_only_fields { + ($field:expr) => { + for meta in $field.iter_mut().map(|manifest| &mut manifest.metadata) { + meta.creation_timestamp = None; + meta.deletion_grace_period_seconds = None; + meta.deletion_timestamp = None; + meta.generation = None; + meta.resource_version = None; + meta.uid = None; + } + }; + ($field:expr, $( $additional_field:ident ),* ) => { + empty_read_only_fields!($field); + + for manifest in $field.iter_mut() { + $( manifest.$additional_field = None; )* + } + }; + ($field:expr, $( $additional_field:ident ),* (spec => $( $additional_field_in_spec:ident ),*)) => { + empty_read_only_fields!($field, $( $additional_field )*); + + for manifest in $field.iter_mut() { + if let Some(spec) = manifest.spec.as_mut() { + $( spec.$additional_field_in_spec = None; )* + } + } + } +} + impl K8sDeploymentUnit { async fn start_bootstrapping_pods( app_name: &AppName, @@ -435,7 +461,7 @@ impl K8sDeploymentUnit { loop { tokio::select! { _ = interval_timer.tick() => { - let pod = api.get_status(&pod_name).await?; + let pod = api.get_status(pod_name).await?; if let Some(phase) = pod.status.and_then(|status| status.phase) { match phase.as_str() { @@ -1029,11 +1055,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(client, &namespace); for role in self.roles { api.delete( - role.metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + role.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1042,12 +1064,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for role_binding in self.role_bindings { api.delete( - role_binding - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + role_binding.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1056,12 +1073,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for config_map in self.config_maps { api.delete( - config_map - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + config_map.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1070,12 +1082,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for secret in self.secrets { api.delete( - secret - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + secret.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1084,11 +1091,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for pvc in self.pvcs { api.delete( - pvc.metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + pvc.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1097,12 +1100,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for service in self.services { api.delete( - service - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + service.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1114,8 +1112,7 @@ impl K8sDeploymentUnit { service_account .metadata() .name - .as_ref() - .map(|n| n.as_str()) + .as_deref() .unwrap_or_default(), &Default::default(), ) @@ -1125,12 +1122,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for policy in self.policies { api.delete( - policy - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + policy.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1139,12 +1131,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for deployment in self.deployments { api.delete( - deployment - .metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + deployment.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1153,11 +1140,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for job in self.jobs { api.delete( - job.metadata() - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + job.metadata().name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1166,12 +1149,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for stateful_set in self.stateful_sets { api.delete( - stateful_set - .metadata - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + stateful_set.metadata.name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1180,12 +1158,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for ingress in self.traefik_ingresses { api.delete( - ingress - .metadata - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + ingress.metadata.name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1194,12 +1167,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for middleware in self.traefik_middlewares { api.delete( - middleware - .metadata - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + middleware.metadata.name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1208,11 +1176,7 @@ impl K8sDeploymentUnit { let api = Api::::namespaced(api.into_client(), &namespace); for pod in self.pods { api.delete( - pod.metadata - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default(), + pod.metadata.name.as_deref().unwrap_or_default(), &Default::default(), ) .await?; @@ -1222,10 +1186,44 @@ impl K8sDeploymentUnit { } pub fn prepare_for_back_up(mut self) -> Self { - self.pvcs.clear(); - self.secrets.clear(); + self.pods.retain(|pod| { + self.deployments.iter().any(|deployment| { + let Some(spec) = deployment.spec.as_ref() else { + return false; + }; + let Some(matches_labels) = spec.selector.match_labels.as_ref() else { + return false; + }; + + pod.metadata + .labels + .as_ref() + .map(|labels| labels.iter().all(|(k, v)| matches_labels.get(k) == Some(v))) + .unwrap_or(false) + }) + }); + + empty_read_only_fields!(self.roles); + empty_read_only_fields!(self.role_bindings); + + empty_read_only_fields!(self.stateful_sets, status); + self.config_maps.clear(); - // TODO more??? + self.secrets.clear(); + self.pvcs.clear(); + + empty_read_only_fields!(self.services, + status (spec => cluster_ip, cluster_ips) + ); + empty_read_only_fields!(self.pods, status); + empty_read_only_fields!(self.deployments, status); + + empty_read_only_fields!(self.jobs); + empty_read_only_fields!(self.service_accounts); + empty_read_only_fields!(self.policies); + empty_read_only_fields!(self.traefik_middlewares); + empty_read_only_fields!(self.traefik_ingresses); + self } @@ -1286,7 +1284,7 @@ impl K8sDeploymentUnit { dyn_obj ); } - Err(_) => todo!(), + Err(err) => anyhow::bail!("{err}"), } } @@ -1309,8 +1307,21 @@ impl K8sDeploymentUnit { } pub fn to_json_vec(&self) -> Vec { - // TODO right capacity - let mut json = Vec::with_capacity(self.deployments.len()); + let mut json = Vec::with_capacity( + self.roles.len() + + self.config_maps.len() + + self.secrets.len() + + self.pvcs.len() + + self.services.len() + + self.service_accounts.len() + + self.policies.len() + + self.deployments.len() + + self.jobs.len() + + self.stateful_sets.len() + + self.traefik_ingresses.len() + + self.traefik_middlewares.len() + + self.pods.len(), + ); for role in self.roles.iter() { json.push(serde_json::to_value(role).unwrap()); @@ -1393,6 +1404,7 @@ where mod tests { use super::*; use crate::{deployment::deployment_unit::DeploymentUnitBuilder, models::State}; + use assert_json_diff::assert_json_include; use chrono::Utc; use k8s_openapi::api::{ apps::v1::DeploymentSpec, @@ -1400,7 +1412,7 @@ mod tests { }; use std::collections::HashMap; - async fn parse_unit(stdout: &'static str) -> K8sDeploymentUnit { + async fn parse_unit_from_log_stream(stdout: &'static str) -> K8sDeploymentUnit { let log_streams = vec![stdout.as_bytes()]; let deployment_unit = DeploymentUnitBuilder::init(AppName::master(), Vec::new()) @@ -1427,7 +1439,7 @@ mod tests { #[tokio::test] async fn parse_unit_from_secret_stdout_where_value_is_base64_encoded() { - let unit = parse_unit( + let unit = parse_unit_from_log_stream( r#" apiVersion: v1 kind: Secret @@ -1484,7 +1496,7 @@ mod tests { #[tokio::test] async fn parse_unit_from_deployment_stdout() { - let unit = parse_unit( + let unit = parse_unit_from_log_stream( r#" apiVersion: apps/v1 kind: Deployment @@ -1554,7 +1566,7 @@ mod tests { #[tokio::test] async fn merge_deployment_into_bootstrapped_deployment() { - let mut unit = parse_unit( + let mut unit = parse_unit_from_log_stream( r#" apiVersion: apps/v1 kind: Deployment @@ -1631,7 +1643,6 @@ mod tests { }], ..Default::default() }), - ..Default::default() }, ..Default::default() }), @@ -1702,7 +1713,7 @@ mod tests { #[tokio::test] async fn filter_by_instances_and_replicas() { - let mut unit = parse_unit( + let mut unit = parse_unit_from_log_stream( r#" apiVersion: apps/v1 kind: Deployment @@ -1742,7 +1753,7 @@ mod tests { #[tokio::test] async fn filter_not_by_instances_and_replicas() { - let mut unit = parse_unit( + let mut unit = parse_unit_from_log_stream( r#" apiVersion: apps/v1 kind: Deployment @@ -1779,4 +1790,892 @@ mod tests { assert!(!unit.deployments.is_empty()); } + + fn captured_example_from_k3s() -> Vec { + vec![ + serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "creationTimestamp": "2025-12-23T09:05:11Z", + "name": "blog", + "namespace": "test", + "resourceVersion": "869", + "uid": "a27286f6-6193-4ab7-a3a8-88f55151e625" + }, + "spec": { + "clusterIP": "10.43.226.56", + "clusterIPs": [ + "10.43.226.56" + ], + "internalTrafficPolicy": "Cluster", + "ipFamilies": [ + "IPv4" + ], + "ipFamilyPolicy": "SingleStack", + "ports": [ + { + "name": "blog", + "port": 80, + "protocol": "TCP", + "targetPort": 80 + } + ], + "selector": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "blog" + }, + "sessionAffinity": "None", + "type": "ClusterIP" + }, + "status": { + "loadBalancer": {} + } + }), + serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "creationTimestamp": "2025-12-23T09:05:11Z", + "name": "db", + "namespace": "test", + "resourceVersion": "865", + "uid": "ed8c87a2-632d-4cd5-9b0e-7d1cc6f08236" + }, + "spec": { + "clusterIP": "10.43.177.211", + "clusterIPs": [ + "10.43.177.211" + ], + "internalTrafficPolicy": "Cluster", + "ipFamilies": [ + "IPv4" + ], + "ipFamilyPolicy": "SingleStack", + "ports": [ + { + "name": "db", + "port": 3306, + "protocol": "TCP", + "targetPort": 3306 + } + ], + "selector": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "db" + }, + "sessionAffinity": "None", + "type": "ClusterIP" + }, + "status": { + "loadBalancer": {} + } + }), + serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "com.aixigo.preview.servant.image": "docker.io/library/wordpress:latest", + "com.aixigo.preview.servant.replicated-env": "{\"WORDPRESS_CONFIG_EXTRA\":{\"replicate\":true,\"templated\":true,\"value\":\"define('WP_HOME','http://localhost');\\ndefine('WP_SITEURL','http://localhost/{{application.name}}/blog');\"},\"WORDPRESS_DB_HOST\":{\"replicate\":true,\"templated\":false,\"value\":\"db\"},\"WORDPRESS_DB_NAME\":{\"replicate\":true,\"templated\":false,\"value\":\"example-database\"},\"WORDPRESS_DB_PASSWORD\":{\"replicate\":true,\"templated\":false,\"value\":\"my_cool_secret\"},\"WORDPRESS_DB_USER\":{\"replicate\":true,\"templated\":false,\"value\":\"example-user\"}}", + "deployment.kubernetes.io/revision": "1" + }, + "creationTimestamp": "2025-12-23T09:05:11Z", + "generation": 1, + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "blog" + }, + "name": "test-blog-deployment", + "namespace": "test", + "resourceVersion": "916", + "uid": "43a087ee-0368-4499-8dfc-57739afa8230" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "blog" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": "25%", + "maxUnavailable": "25%" + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "annotations": { + "date": "2025-12-23T09:05:11.886481258+00:00" + }, + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "blog" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "WORDPRESS_CONFIG_EXTRA", + "value": "define('WP_HOME','http://localhost');\ndefine('WP_SITEURL','http://localhost/test/blog');" + }, + { + "name": "WORDPRESS_DB_HOST", + "value": "db" + }, + { + "name": "WORDPRESS_DB_NAME", + "value": "example-database" + }, + { + "name": "WORDPRESS_DB_PASSWORD", + "value": "my_cool_secret" + }, + { + "name": "WORDPRESS_DB_USER", + "value": "example-user" + } + ], + "image": "docker.io/library/wordpress:latest", + "imagePullPolicy": "Always", + "name": "blog", + "ports": [ + { + "containerPort": 80, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File" + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "terminationGracePeriodSeconds": 30 + } + } + }, + "status": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "lastUpdateTime": "2025-12-23T09:05:14Z", + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2025-12-23T09:05:11Z", + "lastUpdateTime": "2025-12-23T09:05:14Z", + "message": "ReplicaSet \"test-blog-deployment-5bb689bcd9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 1, + "readyReplicas": 1, + "replicas": 1, + "updatedReplicas": 1 + } + }), + serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "com.aixigo.preview.servant.image": "docker.io/library/mariadb:latest", + "com.aixigo.preview.servant.replicated-env": "{\"MARIADB_DATABASE\":{\"replicate\":true,\"templated\":false,\"value\":\"example-database\"},\"MARIADB_PASSWORD\":{\"replicate\":true,\"templated\":false,\"value\":\"my_cool_secret\"},\"MARIADB_ROOT_PASSWORD\":{\"replicate\":true,\"templated\":false,\"value\":\"example\"},\"MARIADB_USER\":{\"replicate\":true,\"templated\":false,\"value\":\"example-user\"}}", + "deployment.kubernetes.io/revision": "1" + }, + "creationTimestamp": "2025-12-23T09:05:11Z", + "generation": 1, + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "db" + }, + "name": "test-db-deployment", + "namespace": "test", + "resourceVersion": "920", + "uid": "7cae9a70-9187-442e-9700-dce0840d25e6" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "db" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": "25%", + "maxUnavailable": "25%" + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "annotations": { + "date": "2025-12-23T09:05:11.873776450+00:00" + }, + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "db" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "MARIADB_DATABASE", + "value": "example-database" + }, + { + "name": "MARIADB_PASSWORD", + "value": "my_cool_secret" + }, + { + "name": "MARIADB_ROOT_PASSWORD", + "value": "example" + }, + { + "name": "MARIADB_USER", + "value": "example-user" + } + ], + "image": "docker.io/library/mariadb:latest", + "imagePullPolicy": "Always", + "name": "db", + "ports": [ + { + "containerPort": 3306, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File" + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "terminationGracePeriodSeconds": 30 + } + } + }, + "status": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "lastUpdateTime": "2025-12-23T09:05:14Z", + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2025-12-23T09:05:11Z", + "lastUpdateTime": "2025-12-23T09:05:14Z", + "message": "ReplicaSet \"test-db-deployment-5fcf85b44b\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 1, + "readyReplicas": 1, + "replicas": 1, + "updatedReplicas": 1 + } + }), + serde_json::json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "date": "2025-12-23T09:05:11.886481258+00:00" + }, + "creationTimestamp": "2025-12-23T09:05:11Z", + "generateName": "test-blog-deployment-5bb689bcd9-", + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "blog", + "pod-template-hash": "5bb689bcd9" + }, + "name": "test-blog-deployment-5bb689bcd9-wj557", + "namespace": "test", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "test-blog-deployment-5bb689bcd9", + "uid": "13227d25-9fc9-46bf-b199-30137d4dfbb6" + } + ], + "resourceVersion": "911", + "uid": "9730567c-8c14-4264-a790-c347d317056d" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "WORDPRESS_CONFIG_EXTRA", + "value": "define('WP_HOME','http://localhost');\ndefine('WP_SITEURL','http://localhost/test/blog');" + }, + { + "name": "WORDPRESS_DB_HOST", + "value": "db" + }, + { + "name": "WORDPRESS_DB_NAME", + "value": "example-database" + }, + { + "name": "WORDPRESS_DB_PASSWORD", + "value": "my_cool_secret" + }, + { + "name": "WORDPRESS_DB_USER", + "value": "example-user" + } + ], + "image": "docker.io/library/wordpress:latest", + "imagePullPolicy": "Always", + "name": "blog", + "ports": [ + { + "containerPort": 80, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-8cdhb", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "k3d-dash-server-0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-8cdhb", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastTransitionTime": "2025-12-23T09:05:12Z", + "status": "True", + "type": "Initialized" + }, + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "Ready" + }, + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastTransitionTime": "2025-12-23T09:05:11Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://a58e137c636cd57c0ce8c392990e59d9ad638cd577cc40317cf19beea771a25c", + "image": "docker.io/library/wordpress:latest", + "imageID": "docker.io/library/wordpress@sha256:c6c44891a684b52c0d183d9d1f182dca1e16d58711670a4d973e9625d903efb3", + "lastState": {}, + "name": "blog", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2025-12-23T09:05:13Z" + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-8cdhb", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "172.24.25.2", + "hostIPs": [ + { + "ip": "172.24.25.2" + } + ], + "phase": "Running", + "podIP": "10.42.0.12", + "podIPs": [ + { + "ip": "10.42.0.12" + } + ], + "qosClass": "BestEffort", + "startTime": "2025-12-23T09:05:12Z" + } + }), + serde_json::json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "date": "2025-12-23T09:05:11.873776450+00:00" + }, + "creationTimestamp": "2025-12-23T09:05:11Z", + "generateName": "test-db-deployment-5fcf85b44b-", + "labels": { + "com.aixigo.preview.servant.app-name": "test", + "com.aixigo.preview.servant.container-type": "instance", + "com.aixigo.preview.servant.service-name": "db", + "pod-template-hash": "5fcf85b44b" + }, + "name": "test-db-deployment-5fcf85b44b-dcfx9", + "namespace": "test", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "test-db-deployment-5fcf85b44b", + "uid": "9b2f3c50-2af2-4b22-8b26-31cc06034769" + } + ], + "resourceVersion": "915", + "uid": "4ff6e7e8-d784-4176-bd48-1cbe8bde8acf" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "MARIADB_DATABASE", + "value": "example-database" + }, + { + "name": "MARIADB_PASSWORD", + "value": "my_cool_secret" + }, + { + "name": "MARIADB_ROOT_PASSWORD", + "value": "example" + }, + { + "name": "MARIADB_USER", + "value": "example-user" + } + ], + "image": "docker.io/library/mariadb:latest", + "imagePullPolicy": "Always", + "name": "db", + "ports": [ + { + "containerPort": 3306, + "protocol": "TCP" + } + ], + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-ltqf9", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "k3d-dash-server-0", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "kube-api-access-ltqf9", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "PodReadyToStartContainers" + }, + { + "lastTransitionTime": "2025-12-23T09:05:11Z", + "status": "True", + "type": "Initialized" + }, + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "Ready" + }, + { + "lastTransitionTime": "2025-12-23T09:05:14Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastTransitionTime": "2025-12-23T09:05:11Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://5e13883138c5dad22ea0279b005678a0dafb8d80a102f1d5cf1b5b75594600d3", + "image": "docker.io/library/mariadb:latest", + "imageID": "docker.io/library/mariadb@sha256:e1bcd6f85781f4a875abefb11c4166c1d79e4237c23de597bf0df81fec225b40", + "lastState": {}, + "name": "db", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2025-12-23T09:05:13Z" + } + }, + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-ltqf9", + "readOnly": true, + "recursiveReadOnly": "Disabled" + } + ] + } + ], + "hostIP": "172.24.25.2", + "hostIPs": [ + { + "ip": "172.24.25.2" + } + ], + "phase": "Running", + "podIP": "10.42.0.13", + "podIPs": [ + { + "ip": "10.42.0.13" + } + ], + "qosClass": "BestEffort", + "startTime": "2025-12-23T09:05:11Z" + } + }), + ] + } + + #[test] + fn parse_from_json() { + let json = captured_example_from_k3s(); + + let unit = K8sDeploymentUnit::parse_from_json(&AppName::master(), json).unwrap(); + + assert_eq!(unit.roles.len(), 0); + assert_eq!(unit.role_bindings.len(), 0); + assert_eq!(unit.stateful_sets.len(), 0); + assert_eq!(unit.config_maps.len(), 0); + assert_eq!(unit.secrets.len(), 0); + assert_eq!(unit.pvcs.len(), 0); + assert_eq!(unit.services.len(), 2); + assert_eq!(unit.pods.len(), 2); + assert_eq!(unit.deployments.len(), 2); + assert_eq!(unit.jobs.len(), 0); + assert_eq!(unit.service_accounts.len(), 0); + assert_eq!(unit.policies.len(), 0); + assert_eq!(unit.traefik_ingresses.len(), 0); + assert_eq!(unit.traefik_middlewares.len(), 0); + } + + #[test] + fn to_json_vec() { + let json = captured_example_from_k3s(); + + let unit = K8sDeploymentUnit::parse_from_json(&AppName::master(), json).unwrap(); + + assert_json_include!( + actual: serde_json::Value::Array(unit.to_json_vec()), + expected: serde_json::Value::Array(captured_example_from_k3s()) + ); + } + + mod prepare_for_back_up { + use super::*; + + #[test] + fn clean_system_populations() { + let unit = + K8sDeploymentUnit::parse_from_json(&AppName::master(), captured_example_from_k3s()) + .unwrap(); + + let prepare_for_back_up = unit.prepare_for_back_up(); + + assert!( + prepare_for_back_up.pods.is_empty(), + "Pods should be empty because they are covered by deployments" + ); + assert!( + !prepare_for_back_up + .services + .iter() + .filter_map(|service| service.spec.as_ref()) + .any(|spec| spec.cluster_ip.is_some() || spec.cluster_ips.is_some()), + "Don't preserve cluster IP(s)" + ); + assert!( + !prepare_for_back_up + .deployments + .iter() + .any(|deployment| deployment.status.is_some()), + "Deployment's status will be set by server" + ); + } + + #[tokio::test] + async fn clean_volume_mounts() { + let unit = parse_unit_from_log_stream( + r#" +apiVersion: v1 +kind: Secret +metadata: + name: secret-tls +type: kubernetes.io/tls +data: + # values are base64 encoded, which obscures them but does NOT provide + # any useful level of confidentiality + tls.crt: | + LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNVakNDQWJzQ0FnMytNQTBHQ1NxR1NJYjNE + UUVCQlFVQU1JR2JNUXN3Q1FZRFZRUUdFd0pLVURFT01Bd0cKQTFVRUNCTUZWRzlyZVc4eEVEQU9C + Z05WQkFjVEIwTm9kVzh0YTNVeEVUQVBCZ05WQkFvVENFWnlZVzVyTkVSRQpNUmd3RmdZRFZRUUxF + dzlYWldKRFpYSjBJRk4xY0hCdmNuUXhHREFXQmdOVkJBTVREMFp5WVc1ck5FUkVJRmRsCllpQkRR + VEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVjM1Z3Y0c5eWRFQm1jbUZ1YXpSa1pDNWpiMjB3SGhjTk1U + TXcKTVRFeE1EUTFNVE01V2hjTk1UZ3dNVEV3TURRMU1UTTVXakJMTVFzd0NRWURWUVFHREFKS1VE + RVBNQTBHQTFVRQpDQXdHWEZSdmEzbHZNUkV3RHdZRFZRUUtEQWhHY21GdWF6UkVSREVZTUJZR0Ex + VUVBd3dQZDNkM0xtVjRZVzF3CmJHVXVZMjl0TUlHYU1BMEdDU3FHU0liM0RRRUJBUVVBQTRHSUFE + Q0JoQUo5WThFaUhmeHhNL25PbjJTbkkxWHgKRHdPdEJEVDFKRjBReTliMVlKanV2YjdjaTEwZjVN + Vm1UQllqMUZTVWZNOU1vejJDVVFZdW4yRFljV29IcFA4ZQpqSG1BUFVrNVd5cDJRN1ArMjh1bklI + QkphVGZlQ09PekZSUFY2MEdTWWUzNmFScG04L3dVVm16eGFLOGtCOWVaCmhPN3F1TjdtSWQxL2pW + cTNKODhDQXdFQUFUQU5CZ2txaGtpRzl3MEJBUVVGQUFPQmdRQU1meTQzeE15OHh3QTUKVjF2T2NS + OEtyNWNaSXdtbFhCUU8xeFEzazlxSGtyNFlUY1JxTVQ5WjVKTm1rWHYxK2VSaGcwTi9WMW5NUTRZ + RgpnWXcxbnlESnBnOTduZUV4VzQyeXVlMFlHSDYyV1hYUUhyOVNVREgrRlowVnQvRGZsdklVTWRj + UUFEZjM4aU9zCjlQbG1kb3YrcE0vNCs5a1h5aDhSUEkzZXZ6OS9NQT09Ci0tLS0tRU5EIENFUlRJ + RklDQVRFLS0tLS0K + # In this example, the key data is not a real PEM-encoded private key + tls.key: | + RXhhbXBsZSBkYXRhIGZvciB0aGUgVExTIGNydCBmaWVsZA== +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: game-demo +data: + # property-like keys; each key maps to a simple value + player_initial_lives: "3" + ui_properties_file_name: "user-interface.properties" + + # file-like keys + game.properties: | + enemy.types=aliens,monsters + player.maximum-lives=5 + user-interface.properties: | + color.good=purple + color.bad=yellow + allow.textmode=true +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: foo-pvc + namespace: foo +spec: + storageClassName: "" + volumeName: foo-pv + "#, + ) + .await; + + assert!(!unit.secrets.is_empty()); + assert!(!unit.config_maps.is_empty()); + assert!(!unit.pvcs.is_empty()); + + let prepared_for_back_up = unit.prepare_for_back_up(); + + assert!(prepared_for_back_up.secrets.is_empty()); + assert!(prepared_for_back_up.config_maps.is_empty()); + assert!(prepared_for_back_up.pvcs.is_empty()); + } + } } diff --git a/api/src/infrastructure/kubernetes/infrastructure.rs b/api/src/infrastructure/kubernetes/infrastructure.rs index 24cf52a0..16090429 100644 --- a/api/src/infrastructure/kubernetes/infrastructure.rs +++ b/api/src/infrastructure/kubernetes/infrastructure.rs @@ -672,6 +672,16 @@ impl Infrastructure for KubernetesInfrastructure { .await } + async fn restore_infrastructure_objects_partially( + &self, + app_name: &AppName, + infrastructure_payload: &[serde_json::Value], + ) -> Result { + let unit = K8sDeploymentUnit::parse_from_json(app_name, infrastructure_payload)?; + unit.deploy(self.client().await?, app_name).await?; + Ok(self.fetch_app(app_name).await?.unwrap_or_else(App::empty)) + } + async fn get_logs<'a>( &'a self, app_name: &'a AppName, diff --git a/api/src/main.rs b/api/src/main.rs index 4a53c90a..63fa5f1a 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -29,7 +29,7 @@ extern crate lazy_static; #[macro_use] extern crate serde_derive; -use crate::apps::Apps; +use crate::apps::{AppRepository, Apps}; use crate::config::{ApiAccessMode, Config, Runtime}; use crate::db::DatabasePool; use crate::infrastructure::{Docker, Infrastructure, Kubernetes}; @@ -231,6 +231,7 @@ async fn main() -> Result<(), StartUpError> { .attach(DatabasePool::fairing()) .attach(Auth::fairing()) .attach(AppProcessingQueue::fairing()) + .attach(AppRepository::fairing()) .attach(TicketsCaching::fairing()) .attach(Apps::fairing(config, infrastructure)) .launch() diff --git a/api/src/models/app.rs b/api/src/models/app.rs index 72c4260f..5bf02889 100644 --- a/api/src/models/app.rs +++ b/api/src/models/app.rs @@ -347,6 +347,31 @@ impl AppWithHostMeta { } } +#[derive(Clone, Debug, PartialEq)] +pub struct AppWithHostMetaAndStatus { + services: Vec, + owners: HashSet, + status: AppStatus, +} + +#[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq)] +pub enum AppStatus { + #[serde(rename = "deployed")] + Deployed, + #[serde(rename = "backed-up")] + BackedUp, +} + +impl From<(AppWithHostMeta, AppStatus)> for AppWithHostMetaAndStatus { + fn from(value: (AppWithHostMeta, AppStatus)) -> Self { + Self { + services: value.0.services, + owners: value.0.owners, + status: value.1, + } + } +} + #[derive(Debug, Default, Deserialize, Clone, Eq, Hash, PartialEq, Serialize)] pub enum ContainerType { #[serde(rename = "instance")] diff --git a/api/src/models/mod.rs b/api/src/models/mod.rs index ea8ddbd5..d3df1f29 100644 --- a/api/src/models/mod.rs +++ b/api/src/models/mod.rs @@ -25,8 +25,8 @@ */ pub use app::{ - App, AppWithHostMeta, ContainerType, Owner, Service, ServiceError, ServiceStatus, - ServiceWithHostMeta, State, + App, AppStatus, AppWithHostMeta, AppWithHostMetaAndStatus, ContainerType, Owner, Service, + ServiceError, ServiceStatus, ServiceWithHostMeta, State, }; pub use app_name::{AppName, AppNameError}; pub use app_status_change_id::{AppStatusChangeId, AppStatusChangeIdError}; From 4763f547070d1a5650dbfd531b761fa1a50b9d2e Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Tue, 23 Dec 2025 15:29:11 +0100 Subject: [PATCH 3/6] Fetch Backed Up via HTTP API The HTTP API provides integration with the backup and restore capability to return the list of backed up applications from the repository. --- api/src/apps/host_meta_cache.rs | 52 +- api/src/apps/mod.rs | 1 + api/src/apps/queue/mod.rs | 30 +- api/src/apps/queue/postgres.rs | 544 --------------- api/src/apps/repository.rs | 656 +++++++++++++++++- api/src/apps/routes/get_apps.rs | 207 ++++-- api/src/apps/routes/mod.rs | 131 +++- api/src/db.rs | 1 + api/src/main.rs | 2 +- api/src/models/app.rs | 17 - .../queue/task.rs => models/app_task.rs} | 2 +- api/src/models/mod.rs | 6 +- 12 files changed, 964 insertions(+), 685 deletions(-) delete mode 100644 api/src/apps/queue/postgres.rs rename api/src/{apps/queue/task.rs => models/app_task.rs} (99%) diff --git a/api/src/apps/host_meta_cache.rs b/api/src/apps/host_meta_cache.rs index ffccb84f..24d8f8cd 100644 --- a/api/src/apps/host_meta_cache.rs +++ b/api/src/apps/host_meta_cache.rs @@ -88,6 +88,40 @@ pub fn new(config: Config) -> (HostMetaCache, HostMetaCrawler) { } impl HostMetaCache { + pub fn assign_host_meta_data_for_app( + &self, + app_name: &AppName, + app: App, + request_info: &RequestInfo, + ) -> AppWithHostMeta { + let reader = self.reader_factory.handle(); + + let mut services_with_host_meta = Vec::with_capacity(app.services().len()); + + let (services, owners) = app.into_services_and_owners(); + for service in services.into_iter() { + let service_id = service.id.clone(); + let key = Key { + app_name: app_name.clone(), + service_id, + }; + + let web_host_meta = match reader.get_one(&key) { + Some(value) => value.web_host_meta.with_base_url(request_info.base_url()), + None => WebHostMeta::empty(), + }; + + services_with_host_meta.push(ServiceWithHostMeta::from_service_and_web_host_meta( + service, + web_host_meta, + request_info.base_url().clone(), + app_name, + )); + } + + AppWithHostMeta::new(services_with_host_meta, owners) + } + pub fn assign_host_meta_data( &self, apps: HashMap, @@ -319,14 +353,16 @@ impl HostMetaCrawler { running_services_without_host_meta.into_iter() { let now = Utc::now(); - debug!( - "Resolving web host meta data for app {app_name} and the services: {}.", - running_services_without_host_meta - .iter() - .map(|(_k, service)| service.service_name()) - .fold(String::new(), |a, b| a + &b + ", ") - .trim_end_matches(", ") - ); + if log::log_enabled!(log::Level::Debug) { + debug!( + "Resolving web host meta data for app {app_name} and the services: {}.", + running_services_without_host_meta + .iter() + .map(|(_k, service)| service.service_name()) + .fold(String::new(), |a, b| a + b + ", ") + .trim_end_matches(", ") + ); + } let duration_prevant_startup = now.signed_duration_since(since_timestamp); let resolved_host_meta_infos = Self::resolve_host_meta( diff --git a/api/src/apps/mod.rs b/api/src/apps/mod.rs index a273b4e3..b0d6b776 100644 --- a/api/src/apps/mod.rs +++ b/api/src/apps/mod.rs @@ -136,6 +136,7 @@ impl Apps { let infrastructure = dyn_clone::clone_box(&*self.infrastructure); let (tx, rx) = tokio::sync::watch::channel::>(HashMap::new()); + // TODO: we should return this and spawn on liftoff tokio::spawn(async move { loop { debug!("Fetching list of apps to send updates."); diff --git a/api/src/apps/queue/mod.rs b/api/src/apps/queue/mod.rs index e8d241af..b2ae6d8e 100644 --- a/api/src/apps/queue/mod.rs +++ b/api/src/apps/queue/mod.rs @@ -1,22 +1,18 @@ -use crate::apps::{queue::task::AppTask, Apps, AppsError}; -use crate::models::{App, AppName, AppStatusChangeId, Owner, ServiceConfig}; +use crate::apps::repository::AppPostgresRepository; +use crate::apps::{Apps, AppsError}; +use crate::models::{App, AppName, AppStatusChangeId, AppTask, Owner, ServiceConfig}; use anyhow::Result; use chrono::{DateTime, TimeDelta, Utc}; -use postgres::PostgresAppTaskQueueDB; use rocket::{ fairing::{Fairing, Info, Kind}, Build, Orbit, Rocket, }; -use sqlx::PgPool; use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration}; use tokio::{ sync::{Mutex, Notify}, time::{sleep, sleep_until, timeout}, }; -mod postgres; -mod task; - pub struct AppProcessingQueue {} impl AppProcessingQueue { @@ -35,9 +31,9 @@ impl Fairing for AppProcessingQueue { } async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { - let db = match rocket.state::() { - Some(pool) => AppTaskQueueDB::db(pool.clone()), - None => AppTaskQueueDB::inmemory(), + let db = match rocket.state::>() { + Some(Some(repository)) => AppTaskQueueDB::db(repository.clone()), + _ => AppTaskQueueDB::inmemory(), }; let producer = AppTaskQueueProducer { @@ -289,9 +285,9 @@ impl AppTaskQueueConsumer { let result = apps .create_or_update( - &app_name, + app_name, replicate_from.clone(), - &service_configs, + service_configs, owners.clone(), user_defined_parameters.clone(), ) @@ -302,7 +298,7 @@ impl AppTaskQueueConsumer { if log::log_enabled!(log::Level::Debug) { log::debug!("Deleting app {app_name}."); } - let result = apps.delete_app(&app_name).await; + let result = apps.delete_app(app_name).await; (task, result) } } @@ -324,7 +320,7 @@ enum AppTaskStatus { enum AppTaskQueueDB { InMemory(Mutex>), - DB(PostgresAppTaskQueueDB), + DB(AppPostgresRepository), } impl AppTaskQueueDB { @@ -332,8 +328,8 @@ impl AppTaskQueueDB { Self::InMemory(Mutex::new(VecDeque::new())) } - fn db(pool: PgPool) -> Self { - Self::DB(PostgresAppTaskQueueDB::new(pool)) + fn db(repository: AppPostgresRepository) -> Self { + Self::DB(repository) } pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { @@ -412,7 +408,7 @@ impl AppTaskQueueDB { Ok(()) } - AppTaskQueueDB::DB(db) => db.execute_tasks(f).await, + AppTaskQueueDB::DB(db) => db.update_queued_tasks_with_executor_result(f).await, } } diff --git a/api/src/apps/queue/postgres.rs b/api/src/apps/queue/postgres.rs deleted file mode 100644 index 26136eb6..00000000 --- a/api/src/apps/queue/postgres.rs +++ /dev/null @@ -1,544 +0,0 @@ -use super::AppTask; -use crate::{ - apps::AppsError, - models::{ - user_defined_parameters::UserDefinedParameters, App, AppStatusChangeId, Owner, Service, - ServiceConfig, ServiceStatus, State, - }, -}; -use anyhow::Result; -use chrono::{DateTime, Utc}; -use sqlx::PgPool; -use std::{collections::HashSet, future::Future}; - -pub struct PostgresAppTaskQueueDB { - pool: PgPool, -} - -#[derive(Serialize, Deserialize)] -struct RawApp { - services: Vec, - owner: HashSet, - user_defined_parameters: Option, -} - -impl From for App { - fn from(value: RawApp) -> Self { - Self::new( - value.services.into_iter().map(Service::from).collect(), - Owner::normalize(value.owner), - value - .user_defined_parameters - .map(|data| unsafe { UserDefinedParameters::without_validation(data) }), - ) - } -} - -#[derive(Serialize, Deserialize)] -struct RawService { - id: String, - status: ServiceStatus, - config: ServiceConfig, -} - -impl From for Service { - fn from(value: RawService) -> Self { - Self { - id: value.id, - state: State { - status: value.status, - started_at: None, - }, - config: value.config, - } - } -} - -impl PostgresAppTaskQueueDB { - pub fn new(pool: PgPool) -> Self { - Self { pool } - } - - pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { - let mut tx = self.pool.begin().await?; - - sqlx::query( - r#" - INSERT INTO app_task (id, app_name, task) - VALUES ($1, $2, $3) - "#, - ) - .bind(task.status_id().as_uuid()) - .bind(task.app_name().as_str()) - .bind(serde_json::to_value(&task).unwrap()) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(()) - } - - pub async fn peek_result( - &self, - status_id: &AppStatusChangeId, - ) -> Option> { - let mut connection = self - .pool - .acquire() - .await - .inspect_err(|err| log::error!("Cannot acquire database connection: {err}")) - .ok()?; - - sqlx::query_as::< - _, - ( - Option>, - Option>, - Option>, - Option>, - ), - >( - r#" - SELECT a.result_success, a.result_error, m.result_success, m.result_error - FROM app_task a - LEFT OUTER JOIN app_task m - ON a.executed_and_merged_with = m.id - WHERE a.id = $1 - AND a.status = 'done' - "#, - ) - .bind(status_id.as_uuid()) - .fetch_optional(&mut *connection) - .await - .inspect_err(|err| log::error!("Cannot peek result for {status_id}: {err}")) - .ok()? - .map(|(app, error, merged_app, merged_error)| { - match ( - app.map(|app| app.0), - error.map(|error| error.0), - merged_app.map(|app| app.0), - merged_error.map(|error| error.0), - ) { - (Some(app), None, None, None) => Ok(app.into()), - (None, Some(err), None, None) => Err(err), - (None, None, Some(app), None) => Ok(app.into()), - (None, None, None, Some(err)) => Err(err), - _ => unreachable!( - "There should be either a result or an error stored in the database" - ), - } - }) - } - - pub async fn execute_tasks(&self, f: F) -> Result<()> - where - F: FnOnce(Vec) -> Fut, - Fut: Future)>, - { - let mut tx = self.pool.begin().await?; - - let tasks = sqlx::query_as::<_, (sqlx::types::Uuid, sqlx::types::Json)>( - r#" - WITH eligible_tasks AS ( - SELECT id, task - FROM app_task - WHERE status = 'queued' - AND app_name = ( - SELECT app_name - FROM app_task - WHERE created_at = (SELECT min(created_at) FROM app_task WHERE status = 'queued') - AND status = 'queued' - ) - ORDER BY created_at - FOR UPDATE SKIP LOCKED - ) - UPDATE app_task - SET status = 'running' - FROM eligible_tasks - WHERE app_task.id = eligible_tasks.id - RETURNING eligible_tasks .id, eligible_tasks.task; - "#, - ) - .fetch_all(&mut *tx) - .await?; - - let tasks_to_work_on = tasks - .iter() - .map(|task_to_work_on| task_to_work_on.1 .0.clone()) - .collect::>(); - - if tasks_to_work_on.is_empty() { - return Ok(()); - } - - let (tasked_worked_on, result) = f(tasks_to_work_on).await; - let is_success = result.is_ok(); - let id = *tasked_worked_on.status_id(); - - let failed_result = result - .as_ref() - .map_or_else(|e| serde_json::to_value(e).ok(), |_| None); - let success_result = result.map_or_else( - |_| None, - |app| { - let (services, owner, user_defined_parameters) = - app.into_services_and_owners_and_user_defined_parameters(); - let raw = RawApp { - owner, - services: services - .into_iter() - .map(|service| RawService { - id: service.id, - status: service.state.status, - config: service.config, - }) - .collect(), - user_defined_parameters: user_defined_parameters.and_then( - |user_defined_parameters| { - serde_json::to_value(user_defined_parameters).ok() - }, - ), - }; - serde_json::to_value(raw).ok() - }, - ); - - sqlx::query( - r#" - UPDATE app_task - SET status = 'done', result_success = $3, result_error = $2 - WHERE id = $1 - "#, - ) - .bind(id.as_uuid()) - .bind(failed_result) - .bind(&success_result) - .execute(&mut *tx) - .await?; - - if is_success { - if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { - app_name, - infrastructure_payload_to_back_up, - .. - } = tasked_worked_on - { - sqlx::query( - r#" - INSERT INTO app_backup (app_name, app, infrastructure_payload) - VALUES ($1, $2, $3); - "#, - ) - .bind(app_name.as_str()) - .bind(success_result) - .bind(serde_json::Value::Array(infrastructure_payload_to_back_up)) - .execute(&mut *tx) - .await?; - } else if let AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } = - tasked_worked_on - { - sqlx::query( - r#" - DELETE FROM app_backup - WHERE app_name = $1; - "#, - ) - .bind(app_name.as_str()) - .execute(&mut *tx) - .await?; - } - } - - // TODO: backup cannot be merged… and we should mark up to this task id. - for (task_id_that_was_merged, _merged_task) in - tasks.iter().filter(|task| task.0 != *id.as_uuid()) - { - sqlx::query( - r#" - UPDATE app_task - SET status = 'done', executed_and_merged_with = $1 - WHERE id = $2 - "#, - ) - .bind(id.as_uuid()) - .bind(task_id_that_was_merged) - .execute(&mut *tx) - .await?; - } - - tx.commit().await?; - - Ok(()) - } - - pub async fn clean_up_done_tasks(&self, older_than: DateTime) -> Result { - let mut tx = self.pool.begin().await?; - - let affected_rows = sqlx::query( - r#" - DELETE FROM app_task - WHERE status = 'done' - AND created_at <= $1 - "#, - ) - .bind(older_than) - .execute(&mut *tx) - .await? - .rows_affected(); - - tx.commit().await?; - - Ok(affected_rows as usize) - } -} - -#[cfg(test)] -mod tests { - use std::{str::FromStr, time::Duration}; - - use super::*; - use crate::{db::DatabasePool, models::AppName, sc}; - use sqlx::postgres::PgConnectOptions; - use testcontainers_modules::{ - postgres::{self}, - testcontainers::{runners::AsyncRunner, ContainerAsync}, - }; - - async fn create_queue() -> (ContainerAsync, PostgresAppTaskQueueDB) { - let postgres_instance = postgres::Postgres::default().start().await.unwrap(); - - let connection = PgConnectOptions::new_without_pgpass() - .application_name("PREvant") - .host(&postgres_instance.get_host().await.unwrap().to_string()) - .port(postgres_instance.get_host_port_ipv4(5432).await.unwrap()) - .username("postgres") - .password("postgres"); - - let pool = DatabasePool::connect_with_exponential_backoff(connection) - .await - .unwrap(); - sqlx::migrate!().run(&pool).await.unwrap(); - - (postgres_instance, PostgresAppTaskQueueDB::new(pool)) - } - - #[tokio::test] - async fn enqueue_and_execute_successfully() { - let (_postgres_instance, queue) = create_queue().await; - - let status_id = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id.clone(), - app_name: AppName::master(), - }) - .await - .unwrap(); - - queue - .execute_tasks(async |tasks| { - let task = tasks.last().unwrap().clone(); - ( - task, - Ok(App::new( - vec![Service { - id: String::from("nginx-1234"), - state: State { - status: ServiceStatus::Paused, - started_at: None, - }, - config: sc!("nginx"), - }], - HashSet::new(), - None, - )), - ) - }) - .await - .unwrap(); - - let result = queue.peek_result(&status_id).await; - assert!(matches!(result, Some(Ok(_)))); - - let cleaned = queue.clean_up_done_tasks(Utc::now()).await.unwrap(); - assert_eq!(cleaned, 1); - } - - #[tokio::test] - async fn execute_all_tasks_per_app_name() { - let (_postgres_instance, queue) = create_queue().await; - - let status_id_1 = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id_1.clone(), - app_name: AppName::master(), - }) - .await - .unwrap(); - let status_id_2 = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id_2.clone(), - app_name: AppName::master(), - }) - .await - .unwrap(); - - // spawning an independent task that asserts that all tasks for AppName::master() are - // blocked by the execute_task below. - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - let spawned_queue = PostgresAppTaskQueueDB { - pool: queue.pool.clone(), - }; - let spawn_handle_1 = tokio::spawn(async move { - let _ = rx.await.unwrap(); - spawned_queue - .execute_tasks(async |tasks| { - unreachable!("There should be no task to be executed here because the spawned task should have blocked it: {tasks:?}") - }) - .await - }); - - let spawned_queue = PostgresAppTaskQueueDB { - pool: queue.pool.clone(), - }; - - let spawn_handle_2 = tokio::spawn(async move { - spawned_queue - .execute_tasks(async |tasks| { - tx.send(()).unwrap(); - - tokio::time::sleep(Duration::from_secs(4)).await; - - let task = tasks.last().unwrap().clone(); - ( - task, - Ok(App::new( - vec![Service { - id: String::from("nginx-1234"), - state: State { - status: ServiceStatus::Paused, - started_at: None, - }, - config: sc!("nginx"), - }], - HashSet::new(), - None, - )), - ) - }) - .await - }); - - let result_1 = spawn_handle_1.await; - assert!(result_1.is_ok()); - let result_2 = spawn_handle_2.await; - assert!(result_2.is_ok()); - - let result_1 = queue.peek_result(&status_id_1).await; - assert!(matches!(result_1, Some(Ok(_)))); - let result_2 = queue.peek_result(&status_id_2).await; - assert_eq!(result_2, result_1); - - let cleaned = queue.clean_up_done_tasks(Utc::now()).await.unwrap(); - assert_eq!(cleaned, 2); - } - - #[tokio::test] - async fn execute_task_with_different_app_names_in_parallel() { - let (_postgres_instance, queue) = create_queue().await; - - let status_id_1 = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id_1.clone(), - app_name: AppName::master(), - }) - .await - .unwrap(); - let status_id_2 = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id_2.clone(), - app_name: AppName::from_str("other").unwrap(), - }) - .await - .unwrap(); - let status_id_3 = AppStatusChangeId::new(); - queue - .enqueue_task(AppTask::Delete { - status_id: status_id_3, - app_name: AppName::master(), - }) - .await - .unwrap(); - - let spawned_queue = PostgresAppTaskQueueDB { - pool: queue.pool.clone(), - }; - let spawn_handle_1 = tokio::spawn(async move { - spawned_queue - .execute_tasks(async |tasks| { - let task = tasks.last().unwrap().clone(); - ( - task, - Ok(App::new( - vec![Service { - id: String::from("nginx-1234"), - state: State { - status: ServiceStatus::Paused, - started_at: None, - }, - config: sc!("nginx"), - }], - HashSet::new(), - None, - )), - ) - }) - .await - .unwrap(); - }); - - let spawned_queue = PostgresAppTaskQueueDB { - pool: queue.pool.clone(), - }; - let spawn_handle_2 = tokio::spawn(async move { - spawned_queue - .execute_tasks(async |tasks| { - let task = tasks.last().unwrap().clone(); - ( - task, - Ok(App::new( - vec![Service { - id: String::from("nginx-1234"), - state: State { - status: ServiceStatus::Paused, - started_at: None, - }, - config: sc!("nginx"), - }], - HashSet::new(), - None, - )), - ) - }) - .await - .unwrap(); - }); - - let result_1 = spawn_handle_1.await; - assert!(result_1.is_ok()); - let result_2 = spawn_handle_2.await; - assert!(result_2.is_ok()); - - let result_from_master = queue.peek_result(&status_id_1).await; - assert!(matches!(result_from_master, Some(Ok(_)))); - let result = queue.peek_result(&status_id_2).await; - assert!(matches!(result, Some(Ok(_)))); - let result = queue.peek_result(&status_id_3).await; - assert_eq!(result, result_from_master); - } -} diff --git a/api/src/apps/repository.rs b/api/src/apps/repository.rs index b8a26d8d..863b7911 100644 --- a/api/src/apps/repository.rs +++ b/api/src/apps/repository.rs @@ -1,16 +1,34 @@ -use crate::models::AppName; +use crate::{ + apps::AppsError, + models::{ + user_defined_parameters::UserDefinedParameters, App, AppName, AppStatusChangeId, AppTask, + Owner, Service, ServiceConfig, ServiceStatus, State, + }, +}; use anyhow::Result; +use chrono::{DateTime, Utc}; use rocket::{ fairing::{Fairing, Info, Kind}, - Build, Rocket, + Build, Orbit, Rocket, +}; +use sqlx::{PgPool, Postgres}; +use std::{ + collections::{HashMap, HashSet}, + future::Future, + str::FromStr, + sync::Mutex, }; -use sqlx::PgPool; +use tokio::sync::watch::Receiver; -pub struct AppRepository {} +pub struct AppRepository { + backup_poller: Mutex>, +} impl AppRepository { pub fn fairing() -> Self { - Self {} + Self { + backup_poller: Mutex::new(None), + } } } @@ -26,16 +44,49 @@ impl Fairing for AppRepository { async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { let repository = rocket .state::() - .map(|pool| AppPostgresRepository { pool: pool.clone() }); - Ok(rocket.manage(repository)) + .map(|pool| AppPostgresRepository::new(pool.clone())); + + match repository + .as_ref() + .map(|repository| repository.backup_updates()) + { + Some((backup_poller, backup_updates)) => { + log::debug!("Database is available, configuring backup poller to stream backup changes into application updates."); + + let mut bp = self.backup_poller.lock().unwrap(); + *bp = Some(backup_poller); + + Ok(rocket + .manage(repository) + .manage(Some(BackupUpdateReceiver(backup_updates)))) + } + None => Ok(rocket + .manage(None::) + .manage(None::)), + } + } + + async fn on_liftoff(&self, _rocket: &Rocket) { + let mut backup_poller = self.backup_poller.lock().unwrap(); + if let Some(backup_poller) = backup_poller.take() { + tokio::task::spawn(backup_poller.0); + } } } +#[derive(Clone)] pub struct AppPostgresRepository { pool: PgPool, } +pub struct BackupUpdateReceiver(pub Receiver>); +struct BackupPoller(std::pin::Pin + Send>>); + impl AppPostgresRepository { + fn new(pool: PgPool) -> Self { + Self { pool } + } + pub async fn fetch_backup(&self, app_name: &AppName) -> Result>> { let mut connection = self.pool.acquire().await?; @@ -57,4 +108,595 @@ impl AppPostgresRepository { }, )) } + + fn backup_updates(&self) -> (BackupPoller, Receiver>) { + let (tx, rx) = tokio::sync::watch::channel::>(HashMap::new()); + + let pool = self.pool.clone(); + let poller = BackupPoller(Box::pin(async move { + loop { + match pool.acquire().await { + Ok(mut connection) => { + log::debug!("Fetching list of backups to send updates."); + match Self::fetch_backed_up_apps_inner(&mut *connection).await { + Ok(apps) => { + tx.send_if_modified(move |state| { + if &apps != state { + log::debug!("List of backups changed, sending updates."); + *state = apps; + true + } else { + false + } + }); + } + Err(err) => { + log::error!("Cannot fetch backups from database: {err}"); + } + } + } + Err(err) => { + log::error!("Fetching list of backups failed: {err}."); + } + } + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + })); + + (poller, rx) + } + + pub async fn fetch_backed_up_apps(&self) -> Result> { + let mut connection = self.pool.acquire().await?; + + Self::fetch_backed_up_apps_inner(&mut *connection).await + } + + async fn fetch_backed_up_apps_inner<'a, E>(executor: E) -> Result> + where + E: sqlx::Executor<'a, Database = Postgres>, + { + let result = sqlx::query_as::<_, (String, sqlx::types::Json)>( + r#" + SELECT app_name, app + FROM app_backup + "#, + ) + .fetch_all(executor) + .await?; + + Ok(result + .into_iter() + .map(|(app_name, app)| (AppName::from_str(&app_name).unwrap(), App::from(app.0))) + .collect::>()) + } + + pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { + let mut tx = self.pool.begin().await?; + + sqlx::query( + r#" + INSERT INTO app_task (id, app_name, task) + VALUES ($1, $2, $3) + "#, + ) + .bind(task.status_id().as_uuid()) + .bind(task.app_name().as_str()) + .bind(serde_json::to_value(&task).unwrap()) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(()) + } + + pub async fn peek_result( + &self, + status_id: &AppStatusChangeId, + ) -> Option> { + let mut connection = self + .pool + .acquire() + .await + .inspect_err(|err| log::error!("Cannot acquire database connection: {err}")) + .ok()?; + + sqlx::query_as::< + _, + ( + Option>, + Option>, + Option>, + Option>, + ), + >( + r#" + SELECT a.result_success, a.result_error, m.result_success, m.result_error + FROM app_task a + LEFT OUTER JOIN app_task m + ON a.executed_and_merged_with = m.id + WHERE a.id = $1 + AND a.status = 'done' + "#, + ) + .bind(status_id.as_uuid()) + .fetch_optional(&mut *connection) + .await + .inspect_err(|err| log::error!("Cannot peek result for {status_id}: {err}")) + .ok()? + .map(|(app, error, merged_app, merged_error)| { + match ( + app.map(|app| app.0), + error.map(|error| error.0), + merged_app.map(|app| app.0), + merged_error.map(|error| error.0), + ) { + (Some(app), None, None, None) => Ok(app.into()), + (None, Some(err), None, None) => Err(err), + (None, None, Some(app), None) => Ok(app.into()), + (None, None, None, Some(err)) => Err(err), + _ => unreachable!( + "There should be either a result or an error stored in the database" + ), + } + }) + } + + /// All queued tasks will be locked by a new transaction (see [PostgreSQL as message + /// queue](https://www.svix.com/resources/guides/postgres-message-queue/#why-use-postgresql-as-a-message-queue)) + /// and the `executor`'s result will be stored for the tasks that could be executed at once. + pub async fn update_queued_tasks_with_executor_result(&self, executor: F) -> Result<()> + where + F: FnOnce(Vec) -> Fut, + Fut: Future)>, + { + let mut tx = self.pool.begin().await?; + + let tasks = sqlx::query_as::<_, (sqlx::types::Uuid, sqlx::types::Json)>( + r#" + WITH eligible_tasks AS ( + SELECT id, task + FROM app_task + WHERE status = 'queued' + AND app_name = ( + SELECT app_name + FROM app_task + WHERE created_at = (SELECT min(created_at) FROM app_task WHERE status = 'queued') + AND status = 'queued' + ) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + ) + UPDATE app_task + SET status = 'running' + FROM eligible_tasks + WHERE app_task.id = eligible_tasks.id + RETURNING eligible_tasks .id, eligible_tasks.task; + "#, + ) + .fetch_all(&mut *tx) + .await?; + + let tasks_to_work_on = tasks + .iter() + .map(|task_to_work_on| task_to_work_on.1 .0.clone()) + .collect::>(); + + if tasks_to_work_on.is_empty() { + return Ok(()); + } + + let (tasked_worked_on, result) = executor(tasks_to_work_on).await; + let is_success = result.is_ok(); + let id = *tasked_worked_on.status_id(); + + let failed_result = result + .as_ref() + .map_or_else(|e| serde_json::to_value(e).ok(), |_| None); + let success_result = result.map_or_else( + |_| None, + |app| { + let (services, owner, user_defined_parameters) = + app.into_services_and_owners_and_user_defined_parameters(); + let raw = RawApp { + owner, + services: services + .into_iter() + .map(|service| RawService { + id: service.id, + status: service.state.status, + config: service.config, + }) + .collect(), + user_defined_parameters: user_defined_parameters.and_then( + |user_defined_parameters| { + serde_json::to_value(user_defined_parameters).ok() + }, + ), + }; + serde_json::to_value(raw).ok() + }, + ); + + sqlx::query( + r#" + UPDATE app_task + SET status = 'done', result_success = $3, result_error = $2 + WHERE id = $1 + "#, + ) + .bind(id.as_uuid()) + .bind(failed_result) + .bind(&success_result) + .execute(&mut *tx) + .await?; + + if is_success { + if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + infrastructure_payload_to_back_up, + .. + } = tasked_worked_on + { + log::debug!("Backing-up infrastructure payload for {app_name}."); + + sqlx::query( + r#" + INSERT INTO app_backup (app_name, app, infrastructure_payload) + VALUES ($1, $2, $3); + "#, + ) + .bind(app_name.as_str()) + .bind(success_result) + .bind(serde_json::Value::Array(infrastructure_payload_to_back_up)) + .execute(&mut *tx) + .await?; + } else if let AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } = + tasked_worked_on + { + log::debug!("Deleting infrastructure payload for {app_name} from backups."); + + sqlx::query( + r#" + DELETE FROM app_backup + WHERE app_name = $1; + "#, + ) + .bind(app_name.as_str()) + .execute(&mut *tx) + .await?; + } + } + + // TODO: backup cannot be merged… and we should mark up to this task id. + for (task_id_that_was_merged, _merged_task) in + tasks.iter().filter(|task| task.0 != *id.as_uuid()) + { + sqlx::query( + r#" + UPDATE app_task + SET status = 'done', executed_and_merged_with = $1 + WHERE id = $2 + "#, + ) + .bind(id.as_uuid()) + .bind(task_id_that_was_merged) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + + Ok(()) + } + + pub async fn clean_up_done_tasks(&self, older_than: DateTime) -> Result { + let mut tx = self.pool.begin().await?; + + let affected_rows = sqlx::query( + r#" + DELETE FROM app_task + WHERE status = 'done' + AND created_at <= $1 + "#, + ) + .bind(older_than) + .execute(&mut *tx) + .await? + .rows_affected(); + + tx.commit().await?; + + Ok(affected_rows as usize) + } +} + +#[derive(Serialize, Deserialize)] +struct RawApp { + services: Vec, + owner: HashSet, + user_defined_parameters: Option, +} + +impl From for App { + fn from(value: RawApp) -> Self { + Self::new( + value.services.into_iter().map(Service::from).collect(), + Owner::normalize(value.owner), + value + .user_defined_parameters + .map(|data| unsafe { UserDefinedParameters::without_validation(data) }), + ) + } +} + +#[derive(Serialize, Deserialize)] +struct RawService { + id: String, + status: ServiceStatus, + config: ServiceConfig, +} + +impl From for Service { + fn from(value: RawService) -> Self { + Self { + id: value.id, + state: State { + status: value.status, + started_at: None, + }, + config: value.config, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{db::DatabasePool, models::AppName, sc}; + use sqlx::postgres::PgConnectOptions; + use std::{str::FromStr, time::Duration}; + use testcontainers_modules::{ + postgres::{self}, + testcontainers::{runners::AsyncRunner, ContainerAsync}, + }; + + async fn create_repository() -> (ContainerAsync, AppPostgresRepository) { + let postgres_instance = postgres::Postgres::default().start().await.unwrap(); + + let connection = PgConnectOptions::new_without_pgpass() + .application_name("PREvant") + .host(&postgres_instance.get_host().await.unwrap().to_string()) + .port(postgres_instance.get_host_port_ipv4(5432).await.unwrap()) + .username("postgres") + .password("postgres"); + + let pool = DatabasePool::connect_with_exponential_backoff(connection) + .await + .unwrap(); + sqlx::migrate!().run(&pool).await.unwrap(); + + (postgres_instance, AppPostgresRepository { pool }) + } + + #[tokio::test] + async fn enqueue_and_execute_successfully() { + let (_postgres_instance, repository) = create_repository().await; + + let status_id = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id, + app_name: AppName::master(), + }) + .await + .unwrap(); + + repository + .update_queued_tasks_with_executor_result(async |tasks| { + let task = tasks.last().unwrap().clone(); + ( + task, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + .unwrap(); + + let result = repository.peek_result(&status_id).await; + assert!(matches!(result, Some(Ok(_)))); + + let cleaned = repository.clean_up_done_tasks(Utc::now()).await.unwrap(); + assert_eq!(cleaned, 1); + } + + #[tokio::test] + async fn execute_all_tasks_per_app_name() { + let (_postgres_instance, repository) = create_repository().await; + + let status_id_1 = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + let status_id_2 = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }) + .await + .unwrap(); + + // spawning an independent task that asserts that all tasks for AppName::master() are + // blocked by the execute_task below. + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let spawned_repository = AppPostgresRepository { + pool: repository.pool.clone(), + }; + let spawn_handle_1 = tokio::spawn(async move { + rx.await.unwrap(); + spawned_repository + .update_queued_tasks_with_executor_result(async |tasks| { + unreachable!("There should be no task to be executed here because the spawned task should have blocked it: {tasks:?}") + }) + .await + }); + + let spawned_repository = AppPostgresRepository { + pool: repository.pool.clone(), + }; + + let spawn_handle_2 = tokio::spawn(async move { + spawned_repository + .update_queued_tasks_with_executor_result(async |tasks| { + tx.send(()).unwrap(); + + tokio::time::sleep(Duration::from_secs(4)).await; + + let task = tasks.last().unwrap().clone(); + ( + task, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + }); + + let result_1 = spawn_handle_1.await; + assert!(result_1.is_ok()); + let result_2 = spawn_handle_2.await; + assert!(result_2.is_ok()); + + let result_1 = repository.peek_result(&status_id_1).await; + assert!(matches!(result_1, Some(Ok(_)))); + let result_2 = repository.peek_result(&status_id_2).await; + assert_eq!(result_2, result_1); + + let cleaned = repository.clean_up_done_tasks(Utc::now()).await.unwrap(); + assert_eq!(cleaned, 2); + } + + #[tokio::test] + async fn execute_task_with_different_app_names_in_parallel() { + let (_postgres_instance, repository) = create_repository().await; + + let status_id_1 = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + let status_id_2 = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::from_str("other").unwrap(), + }) + .await + .unwrap(); + let status_id_3 = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id: status_id_3, + app_name: AppName::master(), + }) + .await + .unwrap(); + + let spawned_repository = AppPostgresRepository { + pool: repository.pool.clone(), + }; + let spawn_handle_1 = tokio::spawn(async move { + spawned_repository + .update_queued_tasks_with_executor_result(async |tasks| { + let task = tasks.last().unwrap().clone(); + ( + task, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + .unwrap(); + }); + + let spawned_repository = AppPostgresRepository { + pool: repository.pool.clone(), + }; + let spawn_handle_2 = tokio::spawn(async move { + spawned_repository + .update_queued_tasks_with_executor_result(async |tasks| { + let task = tasks.last().unwrap().clone(); + ( + task, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + .unwrap(); + }); + + let result_1 = spawn_handle_1.await; + assert!(result_1.is_ok()); + let result_2 = spawn_handle_2.await; + assert!(result_2.is_ok()); + + let result_from_master = repository.peek_result(&status_id_1).await; + assert!(matches!(result_from_master, Some(Ok(_)))); + let result = repository.peek_result(&status_id_2).await; + assert!(matches!(result, Some(Ok(_)))); + let result = repository.peek_result(&status_id_3).await; + assert_eq!(result, result_from_master); + } } diff --git a/api/src/apps/routes/get_apps.rs b/api/src/apps/routes/get_apps.rs index 24ea1138..2d1dae9d 100644 --- a/api/src/apps/routes/get_apps.rs +++ b/api/src/apps/routes/get_apps.rs @@ -1,15 +1,21 @@ use crate::{ - apps::{Apps, HostMetaCache}, - http_result::HttpResult, - models::{App, AppName, AppWithHostMeta, Owner, RequestInfo, ServiceWithHostMeta}, + apps::{ + repository::{AppPostgresRepository, BackupUpdateReceiver}, + routes::AppV2, + Apps, HostMetaCache, + }, + http_result::{HttpApiError, HttpResult}, + models::{App, AppName, AppWithHostMeta, RequestInfo}, }; +use http::StatusCode; +use http_api_problem::HttpApiProblem; use rocket::{ response::stream::{Event, EventStream}, serde::json::Json, Shutdown, State, }; use serde::{ser::SerializeMap as _, Serialize}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use tokio::{select, sync::watch::Receiver}; use tokio_stream::StreamExt; @@ -30,7 +36,7 @@ impl Serialize for AppsV1 { } } -pub(super) struct AppsV2(HashMap); +pub(super) struct AppsV2(HashMap); impl Serialize for AppsV2 { fn serialize(&self, serializer: S) -> Result @@ -39,21 +45,8 @@ impl Serialize for AppsV2 { { let mut map = serializer.serialize_map(Some(self.0.len()))?; - #[derive(Serialize)] - struct App<'a> { - services: &'a [ServiceWithHostMeta], - #[serde(skip_serializing_if = "HashSet::is_empty")] - owners: &'a HashSet, - } - - for (app_name, services_with_host_meta) in self.0.iter() { - map.serialize_entry( - app_name, - &App { - services: services_with_host_meta.services(), - owners: services_with_host_meta.owners(), - }, - )?; + for (app_name, app) in self.0.iter() { + map.serialize_entry(app_name, app)?; } map.end() @@ -66,51 +59,62 @@ pub(super) async fn apps_v1( request_info: RequestInfo, host_meta_cache: &State, ) -> HttpResult> { + // We don't fetch app backups here because the deprecated API wouldn't have an option to + // show the outside what kind of application the consumer received. Seeing the backed up + // applications on the receivers' ends would be a semantic breaking change. let apps = apps.fetch_apps().await?; Ok(Json(AppsV1( host_meta_cache.assign_host_meta_data(apps, &request_info), ))) } +fn merge( + deployed_apps: HashMap, + backed_up_apps: HashMap, +) -> HashMap { + let mut deployed_apps = deployed_apps + .into_iter() + .map(|(app_name, app)| (app_name, AppV2::Deployed(app))) + .collect::>(); + + let backed_up_apps = backed_up_apps + .into_iter() + .map(|(app_name, app)| (app_name, AppV2::BackedUp(app))) + .collect::>(); + + deployed_apps.extend(backed_up_apps); + + deployed_apps +} + #[rocket::get("/", format = "application/vnd.prevant.v2+json", rank = 2)] pub(super) async fn apps_v2( apps: &State, + app_repository: &State>, request_info: RequestInfo, host_meta_cache: &State, ) -> HttpResult> { - let apps = apps.fetch_apps().await?; - Ok(Json(AppsV2( - host_meta_cache.assign_host_meta_data(apps, &request_info), - ))) -} - -macro_rules! stream_apps { - ($apps_updates:ident, $host_meta_cache:ident, $request_info:ident, $end:ident, $app_version_type:ty) => {{ - let mut services = $apps_updates.inner().borrow().clone(); - - let mut app_changes = - tokio_stream::wrappers::WatchStream::from_changes($apps_updates.inner().clone()); - let mut host_meta_cache_updates = $host_meta_cache.cache_updates(); - - EventStream! { - yield Event::json(&$app_version_type($host_meta_cache.assign_host_meta_data(services.clone(), &$request_info))); - - loop { - select! { - Some(new_services) = app_changes.next() => { - log::debug!("New app list update: sending app service update"); - services = new_services; - } - Some(_t) = host_meta_cache_updates.next() => { - log::debug!("New host meta cache update: sending app service update"); - } - _ = &mut $end => break, - }; - - yield Event::json(&$app_version_type($host_meta_cache.assign_host_meta_data(services.clone(), &$request_info))); + let (apps, app_backups) = futures::try_join!( + async { + apps.fetch_apps() + .await + .map_err(HttpApiError::from) + .map(|apps| host_meta_cache.assign_host_meta_data(apps, &request_info)) + }, + async { + match &**app_repository { + Some(app_repository) => app_repository.fetch_backed_up_apps().await.map_err(|e| { + HttpApiError::from( + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()), + ) + }), + None => Ok(HashMap::new()), } } - }}; + )?; + + Ok(Json(AppsV2(merge(apps, app_backups)))) } #[rocket::get("/", format = "text/event-stream", rank = 3)] @@ -120,26 +124,98 @@ pub(super) async fn stream_apps_v1( request_info: RequestInfo, host_meta_cache: HostMetaCache, ) -> EventStream![] { - stream_apps!(apps_updates, host_meta_cache, request_info, end, AppsV1) + // We don't fetch app backups here because the deprecated API wouldn't have an option to + // show the outside what kind of application the consumer received. Seeing the backed up + // applications on the receivers' ends would be a semantic breaking change. + let mut deployed_apps = apps_updates.inner().borrow().clone(); + + let mut app_changes = + tokio_stream::wrappers::WatchStream::from_changes(apps_updates.inner().clone()); + let mut host_meta_cache_updates = host_meta_cache.cache_updates(); + + EventStream! { + yield Event::json(&AppsV1(host_meta_cache.assign_host_meta_data(deployed_apps.clone(), &request_info))); + + loop { + select! { + Some(new_apps) = app_changes.next() => { + log::debug!("New app list update: sending app service update"); + deployed_apps = new_apps; + } + Some(_t) = host_meta_cache_updates.next() => { + log::debug!("New host meta cache update: sending app service update"); + } + _ = &mut end => break, + }; + + yield Event::json(&AppsV1(host_meta_cache.assign_host_meta_data(deployed_apps.clone(), &request_info))); + } + } } #[rocket::get("/", format = "text/vnd.prevant.v2+event-stream", rank = 4)] pub(super) async fn stream_apps_v2( apps_updates: &State>>, + backup_updates: &State>, mut end: Shutdown, request_info: RequestInfo, host_meta_cache: HostMetaCache, ) -> EventStream![] { - stream_apps!(apps_updates, host_meta_cache, request_info, end, AppsV2) + let mut deployed_apps = apps_updates.inner().borrow().clone(); + let mut backed_up_apps = match &**backup_updates { + Some(backup_updates) => backup_updates.0.borrow().clone(), + None => HashMap::new(), + }; + + let mut app_changes = + tokio_stream::wrappers::WatchStream::from_changes(apps_updates.inner().clone()); + let mut backup_changes = match &**backup_updates { + Some(backup_updates) => { + tokio_stream::wrappers::WatchStream::from_changes(backup_updates.0.clone()) + } + None => { + let (_tx, rx) = tokio::sync::watch::channel(HashMap::new()); + tokio_stream::wrappers::WatchStream::from_changes(rx) + } + }; + let mut host_meta_cache_updates = host_meta_cache.cache_updates(); + EventStream! { + yield Event::json(&AppsV2(merge( + host_meta_cache.assign_host_meta_data(deployed_apps.clone(), &request_info), + backed_up_apps.clone(), + ))); + + loop { + select! { + Some(new_apps) = app_changes.next() => { + log::debug!("New app list update: sending app service update"); + deployed_apps = new_apps; + } + Some(new_backups) = backup_changes.next() => { + log::debug!("New backup list update: sending app service update"); + backed_up_apps = new_backups; + } + Some(_t) = host_meta_cache_updates.next() => { + log::debug!("New host meta cache update: sending app service update"); + } + _ = &mut end => break, + }; + + yield Event::json(&AppsV2(merge( + host_meta_cache.assign_host_meta_data(deployed_apps.clone(), &request_info), + backed_up_apps.clone(), + ))); + } + } } #[cfg(test)] mod tests { use super::*; - use crate::models::{Service, ServiceStatus, ServiceWithHostMeta, WebHostMeta}; + use crate::models::{Owner, Service, ServiceStatus, ServiceWithHostMeta, WebHostMeta}; use assert_json_diff::assert_json_eq; use chrono::Utc; - use std::str::FromStr; + use std::{collections::HashSet, str::FromStr}; use url::Url; #[test] @@ -213,6 +289,7 @@ mod tests { assert_json_eq!( serde_json::json!({ "master": { + "status": "deployed", "services": [{ "name": "mariadb", "type": "instance", @@ -234,7 +311,7 @@ mod tests { }), serde_json::to_value(AppsV2(HashMap::from([( app_name.clone(), - AppWithHostMeta::new( + AppV2::Deployed(AppWithHostMeta::new( vec![ ServiceWithHostMeta::from_service_and_web_host_meta( Service { @@ -264,7 +341,7 @@ mod tests { ) ], HashSet::new() - ) + )) )]))) .unwrap() ); @@ -278,6 +355,7 @@ mod tests { assert_json_eq!( serde_json::json!({ "master": { + "status": "deployed", "owners": [{ "sub": "some-sub", "iss": "https://openid.example.com" @@ -303,7 +381,7 @@ mod tests { }), serde_json::to_value(AppsV2(HashMap::from([( app_name.clone(), - AppWithHostMeta::new( + AppV2::Deployed(AppWithHostMeta::new( vec![ ServiceWithHostMeta::from_service_and_web_host_meta( Service { @@ -340,7 +418,7 @@ mod tests { .unwrap(), name: None, }]) - ) + )) )]))) .unwrap() ); @@ -348,7 +426,7 @@ mod tests { mod url_rendering { use super::apps_v1; - use crate::apps::repository::AppPostgresRepository; + use crate::apps::repository::{AppPostgresRepository, BackupUpdateReceiver}; use crate::apps::{AppProcessingQueue, Apps, HostMetaCache}; use crate::config::Config; use crate::infrastructure::Dummy; @@ -368,19 +446,14 @@ mod tests { let infrastructure = Box::new(Dummy::new()); let apps = Apps::new(Default::default(), infrastructure).unwrap(); let _result = apps - .create_or_update( - &AppName::master(), - None, - &[sc!("service-a")], - vec![], - None, - ) + .create_or_update(&AppName::master(), None, &[sc!("service-a")], vec![], None) .await?; let rocket = rocket::build() .manage(host_meta_cache) .manage(apps) .manage(Config::default()) + .manage(None::) .manage(None::) .manage(tokio::sync::watch::channel::>(HashMap::new()).1) .mount("/", rocket::routes![apps_v1]) diff --git a/api/src/apps/routes/mod.rs b/api/src/apps/routes/mod.rs index 66d4eacc..6d58fead 100644 --- a/api/src/apps/routes/mod.rs +++ b/api/src/apps/routes/mod.rs @@ -26,14 +26,14 @@ use super::queue::AppTaskQueueProducer; use crate::apps::repository::AppPostgresRepository; -use crate::apps::{Apps, AppsError}; +use crate::apps::{Apps, AppsError, HostMetaCache}; use crate::auth::UserValidatedByAccessMode; use crate::config::Config; use crate::deployment::hooks::Hooks; use crate::http_result::{HttpApiError, HttpResult}; use crate::models::{ - App, AppName, AppNameError, AppStatus, AppStatusChangeId, AppStatusChangeIdError, Owner, - Service, ServiceStatus, + App, AppName, AppNameError, AppStatus, AppStatusChangeId, AppStatusChangeIdError, + AppWithHostMeta, Owner, RequestInfo, Service, ServiceStatus, ServiceWithHostMeta, }; use create_app_payload::CreateAppPayload; use http_api_problem::{HttpApiProblem, StatusCode}; @@ -91,23 +91,53 @@ impl Serialize for AppV1 { } } -pub struct AppV2(App); +pub enum AppV2 { + Deployed(AppWithHostMeta), + BackedUp(App), +} impl Serialize for AppV2 { fn serialize(&self, serializer: S) -> Result where S: Serializer, { + #[derive(Serialize)] + #[serde(untagged)] + enum Service<'a> { + ServiceWithHostMeta(&'a ServiceWithHostMeta), + ServiceWithoutStatus(Box), + } #[derive(Serialize)] struct App<'a> { - services: &'a [Service], + services: Vec>, #[serde(skip_serializing_if = "HashSet::is_empty")] owners: &'a HashSet, + status: AppStatus, } - let app = App { - services: self.0.services(), - owners: self.0.owners(), + let app = match self { + AppV2::Deployed(app_with_host_meta) => App { + services: app_with_host_meta + .services() + .iter() + .map(Service::ServiceWithHostMeta) + .collect(), + owners: app_with_host_meta.owners(), + status: AppStatus::Deployed, + }, + AppV2::BackedUp(app) => App { + services: app + .services() + .iter() + .map(|s| { + let mut s = s.clone(); + s.state.status = ServiceStatus::Paused; + Service::ServiceWithoutStatus(Box::new(s)) + }) + .collect(), + owners: app.owners(), + status: AppStatus::BackedUp, + }, }; app.serialize(serializer) @@ -141,15 +171,24 @@ async fn status_change_v2( status_id: Result, app_queue: &State, options: WaitForQueueOptions, + host_meta_cache: &State, + request_info: RequestInfo, ) -> HttpResult>> { let app_name = app_name?; let status_id = status_id?; - try_wait_for_task(app_queue, app_name, status_id, options, AppV2).await + try_wait_for_task(app_queue, app_name.clone(), status_id, options, |app| { + AppV2::Deployed(host_meta_cache.assign_host_meta_data_for_app( + &app_name, + app, + &request_info, + )) + }) + .await } async fn delete_app( - app_name: Result, + app_name: AppName, app_queue: &State, options: WaitForQueueOptions, user: Result, @@ -161,8 +200,6 @@ where // TODO: authorization hook to verify e.g. if a user is member of a GitLab group let _user = user.map_err(HttpApiError::from)?; - let app_name = app_name?; - let status_id = app_queue .enqueue_delete_task(app_name.clone()) .await @@ -183,6 +220,8 @@ pub async fn delete_app_v1( options: WaitForQueueOptions, user: Result, ) -> HttpResult>> { + let app_name = app_name?; + delete_app(app_name, app_queue, options, user, AppV1).await } @@ -192,8 +231,19 @@ pub async fn delete_app_v2( app_queue: &State, options: WaitForQueueOptions, user: Result, + host_meta_cache: &State, + request_info: RequestInfo, ) -> HttpResult>> { - delete_app(app_name, app_queue, options, user, AppV2).await + let app_name = app_name?; + + delete_app(app_name.clone(), app_queue, options, user, |app| { + AppV2::Deployed(host_meta_cache.assign_host_meta_data_for_app( + &app_name, + app, + &request_info, + )) + }) + .await } pub async fn delete_app_sync( @@ -210,7 +260,7 @@ pub async fn delete_app_sync( } async fn create_app( - app_name: Result, + app_name: AppName, app_queue: &State, create_app_form: CreateAppOptions, payload: Result, @@ -226,7 +276,6 @@ where // TODO: authorization hook to verify e.g. if a user is member of a GitLab group let user = user.map_err(HttpApiError::from)?; - let app_name = app_name?; let replicate_from = create_app_form.replicate_from().clone(); let owner = hooks @@ -269,6 +318,8 @@ pub async fn create_app_v1( hooks: Hooks<'_>, user: Result, ) -> HttpResult>> { + let app_name = app_name?; + create_app( app_name, app_queue, @@ -296,16 +347,26 @@ pub async fn create_app_v2( options: WaitForQueueOptions, hooks: Hooks<'_>, user: Result, + host_meta_cache: &State, + request_info: RequestInfo, ) -> HttpResult>> { + let app_name = app_name?; + create_app( - app_name, + app_name.clone(), app_queue, create_app_form, payload, options, &hooks, user, - AppV2, + |app| { + AppV2::Deployed(host_meta_cache.assign_host_meta_data_for_app( + &app_name, + app, + &request_info, + )) + }, ) .await } @@ -324,6 +385,8 @@ pub async fn change_app_status( user: Result, options: WaitForQueueOptions, payload: Json, + host_meta_cache: &State, + request_info: RequestInfo, ) -> HttpResult>> { // TODO: authorization hook to verify e.g. if a user is member of a GitLab group let _user = user.map_err(HttpApiError::from)?; @@ -358,6 +421,17 @@ pub async fn change_app_status( })? } AppStatus::BackedUp => { + if let Some(backup) = app_repository + .fetch_backed_up_app(&app_name) + .await + .map_err(|e| { + HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) + .detail(e.to_string()) + })? + { + return Ok(AsyncCompletion::Ready(Json(AppV2::BackedUp(backup)))); + } + let Some(infrastructure_payload) = apps .fetch_app_as_backup_based_infrastructure_payload(&app_name) .await? @@ -375,7 +449,21 @@ pub async fn change_app_status( } }; - try_wait_for_task(app_queue, app_name, status_id, options, AppV2).await + try_wait_for_task( + app_queue, + app_name.clone(), + status_id, + options, + |app| match payload.status { + AppStatus::Deployed => AppV2::Deployed(host_meta_cache.assign_host_meta_data_for_app( + &app_name, + app, + &request_info, + )), + AppStatus::BackedUp => AppV2::BackedUp(app), + }, + ) + .await } #[rocket::put( @@ -1036,17 +1124,18 @@ mod tests { fn serialize_app_v2() { assert_json_eq!( serde_json::json!({ + "status": "backed-up", "services": [{ "name": "mariadb", "type": "instance", "state": { - "status": "running" + "status": "paused" } }, { "name": "postgres", "type": "instance", "state": { - "status": "running" + "status": "paused" } }], "owners": [{ @@ -1054,7 +1143,7 @@ mod tests { "iss": "https://openid.example.com" }], }), - serde_json::to_value(AppV2(App::new( + serde_json::to_value(AppV2::BackedUp(App::new( vec![ Service { id: String::from("some id"), diff --git a/api/src/db.rs b/api/src/db.rs index 83a41a6e..4ab0bafe 100644 --- a/api/src/db.rs +++ b/api/src/db.rs @@ -19,6 +19,7 @@ impl DatabasePool { let min = std::time::Duration::from_millis(100); let max = std::time::Duration::from_secs(10); for duration in exponential_backoff::Backoff::new(5, min, max) { + log::debug!("Connecting to database…"); let pool = match PgPool::connect_with(database_options.clone()).await { Ok(pool) => pool, Err(err) => match duration { diff --git a/api/src/main.rs b/api/src/main.rs index 63fa5f1a..5998bfd4 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -230,8 +230,8 @@ async fn main() -> Result<(), StartUpError> { .mount("/auth", crate::auth::auth_routes()) .attach(DatabasePool::fairing()) .attach(Auth::fairing()) - .attach(AppProcessingQueue::fairing()) .attach(AppRepository::fairing()) + .attach(AppProcessingQueue::fairing()) .attach(TicketsCaching::fairing()) .attach(Apps::fairing(config, infrastructure)) .launch() diff --git a/api/src/models/app.rs b/api/src/models/app.rs index 5bf02889..7e981b52 100644 --- a/api/src/models/app.rs +++ b/api/src/models/app.rs @@ -347,13 +347,6 @@ impl AppWithHostMeta { } } -#[derive(Clone, Debug, PartialEq)] -pub struct AppWithHostMetaAndStatus { - services: Vec, - owners: HashSet, - status: AppStatus, -} - #[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq)] pub enum AppStatus { #[serde(rename = "deployed")] @@ -362,16 +355,6 @@ pub enum AppStatus { BackedUp, } -impl From<(AppWithHostMeta, AppStatus)> for AppWithHostMetaAndStatus { - fn from(value: (AppWithHostMeta, AppStatus)) -> Self { - Self { - services: value.0.services, - owners: value.0.owners, - status: value.1, - } - } -} - #[derive(Debug, Default, Deserialize, Clone, Eq, Hash, PartialEq, Serialize)] pub enum ContainerType { #[serde(rename = "instance")] diff --git a/api/src/apps/queue/task.rs b/api/src/models/app_task.rs similarity index 99% rename from api/src/apps/queue/task.rs rename to api/src/models/app_task.rs index a40c9536..8f25a14b 100644 --- a/api/src/apps/queue/task.rs +++ b/api/src/models/app_task.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq)] #[serde(untagged)] -pub(super) enum AppTask { +pub enum AppTask { MovePayloadToBackUpAndDeleteFromInfrastructure { status_id: AppStatusChangeId, app_name: AppName, diff --git a/api/src/models/mod.rs b/api/src/models/mod.rs index d3df1f29..9f1691aa 100644 --- a/api/src/models/mod.rs +++ b/api/src/models/mod.rs @@ -25,11 +25,12 @@ */ pub use app::{ - App, AppStatus, AppWithHostMeta, AppWithHostMetaAndStatus, ContainerType, Owner, Service, - ServiceError, ServiceStatus, ServiceWithHostMeta, State, + App, AppStatus, AppWithHostMeta, ContainerType, Owner, Service, ServiceError, ServiceStatus, + ServiceWithHostMeta, State, }; pub use app_name::{AppName, AppNameError}; pub use app_status_change_id::{AppStatusChangeId, AppStatusChangeIdError}; +pub use app_task::AppTask; pub use image::Image; pub use logs_chunks::LogChunk; pub use request_info::RequestInfo; @@ -40,6 +41,7 @@ pub use web_host_meta::WebHostMeta; mod app; mod app_name; mod app_status_change_id; +mod app_task; mod image; mod logs_chunks; pub mod request_info; From 9e7c91eb432d72db5a8940f6d2f1e268dd5a2c92 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Tue, 30 Dec 2025 15:02:49 +0100 Subject: [PATCH 4/6] Merge back-up and restore tasks This change makes sure that the back-up and restore tasks are merged if possible. If they cannot be merged, they will be executed until without merging to make sure that the application state is the same as if they are executed in singular tasks. Also, this refactoring fixes #277 for queue processing without persistent database queue. --- api/src/apps/queue.rs | 836 ++++++++++++++++++ api/src/apps/queue/mod.rs | 430 --------- api/src/apps/repository.rs | 321 ++++++- api/src/apps/routes/mod.rs | 6 +- .../kubernetes/deployment_unit.rs | 43 +- api/src/models/app_task.rs | 731 ++++++++++++++- api/src/models/mod.rs | 2 +- api/src/models/service_config/mod.rs | 37 +- api/src/models/service_config/templating.rs | 2 +- 9 files changed, 1904 insertions(+), 504 deletions(-) create mode 100644 api/src/apps/queue.rs delete mode 100644 api/src/apps/queue/mod.rs diff --git a/api/src/apps/queue.rs b/api/src/apps/queue.rs new file mode 100644 index 00000000..15f90cc6 --- /dev/null +++ b/api/src/apps/queue.rs @@ -0,0 +1,836 @@ +use crate::apps::repository::AppPostgresRepository; +use crate::apps::{Apps, AppsError}; +use crate::models::{ + App, AppName, AppStatusChangeId, AppTask, MergedAppTask, Owner, ServiceConfig, +}; +use anyhow::Result; +use chrono::{DateTime, TimeDelta, Utc}; +use rocket::{ + fairing::{Fairing, Info, Kind}, + Build, Orbit, Rocket, +}; +use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration}; +use tokio::{ + sync::{Mutex, Notify}, + time::{sleep, sleep_until, timeout}, +}; + +pub struct AppProcessingQueue {} + +impl AppProcessingQueue { + pub fn fairing() -> Self { + Self {} + } +} + +#[rocket::async_trait] +impl Fairing for AppProcessingQueue { + fn info(&self) -> Info { + Info { + name: "app-queue", + kind: Kind::Ignite | Kind::Liftoff, + } + } + + async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { + let db = match rocket.state::>() { + Some(Some(repository)) => AppTaskQueueDB::db(repository.clone()), + _ => AppTaskQueueDB::inmemory(), + }; + + let producer = AppTaskQueueProducer { + db: Arc::new(db), + notify: Arc::new(Notify::new()), + }; + + Ok(rocket.manage(producer)) + } + + async fn on_liftoff(&self, rocket: &Rocket) { + let apps = rocket.state::().unwrap(); + let producer = rocket.state::().unwrap(); + let consumer = AppTaskQueueConsumer { + db: producer.db.clone(), + notify: producer.notify.clone(), + }; + let mut shutdown = rocket.shutdown(); + + let apps = apps.clone(); + rocket::tokio::spawn(async move { + loop { + tokio::select! { + res = consumer.process_next_task(&apps) => { + if let Err(err) = res { + log::error!("Cannot process task: {err}"); + } + } + _ = &mut shutdown => { + log::info!("Shutting down queue processing"); + break; + } + }; + + match consumer.clean_up_done_tasks().await { + Ok(number_of_deleted_tasks) if number_of_deleted_tasks > 0 => { + log::debug!("Deleted {number_of_deleted_tasks} done tasks"); + } + Err(err) => { + log::error!("Cannot cleanup done task: {err}"); + } + _ => {} + } + } + }); + } +} + +pub struct AppTaskQueueProducer { + db: Arc, + notify: Arc, +} +impl AppTaskQueueProducer { + pub async fn enqueue_create_or_update_task( + &self, + app_name: AppName, + replicate_from: Option, + service_configs: Vec, + owner: Option, + user_defined_parameters: Option, + ) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::CreateOrUpdate { + app_name, + status_id, + replicate_from, + service_configs, + owners: owner.into_iter().collect(), + user_defined_parameters, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new create or update task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + + pub async fn enqueue_delete_task(&self, app_name: AppName) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::Delete { + app_name, + status_id, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new delete task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + + pub async fn enqueue_backup_task( + &self, + app_name: AppName, + infrastructure_payload: Vec, + ) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name, + infrastructure_payload_to_back_up: infrastructure_payload, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new backup task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + + pub async fn enqueue_restore_task( + &self, + app_name: AppName, + infrastructure_payload: Vec, + ) -> Result { + let status_id = AppStatusChangeId::new(); + self.db + .enqueue_task(AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id, + app_name, + infrastructure_payload_to_restore: infrastructure_payload, + }) + .await?; + + if log::log_enabled!(log::Level::Debug) { + log::debug!("Notify about new restore task: {status_id}."); + } + self.notify.notify_one(); + + Ok(status_id) + } + + pub async fn try_wait_for_task( + &self, + status_id: &AppStatusChangeId, + wait_timeout: Duration, + ) -> Option> { + let interval = Duration::from_secs(2); + + let mut interval_timer = tokio::time::interval(interval); + let start_time = tokio::time::Instant::now(); + + loop { + tokio::select! { + _ = interval_timer.tick() => { + match timeout(wait_timeout, self.db.peek_result(status_id)).await { + Ok(Some(result)) => return Some(result), + Ok(None) => continue, + Err(err) => { + log::debug!("Did not receive result within {} sec: {err}", wait_timeout.as_secs()); + break; + } + } + } + _ = sleep_until(start_time + wait_timeout) => { + log::debug!("Timeout reached, stopping querying the queue"); + break; + } + } + } + + None + } +} + +struct AppTaskQueueConsumer { + db: Arc, + notify: Arc, +} + +impl AppTaskQueueConsumer { + pub async fn process_next_task(&self, apps: &Apps) -> Result<()> { + tokio::select! { + _ = self.notify.notified() => { + log::debug!("Got notified by another thread to check for new items in the queue."); + } + _ = sleep(Duration::from_secs(30)) => { + log::debug!("Regular task check."); + } + } + + self.db + .execute_tasks(async |tasks| { + let merged = AppTask::merge_tasks(tasks); + + if log::log_enabled!(log::Level::Debug) { + log::debug!( + "Processing task {} for {}.", + merged.task_to_work_on.status_id(), + merged.task_to_work_on.app_name() + ); + } + + let result = match &merged.task_to_work_on { + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + infrastructure_payload_to_back_up, + .. + } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!( + "Dropping infrastructure objects for {app_name} due to back up." + ); + } + + apps + .delete_app_partially(app_name, infrastructure_payload_to_back_up) + .await + } + AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + app_name, + infrastructure_payload_to_restore, + .. + } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!( + "Restoring infrastructure objects for {app_name} due to restore task." + ); + } + + apps + .restore_app_partially(app_name, infrastructure_payload_to_restore) + .await + }, + AppTask::CreateOrUpdate { + app_name, + replicate_from, + service_configs, + owners, + user_defined_parameters, + .. + } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!("Creating or updating app {app_name}."); + } + + apps + .create_or_update( + app_name, + replicate_from.clone(), + service_configs, + owners.clone(), + user_defined_parameters.clone(), + ) + .await + } + AppTask::Delete { app_name, .. } => { + if log::log_enabled!(log::Level::Debug) { + log::debug!("Deleting app {app_name}."); + } + apps.delete_app(app_name).await + } + }; + + (merged, result) + }) + .await + } + + pub async fn clean_up_done_tasks(&self) -> Result { + let an_hour_ago = Utc::now() - TimeDelta::hours(1); + self.db.clean_up_done_tasks(an_hour_ago).await + } +} + +enum AppTaskStatus { + New, + InProcess, + Done((DateTime, std::result::Result)), +} + +enum AppTaskQueueDB { + InMemory(Mutex>), + DB(AppPostgresRepository), +} + +impl AppTaskQueueDB { + fn inmemory() -> Self { + Self::InMemory(Mutex::new(VecDeque::new())) + } + + fn db(repository: AppPostgresRepository) -> Self { + Self::DB(repository) + } + + pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { + match self { + AppTaskQueueDB::InMemory(mutex) => { + if matches!( + task, + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { .. } + | AppTask::RestoreOnInfrastructureAndDeleteFromBackup { .. } + ) { + anyhow::bail!( + "Backup or restore is not supported by in-memory app queue processing." + ); + } + + let mut queue = mutex.lock().await; + queue.push_back((task, AppTaskStatus::New)); + } + AppTaskQueueDB::DB(db) => { + db.enqueue_task(task).await?; + } + } + + Ok(()) + } + + async fn peek_result( + &self, + status_id: &AppStatusChangeId, + ) -> Option> { + log::debug!("Checking for results for {status_id}."); + match self { + AppTaskQueueDB::InMemory(mutex) => { + let queue = mutex.lock().await; + + for (task, status) in queue.iter() { + if task.status_id() == status_id { + if let AppTaskStatus::Done((_, result)) = status { + return Some(result.clone()); + } + } + } + None + } + AppTaskQueueDB::DB(db) => db.peek_result(status_id).await, + } + } + + async fn execute_tasks(&self, executor: F) -> Result<()> + where + F: FnOnce(Vec) -> Fut, + Fut: Future)>, + { + match self { + AppTaskQueueDB::InMemory(mutex) => { + let tasks = { + let mut queue = mutex.lock().await; + + let mut iter = queue.iter_mut(); + + let mut tasks = Vec::new(); + let mut app_name = None; + for (task, s) in iter.by_ref() { + app_name = match (&s, app_name.take()) { + (AppTaskStatus::New, current_app_name) => { + if current_app_name.is_some() + && Some(task.app_name()) != current_app_name + { + break; + } + *s = AppTaskStatus::InProcess; + tasks.push(task.clone()); + Some(task.app_name()) + } + (AppTaskStatus::InProcess, None) => { + log::warn!("Trying to find tasks to be process but there is currently {} in process", task.app_name()); + tasks.clear(); + break; + } + (AppTaskStatus::InProcess, Some(app_name)) => { + log::error!( + "The interior queue status seem to be messed up while searching for tasks for {app_name} we found a in-process task for {}", + task.app_name() + ); + tasks.clear(); + break; + } + (AppTaskStatus::Done(_), _) => continue, + } + } + + tasks + }; + + if tasks.is_empty() { + return Ok(()); + } + + let (merged, result) = executor(tasks).await; + let done_timestamp = Utc::now(); + + let mut queue = mutex.lock().await; + let task_worked_on = merged.task_to_work_on; + let status_id = task_worked_on.status_id(); + + let Some(task) = queue + .iter_mut() + .find(|(task, _)| task.status_id() == status_id) + else { + anyhow::bail!("Cannot update {status_id} in queue which should be present"); + }; + + task.1 = AppTaskStatus::Done((done_timestamp, result.clone())); + + for (task, s) in queue.iter_mut() { + if merged.tasks_to_be_marked_as_done.contains(task.status_id()) { + *s = AppTaskStatus::Done((done_timestamp, result.clone())); + } + if merged.tasks_to_stay_untouched.contains(task.status_id()) { + *s = AppTaskStatus::New; + } + } + + Ok(()) + } + AppTaskQueueDB::DB(db) => db.lock_queued_tasks_and_perform_executor(executor).await, + } + } + + async fn clean_up_done_tasks(&self, older_than: DateTime) -> Result { + match self { + AppTaskQueueDB::InMemory(mutex) => { + let mut queue = mutex.lock().await; + + let before = queue.len(); + queue.retain( + |(_, status)| !matches!(status, AppTaskStatus::Done((timestamp, _)) if timestamp < &older_than), + ); + + Ok(before - queue.len()) + } + AppTaskQueueDB::DB(db) => db.clean_up_done_tasks(older_than).await, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{db::DatabasePool, models::AppName, sc}; + use rstest::rstest; + use std::collections::HashSet; + use testcontainers_modules::{ + postgres::{self}, + testcontainers::{runners::AsyncRunner, ContainerAsync}, + }; + + enum TestQueue { + InMemory(Box), + DB(Box<(ContainerAsync, AppTaskQueueDB)>), + } + + impl TestQueue { + fn inmemory() -> Self { + Self::InMemory(Box::new(AppTaskQueueDB::inmemory())) + } + + async fn postgres_queue() -> Self { + use sqlx::postgres::PgConnectOptions; + let postgres_instance = postgres::Postgres::default().start().await.unwrap(); + + let connection = PgConnectOptions::new_without_pgpass() + .application_name("PREvant") + .host(&postgres_instance.get_host().await.unwrap().to_string()) + .port(postgres_instance.get_host_port_ipv4(5432).await.unwrap()) + .username("postgres") + .password("postgres"); + + let pool = DatabasePool::connect_with_exponential_backoff(connection) + .await + .unwrap(); + sqlx::migrate!().run(&pool).await.unwrap(); + + Self::DB(Box::new(( + postgres_instance, + AppTaskQueueDB::db(AppPostgresRepository::new(pool)), + ))) + } + + async fn enqueue_task(&self, task: AppTask) -> Result<()> { + match self { + TestQueue::InMemory(queue) => queue.enqueue_task(task).await, + TestQueue::DB(b) => b.1.enqueue_task(task).await, + } + } + + async fn execute_tasks(&self, executor: F) -> Result<()> + where + F: FnOnce(Vec) -> Fut, + Fut: Future)>, + { + match self { + TestQueue::InMemory(queue) => queue.execute_tasks(executor).await, + TestQueue::DB(b) => b.1.execute_tasks(executor).await, + } + } + + async fn peek_result( + &self, + status_id: &AppStatusChangeId, + ) -> Option> { + match self { + TestQueue::InMemory(queue) => queue.peek_result(status_id).await, + TestQueue::DB(b) => b.1.peek_result(status_id).await, + } + } + + async fn clean_up_done_tasks(&self, older_than: DateTime) -> Result { + match self { + TestQueue::InMemory(queue) => queue.clean_up_done_tasks(older_than).await, + TestQueue::DB(b) => b.1.clean_up_done_tasks(older_than).await, + } + } + } + + fn simulate_result(tasks: Vec) -> (MergedAppTask, Result) { + ( + AppTask::merge_tasks(tasks), + Ok(App::new( + vec![crate::models::Service { + id: String::from("nginx-1234"), + state: crate::models::State { + status: crate::models::ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[tokio::test] + async fn inmemory_queue_cannot_handle_back_up_and_restore( + #[future] + #[case] + queue: TestQueue, + ) { + let queue = queue.await; + + let err = queue + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: AppStatusChangeId::new(), + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({})], + }) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + "Backup or restore is not supported by in-memory app queue processing." + ); + + let err = queue + .enqueue_task(AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: AppStatusChangeId::new(), + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({})], + }) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + "Backup or restore is not supported by in-memory app queue processing." + ); + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_nothing_and_execute_single_task( + #[future] + #[case] + queue: TestQueue, + ) { + let queue = queue.await; + + let result = queue + .execute_tasks(async |_tasks| unreachable!("Empty queue shouldn't trigger this code")) + .await; + + assert!(result.is_ok()) + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_and_execute_single_task( + #[future] + #[case] + queue: TestQueue, + ) { + let queue = queue.await; + + let status_id = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id, + app_name: AppName::master(), + }) + .await + .unwrap(); + + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + + let result = queue.peek_result(&status_id).await; + assert!(matches!(result, Some(Ok(_)))); + + let cleaned = queue.clean_up_done_tasks(Utc::now()).await.unwrap(); + assert_eq!(cleaned, 1); + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_and_execute_merged_task( + #[future] + #[case] + queue: TestQueue, + ) { + let queue = queue.await; + + let status_id_1 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + let status_id_2 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }) + .await + .unwrap(); + + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + + let result = queue.peek_result(&status_id_1).await; + assert!(matches!(result, Some(Ok(_)))); + let result = queue.peek_result(&status_id_2).await; + assert!(matches!(result, Some(Ok(_)))); + + let cleaned = queue.clean_up_done_tasks(Utc::now()).await.unwrap(); + assert_eq!(cleaned, 2); + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_and_handle_one_app_at_the_time( + #[future] + #[case] + queue: TestQueue, + ) { + use std::str::FromStr; + + let queue = queue.await; + + let status_id_1 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + + let status_id_2 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::from_str("other").unwrap(), + }) + .await + .unwrap(); + + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + + let result = queue.peek_result(&status_id_1).await; + assert!(matches!(result, Some(Ok(_)))); + let result = queue.peek_result(&status_id_2).await; + assert!(result.is_none()); + } + + #[rstest] + #[case::inmemory(async { TestQueue::inmemory() })] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_and_handle_one_app_after_the_other( + #[future] + #[case] + queue: TestQueue, + ) { + use std::str::FromStr; + + let queue = queue.await; + + let status_id_1 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + + let status_id_2 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::from_str("other").unwrap(), + }) + .await + .unwrap(); + + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + + let result = queue.peek_result(&status_id_1).await; + assert!(matches!(result, Some(Ok(_)))); + let result = queue.peek_result(&status_id_2).await; + assert!(matches!(result, Some(Ok(_)))); + } + + #[rstest] + #[case::postgres(async { TestQueue::postgres_queue().await })] + #[tokio::test] + async fn enqueue_and_handle_none_mergeable_tasks( + #[future] + #[case] + queue: TestQueue, + ) { + let _ = env_logger::builder().is_test(true).try_init(); + + let queue = queue.await; + + let status_id_1 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }) + .await + .unwrap(); + + let status_id_2 = AppStatusChangeId::new(); + queue + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }) + .await + .unwrap(); + + queue + .execute_tasks(async |tasks| simulate_result(tasks)) + .await + .unwrap(); + + let result = queue.peek_result(&status_id_1).await; + assert!(matches!(result, Some(Ok(_)))); + let result = queue.peek_result(&status_id_2).await; + assert!(result.is_none()); + } +} diff --git a/api/src/apps/queue/mod.rs b/api/src/apps/queue/mod.rs deleted file mode 100644 index b2ae6d8e..00000000 --- a/api/src/apps/queue/mod.rs +++ /dev/null @@ -1,430 +0,0 @@ -use crate::apps::repository::AppPostgresRepository; -use crate::apps::{Apps, AppsError}; -use crate::models::{App, AppName, AppStatusChangeId, AppTask, Owner, ServiceConfig}; -use anyhow::Result; -use chrono::{DateTime, TimeDelta, Utc}; -use rocket::{ - fairing::{Fairing, Info, Kind}, - Build, Orbit, Rocket, -}; -use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration}; -use tokio::{ - sync::{Mutex, Notify}, - time::{sleep, sleep_until, timeout}, -}; - -pub struct AppProcessingQueue {} - -impl AppProcessingQueue { - pub fn fairing() -> Self { - Self {} - } -} - -#[rocket::async_trait] -impl Fairing for AppProcessingQueue { - fn info(&self) -> Info { - Info { - name: "app-queue", - kind: Kind::Ignite | Kind::Liftoff, - } - } - - async fn on_ignite(&self, rocket: Rocket) -> rocket::fairing::Result { - let db = match rocket.state::>() { - Some(Some(repository)) => AppTaskQueueDB::db(repository.clone()), - _ => AppTaskQueueDB::inmemory(), - }; - - let producer = AppTaskQueueProducer { - db: Arc::new(db), - notify: Arc::new(Notify::new()), - }; - - Ok(rocket.manage(producer)) - } - - async fn on_liftoff(&self, rocket: &Rocket) { - let apps = rocket.state::().unwrap(); - let producer = rocket.state::().unwrap(); - let consumer = AppTaskQueueConsumer { - db: producer.db.clone(), - notify: producer.notify.clone(), - }; - let mut shutdown = rocket.shutdown(); - - let apps = apps.clone(); - rocket::tokio::spawn(async move { - loop { - tokio::select! { - res = consumer.process_next_task(&apps) => { - if let Err(err) = res { - log::error!("Cannot process task: {err}"); - } - } - _ = &mut shutdown => { - log::info!("Shutting down queue processing"); - break; - } - }; - - match consumer.clean_up_done_tasks().await { - Ok(number_of_deleted_tasks) if number_of_deleted_tasks > 0 => { - log::debug!("Deleted {number_of_deleted_tasks} done tasks"); - } - Err(err) => { - log::error!("Cannot cleanup done task: {err}"); - } - _ => {} - } - } - }); - } -} - -pub struct AppTaskQueueProducer { - db: Arc, - notify: Arc, -} -impl AppTaskQueueProducer { - pub async fn enqueue_create_or_update_task( - &self, - app_name: AppName, - replicate_from: Option, - service_configs: Vec, - owner: Option, - user_defined_parameters: Option, - ) -> Result { - let status_id = AppStatusChangeId::new(); - self.db - .enqueue_task(AppTask::CreateOrUpdate { - app_name, - status_id, - replicate_from, - service_configs, - owners: owner.into_iter().collect(), - user_defined_parameters, - }) - .await?; - - if log::log_enabled!(log::Level::Debug) { - log::debug!("Notify about new create or update task: {status_id}."); - } - self.notify.notify_one(); - - Ok(status_id) - } - - pub async fn enqueue_delete_task(&self, app_name: AppName) -> Result { - let status_id = AppStatusChangeId::new(); - self.db - .enqueue_task(AppTask::Delete { - app_name, - status_id, - }) - .await?; - - if log::log_enabled!(log::Level::Debug) { - log::debug!("Notify about new delete task: {status_id}."); - } - self.notify.notify_one(); - - Ok(status_id) - } - - pub async fn enqueue_backup_task( - &self, - app_name: AppName, - infrastructure_payload: Vec, - ) -> Result { - let status_id = AppStatusChangeId::new(); - self.db - .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { - status_id, - app_name, - infrastructure_payload_to_back_up: infrastructure_payload, - }) - .await?; - - if log::log_enabled!(log::Level::Debug) { - log::debug!("Notify about new backup task: {status_id}."); - } - self.notify.notify_one(); - - Ok(status_id) - } - - pub async fn enqueue_restore_task( - &self, - app_name: AppName, - infrastructure_payload: Vec, - ) -> Result { - let status_id = AppStatusChangeId::new(); - self.db - .enqueue_task(AppTask::RestoreOnInfrastructureAndDeleteFromBackup { - status_id, - app_name, - infrastructure_payload_to_restore: infrastructure_payload, - }) - .await?; - - if log::log_enabled!(log::Level::Debug) { - log::debug!("Notify about new restore task: {status_id}."); - } - self.notify.notify_one(); - - Ok(status_id) - } - - pub async fn try_wait_for_task( - &self, - status_id: &AppStatusChangeId, - wait_timeout: Duration, - ) -> Option> { - let interval = Duration::from_secs(2); - - let mut interval_timer = tokio::time::interval(interval); - let start_time = tokio::time::Instant::now(); - - loop { - tokio::select! { - _ = interval_timer.tick() => { - match timeout(wait_timeout, self.db.peek_result(status_id)).await { - Ok(Some(result)) => return Some(result), - Ok(None) => continue, - Err(err) => { - log::debug!("Did not receive result within {} sec: {err}", wait_timeout.as_secs()); - break; - } - } - } - _ = sleep_until(start_time + wait_timeout) => { - log::debug!("Timeout reached, stopping querying the queue"); - break; - } - } - } - - None - } -} - -struct AppTaskQueueConsumer { - db: Arc, - notify: Arc, -} - -impl AppTaskQueueConsumer { - pub async fn process_next_task(&self, apps: &Apps) -> Result<()> { - tokio::select! { - _ = self.notify.notified() => { - log::debug!("Got notified by another thread to check for new items in the queue."); - } - _ = sleep(Duration::from_secs(30)) => { - log::debug!("Regular task check."); - } - } - - self.db - .execute_tasks(async |tasks| { - let Some(task) = tasks.into_iter().reduce(|acc, e| acc.merge_with(e)) else { - panic!("tasks must not be empty"); - }; - - if log::log_enabled!(log::Level::Debug) { - log::debug!( - "Processing task {} for {}.", - task.status_id(), - task.app_name() - ); - } - match &task { - AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { - app_name, - infrastructure_payload_to_back_up, - .. - } => { - if log::log_enabled!(log::Level::Debug) { - log::debug!( - "Dropping infrastructure objects for {app_name} due to back up." - ); - } - - let result = apps - .delete_app_partially(app_name, infrastructure_payload_to_back_up) - .await; - (task, result) - } - AppTask::RestoreOnInfrastructureAndDeleteFromBackup { - app_name, - infrastructure_payload_to_restore, - .. - } => { - if log::log_enabled!(log::Level::Debug) { - log::debug!( - "Restoring infrastructure objects for {app_name} due to restore task." - ); - } - - let result = apps - .restore_app_partially(app_name, infrastructure_payload_to_restore) - .await; - (task, result) - }, - AppTask::CreateOrUpdate { - app_name, - replicate_from, - service_configs, - owners, - user_defined_parameters, - .. - } => { - if log::log_enabled!(log::Level::Debug) { - log::debug!("Creating or updating app {app_name}."); - } - - let result = apps - .create_or_update( - app_name, - replicate_from.clone(), - service_configs, - owners.clone(), - user_defined_parameters.clone(), - ) - .await; - (task, result) - } - AppTask::Delete { app_name, .. } => { - if log::log_enabled!(log::Level::Debug) { - log::debug!("Deleting app {app_name}."); - } - let result = apps.delete_app(app_name).await; - (task, result) - } - } - }) - .await - } - - pub async fn clean_up_done_tasks(&self) -> Result { - let an_hour_ago = Utc::now() - TimeDelta::hours(1); - self.db.clean_up_done_tasks(an_hour_ago).await - } -} - -enum AppTaskStatus { - New, - InProcess, - Done((DateTime, std::result::Result)), -} - -enum AppTaskQueueDB { - InMemory(Mutex>), - DB(AppPostgresRepository), -} - -impl AppTaskQueueDB { - fn inmemory() -> Self { - Self::InMemory(Mutex::new(VecDeque::new())) - } - - fn db(repository: AppPostgresRepository) -> Self { - Self::DB(repository) - } - - pub async fn enqueue_task(&self, task: AppTask) -> Result<()> { - match self { - AppTaskQueueDB::InMemory(mutex) => { - // TODO: fail for in memory backup - let mut queue = mutex.lock().await; - queue.push_back((task, AppTaskStatus::New)); - } - AppTaskQueueDB::DB(db) => { - db.enqueue_task(task).await?; - } - } - - Ok(()) - } - - async fn peek_result( - &self, - status_id: &AppStatusChangeId, - ) -> Option> { - log::debug!("Checking for results for {status_id}."); - match self { - AppTaskQueueDB::InMemory(mutex) => { - let queue = mutex.lock().await; - - for (task, status) in queue.iter() { - if task.status_id() == status_id { - if let AppTaskStatus::Done((_, result)) = status { - return Some(result.clone()); - } - } - } - None - } - AppTaskQueueDB::DB(db) => db.peek_result(status_id).await, - } - } - - async fn execute_tasks(&self, f: F) -> Result<()> - where - F: FnOnce(Vec) -> Fut, - Fut: Future)>, - { - match self { - AppTaskQueueDB::InMemory(mutex) => { - let task = { - let mut queue = mutex.lock().await; - - // TODO: we should process multiple tasks here too - let Some(task) = queue - .iter_mut() - .find(|(_, s)| matches!(s, AppTaskStatus::New)) - else { - return Ok(()); - }; - - task.1 = AppTaskStatus::InProcess; - - task.0.clone() - }; - - let status_id = *task.status_id(); - let (task_worked_on, result) = f(vec![task]).await; - - let mut queue = mutex.lock().await; - let Some(task) = queue - .iter_mut() - .find(|(task, _)| task.status_id() == &status_id) - else { - anyhow::bail!("Cannot update {status_id} in queue which should be present"); - }; - - assert!(task_worked_on.status_id() == task.0.status_id()); - task.1 = AppTaskStatus::Done((Utc::now(), result)); - - Ok(()) - } - AppTaskQueueDB::DB(db) => db.update_queued_tasks_with_executor_result(f).await, - } - } - - async fn clean_up_done_tasks(&self, older_than: DateTime) -> Result { - match self { - AppTaskQueueDB::InMemory(mutex) => { - let mut queue = mutex.lock().await; - - let before = queue.len(); - queue.retain( - |(_, status)| !matches!(status, AppTaskStatus::Done((timestamp, _)) if timestamp < &older_than), - ); - - Ok(before - queue.len()) - } - AppTaskQueueDB::DB(db) => db.clean_up_done_tasks(older_than).await, - } - } -} diff --git a/api/src/apps/repository.rs b/api/src/apps/repository.rs index 863b7911..60214caf 100644 --- a/api/src/apps/repository.rs +++ b/api/src/apps/repository.rs @@ -2,7 +2,7 @@ use crate::{ apps::AppsError, models::{ user_defined_parameters::UserDefinedParameters, App, AppName, AppStatusChangeId, AppTask, - Owner, Service, ServiceConfig, ServiceStatus, State, + MergedAppTask, Owner, Service, ServiceConfig, ServiceStatus, State, }, }; use anyhow::Result; @@ -11,7 +11,7 @@ use rocket::{ fairing::{Fairing, Info, Kind}, Build, Orbit, Rocket, }; -use sqlx::{PgPool, Postgres}; +use sqlx::{PgPool, Postgres, Transaction}; use std::{ collections::{HashMap, HashSet}, future::Future, @@ -83,11 +83,14 @@ pub struct BackupUpdateReceiver(pub Receiver>); struct BackupPoller(std::pin::Pin + Send>>); impl AppPostgresRepository { - fn new(pool: PgPool) -> Self { + pub fn new(pool: PgPool) -> Self { Self { pool } } - pub async fn fetch_backup(&self, app_name: &AppName) -> Result>> { + pub async fn fetch_backup_infrastructure_payload( + &self, + app_name: &AppName, + ) -> Result>> { let mut connection = self.pool.acquire().await?; let result = sqlx::query_as::<_, (sqlx::types::Json,)>( @@ -147,6 +150,23 @@ impl AppPostgresRepository { (poller, rx) } + pub async fn fetch_backed_up_app(&self, app_name: &AppName) -> Result> { + let mut connection = self.pool.acquire().await?; + + let result = sqlx::query_as::<_, (String, sqlx::types::Json)>( + r#" + SELECT app_name, app + FROM app_backup + WHERE app_name = $1 + "#, + ) + .bind(app_name.as_str()) + .fetch_optional(&mut *connection) + .await?; + + Ok(result.map(|(_app_name, app)| App::from(app.0))) + } + pub async fn fetch_backed_up_apps(&self) -> Result> { let mut connection = self.pool.acquire().await?; @@ -247,10 +267,10 @@ impl AppPostgresRepository { /// All queued tasks will be locked by a new transaction (see [PostgreSQL as message /// queue](https://www.svix.com/resources/guides/postgres-message-queue/#why-use-postgresql-as-a-message-queue)) /// and the `executor`'s result will be stored for the tasks that could be executed at once. - pub async fn update_queued_tasks_with_executor_result(&self, executor: F) -> Result<()> + pub async fn lock_queued_tasks_and_perform_executor(&self, executor: F) -> Result<()> where F: FnOnce(Vec) -> Fut, - Fut: Future)>, + Fut: Future)>, { let mut tx = self.pool.begin().await?; @@ -288,8 +308,34 @@ impl AppPostgresRepository { return Ok(()); } - let (tasked_worked_on, result) = executor(tasks_to_work_on).await; + let (merged_tasks, result) = executor(tasks_to_work_on).await; + Self::store_result( + &mut tx, + merged_tasks.task_to_work_on, + result, + merged_tasks.tasks_to_be_marked_as_done, + ) + .await?; + + Self::move_untouched_tasks_back_into_queue(&mut tx, merged_tasks.tasks_to_stay_untouched) + .await?; + + tx.commit().await?; + + Ok(()) + } + + async fn store_result( + tx: &mut Transaction<'_, Postgres>, + tasked_worked_on: AppTask, + result: Result, + tasks_to_be_marked_as_done: HashSet, + ) -> Result<()> { let is_success = result.is_ok(); + let is_failed_deletion_due_to_app_not_found = result + .as_ref() + .map_err(|err| matches!(err, AppsError::AppNotFound { .. })) + .map_or_else(|e| e, |_| false); let id = *tasked_worked_on.status_id(); let failed_result = result @@ -330,16 +376,15 @@ impl AppPostgresRepository { .bind(id.as_uuid()) .bind(failed_result) .bind(&success_result) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - if is_success { - if let AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + match tasked_worked_on { + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { app_name, infrastructure_payload_to_back_up, .. - } = tasked_worked_on - { + } if is_success => { log::debug!("Backing-up infrastructure payload for {app_name}."); sqlx::query( @@ -351,11 +396,10 @@ impl AppPostgresRepository { .bind(app_name.as_str()) .bind(success_result) .bind(serde_json::Value::Array(infrastructure_payload_to_back_up)) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - } else if let AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } = - tasked_worked_on - { + } + AppTask::RestoreOnInfrastructureAndDeleteFromBackup { app_name, .. } if is_success => { log::debug!("Deleting infrastructure payload for {app_name} from backups."); sqlx::query( @@ -365,15 +409,28 @@ impl AppPostgresRepository { "#, ) .bind(app_name.as_str()) - .execute(&mut *tx) + .execute(&mut **tx) .await?; } + AppTask::Delete { app_name, .. } + if is_success || is_failed_deletion_due_to_app_not_found => + { + log::debug!("Deleting infrastructure payload for {app_name} from backups due to deletion request."); + + sqlx::query( + r#" + DELETE FROM app_backup + WHERE app_name = $1; + "#, + ) + .bind(app_name.as_str()) + .execute(&mut **tx) + .await?; + } + _ => {} } - // TODO: backup cannot be merged… and we should mark up to this task id. - for (task_id_that_was_merged, _merged_task) in - tasks.iter().filter(|task| task.0 != *id.as_uuid()) - { + for task_id_that_was_merged in tasks_to_be_marked_as_done { sqlx::query( r#" UPDATE app_task @@ -382,12 +439,30 @@ impl AppPostgresRepository { "#, ) .bind(id.as_uuid()) - .bind(task_id_that_was_merged) - .execute(&mut *tx) + .bind(task_id_that_was_merged.as_uuid()) + .execute(&mut **tx) .await?; } - tx.commit().await?; + Ok(()) + } + + async fn move_untouched_tasks_back_into_queue( + tx: &mut Transaction<'_, Postgres>, + tasks_to_stay_untouched: HashSet, + ) -> Result<()> { + for task_id in tasks_to_stay_untouched { + sqlx::query( + r#" + UPDATE app_task + SET status = 'queued' + WHERE id = $1 + "#, + ) + .bind(task_id.as_uuid()) + .execute(&mut **tx) + .await?; + } Ok(()) } @@ -495,10 +570,10 @@ mod tests { .unwrap(); repository - .update_queued_tasks_with_executor_result(async |tasks| { - let task = tasks.last().unwrap().clone(); + .lock_queued_tasks_and_perform_executor(async |tasks| { + let merged = AppTask::merge_tasks(tasks); ( - task, + merged, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -523,6 +598,178 @@ mod tests { assert_eq!(cleaned, 1); } + #[tokio::test] + #[rstest::rstest] + #[case(Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )))] + // simulate that app has been deleted via kubectl or another while the update was in the + // database + #[case(Err(AppsError::AppNotFound { app_name: AppName::master() }))] + async fn clean_up_back_up_after_deletion(#[case] delete_task_result: Result) { + let (_postgres_instance, repository) = create_repository().await; + + let status_id = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({})], + }) + .await + .unwrap(); + + repository + .lock_queued_tasks_and_perform_executor(async |tasks| { + assert_eq!(tasks.len(), 1); + + let merged = AppTask::merge_tasks(tasks); + assert!(matches!( + merged.task_to_work_on, + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { .. } + )); + + ( + merged, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + .unwrap(); + + let status_id = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id, + app_name: AppName::master(), + }) + .await + .unwrap(); + repository + .lock_queued_tasks_and_perform_executor(async |tasks| { + assert_eq!(tasks.len(), 1); + assert!(matches!(tasks[0], AppTask::Delete { .. })); + + let merged = AppTask::merge_tasks(tasks); + + (merged, delete_task_result) + }) + .await + .unwrap(); + + let backups = repository.fetch_backed_up_apps().await.unwrap(); + assert!(backups.is_empty()); + } + + #[tokio::test] + async fn do_not_clean_up_back_up_after_failed_deletion() { + let (_postgres_instance, repository) = create_repository().await; + + let status_id = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({})], + }) + .await + .unwrap(); + + repository + .lock_queued_tasks_and_perform_executor(async |tasks| { + assert_eq!(tasks.len(), 1); + + let merged = AppTask::merge_tasks(tasks); + assert!(matches!( + merged.task_to_work_on, + AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { .. } + )); + + ( + merged, + Ok(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx"), + }], + HashSet::new(), + None, + )), + ) + }) + .await + .unwrap(); + + let status_id = AppStatusChangeId::new(); + repository + .enqueue_task(AppTask::Delete { + status_id, + app_name: AppName::master(), + }) + .await + .unwrap(); + repository + .lock_queued_tasks_and_perform_executor(async |tasks| { + assert_eq!(tasks.len(), 1); + assert!(matches!(tasks[0], AppTask::Delete { .. })); + + let merged = AppTask::merge_tasks(tasks); + + ( + merged, + Err(AppsError::InfrastructureError { + error: String::from("unexpected"), + }), + ) + }) + .await + .unwrap(); + + let backup = repository + .fetch_backed_up_app(&AppName::master()) + .await + .unwrap(); + assert_eq!( + backup, + Some(App::new( + vec![Service { + id: String::from("nginx-1234"), + state: State { + status: ServiceStatus::Paused, + started_at: None, + }, + config: sc!("nginx").with_port(0), + }], + HashSet::new(), + None, + )) + ); + } + #[tokio::test] async fn execute_all_tasks_per_app_name() { let (_postgres_instance, repository) = create_repository().await; @@ -553,7 +800,7 @@ mod tests { let spawn_handle_1 = tokio::spawn(async move { rx.await.unwrap(); spawned_repository - .update_queued_tasks_with_executor_result(async |tasks| { + .lock_queued_tasks_and_perform_executor(async |tasks| { unreachable!("There should be no task to be executed here because the spawned task should have blocked it: {tasks:?}") }) .await @@ -565,14 +812,14 @@ mod tests { let spawn_handle_2 = tokio::spawn(async move { spawned_repository - .update_queued_tasks_with_executor_result(async |tasks| { + .lock_queued_tasks_and_perform_executor(async |tasks| { tx.send(()).unwrap(); tokio::time::sleep(Duration::from_secs(4)).await; - let task = tasks.last().unwrap().clone(); + let merged = AppTask::merge_tasks(tasks); ( - task, + merged, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -638,10 +885,10 @@ mod tests { }; let spawn_handle_1 = tokio::spawn(async move { spawned_repository - .update_queued_tasks_with_executor_result(async |tasks| { - let task = tasks.last().unwrap().clone(); + .lock_queued_tasks_and_perform_executor(async |tasks| { + let merged = AppTask::merge_tasks(tasks); ( - task, + merged, Ok(App::new( vec![Service { id: String::from("nginx-1234"), @@ -665,10 +912,10 @@ mod tests { }; let spawn_handle_2 = tokio::spawn(async move { spawned_repository - .update_queued_tasks_with_executor_result(async |tasks| { - let task = tasks.last().unwrap().clone(); + .lock_queued_tasks_and_perform_executor(async |tasks| { + let merged = AppTask::merge_tasks(tasks); ( - task, + merged, Ok(App::new( vec![Service { id: String::from("nginx-1234"), diff --git a/api/src/apps/routes/mod.rs b/api/src/apps/routes/mod.rs index 6d58fead..975b009d 100644 --- a/api/src/apps/routes/mod.rs +++ b/api/src/apps/routes/mod.rs @@ -403,8 +403,10 @@ pub async fn change_app_status( let status_id = match payload.status { AppStatus::Deployed => { - let Some(infrastructure_payload) = - app_repository.fetch_backup(&app_name).await.map_err(|e| { + let Some(infrastructure_payload) = app_repository + .fetch_backup_infrastructure_payload(&app_name) + .await + .map_err(|e| { HttpApiProblem::with_title_and_type(StatusCode::INTERNAL_SERVER_ERROR) .detail(e.to_string()) })? diff --git a/api/src/infrastructure/kubernetes/deployment_unit.rs b/api/src/infrastructure/kubernetes/deployment_unit.rs index de0294a1..ef2bc57d 100644 --- a/api/src/infrastructure/kubernetes/deployment_unit.rs +++ b/api/src/infrastructure/kubernetes/deployment_unit.rs @@ -1185,7 +1185,12 @@ impl K8sDeploymentUnit { Ok(()) } - pub fn prepare_for_back_up(mut self) -> Self { + /// Clears out any Kubernetes object that shouldn't be put into the backup. For example, + /// [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) are + /// removed from `self` and then `self` can be deleted from the infrastructure with + /// [`Self::delete`]. + pub(super) fn prepare_for_back_up(mut self) -> Self { + // Keep only the pods that aren't created by a deployment self.pods.retain(|pod| { self.deployments.iter().any(|deployment| { let Some(spec) = deployment.spec.as_ref() else { @@ -1208,6 +1213,9 @@ impl K8sDeploymentUnit { empty_read_only_fields!(self.stateful_sets, status); + // Clear the volume mounts and keep them on the Kubernetes infrastructure because they + // might contain data that a tester of the application crafted for a long time and this + // should be preserved. self.config_maps.clear(); self.secrets.clear(); self.pvcs.clear(); @@ -1218,7 +1226,11 @@ impl K8sDeploymentUnit { empty_read_only_fields!(self.pods, status); empty_read_only_fields!(self.deployments, status); - empty_read_only_fields!(self.jobs); + // The jobs won't be contained in the back-up because “Jobs represent one-off tasks that + // run to completion and then stop.” If they are part of the back-up they would restart and + // try to the same thing again which might be already done. + self.jobs.clear(); + empty_read_only_fields!(self.service_accounts); empty_read_only_fields!(self.policies); empty_read_only_fields!(self.traefik_middlewares); @@ -2604,6 +2616,33 @@ mod tests { ); } + #[tokio::test] + async fn clean_jobs() { + let unit = parse_unit_from_log_stream( + r#" +apiVersion: batch/v1 +kind: Job +metadata: + name: pi +spec: + template: + spec: + containers: + - name: pi + image: perl:5.34.0 + command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] + restartPolicy: Never + backoffLimit: 4 "#, + ) + .await; + + assert!(!unit.jobs.is_empty()); + + let prepared_for_back_up = unit.prepare_for_back_up(); + + assert!(prepared_for_back_up.jobs.is_empty()); + } + #[tokio::test] async fn clean_volume_mounts() { let unit = parse_unit_from_log_stream( diff --git a/api/src/models/app_task.rs b/api/src/models/app_task.rs index 8f25a14b..e5a0a5fc 100644 --- a/api/src/models/app_task.rs +++ b/api/src/models/app_task.rs @@ -31,6 +31,24 @@ pub enum AppTask { }, } +#[derive(Debug, thiserror::Error, PartialEq)] +pub enum AppTaskMergeError { + #[error("Merge of {status_id_1} and {status_id_2} for {app_name} is not possible.")] + NotMergable { + app_name: AppName, + status_id_1: AppStatusChangeId, + status_id_2: AppStatusChangeId, + }, +} + +/// Provides a summary of [`AppTask::fold`]. +#[derive(Debug, PartialEq)] +pub struct MergedAppTask { + pub task_to_work_on: AppTask, + pub tasks_to_be_marked_as_done: HashSet, + pub tasks_to_stay_untouched: HashSet, +} + impl AppTask { pub fn app_name(&self) -> &AppName { match self { @@ -49,7 +67,51 @@ impl AppTask { } } - pub fn merge_with(self, other: AppTask) -> Self { + pub fn merge_tasks(tasks: I) -> MergedAppTask + where + I: IntoIterator, + { + let mut iter = tasks.into_iter(); + + let Some(task_to_work_on) = iter.next() else { + panic!("At least one task must be provided"); + }; + + let mut merged_app_task = MergedAppTask { + task_to_work_on, + tasks_to_be_marked_as_done: HashSet::new(), + tasks_to_stay_untouched: HashSet::new(), + }; + + for task in iter.by_ref() { + let previous_task = merged_app_task.task_to_work_on; + let previous_task_id = *previous_task.status_id(); + + match previous_task.clone().merge_with(task) { + Ok(task) => { + merged_app_task + .tasks_to_be_marked_as_done + .insert(previous_task_id); + merged_app_task.task_to_work_on = task; + } + Err(AppTaskMergeError::NotMergable { status_id_2, .. }) => { + merged_app_task.task_to_work_on = previous_task; + merged_app_task.tasks_to_stay_untouched.insert(status_id_2); + break; + } + }; + } + + for task_to_stay_untouched in iter { + merged_app_task + .tasks_to_stay_untouched + .insert(*task_to_stay_untouched.status_id()); + } + + merged_app_task + } + + pub fn merge_with(self, other: AppTask) -> Result { assert_eq!(self.app_name(), other.app_name()); match (self, other) { ( @@ -96,7 +158,7 @@ impl AppTask { .collect::>(); owners.sort_unstable_by(|o1, o2| o1.sub.cmp(&o2.sub)); - Self::CreateOrUpdate { + Ok(Self::CreateOrUpdate { app_name, status_id, replicate_from, @@ -114,7 +176,7 @@ impl AppTask { Some(value) } }, - } + }) } ( Self::CreateOrUpdate { .. }, @@ -122,10 +184,10 @@ impl AppTask { status_id, app_name, }, - ) => Self::Delete { + ) => Ok(Self::Delete { status_id, app_name, - }, + }), ( Self::Delete { .. }, Self::CreateOrUpdate { @@ -136,25 +198,191 @@ impl AppTask { owners, user_defined_parameters, }, - ) => Self::CreateOrUpdate { + ) => Ok(Self::CreateOrUpdate { app_name, status_id, replicate_from, service_configs, owners, user_defined_parameters, - }, + }), ( Self::Delete { .. }, Self::Delete { status_id, app_name, }, - ) => Self::Delete { + ) => Ok(Self::Delete { status_id, app_name, - }, - _ => unimplemented!(), + }), + ( + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { .. }, + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name, + infrastructure_payload_to_back_up, + }, + ) => Ok(Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id, + app_name, + infrastructure_payload_to_back_up, + }), + ( + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + app_name, + status_id: status_id_1, + .. + }, + Self::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { .. }, + Self::CreateOrUpdate { + app_name, + status_id, + replicate_from, + service_configs, + owners, + user_defined_parameters, + }, + ) => Ok(Self::CreateOrUpdate { + app_name, + status_id, + replicate_from, + service_configs, + owners, + user_defined_parameters, + }), + ( + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { .. }, + Self::Delete { + status_id, + app_name, + }, + ) => Ok(Self::Delete { + status_id, + app_name, + }), + ( + Self::RestoreOnInfrastructureAndDeleteFromBackup { + app_name, + status_id: status_id_1, + .. + }, + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::RestoreOnInfrastructureAndDeleteFromBackup { .. }, + Self::RestoreOnInfrastructureAndDeleteFromBackup { + status_id, + app_name, + infrastructure_payload_to_restore, + }, + ) => Ok(Self::RestoreOnInfrastructureAndDeleteFromBackup { + status_id, + app_name, + infrastructure_payload_to_restore, + }), + ( + Self::RestoreOnInfrastructureAndDeleteFromBackup { .. }, + Self::Delete { + status_id, + app_name, + }, + ) => Ok(Self::Delete { + status_id, + app_name, + }), + ( + Self::RestoreOnInfrastructureAndDeleteFromBackup { + app_name, + status_id: status_id_1, + .. + }, + Self::CreateOrUpdate { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::CreateOrUpdate { + app_name, + status_id: status_id_1, + .. + }, + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::CreateOrUpdate { + app_name, + status_id: status_id_1, + .. + }, + Self::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::Delete { + app_name, + status_id: status_id_1, + .. + }, + Self::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), + ( + Self::Delete { + app_name, + status_id: status_id_1, + .. + }, + Self::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + .. + }, + ) => Err(AppTaskMergeError::NotMergable { + app_name, + status_id_1, + status_id_2, + }), } } } @@ -181,10 +409,10 @@ mod tests { assert_eq!( merged, - AppTask::Delete { + Ok(AppTask::Delete { status_id: status_id_2, app_name: AppName::master(), - }, + }), ); } @@ -208,14 +436,14 @@ mod tests { assert_eq!( merged, - AppTask::CreateOrUpdate { + Ok(AppTask::CreateOrUpdate { status_id: status_id_2, app_name: AppName::master(), replicate_from: None, service_configs: vec![sc!("nginx")], owners: Vec::new(), user_defined_parameters: None, - }, + }), ); } @@ -239,10 +467,10 @@ mod tests { assert_eq!( merged, - AppTask::Delete { + Ok(AppTask::Delete { status_id: status_id_2, app_name: AppName::master(), - }, + }), ); } @@ -287,7 +515,7 @@ mod tests { assert_eq!( merged, - AppTask::CreateOrUpdate { + Ok(AppTask::CreateOrUpdate { status_id: status_id_2, app_name: AppName::master(), replicate_from: None, @@ -311,7 +539,474 @@ mod tests { "string-key": "test-overwrite", "array-key": [1, 2, 3, 4, 5, 6] })), - }, + }), ); } + + #[test] + fn merge_back_up_with_backup() { + let t1 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: AppStatusChangeId::new(), + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test-overwrite", + "array-key": [4, 5, 6] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Ok(AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test-overwrite", + "array-key": [4, 5, 6] + })], + }), + ); + } + + #[test] + fn merge_back_up_with_delete() { + let t1 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: AppStatusChangeId::new(), + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Ok(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }), + ); + } + + #[test] + fn merge_back_up_with_create_or_update() { + let t1 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: AppStatusChangeId::new(), + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::CreateOrUpdate { + status_id: status_id_2, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![sc!("nginx")], + owners: Vec::new(), + user_defined_parameters: None, + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Ok(AppTask::CreateOrUpdate { + status_id: status_id_2, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![sc!("nginx")], + owners: Vec::new(), + user_defined_parameters: None, + }), + ); + } + + #[test] + fn merge_back_up_with_restore() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_1, + status_id_2 + }) + ); + } + + #[test] + fn merge_restore_with_back_up() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_1, + status_id_2 + }) + ); + } + + #[test] + fn merge_restore_with_restore() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Ok(AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }) + ); + } + + #[test] + fn merge_restore_with_delete() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Ok(AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }) + ); + } + + #[test] + fn merge_restore_with_create_or_update() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::CreateOrUpdate { + status_id: status_id_2, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![sc!("nginx")], + owners: Vec::new(), + user_defined_parameters: None, + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_1, + status_id_2, + }) + ); + } + + #[test] + fn merge_create_or_update_with_back_up() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::CreateOrUpdate { + status_id: status_id_1, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![sc!("nginx")], + owners: Vec::new(), + user_defined_parameters: None, + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_2, + status_id_1, + }) + ); + } + + #[test] + fn merge_create_or_update_with_restore() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::CreateOrUpdate { + status_id: status_id_1, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![sc!("nginx")], + owners: Vec::new(), + user_defined_parameters: None, + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_2, + status_id_1, + }) + ); + } + + #[test] + fn merge_delete_with_back_up() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::MovePayloadToBackUpAndDeleteFromInfrastructure { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_back_up: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_2, + status_id_1, + }) + ); + } + + #[test] + fn merge_delete_with_restore() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + + let merged = t1.merge_with(t2); + + assert_eq!( + merged, + Err(AppTaskMergeError::NotMergable { + app_name: AppName::master(), + status_id_2, + status_id_1, + }) + ); + } + + mod merge_tasks { + use super::super::*; + + #[test] + #[should_panic] + fn empty() { + AppTask::merge_tasks(Vec::new()); + } + + #[test] + fn all_mergeable() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_1, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }; + + let merge = AppTask::merge_tasks(vec![t1, t2]); + + assert_eq!( + merge, + MergedAppTask { + task_to_work_on: AppTask::Delete { + status_id: status_id_2, + app_name: AppName::master(), + }, + tasks_to_be_marked_as_done: HashSet::from([status_id_1]), + tasks_to_stay_untouched: HashSet::new(), + } + ) + } + + #[test] + fn with_none_mergable_tasks() { + let status_id_1 = AppStatusChangeId::new(); + let t1 = AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }; + let status_id_2 = AppStatusChangeId::new(); + let t2 = AppTask::RestoreOnInfrastructureAndDeleteFromBackup { + status_id: status_id_2, + app_name: AppName::master(), + infrastructure_payload_to_restore: vec![serde_json::json!({ + "string-key": "test", + "array-key": [1, 2, 3] + })], + }; + let status_id_3 = AppStatusChangeId::new(); + let t3 = AppTask::CreateOrUpdate { + status_id: status_id_3, + app_name: AppName::master(), + replicate_from: None, + service_configs: vec![crate::sc!("nginx")], + owners: vec![], + user_defined_parameters: None, + }; + + let merge = AppTask::merge_tasks(vec![t1, t2, t3]); + + assert_eq!( + merge, + MergedAppTask { + task_to_work_on: AppTask::Delete { + status_id: status_id_1, + app_name: AppName::master(), + }, + tasks_to_stay_untouched: HashSet::from([status_id_2, status_id_3]), + tasks_to_be_marked_as_done: HashSet::new(), + } + ) + } + } } diff --git a/api/src/models/mod.rs b/api/src/models/mod.rs index 9f1691aa..4c10d46c 100644 --- a/api/src/models/mod.rs +++ b/api/src/models/mod.rs @@ -30,7 +30,7 @@ pub use app::{ }; pub use app_name::{AppName, AppNameError}; pub use app_status_change_id::{AppStatusChangeId, AppStatusChangeIdError}; -pub use app_task::AppTask; +pub use app_task::{AppTask, MergedAppTask}; pub use image::Image; pub use logs_chunks::LogChunk; pub use request_info::RequestInfo; diff --git a/api/src/models/service_config/mod.rs b/api/src/models/service_config/mod.rs index 6af7c122..f92dd798 100644 --- a/api/src/models/service_config/mod.rs +++ b/api/src/models/service_config/mod.rs @@ -80,8 +80,8 @@ impl ServiceConfig { &self.image } - pub fn set_service_name(&mut self, service_name: &String) { - self.service_name = service_name.clone() + pub fn set_service_name(&mut self, service_name: String) { + self.service_name = service_name } pub fn service_name(&self) -> &String { @@ -139,6 +139,15 @@ impl ServiceConfig { self.port } + /// TODO: This method is only available in tests due to the fact that there are mixed + /// responsibilities for [`ServiceConfig`]: deserializing at REST API level and internal + /// handling within. We have to decouple this responsibility. + #[cfg(test)] + pub fn with_port(mut self, port: u16) -> Self { + self.port = port; + self + } + pub fn set_routing(&mut self, routing: Routing) { self.routing = Some(routing); } @@ -218,7 +227,7 @@ macro_rules! sc { let img_hash = &format!("sha256:{:x}", hasher.finalize()); let mut config = - ServiceConfig::new(String::from($name), crate::models::Image::from_str(img_hash).unwrap()); + ServiceConfig::new(String::from($name), $crate::models::Image::from_str(img_hash).unwrap()); let mut _labels = std::collections::BTreeMap::new(); $( _labels.insert(String::from($l_key), String::from($l_value)); )* @@ -228,9 +237,10 @@ macro_rules! sc { $( _files.insert(std::path::PathBuf::from($v_key), String::from($v_value)); )* config.set_files(Some(_files)); - let mut _env = Vec::new(); - $( _env.push(crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value))); )* - config.set_env(Some(crate::models::Environment::new(_env))); + let mut _env = vec![ + $( $crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value)), )* + ]; + config.set_env(Some($crate::models::Environment::new(_env))); config }}; @@ -240,11 +250,12 @@ macro_rules! sc { ) => {{ use std::str::FromStr; let mut config = - ServiceConfig::new(String::from($name), crate::models::Image::from_str($img).unwrap()); + ServiceConfig::new(String::from($name), $crate::models::Image::from_str($img).unwrap()); - let mut _env = Vec::new(); - $( _env.push(crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value))); )* - config.set_env(Some(crate::models::Environment::new(_env))); + let mut _env = vec![ + $( $crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value)), )* + ]; + config.set_env(Some($crate::models::Environment::new(_env))); config }}; @@ -255,7 +266,7 @@ macro_rules! sc { files = ($($v_key:expr_2021 => $v_value:expr_2021),*) ) => {{ use std::str::FromStr; let mut config = - ServiceConfig::new(String::from($name), crate::models::Image::from_str($img).unwrap()); + ServiceConfig::new(String::from($name), $crate::models::Image::from_str($img).unwrap()); let mut _labels = std::collections::BTreeMap::new(); $( _labels.insert(String::from($l_key), String::from($l_value)); )* @@ -266,8 +277,8 @@ macro_rules! sc { config.set_files(Some(_files)); let mut _env = Vec::new(); - $( _env.push(crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value))); )* - config.set_env(Some(crate::models::Environment::new(_env))); + $( _env.push($crate::models::EnvironmentVariable::new(String::from($env_key), secstr::SecUtf8::from($env_value))); )* + config.set_env(Some($crate::models::Environment::new(_env))); config }}; diff --git a/api/src/models/service_config/templating.rs b/api/src/models/service_config/templating.rs index d95de270..80759e29 100644 --- a/api/src/models/service_config/templating.rs +++ b/api/src/models/service_config/templating.rs @@ -120,7 +120,7 @@ impl ServiceConfig { reg.register_helper("isNotCompanion", Box::new(is_not_companion)); let mut templated_config = self.clone(); - templated_config.set_service_name(®.render_template(self.service_name(), ¶meters)?); + templated_config.set_service_name(reg.render_template(self.service_name(), ¶meters)?); if let Some(env) = self.env() { templated_config.set_env(Some(env.apply_templating(parameters, &mut reg)?)); From b70b8f5c265711d5c8dd9636d62a403ad7881496 Mon Sep 17 00:00:00 2001 From: Marc Schreiber Date: Tue, 23 Dec 2025 23:57:18 +0100 Subject: [PATCH 5/6] Display Backups in Frontend --- frontend/src/components/ReviewAppCard.vue | 2 +- frontend/src/store/index.js | 27 ++++++++++++++--------- frontend/src/views/Apps.vue | 12 ++++++++++ frontend/tests/fixtures/apps.js | 3 ++- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/frontend/src/components/ReviewAppCard.vue b/frontend/src/components/ReviewAppCard.vue index 6bd8e98b..8267ae57 100644 --- a/frontend/src/components/ReviewAppCard.vue +++ b/frontend/src/components/ReviewAppCard.vue @@ -98,7 +98,7 @@ {{ container.name }} {{ container.name }} -