From a25cb5570f897152b75f5b0a70d6f19ade00cc1b Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Mon, 28 Oct 2024 15:04:53 -0400 Subject: [PATCH 01/60] remove aid info from app identity token --- appstore/api/v1/views.py | 4 ++-- appstore/core/models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/appstore/api/v1/views.py b/appstore/api/v1/views.py index 66d46423..0723cc95 100644 --- a/appstore/api/v1/views.py +++ b/appstore/api/v1/views.py @@ -604,7 +604,7 @@ def create(self, request): host = get_host(request) system = tycho.start(principal, app_id, resource_request.resources, host, env) - identity_token.consumer_id = identity_token.compute_app_consumer_id(app_id, system.identifier) + identity_token.consumer_id = identity_token.compute_app_consumer_id(system.identifier) identity_token.save() s = InstanceSpec( @@ -690,7 +690,7 @@ def destroy(self, request, sid=None): if status.services[0].username == request.user.username: response = tycho.delete({"name": serializer.validated_data["sid"]}) # Delete all the tokens the user had associated with that app - consumer_id = UserIdentityToken.compute_app_consumer_id(serializer.validated_data["aid"], serializer.validated_data["sid"]) + consumer_id = UserIdentityToken.compute_app_consumer_id(serializer.validated_data["sid"]) tokens = UserIdentityToken.objects.filter(user=request.user, consumer_id=consumer_id) tokens.delete() # TODO How can we avoid this sleep? Do we need an immediate response beyond diff --git a/appstore/core/models.py b/appstore/core/models.py index 1f0eb9a3..77ebcbc8 100644 --- a/appstore/core/models.py +++ b/appstore/core/models.py @@ -62,8 +62,8 @@ def valid(self): return timezone.now() <= self.expires @staticmethod - def compute_app_consumer_id(app_id, system_id): - return f"{ app_id }-{ system_id }" + def compute_app_consumer_id(system_id): + return f"{ system_id }" def __str__(self): return f"{ self.user.get_username() }-token-{ self.pk }" From f4e92117d7d8eada9941b9bafd85e189c33d70ba Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 19 Mar 2025 13:02:19 -0400 Subject: [PATCH 02/60] Fix log level bug in makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 70013d7a..42a2f426 100644 --- a/Makefile +++ b/Makefile @@ -51,11 +51,11 @@ DEBUG := false endif ifndef LOG_LEVEL -LOG_LEVEL := "info" +LOG_LEVEL := "INFO" endif ifeq "${DEBUG}" "true" -LOG_LEVEL := "debug" +LOG_LEVEL := "DEBUG" endif ifeq "${ENVS_FROM_FILE}" "true" From 3e39569330283a8a4aaa447955f30d112240e73d Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 19 Mar 2025 14:25:22 -0400 Subject: [PATCH 03/60] change tycho info logging to debug level --- appstore/tycho/actions.py | 2 +- appstore/tycho/client.py | 10 +++++----- appstore/tycho/config.py | 2 +- appstore/tycho/context.py | 14 +++++++------- appstore/tycho/kube.py | 8 ++++---- appstore/tycho/model.py | 8 ++++---- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/appstore/tycho/actions.py b/appstore/tycho/actions.py index 61e85048..be431278 100644 --- a/appstore/tycho/actions.py +++ b/appstore/tycho/actions.py @@ -80,7 +80,7 @@ class StartSystemResource(TychoResource): def post(self, request): response = {} try: - logger.info(f"actions.StartSystemResource.post - start-system: {json.dumps(request, indent=2)}") + logger.debug(f"actions.StartSystemResource.post - start-system: {json.dumps(request, indent=2)}") self.validate(request, component="System") system = tycho().parse(request) response = self.create_response( diff --git a/appstore/tycho/client.py b/appstore/tycho/client.py index 85ce49eb..9c1a1bbd 100644 --- a/appstore/tycho/client.py +++ b/appstore/tycho/client.py @@ -350,7 +350,7 @@ def patch(self, mod_items): :returns: A list of all the patches applied to the system :rtype: A list """ - logger.info(f"System specifications and metadata to be modified: {mod_items}") + logger.debug(f"System specifications and metadata to be modified: {mod_items}") try: response = self.modify(mod_items) logger.debug(f"TychoClient.patch - {json.dumps(response, indent=2)}") @@ -422,7 +422,7 @@ def get_client (self, name="tycho-api", namespace="default", default_url="http:/ if len(ip) > 0: try: ipaddress.ip_address (ip) - logger.info (f"configuring minikube ip: {ip}") + logger.debug (f"configuring minikube ip: {ip}") port = service.spec.ports[0].node_port logger.debug (f"located tycho api instance in minikube") url = f"http://{ip}:{port}" @@ -433,9 +433,9 @@ def get_client (self, name="tycho-api", namespace="default", default_url="http:/ except Exception as e: url = default_url print(f"url: {url}") - logger.info (f"cannot find {name} in namespace {namespace}") + logger.debug (f"cannot find {name} in namespace {namespace}") - logger.info (f"creating tycho client with url: {url}") + logger.debug (f"creating tycho client with url: {url}") return TychoClient (url=url) @@ -578,7 +578,7 @@ def getmetadata(self): """ That didn't work so use the default value. """ client = TychoClient (url=args.service) if not client: - logger.info (f"creating client directly {args.service}") + logger.debug (f"creating client directly {args.service}") client = TychoClient (url=args.service) if args.up: diff --git a/appstore/tycho/config.py b/appstore/tycho/config.py index f53091de..7f47ab7a 100644 --- a/appstore/tycho/config.py +++ b/appstore/tycho/config.py @@ -31,7 +31,7 @@ def __init__(self, config="conf/tycho.yaml"): if len(ip) > 0: try: ipaddress.ip_address (ip) - logger.info (f"configuring minikube ip: {ip}") + logger.debug (f"configuring minikube ip: {ip}") self.conf['tycho']['compute']['platform']['kube']['ip'] = ip except ValueError as e: logger.error ("unable to get minikube ip address") diff --git a/appstore/tycho/context.py b/appstore/tycho/context.py index bfb132ea..20e56977 100644 --- a/appstore/tycho/context.py +++ b/appstore/tycho/context.py @@ -47,7 +47,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app if tycho_config_url != "": tycho_config_url += "/" if not tycho_config_url.endswith("/") else "" self.tycho_config_url = tycho_config_url - logger.info (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") + logger.debug (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") self.http_session = CachedSession (cache_name='tycho-registry') self.registry = self._get_config(registry_config) self.app_defaults = self._get_config(app_defaults_config) @@ -62,7 +62,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app def _get_config(self, file_name): """ Load the registry metadata. """ - logger.info (f"-- loading config:\n file_name: {file_name}\ntycho_config_url: {self.tycho_config_url}") + logger.debug (f"-- loading config:\n file_name: {file_name}\ntycho_config_url: {self.tycho_config_url}") config = {} if self.tycho_config_url == "": """ Load it from the Tycho conf directory for now. Perhaps more dynamic in the future. """ @@ -113,7 +113,7 @@ def mixin(self,contexts,context,apps): for mixer in context.get("mixin", []): for app in apps: if contexts.get(mixer,None) != None and contexts[mixer].get("apps",None) != None and contexts[mixer]["apps"].get(app,None) != None: - logger.info("mixing " + app) + logger.debug("mixing " + app) apps[app] = mixin_merge.merge(copy.deepcopy(apps[app]),copy.deepcopy(contexts[mixer]["apps"].get(app))) return apps @@ -123,7 +123,7 @@ def _grok (self): contexts = self.registry.get ('contexts', {}) if not self.product in contexts: raise ContextException (f"undefined product {self.product} not found in contexts.") - logger.info (f"-- load-context: id:{self.product}") + logger.debug (f"-- load-context: id:{self.product}") ''' context = contexts[self.product] apps = context.get ('apps', {}) @@ -286,7 +286,7 @@ def update(self, request): def start (self, principal, app_id, resource_request, host, extra_container_env={}): """ Get application metadata, docker-compose structure, settings, and compose API request. """ - logger.info(f"\nprincipal: {principal}\napp_id: {app_id}\n" + logger.debug(f"\nprincipal: {principal}\napp_id: {app_id}\n" f"resource_request: {resource_request}\nhost: {host}") spec = self.get_spec (app_id) logger.debug(f"context.start - \nspec: {spec}") @@ -358,7 +358,7 @@ def start (self, principal, app_id, resource_request, host, extra_container_env= running = { v.name : v.port for v in system.services } for name, port in services.items (): assert name in running, f"Svc {name} expected but {services.keys()} actually running." - logger.info ( + logger.debug ( f" -- started app id:{app_id} user:{principal.username} id:{system.identifier} services:{list(running.items ())}") return system @@ -429,7 +429,7 @@ def __init__(self): logger.debug("ContextFactory.__init__: creating contexts dictionary") self.contexts = {} def get (self, product, registry_config="app-registry.yaml", app_defaults_config="app-defaults.yaml", context_type="null", tycho_config_url=""): - logger.info (f"-- ContextFactory.get: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {tycho_config_url} | context_type: {context_type}") + logger.debug (f"-- ContextFactory.get: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {tycho_config_url} | context_type: {context_type}") if context_type in self.contexts: logger.debug(f"ContextFactory.get: returning existing context for {context_type}") returnContext = self.contexts[context_type] diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index ff9f3fea..1fd1ae97 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -74,7 +74,7 @@ def check_volumes(self, volumes, namespace): continue else: notExists = False - logger.info(f"PVC {volume['volume_name']} exists.") + logger.debug(f"PVC {volume['volume_name']} exists.") break if notExists and volume["volume_name"] != 'stdnfs': volumesNA.append(index) @@ -89,7 +89,7 @@ def is_ambassador_context(self, namespace, ambassador_service_name): field_sel_api_response = self.api.list_namespaced_service(field_selector=f"metadata.name={ambassador_service_name}", namespace=namespace) return len(field_sel_api_response.items) == 1 except ApiException as e: - logger.info(f"There was a problem assessing whether the ambassador service is running.", e) + logger.debug(f"There was a problem assessing whether the ambassador service is running.", e) def start (self, system, namespace="default"): """ Start an abstractly described distributed system on the cluster. @@ -243,7 +243,7 @@ def start (self, system, namespace="default"): message=f"Unable to start system: {system.name}", details=text) - logger.info (f"result of the app launch: {json.dumps(result,indent=2)}") + logger.debug (f"result of the app launch: {json.dumps(result,indent=2)}") return result def get_service_ip_address (self, service_metadata): @@ -258,7 +258,7 @@ def get_service_ip_address (self, service_metadata): ip_address = None if os.environ.get("DEV_PHASE", "prod") != "test" else "127.0.0.1" try: app_id = service_metadata.metadata.labels["tycho-app"] - logger.info (f"-================================> *** {app_id}") + logger.debug (f"-================================> *** {app_id}") if not app_id in port_forwards: port_forwards[app_id] = app_id #process.pid sleep (3) diff --git a/appstore/tycho/model.py b/appstore/tycho/model.py index bdd80f79..8d340f02 100644 --- a/appstore/tycho/model.py +++ b/appstore/tycho/model.py @@ -184,7 +184,7 @@ def __init__(self, config, name, principal, service_account, conn_string, proxy_ containers_exist = len(containers) > 0 none_are_null = not any([ c for c in containers if c == None ]) assert containers_exist and none_are_null, "System container elements may not be null." - logger.info(f"=======> Constructing system from containers = {containers}") + logger.debug(f"=======> Constructing system from containers = {containers}") self.containers = list(map(lambda v : Container(**v), containers)) \ if isinstance(containers[0], dict) else \ containers @@ -242,11 +242,11 @@ def __init__(self, config, name, principal, service_account, conn_string, proxy_ self.proxy_rewrite = proxy_rewrite # """Flag for checking if an IRODS connection is enabled""" if os.environ.get("IROD_HOST") != None: - logger.info("Irods host enabled") + logger.debug("Irods host enabled") self.irods_enabled = True self.nfsrods_host = os.environ.get('NFSRODS_HOST', '') else: - logger.info("Irods host not enabled") + logger.debug("Irods host not enabled") """gitea settings""" self.gitea_integration = gitea_integration self.gitea_host = os.environ.get("GITEA_HOST", " ") @@ -403,7 +403,7 @@ def parse (config, name, principal, system, service_account, env={}, services={} volume = volume.replace(k, v) spec.get('volumes', []).append(volume) except Exception as e: - logger.info("No volumes specified in the configuration.") + logger.debug("No volumes specified in the configuration.") """ Adding entrypoint to container if exists """ if isinstance(entrypoint, str): entrypoint = entrypoint.split () From f94156b75d5b0e0434340999190266233e35e504 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 19 Mar 2025 14:36:38 -0400 Subject: [PATCH 04/60] add select info logs back to tycho --- appstore/tycho/context.py | 2 +- appstore/tycho/kube.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/appstore/tycho/context.py b/appstore/tycho/context.py index 20e56977..70c01b3f 100644 --- a/appstore/tycho/context.py +++ b/appstore/tycho/context.py @@ -47,7 +47,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app if tycho_config_url != "": tycho_config_url += "/" if not tycho_config_url.endswith("/") else "" self.tycho_config_url = tycho_config_url - logger.debug (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") + logger.info (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") self.http_session = CachedSession (cache_name='tycho-registry') self.registry = self._get_config(registry_config) self.app_defaults = self._get_config(app_defaults_config) diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index 1fd1ae97..68d96bfe 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -243,6 +243,7 @@ def start (self, system, namespace="default"): message=f"Unable to start system: {system.name}", details=text) + logger.info (f"launched app { system.name } with sid { system.identifier }") logger.debug (f"result of the app launch: {json.dumps(result,indent=2)}") return result From aca28b939d565711875085cb622fded4a4d325eb Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 19 Mar 2025 14:39:57 -0400 Subject: [PATCH 05/60] add info log for app termination --- appstore/tycho/kube.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index 68d96bfe..bd4491a6 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -354,6 +354,8 @@ def delete (self, name, namespace="default"): raise DeleteException ( message=f"Failed to delete system: {name}", details=text) + + logger.info (f"terminated app with sid { name }") return { } From 06529c861731784187688d217362a174e2f441aa Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 16:28:23 -0400 Subject: [PATCH 06/60] add further logging changes and authentication signal logging --- appstore/api/v1/views.py | 9 ++-- appstore/appstore/adapter.py | 15 +++++- appstore/appstore/settings/base.py | 6 +++ ...ser_alter_authorizeduser_email_and_more.py | 50 +++++++++++++++++++ appstore/core/signals.py | 21 ++++++++ appstore/tycho/actions.py | 2 +- appstore/tycho/client.py | 13 +++-- appstore/tycho/context.py | 2 +- appstore/tycho/kube.py | 2 +- 9 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 appstore/core/migrations/0003_irodauthorizeduser_alter_authorizeduser_email_and_more.py diff --git a/appstore/api/v1/views.py b/appstore/api/v1/views.py index 66d46423..77e1581e 100644 --- a/appstore/api/v1/views.py +++ b/appstore/api/v1/views.py @@ -1,9 +1,10 @@ import functools import logging -from dataclasses import asdict import time import os import re +from typing import Optional +from dataclasses import asdict from django.conf import settings from django.contrib.contenttypes.models import ContentType @@ -358,7 +359,7 @@ def list(self, request): # TODO change this to serializer.data after discovery on nested object data return Response(apps) - def retrieve(self, request, app_id=None): + def retrieve(self, request, app_id: Optional[str]=None): """ Provide app details. """ @@ -790,7 +791,9 @@ def _get_access_token(self, request): if request.session.get("Authorization", None): return request.session["Authorization"].split(" ")[1] else: - logger.error(f"Authorization not set for {request.user.username}") + # This is not necessarily an error, since authorization (access token) + # may or may not be used, i.e., with authentication via sessionid. + logger.debug(f"Authorization not set for {request.user.username}") return None def list(self, request): diff --git a/appstore/appstore/adapter.py b/appstore/appstore/adapter.py index 3896af03..b2816809 100644 --- a/appstore/appstore/adapter.py +++ b/appstore/appstore/adapter.py @@ -1,9 +1,11 @@ +import logging from allauth.account.adapter import DefaultAccountAdapter from allauth.socialaccount.adapter import DefaultSocialAccountAdapter from django.conf import settings from django.forms import ValidationError +logger = logging.getLogger(__file__) class RestrictEmailAdapter(DefaultAccountAdapter): def clean_email(self, email): @@ -55,4 +57,15 @@ def get_logout_redirect_url(self, request): # an error and returning of the route if request.session.get("helx_frontend"): del request.session["helx_frontend"] - return url \ No newline at end of file + return url + +class SocialAccountAdapter(DefaultSocialAccountAdapter): + def on_authentication_error(self, request, provider, error=None, exception=None, extra_context=None): + provider_id = provider.id if provider else "unknown" + error_code = error.name if error else "unknown" + exception_str = str(exception) if exception else "No exception details" + + logger.info(f"User failed to login using allauth:\nprovider id: { provider_id}\nerror code: { error_code }\nexception: { exception_str }") + + # Note: this is a no-op, since this hook is unimplemented in the default (super) adapter class. + return super().on_authentication_error(request, provider, error, exception, extra_context) \ No newline at end of file diff --git a/appstore/appstore/settings/base.py b/appstore/appstore/settings/base.py index 48931e2b..865c49d1 100644 --- a/appstore/appstore/settings/base.py +++ b/appstore/appstore/settings/base.py @@ -159,6 +159,7 @@ SAML_URL = "/accounts/saml" SAML_ACS_URL = "/saml2_auth/acs/" #SAML_ACS_URL = "/sso/acs/" +SOCIALACCOUNT_ADAPATER = "appstore.adapter.SocialAccountAdapter" SOCIALACCOUNT_QUERY_EMAIL = ACCOUNT_EMAIL_REQUIRED SOCIALACCOUNT_STORE_TOKENS = True SOCIALACCOUNT_PROVIDERS = { @@ -344,6 +345,11 @@ "handlers": ["console"], "level": LOG_LEVEL, }, + # Info logs coming from xmlschema are generally irrelevant and crowd the logs + "xmlschema": { + "handlers": ["console"], + "level": "WARNING" + } }, } diff --git a/appstore/core/migrations/0003_irodauthorizeduser_alter_authorizeduser_email_and_more.py b/appstore/core/migrations/0003_irodauthorizeduser_alter_authorizeduser_email_and_more.py new file mode 100644 index 00000000..a686d4cc --- /dev/null +++ b/appstore/core/migrations/0003_irodauthorizeduser_alter_authorizeduser_email_and_more.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2 on 2024-10-28 18:45 + +import core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('core', '0002_auto_20200430_1711'), + ] + + operations = [ + migrations.CreateModel( + name='IrodAuthorizedUser', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('user', models.TextField(max_length=254)), + ('uid', models.IntegerField()), + ], + ), + migrations.AlterField( + model_name='authorizeduser', + name='email', + field=models.EmailField(blank=True, max_length=254), + ), + migrations.AlterField( + model_name='authorizeduser', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='authorizeduser', + name='username', + field=models.CharField(blank=True, max_length=128), + ), + migrations.CreateModel( + name='UserIdentityToken', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(default=core.models.generate_token, max_length=256, unique=True)), + ('consumer_id', models.CharField(default=None, max_length=256, null=True)), + ('expires', models.DateTimeField(default=core.models.user_token_expires)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/appstore/core/signals.py b/appstore/core/signals.py index e69de29b..9c83a634 100644 --- a/appstore/core/signals.py +++ b/appstore/core/signals.py @@ -0,0 +1,21 @@ +import logging +from django.contrib.auth.signals import user_logged_in, user_login_failed, user_logged_out +from django.dispatch import receiver + +logger = logging.getLogger("django") + +@receiver(user_logged_in) +def on_user_logged_in(sender, request, user, **kwargs): + logger.info(f"User { user.username } logged in") + +@receiver(user_logged_out) +def on_user_logged_out(sender, request, user, **kwargs): + # The user object may be None if their session expired prior to logout. + if user: logger.info(f"User { user.username } logged out") + else: logger.info("User logged out (identity unavailable)") + +@receiver(user_login_failed) +def on_user_failed_login(sender, credentials, request): + # This will generally only work for form-based login (i.e., not for allauth). + # Allauth failures are logged within the LoginRedirectAdapter `on_authentication_error` hook. + logger.info(f"User failed to login with username { credentials.get('username', '') }") \ No newline at end of file diff --git a/appstore/tycho/actions.py b/appstore/tycho/actions.py index be431278..f96ca113 100644 --- a/appstore/tycho/actions.py +++ b/appstore/tycho/actions.py @@ -130,7 +130,7 @@ def post(self, request): response = self.create_response( exception=e, message=f"Failed to get system status.") - print(json.dumps(response, indent=2)) + logger.debug (f" { json.dumps(response, indent=2) }") return response diff --git a/appstore/tycho/client.py b/appstore/tycho/client.py index 9c1a1bbd..bcb854b9 100644 --- a/appstore/tycho/client.py +++ b/appstore/tycho/client.py @@ -186,8 +186,7 @@ def delete (self, request): :param request: A request formatted as above. :type request: JSON """ - logger.error (f"-- delete: {json.dumps(request, indent=2)}") - print (f"-- delete: {json.dumps(request, indent=2)}") + logger.debug (f"-- delete service: {json.dumps(request, indent=2)}") return self.request ("delete", request) def status (self, request): @@ -259,10 +258,10 @@ def up (self, name, system, settings=""): response = self.start (request) logger.debug (f"client.up - response: {response}") if response.status == 'error': - print (response.message) + logger.error (f"failed to bring up service: { response.message }") else: format_string = '{:<30} {:<35} {:<15} {:<7}' - print (format_string.format("SERVICE", "GUID", "IP_ADDRESS", "PORT")) + logger.debug (f"brought up service: { format_string.format('SERVICE', 'GUID', 'IP_ADDRESS', 'PORT') }") for service in response.services: print (format_string.format ( TemplateUtils.trunc (service.name, max_len=28), @@ -429,11 +428,11 @@ def get_client (self, name="tycho-api", namespace="default", default_url="http:/ except ValueError as e: logger.error ("unable to get minikube ip address") traceback.print_exc() - print(f"URL: {url}") + logger.debug (f"using client URL: {url}") except Exception as e: url = default_url - print(f"url: {url}") logger.debug (f"cannot find {name} in namespace {namespace}") + logger.debug (f"using fallback client URL: {url}") logger.debug (f"creating tycho client with url: {url}") return TychoClient (url=url) @@ -522,7 +521,7 @@ def getmetadata(self): system = metadata['System'] if 'Settings' in metadata.keys(): settings = metadata['Settings'] - print(f"settings: {settings}") + logger.debug(f"using settings: {settings}") elif args.file: if not args.name: diff --git a/appstore/tycho/context.py b/appstore/tycho/context.py index 70c01b3f..20e56977 100644 --- a/appstore/tycho/context.py +++ b/appstore/tycho/context.py @@ -47,7 +47,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app if tycho_config_url != "": tycho_config_url += "/" if not tycho_config_url.endswith("/") else "" self.tycho_config_url = tycho_config_url - logger.info (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") + logger.debug (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") self.http_session = CachedSession (cache_name='tycho-registry') self.registry = self._get_config(registry_config) self.app_defaults = self._get_config(app_defaults_config) diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index bd4491a6..dfb6318a 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -89,7 +89,7 @@ def is_ambassador_context(self, namespace, ambassador_service_name): field_sel_api_response = self.api.list_namespaced_service(field_selector=f"metadata.name={ambassador_service_name}", namespace=namespace) return len(field_sel_api_response.items) == 1 except ApiException as e: - logger.debug(f"There was a problem assessing whether the ambassador service is running.", e) + logger.info(f"There was a problem assessing whether the ambassador service is running.", e) def start (self, system, namespace="default"): """ Start an abstractly described distributed system on the cluster. From 2197af1d81125cdf72596034c129e0863aa010ad Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 17:20:16 -0400 Subject: [PATCH 07/60] add default helx favicon for api logging purposes --- appstore/core/static/images/favicon.ico | Bin 0 -> 99678 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 appstore/core/static/images/favicon.ico diff --git a/appstore/core/static/images/favicon.ico b/appstore/core/static/images/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..491d22e5026aae3b034515ebda7546daf35dc9cc GIT binary patch literal 99678 zcmeI534C2ey~j^lXv)@LD>vI@A8%2H6409uR2-1q&S%v?@xZ<3odNo$$;e7?+?{Xf6|%$$?M zftzIw*^1f3i5YQAcESo-c6gR$EiFCi>&d@^yeU&cdh;whZtX0aF{3BF)!tb)VfQR+ zYYXY|4cXN5vTQ5*pea4}MZ34bI)bn2f#iinFo8o`vdPJelHvXN5LClD&%$Tfa}%axR9?6XTxkb0uF=yL3>t% zkHCL`ZM4VsvL~DYhk|WSgtNf0XMlCKg9G6p*cvv3ZNai>urt`_yI?c0YzpiRE5hzz zA6vo>@Fw^o*nSI$^WT+gwKIP`*dFvx3+xEH!CtT*>;%@UXd14~i&V7kQ9S*#s6e@# zWks4vPaN9=2U6BH188ssT-DmUuI)T$T3`lve&xY~CxxUyQXna?3@K2!C&YV09*>5J zj6DIoe|bM^hE-uS$G?&BX2L{>&nlxi{{4*W^Mqr1|5t-NguD%iy`TFG;d93Ma5`)X zO)v|5Mlnyk_Bm;7$mc=qbBE6%i_MEXpEte^?*{vRCltruhxCQuGtw`>dOj=N06t57 z7mkJ_1LoP!m9QpEhU>uRty<%ow;`MZKZEM=eNJ+$o8T*O5{v<#;VyztfzLrjn^z!h zy=&oXa46(;h!+dyfzLzsXWe37Gf2;c%fRO@pUu?SF?I&$Sv|h}`7Eejx5K)y2I$)+ zaI9XzG<+C73;zzs!Z9!hoWnO^GN`FGoC>as%J^K{hgk2ca5$)gbzBdtfNRWWWv?s2 zYd9A^2eaUKa2;I%7lVHP7Ffr=wP6~38YK*zQpH;Yi)lIybV?Z)7A6$8F@2sjl@3uTcf2a?}Mc8=ozU}Z=msG_1N!@ z^idq&^`YJ~;UFl1Yh)YnS$;d14$l2P_*LwXYx{KkgS|F{Jm=pQu6qhzD|7xu{XHWb z)^7OsKr4m`oytq(D+2DUcLM3M2)R0!e|SKvH1H3WRqdz9*Rg zz6bOjRST>IHQ%2Md9V>^h>ersBXA!0e#!S)>p=b@3g=j(en*PQQHeK+g7!xh1@aj+x!?*jV5ceZ1p61;|$ zK|AvIdX>Ve3(9?`{Sb^= zz&-CfZ0)=T_JBuWPgoy(=UQX~(kH<0KpmZf{r@MZhwqdfv(FIAiT%#xOK>kd4W~n| zU>fZIhj1Qf=S}b%I1X}O6YmZW!O3txct$K1%mdr#YyITA=S!h3+lw92#=cw+`-6I% z0lk80(EcC5^`LE&!13pU|87HjT+eTSeZhIuwBODt6LmikUWPxy3^0EU@Laciz+fF% z0R~z>+OO`vfOWzC)j!T5-;bnshKFGw`_;t#_ar6eFoR>h2nm&pEaQbZvpq1d5eXR zul_f~v*7x0Ul(ybyaMlrPr)z2^}b@@V&WIUv7Bp7`^ndal|bE#>s=jYf#=vUkZVqC z{x)zL>=n|z=0Ki(YX2=@zt-Qrim9aUg`+?%z6xr78+;dj4esx)U?*72e)4kth>r#J zP}`VitgR=&gW&z7Smt`Y63&G2#R-IZ(f(6S7egr5-=3dGhx97MYTXL1sW>mk+!^|_ ze{0Gs_QkQtv#tKL{pzM;@EYC-e}s?5eB)3yDEo`;i`SvA#q2MP;ogk(Ex; z91eehKNl;9^3ky0HF*Pk4K9Zd!Av*^TuaV-Ti6m@hn_$CfNSbHcme(nHxJSNAJf+T z+h?#0`qFihu|^ryt@akg(G2&q5-5GQ=dA=_n9sR z)8N?d(?d=76)Xe)o$owoG2K_NEZ~}VKdl8D!Dg@(c!t}Ddv{fE4drc$ea1ZEO<_&b z#SqFp&nCmf>hczDmXjckJfuulEASsX(ND3qck^)JAq`*>Epyzuc|2rqc=*4Ye zyoL}rhj{&0$7Fl(-(59yX4!fn&ai84=lD_s@%+xLbU`W0Ug{iEYHP}}&TQ{eHrqIx zK5?&}c-k41zcgn0N6#_to>r#NSg8wfse}LS>(!UKOQSjOe45N7#(t+Y1KObSot7CX zBn6TJNr9w5QXnai6i5mr1(E_ufuulEASsX(ND3qck^-+g1^kX8;CDWL%i%YQe(< zPJVl`Tn6U_els-#P659!^BWw$rStou7Fce-GhuFiLwq~%H)&tl1iQry%3=RXoReT&i3HeyH`U_>=H{$$H z;M(=%o@#~s-bw5p*%oerzrjqH3a`NXVW2VeulJ=t!(1q8;M|`9b<_ua_SMfE>|cAn z2YyF;8+1ZF{&)V`wLiFqJyY+4=fM4A-?>f1EpR`0zc~P2f?3dSupEjSw3Ai?*Zh#@ z9tWk5Xy2+_`wj5Vp$wNnUKZ;X<51@OU3=%kLO2Pw0{!8-sMbK6-6Q+K%P`>l9g}(z zHE_-k!veSmwgbyuZ@Jz@JL|m;eb|rZfPL5Nf9L-h@cZ$-!9KL9mIiS>OryN+`4=>B zj@qSv9Pc1-Jn{SF9l>YUA?ZLn`-$gYz5Cz!yH@o7fe`KKO9RUXHUC1Nt{>0R-@>2Z z#}Lm0&zZGhH_#6ogL!oa>q4CW6`}nK#I>;Q>7X6Q!HX~l;{1*K(xC477y2unb=#AF z1k^ec^}YAwcZ53XklQqn5TTy)|2=eoXJTL6LfIpr4Qg&Z*TTL~)WH38H^lS6vbyG9 z=tsX-pLNdjNO1gii0dGZ?OvGzo=w(U1@f}KbO_~f{!fv1ec5*{Ec*jI2iwA%;0c%y zC&8w$25bVK2KTIOR)$^SWtasugZTmXxc6tz6W3GBH&%mx1GUx0$xtl!dLn!fJ_lPs zUS4a^Q0Dv}2lw=fFuo7m6K8^ByB_U#f4C3+0qXZF@GSj1xCTmKy#f6n>a9=w5Znuu zxkuYW9X0Si^8#EN@?w>IuH-q7kHP=J45%5*hdBRFgmiy(PoYk6{;~c0$o~@f+rV_q1e&-sV))rofj&k*;o>o;FJv7dV4&_?~& zg%86k;QU{x)t>ygejn$0(CfcepVfI`jDhCA5A{4degp2=TfzNw0h|h+c?ZCr@CNYw z-74VqRIvPH_yTx_bb)j4giEUJGEM%nGJnrp=kpdg5%k5^;STV8{VhBJ-WQ$$&sgJU z;W_Z$@+$lj{tli|_k&ufE&pr0#RK_q{vBK|75`UT6>QQj*Ri^|H+P(I z&0Sf;ou&DWja{f)ns?#6u5A36((HNjyISXrD_#CXXIJaxlQ6-!dwNs3J&&ndj7y!_ zSku%icZWEbxy(lXeX4b?fb|-o^81%ur|>VRfPRCO;3&8g7J|R4_!D>!_#3rjAcf^n0sOiKTmkys-w*Zo zG%p8#XFWd`mcwbMDFp@a@g<7prYM#duM2fovn;pW!C|?PtdrY6$k_b6Apo`Wv5q_k{%z&j5c%{)14{ujFfw zWru@n=XhA6-?^rJoTKj=Zh~{bze#!p(*gZ^G<*Xd0e=(zmB62fAB1b*o#1<_yj`)g zkZ0M2aO*JRGt_rLjRvg^ZQ?w);rdeWU5md%KJ0-${oC$bC_V!|OZrggOE>>!_&K;{ zI$$#xaC3p?ic^7y_743~{w>thC^?n2gg8RD?yoUY3I@RrlEByj} z&xD$LsbWmwg1XuzJAm(Av}F;z9fm#FC#(wo%~#a)Nz#WvFWX65-!u17&^CRwGYmP{ z7JR2X7wq#dFdgz>4DQw0@G=zr@m!KPpbvjA6g7Yl)c?4=zq^$>$%29 z%>VT5c>3+LP}J2maw~ZMnF^bL=g^zrG;rS+_qINr1NnW0*nSU(*r)5+ciH>HTbBUa z&x8jd>NwyUpsw~vweMZ;`d~!g!G*E)^GWbDMBD$$^_8#@L>oub|IYgy(0|b;<11ig z$Zd-IBBqHaz_IW%6AAdcPV8Yr}><@+-zUt+uJ znM1I{b80WR3p`u3-}BUa(ZQfC>%t>o*$Xg|`+s-(Ul7KMS{gfkevcqtESMMAjkvfM z|3F%uEk7LM{=b#$VbMACZU0lD-!)LEyHtH21uWRJ0oRwpa}ag+-uiji7aj}wBjx`+ z>EFG7T^L6jJo6V8(%Mjh+}CjqF^>7f+WQQ|d3-9AjfnppKlDG~8VL243ExNE3ihvn zefz-o;Z;yS&jPiK_tuf}zx#gsFqY%GPtS!j!Fyu#Wj$-aGwq^6n_qM78Rq?d*!O?b zk-pkCgv?jynF^mQ8iGy7OlWFrX*xy~y z1S9JI!ak~X4!DQTgS#Ql(K(IU{XY76pctFtI^KwDpT&Ly(dLoze~B@^P#EV+T)U68 z#rt&KYaq0--L7y7c&?ZpvGc#EX&B4#s?Pz>(x{`pKOKhU8KgM=a9v}YbCBI2K9h}v z|6ig1H^Z$E^*M{{IKJ_q)nq{@LSD@^5GPDM=kGnjb#OLZ1SM#MQ3;L%qv3!5{`&w#z3%0DEm$m=2ZLGzq0NZ; zKa8Pm)xM9~{?~dFqPmU>aNo?hV($h@S_H;n+q0KTmoj-v8-;pEKPj zYO@HAgna(hiy-9Jy#_-45%Yhvlem`eJ-@Xz`f|`~ps)_c)6cP&1ehb*+I=cSQUj`qq|yeP8gQ zb(drf*nVv|2mJT)m%=7s{;-D7XL0{W-wbE0|EGtsd`I(Fi2Yy3b$%bPygHZ%eXRk{ zlw6Y#54-;h{cp?lJrM2o%&+YGqTPiyLs|p&75eb+L(k0Hzve{$_vw;j~{dzgIZYrC&NY-jwxuo^_YhSmRte*GJ5F2u3>^nF|hg|@DNhYD>6 zd>*LL$C|YF40r*q2elfOH4yqK`hO95UxzvHUrWGMv~{j(R^0#E-3;+vK-Bd2Jecujo+yQ-G!+)NAj}`i;dkxqZ?ZjunlfdUE z-!~l%OZxLXcpQ{qz3F=nuf5%i;p9Q)hYU{OY{C;q6{|Fw2mmt3u4mnHB%rXDByzT<74Liy99U`%LNI%1uz2ceoyI|GN*K z0c{$z@1uUfHf=4gfkmVbu&mw?`tk3?3&H!Qct5D@gL?al^=KRY@Cet=bHx2$HMaBF z6+Q_*iwUNOFed-*1-aQIVgEQe6D1kOE=J(ha zWzJ*R{BOUZPkruvpwF?R4FkoYKl`<>n$Nm}=;NdG5&iETxf~h?YF}MWo2BmmYMX|V z4)YjR{}08E(P$Iq>T^K!zwwLkuECBU`dX^{f3P;!svqXD-1|Sw)AQpqQ1rjoKdhxq ze|e$5m?VOb;eqd&ywPX8UiPB0U$hR4BYy;{Eao%&hH|AYVj7u}-fAL9CHSO|UT?7Ju5 z!94}85C1m29linHlP?DQ@%hYi`t7g}>o~Hq3;#fc|!ze6JHP zaj*~2|Br%e?-|$zh8%1QYT~@?)BT_(BNpr%j({$3%v)haSn5F=0u98TC)(w{RinsC zAz$D6ENR`m-AGO#jIkE+eV}enz+@P5uq`x#>)AeiUfK;tEZ8@AHhWGw=8d4%DfFX& z`he^0B)A)VudMHe9BdmnjM!%<&%P$}Mk?4htO;KP=i&LA+c8pOE-(ECom>N}L4I#p z(z%5`J;UsC#LwX{p8ob3-nEvWpG!KA)IKSY6i5mr1(E_ufuulEASsX(ND3qck^)JA zq(D+2DUcLM3M2)R0!e|SKvEznkQ7J?Bn6TJNr9w5QXnai6i5mr1(E_ufuulEASsX( zND3qck^)JAq(D+2DUcLM3M2)R0!e|SKvEznkQ7J?Bn6TJNr9w5QXnai6i5mr1(E_u zfuulEASsX(ND3qck^)JAq(D+2DNt0PoI+AySyG^*xtBFvS?_d5*4nGQt<)=7Zs?sX zkF82K4v-$(-CBO7yP=$AJv}unDwXFi%09-PY3*(3fpqgC z((Thow`JoEy<#HG4_N?`SCc17bm&@IG zy3|Jc{M>>z%bVLtx94-|AU(e6kEA>DxnNKGMvdL1JDd1N+4Q8bdAivKrkfi~cQm%g zberk+v7M!!@}~BWk#1|~;-A)U>X^smI>AdAecjq6t0o>1egQA)ntO z=1XJK%)hBjs)-KCQlF-BN2#l+yD?9rAL-WaCI+y)Gh@7ta=EE|exB~iD0-<}zNXxs zr;!DF4Atqjs`9Lo?soEBucf?zpK?liLRV|~39gz$GDS&&Wl4em2mLo$_5c6? literal 0 HcmV?d00001 From 885bb27acdb627e7e9c3ed184da8e7a1f7a36eb2 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 18:51:37 -0400 Subject: [PATCH 08/60] downgrade 403 list users endpoint to debug log, fix signal kwarg bug, upgrade debug_toolbar and fix associated bug --- appstore/appstore/logging.py | 29 +++++++++++++++++++ appstore/appstore/settings/base.py | 11 +++++++ appstore/core/signals.py | 2 +- .../middleware/filter_whitelist_middleware.py | 2 ++ requirements.txt | 2 +- 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 appstore/appstore/logging.py diff --git a/appstore/appstore/logging.py b/appstore/appstore/logging.py new file mode 100644 index 00000000..13bde959 --- /dev/null +++ b/appstore/appstore/logging.py @@ -0,0 +1,29 @@ +import logging + +""" Used to filter out superfluous logs relating to particular API endpoints. """ +class SuperfluousEndpointLogFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + # We do not want to pollute logs with 403s here when all they indicate is that the user is logged out. + if self.is_forbidden_user_list(record): + # Downgrade from a WARNING to a DEBUG. We could also return False to filter it out entirely. + record.levelname = "DEBUG" + record.levelno = 10 + return True + + """ Does the log originate from a request endpoint? + (This should always be true with proper logging configuration, but worthwhile adding the check.) + """ + def is_request_log(self, record) -> bool: + return hasattr(record, "request") and hasattr(record, "status_code") + + """ + A 403 Forbidden is returned by the `GET users` endpoint when a user is logged out/not authenticated. + This response is expected and frequent behavior, since it is the endpoint the frontend uses to + assess authentication status and thus is frequently called. + """ + def is_forbidden_user_list(self, record) -> bool: + return ( + self.is_request_log(record) and + record.request.path_info == "/api/v1/users/" and + record.status_code == 403 + ) diff --git a/appstore/appstore/settings/base.py b/appstore/appstore/settings/base.py index 865c49d1..5406b741 100644 --- a/appstore/appstore/settings/base.py +++ b/appstore/appstore/settings/base.py @@ -323,6 +323,12 @@ "level": LOG_LEVEL, "propagate": False, }, + "django.request": { + "handlers": ["console"], + "level": LOG_LEVEL, + "propagate": False, + "filters": ["skip_superfluous_endpoint_logs"] + }, "django.template": { "handlers": ["console"], "level": LOG_LEVEL, @@ -351,6 +357,11 @@ "level": "WARNING" } }, + "filters": { + "skip_superfluous_endpoint_logs": { + "()": "appstore.logging.SuperfluousEndpointLogFilter" + } + } } csrf_strings = os.environ.get("CSRF_DOMAINS", "") diff --git a/appstore/core/signals.py b/appstore/core/signals.py index 9c83a634..04a70a61 100644 --- a/appstore/core/signals.py +++ b/appstore/core/signals.py @@ -15,7 +15,7 @@ def on_user_logged_out(sender, request, user, **kwargs): else: logger.info("User logged out (identity unavailable)") @receiver(user_login_failed) -def on_user_failed_login(sender, credentials, request): +def on_user_failed_login(sender, credentials, request, **kwargs): # This will generally only work for form-based login (i.e., not for allauth). # Allauth failures are logged within the LoginRedirectAdapter `on_authentication_error` hook. logger.info(f"User failed to login with username { credentials.get('username', '') }") \ No newline at end of file diff --git a/appstore/middleware/filter_whitelist_middleware.py b/appstore/middleware/filter_whitelist_middleware.py index d65e8470..e3e98885 100644 --- a/appstore/middleware/filter_whitelist_middleware.py +++ b/appstore/middleware/filter_whitelist_middleware.py @@ -23,6 +23,8 @@ def __init__(self, get_response=None): self.get_response = get_response else: self.get_response = self._get_response + + super().__init__(get_response) def process_request(self, request): user = request.user diff --git a/requirements.txt b/requirements.txt index aba937ce..6a2cf52b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Django==4.2 django-allauth==0.61.1 django-cors-headers==4.3.1 django-crispy-forms==2.1 -django-debug-toolbar==4.3.0 +django-debug-toolbar==5.1.0 django-extensions==3.2.3 grafana-django-saml2-auth==3.12.0 djangorestframework==3.14.0 From 04f3299474cfb843141ac8bb00e8251ade5297c4 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 18:58:31 -0400 Subject: [PATCH 09/60] downgrade debug-toolbar back to original version due to version conflict out of scope of pr --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a2cf52b..aba937ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Django==4.2 django-allauth==0.61.1 django-cors-headers==4.3.1 django-crispy-forms==2.1 -django-debug-toolbar==5.1.0 +django-debug-toolbar==4.3.0 django-extensions==3.2.3 grafana-django-saml2-auth==3.12.0 djangorestframework==3.14.0 From 3d1cf615bf47aa974ee07c1c1a5a6a1239b59e20 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 19:35:35 -0400 Subject: [PATCH 10/60] Restore tycho logging. Set tycho log level to warning+. Move app start/delete to appstore internals --- appstore/api/v1/views.py | 15 ++++++++++----- appstore/appstore/settings/base.py | 8 ++------ appstore/tycho/actions.py | 4 ++-- appstore/tycho/client.py | 23 ++++++++++++----------- appstore/tycho/config.py | 2 +- appstore/tycho/context.py | 14 +++++++------- appstore/tycho/kube.py | 9 +++------ appstore/tycho/model.py | 8 ++++---- 8 files changed, 41 insertions(+), 42 deletions(-) diff --git a/appstore/api/v1/views.py b/appstore/api/v1/views.py index 96f28c82..c0ae4d97 100644 --- a/appstore/api/v1/views.py +++ b/appstore/api/v1/views.py @@ -623,11 +623,13 @@ def create(self, request): if s: serializer = InstanceSpecSerializer(data=asdict(s)) try: + logger.info(f"Launched app { app_id }-{ system.identifier } for user { username }") serializer.is_valid(raise_exception=True) return Response(serializer.validated_data) - except serializers.ValidationError: + except serializers.ValidationError as e: # Delete invalid instance configuration that we won't be tracking # for the user. + logger.error(f"Failed to launch app { app_id } for user { username }; exception: { str(e) }") tycho.delete({"name": system.services[0].identifier}) return Response( serializer.errors, status=drf_status.HTTP_400_BAD_REQUEST @@ -635,6 +637,7 @@ def create(self, request): else: # Failed to construct a tracked instance, attempt to remove # potentially created instance rather than leaving it hanging. + logger.error(f"Failed to launch app { app_id } for user { username }; null instance spec") tycho.delete({"name": system.services[0].identifier}) identity_token.delete() return Response( @@ -683,12 +686,12 @@ def destroy(self, request, sid=None): """ serializer = self.get_serializer(data={"sid": sid}) serializer.is_valid(raise_exception=True) - logger.debug(f"\nDeleting: {sid}") status = tycho.status({"name": serializer.validated_data["sid"]}) if status.services != None and len(status.services) == 1: - logger.info("service username: " + str(status.services[0].username)) - logger.info("request username: " + str(request.user.username)) + logger.debug("service username: " + str(status.services[0].username)) + logger.debug("request username: " + str(request.user.username)) if status.services[0].username == request.user.username: + logger.info(f"Terminating app id { sid } for user { request.user.username }") response = tycho.delete({"name": serializer.validated_data["sid"]}) # Delete all the tokens the user had associated with that app consumer_id = UserIdentityToken.compute_app_consumer_id(serializer.validated_data["sid"]) @@ -699,7 +702,9 @@ def destroy(self, request, sid=None): # to the front end? time.sleep(2) return Response(response) - else: return Response(status=drf_status.HTTP_403_FORBIDDEN) + else: + logger.warning(f"User { request.user.username } attempted to terminate app id { sid } owned by user { status.services[0].username }") + return Response(status=drf_status.HTTP_403_FORBIDDEN) else: return Response(status=drf_status.HTTP_404_NOT_FOUND) def partial_update(self, request, sid=None): diff --git a/appstore/appstore/settings/base.py b/appstore/appstore/settings/base.py index 5406b741..f3ab83c8 100644 --- a/appstore/appstore/settings/base.py +++ b/appstore/appstore/settings/base.py @@ -343,13 +343,9 @@ "handlers": ["console"], "level": LOG_LEVEL, }, - "tycho.client": { + "tycho": { "handlers": ["console"], - "level": LOG_LEVEL, - }, - "tycho.kube": { - "handlers": ["console"], - "level": LOG_LEVEL, + "level": "WARNING", }, # Info logs coming from xmlschema are generally irrelevant and crowd the logs "xmlschema": { diff --git a/appstore/tycho/actions.py b/appstore/tycho/actions.py index f96ca113..a92f311f 100644 --- a/appstore/tycho/actions.py +++ b/appstore/tycho/actions.py @@ -80,7 +80,7 @@ class StartSystemResource(TychoResource): def post(self, request): response = {} try: - logger.debug(f"actions.StartSystemResource.post - start-system: {json.dumps(request, indent=2)}") + logger.info(f"actions.StartSystemResource.post - start-system: {json.dumps(request, indent=2)}") self.validate(request, component="System") system = tycho().parse(request) response = self.create_response( @@ -130,7 +130,7 @@ def post(self, request): response = self.create_response( exception=e, message=f"Failed to get system status.") - logger.debug (f" { json.dumps(response, indent=2) }") + # print(json.dumps(response, indent=2)) return response diff --git a/appstore/tycho/client.py b/appstore/tycho/client.py index bcb854b9..85ce49eb 100644 --- a/appstore/tycho/client.py +++ b/appstore/tycho/client.py @@ -186,7 +186,8 @@ def delete (self, request): :param request: A request formatted as above. :type request: JSON """ - logger.debug (f"-- delete service: {json.dumps(request, indent=2)}") + logger.error (f"-- delete: {json.dumps(request, indent=2)}") + print (f"-- delete: {json.dumps(request, indent=2)}") return self.request ("delete", request) def status (self, request): @@ -258,10 +259,10 @@ def up (self, name, system, settings=""): response = self.start (request) logger.debug (f"client.up - response: {response}") if response.status == 'error': - logger.error (f"failed to bring up service: { response.message }") + print (response.message) else: format_string = '{:<30} {:<35} {:<15} {:<7}' - logger.debug (f"brought up service: { format_string.format('SERVICE', 'GUID', 'IP_ADDRESS', 'PORT') }") + print (format_string.format("SERVICE", "GUID", "IP_ADDRESS", "PORT")) for service in response.services: print (format_string.format ( TemplateUtils.trunc (service.name, max_len=28), @@ -349,7 +350,7 @@ def patch(self, mod_items): :returns: A list of all the patches applied to the system :rtype: A list """ - logger.debug(f"System specifications and metadata to be modified: {mod_items}") + logger.info(f"System specifications and metadata to be modified: {mod_items}") try: response = self.modify(mod_items) logger.debug(f"TychoClient.patch - {json.dumps(response, indent=2)}") @@ -421,20 +422,20 @@ def get_client (self, name="tycho-api", namespace="default", default_url="http:/ if len(ip) > 0: try: ipaddress.ip_address (ip) - logger.debug (f"configuring minikube ip: {ip}") + logger.info (f"configuring minikube ip: {ip}") port = service.spec.ports[0].node_port logger.debug (f"located tycho api instance in minikube") url = f"http://{ip}:{port}" except ValueError as e: logger.error ("unable to get minikube ip address") traceback.print_exc() - logger.debug (f"using client URL: {url}") + print(f"URL: {url}") except Exception as e: url = default_url - logger.debug (f"cannot find {name} in namespace {namespace}") - logger.debug (f"using fallback client URL: {url}") + print(f"url: {url}") + logger.info (f"cannot find {name} in namespace {namespace}") - logger.debug (f"creating tycho client with url: {url}") + logger.info (f"creating tycho client with url: {url}") return TychoClient (url=url) @@ -521,7 +522,7 @@ def getmetadata(self): system = metadata['System'] if 'Settings' in metadata.keys(): settings = metadata['Settings'] - logger.debug(f"using settings: {settings}") + print(f"settings: {settings}") elif args.file: if not args.name: @@ -577,7 +578,7 @@ def getmetadata(self): """ That didn't work so use the default value. """ client = TychoClient (url=args.service) if not client: - logger.debug (f"creating client directly {args.service}") + logger.info (f"creating client directly {args.service}") client = TychoClient (url=args.service) if args.up: diff --git a/appstore/tycho/config.py b/appstore/tycho/config.py index 7f47ab7a..f53091de 100644 --- a/appstore/tycho/config.py +++ b/appstore/tycho/config.py @@ -31,7 +31,7 @@ def __init__(self, config="conf/tycho.yaml"): if len(ip) > 0: try: ipaddress.ip_address (ip) - logger.debug (f"configuring minikube ip: {ip}") + logger.info (f"configuring minikube ip: {ip}") self.conf['tycho']['compute']['platform']['kube']['ip'] = ip except ValueError as e: logger.error ("unable to get minikube ip address") diff --git a/appstore/tycho/context.py b/appstore/tycho/context.py index 20e56977..bfb132ea 100644 --- a/appstore/tycho/context.py +++ b/appstore/tycho/context.py @@ -47,7 +47,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app if tycho_config_url != "": tycho_config_url += "/" if not tycho_config_url.endswith("/") else "" self.tycho_config_url = tycho_config_url - logger.debug (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") + logger.info (f"-- TychoContext.__init__: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {self.tycho_config_url} | stub: {stub}") self.http_session = CachedSession (cache_name='tycho-registry') self.registry = self._get_config(registry_config) self.app_defaults = self._get_config(app_defaults_config) @@ -62,7 +62,7 @@ def __init__(self, registry_config="app-registry.yaml", app_defaults_config="app def _get_config(self, file_name): """ Load the registry metadata. """ - logger.debug (f"-- loading config:\n file_name: {file_name}\ntycho_config_url: {self.tycho_config_url}") + logger.info (f"-- loading config:\n file_name: {file_name}\ntycho_config_url: {self.tycho_config_url}") config = {} if self.tycho_config_url == "": """ Load it from the Tycho conf directory for now. Perhaps more dynamic in the future. """ @@ -113,7 +113,7 @@ def mixin(self,contexts,context,apps): for mixer in context.get("mixin", []): for app in apps: if contexts.get(mixer,None) != None and contexts[mixer].get("apps",None) != None and contexts[mixer]["apps"].get(app,None) != None: - logger.debug("mixing " + app) + logger.info("mixing " + app) apps[app] = mixin_merge.merge(copy.deepcopy(apps[app]),copy.deepcopy(contexts[mixer]["apps"].get(app))) return apps @@ -123,7 +123,7 @@ def _grok (self): contexts = self.registry.get ('contexts', {}) if not self.product in contexts: raise ContextException (f"undefined product {self.product} not found in contexts.") - logger.debug (f"-- load-context: id:{self.product}") + logger.info (f"-- load-context: id:{self.product}") ''' context = contexts[self.product] apps = context.get ('apps', {}) @@ -286,7 +286,7 @@ def update(self, request): def start (self, principal, app_id, resource_request, host, extra_container_env={}): """ Get application metadata, docker-compose structure, settings, and compose API request. """ - logger.debug(f"\nprincipal: {principal}\napp_id: {app_id}\n" + logger.info(f"\nprincipal: {principal}\napp_id: {app_id}\n" f"resource_request: {resource_request}\nhost: {host}") spec = self.get_spec (app_id) logger.debug(f"context.start - \nspec: {spec}") @@ -358,7 +358,7 @@ def start (self, principal, app_id, resource_request, host, extra_container_env= running = { v.name : v.port for v in system.services } for name, port in services.items (): assert name in running, f"Svc {name} expected but {services.keys()} actually running." - logger.debug ( + logger.info ( f" -- started app id:{app_id} user:{principal.username} id:{system.identifier} services:{list(running.items ())}") return system @@ -429,7 +429,7 @@ def __init__(self): logger.debug("ContextFactory.__init__: creating contexts dictionary") self.contexts = {} def get (self, product, registry_config="app-registry.yaml", app_defaults_config="app-defaults.yaml", context_type="null", tycho_config_url=""): - logger.debug (f"-- ContextFactory.get: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {tycho_config_url} | context_type: {context_type}") + logger.info (f"-- ContextFactory.get: registry_config: {registry_config} | app_defaults_config: {app_defaults_config} | product: {product} | tycho_config_url: {tycho_config_url} | context_type: {context_type}") if context_type in self.contexts: logger.debug(f"ContextFactory.get: returning existing context for {context_type}") returnContext = self.contexts[context_type] diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index dfb6318a..ff9f3fea 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -74,7 +74,7 @@ def check_volumes(self, volumes, namespace): continue else: notExists = False - logger.debug(f"PVC {volume['volume_name']} exists.") + logger.info(f"PVC {volume['volume_name']} exists.") break if notExists and volume["volume_name"] != 'stdnfs': volumesNA.append(index) @@ -243,8 +243,7 @@ def start (self, system, namespace="default"): message=f"Unable to start system: {system.name}", details=text) - logger.info (f"launched app { system.name } with sid { system.identifier }") - logger.debug (f"result of the app launch: {json.dumps(result,indent=2)}") + logger.info (f"result of the app launch: {json.dumps(result,indent=2)}") return result def get_service_ip_address (self, service_metadata): @@ -259,7 +258,7 @@ def get_service_ip_address (self, service_metadata): ip_address = None if os.environ.get("DEV_PHASE", "prod") != "test" else "127.0.0.1" try: app_id = service_metadata.metadata.labels["tycho-app"] - logger.debug (f"-================================> *** {app_id}") + logger.info (f"-================================> *** {app_id}") if not app_id in port_forwards: port_forwards[app_id] = app_id #process.pid sleep (3) @@ -354,8 +353,6 @@ def delete (self, name, namespace="default"): raise DeleteException ( message=f"Failed to delete system: {name}", details=text) - - logger.info (f"terminated app with sid { name }") return { } diff --git a/appstore/tycho/model.py b/appstore/tycho/model.py index 8d340f02..bdd80f79 100644 --- a/appstore/tycho/model.py +++ b/appstore/tycho/model.py @@ -184,7 +184,7 @@ def __init__(self, config, name, principal, service_account, conn_string, proxy_ containers_exist = len(containers) > 0 none_are_null = not any([ c for c in containers if c == None ]) assert containers_exist and none_are_null, "System container elements may not be null." - logger.debug(f"=======> Constructing system from containers = {containers}") + logger.info(f"=======> Constructing system from containers = {containers}") self.containers = list(map(lambda v : Container(**v), containers)) \ if isinstance(containers[0], dict) else \ containers @@ -242,11 +242,11 @@ def __init__(self, config, name, principal, service_account, conn_string, proxy_ self.proxy_rewrite = proxy_rewrite # """Flag for checking if an IRODS connection is enabled""" if os.environ.get("IROD_HOST") != None: - logger.debug("Irods host enabled") + logger.info("Irods host enabled") self.irods_enabled = True self.nfsrods_host = os.environ.get('NFSRODS_HOST', '') else: - logger.debug("Irods host not enabled") + logger.info("Irods host not enabled") """gitea settings""" self.gitea_integration = gitea_integration self.gitea_host = os.environ.get("GITEA_HOST", " ") @@ -403,7 +403,7 @@ def parse (config, name, principal, system, service_account, env={}, services={} volume = volume.replace(k, v) spec.get('volumes', []).append(volume) except Exception as e: - logger.debug("No volumes specified in the configuration.") + logger.info("No volumes specified in the configuration.") """ Adding entrypoint to container if exists """ if isinstance(entrypoint, str): entrypoint = entrypoint.split () From 0af823a7070bb33d28dc7530fa9bae53fbed5580 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 3 Apr 2025 19:55:34 -0400 Subject: [PATCH 11/60] move log order in app launch order, hide delete prints in tycho --- appstore/api/v1/views.py | 2 +- appstore/tycho/client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appstore/api/v1/views.py b/appstore/api/v1/views.py index c0ae4d97..e2908945 100644 --- a/appstore/api/v1/views.py +++ b/appstore/api/v1/views.py @@ -623,8 +623,8 @@ def create(self, request): if s: serializer = InstanceSpecSerializer(data=asdict(s)) try: - logger.info(f"Launched app { app_id }-{ system.identifier } for user { username }") serializer.is_valid(raise_exception=True) + logger.info(f"Launched app { app_id }-{ system.identifier } for user { username }") return Response(serializer.validated_data) except serializers.ValidationError as e: # Delete invalid instance configuration that we won't be tracking diff --git a/appstore/tycho/client.py b/appstore/tycho/client.py index 85ce49eb..12458fb0 100644 --- a/appstore/tycho/client.py +++ b/appstore/tycho/client.py @@ -186,8 +186,8 @@ def delete (self, request): :param request: A request formatted as above. :type request: JSON """ - logger.error (f"-- delete: {json.dumps(request, indent=2)}") - print (f"-- delete: {json.dumps(request, indent=2)}") + logger.info (f"-- delete: {json.dumps(request, indent=2)}") + # print (f"-- delete: {json.dumps(request, indent=2)}") return self.request ("delete", request) def status (self, request): From 3295ab465595c4cbe1fff993f71384b61359eaf3 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Fri, 4 Apr 2025 12:25:01 -0400 Subject: [PATCH 12/60] update dockerfile --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff85daaa..3d967871 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ -FROM python:3.9.18-slim-bullseye +FROM python:3.10.16-slim-bullseye -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 # Least privilege: Run as a non-root user. -ENV USER appstore -ENV APP_HOME /usr/src/inst-mgmt -ENV HOME /home/$USER -ENV UID 1000 +ENV USER=appstore +ENV APP_HOME=/usr/src/inst-mgmt +ENV HOME=/home/$USER +ENV UID=1000 RUN mkdir $APP_HOME From 4dcd98bb7362bd54d45e8da422974d2e57d5305d Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Fri, 4 Apr 2025 14:32:47 -0400 Subject: [PATCH 13/60] revert to python3.9 to avoid pyyaml/cython conflict --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3d967871..30fe1eef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.16-slim-bullseye +FROM python:3.9.18-slim-bullseye ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 6f61803c6ac2946f9e370354b803959cb033c5d8 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Fri, 4 Apr 2025 14:38:03 -0400 Subject: [PATCH 14/60] upgrade from python 3.9 patch 18 to 21 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30fe1eef..afb45d68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.18-slim-bullseye +FROM python:3.9.21-slim-bullseye ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 59209ffd47b8e5bea02d899ab879f006b5431c22 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Fri, 4 Apr 2025 15:15:23 -0400 Subject: [PATCH 15/60] remove pvc collectiondelete that lacked permission --- appstore/tycho/kube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appstore/tycho/kube.py b/appstore/tycho/kube.py index ff9f3fea..8c4932c8 100644 --- a/appstore/tycho/kube.py +++ b/appstore/tycho/kube.py @@ -336,7 +336,7 @@ def delete (self, name, namespace="default"): "deployment" : self.extensions_api.delete_collection_namespaced_deployment, "replica_set" : self.extensions_api.delete_collection_namespaced_replica_set, "pod" : self.api.delete_collection_namespaced_pod, - "persistentvolumeclaim" : self.api.delete_collection_namespaced_persistent_volume_claim, + # "persistentvolumeclaim" : self.api.delete_collection_namespaced_persistent_volume_claim, #"networkpolicy" : self.networking_api.delete_collection_namespaced_network_policy } for object_type, finalizer in finalizers.items (): From 3e9aec324f800908891bfe9ee4fecf18fe08bea2 Mon Sep 17 00:00:00 2001 From: waTeim Date: Thu, 1 May 2025 10:09:25 -0400 Subject: [PATCH 16/60] Set the enableServiceLins to false without exception --- appstore/tycho/template/pod.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/appstore/tycho/template/pod.yaml b/appstore/tycho/template/pod.yaml index 408a1ded..fd376886 100644 --- a/appstore/tycho/template/pod.yaml +++ b/appstore/tycho/template/pod.yaml @@ -21,6 +21,7 @@ metadata: tycho-guid: {{ system.identifier }} tycho-app-id: {{ system.app_id }} spec: + enableServiceLinks: false {% if system.serviceaccount %} serviceAccountName: {{system.serviceaccount}} {% endif %} From 13fdf5cb6c2567a828cb303fe292bfee33323c8e Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:23:46 -0400 Subject: [PATCH 17/60] Upgrade Django to latest 4.2 patch, remove superfluous flask requirements --- requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index aba937ce..a00cb3b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2 +Django==4.2.23 django-allauth==0.61.1 django-cors-headers==4.3.1 django-crispy-forms==2.1 @@ -22,9 +22,6 @@ psycopg2-binary python-irodsclient==1.1.5 deepmerge==1.0.1 flasgger==0.9.5 -Flask==2.0.3 -flask_cors==3.0.10 -flask_restful==0.3.9 GitPython==3.1.11 Jinja2==3.0.3 jsonschema==3.2.0 From 5db139b588aa053a88899ecb7c1842a7d380e510 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:25:56 -0400 Subject: [PATCH 18/60] update gitpython patch --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a00cb3b1..6b187d61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,8 +21,7 @@ asgiref==3.7.2 psycopg2-binary python-irodsclient==1.1.5 deepmerge==1.0.1 -flasgger==0.9.5 -GitPython==3.1.11 +GitPython==3.1.44 Jinja2==3.0.3 jsonschema==3.2.0 kubernetes==25.3.0 From ef725bf833969f36b799f9a131c9fea552ac9306 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:26:49 -0400 Subject: [PATCH 19/60] delete superfluous sqlparse req --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b187d61..2a62bfab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,6 @@ requests==2.31.0 requests-oauthlib==1.4.0 selenium==3.141.0 webdriver-manager==3.2.1 -sqlparse==0.4.4 asgiref==3.7.2 psycopg2-binary python-irodsclient==1.1.5 From 6cf27b724e12353ab72373710937b40b1867e1a5 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:32:28 -0400 Subject: [PATCH 20/60] Upgrade to python 3.9.23 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index afb45d68..1c499a64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.21-slim-bullseye +FROM python:3.9.23-slim-bullseye ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From bed836ae724a0f64f78300bdbe424b5402ce53f8 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:40:38 -0400 Subject: [PATCH 21/60] upgrade gunicorn to 23.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a62bfab..81d16ebc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ grafana-django-saml2-auth==3.12.0 djangorestframework==3.14.0 drf-spectacular==0.27.1 flake8==3.9.0 -gunicorn==20.1.0 +gunicorn==23.0.0 mock==4.0.2 pysaml2==7.4.2 python3-openid==3.2.0 From 8e3f274ec87ac0e90500cb91e1061d1b395e51f3 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:46:29 -0400 Subject: [PATCH 22/60] upgrade requests, jinja --- requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 81d16ebc..ee616e94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ gunicorn==23.0.0 mock==4.0.2 pysaml2==7.4.2 python3-openid==3.2.0 -requests==2.31.0 requests-oauthlib==1.4.0 selenium==3.141.0 webdriver-manager==3.2.1 @@ -21,11 +20,11 @@ psycopg2-binary python-irodsclient==1.1.5 deepmerge==1.0.1 GitPython==3.1.44 -Jinja2==3.0.3 +Jinja2==3.1.6 jsonschema==3.2.0 kubernetes==25.3.0 netifaces==0.11.0 PyYAML==5.4.1 -requests==2.31.0 +requests==2.32.4 docker-compose==1.29.2 requests_cache==0.9.2 From 68247f8e1925274fa8d96dbfc2c6bf05a52bb047 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 13:51:16 -0400 Subject: [PATCH 23/60] upgrade drf --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ee616e94..eee7f2b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ django-crispy-forms==2.1 django-debug-toolbar==4.3.0 django-extensions==3.2.3 grafana-django-saml2-auth==3.12.0 -djangorestframework==3.14.0 +djangorestframework==3.15.2 drf-spectacular==0.27.1 flake8==3.9.0 gunicorn==23.0.0 From 0cf60f0518ea6013303ca052254a1c3c637e3d75 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 14:28:24 -0400 Subject: [PATCH 24/60] test bookworm image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c499a64..aa4246be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.23-slim-bullseye +FROM python:3.9.23-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 5f3ffd2a058cfc7aa2d6e04d473b44c8bf2b1e00 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 16:54:59 -0400 Subject: [PATCH 25/60] test remove build --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa4246be..3a1bd9f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ - && make install \ + # && make install \ && unset SET_BUILD_ENV_FROM_FILE RUN chown -R 1000:0 /usr/src/inst-mgmt From 592b83dfc99d5302e51cef5b465b837f1315f899 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 16:58:52 -0400 Subject: [PATCH 26/60] also remove installs --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a1bd9f6..e5f56f96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,9 @@ RUN mkdir $APP_HOME RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && \ chown -R $UID:$UID $HOME -RUN set -x && apt-get update && \ - chown -R $UID:$UID $APP_HOME && \ - apt-get install -y build-essential git xmlsec1 libpq5 gcc +# RUN set -x && apt-get update && \ +# chown -R $UID:$UID $APP_HOME && \ +# apt-get install -y build-essential git xmlsec1 libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 61df9399f418fc7b571fa60eebf4b3aa81184f21 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:08:40 -0400 Subject: [PATCH 27/60] test removing libpq5 and gcc --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5f56f96..285e58bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,9 @@ RUN mkdir $APP_HOME RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && \ chown -R $UID:$UID $HOME -# RUN set -x && apt-get update && \ -# chown -R $UID:$UID $APP_HOME && \ -# apt-get install -y build-essential git xmlsec1 libpq5 gcc +RUN set -x && apt-get update && \ + chown -R $UID:$UID $APP_HOME && \ + apt-get install -y build-essential git xmlsec1 # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be @@ -29,7 +29,7 @@ COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ - # && make install \ + && make install \ && unset SET_BUILD_ENV_FROM_FILE RUN chown -R 1000:0 /usr/src/inst-mgmt From b379d344d8250027dd07c7dbb81f8b00842733ab Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:17:20 -0400 Subject: [PATCH 28/60] test removing build-essential --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 285e58bc..5b2b095f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y build-essential git xmlsec1 + apt-get install -y git xmlsec1 libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 7df5cda50f380b0259fe219f0b07bd5d9b07b624 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:24:02 -0400 Subject: [PATCH 29/60] also remove build so scan runs --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b2b095f..3a1bd9f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y git xmlsec1 libpq5 gcc + apt-get install -y build-essential git xmlsec1 libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be @@ -29,7 +29,7 @@ COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ - && make install \ + # && make install \ && unset SET_BUILD_ENV_FROM_FILE RUN chown -R 1000:0 /usr/src/inst-mgmt From 00f64d6563ed293d3c486baf2630c675017cc88f Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:26:50 -0400 Subject: [PATCH 30/60] remove build again --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3a1bd9f6..b13853e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y build-essential git xmlsec1 libpq5 gcc + apt-get install -y git xmlsec1 libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From fc99160b5515e3c101593bf9d6b5aef53d9aa0fe Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:34:12 -0400 Subject: [PATCH 31/60] remove git --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b13853e7..42555290 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y git xmlsec1 libpq5 gcc + apt-get install -y xmlsec1 libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 2175b3358e3d210384dca70334e51a511fe0884a Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:35:39 -0400 Subject: [PATCH 32/60] remove xmlsec --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42555290..a445bb31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y xmlsec1 libpq5 gcc + apt-get install -y libpq5 gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From c658fdece3742a12b2fb92c568c1b59c57da5e9d Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:43:11 -0400 Subject: [PATCH 33/60] remove libpq5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a445bb31..21b82332 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && RUN set -x && apt-get update && \ chown -R $UID:$UID $APP_HOME && \ - apt-get install -y libpq5 gcc + apt-get install -y gcc # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 57cb0a6d3449b25dc459ae9cf5b9c506d71f29cb Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 11 Jun 2025 17:45:57 -0400 Subject: [PATCH 34/60] remove gcc --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21b82332..56067b94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && chown -R $UID:$UID $HOME RUN set -x && apt-get update && \ - chown -R $UID:$UID $APP_HOME && \ - apt-get install -y gcc + chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 016aaf1cb649e951e452d2171cc7d6ca5c1c5396 Mon Sep 17 00:00:00 2001 From: Joshua Seals Date: Thu, 12 Jun 2025 10:27:26 -0400 Subject: [PATCH 35/60] trying apt-get install make --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 56067b94..752e4b5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && chown -R $UID:$UID $HOME RUN set -x && apt-get update && \ + apt-get -y install make \ chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. @@ -28,7 +29,7 @@ COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ - # && make install \ + && make install \ && unset SET_BUILD_ENV_FROM_FILE RUN chown -R 1000:0 /usr/src/inst-mgmt From 64a5a4c7e1251a2f663dcaae2b4977da076e9d70 Mon Sep 17 00:00:00 2001 From: Joshua Seals Date: Thu, 12 Jun 2025 10:29:49 -0400 Subject: [PATCH 36/60] tweak syntax --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 752e4b5d..c32e6da2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && chown -R $UID:$UID $HOME RUN set -x && apt-get update && \ - apt-get -y install make \ + apt-get install -y make && \ chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. From 5bb8e8b6b87d55bc0a683cfb8f9570e2bd4cbd86 Mon Sep 17 00:00:00 2001 From: Joshua Seals Date: Thu, 12 Jun 2025 10:35:19 -0400 Subject: [PATCH 37/60] Adding git back --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c32e6da2..9c35f46a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && chown -R $UID:$UID $HOME RUN set -x && apt-get update && \ - apt-get install -y make && \ + apt-get install -y make git && \ chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. From f5433dfc508dbc54f62710a6809e6f5eeba4932f Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 12 Jun 2025 15:43:43 -0400 Subject: [PATCH 38/60] add xmlsec back --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9c35f46a..4e0d0095 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && chown -R $UID:$UID $HOME RUN set -x && apt-get update && \ - apt-get install -y make git && \ + apt-get install -y make git xmlsec1 && \ chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. From 543ab5a54e3ad74029b7850b8817418444956b76 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 14:44:21 -0400 Subject: [PATCH 39/60] test appstore on alpine --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4e0d0095..de33541c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.23-slim-bookworm +FROM python:3.9.23-alpine ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From aa2dcd0be85cc35c5b7b1b57ded5f949afb36413 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 14:52:54 -0400 Subject: [PATCH 40/60] Change dockerfile to hopefully work on alpine --- Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index de33541c..b95d4c3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,12 +11,13 @@ ENV UID=1000 RUN mkdir $APP_HOME -RUN adduser --disabled-login --home $HOME --shell /bin/bash --uid $UID $USER && \ - chown -R $UID:$UID $HOME - -RUN set -x && apt-get update && \ - apt-get install -y make git xmlsec1 && \ - chown -R $UID:$UID $APP_HOME +# RUN set -x && apt-get update && \ +# apt-get install -y make git xmlsec1 && \ +# chown -R $UID:$UID $APP_HOME +RUN set -x && \ + apk add --no-cache make git bash xmlsec libxml2-dev && \ + adduser -D -s /bin/bash -h $HOME -u $UID $USER && \ + chown -R $UID:$UID $HOME # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be From 9f02cee7ae58ee2bb9dd18ee69b3488f8754b120 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 15:44:22 -0400 Subject: [PATCH 41/60] fix pyyaml wheel build? --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b95d4c3e..2d0a2765 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN mkdir $APP_HOME # apt-get install -y make git xmlsec1 && \ # chown -R $UID:$UID $APP_HOME RUN set -x && \ - apk add --no-cache make git bash xmlsec libxml2-dev && \ + apk add --no-cache make git bash build-base xmlsec libxml2-dev && \ adduser -D -s /bin/bash -h $HOME -u $UID $USER && \ chown -R $UID:$UID $HOME @@ -30,6 +30,8 @@ COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ + && pip install "cython<3.0.0" wheel \ + && pip install "pyyaml==5.4.1" --no-build-isolation \ && make install \ && unset SET_BUILD_ENV_FROM_FILE From e224c15ac6656224eb7445b1a423e9abd303c6a6 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 15:48:23 -0400 Subject: [PATCH 42/60] Add linux-headers --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d0a2765..336fc9d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN mkdir $APP_HOME # apt-get install -y make git xmlsec1 && \ # chown -R $UID:$UID $APP_HOME RUN set -x && \ - apk add --no-cache make git bash build-base xmlsec libxml2-dev && \ + apk add --no-cache make git bash build-base xmlsec libxml2-dev linux-headers && \ adduser -D -s /bin/bash -h $HOME -u $UID $USER && \ chown -R $UID:$UID $HOME From c67c1f929802085e5acceed398eec93da43dd264 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 15:53:18 -0400 Subject: [PATCH 43/60] remove netifaces --- appstore/tycho/actions.py | 2 -- appstore/tycho/tycho_utils.py | 14 -------------- requirements.txt | 1 - 3 files changed, 17 deletions(-) diff --git a/appstore/tycho/actions.py b/appstore/tycho/actions.py index a92f311f..fc08fc31 100644 --- a/appstore/tycho/actions.py +++ b/appstore/tycho/actions.py @@ -3,14 +3,12 @@ import json import jsonschema import logging -import netifaces import os import requests import sys import traceback import yaml from tycho.core import Tycho -from tycho.tycho_utils import NetworkUtils """ Provides actions for creating, monitoring, and deleting distributed systems of cloud native diff --git a/appstore/tycho/tycho_utils.py b/appstore/tycho/tycho_utils.py index 812b23a9..fcb813e9 100644 --- a/appstore/tycho/tycho_utils.py +++ b/appstore/tycho/tycho_utils.py @@ -86,20 +86,6 @@ def apply_environment (environment, text): def trunc (a_string, max_len=80): return (a_string[:max_len] + '..') if len(a_string) > max_len else a_string -class NetworkUtils: - @staticmethod - def get_client_ip (request, debug=False): - """ Get the IP address of the client. Account for requests from proxies. - In debug mode, ignore loopback and try to get an IP from a n interface.""" - ip_addr = request.remote_addr - if request.headers.getlist("X-Forwarded-For"): - ip_addr = request.headers.getlist("X-Forwarded-For")[0] - if debug: - interface = netifaces.ifaddresses ('en0') - ip_addr = interface[2][0]['addr'] - logger.debug (f"(debug mode ip addr:)--> {ip_addr}") - return ip_addr - class Resource: @staticmethod def get_resource_path(resource_name): diff --git a/requirements.txt b/requirements.txt index eee7f2b5..22d350a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,6 @@ GitPython==3.1.44 Jinja2==3.1.6 jsonschema==3.2.0 kubernetes==25.3.0 -netifaces==0.11.0 PyYAML==5.4.1 requests==2.32.4 docker-compose==1.29.2 From ef404ff62ba11a3ed54c1f1c8f0ffd5451ee5504 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 16:56:38 -0400 Subject: [PATCH 44/60] remove netifaces import --- appstore/tycho/tycho_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/appstore/tycho/tycho_utils.py b/appstore/tycho/tycho_utils.py index fcb813e9..441aff74 100644 --- a/appstore/tycho/tycho_utils.py +++ b/appstore/tycho/tycho_utils.py @@ -1,7 +1,6 @@ import datetime import json import logging -import netifaces import os import string import traceback From c7320820ce9843a0549a70024fe8fd477f961192 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Wed, 18 Jun 2025 17:52:05 -0400 Subject: [PATCH 45/60] add debug --- appstore/appstore/wsgi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appstore/appstore/wsgi.py b/appstore/appstore/wsgi.py index 211ac797..8e1e3322 100644 --- a/appstore/appstore/wsgi.py +++ b/appstore/appstore/wsgi.py @@ -8,6 +8,8 @@ from django.core.wsgi import get_wsgi_application +print(os.environ) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "appstore.settings") application = get_wsgi_application() From 071cf3474a3df37f2147dc538ffb3ab14ec8386b Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 13:22:00 -0400 Subject: [PATCH 46/60] add openssl --- Dockerfile | 2 +- appstore/appstore/wsgi.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 336fc9d6..20373b5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN mkdir $APP_HOME # apt-get install -y make git xmlsec1 && \ # chown -R $UID:$UID $APP_HOME RUN set -x && \ - apk add --no-cache make git bash build-base xmlsec libxml2-dev linux-headers && \ + apk add --no-cache make git bash build-base xmlsec libxml2-dev linux-headers openssl && \ adduser -D -s /bin/bash -h $HOME -u $UID $USER && \ chown -R $UID:$UID $HOME diff --git a/appstore/appstore/wsgi.py b/appstore/appstore/wsgi.py index 8e1e3322..211ac797 100644 --- a/appstore/appstore/wsgi.py +++ b/appstore/appstore/wsgi.py @@ -8,8 +8,6 @@ from django.core.wsgi import get_wsgi_application -print(os.environ) - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "appstore.settings") application = get_wsgi_application() From a6f48fd49cf5531a662f86d278d8a0fc7ce3670a Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 13:27:50 -0400 Subject: [PATCH 47/60] upgrade urllib3 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 22d350a6..0ece2d1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ PyYAML==5.4.1 requests==2.32.4 docker-compose==1.29.2 requests_cache==0.9.2 +urllib3==2.5.0 \ No newline at end of file From e99170ca829af14ab34466fe3a6eb58dfdf79134 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 13:48:56 -0400 Subject: [PATCH 48/60] dockerfile alpine permission fixes --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 20373b5e..9cdfd210 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,8 @@ RUN set -x && \ # RUN apt-get install -y nodejs WORKDIR $APP_HOME -COPY . . +COPY --chown=$USER:$USER . . +USER $USER RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ @@ -35,8 +36,10 @@ RUN export SET_BUILD_ENV_FROM_FILE=false \ && make install \ && unset SET_BUILD_ENV_FROM_FILE -RUN chown -R 1000:0 /usr/src/inst-mgmt -RUN chmod -R g+w /usr/src/inst-mgmt +USER root + +RUN chown -R $USER:0 $APP_HOME && \ + chmod -R g+w $APP_HOME EXPOSE 8000 CMD ["make","start"] From ef1f992d0c5cceb0ac327c86b918d1d4a70a7e88 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 14:11:06 -0400 Subject: [PATCH 49/60] upgrade requests-cache to resolve urllib3 conflict --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ece2d1a..1263990d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,5 +26,5 @@ kubernetes==25.3.0 PyYAML==5.4.1 requests==2.32.4 docker-compose==1.29.2 -requests_cache==0.9.2 +requests_cache==1.2.1 urllib3==2.5.0 \ No newline at end of file From 462f1077d3dcf13953fe7d058d824dc76306cce4 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 14:43:30 -0400 Subject: [PATCH 50/60] try to force cache invalidate --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1263990d..10913a9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,4 +27,4 @@ PyYAML==5.4.1 requests==2.32.4 docker-compose==1.29.2 requests_cache==1.2.1 -urllib3==2.5.0 \ No newline at end of file +urllib3==2.5.0 From 7a73a63bf55b46a41281fc5d786190c6cf40b065 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:00:23 -0400 Subject: [PATCH 51/60] remove permission change --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9cdfd210..a24d024e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,6 @@ RUN set -x && \ WORKDIR $APP_HOME COPY --chown=$USER:$USER . . -USER $USER RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ @@ -36,10 +35,8 @@ RUN export SET_BUILD_ENV_FROM_FILE=false \ && make install \ && unset SET_BUILD_ENV_FROM_FILE -USER root - -RUN chown -R $USER:0 $APP_HOME && \ - chmod -R g+w $APP_HOME +RUN chown -R 1000:0 /usr/src/inst-mgmt +RUN chmod -R g+w /usr/src/inst-mgmt EXPOSE 8000 CMD ["make","start"] From 8f106a8a4b72f7dcf8eefa030c2bfe893190daa0 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:09:20 -0400 Subject: [PATCH 52/60] fix chown and remove debian install step --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a24d024e..1ecf352c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,13 +11,10 @@ ENV UID=1000 RUN mkdir $APP_HOME -# RUN set -x && apt-get update && \ -# apt-get install -y make git xmlsec1 && \ -# chown -R $UID:$UID $APP_HOME RUN set -x && \ apk add --no-cache make git bash build-base xmlsec libxml2-dev linux-headers openssl && \ adduser -D -s /bin/bash -h $HOME -u $UID $USER && \ - chown -R $UID:$UID $HOME + chown -R $UID:$UID $APP_HOME # Removing but leaving commented in case Tycho needs this for swagger. # Version 3.3.1 currently, if not complaints v3.3.3 this can be @@ -26,7 +23,7 @@ RUN set -x && \ # RUN apt-get install -y nodejs WORKDIR $APP_HOME -COPY --chown=$USER:$USER . . +COPY . . RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ From 7b12f49044bbb8855844a06c90947c495c3a262c Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:13:07 -0400 Subject: [PATCH 53/60] run on non-root user --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1ecf352c..23636a64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,9 @@ RUN set -x && \ # RUN apt-get install -y nodejs WORKDIR $APP_HOME -COPY . . +COPY --chown $UID:$UID . . + +USER $USER RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ From 8d4939c9d9bddccc729f574261efd6f386df40db Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:14:00 -0400 Subject: [PATCH 54/60] fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 23636a64..e58f2ca4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN set -x && \ # RUN apt-get install -y nodejs WORKDIR $APP_HOME -COPY --chown $UID:$UID . . +COPY --chown=$UID:$UID . . USER $USER From c4e60af844b8ec4342f80f4b22a940175ca29504 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:19:16 -0400 Subject: [PATCH 55/60] more user perm tweaking. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e58f2ca4..5f818030 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,9 @@ RUN set -x && \ WORKDIR $APP_HOME COPY --chown=$UID:$UID . . +RUN chown -R $USER:0 $APP_HOME && \ + chmod -R g+w $APP_HOME + USER $USER RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi @@ -34,8 +37,5 @@ RUN export SET_BUILD_ENV_FROM_FILE=false \ && make install \ && unset SET_BUILD_ENV_FROM_FILE -RUN chown -R 1000:0 /usr/src/inst-mgmt -RUN chmod -R g+w /usr/src/inst-mgmt - EXPOSE 8000 CMD ["make","start"] From feee5af672f833d3d51253e14979385776c13341 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Thu, 19 Jun 2025 15:29:18 -0400 Subject: [PATCH 56/60] install as root --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5f818030..b0e3742b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,6 @@ COPY --chown=$UID:$UID . . RUN chown -R $USER:0 $APP_HOME && \ chmod -R g+w $APP_HOME -USER $USER - RUN if [ -d whl -a "$(ls -A whl/*.whl)" ]; then pip install whl/*.whl; fi RUN export SET_BUILD_ENV_FROM_FILE=false \ && pip install "cython<3.0.0" wheel \ @@ -37,5 +35,7 @@ RUN export SET_BUILD_ENV_FROM_FILE=false \ && make install \ && unset SET_BUILD_ENV_FROM_FILE +USER $USER + EXPOSE 8000 CMD ["make","start"] From a3835ffa3059866c1b36b7ba0c110b91056f5a78 Mon Sep 17 00:00:00 2001 From: frostyfan109 Date: Tue, 8 Jul 2025 16:32:24 -0400 Subject: [PATCH 57/60] create symlink to /shared under /home/user/shared for ordrd deployments --- appstore/tycho/template/pod.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appstore/tycho/template/pod.yaml b/appstore/tycho/template/pod.yaml index 408a1ded..436ffe4d 100644 --- a/appstore/tycho/template/pod.yaml +++ b/appstore/tycho/template/pod.yaml @@ -74,6 +74,9 @@ spec: mkdir -p {{ system.parent_dir }}/{{ system.subpath_dir }}/.ssh && echo -e "Host {{ system.gitea_host }}\n Hostname {{ system.gitea_service_name }}\n User {{ system.gitea_user }}\n IdentityFile ~/.ssh/id_gitea" > {{ system.parent_dir }}/{{ system.subpath_dir }}/.ssh/config && {% endif %} + if [ -d /shared ]; then + ln -sfn /shared {{ system.parent_dir }}/{{ system.subpath_dir }}/shared; + fi && ls -aln {{ system.parent_dir }} && echo OK volumeMounts: From 0c624250e1eaccd42d619db6cefae5974547e66e Mon Sep 17 00:00:00 2001 From: waTeim Date: Thu, 24 Jul 2025 10:49:36 -0400 Subject: [PATCH 58/60] Feature; add ldap check for whitelist inclusion --- appstore/frontend/views.py | 29 ++-- .../middleware/filter_whitelist_middleware.py | 156 +++++++++++++----- requirements.txt | 1 + 3 files changed, 134 insertions(+), 52 deletions(-) diff --git a/appstore/frontend/views.py b/appstore/frontend/views.py index ad71bd0b..c89b99f8 100644 --- a/appstore/frontend/views.py +++ b/appstore/frontend/views.py @@ -10,6 +10,7 @@ from django.shortcuts import redirect from django.views.generic.base import TemplateView from rest_framework.test import APIRequestFactory +from django.contrib.sites.models import Site """ ####### @@ -124,16 +125,24 @@ def dispatch(self, *args, **kwargs): class LoginWhitelistView(TemplateView): - template_name = "frontend/whitelist.html" + brand_context = get_brand_details() - brand = brand_context["brand"] - full_brand = brand_context["title"] - brand_logo = brand_context["logo_url"] + brand = brand_context["brand"] + full_brand = brand_context["title"] + brand_logo = brand_context["logo_url"] + brand_links = brand_context["links"] # NEW + color_scheme = brand_context.get("color_scheme") # optional - def get_context_data(self, *args, **kwargs): - context = super(LoginWhitelistView, self).get_context_data(*args, **kwargs) - context["brand"] = self.brand - context["full_brand"] = self.full_brand - context["brand_logo"] = self.brand_logo - return context + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + current_site = Site.objects.get_current(request=self.request) + ctx.update( + brand=self.brand, + full_brand=self.full_brand, + brand_logo=self.brand_logo, + brand_links=self.brand_links, + brand_color_scheme=self.color_scheme, + site=current_site, # ← add this + ) + return ctx diff --git a/appstore/middleware/filter_whitelist_middleware.py b/appstore/middleware/filter_whitelist_middleware.py index e3e98885..e8c936ba 100644 --- a/appstore/middleware/filter_whitelist_middleware.py +++ b/appstore/middleware/filter_whitelist_middleware.py @@ -1,5 +1,7 @@ +import os import logging import re +from functools import lru_cache from django.conf import settings from django.contrib.auth.models import Group @@ -7,72 +9,125 @@ from django.http import HttpResponseRedirect from django.utils.deprecation import MiddlewareMixin from django.core.mail import send_mail - from smtplib import SMTPSenderRefused, SMTPResponseException +from ldap3 import Server, Connection, SUBTREE # NEW from core.models import AuthorizedUser + logger = logging.getLogger(__name__) FORMAT = "%(asctime)-15s %(clientip)s %(user)-8s %(message)s" logging.basicConfig(format=FORMAT) class AllowWhiteListedUserOnly(MiddlewareMixin): - def __init__(self, get_response=None): - if get_response is not None: - self.get_response = get_response + + @staticmethod + @lru_cache(maxsize=1) + def _ldap_conn(): + """One connection per worker, with DEBUG breadcrumbs.""" + uri = os.getenv("LDAP_URI") + if not uri: + logger.debug("[LDAP] LDAP_URI not set → LDAP feature disabled") + return None + + bind_dn = os.getenv("LDAP_BIND_DN", "") + bind_pw = os.getenv("LDAP_BIND_PASSWORD", "") + + logger.debug("[LDAP] Connecting to %s (bind_dn=%s)", uri, bind_dn or "anonymous") + try: + server = Server(uri, get_info=None) + conn = Connection(server, user=bind_dn, password=bind_pw, auto_bind=True) + logger.debug("[LDAP] Connection established. Server info: %s", server) + return conn + except Exception as exc: # noqa: BLE001 + logger.error("[LDAP] Connection failed: %s", exc) + return None + + @classmethod + def _ldap_group_member(cls, user): + """Return True when user is in LDAP_GROUP_DN; log every step.""" + conn = cls._ldap_conn() + group_dn = os.getenv("LDAP_GROUP_DN") + if not conn: + logger.debug("[LDAP] No connection – skipping LDAP checks") + return False + if not group_dn: + logger.debug("[LDAP] LDAP_GROUP_DN unset – skipping LDAP checks") + return False + + base = os.getenv("LDAP_SEARCH_BASE", group_dn.split(",", 1)[1]) + flt = f"(|(mail={user.email})(uid={user.username}))" + logger.debug("[LDAP] Searching base=%s filter=%s", base, flt) + + try: + ok = conn.search(base, flt, SUBTREE, attributes=[]) + logger.debug("[LDAP] Search ok=%s; hits=%s", ok, len(conn.entries)) + if not ok or not conn.entries: + return False + user_dn = conn.entries[0].entry_dn + logger.debug("[LDAP] Resolved user_dn=%s", user_dn) + except Exception as exc: # noqa: BLE001 + logger.error("[LDAP] Search failed: %s", exc) + return False + + try: + result = conn.compare(group_dn, "member", user_dn) + logger.debug("[LDAP] compare(%s member %s) → %s", group_dn, user_dn, result) + return result + except Exception as exc: # noqa: BLE001 + logger.error("[LDAP] Compare failed: %s", exc) + return False + + def _get_response(self, request): + """ + Call the next middleware in the chain to get a response. + """ + # Call the next middleware in the chain to get a response + if hasattr(self, 'process_response'): + return self.process_response(request) else: - self.get_response = self._get_response - + # If there's no process_response method, return None + return None + + def __init__(self, get_response=None): + self.get_response = get_response or self._get_response super().__init__(get_response) + def process_request(self, request): user = request.user - logger.debug(f"testing user: {user}") + logger.debug("testing user: %s", user) if user.is_authenticated and not user.is_superuser: if not any( - [ - request.path.startswith(settings.LOGIN_URL), - request.path.startswith(settings.LOGIN_WHITELIST_URL), - request.path.startswith(settings.ADMIN_URL), - request.path.startswith(settings.STATIC_URL), - request.path.startswith(settings.SAML_URL), - request.path.startswith(settings.SAML_ACS_URL), - request.path.startswith("/api/v1/context"), - request.path.startswith("/api/v1/providers"), - ] + request.path.startswith(p) + for p in ( + settings.LOGIN_URL, + settings.LOGIN_WHITELIST_URL, + settings.ADMIN_URL, + settings.STATIC_URL, + settings.SAML_URL, + settings.SAML_ACS_URL, + "/api/v1/context", + "/api/v1/providers", + ) ): if self.is_authorized(user): - logger.debug(f"Adding user {user} to whitelist") + logger.info("Adding user %s to whitelist", user) whitelist_group = Group.objects.get(name="whitelisted") user.groups.add(whitelist_group) else: - logger.debug(f"Filtering user {user} is not authorized") + logger.info("Filtering user %s is not authorized", user) self.clear_session(request) try: - # This will fail if email isn't setup correctly and won't - # route the user correctly. self.send_whitelist_email(request, user) except (SMTPSenderRefused, SMTPResponseException) as err: - logger.error( - f"SMTP misconfigured, please check settings.\n{err}\n" - ) + logger.error("SMTP misconfigured: %s", err) finally: - # Make sure to always run the redirect. - return HttpResponseRedirect(settings.LOGIN_WHITELIST_URL) + return HttpResponseRedirect(settings.LOGIN_WHITELIST_URL) return None - def _get_response(self, request): - """ - Call the next middleware in the chain to get a response. - """ - # Call the next middleware in the chain to get a response - if hasattr(self, 'process_response'): - return self.process_response(request) - else: - # If there's no process_response method, return None - return None @staticmethod def is_whitelisted(user): @@ -80,6 +135,7 @@ def is_whitelisted(user): return True return False + @staticmethod def is_auto_whitelisted_email(user): email = user.email @@ -95,25 +151,41 @@ def is_whitelisted_username(user): @staticmethod def is_authorized(user): + logger.debug("[AUTHZ] Testing %s / %s", user.username, user.email) + if AuthorizedUser.objects.filter(email=user.email).exists(): - logger.debug(f"found user email {user.email} in AuthorizedUser") + logger.debug("[AUTHZ] email match in AuthorizedUser") return True + if AllowWhiteListedUserOnly.is_auto_whitelisted_email(user): - # authorize the user automatically, and allow them through. - AuthorizedUser.objects.create(email=user.email) + logger.debug("[AUTHZ] AUTO-pattern match → persisting email") + AuthorizedUser.objects.get_or_create( + email=user.email, defaults={"username": user.username} + ) return True + if AuthorizedUser.objects.filter(username=user.username).exists(): - logger.debug(f"found user with username {user.username} in AuthorizedUser") + logger.debug("[AUTHZ] username match in AuthorizedUser") + return True + + if AllowWhiteListedUserOnly._ldap_group_member(user): + logger.debug("[AUTHZ] LDAP group match → persisting email") + AuthorizedUser.objects.get_or_create( + email=user.email, defaults={"username": user.username} + ) return True - logger.debug(f"user email {user.email} not found in AuthorizedUser") + + logger.debug("[AUTHZ] No rule matched; user is NOT authorised") return False + @staticmethod def clear_session(request): session_key = request.session.session_key session = Session.objects.get(session_key=session_key) Session.objects.filter(session_key=session).delete() + @staticmethod def send_whitelist_email(request, user): logger.debug("sending email") @@ -144,4 +216,4 @@ def send_whitelist_email(request, user): settings.EMAIL_HOST_USER, recipient_list, fail_silently=False, - ) + ) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aba937ce..1773aef9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,3 +34,4 @@ PyYAML==5.4.1 requests==2.31.0 docker-compose==1.29.2 requests_cache==0.9.2 +ldap3 From bb00354ba359270b7320d8ae7330a81c78a648b8 Mon Sep 17 00:00:00 2001 From: waTeim Date: Thu, 7 Aug 2025 10:02:31 -0400 Subject: [PATCH 59/60] resolve requests_cache conflict --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1773aef9..6d762508 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,5 +33,5 @@ netifaces==0.11.0 PyYAML==5.4.1 requests==2.31.0 docker-compose==1.29.2 -requests_cache==0.9.2 +requests_cache==1.2.1 ldap3 From d8b83e532f918a3498a7c368f6aa6de4bf876bd5 Mon Sep 17 00:00:00 2001 From: waTeim Date: Wed, 13 Aug 2025 13:40:46 -0400 Subject: [PATCH 60/60] README++ --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 5e571791..313671be 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,13 @@ each in an application specific subfolder. Along with the docker compose, a `.en file specifies environment variables for the application. If a file called icon.png is provided, that is used as the application's icon. +### App Deployment features + +1. Follows docker-compose as a baseline description +2. Best practices in constructing deployment definitions + a. disables service env file generation to avoid information leak + + ## Development Environment ### Prerequisites