diff --git a/azurelinuxagent/common/logger.py b/azurelinuxagent/common/logger.py index 07e3f2393..3d0dc617d 100644 --- a/azurelinuxagent/common/logger.py +++ b/azurelinuxagent/common/logger.py @@ -45,6 +45,7 @@ def __init__(self, logger=None, prefix=None): self.logger = self if logger is None else logger self.periodic_messages = {} self.prefix = prefix + self.silent = False def reset_periodic(self): self.logger.periodic_messages = {} @@ -124,6 +125,9 @@ def write_log(log_appender): # pylint: disable=W0612 finally: log_appender.appender_lock = False + if self.silent: + return + # if msg_format is not unicode convert it to unicode if type(msg_format) is not ustr: msg_format = ustr(msg_format, errors="backslashreplace") diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py index 2ed4be78b..f14fdfbb5 100644 --- a/azurelinuxagent/common/osutil/factory.py +++ b/azurelinuxagent/common/osutil/factory.py @@ -34,7 +34,7 @@ from .nsbsd import NSBSDOSUtil from .openbsd import OpenBSDOSUtil from .openwrt import OpenWRTOSUtil -from .redhat import RedhatOSUtil, Redhat6xOSUtil +from .redhat import RedhatOSUtil, Redhat6xOSUtil, RedhatOSModernUtil from .suse import SUSEOSUtil, SUSE11OSUtil from .photonos import PhotonOSUtil from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, \ @@ -107,6 +107,9 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) if Version(distro_version) < Version("7"): return Redhat6xOSUtil() + if Version(distro_version) >= Version("8.6"): + return RedhatOSModernUtil() + return RedhatOSUtil() if distro_name == "euleros": diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py index 9759d1136..312dd1608 100644 --- a/azurelinuxagent/common/osutil/redhat.py +++ b/azurelinuxagent/common/osutil/redhat.py @@ -142,3 +142,25 @@ def get_dhcp_lease_endpoint(self): endpoint = self.get_endpoint_from_leases_path('/var/lib/NetworkManager/dhclient-*.lease') return endpoint + + +class RedhatOSModernUtil(RedhatOSUtil): + def __init__(self): # pylint: disable=W0235 + super(RedhatOSModernUtil, self).__init__() + + def restart_if(self, ifname, retries=3, wait=5): + """ + Restart an interface by bouncing the link. systemd-networkd observes + this event, and forces a renew of DHCP. + """ + retry_limit = retries + 1 + for attempt in range(1, retry_limit): + return_code = shellutil.run("ip link set {0} down && ip link set {0} up".format(ifname)) + if return_code == 0: + return + logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code)) + if attempt < retry_limit: + logger.info("retrying in {0} seconds".format(wait)) + time.sleep(wait) + else: + logger.warn("exceeded restart retries") diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py index c7e01dd20..8dce261ce 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py @@ -25,6 +25,7 @@ from azurelinuxagent.common.future import ustr from azurelinuxagent.common.protocol.extensions_goal_state import ExtensionsGoalState, GoalStateChannel, GoalStateSource from azurelinuxagent.common.protocol.restapi import ExtensionSettings, Extension, VMAgentManifest, ExtensionState, InVMGoalStateMetaData +from azurelinuxagent.common.utils import restutil from azurelinuxagent.common.utils.textutil import parse_doc, parse_json, findall, find, findtext, getattrib, gettext, format_exception, \ is_str_none_or_whitespace, is_str_empty @@ -99,7 +100,7 @@ def fetch_direct(): def fetch_through_host(): host = wire_client.get_host_plugin() uri, headers = host.get_artifact_request(artifacts_profile_blob) - content, _ = wire_client.fetch(uri, headers, use_proxy=False) + content, _ = wire_client.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return content logger.verbose("Retrieving the artifacts profile") diff --git a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py index ce99a2607..10e036c9c 100644 --- a/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py +++ b/azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py @@ -266,7 +266,9 @@ def _parse_agent_manifests(self, vm_settings): for family in families: name = family["name"] version = family.get("version") - uris = family["uris"] + uris = family.get("uris") + if uris is None: + uris = [] manifest = VMAgentManifest(name, version) for u in uris: manifest.uris.append(u) @@ -289,7 +291,7 @@ def _parse_extensions(self, vm_settings): # "settingsSeqNo": 0, # "settings": [ # { - # "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + # "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", # "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", # "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" # } diff --git a/azurelinuxagent/common/protocol/goal_state.py b/azurelinuxagent/common/protocol/goal_state.py index ae01e4d22..97ae270f8 100644 --- a/azurelinuxagent/common/protocol/goal_state.py +++ b/azurelinuxagent/common/protocol/goal_state.py @@ -15,9 +15,11 @@ # limitations under the License. # # Requires Python 2.6+ and Openssl 1.0+ +import datetime import os import re import time +import json import azurelinuxagent.common.conf as conf import azurelinuxagent.common.logger as logger @@ -30,8 +32,8 @@ from azurelinuxagent.common.protocol.extensions_goal_state import VmSettingsParseError, GoalStateSource from azurelinuxagent.common.protocol.hostplugin import VmSettingsNotSupported, VmSettingsSupportStopped from azurelinuxagent.common.protocol.restapi import Cert, CertList, RemoteAccessUser, RemoteAccessUsersList -from azurelinuxagent.common.utils import fileutil, timeutil -from azurelinuxagent.common.utils.archive import GoalStateHistory +from azurelinuxagent.common.utils import fileutil +from azurelinuxagent.common.utils.archive import GoalStateHistory, SHARED_CONF_FILE_NAME from azurelinuxagent.common.utils.cryptutil import CryptUtil from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, findtext, getattrib @@ -46,8 +48,16 @@ _GET_GOAL_STATE_MAX_ATTEMPTS = 6 +class GoalStateInconsistentError(ProtocolError): + """ + Indicates an inconsistency in the goal state (e.g. missing tenant certificate) + """ + def __init__(self, msg, inner=None): + super(GoalStateInconsistentError, self).__init__(msg, inner) + + class GoalState(object): - def __init__(self, wire_client): + def __init__(self, wire_client, silent=False): """ Fetches the goal state using the given wire client. @@ -62,6 +72,8 @@ def __init__(self, wire_client): self._wire_client = wire_client self._history = None self._extensions_goal_state = None # populated from vmSettings or extensionsConfig + self.logger = logger.Logger(logger.DEFAULT_LOGGER) + self.logger.silent = silent # These properties hold the goal state from the WireServer and are initialized by self._fetch_full_wire_server_goal_state() self._incarnation = None @@ -70,11 +82,13 @@ def __init__(self, wire_client): self._container_id = None self._hosting_env = None self._shared_conf = None - self._certs = None + self._certs = EmptyCertificates() self._remote_access = None - self.update() + self.update(silent=silent) + except ProtocolError: + raise except Exception as exception: # We don't log the error here since fetching the goal state is done every few seconds raise ProtocolError(msg="Error fetching goal state", inner=exception) @@ -123,33 +137,47 @@ def update_host_plugin_headers(wire_client): # Fetching the goal state updates the HostGAPlugin so simply trigger the request GoalState._fetch_goal_state(wire_client) - def update(self): + def update(self, silent=False): """ Updates the current GoalState instance fetching values from the WireServer/HostGAPlugin as needed """ + self.logger.silent = silent + + try: + self._update(force_update=False) + except GoalStateInconsistentError as e: + self.logger.warn("Detected an inconsistency in the goal state: {0}", ustr(e)) + self._update(force_update=True) + self.logger.info("The goal state is consistent") + + def _update(self, force_update): # # Fetch the goal state from both the HGAP and the WireServer # - timestamp = timeutil.create_timestamp() + timestamp = datetime.datetime.utcnow() + + if force_update: + self.logger.info("Refreshing goal state and vmSettings") incarnation, xml_text, xml_doc = GoalState._fetch_goal_state(self._wire_client) - goal_state_updated = incarnation != self._incarnation + goal_state_updated = force_update or incarnation != self._incarnation if goal_state_updated: - logger.info('Fetched a new incarnation for the WireServer goal state [incarnation {0}]', incarnation) + self.logger.info('Fetched a new incarnation for the WireServer goal state [incarnation {0}]', incarnation) vm_settings, vm_settings_updated = None, False try: - vm_settings, vm_settings_updated = GoalState._fetch_vm_settings(self._wire_client) + vm_settings, vm_settings_updated = GoalState._fetch_vm_settings(self._wire_client, force_update=force_update) except VmSettingsSupportStopped as exception: # If the HGAP stopped supporting vmSettings, we need to use the goal state from the WireServer self._restore_wire_server_goal_state(incarnation, xml_text, xml_doc, exception) return if vm_settings_updated: - logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) + self.logger.info('') + self.logger.info("Fetched new vmSettings [HostGAPlugin correlation ID: {0} eTag: {1} source: {2}]", vm_settings.hostga_plugin_correlation_id, vm_settings.etag, vm_settings.source) # Ignore the vmSettings if their source is Fabric (processing a Fabric goal state may require the tenant certificate and the vmSettings don't include it.) if vm_settings is not None and vm_settings.source == GoalStateSource.Fabric: if vm_settings_updated: - logger.info("The vmSettings originated via Fabric; will ignore them.") + self.logger.info("The vmSettings originated via Fabric; will ignore them.") vm_settings, vm_settings_updated = None, False # If neither goal state has changed we are done with the update @@ -181,19 +209,41 @@ def update(self): else: # vm_settings_updated most_recent = vm_settings - if self._extensions_goal_state is None or most_recent.created_on_timestamp > self._extensions_goal_state.created_on_timestamp: + if self._extensions_goal_state is None or most_recent.created_on_timestamp >= self._extensions_goal_state.created_on_timestamp: self._extensions_goal_state = most_recent + # + # For Fast Track goal states, verify that the required certificates are in the goal state. + # + # Some scenarios can produce inconsistent goal states. For example, during hibernation/resume, the Fabric goal state changes (the + # tenant certificate is re-generated when the VM is restarted) *without* the incarnation necessarily changing (e.g. if the incarnation + # is 1 before the hibernation; on resume the incarnation is set to 1 even though the goal state has a new certificate). If a Fast + # Track goal state comes after that, the extensions will need the new certificate. The Agent needs to refresh the goal state in that + # case, to ensure it fetches the new certificate. + # + if self._extensions_goal_state.source == GoalStateSource.FastTrack: + self._check_certificates() + + def _check_certificates(self): + for extension in self.extensions_goal_state.extensions: + for settings in extension.settings: + if settings.protectedSettings is None: + continue + certificates = self.certs.summary + if not any(settings.certificateThumbprint == c['thumbprint'] for c in certificates): + message = "Certificate {0} needed by {1} is missing from the goal state".format(settings.certificateThumbprint, extension.name) + raise GoalStateInconsistentError(message) + def _restore_wire_server_goal_state(self, incarnation, xml_text, xml_doc, vm_settings_support_stopped_error): - logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') - self._history = GoalStateHistory(timeutil.create_timestamp(), incarnation) + self.logger.info('The HGAP stopped supporting vmSettings; will fetched the goal state from the WireServer.') + self._history = GoalStateHistory(datetime.datetime.utcnow(), incarnation) self._history.save_goal_state(xml_text) self._extensions_goal_state = self._fetch_full_wire_server_goal_state(incarnation, xml_doc) if self._extensions_goal_state.created_on_timestamp < vm_settings_support_stopped_error.timestamp: self._extensions_goal_state.is_outdated = True msg = "Fetched a Fabric goal state older than the most recent FastTrack goal state; will skip it.\nFabric: {0}\nFastTrack: {1}".format( self._extensions_goal_state.created_on_timestamp, vm_settings_support_stopped_error.timestamp) - logger.info(msg) + self.logger.info(msg) add_event(op=WALAEventOperation.VmSettings, message=msg, is_success=True) def save_to_history(self, data, file_name): @@ -235,7 +285,7 @@ def _fetch_goal_state(wire_client): return incarnation, xml_text, xml_doc @staticmethod - def _fetch_vm_settings(wire_client): + def _fetch_vm_settings(wire_client, force_update=False): """ Issues an HTTP request (HostGAPlugin) for the vm settings and returns the response as an ExtensionsGoalState. """ @@ -244,11 +294,11 @@ def _fetch_vm_settings(wire_client): if conf.get_enable_fast_track(): try: try: - vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings() + vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings(force_update=force_update) except ResourceGoneError: # retry after refreshing the HostGAPlugin GoalState.update_host_plugin_headers(wire_client) - vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings() + vm_settings, vm_settings_updated = wire_client.get_host_plugin().fetch_vm_settings(force_update=force_update) except VmSettingsSupportStopped: raise @@ -257,7 +307,7 @@ def _fetch_vm_settings(wire_client): except VmSettingsParseError as exception: # ensure we save the vmSettings if there were parsing errors, but save them only once per ETag if not GoalStateHistory.tag_exists(exception.etag): - GoalStateHistory(timeutil.create_timestamp(), exception.etag).save_vm_settings(exception.vm_settings_text) + GoalStateHistory(datetime.datetime.utcnow(), exception.etag).save_vm_settings(exception.vm_settings_text) raise return vm_settings, vm_settings_updated @@ -270,7 +320,8 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): Returns the value of ExtensionsConfig. """ try: - logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) + self.logger.info('') + self.logger.info('Fetching full goal state from the WireServer [incarnation {0}]', incarnation) role_instance = find(xml_doc, "RoleInstance") role_instance_id = findtext(role_instance, "InstanceId") @@ -294,15 +345,26 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): shared_conf_uri = findtext(xml_doc, "SharedConfig") xml_text = self._wire_client.fetch_config(shared_conf_uri, self._wire_client.get_header()) - shared_conf = SharedConfig(xml_text) + shared_config = SharedConfig(xml_text) self._history.save_shared_conf(xml_text) + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband), so save it to the agent's root directory as well + shared_config_file = os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME) + try: + fileutil.write_file(shared_config_file, xml_text) + except Exception as e: + logger.warn("Failed to save {0}: {1}".format(shared_config, e)) - certs = None + certs = EmptyCertificates() certs_uri = findtext(xml_doc, "Certificates") if certs_uri is not None: - # Note that we do not save the certificates to the goal state history xml_text = self._wire_client.fetch_config(certs_uri, self._wire_client.get_header_for_cert()) - certs = Certificates(xml_text) + certs = Certificates(xml_text, self.logger) + # Log and save the certificates summary (i.e. the thumbprint but not the certificate itself) to the goal state history + for c in certs.summary: + self.logger.info("Downloaded certificate {0}".format(c)) + if len(certs.warnings) > 0: + self.logger.warn(certs.warnings) + self._history.save_certificates(json.dumps(certs.summary)) remote_access = None remote_access_uri = findtext(container, "RemoteAccessInfo") @@ -316,17 +378,17 @@ def _fetch_full_wire_server_goal_state(self, incarnation, xml_doc): self._role_config_name = role_config_name self._container_id = container_id self._hosting_env = hosting_env - self._shared_conf = shared_conf + self._shared_conf = shared_config self._certs = certs self._remote_access = remote_access return extensions_config except Exception as exception: - logger.warn("Fetching the goal state failed: {0}", ustr(exception)) + self.logger.warn("Fetching the goal state failed: {0}", ustr(exception)) raise ProtocolError(msg="Error fetching goal state", inner=exception) finally: - logger.info('Fetch goal state completed') + self.logger.info('Fetch goal state completed') class HostingEnv(object): @@ -347,8 +409,10 @@ def __init__(self, xml_text): class Certificates(object): - def __init__(self, xml_text): + def __init__(self, xml_text, my_logger): self.cert_list = CertList() + self.summary = [] # debugging info + self.warnings = [] # Save the certificates local_file = os.path.join(conf.get_lib_dir(), CERTS_FILE_NAME) @@ -363,7 +427,7 @@ def __init__(self, xml_text): # if the certificates format is not Pkcs7BlobWithPfxContents do not parse it certificateFormat = findtext(xml_doc, "Format") if certificateFormat and certificateFormat != "Pkcs7BlobWithPfxContents": - logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) + my_logger.warn("The Format is not Pkcs7BlobWithPfxContents. Format is " + certificateFormat) return cryptutil = CryptUtil(conf.get_openssl_cmd()) @@ -428,18 +492,14 @@ def __init__(self, xml_text): tmp_file = prvs[pubkey] prv = "{0}.prv".format(thumbprint) os.rename(tmp_file, os.path.join(conf.get_lib_dir(), prv)) - logger.info("Found private key matching thumbprint {0}".format(thumbprint)) else: # Since private key has *no* matching certificate, # it will not be named correctly - logger.warn("Found NO matching cert/thumbprint for private key!") + self.warnings.append("Found NO matching cert/thumbprint for private key!") - # Log if any certificates were found without matching private keys - # This can happen (rarely), and is useful to know for debugging - for pubkey in thumbprints: - if not pubkey in prvs: - msg = "Certificate with thumbprint {0} has no matching private key." - logger.info(msg.format(thumbprints[pubkey])) + for pubkey, thumbprint in thumbprints.items(): + has_private_key = pubkey in prvs + self.summary.append({"thumbprint": thumbprint, "hasPrivateKey": has_private_key}) for v1_cert in v1_cert_list: cert = Cert() @@ -452,6 +512,11 @@ def _write_to_tmp_file(index, suffix, buf): fileutil.write_file(file_name, "".join(buf)) return file_name +class EmptyCertificates: + def __init__(self): + self.cert_list = CertList() + self.summary = [] # debugging info + self.warnings = [] class RemoteAccess(object): """ diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py index cf254be30..f79076f8e 100644 --- a/azurelinuxagent/common/protocol/hostplugin.py +++ b/azurelinuxagent/common/protocol/hostplugin.py @@ -95,7 +95,7 @@ def __init__(self, endpoint): if not os.path.exists(self._get_fast_track_state_file()): self._supports_vm_settings = False self._supports_vm_settings_next_check = datetime.datetime.now() - self._fast_track_timestamp = None + self._fast_track_timestamp = timeutil.create_timestamp(datetime.datetime.min) else: self._supports_vm_settings = True self._supports_vm_settings_next_check = datetime.datetime.now() @@ -443,7 +443,7 @@ def get_fast_track_timestamp(): goal state was Fabric or fetch_vm_settings() has not been invoked. """ if not os.path.exists(HostPluginProtocol._get_fast_track_state_file()): - return None + return timeutil.create_timestamp(datetime.datetime.min) try: with open(HostPluginProtocol._get_fast_track_state_file(), "r") as file_: @@ -453,7 +453,7 @@ def get_fast_track_timestamp(): HostPluginProtocol._get_fast_track_state_file(), ustr(e)) return timeutil.create_timestamp(datetime.datetime.utcnow()) - def fetch_vm_settings(self): + def fetch_vm_settings(self, force_update=False): """ Queries the vmSettings from the HostGAPlugin and returns an (ExtensionsGoalState, bool) tuple with the vmSettings and a boolean indicating if they are an updated (True) or a cached value (False). @@ -491,7 +491,7 @@ def format_message(msg): # Raise VmSettingsNotSupported directly instead of using raise_not_supported() to avoid resetting the timestamp for the next check raise VmSettingsNotSupported() - etag = None if self._cached_vm_settings is None else self._cached_vm_settings.etag + etag = None if force_update or self._cached_vm_settings is None else self._cached_vm_settings.etag correlation_id = str(uuid.uuid4()) self._vm_settings_error_reporter.report_request() @@ -547,9 +547,8 @@ def format_message(msg): logger.info(message) add_event(op=WALAEventOperation.HostPlugin, message=message, is_success=True) - # Don't support HostGAPlugin versions older than 123 - # TODO: update the minimum version to 1.0.8.123 before release - if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.117"): + # Don't support HostGAPlugin versions older than 124 + if vm_settings.host_ga_plugin_version < FlexibleVersion("1.0.8.124"): raise_not_supported() self._supports_vm_settings = True diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py index 40e58cc0f..b8b05c98b 100644 --- a/azurelinuxagent/common/protocol/wire.py +++ b/azurelinuxagent/common/protocol/wire.py @@ -83,8 +83,8 @@ def detect(self): logger.info('Initializing goal state during protocol detection') self.client.update_goal_state(force_update=True) - def update_goal_state(self): - self.client.update_goal_state() + def update_goal_state(self, silent=False): + self.client.update_goal_state(silent=silent) def update_host_plugin_from_goal_state(self): self.client.update_host_plugin_from_goal_state() @@ -130,7 +130,7 @@ def get_goal_state(self): def _download_ext_handler_pkg_through_host(self, uri, destination): host = self.client.get_host_plugin() uri, headers = host.get_artifact_request(uri, host.manifest_uri) - success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) + success = self.client.stream(uri, destination, headers=headers, use_proxy=False, max_retry=1) # set max_retry to 1 because extension packages already have a retry loop (see ExtHandlerInstance.download()) return success def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True): # pylint: disable=W0613 @@ -626,7 +626,7 @@ def call_storage_service(http_req, *args, **kwargs): def fetch_manifest_through_host(self, uri): host = self.get_host_plugin() uri, headers = host.get_artifact_request(uri) - response, _ = self.fetch(uri, headers, use_proxy=False, max_retry=1) + response, _ = self.fetch(uri, headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES) return response def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): @@ -649,9 +649,11 @@ def fetch_manifest(self, version_uris, timeout_in_minutes=5, timeout_in_ms=0): logger.verbose('The specified manifest URL is empty, ignored.') continue - direct_func = lambda: self.fetch(version_uri, max_retry=1)[0] # pylint: disable=W0640 + # Disable W0640: OK to use version_uri in a lambda within the loop's body + direct_func = lambda: self.fetch(version_uri)[0] # pylint: disable=W0640 # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data # in the lambda. + # Disable W0640: OK to use version_uri in a lambda within the loop's body host_func = lambda: self.fetch_manifest_through_host(version_uri) # pylint: disable=W0640 try: @@ -690,7 +692,7 @@ def stream(self, uri, destination, headers=None, use_proxy=None, max_retry=None) return success - def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, ok_codes=None): + def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used @@ -699,14 +701,14 @@ def fetch(self, uri, headers=None, use_proxy=None, decode=True, max_retry=None, logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers) content = None response_headers = None - response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, ok_codes=ok_codes) + response = self._fetch_response(uri, headers, use_proxy, max_retry=max_retry, retry_codes=retry_codes, ok_codes=ok_codes) if response is not None and not restutil.request_failed(response, ok_codes=ok_codes): response_content = response.read() content = self.decode_config(response_content) if decode else response_content response_headers = response.getheaders() return content, response_headers - def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_codes=None): + def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, retry_codes=None, ok_codes=None): """ max_retry indicates the maximum number of retries for the HTTP request; None indicates that the default value should be used """ @@ -717,7 +719,8 @@ def _fetch_response(self, uri, headers=None, use_proxy=None, max_retry=None, ok_ uri, headers=headers, use_proxy=use_proxy, - max_retry=max_retry) + max_retry=max_retry, + retry_codes=retry_codes) host_plugin = self.get_host_plugin() @@ -759,18 +762,18 @@ def update_host_plugin(self, container_id, role_config_name): self._host_plugin.update_container_id(container_id) self._host_plugin.update_role_config_name(role_config_name) - def update_goal_state(self, force_update=False): + def update_goal_state(self, force_update=False, silent=False): """ Updates the goal state if the incarnation or etag changed or if 'force_update' is True """ try: - if force_update: - logger.info("Forcing an update of the goal state..") + if force_update and not silent: + logger.info("Forcing an update of the goal state.") if self._goal_state is None or force_update: - self._goal_state = GoalState(self) + self._goal_state = GoalState(self, silent=silent) else: - self._goal_state.update() + self._goal_state.update(silent=silent) except ProtocolError: raise @@ -967,11 +970,13 @@ def upload_status_blob(self): if extensions_goal_state.status_upload_blob is None: # the status upload blob is in ExtensionsConfig so force a full goal state refresh - self.update_goal_state(force_update=True) + self.update_goal_state(force_update=True, silent=True) extensions_goal_state = self.get_goal_state().extensions_goal_state - if extensions_goal_state.status_upload_blob is None: - raise ProtocolNotFoundError("Status upload uri is missing") + if extensions_goal_state.status_upload_blob is None: + raise ProtocolNotFoundError("Status upload uri is missing") + + logger.info("Refreshed the goal state to get the status upload blob. New Goal State ID: {0}", extensions_goal_state.id) blob_type = extensions_goal_state.status_upload_blob_type diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py index 299b1a8a5..21a7af860 100644 --- a/azurelinuxagent/common/rdma.py +++ b/azurelinuxagent/common/rdma.py @@ -424,10 +424,7 @@ def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec): n = 0 found_ib0 = None while not found_ib0 and n < total_retries: - ret, output = shellutil.run_get_output("ifconfig -a") - if ret != 0: - raise Exception("Failed to list network interfaces") - found_ib0 = re.search("ib0", output, re.IGNORECASE) + found_ib0 = os.path.exists('/sys/class/net/ib0') if found_ib0: break time.sleep(check_interval_sec) @@ -439,8 +436,8 @@ def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec): netmask = 16 logger.info("RDMA: configuring IPv4 addr and netmask on ipoib interface") addr = '{0}/{1}'.format(ipv4_addr, netmask) - if shellutil.run("ifconfig ib0 {0}".format(addr)) != 0: - raise Exception("Could set addr to {0} on ib0".format(addr)) + if shellutil.run("ip addr add {0} dev ib0".format(addr)) != 0: + raise Exception("Could not set addr to {0} on ib0".format(addr)) logger.info("RDMA: ipoib address and netmask configured on interface") @staticmethod @@ -533,27 +530,20 @@ def update_network_interface(mac_addr, ipv4_addr): if_name = RDMADeviceHandler.get_interface_by_mac(mac_addr) logger.info("RDMA: network interface found: {0}", if_name) logger.info("RDMA: bringing network interface up") - if shellutil.run("ifconfig {0} up".format(if_name)) != 0: + if shellutil.run("ip link set dev {0} up".format(if_name)) != 0: raise Exception("Could not bring up RMDA interface: {0}".format(if_name)) logger.info("RDMA: configuring IPv4 addr and netmask on interface") addr = '{0}/{1}'.format(ipv4_addr, netmask) - if shellutil.run("ifconfig {0} {1}".format(if_name, addr)) != 0: + if shellutil.run("ip addr add {0} dev {1}".format(addr, if_name)) != 0: raise Exception("Could set addr to {1} on {0}".format(if_name, addr)) logger.info("RDMA: network address and netmask configured on interface") @staticmethod def get_interface_by_mac(mac): - ret, output = shellutil.run_get_output("ifconfig -a") - if ret != 0: - raise Exception("Failed to list network interfaces") - output = output.replace('\n', '') - match = re.search(r"(eth\d).*(HWaddr|ether) {0}".format(mac), - output, re.IGNORECASE) - if match is None: - raise Exception("Failed to get ifname with mac: {0}".format(mac)) - output = match.group(0) - eths = re.findall(r"eth\d", output) - if eths is None or len(eths) == 0: - raise Exception("ifname with mac: {0} not found".format(mac)) - return eths[-1] + for iface in os.scandir('/sys/class/net'): + addr_file = os.path.join('/sys/class/net', iface, 'address') + if os.path.exists(addr_file): + if open(addr_file, 'r').read() == mac.lower(): + return iface + raise Exception("Failed to get ifname with mac: {0}".format(mac)) diff --git a/azurelinuxagent/common/utils/archive.py b/azurelinuxagent/common/utils/archive.py index ed8122e97..1f8fdd931 100644 --- a/azurelinuxagent/common/utils/archive.py +++ b/azurelinuxagent/common/utils/archive.py @@ -9,7 +9,7 @@ import azurelinuxagent.common.logger as logger import azurelinuxagent.common.conf as conf -from azurelinuxagent.common.utils import fileutil +from azurelinuxagent.common.utils import fileutil, timeutil # pylint: disable=W0105 @@ -39,16 +39,23 @@ ARCHIVE_DIRECTORY_NAME = 'history' +# TODO: See comment in GoalStateHistory._save_placeholder and remove this code when no longer needed +_PLACEHOLDER_FILE_NAME = 'GoalState.1.xml' +# END TODO + _MAX_ARCHIVED_STATES = 50 _CACHE_PATTERNS = [ - re.compile(r"^VmSettings.\d+\.json$"), + # + # Note that SharedConfig.xml is not included here; this file is used by other components (Azsec and Singularity/HPC Infiniband) + # + re.compile(r"^VmSettings\.\d+\.json$"), re.compile(r"^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE), re.compile(r"^(.*)\.(\d+)\.(xml)$", re.IGNORECASE), - re.compile(r"^SharedConfig\.xml$", re.IGNORECASE), re.compile(r"^HostingEnvironmentConfig\.xml$", re.IGNORECASE), - re.compile(r"^RemoteAccess\.xml$", re.IGNORECASE) + re.compile(r"^RemoteAccess\.xml$", re.IGNORECASE), + re.compile(r"^waagent_status\.\d+\.json$"), ] # @@ -57,25 +64,28 @@ # 2018-04-06T08:21:37.142697.zip # 2018-04-06T08:21:37.142697_incarnation_N # 2018-04-06T08:21:37.142697_incarnation_N.zip +# 2018-04-06T08:21:37.142697_N-M +# 2018-04-06T08:21:37.142697_N-M.zip # # Current names # -# 2018-04-06T08:21:37.142697_N-M -# 2018-04-06T08:21:37.142697_N-M.zip +# 2018-04-06T08-21-37__N-M +# 2018-04-06T08-21-37__N-M.zip # -_ARCHIVE_BASE_PATTERN = r"\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+((_incarnation)?_(\d+|status)(-\d+)?)?" +_ARCHIVE_BASE_PATTERN = r"\d{4}\-\d{2}\-\d{2}T\d{2}[:-]\d{2}[:-]\d{2}(\.\d+)?((_incarnation)?_+(\d+|status)(-\d+)?)?" _ARCHIVE_PATTERNS_DIRECTORY = re.compile(r'^{0}$'.format(_ARCHIVE_BASE_PATTERN)) _ARCHIVE_PATTERNS_ZIP = re.compile(r'^{0}\.zip$'.format(_ARCHIVE_BASE_PATTERN)) _GOAL_STATE_FILE_NAME = "GoalState.xml" _VM_SETTINGS_FILE_NAME = "VmSettings.json" +_CERTIFICATES_FILE_NAME = "Certificates.json" _HOSTING_ENV_FILE_NAME = "HostingEnvironmentConfig.xml" -_SHARED_CONF_FILE_NAME = "SharedConfig.xml" _REMOTE_ACCESS_FILE_NAME = "RemoteAccess.xml" _EXT_CONF_FILE_NAME = "ExtensionsConfig.xml" _MANIFEST_FILE_NAME = "{0}.manifest.xml" AGENT_STATUS_FILE = "waagent_status.json" +SHARED_CONF_FILE_NAME = "SharedConfig.xml" # TODO: use @total_ordering once RHEL/CentOS and SLES 11 are EOL. # @total_ordering first appeared in Python 2.7 and 3.2 @@ -154,22 +164,15 @@ def __init__(self, lib_dir): if exception.errno != errno.EEXIST: logger.warn("{0} : {1}", self._source, exception.strerror) - def purge(self): - """ - Delete "old" archive directories and .zip archives. Old - is defined as any directories or files older than the X - newest ones. Also, clean up any legacy history files. - """ - states = self._get_archive_states() - states.sort(reverse=True) - - for state in states[_MAX_ARCHIVED_STATES:]: - state.delete() - @staticmethod def purge_legacy_goal_state_history(): lib_dir = conf.get_lib_dir() for current_file in os.listdir(lib_dir): + # Don't remove the placeholder goal state file. + # TODO: See comment in GoalStateHistory._save_placeholder and remove this code when no longer needed + if current_file == _PLACEHOLDER_FILE_NAME: + continue + # END TODO full_path = os.path.join(lib_dir, current_file) for pattern in _CACHE_PATTERNS: match = pattern.match(current_file) @@ -182,7 +185,6 @@ def purge_legacy_goal_state_history(): def archive(self): states = self._get_archive_states() - states.sort(reverse=True) if len(states) > 0: # Skip the most recent goal state, since it may still be in use @@ -201,13 +203,18 @@ def _get_archive_states(self): if match is not None: states.append(StateZip(full_path, match.group(0))) + states.sort(key=lambda state: os.path.getctime(state._path), reverse=True) + return states class GoalStateHistory(object): - def __init__(self, timestamp, tag): + def __init__(self, time, tag): self._errors = False - self._root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME, "{0}_{1}".format(timestamp, tag) if tag is not None else timestamp) + timestamp = timeutil.create_history_timestamp(time) + self._root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME, "{0}__{1}".format(timestamp, tag) if tag is not None else timestamp) + + GoalStateHistory._purge() @staticmethod def tag_exists(tag): @@ -227,8 +234,60 @@ def save(self, data, file_name): self._errors = True logger.warn("Failed to save {0} to the goal state history: {1} [no additional errors saving the goal state will be reported]".format(file_name, e)) + _purge_error_count = 0 + + @staticmethod + def _purge(): + """ + Delete "old" history directories and .zip archives. Old is defined as any directories or files older than the X newest ones. + """ + try: + history_root = os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME) + + if not os.path.exists(history_root): + return + + items = [] + for current_item in os.listdir(history_root): + full_path = os.path.join(history_root, current_item) + items.append(full_path) + items.sort(key=os.path.getctime, reverse=True) + + for current_item in items[_MAX_ARCHIVED_STATES:]: + if os.path.isfile(current_item): + os.remove(current_item) + else: + shutil.rmtree(current_item) + + if GoalStateHistory._purge_error_count > 0: + GoalStateHistory._purge_error_count = 0 + # Log a success message when we are recovering from errors. + logger.info("Successfully cleaned up the goal state history directory") + + except Exception as e: + GoalStateHistory._purge_error_count += 1 + if GoalStateHistory._purge_error_count < 5: + logger.warn("Failed to clean up the goal state history directory: {0}".format(e)) + elif GoalStateHistory._purge_error_count == 5: + logger.warn("Failed to clean up the goal state history directory [will stop reporting these errors]: {0}".format(e)) + + + @staticmethod + def _save_placeholder(): + """ + Some internal components took a dependency in the legacy GoalState.*.xml file. We create it here while those components are updated to remove the dependency. + When removing this code, also remove the check in StateArchiver.purge_legacy_goal_state_history, and the definition of _PLACEHOLDER_FILE_NAME + """ + try: + placeholder = os.path.join(conf.get_lib_dir(), _PLACEHOLDER_FILE_NAME) + with open(placeholder, "w") as handle: + handle.write("empty placeholder file") + except Exception as e: + logger.warn("Failed to save placeholder file ({0}): {1}".format(_PLACEHOLDER_FILE_NAME, e)) + def save_goal_state(self, text): self.save(text, _GOAL_STATE_FILE_NAME) + self._save_placeholder() def save_extensions_config(self, text): self.save(text, _EXT_CONF_FILE_NAME) @@ -239,8 +298,11 @@ def save_vm_settings(self, text): def save_remote_access(self, text): self.save(text, _REMOTE_ACCESS_FILE_NAME) + def save_certificates(self, text): + self.save(text, _CERTIFICATES_FILE_NAME) + def save_hosting_env(self, text): self.save(text, _HOSTING_ENV_FILE_NAME) def save_shared_conf(self, text): - self.save(text, _SHARED_CONF_FILE_NAME) + self.save(text, SHARED_CONF_FILE_NAME) diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py index 0c6d6d9ad..8c2fc4e4e 100644 --- a/azurelinuxagent/common/utils/restutil.py +++ b/azurelinuxagent/common/utils/restutil.py @@ -56,6 +56,15 @@ 429, # Request Rate Limit Exceeded ] +# +# Currently the HostGAPlugin has an issue its cache that may produce a BAD_REQUEST failure for valid URIs when using the extensionArtifact API. +# Add this status to the retryable codes, but use it only when requesting downloads via the HostGAPlugin. The retry logic in the download code +# would give enough time to the HGAP to refresh its cache. Once the fix to address that issue is deployed, consider removing the use of +# HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES. +# +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES = RETRY_CODES[:] # make a copy of RETRY_CODES +HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES.append(httpclient.BAD_REQUEST) + RESOURCE_GONE_CODES = [ httpclient.GONE ] diff --git a/azurelinuxagent/common/utils/timeutil.py b/azurelinuxagent/common/utils/timeutil.py index c4dd755a0..c8fa37647 100644 --- a/azurelinuxagent/common/utils/timeutil.py +++ b/azurelinuxagent/common/utils/timeutil.py @@ -5,7 +5,7 @@ def create_timestamp(dt=None): """ - Returns a string with the given datetime iso format. If no datetime is given as parameter, it + Returns a string with the given datetime in iso format. If no datetime is given as parameter, it uses datetime.utcnow(). """ if dt is None: @@ -13,6 +13,15 @@ def create_timestamp(dt=None): return dt.isoformat() +def create_history_timestamp(dt=None): + """ + Returns a string with the given datetime formatted as a timestamp for the agent's history folder + """ + if dt is None: + dt = datetime.datetime.utcnow() + return dt.strftime('%Y-%m-%dT%H-%M-%S') + + def datetime_to_ticks(dt): """ Converts 'dt', a datetime, to the number of ticks (1 tick == 1/10000000 sec) since datetime.min (0001-01-01 00:00:00). diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py index ff9c903b9..099ebc542 100644 --- a/azurelinuxagent/common/version.py +++ b/azurelinuxagent/common/version.py @@ -207,9 +207,9 @@ def has_logrotate(): # IMPORTANT: Please be sure that the version is always 9.9.9.9 on the develop branch. Automation requires this, otherwise # DCR may test the wrong agent version. # -# When doing a release, be sure to use the actual agent version. Current agent version: 2.4.0.0 +# When doing a release, be sure to use the actual agent version. # -AGENT_VERSION = '9.9.9.9' +AGENT_VERSION = '2.8.0.11' AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION) AGENT_DESCRIPTION = """ The Azure Linux Agent supports the provisioning and running of Linux diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py index 91685bc64..c608768a6 100644 --- a/azurelinuxagent/daemon/main.py +++ b/azurelinuxagent/daemon/main.py @@ -64,7 +64,7 @@ def run(self, child_args=None): # # Be aware that telemetry events emitted before that will not include the Container ID. # - logger.info("{0} Version:{1}", AGENT_LONG_NAME, AGENT_VERSION) + logger.info("{0} Version: {1}", AGENT_LONG_NAME, AGENT_VERSION) logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py index cc9d0afc3..3e8dbc23d 100644 --- a/azurelinuxagent/ga/exthandlers.py +++ b/azurelinuxagent/ga/exthandlers.py @@ -308,6 +308,7 @@ def run(self): error = None message = "ProcessExtensionsGoalState started [{0} channel: {1} source: {2} activity: {3} correlation {4} created: {5}]".format( egs.id, egs.channel, egs.source, egs.activity_id, egs.correlation_id, egs.created_on_timestamp) + logger.info('') logger.info(message) add_event(op=WALAEventOperation.ExtensionProcessing, message=message) @@ -319,7 +320,7 @@ def run(self): finally: duration = elapsed_milliseconds(utc_start) if error is None: - message = 'ProcessExtensionsGoalState completed [{0} {1} ms]'.format(egs.id, duration) + message = 'ProcessExtensionsGoalState completed [{0} {1} ms]\n'.format(egs.id, duration) logger.info(message) else: message = 'ProcessExtensionsGoalState failed [{0} {1} ms]\n{2}'.format(egs.id, duration, error) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 2118ca683..d17fff6a4 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -54,7 +54,7 @@ from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.utils.networkutil import AddFirewallRules from azurelinuxagent.common.utils.shellutil import CommandError -from azurelinuxagent.common.version import AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, \ +from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_NAME, AGENT_DIR_PATTERN, CURRENT_AGENT, AGENT_VERSION, \ CURRENT_VERSION, DISTRO_NAME, DISTRO_VERSION, get_lis_version, \ has_logrotate, PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO, get_daemon_version from azurelinuxagent.ga.collect_logs import get_collect_logs_handler, is_log_collection_allowed @@ -168,7 +168,8 @@ def __init__(self): # these members are used to avoid reporting errors too frequently self._heartbeat_update_goal_state_error_count = 0 - self._last_try_update_goal_state_failed = False + self._update_goal_state_error_count = 0 + self._update_goal_state_last_error_report = datetime.min self._report_status_last_failed_goal_state = None # incarnation of the last goal state that has been fully processed @@ -324,20 +325,10 @@ def run(self, debug=False): """ try: - logger.info(u"Agent {0} is running as the goal state agent", CURRENT_AGENT) + logger.info("{0} (Goal State Agent version {1})", AGENT_LONG_NAME, AGENT_VERSION) + logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) + logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) - # - # Initialize the goal state; some components depend on information provided by the goal state and this - # call ensures the required info is initialized (e.g. telemetry depends on the container ID.) - # - protocol = self.protocol_util.get_protocol() - - self._initialize_goal_state(protocol) - - # Initialize the common parameters for telemetry events - initialize_event_logger_vminfo_common_parameters(protocol) - - # Log OS-specific info. os_info_msg = u"Distro: {dist_name}-{dist_ver}; "\ u"OSUtil: {util_name}; AgentService: {service_name}; "\ u"Python: {py_major}.{py_minor}.{py_micro}; "\ @@ -351,8 +342,20 @@ def run(self, debug=False): py_micro=PY_VERSION_MICRO, systemd=systemd.is_systemd(), lis_ver=get_lis_version(), has_logrotate=has_logrotate() ) - logger.info(os_info_msg) + + # + # Initialize the goal state; some components depend on information provided by the goal state and this + # call ensures the required info is initialized (e.g. telemetry depends on the container ID.) + # + protocol = self.protocol_util.get_protocol() + + self._initialize_goal_state(protocol) + + # Initialize the common parameters for telemetry events + initialize_event_logger_vminfo_common_parameters(protocol) + + # Send telemetry for the OS-specific info. add_event(AGENT_NAME, op=WALAEventOperation.OSInfo, message=os_info_msg) # @@ -375,6 +378,7 @@ def run(self, debug=False): self._ensure_firewall_rules_persisted(dst_ip=protocol.get_endpoint()) self._add_accept_tcp_firewall_rule_if_not_enabled(dst_ip=protocol.get_endpoint()) self._reset_legacy_blacklisted_agents() + self._cleanup_legacy_goal_state_history() # Get all thread handlers telemetry_handler = get_send_telemetry_events_handler(self.protocol_util) @@ -393,8 +397,6 @@ def run(self, debug=False): logger.info("Goal State Period: {0} sec. This indicates how often the agent checks for new goal states and reports status.", self._goal_state_period) - self._cleanup_legacy_goal_state_history() - while self.is_running: self._check_daemon_running(debug) self._check_threads_running(all_thread_handlers) @@ -479,15 +481,18 @@ def _try_update_goal_state(self, protocol): Attempts to update the goal state and returns True on success or False on failure, sending telemetry events about the failures. """ try: - protocol.update_goal_state() + max_errors_to_log = 3 + + protocol.update_goal_state(silent=self._update_goal_state_error_count >= max_errors_to_log) self._goal_state = protocol.get_goal_state() - if self._last_try_update_goal_state_failed: - self._last_try_update_goal_state_failed = False - message = u"Retrieving the goal state recovered from previous errors" + if self._update_goal_state_error_count > 0: + message = u"Fetching the goal state recovered from previous errors. Fetched {0} (certificates: {1})".format( + self._goal_state.extensions_goal_state.id, self._goal_state.certs.summary) add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=True, message=message, log_event=False) logger.info(message) + self._update_goal_state_error_count = 0 try: self._supports_fast_track = conf.get_enable_fast_track() and protocol.client.get_host_plugin().check_vm_settings_support() @@ -495,15 +500,21 @@ def _try_update_goal_state(self, protocol): self._supports_fast_track = False except Exception as e: - if not self._last_try_update_goal_state_failed: - self._last_try_update_goal_state_failed = True - message = u"An error occurred while retrieving the goal state: {0}".format(textutil.format_exception(e)) - logger.warn(message) - add_event(AGENT_NAME, op=WALAEventOperation.FetchGoalState, version=CURRENT_VERSION, is_success=False, message=message, log_event=False) - message = u"Attempts to retrieve the goal state are failing: {0}".format(ustr(e)) - logger.periodic_warn(logger.EVERY_SIX_HOURS, "[PERIODIC] {0}".format(message)) + self._update_goal_state_error_count += 1 self._heartbeat_update_goal_state_error_count += 1 + if self._update_goal_state_error_count <= max_errors_to_log: + message = u"Error fetching the goal state: {0}".format(textutil.format_exception(e)) + logger.error(message) + add_event(op=WALAEventOperation.FetchGoalState, is_success=False, message=message, log_event=False) + self._update_goal_state_last_error_report = datetime.now() + else: + if self._update_goal_state_last_error_report + timedelta(hours=6) > datetime.now(): + self._update_goal_state_last_error_report = datetime.now() + message = u"Fetching the goal state is still failing: {0}".format(textutil.format_exception(e)) + logger.error(message) + add_event(op=WALAEventOperation.FetchGoalState, is_success=False, message=message, log_event=False) return False + return True def __update_guest_agent(self, protocol): @@ -557,8 +568,8 @@ def handle_updates_for_requested_version(): raise AgentUpgradeExitException( "Exiting current process to {0} to the request Agent version {1}".format(prefix, requested_version)) - # Ignore new agents if updating is disabled - if not conf.get_autoupdate_enabled(): + # Skip the update if there is no goal state yet or auto-update is disabled + if self._goal_state is None or not conf.get_autoupdate_enabled(): return False if self._download_agent_if_upgrade_available(protocol): @@ -594,16 +605,19 @@ def _processing_new_extensions_goal_state(self): return self._goal_state is not None and egs.id != self._last_extensions_gs_id and not egs.is_outdated def _process_goal_state(self, exthandlers_handler, remote_access_handler): - try: - protocol = exthandlers_handler.protocol + protocol = exthandlers_handler.protocol - # update self._goal_state - self._try_update_goal_state(protocol) + # update self._goal_state + if not self._try_update_goal_state(protocol): + # agent updates and status reporting should be done even when the goal state is not updated + self.__update_guest_agent(protocol) + self._report_status(exthandlers_handler) + return - # Update the Guest Agent if a new version is available - if self._goal_state is not None: - self.__update_guest_agent(protocol) + # check for agent updates + self.__update_guest_agent(protocol) + try: if self._processing_new_extensions_goal_state(): if not self._extensions_summary.converged: message = "A new goal state was received, but not all the extensions in the previous goal state have completed: {0}".format(self._extensions_summary) @@ -614,16 +628,15 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): self._extensions_summary = ExtensionsSummary() exthandlers_handler.run() - # always report status, even if the goal state did not change - # do it before processing the remote access, since that operation can take a long time + # report status before processing the remote access, since that operation can take a long time self._report_status(exthandlers_handler) if self._processing_new_incarnation(): remote_access_handler.run() - # lastly, cleanup the goal state history (but do it only on new goal states - no need to do it on every iteration) + # lastly, archive the goal state history (but do it only on new goal states - no need to do it on every iteration) if self._processing_new_extensions_goal_state(): - UpdateHandler._cleanup_goal_state_history() + UpdateHandler._archive_goal_state_history() finally: if self._goal_state is not None: @@ -631,10 +644,9 @@ def _process_goal_state(self, exthandlers_handler, remote_access_handler): self._last_extensions_gs_id = self._goal_state.extensions_goal_state.id @staticmethod - def _cleanup_goal_state_history(): + def _archive_goal_state_history(): try: archiver = StateArchiver(conf.get_lib_dir()) - archiver.purge() archiver.archive() except Exception as exception: logger.warn("Error cleaning up the goal state history: {0}", ustr(exception)) @@ -734,7 +746,7 @@ def forward_signal(self, signum, frame): return logger.info( - u"Agent {0} forwarding signal {1} to {2}", + u"Agent {0} forwarding signal {1} to {2}\n", CURRENT_AGENT, signum, self.child_agent.name if self.child_agent is not None else CURRENT_AGENT) @@ -823,6 +835,9 @@ def log_if_op_disabled(name, value): if conf.get_autoupdate_enabled(): log_if_int_changed_from_default("Autoupdate.Frequency", conf.get_autoupdate_frequency()) + if conf.get_enable_fast_track(): + log_if_op_disabled("Debug.EnableFastTrack", conf.get_enable_fast_track()) + if conf.get_lib_dir() != "/var/lib/waagent": log_event("lib dir is in an unexpected location: {0}".format(conf.get_lib_dir())) @@ -1597,7 +1612,7 @@ def _download(self): uri, headers = self.host.get_artifact_request(uri, self.host.manifest_uri) try: - if self._fetch(uri, headers=headers, use_proxy=False): + if self._fetch(uri, headers=headers, use_proxy=False, retry_codes=restutil.HGAP_GET_EXTENSION_ARTIFACT_RETRY_CODES): if not HostPluginProtocol.is_default_channel: logger.verbose("Setting host plugin as default channel") HostPluginProtocol.is_default_channel = True @@ -1624,12 +1639,12 @@ def _download(self): message=msg) raise UpdateError(msg) - def _fetch(self, uri, headers=None, use_proxy=True): + def _fetch(self, uri, headers=None, use_proxy=True, retry_codes=None): package = None try: is_healthy = True error_response = '' - resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=1) + resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers, max_retry=3, retry_codes=retry_codes) # Use only 3 retries, since there are usually 5 or 6 URIs and we try all of them if restutil.request_succeeded(resp): package = resp.read() fileutil.write_file(self.get_agent_pkg_path(), diff --git a/tests/data/hostgaplugin/ext_conf-requested_version.xml b/tests/data/hostgaplugin/ext_conf-requested_version.xml index c3bd92823..bbb8a20fe 100644 --- a/tests/data/hostgaplugin/ext_conf-requested_version.xml +++ b/tests/data/hostgaplugin/ext_conf-requested_version.xml @@ -60,7 +60,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": {"GCS_AUTO_CONFIG":true} } @@ -73,7 +73,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": {"enableGenevaUpload":true} } diff --git a/tests/data/hostgaplugin/ext_conf.xml b/tests/data/hostgaplugin/ext_conf.xml index eac5d6364..ebd90aa0b 100644 --- a/tests/data/hostgaplugin/ext_conf.xml +++ b/tests/data/hostgaplugin/ext_conf.xml @@ -58,7 +58,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Monitor.AzureMonitorLinuxAgent==", "publicSettings": {"GCS_AUTO_CONFIG":true} } @@ -71,7 +71,7 @@ "runtimeSettings": [ { "handlerSettings": { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent==", "publicSettings": {"enableGenevaUpload":true} } diff --git a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json index 9cfb42752..560126870 100644 --- a/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json +++ b/tests/data/hostgaplugin/vm_settings-difference_in_required_features.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -76,7 +76,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"enableGenevaUpload\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json index 6fa93452c..94d9f0eb1 100644 --- a/tests/data/hostgaplugin/vm_settings-empty_depends_on.json +++ b/tests/data/hostgaplugin/vm_settings-empty_depends_on.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json new file mode 100644 index 000000000..bbd945933 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-fabric-no_thumbprints.json @@ -0,0 +1,192 @@ +{ + "hostGAPluginVersion": "1.0.8.124", + "vmSettingsSchemaVersion": "0.0", + "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", + "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", + "inSvdSeqNo": 1, + "extensionsLastModifiedTickCount": 637726657706205299, + "extensionGoalStatesSource": "Fabric", + "onHold": true, + "statusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "name": "Test", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.Azure.Monitor.AzureMonitorLinuxAgent", + "version": "1.9.1", + "location": "https://zrdfepirv2cbn04prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "failoverlocation": "https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml", + "additionalLocations": ["https://zrdfepirv2cbn09pr02a.blob.core.windows.net/a47f0806d764480a8d989d009c75007d/Microsoft.Azure.Monitor_AzureMonitorLinuxAgent_useast2euap_manifest.xml"], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "settings": [ + { + "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" + } + ] + }, + { + "name": "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent", + "version": "2.15.112", + "location": "https://zrdfepirv2cbn04prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml", + "failoverlocation": "https://zrdfepirv2cbz06prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml", + "additionalLocations": ["https://zrdfepirv2cbn06prdstr01a.blob.core.windows.net/4ef06ad957494df49c807a5334f2b5d2/Microsoft.Azure.Security.Monitoring_AzureSecurityLinuxAgent_useast2euap_manifest.xml"], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "settings": [ + { + "publicSettings": "{\"enableGenevaUpload\":true}" + } + ] + }, + { + "name": "Microsoft.Azure.Extensions.CustomScript", + "version": "2.1.6", + "location": "https://umsavwggj2v40kvqhc0w.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "failoverlocation": "https://umsafwzhkbm1rfrhl0ws.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml", + "additionalLocations": [ + "https://umsanh4b5rfz0q0p4pwm.blob.core.windows.net/5237dd14-0aad-f051-0fad-1e33e1b63091/5237dd14-0aad-f051-0fad-1e33e1b63091_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "publicSettings": "{\"commandToExecute\":\"echo 'cee174d4-4daa-4b07-9958-53b9649445c2'\"}" + } + ], + "dependsOn": [ + { + "DependsOnExtension": [ + { + "handler": "Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent" + } + ], + "dependencyLevel": 1 + } + ] + }, + { + "name": "Microsoft.CPlat.Core.RunCommandHandlerLinux", + "version": "1.2.0", + "location": "https://umsavbvncrpzbnxmxzmr.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml", + "failoverlocation": "https://umsajbjtqrb3zqjvgb2z.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml", + "additionalLocations": [ + "https://umsawqtlsshtn5v2nfgh.blob.core.windows.net/f4086d41-69f9-3103-78e0-8a2c7e789d0f/f4086d41-69f9-3103-78e0-8a2c7e789d0f_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": true, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": true, + "settings": [ + { + "publicSettings": "{\"source\":{\"script\":\"echo '4abb1e88-f349-41f8-8442-247d9fdfcac5'\"}}", + "seqNo": 0, + "extensionName": "MCExt1", + "extensionState": "enabled" + }, + { + "publicSettings": "{\"source\":{\"script\":\"echo 'e865c9bc-a7b3-42c6-9a79-cfa98a1ee8b3'\"}}", + "seqNo": 0, + "extensionName": "MCExt2", + "extensionState": "enabled" + }, + { + "publicSettings": "{\"source\":{\"script\":\"echo 'f923e416-0340-485c-9243-8b84fb9930c6'\"}}", + "seqNo": 0, + "extensionName": "MCExt3", + "extensionState": "enabled" + } + ], + "dependsOn": [ + { + "dependsOnExtension": [ + { + "extension": "...", + "handler": "..." + }, + { + "extension": "...", + "handler": "..." + } + ], + "dependencyLevel": 2, + "name": "MCExt1" + }, + { + "dependsOnExtension": [ + { + "extension": "...", + "handler": "..." + } + ], + "dependencyLevel": 1, + "name": "MCExt2" + } + ] + }, + { + "name": "Microsoft.OSTCExtensions.VMAccessForLinux", + "version": "1.5.11", + "location": "https://umsasc25p0kjg0c1dg4b.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "failoverlocation": "https://umsamfwlmfshvxx2lsjm.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "additionalLocations": [ + "https://umsah3cwjlctnmhsvzqv.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": false, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ ] + } + ] +} diff --git a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json index 62314a403..e7945845a 100644 --- a/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json +++ b/tests/data/hostgaplugin/vm_settings-invalid_blob_type.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "2e7f8b5d-f637-4721-b757-cb190d49b4e9", "correlationId": "1bef4c48-044e-4225-8f42-1d1eac1eb158", diff --git a/tests/data/hostgaplugin/vm_settings-missing_cert.json b/tests/data/hostgaplugin/vm_settings-missing_cert.json new file mode 100644 index 000000000..a7192e942 --- /dev/null +++ b/tests/data/hostgaplugin/vm_settings-missing_cert.json @@ -0,0 +1,68 @@ +{ + "hostGAPluginVersion": "1.0.8.124", + "vmSettingsSchemaVersion": "0.0", + "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", + "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", + "inSvdSeqNo": 1, + "extensionsLastModifiedTickCount": 637726657706205299, + "extensionGoalStatesSource": "FastTrack", + "onHold": true, + "statusUploadBlob": { + "statusBlobType": "BlockBlob", + "value": "https://dcrcl3a0xs.blob.core.windows.net/$system/edp0plkw2b.86f4ae0a-61f8-48ae-9199-40f402d56864.status?sv=2018-03-28&sr=b&sk=system-1&sig=KNWgC2%3d&se=9999-01-01T00%3a00%3a00Z&sp=w" + }, + "inVMMetadata": { + "subscriptionId": "8e037ad4-618f-4466-8bc8-5099d41ac15b", + "resourceGroupName": "rg-dc-86fjzhp", + "vmName": "edp0plkw2b", + "location": "CentralUSEUAP", + "vmId": "86f4ae0a-61f8-48ae-9199-40f402d56864", + "vmSize": "Standard_B2s", + "osType": "Linux" + }, + "requiredFeatures": [ + { + "name": "MultipleExtensionsPerHandler" + } + ], + "gaFamilies": [ + { + "name": "Prod", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentraleuap_manifest.xml" + ] + }, + { + "name": "Test", + "uris": [ + "https://zrdfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml", + "https://ardfepirv2cdm03prdstr01a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Test_uscentraleuap_manifest.xml" + ] + } + ], + "extensionGoalStates": [ + { + "name": "Microsoft.OSTCExtensions.VMAccessForLinux", + "version": "1.5.11", + "location": "https://umsasc25p0kjg0c1dg4b.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "failoverlocation": "https://umsamfwlmfshvxx2lsjm.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml", + "additionalLocations": [ + "https://umsah3cwjlctnmhsvzqv.blob.core.windows.net/2bbece4f-0283-d415-b034-cc0adc6997a1/2bbece4f-0283-d415-b034-cc0adc6997a1_manifest.xml" + ], + "state": "enabled", + "autoUpgrade": false, + "runAsStartupTask": false, + "isJson": true, + "useExactVersion": true, + "settingsSeqNo": 0, + "isMultiConfig": false, + "settings": [ + { + "protectedSettingsCertThumbprint": "59A10F50FFE2A0408D3F03FE336C8FD5716CF25C", + "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpddesZQewdDBgegkxNzA1BgoJkgergres/Microsoft.OSTCExtensions.VMAccessForLinux==" + } + ] + } + ] +} diff --git a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json b/tests/data/hostgaplugin/vm_settings-no_manifests.json similarity index 82% rename from tests/data/hostgaplugin/vm_settings-no_extension_manifests.json rename to tests/data/hostgaplugin/vm_settings-no_manifests.json index 7deff8d5e..7ec3a5c3d 100644 --- a/tests/data/hostgaplugin/vm_settings-no_extension_manifests.json +++ b/tests/data/hostgaplugin/vm_settings-no_manifests.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "89d50bf1-fa55-4257-8af3-3db0c9f81ab4", "correlationId": "c143f8f0-a66b-4881-8c06-1efd278b0b02", @@ -27,12 +27,7 @@ }, "gaFamilies": [ { - "name": "Prod", - "uris": [ - "https://zrdfepirv2dz5prdstr07a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml", - "https://rdfepirv2dm1prdstr09.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml", - "https://zrdfepirv2dm5prdstr06a.blob.core.windows.net/7d89d439b79f4452950452399add2c90/Microsoft.OSTCLinuxAgent_Prod_uscentral_manifest.xml" - ] + "name": "Prod" } ], "extensionGoalStates": [ diff --git a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json index 27ebebcef..2f70b5576 100644 --- a/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json +++ b/tests/data/hostgaplugin/vm_settings-no_status_upload_blob.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-out-of-sync.json b/tests/data/hostgaplugin/vm_settings-out-of-sync.json index 737350d69..1f369ae5b 100644 --- a/tests/data/hostgaplugin/vm_settings-out-of-sync.json +++ b/tests/data/hostgaplugin/vm_settings-out-of-sync.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "AAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE", "correlationId": "EEEEEEEE-DDDD-CCCC-BBBB-AAAAAAAAAAAA", @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings-parse_error.json b/tests/data/hostgaplugin/vm_settings-parse_error.json index e817a1e88..bae5de4cb 100644 --- a/tests/data/hostgaplugin/vm_settings-parse_error.json +++ b/tests/data/hostgaplugin/vm_settings-parse_error.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": THIS_IS_A_SYNTAX_ERROR, "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", diff --git a/tests/data/hostgaplugin/vm_settings-requested_version.json b/tests/data/hostgaplugin/vm_settings-requested_version.json index 1b5023b11..98959dd4e 100644 --- a/tests/data/hostgaplugin/vm_settings-requested_version.json +++ b/tests/data/hostgaplugin/vm_settings-requested_version.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -74,7 +74,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/IsZAEZFidXaW5kb3dzIEF6dXJlIENSUCBDZXJ0aWZpY2F0ZSBHZW5lcmF0b3ICEFpB/HKM/7evRk+DBz754wUwDQYJKoZIhvcNAQEBBQAEggEADPJwniDeIUXzxNrZCloitFdscQ59Bz1dj9DLBREAiM8jmxM0LLicTJDUv272Qm/4ZQgdqpFYBFjGab/9MX+Ih2x47FkVY1woBkckMaC/QOFv84gbboeQCmJYZC/rZJdh8rCMS+CEPq3uH1PVrvtSdZ9uxnaJ+E4exTPPviIiLIPtqWafNlzdbBt8HZjYaVw+SSe+CGzD2pAQeNttq3Rt/6NjCzrjG8ufKwvRoqnrInMs4x6nnN5/xvobKIBSv4/726usfk8Ug+9Q6Benvfpmre2+1M5PnGTfq78cO3o6mI3cPoBUjp5M0iJjAMGeMt81tyHkimZrEZm6pLa4NQMOEjArBgkqhkiG9w0BBwEwFAYIKoZIhvcNAwcECC5nVaiJaWt+gAhgeYvxUOYHXw==", "publicSettings": "{\"enableGenevaUpload\":true}" } diff --git a/tests/data/hostgaplugin/vm_settings.json b/tests/data/hostgaplugin/vm_settings.json index b67ee0a23..a4ef0f785 100644 --- a/tests/data/hostgaplugin/vm_settings.json +++ b/tests/data/hostgaplugin/vm_settings.json @@ -1,5 +1,5 @@ { - "hostGAPluginVersion": "1.0.8.123", + "hostGAPluginVersion": "1.0.8.124", "vmSettingsSchemaVersion": "0.0", "activityId": "a33f6f53-43d6-4625-b322-1a39651a00c9", "correlationId": "9a47a2a2-e740-4bfc-b11b-4f2f7cfe7d2e", @@ -56,7 +56,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Monitor.AzureMonitorLinuxAgent==", "publicSettings": "{\"GCS_AUTO_CONFIG\":true}" } @@ -76,7 +76,7 @@ "settingsSeqNo": 0, "settings": [ { - "protectedSettingsCertThumbprint": "4C4F304667711036E64AF4894B76EB208A863BD4", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpMIIBZQIBADBNMDkxNzA1BgoJkiaJk/Microsoft.Azure.Security.Monitoring.AzureSecurityLinuxAgent==", "publicSettings": "{\"enableGenevaUpload\":true}" } @@ -192,7 +192,7 @@ "isMultiConfig": false, "settings": [ { - "protectedSettingsCertThumbprint": "59A10F50FFE2A0408D3F03FE336C8FD5716CF25C", + "protectedSettingsCertThumbprint": "4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3", "protectedSettings": "MIIBsAYJKoZIhvcNAQcDoIIBoTCCAZ0CAQAxggFpddesZQewdDBgegkxNzA1BgoJkgergres/Microsoft.OSTCExtensions.VMAccessForLinux==" } ] diff --git a/tests/data/wire/certs-2.xml b/tests/data/wire/certs-2.xml new file mode 100644 index 000000000..66a231ee8 --- /dev/null +++ b/tests/data/wire/certs-2.xml @@ -0,0 +1,85 @@ + + + 2012-11-30 + 5 + Pkcs7BlobWithPfxContents + MIIOgwYJKoZIhvcNAQcDoIIOdDCCDnACAQIxggEwMIIBLAIBAoAUiF8ZYMs9mMa8 +QOEMxDaIhGza+0IwDQYJKoZIhvcNAQEBBQAEggEAQW7GyeRVEhHSU1/dzV0IndH0 +rDQk+27MvlsWTcpNcgGFtfRYxu5bzmp0+DoimX3pRBlSFOpMJ34jpg4xs78EsSWH +FRhCf3EGuEUBHo6yR8FhXDTuS7kZ0UmquiCI2/r8j8gbaGBNeP8IRizcAYrPMA5S +E8l1uCrw7DHuLscbVni/7UglGaTfFS3BqS5jYbiRt2Qh3p+JPUfm51IG3WCIw/WS +2QHebmHxvMFmAp8AiBWSQJizQBEJ1lIfhhBMN4A7NadMWAe6T2DRclvdrQhJX32k +amOiogbW4HJsL6Hphn7Frrw3CENOdWMAvgQBvZ3EjAXgsJuhBA1VIrwofzlDljCC +DTUGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIxcvw9qx4y0qAgg0QrINXpC23BWT2 +Fb9N8YS3Be9eO3fF8KNdM6qGf0kKR16l/PWyP2L+pZxCcCPk83d070qPdnJK9qpJ +6S1hI80Y0oQnY9VBFrdfkc8fGZHXqm5jNS9G32v/AxYpJJC/qrAQnWuOdLtOZaGL +94GEh3XRagvz1wifv8SRI8B1MzxrpCimeMxHkL3zvJFg9FjLGdrak868feqhr6Nb +pqH9zL7bMq8YP788qTRELUnL72aDzGAM7HEj7V4yu2uD3i3Ryz3bqWaj9IF38Sa0 +6rACBkiNfZBPgExoMUm2GNVyx8hTis2XKRgz4NLh29bBkKrArK9sYDncE9ocwrrX +AQ99yn03Xv6TH8bRp0cSj4jzBXc5RFsUQG/LxzJVMjvnkDbwNE41DtFiYz5QVcv1 +cMpTH16YfzSL34a479eNq/4+JAs/zcb2wjBskJipMUU4hNx5fhthvfKwDOQbLTqN +HcP23iPQIhjdUXf6gpu5RGu4JZ0dAMHMHFKvNL6TNejwx/H6KAPp6rCRsYi6QhAb +42SXdZmhAyQsFpGD9U5ieJApqeCHfj9Xhld61GqLJA9+WLVhDPADjqHoAVvrOkKH +OtPegId/lWnCB7p551klAjiEA2/DKxFBIAEhqZpiLl+juZfMXovkdmGxMP4gvNNF +gbS2k5A0IJ8q51gZcH1F56smdAmi5kvhPnFdy/9gqeI/F11F1SkbPVLImP0mmrFi +zQD5JGfEu1psUYvhpOdaYDkmAK5qU5xHSljqZFz5hXNt4ebvSlurHAhunJb2ln3g +AJUHwtZnVBrtYMB0w6fdwYqMxXi4vLeqUiHtIQtbOq32zlSryNPQqG9H0iP9l/G1 +t7oUfr9woI/B0kduaY9jd5Qtkqs1DoyfNMSaPNohUK/CWOTD51qOadzSvK0hJ+At +033PFfv9ilaX6GmzHdEVEanrn9a+BoBCnGnuysHk/8gdswj9OzeCemyIFJD7iObN +rNex3SCf3ucnAejJOA0awaLx88O1XTteUjcFn26EUji6DRK+8JJiN2lXSyQokNeY +ox6Z4hFQDmw/Q0k/iJqe9/Dq4zA0l3Krkpra0DZoWh5kzYUA0g5+Yg6GmRNRa8YG +tuuD6qK1SBEzmCYff6ivjgsXV5+vFBSjEpx2dPEaKdYxtHMOjkttuTi1mr+19dVf +hSltbzfISbV9HafX76dhwZJ0QwsUx+aOW6OrnK8zoQc5AFOXpe9BrrOuEX01qrM0 +KX5tS8Zx5HqDLievjir194oi3r+nAiG14kYlGmOTHshu7keGCgJmzJ0iVG/i+TnV +ZSLyd8OqV1F6MET1ijgR3OPL3kt81Zy9lATWk/DgKbGBkkKAnXO2HUw9U34JFyEy +vEc81qeHci8sT5QKSFHiP3r8EcK8rT5k9CHpnbFmg7VWSMVD0/wRB/C4BiIw357a +xyJ/q1NNvOZVAyYzIzf9TjwREtyeHEo5kS6hyWSn7fbFf3sNGO2I30veWOvE6kFA +HMtF3NplOrTYcM7fAK5zJCBK20oU645TxI8GsICMog7IFidFMdRn4MaXpwAjEZO4 +44m2M+4XyeRCAZhp1Fu4mDiHGqgd44mKtwvLACVF4ygWZnACDpI17X88wMnwL4uU +vgehLZdAE89gvukSCsET1inVBnn/hVenCRbbZ++IGv2XoYvRfeezfOoNUcJXyawQ +JFqN0CRB5pliuCesTO2urn4HSwGGoeBd507pGWZmOAjbNjGswlJJXF0NFnNW/zWw +UFYy+BI9axuhWTSnCXbNbngdNQKHznKe1Lwit6AI3U9jS33pM3W+pwUAQegVdtpG +XT01YgiMCBX+b8B/xcWTww0JbeUwKXudzKsPhQmaA0lubAo04JACMfON8jSZCeRV +TyIzgacxGU6YbEKH4PhYTGl9srcWIT9iGSYD53V7Kyvjumd0Y3Qc3JLnuWZT6Oe3 +uJ4xz9jJtoaTDvPJQNK3igscjZnWZSP8XMJo1/f7vbvD57pPt1Hqdirp1EBQNshk +iX9CUh4fuGFFeHf6MtGxPofbXmvA2GYcFsOez4/2eOTEmo6H3P4Hrya97XHS0dmD +zFSAjzAlacTrn1uuxtxFTikdOwvdmQJJEfyYWCB1lqWOZi97+7nzqyXMLvMgmwug +ZF/xHFMhFTR8Wn7puuwf36JpPQiM4oQ/Lp66zkS4UlKrVsmSXIXudLMg8SQ5WqK8 +DjevEZwsHHaMtfDsnCAhAdRc2jCpyHKKnmhCDdkcdJJEymWKILUJI5PJ3XtiMHnR +Sa35OOICS0lTq4VwhUdkGwGjRoY1GsriPHd6LOt1aom14yJros1h7ta604hSCn4k +zj9p7wY9gfgkXWXNfmarrZ9NNwlHxzgSva+jbJcLmE4GMX5OFHHGlRj/9S1xC2Wf +MY9orzlooGM74NtmRi4qNkFj3dQCde8XRR4wh2IvPUCsr4j+XaoCoc3R5Rn/yNJK +zIkccJ2K14u9X/A0BLXHn5Gnd0tBYcVOqP6dQlW9UWdJC/Xooh7+CVU5cZIxuF/s +Vvg+Xwiv3XqekJRu3cMllJDp5rwe5EWZSmnoAiGKjouKAIszlevaRiD/wT6Zra3c +Wn/1U/sGop6zRscHR7pgI99NSogzpVGThUs+ez7otDBIdDbLpMjktahgWoi1Vqhc +fNZXjA6ob4zTWY/16Ys0YWxHO+MtyWTMP1dnsqePDfYXGUHe8yGxylbcjfrsVYta +4H6eYR86eU3eXB+MpS/iA4jBq4QYWR9QUkd6FDfmRGgWlMXhisPv6Pfnj384NzEV +Emeg7tW8wzWR64EON9iGeGYYa2BBl2FVaayMEoUhthhFcDM1r3/Mox5xF0qnlys4 +goWkMzqbzA2t97bC0KDGzkcHT4wMeiJBLDZ7S2J2nDAEhcTLY0P2zvOB4879pEWx +Bd15AyG1DvNssA5ooaDzKi/Li6NgDuMJ8W7+tmsBwDvwuf2N3koqBeXfKhR4rTqu +Wg1k9fX3+8DzDf0EjtDZJdfWZAynONi1PhZGbNbaMKsQ+6TflkCACInRdOADR5GM +rL7JtrgF1a9n0HD9vk2WGZqKI71tfS8zODkOZDD8aAusD2DOSmVZl48HX/t4i4Wc +3dgi/gkCMrfK3wOujb8tL4zjnlVkM7kzKk0MgHuA1w81zFjeMFvigHes4IWhQVcz +ek3l4bGifI2kzU7bGIi5e/019ppJzGsVcrOE/3z4GS0DJVk6fy7MEMIFx0LhJPlL +T+9HMH85sSYb97PTiMWpfBvNw3FSC7QQT9FC3L8d/XtMY3NvZoc7Fz7cSGaj7NXG +1OgVnAzMunPa3QaduoxMF9346s+4a+FrpRxL/3bb4skojjmmLqP4dsbD1uz0fP9y +xSifnTnrtjumYWMVi+pEb5kR0sTHl0XS7qKRi3SEfv28uh72KdvcufonIA5rnEb5 ++yqAZiqW2OxVsRoVLVODPswP4VIDiun2kCnfkQygPzxlZUeDZur0mmZ3vwC81C1Q +dZcjlukZcqUaxybUloUilqfNeby+2Uig0krLh2+AM4EqR63LeZ/tk+zCitHeRBW0 +wl3Bd7ShBFg6kN5tCJlHf/G6suIJVr+A9BXfwekO9+//CutKakCwmJTUiNWbQbtN +q3aNCnomyD3WjvUbitVO0CWYjZrmMLIsPtzyLQydpT7tjXpHgvwm5GYWdUGnNs4y +NbA262sUl7Ku/GDw1CnFYXbxl+qxbucLtCdSIFR2xUq3rEO1MXlD/txdTxn6ANax +hi9oBg8tHzuGYJFiCDCvbVVTHgWUSnm/EqfclpJzGmxt8g7vbaohW7NMmMQrLBFP +G6qBypgvotx1iJWaHVLNNiXvyqQwTtelNPAUweRoNawBp/5KTwwy/tHeF0gsVQ7y +mFX4umub9YT34Lpe7qUPKNxXzFcUgAf1SA6vyZ20UI7p42S2OT2PrahJ+uO6LQVD ++REhtN0oyS3G6HzAmKkBgw7LcV3XmAr39iSR7mdmoHSJuI9bjveAPhniK+N6uuln +xf17Qnw5NWfr9MXcLli7zqwMglU/1bNirkwVqf/ogi/zQ3JYCo6tFGf/rnGQAORJ +hvOq2SEYXnizPPIH7VrpE16+jUXwgpiQ8TDyeLPmpZVuhXTXiCaJO5lIwmLQqkmg +JqNiT9V44sksNFTGNKgZo5O9rEqfqX4dLjfv6pGJL+MFXD9if4f1JQiXJfhcRcDh +Ff9B6HukgbJ1H96eLUUNj8sL1+WPOqawkS4wg7tVaERE8CW7mqk15dCysn9shSut +I+7JU7+dZsxpj0ownrxuPAFuT8ZlcBPrFzPUwTlW1G0CbuEco8ijfy5IfbyGCn5s +K/0bOfAuNVGoOpLZ1dMki2bGdBwQOQlkLKhAxYcCVQ0/urr1Ab+VXU9kBsIU8ssN +GogKngYpuUV0PHmpzmobielOHLjNqA2v9vQSV3Ed48wRy5OCwLX1+vYmYlggMDGt +wfl+7QbXYf+k5WnELf3IqYvh8ZWexa0= + + \ No newline at end of file diff --git a/tests/data/wire/goal_state.xml b/tests/data/wire/goal_state.xml index 579b5e87a..0ccff211c 100644 --- a/tests/data/wire/goal_state.xml +++ b/tests/data/wire/goal_state.xml @@ -15,12 +15,12 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/extensionsconfiguri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/data/wire/goal_state_no_certs.xml b/tests/data/wire/goal_state_no_certs.xml new file mode 100644 index 000000000..1ab7fa217 --- /dev/null +++ b/tests/data/wire/goal_state_no_certs.xml @@ -0,0 +1,27 @@ + + + 2010-12-15 + 1 + + Started + + 16001 + + + + c6d5526c-5ac2-4200-b6e2-56f2b70c5ab2 + + + b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 + Started + + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml + + + + + diff --git a/tests/data/wire/goal_state_no_ext.xml b/tests/data/wire/goal_state_no_ext.xml index ef7e3989e..e9048daf6 100644 --- a/tests/data/wire/goal_state_no_ext.xml +++ b/tests/data/wire/goal_state_no_ext.xml @@ -15,11 +15,11 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/data/wire/goal_state_remote_access.xml b/tests/data/wire/goal_state_remote_access.xml index c2840645f..279006f21 100644 --- a/tests/data/wire/goal_state_remote_access.xml +++ b/tests/data/wire/goal_state_remote_access.xml @@ -17,12 +17,13 @@ b61f93d0-e1ed-40b2-b067-22c243233448.MachineRole_IN_0 Started - http://168.63.129.16:80/hostingenvuri/ - http://168.63.129.16:80/sharedconfiguri/ - http://168.63.129.16:80/certificatesuri/ - http://168.63.129.16:80/extensionsconfiguri/ - http://168.63.129.16:80/fullconfiguri/ - b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + b61f93d0-e1ed-40b2-b067-22c243233448.1.b61f93d0-e1ed-40b2-b067-22c243233448.2.MachineRole_IN_0.xml + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=hostingEnvironmentConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=sharedConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=extensionsConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=config&type=fullConfig&incarnation=1 + http://168.63.129.16:80/machine/865a6683-91d8-450f-99ae/bc8b9d47%2Db5ed%2D4704%2D85d9%2Dfd74cc967ec2.%5Fcanary?comp=certificates&incarnation=1 + bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5.bc8b9d47-b5ed-4704-85d9-fd74cc967ec2.5._canary.1.xml diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py index 8a648a8c4..cd5595569 100644 --- a/tests/ga/test_update.py +++ b/tests/ga/test_update.py @@ -40,7 +40,7 @@ VMAgentUpdateStatuses from azurelinuxagent.common.protocol.util import ProtocolUtil from azurelinuxagent.common.protocol.wire import WireProtocol -from azurelinuxagent.common.utils import fileutil, restutil, textutil +from azurelinuxagent.common.utils import fileutil, restutil, textutil, timeutil from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME, AGENT_STATUS_FILE from azurelinuxagent.common.utils.flexible_version import FlexibleVersion from azurelinuxagent.common.utils.networkutil import FirewallCmdDirectCommands, AddFirewallRules @@ -54,7 +54,7 @@ READONLY_FILE_GLOBS, ExtensionsSummary, AgentUpgradeType from tests.ga.mocks import mock_update_handler from tests.protocol.mocks import mock_wire_protocol, MockHttpResponse -from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT +from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT, DATA_FILE_VM_SETTINGS from tests.tools import AgentTestCase, AgentTestCaseWithGetVmSizeMock, data_dir, DEFAULT, patch, load_bin_data, Mock, MagicMock, \ clear_singleton_instances, mock_sleep from tests.protocol import mockwiredata @@ -1564,6 +1564,53 @@ def _get_test_ext_handler_instance(protocol, name="OSTCExtensions.ExampleHandler eh = Extension(name=name) eh.version = version return ExtHandlerInstance(eh, protocol) + + def test_update_handler_recovers_from_error_with_no_certs(self): + data = DATA_FILE.copy() + data['goal_state'] = 'wire/goal_state_no_certs.xml' + + def fail_gs_fetch(url, *_, **__): + if HttpRequestPredicates.is_goal_state_request(url): + return MockHttpResponse(status=500) + return None + + with mock_wire_protocol(data) as protocol: + + def fail_fetch_on_second_iter(iteration): + if iteration == 2: + protocol.set_http_handlers(http_get_handler=fail_gs_fetch) + if iteration > 2: # Zero out the fail handler for subsequent iterations. + protocol.set_http_handlers(http_get_handler=None) + + with mock_update_handler(protocol, 3, on_new_iteration=fail_fetch_on_second_iter) as update_handler: + with patch("azurelinuxagent.ga.update.logger.error") as patched_error: + with patch("azurelinuxagent.ga.update.logger.info") as patched_info: + def match_unexpected_errors(): + unexpected_msg_fragment = "Error fetching the goal state:" + + matching_errors = [] + for (args, _) in filter(lambda a: len(a) > 0, patched_error.call_args_list): + if unexpected_msg_fragment in args[0]: + matching_errors.append(args[0]) + + if len(matching_errors) > 1: + self.fail("Guest Agent did not recover, with new error(s): {}"\ + .format(matching_errors[1:])) + + def match_expected_info(): + expected_msg_fragment = "Fetching the goal state recovered from previous errors" + + for (call_args, _) in filter(lambda a: len(a) > 0, patched_info.call_args_list): + if expected_msg_fragment in call_args[0]: + break + else: + self.fail("Expected the guest agent to recover with '{}', but it didn't"\ + .format(expected_msg_fragment)) + + update_handler.run(debug=True) + match_unexpected_errors() # Match on errors first, they can provide more info. + match_expected_info() + def test_it_should_recreate_handler_env_on_service_startup(self): iterations = 5 @@ -1898,7 +1945,7 @@ def update_goal_state_and_run_handler(): def test_it_should_wait_to_fetch_first_goal_state(self): with _get_update_handler() as (update_handler, protocol): - with patch("azurelinuxagent.common.logger.warn") as patch_warn: + with patch("azurelinuxagent.common.logger.error") as patch_error: with patch("azurelinuxagent.common.logger.info") as patch_info: # Fail GS fetching for the 1st 5 times the agent asks for it update_handler._fail_gs_count = 5 @@ -1914,13 +1961,13 @@ def get_handler(url, **kwargs): self.assertTrue(update_handler.exit_mock.called, "The process should have exited") exit_args, _ = update_handler.exit_mock.call_args - self.assertEqual(exit_args[0], 0, "Exit code should be 0; List of all warnings logged by the agent: {0}".format( - patch_warn.call_args_list)) - warn_msgs = [args[0] for (args, _) in patch_warn.call_args_list if - "An error occurred while retrieving the goal state" in args[0]] - self.assertTrue(len(warn_msgs) > 0, "Error should've been reported when failed to retrieve GS") + self.assertEqual(exit_args[0], 0, "Exit code should be 0; List of all errors logged by the agent: {0}".format( + patch_error.call_args_list)) + error_msgs = [args[0] for (args, _) in patch_error.call_args_list if + "Error fetching the goal state" in args[0]] + self.assertTrue(len(error_msgs) > 0, "Error should've been reported when failed to retrieve GS") info_msgs = [args[0] for (args, _) in patch_info.call_args_list if - "Retrieving the goal state recovered from previous errors" in args[0]] + "Fetching the goal state recovered from previous errors." in args[0]] self.assertTrue(len(info_msgs) > 0, "Agent should've logged a message when recovered from GS errors") def test_it_should_reset_legacy_blacklisted_agents_on_process_start(self): @@ -2684,9 +2731,9 @@ def create_log_and_telemetry_mocks(): calls_to_strings = lambda calls: (str(c) for c in calls) filter_calls = lambda calls, regex=None: (c for c in calls_to_strings(calls) if regex is None or re.match(regex, c)) logger_calls = lambda regex=None: [m for m in filter_calls(logger.method_calls, regex)] # pylint: disable=used-before-assignment,unnecessary-comprehension - warnings = lambda: logger_calls(r'call.warn\(.*An error occurred while retrieving the goal state.*') - periodic_warnings = lambda: logger_calls(r'call.periodic_warn\(.*Attempts to retrieve the goal state are failing.*') - success_messages = lambda: logger_calls(r'call.info\(.*Retrieving the goal state recovered from previous errors.*') + errors = lambda: logger_calls(r'call.error\(.*Error fetching the goal state.*') + periodic_errors = lambda: logger_calls(r'call.error\(.*Fetching the goal state is still failing*') + success_messages = lambda: logger_calls(r'call.info\(.*Fetching the goal state recovered from previous errors.*') telemetry_calls = lambda regex=None: [m for m in filter_calls(add_event.mock_calls, regex)] # pylint: disable=used-before-assignment,unnecessary-comprehension goal_state_events = lambda: telemetry_calls(r".*op='FetchGoalState'.*") @@ -2711,10 +2758,8 @@ def create_log_and_telemetry_mocks(): with create_log_and_telemetry_mocks() as (logger, add_event): update_handler._try_update_goal_state(protocol) - w = warnings() - pw = periodic_warnings() - self.assertEqual(1, len(w), "A failure should have produced a warning: [{0}]".format(w)) - self.assertEqual(1, len(pw), "A failure should have produced a periodic warning: [{0}]".format(pw)) + e = errors() + self.assertEqual(1, len(e), "A failure should have produced an error: [{0}]".format(e)) gs = goal_state_events() self.assertTrue(len(gs) == 1 and 'is_success=False' in gs[0], "A failure should produce a telemetry event (success=false): [{0}]".format(gs)) @@ -2723,17 +2768,17 @@ def create_log_and_telemetry_mocks(): # ... and errors continue happening... # with create_log_and_telemetry_mocks() as (logger, add_event): - update_handler._try_update_goal_state(protocol) - update_handler._try_update_goal_state(protocol) - update_handler._try_update_goal_state(protocol) + for _ in range(5): + update_handler._update_goal_state_last_error_report = datetime.now() + timedelta(days=1) + update_handler._try_update_goal_state(protocol) - w = warnings() - pw = periodic_warnings() - self.assertTrue(len(w) == 0, "Subsequent failures should not produce warnings: [{0}]".format(w)) - self.assertEqual(len(pw), 3, "Subsequent failures should produce periodic warnings: [{0}]".format(pw)) + e = errors() + pe = periodic_errors() + self.assertEqual(2, len(e), "Two additional errors should have been reported: [{0}]".format(e)) + self.assertEqual(len(pe), 3, "Subsequent failures should produce periodic errors: [{0}]".format(pe)) tc = telemetry_calls() - self.assertTrue(len(tc) == 0, "Subsequent failures should not produce any telemetry events: [{0}]".format(tc)) + self.assertTrue(len(tc) == 5, "The failures should have produced telemetry events. Got: [{0}]".format(tc)) # # ... until we finally succeed @@ -2743,10 +2788,10 @@ def create_log_and_telemetry_mocks(): update_handler._try_update_goal_state(protocol) s = success_messages() - w = warnings() - pw = periodic_warnings() + e = errors() + pe = periodic_errors() self.assertEqual(len(s), 1, "Recovering after failures should have produced an info message: [{0}]".format(s)) - self.assertTrue(len(w) == 0 and len(pw) == 0, "Recovering after failures should have not produced any warnings: [{0}] [{1}]".format(w, pw)) + self.assertTrue(len(e) == 0 and len(pe) == 0, "Recovering after failures should have not produced any errors: [{0}] [{1}]".format(e, pe)) gs = goal_state_events() self.assertTrue(len(gs) == 1 and 'is_success=True' in gs[0], "Recovering after failures should produce a telemetry event (success=true): [{0}]".format(gs)) @@ -2874,7 +2919,7 @@ def test_it_should_mark_outdated_goal_states_on_service_restart_when_host_ga_plu def test_it_should_clear_the_timestamp_for_the_most_recent_fast_track_goal_state(self): data_file = self._prepare_fast_track_goal_state() - if HostPluginProtocol.get_fast_track_timestamp() is None: + if HostPluginProtocol.get_fast_track_timestamp() == timeutil.create_timestamp(datetime.min): raise Exception("The test setup did not save the Fast Track state") with patch("azurelinuxagent.common.conf.get_enable_fast_track", return_value=False): @@ -2882,8 +2927,55 @@ def test_it_should_clear_the_timestamp_for_the_most_recent_fast_track_goal_state with mock_update_handler(protocol) as update_handler: update_handler.run() - self.assertIsNone(HostPluginProtocol.get_fast_track_timestamp(), "The Fast Track state was not cleared") + self.assertEqual(HostPluginProtocol.get_fast_track_timestamp(), timeutil.create_timestamp(datetime.min), + "The Fast Track state was not cleared") + + def test_it_should_default_fast_track_timestamp_to_datetime_min(self): + data = DATA_FILE_VM_SETTINGS.copy() + # TODO: Currently, there's a limitation in the mocks where bumping the incarnation but the goal + # state will cause the agent to error out while trying to write the certificates to disk. These + # files have no dependencies on certs, so using them does not present that issue. + # + # Note that the scenario this test is representing does not depend on certificates at all, and + # can be changed to use the default files when the above limitation is addressed. + data["vm_settings"] = "hostgaplugin/vm_settings-fabric-no_thumbprints.json" + data['goal_state'] = 'wire/goal_state_no_certs.xml' + + def vm_settings_no_change(url, *_, **__): + if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): + return MockHttpResponse(httpclient.NOT_MODIFIED) + return None + + def vm_settings_not_supported(url, *_, **__): + if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): + return MockHttpResponse(404) + return None + + with mock_wire_protocol(data) as protocol: + + def mock_live_migration(iteration): + if iteration == 1: + protocol.mock_wire_data.set_incarnation(2) + protocol.set_http_handlers(http_get_handler=vm_settings_no_change) + elif iteration == 2: + protocol.mock_wire_data.set_incarnation(3) + protocol.set_http_handlers(http_get_handler=vm_settings_not_supported) + + with mock_update_handler(protocol, 3, on_new_iteration=mock_live_migration) as update_handler: + with patch("azurelinuxagent.ga.update.logger.error") as patched_error: + def check_for_errors(): + msg_fragment = "Error fetching the goal state:" + for (args, _) in filter(lambda a: len(a) > 0, patched_error.call_args_list): + if msg_fragment in args[0]: + self.fail("Found error: {}".format(args[0])) + + update_handler.run(debug=True) + check_for_errors() + + timestamp = protocol.client.get_host_plugin()._fast_track_timestamp + self.assertEqual(timestamp, timeutil.create_timestamp(datetime.min), + "Expected fast track time stamp to be set to {0}, got {1}".format(datetime.min, timestamp)) class HeartbeatTestCase(AgentTestCase): diff --git a/tests/protocol/HttpRequestPredicates.py b/tests/protocol/HttpRequestPredicates.py index 39243d543..db3ab8b2a 100644 --- a/tests/protocol/HttpRequestPredicates.py +++ b/tests/protocol/HttpRequestPredicates.py @@ -11,6 +11,22 @@ class HttpRequestPredicates(object): def is_goal_state_request(url): return url.lower() == 'http://{0}/machine/?comp=goalstate'.format(restutil.KNOWN_WIRESERVER_IP) + @staticmethod + def is_certificates_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=certificates'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_extensions_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=extensionsConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_hosting_environment_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=hostingEnvironmentConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + + @staticmethod + def is_shared_config_request(url): + return re.match(r'http://{0}(:80)?/machine/.*?comp=config&type=sharedConfig'.format(restutil.KNOWN_WIRESERVER_IP), url, re.IGNORECASE) + @staticmethod def is_telemetry_request(url): return url.lower() == 'http://{0}/machine?comp=telemetrydata'.format(restutil.KNOWN_WIRESERVER_IP) diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py index 218bd2937..7ec311af4 100644 --- a/tests/protocol/mockwiredata.py +++ b/tests/protocol/mockwiredata.py @@ -135,10 +135,10 @@ def __init__(self, data_files=None): "/HealthService": 0, "/vmAgentLog": 0, "goalstate": 0, - "hostingenvuri": 0, - "sharedconfiguri": 0, - "certificatesuri": 0, - "extensionsconfiguri": 0, + "hostingEnvironmentConfig": 0, + "sharedConfig": 0, + "certificates": 0, + "extensionsConfig": 0, "remoteaccessinfouri": 0, "extensionArtifact": 0, "agentArtifact": 0, @@ -198,6 +198,10 @@ def reload(self): if in_vm_artifacts_profile_file is not None: self.in_vm_artifacts_profile = load_data(in_vm_artifacts_profile_file) + def reset_call_counts(self): + for counter in self.call_counts: + self.call_counts[counter] = 0 + def mock_http_get(self, url, *_, **kwargs): content = '' response_headers = [] @@ -217,18 +221,18 @@ def mock_http_get(self, url, *_, **kwargs): elif "goalstate" in url: content = self.goal_state self.call_counts["goalstate"] += 1 - elif "hostingenvuri" in url: + elif HttpRequestPredicates.is_hosting_environment_config_request(url): content = self.hosting_env - self.call_counts["hostingenvuri"] += 1 - elif "sharedconfiguri" in url: + self.call_counts["hostingEnvironmentConfig"] += 1 + elif HttpRequestPredicates.is_shared_config_request(url): content = self.shared_config - self.call_counts["sharedconfiguri"] += 1 - elif "certificatesuri" in url: + self.call_counts["sharedConfig"] += 1 + elif HttpRequestPredicates.is_certificates_request(url): content = self.certs - self.call_counts["certificatesuri"] += 1 - elif "extensionsconfiguri" in url: + self.call_counts["certificates"] += 1 + elif HttpRequestPredicates.is_extensions_config_request(url): content = self.ext_conf - self.call_counts["extensionsconfiguri"] += 1 + self.call_counts["extensionsConfig"] += 1 elif "remoteaccessinfouri" in url: content = self.remote_access self.call_counts["remoteaccessinfouri"] += 1 diff --git a/tests/protocol/test_extensions_goal_state_from_vm_settings.py b/tests/protocol/test_extensions_goal_state_from_vm_settings.py index 9bcba5ece..8cdfa81bf 100644 --- a/tests/protocol/test_extensions_goal_state_from_vm_settings.py +++ b/tests/protocol/test_extensions_goal_state_from_vm_settings.py @@ -69,9 +69,17 @@ def test_it_should_parse_missing_status_upload_blob_as_none(self): self.assertIsNone(extensions_goal_state.status_upload_blob, "Expected status upload blob to be None") self.assertEqual("BlockBlob", extensions_goal_state.status_upload_blob_type, "Expected status upload blob to be Block") + def test_it_should_parse_missing_agent_manifests_as_empty(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + data_file["vm_settings"] = "hostgaplugin/vm_settings-no_manifests.json" + with mock_wire_protocol(data_file) as protocol: + extensions_goal_state = protocol.get_goal_state().extensions_goal_state + self.assertEqual(1, len(extensions_goal_state.agent_manifests), "Expected exactly one agent manifest. Got: {0}".format(extensions_goal_state.agent_manifests)) + self.assertListEqual([], extensions_goal_state.agent_manifests[0].uris, "Expected an empty list of agent manifests") + def test_it_should_parse_missing_extension_manifests_as_empty(self): data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() - data_file["vm_settings"] = "hostgaplugin/vm_settings-no_extension_manifests.json" + data_file["vm_settings"] = "hostgaplugin/vm_settings-no_manifests.json" with mock_wire_protocol(data_file) as protocol: extensions_goal_state = protocol.get_goal_state().extensions_goal_state diff --git a/tests/protocol/test_goal_state.py b/tests/protocol/test_goal_state.py index 65d050d86..87a1db50e 100644 --- a/tests/protocol/test_goal_state.py +++ b/tests/protocol/test_goal_state.py @@ -8,12 +8,13 @@ import re import time +from azurelinuxagent.common import conf from azurelinuxagent.common.future import httpclient from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource, GoalStateChannel from azurelinuxagent.common.protocol.extensions_goal_state_from_extensions_config import ExtensionsGoalStateFromExtensionsConfig from azurelinuxagent.common.protocol.extensions_goal_state_from_vm_settings import ExtensionsGoalStateFromVmSettings from azurelinuxagent.common.protocol import hostplugin -from azurelinuxagent.common.protocol.goal_state import GoalState, _GET_GOAL_STATE_MAX_ATTEMPTS +from azurelinuxagent.common.protocol.goal_state import GoalState, GoalStateInconsistentError, _GET_GOAL_STATE_MAX_ATTEMPTS from azurelinuxagent.common.exception import ProtocolError from azurelinuxagent.common.utils import fileutil from azurelinuxagent.common.utils.archive import ARCHIVE_DIRECTORY_NAME @@ -96,7 +97,15 @@ def test_fetch_goal_state_should_raise_on_incomplete_goal_state(self): GoalState(protocol.client) self.assertEqual(_GET_GOAL_STATE_MAX_ATTEMPTS, mock_sleep.call_count, "Unexpected number of retries") - def test_instantiating_goal_state_should_save_the_goal_state_to_the_history_directory(self): + def test_fetching_the_goal_state_should_save_the_shared_config(self): + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband); verify that we do not delete it + with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: + _ = GoalState(protocol.client) + + shared_config = os.path.join(conf.get_lib_dir(), 'SharedConfig.xml') + self.assertTrue(os.path.exists(shared_config), "{0} should have been created".format(shared_config)) + + def test_fetching_the_goal_state_should_save_the_goal_state_to_the_history_directory(self): with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: protocol.mock_wire_data.set_incarnation(999) protocol.mock_wire_data.set_etag(888) @@ -105,7 +114,7 @@ def test_instantiating_goal_state_should_save_the_goal_state_to_the_history_dire self._assert_directory_contents( self._find_history_subdirectory("999-888"), - ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) def _find_history_subdirectory(self, tag): matches = glob.glob(os.path.join(self.tmp_dir, ARCHIVE_DIRECTORY_NAME, "*_{0}".format(tag))) @@ -128,7 +137,7 @@ def test_update_should_create_new_history_subdirectories(self): goal_state = GoalState(protocol.client) self._assert_directory_contents( self._find_history_subdirectory("123-654"), - ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "VmSettings.json", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) def http_get_handler(url, *_, **__): if HttpRequestPredicates.is_host_plugin_vm_settings_request(url): @@ -140,7 +149,7 @@ def http_get_handler(url, *_, **__): goal_state.update() self._assert_directory_contents( self._find_history_subdirectory("234-654"), - ["GoalState.xml", "ExtensionsConfig.xml", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) + ["GoalState.xml", "ExtensionsConfig.xml", "Certificates.json", "SharedConfig.xml", "HostingEnvironmentConfig.xml"]) protocol.mock_wire_data.set_etag(987) protocol.set_http_handlers(http_get_handler=None) @@ -358,3 +367,55 @@ def http_get_handler(url, *_, **__): self._assert_goal_state(goal_state, initial_incarnation, channel=GoalStateChannel.WireServer, source=GoalStateSource.Fabric) self.assertEqual(initial_timestamp, goal_state.extensions_goal_state.created_on_timestamp, "The timestamp of the updated goal state is incorrect") self.assertTrue(goal_state.extensions_goal_state.is_outdated, "The updated goal state should be marked as outdated") + + def test_it_should_raise_when_the_tenant_certificate_is_missing(self): + data_file = mockwiredata.DATA_FILE_VM_SETTINGS.copy() + + with mock_wire_protocol(data_file) as protocol: + data_file["vm_settings"] = "hostgaplugin/vm_settings-missing_cert.json" + protocol.mock_wire_data.reload() + + with self.assertRaises(GoalStateInconsistentError) as context: + _ = GoalState(protocol.client) + + expected_message = "Certificate 59A10F50FFE2A0408D3F03FE336C8FD5716CF25C needed by Microsoft.OSTCExtensions.VMAccessForLinux is missing from the goal state" + self.assertIn(expected_message, str(context.exception)) + + def test_it_should_refresh_the_goal_state_when_it_is_inconsistent(self): + # + # Some scenarios can produce inconsistent goal states. For example, during hibernation/resume, the Fabric goal state changes (the + # tenant certificate is re-generated when the VM is restarted) *without* the incarnation changing. If a Fast Track goal state + # comes after that, the extensions will need the new certificate. This test simulates that scenario by mocking the certificates + # request and returning first a set of certificates (certs-2.xml) that do not match those needed by the extensions, and then a + # set (certs.xml) that does match. The test then ensures that the goal state was refreshed and the correct certificates were + # fetched. + # + data_files = [ + "wire/certs-2.xml", + "wire/certs.xml" + ] + + def http_get_handler(url, *_, **__): + if HttpRequestPredicates.is_certificates_request(url): + http_get_handler.certificate_requests += 1 + if http_get_handler.certificate_requests < len(data_files): + data = load_data(data_files[http_get_handler.certificate_requests - 1]) + return MockHttpResponse(status=200, body=data.encode('utf-8')) + return None + http_get_handler.certificate_requests = 0 + + with mock_wire_protocol(mockwiredata.DATA_FILE_VM_SETTINGS) as protocol: + protocol.set_http_handlers(http_get_handler=http_get_handler) + protocol.mock_wire_data.reset_call_counts() + + goal_state = GoalState(protocol.client) + + self.assertEqual(2, protocol.mock_wire_data.call_counts['goalstate'], "There should have been exactly 2 requests for the goal state (original + refresh)") + self.assertEqual(2, http_get_handler.certificate_requests, "There should have been exactly 2 requests for the goal state certificates (original + refresh)") + + thumbprints = [c.thumbprint for c in goal_state.certs.cert_list.certificates] + + for extension in goal_state.extensions_goal_state.extensions: + for settings in extension.settings: + if settings.protectedSettings is not None: + self.assertIn(settings.certificateThumbprint, thumbprints, "Certificate is missing from the goal state.") diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py index 16bb7ef0b..9f96f7d55 100644 --- a/tests/protocol/test_hostplugin.py +++ b/tests/protocol/test_hostplugin.py @@ -257,9 +257,8 @@ def test_default_channel(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is only called once, non-forced + # assert update goal state is only called once self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Unexpected parameters") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) @@ -291,9 +290,8 @@ def test_fallback_channel_503(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is only called once, non-forced + # assert update goal state is only called once self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected call count") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) @@ -326,9 +324,8 @@ def test_fallback_channel_410(self, patch_refresh_host_plugin, patch_put, patch_ # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is called with no arguments (forced=False), then update_host_plugin_from_goal_state is called + # assert update goal state is called, then update_host_plugin_from_goal_state is called self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected argument count") self.assertEqual(1, patch_refresh_host_plugin.call_count, "Refresh host plugin unexpected call count") # ensure the correct url is used @@ -361,9 +358,8 @@ def test_fallback_channel_failure(self, patch_put, patch_upload, _): # assert host plugin route is called self.assertEqual(1, patch_put.call_count, "Host plugin was not used") - # assert update goal state is called twice, forced=True on the second + # assert update goal state is called twice self.assertEqual(1, wire_protocol.client.update_goal_state.call_count, "Update goal state unexpected call count") - self.assertEqual(0, len(wire_protocol.client.update_goal_state.call_args[1]), "Update goal state unexpected call count") # ensure the correct url is used self.assertEqual(sas_url, patch_put.call_args[0][0]) diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py index 0cc8a01e9..c564af721 100644 --- a/tests/protocol/test_wire.py +++ b/tests/protocol/test_wire.py @@ -160,7 +160,7 @@ def test_getters_with_stale_goal_state(self, patch_report, *args): # -- Tracking calls to retrieve GoalState is problematic since it is # fetched often; however, the dependent documents, such as the # HostingEnvironmentConfig, will be retrieved the expected number - self.assertEqual(1, test_data.call_counts["hostingenvuri"]) + self.assertEqual(1, test_data.call_counts["hostingEnvironmentConfig"]) self.assertEqual(1, patch_report.call_count) def test_call_storage_kwargs(self, *args): # pylint: disable=unused-argument diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py index 466d674c7..54766862f 100644 --- a/tests/utils/test_archive.py +++ b/tests/utils/test_archive.py @@ -6,8 +6,9 @@ from datetime import datetime, timedelta import azurelinuxagent.common.logger as logger +from azurelinuxagent.common import conf from azurelinuxagent.common.utils import fileutil, timeutil -from azurelinuxagent.common.utils.archive import StateArchiver, _MAX_ARCHIVED_STATES +from azurelinuxagent.common.utils.archive import GoalStateHistory, StateArchiver, _MAX_ARCHIVED_STATES, ARCHIVE_DIRECTORY_NAME from tests.tools import AgentTestCase, patch debug = False @@ -28,7 +29,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp(prefix=prefix) def _write_file(self, filename, contents=None): - full_name = os.path.join(self.tmp_dir, filename) + full_name = os.path.join(conf.get_lib_dir(), filename) fileutil.mkdir(os.path.dirname(full_name)) with open(full_name, 'w') as file_handler: @@ -38,7 +39,7 @@ def _write_file(self, filename, contents=None): @property def history_dir(self): - return os.path.join(self.tmp_dir, 'history') + return os.path.join(conf.get_lib_dir(), ARCHIVE_DIRECTORY_NAME) @staticmethod def _parse_archive_name(name): @@ -66,8 +67,11 @@ def test_archive_should_zip_all_but_the_latest_goal_state_in_the_history_folder( self._write_file(os.path.join(directory, current_file)) test_directories.append(directory) - test_subject = StateArchiver(self.tmp_dir) - test_subject.archive() + test_subject = StateArchiver(conf.get_lib_dir()) + # NOTE: StateArchiver sorts the state directories by creation time, but the test files are created too fast and the + # time resolution is too coarse, so instead we mock getctime to simply return the path of the file + with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): + test_subject.archive() for directory in test_directories[0:2]: zip_file = directory + ".zip" @@ -80,9 +84,9 @@ def test_archive_should_zip_all_but_the_latest_goal_state_in_the_history_folder( self.assertTrue(os.path.exists(test_directories[2]), "{0}, the latest goal state, should not have being removed".format(test_directories[2])) - def test_archive02(self): + def test_goal_state_history_init_should_purge_old_items(self): """ - StateArchiver should purge the MAX_ARCHIVED_STATES oldest files + GoalStateHistory.__init__ should _purge the MAX_ARCHIVED_STATES oldest files or directories. The oldest timestamps are purged first. This test case creates a mixture of archive files and directories. @@ -109,8 +113,10 @@ def test_archive02(self): self.assertEqual(total, len(os.listdir(self.history_dir))) - test_subject = StateArchiver(self.tmp_dir) - test_subject.purge() + # NOTE: The purge method sorts the items by creation time, but the test files are created too fast and the + # time resolution is too coarse, so instead we mock getctime to simply return the path of the file + with patch("azurelinuxagent.common.utils.archive.os.path.getctime", side_effect=lambda path: path): + GoalStateHistory(datetime.utcnow(), 'test') archived_entries = os.listdir(self.history_dir) self.assertEqual(_MAX_ARCHIVED_STATES, len(archived_entries)) @@ -127,66 +133,32 @@ def test_archive02(self): def test_purge_legacy_goal_state_history(self): with patch("azurelinuxagent.common.conf.get_lib_dir", return_value=self.tmp_dir): + # SharedConfig.xml is used by other components (Azsec and Singularity/HPC Infiniband); verify that we do not delete it + shared_config = os.path.join(self.tmp_dir, 'SharedConfig.xml') + legacy_files = [ - 'GoalState.1.xml', - 'VmSettings.1.json', - 'Prod.1.manifest.xml', - 'ExtensionsConfig.1.xml', + 'GoalState.2.xml', + 'VmSettings.2.json', + 'Prod.2.manifest.xml', + 'ExtensionsConfig.2.xml', 'Microsoft.Azure.Extensions.CustomScript.1.xml', - 'SharedConfig.xml', 'HostingEnvironmentConfig.xml', - 'RemoteAccess.xml' + 'RemoteAccess.xml', + 'waagent_status.1.json' ] legacy_files = [os.path.join(self.tmp_dir, f) for f in legacy_files] + + self._write_file(shared_config) for f in legacy_files: self._write_file(f) StateArchiver.purge_legacy_goal_state_history() + self.assertTrue(os.path.exists(shared_config), "{0} should not have been removed".format(shared_config)) + for f in legacy_files: self.assertFalse(os.path.exists(f), "Legacy file {0} was not removed".format(f)) - def test_archive03(self): - """ - All archives should be purged, both with the legacy naming (with incarnation number) and with the new naming. - """ - start = datetime.now() - timestamp1 = start + timedelta(seconds=5) - timestamp2 = start + timedelta(seconds=10) - timestamp3 = start + timedelta(seconds=10) - - dir_old = timestamp1.isoformat() - dir_new = "{0}_incarnation_1".format(timestamp2.isoformat()) - - archive_old = "{0}.zip".format(timestamp1.isoformat()) - archive_new = "{0}_incarnation_1.zip".format(timestamp2.isoformat()) - - status = "{0}.zip".format(timestamp3.isoformat()) - - self._write_file(os.path.join("history", dir_old, "Prod.manifest.xml")) - self._write_file(os.path.join("history", dir_new, "Prod.manifest.xml")) - self._write_file(os.path.join("history", archive_old)) - self._write_file(os.path.join("history", archive_new)) - self._write_file(os.path.join("history", status)) - - self.assertEqual(5, len(os.listdir(self.history_dir)), "Not all entries were archived!") - - test_subject = StateArchiver(self.tmp_dir) - with patch("azurelinuxagent.common.utils.archive._MAX_ARCHIVED_STATES", 0): - test_subject.purge() - - archived_entries = os.listdir(self.history_dir) - self.assertEqual(0, len(archived_entries), "Not all entries were purged!") - - def test_archive04(self): - """ - The archive directory is created if it does not exist. - - This failure was caught when .purge() was called before .archive(). - """ - test_subject = StateArchiver(os.path.join(self.tmp_dir, 'does-not-exist')) - test_subject.purge() - @staticmethod def parse_isoformat(timestamp_str): return datetime.strptime(timestamp_str, '%Y-%m-%dT%H:%M:%S.%f')