From 6c8483a0e36b4625f84db700974de037ad7cfa23 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Wed, 13 Aug 2025 14:04:07 -0700 Subject: [PATCH 01/11] add scenario for leader change case --- enos/enos-scenario-openldap-leader-change.hcl | 827 ++++++++++++++++++ 1 file changed, 827 insertions(+) create mode 100644 enos/enos-scenario-openldap-leader-change.hcl diff --git a/enos/enos-scenario-openldap-leader-change.hcl b/enos/enos-scenario-openldap-leader-change.hcl new file mode 100644 index 0000000..e1c32b3 --- /dev/null +++ b/enos/enos-scenario-openldap-leader-change.hcl @@ -0,0 +1,827 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +scenario "openldap_leader_change" { + description = <<-EOF + The scenario verifies that the Vault OpenLDAP secrets engine plugin works correctly after a leader change. + + This scenario creates a Vault cluster with the OpenLDAP secrets engine plugin installed and configured, and starts an OpenLDAP server. + It then tests the plugin by creating static and dynamic roles, verifying that they can be created, read, updated, and deleted via the Vault API. + After that, it forces a Vault leader stepdown followed by a leader election and verifies that the plugin still works correctly after the leader change + + # How to run this scenario + + For general instructions on running a scenario, refer to the Enos docs: https://eng-handbook.hashicorp.services/internal-tools/enos/running-a-scenario/ + For troubleshooting tips and common errors, see https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/. + + Variables required for all scenario variants: + - aws_ssh_private_key_path (more info about AWS SSH keypairs: https://eng-handbook.hashicorp.services/internal-tools/enos/getting-started/#set-your-aws-key-pair-name-and-private-key) + - aws_ssh_keypair_name + - vault_build_date* + - vault_product_version + - vault_revision* + + * If you don't already know what build date and revision you should be using, see + https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/#execution-error-expected-vs-got-for-vault-versioneditionrevisionbuild-date. + + Variables required for some scenario variants: + - artifactory_token (if using `artifact_source:artifactory` in your filter) + - aws_region (if different from the default value in enos-variables.hcl) + - distro_version_ (if different from the default version for your target + distro. See supported distros and default versions in the distro_version_ + definitions in enos-variables.hcl) + - vault_artifact_path (the path to where you have a Vault artifact already downloaded, + if using `artifact_source:crt` in your filter) + - vault_license_path (if using an ENT edition of Vault) + EOF + + matrix { + arch = global.archs + artifact_source = global.artifact_sources + ldap_artifact_source = global.ldap_artifact_sources + artifact_type = global.artifact_types + backend = global.backends + config_mode = global.config_modes + distro = global.distros + edition = global.editions + ip_version = global.ip_versions + seal = global.seals + + // Our local builder always creates bundles + exclude { + artifact_source = ["local"] + ldap_artifact_source = ["local"] + artifact_type = ["package"] + } + + // PKCS#11 can only be used on ent.hsm and ent.hsm.fips1403. + exclude { + seal = ["pkcs11"] + edition = [for e in matrix.edition : e if !strcontains(e, "hsm")] + } + + // softhsm packages not available for leap/sles. + exclude { + seal = ["pkcs11"] + distro = ["leap", "sles"] + } + } + + terraform_cli = terraform_cli.default + terraform = terraform.default + providers = [ + provider.aws.default, + provider.enos.ec2_user, + provider.enos.ubuntu + ] + + locals { + artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null + ldap_artifact_path = matrix.ldap_artifact_source != "artifactory" ? abspath(var.ldap_artifact_path) : null + enos_provider = { + amzn = provider.enos.ec2_user + leap = provider.enos.ec2_user + rhel = provider.enos.ec2_user + sles = provider.enos.ec2_user + ubuntu = provider.enos.ubuntu + } + manage_service = matrix.artifact_type == "bundle" + } + + step "build_vault" { + description = global.description.build_vault + module = "build_vault_${matrix.artifact_source}" + + variables { + build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] + artifact_path = local.artifact_path + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null + artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.artifact_source == "artifactory" ? matrix.arch : null + product_version = var.vault_product_version + artifact_type = matrix.artifact_type + distro = matrix.artifact_source == "artifactory" ? matrix.distro : null + edition = matrix.artifact_source == "artifactory" ? matrix.edition : null + revision = var.vault_revision + } + } + + step "ec2_info" { + description = global.description.ec2_info + module = module.ec2_info + } + + step "create_vpc" { + description = global.description.create_vpc + module = module.create_vpc + + variables { + common_tags = global.tags + ip_version = matrix.ip_version + } + } + + step "read_vault_license" { + description = global.description.read_vault_license + skip_step = matrix.edition == "ce" + module = module.read_license + + variables { + file_name = global.vault_license_path + } + } + + step "create_seal_key" { + description = global.description.create_seal_key + module = "seal_${matrix.seal}" + depends_on = [step.create_vpc] + + providers = { + enos = provider.enos.ubuntu + } + + variables { + cluster_id = step.create_vpc.id + common_tags = global.tags + } + } + + step "create_vault_cluster_targets" { + description = global.description.create_vault_cluster_targets + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.vault_tag_key + common_tags = global.tags + seal_key_names = step.create_seal_key.resource_names + vpc_id = step.create_vpc.id + } + } + + step "create_vault_cluster" { + description = global.description.create_vault_cluster + module = module.vault_cluster + depends_on = [ + step.build_vault, + step.create_vault_cluster_targets, + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + // verified in modules + quality.vault_artifact_bundle, + quality.vault_artifact_deb, + quality.vault_artifact_rpm, + quality.vault_audit_log, + quality.vault_audit_socket, + quality.vault_audit_syslog, + quality.vault_autojoin_aws, + quality.vault_config_env_variables, + quality.vault_config_file, + quality.vault_config_log_level, + quality.vault_init, + quality.vault_license_required_ent, + quality.vault_listener_ipv4, + quality.vault_listener_ipv6, + quality.vault_service_start, + quality.vault_storage_backend_raft, + // verified in enos_vault_start resource + quality.vault_api_sys_config_read, + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_health_read, + quality.vault_api_sys_host_info_read, + quality.vault_api_sys_replication_status_read, + quality.vault_api_sys_seal_status_api_read_matches_sys_health, + quality.vault_api_sys_storage_raft_autopilot_configuration_read, + quality.vault_api_sys_storage_raft_autopilot_state_read, + quality.vault_api_sys_storage_raft_configuration_read, + quality.vault_cli_status_exit_code, + quality.vault_service_systemd_notified, + quality.vault_service_systemd_unit, + ] + + variables { + artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null + backend_cluster_name = null + backend_cluster_tag_key = global.backend_tag_key + cluster_name = step.create_vault_cluster_targets.cluster_name + config_mode = matrix.config_mode + enable_audit_devices = var.vault_enable_audit_devices + hosts = step.create_vault_cluster_targets.hosts + install_dir = global.vault_install_dir[matrix.artifact_type] + ip_version = matrix.ip_version + license = matrix.edition != "ce" ? step.read_vault_license.license : null + local_artifact_path = local.artifact_path + manage_service = local.manage_service + packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) + seal_attributes = step.create_seal_key.attributes + seal_type = matrix.seal + storage_backend = matrix.backend + } + } + + step "bootstrap_vault_cluster_targets" { + description = global.description.bootstrap_vault_cluster_targets + module = module.bootstrap_vault_cluster_targets + depends_on = [step.create_vault_cluster] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + unseal_keys = step.create_vault_cluster.unseal_keys_b64 + threshold = step.create_vault_cluster.unseal_threshold + } + } + + step "get_local_metadata" { + description = global.description.get_local_metadata + skip_step = matrix.artifact_source != "local" + module = module.get_local_metadata + } + + // Wait for our cluster to elect a leader + step "wait_for_new_leader" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.create_vault_cluster, + step.bootstrap_vault_cluster_targets] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_unseal_ha_leader_election, + ] + + variables { + timeout = 120 // seconds + ip_version = matrix.ip_version + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "get_vault_cluster_ips" { + description = global.description.get_vault_cluster_ip_addresses + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_new_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_members, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + + step "verify_vault_unsealed" { + description = global.description.verify_vault_unsealed + module = module.vault_wait_for_cluster_unsealed + depends_on = [step.wait_for_new_leader] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_seal_awskms, + quality.vault_seal_pkcs11, + quality.vault_seal_shamir, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "verify_vault_version" { + description = global.description.verify_vault_version + module = module.vault_verify_version + depends_on = [step.verify_vault_unsealed] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_version_history_keys, + quality.vault_api_sys_version_history_key_info, + quality.vault_version_build_date, + quality.vault_version_edition, + quality.vault_version_release, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_edition = matrix.edition + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version + vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision + vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_raft_auto_join_voter" { + description = global.description.verify_raft_cluster_all_nodes_are_voters + skip_step = matrix.backend != "raft" + module = module.vault_verify_raft_auto_join_voter + depends_on = [ + step.verify_vault_unsealed, + step.get_vault_cluster_ips + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = quality.vault_raft_voters + + variables { + hosts = step.create_vault_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "build_ldap" { + description = global.description.build_ldap + module = "build_ldap_${matrix.ldap_artifact_source}" + + variables { + goarch = matrix.arch + goos = "linux" + artifactory_host = matrix.ldap_artifact_source == "artifactory" ? var.artifactory_host : null + artifactory_repo = matrix.ldap_artifact_source == "artifactory" ? var.plugin_artifactory_repo : null + artifactory_token = matrix.ldap_artifact_source == "artifactory" ? var.artifactory_token : null + arch = matrix.ldap_artifact_source == "artifactory" ? matrix.arch : null + artifact_type = matrix.ldap_artifact_source == "artifactory" ? "bundle" : null + product_version = var.ldap_plugin_version + revision = var.ldap_revision + plugin_name = var.plugin_name + makefile_dir = matrix.ldap_artifact_source == "local" ? var.makefile_dir : null + plugin_dest_dir = matrix.ldap_artifact_source == "local" ? var.plugin_dest_dir : null + } + } + + step "create_ldap_server_target" { + description = global.description.create_ldap_server_target + module = module.target_ec2_instances + depends_on = [step.create_vpc] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] + cluster_tag_key = global.ldap_tag_key + common_tags = global.tags + vpc_id = step.create_vpc.id + instance_count = 1 + } + } + + step "create_ldap_server" { + description = global.description.create_ldap_server + module = module.create_backend_server + depends_on = [step.create_ldap_server_target] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + hosts = step.create_ldap_server_target.hosts + ldap_tag = var.ldap_tag + ldap_port = global.ports.ldap.port + } + } + + step "setup_plugin" { + description = global.description.setup_plugin + module = module.setup_plugin + depends_on = [ + step.get_vault_cluster_ips, + step.create_ldap_server, + step.verify_vault_unsealed, + step.build_ldap + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + artifactory_release = matrix.ldap_artifact_source == "artifactory" ? step.build_ldap.ldap_artifactory_release : null + release = matrix.ldap_artifact_source == "releases" ? { version = var.ldap_plugin_version, edition = "ce" } : null + hosts = step.create_vault_cluster_targets.hosts + local_artifact_path = matrix.ldap_artifact_source == "local" ? local.ldap_artifact_path : null + + + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + + plugin_name = var.plugin_name + plugin_dir_vault = var.plugin_dir_vault + plugin_mount_path = var.plugin_mount_path + } + } + + step "configure_plugin" { + description = global.description.configure_plugin + module = module.configure_plugin + depends_on = [step.setup_plugin] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + ldap_bind_pass = var.ldap_bind_pass + ldap_schema = var.ldap_schema + } + } + + step "test_static_role_crud_api" { + description = global.description.static_role_crud_api + module = module.static_role_crud_api + depends_on = [step.configure_plugin] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + ldap_bind_pass = var.ldap_bind_pass + ldap_user_role_name = var.ldap_user_role_name + ldap_username = var.ldap_username + ldap_user_old_password = var.ldap_user_old_password + } + } + + step "test_dynamic_role_crud_api" { + description = global.description.dynamic_role_crud_api + module = module.dynamic_role_crud_api + depends_on = [step.configure_plugin] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + hosts = step.create_vault_cluster_targets.hosts + + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + dynamic_role_ldif_templates_path = var.dynamic_role_ldif_templates_path + ldap_dynamic_user_role_name = var.ldap_dynamic_user_role_name + } + } + + step "test_library_crud_api" { + description = global.description.library_crud_api + module = module.library_crud_api + depends_on = [ + step.configure_plugin, + step.test_static_role_crud_api + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + library_set_name = var.library_set_name + service_account_names = var.service_account_names + } + } + + step "verify_log_secrets" { + skip_step = !var.vault_enable_audit_devices || !var.verify_log_secrets + + description = global.description.verify_log_secrets + module = module.verify_log_secrets + depends_on = [ + step.verify_vault_unsealed, + step.test_static_role_crud_api, + step.test_dynamic_role_crud_api, + step.test_library_crud_api + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_audit_log_secrets, + quality.vault_journal_secrets, + quality.vault_radar_index_create, + quality.vault_radar_scan_file, + ] + + variables { + audit_log_file_path = step.create_vault_cluster.audit_device_file_path + leader_host = step.get_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Force a step down to trigger a new leader election + step "vault_leader_step_down" { + description = global.description.vault_leader_step_down + module = module.vault_step_down + depends_on = [ + step.get_vault_cluster_ips, + step.test_static_role_crud_api, + step.test_dynamic_role_crud_api, + step.test_library_crud_api + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_step_down_steps_down, + quality.vault_cli_operator_step_down, + ] + + variables { + leader_host = step.get_vault_cluster_ips.leader_host + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + // Wait for our cluster to elect a leader + step "wait_for_leader_after_step_down" { + description = global.description.wait_for_cluster_to_have_leader + module = module.vault_wait_for_leader + depends_on = [step.vault_leader_step_down] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_step_down, + ] + + variables { + timeout = 120 // seconds + ip_version = matrix.ip_version + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "get_vault_cluster_ips_after_step_down" { + description = global.description.get_vault_cluster_ip_addresses + module = module.vault_get_cluster_ips + depends_on = [step.wait_for_leader_after_step_down] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_api_sys_ha_status_read, + quality.vault_api_sys_leader_read, + quality.vault_cli_operator_members, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + ip_version = matrix.ip_version + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + vault_root_token = step.create_vault_cluster.root_token + } + } + + step "verify_vault_unsealed_after_step_down" { + description = global.description.verify_vault_unsealed + module = module.vault_wait_for_cluster_unsealed + depends_on = [step.wait_for_leader_after_step_down] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + verifies = [ + quality.vault_seal_awskms, + quality.vault_seal_pkcs11, + quality.vault_seal_shamir, + ] + + variables { + hosts = step.create_vault_cluster_targets.hosts + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_install_dir = global.vault_install_dir[matrix.artifact_type] + } + } + + step "test_static_role_crud_api_after_step_down" { + description = global.description.static_role_crud_api + module = module.static_role_crud_api + depends_on = [step.verify_vault_unsealed_after_step_down] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips_after_step_down.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + ldap_bind_pass = var.ldap_bind_pass + ldap_user_role_name = var.ldap_user_role_name + ldap_username = var.ldap_username + ldap_user_old_password = var.ldap_user_old_password + } + } + + step "test_dynamic_role_crud_api_after_step_down" { + description = global.description.dynamic_role_crud_api + module = module.dynamic_role_crud_api + depends_on = [step.verify_vault_unsealed_after_step_down] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips_after_step_down.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + hosts = step.create_vault_cluster_targets.hosts + + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + dynamic_role_ldif_templates_path = var.dynamic_role_ldif_templates_path + ldap_dynamic_user_role_name = var.ldap_dynamic_user_role_name + } + } + + step "test_library_crud_api_after_step_down" { + description = global.description.library_crud_api + module = module.library_crud_api + depends_on = [ + step.verify_vault_unsealed_after_step_down, + step.test_static_role_crud_api_after_step_down + ] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips_after_step_down.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + plugin_mount_path = var.plugin_mount_path + ldap_host = step.create_ldap_server.ldap_ip_address + ldap_port = step.create_ldap_server.ldap_port + ldap_base_dn = var.ldap_base_dn + library_set_name = var.library_set_name + service_account_names = var.service_account_names + } + } + + output "audit_device_file_path" { + description = "The file path for the file audit device, if enabled" + value = step.create_vault_cluster.audit_device_file_path + } + + output "cluster_name" { + description = "The Vault cluster name" + value = step.create_vault_cluster.cluster_name + } + + output "hosts" { + description = "The Vault cluster target hosts" + value = step.create_vault_cluster.hosts + } + + output "private_ips" { + description = "The Vault cluster private IPs" + value = step.create_vault_cluster.private_ips + } + + output "public_ips" { + description = "The Vault cluster public IPs" + value = step.create_vault_cluster.public_ips + } + + output "root_token" { + description = "The Vault cluster root token" + value = step.create_vault_cluster.root_token + } + + output "recovery_key_shares" { + description = "The Vault cluster recovery key shares" + value = step.create_vault_cluster.recovery_key_shares + } + + output "recovery_keys_b64" { + description = "The Vault cluster recovery keys b64" + value = step.create_vault_cluster.recovery_keys_b64 + } + + output "recovery_keys_hex" { + description = "The Vault cluster recovery keys hex" + value = step.create_vault_cluster.recovery_keys_hex + } + + output "seal_key_attributes" { + description = "The Vault cluster seal attributes" + value = step.create_seal_key.attributes + } + + output "unseal_keys_b64" { + description = "The Vault cluster unseal keys" + value = step.create_vault_cluster.unseal_keys_b64 + } + + output "unseal_keys_hex" { + description = "The Vault cluster unseal keys hex" + value = step.create_vault_cluster.unseal_keys_hex + } +} From 4a944c772e3d13e8ac7b00ec157f61c80ec476d9 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Wed, 13 Aug 2025 14:04:52 -0700 Subject: [PATCH 02/11] add modules refs, descriptions, and qualities for leader change case --- enos/enos-descriptions.hcl | 7 ++++++- enos/enos-modules.hcl | 6 ++++++ enos/enos-qualities.hcl | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/enos/enos-descriptions.hcl b/enos/enos-descriptions.hcl index cf5cd00..e1597de 100644 --- a/enos/enos-descriptions.hcl +++ b/enos/enos-descriptions.hcl @@ -111,7 +111,12 @@ globals { unseal_vault = <<-EOF Unseal the Vault cluster using the configured seal mechanism. -EOF + EOF + + vault_leader_step_down = <<-EOF + Force the Vault cluster leader to step down which forces the Vault cluster to perform a leader + election. + EOF verify_log_secrets = <<-EOF Verify that the vault audit log and systemd journal do not leak secret values. diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 4ddc632..868cf9d 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -163,6 +163,12 @@ module "vault_get_cluster_ips" { vault_install_dir = var.vault_install_dir } +module "vault_step_down" { + source = "git::https://github.com/hashicorp/vault.git//enos/modules/vault_step_down?ref=${var.vault_repo_ref}" + + vault_install_dir = var.vault_install_dir +} + module "vault_unseal_replication_followers" { source = "git::https://github.com/hashicorp/vault.git//enos/modules/vault_unseal_replication_followers?ref=${var.vault_repo_ref}" diff --git a/enos/enos-qualities.hcl b/enos/enos-qualities.hcl index 6731a6b..6ca45e9 100644 --- a/enos/enos-qualities.hcl +++ b/enos/enos-qualities.hcl @@ -41,6 +41,13 @@ quality "vault_api_sys_seal_status_api_read_matches_sys_health" { EOF } +quality "vault_api_sys_step_down_steps_down" { + description = <<-EOF + The v1/sys/step-down Vault API forces the cluster leader to step down and intiates a new leader + election + EOF +} + quality "vault_api_sys_storage_raft_autopilot_configuration_read" { description = <<-EOF The /sys/storage/raft/autopilot/configuration Vault API returns the autopilot configuration of @@ -112,6 +119,10 @@ quality "vault_cli_operator_members" { description = "The 'vault operator members' command returns the expected list of members" } +quality "vault_cli_operator_step_down" { + description = "The 'vault operator step-down' command forces the cluster leader to step down" +} + quality "vault_cli_status_exit_code" { description = <<-EOF The 'vault status' command exits with the correct code depending on expected seal status From 79b47f4429748e99331f2e0fe6850bf52f353657 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:11:32 -0700 Subject: [PATCH 03/11] add tf module for testing ldap secrets engine manual root_rotation --- enos/modules/root_rotation_manual/main.tf | 30 ++++++++++++ .../scripts/test-root-rotation-manual.sh | 46 +++++++++++++++++++ .../modules/root_rotation_manual/variables.tf | 22 +++++++++ 3 files changed, 98 insertions(+) create mode 100644 enos/modules/root_rotation_manual/main.tf create mode 100755 enos/modules/root_rotation_manual/scripts/test-root-rotation-manual.sh create mode 100644 enos/modules/root_rotation_manual/variables.tf diff --git a/enos/modules/root_rotation_manual/main.tf b/enos/modules/root_rotation_manual/main.tf new file mode 100644 index 0000000..bbac5b0 --- /dev/null +++ b/enos/modules/root_rotation_manual/main.tf @@ -0,0 +1,30 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "rotation_period" { default = null } +variable "rotation_window" { default = null } + +resource "enos_remote_exec" "root_rotation_manual_test" { + scripts = [abspath("${path.module}/scripts/test-root-rotation-manual.sh")] + + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + PLUGIN_PATH = var.plugin_mount_path + } + + transport = { + ssh = { + host = var.vault_leader_ip + } + } +} + diff --git a/enos/modules/root_rotation_manual/scripts/test-root-rotation-manual.sh b/enos/modules/root_rotation_manual/scripts/test-root-rotation-manual.sh new file mode 100755 index 0000000..e1c30e3 --- /dev/null +++ b/enos/modules/root_rotation_manual/scripts/test-root-rotation-manual.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +PLUGIN_PATH=local-secrets-ldap + +# Required env vars: PLUGIN_PATH +if [[ -z "${PLUGIN_PATH:-}" ]]; then + fail "PLUGIN_PATH env variable has not been set" +fi + +# Configure plugin for manual rotation +vault write -format=json "${PLUGIN_PATH}/config" \ + disable_automated_rotation=true \ + rotation_period=0 \ + rotation_schedule="" \ + rotation_window=0 >/dev/null + +# Read disable_automated_rotation from config +disable_automated_rotation=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.disable_automated_rotation') + +# Validate disable_automated_rotation +if [[ "$disable_automated_rotation" != "true" ]]; then + fail "[ERROR] Expected rotation_schedule=true, got $disable_automated_rotation" +fi + +# Read pre-rotation timestamp +before=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') + +# Trigger manual rotation +vault write -format=json -f "${PLUGIN_PATH}/rotate-root" >/dev/null + +# Read post-rotation timestamp after a brief pause +after=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') + +if [[ "$after" == "$before" ]]; then + fail "[ERROR] Manual rotation failed: timestamp did not change (before=$before, after=$after)" +fi + +echo "[OK] Manual rotation succeeded: timestamp updated (before=$before, after=$after)" diff --git a/enos/modules/root_rotation_manual/variables.tf b/enos/modules/root_rotation_manual/variables.tf new file mode 100644 index 0000000..0cf21c6 --- /dev/null +++ b/enos/modules/root_rotation_manual/variables.tf @@ -0,0 +1,22 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +variable "vault_addr" { + type = string + description = "Vault API address" +} + +variable "vault_root_token" { + type = string + description = "Vault cluster root token" +} + +variable "vault_leader_ip" { + type = string + description = "SSH host/IP of Vault leader for remote exec" +} + +variable "plugin_mount_path" { + type = string + description = "Mount path of the LDAP plugin in Vault" +} From b3cc5d0b9a22ec5045ed4bb9d7cda90214da45f0 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:12:00 -0700 Subject: [PATCH 04/11] add tf module for testing ldap secrets engine periodic root_rotation --- enos/modules/root_rotation_period/main.tf | 27 +++++++ .../scripts/test-root-rotation-period.sh | 80 +++++++++++++++++++ .../modules/root_rotation_period/variables.tf | 27 +++++++ 3 files changed, 134 insertions(+) create mode 100644 enos/modules/root_rotation_period/main.tf create mode 100755 enos/modules/root_rotation_period/scripts/test-root-rotation-period.sh create mode 100644 enos/modules/root_rotation_period/variables.tf diff --git a/enos/modules/root_rotation_period/main.tf b/enos/modules/root_rotation_period/main.tf new file mode 100644 index 0000000..1dbfd65 --- /dev/null +++ b/enos/modules/root_rotation_period/main.tf @@ -0,0 +1,27 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "rotation_window" { default = null } + +resource "enos_remote_exec" "root_rotation_period_test" { + scripts = [abspath("${path.module}/scripts/test-root-rotation-period.sh")] + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + PLUGIN_PATH = var.plugin_mount_path + ROTATION_PERIOD = var.rotation_period + } + transport = { + ssh = { + host = var.vault_leader_ip + } + } +} diff --git a/enos/modules/root_rotation_period/scripts/test-root-rotation-period.sh b/enos/modules/root_rotation_period/scripts/test-root-rotation-period.sh new file mode 100755 index 0000000..78c0005 --- /dev/null +++ b/enos/modules/root_rotation_period/scripts/test-root-rotation-period.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +# Required env vars: PLUGIN_PATH, ROTATION_PERIOD +if [[ -z "${PLUGIN_PATH:-}" ]]; then fail "PLUGIN_PATH not set"; fi +if [[ -z "${ROTATION_PERIOD:-}" ]]; then fail "ROTATION_PERIOD not set"; fi + +# Configure plugin for rotation period +vault write -format=json "${PLUGIN_PATH}/config" \ + disable_automated_rotation=false \ + rotation_period="${ROTATION_PERIOD}" \ + rotation_schedule="" \ + rotation_window=0 >/dev/null + +# Add cross-platform parse_epoch helper +parse_epoch() { + python3 -c " +import sys, datetime, re +ts = sys.argv[1] +if ts == 'null': + print(0) + sys.exit(0) +# Remove Z and handle nanoseconds +if ts.endswith('Z'): + ts = ts[:-1] +match = re.match(r'(.*\.\d{6})\d*(.*)', ts) +if match: + ts = match.group(1) + match.group(2) +dt = datetime.datetime.fromisoformat(ts) +print(int(dt.timestamp())) +" "$1" +} + +# Read rotation_period from config +rotation_period=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.rotation_period') + +# Validate rotation_period +if [[ "$rotation_period" != "$ROTATION_PERIOD" ]]; then + fail "[ERROR] Expected rotation_period=$ROTATION_PERIOD, got $rotation_period" +fi + +# Read timestamp before rotation +before=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') + +# Convert to epoch +before_epoch=$(parse_epoch "$before") + +# Wait for rotation_period + 1 seconds +echo "==> Sleeping for $((ROTATION_PERIOD + 1)) seconds for automated rotation" +sleep $((ROTATION_PERIOD + 1)) + +# Read timestamp after rotation +after=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') + +after_epoch=$(parse_epoch "$after") + +# Assert a rotation occurred +if [[ "$before" == "null" ]]; then + echo "[INFO] No previous rotation timestamp found (before=null), first rotation expected." +fi +if [[ "$after" == "null" ]]; then + fail "[ERROR] No rotation occurred, after=null" +fi + +# Compute difference +diff=$((after_epoch - before_epoch)) +if [[ "$diff" -lt "$ROTATION_PERIOD" ]]; then + fail "[ERROR] Automated rotation did not occur: delta $diff < $ROTATION_PERIOD" +fi + +#final check: + +echo "[OK] Automated rotation succeeded: delta $diff >= $ROTATION_PERIOD" diff --git a/enos/modules/root_rotation_period/variables.tf b/enos/modules/root_rotation_period/variables.tf new file mode 100644 index 0000000..3434a88 --- /dev/null +++ b/enos/modules/root_rotation_period/variables.tf @@ -0,0 +1,27 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +variable "vault_addr" { + type = string + description = "Vault API address" +} + +variable "vault_root_token" { + type = string + description = "Vault cluster root token" +} + +variable "vault_leader_ip" { + type = string + description = "SSH host/IP of Vault leader for remote exec" +} + +variable "plugin_mount_path" { + type = string + description = "Mount path of the LDAP plugin in Vault" +} + +variable "rotation_period" { + type = number + description = "Automated rotation period in seconds for the LDAP root credentials" +} From 490c685fcd21f7c312b9c3df5cf28a7898324822 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:12:14 -0700 Subject: [PATCH 05/11] add tf module for testing ldap secrets engine scheduled root_rotation --- enos/modules/root_rotation_schedule/main.tf | 27 ++++++ .../scripts/test-root-rotation-schedule.sh | 89 +++++++++++++++++++ .../root_rotation_schedule/variables.tf | 33 +++++++ 3 files changed, 149 insertions(+) create mode 100644 enos/modules/root_rotation_schedule/main.tf create mode 100755 enos/modules/root_rotation_schedule/scripts/test-root-rotation-schedule.sh create mode 100644 enos/modules/root_rotation_schedule/variables.tf diff --git a/enos/modules/root_rotation_schedule/main.tf b/enos/modules/root_rotation_schedule/main.tf new file mode 100644 index 0000000..c4e7ad4 --- /dev/null +++ b/enos/modules/root_rotation_schedule/main.tf @@ -0,0 +1,27 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +terraform { + required_providers { + enos = { + source = "registry.terraform.io/hashicorp-forge/enos" + } + } +} + +variable "rotation_period" { default = null } + +resource "enos_remote_exec" "root_rotation_schedule_test" { + scripts = [abspath("${path.module}/scripts/test-root-rotation-schedule.sh")] + environment = { + VAULT_ADDR = var.vault_addr + VAULT_TOKEN = var.vault_root_token + PLUGIN_PATH = var.plugin_mount_path + ROTATION_WINDOW = var.rotation_window + } + transport = { + ssh = { + host = var.vault_leader_ip + } + } +} diff --git a/enos/modules/root_rotation_schedule/scripts/test-root-rotation-schedule.sh b/enos/modules/root_rotation_schedule/scripts/test-root-rotation-schedule.sh new file mode 100755 index 0000000..122b022 --- /dev/null +++ b/enos/modules/root_rotation_schedule/scripts/test-root-rotation-schedule.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 +set -euo pipefail + +fail() { + echo "$1" 1>&2 + exit 1 +} + +# Required env vars: PLUGIN_PATH, ROTATION_WINDOW +if [[ -z "${PLUGIN_PATH:-}" ]]; then fail "PLUGIN_PATH not set"; fi +if [[ -z "${ROTATION_WINDOW:-}" ]]; then fail "ROTATION_WINDOW not set"; fi + +# Compute cron schedule one minute from now +schedule=$(python3 - < Using cron schedule: $schedule" + +# Configure plugin for schedule-based rotation +vault write -format=json "${PLUGIN_PATH}/config" \ + disable_automated_rotation=false \ + rotation_schedule="$schedule" \ + rotation_window="${ROTATION_WINDOW}" \ + rotation_period=0 >/dev/null + +# Read rotation_schedule from config +rotation_schedule=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.rotation_schedule') + +# Validate rotation_schedule +if [[ "$rotation_schedule" != "$schedule" ]]; then + fail "[ERROR] Expected rotation_schedule=$schedule, got $rotation_schedule" +fi + +# Read rotation_window from config +rotation_window=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.rotation_window') + +# Validate rotation_window +if [[ "$rotation_window" != "$ROTATION_WINDOW" ]]; then + fail "[ERROR] Expected rotation_period=$ROTATION_WINDOW, got $rotation_window" +fi + +# Cross-platform parse_epoch helper +parse_epoch() { + python3 -c " +import sys, datetime, re +ts = sys.argv[1] +if ts == 'null': + print(0) + sys.exit(0) +# Remove Z and handle nanoseconds +if ts.endswith('Z'): + ts = ts[:-1] +match = re.match(r'(.*\.\d{6})\d*(.*)', ts) +if match: + ts = match.group(1) + match.group(2) +dt = datetime.datetime.fromisoformat(ts) +print(int(dt.timestamp())) +" "$1" +} + +# Read timestamp before window expiration +before=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') +before_epoch=$(parse_epoch "$before") + +sleep 61 # Wait for the cron job to trigger + +# Read timestamp after window expiration +after=$(vault read -format=json "${PLUGIN_PATH}/config" | jq -r '.data.last_bind_password_rotation') +after_epoch=$(parse_epoch "$after") + +# Assert a rotation occurred +if [[ "$before" == "null" ]]; then + echo "[INFO] No previous rotation timestamp found (before=null), first rotation expected." +fi +if [[ "$after" == "null" ]]; then + fail "[ERROR] No rotation occurred, after=null" +fi + +diff=$((after_epoch - before_epoch)) +if [[ "$diff" -eq 0 ]]; then + fail "[ERROR] No rotation occurred at $after" +fi + +echo "[OK] Rotation occurred at $after" diff --git a/enos/modules/root_rotation_schedule/variables.tf b/enos/modules/root_rotation_schedule/variables.tf new file mode 100644 index 0000000..033f7de --- /dev/null +++ b/enos/modules/root_rotation_schedule/variables.tf @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +variable "vault_addr" { + type = string + description = "Vault API address" +} + +variable "vault_root_token" { + type = string + description = "Vault cluster root token" +} + +variable "vault_leader_ip" { + type = string + description = "SSH host/IP of Vault leader for remote exec" +} + +variable "plugin_mount_path" { + type = string + description = "Mount path of the LDAP plugin in Vault" +} + +variable "rotation_window" { + type = number + description = "Maximum time in seconds allowed to complete a scheduled rotation" + default = 3600 + + validation { + condition = var.rotation_window >= 3600 + error_message = "rotation_window must be at least 3600 seconds (1 hour)." + } +} \ No newline at end of file From 0759ac3b6cb1c6edb659d77ad4517e939b41aac5 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:13:12 -0700 Subject: [PATCH 06/11] add setup for integrating root rotation modules --- enos/enos-descriptions.hcl | 4 ++++ enos/enos-globals.hcl | 13 +++++++------ enos/enos-modules.hcl | 12 ++++++++++++ enos/enos-variables.hcl | 18 ++++++++++++++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/enos/enos-descriptions.hcl b/enos/enos-descriptions.hcl index e1597de..e99ed00 100644 --- a/enos/enos-descriptions.hcl +++ b/enos/enos-descriptions.hcl @@ -109,6 +109,10 @@ globals { Build, register, and enable the Vault plugin. EOF + ldap_config_root_rotation = <<-EOF + Test the LDAP secrets engine's config endpoint root rotation functionality. + EOF + unseal_vault = <<-EOF Unseal the Vault cluster using the configured seal mechanism. EOF diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index 0504052..67fa2f5 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -2,12 +2,13 @@ // SPDX-License-Identifier: MPL-2.0 globals { - archs = ["amd64", "arm64"] - artifact_sources = ["local", "crt", "artifactory"] - ldap_artifact_sources = ["local", "releases", "artifactory"] - artifact_types = ["bundle", "package"] - backends = ["raft"] - backend_tag_key = "VaultStorage" + archs = ["amd64", "arm64"] + artifact_sources = ["local", "crt", "artifactory"] + ldap_artifact_sources = ["local", "releases", "artifactory"] + ldap_config_root_rotation_methods = ["period", "schedule", "manual"] + artifact_types = ["bundle", "package"] + backends = ["raft"] + backend_tag_key = "VaultStorage" build_tags = { "ce" = ["ui"] "ent" = ["ui", "enterprise", "ent"] diff --git a/enos/enos-modules.hcl b/enos/enos-modules.hcl index 868cf9d..50e923d 100644 --- a/enos/enos-modules.hcl +++ b/enos/enos-modules.hcl @@ -90,6 +90,18 @@ module "restart_vault" { vault_install_dir = var.vault_install_dir } +module "root_rotation_period" { + source = "./modules/root_rotation_period" +} + +module "root_rotation_schedule" { + source = "./modules/root_rotation_schedule" +} + +module "root_rotation_manual" { + source = "./modules/root_rotation_manual" +} + module "seal_awskms" { source = "git::https://github.com/hashicorp/vault.git//enos/modules/seal_awskms?ref=${var.vault_repo_ref}" diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl index a178293..ba54bfa 100644 --- a/enos/enos-variables.hcl +++ b/enos/enos-variables.hcl @@ -91,6 +91,12 @@ variable "ldap_bind_pass" { default = null } +variable "ldap_disable_automated_rotation" { + type = bool + default = false + description = "Enterprise: cancel upcoming rotations until unset" +} + variable "ldap_dynamic_user_role_name" { description = "The name of the LDAP dynamic user role to create" type = string @@ -109,6 +115,18 @@ variable "ldap_revision" { default = null } +variable "ldap_rotation_period" { + type = number + default = 0 + description = "Enterprise: time in seconds before rotating the LDAP secret engine root credential. 0 disables rotation" +} + +variable "ldap_rotation_window" { + type = number + default = 0 + description = "Enterprise: max time in seconds to complete scheduled rotation" +} + variable "ldap_schema" { description = "LDAP schema type" type = string From 58ceb447a69a1dee44946ec9ac08e0983848a6e7 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:13:56 -0700 Subject: [PATCH 07/11] fmt --- enos/enos-scenario-openldap-leader-change.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enos/enos-scenario-openldap-leader-change.hcl b/enos/enos-scenario-openldap-leader-change.hcl index e1c32b3..706e361 100644 --- a/enos/enos-scenario-openldap-leader-change.hcl +++ b/enos/enos-scenario-openldap-leader-change.hcl @@ -260,7 +260,7 @@ scenario "openldap_leader_change" { description = global.description.wait_for_cluster_to_have_leader module = module.vault_wait_for_leader depends_on = [step.create_vault_cluster, - step.bootstrap_vault_cluster_targets] + step.bootstrap_vault_cluster_targets] providers = { enos = local.enos_provider[matrix.distro] @@ -597,7 +597,7 @@ scenario "openldap_leader_change" { step "vault_leader_step_down" { description = global.description.vault_leader_step_down module = module.vault_step_down - depends_on = [ + depends_on = [ step.get_vault_cluster_ips, step.test_static_role_crud_api, step.test_dynamic_role_crud_api, From 2443885fc075dddfebd0790323134dbfaad08d6f Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:19:51 -0700 Subject: [PATCH 08/11] takeout root rotation from scripts --- enos/modules/dynamic_role_crud_api/scripts/dynamic-role.sh | 3 --- enos/modules/library_crud_api/scripts/library.sh | 4 ---- enos/modules/static_role_crud_api/scripts/static-role.sh | 3 --- 3 files changed, 10 deletions(-) diff --git a/enos/modules/dynamic_role_crud_api/scripts/dynamic-role.sh b/enos/modules/dynamic_role_crud_api/scripts/dynamic-role.sh index b2c326a..fa6d58d 100755 --- a/enos/modules/dynamic_role_crud_api/scripts/dynamic-role.sh +++ b/enos/modules/dynamic_role_crud_api/scripts/dynamic-role.sh @@ -29,9 +29,6 @@ fail() { export VAULT_ADDR export VAULT_TOKEN -echo "==> Rotating root credentials" -vault write -f "${PLUGIN_PATH}/rotate-root" - ROLE_PATH="${PLUGIN_PATH}/role/${ROLE_NAME}" echo "==> Creating dynamic role: ${ROLE_NAME}" diff --git a/enos/modules/library_crud_api/scripts/library.sh b/enos/modules/library_crud_api/scripts/library.sh index 42915ec..f4f8144 100644 --- a/enos/modules/library_crud_api/scripts/library.sh +++ b/enos/modules/library_crud_api/scripts/library.sh @@ -33,10 +33,6 @@ if [[ ${#SA_LIST[@]} -lt 1 ]]; then fail "SERVICE_ACCOUNT_NAMES must contain at least one account" fi -# Rotate root credentials -echo "==> Rotating root credentials" -vault write -f "${PLUGIN_PATH}/rotate-root" - # Create library set echo "==> Creating library set ${LIBRARY_SET_NAME}" vault write "${LIB_PATH}" \ diff --git a/enos/modules/static_role_crud_api/scripts/static-role.sh b/enos/modules/static_role_crud_api/scripts/static-role.sh index 0e15567..9c92d7e 100644 --- a/enos/modules/static_role_crud_api/scripts/static-role.sh +++ b/enos/modules/static_role_crud_api/scripts/static-role.sh @@ -39,9 +39,6 @@ CRED_PATH="${PLUGIN_PATH}/static-cred/${ROLE_NAME}" echo "==> LDAP_HOST: ${LDAP_HOST}" echo "==> LDAP_PORT: ${LDAP_PORT}" -echo "==> Rotating root credentials" -vault write -f "${PLUGIN_PATH}/rotate-root" - echo "==> Creating static role ${ROLE_NAME}" vault write "${ROLE_PATH}" \ dn="${LDAP_DN}" \ From 1afde88f945096bcc6ad32a1a927a26587c31d3a Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 14 Aug 2025 18:21:18 -0700 Subject: [PATCH 09/11] integrate root rotation modules with smoke scenario --- enos/enos-scenario-openldap-smoke.hcl | 53 ++++++++++++++++++++------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/enos/enos-scenario-openldap-smoke.hcl b/enos/enos-scenario-openldap-smoke.hcl index eaaee2f..92e4d48 100644 --- a/enos/enos-scenario-openldap-smoke.hcl +++ b/enos/enos-scenario-openldap-smoke.hcl @@ -37,16 +37,17 @@ scenario "openldap_smoke" { EOF matrix { - arch = global.archs - artifact_source = global.artifact_sources - ldap_artifact_source = global.ldap_artifact_sources - artifact_type = global.artifact_types - backend = global.backends - config_mode = global.config_modes - distro = global.distros - edition = global.editions - ip_version = global.ip_versions - seal = global.seals + arch = global.archs + artifact_source = global.artifact_sources + ldap_artifact_source = global.ldap_artifact_sources + artifact_type = global.artifact_types + backend = global.backends + config_mode = global.config_modes + distro = global.distros + edition = global.editions + ip_version = global.ip_versions + seal = global.seals + ldap_config_root_rotation_method = global.ldap_config_root_rotation_methods // Our local builder always creates bundles exclude { @@ -66,6 +67,12 @@ scenario "openldap_smoke" { seal = ["pkcs11"] distro = ["leap", "sles"] } + + // rotation manager capabilities not supported in Vault community edition + exclude { + edition = ["ce"] + ldap_config_root_rotation_method = ["period", "schedule"] + } } terraform_cli = terraform_cli.default @@ -490,10 +497,30 @@ scenario "openldap_smoke" { } } + step "test_ldap_config_root_rotation" { + description = global.description.ldap_config_root_rotation + module = "root_rotation_${matrix.ldap_config_root_rotation_method}" + depends_on = [step.configure_plugin] + + providers = { + enos = local.enos_provider[matrix.distro] + } + + variables { + vault_leader_ip = step.get_vault_cluster_ips.leader_host.public_ip + vault_addr = step.create_vault_cluster.api_addr_localhost + vault_root_token = step.create_vault_cluster.root_token + plugin_mount_path = var.plugin_mount_path + + rotation_period = matrix.ldap_config_root_rotation_method == "period" ? var.ldap_rotation_period : null + rotation_window = matrix.ldap_config_root_rotation_method == "schedule" ? var.ldap_rotation_window : null + } + } + step "test_static_role_crud_api" { description = global.description.static_role_crud_api module = module.static_role_crud_api - depends_on = [step.configure_plugin] + depends_on = [step.test_ldap_config_root_rotation] providers = { enos = local.enos_provider[matrix.distro] @@ -517,7 +544,7 @@ scenario "openldap_smoke" { step "test_dynamic_role_crud_api" { description = global.description.dynamic_role_crud_api module = module.dynamic_role_crud_api - depends_on = [step.configure_plugin] + depends_on = [step.test_ldap_config_root_rotation] providers = { enos = local.enos_provider[matrix.distro] @@ -542,7 +569,7 @@ scenario "openldap_smoke" { description = global.description.library_crud_api module = module.library_crud_api depends_on = [ - step.configure_plugin, + step.test_ldap_config_root_rotation, step.test_static_role_crud_api ] From 40850970423e7a192fc6709a72c0f35efaa3e639 Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 21 Aug 2025 00:41:39 -0500 Subject: [PATCH 10/11] wip: add CI for enos scenarios --- .github/workflows/enos-tests.yaml | 58 +++++++++++++ .github/workflows/run-sample.yml | 69 ++++++++++++++++ .github/workflows/run-scenario.yml | 127 +++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 .github/workflows/enos-tests.yaml create mode 100644 .github/workflows/run-sample.yml create mode 100644 .github/workflows/run-scenario.yml diff --git a/.github/workflows/enos-tests.yaml b/.github/workflows/enos-tests.yaml new file mode 100644 index 0000000..4892ef5 --- /dev/null +++ b/.github/workflows/enos-tests.yaml @@ -0,0 +1,58 @@ +--- +name: test + +on: + pull_request: + +concurrency: + group: ${{ github.head_ref || github.run_id }}-test + cancel-in-progress: true + +jobs: + build: + name: build + runs-on: ubuntu-latest + outputs: + linux-amd64-artifact: ${{ steps.outputs.outputs.linux-amd64-artifact }} + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + - id: build + run: | + GOOS=linux GOARCH=amd64 make dev + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: vault-plugin-secrets-openldap_linux_amd64 + path: bin/vault-plugin-secrets-openldap + if-no-files-found: error + retention-days: 1 + - id: outputs + run: | + echo "linux-amd64-artifact=vault-plugin-secrets-openldap_linux_amd64" | tee -a "$GITHUB_OUTPUT" + + scenarios: + name: enos scenario + needs: build + uses: ./.github/workflows/run-sample.yml + secrets: inherit + with: + sample-name: build_ent_linux_amd64_deb + download: ${{ needs.build.outputs.linux-amd64-artifact }} + max: 1 + + completed-successfully: + if: always() + runs-on: ubuntu-latest + needs: + - build + - scenarios + steps: + - id: status + name: Determine status + run: | + results=$(tr -d '\n' <<< '${{ toJSON(needs.*.result) }}') + if ! grep -q -v -E '(failure|cancelled)' <<< "$results"; then + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/run-sample.yml b/.github/workflows/run-sample.yml new file mode 100644 index 0000000..43a056b --- /dev/null +++ b/.github/workflows/run-sample.yml @@ -0,0 +1,69 @@ +--- +name: run-samples + +on: + workflow_dispatch: + inputs: + max: + description: The maximum number of scenarios to sample + type: number + default: 8 + min: + description: The minimum number of scenarios to sample + type: number + default: 1 + sample-name: + description: The sample name + required: true + type: string + workflow_call: + inputs: + download: + description: The name of the artifact to download + type: string + required: true + max: + description: The maximum number of scenarios to sample + type: number + default: 8 + min: + description: The minimum number of scenarios to sample + type: number + default: 1 + sample-name: + description: The sample name + required: true + type: string + +jobs: + sample: + name: sample observe + runs-on: ubuntu-latest + outputs: + sample: ${{ steps.metadata.outputs.sample }} + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: hashicorp/action-setup-enos@v1 + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - id: metadata + run: | + sample_seed=$(date +%s%N) + sample=$(enos scenario sample observe ${{ inputs.sample-name }} --chdir ./enos --min ${{ inputs.min }} --max ${{ inputs.max }} --seed "${sample_seed}" --format json | jq -c ".observation.elements") + { + echo "sample=${sample}" + echo "sample-seed=${sample_seed}" # This isn't used outside of here but is nice to know for duplicating observations + } | tee -a "$GITHUB_OUTPUT" + + run: + needs: sample + name: run ${{ matrix.scenario.id.filter }} + strategy: + fail-fast: false # don't fail as that can skip required cleanup steps for jobs + matrix: + include: ${{ fromJSON(needs.sample.outputs.sample) }} + uses: ./.github/workflows/run-scenario.yml + secrets: inherit + with: + scenario-filter: ${{ matrix.scenario.id.filter }} + download: ${{ inputs.download }} \ No newline at end of file diff --git a/.github/workflows/run-scenario.yml b/.github/workflows/run-scenario.yml new file mode 100644 index 0000000..466a247 --- /dev/null +++ b/.github/workflows/run-scenario.yml @@ -0,0 +1,127 @@ +--- +name: run-scenario + +on: + workflow_dispatch: + inputs: + scenario-filter: + description: The filter of the scenario to run + required: true + type: string + workflow_call: + inputs: + download: + type: string + required: false + scenario-filter: + required: true + type: string + +jobs: + run: + name: ${{ inputs.scenario-filter }} + runs-on: ubuntu-latest + env: + ENOS_DEBUG_DATA_ROOT_DIR: /tmp/enos/logs + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + # The github actions service user creds for this account managed in hashicorp/enos-ci + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: "us-east-1" + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + role-skip-session-tagging: true + - uses: hashicorp/setup-terraform@v3 + with: + # the terraform wrapper will break terraform execution in enos because + # it changes the output to text when we expect it to be JSON. + terraform_wrapper: false + - uses: hashicorp/action-setup-enos@v1 + with: + github-token: + ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - name: Set up support files + run: | + mkdir -p enos/support + mkdir -p enos/support/vault-plugins + echo "${{ secrets.ENOS_CI_SSH_KEY }}" > enos/support/enos-ci-ssh-key.pem + echo "${{ secrets.VAULT_LICENSE }}" > enos/support/vault.hclic + chmod 600 enos/support/enos-ci-ssh-key.pem + chmod 600 enos/support/vault.hclic + - name: Download plugin artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: dist + name: ${{ inputs.download }} + - run: chmod +x ./dist/* + - name: Export OpenLDAP plugin vars + run: | + # Configure input environment variables. + { + echo "GITHUB_TOKEN=${ { steps.secrets.outputs.github-token } }" + echo "ENOS_DEBUG_DATA_ROOT_DIR=/tmp/enos-debug-data" + echo "ENOS_VAR_artifactory_token=${ { steps.secrets.outputs.artifactory-token } }" + echo "ENOS_VAR_artifactory_host=https://artifactory.hashicorp.engineering/artifactory" + echo "ENOS_VAR_artifactory_repo=hashicorp-crt-stable-local*" + echo "ENOS_VAR_aws_region="us-east-1" + echo "ENOS_VAR_aws_ssh_keypair_name=${ { inputs.ssh-key-name } }" + echo "ENOS_VAR_aws_ssh_private_key_path=./support/private_key.pem" + echo "ENOS_VAR_project_name=vault-openldap-se-enos-integration" + echo "ENOS_VAR_backend_license_path=./support/vault.hclic" + echo "ENOS_VAR_distro_version_amzn=2" + echo "ENOS_VAR_distro_version_ubuntu=22.04" + echo "ENOS_VAR_tags={\"Project Name\":\"Vault\",\"Something Cool\":\"Value\"}" + echo "ENOS_VAR_terraform_plugin_cache_dir=./support/terraform-plugin-cache" + echo "ENOS_VAR_vault_artifact_path=./support/downloads/${ { inputs.build-artifact-name } }" + echo "ENOS_VAR_vault_artifact_type=bundle" + echo "ENOS_VAR_vault_build_date=${ { needs.metadata.outputs.build-date } }" + echo "ENOS_VAR_vault_license_path=./support/vault.hclic" + echo "ENOS_VAR_vault_product_version=${ { needs.metadata.outputs.vault-version } }" + echo "ENOS_VAR_vault_revision=${ { inputs.vault-revision } }" + echo "ENOS_VAR_vault_enable_audit_devices=true" + echo "ENOS_VAR_vault_install_dir=/opt/vault/bin" + echo "ENOS_VAR_vault_instance_type=t3.small" + echo "ENOS_VAR_vault_log_level=trace" + echo "ENOS_VAR_verify_aws_secrets_engine=false" + echo "ENOS_VAR_verify_log_secrets=true" + + # Default LDAP settings from enos.vars.hcl + echo "ENOS_VAR_plugin_name=vault-plugin-secrets-openldap" + echo "ENOS_VAR_plugin_dest_dir=$(pwd)/enos/support/vault-plugins" + echo "ENOS_VAR_ldap_artifact_path=$(pwd)/dist/${{ inputs.download }}" + echo "ENOS_VAR_makefile_dir=$(pwd)" + echo "ENOS_VAR_plugin_dir_vault=/etc/vault/plugins" + echo "ENOS_VAR_plugin_mount_path=local-secrets-ldap" + echo "ENOS_VAR_ldap_bind_dn=cn=admin,dc=example,dc=com" + echo "ENOS_VAR_ldap_bind_pass=adminpassword" + echo "ENOS_VAR_ldap_user_dn=ou=users,dc=example,dc=com" + echo "ENOS_VAR_ldap_schema=openldap" + echo "ENOS_VAR_ldap_tag=${ { inputs.ldap-tag } }" + echo "ENOS_VAR_ldap_revision=${ { inputs.ldap-revision } }" + echo "ENOS_VAR_ldap_artifactory_repo=hashicorp-vault-ecosystem-staging-local" + echo "ENOS_VAR_ldap_plugin_version=${ { inputs.ldap-plugin-version } }" + echo "ENOS_VAR_ldap_base_dn=dc=example,dc=com" + echo "ENOS_VAR_ldap_user_role_name=mary" + echo "ENOS_VAR_ldap_username=mary.smith" + echo "ENOS_VAR_ldap_user_old_password=defaultpassword" + echo "ENOS_VAR_ldap_dynamic_user_role_name=adam" + echo "ENOS_VAR_dynamic_role_ldif_templates_path=/tmp" + echo "ENOS_VAR_library_set_name=dev-team" + echo "ENOS_VAR_service_account_names=[ \"staticuser\",\"bob.johnson\",\"mary.smith\" ]" + echo "ENOS_VAR_ldap_rotation_period=10" + echo "ENOS_VAR_ldap_rotation_window=3600" + } | tee -a "$GITHUB_ENV" + - run: enos scenario run ${{ inputs.scenario-filter }} + working-directory: enos + - if: ${{ always() }} + run: enos scenario destroy ${{ inputs.scenario-filter }} + working-directory: enos + - name: Remove enos runtime directories + if: ${{ always() }} + run: | + rm -rf /tmp/enos* + rm -rf enos/support + rm -rf enos/.enos + rm -rf "$HOME/.terraform.d" \ No newline at end of file From 65dfe6bb2dad78698fc88e4e4953d7cec242ac8b Mon Sep 17 00:00:00 2001 From: HamzaShili65 Date: Thu, 21 Aug 2025 00:42:57 -0500 Subject: [PATCH 11/11] keep only amzn and ubuntu distros --- enos/enos-globals.hcl | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index 67fa2f5..656e04d 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -17,26 +17,13 @@ globals { "ent.hsm.fips1403" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_3", "ent.hsm.fips1403"] } config_modes = ["env", "file"] - distros = ["amzn", "leap", "rhel", "sles", "ubuntu"] + distros = ["amzn", "ubuntu"] // Different distros may require different packages, or use different aliases for the same package distro_packages = { amzn = { "2" = ["nc"] "2023" = ["nc"] } - leap = { - "15.6" = ["netcat", "openssl"] - } - rhel = { - "8.10" = ["nc"] - "9.5" = ["nc"] - } - sles = { - // When installing Vault RPM packages on a SLES AMI, the openssl package provided - // isn't named "openssl, which rpm doesn't know how to handle. Therefore we add the - // "correctly" named one in our package installation before installing Vault. - "15.6" = ["netcat-openbsd", "openssl"] - } ubuntu = { "20.04" = ["netcat"] "22.04" = ["netcat"] @@ -45,9 +32,6 @@ globals { } distro_version = { amzn = var.distro_version_amzn - leap = var.distro_version_leap - rhel = var.distro_version_rhel - sles = var.distro_version_sles ubuntu = var.distro_version_ubuntu } editions = ["ce", "ent", "ent.fips1403", "ent.hsm", "ent.hsm.fips1403"] @@ -55,9 +39,6 @@ globals { ip_versions = ["4", "6"] package_manager = { "amzn" = "yum" - "leap" = "zypper" - "rhel" = "yum" - "sles" = "zypper" "ubuntu" = "apt" } packages = ["jq"]