diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..cd3a98b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,196 @@ +--- +name: CI +on: + push: + branches: + - master + paths: + - '**.yml' + - '**.yaml' + pull_request: + paths: + - '**.yml' + - '**.yaml' + schedule: + - cron: "0 7 * * 1" + +jobs: + lint: + name: Run Ansible Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3 + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Ansible, Ansible-Lint, Molecule and dependencies + run: pip3 install ansible ansible-lint + + - name: Run ansible-lint + run: ansible-lint + + molecule_zookeeper_default: + uses: ./.github/workflows/test-molecule.yml + needs: lint + with: + collection_path: bigdata/zookeeper + scenario: default + + molecule_zookeeper_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + with: + collection_path: bigdata/zookeeper + scenario: ha_setup + + molecule_ldap_default: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + with: + collection_path: authorization/ldap + scenario: default + + molecule_ldap_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + with: + collection_path: authorization/ldap + scenario: ha_setup + + molecule_kerberos_default: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + with: + collection_path: authentication/kerberos + scenario: default + + molecule_kerberos_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + with: + collection_path: authentication/kerberos + scenario: ha_setup + + molecule_postgres_default: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + with: + collection_path: rdbms/postgres + scenario: default + + molecule_postgres_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + - molecule_postgres_default + with: + collection_path: rdbms/postgres + scenario: ha_setup + + molecule_hdfs_default: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + - molecule_postgres_default + - molecule_postgres_ha_setup + with: + collection_path: hadoop/hdfs + scenario: default + + molecule_hdfs_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + - molecule_postgres_default + - molecule_postgres_ha_setup + - molecule_hdfs_default + with: + collection_path: hadoop/hdfs + scenario: ha_setup + + molecule_yarn_default: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + - molecule_postgres_default + - molecule_postgres_ha_setup + - molecule_hdfs_default + - molecule_hdfs_ha_setup + with: + collection_path: hadoop/yarn + scenario: default + + molecule_yarn_ha_setup: + uses: ./.github/workflows/test-molecule.yml + needs: + - lint + - molecule_zookeeper_default + - molecule_zookeeper_ha_setup + - molecule_ldap_default + - molecule_ldap_ha_setup + - molecule_kerberos_default + - molecule_kerberos_ha_setup + - molecule_postgres_default + - molecule_postgres_ha_setup + - molecule_hdfs_default + - molecule_hdfs_ha_setup + - molecule_yarn_default + with: + collection_path: hadoop/yarn + scenario: ha_setup diff --git a/.github/workflows/test-molecule.yml b/.github/workflows/test-molecule.yml new file mode 100644 index 0000000..d3a4c7f --- /dev/null +++ b/.github/workflows/test-molecule.yml @@ -0,0 +1,30 @@ +# .github/workflows/test-molecule.yml +name: Molecule test (reusable) + +on: + workflow_call: + inputs: + collection_path: + required: true + type: string + scenario: + required: false + type: string + default: default + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - run: pip install ansible molecule molecule-plugins[docker] docker + - name: Run Molecule scenario + working-directory: collections/ansible_collections/${{ inputs.collection_path }}/extensions + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + ANSIBLE_COLLECTIONS_PATH: ${{ github.workspace }}/collections + run: molecule test -s ${{ inputs.scenario }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d03eb51..fad5fbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- communitylab#61: Add GitHub Action to lint and test all Ansible collections - communitylab#59: Upgrading to PostgreSQL 17 - communitylab#57: Add new kernels for JupyterLab and upgrade software components - communitylab#55: Enable /bin/bash as default shell in JupyterLab diff --git a/ansible.cfg b/ansible.cfg index edd9eb8..d1012ed 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -7,7 +7,6 @@ force_valid_group_names = silently interpreter_python = /usr/bin/python3 retry_files_enabled = False inventory = ./inventory -vault_password_file = ~/.vault_pass.txt private_key_file = ~/.ssh/id_rsa collections_path = ./collections/ansible_collections roles_path = ./roles diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/default/molecule.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/default/molecule.yml index 8163a99..6155a83 100644 --- a/collections/ansible_collections/authentication/kerberos/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -21,3 +34,9 @@ provisioner: vvv: false playbooks: converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + realm: COMMUNITY.LAB + domain: example.com diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml index 9eaa598..9ab432e 100644 --- a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -52,11 +65,11 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/authentication/kerberos/roles/check/tasks/main.yml b/collections/ansible_collections/authentication/kerberos/roles/check/tasks/main.yml index 640dc30..11a3841 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/check/tasks/main.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/check/tasks/main.yml @@ -1,5 +1,6 @@ --- - name: Delete existing test kerberos principal + changed_when: true ansible.builtin.command: kadmin.local -q "delprinc -force {{ check_kerberos_propagation_principal }}" when: "'kerberos1' in group_names" @@ -7,15 +8,17 @@ when: "'kerberos1' in group_names" block: - name: Create test kerberos principal on primary KDC + changed_when: true ansible.builtin.command: kadmin.local -q "addprinc -randkey {{ check_kerberos_propagation_principal }}" - name: Get principals of Kerberos primary KDC + changed_when: false ansible.builtin.command: kadmin.local -q "listprincs" register: kerberos_primary_principals - name: Print message depending on listprincs output ansible.builtin.assert: - that: "'{{ check_kerberos_propagation_principal }}@{{ setup_realm }}' in kerberos_primary_principals.stdout" + that: "'{{ check_kerberos_propagation_principal }}@{{ realm }}' in kerberos_primary_principals.stdout" fail_msg: Kerberos Principal was not created, see Logs for details success_msg: Kerberos Principal was successfully created @@ -23,14 +26,16 @@ when: "'kerberos2' in group_names" block: - name: Get principals of Kerberos secondary KDC + changed_when: false ansible.builtin.command: kadmin.local -q "listprincs" register: kerberos_secondary_principals - name: Print message depending on listprincs output ansible.builtin.assert: - that: "'{{ check_kerberos_propagation_principal }}@{{ setup_realm }}' in kerberos_secondary_principals.stdout" + that: "'{{ check_kerberos_propagation_principal }}@{{ realm }}' in kerberos_secondary_principals.stdout" fail_msg: Kerberos database replication failed, see Logs for details success_msg: Kerberos primary and secondary KDC running and Kerberos database replication successfull - name: Delete test kerberos principal + changed_when: true ansible.builtin.command: kadmin.local -q "delprinc -force {{ check_kerberos_propagation_principal }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/client/defaults/main.yml b/collections/ansible_collections/authentication/kerberos/roles/client/defaults/main.yml deleted file mode 100644 index da8bd50..0000000 --- a/collections/ansible_collections/authentication/kerberos/roles/client/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -setup_realm: "{{ realm | default('COMMUNITY.LAB') }}" -setup_domain: "{{ domain | default('example.com') }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/keytab/defaults/main.yml b/collections/ansible_collections/authentication/kerberos/roles/keytab/defaults/main.yml index c1b0b67..07da3cc 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/keytab/defaults/main.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/keytab/defaults/main.yml @@ -1,3 +1,8 @@ --- keytab_user: "{% if keytab.keytab_user is defined %}{{ keytab.keytab_user }}{% else %}{{ keytab.principal }}{% endif %}" keytab_group: "{{ ide_services_group | default('root') }}" +keytab_hdfs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/keytab/tasks/create-keytab.yml b/collections/ansible_collections/authentication/kerberos/roles/keytab/tasks/create-keytab.yml index 07cc4ee..73c8293 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/keytab/tasks/create-keytab.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/keytab/tasks/create-keytab.yml @@ -4,8 +4,9 @@ keytab_hostname: "{{ ansible_fqdn if molecule_deployment is defined and molecule_deployment else inventory_hostname }}" - name: Check if keytab is already present and kinit possible using specific principal - ansible.builtin.command: kinit -k {{ keytab.principal }}/{{ keytab_hostname }}@{{ realm }} -t {{ keytab_folder }}/{{ keytab.principal }}.keytab failed_when: false + changed_when: false + ansible.builtin.command: kinit -k {{ keytab.principal }}/{{ keytab_hostname }}@{{ realm }} -t {{ keytab_folder }}/{{ keytab.principal }}.keytab register: keytab_possible - name: Create principal on kerberos primary kdc when kinit not possible and fetch it to localhost @@ -13,13 +14,16 @@ when: keytab_possible.rc != 0 block: - name: Add specific principal + changed_when: true ansible.builtin.command: kadmin.local -q "addprinc -randkey {{ keytab.principal }}/{{ keytab_hostname }}" - name: Add additional non-fqdn principal if additional_principal true + changed_when: true ansible.builtin.command: kadmin.local -q "addprinc -randkey {{ keytab.principal }}/{{ keytab.additional_principal_name }}" when: keytab.additional_principal_name is defined - name: Change random principal password for specific principal if set + changed_when: true ansible.builtin.command: kadmin.local -q "change_password -w {{ keytab.principal_password }} {{ keytab.principal }}/{{ keytab_hostname }}" when: keytab.principal_password is defined @@ -29,10 +33,12 @@ state: absent - name: Create keytab in /tmp + changed_when: true ansible.builtin.command: kadmin.local \ -q "xst -norandkey -k /tmp/{{ keytab.principal }}_{{ keytab_hostname }}.keytab {{ keytab.principal }}/{{ keytab_hostname }}" - name: Add additional non-fqdn principal to keytab if additional_principal_name is defined + changed_when: true ansible.builtin.command: kadmin.local \ -q "ktadd -norandkey -k /tmp/{{ keytab.principal }}_{{ keytab_hostname }}.keytab {{ keytab.principal }}/{{ keytab.additional_principal_name }}" when: keytab.additional_principal_name is defined diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml index b3b581d..7d7be93 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml @@ -15,9 +15,11 @@ remote_src: true - name: Decompress Kerberos schema file using gunzip + changed_when: true ansible.builtin.command: gunzip /etc/ldap/schema/kerberos.schema.gz - name: Import Kerberos schema + changed_when: true ansible.builtin.command: ldap-schema-manager -i kerberos.schema - name: Copy krb5_principal_name.ldif to /tmp @@ -28,6 +30,7 @@ - name: Import index krbPrincipalName to LDAP database failed_when: ldap_modify.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapmodify -Q -Y EXTERNAL -H ldapi:/// -w {{ ldap_password }} -f /tmp/krb5_principal_name.ldif register: ldap_modify @@ -39,14 +42,17 @@ - name: Create LDAP entries for the Kerberos administrative entities that will contact the OpenLDAP server failed_when: ldap_add.rc not in [0,68] + changed_when: true ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/krb5_administrative_entities.ldif register: ldap_add - name: Set password for 'uid=kdc-service,{{ ldap_organization }}' + changed_when: true ansible.builtin.command: ldappasswd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ -s {{ ldap_kdc_service_password }} uid=kdc-service,{{ ldap_organization }} - name: Set password for 'uid=kadmin-service,{{ ldap_organization }}' + changed_when: true ansible.builtin.command: ldappasswd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ -s {{ ldap_kadmin_service_password }} uid=kadmin-service,{{ ldap_organization }} @@ -58,6 +64,7 @@ - name: Add Kerberos ACLs to LDAP database failed_when: ldap_modify.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapmodify -Q -Y EXTERNAL -H ldapi:/// -w {{ ldap_password }} -f /tmp/kerberos_ldap_acls.ldif register: ldap_modify @@ -76,6 +83,7 @@ - name: Create Kerberos LDAP database failed_when: false + changed_when: true ansible.builtin.command: kdb5_ldap_util -D cn=admin,{{ ldap_organization }} create \ -subtrees {{ ldap_organization }} -r {{ realm }} -s -H ldapi:/// -w {{ ldap_password }} -P {{ realm_password }} @@ -86,7 +94,9 @@ mode: "0555" - name: Create a stash of the password used to bind to the LDAP server + changed_when: true ansible.builtin.command: /tmp/create_stash_file.sh + - name: Stop Kerberos services ansible.builtin.systemd: name: "{{ item }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml index 747ea56..6642435 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml @@ -15,9 +15,11 @@ remote_src: true - name: Decompress Kerberos schema file using gunzip + changed_when: true ansible.builtin.command: gunzip /etc/ldap/schema/kerberos.schema.gz - name: Import Kerberos schema + changed_when: true ansible.builtin.command: ldap-schema-manager -i kerberos.schema - name: Copy krb5_principal_name.ldif to /tmp @@ -28,6 +30,7 @@ - name: Import index krbPrincipalName to LDAP database failed_when: ldap_modify.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapmodify -Q -Y EXTERNAL -H ldapi:/// -w {{ ldap_password }} -f /tmp/krb5_principal_name.ldif register: ldap_modify diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure.yml index 56daab2..8577441 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure.yml @@ -15,8 +15,10 @@ label: "{{ item.file }}" - name: Delete existing kerberos database - ansible.builtin.command: kdb5_util destroy -f failed_when: false + changed_when: true + ansible.builtin.command: kdb5_util destroy -f - name: Create database + changed_when: true ansible.builtin.command: kdb5_util create -s -P {{ setup_realm_password }} diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/templates/kadm5.acl b/collections/ansible_collections/authentication/kerberos/roles/server/templates/kadm5.acl index e54a25e..a8daf20 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/templates/kadm5.acl +++ b/collections/ansible_collections/authentication/kerberos/roles/server/templates/kadm5.acl @@ -1 +1 @@ -kadmin/admin@{{ setup_realm }} * +kadmin/admin@{{ realm }} * diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/templates/kdc.conf b/collections/ansible_collections/authentication/kerberos/roles/server/templates/kdc.conf index 6f7657d..0724d8b 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/templates/kdc.conf +++ b/collections/ansible_collections/authentication/kerberos/roles/server/templates/kdc.conf @@ -2,7 +2,7 @@ kdc_ports = 750,88 [realms] - {{ setup_realm }} = { + {{ realm }} = { database_name = /var/lib/krb5kdc/principal admin_keytab = FILE:/etc/krb5kdc/kadm5.keytab acl_file = /etc/krb5kdc/kadm5.acl diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/templates/krb5.conf b/collections/ansible_collections/authentication/kerberos/roles/server/templates/krb5.conf index d3d6a95..045fdf2 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/templates/krb5.conf +++ b/collections/ansible_collections/authentication/kerberos/roles/server/templates/krb5.conf @@ -1,19 +1,19 @@ [libdefaults] - default_realm = {{ setup_realm }} + default_realm = {{ realm }} spake_preauth_groups = edwards25519 [kdcdefaults] spake_preauth_kdc_challenge = edwards25519 [realms] - {{ setup_realm }} = { + {{ realm }} = { kdc = {{ groups['kerberos1'][0] }} {% if groups.kerberos | length == 2 %} kdc = {{ groups['kerberos2'][0] }} database_module = openldap_ldapconf {% endif %} admin_server = {{ groups['kerberos1'][0] }} - default_domain = {{ setup_domain }} + default_domain = {{ domain }} master_key_type = aes256-cts disable_encrypted_timestamp = true } @@ -30,7 +30,7 @@ disable_lockout = true # this object needs to have read rights on - # the setup_realm container, principal container and setup_realm sub-trees + # the realm container, principal container and realm sub-trees ldap_kdc_dn = "uid=kdc-service,{{ ldap_organization }}" # this object needs to have read and write rights on @@ -44,5 +44,5 @@ {% endif %} [domain_realm] - .{{ setup_realm | lower }} = {{ setup_realm | upper }} - {{ setup_realm | lower }} = {{ setup_realm | upper }} + .{{ realm | lower }} = {{ realm | upper }} + {{ realm | lower }} = {{ realm | upper }} diff --git a/collections/ansible_collections/authentication/kerberos/roles/setup/defaults/main.yml b/collections/ansible_collections/authentication/kerberos/roles/setup/defaults/main.yml index 0102f90..c26c558 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/setup/defaults/main.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/setup/defaults/main.yml @@ -1,4 +1,2 @@ --- -setup_realm: "{{ realm | default('COMMUNITY.LAB') }}" setup_realm_password: "{{ realm_password | default('changeit') }}" -setup_domain: "{{ domain | default('example.com') }}" diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml index 4143f23..f266a47 100644 --- a/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -42,11 +55,11 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml index ec0bd4d..b649b41 100644 --- a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -52,11 +65,11 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml index fea10a7..ec144df 100644 --- a/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml @@ -15,6 +15,7 @@ when: ldap_service_status.status.ActiveState == "active" block: - name: Check if IDE users are added to IDE group + changed_when: false ansible.builtin.shell: cmd: 'set -o pipefail && ldapsearch -H ldapi:/// -Y EXTERNAL -LLL -b "{{ ldap_organization }}" filter uid | grep uid:' executable: /bin/bash @@ -47,6 +48,7 @@ - molecule_deployment block: - name: Check if ldap standard replication is active + changed_when: false ansible.builtin.command: ldapsearch -z1 -LLL -H ldapi:/// -s base -b {{ ldap_organization }} contextCSN register: ldap_search_context diff --git a/collections/ansible_collections/authorization/ldap/roles/client/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/client/tasks/main.yml index 22ed731..9d956e4 100644 --- a/collections/ansible_collections/authorization/ldap/roles/client/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/client/tasks/main.yml @@ -24,6 +24,7 @@ mode: "0644" - name: Configure /etc/nsswitch and pam files for sssd + changed_when: false ansible.builtin.command: pam-auth-update --enable mkhomedir - name: Configure sssd with main configuration file sssd.conf diff --git a/collections/ansible_collections/authorization/ldap/roles/database/tasks/add-ldap-users.yml b/collections/ansible_collections/authorization/ldap/roles/database/tasks/add-ldap-users.yml index 7d90772..8e5e53d 100644 --- a/collections/ansible_collections/authorization/ldap/roles/database/tasks/add-ldap-users.yml +++ b/collections/ansible_collections/authorization/ldap/roles/database/tasks/add-ldap-users.yml @@ -13,9 +13,11 @@ - name: Add specific IDE user failed_when: ldapadd.rc not in [0,68] + changed_when: true ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/user.ldif register: ldapadd - name: Set password for specific IDE user + changed_when: true ansible.builtin.command: ldappasswd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ -s {{ item.password }} uid={{ item.name }},ou=people,{{ ldap_organization }} diff --git a/collections/ansible_collections/authorization/ldap/roles/database/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/database/tasks/main.yml index 035829a..fccb6a7 100644 --- a/collections/ansible_collections/authorization/ldap/roles/database/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/database/tasks/main.yml @@ -10,22 +10,25 @@ - group.ldif - name: Create organization unit for IDE users + changed_when: false + failed_when: ldapadd.rc not in [0,68] ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/org_people.ldif register: ldapadd until: ldapadd.rc in [0,68] retries: 5 delay: 3 - failed_when: ldapadd.rc not in [0,68] - name: Create organization unit for IDE groups + changed_when: false + failed_when: ldapadd.rc not in [0,68] ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/org_groups.ldif register: ldapadd - failed_when: ldapadd.rc not in [0,68] - name: Create group for IDE users + changed_when: false + failed_when: ldapadd.rc not in [0,68] ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/group.ldif register: ldapadd - failed_when: ldapadd.rc not in [0,68] - name: Add IDE users to organization unit and ldap group ansible.builtin.include_tasks: add-ldap-users.yml diff --git a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-consumer.yml b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-consumer.yml index d897e82..2b23a7f 100644 --- a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-consumer.yml +++ b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-consumer.yml @@ -7,5 +7,6 @@ - name: Add consumer_simple_sync.ldif to ldap consumer failed_when: ldapadd.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapadd -Q -Y EXTERNAL -H ldapi:/// -f /tmp/consumer_simple_sync.ldif register: ldapadd diff --git a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml index 37aea07..3cd621d 100644 --- a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml +++ b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml @@ -7,10 +7,12 @@ - name: Add replication user '{{ ldap_replication_user }}' failed_when: ldapadd.rc not in [0,68] + changed_when: true ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/replicator.ldif register: ldapadd - name: Set password for IDE user '{{ ldap_replication_user }}' + changed_when: true ansible.builtin.command: ldappasswd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ -s {{ ldap_replication_password }} {{ ldap_replication_user }} @@ -22,6 +24,7 @@ - name: Add replicator-acl-limits.ldif to ldap provider failed_when: ldapmodify.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapmodify -Q -Y EXTERNAL -H ldapi:/// -f /tmp/replicator-acl-limits.ldif register: ldapmodify @@ -33,5 +36,6 @@ - name: Add provider_simple_sync.ldif to ldap provider failed_when: ldapmodify.rc not in [0,20] + changed_when: true ansible.builtin.command: ldapmodify -Q -Y EXTERNAL -H ldapi:/// -f /tmp/provider_simple_sync.ldif register: ldapmodify diff --git a/collections/ansible_collections/authorization/ldap/roles/server/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/server/tasks/main.yml index 08c0a09..fefacd2 100644 --- a/collections/ansible_collections/authorization/ldap/roles/server/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/server/tasks/main.yml @@ -21,9 +21,11 @@ - name: Reconfigure slapd failed_when: false ansible.builtin.command: dpkg-reconfigure -f noninteractive slapd --force + changed_when: true - name: Change ownership of ldap folders ansible.builtin.command: chown -R {{ ldap_user }}:{{ ldap_group }} {{ item }} + changed_when: true loop: - /etc/ldap/slapd.d - /var/lib/ldap @@ -48,6 +50,7 @@ - name: Configure ldap server for TLS failed_when: ldapmodify.rc not in [20,0] ansible.builtin.command: ldapmodify -Y EXTERNAL -H ldapi:/// -f /tmp/ssl.ldif + changed_when: true register: ldapmodify - name: Copy update-module.ldif @@ -59,6 +62,7 @@ - name: Load ldap module memberOf failed_when: ldapadd.rc not in [20,0] ansible.builtin.command: ldapadd -Q -Y EXTERNAL -H ldapi:/// -f /tmp/update-module.ldif + changed_when: true register: ldapadd - name: Copy memberof-overlay.ldif @@ -69,6 +73,7 @@ - name: Add overlay to database ansible.builtin.command: ldapadd -Q -Y EXTERNAL -H ldapi:/// -f /tmp/add-memberof-overlay.ldif + changed_when: true - name: Copy add-refint.ldif ansible.builtin.copy: @@ -79,6 +84,7 @@ - name: Ldapadd add-refint.ldif failed_when: ldapadd.rc not in [20,0] ansible.builtin.command: ldapadd -Q -Y EXTERNAL -H ldapi:/// -f /tmp/add-refint.ldif + changed_when: true register: ldapadd - name: Configure ldap.conf diff --git a/collections/ansible_collections/bigdata/spark/extensions/molecule/default/molecule.yml b/collections/ansible_collections/bigdata/spark/extensions/molecule/default/molecule.yml index dac89c9..e4fdb3a 100644 --- a/collections/ansible_collections/bigdata/spark/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/bigdata/spark/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml index 331e0c5..f1feb3c 100644 --- a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -41,11 +54,11 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml index 31c4ad2..3c647d7 100644 --- a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -61,11 +74,11 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/check/tasks/main.yml b/collections/ansible_collections/bigdata/zookeeper/roles/check/tasks/main.yml index c42d710..4c5757d 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/check/tasks/main.yml +++ b/collections/ansible_collections/bigdata/zookeeper/roles/check/tasks/main.yml @@ -7,6 +7,7 @@ -server {% for host in groups.zookeeper %}\ {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:2281{% if not loop.last %},{% endif %}\ {% endfor %} get /zookeeper + changed_when: false register: main_znode_get retries: 5 until: main_znode_get.rc == 0 diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/install/defaults/main.yml b/collections/ansible_collections/bigdata/zookeeper/roles/install/defaults/main.yml index 026bf82..eeb128b 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/install/defaults/main.yml +++ b/collections/ansible_collections/bigdata/zookeeper/roles/install/defaults/main.yml @@ -2,3 +2,8 @@ install_zookeeper_version: 3.9.3 install_zookeeper_data_dir: /var/zookeeper install_zookeeper_log_dir: /var/log/zookeeper + +install_zookeeper_user: "{{ zookeeper_user | default('zookeeper') }}" +install_zookeeper_uid: "{{ zookeeper_uid | default('5005') }}" +install_zookeeper_group: "{{ zookeeper_group | default('hadoop') }}" +install_zookeeper_gid: "{{ zookeeper_gid | default('4001') }}" diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/configure.yml b/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/configure.yml index 9b6654f..0b1c5d3 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/configure.yml +++ b/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/configure.yml @@ -5,8 +5,8 @@ path: "{{ item }}" state: directory mode: "0755" - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" loop: - "{{ install_zookeeper_data_dir }}" - "{{ install_zookeeper_log_dir }}" @@ -17,8 +17,8 @@ content: "{{ zookeeper_id }}" dest: "{{ install_zookeeper_data_dir }}/myid" mode: "0644" - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" - name: Configure zookeeper with zoo.cfg and zookeeper-env.sh become: true @@ -26,8 +26,8 @@ src: "{{ item }}" dest: /opt/apache-zookeeper/zookeeper/conf/{{ item }} mode: "0644" - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" loop: - zoo.cfg - zookeeper-env.sh diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/install.yml b/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/install.yml index ab613e0..68fefbd 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/install.yml +++ b/collections/ansible_collections/bigdata/zookeeper/roles/install/tasks/install.yml @@ -7,16 +7,16 @@ - name: Create zookeeper group ansible.builtin.group: - name: "{{ setup_zookeeper_group }}" - gid: "{{ setup_zookeeper_gid }}" + name: "{{ install_zookeeper_group }}" + gid: "{{ install_zookeeper_gid }}" state: present - name: Create zookeeper user ansible.builtin.user: - name: "{{ setup_zookeeper_user }}" + name: "{{ install_zookeeper_user }}" comment: User for Apache Zookeeper - uid: "{{ setup_zookeeper_uid }}" - group: "{{ setup_zookeeper_gid }}" + uid: "{{ install_zookeeper_uid }}" + group: "{{ install_zookeeper_gid }}" state: present shell: /bin/bash @@ -26,8 +26,8 @@ path: /opt/apache-zookeeper state: directory mode: "0755" - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" - name: Unpack Apache Zookeeper tarball become: true @@ -35,8 +35,8 @@ copy: false src: /var/tmp/apache-zookeeper-{{ install_zookeeper_version }}-bin.tar.gz dest: /opt/apache-zookeeper - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" creates: /opt/apache-zookeeper/apache-zookeeper-{{ install_zookeeper_version }}-bin - name: Create symbolic link @@ -45,5 +45,5 @@ src: /opt/apache-zookeeper/apache-zookeeper-{{ install_zookeeper_version }}-bin dest: /opt/apache-zookeeper/zookeeper state: link - owner: "{{ setup_zookeeper_user }}" - group: "{{ setup_zookeeper_group }}" + owner: "{{ install_zookeeper_user }}" + group: "{{ install_zookeeper_group }}" diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zookeeper.service b/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zookeeper.service index a3c214f..43fbc62 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zookeeper.service +++ b/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zookeeper.service @@ -6,8 +6,8 @@ After=network.target network-online.target [Service] Type=forking WorkingDirectory=/opt/apache-zookeeper/zookeeper -User={{ setup_zookeeper_user }} -Group={{ setup_zookeeper_group }} +User={{ install_zookeeper_user }} +Group={{ install_zookeeper_group }} ExecStart=/opt/apache-zookeeper/zookeeper/bin/zkServer.sh start ExecStop=/opt/apache-zookeeper/zookeeper/bin/zkServer.sh stop TimeoutSec=30 diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/setup/defaults/main.yml b/collections/ansible_collections/bigdata/zookeeper/roles/setup/defaults/main.yml deleted file mode 100644 index bf454cb..0000000 --- a/collections/ansible_collections/bigdata/zookeeper/roles/setup/defaults/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -setup_zookeeper_user: "{{ zookeeper_user | default('zookeeper') }}" -setup_zookeeper_uid: "{{ zookeeper_uid | default('5005') }}" -setup_zookeeper_group: "{{ zookeeper_group | default('hadoop') }}" -setup_zookeeper_gid: "{{ zookeeper_gid | default('4001') }}" diff --git a/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml b/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml index 1c8390e..f384d43 100644 --- a/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml +++ b/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml @@ -1,15 +1,7 @@ --- setup_hadoop_version: 3.4.1 -hadoop_client_user: root -hadoop_client_group: root -hdfs_data_dir: /var/hadoop/hdfs setup_hadoop_conf_dir: /opt/apache-hadoop setup_hadoop_log_dir: /var/log/hadoop setup_core_site_name: "{{ 'core-site-ha.xml' if (groups.namenodes | length == 3) else 'core-site.xml' }}" setup_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (groups.namenodes | length == 3) else 'hdfs-site.xml' }}" setup_yarn_site_name: "{{ 'yarn-site-ha.xml' if (groups.namenodes | length == 3) else 'yarn-site.xml' }}" - -keystore_file: /etc/ssl/private/{{ inventory_hostname }}.jks -keystore_password: changeit -truststore_file: /etc/ssl/certs/truststore.jks -truststore_password: changeit diff --git a/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml index b72acab..ae555b7 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml +++ b/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml @@ -6,21 +6,4 @@ setup_hdfs_group: "{{ hdfs_group | default('hadoop') }}" setup_hdfs_gid: "{{ hdfs_gid | default('4001') }}" setup_hadoop_conf_dir: /opt/apache-hadoop setup_hadoop_log_dir: /var/log/hadoop -keytab_folder: /etc/keytabs -keytab_user_hdfs: hdfs -keytab_user_yarn: yarn -keytab_user_jupyter: jupyterhub -keytab_user_http: HTTP setup_core_site_name: "{{ 'core-site-ha.xml' if (groups.namenodes | length == 3) else 'core-site.xml' }}" - -keystore_file: /etc/ssl/private/{{ ansible_fqdn if molecule_deployment is defined and molecule_deployment else inventory_hostname }}.jks -keystore_password: changeit -truststore_file: /etc/ssl/certs/truststore.jks -truststore_password: changeit - -ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" -ldap_organization: dc=example,dc=com -ldap_user_search_base: ou=people,{{ ldap_organization }} -ldap_group_search_base: ou=groups,{{ ldap_organization }} -ldap_bind_user: cn=admin,{{ ldap_organization }} -ldap_password: changeit diff --git a/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml index 8d6c728..5091e30 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml @@ -22,6 +22,12 @@ - not kerberos_external ansible.builtin.include_role: name: authentication.kerberos.keytab + vars: + kerberos_keytabs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" - name: Create hadoop configuration and log folder ansible.builtin.file: @@ -90,6 +96,7 @@ - name: Generate secret key used for signing authentication tokens ansible.builtin.command: dd if=/dev/urandom of={{ keytab_folder }}/hadoop_secret bs=1024 count=1 + changed_when: false - name: Change permissions of generated secret key ansible.builtin.file: diff --git a/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml deleted file mode 100644 index 92d9e17..0000000 --- a/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -kerberos_keytabs: - - principal: "{{ hdfs_user }}" - - principal: HTTP - keytab_user: "{{ hdfs_user }}" diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/molecule.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/molecule.yml index 814a3a0..8f6fccd 100644 --- a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -72,6 +85,7 @@ provisioner: self_signed_certificates: true domain: example.com keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs realm: COMMUNITY.LAB ldap_organization: dc=example,dc=com ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" @@ -95,6 +109,14 @@ provisioner: uid_number: 6002 gid_number: "{{ ide_users_gid }}" password: datascience + keytab_user_hdfs: hdfs + keytab_user_http: HTTP + keytab_user_yarn: yarn + keytab_user_jupyter: jupyterhub + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit ldap: ldap_user: openldap ldap_uid: "5001" @@ -115,6 +137,11 @@ provisioner: service_gid: "4002" tls_user: hdfs tls_group: hadoop + kerberos_keytabs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" datanodes: service_user: hdfs service_name: Apache Hadoop @@ -123,3 +150,8 @@ provisioner: service_gid: "4002" tls_user: hdfs tls_group: hadoop + kerberos_keytabs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml index a2c5071..e39034d 100644 --- a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -90,11 +103,11 @@ provisioner: hdfs_user: hdfs hdfs_group: hadoop hadoop_nameservice: communitylab - keytab_user_journalnode: journalnode ide_services_group: hadoop self_signed_certificates: true domain: example.com keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs realm: COMMUNITY.LAB ldap_organization: dc=example,dc=com ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" @@ -122,6 +135,11 @@ provisioner: keystore_password: changeit truststore_file: /etc/ssl/certs/truststore.jks truststore_password: changeit + keytab_user_hdfs: hdfs + keytab_user_http: HTTP + keytab_user_yarn: yarn + keytab_user_jupyter: jupyterhub + keytab_user_journalnode: journalnode ldap: ldap_user: openldap ldap_uid: "5001" @@ -142,6 +160,11 @@ provisioner: service_gid: "4001" tls_user: hdfs tls_group: hadoop + kerberos_keytabs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" datanodes: service_user: hdfs service_name: Apache Hadoop @@ -150,6 +173,11 @@ provisioner: service_gid: "4001" tls_user: hdfs tls_group: hadoop + kerberos_keytabs: + - principal: "{{ hdfs_user }}" + keytab_user: "{{ hdfs_user }}" + - principal: HTTP + keytab_user: "{{ hdfs_user }}" zookeeper1: zookeeper_id: 1 zookeeper2: diff --git a/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml index afaf87b..0ca2ea3 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml @@ -5,6 +5,7 @@ - name: Get valid keytab for HDFS user ansible.builtin.command: kinit -k hdfs/{{ keytab_principal_hostname }}@{{ realm }} -t {{ keytab_folder }}/hdfs.keytab + changed_when: false - name: Block for IDE HA setup when: groups.namenodes | length == 3 @@ -12,6 +13,7 @@ - name: Check if HDFS namenodes are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs haadmin -getAllServiceState register: ha_namenodes + changed_when: false - name: Print message depending on hdfs haadmin -getAllServiceState output ansible.builtin.assert: @@ -22,6 +24,7 @@ - name: Check if HDFS Datanodes are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfsadmin -report register: hdfs_report + changed_when: false - name: Print message depending on hdfs dfsadmin -report output ansible.builtin.assert: @@ -34,6 +37,7 @@ /opt/apache-hadoop/hadoop/bin/hdfs dfs -mkdir /test /opt/apache-hadoop/hadoop/bin/hdfs dfs -rm -r /test register: hdfs_writable + changed_when: false - name: Print message depending on hdfs writable commands ansible.builtin.assert: diff --git a/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml index bdc17fe..ef7506a 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml @@ -1,10 +1,4 @@ --- common_hdfs_user: "{{ hdfs_user | default('hdfs') }}" common_hdfs_group: "{{ hdfs_group | default('hadoop') }}" -keytab_folder: /etc/keytabs -hdfs_data_dir: /var/hadoop/hdfs common_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (groups.namenodes | length == 3) else 'hdfs-site.xml' }}" - -keytab_user_hdfs: hdfs -keytab_user_http: HTTP -setup_realm: COMMUNITY.LAB diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/defaults/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/namenode/defaults/main.yml index 39e835b..fc2d5b0 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/defaults/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/defaults/main.yml @@ -1,5 +1,4 @@ --- -setup_hdfs_user: "{{ hdfs_user | default('hdfs') }}" -setup_hdfs_group: "{{ hdfs_group | default('hadoop') }}" +namenode_hdfs_user: "{{ hdfs_user | default('hdfs') }}" +namenode_hdfs_group: "{{ hdfs_group | default('hadoop') }}" namenode_hdfs_data_dir: /var/hadoop/hdfs -hdfs_user: hdfs diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/configure-namenode.yml b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/configure-namenode.yml index 74e54d9..901398b 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/configure-namenode.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/configure-namenode.yml @@ -9,14 +9,14 @@ ansible.builtin.file: path: "{{ namenode_hdfs_data_dir }}/namenode" state: directory - owner: "{{ setup_hdfs_user }}" - group: "{{ setup_hdfs_group }}" + owner: "{{ namenode_hdfs_user }}" + group: "{{ namenode_hdfs_group }}" mode: "0775" - name: Create hosts file ansible.builtin.template: src: hosts.json.j2 dest: /opt/apache-hadoop/hadoop/etc/hadoop/hosts.json - owner: "{{ setup_hdfs_user }}" - group: "{{ setup_hdfs_group }}" + owner: "{{ namenode_hdfs_user }}" + group: "{{ namenode_hdfs_group }}" mode: "0644" diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml index 223a4f2..0cb6f27 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml @@ -9,8 +9,8 @@ - name: Set fact for initializing process ansible.builtin.set_fact: - namenode_hdfs_data_dir_current: "{{ namenode_hdfs_data_dir + '/namenode/storage1/current' if groups.namenodes | length == 3 else namenode_hdfs_data_dir + '/namenode/current' - }}" + namenode_hdfs_data_dir_current: "{{ namenode_hdfs_data_dir + '/namenode/storage1/current' if groups.namenodes | length == 3 else + namenode_hdfs_data_dir + '/namenode/current' }}" - name: Check HDFS is already initialized ansible.builtin.stat: @@ -43,8 +43,9 @@ - name: Format HDFS ZKFC become: true - become_user: "{{ hdfs_user }}" + become_user: "{{ namenode_hdfs_user }}" ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs zkfc -formatZK -force + changed_when: true when: - "'namenode1' in group_names" - not namenode_initialized_dir.stat.exists @@ -52,8 +53,9 @@ - name: Format HDFS namenode become: true - become_user: "{{ hdfs_user }}" + become_user: "{{ namenode_hdfs_user }}" ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs namenode -format -force + changed_when: true when: - "'namenode1' in group_names" - not namenode_initialized_dir.stat.exists @@ -73,8 +75,9 @@ - name: Bootstrap HDFS secondary and tertiary namenode become: true - become_user: "{{ hdfs_user }}" + become_user: "{{ namenode_hdfs_user }}" ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs namenode -bootstrapStandby -nonInteractive -force + changed_when: true when: - "'namenode1' not in group_names" - not namenode_initialized_dir.stat.exists diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service index e27883f..170bbe0 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service @@ -11,7 +11,7 @@ Wants=hdfs-journalnode.service Type=forking ExecStart=/opt/apache-hadoop/hadoop/bin/hdfs --daemon start namenode ExecStop=/opt/apache-hadoop/hadoop/bin/hdfs --daemon stop namenode -User={{ hdfs_user }} +User={{ namenode_hdfs_user }} Group={{ hdfs_group }} TimeoutSec=30 Restart=always diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-zkfc.service b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-zkfc.service index c4b04df..2de7b31 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-zkfc.service +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-zkfc.service @@ -7,7 +7,7 @@ Wants=zookeeper.service Type=forking ExecStart=/opt/apache-hadoop/hadoop/bin/hdfs --daemon start zkfc ExecStop=/opt/apache-hadoop/hadoop/bin/hdfs --daemon stop zkfc -User={{ hdfs_user }} +User={{ namenode_hdfs_user }} Group={{ hdfs_group }} TimeoutSec=30 Restart=always diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/default/molecule.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/default/molecule.yml index 27c2b12..74ea7ed 100644 --- a/collections/ansible_collections/hadoop/yarn/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -74,10 +87,13 @@ provisioner: self_signed_certificates: true domain: example.com keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs realm: COMMUNITY.LAB ide_services_group: hadoop + keytab_user_hdfs: hdfs keytab_user_yarn: yarn keytab_user_http: HTTP + keytab_user_jupyter: jupyterhub ldap_organization: dc=example,dc=com ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" ldap_user_search_base: ou=people,{{ ldap_organization }} @@ -100,6 +116,10 @@ provisioner: uid_number: 6002 gid_number: "{{ ide_users_gid }}" password: datascience + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit ldap: ldap_user: openldap ldap_uid: "5001" @@ -120,6 +140,9 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" nodemanagers: service_user: yarn service_name: Apache Hadoop @@ -128,3 +151,6 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml index 8e15ab7..3e1ae88 100644 --- a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -90,16 +103,19 @@ provisioner: hdfs_user: hdfs hdfs_group: hadoop hadoop_nameservice: communitylab - keytab_user_journalnode: journalnode yarn_user: yarn yarn_group: hadoop self_signed_certificates: true domain: example.com keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs realm: COMMUNITY.LAB ide_services_group: hadoop - keytab_user_yarn: yarn + keytab_user_hdfs: hdfs keytab_user_http: HTTP + keytab_user_yarn: yarn + keytab_user_jupyter: jupyterhub + keytab_user_journalnode: journalnode ldap_organization: dc=example,dc=com ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" ldap_user_search_base: ou=people,{{ ldap_organization }} @@ -149,6 +165,8 @@ provisioner: journalnode_user: journalnode journalnode_group: hadoop journalnode_uid: "5006" + kerberos_keytabs: + - principal: "{{ yarn_user }}" nodemanagers: service_user: yarn service_name: Apache Hadoop @@ -157,6 +175,8 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" zookeeper1: zookeeper_id: 1 zookeeper2: diff --git a/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml b/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml index ffca378..1ebbb72 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml @@ -5,6 +5,7 @@ - name: Get valid keytab for YARN user ansible.builtin.command: kinit -k yarn/{{ keytab_principal_hostname }}@{{ realm }} -t {{ keytab_folder }}/yarn.keytab + changed_when: false - name: Block for IDE HA setup when: @@ -12,6 +13,7 @@ block: - name: Check if YARN resourcemanagers are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/yarn rmadmin -getAllServiceState + changed_when: false register: ha_resourcemanagers - name: Print message depending on yarn rmadmin -getAllServiceState output @@ -22,6 +24,7 @@ - name: Check if YARN nodemanagers are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/yarn node -list + changed_when: false register: nodemanagers until: nodemanagers.rc == 0 retries: 3 diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml b/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml index ef4baf3..56e6885 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml @@ -1,6 +1,6 @@ --- common_yarn_user: "{{ yarn_user | default('yarn') }}" common_yarn_group: "{{ yarn_group | default('yarn') }}" -hdfs_data_dir: /var/hadoop/hdfs -yarn_data_dir: /var/hadoop/yarn +common_hdfs_data_dir: /var/hadoop/hdfs +common_yarn_data_dir: /var/hadoop/yarn common_yarn_site_name: "{{ 'yarn-site-ha.xml' if (groups.resourcemanagers | length == 3) else 'yarn-site.xml' }}" diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/tasks/main.yml b/collections/ansible_collections/hadoop/yarn/roles/common/tasks/main.yml index c92e6b6..8bf5355 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/tasks/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Create data folder for yarn ansible.builtin.file: - path: "{{ yarn_data_dir }}" + path: "{{ common_yarn_data_dir }}" state: directory owner: "{{ common_yarn_user }}" group: "{{ common_yarn_group }}" diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml index c700612..7ee3b03 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml @@ -106,7 +106,7 @@ yarn.nodemanager.log.retain-seconds diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site.xml b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site.xml index fce911b..c5d20f9 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site.xml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site.xml @@ -86,7 +86,7 @@ yarn.nodemanager.log.retain-seconds diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/vars/main.yml b/collections/ansible_collections/hadoop/yarn/roles/common/vars/main.yml deleted file mode 100644 index d118856..0000000 --- a/collections/ansible_collections/hadoop/yarn/roles/common/vars/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -kerberos_keytabs: - - principal: "{{ common_yarn_user }}" diff --git a/collections/ansible_collections/ide/environment/roles/variables/tasks/main.yml b/collections/ansible_collections/ide/environment/roles/variables/tasks/main.yml index 3a2eef4..e293a85 100644 --- a/collections/ansible_collections/ide/environment/roles/variables/tasks/main.yml +++ b/collections/ansible_collections/ide/environment/roles/variables/tasks/main.yml @@ -1,6 +1,7 @@ --- - name: Get Java Home ansible.builtin.shell: dirname $(dirname $(readlink -f $(which java))) + changed_when: false register: java_home - name: Set Java Home variable diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml index 4ab2ac1..025dd7b 100644 --- a/collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -90,6 +103,7 @@ provisioner: truststore_file: /etc/ssl/certs/truststore.jks truststore_password: changeit keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs realm: COMMUNITY.LAB ide_services_group: hadoop ide_users_group: ide_users @@ -138,6 +152,13 @@ provisioner: port: 5432 hostname: "*" scheme: jupyterhub + miniforge_user: "{{ jupyterhub_user }}" + miniforge_group: "{{ jupyterhub_group }}" + hadoop_client_user: "{{ jupyterhub_user }}" + hadoop_client_group: "{{ jupyterhub_group }}" + kerberos_keytabs: + - principal: "{{ jupyterhub_user }}" + keytab_user: "{{ jupyterhub_user }}" ldap: ldap_user: openldap ldap_uid: "5001" @@ -158,6 +179,9 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" nodemanagers: service_user: yarn service_name: Apache Hadoop @@ -166,5 +190,11 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" + jupyterlab: + miniforge_user: "{{ yarn_user }}" + miniforge_group: "{{ yarn_group }}" zookeeper1: zookeeper_id: 1 diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml index 5829860..2bd490e 100644 --- a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -130,6 +143,7 @@ provisioner: truststore_file: /etc/ssl/certs/truststore.jks truststore_password: changeit keytab_folder: /etc/keytabs + hdfs_data_dir: /var/hadoop/hdfs ide_services_group: hadoop ide_users_group: ide_users ide_users_gid: "5001" @@ -204,6 +218,13 @@ provisioner: haproxy_pem_file: /etc/ssl/private/haproxy.pem jupyterhub_domain_ip: true haproxy_admin_password: changeit + miniforge_user: "{{ jupyterhub_user }}" + miniforge_group: "{{ jupyterhub_group }}" + hadoop_client_user: "{{ jupyterhub_user }}" + hadoop_client_group: "{{ jupyterhub_group }}" + kerberos_keytabs: + - principal: "{{ jupyterhub_user }}" + keytab_user: "{{ jupyterhub_user }}" ldap: ldap_user: openldap ldap_uid: "5001" @@ -224,6 +245,9 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" nodemanagers: service_user: yarn service_name: Apache Hadoop @@ -232,16 +256,22 @@ provisioner: service_gid: "4001" tls_user: yarn tls_group: hadoop + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" + jupyterlab: + miniforge_user: "{{ yarn_user }}" + miniforge_group: "{{ yarn_group }}" postgres: certs_source: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_dest: + distribute_certs_dest: - cert.pem - chain.pem - key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/jupyter/hub/roles/configure/tasks/main.yml b/collections/ansible_collections/jupyter/hub/roles/configure/tasks/main.yml index b64cb0a..4b3422e 100644 --- a/collections/ansible_collections/jupyter/hub/roles/configure/tasks/main.yml +++ b/collections/ansible_collections/jupyter/hub/roles/configure/tasks/main.yml @@ -38,6 +38,7 @@ - name: Generate jupyterhub cookie secret ansible.builtin.shell: openssl rand -hex 32 > /etc/jupyterhub/jupyterhub_cookie_secret + changed_when: true when: not jupyterhub_cookie_secret.stat.exists - name: Change permissions for jupyterhub cookie secret diff --git a/collections/ansible_collections/jupyter/hub/roles/install/tasks/main.yml b/collections/ansible_collections/jupyter/hub/roles/install/tasks/main.yml index 3139e88..ecf2315 100644 --- a/collections/ansible_collections/jupyter/hub/roles/install/tasks/main.yml +++ b/collections/ansible_collections/jupyter/hub/roles/install/tasks/main.yml @@ -17,11 +17,13 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate conda create -n jupyterhub python={{ install_environment_python_version }} -y + changed_when: true - name: Install jupyterhub conda packages in jupyterhub environment ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterhub conda install -c conda-forge {% for conda_package in install_conda_packages %}{{ conda_package }}{% if not loop.last %} {% endif %}{% endfor %} -y + changed_when: true args: executable: /bin/bash @@ -29,8 +31,10 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterhub pip install {% for pip_package in install_pip_packages %}{{ pip_package }}{% if not loop.last %} {% endif %}{% endfor %} + changed_when: true args: executable: /bin/bash - name: Change owner to jupyterhub user and group - use shell module because of better performance ansible.builtin.command: chown {{ jupyterhub_user }}:{{ jupyterhub_group }} -R {{ install_miniforge_base_path }}/envs/jupyterhub + changed_when: true diff --git a/collections/ansible_collections/jupyter/hub/roles/install/vars/main.yml b/collections/ansible_collections/jupyter/hub/roles/install/vars/main.yml deleted file mode 100644 index 1e4defb..0000000 --- a/collections/ansible_collections/jupyter/hub/roles/install/vars/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -miniforge_user: "{{ jupyterhub_user }}" -miniforge_group: "{{ jupyterhub_group }}" - -kerberos_keytabs: - - principal: "{{ jupyterhub_user }}" - -hadoop_client_user: "{{ jupyterhub_user }}" -hadoop_client_group: "{{ jupyterhub_group }}" diff --git a/collections/ansible_collections/jupyter/lab/roles/setup/tasks/copy-example-notebooks.yml b/collections/ansible_collections/jupyter/lab/roles/setup/tasks/copy-example-notebooks.yml index 7790551..d67683e 100644 --- a/collections/ansible_collections/jupyter/lab/roles/setup/tasks/copy-example-notebooks.yml +++ b/collections/ansible_collections/jupyter/lab/roles/setup/tasks/copy-example-notebooks.yml @@ -1,10 +1,12 @@ - name: Check if example notebook is already present in /share of HDFS failed_when: example_notebook_present.rc not in [0,1] ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs \ - -test -e /share/{{ item }} + -test -e /share/{{ item }} + changed_when: false register: example_notebook_present -- name: Copy example notebook to /share of HDFS +- name: Copy example notebook to /share of HDFS ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs \ - -copyFromLocal /tmp/{{ item }} /share/{{ item }} + -copyFromLocal /tmp/{{ item }} /share/{{ item }} + changed_when: true when: example_notebook_present.rc != 0 diff --git a/collections/ansible_collections/jupyter/lab/roles/setup/tasks/main.yml b/collections/ansible_collections/jupyter/lab/roles/setup/tasks/main.yml index 1a39f7a..33c2e11 100644 --- a/collections/ansible_collections/jupyter/lab/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/jupyter/lab/roles/setup/tasks/main.yml @@ -19,6 +19,7 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate conda create -n jupyterlab python={{ setup_environment_python_version }} -y + changed_when: true - name: Install jupyterlab conda packages in jupyterlab environment tags: @@ -26,6 +27,7 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterlab conda install -c conda-forge {% for conda_package in setup_conda_packages %}{{ conda_package }}{% if not loop.last %} {% endif %}{% endfor %} -y + changed_when: true args: executable: /bin/bash async: 6800 @@ -35,6 +37,7 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterlab pip install {% for pip_package in setup_pip_packages %}{{ pip_package }}{% if not loop.last %} {% endif %}{% endfor %} + changed_when: true args: executable: /bin/bash @@ -42,6 +45,7 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterlab python -m bash_kernel.install + changed_when: true args: executable: /bin/bash @@ -49,6 +53,7 @@ ansible.builtin.shell: | . /opt/miniforge/miniforge/bin/activate jupyterlab jupyter toree install --spark_home=/opt/apache-spark/spark --sys-prefix /opt/miniforge/miniforge/envs/jupyterlab + changed_when: true args: executable: /bin/bash @@ -61,9 +66,11 @@ - name: Change mode of environment - use shell module because of better performance ansible.builtin.command: chmod -R 755 {{ setup_miniforge_base_path }} + changed_when: true - name: Change owner to yarn user and group in environment - use shell module because of better performance ansible.builtin.command: chown -R {{ yarn_user }}:{{ yarn_group }} -R {{ setup_miniforge_base_path }} + changed_when: true - name: Create folders for environment variables in jupyterlab environment ansible.builtin.file: @@ -109,22 +116,28 @@ ansible.builtin.command: kinit \ -k {{ hdfs_user }}/{{ keytab_principal_host }}@{{ realm }} \ -t {{ keytab_folder }}/{{ hdfs_user }}.keytab + changed_when: false - name: Set ACL in root folder of HDFS ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs -setfacl -m "group:{{ ide_users_group }}:rwx" / + changed_when: true - name: Create HDFS folder /user ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs -mkdir -p /user + changed_when: true - name: Set ACL in /user folder of HDFS ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs -setfacl -m "group:{{ ide_users_group }}:rwx" /user + changed_when: true - name: Create HDFS folder /share ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs -mkdir -p /share + changed_when: true - name: Set ACL in /share folder of HDFS ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs dfs \ -setfacl -m "group:{{ ide_users_group }}:rwx,default:group:{{ ide_users_group }}:rwx" /share + changed_when: true - name: Copy example notebooks to /share folder of HDFS ansible.builtin.include_tasks: copy-example-notebooks.yml diff --git a/collections/ansible_collections/jupyter/lab/roles/setup/vars/main.yml b/collections/ansible_collections/jupyter/lab/roles/setup/vars/main.yml deleted file mode 100644 index e03cd3e..0000000 --- a/collections/ansible_collections/jupyter/lab/roles/setup/vars/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -miniforge_user: "{{ yarn_user }}" -miniforge_group: "{{ yarn_group }}" diff --git a/collections/ansible_collections/jupyter/miniforge/roles/install/tasks/main.yml b/collections/ansible_collections/jupyter/miniforge/roles/install/tasks/main.yml index b462ea7..b5c292e 100644 --- a/collections/ansible_collections/jupyter/miniforge/roles/install/tasks/main.yml +++ b/collections/ansible_collections/jupyter/miniforge/roles/install/tasks/main.yml @@ -13,10 +13,12 @@ - name: Install miniforge using miniforge installation script when not already present ansible.builtin.command: bash /var/tmp/{{ install_miniforge_version }}.sh -b -p /opt/miniforge/{{ install_miniforge_version }} + changed_when: true when: not miniforge_install.stat.exists - name: Change ownership to miniforge user and group ansible.builtin.command: chown -R {{ miniforge_user }}:{{ miniforge_group }} /opt/miniforge + changed_when: true when: not miniforge_install.stat.exists - name: Create symbolic link diff --git a/collections/ansible_collections/loadbalancing/haproxy/roles/setup/tasks/main.yml b/collections/ansible_collections/loadbalancing/haproxy/roles/setup/tasks/main.yml index cdec7f3..35d0bf1 100644 --- a/collections/ansible_collections/loadbalancing/haproxy/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/loadbalancing/haproxy/roles/setup/tasks/main.yml @@ -13,6 +13,7 @@ - name: Create haproxy.pem ansible.builtin.shell: cat /etc/ssl/private/cert.pem /etc/ssl/private/key.pem > /etc/ssl/private/haproxy.pem + changed_when: true - name: Enable haproxy service ansible.builtin.systemd: diff --git a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml index 7791038..50a832c 100644 --- a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml @@ -8,6 +8,7 @@ - name: Get ip address of remote host - use bash to avoid dns python conflict for dig lookup when: jupyterhub_domain_ip_address is not defined ansible.builtin.command: dig +short jupyterhub.{{ domain }} + changed_when: false register: get_jupyterhub_domain_ip_address - name: Configure keepalived for Hetzner Cloud diff --git a/collections/ansible_collections/rdbms/postgres/extensions/molecule/default/molecule.yml b/collections/ansible_collections/rdbms/postgres/extensions/molecule/default/molecule.yml index 9c72b04..9568886 100644 --- a/collections/ansible_collections/rdbms/postgres/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/rdbms/postgres/extensions/molecule/default/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: default + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -35,7 +48,7 @@ provisioner: - /opt/selfsigned/{{ ansible_fqdn }}/cert.pem - /opt/selfsigned/{{ ansible_fqdn }}/RootCA.pem - /opt/selfsigned/{{ ansible_fqdn }}/key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml index e8bea59..90c9505 100644 --- a/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml @@ -1,4 +1,17 @@ --- +scenario: + name: ha_setup + test_sequence: + - cleanup + - destroy + - syntax + - create + - prepare + - converge + - verify + - cleanup + - converge + - destroy dependency: name: galaxy options: @@ -61,6 +74,10 @@ provisioner: self_signed_certificates: true tls_external: false domain: example.com + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit loadbalancers: haproxy_admin_password: changeit haproxy_pem_file: /etc/ssl/private/haproxy.pem @@ -76,10 +93,6 @@ provisioner: service_gid: "4001" tls_user: zookeeper tls_group: zookeeper - keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks - keystore_password: changeit - truststore_file: /etc/ssl/certs/truststore.jks - truststore_password: changeit zookeeper1: zookeeper_id: 1 postgres: @@ -89,7 +102,7 @@ provisioner: - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/defaults/main.yml b/collections/ansible_collections/rdbms/postgres/roles/install/defaults/main.yml index 9365e03..23024f1 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/defaults/main.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/install/defaults/main.yml @@ -1,5 +1,5 @@ --- -postgresql_major_version: 17 +install_postgresql_major_version: 17 install_ssl_files: - "{{ chain_file_postgres }}" diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres-ha.yml b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres-ha.yml index 8acb25d..594721d 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres-ha.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres-ha.yml @@ -4,7 +4,7 @@ manager: auto - name: Install PostgreSQL - when: "'postgresql-{{ postgresql_major_version }}' not in ansible_facts.packages" + when: "'postgresql-17' not in ansible_facts.packages" block: - name: Install required packages ansible.builtin.apt: @@ -15,21 +15,21 @@ state: present update_cache: true - - name: Install PostgreSQL {{ postgresql_major_version }} key + - name: Install PostgreSQL key ansible.builtin.get_url: url: https://www.postgresql.org/media/keys/ACCC4CF8.asc dest: /etc/apt/trusted.gpg.d/postgresql.asc mode: "0644" - - name: Add PostgreSQL {{ postgresql_major_version }} repository + - name: Add PostgreSQL repository ansible.builtin.apt_repository: repo: deb [signed-by=/etc/apt/trusted.gpg.d/postgresql.asc] http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main state: present update_cache: true - - name: Install PostgreSQL {{ postgresql_major_version }} + - name: Install PostgreSQL ansible.builtin.apt: - name: postgresql-{{ postgresql_major_version }} + name: postgresql-{{ install_postgresql_major_version }} state: present update_cache: true @@ -48,11 +48,11 @@ - name: Install requirements for patroni in virtualenv ansible.builtin.pip: requirements: /tmp/requirements.txt - virtualenv: /usr/lib/postgresql/{{ postgresql_major_version }}/venv + virtualenv: /usr/lib/postgresql/{{ install_postgresql_major_version }}/venv - name: Change owner of virtualenv to postgres user ansible.builtin.file: - path: /usr/lib/postgresql/{{ postgresql_major_version }}/venv + path: /usr/lib/postgresql/{{ install_postgresql_major_version }}/venv state: directory owner: postgres group: postgres @@ -89,7 +89,7 @@ with_together: - "{{ certs_source }}" - "{{ certs_dest_postgres }}" - - "{{ certs_mode }}" + - "{{ distribute_certs_mode }}" loop_control: label: "{{ item.1 }}" when: (custom_inventory_file is defined and not custom_inventory_file) or (tls_external is defined and not tls_external) @@ -120,16 +120,16 @@ mode: "0644" - name: Delete present PostgreSQL data - when: "'postgresql-{{ postgresql_major_version }}' not in ansible_facts.packages" + when: "'postgresql-17' not in ansible_facts.packages" block: - name: Delete PostgreSQL data for Patroni to initialize ansible.builtin.file: - path: /var/lib/postgresql/{{ postgresql_major_version }}/main + path: /var/lib/postgresql/{{ install_postgresql_major_version }}/main state: absent - name: Create PostgreSQL data folder ansible.builtin.file: - path: /var/lib/postgresql/{{ postgresql_major_version }}/main + path: /var/lib/postgresql/{{ install_postgresql_major_version }}/main state: directory owner: postgres group: postgres diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml index 1a8adab..1f314c0 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml @@ -4,7 +4,7 @@ manager: auto - name: Install PostgreSQL - when: "'postgresql-{{ postgresql_major_version }}' not in ansible_facts.packages" + when: "'postgresql' not in ansible_facts.packages" block: - name: Install required packages ansible.builtin.apt: @@ -15,21 +15,21 @@ state: present update_cache: true - - name: Install PostgreSQL {{ postgresql_major_version }} key + - name: Install PostgreSQL key ansible.builtin.get_url: url: https://www.postgresql.org/media/keys/ACCC4CF8.asc dest: /etc/apt/trusted.gpg.d/postgresql.asc mode: "0644" - - name: Add PostgreSQL {{ postgresql_major_version }} repository + - name: Add PostgreSQL repository ansible.builtin.apt_repository: repo: deb [signed-by=/etc/apt/trusted.gpg.d/postgresql.asc] http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main state: present update_cache: true - - name: Install PostgreSQL {{ postgresql_major_version }} + - name: Install PostgreSQL ansible.builtin.apt: - name: postgresql-{{ postgresql_major_version }} + name: postgresql-{{ install_postgresql_major_version }} state: present update_cache: true diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.service b/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.service index 0a6c231..26d081b 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.service +++ b/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.service @@ -6,7 +6,7 @@ After=syslog.target network.target Type=simple User=postgres Group=postgres -ExecStart=/usr/lib/postgresql/{{ postgresql_major_version }}/venv/bin/patroni /etc/patroni.yml +ExecStart=/usr/lib/postgresql/{{ install_postgresql_major_version }}/venv/bin/patroni /etc/patroni.yml KillMode=process RestartSec=30 TimeoutSec=30 diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.yml b/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.yml index 38df380..38f84ce 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/install/templates/patroni.yml @@ -82,9 +82,9 @@ bootstrap: postgresql: listen: "{{ ansible_fqdn if molecule_deployment is defined and molecule_deployment else inventory_hostname }}:5432" connect_address: "{{ ansible_fqdn if molecule_deployment is defined and molecule_deployment else inventory_hostname }}:5432" - data_dir: "/var/lib/postgresql/{{ postgresql_major_version }}/main" - config_dir: "/etc/postgresql/{{ postgresql_major_version }}/main" - bin_dir: "/usr/lib/postgresql/{{ postgresql_major_version }}/bin" + data_dir: "/var/lib/postgresql/{{ install_postgresql_major_version }}/main" + config_dir: "/etc/postgresql/{{ install_postgresql_major_version }}/main" + bin_dir: "/usr/lib/postgresql/{{ install_postgresql_major_version }}/bin" authentication: replication: username: repl diff --git a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/create-user-and-schemes.yml b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/create-user-and-schemes.yml index fce7518..e1289a4 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/create-user-and-schemes.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/create-user-and-schemes.yml @@ -6,16 +6,18 @@ when: item.scheme is defined block: - name: Check if PostgreSQL is available - localhost - when: groups.postgres | length == 1 ansible.builtin.command: psql -c "SELECT 1" + when: groups.postgres | length == 1 + changed_when: false register: psql_check until: psql_check.rc == 0 retries: 10 delay: 5 - name: Check if PostgreSQL is available - Patroni Cluster - when: groups.postgres | length == 3 ansible.builtin.command: psql -h {{ postgres_host }} -c "SELECT 1" + when: groups.postgres | length == 3 + changed_when: false register: psql_check until: psql_check.rc == 0 retries: 10 diff --git a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml index 371c269..956f0ef 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml @@ -1,10 +1,11 @@ --- - name: Get ip address of remote host if jupyterhub_domain_ip_address is undefined - use bash to avoid dns python conflict for dig lookup + ansible.builtin.command: dig +short jupyterhub.{{ domain }} when: - jupyterhub_domain_ip_address is not defined - groups.hubs is defined - groups.hubs | length == 2 - ansible.builtin.command: dig +short jupyterhub.{{ domain }} + changed_when: false register: get_jupyterhub_domain_ip_address - name: Set fact if jupyterhub_domain_ip_address is undefined diff --git a/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml b/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml index 6f0bd6d..c8ae199 100644 --- a/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml +++ b/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml @@ -1,17 +1,14 @@ --- -self_signed_certificates: true +distribute_self_signed_certificates: true # Names of issued self signed certificate and key on remote hosts - used for Ansible Role tls.certs.distribute -certs_dest: +distribute_certs_dest: - "{{ ansible_fqdn }}.cert.pem" - "{{ ansible_fqdn }}.csr.pem" - "{{ ansible_fqdn }}.key" # Mode of issued self signed certificate and key on remote hosts - used for Ansible Role tls.certs.distribute -certs_mode: +distribute_certs_mode: - "0660" - "0660" - "0400" - -keystore_password: changeit -truststore_password: changeit diff --git a/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml b/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml index 87c1b2a..9fa095f 100644 --- a/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml +++ b/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml @@ -11,7 +11,7 @@ mode: "0770" - name: Change owner of certs and key on remote server - when: self_signed_certificates + when: distribute_self_signed_certificates ansible.builtin.file: path: "/etc/ssl/private/{{ item.0 }}" state: file @@ -19,13 +19,13 @@ group: "{{ tls_group }}" mode: "{{ item.1 }}" with_together: - - "{{ certs_dest }}" - - "{{ certs_mode }}" + - "{{ distribute_certs_dest }}" + - "{{ distribute_certs_mode }}" loop_control: label: "{{ item.0 }}" - name: Copy cert, fullchain and private key to server - when: not self_signed_certificates + when: not distribute_self_signed_certificates ansible.builtin.copy: src: "{{ item.0 }}" dest: /etc/ssl/private/{{ item.1 }} @@ -34,7 +34,7 @@ mode: "{{ item.2 }}" with_together: - "{{ certs_source }}" - - "{{ certs_dest }}" - - "{{ certs_mode }}" + - "{{ distribute_certs_dest }}" + - "{{ distribute_certs_mode }}" loop_control: label: "{{ item.1 }}" diff --git a/collections/ansible_collections/tls/certs/roles/issue/defaults/main.yml b/collections/ansible_collections/tls/certs/roles/issue/defaults/main.yml index 00588bf..ed80acb 100644 --- a/collections/ansible_collections/tls/certs/roles/issue/defaults/main.yml +++ b/collections/ansible_collections/tls/certs/roles/issue/defaults/main.yml @@ -1,2 +1,3 @@ --- -self_signed_certificates: true +issue_serial_number: 1000 +issue_self_signed_certificates: "{{ self_signed_certificates | default('true') }}" diff --git a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-certs-using-certbot.yml b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-certs-using-certbot.yml index 23f00b4..156ff2a 100644 --- a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-certs-using-certbot.yml +++ b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-certs-using-certbot.yml @@ -2,6 +2,7 @@ - name: Set fact for no root user - use command to avoid ansible_fact_caching for root user become: false ansible.builtin.command: whoami + changed_when: false register: no_root_user - name: Create folder for certificates and keys @@ -32,6 +33,7 @@ - name: Issue certificate and key using certbot when shared IP address for JupyterHub frontend is undefined ansible.builtin.command: certbot certonly --standalone --preferred-challenges http --agree-tos --email {{ my_email }} -d {{ inventory_hostname }} + changed_when: false when: jupyterhub_domain_ip is not defined - name: Issue certificate and key using certbot when shared IP address for JupyterHub frontend is defined @@ -70,6 +72,7 @@ --manual-cleanup-hook /usr/local/bin/certbot-hetzner-cleanup.sh \ -d {{ inventory_hostname }} -d jupyterhub.{{ domain }} executable: /bin/bash + changed_when: false - name: Fetch certificate and key to ansible server ansible.posix.synchronize: diff --git a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml index 69d15f6..c9dd539 100644 --- a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml +++ b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml @@ -37,8 +37,8 @@ - /tmp/selfsigned_certs/myCA/intermediateCA/index.txt - name: Create initial files - ansible.builtin.copy: - content: 1000 + ansible.builtin.template: + src: serialnumber.j2 dest: "{{ item }}" mode: "0644" loop: @@ -62,6 +62,7 @@ - name: Create a key - Root CA certificate args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl genpkey \ -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ -out rootCA.key @@ -70,6 +71,7 @@ - name: Create a self-signed certificate args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl req -config openssl_root.cnf \ -key rootCA.key -new -x509 -days 7300 \ -sha256 -extensions v3_ca -out myCA/rootCA/certs/ca.cert.pem \ @@ -78,6 +80,7 @@ - name: Create a key - Intermediate CA certificate args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl genpkey \ -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ -out intermediateCA.key @@ -85,6 +88,7 @@ - name: Create a certificate signing request (CSR) args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl req -config openssl_intermediate.cnf \ -key intermediateCA.key \ -new -sha256 \ @@ -95,6 +99,7 @@ - name: As root CA, sign the CSR for the intermediate CA with the root CA key args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl ca -batch -config openssl_root.cnf \ -keyfile rootCA.key \ -extensions v3_intermediate_ca -days 750 -notext -md sha256 \ @@ -104,12 +109,14 @@ - name: Verify the signature on the intermediate certificate args: chdir: /tmp/selfsigned_certs + changed_when: false ansible.builtin.command: openssl verify -CAfile myCA/rootCA/certs/ca.cert.pem \ myCA/intermediateCA/certs/intermediate.cert.pem - name: Create a key - Server certificate args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl genpkey \ -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ -out {{ ansible_fqdn }}.key @@ -117,6 +124,7 @@ - name: Create a certificate signing request (CSR) args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl req -copy_extensions=copyall \ -key {{ ansible_fqdn }}.key \ -new -sha256 \ @@ -127,6 +135,7 @@ - name: As intermediate CA, sign the server CSR with the intermediate CA key args: chdir: /tmp/selfsigned_certs + changed_when: true ansible.builtin.command: openssl ca -batch -config openssl_intermediate.cnf \ -extensions v3_server_cert \ -keyfile intermediateCA.key \ @@ -137,6 +146,7 @@ - name: Verify the certificate by checking the signatures using OpenSSL args: chdir: /tmp/selfsigned_certs + changed_when: false ansible.builtin.shell: | cat myCA/intermediateCA/certs/intermediate.cert.pem \ myCA/rootCA/certs/ca.cert.pem \ @@ -162,6 +172,7 @@ mode: "0644" - name: Update ca-certificates + changed_when: true ansible.builtin.command: update-ca-certificates - name: Change ownership of /etc/ssl/private diff --git a/collections/ansible_collections/tls/certs/roles/issue/tasks/main.yml b/collections/ansible_collections/tls/certs/roles/issue/tasks/main.yml index f99c1ed..35c3f76 100644 --- a/collections/ansible_collections/tls/certs/roles/issue/tasks/main.yml +++ b/collections/ansible_collections/tls/certs/roles/issue/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: Issue certificates using Certbot ansible.builtin.include_tasks: issue-certs-using-certbot.yml - when: not self_signed_certificates + when: not issue_self_signed_certificates - name: Issue self signed certificates ansible.builtin.include_tasks: issue-selfsigned-certs.yml - when: self_signed_certificates + when: issue_self_signed_certificates diff --git a/collections/ansible_collections/tls/certs/roles/issue/templates/serialnumber.j2 b/collections/ansible_collections/tls/certs/roles/issue/templates/serialnumber.j2 new file mode 100644 index 0000000..6a63c2a --- /dev/null +++ b/collections/ansible_collections/tls/certs/roles/issue/templates/serialnumber.j2 @@ -0,0 +1 @@ +{{ issue_serial_number }} diff --git a/collections/ansible_collections/tls/java/roles/keystore/tasks/main.yml b/collections/ansible_collections/tls/java/roles/keystore/tasks/main.yml index b8005af..0561768 100644 --- a/collections/ansible_collections/tls/java/roles/keystore/tasks/main.yml +++ b/collections/ansible_collections/tls/java/roles/keystore/tasks/main.yml @@ -16,27 +16,33 @@ - name: Generate PKCS 12 archive for building java keystore if CA name is Let's Encrypt when: not self_signed_certificates + changed_when: true ansible.builtin.command: openssl pkcs12 -export -in /etc/ssl/private/cert.pem \ -inkey /etc/ssl/private/key.pem \ -out /etc/ssl/private/{{ tls_hostname }}.p12 \ -name {{ tls_hostname }} \ -CAfile /etc/ssl/private/chain.pem \ -caname "Let's Encrypt Authority X3" -password pass:{{ keystore_password }} + - name: Generate PKCS 12 archive for building java keystore if CA name is not Let's Encrypt when: self_signed_certificates + changed_when: true ansible.builtin.command: openssl pkcs12 -export -in /etc/ssl/private/cert.pem \ -inkey /etc/ssl/private/key.pem \ -out /etc/ssl/private/{{ tls_hostname }}.p12 \ -name {{ tls_hostname }} \ -CAfile /etc/ssl/private/chain.pem \ -caname "Self Signed Authority X3" -password pass:{{ keystore_password }} + - name: Create java keystore + changed_when: true ansible.builtin.command: keytool -importkeystore -deststorepass {{ keystore_password }} \ -destkeypass {{ keystore_password }} -deststoretype pkcs12 \ -srckeystore /etc/ssl/private/{{ tls_hostname }}.p12 \ -srcstoretype PKCS12 -srcstorepass {{ keystore_password }} \ -destkeystore /etc/ssl/private/{{ tls_hostname }}.jks \ -alias {{ tls_hostname }} + - name: Give access to created java keystore ansible.builtin.file: path: "{{ item }}" diff --git a/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml b/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml index 275fc36..d6c116c 100644 --- a/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml +++ b/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml @@ -1,3 +1,3 @@ --- -local_cert_path: "{{ '/tmp/selfsigned_certs' if self_signed_certificates is defined and self_signed_certificates == true else '/opt/letsencrypt' }}" -cert_name: "{{ 'cert.pem' if self_signed_certificates is defined and self_signed_certificates == true else 'cert1.pem' }}" +truststore_local_cert_path: "{{ '/tmp/selfsigned_certs' if self_signed_certificates is defined and self_signed_certificates == true else '/opt/letsencrypt' }}" +truststore_cert_name: "{{ 'cert.pem' if self_signed_certificates is defined and self_signed_certificates == true else 'cert1.pem' }}" diff --git a/collections/ansible_collections/tls/java/roles/truststore/tasks/main.yml b/collections/ansible_collections/tls/java/roles/truststore/tasks/main.yml index ca72f8a..3b6ee80 100644 --- a/collections/ansible_collections/tls/java/roles/truststore/tasks/main.yml +++ b/collections/ansible_collections/tls/java/roles/truststore/tasks/main.yml @@ -16,7 +16,8 @@ - name: Copy temporary all certs of hosts to /var/tmp of each host ansible.builtin.copy: - src: "{{ local_cert_path }}/{{ item + '.' + domain if molecule_deployment is defined and molecule_deployment else item }}/{{ cert_name }}" + src: "{{ truststore_local_cert_path }}/{{ item + '.' + domain if molecule_deployment is defined and molecule_deployment else + item }}/{{ truststore_cert_name }}" dest: /var/tmp/{{ item }}_cert.pem mode: "0644" loop: "{{ groups.all }}" @@ -26,6 +27,7 @@ -alias {{ item + '.' + domain if molecule_deployment is defined and molecule_deployment else item }} \ -keystore /etc/ssl/certs/truststore.jks \ -file /var/tmp/{{ item }}_cert.pem -storepass {{ truststore_password }} -noprompt + changed_when: true loop: "{{ groups.all }}" - name: Import chain.pem into /etc/ssl/certs/truststore.jks @@ -34,6 +36,8 @@ -keystore /etc/ssl/certs/truststore.jks \ -file /etc/ssl/private/chain.pem \ -storepass {{ truststore_password }} -noprompt + changed_when: true + - name: Give access to created Java truststore ansible.builtin.file: path: /etc/ssl/certs/truststore.jks diff --git a/roles/requirements.yml b/roles/requirements.yml deleted file mode 100644 index 2f4c5e6..0000000 --- a/roles/requirements.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- src: git@github.com:GeorgSchulz/CommunityLab.git - scm: git - version: master - name: CommunityLab diff --git a/setup.yml b/setup.yml index deb6355..d9acc59 100644 --- a/setup.yml +++ b/setup.yml @@ -26,6 +26,7 @@ tasks: - name: Set hostnames and fqdn for all hosts on global level ansible.builtin.command: hostnamectl set-hostname {{ inventory_hostname }} + changed_when: true when: - custom_inventory_file is defined - not custom_inventory_file diff --git a/terraform/inventory_ha_ide.tpl b/terraform/inventory_ha_ide.tpl index e88f9c3..43ccae9 100644 --- a/terraform/inventory_ha_ide.tpl +++ b/terraform/inventory_ha_ide.tpl @@ -53,6 +53,13 @@ all: tls_user: "{{ jupyterhub_user }}" tls_group: "{{ jupyterhub_group }}" keytab_group: "{{ jupyterhub_group }}" + miniforge_user: "{{ jupyterhub_user }}" + miniforge_group: "{{ jupyterhub_group }}" + hadoop_client_user: "{{ jupyterhub_user }}" + hadoop_client_group: "{{ jupyterhub_group }}" + kerberos_keytabs: + - principal: "{{ jupyterhub_user }}" + keytab_user: "{{ jupyterhub_user }}" loadbalancers: children: hub1: @@ -98,6 +105,10 @@ all: resourcemanager1: resourcemanager2: resourcemanager3: + vars: + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" zookeeper: children: master1: @@ -121,6 +132,10 @@ all: worker1: worker2: worker3: + vars: + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" spark: children: worker1: @@ -158,6 +173,9 @@ all: worker1: worker2: worker3: + vars: + miniforge_user: "{{ yarn_user }}" + miniforge_group: "{{ yarn_group }}" ldap1: children: security1: @@ -208,11 +226,11 @@ all: - "/opt/letsencrypt/{{ inventory_hostname }}/cert1.pem" - "/opt/letsencrypt/{{ inventory_hostname }}/chain1.pem" - "/opt/letsencrypt/{{ inventory_hostname }}/privkey1.pem" - certs_dest: + distribute_certs_dest: - "cert.pem" - "chain.pem" - "key.pem" - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" @@ -247,3 +265,4 @@ all: hadoop_nameservice: "communitylab" jupyterhub_domain_ip: true postgres_host: "{{ jupyterhub_domain_ip_address }}" + hdfs_data_dir: /var/hadoop/hdfs diff --git a/terraform/inventory_non_ha_ide.tpl b/terraform/inventory_non_ha_ide.tpl index 2ffdb24..ba191d1 100644 --- a/terraform/inventory_non_ha_ide.tpl +++ b/terraform/inventory_non_ha_ide.tpl @@ -26,6 +26,13 @@ all: tls_user: "{{ jupyterhub_user }}" tls_group: "{{ jupyterhub_group }}" keytab_group: "{{ jupyterhub_group }}" + miniforge_user: "{{ jupyterhub_user }}" + miniforge_group: "{{ jupyterhub_group }}" + hadoop_client_user: "{{ jupyterhub_user }}" + hadoop_client_group: "{{ jupyterhub_group }}" + kerberos_keytabs: + - principal: "{{ jupyterhub_user }}" + keytab_user: "{{ jupyterhub_user }}" postgres: children: hub1: @@ -57,6 +64,10 @@ all: resourcemanagers: children: resourcemanager1: + vars: + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" zookeeper: children: master1: @@ -73,11 +84,16 @@ all: service_gid: "4001" tls_user: "{{ yarn_user }}" tls_group: "{{ yarn_group }}" + keytab_user: "{{ hdfs_user }}" nodemanagers: children: worker1: worker2: worker3: + vars: + kerberos_keytabs: + - principal: "{{ yarn_user }}" + keytab_user: "{{ yarn_user }}" spark: children: worker1: @@ -88,6 +104,9 @@ all: worker1: worker2: worker3: + vars: + miniforge_user: "{{ yarn_user }}" + miniforge_group: "{{ yarn_group }}" ldap1: children: security1: @@ -130,11 +149,11 @@ all: - "/opt/letsencrypt/{{ inventory_hostname }}/cert1.pem" - "/opt/letsencrypt/{{ inventory_hostname }}/chain1.pem" - "/opt/letsencrypt/{{ inventory_hostname }}/privkey1.pem" - certs_dest: + distribute_certs_dest: - "cert.pem" - "chain.pem" - "key.pem" - certs_mode: + distribute_certs_mode: - "0660" - "0660" - "0400" @@ -163,3 +182,4 @@ all: keytab_folder: "/etc/keytabs" hadoop_nameservice: "communitylab" postgres_host: "" + hdfs_data_dir: /var/hadoop/hdfs