Explicitly set update_domains and fault_domains to match prod VMs on …

Jenkinsfile_CNP

@@ -82,11 +82,11 @@ def runAnsible() {
         """
 
         // for (t in targets) {
-            def t = 'demo_int'
+            def t = 'demo'
             def clusterName = "ccd-elastic-search-${t.replace('_','-')}"
-            def rollingUpgrade = (t == 'demo_int' && params.ROLLING_UPGRADE) ? "true" : "false"
-            def reindexFlag    = (t == 'demo_int' && params.REINDEX) ? "true" : "false"
-            def upgradeFlag    = (t == 'demo_int' && params.UPGRADE_ES) ? "true" : "false"
+            def rollingUpgrade = (t == 'demo' && params.ROLLING_UPGRADE) ? "true" : "false"
+            def reindexFlag    = (t == 'demo' && params.REINDEX) ? "true" : "false"
+            def upgradeFlag    = (t == 'demo' && params.UPGRADE_ES) ? "true" : "false"
 
             sh """
                 ANSIBLE_HOST_KEY_CHECKING=False /tmp/ansible-venv/bin/ansible-playbook \

ansible/elasticsearch.yml.j2

@@ -2,20 +2,28 @@
 cluster.name: "{{ elastic_clustername }}"
 node.name: "{{ ansible_hostname }}"
 path.logs: /var/log/elasticsearch
-path.data: /datadisks/disk1/elasticsearch/data
-discovery.seed_hosts: ["ccd-data-0", "ccd-data-1", "ccd-data-2", "ccd-data-3"]
-cluster.initial_master_nodes: ["ccd-data-0", "ccd-data-1", "ccd-data-2", "ccd-data-3"]
-node.master: true
-node.data: true
+path.data: {{ es_data_path }}
+
+# Discovery settings
+discovery.seed_hosts: {{ groups[group_names[0]] | map('extract', hostvars, 'ansible_hostname') | list | to_json }}
+{% if bootstrap %}
+cluster.initial_master_nodes: {{ groups[group_names[0]] | map('extract', hostvars, 'ansible_hostname') | list | to_json }}
+{% endif %}
+
+node.roles: [master, data, ingest]
+
 network.host: [_site_, _local_]
-node.max_local_storage_nodes: 1
-node.attr.fault_domain: 1
-node.attr.update_domain: 1
+
+# Availability zone awareness
+node.attr.fault_domain: {{ fault_domain }}
+node.attr.update_domain: {{ update_domain }}
 cluster.routing.allocation.awareness.attributes: fault_domain,update_domain
+
 action.auto_create_index: .security*,.monitoring*,.kibana*,.watches,.triggered_watches,.watcher-history*,.logstash_dead_letter,.ml*
-xpack.monitoring.collection.enabled: true
 xpack.security.enabled: false
+
 script.allowed_types: inline
-script.allowed_contexts: template, ingest
+script.allowed_contexts: template, ingest, watcher_condition
+
 ingest.geoip.downloader.enabled: false
-bootstrap.memory_lock: true
+bootstrap.memory_lock: true

ansible/group_vars/demo_int.yml

@@ -1,3 +1 @@
-elasticsearch_version: 7.17.28
-
-elasticsearch_config: "elasticsearch-demo-int.yml.j2"
+elasticsearch_version: 9.2.2

ansible/group_vars/demo_int2.yml

@@ -1,3 +1 @@
-elasticsearch_version: 9.2.2
-
-elasticsearch_config: "elasticsearch-demo-int.yml.j2"
+elasticsearch_version: 9.2.2

ansible/host_vars/ccd-data-0.yml

@@ -0,0 +1,2 @@
+fault_domain: 1
+update_domain: 2

ansible/host_vars/ccd-data-1.yml

@@ -0,0 +1,2 @@
+fault_domain: 0
+update_domain: 0

ansible/host_vars/ccd-data-2.yml

@@ -0,0 +1,2 @@
+fault_domain: 1
+update_domain: 1

ansible/host_vars/ccd-data-3.yml

@@ -0,0 +1,2 @@
+fault_domain: 0
+update_domain: 3

ansible/roles/elasticsearch/defaults/main.yml

@@ -1,7 +1,7 @@
 # Override this variable in ansible/group_vars/group_name.yml file
 # where group_name is the inventory group to set different version
 # on different environments
-elasticsearch_version: 7.17.28
+elasticsearch_version: 9.2.4
 
 
 # Elasticsearch default vars

ansible/roles/elasticsearch/filter_plugins/es_filters.py

@@ -35,6 +35,14 @@ def filter_readonly_parts(index_def):
     index.pop("provided_name", None)
     index.pop("blocks", None)
     stripped.pop("aliases", None)
+
+    mappings = stripped.get("mappings", {})
+    props = mappings.get("properties", {})
+    if "@timestamp" in props:
+        props["@timestamp"] = {
+            "type": "date",
+            "ignore_malformed": True,
+        }
     return stripped
 
 

ansible/roles/elasticsearch/tasks/1_install.yml

@@ -7,15 +7,6 @@
         {{ elasticsearch_package }}
       {%- endif -%}
 
-- name: Determine Elasticsearch config template based on cluster type
-  ansible.builtin.set_fact:
-    es_config_template: >-
-      {%- if group_names | select('search', '_upgrade$') | list | length > 0 -%}
-        elasticsearch-demo-int.yml.j2
-      {%- else -%}
-        {{ elasticsearch_config }}
-      {%- endif -%}
-
 - name: Download Elasticsearch package
   ansible.builtin.get_url:
     url: "{{ elasticsearch_url_base }}/{{ elasticsearch_package }}"
@@ -24,13 +15,21 @@
     checksum: "sha512:{{ lookup('url', checksum_url).split()[0] }}"
   check_mode: no
 
+- name: Create Elasticsearch config directory
+  ansible.builtin.file:
+    path: /etc/elasticsearch/jvm.options.d
+    state: directory
+    owner: root
+    group: elasticsearch
+    mode: "2750"
+
 - name: Configure Elasticsearch settings
   ansible.builtin.template:
-    src: "{{ es_config_template }}"
+    src: "{{ elasticsearch_config }}"
     dest: "/etc/elasticsearch/elasticsearch.yml"
     mode: "0644"
     owner: root
-    group: root
+    group: elasticsearch
   register: es_config
 
 - name: Create systemd override directory for Elasticsearch
@@ -57,6 +56,13 @@
   when: not ansible_check_mode
   register: es_jvm_opts
 
+- name: Remove java security manager options
+  ansible.builtin.lineinfile:
+    path: /etc/elasticsearch/jvm.options
+    regexp: '^-Djava\.security\.(manager|policy)'
+    state: absent
+    backup: yes
+
 - name: Disable shard allocation
   ansible.builtin.uri:
     url: "http://localhost:9200/_cluster/settings"
@@ -103,7 +109,6 @@
     daemon_reload: yes
   when:
     - not ansible_check_mode
-    - es_systemd_override is changed
 
 - name: Restart Elasticsearch to apply changes
   ansible.builtin.systemd:

ansible/roles/elasticsearch/tasks/process_data_streams.yml

@@ -8,8 +8,10 @@
   vars:
     index_name: "{{ backing_indices.json[-1].index }}"
 
+# !!! can potentially timeout
 - name: Rollover DS streams
   uri:
     url: "{{ es_url }}/{{ ds_index }}/_rollover"
     method: POST
+    timeout: 3600
   when: index_version < elasticsearch_major_version

ansible/roles/elasticsearch/tasks/process_index.yml

@@ -2,6 +2,7 @@
 - include_tasks: get_index_version.yml
 
 - name: Process index only if upgrade needed
+  when: index_version < elasticsearch_major_version
   block:
 
     ## Make readonly for cloning
@@ -25,9 +26,18 @@
         - not index_name.endswith(clone_suffix)
 
 
+    # !!! can potentially timeout
     - name: Wait for cloned index health to be green
       uri:
-        url: "{{ es_url }}/_cluster/health/{{ index_name }}{{ clone_suffix }}?wait_for_status=green&timeout=120s"
+        url: "{{ es_url }}/_cluster/health/{{ index_name }}{{ clone_suffix }}?format=json"
+        timeout: 30
+        return_content: yes
+      register: cloned_health
+      until: >
+        cloned_health.json.status == 'green'
+        and (cloned_health.json.unassigned_shards | default(0) == 0)
+      retries: 60
+      delay: 30
       when:
         - not index_name.endswith(clone_suffix)
 
@@ -77,11 +87,12 @@
         body: "{{ cloned_index_def.json | filter_readonly_parts }}"
         status_code: [200]
 
-
+    # !!! can potentially timeout
     - name: Reindex from cloned index back into original
       uri:
         url: "{{ es_url }}/_reindex?wait_for_completion=true"
         method: POST
+        timeout: 3600
         body_format: json
         body:
           source:
@@ -118,11 +129,12 @@
 
 
     ## Optimise user indices
-    - name: Force merge user indices to 3 segment (optional)
-      uri:
-        url: "{{ es_url }}/{{ index_name }}/_forcemerge?max_num_segments=3"
-        method: POST
-      when: do_forcemerge
+    # usually takes a long time and very IO heavy, so skip
+    # - name: Force merge user indices to 3 segment (optional)
+    #   uri:
+    #     url: "{{ es_url }}/{{ index_name }}/_forcemerge?max_num_segments=3"
+    #     method: POST
+    #   when: do_forcemerge
 
 
     ## Delete cloned index
@@ -132,4 +144,3 @@
         method: DELETE
         status_code: [200, 404]  # allow success if index doesn't exist
       when: delete_clones
-  when: index_version < elasticsearch_major_version