diff --git a/.circleci/ansible-vars-ci.yml b/.circleci/ansible-vars-ci.yml index aed8ee6e..3f42f390 100644 --- a/.circleci/ansible-vars-ci.yml +++ b/.circleci/ansible-vars-ci.yml @@ -5,11 +5,6 @@ fail2ban_enable: false enable_aws_checks: false service_mgr: sysvinit node_exporter_enabled: false -datadog_install: false -datadog_enabled: false -datadog_skip_running_check: true -datadog_api_key: "" -datadog_app_key: "" project_root: /root/node packages_path: /tmp diff --git a/.dockerignore b/.dockerignore index 46aec117..193e117d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,7 +12,6 @@ # Copy from .gitignore *.retry ansible/inventory/*.rc.sh -ansible/callback_plugins/datadog_callback.yml .virtualenv/ .DS_Store *.pyc diff --git a/.gitignore b/.gitignore index 87cea45e..271896b7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ *.retry ansible/group_vars/**/*.local.* ansible/inventory/*.rc.sh -ansible/callback_plugins/datadog_callback.yml .virtualenv/ .DS_Store *.pyc diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 49404c9d..1b363ce5 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,6 +1,6 @@ [defaults] stdout_callback = yaml -callbacks_enabled = profile_tasks,datadog +callbacks_enabled = profile_tasks host_key_checking = False inventory = inventory/ timeout = 30 diff --git a/ansible/callback_plugins/datadog.py b/ansible/callback_plugins/datadog.py deleted file mode 100644 index 81c19ceb..00000000 --- a/ansible/callback_plugins/datadog.py +++ /dev/null @@ -1,370 +0,0 @@ -# Source: https://github.com/DataDog/ansible-datadog-callback/commit/6587078da426b6383675dd83f57cf82d2c79bae8 - -from __future__ import absolute_import, print_function - -import getpass -import logging -import os -import time - -IMPORT_ERROR = None -try: - import datadog - import yaml - from packaging import version -except ImportError as e: - IMPORT_ERROR = str(e) - - -import ansible -from ansible.plugins.callback import CallbackBase -try: - from __main__ import cli -except ImportError: - cli = False - -ANSIBLE_ABOVE_28 = False -if IMPORT_ERROR is None and version.parse(ansible.__version__) >= version.parse('2.8.0'): - ANSIBLE_ABOVE_28 = True - from ansible.context import CLIARGS - -DEFAULT_DD_URL = "https://api.datadoghq.com" - -class CallbackModule(CallbackBase): - def __init__(self): - if IMPORT_ERROR is not None: - self.disabled = True - print( - 'Datadog callback disabled because of a dependency problem: {}. ' - 'Please install requirements with "pip install -r requirements.txt"' - .format(IMPORT_ERROR) - ) - else: - self.disabled = False - # Set logger level - datadog api and urllib3 - for log_name in ['requests.packages.urllib3', 'datadog.api']: - self._set_logger_level(log_name) - - self._playbook_name = None - self._start_time = time.time() - self._options = None - if IMPORT_ERROR is None: - if ANSIBLE_ABOVE_28: - self._options = CLIARGS - elif cli: - self._options = cli.options - - # self.playbook is set in the `v2_playbook_on_start` callback method - self.playbook = None - # self.play is set in the `playbook_on_play_start` callback method - self.play = None - - # Set logger level - def _set_logger_level(self, name, level=logging.WARNING): - try: - log = logging.getLogger(name) - log.setLevel(level) - log.propagate = False - except Exception as e: - # We don't want Ansible to fail on an API error - print("Couldn't get logger - %s" % name) - print(e) - - # Load parameters from conf file - def _load_conf(self, file_path): - conf_dict = {} - if os.path.isfile(file_path): - try: - loader = yaml.FullLoader - except AttributeError: - # on pyyaml < 5.1, there's no FullLoader, - # but we can still use SafeLoader - loader = yaml.SafeLoader - with open(file_path, 'r') as conf_file: - conf_dict = yaml.load(conf_file, Loader=loader) - - api_key = os.environ.get('DATADOG_API_KEY', conf_dict.get('api_key', '')) - dd_url = os.environ.get('DATADOG_URL', conf_dict.get('url', '')) - dd_site = os.environ.get('DATADOG_SITE', conf_dict.get('site', '')) - return api_key, dd_url, dd_site - - # Send event to Datadog - def _send_event(self, title, alert_type=None, text=None, tags=None, host=None, event_type=None, event_object=None): - if tags is None: - tags = [] - tags.extend(self.default_tags) - priority = 'normal' if alert_type == 'error' else 'low' - try: - datadog.api.Event.create( - title=title, - text=text.replace('@','(@)'), # avoid notifying @ mentions - alert_type=alert_type, - priority=priority, - tags=tags, - host=host, - source_type_name='ansible', - event_type=event_type, - event_object=event_object, - ) - except Exception as e: - # We don't want Ansible to fail on an API error - print('Couldn\'t send event "{0}" to Datadog'.format(title)) - print(e) - - # Send event, aggregated with other task-level events from the same host - def send_task_event(self, title, alert_type='info', text='', tags=None, host=None): - if getattr(self, 'play', None): - if tags is None: - tags = [] - tags.append('play:{0}'.format(self.play.name)) - self._send_event( - title, - alert_type=alert_type, - text=text, - tags=tags, - host=host, - event_type='config_management.task', - event_object=host, - ) - - # Send event, aggregated with other playbook-level events from the same playbook and of the same type - def send_playbook_event(self, title, alert_type='info', text='', tags=None, event_type=''): - self._send_event( - title, - alert_type=alert_type, - text=text, - tags=tags, - event_type='config_management.run.{0}'.format(event_type), - event_object=self._playbook_name, - ) - - # Send ansible metric to Datadog - def send_metric(self, metric, value, tags=None, host=None): - if tags is None: - tags = [] - tags.extend(self.default_tags) - try: - datadog.api.Metric.send( - metric="ansible.{0}".format(metric), - points=value, - tags=tags, - host=host, - ) - except Exception as e: - # We don't want Ansible to fail on an API error - print('Couldn\'t send metric "{0}" to Datadog'.format(metric)) - print(e) - - # Start timer to measure playbook running time - def start_timer(self): - self._start_time = time.time() - - # Get the time elapsed since the timer was started - def get_elapsed_time(self): - return time.time() - self._start_time - - # Default tags sent with events and metrics - @property - def default_tags(self): - return ['playbook:{0}'.format(self._playbook_name)] - - @staticmethod - def pluralize(number, noun): - if number == 1: - return "{0} {1}".format(number, noun) - - return "{0} {1}s".format(number, noun) - - # format helper for event_text - @staticmethod - def format_result(res): - msg = "$$$\n{0}\n$$$\n".format(res['msg']) if res.get('msg') else "" - module_name = 'undefined' - - if res.get('censored'): - event_text = res.get('censored') - elif not res.get('invocation'): - event_text = msg - else: - invocation = res['invocation'] - module_name = invocation.get('module_name', 'undefined') - event_text = "$$$\n{0}[{1}]\n$$$\n".format(module_name, invocation.get('module_args', '')) - event_text += msg - if 'module_stdout' in res: - # On Ansible v2, details on internal failures of modules are not reported in the `msg`, - # so we have to extract the info differently - event_text += "$$$\n{0}\n{1}\n$$$\n".format( - res.get('module_stdout', ''), res.get('module_stderr', '')) - - module_name_tag = 'module:{0}'.format(module_name) - - return event_text, module_name_tag - - def get_dd_hostname(self, ansible_hostname): - """ This function allows providing custom logic that transforms an Ansible - inventory hostname to a Datadog hostname. - """ - dd_hostname = ansible_hostname - # provide your code to obtain Datadog hostname from Ansible inventory hostname - return dd_hostname - - ### Ansible callbacks ### - def runner_on_failed(self, host, res, ignore_errors=False): - host = self.get_dd_hostname(host) - # don't post anything if user asked to ignore errors - if ignore_errors: - return - - event_text, module_name_tag = self.format_result(res) - self.send_task_event( - 'Ansible task failed on "{0}"'.format(host), - alert_type='error', - text=event_text, - tags=[module_name_tag], - host=host, - ) - - def runner_on_ok(self, host, res): - host = self.get_dd_hostname(host) - # Only send an event when the task has changed on the host - if res.get('changed'): - event_text, module_name_tag = self.format_result(res) - self.send_task_event( - 'Ansible task changed on "{0}"'.format(host), - alert_type='success', - text=event_text, - tags=[module_name_tag], - host=host, - ) - - def runner_on_unreachable(self, host, res): - host = self.get_dd_hostname(host) - event_text = "\n$$$\n{0}\n$$$\n".format(res) - self.send_task_event( - 'Ansible failed on unreachable host "{0}"'.format(host), - alert_type='error', - text=event_text, - host=host, - ) - - # Implementation compatible with Ansible v2 only - def v2_playbook_on_start(self, playbook): - # On Ansible v2, Ansible doesn't set `self.playbook` automatically - self.playbook = playbook - - playbook_file_name = self.playbook._file_name - if ANSIBLE_ABOVE_28: - inventory = self._options['inventory'] - else: - inventory = self._options.inventory - - self.start_timer() - - # Set the playbook name from its filename - self._playbook_name, _ = os.path.splitext( - os.path.basename(playbook_file_name)) - if isinstance(inventory, (list, tuple)): - inventory = ','.join(inventory) - self._inventory_name = ','.join([os.path.basename(os.path.realpath(name)) for name in inventory.split(',') if name]) - - def v2_playbook_on_play_start(self, play): - # On Ansible v2, Ansible doesn't set `self.play` automatically - self.play = play - if self.disabled: - return - - # Read config and hostvars - config_path = os.environ.get('ANSIBLE_DATADOG_CALLBACK_CONF_FILE', os.path.join(os.path.dirname(__file__), "datadog_callback.yml")) - api_key, dd_url, dd_site = self._load_conf(config_path) - - # If there is no api key defined in config file, try to get it from hostvars - if api_key == '': - hostvars = self.play.get_variable_manager()._hostvars - - if not hostvars: - print("No api_key found in the config file ({0}) and hostvars aren't set: disabling Datadog callback plugin".format(config_path)) - self.disabled = True - else: - try: - api_key = hostvars['localhost']['datadog_api_key'] - if not dd_url: - dd_url = hostvars['localhost'].get('datadog_url') - if not dd_site: - dd_site = hostvars['localhost'].get('datadog_site') - except Exception as e: - print('No "api_key" found in the config file ({0}) and "datadog_api_key" is not set in the hostvars: disabling Datadog callback plugin'.format(config_path)) - self.disabled = True - - if not dd_url: - if dd_site: - dd_url = "https://api."+ dd_site - else: - dd_url = DEFAULT_DD_URL # default to Datadog US - - # Set up API client and send a start event - if not self.disabled: - datadog.initialize(api_key=str(api_key), api_host=dd_url) - - self.send_playbook_event( - 'Ansible play "{0}" started in playbook "{1}" by "{2}" against "{3}"'.format( - self.play.name, - self._playbook_name, - getpass.getuser(), - self._inventory_name), - event_type='start', - ) - - def playbook_on_stats(self, stats): - total_tasks = 0 - total_updated = 0 - total_errors = 0 - error_hosts = [] - for host in stats.processed: - host = self.get_dd_hostname(host) - # Aggregations for the event text - summary = stats.summarize(host) - total_tasks += sum([summary['ok'], summary['failures'], summary['skipped']]) - total_updated += summary['changed'] - errors = sum([summary['failures'], summary['unreachable']]) - if errors > 0: - error_hosts.append((host, summary['failures'], summary['unreachable'])) - total_errors += errors - - # Send metrics for this host - for metric, value in summary.items(): - self.send_metric('task.{0}'.format(metric), value, host=host) - - # Send playbook elapsed time - self.send_metric('elapsed_time', self.get_elapsed_time()) - - # Generate basic "Completed" event - event_title = 'Ansible playbook "{0}" completed in {1}'.format( - self._playbook_name, - self.pluralize(int(self.get_elapsed_time()), 'second')) - event_text = 'Ansible updated {0} out of {1} total, on {2}. {3} occurred.'.format( - self.pluralize(total_updated, 'task'), - self.pluralize(total_tasks, 'task'), - self.pluralize(len(stats.processed), 'host'), - self.pluralize(total_errors, 'error')) - alert_type = 'success' - - # Add info to event if errors occurred - if total_errors > 0: - alert_type = 'error' - event_title += ' with errors' - event_text += "\nErrors occurred on the following hosts:\n%%%\n" - for host, failures, unreachable in error_hosts: - event_text += "- `{0}` (failure: {1}, unreachable: {2})\n".format( - host, - failures, - unreachable) - event_text += "\n%%%\n" - else: - event_title += ' successfully' - - self.send_playbook_event( - event_title, - alert_type=alert_type, - text=event_text, - event_type='end', - ) diff --git a/ansible/deploy-aemdw.yml b/ansible/deploy-aemdw.yml index a8bb9d1a..364b416f 100644 --- a/ansible/deploy-aemdw.yml +++ b/ansible/deploy-aemdw.yml @@ -11,10 +11,6 @@ vars: public_ipv4: "{{ ansible_ec2_public_ipv4|default(ansible_ssh_host) | default(ansible_host)|default(inventory_hostname) }}" aemdw_project_root: "/home/{{ ansible_user_id }}/aemdw" - datadog_api_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:api_key') }}" - datadog_app_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:app_key') }}" - datadog_api_url: https://api.datadoghq.com/api/v1/downtime?api_key={{ datadog_api_key }}&application_key={{ datadog_app_key }} - downtime: "0" enable_aws_checks: true health_check: true healthcheck_port: 3013 @@ -25,7 +21,7 @@ docker_tag: "latest" accounts_path: /home/aeternity/node/data/aecore/.genesis/accounts_test.json contracts_path: /home/aeternity/node/data/aecore/.genesis/contacts_test.json - # Label used for datadog tagging and ENV file planting + # Label used for ENV file planting env: unknown tasks: @@ -55,23 +51,6 @@ - name: Deploy block: - - name: Schedule downtime - run_once: true - ansible.builtin.uri: - url: "https://api.datadoghq.com/api/v1/downtime?api_key={{ datadog_api_key }}&application_key={{ datadog_app_key }}" - method: POST - status_code: 200 - timeout: 10 - body_format: json - body: - scope: "role:aemdw,env:{{ env }}" - message: "Deploy aemdw - {{ docker_repository }}:{{ docker_tag }}" - end: "{{ lookup('pipe', 'date +%s') | int + downtime | int }}" - register: datadog_downtime - when: - - downtime is defined - - downtime != "0" - - name: Make sure 'aemdw_project_root' exists ansible.builtin.file: path: "{{ aemdw_project_root }}" @@ -164,50 +143,7 @@ - name: Run node health checks ansible.builtin.include_tasks: tasks/health_check.yml - - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ec2_instance_id | default(ansible_hostname) }}" - alert_type: success - title: Middleware Deployment - text: | - %%% - Version: [{{ docker_tag }}](https://github.com/aeternity/ae_mdw/releases/tag/v{{ docker_tag }}) - %%% - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "version:{{ docker_tag }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - become: false - connection: local - tags: [datadog-event] rescue: - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ec2_instance_id | default(ansible_hostname) }}" - alert_type: error - title: Middleware Deployment - text: | - %%% - Version: [{{ docker_tag }}](https://github.com/aeternity/ae_mdw/releases/tag/v{{ docker_tag }}) - - Task: {{ ansible_failed_task.name }}\ - Error Message:\ - {{ ansible_failed_result.msg }} - %%% - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "version:{{ docker_tag }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - connection: local - become: false - tags: [datadog-event] - name: Fail the playbook ansible.builtin.fail: msg: "Failed deploy" diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 8140a62a..b7f2e553 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -11,17 +11,13 @@ public_ipv4: "{{ ansible_ec2_public_ipv4|default(ansible_ssh_host)|default(ansible_host)|default(inventory_hostname) }}" project_root: "/home/aeternity/node" packages_path: "/home/aeternity" - datadog_api_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:api_key') }}" - datadog_app_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:app_key') }}" - datadog_api_url: https://api.datadoghq.com/api/v1/downtime?api_key={{ datadog_api_key }}&application_key={{ datadog_app_key }} - downtime: "0" health_check: true healthcheck_port: 3013 enable_aws_checks: true db_path: data/ keys_dir: keys/ - # Label used for datadog tagging and ENV file planting + # Label used ENV file planting env: unknown tasks: @@ -51,22 +47,6 @@ - name: Deploy block: - - name: Schedule downtime - run_once: true - ansible.builtin.uri: - url: "{{ datadog_api_url }}" - method: POST - status_code: 200 - timeout: 10 - body_format: json - body: - scope: "env:{{ env }}" - message: "Deploy" - end: "{{ lookup('pipe', 'date +%s') | int + downtime | int }}" - when: - - downtime is defined - - downtime != "0" - - name: Fail on missing package variable ansible.builtin.fail: msg: '"package" variable must be defined' @@ -210,51 +190,7 @@ - name: Run health checks ansible.builtin.include_tasks: tasks/health_check.yml - - - name: Refresh aeternity facts - ansible.builtin.setup: - tags: [datadog-event] - - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ec2_instance_id | default(ansible_hostname) }}" - alert_type: success - title: Node Deployment - text: | - %%% - Revision: [{{ ansible_local.aeternity.revision }}](https://github.com/aeternity/aeternity/commit/{{ ansible_local.aeternity.revision }}) - Package: {{ package | default('unknown') }} - %%% - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "env:{{ env }}" - - "version:{{ ansible_local.aeternity.version }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - connection: local - # tags: [datadog-event] rescue: - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ec2_instance_id | default(ansible_hostname) }}" - alert_type: error - title: Node Deployment - text: | - %%% - Package: {{ package | default('unknown') }} - Task {{ ansible_failed_task.name }} failed with error {{ ansible_failed_result.msg }} - %%% - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "env:{{ env }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - connection: local - tags: [datadog-event] - name: Fail the playbook ansible.builtin.fail: msg: "Failed deploy" diff --git a/ansible/mnesia_reset_once.yml b/ansible/mnesia_reset_once.yml index 15b64bc9..3ea45b46 100644 --- a/ansible/mnesia_reset_once.yml +++ b/ansible/mnesia_reset_once.yml @@ -7,12 +7,10 @@ project_root: "{{ ansible_env.HOME }}/node" aeternity_bin: "{{ project_root }}/bin/aeternity" public_ipv4: "{{ ansible_ec2_public_ipv4|default(ansible_ssh_host)|default(ansible_host) | default(inventory_hostname) }}" - datadog_api_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:api_key') }}" - datadog_app_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:app_key') }}" healthcheck_port: 3013 tasks: - - name: Get nodes info and send Datadog event + - name: Check synced nodes run_once: true block: - name: Get nodes status @@ -32,32 +30,7 @@ | selectattr('sync_progress', 'equalto', 100) | list | length == 0 - - - name: Send Datadog event - community.general.datadog_event: - host: "{{ public_ipv4 }}" - alert_type: success - title: Reset mnesia database - text: "Synced node found" - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - connection: local rescue: - - name: Send Datadog event - community.general.datadog_event: - host: "{{ public_ipv4 }}" - alert_type: error - title: Reset mnesia database - text: "No synced node found" - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - connection: local - name: Fail the playbook ansible.builtin.fail: msg: "No synced node found" diff --git a/ansible/mnesia_snapshot.yml b/ansible/mnesia_snapshot.yml index fbb2e7d5..ee82a871 100644 --- a/ansible/mnesia_snapshot.yml +++ b/ansible/mnesia_snapshot.yml @@ -18,10 +18,6 @@ aws_access_key: "{{ lookup('env','AWS_ACCESS_KEY_ID') }}" aws_secret_key: "{{ lookup('env','AWS_SECRET_ACCESS_KEY') }}" security_token: "{{ lookup('env','AWS_SESSION_TOKEN') }}" - datadog_api_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:api_key') }}" - datadog_app_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/deploy:app_key') }}" - datadog_api_url: https://api.datadoghq.com/api/v1/downtime?api_key={{ datadog_api_key }}&application_key={{ datadog_app_key }} - downtime: "0" db_path: data/ healthcheck_port: 3013 @@ -34,22 +30,6 @@ register: status failed_when: status.json.syncing == true - - name: Schedule downtime - run_once: true - ansible.builtin.uri: - url: "{{ datadog_api_url }}" - method: POST - status_code: 200 - timeout: 10 - body_format: json - body: - scope: "env:{{ env }}" - message: "Snapshot" - end: "{{ lookup('pipe', 'date +%s') | int + downtime | int }}" - when: - - downtime is defined - - downtime != "0" - - name: Stop node ansible.builtin.command: "/bin/true" changed_when: true @@ -145,40 +125,7 @@ notify: "clean snapshot" tags: [clean] - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ssh_host | default(inventory_hostname) }}" - alert_type: success - title: Snapshot - text: "Filename: {{ snapshot_filename }}" - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "env:{{ env }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - tags: [datadog-event] - rescue: - - name: Send Datadog event - community.general.datadog_event: - host: "{{ ansible_ssh_host | default(inventory_hostname) }}" - alert_type: error - title: Snapshot - text: | - %%% - Filename: {{ snapshot_filename }} - Task {{ ansible_failed_task.name }} failed with error {{ ansible_failed_result.msg }} - %%% - api_key: "{{ datadog_api_key }}" - app_key: "{{ datadog_app_key }}" - tags: - - "env:{{ env }}" - when: - - datadog_api_key != "" - - datadog_app_key != "" - tags: [datadog-event] - name: Fail the playbook ansible.builtin.fail: msg: "Failed snapshot" diff --git a/ansible/monitoring.yml b/ansible/monitoring.yml index 1daea8fb..5d8c2ec4 100644 --- a/ansible/monitoring.yml +++ b/ansible/monitoring.yml @@ -1,34 +1,3 @@ -- name: Collect cloud instance metadata - hosts: all - remote_user: master - become: true - gather_facts: true - tags: [datadog] - vars: - enable_aws_checks: true - tasks: - - name: Check if running on AWS instance - ansible.builtin.uri: - url: http://169.254.169.254/latest/meta-data - timeout: 2 - register: aws_uri_check - failed_when: false - when: - - enable_aws_checks|bool - - packer_build_name is not defined - - - name: Set AWS check fact - ansible.builtin.set_fact: - is_aws: "{{ packer_build_name is not defined and aws_uri_check.status == 200 }}" - when: - - enable_aws_checks|bool - - - name: Get instance metadata facts - amazon.aws.ec2_metadata_facts: - when: - - enable_aws_checks|bool - - is_aws - - name: Configure health check services (goss) hosts: all remote_user: master @@ -155,98 +124,15 @@ tags: - goss-install -- name: Configure monitoring services (DataDog) +- name: Configure monitoring services (Rsyslog) hosts: all remote_user: master become: true gather_facts: false - tags: [datadog] + tags: [rsyslog] vars: - project_user: aeternity - public_ipv4: "{{ ansible_ec2_public_ipv4|default(ansible_ssh_host)|default(ansible_host)|default(inventory_hostname) }}" fail2ban_enable: true - datadog_api_key: "{{ lookup('hashi_vault', 'secret=secret/datadog/agent:api_key') }}" - datadog_default_tags: - - "lsb:{{ ansible_lsb.description }}" - - "public_ipv4:{{ public_ipv4|default('unknown') }}" - - "network_id:{{ network_id|default('unknown') }}" - datadog_agent_major_version: 6 - datadog_config: - log_level: warning - log_to_console: false - apm_enabled: false - use_dogstatsd: true - process_config: - enabled: "true" # has to be set as a string - apm_config: - enabled: false - logs_enabled: true - tags: "{{ (datadog_tags + datadog_default_tags) if datadog_tags is defined else datadog_default_tags }}" - datadog_checks: - system_core: - init_config: - instances: - # The Agent just needs one item in instances in order to enable the check. - # The content of the item doesn’t matter. - - foo: bar - process: - init_config: - instances: - - name: ssh - search_string: ['sshd'] - - name: aeternity - search_string: ['beam.smp'] - - name: epmd - search_string: ['epmd'] - thresholds: - warning: [1, 1] - http_check: - init_config: - instances: - - name: API - url: "http://localhost:8080/healthz" - # Default is false, i.e. emit events instead of service checks. - # Recommend to set to true. - skip_event: true - logs_agent: - init_config: - instances: - logs: - - type: tcp - port: 10518 - service: aeternity - source: syslog - - type: file - path: "/home/{{ project_user }}/node/log/aeternity.log" - service: aeternity - source: lager - sourcecategory: aeternity - - type: file - path: "/home/{{ project_user }}/node/log/aeternity_mining.log" - service: aeternity - source: lager - sourcecategory: mining - - type: file - path: "/home/{{ project_user }}/node/log/aeternity_pow_cuckoo.log" - service: aeternity - source: lager - sourcecategory: pow - - type: file - path: "/home/{{ project_user }}/node/log/aeternity_sync.log" - service: aeternity - source: lager - sourcecategory: sync - - pre_tasks: - - name: "Add dd-agent to {{ project_user ~ 'group' }}" - ansible.builtin.user: - name: dd-agent - groups: "{{ project_user }}" - append: true - - roles: - - { role: Datadog.datadog } post_tasks: - name: Check is systemd enabled @@ -259,12 +145,6 @@ update_cache: true pkg: - rsyslog - - name: Copy datadog.conf to rsyslog - ansible.builtin.copy: - src: files/rsyslog/datadog.conf - dest: /etc/rsyslog.d/datadog.conf - mode: '0600' - notify: restart rsyslogd - name: Copy fail2ban.conf to rsyslog ansible.builtin.copy: src: files/rsyslog/fail2ban.conf diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 713bab2d..51f23ed0 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,5 +1,3 @@ -- src: Datadog.datadog - version: "4.13.0" - src: https://github.com/geerlingguy/ansible-role-node_exporter.git version: "2.1.0" name: node_exporter diff --git a/ansible/setup.yml b/ansible/setup.yml index 62a2ada0..d41d1c35 100644 --- a/ansible/setup.yml +++ b/ansible/setup.yml @@ -127,7 +127,6 @@ - awscli==1.32.45 - boto3==1.34.45 - boto==2.49.0 - - datadog==0.43.0 - hvac==1.2.1 - wheel==0.37.1 # temporary fix for https://github.com/docker/docker-py/issues/3256 diff --git a/import-secrets.sh b/import-secrets.sh index eec66422..6fb10565 100755 --- a/import-secrets.sh +++ b/import-secrets.sh @@ -48,7 +48,6 @@ if [ -n "$VAULT_TOKEN" ]; then export DOCKER_USER=$(echo $DOCKERHUB_CREDS | grep -o 'username [^ ]*' | awk '{print $2}') export DOCKER_PASS=$(echo $DOCKERHUB_CREDS | grep -o 'password [^ ]*' | awk '{print $2}') - export DATADOG_API_KEY=$(vault read -field=api_key secret/datadog/deploy) export ROCKET_HOOK_URL=$(vault read -field=core-alerts secret/rocketchat/prod/hooks) export ROCKET_HOOK_DEVOPS_URL=$(vault read -field=core-alerts-devops secret/rocketchat/prod/hooks) fi diff --git a/requirements-lock.txt b/requirements-lock.txt index 3f3b8c27..b2719e68 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -16,7 +16,6 @@ click==8.1.7 colorama==0.4.3 commonmark==0.9.1 cryptography==41.0.6 -datadog==0.43.0 decorator==5.1.1 docutils==0.15.2 envdir==1.0.1 diff --git a/requirements.txt b/requirements.txt index 4b41308c..06ce7f0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ ansible==9.2.0 awscli==1.32.45 boto3==1.34.45 boto==2.49.0 -datadog==0.48.0 envdir==1.0.1 hvac==1.2.1 wheel==0.37.1 diff --git a/scripts/inspect-miners.sh b/scripts/inspect-miners.sh new file mode 100755 index 00000000..92f7fb78 --- /dev/null +++ b/scripts/inspect-miners.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +START_HEIGHT=$1 +OFFSET=${2:-10} +API="https://mainnet.aeternity.io" + +TWO_MINERS_ADDR="ak_dArxCkAsk1mZB1L9CX3cdz1GDN4hN84L3Q8dMLHN4v8cU85TF" +WOOLY_POOLY_ADDR="ak_wM8yFU8eSETXU7VSN48HMDmevGoCMiuveQZgkPuRn1nTiRqyv" + +echo "2Miners: $TWO_MINERS_ADDR https://ae.2miners.com" +echo "Wooly Pooly: $WOOLY_POOLY_ADDR https://woolypooly.com/en/coin/ae" +echo "-----------------------------------------------------------------" + +printf "%-20s %-7s %-53s %-12s %-53s %-12s \n" "Time" "Height" "Keyblock" "Beneficiary" "Miner" "Micro-blocks" +echo -n "----------------------------------------------------------------------------------------------------" +echo "----------------------------------------------------------" +for ((HEIGHT = $START_HEIGHT; HEIGHT <= $START_HEIGHT+$OFFSET; HEIGHT++ )); do + GEN=$(curl -s $API/v3/generations/height/$HEIGHT) + KB=$(curl -s $API/v3/key-blocks/height/$HEIGHT) + # echo $GEN | jq -r '.hash' + + read TIME KB_HASH BENEFICIARY_ADDR MINER < <( echo $KB | jq -r '.time,.hash,.beneficiary,.miner' | tr \\n ' ') + MB_COUNT=$(echo $GEN | jq -r '.micro_blocks | length') + + DTIME=$(date -r $(($TIME/1000)) +'%Y-%m-%dT%H:%M:%S') + BENEFICIARY=$(echo $BENEFICIARY_ADDR | sed "s/$TWO_MINERS_ADDR/2Miners/" | sed "s/$WOOLY_POOLY_ADDR/Wooly-Pooly/") + printf "%-20s %-7s %-53s %-12s %-53s %-4s \n" $DTIME $HEIGHT $KB_HASH $BENEFICIARY $MINER $MB_COUNT +done diff --git a/scripts/secrets/policies/ci.sh b/scripts/secrets/policies/ci.sh index 49ba4817..f02210ff 100644 --- a/scripts/secrets/policies/ci.sh +++ b/scripts/secrets/policies/ci.sh @@ -9,8 +9,6 @@ DOCKERHUB_CREDS=$(vault read secret/dockerhub/prod) DOCKER_USER=$(echo $DOCKERHUB_CREDS | grep -o 'username [^ ]*' | awk '{print $2}') DOCKER_PASS=$(echo $DOCKERHUB_CREDS | grep -o 'password [^ ]*' | awk '{print $2}') -DATADOG_API_KEY=$(vault read -field=api_key secret/datadog/deploy) - dump_var "VAULT_ADDR" dump_var "VAULT_TOKEN" dump_var "AWS_ACCESS_KEY_ID" @@ -18,4 +16,3 @@ dump_var "AWS_SECRET_ACCESS_KEY" dump_var "AWS_SESSION_TOKEN" dump_var "DOCKER_USER" dump_var "DOCKER_PASS" -dump_var "DATADOG_API_KEY" diff --git a/scripts/secrets/policies/dev.sh b/scripts/secrets/policies/dev.sh index cd9e1ef6..8d221696 100644 --- a/scripts/secrets/policies/dev.sh +++ b/scripts/secrets/policies/dev.sh @@ -5,11 +5,8 @@ AWS_ACCESS_KEY_ID=$(echo $AWS_CREDS | grep -o 'access_key [^ ]*' | awk '{print $ AWS_SECRET_ACCESS_KEY=$(echo $AWS_CREDS | grep -o 'secret_key [^ ]*' | awk '{print $2}') AWS_SESSION_TOKEN=$(echo $AWS_CREDS | grep -o 'security_token [^ ]*' | awk '{print $2}') -DATADOG_API_KEY=$(vault read -field=api_key secret/datadog/deploy) - dump_var "VAULT_ADDR" dump_var "VAULT_TOKEN" dump_var "AWS_ACCESS_KEY_ID" dump_var "AWS_SECRET_ACCESS_KEY" dump_var "AWS_SESSION_TOKEN" -dump_var "DATADOG_API_KEY"