From 866b640b6f865470fe28a492fd677e1b3106e81d Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Wed, 20 Aug 2025 09:56:46 +0100 Subject: [PATCH] ENH: Add filebeat Install filebeat on the stack VM and then add filebeat to config to all services that provide logs --- chatops_deployment/ansible/configure.yml | 9 +++ .../files/alertmanager.filebeat.yml | 9 +++ .../ansible/roles/alertmanager/tasks/main.yml | 9 +++ .../roles/chatops/files/docker.filebeat.yml | 18 +++++ .../ansible/roles/chatops/tasks/main.yml | 9 +++ .../ansible/roles/filebeat/handlers/main.yml | 6 ++ .../ansible/roles/filebeat/tasks/main.yml | 68 +++++++++++++++++++ .../roles/filebeat/templates/filebeat.yml.j2 | 31 +++++++++ .../roles/grafana/files/grafana.filebeat.yml | 9 +++ .../ansible/roles/grafana/tasks/main.yml | 9 +++ .../roles/haproxy/files/haproxy.filebeat.yml | 9 +++ .../ansible/roles/haproxy/tasks/haproxy.yml | 9 +++ .../prometheus/files/prometheus.filebeat.yml | 9 +++ .../ansible/roles/prometheus/tasks/main.yml | 9 +++ 14 files changed, 213 insertions(+) create mode 100644 chatops_deployment/ansible/roles/alertmanager/files/alertmanager.filebeat.yml create mode 100644 chatops_deployment/ansible/roles/chatops/files/docker.filebeat.yml create mode 100644 chatops_deployment/ansible/roles/filebeat/handlers/main.yml create mode 100644 chatops_deployment/ansible/roles/filebeat/tasks/main.yml create mode 100644 chatops_deployment/ansible/roles/filebeat/templates/filebeat.yml.j2 create mode 100644 chatops_deployment/ansible/roles/grafana/files/grafana.filebeat.yml create mode 100644 chatops_deployment/ansible/roles/haproxy/files/haproxy.filebeat.yml create mode 100644 chatops_deployment/ansible/roles/prometheus/files/prometheus.filebeat.yml diff --git a/chatops_deployment/ansible/configure.yml b/chatops_deployment/ansible/configure.yml index e44e1c1a..0c84c983 100644 --- a/chatops_deployment/ansible/configure.yml +++ b/chatops_deployment/ansible/configure.yml @@ -8,6 +8,15 @@ ansible.builtin.include_role: name: add_ssh_key + +- name: Set up filebeat + hosts: stack + gather_facts: false + roles: + - role: filebeat + tags: + - filebeat + - name: Configure load balancer hosts: stack roles: diff --git a/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.filebeat.yml b/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.filebeat.yml new file mode 100644 index 00000000..e54a731d --- /dev/null +++ b/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.filebeat.yml @@ -0,0 +1,9 @@ +--- +- type: filestream + id: alertmanager + enabled: true + paths: + - /opt/alertmanager/alertmanager.log + fields: + service.name: alertmanager + fields_under_root: true diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index 8f2581a7..e98bf5d1 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -66,3 +66,12 @@ owner: alertmanager group: alertmanager mode: "0770" + +- name: Copy filebeat external config + become: true + ansible.builtin.copy: + src: alertmanager.filebeat.yml + dest: /var/filebeat/alertmanager.filebeat.yml + owner: root + group: root + mode: "0640" diff --git a/chatops_deployment/ansible/roles/chatops/files/docker.filebeat.yml b/chatops_deployment/ansible/roles/chatops/files/docker.filebeat.yml new file mode 100644 index 00000000..3b14d5d6 --- /dev/null +++ b/chatops_deployment/ansible/roles/chatops/files/docker.filebeat.yml @@ -0,0 +1,18 @@ +--- +- type: filestream + id: docker + prospector.scanner.symlinks: true + paths: + - "/var/lib/docker/containers/*/*.log" + parsers: + - container: + stream: all + format: docker + - multiline: + type: pattern + pattern: "^\\[\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}\\]" + negate: true + match: after + fields: + service.name: chatops + fields_under_root: true diff --git a/chatops_deployment/ansible/roles/chatops/tasks/main.yml b/chatops_deployment/ansible/roles/chatops/tasks/main.yml index de358a5c..2ac51198 100644 --- a/chatops_deployment/ansible/roles/chatops/tasks/main.yml +++ b/chatops_deployment/ansible/roles/chatops/tasks/main.yml @@ -54,3 +54,12 @@ - /etc/chatops/config.yml:/usr/src/app/cloud_chatops/config/config.yml - /etc/chatops/secrets.yml:/usr/src/app/cloud_chatops/secrets/secrets.yml network_mode: host + +- name: Copy filebeat external config + become: true + ansible.builtin.copy: + src: docker.filebeat.yml + dest: /var/filebeat/docker.filebeat.yml + owner: root + group: root + mode: "0640" diff --git a/chatops_deployment/ansible/roles/filebeat/handlers/main.yml b/chatops_deployment/ansible/roles/filebeat/handlers/main.yml new file mode 100644 index 00000000..d9541aed --- /dev/null +++ b/chatops_deployment/ansible/roles/filebeat/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart Filebeat + become: true + ansible.builtin.systemd_service: + name: filebeat.service + state: restarted diff --git a/chatops_deployment/ansible/roles/filebeat/tasks/main.yml b/chatops_deployment/ansible/roles/filebeat/tasks/main.yml new file mode 100644 index 00000000..1aaaa0e7 --- /dev/null +++ b/chatops_deployment/ansible/roles/filebeat/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: Install prerequisite packages + become: true + ansible.builtin.apt: + pkg: + - apt-transport-https + - software-properties-common + - wget + update_cache: true + +- name: Create key directory + become: true + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Add Elasticsearch key and repository to apt + become: true + block: + - name: Add key + ansible.builtin.get_url: + url: https://artifacts.elastic.co/GPG-KEY-elasticsearch + dest: /etc/apt/keyrings/elasticsearch.asc + mode: "0755" + + - name: Add repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/9.x/apt stable main" + state: present + +- name: Install Filebeat + become: true + ansible.builtin.apt: + name: filebeat + state: latest # noqa: package-latest + update_cache: true + +- name: Template filebeat config + become: true + ansible.builtin.template: + src: filebeat.yml.j2 + dest: "/etc/filebeat/filebeat.yml" + owner: root + group: root + mode: "0640" + notify: + - Restart Filebeat + +- name: Copy Logstash SSL certificate + become: true + ansible.builtin.copy: + src: "./{{ env }}_ssl/logstash.crt" + dest: "/etc/filebeat" + owner: root + group: root + mode: "0400" + notify: + - Restart Filebeat + +- name: Create filebeat external config directory + become: true + ansible.builtin.file: + path: /var/filebeat + state: directory + owner: root + group: root + mode: "0770" diff --git a/chatops_deployment/ansible/roles/filebeat/templates/filebeat.yml.j2 b/chatops_deployment/ansible/roles/filebeat/templates/filebeat.yml.j2 new file mode 100644 index 00000000..5cf0c32e --- /dev/null +++ b/chatops_deployment/ansible/roles/filebeat/templates/filebeat.yml.j2 @@ -0,0 +1,31 @@ +output.logstash: + hosts: ["localhost:5044"] + ssl: + enabled: true + certificate_authorities: ["/etc/filebeat/logstash.crt"] + +filebeat.config.inputs: + enabled: true + path: /var/filebeat/*.filebeat.yml + reload.enabled: true + reload.period: 10s + +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + +logging.level: info +logging.to_files: true +logging.files: + path: /var/log/filebeat + name: filebeat + keepfiles: 7 + permissions: 0640 + +filebeat.inputs: + - type: filestream + id: filebeat + enabled: true + paths: + - /etc/filebeat/filebeat.log + - /etc/filebeat/logs/*.ndjson diff --git a/chatops_deployment/ansible/roles/grafana/files/grafana.filebeat.yml b/chatops_deployment/ansible/roles/grafana/files/grafana.filebeat.yml new file mode 100644 index 00000000..cd07a598 --- /dev/null +++ b/chatops_deployment/ansible/roles/grafana/files/grafana.filebeat.yml @@ -0,0 +1,9 @@ +--- +- type: filestream + id: grafana + enabled: true + paths: + - /var/log/grafana/grafana.log + fields: + service.name: grafana + fields_under_root: true diff --git a/chatops_deployment/ansible/roles/grafana/tasks/main.yml b/chatops_deployment/ansible/roles/grafana/tasks/main.yml index 712c6695..637d011d 100644 --- a/chatops_deployment/ansible/roles/grafana/tasks/main.yml +++ b/chatops_deployment/ansible/roles/grafana/tasks/main.yml @@ -81,3 +81,12 @@ ansible.builtin.systemd_service: state: restarted name: grafana-server.service + +- name: Copy filebeat external config + become: true + ansible.builtin.copy: + src: grafana.filebeat.yml + dest: /var/filebeat/grafana.filebeat.yml + owner: root + group: root + mode: "0640" diff --git a/chatops_deployment/ansible/roles/haproxy/files/haproxy.filebeat.yml b/chatops_deployment/ansible/roles/haproxy/files/haproxy.filebeat.yml new file mode 100644 index 00000000..e77c1300 --- /dev/null +++ b/chatops_deployment/ansible/roles/haproxy/files/haproxy.filebeat.yml @@ -0,0 +1,9 @@ +--- +- type: filestream + id: haproxy + enabled: true + paths: + - /var/log/haproxy.log + fields: + service.name: haproxy + fields_under_root: true diff --git a/chatops_deployment/ansible/roles/haproxy/tasks/haproxy.yml b/chatops_deployment/ansible/roles/haproxy/tasks/haproxy.yml index 06e730de..dcc39907 100644 --- a/chatops_deployment/ansible/roles/haproxy/tasks/haproxy.yml +++ b/chatops_deployment/ansible/roles/haproxy/tasks/haproxy.yml @@ -27,3 +27,12 @@ state: restarted name: haproxy.service when: haproxy_certificate_file.stat.exists + +- name: Copy filebeat external config + become: true + ansible.builtin.copy: + src: haproxy.filebeat.yml + dest: /var/filebeat/haproxy.filebeat.yml + owner: root + group: root + mode: "0640" diff --git a/chatops_deployment/ansible/roles/prometheus/files/prometheus.filebeat.yml b/chatops_deployment/ansible/roles/prometheus/files/prometheus.filebeat.yml new file mode 100644 index 00000000..3450fbfd --- /dev/null +++ b/chatops_deployment/ansible/roles/prometheus/files/prometheus.filebeat.yml @@ -0,0 +1,9 @@ +--- +- type: filestream + id: prometheus + enabled: true + paths: + - /opt/prometheus/prometheus.log + fields: + service.name: prometheus + fields_under_root: true diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index 9c5485de..2a4d33bc 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -90,3 +90,12 @@ owner: prometheus group: prometheus mode: "0770" + +- name: Copy filebeat external config + become: true + ansible.builtin.copy: + src: prometheus.filebeat.yml + dest: /var/filebeat/prometheus.filebeat.yml + owner: root + group: root + mode: "0640"