ENH: Add filebeat

Install filebeat on the stack VM and then add filebeat to config to all services that provide logs

Author: khalford

chatops_deployment/ansible/configure.yml

@@ -8,6 +8,15 @@
       ansible.builtin.include_role:
         name: add_ssh_key
 
+
+- name: Set up filebeat
+  hosts: stack
+  gather_facts: false
+  roles:
+    - role: filebeat
+      tags:
+        - filebeat
+
 - name: Configure load balancer
   hosts: stack
   roles:

chatops_deployment/ansible/roles/alertmanager/files/alertmanager.filebeat.yml

@@ -0,0 +1,9 @@
+---
+- type: filestream
+  id: alertmanager
+  enabled: true
+  paths:
+    - /opt/alertmanager/alertmanager.log
+  fields:
+    service.name: alertmanager
+  fields_under_root: true

chatops_deployment/ansible/roles/alertmanager/tasks/main.yml

@@ -66,3 +66,12 @@
     owner: alertmanager
     group: alertmanager
     mode: "0770"
+
+- name: Copy filebeat external config
+  become: true
+  ansible.builtin.copy:
+    src: alertmanager.filebeat.yml
+    dest: /var/filebeat/alertmanager.filebeat.yml
+    owner: root
+    group: root
+    mode: "0640"

chatops_deployment/ansible/roles/chatops/files/docker.filebeat.yml

@@ -0,0 +1,18 @@
+---
+- type: filestream
+  id: docker
+  prospector.scanner.symlinks: true
+  paths:
+    - "/var/lib/docker/containers/*/*.log"
+  parsers:
+    - container:
+        stream: all
+        format: docker
+    - multiline:
+        type: pattern
+        pattern: "^\\[\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}\\]"
+        negate: true
+        match: after
+  fields:
+    service.name: chatops
+  fields_under_root: true

chatops_deployment/ansible/roles/chatops/tasks/main.yml

@@ -54,3 +54,12 @@
       - /etc/chatops/config.yml:/usr/src/app/cloud_chatops/config/config.yml
       - /etc/chatops/secrets.yml:/usr/src/app/cloud_chatops/secrets/secrets.yml
     network_mode: host
+
+- name: Copy filebeat external config
+  become: true
+  ansible.builtin.copy:
+    src: docker.filebeat.yml
+    dest: /var/filebeat/docker.filebeat.yml
+    owner: root
+    group: root
+    mode: "0640"

chatops_deployment/ansible/roles/filebeat/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart Filebeat
+  become: true
+  ansible.builtin.systemd_service:
+    name: filebeat.service
+    state: restarted

chatops_deployment/ansible/roles/filebeat/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+- name: Install prerequisite packages
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - apt-transport-https
+      - software-properties-common
+      - wget
+    update_cache: true
+
+- name: Create key directory
+  become: true
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Add Elasticsearch key and repository to apt
+  become: true
+  block:
+    - name: Add key
+      ansible.builtin.get_url:
+        url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+        dest: /etc/apt/keyrings/elasticsearch.asc
+        mode: "0755"
+
+    - name: Add repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/9.x/apt stable main"
+        state: present
+
+- name: Install Filebeat
+  become: true
+  ansible.builtin.apt:
+    name: filebeat
+    state: latest # noqa: package-latest
+    update_cache: true
+
+- name: Template filebeat config
+  become: true
+  ansible.builtin.template:
+    src: filebeat.yml.j2
+    dest: "/etc/filebeat/filebeat.yml"
+    owner: root
+    group: root
+    mode: "0640"
+  notify:
+    - Restart Filebeat
+
+- name: Copy Logstash SSL certificate
+  become: true
+  ansible.builtin.copy:
+    src: "./{{ env }}_ssl/logstash.crt"
+    dest: "/etc/filebeat"
+    owner: root
+    group: root
+    mode: "0400"
+  notify:
+    - Restart Filebeat
+
+- name: Create filebeat external config directory
+  become: true
+  ansible.builtin.file:
+    path: /var/filebeat
+    state: directory
+    owner: root
+    group: root
+    mode: "0770"

chatops_deployment/ansible/roles/filebeat/templates/filebeat.yml.j2

@@ -0,0 +1,31 @@
+output.logstash:
+  hosts: ["localhost:5044"]
+  ssl:
+    enabled: true
+    certificate_authorities: ["/etc/filebeat/logstash.crt"]
+
+filebeat.config.inputs:
+  enabled: true
+  path: /var/filebeat/*.filebeat.yml
+  reload.enabled: true
+  reload.period: 10s
+
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+
+logging.level: info
+logging.to_files: true
+logging.files:
+  path: /var/log/filebeat
+  name: filebeat
+  keepfiles: 7
+  permissions: 0640
+
+filebeat.inputs:
+  - type: filestream
+    id: filebeat
+    enabled: true
+    paths:
+      - /etc/filebeat/filebeat.log
+      - /etc/filebeat/logs/*.ndjson

chatops_deployment/ansible/roles/grafana/files/grafana.filebeat.yml

@@ -0,0 +1,9 @@
+---
+- type: filestream
+  id: grafana
+  enabled: true
+  paths:
+    - /var/log/grafana/grafana.log
+  fields:
+    service.name: grafana
+  fields_under_root: true

chatops_deployment/ansible/roles/grafana/tasks/main.yml

@@ -81,3 +81,12 @@
   ansible.builtin.systemd_service:
     state: restarted
     name: grafana-server.service
+
+- name: Copy filebeat external config
+  become: true
+  ansible.builtin.copy:
+    src: grafana.filebeat.yml
+    dest: /var/filebeat/grafana.filebeat.yml
+    owner: root
+    group: root
+    mode: "0640"