diff --git a/ansible/plays/setup-gitea-runner-logging.yml b/ansible/plays/setup-gitea-runner-logging.yml new file mode 100644 index 00000000..bbe275c3 --- /dev/null +++ b/ansible/plays/setup-gitea-runner-logging.yml @@ -0,0 +1,72 @@ +--- +# Setup log collection for Gitea runners +# Deploys Promtail to collect act_runner logs and ship to Loki +# +# Usage: +# ansible-playbook -i inventory/ plays/setup-gitea-runner-logging.yml + +- name: Setup Gitea Runner Log Collection + hosts: proxmox_gitea_runner + become: yes + vars: + promtail_loki_url: "http://loki.srv.renner.uno:3100/loki/api/v1/push" + promtail_scrape_configs: + # Collect act_runner service logs via journald + - job_name: act_runner + journal: + max_age: 12h + matches: _SYSTEMD_UNIT=act_runner.service + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: unit + - source_labels: ['__journal__hostname'] + target_label: host + + # Collect system logs + - job_name: syslog + static_configs: + - targets: + - localhost + labels: + job: syslog + host: "{{ inventory_hostname }}" + service: gitea-runner + __path__: /var/log/syslog + + # Collect Docker daemon logs (runners use Docker) + - job_name: docker + journal: + max_age: 12h + matches: _SYSTEMD_UNIT=docker.service + relabel_configs: + - source_labels: ['__journal__systemd_unit'] + target_label: unit + - source_labels: ['__journal__hostname'] + target_label: host + + roles: + - promtail + + post_tasks: + - name: Verify promtail is running + systemd: + name: promtail + state: started + register: promtail_status + + - name: Display promtail status + debug: + msg: "Promtail is {{ 'running' if promtail_status.status.ActiveState == 'active' else 'not running' }} on {{ inventory_hostname }}" + + - name: Test Loki connectivity + uri: + url: "http://loki.srv.renner.uno:3100/ready" + method: GET + status_code: 200 + timeout: 10 + register: loki_health + ignore_errors: yes + + - name: Display Loki connectivity status + debug: + msg: "Loki server is {{ 'reachable' if loki_health.status == 200 else 'NOT reachable - check network' }}" diff --git a/ansible/roles/promtail/defaults/main.yml b/ansible/roles/promtail/defaults/main.yml new file mode 100644 index 00000000..11f7ae3b --- /dev/null +++ b/ansible/roles/promtail/defaults/main.yml @@ -0,0 +1,36 @@ +--- +# Promtail Configuration + +# Version and download +promtail_version: "3.4.2" +promtail_arch: "linux-amd64" +promtail_download_url: "https://github.com/grafana/loki/releases/download/v{{ promtail_version }}/promtail-{{ promtail_arch }}.zip" +promtail_bin_path: "/usr/local/bin/promtail" + +# Directories +promtail_config_dir: "/etc/promtail" +promtail_config_file: "{{ promtail_config_dir }}/promtail-config.yaml" +promtail_positions_file: "/var/lib/promtail/positions.yaml" + +# Server configuration +promtail_http_listen_port: 9080 +promtail_http_listen_address: "0.0.0.0" + +# Loki server +promtail_loki_url: "http://loki.srv.renner.uno:3100/loki/api/v1/push" + +# Log scrape configurations +# Override in playbook or host_vars for specific services +promtail_scrape_configs: + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: syslog + host: "{{ inventory_hostname }}" + __path__: /var/log/syslog + +# Systemd service +promtail_user: "promtail" +promtail_group: "promtail" diff --git a/ansible/roles/promtail/handlers/main.yml b/ansible/roles/promtail/handlers/main.yml new file mode 100644 index 00000000..141c9a53 --- /dev/null +++ b/ansible/roles/promtail/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart promtail + systemd: + name: promtail + state: restarted + daemon_reload: yes diff --git a/ansible/roles/promtail/tasks/main.yml b/ansible/roles/promtail/tasks/main.yml new file mode 100644 index 00000000..51a22335 --- /dev/null +++ b/ansible/roles/promtail/tasks/main.yml @@ -0,0 +1,112 @@ +--- +- name: Install dependencies + apt: + name: + - unzip + - curl + state: present + update_cache: yes + +- name: Create promtail group + group: + name: "{{ promtail_group }}" + system: yes + state: present + +- name: Create promtail user + user: + name: "{{ promtail_user }}" + group: "{{ promtail_group }}" + system: yes + shell: /usr/sbin/nologin + home: /var/lib/promtail + create_home: yes + state: present + +- name: Add promtail user to systemd-journal group + user: + name: "{{ promtail_user }}" + groups: systemd-journal + append: yes + +- name: Create config directory + file: + path: "{{ promtail_config_dir }}" + state: directory + owner: "{{ promtail_user }}" + group: "{{ promtail_group }}" + mode: '0755' + +- name: Create positions directory + file: + path: "{{ promtail_positions_file | dirname }}" + state: directory + owner: "{{ promtail_user }}" + group: "{{ promtail_group }}" + mode: '0755' + +- name: Check if promtail binary exists + stat: + path: "{{ promtail_bin_path }}" + register: promtail_binary + +- name: Get installed promtail version + command: "{{ promtail_bin_path }} --version" + register: installed_version + changed_when: false + failed_when: false + when: promtail_binary.stat.exists + +- name: Download promtail + get_url: + url: "{{ promtail_download_url }}" + dest: /tmp/promtail.zip + mode: '0644' + when: not promtail_binary.stat.exists or (installed_version.stdout is defined and promtail_version not in installed_version.stdout) + +- name: Extract promtail binary + unarchive: + src: /tmp/promtail.zip + dest: /tmp/ + remote_src: yes + when: not promtail_binary.stat.exists or (installed_version.stdout is defined and promtail_version not in installed_version.stdout) + +- name: Install promtail binary + copy: + src: "/tmp/promtail-{{ promtail_arch }}" + dest: "{{ promtail_bin_path }}" + remote_src: yes + mode: '0755' + when: not promtail_binary.stat.exists or (installed_version.stdout is defined and promtail_version not in installed_version.stdout) + notify: Restart promtail + +- name: Clean up downloaded files + file: + path: "{{ item }}" + state: absent + loop: + - /tmp/promtail.zip + - "/tmp/promtail-{{ promtail_arch }}" + +- name: Configure promtail + template: + src: promtail-config.yaml.j2 + dest: "{{ promtail_config_file }}" + owner: "{{ promtail_user }}" + group: "{{ promtail_group }}" + mode: '0644' + notify: Restart promtail + +- name: Create systemd service + template: + src: promtail.service.j2 + dest: /etc/systemd/system/promtail.service + mode: '0644' + notify: Restart promtail + +- name: Enable and start promtail + systemd: + name: promtail + state: started + enabled: yes + daemon_reload: yes diff --git a/ansible/roles/promtail/templates/promtail-config.yaml.j2 b/ansible/roles/promtail/templates/promtail-config.yaml.j2 new file mode 100644 index 00000000..98a96f37 --- /dev/null +++ b/ansible/roles/promtail/templates/promtail-config.yaml.j2 @@ -0,0 +1,79 @@ +server: + http_listen_port: {{ promtail_http_listen_port }} + http_listen_address: {{ promtail_http_listen_address }} + grpc_listen_port: 0 + +positions: + filename: {{ promtail_positions_file }} + +clients: + - url: {{ promtail_loki_url }} + tenant_id: default + +scrape_configs: +{% for config in promtail_scrape_configs %} + - job_name: {{ config.job_name }} +{% if config.journal is defined %} + journal: + max_age: {{ config.journal.max_age | default('12h') }} + labels: + job: {{ config.job_name }} + host: {{ inventory_hostname }} +{% if config.journal.matches is defined %} + matches: {{ config.journal.matches }} +{% endif %} +{% if config.relabel_configs is defined %} + relabel_configs: +{% for relabel in config.relabel_configs %} + - source_labels: {{ relabel.source_labels | to_json }} + target_label: {{ relabel.target_label }} +{% if relabel.regex is defined %} + regex: '{{ relabel.regex }}' +{% endif %} +{% if relabel.replacement is defined %} + replacement: '{{ relabel.replacement }}' +{% endif %} +{% endfor %} +{% endif %} +{% endif %} +{% if config.static_configs is defined %} + static_configs: +{% for static in config.static_configs %} + - targets: +{% for target in static.targets %} + - {{ target }} +{% endfor %} + labels: +{% for key, value in static.labels.items() %} + {{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% if config.pipeline_stages is defined %} + pipeline_stages: +{% for stage in config.pipeline_stages %} +{% if stage.multiline is defined %} + - multiline: + firstline: '{{ stage.multiline.firstline }}' +{% if stage.multiline.max_wait_time is defined %} + max_wait_time: {{ stage.multiline.max_wait_time }} +{% endif %} +{% endif %} +{% if stage.regex is defined %} + - regex: + expression: '{{ stage.regex.expression }}' +{% endif %} +{% if stage.labels is defined %} + - labels: +{% for label in stage.labels %} + {{ label }}: +{% endfor %} +{% endif %} +{% if stage.timestamp is defined %} + - timestamp: + source: {{ stage.timestamp.source }} + format: '{{ stage.timestamp.format }}' +{% endif %} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/ansible/roles/promtail/templates/promtail.service.j2 b/ansible/roles/promtail/templates/promtail.service.j2 new file mode 100644 index 00000000..b4def0b2 --- /dev/null +++ b/ansible/roles/promtail/templates/promtail.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Promtail Log Collector +Documentation=https://grafana.com/docs/loki/latest/clients/promtail/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ promtail_user }} +Group={{ promtail_group }} +ExecStart={{ promtail_bin_path }} -config.file={{ promtail_config_file }} +Restart=always +RestartSec=5 +TimeoutStopSec=30 + +# Security hardening +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/promtail +CapabilityBoundingSet=CAP_DAC_READ_SEARCH + +[Install] +WantedBy=multi-user.target