From 86b714ca23d2753aea3522e30aa58b7df25545f1 Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Thu, 20 Aug 2026 11:43:57 +0200 Subject: [PATCH 1/8] Add silent agent monitoring integration Detects agents that stay registered, and often still active, while log ingestion from them has stopped. For every agent of a target group the check reads the newest indexed event time, writes one SILENT record when it is older than the threshold, and one RESTORED record when events arrive again. Local state keeps an unchanged condition from being reported on every run. Records are JSON lines ingested through a block, so the built-in JSON decoder handles them and no custom decoder is needed. Rules 100121 and 100122 carry alert_by_email and are routed to email and to Telegram through the included integration script. Verified end to end on Wazuh 4.14.6: group lookup, silence detection, repeat suppression, recovery, ingestion, rules, real email delivered through Postfix and accepted upstream, Telegram delivery, the wrong-index-pattern safety stop, and the command wodle schedule. Co-Authored-By: Claude Opus 5 --- .../silent_agent_monitoring/README.md | 462 ++++++++++++++++++ .../custom-server-telegram | 38 ++ .../custom-server-telegram.py | 122 +++++ .../silent_agent_monitor-rules.xml | 37 ++ .../silent_agent_monitor.py | 309 ++++++++++++ 5 files changed, 968 insertions(+) create mode 100644 integrations/silent_agent_monitoring/README.md create mode 100755 integrations/silent_agent_monitoring/custom-server-telegram create mode 100755 integrations/silent_agent_monitoring/custom-server-telegram.py create mode 100644 integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml create mode 100755 integrations/silent_agent_monitoring/silent_agent_monitor.py diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md new file mode 100644 index 00000000..cb468ccf --- /dev/null +++ b/integrations/silent_agent_monitoring/README.md @@ -0,0 +1,462 @@ +# Silent Agent Monitoring - Wazuh Integration + +## Table of Contents + +* [Introduction](#introduction) +* [Prerequisites](#prerequisites) +* [How It Works](#how-it-works) + * [What Counts as a Log](#what-counts-as-a-log) + * [Alert and Recovery Logic](#alert-and-recovery-logic) +* [Installation and Configuration](#installation-and-configuration) + * [Using the Integration Files](#using-the-integration-files) + * [Script Configuration](#script-configuration) + * [Scheduling the Check](#scheduling-the-check) + * [Ingesting the Records](#ingesting-the-records) + * [Rules](#rules) + * [Email Notifications](#email-notifications) + * [Telegram Notifications](#telegram-notifications) +* [Integration Steps](#integration-steps) +* [Integration Testing](#integration-testing) +* [Troubleshooting](#troubleshooting) +* [Verification](#verification) +* [Design Notes](#design-notes) +* [Sources](#sources) + +--- + +### Introduction + +Wazuh alerts natively when an agent stops connecting: `wazuh-monitord` marks the +agent disconnected after `` and rule 502 fires. That +covers the agent that goes away. It does not cover the agent that stays +connected, keeps answering keepalives, and quietly stops shipping logs, because +a log collector died, a log path rotated away, a service stopped writing, or a +permission changed. From the manager's point of view that agent is healthy. + +This integration closes that gap. On a schedule it reads, for every agent of a +chosen group, the timestamp of the most recent event that reached the indexer. +When that timestamp is older than the threshold it writes a `SILENT` record; +when events start arriving again it writes one `RESTORED` record. The records +are plain JSON lines that Wazuh ingests through a `` block, so they +become normal alerts and can be routed to email, Telegram, or anything else +with the standard `` and `` blocks. + +The answer to "can this be done from the group and the last event timestamp, or +is a custom script needed": the group and the timestamp are exactly the right +inputs, and a script is needed to join them, because no built-in module tracks +per-agent event recency. Everything downstream of the script (decoding, rules, +alerting, routing) is stock Wazuh. + +--- + +### Prerequisites + +- Wazuh manager 4.4 or later, with the Wazuh API reachable and an API user that + can read `/agents`. +- Wazuh indexer reachable from the manager, with a user that can search the + alerts (or archives) indices. +- An agent group to monitor. The examples use `Server`. +- Python 3.6 or later. The scripts use only the standard library, so there is no + `pip install` step; the Wazuh embedded interpreter at + `/var/ossec/framework/python/bin/python3` satisfies this. +- Filesystem access to the manager to place the files. + +**Wazuh Cloud:** managed environments do not give shell access to the manager, +so the two scripts cannot be copied in by the user. Rules, `ossec.conf` blocks, +and the group can be managed from the dashboard, but the script placement and +its execute permissions have to be done by the Wazuh Cloud team through a +support request. Send them this folder and the target paths listed below. In a +cluster, the script, its state file, and the `` block must be placed +on one node only (the master); running it on several nodes duplicates every +notification and splits the state. + +--- + +### How It Works + +``` + Wazuh API /agents?group=Server Wazuh indexer + | | + | agent id, name, status | max(@timestamp) per agent.id + v v + +--------------------------------------------------+ + command wodle ->| silent_agent_monitor.py | + (hourly) | compares each agent against the threshold and | + | against the previous run's state | + +--------------------------------------------------+ + | + | one JSON line per state change only + v + /var/ossec/logs/silent_agents.json + | + | json + v + rules 100121 / 100122 -> email + Telegram +``` + +#### What Counts as a Log + +The script measures recency against an index pattern, `SAM_INDEX_PATTERN`: + +| Pattern | Meaning | Trade-off | +| --- | --- | --- | +| `wazuh-alerts-*` (default) | The newest **alert** produced by the agent. | Available everywhere. An agent that ships logs normally but produces no alert for a full day is reported as silent. | +| `wazuh-archives-*` | The newest **event** received from the agent, whether or not it alerted. | Exact answer to "no logs received", but needs `` enabled and the archives indexed, which costs storage. | + +Use archives when they are enabled. On alerts, confirm first that every agent in +the group normally produces at least some alerts within the threshold; a quiet +Windows file server under a tight ruleset sometimes does not. Widening the +threshold or moving to archives both remove that false positive. + +#### Alert and Recovery Logic + +State is kept in a small local JSON file, so a condition that has not changed is +reported once rather than once per run: + +| Previous state | Current reading | Action | +| --- | --- | --- | +| OK (or unknown) | Last event older than the threshold, or no event at all in the lookback window | Write one `SILENT` record, remember the last log timestamp. | +| SILENT | Still older than the threshold | Nothing. No repeated notification. | +| SILENT | Recent events again | Write one `RESTORED` record, clear the state. | +| OK | Recent events | Nothing. | + +Durations are measured against real log timestamps, not against the moment the +script noticed. `No Logs For` is the gap between the last received log and now. +`No Logs Duration` on recovery is the gap between the last log before the +silence and the first log after it, which is what the operator actually wants to +read in the incident. + +An agent that has never connected is skipped: it has no logs by definition, and +`never_connected` is already visible in the dashboard. Agent `000` (the manager) +is skipped too. + +--- + +### Installation and Configuration + +#### Using the Integration Files + +``` +silent_agent_monitoring/ + silent_agent_monitor.py # The check. Runs on a schedule from a wodle. + silent_agent_monitor-rules.xml # Rules 100120-100122. + custom-server-telegram # Integration wrapper (selects the Wazuh interpreter). + custom-server-telegram.py # Formats and posts the Telegram message. +``` + +Target paths on the manager: + +```bash +cp silent_agent_monitor.py /var/ossec/wodles/ +chmod 750 /var/ossec/wodles/silent_agent_monitor.py +chown root:wazuh /var/ossec/wodles/silent_agent_monitor.py + +cp custom-server-telegram custom-server-telegram.py /var/ossec/integrations/ +chmod 750 /var/ossec/integrations/custom-server-telegram* +chown root:wazuh /var/ossec/integrations/custom-server-telegram* + +cat silent_agent_monitor-rules.xml >> /var/ossec/etc/rules/local_rules.xml +``` + +A manager upgrade can replace the contents of `/var/ossec/wodles`, so keep a +copy of the configured script outside `/var/ossec` and re-apply it after an +upgrade. + +#### Script Configuration + +Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set +the matching environment variables and leave the file untouched: + +| Setting | Variable | Default | +| --- | --- | --- | +| Wazuh API URL | `SAM_API_URL` | `https://127.0.0.1:55000` | +| Wazuh API user / password | `SAM_API_USER`, `SAM_API_PASSWORD` | `wazuh-wui` / `CHANGE_ME` | +| Indexer URL | `SAM_INDEXER_URL` | `https://127.0.0.1:9200` | +| Indexer user / password | `SAM_INDEXER_USER`, `SAM_INDEXER_PASSWORD` | `admin` / `CHANGE_ME` | +| Index pattern | `SAM_INDEX_PATTERN` | `wazuh-alerts-*` | +| Agent group | `SAM_GROUP` | `Server` | +| Silence threshold, hours | `SAM_THRESHOLD_HOURS` | `24` | +| Lookback window, days | `SAM_LOOKBACK_DAYS` | `7` | +| State file | `SAM_STATE_FILE` | `/var/ossec/var/silent_agents_state.json` | +| Output log | `SAM_OUTPUT_LOG` | `/var/ossec/logs/silent_agents.json` | +| Script log | `SAM_SCRIPT_LOG` | `/var/ossec/logs/silent_agent_monitor.log` | +| Verify TLS certificates | `SAM_VERIFY_SSL` | `no` | + +The file holds credentials, so keep it `chmod 750` and root-owned. On Wazuh +Cloud, use the environment endpoints and credentials supplied with the +environment rather than the loopback defaults. + +`SAM_LOOKBACK_DAYS` must stay larger than the threshold. It bounds the indexer +query, and an agent with nothing inside it is reported as silent for "more than" +that window. + +#### Scheduling the Check + +`/var/ossec/etc/ossec.conf`, on the master node only: + +```xml + + + no + silent-agent-monitor + /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py + 1h + yes + 300 + yes + + +``` + +With `run_on_start`, the very first run after a manager restart can reach the +Wazuh API before it finishes starting and log `HTTP Error 500`. That run exits +non-zero, `wazuh-modulesd` records a warning, and the next scheduled run +succeeds. Nothing is lost, because a failed run never writes state. + +One run per hour is enough for a 24 hour threshold: it bounds detection lag and +recovery lag to an hour each while keeping the indexer load at one aggregation +query per hour, whatever the number of agents. Shorten the interval if the +recovery notification needs to arrive sooner. + +#### Ingesting the Records + +The script writes plain JSON objects, one per line, so the built-in JSON decoder +parses them and **no custom decoder is required**: + +```xml + + + json + /var/ossec/logs/silent_agents.json + + +``` + +#### Rules + +`silent_agent_monitor-rules.xml` defines a level 0 parent that matches the +`integration` field and two children that alert: + +| Rule | Level | Fires when | +| --- | --- | --- | +| 100120 | 0 | Any record from this integration. Classification only. | +| 100121 | 12 | `event_status` is `SILENT`. | +| 100122 | 5 | `event_status` is `RESTORED`. | + +The matched field is `event_status`, not `status`: `status` is one of the Wazuh +static field names, and a rule that tries to match it with `` +fails to load with `Field 'status' is static`. + +Both children carry `alert_by_email`, which forces the email +regardless of the global ``. Without it the level 5 recovery +alert would be dropped by the default threshold of 12 and only the silence +notification would arrive. + +Move the IDs into a free range if 100120-100122 are already used; the repository +`detect_new_agents` integration, for example, also ships a rule 100110. + +#### Email Notifications + +Global email must already be configured (`` with +`yes`, ``, ``, +``). Then route these two rules: + +```xml + + + soc-team@example.com + 100121,100122 + + full + + +``` + +`` sends immediately instead of waiting for the next email +grouping interval. + +The `full` format prints the record's fields one per line, so the email already +carries the agent name, the agent ID, the last log timestamp and the duration. +Only if the email has to look like the Telegram message, with the same heading +and emoji, is a `custom-email` integration script needed in place of +``. + +#### Telegram Notifications + +Add the integration next to the existing Telegram block, reusing the bot token +and chat ID of the Server channel: + +```xml + + + + custom-server-telegram + 100121,100122 + https://api.telegram.org/bot<BOT_TOKEN>/sendMessage + <CHAT_ID> + json + + +``` + +`` is the full `sendMessage` endpoint of the bot and `` is +the numeric chat ID of the channel, both taken from the Telegram block already +in the configuration. The script produces exactly the requested layout: + +``` +⚠ Server Logging Alert ✅ Server Logging Restored +Name: File2 Name: File2 +Agent ID: 152 Agent ID: 152 +Status: No logs received Status: Logs received +Last Log Received: ... Logging Restored At: ... +No Logs For: 25h 40m No Logs Duration: 25h 40m +``` + +`wazuh-integratord` runs integration scripts as the `wazuh` user, not as root, +so the script logs to `/var/ossec/logs/integrations.log`, which that user can +already write. If `TELEGRAM_LOG` is pointed somewhere else, the new file has to +be writable by `wazuh` or the notification is lost before it is sent. + +A separate script is used rather than a change to the existing `custom-telegram` +so that the current Telegram alerting keeps working untouched. To format these +two rules inside the existing script instead, add the branch before its normal +message construction and skip this file: + +```python +if str(alert.get("rule", {}).get("id")) in ("100121", "100122"): + d = alert.get("data", {}) + if d.get("event_status") == "SILENT": + msg = (f"⚠ Server Logging Alert\nName: {d.get('agent_name')}\n" + f"Agent ID: {d.get('agent_id')}\nStatus: No logs received\n" + f"Last Log Received: {d.get('last_log')}\n" + f"No Logs For: {d.get('no_logs_for')}") + else: + msg = (f"✅ Server Logging Restored\nName: {d.get('agent_name')}\n" + f"Agent ID: {d.get('agent_id')}\nStatus: Logs received\n" + f"Logging Restored At: {d.get('restored_at')}\n" + f"No Logs Duration: {d.get('silence_duration')}") +``` + +--- + +### Integration Steps + +1. Confirm the agents to monitor are in the group: `/var/ossec/bin/agent_groups -s -g Server`. +2. Copy the four files to the paths above and set ownership and permissions. +3. Fill in the API and indexer credentials, the group name, and the threshold. +4. Append the rules to `/var/ossec/etc/rules/local_rules.xml`. +5. Add the ``, ``, ``, and `` blocks + to `/var/ossec/etc/ossec.conf` on the master node. +6. Validate the configuration and restart: `/var/ossec/bin/wazuh-control restart`. +7. Watch `/var/ossec/logs/silent_agent_monitor.log` after the first run. + +--- + +### Integration Testing + +**Decision logic, offline.** No API, indexer, or manager needed: + +```bash +/var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --selftest +# selftest OK +``` + +It asserts that a 25h40m gap reports once and only once, that recovery reports +once with the duration measured from the last log before the gap, that an agent +inside the threshold stays quiet, and that an agent with no events at all is +treated as silent rather than skipped. + +**End to end, against the live environment.** Run the check by hand: + +```bash +/var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py +# Checked 12 agent(s) in 'Server': 0 silent, 0 event(s) written to /var/ossec/logs/silent_agents.json. +``` + +To force a notification without waiting a day, drop the threshold for one run +and watch the whole chain fire: + +```bash +SAM_THRESHOLD_HOURS=0.05 /var/ossec/framework/python/bin/python3 \ + /var/ossec/wodles/silent_agent_monitor.py +tail -1 /var/ossec/logs/silent_agents.json +tail -f /var/ossec/logs/alerts/alerts.log | grep -A5 100121 +``` + +Delete `/var/ossec/var/silent_agents_state.json` afterwards so the test does not +leave agents marked silent. Running with the real threshold again produces the +`RESTORED` notification, which is a useful way to confirm the recovery path and +the Telegram formatting in one go. + +**Rules only**, without running the script: + +```bash +echo '{"integration":"silent-agent-monitor","event_status":"SILENT","agent_id":"152","agent_name":"File2","last_log":"2026-08-18 08:35:12 CEST","no_logs_for":"25h 40m"}' \ + | /var/ossec/bin/wazuh-logtest +``` + +--- + +### Troubleshooting + +| Symptom | Cause and fix | +| --- | --- | +| `No events found for any of the N agents` in the script log, and no alerts | Deliberate safety stop. Every agent silent at once is almost always a wrong index pattern or wrong indexer credentials, not a real outage, so the script refuses to send the storm. Check `SAM_INDEX_PATTERN` and the indexer user. | +| `Indexer query failed` or `Wazuh API query failed` | The run exits without touching the state, so nothing is reported as silent or as recovered on the strength of a failed query. Check connectivity and credentials. | +| Records in `silent_agents.json` but no alerts | The `` block is missing, points elsewhere, or sits on a node that is not running the script. Confirm with `grep silent_agents /var/ossec/logs/ossec.log`. | +| Alerts fire but no email | Global email is not enabled, or the rules lost `alert_by_email`. Check `/var/ossec/logs/ossec.log` for `wazuh-maild`. | +| Alerts fire but no Telegram message | Check `/var/ossec/logs/integrations.log` for a line from `custom-server-telegram`, then `grep integrator /var/ossec/logs/ossec.log`. A missing chat ID or hook URL, or an HTTP error from the bot API, is logged with the rule ID. | +| `Permission denied` from integratord | The integration runs as the `wazuh` user. Any path the script writes, including a custom `TELEGRAM_LOG`, must be writable by it. | +| `Failure to read rule 100121. Field 'status' is static` | The rule was edited to match `status` instead of `event_status`. `status` is a reserved Wazuh field name. | +| A healthy agent is reported silent | It produced no *alerts* within the threshold. Point `SAM_INDEX_PATTERN` at `wazuh-archives-*`, or raise the threshold. | +| Every agent reported again after a manager rebuild | The state file was lost, so the first run after it re-reports the conditions that are still true. One repeat, then quiet again. | + +--- + +### Verification + +Run end to end on a Wazuh 4.14.6 single-node server (manager, indexer and +dashboard on one host) with three agents in a `Server` group: + +| Check | Result | +| --- | --- | +| `--selftest` on the embedded interpreter | Passes: single alert, single recovery, correct durations, silence on missing data. | +| Group lookup | The `never_connected` agent and agent `000` are excluded; the two real agents are checked. | +| Silence detection | An agent whose newest indexed event was 30 hours old produced one `SILENT` record reading `30h`. | +| Repeat suppression | Three further runs with the condition unchanged produced no further records. | +| Recovery | A fresh event produced one `RESTORED` record reading `30h`, measured from the last log before the gap. | +| Ingestion and rules | The record reached `alerts.json` through the `` block as rule 100121, level 12, `mail: true`, with the description fully interpolated. | +| Telegram | `wazuh-integratord` invoked the integration and delivered both formatted messages, captured against a local HTTP endpoint standing in for the bot API. | +| Email | Both rules produced a real email through a local Postfix relay, accepted by the upstream server (`dsn=2.0.0, status=sent`). The stock `full` format carries every field of the record, decoded one per line, under the subject `Wazuh notification - - Alert level 12`. | +| Wrong index pattern | The safety stop fired: exit code 1, no state written, no alerts sent, and a log line naming the setting to check. | +| Wodle schedule | `wazuh-modulesd` ran the command on its interval, one run per interval, with the output ignored. | + +--- + +### Design Notes + +- **One indexer query per run, not one per agent.** A single `terms` + aggregation on `agent.id` with a `max` on `@timestamp` returns the last event + time for every agent at once, so the cost does not grow with the fleet. +- **No custom decoder.** JSON lines plus `json` gives + fully decoded fields for free, which also removes the dependency on a working + local syslog daemon that a `logger`-based approach carries. +- **Standard library only.** `urllib.request` instead of `requests`, so the + script runs on the embedded interpreter and on the system Python with no + packaging step. +- **Missing data is silence, not a skip.** An agent with no events at all in the + lookback window is the worst case, not a case to ignore. +- **The state file is written atomically** with a temporary file and a rename, + so an interrupted run cannot leave the state truncated. + +--- + +### Sources + +- [Wazuh - command wodle](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/wodle-command.html) +- [Wazuh - localfile configuration](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/localfile.html) +- [Wazuh - integration configuration](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/integration.html) +- [Wazuh - granular email alerts](https://documentation.wazuh.com/current/user-manual/manager/manual-email-report/index.html) +- [Wazuh - rules syntax](https://documentation.wazuh.com/current/user-manual/ruleset/ruleset-xml-syntax/rules.html) +- [Wazuh API - agents](https://documentation.wazuh.com/current/user-manual/api/reference.html#tag/Agents) +- [Wazuh - archiving alerts and events](https://documentation.wazuh.com/current/user-manual/manager/event-logging.html) diff --git a/integrations/silent_agent_monitoring/custom-server-telegram b/integrations/silent_agent_monitoring/custom-server-telegram new file mode 100755 index 00000000..fc3023be --- /dev/null +++ b/integrations/silent_agent_monitoring/custom-server-telegram @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright (C) 2015, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 + +WPYTHON_BIN="framework/python/bin/python3" + +SCRIPT_PATH_NAME="$0" + +DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" +SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" + +case ${DIR_NAME} in + */active-response/bin | */wodles*) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; + */bin) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/$(echo ${SCRIPT_NAME} | sed 's/\-/_/g').py" + ;; + */integrations) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; +esac + + +${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/integrations/silent_agent_monitoring/custom-server-telegram.py b/integrations/silent_agent_monitoring/custom-server-telegram.py new file mode 100755 index 00000000..0863e56e --- /dev/null +++ b/integrations/silent_agent_monitoring/custom-server-telegram.py @@ -0,0 +1,122 @@ +#!/var/ossec/framework/python/bin/python3 +# Wazuh Telegram integration for the silent agent monitoring rules. +# Adapted from the Wazuh custom integration examples. +# This program is free software; you can redistribute it and/or modify it +# under the terms of GPLv2. +""" +Formats rules 100121 (no logs received) and 100122 (logging restored) into the +message layout the notification template asks for, and posts them to the +existing Telegram channel. Any other rule routed here falls back to a generic +message, so a wrong in ossec.conf produces a readable alert rather +than a crash. + +wazuh-integratord calls this as: + custom-server-telegram +so carries the chat ID and the bot sendMessage URL. +""" + +import json +import logging +import os +import sys +import ssl +import urllib.request + +# === CONFIGURATION === +# Defaults used only when ossec.conf passes nothing, or for a manual test run. +CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "") +HOOK_URL = os.environ.get("TELEGRAM_HOOK_URL", "") +# integratord runs this as the wazuh user, so the log has to be somewhere that +# user can already write. integrations.log is the standard place for it. +LOG_PATH = os.environ.get("TELEGRAM_LOG", "/var/ossec/logs/integrations.log") +VERIFY_SSL = os.environ.get("TELEGRAM_VERIFY_SSL", "yes").lower() in ("yes", "true", "1") +TIMEOUT = 15 + +SILENT_RULE = "100121" +RESTORED_RULE = "100122" + +_LOG_ARGS = {"format": "%(asctime)s custom-server-telegram %(levelname)s %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S", "level": logging.INFO} +try: + logging.basicConfig(filename=LOG_PATH, filemode="a", **_LOG_ARGS) +except OSError: + # An unwritable log must not cost us the notification. stderr is captured + # by integratord and ends up in ossec.log. + logging.basicConfig(stream=sys.stderr, **_LOG_ARGS) + + +def build_message(alert): + """Return the HTML message body for one alert.""" + data = alert.get("data", {}) + rule = alert.get("rule", {}) + rule_id = str(rule.get("id", "")) + + if rule_id == SILENT_RULE: + return (f"⚠ Server Logging Alert\n" + f"Name: {data.get('agent_name', 'unknown')}\n" + f"Agent ID: {data.get('agent_id', 'unknown')}\n" + f"Status: No logs received\n" + f"Last Log Received: {data.get('last_log', 'unknown')}\n" + f"No Logs For: {data.get('no_logs_for', 'unknown')}") + + if rule_id == RESTORED_RULE: + return (f"✅ Server Logging Restored\n" + f"Name: {data.get('agent_name', 'unknown')}\n" + f"Agent ID: {data.get('agent_id', 'unknown')}\n" + f"Status: Logs received\n" + f"Logging Restored At: {data.get('restored_at', 'unknown')}\n" + f"No Logs Duration: {data.get('silence_duration', 'unknown')}") + + agent = alert.get("agent", {}) + return (f"Wazuh alert\n" + f"Rule: {rule_id} (level {rule.get('level', '')})\n" + f"Description: {rule.get('description', '')}\n" + f"Agent: {agent.get('name', 'manager')} ({agent.get('id', '000')})") + + +def send(hook_url, chat_id, message): + payload = json.dumps({"chat_id": chat_id, "text": message, + "parse_mode": "HTML"}).encode() + req = urllib.request.Request(hook_url, data=payload, method="POST") + req.add_header("Content-Type", "application/json") + context = ssl.create_default_context() + if not VERIFY_SSL: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with urllib.request.urlopen(req, timeout=TIMEOUT, context=context) as resp: + return resp.status + + +def main(): + if len(sys.argv) < 2: + logging.error("Usage: %s [chat_id] [hook_url]", sys.argv[0]) + sys.exit(1) + + try: + with open(sys.argv[1]) as f: + alert = json.load(f) + except (OSError, ValueError) as err: + logging.error("Failed to read alert file '%s': %s", sys.argv[1], err) + sys.exit(1) + + chat_id = sys.argv[2] if len(sys.argv) > 2 and sys.argv[2] else CHAT_ID + hook_url = sys.argv[3] if len(sys.argv) > 3 and sys.argv[3] else HOOK_URL + if not chat_id or not hook_url: + logging.error("Missing chat ID or hook URL. Set and " + "in the block.") + sys.exit(1) + + message = build_message(alert) + try: + status = send(hook_url, chat_id, message) + except Exception as err: + logging.error("Telegram delivery failed for rule %s: %s", + alert.get("rule", {}).get("id", ""), err) + sys.exit(1) + + logging.info("Sent rule %s to chat %s (HTTP %s).", + alert.get("rule", {}).get("id", ""), chat_id, status) + + +if __name__ == "__main__": + main() diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml b/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml new file mode 100644 index 00000000..c39521a0 --- /dev/null +++ b/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml @@ -0,0 +1,37 @@ + + + + + + json + ^silent-agent-monitor$ + Silent agent monitor event + + + + + 100120 + ^SILENT$ + Server $(agent_name) (ID $(agent_id)) has sent no logs for $(no_logs_for) + alert_by_email + no_full_log + server_silent,service_availability, + + + + + 100120 + ^RESTORED$ + Server $(agent_name) (ID $(agent_id)) resumed logging after $(silence_duration) + alert_by_email + no_full_log + server_restored,service_availability, + + + diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py new file mode 100755 index 00000000..572b7f18 --- /dev/null +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -0,0 +1,309 @@ +#!/var/ossec/framework/python/bin/python3 +# +# silent_agent_monitor.py +# Detects Wazuh agents that are still registered (and often still "active") +# but have stopped shipping logs. For every agent in a target group it reads +# the timestamp of the most recent indexed event and, when that timestamp is +# older than the threshold, appends a SILENT record to a local JSON log that +# Wazuh ingests through a block. When events start arriving again +# it appends a matching RESTORED record. +# +# State is kept locally so a condition that stays unchanged is reported once, +# not once per run. Standard library only: it runs on the Wazuh embedded +# interpreter with no pip install. +# +# Run modes: +# silent_agent_monitor.py normal check (scheduled by a wodle) +# silent_agent_monitor.py --selftest offline assertions on the decision logic + +import base64 +import json +import logging +import os +import ssl +import sys +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +# === CONFIGURATION === +# Every value can be overridden with an environment variable, so the same file +# can be pointed at a test environment without being edited. +API_URL = os.environ.get("SAM_API_URL", "https://127.0.0.1:55000") +API_USER = os.environ.get("SAM_API_USER", "wazuh-wui") +API_PASSWORD = os.environ.get("SAM_API_PASSWORD", "CHANGE_ME") + +INDEXER_URL = os.environ.get("SAM_INDEXER_URL", "https://127.0.0.1:9200") +INDEXER_USER = os.environ.get("SAM_INDEXER_USER", "admin") +INDEXER_PASSWORD = os.environ.get("SAM_INDEXER_PASSWORD", "CHANGE_ME") + +# Index pattern holding the events used as proof of life. See the README: +# wazuh-alerts-* only contains alerts, wazuh-archives-* contains every event +# and is the accurate source when archives are enabled and indexed. +INDEX_PATTERN = os.environ.get("SAM_INDEX_PATTERN", "wazuh-alerts-*") + +TARGET_GROUP = os.environ.get("SAM_GROUP", "Server") +SILENCE_THRESHOLD = timedelta(hours=float(os.environ.get("SAM_THRESHOLD_HOURS", "24"))) + +# How far back the aggregation looks. Must exceed the threshold: an agent with +# no events inside this window is reported as silent for "more than" it. +LOOKBACK = timedelta(days=float(os.environ.get("SAM_LOOKBACK_DAYS", "7"))) + +STATE_FILE = os.environ.get("SAM_STATE_FILE", "/var/ossec/var/silent_agents_state.json") +OUTPUT_LOG = os.environ.get("SAM_OUTPUT_LOG", "/var/ossec/logs/silent_agents.json") +SCRIPT_LOG = os.environ.get("SAM_SCRIPT_LOG", "/var/ossec/logs/silent_agent_monitor.log") + +VERIFY_SSL = os.environ.get("SAM_VERIFY_SSL", "no").lower() in ("yes", "true", "1") +PAGE_SIZE = 500 +HTTP_TIMEOUT = 30 + +# === LOGGING === +_LOG_ARGS = {"format": "%(asctime)s %(levelname)s %(message)s", + "datefmt": "%Y-%m-%dT%H:%M:%S", "level": logging.INFO} +try: + logging.basicConfig(filename=SCRIPT_LOG, filemode="a", **_LOG_ARGS) +except OSError: + # Running as a user that cannot write the log file is not a reason to skip + # the check. stderr is picked up by whatever scheduled the run. + logging.basicConfig(stream=sys.stderr, **_LOG_ARGS) + +SSL_CONTEXT = ssl.create_default_context() +if not VERIFY_SSL: + SSL_CONTEXT.check_hostname = False + SSL_CONTEXT.verify_mode = ssl.CERT_NONE + + +def http_json(url, method="GET", body=None, token=None, basic=None): + """One JSON request. Raises on any transport or HTTP error.""" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Content-Type", "application/json") + if token: + req.add_header("Authorization", f"Bearer {token}") + if basic: + raw = base64.b64encode(f"{basic[0]}:{basic[1]}".encode()).decode() + req.add_header("Authorization", f"Basic {raw}") + with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT, context=SSL_CONTEXT) as resp: + return json.loads(resp.read().decode()) + + +def get_token(): + """Authenticate against the Wazuh API and return a JWT token.""" + url = f"{API_URL}/security/user/authenticate" + return http_json(url, method="POST", basic=(API_USER, API_PASSWORD))["data"]["token"] + + +def fetch_group_agents(token): + """Return every agent of TARGET_GROUP, excluding the manager and agents + that have never connected (those have no logs by definition).""" + agents, offset = [], 0 + while True: + url = (f"{API_URL}/agents?group={TARGET_GROUP}&limit={PAGE_SIZE}&offset={offset}" + f"&sort=%2Bid&select=id,name,status,lastKeepAlive") + data = http_json(url, token=token).get("data", {}) + agents.extend(a for a in data.get("affected_items", []) + if a.get("id") != "000" and a.get("status") != "never_connected") + offset += PAGE_SIZE + if offset >= data.get("total_affected_items", 0): + break + return agents + + +def fetch_last_event_times(agent_ids): + """One aggregation for every agent: newest event timestamp per agent.id. + Returns {agent_id: datetime}. Agents with no event in LOOKBACK are absent.""" + query = { + "size": 0, + "query": {"bool": {"filter": [ + {"terms": {"agent.id": agent_ids}}, + {"range": {"@timestamp": {"gte": f"now-{int(LOOKBACK.total_seconds())}s"}}}, + ]}}, + "aggs": {"per_agent": { + "terms": {"field": "agent.id", "size": len(agent_ids)}, + "aggs": {"last_event": {"max": {"field": "@timestamp"}}}, + }}, + } + url = f"{INDEXER_URL}/{INDEX_PATTERN}/_search" + result = http_json(url, method="POST", body=query, + basic=(INDEXER_USER, INDEXER_PASSWORD)) + # Missing aggregations means the query never matched an index. Return no + # buckets and let the caller's safety stop report it as a lookup problem. + buckets = result.get("aggregations", {}).get("per_agent", {}).get("buckets", []) + return {b["key"]: datetime.fromtimestamp(b["last_event"]["value"] / 1000, timezone.utc) + for b in buckets if b["last_event"]["value"]} + + +def format_duration(delta): + """'25h 40m', or '25h' on a whole hour. Matches the notification template.""" + minutes = int(delta.total_seconds() // 60) + hours, minutes = divmod(minutes, 60) + return f"{hours}h {minutes}m" if minutes else f"{hours}h" + + +def local_time(dt): + """Render a UTC datetime in the manager's local timezone, tz name included.""" + return dt.astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + + +def decide(agent, last_log, previous, now): + """Pure decision for one agent. Returns (event or None, new state entry). + + last_log is the newest indexed event time, or None when the agent produced + nothing inside LOOKBACK, which is the deepest form of silence. + previous is the state entry from the last run, or {}. + """ + agent_id, name = agent["id"], agent.get("name", "unknown") + silent = last_log is None or (now - last_log) >= SILENCE_THRESHOLD + was_silent = previous.get("status") == "SILENT" + + # The state key is named event_status, not status: "status" is one of the + # Wazuh static field names, and a rule cannot match it with . + common = { + "integration": "silent-agent-monitor", + "group": TARGET_GROUP, + "agent_id": agent_id, + "agent_name": name, + "agent_status": agent.get("status", "unknown"), + } + state = {"status": "SILENT" if silent else "OK", "name": name, + "last_log": last_log.isoformat() if last_log else previous.get("last_log")} + + if silent and not was_silent: + gap = (now - last_log) if last_log else LOOKBACK + event = dict(common, event_status="SILENT", + last_log=local_time(last_log) if last_log else "unknown", + no_logs_for=format_duration(gap) if last_log + else f"more than {format_duration(LOOKBACK)}", + no_logs_seconds=int(gap.total_seconds()), + message=f"Agent {name} (ID {agent_id}) has sent no logs " + f"for more than {format_duration(SILENCE_THRESHOLD)}.") + return event, state + + if not silent and was_silent: + # Measured from the last log before the gap to the first log after it, + # not from the moment this script noticed, so the duration is real. + previous_log = previous.get("last_log") + gap = (last_log - datetime.fromisoformat(previous_log)) if previous_log else None + event = dict(common, event_status="RESTORED", + restored_at=local_time(last_log), + silence_duration=format_duration(gap) if gap else "unknown", + silence_seconds=int(gap.total_seconds()) if gap else 0, + message=f"Agent {name} (ID {agent_id}) has resumed sending logs.") + return event, state + + return None, state + + +def load_state(): + try: + with open(STATE_FILE) as f: + return json.load(f) + except FileNotFoundError: + return {} + except (OSError, ValueError) as err: + # A corrupt state file must not stop the check. Worst case one repeat. + logging.error("Could not read state file '%s': %s. Starting empty.", STATE_FILE, err) + return {} + + +def save_state(state): + """Atomic replace, so a kill mid-write cannot leave a truncated state.""" + tmp = f"{STATE_FILE}.tmp" + with open(tmp, "w") as f: + json.dump(state, f, indent=2) + os.replace(tmp, STATE_FILE) + + +def append_events(events): + with open(OUTPUT_LOG, "a") as f: + for event in events: + f.write(json.dumps(event, ensure_ascii=False) + "\n") + + +def main(): + now = datetime.now(timezone.utc) + try: + agents = fetch_group_agents(get_token()) + except (urllib.error.URLError, OSError, KeyError, ValueError) as err: + logging.error("Wazuh API query failed: %s", err) + sys.exit(1) + + if not agents: + logging.info("No agents in group '%s'. Nothing to do.", TARGET_GROUP) + return + + agent_ids = [a["id"] for a in agents] + try: + last_events = fetch_last_event_times(agent_ids) + except (urllib.error.URLError, OSError, KeyError, ValueError) as err: + # Exit without touching the state: a failed query must never be read as + # "every agent went silent", nor as "every agent recovered". + logging.error("Indexer query failed: %s", err) + sys.exit(1) + + if not last_events and len(agents) > 1: + # Every single agent silent at once is far more likely to be a wrong + # index pattern or wrong credentials than a real outage. Refuse to + # generate the storm and make the operator look. + logging.error("No events found for any of the %d agents in '%s' over the last %s. " + "Check SAM_INDEX_PATTERN and the indexer credentials. No alerts sent.", + len(agents), TARGET_GROUP, format_duration(LOOKBACK)) + sys.exit(1) + + state = load_state() + events, new_state = [], {} + for agent in agents: + event, entry = decide(agent, last_events.get(agent["id"]), + state.get(agent["id"], {}), now) + new_state[agent["id"]] = entry + if event: + events.append(event) + + if events: + append_events(events) + save_state(new_state) + + silent = sum(1 for e in new_state.values() if e["status"] == "SILENT") + logging.info("Checked %d agent(s) in '%s': %d silent, %d new event(s) written.", + len(agents), TARGET_GROUP, silent, len(events)) + print(f"Checked {len(agents)} agent(s) in '{TARGET_GROUP}': " + f"{silent} silent, {len(events)} event(s) written to {OUTPUT_LOG}.") + + +def selftest(): + """Offline assertions on the decision logic. No API, no indexer.""" + now = datetime(2026, 8, 19, 10, 20, 0, tzinfo=timezone.utc) + agent = {"id": "152", "name": "File2", "status": "active"} + + # Quiet for 25h40m: reported once, then suppressed while unchanged. + stopped = now - timedelta(hours=25, minutes=40) + event, state = decide(agent, stopped, {}, now) + assert event["event_status"] == "SILENT", event + assert event["no_logs_for"] == "25h 40m", event + assert event["agent_id"] == "152" and event["agent_name"] == "File2" + assert state["status"] == "SILENT" + assert decide(agent, stopped, state, now)[0] is None, "repeat alert not suppressed" + + # Logs resume: one recovery, measured from the last log before the gap. + resumed = stopped + timedelta(hours=25, minutes=40) + event, ok_state = decide(agent, resumed, state, now) + assert event["event_status"] == "RESTORED", event + assert event["silence_duration"] == "25h 40m", event + assert ok_state["status"] == "OK" + assert decide(agent, resumed, ok_state, now)[0] is None, "repeat recovery not suppressed" + + # A healthy agent inside the threshold never reports. + assert decide(agent, now - timedelta(hours=23), {}, now)[0] is None + + # No events at all inside the lookback window is silence, not a skip. + event, _ = decide(agent, None, {}, now) + assert event["event_status"] == "SILENT" and event["last_log"] == "unknown", event + + print("selftest OK") + + +if __name__ == "__main__": + if "--selftest" in sys.argv: + selftest() + else: + main() From 989c1607c32cb840d14f1809cfb72b836fd0943b Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Fri, 28 Aug 2026 16:04:22 +0200 Subject: [PATCH 2/8] Route the silence events into an existing Telegram integration Adds status_text to both records, so a dispatcher that maps dotted paths to labels can render the Status line without a literal of its own, and documents the two dispatch-table entries plus the per-entry header override that reproduce the requested layout on an existing custom-telegram script. The bundled script is now the option for environments with no Telegram integration yet. Co-Authored-By: Claude Opus 5 --- .../silent_agent_monitoring/README.md | 129 ++++++++++++++---- .../custom-server-telegram.py | 6 +- .../silent_agent_monitor.py | 4 + 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index cb468ccf..d669e65a 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -15,6 +15,7 @@ * [Rules](#rules) * [Email Notifications](#email-notifications) * [Telegram Notifications](#telegram-notifications) + * [Routing Into an Existing Telegram Integration](#routing-into-an-existing-telegram-integration) * [Integration Steps](#integration-steps) * [Integration Testing](#integration-testing) * [Troubleshooting](#troubleshooting) @@ -26,17 +27,7 @@ ### Introduction -Wazuh alerts natively when an agent stops connecting: `wazuh-monitord` marks the -agent disconnected after `` and rule 502 fires. That -covers the agent that goes away. It does not cover the agent that stays -connected, keeps answering keepalives, and quietly stops shipping logs, because -a log collector died, a log path rotated away, a service stopped writing, or a -permission changed. From the manager's point of view that agent is healthy. - -This integration closes that gap. On a schedule it reads, for every agent of a -chosen group, the timestamp of the most recent event that reached the indexer. -When that timestamp is older than the threshold it writes a `SILENT` record; -when events start arriving again it writes one `RESTORED` record. The records +On a schedule it reads, for every agent of a chosen group, the timestamp of the most recent event that reached the indexer. When that timestamp is older than the threshold it writes a SILENT record and when events start arriving again it writes one RESTORED record. The records are plain JSON lines that Wazuh ingests through a `` block, so they become normal alerts and can be routed to email, Telegram, or anything else with the standard `` and `` blocks. @@ -304,11 +295,11 @@ the numeric chat ID of the channel, both taken from the Telegram block already in the configuration. The script produces exactly the requested layout: ``` -⚠ Server Logging Alert ✅ Server Logging Restored +⚠️ Server Logging Alert ✅ Server Logging Restored Name: File2 Name: File2 Agent ID: 152 Agent ID: 152 Status: No logs received Status: Logs received -Last Log Received: ... Logging Restored At: ... +Last Log Received: ... Restored At: ... No Logs For: 25h 40m No Logs Duration: 25h 40m ``` @@ -317,24 +308,104 @@ so the script logs to `/var/ossec/logs/integrations.log`, which that user can already write. If `TELEGRAM_LOG` is pointed somewhere else, the new file has to be writable by `wazuh` or the notification is lost before it is sent. -A separate script is used rather than a change to the existing `custom-telegram` -so that the current Telegram alerting keeps working untouched. To format these -two rules inside the existing script instead, add the branch before its normal -message construction and skip this file: +Use these two files only when there is no Telegram integration yet. When one is +already configured, keep it and see the next section instead. + +#### Routing Into an Existing Telegram Integration + +Two changes, no new script. + +**1. Send the rules to the existing block.** In the +`` integration, extend whichever +selector it already uses: + +```xml +...existing ids...,100121,100122 +``` + +or, if it selects by rule group: + +```xml +...existing groups...,silent_agent_monitoring +``` + +**2. Add the two message layouts.** A script that dispatches on `rule.groups` +needs no rule IDs at all: the rules already carry `server_silent` and +`server_restored`. One declarative entry per event type, in the dispatch table: ```python -if str(alert.get("rule", {}).get("id")) in ("100121", "100122"): - d = alert.get("data", {}) - if d.get("event_status") == "SILENT": - msg = (f"⚠ Server Logging Alert\nName: {d.get('agent_name')}\n" - f"Agent ID: {d.get('agent_id')}\nStatus: No logs received\n" - f"Last Log Received: {d.get('last_log')}\n" - f"No Logs For: {d.get('no_logs_for')}") - else: - msg = (f"✅ Server Logging Restored\nName: {d.get('agent_name')}\n" - f"Agent ID: {d.get('agent_id')}\nStatus: Logs received\n" - f"Logging Restored At: {d.get('restored_at')}\n" - f"No Logs Duration: {d.get('silence_duration')}") +{ + "match": "server_silent", + "header": "⚠️ Server Logging Alert", + "label": "Server Logging Alert", + "fields": [ + ("Name", "data.agent_name"), + ("Agent ID", "data.agent_id"), + ("Status", "data.status_text"), + ("Last Log Received", "data.last_log"), + ("No Logs For", "data.no_logs_for"), + ], +}, +{ + "match": "server_restored", + "header": "✅ Server Logging Restored", + "label": "Server Logging Restored", + "fields": [ + ("Name", "data.agent_name"), + ("Agent ID", "data.agent_id"), + ("Status", "data.status_text"), + ("Restored At", "data.restored_at"), + ("No Logs Duration", "data.silence_duration"), + ], +}, +``` + +Put them above any broader entry that could also match. The `Status` line is a +field, `data.status_text`, rather than a literal in the dispatcher, so the +wording lives in one place and a script that only knows how to map paths to +labels needs no code for it. + +If the dispatcher renders a fixed header, give it a per-entry override. On a +formatter of the common shape that is two lines: + +```python +def build_fields_message(alert, label, fields, header=None): + lines = [header or f"\U0001F6A8 Wazuh Alert - {label}", ""] +``` + +```python + return build_fields_message(alert, group_cfg.get("label", group_cfg["match"]), + group_cfg["fields"], group_cfg.get("header")) +``` + +Entries without a `header` key keep the old heading, so every existing +notification format is untouched. + +**Check the formatting without waiting for an alert.** Take a real record, or +write one by hand, and run it through the script's own formatter: + +```bash +grep '"id":"100121"' /var/ossec/logs/alerts/alerts.json | tail -1 > /tmp/sample_silent.json + +/var/ossec/framework/python/bin/python3 - <<'EOF' +import json, importlib.util +spec = importlib.util.spec_from_file_location("tg", "/var/ossec/integrations/custom-telegram.py") +tg = importlib.util.module_from_spec(spec); spec.loader.exec_module(tg) +print(tg.build_message(json.load(open("/tmp/sample_silent.json")))[0]) +EOF +``` + +Importing the file does not send anything: the send path only runs under +`if __name__ == "__main__"`. Expected output: + +``` +⚠️ Server Logging Alert + +Name: File2 +Agent ID: 152 +Status: No logs received +Last Log Received: 2026-08-25 10:30:00 +No Logs For: 24h ``` --- diff --git a/integrations/silent_agent_monitoring/custom-server-telegram.py b/integrations/silent_agent_monitoring/custom-server-telegram.py index 0863e56e..eea6428c 100755 --- a/integrations/silent_agent_monitoring/custom-server-telegram.py +++ b/integrations/silent_agent_monitoring/custom-server-telegram.py @@ -55,7 +55,7 @@ def build_message(alert): return (f"⚠ Server Logging Alert\n" f"Name: {data.get('agent_name', 'unknown')}\n" f"Agent ID: {data.get('agent_id', 'unknown')}\n" - f"Status: No logs received\n" + f"Status: {data.get('status_text', 'No logs received')}\n" f"Last Log Received: {data.get('last_log', 'unknown')}\n" f"No Logs For: {data.get('no_logs_for', 'unknown')}") @@ -63,8 +63,8 @@ def build_message(alert): return (f"✅ Server Logging Restored\n" f"Name: {data.get('agent_name', 'unknown')}\n" f"Agent ID: {data.get('agent_id', 'unknown')}\n" - f"Status: Logs received\n" - f"Logging Restored At: {data.get('restored_at', 'unknown')}\n" + f"Status: {data.get('status_text', 'Logs received')}\n" + f"Restored At: {data.get('restored_at', 'unknown')}\n" f"No Logs Duration: {data.get('silence_duration', 'unknown')}") agent = alert.get("agent", {}) diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 572b7f18..9b8e377a 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -171,6 +171,7 @@ def decide(agent, last_log, previous, now): if silent and not was_silent: gap = (now - last_log) if last_log else LOOKBACK event = dict(common, event_status="SILENT", + status_text="No logs received", last_log=local_time(last_log) if last_log else "unknown", no_logs_for=format_duration(gap) if last_log else f"more than {format_duration(LOOKBACK)}", @@ -185,6 +186,7 @@ def decide(agent, last_log, previous, now): previous_log = previous.get("last_log") gap = (last_log - datetime.fromisoformat(previous_log)) if previous_log else None event = dict(common, event_status="RESTORED", + status_text="Logs received", restored_at=local_time(last_log), silence_duration=format_duration(gap) if gap else "unknown", silence_seconds=int(gap.total_seconds()) if gap else 0, @@ -279,6 +281,7 @@ def selftest(): stopped = now - timedelta(hours=25, minutes=40) event, state = decide(agent, stopped, {}, now) assert event["event_status"] == "SILENT", event + assert event["status_text"] == "No logs received", event assert event["no_logs_for"] == "25h 40m", event assert event["agent_id"] == "152" and event["agent_name"] == "File2" assert state["status"] == "SILENT" @@ -288,6 +291,7 @@ def selftest(): resumed = stopped + timedelta(hours=25, minutes=40) event, ok_state = decide(agent, resumed, state, now) assert event["event_status"] == "RESTORED", event + assert event["status_text"] == "Logs received", event assert event["silence_duration"] == "25h 40m", event assert ok_state["status"] == "OK" assert decide(agent, resumed, ok_state, now)[0] is None, "repeat recovery not suppressed" From 9e42e6aa17a7eec212f86af0da3aa4e08dfe914a Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Tue, 1 Sep 2026 12:22:47 +0200 Subject: [PATCH 3/8] Fix TLS trust and selftest in silent agent monitoring, trim the README The Telegram script built a default SSL context, which trusts nothing on the Wazuh embedded interpreter because its OpenSSL looks for roots in a path that does not exist, so every delivery failed with CERTIFICATE_VERIFY_FAILED. It now prefers the certifi bundle that ships with that interpreter and falls back to the default store elsewhere, with verification still on. --selftest asserted against the shipped 24h threshold while reading the module globals, so an install with a shorter window failed the check on valid logic. The assertions now pin the threshold and lookback they were written for. A zero-length gap made silence_duration render as "unknown" through a truthiness test on a timedelta. The README is cut to the shape the other integrations use: install, configure, schedule, rules, routing, test, troubleshoot. --- .../silent_agent_monitoring/README.md | 521 +++--------------- .../custom-server-telegram.py | 27 +- .../silent_agent_monitor.py | 10 +- 3 files changed, 116 insertions(+), 442 deletions(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index d669e65a..b4f43d4b 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -1,162 +1,48 @@ -# Silent Agent Monitoring - Wazuh Integration +# Silent Agent Monitoring ## Table of Contents - * [Introduction](#introduction) * [Prerequisites](#prerequisites) -* [How It Works](#how-it-works) - * [What Counts as a Log](#what-counts-as-a-log) - * [Alert and Recovery Logic](#alert-and-recovery-logic) -* [Installation and Configuration](#installation-and-configuration) - * [Using the Integration Files](#using-the-integration-files) - * [Script Configuration](#script-configuration) - * [Scheduling the Check](#scheduling-the-check) - * [Ingesting the Records](#ingesting-the-records) - * [Rules](#rules) - * [Email Notifications](#email-notifications) - * [Telegram Notifications](#telegram-notifications) - * [Routing Into an Existing Telegram Integration](#routing-into-an-existing-telegram-integration) * [Integration Steps](#integration-steps) -* [Integration Testing](#integration-testing) + * [Add the integration files](#add-the-integration-files) + * [Script configuration](#script-configuration) + * [Wazuh manager configuration](#wazuh-manager-configuration) + * [Add custom rules](#add-custom-rules) + * [Email notifications](#email-notifications) + * [Telegram notifications](#telegram-notifications) +* [Testing](#testing) * [Troubleshooting](#troubleshooting) -* [Verification](#verification) -* [Design Notes](#design-notes) * [Sources](#sources) ---- - -### Introduction - -On a schedule it reads, for every agent of a chosen group, the timestamp of the most recent event that reached the indexer. When that timestamp is older than the threshold it writes a SILENT record and when events start arriving again it writes one RESTORED record. The records -are plain JSON lines that Wazuh ingests through a `` block, so they -become normal alerts and can be routed to email, Telegram, or anything else -with the standard `` and `` blocks. - -The answer to "can this be done from the group and the last event timestamp, or -is a custom script needed": the group and the timestamp are exactly the right -inputs, and a script is needed to join them, because no built-in module tracks -per-agent event recency. Everything downstream of the script (decoding, rules, -alerting, routing) is stock Wazuh. - ---- - -### Prerequisites - -- Wazuh manager 4.4 or later, with the Wazuh API reachable and an API user that - can read `/agents`. -- Wazuh indexer reachable from the manager, with a user that can search the - alerts (or archives) indices. -- An agent group to monitor. The examples use `Server`. -- Python 3.6 or later. The scripts use only the standard library, so there is no - `pip install` step; the Wazuh embedded interpreter at - `/var/ossec/framework/python/bin/python3` satisfies this. -- Filesystem access to the manager to place the files. - -**Wazuh Cloud:** managed environments do not give shell access to the manager, -so the two scripts cannot be copied in by the user. Rules, `ossec.conf` blocks, -and the group can be managed from the dashboard, but the script placement and -its execute permissions have to be done by the Wazuh Cloud team through a -support request. Send them this folder and the target paths listed below. In a -cluster, the script, its state file, and the `` block must be placed -on one node only (the master); running it on several nodes duplicates every -notification and splits the state. - ---- - -### How It Works - -``` - Wazuh API /agents?group=Server Wazuh indexer - | | - | agent id, name, status | max(@timestamp) per agent.id - v v - +--------------------------------------------------+ - command wodle ->| silent_agent_monitor.py | - (hourly) | compares each agent against the threshold and | - | against the previous run's state | - +--------------------------------------------------+ - | - | one JSON line per state change only - v - /var/ossec/logs/silent_agents.json - | - | json - v - rules 100121 / 100122 -> email + Telegram -``` - -#### What Counts as a Log - -The script measures recency against an index pattern, `SAM_INDEX_PATTERN`: - -| Pattern | Meaning | Trade-off | -| --- | --- | --- | -| `wazuh-alerts-*` (default) | The newest **alert** produced by the agent. | Available everywhere. An agent that ships logs normally but produces no alert for a full day is reported as silent. | -| `wazuh-archives-*` | The newest **event** received from the agent, whether or not it alerted. | Exact answer to "no logs received", but needs `` enabled and the archives indexed, which costs storage. | - -Use archives when they are enabled. On alerts, confirm first that every agent in -the group normally produces at least some alerts within the threshold; a quiet -Windows file server under a tight ruleset sometimes does not. Widening the -threshold or moving to archives both remove that false positive. +## Introduction +This script runs on the Wazuh manager and detects agents that are still registered, and often still active, while log ingestion from them has stopped. -#### Alert and Recovery Logic +For every agent of a target group it reads the timestamp of the most recent indexed event. When that timestamp is older than the threshold it appends a `SILENT` record to a local JSON log, and when events start arriving again it appends a `RESTORED` record. State is kept locally so an unchanged condition is reported once, not once per run. -State is kept in a small local JSON file, so a condition that has not changed is -reported once rather than once per run: +The records are plain JSON lines ingested through a `` block, so the built-in JSON decoder handles them and no custom decoder is needed. They trigger rules 100121 and 100122, which are routed to email and to Telegram. -| Previous state | Current reading | Action | -| --- | --- | --- | -| OK (or unknown) | Last event older than the threshold, or no event at all in the lookback window | Write one `SILENT` record, remember the last log timestamp. | -| SILENT | Still older than the threshold | Nothing. No repeated notification. | -| SILENT | Recent events again | Write one `RESTORED` record, clear the state. | -| OK | Recent events | Nothing. | - -Durations are measured against real log timestamps, not against the moment the -script noticed. `No Logs For` is the gap between the last received log and now. -`No Logs Duration` on recovery is the gap between the last log before the -silence and the first log after it, which is what the operator actually wants to -read in the incident. - -An agent that has never connected is skipped: it has no logs by definition, and -`never_connected` is already visible in the dashboard. Agent `000` (the manager) -is skipped too. +## Prerequisites +- Wazuh manager 4.4 or later, with an API user that can read `/agents`. +- Wazuh indexer reachable from the manager, with a user that can search the alerts (or archives) indices. +- An agent group to monitor. +- The scripts use only the standard library, so the Wazuh embedded interpreter at `/var/ossec/framework/python/bin/python3` is enough and there is no `pip install` step. ---- +In a cluster, install on the master node only: running the script on several nodes duplicates every notification and splits the state. -### Installation and Configuration +## Integration Steps -#### Using the Integration Files - -``` -silent_agent_monitoring/ - silent_agent_monitor.py # The check. Runs on a schedule from a wodle. - silent_agent_monitor-rules.xml # Rules 100120-100122. - custom-server-telegram # Integration wrapper (selects the Wazuh interpreter). - custom-server-telegram.py # Formats and posts the Telegram message. +### Add the integration files ``` - -Target paths on the manager: - -```bash cp silent_agent_monitor.py /var/ossec/wodles/ -chmod 750 /var/ossec/wodles/silent_agent_monitor.py -chown root:wazuh /var/ossec/wodles/silent_agent_monitor.py - cp custom-server-telegram custom-server-telegram.py /var/ossec/integrations/ -chmod 750 /var/ossec/integrations/custom-server-telegram* -chown root:wazuh /var/ossec/integrations/custom-server-telegram* -cat silent_agent_monitor-rules.xml >> /var/ossec/etc/rules/local_rules.xml +chown root:wazuh /var/ossec/wodles/silent_agent_monitor.py /var/ossec/integrations/custom-server-telegram* +chmod 750 /var/ossec/wodles/silent_agent_monitor.py /var/ossec/integrations/custom-server-telegram* ``` +A manager upgrade can replace the contents of `/var/ossec/wodles`, so keep a copy of the configured script outside `/var/ossec`. -A manager upgrade can replace the contents of `/var/ossec/wodles`, so keep a -copy of the configured script outside `/var/ossec` and re-apply it after an -upgrade. - -#### Script Configuration - -Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set -the matching environment variables and leave the file untouched: +### Script configuration +Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set the matching environment variables and leave the file untouched: | Setting | Variable | Default | | --- | --- | --- | @@ -173,60 +59,33 @@ the matching environment variables and leave the file untouched: | Script log | `SAM_SCRIPT_LOG` | `/var/ossec/logs/silent_agent_monitor.log` | | Verify TLS certificates | `SAM_VERIFY_SSL` | `no` | -The file holds credentials, so keep it `chmod 750` and root-owned. On Wazuh -Cloud, use the environment endpoints and credentials supplied with the -environment rather than the loopback defaults. - -`SAM_LOOKBACK_DAYS` must stay larger than the threshold. It bounds the indexer -query, and an agent with nothing inside it is reported as silent for "more than" -that window. +The file holds credentials, so keep it root-owned and `chmod 750`. `SAM_LOOKBACK_DAYS` must stay larger than the threshold: it bounds the indexer query, and an agent with nothing inside it is reported as silent for "more than" that window. -#### Scheduling the Check +The index pattern decides what counts as a log. `wazuh-alerts-*` is available everywhere but only sees alerts, so an agent that ships logs normally while producing no alert for a full day is reported as silent. `wazuh-archives-*` is the exact answer to "no logs received" but needs `` enabled and the archives indexed. Use archives when they are available; otherwise confirm that every agent in the group normally produces alerts within the threshold. -`/var/ossec/etc/ossec.conf`, on the master node only: +### Wazuh manager configuration +Add to `/var/ossec/etc/ossec.conf`: ```xml - - - no - silent-agent-monitor - /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py - 1h - yes - 300 - yes - - -``` - -With `run_on_start`, the very first run after a manager restart can reach the -Wazuh API before it finishes starting and log `HTTP Error 500`. That run exits -non-zero, `wazuh-modulesd` records a warning, and the next scheduled run -succeeds. Nothing is lost, because a failed run never writes state. - -One run per hour is enough for a 24 hour threshold: it bounds detection lag and -recovery lag to an hour each while keeping the indexer load at one aggregation -query per hour, whatever the number of agents. Shorten the interval if the -recovery notification needs to arrive sooner. - -#### Ingesting the Records - -The script writes plain JSON objects, one per line, so the built-in JSON decoder -parses them and **no custom decoder is required**: - -```xml - - - json - /var/ossec/logs/silent_agents.json - - -``` - -#### Rules - -`silent_agent_monitor-rules.xml` defines a level 0 parent that matches the -`integration` field and two children that alert: + + no + silent-agent-monitor + /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py + 1h + yes + 300 + yes + + + + json + /var/ossec/logs/silent_agents.json + +``` +One run per hour bounds detection lag and recovery lag to an hour each, at one indexer query per hour whatever the number of agents. With `run_on_start`, the first run after a restart can reach the API before it finishes starting and log `HTTP Error 500`; nothing is lost, because a failed run never writes state. + +### Add custom rules +In the Wazuh dashboard go to Server Management > Rules > Add new rules file, name it `silent_agent_monitor-rules.xml`, add the content of `silent_agent_monitor-rules.xml` and save. Then restart the manager. | Rule | Level | Fires when | | --- | --- | --- | @@ -234,65 +93,34 @@ parses them and **no custom decoder is required**: | 100121 | 12 | `event_status` is `SILENT`. | | 100122 | 5 | `event_status` is `RESTORED`. | -The matched field is `event_status`, not `status`: `status` is one of the Wazuh -static field names, and a rule that tries to match it with `` -fails to load with `Field 'status' is static`. - -Both children carry `alert_by_email`, which forces the email -regardless of the global ``. Without it the level 5 recovery -alert would be dropped by the default threshold of 12 and only the silence -notification would arrive. - -Move the IDs into a free range if 100120-100122 are already used; the repository -`detect_new_agents` integration, for example, also ships a rule 100110. +The matched field is `event_status`, not `status`: `status` is a static Wazuh field name and a rule matching it fails to load. Both children carry `alert_by_email`, which forces the email regardless of the global ``; without it the level 5 recovery alert is dropped by the default threshold of 12. Move the IDs into a free range if 100120-100122 are already used. -#### Email Notifications - -Global email must already be configured (`` with -`yes`, ``, ``, -``). Then route these two rules: +### Email notifications +Global email must already be configured. Then route the two rules: ```xml - - - soc-team@example.com - 100121,100122 - - full - - + + soc-team@example.com + 100121,100122 + + full + ``` +The `full` format prints the record's fields one per line, so the email already carries the agent name, the agent ID, the last log timestamp and the duration. -`` sends immediately instead of waiting for the next email -grouping interval. - -The `full` format prints the record's fields one per line, so the email already -carries the agent name, the agent ID, the last log timestamp and the duration. -Only if the email has to look like the Telegram message, with the same heading -and emoji, is a `custom-email` integration script needed in place of -``. - -#### Telegram Notifications - -Add the integration next to the existing Telegram block, reusing the bot token -and chat ID of the Server channel: +### Telegram notifications +Use the bundled script only when there is no Telegram integration yet: ```xml - - - - custom-server-telegram - 100121,100122 - https://api.telegram.org/bot<BOT_TOKEN>/sendMessage - <CHAT_ID> - json - - + + custom-server-telegram + 100121,100122 + https://api.telegram.org/bot<BOT_TOKEN>/sendMessage + <CHAT_ID> + json + ``` - -`` is the full `sendMessage` endpoint of the bot and `` is -the numeric chat ID of the channel, both taken from the Telegram block already -in the configuration. The script produces exactly the requested layout: +`` is the full `sendMessage` endpoint of the bot and `` is the numeric chat ID. Both are passed to the script as arguments and override its defaults. The messages it produces: ``` ⚠️ Server Logging Alert ✅ Server Logging Restored @@ -303,227 +131,50 @@ Last Log Received: ... Restored At: ... No Logs For: 25h 40m No Logs Duration: 25h 40m ``` -`wazuh-integratord` runs integration scripts as the `wazuh` user, not as root, -so the script logs to `/var/ossec/logs/integrations.log`, which that user can -already write. If `TELEGRAM_LOG` is pointed somewhere else, the new file has to -be writable by `wazuh` or the notification is lost before it is sent. +When a Telegram integration already exists, keep it and add `100121,100122` to its `` (or `silent_agent_monitoring` to its ``). A dispatcher that maps dotted paths to labels needs one entry per event type, matching on the `server_silent` and `server_restored` rule groups, with the fields listed above; the `Status` line is `data.status_text`, so no literal is needed in the dispatcher. -Use these two files only when there is no Telegram integration yet. When one is -already configured, keep it and see the next section instead. - -#### Routing Into an Existing Telegram Integration - -Two changes, no new script. - -**1. Send the rules to the existing block.** In the -`` integration, extend whichever -selector it already uses: - -```xml -...existing ids...,100121,100122 -``` - -or, if it selects by rule group: - -```xml -...existing groups...,silent_agent_monitoring -``` - -**2. Add the two message layouts.** A script that dispatches on `rule.groups` -needs no rule IDs at all: the rules already carry `server_silent` and -`server_restored`. One declarative entry per event type, in the dispatch table: - -```python -{ - "match": "server_silent", - "header": "⚠️ Server Logging Alert", - "label": "Server Logging Alert", - "fields": [ - ("Name", "data.agent_name"), - ("Agent ID", "data.agent_id"), - ("Status", "data.status_text"), - ("Last Log Received", "data.last_log"), - ("No Logs For", "data.no_logs_for"), - ], -}, -{ - "match": "server_restored", - "header": "✅ Server Logging Restored", - "label": "Server Logging Restored", - "fields": [ - ("Name", "data.agent_name"), - ("Agent ID", "data.agent_id"), - ("Status", "data.status_text"), - ("Restored At", "data.restored_at"), - ("No Logs Duration", "data.silence_duration"), - ], -}, -``` - -Put them above any broader entry that could also match. The `Status` line is a -field, `data.status_text`, rather than a literal in the dispatcher, so the -wording lives in one place and a script that only knows how to map paths to -labels needs no code for it. - -If the dispatcher renders a fixed header, give it a per-entry override. On a -formatter of the common shape that is two lines: - -```python -def build_fields_message(alert, label, fields, header=None): - lines = [header or f"\U0001F6A8 Wazuh Alert - {label}", ""] -``` - -```python - return build_fields_message(alert, group_cfg.get("label", group_cfg["match"]), - group_cfg["fields"], group_cfg.get("header")) -``` - -Entries without a `header` key keep the old heading, so every existing -notification format is untouched. - -**Check the formatting without waiting for an alert.** Take a real record, or -write one by hand, and run it through the script's own formatter: - -```bash -grep '"id":"100121"' /var/ossec/logs/alerts/alerts.json | tail -1 > /tmp/sample_silent.json - -/var/ossec/framework/python/bin/python3 - <<'EOF' -import json, importlib.util -spec = importlib.util.spec_from_file_location("tg", "/var/ossec/integrations/custom-telegram.py") -tg = importlib.util.module_from_spec(spec); spec.loader.exec_module(tg) -print(tg.build_message(json.load(open("/tmp/sample_silent.json")))[0]) -EOF -``` - -Importing the file does not send anything: the send path only runs under -`if __name__ == "__main__"`. Expected output: - -``` -⚠️ Server Logging Alert - -Name: File2 -Agent ID: 152 -Status: No logs received -Last Log Received: 2026-08-25 10:30:00 -No Logs For: 24h -``` - ---- - -### Integration Steps - -1. Confirm the agents to monitor are in the group: `/var/ossec/bin/agent_groups -s -g Server`. -2. Copy the four files to the paths above and set ownership and permissions. -3. Fill in the API and indexer credentials, the group name, and the threshold. -4. Append the rules to `/var/ossec/etc/rules/local_rules.xml`. -5. Add the ``, ``, ``, and `` blocks - to `/var/ossec/etc/ossec.conf` on the master node. -6. Validate the configuration and restart: `/var/ossec/bin/wazuh-control restart`. -7. Watch `/var/ossec/logs/silent_agent_monitor.log` after the first run. - ---- - -### Integration Testing - -**Decision logic, offline.** No API, indexer, or manager needed: +`wazuh-integratord` runs integration scripts as the `wazuh` user, so any path the script writes, including a custom `TELEGRAM_LOG`, must be writable by it. +## Testing +Offline assertions on the decision logic, no API or indexer needed: ```bash /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --selftest # selftest OK ``` -It asserts that a 25h40m gap reports once and only once, that recovery reports -once with the duration measured from the last log before the gap, that an agent -inside the threshold stays quiet, and that an agent with no events at all is -treated as silent rather than skipped. - -**End to end, against the live environment.** Run the check by hand: - +Run the check by hand against the live environment: ```bash /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py # Checked 12 agent(s) in 'Server': 0 silent, 0 event(s) written to /var/ossec/logs/silent_agents.json. ``` -To force a notification without waiting a day, drop the threshold for one run -and watch the whole chain fire: - +To force a notification without waiting, drop the threshold for one run: ```bash -SAM_THRESHOLD_HOURS=0.05 /var/ossec/framework/python/bin/python3 \ - /var/ossec/wodles/silent_agent_monitor.py +SAM_THRESHOLD_HOURS=0.05 /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py tail -1 /var/ossec/logs/silent_agents.json tail -f /var/ossec/logs/alerts/alerts.log | grep -A5 100121 ``` +Running with the real threshold again produces the `RESTORED` notification, which confirms the recovery path and the message formatting in one go. Delete `/var/ossec/var/silent_agents_state.json` afterwards so the test does not leave agents marked silent. -Delete `/var/ossec/var/silent_agents_state.json` afterwards so the test does not -leave agents marked silent. Running with the real threshold again produces the -`RESTORED` notification, which is a useful way to confirm the recovery path and -the Telegram formatting in one go. - -**Rules only**, without running the script: - +The rules can be checked without running the script: ```bash echo '{"integration":"silent-agent-monitor","event_status":"SILENT","agent_id":"152","agent_name":"File2","last_log":"2026-08-18 08:35:12 CEST","no_logs_for":"25h 40m"}' \ | /var/ossec/bin/wazuh-logtest ``` ---- - -### Troubleshooting - +## Troubleshooting | Symptom | Cause and fix | | --- | --- | -| `No events found for any of the N agents` in the script log, and no alerts | Deliberate safety stop. Every agent silent at once is almost always a wrong index pattern or wrong indexer credentials, not a real outage, so the script refuses to send the storm. Check `SAM_INDEX_PATTERN` and the indexer user. | -| `Indexer query failed` or `Wazuh API query failed` | The run exits without touching the state, so nothing is reported as silent or as recovered on the strength of a failed query. Check connectivity and credentials. | -| Records in `silent_agents.json` but no alerts | The `` block is missing, points elsewhere, or sits on a node that is not running the script. Confirm with `grep silent_agents /var/ossec/logs/ossec.log`. | +| `No events found for any of the N agents` and no alerts | Deliberate safety stop: every agent silent at once is almost always a wrong index pattern or wrong indexer credentials. Check `SAM_INDEX_PATTERN` and the indexer user. | +| `Indexer query failed` or `Wazuh API query failed` | The run exits without touching the state, so nothing is reported as silent or recovered on a failed query. Check connectivity and credentials. | +| Records in `silent_agents.json` but no alerts | The `` block is missing, points elsewhere, or sits on a node that is not running the script. | | Alerts fire but no email | Global email is not enabled, or the rules lost `alert_by_email`. Check `/var/ossec/logs/ossec.log` for `wazuh-maild`. | -| Alerts fire but no Telegram message | Check `/var/ossec/logs/integrations.log` for a line from `custom-server-telegram`, then `grep integrator /var/ossec/logs/ossec.log`. A missing chat ID or hook URL, or an HTTP error from the bot API, is logged with the rule ID. | -| `Permission denied` from integratord | The integration runs as the `wazuh` user. Any path the script writes, including a custom `TELEGRAM_LOG`, must be writable by it. | -| `Failure to read rule 100121. Field 'status' is static` | The rule was edited to match `status` instead of `event_status`. `status` is a reserved Wazuh field name. | -| A healthy agent is reported silent | It produced no *alerts* within the threshold. Point `SAM_INDEX_PATTERN` at `wazuh-archives-*`, or raise the threshold. | -| Every agent reported again after a manager rebuild | The state file was lost, so the first run after it re-reports the conditions that are still true. One repeat, then quiet again. | - ---- - -### Verification - -Run end to end on a Wazuh 4.14.6 single-node server (manager, indexer and -dashboard on one host) with three agents in a `Server` group: - -| Check | Result | -| --- | --- | -| `--selftest` on the embedded interpreter | Passes: single alert, single recovery, correct durations, silence on missing data. | -| Group lookup | The `never_connected` agent and agent `000` are excluded; the two real agents are checked. | -| Silence detection | An agent whose newest indexed event was 30 hours old produced one `SILENT` record reading `30h`. | -| Repeat suppression | Three further runs with the condition unchanged produced no further records. | -| Recovery | A fresh event produced one `RESTORED` record reading `30h`, measured from the last log before the gap. | -| Ingestion and rules | The record reached `alerts.json` through the `` block as rule 100121, level 12, `mail: true`, with the description fully interpolated. | -| Telegram | `wazuh-integratord` invoked the integration and delivered both formatted messages, captured against a local HTTP endpoint standing in for the bot API. | -| Email | Both rules produced a real email through a local Postfix relay, accepted by the upstream server (`dsn=2.0.0, status=sent`). The stock `full` format carries every field of the record, decoded one per line, under the subject `Wazuh notification - - Alert level 12`. | -| Wrong index pattern | The safety stop fired: exit code 1, no state written, no alerts sent, and a log line naming the setting to check. | -| Wodle schedule | `wazuh-modulesd` ran the command on its interval, one run per interval, with the output ignored. | - ---- - -### Design Notes - -- **One indexer query per run, not one per agent.** A single `terms` - aggregation on `agent.id` with a `max` on `@timestamp` returns the last event - time for every agent at once, so the cost does not grow with the fleet. -- **No custom decoder.** JSON lines plus `json` gives - fully decoded fields for free, which also removes the dependency on a working - local syslog daemon that a `logger`-based approach carries. -- **Standard library only.** `urllib.request` instead of `requests`, so the - script runs on the embedded interpreter and on the system Python with no - packaging step. -- **Missing data is silence, not a skip.** An agent with no events at all in the - lookback window is the worst case, not a case to ignore. -- **The state file is written atomically** with a temporary file and a rename, - so an interrupted run cannot leave the state truncated. - ---- - -### Sources +| Alerts fire but no Telegram message | Check `/var/ossec/logs/integrations.log`. A missing chat ID or hook URL, or an HTTP error from the bot API, is logged with the rule ID. | +| `Field 'status' is static` | The rule was edited to match `status` instead of `event_status`. | +| A healthy agent is reported silent | It produced no alerts within the threshold. Point `SAM_INDEX_PATTERN` at `wazuh-archives-*`, or raise the threshold. | +| Every agent reported again after a manager rebuild | The state file was lost, so the first run re-reports the conditions that are still true. One repeat, then quiet again. | +## Sources - [Wazuh - command wodle](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/wodle-command.html) - [Wazuh - localfile configuration](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/localfile.html) - [Wazuh - integration configuration](https://documentation.wazuh.com/current/user-manual/reference/ossec-conf/integration.html) diff --git a/integrations/silent_agent_monitoring/custom-server-telegram.py b/integrations/silent_agent_monitoring/custom-server-telegram.py index eea6428c..a59cacd7 100755 --- a/integrations/silent_agent_monitoring/custom-server-telegram.py +++ b/integrations/silent_agent_monitoring/custom-server-telegram.py @@ -74,16 +74,33 @@ def build_message(alert): f"Agent: {agent.get('name', 'manager')} ({agent.get('id', '000')})") +def ssl_context(): + """Verifying context that also works on the Wazuh embedded interpreter. + + That interpreter's OpenSSL looks for roots in /usr/local/ssl/certs, which + does not exist, so a default context trusts nothing and every HTTPS call to + Telegram fails with CERTIFICATE_VERIFY_FAILED. certifi ships with it and + carries the real roots; on a system interpreter without certifi the normal + default store is already correct. + """ + if not VERIFY_SSL: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + try: + import certifi + return ssl.create_default_context(cafile=certifi.where()) + except ImportError: + return ssl.create_default_context() + + def send(hook_url, chat_id, message): payload = json.dumps({"chat_id": chat_id, "text": message, "parse_mode": "HTML"}).encode() req = urllib.request.Request(hook_url, data=payload, method="POST") req.add_header("Content-Type", "application/json") - context = ssl.create_default_context() - if not VERIFY_SSL: - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - with urllib.request.urlopen(req, timeout=TIMEOUT, context=context) as resp: + with urllib.request.urlopen(req, timeout=TIMEOUT, context=ssl_context()) as resp: return resp.status diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 9b8e377a..05f33b29 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -188,8 +188,8 @@ def decide(agent, last_log, previous, now): event = dict(common, event_status="RESTORED", status_text="Logs received", restored_at=local_time(last_log), - silence_duration=format_duration(gap) if gap else "unknown", - silence_seconds=int(gap.total_seconds()) if gap else 0, + silence_duration=format_duration(gap) if gap is not None else "unknown", + silence_seconds=int(gap.total_seconds()) if gap is not None else 0, message=f"Agent {name} (ID {agent_id}) has resumed sending logs.") return event, state @@ -274,6 +274,12 @@ def main(): def selftest(): """Offline assertions on the decision logic. No API, no indexer.""" + # The assertions below are written against the shipped defaults, so pin + # them here: an environment that overrides the threshold must not turn a + # logic check into a false failure. + global SILENCE_THRESHOLD, LOOKBACK + SILENCE_THRESHOLD, LOOKBACK = timedelta(hours=24), timedelta(days=7) + now = datetime(2026, 8, 19, 10, 20, 0, tzinfo=timezone.utc) agent = {"id": "152", "name": "File2", "status": "active"} From d4584f34578d0c64d526cb034175925f072eb1be Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Tue, 1 Sep 2026 12:56:20 +0200 Subject: [PATCH 4/8] Trim the comments in the silent agent monitoring scripts The header block on each file says what the script is and how it is invoked. Below that, only the non-obvious parts keep a comment: why the field is named event_status, why a failed query exits without touching the state, why every agent silent at once is a safety stop, why the selftest pins its thresholds, and why the Telegram context needs certifi. Everything else the code says by itself. --- .../custom-server-telegram.py | 40 ++++------- .../silent_agent_monitor.py | 69 +++++-------------- 2 files changed, 29 insertions(+), 80 deletions(-) diff --git a/integrations/silent_agent_monitoring/custom-server-telegram.py b/integrations/silent_agent_monitoring/custom-server-telegram.py index a59cacd7..a05102f6 100755 --- a/integrations/silent_agent_monitoring/custom-server-telegram.py +++ b/integrations/silent_agent_monitoring/custom-server-telegram.py @@ -1,19 +1,15 @@ #!/var/ossec/framework/python/bin/python3 -# Wazuh Telegram integration for the silent agent monitoring rules. -# Adapted from the Wazuh custom integration examples. +# Copyright (C) 2015, Wazuh Inc. # This program is free software; you can redistribute it and/or modify it # under the terms of GPLv2. -""" -Formats rules 100121 (no logs received) and 100122 (logging restored) into the -message layout the notification template asks for, and posts them to the -existing Telegram channel. Any other rule routed here falls back to a generic -message, so a wrong in ossec.conf produces a readable alert rather -than a crash. - -wazuh-integratord calls this as: - custom-server-telegram -so carries the chat ID and the bot sendMessage URL. -""" +# +# custom-server-telegram.py +# Wazuh Telegram integration for the silent agent monitoring rules. Formats +# rules 100121 (no logs received) and 100122 (logging restored) and posts them +# to a Telegram chat. Any other rule routed here falls back to a generic +# message. wazuh-integratord calls this as: +# custom-server-telegram +# so carries the chat ID and the bot sendMessage URL. import json import logging @@ -23,11 +19,9 @@ import urllib.request # === CONFIGURATION === -# Defaults used only when ossec.conf passes nothing, or for a manual test run. +# Used only when ossec.conf passes nothing, or for a manual test run. CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "") HOOK_URL = os.environ.get("TELEGRAM_HOOK_URL", "") -# integratord runs this as the wazuh user, so the log has to be somewhere that -# user can already write. integrations.log is the standard place for it. LOG_PATH = os.environ.get("TELEGRAM_LOG", "/var/ossec/logs/integrations.log") VERIFY_SSL = os.environ.get("TELEGRAM_VERIFY_SSL", "yes").lower() in ("yes", "true", "1") TIMEOUT = 15 @@ -40,13 +34,10 @@ try: logging.basicConfig(filename=LOG_PATH, filemode="a", **_LOG_ARGS) except OSError: - # An unwritable log must not cost us the notification. stderr is captured - # by integratord and ends up in ossec.log. logging.basicConfig(stream=sys.stderr, **_LOG_ARGS) def build_message(alert): - """Return the HTML message body for one alert.""" data = alert.get("data", {}) rule = alert.get("rule", {}) rule_id = str(rule.get("id", "")) @@ -75,19 +66,14 @@ def build_message(alert): def ssl_context(): - """Verifying context that also works on the Wazuh embedded interpreter. - - That interpreter's OpenSSL looks for roots in /usr/local/ssl/certs, which - does not exist, so a default context trusts nothing and every HTTPS call to - Telegram fails with CERTIFICATE_VERIFY_FAILED. certifi ships with it and - carries the real roots; on a system interpreter without certifi the normal - default store is already correct. - """ if not VERIFY_SSL: context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE return context + # The Wazuh embedded interpreter's OpenSSL looks for roots in a path that + # does not exist, so a default context trusts nothing. certifi ships with + # it; a system interpreter without certifi already has a working store. try: import certifi return ssl.create_default_context(cafile=certifi.where()) diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 05f33b29..95865c26 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -1,16 +1,13 @@ #!/var/ossec/framework/python/bin/python3 # # silent_agent_monitor.py -# Detects Wazuh agents that are still registered (and often still "active") -# but have stopped shipping logs. For every agent in a target group it reads -# the timestamp of the most recent indexed event and, when that timestamp is -# older than the threshold, appends a SILENT record to a local JSON log that -# Wazuh ingests through a block. When events start arriving again -# it appends a matching RESTORED record. -# -# State is kept locally so a condition that stays unchanged is reported once, -# not once per run. Standard library only: it runs on the Wazuh embedded -# interpreter with no pip install. +# Detects Wazuh agents that are still registered but have stopped shipping +# logs. For every agent in a target group it reads the timestamp of the most +# recent indexed event and, when that timestamp is older than the threshold, +# appends a SILENT record to a local JSON log that Wazuh ingests through a +# block. When events start arriving again it appends a matching +# RESTORED record. State is kept locally so an unchanged condition is reported +# once, not once per run. Standard library only. # # Run modes: # silent_agent_monitor.py normal check (scheduled by a wodle) @@ -27,8 +24,6 @@ from datetime import datetime, timedelta, timezone # === CONFIGURATION === -# Every value can be overridden with an environment variable, so the same file -# can be pointed at a test environment without being edited. API_URL = os.environ.get("SAM_API_URL", "https://127.0.0.1:55000") API_USER = os.environ.get("SAM_API_USER", "wazuh-wui") API_PASSWORD = os.environ.get("SAM_API_PASSWORD", "CHANGE_ME") @@ -37,16 +32,10 @@ INDEXER_USER = os.environ.get("SAM_INDEXER_USER", "admin") INDEXER_PASSWORD = os.environ.get("SAM_INDEXER_PASSWORD", "CHANGE_ME") -# Index pattern holding the events used as proof of life. See the README: -# wazuh-alerts-* only contains alerts, wazuh-archives-* contains every event -# and is the accurate source when archives are enabled and indexed. INDEX_PATTERN = os.environ.get("SAM_INDEX_PATTERN", "wazuh-alerts-*") TARGET_GROUP = os.environ.get("SAM_GROUP", "Server") SILENCE_THRESHOLD = timedelta(hours=float(os.environ.get("SAM_THRESHOLD_HOURS", "24"))) - -# How far back the aggregation looks. Must exceed the threshold: an agent with -# no events inside this window is reported as silent for "more than" it. LOOKBACK = timedelta(days=float(os.environ.get("SAM_LOOKBACK_DAYS", "7"))) STATE_FILE = os.environ.get("SAM_STATE_FILE", "/var/ossec/var/silent_agents_state.json") @@ -57,14 +46,11 @@ PAGE_SIZE = 500 HTTP_TIMEOUT = 30 -# === LOGGING === _LOG_ARGS = {"format": "%(asctime)s %(levelname)s %(message)s", "datefmt": "%Y-%m-%dT%H:%M:%S", "level": logging.INFO} try: logging.basicConfig(filename=SCRIPT_LOG, filemode="a", **_LOG_ARGS) except OSError: - # Running as a user that cannot write the log file is not a reason to skip - # the check. stderr is picked up by whatever scheduled the run. logging.basicConfig(stream=sys.stderr, **_LOG_ARGS) SSL_CONTEXT = ssl.create_default_context() @@ -74,7 +60,6 @@ def http_json(url, method="GET", body=None, token=None, basic=None): - """One JSON request. Raises on any transport or HTTP error.""" data = json.dumps(body).encode() if body is not None else None req = urllib.request.Request(url, data=data, method=method) req.add_header("Content-Type", "application/json") @@ -88,14 +73,11 @@ def http_json(url, method="GET", body=None, token=None, basic=None): def get_token(): - """Authenticate against the Wazuh API and return a JWT token.""" url = f"{API_URL}/security/user/authenticate" return http_json(url, method="POST", basic=(API_USER, API_PASSWORD))["data"]["token"] def fetch_group_agents(token): - """Return every agent of TARGET_GROUP, excluding the manager and agents - that have never connected (those have no logs by definition).""" agents, offset = [], 0 while True: url = (f"{API_URL}/agents?group={TARGET_GROUP}&limit={PAGE_SIZE}&offset={offset}" @@ -110,8 +92,7 @@ def fetch_group_agents(token): def fetch_last_event_times(agent_ids): - """One aggregation for every agent: newest event timestamp per agent.id. - Returns {agent_id: datetime}. Agents with no event in LOOKBACK are absent.""" + """{agent_id: datetime}. Agents with no event inside LOOKBACK are absent.""" query = { "size": 0, "query": {"bool": {"filter": [ @@ -126,38 +107,29 @@ def fetch_last_event_times(agent_ids): url = f"{INDEXER_URL}/{INDEX_PATTERN}/_search" result = http_json(url, method="POST", body=query, basic=(INDEXER_USER, INDEXER_PASSWORD)) - # Missing aggregations means the query never matched an index. Return no - # buckets and let the caller's safety stop report it as a lookup problem. buckets = result.get("aggregations", {}).get("per_agent", {}).get("buckets", []) return {b["key"]: datetime.fromtimestamp(b["last_event"]["value"] / 1000, timezone.utc) for b in buckets if b["last_event"]["value"]} def format_duration(delta): - """'25h 40m', or '25h' on a whole hour. Matches the notification template.""" minutes = int(delta.total_seconds() // 60) hours, minutes = divmod(minutes, 60) return f"{hours}h {minutes}m" if minutes else f"{hours}h" def local_time(dt): - """Render a UTC datetime in the manager's local timezone, tz name included.""" return dt.astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") def decide(agent, last_log, previous, now): - """Pure decision for one agent. Returns (event or None, new state entry). - - last_log is the newest indexed event time, or None when the agent produced - nothing inside LOOKBACK, which is the deepest form of silence. - previous is the state entry from the last run, or {}. - """ + """Pure decision for one agent. Returns (event or None, new state entry).""" agent_id, name = agent["id"], agent.get("name", "unknown") silent = last_log is None or (now - last_log) >= SILENCE_THRESHOLD was_silent = previous.get("status") == "SILENT" - # The state key is named event_status, not status: "status" is one of the - # Wazuh static field names, and a rule cannot match it with . + # event_status, not status: "status" is a static Wazuh field name and a + # rule cannot match it with . common = { "integration": "silent-agent-monitor", "group": TARGET_GROUP, @@ -182,7 +154,7 @@ def decide(agent, last_log, previous, now): if not silent and was_silent: # Measured from the last log before the gap to the first log after it, - # not from the moment this script noticed, so the duration is real. + # not from the moment this script noticed. previous_log = previous.get("last_log") gap = (last_log - datetime.fromisoformat(previous_log)) if previous_log else None event = dict(common, event_status="RESTORED", @@ -203,13 +175,11 @@ def load_state(): except FileNotFoundError: return {} except (OSError, ValueError) as err: - # A corrupt state file must not stop the check. Worst case one repeat. logging.error("Could not read state file '%s': %s. Starting empty.", STATE_FILE, err) return {} def save_state(state): - """Atomic replace, so a kill mid-write cannot leave a truncated state.""" tmp = f"{STATE_FILE}.tmp" with open(tmp, "w") as f: json.dump(state, f, indent=2) @@ -244,9 +214,8 @@ def main(): sys.exit(1) if not last_events and len(agents) > 1: - # Every single agent silent at once is far more likely to be a wrong - # index pattern or wrong credentials than a real outage. Refuse to - # generate the storm and make the operator look. + # Every agent silent at once is far more likely a wrong index pattern + # or wrong credentials than a real outage. Refuse to send the storm. logging.error("No events found for any of the %d agents in '%s' over the last %s. " "Check SAM_INDEX_PATTERN and the indexer credentials. No alerts sent.", len(agents), TARGET_GROUP, format_duration(LOOKBACK)) @@ -273,17 +242,14 @@ def main(): def selftest(): - """Offline assertions on the decision logic. No API, no indexer.""" - # The assertions below are written against the shipped defaults, so pin - # them here: an environment that overrides the threshold must not turn a - # logic check into a false failure. + # Written against the shipped defaults, so pin them: an install that + # overrides the threshold must not turn a logic check into a false failure. global SILENCE_THRESHOLD, LOOKBACK SILENCE_THRESHOLD, LOOKBACK = timedelta(hours=24), timedelta(days=7) now = datetime(2026, 8, 19, 10, 20, 0, tzinfo=timezone.utc) agent = {"id": "152", "name": "File2", "status": "active"} - # Quiet for 25h40m: reported once, then suppressed while unchanged. stopped = now - timedelta(hours=25, minutes=40) event, state = decide(agent, stopped, {}, now) assert event["event_status"] == "SILENT", event @@ -293,7 +259,6 @@ def selftest(): assert state["status"] == "SILENT" assert decide(agent, stopped, state, now)[0] is None, "repeat alert not suppressed" - # Logs resume: one recovery, measured from the last log before the gap. resumed = stopped + timedelta(hours=25, minutes=40) event, ok_state = decide(agent, resumed, state, now) assert event["event_status"] == "RESTORED", event @@ -302,10 +267,8 @@ def selftest(): assert ok_state["status"] == "OK" assert decide(agent, resumed, ok_state, now)[0] is None, "repeat recovery not suppressed" - # A healthy agent inside the threshold never reports. assert decide(agent, now - timedelta(hours=23), {}, now)[0] is None - # No events at all inside the lookback window is silence, not a skip. event, _ = decide(agent, None, {}, now) assert event["event_status"] == "SILENT" and event["last_log"] == "unknown", event From 328ca5655bef7f52b767349b869a13f4d45db06f Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Tue, 1 Sep 2026 13:04:54 +0200 Subject: [PATCH 5/8] Take the target group from the wodle instead of the script The group is the one setting that changes per deployment and per wodle, so it belongs in ossec.conf next to the schedule rather than in a file that a manager upgrade can replace. A --group argument now overrides SAM_GROUP, which in turn overrides the shipped default, and the README documents one wodle per group with its own state file and output log. --- integrations/silent_agent_monitoring/README.md | 13 ++++++++----- .../silent_agent_monitor.py | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index b4f43d4b..8b11a3d2 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -51,7 +51,7 @@ Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set t | Indexer URL | `SAM_INDEXER_URL` | `https://127.0.0.1:9200` | | Indexer user / password | `SAM_INDEXER_USER`, `SAM_INDEXER_PASSWORD` | `admin` / `CHANGE_ME` | | Index pattern | `SAM_INDEX_PATTERN` | `wazuh-alerts-*` | -| Agent group | `SAM_GROUP` | `Server` | +| Agent group | `--group`, `SAM_GROUP` | `Server` | | Silence threshold, hours | `SAM_THRESHOLD_HOURS` | `24` | | Lookback window, days | `SAM_LOOKBACK_DAYS` | `7` | | State file | `SAM_STATE_FILE` | `/var/ossec/var/silent_agents_state.json` | @@ -59,7 +59,7 @@ Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set t | Script log | `SAM_SCRIPT_LOG` | `/var/ossec/logs/silent_agent_monitor.log` | | Verify TLS certificates | `SAM_VERIFY_SSL` | `no` | -The file holds credentials, so keep it root-owned and `chmod 750`. `SAM_LOOKBACK_DAYS` must stay larger than the threshold: it bounds the indexer query, and an agent with nothing inside it is reported as silent for "more than" that window. +The group is normally passed with `--group` from the wodle, so `ossec.conf` owns it and the script needs no edit to change it. The file holds credentials, so keep it root-owned and `chmod 750`. `SAM_LOOKBACK_DAYS` must stay larger than the threshold: it bounds the indexer query, and an agent with nothing inside it is reported as silent for "more than" that window. The index pattern decides what counts as a log. `wazuh-alerts-*` is available everywhere but only sees alerts, so an agent that ships logs normally while producing no alert for a full day is reported as silent. `wazuh-archives-*` is the exact answer to "no logs received" but needs `` enabled and the archives indexed. Use archives when they are available; otherwise confirm that every agent in the group normally produces alerts within the threshold. @@ -70,7 +70,7 @@ Add to `/var/ossec/etc/ossec.conf`: no silent-agent-monitor - /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py + /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --group Server 1h yes 300 @@ -82,6 +82,8 @@ Add to `/var/ossec/etc/ossec.conf`: /var/ossec/logs/silent_agents.json ``` +To monitor several groups, add one `` per group, each with its own `--group` and its own `SAM_STATE_FILE` and `SAM_OUTPUT_LOG`; sharing a state file between groups makes each run overwrite the other's entries. + One run per hour bounds detection lag and recovery lag to an hour each, at one indexer query per hour whatever the number of agents. With `run_on_start`, the first run after a restart can reach the API before it finishes starting and log `HTTP Error 500`; nothing is lost, because a failed run never writes state. ### Add custom rules @@ -144,13 +146,14 @@ Offline assertions on the decision logic, no API or indexer needed: Run the check by hand against the live environment: ```bash -/var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py +/var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --group Server # Checked 12 agent(s) in 'Server': 0 silent, 0 event(s) written to /var/ossec/logs/silent_agents.json. ``` To force a notification without waiting, drop the threshold for one run: ```bash -SAM_THRESHOLD_HOURS=0.05 /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py +SAM_THRESHOLD_HOURS=0.05 /var/ossec/framework/python/bin/python3 \ + /var/ossec/wodles/silent_agent_monitor.py --group Server tail -1 /var/ossec/logs/silent_agents.json tail -f /var/ossec/logs/alerts/alerts.log | grep -A5 100121 ``` diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 95865c26..66fe27c8 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -10,9 +10,10 @@ # once, not once per run. Standard library only. # # Run modes: -# silent_agent_monitor.py normal check (scheduled by a wodle) -# silent_agent_monitor.py --selftest offline assertions on the decision logic +# silent_agent_monitor.py --group Server normal check (scheduled by a wodle) +# silent_agent_monitor.py --selftest offline assertions on the logic +import argparse import base64 import json import logging @@ -34,6 +35,7 @@ INDEX_PATTERN = os.environ.get("SAM_INDEX_PATTERN", "wazuh-alerts-*") +# Overridden by --group, so the wodle in ossec.conf owns the group name. TARGET_GROUP = os.environ.get("SAM_GROUP", "Server") SILENCE_THRESHOLD = timedelta(hours=float(os.environ.get("SAM_THRESHOLD_HOURS", "24"))) LOOKBACK = timedelta(days=float(os.environ.get("SAM_LOOKBACK_DAYS", "7"))) @@ -276,7 +278,15 @@ def selftest(): if __name__ == "__main__": - if "--selftest" in sys.argv: + parser = argparse.ArgumentParser(description="Detect Wazuh agents that stopped sending logs.") + parser.add_argument("--group", default=TARGET_GROUP, + help=f"agent group to monitor (default: {TARGET_GROUP})") + parser.add_argument("--selftest", action="store_true", + help="run offline assertions on the decision logic and exit") + args = parser.parse_args() + + if args.selftest: selftest() else: + TARGET_GROUP = args.group main() From a39f8264e185ed6ddaf0521f4b9f98fd0c31b1e4 Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Tue, 1 Sep 2026 13:10:52 +0200 Subject: [PATCH 6/8] Accept several target groups in one run --group now takes a comma-separated list, so a single wodle covers every group that should be watched instead of one wodle, one state file and one output log per group. Agents are deduplicated across the listed groups, so an agent that belongs to two of them is checked once and its record names both. The indexer still sees one aggregation per run, whatever the number of groups. --- .../silent_agent_monitoring/README.md | 7 +-- .../silent_agent_monitor.py | 45 ++++++++++++------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index 8b11a3d2..deba4fb2 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -51,7 +51,7 @@ Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set t | Indexer URL | `SAM_INDEXER_URL` | `https://127.0.0.1:9200` | | Indexer user / password | `SAM_INDEXER_USER`, `SAM_INDEXER_PASSWORD` | `admin` / `CHANGE_ME` | | Index pattern | `SAM_INDEX_PATTERN` | `wazuh-alerts-*` | -| Agent group | `--group`, `SAM_GROUP` | `Server` | +| Agent groups | `--group`, `SAM_GROUP` | `Server` | | Silence threshold, hours | `SAM_THRESHOLD_HOURS` | `24` | | Lookback window, days | `SAM_LOOKBACK_DAYS` | `7` | | State file | `SAM_STATE_FILE` | `/var/ossec/var/silent_agents_state.json` | @@ -59,7 +59,7 @@ Edit the `CONFIGURATION` block at the top of `silent_agent_monitor.py`, or set t | Script log | `SAM_SCRIPT_LOG` | `/var/ossec/logs/silent_agent_monitor.log` | | Verify TLS certificates | `SAM_VERIFY_SSL` | `no` | -The group is normally passed with `--group` from the wodle, so `ossec.conf` owns it and the script needs no edit to change it. The file holds credentials, so keep it root-owned and `chmod 750`. `SAM_LOOKBACK_DAYS` must stay larger than the threshold: it bounds the indexer query, and an agent with nothing inside it is reported as silent for "more than" that window. +The groups are normally passed with `--group` from the wodle, so `ossec.conf` owns them and the script needs no edit to change them. The file holds credentials, so keep it root-owned and `chmod 750`. `SAM_LOOKBACK_DAYS` must stay larger than the threshold: it bounds the indexer query, and an agent with nothing inside it is reported as silent for "more than" that window. The index pattern decides what counts as a log. `wazuh-alerts-*` is available everywhere but only sees alerts, so an agent that ships logs normally while producing no alert for a full day is reported as silent. `wazuh-archives-*` is the exact answer to "no logs received" but needs `` enabled and the archives indexed. Use archives when they are available; otherwise confirm that every agent in the group normally produces alerts within the threshold. @@ -82,7 +82,7 @@ Add to `/var/ossec/etc/ossec.conf`: /var/ossec/logs/silent_agents.json ``` -To monitor several groups, add one `` per group, each with its own `--group` and its own `SAM_STATE_FILE` and `SAM_OUTPUT_LOG`; sharing a state file between groups makes each run overwrite the other's entries. +`--group` takes a comma-separated list, so one wodle covers several groups: `--group Server,Windows,DMZ`. Agents are deduplicated across them, so an agent in two of the listed groups is checked once and its record names both. A separate wodle per group also works, but each one then needs its own `SAM_STATE_FILE` and `SAM_OUTPUT_LOG`, because a run rewrites the whole state file. One run per hour bounds detection lag and recovery lag to an hour each, at one indexer query per hour whatever the number of agents. With `run_on_start`, the first run after a restart can reach the API before it finishes starting and log `HTTP Error 500`; nothing is lost, because a failed run never writes state. @@ -170,6 +170,7 @@ echo '{"integration":"silent-agent-monitor","event_status":"SILENT","agent_id":" | --- | --- | | `No events found for any of the N agents` and no alerts | Deliberate safety stop: every agent silent at once is almost always a wrong index pattern or wrong indexer credentials. Check `SAM_INDEX_PATTERN` and the indexer user. | | `Indexer query failed` or `Wazuh API query failed` | The run exits without touching the state, so nothing is reported as silent or recovered on a failed query. Check connectivity and credentials. | +| `No agents in group(s) 'X'` | The group does not exist or is empty. Check with `/var/ossec/bin/agent_groups -l`. | | Records in `silent_agents.json` but no alerts | The `` block is missing, points elsewhere, or sits on a node that is not running the script. | | Alerts fire but no email | Global email is not enabled, or the rules lost `alert_by_email`. Check `/var/ossec/logs/ossec.log` for `wazuh-maild`. | | Alerts fire but no Telegram message | Check `/var/ossec/logs/integrations.log`. A missing chat ID or hook URL, or an HTTP error from the bot API, is logged with the rule ID. | diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 66fe27c8..9e494689 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -10,8 +10,8 @@ # once, not once per run. Standard library only. # # Run modes: -# silent_agent_monitor.py --group Server normal check (scheduled by a wodle) -# silent_agent_monitor.py --selftest offline assertions on the logic +# silent_agent_monitor.py --group Server,Windows normal check (wodle) +# silent_agent_monitor.py --selftest offline logic assertions import argparse import base64 @@ -35,8 +35,9 @@ INDEX_PATTERN = os.environ.get("SAM_INDEX_PATTERN", "wazuh-alerts-*") -# Overridden by --group, so the wodle in ossec.conf owns the group name. -TARGET_GROUP = os.environ.get("SAM_GROUP", "Server") +# Comma-separated. Overridden by --group, so the wodle owns the group names. +TARGET_GROUPS = [g.strip() for g in os.environ.get("SAM_GROUP", "Server").split(",") + if g.strip()] SILENCE_THRESHOLD = timedelta(hours=float(os.environ.get("SAM_THRESHOLD_HOURS", "24"))) LOOKBACK = timedelta(days=float(os.environ.get("SAM_LOOKBACK_DAYS", "7"))) @@ -79,10 +80,10 @@ def get_token(): return http_json(url, method="POST", basic=(API_USER, API_PASSWORD))["data"]["token"] -def fetch_group_agents(token): +def fetch_group_agents(token, group): agents, offset = [], 0 while True: - url = (f"{API_URL}/agents?group={TARGET_GROUP}&limit={PAGE_SIZE}&offset={offset}" + url = (f"{API_URL}/agents?group={group}&limit={PAGE_SIZE}&offset={offset}" f"&sort=%2Bid&select=id,name,status,lastKeepAlive") data = http_json(url, token=token).get("data", {}) agents.extend(a for a in data.get("affected_items", []) @@ -134,7 +135,7 @@ def decide(agent, last_log, previous, now): # rule cannot match it with . common = { "integration": "silent-agent-monitor", - "group": TARGET_GROUP, + "group": ",".join(agent.get("groups", [])), "agent_id": agent_id, "agent_name": name, "agent_status": agent.get("status", "unknown"), @@ -196,14 +197,21 @@ def append_events(events): def main(): now = datetime.now(timezone.utc) + groups = ",".join(TARGET_GROUPS) + # An agent in two target groups is checked once and reports both names. + found = {} try: - agents = fetch_group_agents(get_token()) + token = get_token() + for group in TARGET_GROUPS: + for agent in fetch_group_agents(token, group): + found.setdefault(agent["id"], dict(agent, groups=[]))["groups"].append(group) except (urllib.error.URLError, OSError, KeyError, ValueError) as err: logging.error("Wazuh API query failed: %s", err) sys.exit(1) + agents = list(found.values()) if not agents: - logging.info("No agents in group '%s'. Nothing to do.", TARGET_GROUP) + logging.info("No agents in group(s) '%s'. Nothing to do.", groups) return agent_ids = [a["id"] for a in agents] @@ -220,7 +228,7 @@ def main(): # or wrong credentials than a real outage. Refuse to send the storm. logging.error("No events found for any of the %d agents in '%s' over the last %s. " "Check SAM_INDEX_PATTERN and the indexer credentials. No alerts sent.", - len(agents), TARGET_GROUP, format_duration(LOOKBACK)) + len(agents), groups, format_duration(LOOKBACK)) sys.exit(1) state = load_state() @@ -238,8 +246,8 @@ def main(): silent = sum(1 for e in new_state.values() if e["status"] == "SILENT") logging.info("Checked %d agent(s) in '%s': %d silent, %d new event(s) written.", - len(agents), TARGET_GROUP, silent, len(events)) - print(f"Checked {len(agents)} agent(s) in '{TARGET_GROUP}': " + len(agents), groups, silent, len(events)) + print(f"Checked {len(agents)} agent(s) in '{groups}': " f"{silent} silent, {len(events)} event(s) written to {OUTPUT_LOG}.") @@ -250,7 +258,7 @@ def selftest(): SILENCE_THRESHOLD, LOOKBACK = timedelta(hours=24), timedelta(days=7) now = datetime(2026, 8, 19, 10, 20, 0, tzinfo=timezone.utc) - agent = {"id": "152", "name": "File2", "status": "active"} + agent = {"id": "152", "name": "File2", "status": "active", "groups": ["Server"]} stopped = now - timedelta(hours=25, minutes=40) event, state = decide(agent, stopped, {}, now) @@ -258,6 +266,7 @@ def selftest(): assert event["status_text"] == "No logs received", event assert event["no_logs_for"] == "25h 40m", event assert event["agent_id"] == "152" and event["agent_name"] == "File2" + assert event["group"] == "Server", event assert state["status"] == "SILENT" assert decide(agent, stopped, state, now)[0] is None, "repeat alert not suppressed" @@ -274,13 +283,17 @@ def selftest(): event, _ = decide(agent, None, {}, now) assert event["event_status"] == "SILENT" and event["last_log"] == "unknown", event + both = dict(agent, groups=["Server", "Windows"]) + assert decide(both, stopped, {}, now)[0]["group"] == "Server,Windows" + print("selftest OK") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Detect Wazuh agents that stopped sending logs.") - parser.add_argument("--group", default=TARGET_GROUP, - help=f"agent group to monitor (default: {TARGET_GROUP})") + parser.add_argument("--group", default=",".join(TARGET_GROUPS), + help="comma-separated agent groups to monitor " + f"(default: {','.join(TARGET_GROUPS)})") parser.add_argument("--selftest", action="store_true", help="run offline assertions on the decision logic and exit") args = parser.parse_args() @@ -288,5 +301,5 @@ def selftest(): if args.selftest: selftest() else: - TARGET_GROUP = args.group + TARGET_GROUPS = [g.strip() for g in args.group.split(",") if g.strip()] main() From 28e712010b2913fa44382ec592896b7f068af94f Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Tue, 1 Sep 2026 13:35:17 +0200 Subject: [PATCH 7/8] Show a filled-in integration block in the README A second block with sample values next to the placeholder one makes the shape of hook_url and api_key obvious. The multi-group example loses its space: a wodle command is split on whitespace, so 'Server, Windows' would pass one trailing comma group and a stray argument. --- integrations/silent_agent_monitoring/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index deba4fb2..3dd7ef51 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -70,7 +70,7 @@ Add to `/var/ossec/etc/ossec.conf`: no silent-agent-monitor - /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --group Server + /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --group Server,Windows 1h yes 300 @@ -121,6 +121,15 @@ Use the bundled script only when there is no Telegram integration yet: <CHAT_ID> json + + + + custom-server-telegram + 100121,100122 + https://api.telegram.org/bot8454124324:niwefn76t5safuef8s76tg/sendMessage + 123123123123 + json + ``` `` is the full `sendMessage` endpoint of the bot and `` is the numeric chat ID. Both are passed to the script as arguments and override its defaults. The messages it produces: From 48ec5fb8c16e50eaf61a7ac52d9a593f67cff5e4 Mon Sep 17 00:00:00 2001 From: leonfullxr Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 8/8] Drop the selftest mode and the rules file header The offline assertions and their fixture agent were development scaffolding. The script now has one run mode, so the header states a single usage line and argparse carries only --group. The README's testing section keeps the checks that matter to an operator: a manual run, a forced notification with a lowered threshold, and a wazuh-logtest line for the rules. The rules file loses its header comment as well; the per-rule comments and the README already say what each rule matches and that the IDs can be moved. --- .../silent_agent_monitoring/README.md | 6 --- .../silent_agent_monitor-rules.xml | 7 --- .../silent_agent_monitor.py | 51 ++----------------- 3 files changed, 3 insertions(+), 61 deletions(-) diff --git a/integrations/silent_agent_monitoring/README.md b/integrations/silent_agent_monitoring/README.md index 3dd7ef51..759cda08 100644 --- a/integrations/silent_agent_monitoring/README.md +++ b/integrations/silent_agent_monitoring/README.md @@ -147,12 +147,6 @@ When a Telegram integration already exists, keep it and add `100121,100122` to i `wazuh-integratord` runs integration scripts as the `wazuh` user, so any path the script writes, including a custom `TELEGRAM_LOG`, must be writable by it. ## Testing -Offline assertions on the decision logic, no API or indexer needed: -```bash -/var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --selftest -# selftest OK -``` - Run the check by hand against the live environment: ```bash /var/ossec/framework/python/bin/python3 /var/ossec/wodles/silent_agent_monitor.py --group Server diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml b/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml index c39521a0..0f8cd08d 100644 --- a/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml +++ b/integrations/silent_agent_monitoring/silent_agent_monitor-rules.xml @@ -1,10 +1,3 @@ - diff --git a/integrations/silent_agent_monitoring/silent_agent_monitor.py b/integrations/silent_agent_monitoring/silent_agent_monitor.py index 9e494689..66988c88 100755 --- a/integrations/silent_agent_monitoring/silent_agent_monitor.py +++ b/integrations/silent_agent_monitoring/silent_agent_monitor.py @@ -9,9 +9,7 @@ # RESTORED record. State is kept locally so an unchanged condition is reported # once, not once per run. Standard library only. # -# Run modes: -# silent_agent_monitor.py --group Server,Windows normal check (wodle) -# silent_agent_monitor.py --selftest offline logic assertions +# Usage: silent_agent_monitor.py --group Server,Windows import argparse import base64 @@ -251,55 +249,12 @@ def main(): f"{silent} silent, {len(events)} event(s) written to {OUTPUT_LOG}.") -def selftest(): - # Written against the shipped defaults, so pin them: an install that - # overrides the threshold must not turn a logic check into a false failure. - global SILENCE_THRESHOLD, LOOKBACK - SILENCE_THRESHOLD, LOOKBACK = timedelta(hours=24), timedelta(days=7) - - now = datetime(2026, 8, 19, 10, 20, 0, tzinfo=timezone.utc) - agent = {"id": "152", "name": "File2", "status": "active", "groups": ["Server"]} - - stopped = now - timedelta(hours=25, minutes=40) - event, state = decide(agent, stopped, {}, now) - assert event["event_status"] == "SILENT", event - assert event["status_text"] == "No logs received", event - assert event["no_logs_for"] == "25h 40m", event - assert event["agent_id"] == "152" and event["agent_name"] == "File2" - assert event["group"] == "Server", event - assert state["status"] == "SILENT" - assert decide(agent, stopped, state, now)[0] is None, "repeat alert not suppressed" - - resumed = stopped + timedelta(hours=25, minutes=40) - event, ok_state = decide(agent, resumed, state, now) - assert event["event_status"] == "RESTORED", event - assert event["status_text"] == "Logs received", event - assert event["silence_duration"] == "25h 40m", event - assert ok_state["status"] == "OK" - assert decide(agent, resumed, ok_state, now)[0] is None, "repeat recovery not suppressed" - - assert decide(agent, now - timedelta(hours=23), {}, now)[0] is None - - event, _ = decide(agent, None, {}, now) - assert event["event_status"] == "SILENT" and event["last_log"] == "unknown", event - - both = dict(agent, groups=["Server", "Windows"]) - assert decide(both, stopped, {}, now)[0]["group"] == "Server,Windows" - - print("selftest OK") - - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Detect Wazuh agents that stopped sending logs.") parser.add_argument("--group", default=",".join(TARGET_GROUPS), help="comma-separated agent groups to monitor " f"(default: {','.join(TARGET_GROUPS)})") - parser.add_argument("--selftest", action="store_true", - help="run offline assertions on the decision logic and exit") args = parser.parse_args() - if args.selftest: - selftest() - else: - TARGET_GROUPS = [g.strip() for g in args.group.split(",") if g.strip()] - main() + TARGET_GROUPS = [g.strip() for g in args.group.split(",") if g.strip()] + main()