diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9ac873..2e2bab9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#876]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#880]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#894]). ### Changed @@ -48,6 +50,10 @@ All notable changes to this project will be documented in this file. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#860]). - The operator now watches all resources that it creates and early-exits the reconcile action when the cluster is marked for deletion ([#867]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#894]). ### Remove @@ -71,6 +77,7 @@ All notable changes to this project will be documented in this file. [#889]: https://github.com/stackabletech/druid-operator/pull/889 [#891]: https://github.com/stackabletech/druid-operator/pull/891 [#892]: https://github.com/stackabletech/druid-operator/pull/892 +[#894]: https://github.com/stackabletech/druid-operator/pull/894 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/druid/pages/usage-guide/logging.adoc b/docs/modules/druid/pages/usage-guide/logging.adoc index 83388b81..c07c2fc7 100644 --- a/docs/modules/druid/pages/usage-guide/logging.adoc +++ b/docs/modules/druid/pages/usage-guide/logging.adoc @@ -30,4 +30,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `druid` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/druid` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index ef3358e1..7af98381 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -7,6 +7,7 @@ use std::str::FromStr; +use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::pod::{ @@ -51,8 +52,13 @@ stackable_operator::constant!(pub(crate) LOG_CONFIG_VOLUME_NAME: VolumeName = "l stackable_operator::constant!(pub(crate) LOG_VOLUME_NAME: VolumeName = "log"); stackable_operator::constant!(pub(crate) RW_CONFIG_VOLUME_NAME: VolumeName = "rwconfig"); -// Needed for the `containerdebug` process to log its tracing information to. -stackable_operator::constant!(pub(crate) CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); +// Let the `containerdebug` process write its tracing information to a file. These are the generic +// `stackable-telemetry` variables and apply to the whole Druid container, so any other +// telemetry-based process in it would log into the same directory and the Vector agent would label +// its events with `container: containerdebug`. +stackable_operator::constant!(pub(crate) FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +stackable_operator::constant!(pub(crate) FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +stackable_operator::constant!(pub(crate) FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); // volume mount directory (not a volume name) const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; @@ -269,10 +275,16 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result = env .iter() - .filter(|env_var| env_var.name == CONTAINERDEBUG_LOG_DIRECTORY.as_ref()) + .filter(|env_var| env_var.name == FILE_LOG_DIRECTORY.as_ref()) .collect(); assert_eq!( containerdebug.len(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 0a6697f9..16bad743 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -485,6 +485,7 @@ impl DruidRole { {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents \"$FILE_LOG_DIRECTORY\" containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & /stackable/druid/bin/run-druid {process_name} {RW_CONFIG_DIRECTORY} & echo \"$!\" >> /tmp/DRUID_PID diff --git a/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 index b8cc3c1a..c613ff19 100644 --- a/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/01_druid-vector-aggregator-values.yaml.j2 @@ -22,191 +22,293 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigBrokerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigBrokerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-broker-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigBrokerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "druid" filteredCustomLogConfigBrokerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigBrokerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-broker-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigBrokerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-broker-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigCoordinatorDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigCoordinatorPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigCoordinatorContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-coordinator-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigCoordinatorVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigCoordinatorDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "druid" filteredCustomLogConfigCoordinatorPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigCoordinatorContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-coordinator-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigCoordinatorVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-coordinator-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigHistoricalDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigHistoricalPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigHistoricalContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-historical-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigHistoricalVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigHistoricalDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "druid" filteredCustomLogConfigHistoricalPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigHistoricalContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-historical-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigHistoricalVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-historical-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigMiddlemanagerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigMiddlemanagerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigMiddlemanagerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-middlemanager-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigMiddlemanagerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigMiddlemanagerDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "druid" filteredCustomLogConfigMiddlemanagerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigMiddlemanagerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-middlemanager-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigMiddlemanagerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-middlemanager-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigRouterDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "druid" filteredAutomaticLogConfigRouterPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "prepare" + filteredAutomaticLogConfigRouterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-router-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigRouterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigRouterDruid: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "druid" filteredCustomLogConfigRouterPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "prepare" + filteredCustomLogConfigRouterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-druid-router-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigRouterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-druid-router-custom-log-config-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -214,11 +316,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: