From 79c5b80946bf130f5cf91cf8cb245d163709418f Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:10:46 +0100 Subject: [PATCH 1/2] Tolerate missing valkey socket in RedisStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When valkey is stopped ahead of cloud_controller_ng draining during an api-group rolling update, its unix socket is removed while CC-NG still serves traffic. Every request then hit RedisStore on the dead socket and raised Redis::CannotConnectError, which surfaced as a 500 with a leaked backtrace. Wrap the RedisStore metrics writes so connection errors are logged and swallowed — a metrics write must never fail a user request. Genuine errors (e.g. Redis::CommandError) still propagate. --- .../metrics/statsd_updater.rb | 20 +++++++-- .../metrics/statsd_updater_spec.rb | 44 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/cloud_controller/metrics/statsd_updater.rb b/lib/cloud_controller/metrics/statsd_updater.rb index 685ee75b7ce..08b373b5b65 100644 --- a/lib/cloud_controller/metrics/statsd_updater.rb +++ b/lib/cloud_controller/metrics/statsd_updater.rb @@ -133,15 +133,29 @@ def initialize(socket) end @prefix = 'metrics' @request_outstanding_gauge_key = "#{@prefix}:cc.requests.outstanding.gauge" - @redis.set(@request_outstanding_gauge_key, 0) + tolerate_connection_errors { @redis.set(@request_outstanding_gauge_key, 0) } end def increment_request_outstanding_gauge - @redis.incr(@request_outstanding_gauge_key) + tolerate_connection_errors { @redis.incr(@request_outstanding_gauge_key) } end def decrement_request_outstanding_gauge - @redis.decr(@request_outstanding_gauge_key) + tolerate_connection_errors { @redis.decr(@request_outstanding_gauge_key) } + end + + private + + # A metrics write must never fail a user request; swallow connection errors (e.g. valkey socket gone during shutdown). + def tolerate_connection_errors + yield + rescue Redis::BaseConnectionError => e + logger.warn("statsd_updater.redis_store.connection_error: #{e.class}: #{e.message}") + nil + end + + def logger + @logger ||= Steno.logger('cc.metrics.statsd_updater.redis_store') end end end diff --git a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb index eb209cce9b4..c1c80d21fdf 100644 --- a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb +++ b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb @@ -378,6 +378,16 @@ module VCAP::CloudController::Metrics StatsdUpdater::RedisStore.new(redis_socket) end end + + context 'when valkey is unreachable' do + before do + allow(redis).to receive(:set).and_raise(Redis::CannotConnectError) + end + + it 'does not raise' do + expect { StatsdUpdater::RedisStore.new(redis_socket) }.not_to raise_error + end + end end it 'increments the gauge in Redis' do @@ -389,6 +399,40 @@ module VCAP::CloudController::Metrics allow(redis).to receive(:decr).with('metrics:cc.requests.outstanding.gauge').and_return(-1) expect(redis_store.decrement_request_outstanding_gauge).to eq(-1) end + + context 'when valkey is unreachable' do + before do + allow(redis).to receive(:incr).and_raise(Redis::CannotConnectError.new('connect(2) for valkey.sock')) + allow(redis).to receive(:decr).and_raise(Redis::TimeoutError) + end + + it 'does not raise when incrementing, returning nil' do + expect(redis_store.increment_request_outstanding_gauge).to be_nil + end + + it 'does not raise when decrementing, returning nil' do + expect(redis_store.decrement_request_outstanding_gauge).to be_nil + end + + it 'logs the connection error' do + logger = instance_double(Steno::Logger, warn: nil) + allow(Steno).to receive(:logger).and_return(logger) + + redis_store.increment_request_outstanding_gauge + + expect(logger).to have_received(:warn).with(/Redis::CannotConnectError/) + end + end + + context 'when Redis raises a non-connection error' do + before do + allow(redis).to receive(:decr).and_raise(Redis::CommandError) + end + + it 'propagates the error' do + expect { redis_store.decrement_request_outstanding_gauge }.to raise_error(Redis::CommandError) + end + end end end end From 2b0fcfaecbc7830934e93005f380aace9a586f48 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:27:11 +0100 Subject: [PATCH 2/2] Fail open on Redis errors, consistent with other Valkey callers StatsdUpdater's RedisStore updates the outstanding-requests gauge in Valkey on every request. Any Redis error propagated out of the request-metrics middleware and failed the request with a 500. Wrap the RedisStore operations in a fail_open helper that logs and swallows Redis errors, matching the existing handling in base_rate_limiter.rb and concurrent_request_counter.rb: rescue Redis::BaseError, log at error with "Redis error: - ". --- lib/cloud_controller/metrics/statsd_updater.rb | 14 +++++++------- .../metrics/statsd_updater_spec.rb | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/cloud_controller/metrics/statsd_updater.rb b/lib/cloud_controller/metrics/statsd_updater.rb index 08b373b5b65..40d85798fdc 100644 --- a/lib/cloud_controller/metrics/statsd_updater.rb +++ b/lib/cloud_controller/metrics/statsd_updater.rb @@ -133,24 +133,24 @@ def initialize(socket) end @prefix = 'metrics' @request_outstanding_gauge_key = "#{@prefix}:cc.requests.outstanding.gauge" - tolerate_connection_errors { @redis.set(@request_outstanding_gauge_key, 0) } + fail_open { @redis.set(@request_outstanding_gauge_key, 0) } end def increment_request_outstanding_gauge - tolerate_connection_errors { @redis.incr(@request_outstanding_gauge_key) } + fail_open { @redis.incr(@request_outstanding_gauge_key) } end def decrement_request_outstanding_gauge - tolerate_connection_errors { @redis.decr(@request_outstanding_gauge_key) } + fail_open { @redis.decr(@request_outstanding_gauge_key) } end private - # A metrics write must never fail a user request; swallow connection errors (e.g. valkey socket gone during shutdown). - def tolerate_connection_errors + # on any Redis error (eg valkey unreachable) log it and carry on. + def fail_open yield - rescue Redis::BaseConnectionError => e - logger.warn("statsd_updater.redis_store.connection_error: #{e.class}: #{e.message}") + rescue Redis::BaseError => e + logger.error("Redis error: #{e.class} - #{e.message}") nil end diff --git a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb index c1c80d21fdf..0258ef70a33 100644 --- a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb +++ b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb @@ -414,23 +414,23 @@ module VCAP::CloudController::Metrics expect(redis_store.decrement_request_outstanding_gauge).to be_nil end - it 'logs the connection error' do - logger = instance_double(Steno::Logger, warn: nil) + it 'logs the redis error' do + logger = instance_double(Steno::Logger, error: nil) allow(Steno).to receive(:logger).and_return(logger) redis_store.increment_request_outstanding_gauge - expect(logger).to have_received(:warn).with(/Redis::CannotConnectError/) + expect(logger).to have_received(:error).with(/Redis error: Redis::CannotConnectError/) end end - context 'when Redis raises a non-connection error' do + context 'when Redis raises a command error' do before do allow(redis).to receive(:decr).and_raise(Redis::CommandError) end - it 'propagates the error' do - expect { redis_store.decrement_request_outstanding_gauge }.to raise_error(Redis::CommandError) + it 'fails open and returns nil' do + expect(redis_store.decrement_request_outstanding_gauge).to be_nil end end end