diff --git a/lib/cloud_controller/metrics/statsd_updater.rb b/lib/cloud_controller/metrics/statsd_updater.rb index 685ee75b7ce..40d85798fdc 100644 --- a/lib/cloud_controller/metrics/statsd_updater.rb +++ b/lib/cloud_controller/metrics/statsd_updater.rb @@ -133,15 +133,29 @@ def initialize(socket) end @prefix = 'metrics' @request_outstanding_gauge_key = "#{@prefix}:cc.requests.outstanding.gauge" - @redis.set(@request_outstanding_gauge_key, 0) + fail_open { @redis.set(@request_outstanding_gauge_key, 0) } end def increment_request_outstanding_gauge - @redis.incr(@request_outstanding_gauge_key) + fail_open { @redis.incr(@request_outstanding_gauge_key) } end def decrement_request_outstanding_gauge - @redis.decr(@request_outstanding_gauge_key) + fail_open { @redis.decr(@request_outstanding_gauge_key) } + end + + private + + # on any Redis error (eg valkey unreachable) log it and carry on. + def fail_open + yield + rescue Redis::BaseError => e + logger.error("Redis error: #{e.class} - #{e.message}") + nil + end + + def logger + @logger ||= Steno.logger('cc.metrics.statsd_updater.redis_store') end end end diff --git a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb index eb209cce9b4..0258ef70a33 100644 --- a/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb +++ b/spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb @@ -378,6 +378,16 @@ module VCAP::CloudController::Metrics StatsdUpdater::RedisStore.new(redis_socket) end end + + context 'when valkey is unreachable' do + before do + allow(redis).to receive(:set).and_raise(Redis::CannotConnectError) + end + + it 'does not raise' do + expect { StatsdUpdater::RedisStore.new(redis_socket) }.not_to raise_error + end + end end it 'increments the gauge in Redis' do @@ -389,6 +399,40 @@ module VCAP::CloudController::Metrics allow(redis).to receive(:decr).with('metrics:cc.requests.outstanding.gauge').and_return(-1) expect(redis_store.decrement_request_outstanding_gauge).to eq(-1) end + + context 'when valkey is unreachable' do + before do + allow(redis).to receive(:incr).and_raise(Redis::CannotConnectError.new('connect(2) for valkey.sock')) + allow(redis).to receive(:decr).and_raise(Redis::TimeoutError) + end + + it 'does not raise when incrementing, returning nil' do + expect(redis_store.increment_request_outstanding_gauge).to be_nil + end + + it 'does not raise when decrementing, returning nil' do + expect(redis_store.decrement_request_outstanding_gauge).to be_nil + end + + it 'logs the redis error' do + logger = instance_double(Steno::Logger, error: nil) + allow(Steno).to receive(:logger).and_return(logger) + + redis_store.increment_request_outstanding_gauge + + expect(logger).to have_received(:error).with(/Redis error: Redis::CannotConnectError/) + end + end + + context 'when Redis raises a command error' do + before do + allow(redis).to receive(:decr).and_raise(Redis::CommandError) + end + + it 'fails open and returns nil' do + expect(redis_store.decrement_request_outstanding_gauge).to be_nil + end + end end end end