Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions lib/cloud_controller/metrics/statsd_updater.rb
Original file line number Diff line number Diff line change
Expand Up @@ -133,15 +133,29 @@ def initialize(socket)
end
@prefix = 'metrics'
@request_outstanding_gauge_key = "#{@prefix}:cc.requests.outstanding.gauge"
@redis.set(@request_outstanding_gauge_key, 0)
fail_open { @redis.set(@request_outstanding_gauge_key, 0) }
end

def increment_request_outstanding_gauge
@redis.incr(@request_outstanding_gauge_key)
fail_open { @redis.incr(@request_outstanding_gauge_key) }
end

def decrement_request_outstanding_gauge
@redis.decr(@request_outstanding_gauge_key)
fail_open { @redis.decr(@request_outstanding_gauge_key) }
end

private

# on any Redis error (eg valkey unreachable) log it and carry on.
def fail_open
yield
rescue Redis::BaseError => e
logger.error("Redis error: #{e.class} - #{e.message}")
nil
end

def logger
@logger ||= Steno.logger('cc.metrics.statsd_updater.redis_store')
end
end
end
Expand Down
44 changes: 44 additions & 0 deletions spec/unit/lib/cloud_controller/metrics/statsd_updater_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,16 @@ module VCAP::CloudController::Metrics
StatsdUpdater::RedisStore.new(redis_socket)
end
end

context 'when valkey is unreachable' do
before do
allow(redis).to receive(:set).and_raise(Redis::CannotConnectError)
end

it 'does not raise' do
expect { StatsdUpdater::RedisStore.new(redis_socket) }.not_to raise_error
end
end
end

it 'increments the gauge in Redis' do
Expand All @@ -389,6 +399,40 @@ module VCAP::CloudController::Metrics
allow(redis).to receive(:decr).with('metrics:cc.requests.outstanding.gauge').and_return(-1)
expect(redis_store.decrement_request_outstanding_gauge).to eq(-1)
end

context 'when valkey is unreachable' do
before do
allow(redis).to receive(:incr).and_raise(Redis::CannotConnectError.new('connect(2) for valkey.sock'))
allow(redis).to receive(:decr).and_raise(Redis::TimeoutError)
end

it 'does not raise when incrementing, returning nil' do
expect(redis_store.increment_request_outstanding_gauge).to be_nil
end

it 'does not raise when decrementing, returning nil' do
expect(redis_store.decrement_request_outstanding_gauge).to be_nil
end

it 'logs the redis error' do
logger = instance_double(Steno::Logger, error: nil)
allow(Steno).to receive(:logger).and_return(logger)

redis_store.increment_request_outstanding_gauge

expect(logger).to have_received(:error).with(/Redis error: Redis::CannotConnectError/)
end
end

context 'when Redis raises a command error' do
before do
allow(redis).to receive(:decr).and_raise(Redis::CommandError)
end

it 'fails open and returns nil' do
expect(redis_store.decrement_request_outstanding_gauge).to be_nil
end
end
end
end
end
Loading