Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions lib/semian/redis.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ def initialize(semian_identifier, *args)

class OutOfMemoryError < Redis::CommandError
include ::Semian::AdapterError

# An OutOfMemoryError is a fast failure. We don't want to open circuits
# because that would block read and dequeue operations that could help
# Redis recover from the OOM state.
def marks_semian_circuits?
false
end
end

class ConnectionError < Redis::BaseConnectionError
Expand Down
8 changes: 8 additions & 0 deletions lib/semian/redis_client.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ def marks_semian_circuits?
end

OutOfMemoryError.include(::Semian::AdapterError)
OutOfMemoryError.class_eval do
# An OutOfMemoryError is a fast failure. We don't want to open circuits
# because that would block read and dequeue operations that could help
# Redis recover from the OOM state.
def marks_semian_circuits?
false
end
end

class ReadOnlyError < RedisClient::ConnectionError
# A ReadOnlyError is a fast failure and we don't want to track these errors so that we can reconnect
Expand Down
20 changes: 14 additions & 6 deletions test/adapters/redis_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -105,37 +105,45 @@ def test_command_errors_does_not_open_the_circuit
end
end

def test_command_errors_because_of_oom_do_open_the_circuit
def test_command_errors_because_of_oom_do_not_open_the_circuit
# OOM errors should NOT open the circuit, because:
# 1. They are fast failures (no benefit from failing fast)
# 2. Blocking reads/dequeues prevents recovery from OOM
client = connect_to_redis!

with_maxmemory(1) do
ERROR_THRESHOLD.times do
(ERROR_THRESHOLD + 1).times do
exception = assert_raises(::Redis::OutOfMemoryError) do
client.set("foo", "bar")
end

assert_equal(:redis_testing, exception.semian_identifier)
end

assert_raises(::Redis::CircuitOpenError) do
# Circuit should NOT be open - we should still get OOM errors, not CircuitOpenError
assert_raises(::Redis::OutOfMemoryError) do
client.set("foo", "bla")
end
end
end

def test_script_errors_because_of_oom_do_open_the_circuit
def test_script_errors_because_of_oom_do_not_open_the_circuit
# OOM errors should NOT open the circuit, because:
# 1. They are fast failures (no benefit from failing fast)
# 2. Blocking reads/dequeues prevents recovery from OOM
client = connect_to_redis!

with_maxmemory(1) do
ERROR_THRESHOLD.times do
(ERROR_THRESHOLD + 1).times do
exception = assert_raises(::Redis::OutOfMemoryError) do
client.eval("return redis.call('set', 'foo', 'bar');")
end

assert_equal(:redis_testing, exception.semian_identifier)
end

assert_raises(::Redis::CircuitOpenError) do
# Circuit should NOT be open - we should still get OOM errors, not CircuitOpenError
assert_raises(::Redis::OutOfMemoryError) do
client.eval("return redis.call('set', 'foo', 'bar');")
end
end
Expand Down
Loading