From b3dd431bee343dd58c8612868333c90d0edbb83a Mon Sep 17 00:00:00 2001 From: Lucas Holmquist Date: Wed, 7 Jul 2021 12:31:33 -0400 Subject: [PATCH] feat: initialize the state of a breaker on creation (#574) * Motivation: If a user is using opossum in a serverless environment where applications are scaling to zero, the CB stats and state are being reset everytime the application comes back up. This adds the ability to initialize a circuit with previous stats and state. Allowing the breaker to pick up where it left off Co-authored-by: Lance Ball --- README.md | 37 +++++++++- lib/circuit.js | 54 +++++++++++++- test/state-test.js | 175 ++++++++++++++++++++++++++++++++++++++++++++ test/status-test.js | 6 +- 4 files changed, 264 insertions(+), 8 deletions(-) create mode 100644 test/state-test.js diff --git a/README.md b/README.md index 8bd81a42..d30a6e1b 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,44 @@ const breaker = new CircuitBreaker(delay); breaker.fire(20000, 1, 2, 3); breaker.fallback((delay, a, b, c) => `Sorry, out of service right now. But your parameters are: ${delay}, ${a}, ${b} and ${c}`); ``` +### Breaker State Initialization + +There may be times where you will need to initialize the state of a Circuit Breaker. Primary use cases for this are in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral. + +The `toJSON` method is a helper function to get the current state and status of a breaker: + +``` +const breakerState = breaker.toJSON(); +``` + +This will return an object that might look similar to this: + +``` +{ + state: { + enabled: true, + name: 'functionName' + closed: true, + open: false, + halfOpen: false, + warmUp: false, + shutdown: false + }, + status: { + ... + } +}; +``` + +A new circuit breaker instance can be created with this state by passing this object in: + +``` +const breaker = new CircuitBreaker({state: state}); +``` + ### Status Initialization -There may be times where you will need to pre-populate the stats of the Circuit Breaker Status Object. Primary use cases for this are in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral. +There may also be times where you will need to pre-populate the stats of the Circuit Breaker Status Object. Primary use cases for this are also in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral. Getting the existing cumalative stats for a breaker can be done like this: diff --git a/lib/circuit.js b/lib/circuit.js index 2a36ae82..9058c4f6 100644 --- a/lib/circuit.js +++ b/lib/circuit.js @@ -161,19 +161,35 @@ class CircuitBreaker extends EventEmitter { if (this.options.status instanceof Status) { this[STATUS] = this.options.status; } else { - // should it error if it is not a Status Object - throw new TypeError('Not a Status Object. The status option must be an instance of Status'); + this[STATUS] = new Status({ stats: this.options.status }); } } else { this[STATUS] = new Status(this.options); } this[STATE] = CLOSED; + + if (options.state) { + this[ENABLED] = options.state.enabled !== false; + this[WARMING_UP] = options.state.warmUp || this[WARMING_UP]; + // Closed if nothing is passed in + this[CLOSED] = options.state.closed !== false; + // These should be in sync + this[HALF_OPEN] = this[PENDING_CLOSE] = options.state.halfOpen || false; + // Open should be the opposite of closed, + // but also the opposite of half_open + this[OPEN] = !this[CLOSED] && !this[HALF_OPEN]; + + this[SHUTDOWN] = options.state.shutdown || false; + + } else { + this[PENDING_CLOSE] = false; + this[ENABLED] = options.enabled !== false; + } + this[FALLBACK_FUNCTION] = null; - this[PENDING_CLOSE] = false; this[NAME] = options.name || action.name || nextName(); this[GROUP] = options.group || this[NAME]; - this[ENABLED] = options.enabled !== false; if (this[WARMING_UP]) { const timer = this[WARMUP_TIMEOUT] = setTimeout( @@ -241,6 +257,21 @@ class CircuitBreaker extends EventEmitter { if (this.options.cache) { CACHE.set(this, undefined); } + + // Prepopulate the State of the Breaker + if (this[SHUTDOWN]) { + this[STATE] = SHUTDOWN; + this.shutdown(); + } else if (this[CLOSED]) { + this.close(); + } else if (this[OPEN]) { + // If the state being passed in is OPEN + // THen we need to start some timers + this.open(); + } else if (this[HALF_OPEN]) { + // Not sure if anythign needs to be done here + this[STATE] = HALF_OPEN; + } } /** @@ -381,6 +412,21 @@ class CircuitBreaker extends EventEmitter { return this[STATUS].stats; } + toJSON () { + return { + state: { + name: this.name, + enabled: this.enabled, + closed: this.closed, + open: this.opened, + halfOpen: this.halfOpen, + warmUp: this.warmUp, + shutdown: this.isShutdown + }, + status: this.status.stats + }; + } + /** * Gets whether the circuit is enabled or not * @type {Boolean} diff --git a/test/state-test.js b/test/state-test.js new file mode 100644 index 00000000..a2187972 --- /dev/null +++ b/test/state-test.js @@ -0,0 +1,175 @@ +'use strict'; + +const test = require('tape'); +const CircuitBreaker = require('../'); +const { timedFailingFunction, passFail } = require('./common'); + +test('CircuitBreaker State - export the state of a breaker instance', t => { + t.plan(7); + const breaker = new CircuitBreaker(passFail); + + t.ok(breaker.toJSON, 'has the toJSON function'); + const breakerState = breaker.toJSON(); + + t.equal(breakerState.state.enabled, true, 'enabled initialized value'); + t.equal(breakerState.state.closed, true, 'closed initialized value'); + t.equal(breakerState.state.open, false, 'open initialized value'); + t.equal(breakerState.state.halfOpen, false, 'half open initialized value'); + t.equal(breakerState.state.warmUp, false, 'warmup initialized value'); + t.equal(breakerState.state.shutdown, false, 'shutdown initialized value'); + t.end(); +}); + +test('CircuitBreaker State - export the state of a breaker instance using toJson', t => { + t.plan(8); + const breaker = new CircuitBreaker(passFail); + + t.ok(breaker.toJSON, 'has the toJSON function'); + const breakerState = breaker.toJSON(); + + t.equal(breakerState.state.name, 'passFail', 'name initialized value'); + t.equal(breakerState.state.enabled, true, 'enabled initialized value'); + t.equal(breakerState.state.closed, true, 'closed initialized value'); + t.equal(breakerState.state.open, false, 'open initialized value'); + t.equal(breakerState.state.halfOpen, false, 'half open initialized value'); + t.equal(breakerState.state.warmUp, false, 'warmup initialized value'); + t.equal(breakerState.state.shutdown, false, 'shutdown initialized value'); + t.end(); +}); + +test('CircuitBreaker State - initalize the breaker as Closed', t => { + t.plan(8); + + const state = { + enabled: true, + closed: true, + halfOpen: false, + warmUp: false, + pendingClose: false, + shutdown: false + }; + + const breaker = new CircuitBreaker(passFail, {state}); + const breakerState = breaker.toJSON(); + + t.equal(breakerState.state.enabled, true, 'enabled primed value'); + t.equal(breakerState.state.closed, true, 'closed primed value'); + t.equal(breakerState.state.open, false, 'open primed value'); + t.equal(breakerState.state.halfOpen, false, 'half open primed value'); + t.equal(breakerState.state.warmUp, false, 'warmup primed value'); + t.equal(breakerState.state.shutdown, false, 'shutdown primed value'); + + breaker.fire(-1).then(() => { + t.fail(); + }).catch(() => { + t.equal(breaker.opened, true, 'breaker should be open'); + t.equal(breaker.closed, false, 'breaker should not be closed'); + + t.end(); + }); +}); + +test('Pre-populate state as Open(Closed === false) - Breaker resets after a configurable amount of time', t => { + t.plan(1); + const resetTimeout = 100; + const breaker = new CircuitBreaker(passFail, { + errorThresholdPercentage: 1, + resetTimeout, + state: { + closed: false + } + }); + + // Now the breaker should be open. Wait for reset and + // fire again. + setTimeout(() => { + breaker.fire(100) + .catch(t.fail) + .then(arg => t.equals(arg, 100, 'breaker has reset')) + .then(_ => breaker.shutdown()) + .then(t.end); + }, resetTimeout * 1.25); +}); + +test('When half-open, the circuit only allows one request through', t => { + t.plan(7); + const options = { + errorThresholdPercentage: 1, + resetTimeout: 100, + state: { + closed: false, + halfOpen: true + } + }; + + const breaker = new CircuitBreaker(timedFailingFunction, options); + t.ok(breaker.halfOpen, 'should be halfOpen on initalize'); + t.ok(breaker.pendingClose, 'should be pending close on initalize'); + + breaker + .fire(500) // fail after a long time, letting other fire()s occur + .catch(e => t.equals(e, 'Failed after 500', 'should fail again')) + .then(() => { + t.ok(breaker.opened, + 'should be open again after long failing function'); + t.notOk(breaker.halfOpen, + 'should not be halfOpen after long failing function'); + t.notOk(breaker.pendingClose, + 'should not be pending close after long failing func'); + }) + .then(_ => { + // fire the breaker again, and be sure it fails as expected + breaker + .fire(1) + .catch(e => t.equals(e.message, 'Breaker is open')); + }) + .then(_ => breaker.shutdown()) + .then(t.end); +}); + +test('Circuit initalized as shutdown', t => { + t.plan(5); + const options = { + state: { + closed: false, + open: false, + shutdown: true + } + }; + const breaker = new CircuitBreaker(passFail, options); + t.ok(breaker.isShutdown, 'breaker is shutdown'); + t.notOk(breaker.enabled, 'breaker has been disabled'); + breaker.fire(1) + .then(t.fail) + .catch(err => { + t.equals('ESHUTDOWN', err.code); + t.equals('The circuit has been shutdown.', err.message); + t.equals( + CircuitBreaker.isOurError(err), true, 'isOurError() should return true' + ); + t.end(); + }); +}); + +test('CircuitBreaker State - imports the state and status of a breaker instance', t => { + const breaker = new CircuitBreaker(passFail); + const exp = breaker.toJSON(); + const state = exp.state; + const status = exp.status; + const clone = new CircuitBreaker(passFail, { state, status }); + // test imported state + for (const stat of ['enabled', 'closed', 'halfOpen', 'warmUp']) { + t.equal(clone[status], state[status], `cloned breaker ${stat} state`); + } + t.equal(clone.opened, state.open, 'cloned breaker open state'); + t.equal(clone.isShutdown, state.shutdown, 'cloned breaker shutdown state'); + // test imported status + const stats = clone.stats; + for (const stat in stats) { + t.equal( + stats[stat].toString(), + status[stat].toString(), + `cloned stat ${stat} value`); + } + t.end(); +}); diff --git a/test/status-test.js b/test/status-test.js index 9ba16be8..c8243bc5 100644 --- a/test/status-test.js +++ b/test/status-test.js @@ -151,9 +151,9 @@ test('CircuitBreaker status - import stats,but not a status object', t => { errorThresholdPercentage: 1, status: prevStats }); - t.fail(); - } catch (err) { - if (err instanceof TypeError) t.pass(); + t.pass(); t.end(); + } catch (err) { + t.fail(); } });