Skip to content

Commit

Permalink
feat: initialize the state of a breaker on creation (#574)
Browse files Browse the repository at this point in the history
* Motivation:  If a user is using opossum in a serverless environment where applications are scaling to zero, the CB stats and state are being reset everytime the application comes back up. This adds the ability to initialize a circuit with previous stats and state.  Allowing the breaker to pick up where it left off

Co-authored-by: Lance Ball <[email protected]>
  • Loading branch information
lholmquist and lance authored Jul 7, 2021
1 parent 16354f2 commit b3dd431
Show file tree
Hide file tree
Showing 4 changed files with 264 additions and 8 deletions.
37 changes: 36 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,44 @@ const breaker = new CircuitBreaker(delay);
breaker.fire(20000, 1, 2, 3);
breaker.fallback((delay, a, b, c) => `Sorry, out of service right now. But your parameters are: ${delay}, ${a}, ${b} and ${c}`);
```
### Breaker State Initialization

There may be times where you will need to initialize the state of a Circuit Breaker. Primary use cases for this are in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral.

The `toJSON` method is a helper function to get the current state and status of a breaker:

```
const breakerState = breaker.toJSON();
```

This will return an object that might look similar to this:

```
{
state: {
enabled: true,
name: 'functionName'
closed: true,
open: false,
halfOpen: false,
warmUp: false,
shutdown: false
},
status: {
...
}
};
```

A new circuit breaker instance can be created with this state by passing this object in:

```
const breaker = new CircuitBreaker({state: state});
```

### Status Initialization

There may be times where you will need to pre-populate the stats of the Circuit Breaker Status Object. Primary use cases for this are in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral.
There may also be times where you will need to pre-populate the stats of the Circuit Breaker Status Object. Primary use cases for this are also in a serverless environment such as Knative or AWS Lambda, or any container based platform, where the container being deployed is ephemeral.

Getting the existing cumalative stats for a breaker can be done like this:

Expand Down
54 changes: 50 additions & 4 deletions lib/circuit.js
Original file line number Diff line number Diff line change
Expand Up @@ -161,19 +161,35 @@ class CircuitBreaker extends EventEmitter {
if (this.options.status instanceof Status) {
this[STATUS] = this.options.status;
} else {
// should it error if it is not a Status Object
throw new TypeError('Not a Status Object. The status option must be an instance of Status');
this[STATUS] = new Status({ stats: this.options.status });
}
} else {
this[STATUS] = new Status(this.options);
}

this[STATE] = CLOSED;

if (options.state) {
this[ENABLED] = options.state.enabled !== false;
this[WARMING_UP] = options.state.warmUp || this[WARMING_UP];
// Closed if nothing is passed in
this[CLOSED] = options.state.closed !== false;
// These should be in sync
this[HALF_OPEN] = this[PENDING_CLOSE] = options.state.halfOpen || false;
// Open should be the opposite of closed,
// but also the opposite of half_open
this[OPEN] = !this[CLOSED] && !this[HALF_OPEN];

this[SHUTDOWN] = options.state.shutdown || false;

} else {
this[PENDING_CLOSE] = false;
this[ENABLED] = options.enabled !== false;
}

this[FALLBACK_FUNCTION] = null;
this[PENDING_CLOSE] = false;
this[NAME] = options.name || action.name || nextName();
this[GROUP] = options.group || this[NAME];
this[ENABLED] = options.enabled !== false;

if (this[WARMING_UP]) {
const timer = this[WARMUP_TIMEOUT] = setTimeout(
Expand Down Expand Up @@ -241,6 +257,21 @@ class CircuitBreaker extends EventEmitter {
if (this.options.cache) {
CACHE.set(this, undefined);
}

// Prepopulate the State of the Breaker
if (this[SHUTDOWN]) {
this[STATE] = SHUTDOWN;
this.shutdown();
} else if (this[CLOSED]) {
this.close();
} else if (this[OPEN]) {
// If the state being passed in is OPEN
// THen we need to start some timers
this.open();
} else if (this[HALF_OPEN]) {
// Not sure if anythign needs to be done here
this[STATE] = HALF_OPEN;
}
}

/**
Expand Down Expand Up @@ -381,6 +412,21 @@ class CircuitBreaker extends EventEmitter {
return this[STATUS].stats;
}

toJSON () {
return {
state: {
name: this.name,
enabled: this.enabled,
closed: this.closed,
open: this.opened,
halfOpen: this.halfOpen,
warmUp: this.warmUp,
shutdown: this.isShutdown
},
status: this.status.stats
};
}

/**
* Gets whether the circuit is enabled or not
* @type {Boolean}
Expand Down
175 changes: 175 additions & 0 deletions test/state-test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
'use strict';

const test = require('tape');
const CircuitBreaker = require('../');
const { timedFailingFunction, passFail } = require('./common');

test('CircuitBreaker State - export the state of a breaker instance', t => {
t.plan(7);
const breaker = new CircuitBreaker(passFail);

t.ok(breaker.toJSON, 'has the toJSON function');
const breakerState = breaker.toJSON();

t.equal(breakerState.state.enabled, true, 'enabled initialized value');
t.equal(breakerState.state.closed, true, 'closed initialized value');
t.equal(breakerState.state.open, false, 'open initialized value');
t.equal(breakerState.state.halfOpen, false, 'half open initialized value');
t.equal(breakerState.state.warmUp, false, 'warmup initialized value');
t.equal(breakerState.state.shutdown, false, 'shutdown initialized value');
t.end();
});

test('CircuitBreaker State - export the state of a breaker instance using toJson', t => {
t.plan(8);
const breaker = new CircuitBreaker(passFail);

t.ok(breaker.toJSON, 'has the toJSON function');
const breakerState = breaker.toJSON();

t.equal(breakerState.state.name, 'passFail', 'name initialized value');
t.equal(breakerState.state.enabled, true, 'enabled initialized value');
t.equal(breakerState.state.closed, true, 'closed initialized value');
t.equal(breakerState.state.open, false, 'open initialized value');
t.equal(breakerState.state.halfOpen, false, 'half open initialized value');
t.equal(breakerState.state.warmUp, false, 'warmup initialized value');
t.equal(breakerState.state.shutdown, false, 'shutdown initialized value');
t.end();
});

test('CircuitBreaker State - initalize the breaker as Closed', t => {
t.plan(8);

const state = {
enabled: true,
closed: true,
halfOpen: false,
warmUp: false,
pendingClose: false,
shutdown: false
};

const breaker = new CircuitBreaker(passFail, {state});
const breakerState = breaker.toJSON();

t.equal(breakerState.state.enabled, true, 'enabled primed value');
t.equal(breakerState.state.closed, true, 'closed primed value');
t.equal(breakerState.state.open, false, 'open primed value');
t.equal(breakerState.state.halfOpen, false, 'half open primed value');
t.equal(breakerState.state.warmUp, false, 'warmup primed value');
t.equal(breakerState.state.shutdown, false, 'shutdown primed value');

breaker.fire(-1).then(() => {
t.fail();
}).catch(() => {
t.equal(breaker.opened, true, 'breaker should be open');
t.equal(breaker.closed, false, 'breaker should not be closed');

t.end();
});
});

test('Pre-populate state as Open(Closed === false) - Breaker resets after a configurable amount of time', t => {
t.plan(1);
const resetTimeout = 100;
const breaker = new CircuitBreaker(passFail, {
errorThresholdPercentage: 1,
resetTimeout,
state: {
closed: false
}
});

// Now the breaker should be open. Wait for reset and
// fire again.
setTimeout(() => {
breaker.fire(100)
.catch(t.fail)
.then(arg => t.equals(arg, 100, 'breaker has reset'))
.then(_ => breaker.shutdown())
.then(t.end);
}, resetTimeout * 1.25);
});

test('When half-open, the circuit only allows one request through', t => {
t.plan(7);
const options = {
errorThresholdPercentage: 1,
resetTimeout: 100,
state: {
closed: false,
halfOpen: true
}
};

const breaker = new CircuitBreaker(timedFailingFunction, options);
t.ok(breaker.halfOpen, 'should be halfOpen on initalize');
t.ok(breaker.pendingClose, 'should be pending close on initalize');

breaker
.fire(500) // fail after a long time, letting other fire()s occur
.catch(e => t.equals(e, 'Failed after 500', 'should fail again'))
.then(() => {
t.ok(breaker.opened,
'should be open again after long failing function');
t.notOk(breaker.halfOpen,
'should not be halfOpen after long failing function');
t.notOk(breaker.pendingClose,
'should not be pending close after long failing func');
})
.then(_ => {
// fire the breaker again, and be sure it fails as expected
breaker
.fire(1)
.catch(e => t.equals(e.message, 'Breaker is open'));
})
.then(_ => breaker.shutdown())
.then(t.end);
});

test('Circuit initalized as shutdown', t => {
t.plan(5);
const options = {
state: {
closed: false,
open: false,
shutdown: true
}
};
const breaker = new CircuitBreaker(passFail, options);
t.ok(breaker.isShutdown, 'breaker is shutdown');
t.notOk(breaker.enabled, 'breaker has been disabled');
breaker.fire(1)
.then(t.fail)
.catch(err => {
t.equals('ESHUTDOWN', err.code);
t.equals('The circuit has been shutdown.', err.message);
t.equals(
CircuitBreaker.isOurError(err), true, 'isOurError() should return true'
);
t.end();
});
});

test('CircuitBreaker State - imports the state and status of a breaker instance', t => {
const breaker = new CircuitBreaker(passFail);
const exp = breaker.toJSON();
const state = exp.state;
const status = exp.status;
const clone = new CircuitBreaker(passFail, { state, status });
// test imported state
for (const stat of ['enabled', 'closed', 'halfOpen', 'warmUp']) {
t.equal(clone[status], state[status], `cloned breaker ${stat} state`);
}
t.equal(clone.opened, state.open, 'cloned breaker open state');
t.equal(clone.isShutdown, state.shutdown, 'cloned breaker shutdown state');
// test imported status
const stats = clone.stats;
for (const stat in stats) {
t.equal(
stats[stat].toString(),
status[stat].toString(),
`cloned stat ${stat} value`);
}
t.end();
});
6 changes: 3 additions & 3 deletions test/status-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ test('CircuitBreaker status - import stats,but not a status object', t => {
errorThresholdPercentage: 1,
status: prevStats
});
t.fail();
} catch (err) {
if (err instanceof TypeError) t.pass();
t.pass();
t.end();
} catch (err) {
t.fail();
}
});

0 comments on commit b3dd431

Please sign in to comment.