Skip to content

Commit f09b51e

Browse files
noblnobl
committed
- API uses its own backoff settings when polling. You can only configure delay between polls. Instead we are using strategy used by: AWS SDK, Google Cloud SDK, Stripe API client, Kubernetes controllers or Distributed message brokers to prevent: retry storms, thundering herd, burst collapse after outage recovery, adapter lockups or permanent dead loops. This leads to: IF (SENEC API down for 2 hours, or Token refresh fails 20 times, or 429 rate limiting kicks in, or Internet drops temporarily) ? (Never dies, never overlaps, never floods API, always recovers)
- API polling no longer honors retries-setting. It will just keep backing off exponentially if errors persist -> we keep trying until you stop the adapter. - 401 won't throw warning anymore
1 parent 69e65db commit f09b51e

2 files changed

Lines changed: 112 additions & 77 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,10 @@ This channel contains values polled from SENEC App-API.
305305
-->
306306

307307
### **WORK IN PROGRESS**
308+
- API uses its own backoff settings when polling. You can only configure delay between polls. Instead we are using strategy used by: AWS SDK, Google Cloud SDK, Stripe API client, Kubernetes controllers or Distributed message brokers to prevent: retry storms, thundering herd, burst collapse after outage recovery, adapter lockups or permanent dead loops. This leads to: IF (SENEC API down for 2 hours, or Token refresh fails 20 times, or 429 rate limiting kicks in, or Internet drops temporarily) ? (Never dies, never overlaps, never floods API, always recovers)
309+
- API polling no longer honors retries-setting. It will just keep backing off exponentially if errors persist -> we keep trying until you stop the adapter.
308310
- 401 won't throw warning anymore
311+
- ReAuth shouldn't stop polling anymore
309312

310313
### 2.4.2 (2026-03-03)
311314
- AuthToken in _api is no longer used. You can safely delete it.

‎main.js‎

Lines changed: 109 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,7 @@ class Senec extends utils.Adapter {
622622
return this.refreshPromise;
623623
}
624624

625-
async pollSenecApi(retry = 0) {
625+
async pollSenecApi() {
626626
if (!this.config.api_use || !apiConnected) {
627627
this.log.info("Usage of SENEC App API not configured or not connected.");
628628
return;
@@ -632,22 +632,26 @@ class Senec extends utils.Adapter {
632632
this.log.warn("API poll still running — skipping overlapping execution.");
633633
return;
634634
}
635+
635636
this.apiPollRunning = true;
636-
const interval = this.config.api_interval * 60000;
637+
638+
const baseInterval = this.config.api_interval * 60000;
639+
let nextDelay = baseInterval;
637640

638641
try {
639642
this.log.info("🔄 Polling SENEC App API...");
643+
640644
// Ensure token exists
641645
if (!this.currentToken) {
642646
await this.refreshTokenSingleFlight();
643647
}
644648

645-
// Read systems once from API and keep them in memory - we will need them for every poll and they don't change that often - this also ensures that we have the correct system IDs in case they change or we have multiple systems
649+
// Read systems once from API and keep them in memory - we will need them for every poll and they don't change that often
646650
if (apiKnownSystems.size === 0) {
647651
this.log.debug("🔄 Reading available systems from API ...");
648652
const sysRes = await this.apiGet(`${HOST_SYSTEMS}/v1/systems`);
649-
if (!sysRes.data || !sysRes.data[0]) {
650-
throw new Error("No Appliances found.");
653+
if (!sysRes?.data?.length) {
654+
throw new Error("No systems returned from API.");
651655
}
652656
for (const sys of sysRes.data) {
653657
this.log.debug(`System found: ${JSON.stringify(sys)}`);
@@ -663,6 +667,7 @@ class Senec extends utils.Adapter {
663667
const utcYear = now.getUTCFullYear();
664668
const utcMonth = now.getUTCMonth();
665669
const utcDate = now.getUTCDate();
670+
666671
const today = new Date(utcYear, utcMonth, utcDate, 0, 0, 0, 0);
667672
const yesterday = new Date(utcYear, utcMonth, utcDate - 1, 0, 0, 0, 0);
668673
const currentMonth = new Date(Date.UTC(utcYear, utcMonth, 1));
@@ -673,62 +678,72 @@ class Senec extends utils.Adapter {
673678
const shouldRunHeavy = nowTs - this.lastHeavyUpdate > heavyInterval;
674679

675680
for (const anlagenId of apiKnownSystems) {
676-
this.log.info(`🔄 Polling system ${anlagenId}...`);
677-
678-
// Dashboard (frequent)
679-
const dashRes = await this.apiGet(`${HOST_MEASUREMENTS}/v1/systems/${anlagenId}/dashboard`);
680-
this.log.silly(`DashRes keys: ${Object.keys(dashRes.data).join(", ")}`);
681-
this.evalPoll(dashRes.data, `${API_PFX}Anlagen.${anlagenId}.Dashboard.`);
682-
683-
// Frequent measurements
684-
await Promise.all([
685-
this.doMeasurementsDay(anlagenId, today, "today"),
686-
this.doMeasurementsDay(anlagenId, today, "today.horly"),
687-
this.doMeasurementsDay(anlagenId, yesterday, "yesterday"),
688-
this.doMeasurementsDay(anlagenId, yesterday, "yesterday.hourly"),
689-
]);
690-
691-
// Heavy measurements (once daily)
692-
if (shouldRunHeavy) {
681+
try {
682+
this.log.info(`🔄 Polling system ${anlagenId}...`);
683+
684+
// Dashboard (frequent)
685+
const dashRes = await this.apiGet(`${HOST_MEASUREMENTS}/v1/systems/${anlagenId}/dashboard`);
686+
this.log.silly(`DashRes keys: ${Object.keys(dashRes.data).join(", ")}`);
687+
this.evalPoll(dashRes.data, `${API_PFX}Anlagen.${anlagenId}.Dashboard.`);
688+
689+
// Frequent measurements
693690
await Promise.all([
694-
this.doMeasurementsMonth(anlagenId, currentMonth, "current_month"),
695-
this.doMeasurementsMonth(anlagenId, currentMonth, "current_month.daily"),
696-
this.doMeasurementsMonth(anlagenId, lastMonth, "previous_month"),
697-
this.doMeasurementsMonth(anlagenId, lastMonth, "previous_month.daily"),
698-
this.doMeasurementsYear(anlagenId, utcYear, false), // Current year
699-
this.doMeasurementsYear(anlagenId, utcYear, true), // Current year
700-
this.doMeasurementsYear(anlagenId, utcYear - 1, false), // check if we need last year too
701-
this.doMeasurementsYear(anlagenId, utcYear - 1, true), // check if we need last year too
691+
this.doMeasurementsDay(anlagenId, today, "today"),
692+
this.doMeasurementsDay(anlagenId, today, "today.horly"),
693+
this.doMeasurementsDay(anlagenId, yesterday, "yesterday"),
694+
this.doMeasurementsDay(anlagenId, yesterday, "yesterday.hourly"),
702695
]);
703-
this.lastHeavyUpdate = nowTs;
704-
await this.updateAllTimeHistory(anlagenId);
705-
}
706696

707-
if (this.config.api_alltimeRebuild) {
708-
// rebuild all-time history if requested - will also pull everying again
709-
await this.doRebuild(anlagenId);
710-
}
711-
}
712-
713-
// schedule next poll
714-
if (!unloaded) {
715-
this.timerAPI = setTimeout(() => this.pollSenecApi(0), interval);
716-
}
717-
} catch (e) {
718-
this.log.error(`❌ API error: ${e.message}`);
697+
// Heavy measurements (once daily)
698+
if (shouldRunHeavy) {
699+
await Promise.all([
700+
this.doMeasurementsMonth(anlagenId, currentMonth, "current_month"),
701+
this.doMeasurementsMonth(anlagenId, currentMonth, "current_month.daily"),
702+
this.doMeasurementsMonth(anlagenId, lastMonth, "previous_month"),
703+
this.doMeasurementsMonth(anlagenId, lastMonth, "previous_month.daily"),
704+
this.doMeasurementsYear(anlagenId, utcYear, false), // Current year
705+
this.doMeasurementsYear(anlagenId, utcYear, true), // Current year
706+
this.doMeasurementsYear(anlagenId, utcYear - 1, false), // check if we need last year too
707+
this.doMeasurementsYear(anlagenId, utcYear - 1, true), // check if we need last year too
708+
]);
709+
this.lastHeavyUpdate = nowTs;
710+
await this.updateAllTimeHistory(anlagenId);
711+
}
719712

720-
if (retry >= this.config.retries) {
721-
this.log.error(`API: Retried ${retry} times. Giving up. Restart adapter.`);
722-
this.setState("info.connection", false, true);
723-
return;
713+
if (this.config.api_alltimeRebuild) {
714+
// rebuild all-time history if requested - will also pull everying again
715+
await this.doRebuild(anlagenId);
716+
}
717+
} catch (systemError) {
718+
// Important: isolate system failure
719+
this.log.error(`❌ Error while polling system ${anlagenId}: ${systemError.message}`);
720+
}
724721
}
725722

726-
retry++;
727-
const delay = interval * this.config.retrymultiplier * retry;
728-
this.log.warn(`API retry ${retry}/${this.config.retries} in ${delay / 1000}s`);
729-
this.timerAPI = setTimeout(() => this.pollSenecApi(retry), delay);
723+
// Success → reset backoff
724+
this.setState("info.connection", true, true);
725+
this.apiFailureCount = 0;
726+
} catch (err) {
727+
this.apiFailureCount = (this.apiFailureCount || 0) + 1;
728+
this.log.error(`❌ API poll error: ${err.message}`);
729+
this.log.warn(`⚠️ Failure count: ${this.apiFailureCount}`);
730+
731+
// Exponential full jitter backoff
732+
nextDelay = computeBackoffDelay(baseInterval, this.apiFailureCount);
733+
734+
// Safety cap (max 8x base interval)
735+
const maxDelay = baseInterval * 8;
736+
nextDelay = Math.min(nextDelay, maxDelay);
737+
this.log.warn(`⏱ Backoff delay: ${(nextDelay / 1000).toFixed(0)}s`);
730738
} finally {
731739
this.apiPollRunning = false;
740+
if (!unloaded) {
741+
if (this.timerAPI) {
742+
clearTimeout(this.timerAPI);
743+
}
744+
this.timerAPI = setTimeout(() => this.pollSenecApi(), nextDelay);
745+
this.log.debug(`⏱ Next API poll scheduled in ${(nextDelay / 1000).toFixed(0)}s`);
746+
}
732747
}
733748
}
734749

@@ -737,33 +752,35 @@ class Senec extends utils.Adapter {
737752
*
738753
* @param {any} url url to call
739754
* @param config config for API call - will be extended by auth header - can be used to pass additional headers or other axios config parameters
740-
* @param attempt number of attempts for retrying in case of 401 - to avoid infinite loops in case something is really broken and token refresh doesn't work
741755
*/
742-
async apiGet(url, config = {}, attempt = 0) {
756+
async apiGet(url, config = {}) {
743757
return this.apiQueue.add(async () => {
744-
try {
745-
return await api_client.get(url, {
746-
...config,
747-
headers: {
748-
Authorization: `Bearer ${this.currentToken}`,
749-
...(config.headers || {}),
750-
},
751-
});
752-
} catch (e) {
753-
if (e.response?.status === 401) {
754-
if (attempt >= 2) {
755-
throw new Error("401 after token refresh retry");
758+
const maxAttempts = 3;
759+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
760+
try {
761+
return await api_client.get(url, {
762+
...config,
763+
headers: {
764+
Authorization: `Bearer ${this.currentToken}`,
765+
...(config.headers || {}),
766+
},
767+
});
768+
} catch (e) {
769+
const status = e.response?.status;
770+
// 🔐 Token expired
771+
if (status === 401 && attempt < maxAttempts - 1) {
772+
this.log.debug("🔐 401 received. Refreshing token...");
773+
await this.refreshTokenSingleFlight();
774+
continue;
775+
}
776+
777+
// 🚦 Rate limited
778+
if (status === 429) {
779+
this.log.warn("🚦 API returned 429 (rate limited)");
756780
}
757-
this.log.debug("🔐 401 received. Refreshing token...");
758-
await this.refreshTokenSingleFlight();
759-
return this.apiGet(url, config, attempt + 1);
760-
}
761781

762-
if (e.response?.status === 429) {
763-
this.log.warn("⚠️ Rate limited (429)");
764782
throw e;
765783
}
766-
throw e;
767784
}
768785
});
769786
}
@@ -975,7 +992,7 @@ class Senec extends utils.Adapter {
975992
});
976993
sums[LAST_UPDATED] = new Date().toISOString();
977994

978-
this.log.debug(`Sums: ${JSON.stringify(sums)}`);
995+
this.log.silly(`Sums: ${JSON.stringify(sums)}`);
979996
let groupBy;
980997
switch (period) {
981998
case "year":
@@ -1013,7 +1030,7 @@ class Senec extends utils.Adapter {
10131030
})
10141031
.then(async (response) => {
10151032
const content = response.data;
1016-
caller.log.debug(`(Poll) received data (${response.status}): ${JSON.stringify(content)}`);
1033+
caller.log.silly(`(Poll) received data (${response.status}): ${JSON.stringify(content)}`);
10171034
resolve(JSON.stringify(content));
10181035
})
10191036
.catch((error) => {
@@ -1069,7 +1086,7 @@ class Senec extends utils.Adapter {
10691086
this.log.debug(`(Poll) Double escapes autofixed! Body out: ${body}`);
10701087
}
10711088
const obj = JSON.parse(body, reviverNumParse);
1072-
this.log.debug(`(Poll) Parsed object: ${JSON.stringify(obj)}`);
1089+
this.log.silly(`(Poll) Parsed object: ${JSON.stringify(obj)}`);
10731090
//await this.evalPollLocal(obj);
10741091
await this.evalPoll(obj, "", "");
10751092

@@ -1530,3 +1547,18 @@ function hasPassword(html) {
15301547
function hasUsernameAndPassword(html) {
15311548
return hasUsername(html) && hasPassword(html);
15321549
}
1550+
1551+
/**
1552+
* Compute a backoff delay with exponential backoff and full jitter.
1553+
*
1554+
* @param {number} baseInterval - Base interval in milliseconds.
1555+
* @param {number} attempt - Attempt count (0-based).
1556+
* @param {number} [maxMultiplier] - Maximum multiplier used to cap the exponent.
1557+
*/
1558+
function computeBackoffDelay(baseInterval, attempt, maxMultiplier = 8) {
1559+
const cappedAttempt = Math.min(attempt, Math.log2(maxMultiplier));
1560+
const expDelay = baseInterval * Math.pow(2, cappedAttempt);
1561+
1562+
// Full jitter
1563+
return Math.floor(Math.random() * expDelay);
1564+
}

0 commit comments

Comments
 (0)