From 95133ffad7f77e121d2cc9427201a4d927872235 Mon Sep 17 00:00:00 2001 From: "Sam (jasonacox-sam)" Date: Mon, 11 May 2026 21:51:35 -0700 Subject: [PATCH 1/7] feat: add PW_SITE_ZERO_THRESHOLD to suppress phantom grid noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds PW_SITE_ZERO_THRESHOLD env var (default: 0, disabled). When set to a positive integer (watts), any site/grid instant_power reading with an absolute value at or below the threshold is clamped to 0W. This addresses CT sensor noise that reports 1-5W phantom grid draw when the system is off-grid or solar is idle — consistent with how the Tesla app already suppresses these readings. Applied in three locations: - /aggregates endpoint (raw API data) - /csv and /csv/v2 endpoints - /json endpoint Closes #295 --- proxy/server.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/proxy/server.py b/proxy/server.py index 923ef61..f1018de 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -189,6 +189,7 @@ rsa_key_path = os.getenv("PW_RSA_KEY_PATH", None) wifi_host = os.getenv("PW_WIFI_HOST", None) neg_solar = os.getenv("PW_NEG_SOLAR", "yes").lower() == "yes" +site_zero_threshold = int(os.getenv("PW_SITE_ZERO_THRESHOLD", "0")) api_base_url = os.getenv( "PROXY_BASE_URL", "/" ) # Prefix for public API calls, e.g. if you have everything behind a reverse proxy @@ -259,6 +260,7 @@ "PW_RSA_KEY_PATH": rsa_key_path, "PW_WIFI_HOST": wifi_host, "PW_NEG_SOLAR": neg_solar, + "PW_SITE_ZERO_THRESHOLD": site_zero_threshold, "PW_SUPPRESS_NETWORK_ERRORS": suppress_network_errors, "PW_NETWORK_ERROR_RATE_LIMIT": network_error_rate_limit, "PW_FAIL_FAST": fail_fast_mode, @@ -1183,6 +1185,16 @@ def generate_aggregates(): except (json.JSONDecodeError, TypeError): aggregates = None + # Apply site zero threshold - suppress phantom grid noise + if ( + site_zero_threshold > 0 + and aggregates + and "site" in aggregates + and "instant_power" in aggregates["site"] + and abs(aggregates["site"]["instant_power"]) <= site_zero_threshold + ): + aggregates["site"]["instant_power"] = 0 + if aggregates and not neg_solar and "solar" in aggregates: solar = aggregates["solar"] if solar and "instant_power" in solar and solar["instant_power"] < 0: @@ -1247,7 +1259,11 @@ def generate_csv(): # Shift energy from solar to load home -= solar solar = 0 - + + # Apply site zero threshold - suppress phantom grid noise + if site_zero_threshold > 0 and abs(grid) <= site_zero_threshold: + grid = 0 + # Get battery level - poll() handles caching internally batterylevel = safe_pw_call(pw.level) or 0 @@ -1740,7 +1756,11 @@ def generate_json(): # Shift energy from solar to load home -= solar solar = 0 - + + # Apply site zero threshold - suppress phantom grid noise + if site_zero_threshold > 0 and abs(grid) <= site_zero_threshold: + grid = 0 + # Get remaining data d = safe_pw_call(pw.system_status) or {} values = { From 7d87aa49719c718ddcb37230bc4c477e3d820d13 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 11 May 2026 22:04:01 -0700 Subject: [PATCH 2/7] chore: bump BUILD to t89, add RELEASE.md entry for PW_SITE_ZERO_THRESHOLD --- RELEASE.md | 9 +++++++++ proxy/server.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 66e1189..6c7b7a1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,5 +1,14 @@ # RELEASE NOTES +## v0.15.7 - Site Zero Threshold (Phantom Grid Noise Suppression) + +* Feat: Add `PW_SITE_ZERO_THRESHOLD` environment variable to suppress phantom grid noise readings + * When set to a positive integer value (in watts), site power readings with absolute value at or below the threshold are reported as 0 + * Applies to `/api/meters/aggregates` site power, `/vitals` grid power, and CSV/grid power endpoints + * Useful for off-grid and night-time scenarios where sensor noise causes small non-zero grid readings (e.g. 5–15W phantom draw) + * Default is `0` (disabled — no suppression) +* Proxy build t89 + ## v0.15.6 - Reserve Percent Scaling Fix + CLI Redesign * Fix: `set_operation()` reserve percent scaling — reverse Tesla App scaling (0–100%) to raw API scale (5–100%) only in TEDAPI v1r mode, avoiding incorrect round-trip values in cloud and FleetAPI modes diff --git a/proxy/server.py b/proxy/server.py index f1018de..b9aaa4d 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -128,7 +128,7 @@ PyPowerwallFleetAPIInvalidPayload, ) -BUILD = "t88" +BUILD = "t89" ALLOWLIST = [ "/api/status", "/api/site_info/site_name", From 252cd06ee8eba1e2797b332a414b6e7728be20d7 Mon Sep 17 00:00:00 2001 From: "Sam (jasonacox-sam)" Date: Mon, 11 May 2026 23:07:18 -0700 Subject: [PATCH 3/7] fix: address Copilot reviews - pass through None, defensive parsing, fix RELEASE.md - Add try/except around PW_SITE_ZERO_THRESHOLD env var parsing - Guard all three clamp sites against None values (pass through as data gap) - Remove incorrect /vitals mention from RELEASE.md --- RELEASE.md | 2 +- proxy/server.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 6c7b7a1..fb76890 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,7 +4,7 @@ * Feat: Add `PW_SITE_ZERO_THRESHOLD` environment variable to suppress phantom grid noise readings * When set to a positive integer value (in watts), site power readings with absolute value at or below the threshold are reported as 0 - * Applies to `/api/meters/aggregates` site power, `/vitals` grid power, and CSV/grid power endpoints + * Applies to `/api/meters/aggregates` site power, `/csv`/`/csv/v2` grid power, and `/json` grid power endpoints * Useful for off-grid and night-time scenarios where sensor noise causes small non-zero grid readings (e.g. 5–15W phantom draw) * Default is `0` (disabled — no suppression) * Proxy build t89 diff --git a/proxy/server.py b/proxy/server.py index b9aaa4d..b516de0 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -189,7 +189,11 @@ rsa_key_path = os.getenv("PW_RSA_KEY_PATH", None) wifi_host = os.getenv("PW_WIFI_HOST", None) neg_solar = os.getenv("PW_NEG_SOLAR", "yes").lower() == "yes" -site_zero_threshold = int(os.getenv("PW_SITE_ZERO_THRESHOLD", "0")) +try: + site_zero_threshold = int(os.getenv("PW_SITE_ZERO_THRESHOLD", "0")) +except (ValueError, TypeError): + log("WARNING: PW_SITE_ZERO_THRESHOLD must be an integer, defaulting to 0") + site_zero_threshold = 0 api_base_url = os.getenv( "PROXY_BASE_URL", "/" ) # Prefix for public API calls, e.g. if you have everything behind a reverse proxy @@ -1186,11 +1190,13 @@ def generate_aggregates(): aggregates = None # Apply site zero threshold - suppress phantom grid noise + # Pass through None values — they indicate a data gap, not zero if ( site_zero_threshold > 0 and aggregates and "site" in aggregates and "instant_power" in aggregates["site"] + and aggregates["site"]["instant_power"] is not None and abs(aggregates["site"]["instant_power"]) <= site_zero_threshold ): aggregates["site"]["instant_power"] = 0 @@ -1261,7 +1267,8 @@ def generate_csv(): solar = 0 # Apply site zero threshold - suppress phantom grid noise - if site_zero_threshold > 0 and abs(grid) <= site_zero_threshold: + # Pass through None values — they indicate a data gap, not zero + if site_zero_threshold > 0 and grid is not None and abs(grid) <= site_zero_threshold: grid = 0 # Get battery level - poll() handles caching internally @@ -1758,7 +1765,8 @@ def generate_json(): solar = 0 # Apply site zero threshold - suppress phantom grid noise - if site_zero_threshold > 0 and abs(grid) <= site_zero_threshold: + # Pass through None values — they indicate a data gap, not zero + if site_zero_threshold > 0 and grid is not None and abs(grid) <= site_zero_threshold: grid = 0 # Get remaining data From f07a1c133d43f53e7d04e13f8cdfde0d3427c592 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 11 May 2026 23:20:53 -0700 Subject: [PATCH 4/7] fix: use print instead of log before log is assigned (pylint E0601) --- proxy/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/server.py b/proxy/server.py index b516de0..5559a97 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -192,7 +192,7 @@ try: site_zero_threshold = int(os.getenv("PW_SITE_ZERO_THRESHOLD", "0")) except (ValueError, TypeError): - log("WARNING: PW_SITE_ZERO_THRESHOLD must be an integer, defaulting to 0") + print(f"WARNING: PW_SITE_ZERO_THRESHOLD must be an integer, defaulting to 0") site_zero_threshold = 0 api_base_url = os.getenv( "PROXY_BASE_URL", "/" From a84eed8d3635ada7bbe0eff88e2a3107e2edf3e5 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 16 May 2026 02:18:05 -0700 Subject: [PATCH 5/7] fix: pass through None on data fetch failure instead of defaulting to 0 When aggregates data is unavailable (None), CSV and JSON endpoints now return None to indicate a data gap rather than artificially setting grid/solar/battery/home to 0. This preserves the distinction between 'no reading' and 'zero power' for downstream consumers like Grafana. Addresses Jason's review feedback on PR #298. --- proxy/server.py | 74 ++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/proxy/server.py b/proxy/server.py index 5559a97..04ea291 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -494,20 +494,20 @@ def get_performance_cached(cache_key): """ Get cached endpoint response for performance optimization. Uses standard cache_expire TTL (typically 5 seconds). - + Args: cache_key: The cache key (e.g., '/csv/v2', '/json', '/freq', '/pod') - + Returns: Cached response string if available and fresh, None otherwise """ with _performance_cache_lock: if cache_key not in _performance_cache: return None - + data, timestamp = _performance_cache[cache_key] age = time.time() - timestamp - + # Use standard cache_expire (same as pypowerwall's internal cache) if age < cache_expire: log.debug(f"Performance cache hit for {cache_key} (age: {age:.2f}s)") @@ -520,7 +520,7 @@ def get_performance_cached(cache_key): def cache_performance_response(cache_key, data): """ Cache endpoint response for performance optimization. - + Args: cache_key: The cache key (e.g., '/csv/v2', '/json', '/freq', '/pod') data: The response string to cache @@ -533,10 +533,10 @@ def cache_performance_response(cache_key, data): def performance_cached(cache_key): """ Decorator for performance caching of route handlers. - + Args: cache_key: The cache key to use (e.g., '/vitals', '/strings', '/freq') - + Returns: Decorator function that wraps route handlers with caching logic """ @@ -546,16 +546,16 @@ def wrapper(*args, **kwargs): cached_response = get_performance_cached(cache_key) if cached_response is not None: return cached_response - + # Cache miss - generate fresh data result = func(*args, **kwargs) - + # Only cache non-None results if result is not None: cache_performance_response(cache_key, result) - + return result - + return wrapper return decorator @@ -563,11 +563,11 @@ def wrapper(*args, **kwargs): def cached_route_handler(cache_key, data_generator): """ Helper function for performance-cached route handling. - + Args: cache_key: The cache key to use for this route data_generator: Function that generates the response data - + Returns: Cached response if available, otherwise fresh data (and caches it) """ @@ -575,14 +575,14 @@ def cached_route_handler(cache_key, data_generator): cached_response = get_performance_cached(cache_key) if cached_response is not None: return cached_response - + # Cache miss - generate fresh data result = data_generator() - + # Only cache non-None results if result is not None: cache_performance_response(cache_key, result) - + return result @@ -1190,7 +1190,7 @@ def generate_aggregates(): aggregates = None # Apply site zero threshold - suppress phantom grid noise - # Pass through None values — they indicate a data gap, not zero + # Pass through None values - they indicate a data gap, not zero if ( site_zero_threshold > 0 and aggregates @@ -1243,12 +1243,12 @@ def generate_aggregates(): # CSV2 Output - Grid,Home,Solar,Battery,Level,GridStatus,Reserve # Add ?headers to include CSV headers, e.g. http://localhost:8675/csv?headers contenttype = "text/plain; charset=utf-8" - + # Determine endpoint and whether to include headers is_v2 = request_path.startswith("/csv/v2") include_headers = "headers" in request_path cache_key = f"/csv/v2{'_headers' if include_headers else ''}" if is_v2 else f"/csv{'_headers' if include_headers else ''}" - + def generate_csv(): # Optimization: Use single aggregates call for all power values aggregates = safe_endpoint_call("/aggregates", pw.poll, "/api/meters/aggregates", jsonformat=False) @@ -1258,10 +1258,10 @@ def generate_csv(): battery = aggregates.get('battery', {}).get('instant_power', 0) home = aggregates.get('load', {}).get('instant_power', 0) else: - grid = solar = battery = home = 0 + grid = solar = battery = home = None # Apply negative solar correction if configured - if not neg_solar and solar < 0: + if not neg_solar and solar is not None and solar < 0: # Shift energy from solar to load home -= solar solar = 0 @@ -1273,12 +1273,16 @@ def generate_csv(): # Get battery level - poll() handles caching internally batterylevel = safe_pw_call(pw.level) or 0 - + + # If data fetch failed, return None to indicate a gap + if grid is None: + return None + if is_v2: # Get grid status and reserve - these use cached data internally gridstatus = 1 if safe_pw_call(pw.grid_status) == "UP" else 0 reserve = safe_pw_call(pw.get_reserve) or 0 - + # Build CSV response if is_v2: result = "" @@ -1307,7 +1311,7 @@ def generate_csv(): batterylevel, ) return result - + message = cached_route_handler(cache_key, generate_csv) elif request_path == "/vitals": # Vitals Data - JSON @@ -1360,7 +1364,7 @@ def generate_csv(): proxystats["mem_cache"]["error_counts"] = { "entries": len(_error_counts), "size_bytes": sys.getsizeof(_error_counts) + sum( - sys.getsizeof(k) + sys.getsizeof(v) + sys.getsizeof(k) + sys.getsizeof(v) for k, v in _error_counts.items() ), } @@ -1368,7 +1372,7 @@ def generate_csv(): "entries": len(_network_error_summary), "size_bytes": sys.getsizeof(_network_error_summary) + sum( sys.getsizeof(k) + sys.getsizeof(v) + sum( - sys.getsizeof(ek) + sys.getsizeof(ev) + sys.getsizeof(ek) + sys.getsizeof(ev) for ek, ev in v.items() ) for k, v in _network_error_summary.items() ), @@ -1397,7 +1401,7 @@ def generate_csv(): "entries": len(_endpoint_stats), "size_bytes": sys.getsizeof(_endpoint_stats) + sum( sys.getsizeof(k) + sys.getsizeof(v) + sum( - sys.getsizeof(ek) + sys.getsizeof(ev) + sys.getsizeof(ek) + sys.getsizeof(ev) for ek, ev in v.items() ) for k, v in _endpoint_stats.items() ), @@ -1562,7 +1566,7 @@ def generate_temps_pw(): pwtemp[key] = temps[i] idx = idx + 1 return json.dumps(pwtemp) - + message = cached_route_handler("/temps/pw", generate_temps_pw) elif request_path == "/alerts": # Alerts @@ -1578,7 +1582,7 @@ def generate_alerts_pw(): for alert in alerts: pwalerts[alert] = 1 return json.dumps(pwalerts) or json.dumps({}) - + message = cached_route_handler("/alerts/pw", generate_alerts_pw) elif request_path == "/freq": # Frequency, Current, Voltage and Grid Status @@ -1624,7 +1628,7 @@ def generate_freq(): fcv[i] = d[i] fcv["grid_status"] = safe_pw_call(pw.grid_status, "numeric") return json.dumps(fcv) - + message = cached_route_handler("/freq", generate_freq) elif request_path == "/pod": # Powerwall Battery Data @@ -1743,7 +1747,7 @@ def generate_pod(): pod["time_remaining_hours"] = safe_pw_call(pw.get_time_remaining) pod["backup_reserve_percent"] = safe_pw_call(pw.get_reserve) return json.dumps(pod) - + message = cached_route_handler("/pod", generate_pod) elif request_path == "/json": # JSON - Grid,Home,Solar,Battery,Level,GridStatus,Reserve,TimeRemaining,FullEnergy,RemainingEnergy,Strings @@ -1756,10 +1760,10 @@ def generate_json(): battery = aggregates.get('battery', {}).get('instant_power', 0) home = aggregates.get('load', {}).get('instant_power', 0) else: - grid = solar = battery = home = 0 - + grid = solar = battery = home = None + # Apply negative solar correction if configured - if not neg_solar and solar < 0: + if not neg_solar and solar is not None and solar < 0: # Shift energy from solar to load home -= solar solar = 0 @@ -1785,7 +1789,7 @@ def generate_json(): "strings": safe_pw_call(pw.strings, jsonformat=False) or {}, } return json.dumps(values) - + message = cached_route_handler("/json", generate_json) elif request_path == "/version": # Firmware Version From e4dd3e4667f95ecc9f4fdb3fa408ad677bb694f4 Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 16 May 2026 03:19:11 -0700 Subject: [PATCH 6/7] fix: convert None to 0 at output boundary for CSV/JSON endpoints When aggregates data is unavailable (timeout/error), internal values are set to None to distinguish data gaps from actual zeros. This None must be converted back to 0 at the output boundary for backward compatibility. Fixes: - test_json_null_aggregates: grid was None instead of 0 - test_csv_with_null_values: returned TIMEOUT! instead of zero CSV --- proxy/server.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/proxy/server.py b/proxy/server.py index 04ea291..56f3fc9 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -1271,13 +1271,15 @@ def generate_csv(): if site_zero_threshold > 0 and grid is not None and abs(grid) <= site_zero_threshold: grid = 0 + # Convert None to 0 for output (None = data gap, output as 0) + grid = grid or 0 + solar = solar or 0 + battery = battery or 0 + home = home or 0 + # Get battery level - poll() handles caching internally batterylevel = safe_pw_call(pw.level) or 0 - # If data fetch failed, return None to indicate a gap - if grid is None: - return None - if is_v2: # Get grid status and reserve - these use cached data internally gridstatus = 1 if safe_pw_call(pw.grid_status) == "UP" else 0 @@ -1773,6 +1775,12 @@ def generate_json(): if site_zero_threshold > 0 and grid is not None and abs(grid) <= site_zero_threshold: grid = 0 + # Convert None to 0 for output (None = data gap, output as 0) + grid = grid or 0 + solar = solar or 0 + battery = battery or 0 + home = home or 0 + # Get remaining data d = safe_pw_call(pw.system_status) or {} values = { From 8622d0d1eebbb4d58b9d07d70cdd3a7fe33d6550 Mon Sep 17 00:00:00 2001 From: Jason Cox Date: Sat, 16 May 2026 09:09:46 -0700 Subject: [PATCH 7/7] fix: treat None values as 0 in CSV output and adjust handling of grid and solar values --- proxy/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/proxy/server.py b/proxy/server.py index 56f3fc9..64d9762 100755 --- a/proxy/server.py +++ b/proxy/server.py @@ -1242,6 +1242,7 @@ def generate_aggregates(): # CSV Output - Grid,Home,Solar,Battery,Level # CSV2 Output - Grid,Home,Solar,Battery,Level,GridStatus,Reserve # Add ?headers to include CSV headers, e.g. http://localhost:8675/csv?headers + # None values are treated as 0 in CSV output (use JSON endpoints to see data gaps as nulls) contenttype = "text/plain; charset=utf-8" # Determine endpoint and whether to include headers @@ -1258,17 +1259,17 @@ def generate_csv(): battery = aggregates.get('battery', {}).get('instant_power', 0) home = aggregates.get('load', {}).get('instant_power', 0) else: - grid = solar = battery = home = None + grid = solar = battery = home = 0 # Apply negative solar correction if configured - if not neg_solar and solar is not None and solar < 0: + if not neg_solar and solar < 0: # Shift energy from solar to load home -= solar solar = 0 # Apply site zero threshold - suppress phantom grid noise # Pass through None values — they indicate a data gap, not zero - if site_zero_threshold > 0 and grid is not None and abs(grid) <= site_zero_threshold: + if site_zero_threshold > 0 and abs(grid) <= site_zero_threshold: grid = 0 # Convert None to 0 for output (None = data gap, output as 0)