From 87d95901c5d9a177c4089aa7fcc06c5192bd904b Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 19 Aug 2026 08:09:14 +0100 Subject: [PATCH 1/5] fix(ml): resolve NaN training loss on startup and preserve database zero entries - Sanitize WindowedFeatures and chunk conversions against NaN, Inf, and None - Use nanmean/nanstd and sanitize scales before np.maximum in feature/target normalization - Add gradient clipping and finite parameter checks in Adam optimizer - Preserve valid 0.0 values in database save/load roundtrips - Fix duplicate training execution and curriculum pass initialization tracking - Add robust unit tests for NaN resilience, zero preservation, and 90-day curriculum --- apps/predbat/load_ml_component.py | 59 +++--- apps/predbat/load_predictor.py | 60 ++++-- apps/predbat/tests/test_load_ml.py | 201 ++++++++++++++++---- apps/predbat/tests/test_ml_training_perf.py | 10 +- apps/predbat/tests/test_sunsynk_api.py | 5 + 5 files changed, 258 insertions(+), 77 deletions(-) diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index 5230dd465..5a8f67c16 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -677,7 +677,16 @@ def _get_predictions(self, now_utc, midnight_utc, exog_features=None): self.log("ML Component: Generating predictions load data age {:.1f} days, {} data points".format(self.load_data_age_days, len(self.load_data) if self.load_data else 0)) if 0: self._log_prediction_input_table(now_utc) - predictions = self.predictor.predict(self.load_data, now_utc, midnight_utc, pv_minutes=self.pv_data, temp_minutes=self.temperature_data, import_rates=self.import_rates_data, export_rates=self.export_rates_data, exog_features=exog_features) + predictions = self.predictor.predict( + self.load_data, + now_utc, + midnight_utc, + pv_minutes=self.pv_data, + temp_minutes=self.temperature_data, + import_rates=self.import_rates_data, + export_rates=self.export_rates_data, + exog_features=exog_features, + ) if predictions: self.current_predictions = predictions @@ -821,7 +830,7 @@ async def save_database_history(self): max_steps = self.load_ml_database_days * 24 * 60 // PREDICT_STEP def dict_to_array(data_dict): - arr = np.zeros(max_steps, dtype=np.float32) + arr = np.full(max_steps, np.nan, dtype=np.float32) if data_dict: for minute, value in data_dict.items(): # Only persist historical data (non-negative integer keys) @@ -886,15 +895,18 @@ async def load_database_history(self): saved_utc = datetime.fromisoformat(metadata["saved_utc"]) age_days = float(metadata.get("age_days", 0)) - max_steps = self.load_ml_database_days * 24 * 60 // PREDICT_STEP - def array_to_dict(arr): """Reconstruct a sparse {minute: value} dict with keys as stored""" result = {} + has_nans = np.isnan(arr).any() for i in range(len(arr)): val = float(arr[i]) - if val != 0.0: - result[i * PREDICT_STEP] = val + if has_nans: + if not np.isnan(val) and np.isfinite(val): + result[i * PREDICT_STEP] = val + else: + if val != 0.0: + result[i * PREDICT_STEP] = val return result self.load_data = array_to_dict(data["load"]) @@ -976,22 +988,22 @@ async def _do_training(self, is_initial): curriculum_step_days=5, max_intermediate_passes=8, ) - # Even if initial was done we need to do one fine tuned curriculum pass too. - val_mae = self.predictor.train_curriculum( - load_data_snap, - now_utc_snap, - pv_minutes=pv_data_snap, - temp_minutes=temp_data_snap, - import_rates=import_rates_snap, - export_rates=export_rates_snap, - epochs=epochs, - time_decay_days=time_decay, - validation_holdout_hours=holdout_hours, - patience=patience, - curriculum_window_days=window_days, - curriculum_step_days=step_days, - max_intermediate_passes=max_intermediate_passes, - ) + else: + val_mae = self.predictor.train_curriculum( + load_data_snap, + now_utc_snap, + pv_minutes=pv_data_snap, + temp_minutes=temp_data_snap, + import_rates=import_rates_snap, + export_rates=export_rates_snap, + epochs=epochs, + time_decay_days=time_decay, + validation_holdout_hours=holdout_hours, + patience=patience, + curriculum_window_days=window_days, + curriculum_step_days=step_days, + max_intermediate_passes=max_intermediate_passes, + ) if val_mae is not None: self.last_train_time = datetime.now(timezone.utc) @@ -1054,7 +1066,8 @@ def _load_baseline_now(self): derived_baseline += self.load_data.get(minute, 0.0) derived_baseline = dp4(derived_baseline) self.log( - "Warn: ML Component: Load baseline of {} kWh was captured on {} which is a previous day, re-derived load so far today as {} kWh".format(dp2(self.load_minutes_now), self.load_minutes_now_time.strftime("%Y-%m-%d %H:%M"), dp2(derived_baseline)) + "Warn: ML Component: Load baseline of {} kWh was captured on {} which is a previous day, " + "re-derived load so far today as {} kWh".format(dp2(self.load_minutes_now), self.load_minutes_now_time.strftime("%Y-%m-%d %H:%M"), dp2(derived_baseline)) ) return derived_baseline diff --git a/apps/predbat/load_predictor.py b/apps/predbat/load_predictor.py index 7da3c3f74..f1ef3ef7e 100644 --- a/apps/predbat/load_predictor.py +++ b/apps/predbat/load_predictor.py @@ -107,6 +107,7 @@ def __array__(self, dtype=None, copy=None): if self.mean is not None and self.std is not None: out -= self.mean out /= self.std + np.nan_to_num(out, copy=False, nan=0.0, posinf=0.0, neginf=0.0) return out.astype(dtype) if dtype is not None and dtype != np.float32 else out def _assemble(self, target_chunks, out): @@ -142,6 +143,7 @@ def __getitem__(self, indices): if self.mean is not None and self.std is not None: batch -= self.mean batch /= self.std + np.nan_to_num(batch, copy=False, nan=0.0, posinf=0.0, neginf=0.0) return batch[0] if scalar else batch @@ -361,6 +363,10 @@ def _backward(self, y_true, activations, pre_activations, sample_weights=None, d if dropout_masks is not None and (i - 1) < len(dropout_masks) and dropout_masks[i - 1] is not None: delta = delta * dropout_masks[i - 1] + # Clip gradients to prevent exploding gradients and NaN weight updates + weight_grads = [np.clip(g, -10.0, 10.0) for g in weight_grads] + bias_grads = [np.clip(g, -10.0, 10.0) for g in bias_grads] + return weight_grads, bias_grads def _adam_update(self, weight_grads, bias_grads, beta1=0.9, beta2=0.999, epsilon=1e-8, lr=None): @@ -405,6 +411,10 @@ def _adam_update(self, weight_grads, bias_grads, beta1=0.9, beta2=0.999, epsilon # Update biases (no weight decay on biases) self.biases[i] -= effective_lr * m_hat / (np.sqrt(v_hat) + epsilon) + # Ensure weights and biases remain finite + self.weights[i] = np.nan_to_num(self.weights[i], copy=False, nan=0.0) + self.biases[i] = np.nan_to_num(self.biases[i], copy=False, nan=0.0) + def _create_time_features(self, minute_of_day, day_of_week, day_of_year=1): """ Create cyclical time features. @@ -533,12 +543,16 @@ def _chunk_energy_to_aligned(self, energy_per_step, now_utc): for j in range(CHUNK_STEPS): m = minute + j * STEP_MINUTES if m in energy_per_step: - total += energy_per_step[m] + val = energy_per_step[m] + if val is None or np.isnan(val) or np.isinf(val): + valid = False + break + total += float(val) else: valid = False break - if valid: - chunked[chunk_idx] = total + if valid and np.isfinite(total): + chunked[chunk_idx] = float(total) chunk_idx += 1 minute += CHUNK_MINUTES @@ -572,9 +586,13 @@ def _chunk_instantaneous_to_aligned(self, values_per_step, alignment_offset): for j in range(CHUNK_STEPS): m = minute + j * STEP_MINUTES if m in values_per_step: - vals.append(values_per_step[m]) + val = values_per_step[m] + if val is not None and not np.isnan(val) and not np.isinf(val): + vals.append(float(val)) if vals: - chunked[chunk_idx] = float(np.mean(vals)) + mean_val = float(np.mean(vals)) + if np.isfinite(mean_val): + chunked[chunk_idx] = mean_val chunk_idx += 1 minute += CHUNK_MINUTES @@ -909,7 +927,7 @@ def _feature_mean_std(X, block=512): total = np.zeros(X.shape[1], dtype=np.float64) for start in range(0, rows, block): total += X[start : start + block].sum(axis=0, dtype=np.float64) - mean = total / rows + mean = np.nan_to_num(total / rows, nan=0.0, posinf=0.0, neginf=0.0) squares = np.zeros(X.shape[1], dtype=np.float64) for start in range(0, rows, block): @@ -918,7 +936,7 @@ def _feature_mean_std(X, block=512): # Square into the deviation buffer rather than allocating a second one of the # same size; the values are identical either way squares += np.square(deviation, out=deviation).sum(axis=0) - std = np.sqrt(squares / rows) + std = np.nan_to_num(np.sqrt(squares / rows), nan=1.0, posinf=1.0, neginf=1.0) return mean.astype(np.float32), std.astype(np.float32) @@ -936,14 +954,14 @@ def _fit_windowed_normalisation(self, dataset, ema_alpha=0.0): total = np.zeros(TOTAL_FEATURES, dtype=np.float64) for block in dataset.blocks(): total += block.sum(axis=0, dtype=np.float64) - mean = total / rows + mean = np.nan_to_num(total / rows, nan=0.0, posinf=0.0, neginf=0.0) squares = np.zeros(TOTAL_FEATURES, dtype=np.float64) for block in dataset.blocks(): deviation = block.astype(np.float64) deviation -= mean squares += np.square(deviation, out=deviation).sum(axis=0) - std = np.sqrt(squares / rows) + std = np.nan_to_num(np.sqrt(squares / rows), nan=1.0, posinf=1.0, neginf=1.0) new_mean = mean.astype(np.float32) new_std = np.maximum(std.astype(np.float32), self._get_min_std_array(TOTAL_FEATURES)) @@ -1054,9 +1072,11 @@ def _normalize_features(self, X, fit=False, ema_alpha=0.0, in_place=False): if in_place: X -= self.feature_mean X /= self.feature_std + np.nan_to_num(X, copy=False, nan=0.0, posinf=0.0, neginf=0.0) return X - return (X - self.feature_mean) / self.feature_std + out = (X - self.feature_mean) / self.feature_std + return np.nan_to_num(out, copy=False, nan=0.0, posinf=0.0, neginf=0.0) def _normalize_targets(self, y, fit=False): """ @@ -1070,14 +1090,17 @@ def _normalize_targets(self, y, fit=False): Normalized target array """ if fit: - self.target_mean = np.mean(y) - self.target_std = np.std(y) + m = np.nanmean(y) + s = np.nanstd(y) + self.target_mean = float(m) if np.isfinite(m) else 0.0 + self.target_std = float(s) if np.isfinite(s) else 1.0 self.target_std = max(self.target_std, 1e-8) if self.target_mean is None or self.target_std is None: return y - return (y - self.target_mean) / self.target_std + out = (y - self.target_mean) / self.target_std + return np.nan_to_num(out, copy=False, nan=0.0, posinf=0.0, neginf=0.0) def _denormalize_predictions(self, y_pred): """ @@ -1364,8 +1387,9 @@ def train( self.validation_mae = best_val_loss self.validation_bias = float(best_val_bias) self.epochs_trained += epochs - - self.log("ML Predictor: Training complete, final val_mae={:.4f} kWh val_bias={:+.4f} kWh ({:+.1f}%)".format(best_val_loss, float(best_val_bias), 100.0 * float(best_val_bias) / (float(np.mean(y_val)) if float(np.mean(y_val)) > 1e-8 else 1e-8))) + mean_y = float(np.mean(y_val)) if float(np.mean(y_val)) > 1e-8 else 1e-8 + pct_bias = 100.0 * float(best_val_bias) / mean_y + self.log("ML Predictor: Training complete, final val_mae={:.4f} kWh val_bias={:+.4f} kWh ({:+.1f}%)".format(best_val_loss, float(best_val_bias), pct_bias)) # Autoregressive diagnostic: run a full AR rollout over the holdout period # to expose compounding error (teacher-forced val_mae won't show this) @@ -1516,7 +1540,7 @@ def train_curriculum( ) total_passes = len(window_sizes) + 1 # intermediate passes + final full pass - self.log("ML Predictor: Curriculum training - {} passes, window {:.1f}→{:.1f} days + final full pass ({:.1f} days)".format(total_passes, window_sizes[0] / day_minutes, window_sizes[-1] / day_minutes, max_minute / day_minutes)) + self.log("ML Predictor: Curriculum training - {} passes, window {:.1f}->{:.1f} days + final full pass ({:.1f} days)".format(total_passes, window_sizes[0] / day_minutes, window_sizes[-1] / day_minutes, max_minute / day_minutes)) val_mae = None for pass_idx, window in enumerate(window_sizes): @@ -1542,7 +1566,7 @@ def train_curriculum( temp_minutes=temp_slice, import_rates=import_slice, export_rates=export_slice, - is_initial=(pass_idx == 0), + is_initial=(pass_idx == 0 and not self.model_initialized), epochs=epochs, time_decay_days=time_decay_days, patience=patience, @@ -1566,7 +1590,7 @@ def train_curriculum( temp_minutes=temp_minutes, import_rates=import_rates, export_rates=export_rates, - is_initial=False, + is_initial=(not self.model_initialized), epochs=epochs, time_decay_days=time_decay_days, patience=patience, diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py index 747cd2467..182b44e30 100644 --- a/apps/predbat/tests/test_load_ml.py +++ b/apps/predbat/tests/test_load_ml.py @@ -63,6 +63,9 @@ def test_load_ml(my_predbat=None): ("car_subtraction_direct", _test_car_subtraction_direct, "Direct car_subtraction method with interpolation and smoothing"), ("component_run_data_merge", _test_component_run_data_merge, "LoadMLComponent run() data fetch, save and merge across two runs"), ("component_init_predictor_last_train_time", _test_component_init_predictor_sets_last_train_time, "LoadMLComponent _init_predictor sets last_train_time from embedded training_timestamp"), + ("nan_inf_robustness", _test_nan_inf_robustness, "LoadPredictor handles NaN, Inf, and None across input channels without NaN loss"), + ("database_zero_preservation", _test_database_zero_preservation, "Database save/load preserves 0.0 values across roundtrip"), + ("curriculum_90day_intermediate_passes", _test_curriculum_90day_intermediate_passes, "Curriculum training across 90-day window with 8 intermediate passes"), # ("real_data_training", _test_real_data_training, "Train on real data with chart"), # ("pretrained_model_prediction", _test_pretrained_model_prediction, "Load pre-trained model and generate predictions with chart"), ] @@ -271,7 +274,7 @@ def _test_pv_energy_conversion(): def _create_synthetic_pv_data(n_days=7, now_utc=None, forecast_hours=48): """Create synthetic PV data for testing (historical + forecast). - Returns per-5-min energy (kWh per step) — not cumulative. + Returns per-5-min energy (kWh per step) -- not cumulative. Positive keys = historical (minute 0 = now), negative keys = future. """ if now_utc is None: @@ -373,7 +376,7 @@ def _create_synthetic_temp_data(n_days=7, now_utc=None, forecast_hours=48): def _create_synthetic_load_data(n_days=7, now_utc=None): """Create synthetic load data for testing. - Returns per-5-min energy (kWh per step) — not cumulative. + Returns per-5-min energy (kWh per step) -- not cumulative. Positive keys = historical, minute 0 = most recent. """ if now_utc is None: @@ -820,7 +823,7 @@ def _test_curriculum_training(): now_utc = datetime.now(timezone.utc) - # ── Happy-path: 28 days of data → 3 intermediate passes + 1 final ──────── + # ── Happy-path: 28 days of data -> 3 intermediate passes + 1 final ──────── np.random.seed(42) load_data_28 = _create_synthetic_load_data(n_days=28, now_utc=now_utc) @@ -884,7 +887,7 @@ def _test_curriculum_training(): now_utc, epochs=3, patience=3, - curriculum_window_days=14, # Requires 14 days; only 5 days available → fallback + curriculum_window_days=14, # Requires 14 days; only 5 days available -> fallback curriculum_step_days=7, ) # Fallback should complete without crashing and return a MAE or None (if still @@ -1081,7 +1084,7 @@ async def mock_save_database_history(): component.save_database_history = mock_save_database_history # ── Run 1 (first=True, seconds=0) ─────────────────────────────────────── - # Expect: data fetched and stored, but training deferred → no save, no training. + # Expect: data fetched and stored, but training deferred -> no save, no training. component._fetch_load_data = AsyncMock(return_value=(fetch_data_1, 28, 5.0, None, None, None, None)) result = await component.run(seconds=0, first=True) @@ -1096,13 +1099,13 @@ async def mock_save_database_history(): assert save_call_count[0] == 0, f"save_database_history should NOT be called on first run (deferred), called {save_call_count[0]} times" assert training_call_count[0] == 0, f"Training should NOT run on first run, ran {training_call_count[0]} times" - print(" \u2713 Run 1: data populated, training deferred, no save") + print(" PASS: Run 1: data populated, training deferred, no save") # ── Advance time by ELAPSED_MINUTES ───────────────────────────────────── mock_base.now_utc = mock_base.now_utc + timedelta(minutes=ELAPSED_MINUTES) # ── Run 2 (first=False, seconds=30) ───────────────────────────────────── - # last_train_time is None → retrain_age_seconds = RETRAIN_INTERVAL_SECONDS → should_train=True + # last_train_time is None -> retrain_age_seconds = RETRAIN_INTERVAL_SECONDS -> should_train=True # Expect: shift old keys, merge fresh data, run initial training, save once. component._fetch_load_data = AsyncMock(return_value=(fetch_data_2, 7, 3.0, None, None, None, None)) @@ -1114,9 +1117,9 @@ async def mock_save_database_history(): assert save_call_count[0] == 1, f"save_database_history should be called once after second run, called {save_call_count[0]} times" # ── Verify time-shift: old keys shifted forward by ELAPSED_MINUTES ── - assert component.load_data.get(SHIFT) == 0.1, f"Old key 0 → key {SHIFT} after shift, got {component.load_data.get(SHIFT)}" - assert component.load_data.get(500 + SHIFT) == 0.1, f"Old key 500 → key {500 + SHIFT}, got {component.load_data.get(500 + SHIFT)}" - assert component.load_data.get(2000 + SHIFT) == 0.05, f"Old key 2000 → key {2000 + SHIFT}, got {component.load_data.get(2000 + SHIFT)}" + assert component.load_data.get(SHIFT) == 0.1, f"Old key 0 -> key {SHIFT} after shift, got {component.load_data.get(SHIFT)}" + assert component.load_data.get(500 + SHIFT) == 0.1, f"Old key 500 -> key {500 + SHIFT}, got {component.load_data.get(500 + SHIFT)}" + assert component.load_data.get(2000 + SHIFT) == 0.05, f"Old key 2000 -> key {2000 + SHIFT}, got {component.load_data.get(2000 + SHIFT)}" assert component.load_data.get(2000) is None, f"Key 2000 should be gone after shift (moved to {2000 + SHIFT})" # ── Verify fresh data from fetch_data_2 is at the expected keys ── @@ -1124,15 +1127,15 @@ async def mock_save_database_history(): actual = component.load_data.get(minute) assert actual == expected_value, f"At minute {minute}: expected {expected_value} (fetch_data_2) but got {actual}" - print(" \u2713 Run 2: initial training fired, keys shifted, fresh data merged, save called") + print(" PASS: Run 2: initial training fired, keys shifted, fresh data merged, save called") # ── Advance time by another ELAPSED_MINUTES ────────────────────────────── mock_base.now_utc = mock_base.now_utc + timedelta(minutes=ELAPSED_MINUTES) # ── Run 3 (first=False, seconds=PREDICTION_INTERVAL_SECONDS) ───────────── # last_train_time = 30 min ago (set in mock_do_training to component.now_utc of Run 2). - # retrain_age_seconds = 30*60 = 1800 < RETRAIN_INTERVAL_SECONDS (7200) → should_train=False. - # seconds % PREDICTION_INTERVAL_SECONDS == 0 → should_fetch=True. + # retrain_age_seconds = 30*60 = 1800 < RETRAIN_INTERVAL_SECONDS (7200) -> should_train=False. + # seconds % PREDICTION_INTERVAL_SECONDS == 0 -> should_fetch=True. # Expect: fetch+predict+save only, no training. component._fetch_load_data = AsyncMock(return_value=(fetch_data_3, 7, 3.0, None, None, None, None)) @@ -1142,7 +1145,7 @@ async def mock_save_database_history(): assert training_call_count[0] == 1, f"Training should NOT run again on third run (model too fresh), ran {training_call_count[0]} times total" assert save_call_count[0] == 2, f"save_database_history should be called again on third run, called {save_call_count[0]} times" - print(" \u2713 Run 3: fetch-only cycle (model fresh), save called without retraining") + print(" PASS: Run 3: fetch-only cycle (model fresh), save called without retraining") run_async(run_test()) @@ -1283,7 +1286,7 @@ def _test_real_data_training(): history_hours = 7 * 24 # 7 days back prediction_hours = 48 # 2 days forward - # Read historical load_data (already per-5-min energy — not cumulative) + # Read historical load_data (already per-5-min energy -- not cumulative) # Going backwards in time: minute 0 is now, higher minutes are past historical_minutes = [] historical_energy = [] @@ -1416,7 +1419,7 @@ def _test_real_data_training(): pred_minutes.append(minute) pred_energy.append(energy_kwh) - # Read PV data (already per-5-min energy — not cumulative) + # Read PV data (already per-5-min energy -- not cumulative) # Historical PV (positive minutes, going back in time) pv_historical_minutes = [] pv_historical_energy = [] @@ -1916,7 +1919,7 @@ async def test_basic_fetch(): assert result_age == 28, f"Expected 28 days, got {result_age}" assert len(result_data) > 0, "Load data should not be empty" assert result_now >= 0, f"Current load should be non-negative, got {result_now}" - print(" ✓ Basic fetch successful") + print(" PASS: Basic fetch successful") # Test 2: Missing sensor (should return None) async def test_missing_sensor(): @@ -1952,7 +1955,7 @@ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=Non assert result_data is None, "Should return None when sensor missing" assert result_age == 0, "Age should be 0 when sensor missing" assert result_now == 0, "Current load should be 0 when sensor missing" - print(" ✓ Missing sensor handled correctly") + print(" PASS: Missing sensor handled correctly") # Test 3: Car charging subtraction async def test_car_charging_subtraction(): @@ -2034,7 +2037,7 @@ def mock_get_arg_with_car(key, default=None, indirect=True, combine=False, attri expected = 0.7 # One step of (1.0 - 0.3) in per-step format assert abs(value_1435 - expected) < 0.01, f"At minute 1435, expected ~{expected} kWh, got {value_1435:.4f}" - print(" ✓ Car charging subtraction works") + print(" PASS: Car charging subtraction works") # Test 3b: Car charging threshold-based detection (without sensor) async def test_car_charging_threshold_detection(): @@ -2133,7 +2136,7 @@ def mock_get_arg_threshold(key, default=None, indirect=True, combine=False, attr expected = 0.375 # Car detected, subtract estimate assert abs(value_1420 - expected) < 0.01, f"At minute 1420 (high load again), expected ~{expected} kWh, got {value_1420:.4f}" - print(" ✓ Car charging threshold detection works") + print(" PASS: Car charging threshold detection works") # Test 4: Load power fill async def test_load_power_fill(): @@ -2174,7 +2177,7 @@ def mock_get_arg_with_power(key, default=None, indirect=True, combine=False, att assert result_data is not None, "Should return load data" assert mock_base_with_power.fill_load_from_power.called, "fill_load_from_power should be called" assert result_now >= 0, f"Current load should be non-negative, got {result_now}" - print(" ✓ Load power fill invoked") + print(" PASS: Load power fill invoked") # Test 5: Exception handling async def test_exception_handling(): @@ -2197,7 +2200,7 @@ async def test_exception_handling(): assert result_data is None, "Should return None on exception" assert result_age == 0, "Age should be 0 on exception" assert result_now == 0, "Current load should be 0 on exception" - print(" ✓ Exception handling works") + print(" PASS: Exception handling works") # Test 6: Empty load data async def test_empty_load_data(): @@ -2221,7 +2224,7 @@ async def test_empty_load_data(): assert result_data is None, "Should return None when load data is empty" assert result_age == 0, "Age should be 0 when load data is empty" assert result_now == 0, "Current load should be 0 when load data is empty" - print(" ✓ Empty load data handled correctly") + print(" PASS: Empty load data handled correctly") # Test 7: Temperature data fetch with future predictions only async def test_temperature_data_fetch(): @@ -2278,7 +2281,7 @@ def mock_get_state_wrapper_side_effect(entity_id, default=None, attribute=None, # Verify get_state_wrapper was called correctly assert mock_base_with_temp.get_state_wrapper.called, "get_state_wrapper should be called" - print(" ✓ Temperature data fetch (future predictions) works") + print(" PASS: Temperature data fetch (future predictions) works") # Test 8: Temperature data with no predictions (None return) async def test_temperature_no_data(): @@ -2308,7 +2311,7 @@ async def test_temperature_no_data(): assert isinstance(result_temp, dict), "Temperature data should be a dict" assert len(result_temp) == 0, "Temperature data should be empty when no predictions available" - print(" ✓ Temperature data with no predictions handled correctly") + print(" PASS: Temperature data with no predictions handled correctly") # Test 9: Step-size calculation correctness (bug #3384 regression test) async def test_step_size_calculation(): @@ -2376,7 +2379,7 @@ def mock_get_arg_no_car_hold(key, default=None, indirect=True, combine=False, at assert abs(value_1440 - expected_value) < 0.01, f"Energy at minute 1440 should be ~{expected_value:.2f} kWh (got {value_1440:.4f}). Bug #3384 would cause 0.0." assert value_1440 > 0.01, f"Energy at minute 1440 should be > 0.01 kWh (got {value_1440:.4f}). Bug #3384 would cause near-zero." - # Check minute 1435 (per-step energy for that interval — NOT accumulated) + # Check minute 1435 (per-step energy for that interval -- NOT accumulated) if 1435 in result_data: value_1435 = result_data[1435] # delta = abs(load_data[1435] - load_data[1440]) = abs(1.0 - 0.5) = 0.5 @@ -2403,7 +2406,7 @@ def mock_get_arg_no_car_hold(key, default=None, indirect=True, combine=False, at # Verify current load is reasonable (not near-zero) assert result_now > 0.05, f"Current load should be > 0.05 kWh (got {result_now:.4f}). Bug #3384 would cause near-zero." - print(" ✓ Step-size calculation correct (bug #3384 regression test passed)") + print(" PASS: Step-size calculation correct (bug #3384 regression test passed)") # Run all sub-tests print(" Running LoadMLComponent._fetch_load_data tests:") @@ -2592,7 +2595,7 @@ def mock_dashboard_item(entity_id, state, attributes, app): assert attrs["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'" assert attrs2["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'" - print(" ✓ Entity published with correct attributes") + print(" PASS: Entity published with correct attributes") # Test 2: Empty predictions mock_base.dashboard_calls = [] @@ -2605,7 +2608,7 @@ def mock_dashboard_item(entity_id, state, attributes, app): assert call2["state"] == 0, "State should be 0 with empty predictions" assert call["attributes"]["results"] == {}, "results should be empty dict" - print(" ✓ Empty predictions handled correctly") + print(" PASS: Empty predictions handled correctly") print(" All _publish_entity tests passed!") @@ -2693,7 +2696,7 @@ def mock_dashboard_item(entity_id, state, attributes, app): assert abs(attrs2["load_today_h8"] - 5.15) < 0.01, f"Expected load_today_h8 5.15 (5.0 + 0.15), got {attrs2['load_today_h8']}" assert any("previous day" in msg for msg in mock_base.log_messages), "Stale baseline should be logged as a warning" - print(" ✓ Stale pre-midnight baseline re-derived from load history") + print(" PASS: Stale pre-midnight baseline re-derived from load history") # Same-day snapshot must be used verbatim, no re-derivation mock_base.dashboard_calls = [] @@ -2706,7 +2709,7 @@ def mock_dashboard_item(entity_id, state, attributes, app): assert abs(attrs2["load_today"] - 0.4) < 0.01, f"Expected load_today 0.4 from the snapshot, got {attrs2['load_today']}" assert abs(attrs2["load_today_h1"] - 1.0) < 0.01, f"Expected load_today_h1 1.0 (0.6 + 0.4), got {attrs2['load_today_h1']}" - print(" ✓ Same-day baseline snapshot used unchanged") + print(" PASS: Same-day baseline snapshot used unchanged") # A missing snapshot timestamp (e.g. loaded from an older state) keeps the old behaviour mock_base.dashboard_calls = [] @@ -2717,7 +2720,7 @@ def mock_dashboard_item(entity_id, state, attributes, app): attrs2 = mock_base.dashboard_calls[1]["attributes"] assert abs(attrs2["load_today"] - 0.4) < 0.01, f"Expected load_today 0.4 when no snapshot time is known, got {attrs2['load_today']}" - print(" ✓ Missing snapshot timestamp falls back to the stored baseline") + print(" PASS: Missing snapshot timestamp falls back to the stored baseline") # Re-fetch after a long training run must re-anchor the baseline before predicting async def run_stale_refetch(): @@ -2766,7 +2769,7 @@ async def mock_do_training(is_initial): run_async(run_stale_refetch()) - print(" ✓ Stale data re-fetched after training before predicting") + print(" PASS: Stale data re-fetched after training before predicting") assert PREDICT_STEP == 5, "Test assumes a 5 minute prediction step" @@ -3086,7 +3089,7 @@ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=Non now_utc = datetime.now(timezone.utc) load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) - # --- Part 1: model with a known timestamp → last_train_time set to that timestamp --- + # --- Part 1: model with a known timestamp -> last_train_time set to that timestamp --- predictor = LoadPredictor(learning_rate=0.01) predictor.train(load_data, now_utc, is_initial=True, epochs=2, time_decay_days=7) known_timestamp = predictor.training_timestamp @@ -3101,7 +3104,7 @@ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=Non assert component.model_valid is True, "Model should be marked valid" assert component.initial_training_done is True, "initial_training_done should be True" - # --- Part 2: model with no timestamp → last_train_time stays None (triggers retrain) --- + # --- Part 2: model with no timestamp -> last_train_time stays None (triggers retrain) --- predictor2 = LoadPredictor(learning_rate=0.01) predictor2.train(load_data, now_utc, is_initial=True, epochs=2, time_decay_days=7) predictor2.training_timestamp = None # Simulate a pre-timestamp model @@ -3112,3 +3115,131 @@ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=Non assert component2.last_train_time is None, "last_train_time should remain None when model has no embedded timestamp (triggers safe retrain)" assert component2.model_valid is True, "Model without timestamp should still be considered valid by is_valid()" + + +def _test_nan_inf_robustness(): + """Test that LoadPredictor handles NaN, Inf, and None across all channels without producing NaN loss.""" + now_utc = datetime.now(timezone.utc) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc) + temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc) + import_rates = {m: 25.0 for m in load_data} + export_rates = {m: 15.0 for m in load_data} + + # Inject NaN, Inf, and None into a subset of minute keys + load_with_nan = dict(load_data) + load_with_nan[30] = float("nan") + load_with_nan[60] = float("inf") + load_with_nan[90] = None + + temp_with_nan = dict(temp_data) + temp_with_nan[30] = float("nan") + temp_with_nan[60] = float("-inf") + + pv_with_nan = dict(pv_data) + pv_with_nan[30] = float("nan") + + import_rates_nan = dict(import_rates) + import_rates_nan[30] = float("nan") + + predictor = LoadPredictor(learning_rate=0.001) + val_mae = predictor.train( + load_with_nan, + now_utc, + pv_minutes=pv_with_nan, + temp_minutes=temp_with_nan, + import_rates=import_rates_nan, + export_rates=export_rates, + is_initial=True, + epochs=3, + ) + + assert val_mae is not None, "Training should produce a valid float val_mae even when NaN/Inf are in inputs" + assert not np.isnan(val_mae), "val_mae must not be NaN" + assert not np.isinf(val_mae), "val_mae must not be Inf" + assert predictor.validation_bias is not None and not np.isnan(predictor.validation_bias), "validation_bias must not be NaN" + + +def _test_database_zero_preservation(): + """Test that save_database_history and load_database_history preserve valid 0.0 entries.""" + import asyncio + import tempfile + from load_ml_component import LoadMLComponent + + class MockBase: + """Minimal base for LoadMLComponent.""" + + def __init__(self, config_root): + self.prefix = "predbat" + self.config_root = config_root + self.now_utc = datetime.now(timezone.utc) + self.midnight_utc = self.now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + self.minutes_now = (self.now_utc - self.midnight_utc).seconds // 60 + self.local_tz = timezone.utc + self.args = {} + self.log_messages = [] + + def log(self, msg): + self.log_messages.append(msg) + + def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return {"load_today": ["sensor.load_today"]}.get(key, default) + + async def run_test(): + with tempfile.TemporaryDirectory() as tmpdir: + base = MockBase(config_root=tmpdir) + component = LoadMLComponent(base, load_ml_enable=True) + component.database_filepath = os.path.join(tmpdir, "predbat_ml_history.npz") + + # Create history with explicit 0.0 values at specific minutes + test_load = {m: (0.0 if m % 30 == 0 else 0.25) for m in range(0, 1440, 5)} + test_temp = {m: 0.0 for m in range(0, 1440, 5)} # Freezing temperature everywhere + test_pv = {m: 0.0 for m in range(0, 1440, 5)} + + component.load_data = test_load + component.temperature_data = test_temp + component.pv_data = test_pv + component.load_data_age_days = 1.0 + + await component.save_database_history() + + # Create a fresh component instance and load history + component2 = LoadMLComponent(base, load_ml_enable=True) + component2.database_filepath = os.path.join(tmpdir, "predbat_ml_history.npz") + + await component2.load_database_history() + + assert component2.load_data is not None, "load_data should be loaded" + assert 0 in component2.load_data, "Minute 0 (value 0.0) must be preserved in load_data" + assert component2.load_data[0] == 0.0, f"Minute 0 load should be 0.0, got {component2.load_data[0]}" + assert 0 in component2.temperature_data, "Minute 0 (0.0°C) must be preserved in temperature_data" + assert component2.temperature_data[0] == 0.0, f"Minute 0 temp should be 0.0, got {component2.temperature_data[0]}" + + asyncio.run(run_test()) + + +def _test_curriculum_90day_intermediate_passes(): + """Test train_curriculum with 90 days of history and 8 intermediate passes.""" + now_utc = datetime.now(timezone.utc) + load_data = _create_synthetic_load_data(n_days=90, now_utc=now_utc) + pv_data = _create_synthetic_pv_data(n_days=28, now_utc=now_utc) # Only 28 days of PV history + temp_data = _create_synthetic_temp_data(n_days=28, now_utc=now_utc) + + predictor = LoadPredictor(learning_rate=0.001) + val_mae = predictor.train_curriculum( + load_data, + now_utc, + pv_minutes=pv_data, + temp_minutes=temp_data, + epochs=1, + time_decay_days=30, + validation_holdout_hours=48, + patience=2, + curriculum_window_days=7, + curriculum_step_days=5, + max_intermediate_passes=8, + ) + + assert val_mae is not None, "90-day curriculum training should succeed" + assert not np.isnan(val_mae), "val_mae must not be NaN" + assert predictor.model_initialized, "Model must be initialized after curriculum training" diff --git a/apps/predbat/tests/test_ml_training_perf.py b/apps/predbat/tests/test_ml_training_perf.py index 87d811a04..8f8075e59 100644 --- a/apps/predbat/tests/test_ml_training_perf.py +++ b/apps/predbat/tests/test_ml_training_perf.py @@ -22,12 +22,16 @@ import gzip import json import os -import resource import subprocess import sys import time from datetime import datetime, timezone +try: + import resource +except ImportError: + resource = None + import numpy as np from load_predictor import LoadPredictor, TOTAL_FEATURES @@ -42,12 +46,16 @@ # interleaved with training's own allocations badly enough to change the result - the same # workload read 967MB with the sampler running and 474MB without. ru_maxrss is the kernel's # own high-water mark, so it needs no sampling at all: read it once before and once after. + + def peak_rss_mb(): """Return this process's peak resident set size in MB. The value only ever rises, so the difference across a section of work is that section's contribution to the peak. ru_maxrss is bytes on macOS and kilobytes on Linux. """ + if resource is None: + return 0.0 value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss return value / 1e6 if sys.platform == "darwin" else value / 1e3 diff --git a/apps/predbat/tests/test_sunsynk_api.py b/apps/predbat/tests/test_sunsynk_api.py index 8d0f45c98..45c55ea17 100644 --- a/apps/predbat/tests/test_sunsynk_api.py +++ b/apps/predbat/tests/test_sunsynk_api.py @@ -707,6 +707,11 @@ def _run_with_hard_timeout(coro, seconds=5): bytecode instructions, so it fires even inside a loop that never truly yields. """ + if not hasattr(signal, "SIGALRM"): + import asyncio + + return run_async_local(asyncio.wait_for(coro, timeout=seconds)) + def _handler(signum, frame): """Convert the SIGALRM into a Python exception so pytest-free assertions can catch it.""" raise _HardTimeout(f"operation exceeded its {seconds}s hard timeout - suspected infinite loop") From 534ea77e5603f4d52a63a9b3ab8527d1090e96fc Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 19 Aug 2026 15:07:17 +0100 Subject: [PATCH 2/5] test: ensure UTF-8 console and LF file compatibility in test runners --- apps/predbat/tests/test_download.py | 12 ++++++------ apps/predbat/tests/test_plan_why_reason.py | 2 +- apps/predbat/unit_test.py | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/predbat/tests/test_download.py b/apps/predbat/tests/test_download.py index b2372e7e2..24b2dfad2 100644 --- a/apps/predbat/tests/test_download.py +++ b/apps/predbat/tests/test_download.py @@ -129,13 +129,13 @@ def test_download(my_predbat): try: test_result = test_func(my_predbat) if test_result: - print(f"✗ FAILED: {test_name}") + print(f"FAILED: {test_name}") failed += 1 else: - print(f"✓ PASSED: {test_name}") + print(f"PASSED: {test_name}") passed += 1 except Exception as e: - print(f"✗ EXCEPTION in {test_name}: {e}") + print(f"EXCEPTION in {test_name}: {e}") import traceback traceback.print_exc() @@ -291,9 +291,9 @@ def _test_compute_file_sha1(my_predbat): """ Test Git blob SHA1 hash computation (matches GitHub's SHA) """ - # Create a temporary file with known content - with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write("test content\n") + # Create a temporary file with known content (exact LF on all platforms) + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content\n") temp_path = f.name try: diff --git a/apps/predbat/tests/test_plan_why_reason.py b/apps/predbat/tests/test_plan_why_reason.py index 506f0b807..06500177b 100644 --- a/apps/predbat/tests/test_plan_why_reason.py +++ b/apps/predbat/tests/test_plan_why_reason.py @@ -493,7 +493,7 @@ def render(): # intended for JS must be doubled. A single "\{" raises SyntaxWarning today and becomes a # SyntaxError in a future Python. print("Test web_helper.py has no invalid escape sequences") - with open(web_helper.__file__, "r") as han: + with open(web_helper.__file__, "r", encoding="utf-8") as han: web_helper_source = han.read() with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py index ca26428dc..330eaef75 100644 --- a/apps/predbat/unit_test.py +++ b/apps/predbat/unit_test.py @@ -14,6 +14,11 @@ import glob import argparse +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") +if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8", errors="replace") + from predbat import PredBat from tests.test_infra import TestHAInterface from tests.test_compute_metric import run_compute_metric_tests From c2cc7da811f52f28392010570e9d7c34bf2e9867 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 19 Aug 2026 16:55:26 +0100 Subject: [PATCH 3/5] style: format load_ml_component and add cspell entries --- .cspell/custom-dictionary-workspace.txt | 2 ++ apps/predbat/load_ml_component.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index a3dba1958..86ef84e4c 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -327,6 +327,8 @@ mtok mult myenergi mypy +nanmean +nanstd nattribute ncalls nearr diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index 5a8f67c16..f34d1bc81 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -1066,8 +1066,7 @@ def _load_baseline_now(self): derived_baseline += self.load_data.get(minute, 0.0) derived_baseline = dp4(derived_baseline) self.log( - "Warn: ML Component: Load baseline of {} kWh was captured on {} which is a previous day, " - "re-derived load so far today as {} kWh".format(dp2(self.load_minutes_now), self.load_minutes_now_time.strftime("%Y-%m-%d %H:%M"), dp2(derived_baseline)) + "Warn: ML Component: Load baseline of {} kWh was captured on {} which is a previous day, " "re-derived load so far today as {} kWh".format(dp2(self.load_minutes_now), self.load_minutes_now_time.strftime("%Y-%m-%d %H:%M"), dp2(derived_baseline)) ) return derived_baseline From dfbeae536a2888312fb96d37567318f0ee1914de Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 19 Aug 2026 16:59:40 +0100 Subject: [PATCH 4/5] docs: add docstrings to ML helper and test mock functions --- apps/predbat/load_ml_component.py | 1 + apps/predbat/tests/test_load_ml.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index f34d1bc81..b477d5641 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -830,6 +830,7 @@ async def save_database_history(self): max_steps = self.load_ml_database_days * 24 * 60 // PREDICT_STEP def dict_to_array(data_dict): + """Convert sparse {minute: value} dictionary into a fixed-size numpy array.""" arr = np.full(max_steps, np.nan, dtype=np.float32) if data_dict: for minute, value in data_dict.items(): diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py index 182b44e30..f654752d6 100644 --- a/apps/predbat/tests/test_load_ml.py +++ b/apps/predbat/tests/test_load_ml.py @@ -3170,6 +3170,7 @@ class MockBase: """Minimal base for LoadMLComponent.""" def __init__(self, config_root): + """Initialize MockBase.""" self.prefix = "predbat" self.config_root = config_root self.now_utc = datetime.now(timezone.utc) @@ -3180,12 +3181,15 @@ def __init__(self, config_root): self.log_messages = [] def log(self, msg): + """Record log message.""" self.log_messages.append(msg) def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + """Mock get_arg.""" return {"load_today": ["sensor.load_today"]}.get(key, default) async def run_test(): + """Run async save and reload assertions.""" with tempfile.TemporaryDirectory() as tmpdir: base = MockBase(config_root=tmpdir) component = LoadMLComponent(base, load_ml_enable=True) From 4ff478a21ce27235cd109582dd7810ea4149b6a6 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 20 Aug 2026 08:35:52 +0100 Subject: [PATCH 5/5] fix(ml): harden DB serialization guards, sentinel metadata, and AdamW inf clamps --- apps/predbat/load_ml_component.py | 101 +++++++++++++++-------------- apps/predbat/load_predictor.py | 4 +- apps/predbat/tests/test_load_ml.py | 55 ++++++++++++++-- 3 files changed, 107 insertions(+), 53 deletions(-) diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index 380b845a2..2e3f7abb7 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -839,8 +839,13 @@ def dict_to_array(data_dict): # Only persist historical data (non-negative integer keys) if isinstance(minute, int) and minute >= 0: idx = minute // PREDICT_STEP - if 0 <= idx < max_steps: - arr[idx] = float(value) + if 0 <= idx < max_steps and value is not None: + try: + val = float(value) + if np.isfinite(val): + arr[idx] = val + except (ValueError, TypeError): + pass return arr total_steps = len(self.load_data) if self.load_data else 0 @@ -852,6 +857,7 @@ def dict_to_array(data_dict): "step_minutes": PREDICT_STEP, "n_steps": total_steps, "age_days": float(self.load_data_age_days), + "nan_sentinel": True, } ) @@ -882,51 +888,52 @@ async def load_database_history(self): try: # np.load is blocking disk IO - run in executor so the event loop stays alive - data = np.load(self.database_filepath, allow_pickle=False) - metadata = json.loads(str(data["metadata_json"])) - - version = metadata.get("version", 0) - if version != DATABASE_VERSION: - self.log("Warn: ML Component: Database version mismatch (saved={}, current={}), discarding".format(version, DATABASE_VERSION)) - return - - step_minutes = metadata.get("step_minutes", PREDICT_STEP) - if step_minutes != PREDICT_STEP: - self.log("Warn: ML Component: Database step size mismatch (saved={}min, current={}min), discarding".format(step_minutes, PREDICT_STEP)) - return - - saved_utc = datetime.fromisoformat(metadata["saved_utc"]) - age_days = float(metadata.get("age_days", 0)) - - def array_to_dict(arr): - """Reconstruct a sparse {minute: value} dict with keys as stored""" - result = {} - has_nans = np.isnan(arr).any() - for i in range(len(arr)): - val = float(arr[i]) - if has_nans: - if not np.isnan(val) and np.isfinite(val): - result[i * PREDICT_STEP] = val - else: - if val != 0.0: - result[i * PREDICT_STEP] = val - return result - - self.load_data = array_to_dict(data["load"]) - self.pv_data = array_to_dict(data["pv"]) - self.temperature_data = array_to_dict(data["temp"]) - self.import_rates_data = array_to_dict(data["import_rate"]) - self.export_rates_data = array_to_dict(data["export_rate"]) - self.load_data_age_days = age_days - # Restore last_data_fetch to the save time so that run()'s _shift() will - # compute the correct elapsed time and apply the shift exactly once. - self.last_data_fetch = saved_utc - - if self.load_data: - self.data_ready = True - - elapsed_minutes = (self.now_utc - saved_utc).total_seconds() / 60.0 - self.log("ML Component: Loaded database history: {} load points, {:.1f}h elapsed since save, age {:.1f} days (shift will be applied on next fetch)".format(len(self.load_data), elapsed_minutes / 60.0, self.load_data_age_days)) + with np.load(self.database_filepath, allow_pickle=False) as data: + metadata = json.loads(str(data["metadata_json"])) + + version = metadata.get("version", 0) + if version != DATABASE_VERSION: + self.log("Warn: ML Component: Database version mismatch (saved={}, current={}), discarding".format(version, DATABASE_VERSION)) + return + + step_minutes = metadata.get("step_minutes", PREDICT_STEP) + if step_minutes != PREDICT_STEP: + self.log("Warn: ML Component: Database step size mismatch (saved={}min, current={}min), discarding".format(step_minutes, PREDICT_STEP)) + return + + saved_utc = datetime.fromisoformat(metadata["saved_utc"]) + age_days = float(metadata.get("age_days", 0)) + use_nan_sentinel = metadata.get("nan_sentinel", False) + + def array_to_dict(arr): + """Reconstruct a sparse {minute: value} dict with keys as stored""" + result = {} + nan_format = use_nan_sentinel or np.isnan(arr).any() + for i in range(len(arr)): + val = float(arr[i]) + if nan_format: + if not np.isnan(val) and np.isfinite(val): + result[i * PREDICT_STEP] = val + else: + if val != 0.0: + result[i * PREDICT_STEP] = val + return result + + self.load_data = array_to_dict(data["load"]) + self.pv_data = array_to_dict(data["pv"]) + self.temperature_data = array_to_dict(data["temp"]) + self.import_rates_data = array_to_dict(data["import_rate"]) + self.export_rates_data = array_to_dict(data["export_rate"]) + self.load_data_age_days = age_days + # Restore last_data_fetch to the save time so that run()'s _shift() will + # compute the correct elapsed time and apply the shift exactly once. + self.last_data_fetch = saved_utc + + if self.load_data: + self.data_ready = True + + elapsed_minutes = (self.now_utc - saved_utc).total_seconds() / 60.0 + self.log("ML Component: Loaded database history: {} load points, {:.1f}h elapsed since save, age {:.1f} days (shift will be applied on next fetch)".format(len(self.load_data), elapsed_minutes / 60.0, self.load_data_age_days)) except Exception as e: self.log("Warn: ML Component: Failed to load database history: {} - {}".format(e, traceback.format_exc())) diff --git a/apps/predbat/load_predictor.py b/apps/predbat/load_predictor.py index f1ef3ef7e..6a5e7dcca 100644 --- a/apps/predbat/load_predictor.py +++ b/apps/predbat/load_predictor.py @@ -412,8 +412,8 @@ def _adam_update(self, weight_grads, bias_grads, beta1=0.9, beta2=0.999, epsilon self.biases[i] -= effective_lr * m_hat / (np.sqrt(v_hat) + epsilon) # Ensure weights and biases remain finite - self.weights[i] = np.nan_to_num(self.weights[i], copy=False, nan=0.0) - self.biases[i] = np.nan_to_num(self.biases[i], copy=False, nan=0.0) + self.weights[i] = np.nan_to_num(self.weights[i], copy=False, nan=0.0, posinf=0.0, neginf=0.0) + self.biases[i] = np.nan_to_num(self.biases[i], copy=False, nan=0.0, posinf=0.0, neginf=0.0) def _create_time_features(self, minute_of_day, day_of_week, day_of_year=1): """ diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py index 689f3939a..b1920846a 100644 --- a/apps/predbat/tests/test_load_ml.py +++ b/apps/predbat/tests/test_load_ml.py @@ -3182,9 +3182,20 @@ def _test_nan_inf_robustness(): assert not np.isinf(val_mae), "val_mae must not be Inf" assert predictor.validation_bias is not None and not np.isnan(predictor.validation_bias), "validation_bias must not be NaN" + # Test that _adam_update safely clamps +/-Inf weights and biases to finite values + predictor._initialize_weights() + predictor.weights[0][0, 0] = float("inf") + predictor.weights[0][0, 1] = float("-inf") + predictor.biases[0][0] = float("inf") + w_grads = [np.zeros_like(w) for w in predictor.weights] + b_grads = [np.zeros_like(b) for b in predictor.biases] + predictor._adam_update(w_grads, b_grads) + assert np.isfinite(predictor.weights[0]).all(), "Weights must be finite after _adam_update with Inf inputs" + assert np.isfinite(predictor.biases[0]).all(), "Biases must be finite after _adam_update with Inf inputs" + def _test_database_zero_preservation(): - """Test that save_database_history and load_database_history preserve valid 0.0 entries.""" + """Test that save_database_history and load_database_history preserve valid 0.0 entries across sparse, fully-populated, and dirty inputs.""" import asyncio import tempfile from load_ml_component import LoadMLComponent @@ -3217,10 +3228,11 @@ async def run_test(): base = MockBase(config_root=tmpdir) component = LoadMLComponent(base, load_ml_enable=True) component.database_filepath = os.path.join(tmpdir, "predbat_ml_history.npz") + component.load_ml_database_days = 1 # 288 5-min steps - # Create history with explicit 0.0 values at specific minutes + # Case 1: Fully-populated history (288 steps, no NaNs in saved array, containing explicit 0.0) test_load = {m: (0.0 if m % 30 == 0 else 0.25) for m in range(0, 1440, 5)} - test_temp = {m: 0.0 for m in range(0, 1440, 5)} # Freezing temperature everywhere + test_temp = {m: 0.0 for m in range(0, 1440, 5)} # Freezing temperature everywhere (all 0.0) test_pv = {m: 0.0 for m in range(0, 1440, 5)} component.load_data = test_load @@ -3230,6 +3242,10 @@ async def run_test(): await component.save_database_history() + # Verify saved array has zero NaNs (fully populated) + with np.load(component.database_filepath, allow_pickle=False) as saved_npz: + assert not np.isnan(saved_npz["load"]).any(), "Test array should have no NaNs to test nan_sentinel metadata flag" + # Create a fresh component instance and load history component2 = LoadMLComponent(base, load_ml_enable=True) component2.database_filepath = os.path.join(tmpdir, "predbat_ml_history.npz") @@ -3237,11 +3253,42 @@ async def run_test(): await component2.load_database_history() assert component2.load_data is not None, "load_data should be loaded" - assert 0 in component2.load_data, "Minute 0 (value 0.0) must be preserved in load_data" + assert len(component2.load_data) == 288, f"Expected all 288 steps in load_data, got {len(component2.load_data)}" + assert 0 in component2.load_data, "Minute 0 (value 0.0) must be preserved in fully populated load_data" assert component2.load_data[0] == 0.0, f"Minute 0 load should be 0.0, got {component2.load_data[0]}" + assert component2.load_data[30] == 0.0, f"Minute 30 load should be 0.0, got {component2.load_data[30]}" + assert len(component2.temperature_data) == 288, f"Expected all 288 steps in temperature_data, got {len(component2.temperature_data)}" assert 0 in component2.temperature_data, "Minute 0 (0.0°C) must be preserved in temperature_data" assert component2.temperature_data[0] == 0.0, f"Minute 0 temp should be 0.0, got {component2.temperature_data[0]}" + # Case 2: Dirty / unparsable history entries (None, "unavailable", NaN, Inf) + test_load_dirty = {0: 0.0, 5: 0.5, 10: None, 15: "unavailable", 20: float("nan"), 25: float("inf"), 30: 0.0} + component.load_data = test_load_dirty + component.temperature_data = {0: 10.0, 5: None, 10: "error"} + component.pv_data = {0: float("nan"), 5: 1.5} + component.database_filepath = os.path.join(tmpdir, "predbat_ml_history_dirty.npz") + + # Must complete cleanly without raising TypeError or ValueError + await component.save_database_history() + + component3 = LoadMLComponent(base, load_ml_enable=True) + component3.database_filepath = os.path.join(tmpdir, "predbat_ml_history_dirty.npz") + await component3.load_database_history() + + assert component3.load_data is not None + assert component3.load_data.get(0) == 0.0, "Valid 0.0 must be preserved" + assert component3.load_data.get(5) == 0.5, "Valid 0.5 must be preserved" + assert component3.load_data.get(30) == 0.0, "Valid 0.0 at minute 30 must be preserved" + assert 10 not in component3.load_data, "None value must not be present in reconstructed dict" + assert 15 not in component3.load_data, "String value must not be present in reconstructed dict" + assert 20 not in component3.load_data, "NaN value must not be present in reconstructed dict" + assert 25 not in component3.load_data, "Inf value must not be present in reconstructed dict" + assert component3.temperature_data.get(0) == 10.0 + assert 5 not in component3.temperature_data + assert 10 not in component3.temperature_data + assert component3.pv_data.get(5) == 1.5 + assert 0 not in component3.pv_data + asyncio.run(run_test())