diff --git a/.gitignore b/.gitignore index 4d9a4a0..9f11515 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ rebar3.crashdump src/grpc/autogen *~ .env -hpr_route_ets/ \ No newline at end of file +hpr_route_ets/ +hpr_devaddr_range_storage/ +CLAUDE.md +*.coverdata \ No newline at end of file diff --git a/config/sys.config.src b/config/sys.config.src index 0e9fcc7..0cb0392 100644 --- a/config/sys.config.src +++ b/config/sys.config.src @@ -34,6 +34,7 @@ }}, {routing_cache_timeout_secs, 15}, {routing_cache_window_secs, 120}, + {devaddr_cache_ttl_ms, 43200000}, {ics_stream_worker_checkpoint_secs, 300} ]}, {aws_credentials, [ diff --git a/include/hpr_metrics.hrl b/include/hpr_metrics.hrl index 1b25eaf..8512b7e 100644 --- a/include/hpr_metrics.hrl +++ b/include/hpr_metrics.hrl @@ -24,6 +24,8 @@ "hpr_iot_config_service_gateway_location_histogram" ). -define(METRICS_DEVICE_GAUGE, "hpr_device_gauge"). +-define(METRICS_DEVADDR_CACHE_HIT_COUNTER, "hpr_devaddr_cache_hit_counter"). +-define(METRICS_DEVADDR_CACHE_MISS_COUNTER, "hpr_devaddr_cache_miss_counter"). -define(METRICS, [ {?METRICS_GRPC_CONNECTION_GAUGE, prometheus_gauge, [], "Number of active GRPC Connections"}, @@ -45,5 +47,7 @@ {?METRICS_ICS_UPDATES_COUNTER, prometheus_counter, [type, action], "ICS updates counter"}, {?METRICS_ICS_GATEWAY_LOCATION_HISTOGRAM, prometheus_histogram, [status], "ICS gateway location req"}, - {?METRICS_DEVICE_GAUGE, prometheus_gauge, [], "Approximate number of devices"} + {?METRICS_DEVICE_GAUGE, prometheus_gauge, [], "Approximate number of devices"}, + {?METRICS_DEVADDR_CACHE_HIT_COUNTER, prometheus_counter, [], "DevAddr cache hits"}, + {?METRICS_DEVADDR_CACHE_MISS_COUNTER, prometheus_counter, [], "DevAddr cache misses"} ]). diff --git a/src/cli/hpr_cli_config.erl b/src/cli/hpr_cli_config.erl index ea78faf..da097e8 100644 --- a/src/cli/hpr_cli_config.erl +++ b/src/cli/hpr_cli_config.erl @@ -18,7 +18,11 @@ -ifdef(TEST). -export([ - config_route_sync/3 + config_route_sync/3, + config_route_activate/3, + config_route_deactivate/3, + config_oui_activate/3, + config_oui_deactivate/3 ]). -endif. @@ -52,6 +56,9 @@ config_usage() -> "config oui - Info for OUI\n", " [--display_euis] default: false (EUIs not included)\n", " [--display_skfs] default: false (SKFs not included)\n", + "config oui activate - Activate all routes for OUI\n", + "config oui deactivate - Deactivate all routes for OUI\n", + "config oui refresh - Refresh all routes for OUI\n", "config route - Info for route\n", " [--display_euis] default: false (EUIs not included)\n", " [--display_skfs] default: false (SKFs not included)\n", @@ -67,6 +74,8 @@ config_usage() -> "config eui --app --dev - List all Routes with EUI pair\n" "\n\n", "config counts - Simple Counts of Configuration\n", + "config cache clear - Clear DevAddr lookup cache\n", + "config cache stats - Show DevAddr cache statistics\n", "config checkpoint next - Time until next writing of configuration to disk\n" "config checkpoint write - Write current configuration to disk\n", "config reset checkpoint [--commit] - Set checkpoint timestamp to beginning of time (0)\n", @@ -87,6 +96,24 @@ config_cmd() -> ], fun config_oui_list/3 ], + [ + ["config", "oui", "activate", '*'], + [], + [], + fun config_oui_activate/3 + ], + [ + ["config", "oui", "deactivate", '*'], + [], + [], + fun config_oui_deactivate/3 + ], + [ + ["config", "oui", "refresh", '*'], + [], + [], + fun config_oui_refresh/3 + ], [ ["config", "route", '*'], [], @@ -157,6 +184,8 @@ config_cmd() -> fun config_eui/3 ], [["config", "counts"], [], [], fun config_counts/3], + [["config", "cache", "clear"], [], [], fun config_cache_clear/3], + [["config", "cache", "stats"], [], [], fun config_cache_stats/3], [["config", "checkpoint", "next"], [], [], fun config_checkpoint_next/3], [["config", "checkpoint", "write"], [], [], fun config_checkpoint_write/3], [ @@ -251,6 +280,49 @@ config_oui_list(["config", "oui", OUIString], [], Flags) -> config_oui_list(_, _, _) -> usage. +config_oui_activate(["config", "oui", "activate", OUIString], [], _Flags) -> + OUI = erlang:list_to_integer(OUIString), + RoutesETS = hpr_route_storage:oui_routes_ets(OUI), + case RoutesETS of + [] -> + c_text("No routes found for OUI ~p", [OUI]); + _ -> + ActivatedCount = set_routes_active(RoutesETS, true), + c_text("Activated ~p routes for OUI ~p", [ActivatedCount, OUI]) + end; +config_oui_activate(_, _, _) -> + usage. + +config_oui_deactivate(["config", "oui", "deactivate", OUIString], [], _Flags) -> + OUI = erlang:list_to_integer(OUIString), + RoutesETS = hpr_route_storage:oui_routes_ets(OUI), + case RoutesETS of + [] -> + c_text("No routes found for OUI ~p", [OUI]); + _ -> + DeactivatedCount = set_routes_active(RoutesETS, false), + c_text("Deactivated ~p routes for OUI ~p", [DeactivatedCount, OUI]) + end; +config_oui_deactivate(_, _, _) -> + usage. + +config_oui_refresh(["config", "oui", "refresh", OUIString], [], _Flags) -> + OUI = erlang:list_to_integer(OUIString), + Pid = erlang:spawn(fun() -> + %% Get all routes for the OUI from the API to ensure we have them all + APIRoutes = get_api_routes_for_oui(OUI), + RouteIDs = [hpr_route:id(Route) || Route <- APIRoutes], + Total = erlang:length(RouteIDs), + lager:info("Found ~p routes to refresh for OUI ~p", [Total, OUI]), + routes_refresh(Total, RouteIDs) + end), + c_text( + "command spawned @ ~p for OUI ~p, look at logs and `tail -F /opt/hpr/log/info.log | grep hpr_cli_config`", + [Pid, OUI] + ); +config_oui_refresh(_, _, _) -> + usage. + config_route(["config", "route", RouteID], [], Flags) -> Options = maps:from_list(Flags), {ok, RouteETS} = hpr_route_storage:lookup(RouteID), @@ -440,9 +512,7 @@ config_route_activate(["config", "route", "activate", RouteID], [], _Flags) -> {error, not_found} -> c_text("Route ~s not found", [RouteID]); {ok, RouteETS} -> - Route0 = hpr_route_ets:route(RouteETS), - Route1 = hpr_route:active(true, Route0), - ok = hpr_route_storage:insert(Route1), + _Count = set_routes_active([RouteETS], true), c_text("Route ~s activated", [RouteID]) end; config_route_activate(_, _, _) -> @@ -453,9 +523,7 @@ config_route_deactivate(["config", "route", "deactivate", RouteID], [], _Flags) {error, not_found} -> c_text("Route ~s not found", [RouteID]); {ok, RouteETS} -> - Route0 = hpr_route_ets:route(RouteETS), - Route1 = hpr_route:active(false, Route0), - ok = hpr_route_storage:insert(Route1), + _Count = set_routes_active([RouteETS], false), c_text("Route ~s deactivated", [RouteID]) end; config_route_deactivate(_, _, _) -> @@ -690,6 +758,22 @@ config_counts(["config", "counts"], [], []) -> config_counts(_, _, _) -> usage. +config_cache_clear(["config", "cache", "clear"], [], []) -> + ok = hpr_devaddr_range_storage:clear_cache(), + c_text("DevAddr cache cleared"); +config_cache_clear(_, _, _) -> + usage. + +config_cache_stats(["config", "cache", "stats"], [], []) -> + CacheSize = hpr_devaddr_range_storage:cache_size(), + c_table([ + [ + {" Cache Entries ", CacheSize} + ] + ]); +config_cache_stats(_, _, _) -> + usage. + config_checkpoint_next(["config", "checkpoint", "next"], [], []) -> Msg = hpr_route_stream_worker:print_next_checkpoint(), c_text(Msg); @@ -734,6 +818,36 @@ config_reset(_, _, _) -> %% Helpers %%-------------------------------------------------------------------- +-spec set_routes_active( + RoutesETS :: list(hpr_route_ets:route()), + Active :: boolean() +) -> non_neg_integer(). +set_routes_active(RoutesETS, Active) -> + lists:foldl( + fun(RouteETS, Count) -> + Route0 = hpr_route_ets:route(RouteETS), + RouteID = hpr_route:id(Route0), + Route1 = hpr_route:active(Active, Route0), + ok = hpr_route_storage:insert(Route1), + case Active of + false -> + %% When deactivating, remove all SKFs, DevAddr ranges, and EUIs + %% but keep the route itself + _ = hpr_skf_storage:delete_route(RouteID), + _ = hpr_devaddr_range_storage:delete_route(RouteID), + _ = hpr_eui_pair_storage:delete_route(RouteID), + ok; + true -> + %% When activating, refresh the route to fetch SKFs, DevAddr ranges, and EUIs + _ = hpr_route_stream_worker:refresh_route(RouteID, 3), + ok + end, + Count + 1 + end, + 0, + RoutesETS + ). + -spec c_table(list(proplists:proplist()) | proplists:proplist()) -> clique_status:status(). c_table(PropLists) -> [clique_status:table(PropLists)]. diff --git a/src/grpc/iot_config/hpr_devaddr_range_storage.erl b/src/grpc/iot_config/hpr_devaddr_range_storage.erl index 932cbbc..4738801 100644 --- a/src/grpc/iot_config/hpr_devaddr_range_storage.erl +++ b/src/grpc/iot_config/hpr_devaddr_range_storage.erl @@ -15,21 +15,30 @@ lookup_for_route/1, count_for_route/1, - delete_all/0 + delete_all/0, + + clear_cache/0, + cache_size/0 ]). -ifdef(TEST). --export([test_delete_ets/0, test_size/0, test_tab_name/0]). +-export([test_delete_ets/0, test_size/0, test_tab_name/0, test_cache_size/0, test_clear_cache/0]). -endif. -define(ETS, hpr_route_devaddr_ranges_ets). -define(DETS, hpr_route_devaddr_ranges_dets). +-define(CACHE_ETS, hpr_devaddr_cache_ets). +% 12 hours +-define(DEFAULT_CACHE_TTL_MS, 43200000). -spec init_ets() -> ok. init_ets() -> ?ETS = ets:new(?ETS, [ public, named_table, bag, {read_concurrency, true} ]), + ?CACHE_ETS = ets:new(?CACHE_ETS, [ + public, named_table, set, {read_concurrency, true} + ]), ok = rehydrate_from_dets(), ok. @@ -45,6 +54,32 @@ foldl(Fun, Acc) -> -spec lookup(DevAddr :: non_neg_integer()) -> [hpr_route_ets:route()]. lookup(DevAddr) -> + case ets:lookup(?CACHE_ETS, DevAddr) of + [{DevAddr, {RouteIDs, CachedAt}}] -> + Now = erlang:system_time(millisecond), + TTL = hpr_utils:get_env_int(devaddr_cache_ttl_ms, ?DEFAULT_CACHE_TTL_MS), + case (Now - CachedAt) < TTL of + true -> + %% Cache hit - fetch routes from storage + ok = hpr_metrics:devaddr_cache_hit(), + [ + Route + || RouteID <- lists:usort(RouteIDs), + {ok, Route} <- [hpr_route_storage:lookup(RouteID)] + ]; + false -> + %% Cache expired + ok = hpr_metrics:devaddr_cache_miss(), + lookup_and_cache(DevAddr) + end; + [] -> + %% Cache miss + ok = hpr_metrics:devaddr_cache_miss(), + lookup_and_cache(DevAddr) + end. + +-spec lookup_and_cache(DevAddr :: non_neg_integer()) -> [hpr_route_ets:route()]. +lookup_and_cache(DevAddr) -> MS = [ { {{'$1', '$2'}, '$3'}, @@ -56,13 +91,16 @@ lookup(DevAddr) -> ], RouteIDs = ets:select(?ETS, MS), - lists:usort( - lists:flatten([ - Route - || RouteID <- RouteIDs, - {ok, Route} <- [hpr_route_storage:lookup(RouteID)] - ]) - ). + %% Cache the RouteIDs + Now = erlang:system_time(millisecond), + true = ets:insert(?CACHE_ETS, {DevAddr, {RouteIDs, Now}}), + + %% Return the full routes (dedup RouteIDs first to avoid redundant lookups) + [ + Route + || RouteID <- lists:usort(RouteIDs), + {ok, Route} <- [hpr_route_storage:lookup(RouteID)] + ]. -spec insert(DevAddrRange :: hpr_devaddr_range:devaddr_range()) -> ok. insert(DevAddrRange) -> @@ -72,6 +110,13 @@ insert(DevAddrRange) -> hpr_devaddr_range:route_id(DevAddrRange) } ]), + %% Invalidate cache entries in the range being inserted (handles empty cached results) + ok = invalidate_cache_for_range( + hpr_devaddr_range:start_addr(DevAddrRange), + hpr_devaddr_range:end_addr(DevAddrRange) + ), + %% Also invalidate all cache entries containing this route (handles existing cached results elsewhere) + ok = invalidate_cache_for_route(hpr_devaddr_range:route_id(DevAddrRange)), lager:debug( [ {start_addr, hpr_utils:int_to_hex_string(hpr_devaddr_range:start_addr(DevAddrRange))}, @@ -88,6 +133,13 @@ delete(DevAddrRange) -> {hpr_devaddr_range:start_addr(DevAddrRange), hpr_devaddr_range:end_addr(DevAddrRange)}, hpr_devaddr_range:route_id(DevAddrRange) }), + %% Invalidate cache entries in the range being deleted (handles empty cached results) + ok = invalidate_cache_for_range( + hpr_devaddr_range:start_addr(DevAddrRange), + hpr_devaddr_range:end_addr(DevAddrRange) + ), + %% Also invalidate all cache entries containing this route (handles existing cached results elsewhere) + ok = invalidate_cache_for_route(hpr_devaddr_range:route_id(DevAddrRange)), lager:debug( [ {start_addr, hpr_utils:int_to_hex_string(hpr_devaddr_range:start_addr(DevAddrRange))}, @@ -103,23 +155,6 @@ delete_all() -> ets:delete_all_objects(?ETS), ok. --ifdef(TEST). - --spec test_delete_ets() -> ok. -test_delete_ets() -> - ets:delete(?ETS), - ok. - --spec test_size() -> non_neg_integer(). -test_size() -> - ets:info(?ETS, size). - --spec test_tab_name() -> atom(). -test_tab_name() -> - ?ETS. - --endif. - %% ------------------------------------------------------------------ %% CLI Functions %% ------------------------------------------------------------------ @@ -135,12 +170,25 @@ count_for_route(RouteID) -> MS = [{{'_', RouteID}, [], [true]}], ets:select_count(?ETS, MS). +-spec clear_cache() -> ok. +clear_cache() -> + ets:delete_all_objects(?CACHE_ETS), + ok. + +-spec cache_size() -> non_neg_integer(). +cache_size() -> + case ets:info(?CACHE_ETS, size) of + undefined -> 0; + Size -> Size + end. + %% ------------------------------------------------------------------- %% Route Stream Helpers %% ------------------------------------------------------------------- -spec delete_route(hpr_route:id()) -> non_neg_integer(). delete_route(RouteID) -> + ok = invalidate_cache_for_route(RouteID), MS1 = [{{'_', RouteID}, [], [true]}], ets:select_delete(?ETS, MS1). @@ -153,6 +201,43 @@ replace_route(RouteID, DevAddrRanges) -> lists:foreach(fun ?MODULE:insert/1, DevAddrRanges), Removed. +-spec invalidate_cache_for_route(RouteID :: hpr_route:id()) -> ok. +invalidate_cache_for_route(RouteID) -> + %% Delete all cache entries that contain this RouteID + %% Since cache stores {DevAddr, {RouteIDs, Timestamp}}, we need to iterate + ets:foldl( + fun({DevAddr, {RouteIDs, _Ts}}, Acc) -> + case lists:member(RouteID, RouteIDs) of + true -> ets:delete(?CACHE_ETS, DevAddr); + false -> ok + end, + Acc + end, + ok, + ?CACHE_ETS + ), + ok. + +-spec invalidate_cache_for_range( + StartAddr :: non_neg_integer(), + EndAddr :: non_neg_integer() +) -> ok. +invalidate_cache_for_range(StartAddr, EndAddr) -> + %% Delete all cache entries for DevAddrs within this range + %% This handles the case where empty results were cached + ets:foldl( + fun({DevAddr, _CachedData}, Acc) -> + case DevAddr >= StartAddr andalso DevAddr =< EndAddr of + true -> ets:delete(?CACHE_ETS, DevAddr); + false -> ok + end, + Acc + end, + ok, + ?CACHE_ETS + ), + ok. + -spec rehydrate_from_dets() -> ok. rehydrate_from_dets() -> with_open_dets(fun() -> @@ -179,3 +264,482 @@ with_open_dets(FN) -> lager:warning("failed to open dets file ~p: ~p, deleted: ~p", [?MODULE, Reason, Deleted]), with_open_dets(FN) end. + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +-spec test_delete_ets() -> ok. +test_delete_ets() -> + ets:delete(?ETS), + ets:delete(?CACHE_ETS), + ok. + +-spec test_size() -> non_neg_integer(). +test_size() -> + ets:info(?ETS, size). + +-spec test_tab_name() -> atom(). +test_tab_name() -> + ?ETS. + +-spec test_cache_size() -> non_neg_integer(). +test_cache_size() -> + ets:info(?CACHE_ETS, size). + +-spec test_clear_cache() -> ok. +test_clear_cache() -> + ets:delete_all_objects(?CACHE_ETS), + ok. + +all_test_() -> + {foreach, fun foreach_setup/0, fun foreach_cleanup/1, [ + {"cache_miss_on_first_lookup", ?_test(cache_miss_on_first_lookup())}, + {"cache_hit_on_second_lookup", ?_test(cache_hit_on_second_lookup())}, + {"cache_expiration", ?_test(cache_expiration())}, + {"cache_invalidation_on_insert", ?_test(cache_invalidation_on_insert())}, + {"cache_invalidation_on_delete", ?_test(cache_invalidation_on_delete())}, + {"cache_invalidation_on_replace", ?_test(cache_invalidation_on_replace())}, + {"cache_deduplicates_route_ids", ?_test(cache_deduplicates_route_ids())}, + {"clear_cache_function", ?_test(clear_cache_function())}, + {"cache_size_function", ?_test(cache_size_function())} + ]}. + +foreach_setup() -> + BaseDirPath = filename:join([ + ?MODULE, + erlang:integer_to_list(erlang:system_time(millisecond)), + "data" + ]), + ok = application:set_env(hpr, data_dir, BaseDirPath), + ok = application:set_env(hpr, devaddr_cache_ttl_ms, 1000), + true = hpr_skf_storage:test_register_heir(), + ok = hpr_route_ets:init(), + meck:new(hpr_metrics, [passthrough]), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), + ok. + +foreach_cleanup(ok) -> + ok = hpr_devaddr_range_storage:test_delete_ets(), + ok = hpr_eui_pair_storage:test_delete_ets(), + ok = hpr_skf_storage:test_delete_ets(), + ok = hpr_route_storage:test_delete_ets(), + true = hpr_skf_storage:test_unregister_heir(), + ?assert(meck:validate(hpr_metrics)), + meck:unload(hpr_metrics), + ok. + +cache_miss_on_first_lookup() -> + %% Setup route + RouteID = "test-route-1", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert devaddr range + DevAddr = 16#00000001, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000000, + end_addr => 16#00000002 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% Cache should be empty before lookup + ?assertEqual(0, cache_size()), + ?assertEqual([], ets:lookup(?CACHE_ETS, DevAddr)), + + %% First lookup - should be a cache miss + Routes = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes)), + [FoundRoute] = Routes, + ?assertEqual(RouteID, hpr_route:id(hpr_route_ets:route(FoundRoute))), + + %% Cache should now have 1 entry with correct data + ?assertEqual(1, cache_size()), + [{DevAddr, {CachedRouteIDs, Timestamp}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs), + ?assert(is_integer(Timestamp)), + ?assert(Timestamp > 0), + + ?assertEqual(1, meck:num_calls(hpr_metrics, devaddr_cache_miss, 0)), + ?assertEqual(0, meck:num_calls(hpr_metrics, devaddr_cache_hit, 0)), + ok. + +cache_hit_on_second_lookup() -> + %% Setup route + RouteID = "test-route-2", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert devaddr range + DevAddr = 16#00000010, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000010, + end_addr => 16#00000020 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% First lookup - cache miss + Routes1 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes1)), + ?assertEqual(1, cache_size()), + + %% Get cached data after first lookup + [{DevAddr, {CachedRouteIDs1, Timestamp1}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs1), + + %% Wait a bit to ensure timestamps would differ if cache was refreshed + timer:sleep(10), + + %% Second lookup - should be cache hit (timestamp should NOT change) + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes2)), + ?assertEqual(1, cache_size()), + + %% Verify cache entry is unchanged (proves it was a cache hit) + [{DevAddr, {CachedRouteIDs2, Timestamp2}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual(CachedRouteIDs1, CachedRouteIDs2), + ?assertEqual(Timestamp1, Timestamp2), + + %% Routes should be the same + ?assertEqual(Routes1, Routes2), + + ?assertEqual(1, meck:num_calls(hpr_metrics, devaddr_cache_miss, 0)), + ?assertEqual(1, meck:num_calls(hpr_metrics, devaddr_cache_hit, 0)), + ok. + +cache_expiration() -> + %% Setup route + RouteID = "test-route-3", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert devaddr range + DevAddr = 16#00000030, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000030, + end_addr => 16#00000040 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% Set short TTL for this test + ok = application:set_env(hpr, devaddr_cache_ttl_ms, 100), + + %% First lookup - cache miss + Routes1 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes1)), + ?assertEqual(1, cache_size()), + + %% Get cached data after first lookup + [{DevAddr, {CachedRouteIDs1, Timestamp1}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs1), + + %% Wait for cache to expire + timer:sleep(150), + + %% Lookup after expiration - should trigger fresh lookup and update timestamp + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes2)), + + %% Cache should still have 1 entry but timestamp should be updated + ?assertEqual(1, cache_size()), + [{DevAddr, {CachedRouteIDs2, Timestamp2}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs2), + ?assert(Timestamp2 > Timestamp1, "Timestamp should be updated after expiration"), + + %% Routes should still be the same + ?assertEqual(Routes1, Routes2), + + %% Reset TTL + ok = application:set_env(hpr, devaddr_cache_ttl_ms, 1000), + ok. + +cache_invalidation_on_insert() -> + %% Setup route + RouteID = "test-route-4", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert first devaddr range + DevAddr = 16#00000050, + DevAddrRange1 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000050, + end_addr => 16#00000060 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange1), + + %% First lookup to populate cache + Routes1 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes1)), + ?assertEqual(1, cache_size()), + + %% Verify cache entry exists + [{DevAddr, {CachedRouteIDs1, _Timestamp1}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs1), + + %% Insert another range for same route - should invalidate cache + DevAddrRange2 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000061, + end_addr => 16#00000070 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange2), + + %% Cache entry for this DevAddr should be cleared + ?assertEqual([], ets:lookup(?CACHE_ETS, DevAddr)), + ?assertEqual(0, cache_size()), + + %% Next lookup should work and repopulate cache + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes2)), + ?assertEqual(1, cache_size()), + ok. + +cache_invalidation_on_delete() -> + %% Setup route + RouteID = "test-route-5", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert devaddr range + DevAddr = 16#00000070, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000070, + end_addr => 16#00000080 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% First lookup to populate cache + Routes1 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes1)), + ?assertEqual(1, cache_size()), + + %% Verify cache entry exists + [{DevAddr, {CachedRouteIDs1, _Timestamp1}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs1), + + %% Delete the devaddr range - should invalidate cache + ok = hpr_devaddr_range_storage:delete(DevAddrRange), + + %% Cache should be cleared + ?assertEqual([], ets:lookup(?CACHE_ETS, DevAddr)), + ?assertEqual(0, cache_size()), + + %% Next lookup should find no routes and cache empty result + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(0, erlang:length(Routes2)), + + %% Cache should now contain empty result + ?assertEqual(1, cache_size()), + [{DevAddr, {CachedRouteIDs2, _Timestamp2}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([], CachedRouteIDs2), + ok. + +cache_invalidation_on_replace() -> + %% Setup route + RouteID = "test-route-6", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert devaddr range + DevAddr = 16#00000090, + DevAddrRange1 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000090, + end_addr => 16#000000A0 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange1), + + %% First lookup to populate cache + Routes1 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes1)), + ?assertEqual(1, cache_size()), + + %% Verify cache entry exists + [{DevAddr, {CachedRouteIDs1, _Timestamp1}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([RouteID], CachedRouteIDs1), + + %% Replace route - should invalidate cache + DevAddrRange2 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#000000A1, + end_addr => 16#000000B0 + }), + Removed = hpr_devaddr_range_storage:replace_route(RouteID, [DevAddrRange2]), + ?assertEqual(1, Removed), + + %% Cache should be cleared + ?assertEqual([], ets:lookup(?CACHE_ETS, DevAddr)), + ?assertEqual(0, cache_size()), + + %% Next lookup should find no routes (range no longer includes DevAddr) + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(0, erlang:length(Routes2)), + + %% Verify empty result was cached + ?assertEqual(1, cache_size()), + [{DevAddr, {CachedRouteIDs2, _Timestamp2}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual([], CachedRouteIDs2), + ok. + +cache_deduplicates_route_ids() -> + %% Setup route + RouteID = "test-route-dedup", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + %% Insert multiple overlapping devaddr ranges for same route + DevAddr = 16#000000B0, + DevAddrRange1 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#000000A0, + end_addr => 16#000000C0 + }), + DevAddrRange2 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#000000B0, + end_addr => 16#000000D0 + }), + DevAddrRange3 = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#00000090, + end_addr => 16#000000B5 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange1), + ok = hpr_devaddr_range_storage:insert(DevAddrRange2), + ok = hpr_devaddr_range_storage:insert(DevAddrRange3), + + %% Lookup should find all 3 ranges but return only 1 route (deduplicated) + Routes = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes)), + [FoundRoute] = Routes, + ?assertEqual(RouteID, hpr_route:id(hpr_route_ets:route(FoundRoute))), + + %% Verify cached RouteIDs contains duplicates (raw ETS select result) + [{DevAddr, {CachedRouteIDs, _Timestamp}}] = ets:lookup(?CACHE_ETS, DevAddr), + ?assertEqual(3, erlang:length(CachedRouteIDs)), + ?assertEqual([RouteID, RouteID, RouteID], CachedRouteIDs), + + %% Second lookup should still return only 1 route (usort applied) + Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, erlang:length(Routes2)), + ok. + +clear_cache_function() -> + %% Setup route and devaddr range + RouteID = "test-route-7", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}}}, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + DevAddr = 16#000000C0, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => 16#000000C0, + end_addr => 16#000000D0 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% Populate cache + _Routes = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, cache_size()), + + %% Clear cache + ok = clear_cache(), + ?assertEqual(0, cache_size()), + + %% Next lookup should work and repopulate cache + _Routes2 = hpr_devaddr_range_storage:lookup(DevAddr), + ?assertEqual(1, cache_size()), + ok. + +cache_size_function() -> + %% Cache should start empty + ?assertEqual(0, cache_size()), + + %% Setup multiple routes and ranges + lists:foreach( + fun(N) -> + RouteID = "test-route-" ++ erlang:integer_to_list(N), + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 1, + oui => 1, + server => #{ + host => "localhost", port => 1234, protocol => {gwmp, #{mapping => []}} + }, + max_copies => 10 + }), + ok = hpr_route_storage:insert(Route), + + DevAddr = 16#00000100 + N, + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, + start_addr => DevAddr, + end_addr => DevAddr + 10 + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange), + + %% Lookup to populate cache + _Routes = hpr_devaddr_range_storage:lookup(DevAddr) + end, + lists:seq(1, 5) + ), + + %% Cache should have 5 entries + ?assertEqual(5, cache_size()), + ok. + +-endif. diff --git a/src/grpc/iot_config/hpr_route_ets.erl b/src/grpc/iot_config/hpr_route_ets.erl index 38ffa45..4c2e3ca 100644 --- a/src/grpc/iot_config/hpr_route_ets.erl +++ b/src/grpc/iot_config/hpr_route_ets.erl @@ -127,6 +127,9 @@ foreach_setup() -> ]), ok = application:set_env(hpr, data_dir, BaseDirPath), true = hpr_skf_storage:test_register_heir(), + meck:new(hpr_metrics, [passthrough]), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), ?MODULE:init(), ok. @@ -135,8 +138,8 @@ foreach_cleanup(ok) -> ok = hpr_eui_pair_storage:test_delete_ets(), ok = hpr_skf_storage:test_delete_ets(), ok = hpr_route_storage:test_delete_ets(), - true = hpr_skf_storage:test_unregister_heir(), + meck:unload(hpr_metrics), ok. test_route() -> diff --git a/src/grpc/packet_router/hpr_packet_router_service.erl b/src/grpc/packet_router/hpr_packet_router_service.erl index f3a6ec5..504348d 100644 --- a/src/grpc/packet_router/hpr_packet_router_service.erl +++ b/src/grpc/packet_router/hpr_packet_router_service.erl @@ -317,19 +317,28 @@ route_packet_test() -> EnvUp = hpr_envelope_up:new(PacketUp), meck:expect(hpr_routing, handle_packet, fun(_PacketUp, _Opts) -> ok end), + meck:new(hpr_metrics, [passthrough]), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), + StreamState = grpcbox_stream:stream_handler_state( #state{}, #handler_state{last_phash = hpr_packet_up:phash(PacketUp)} ), ?assertEqual({ok, StreamState}, ?MODULE:route(EnvUp, StreamState)), + %% Wait for the spawned process to execute (handle_packet spawns a process) + timer:sleep(100), ?assertEqual(1, meck:num_calls(hpr_routing, handle_packet, 2)), meck:unload(hpr_routing), + meck:unload(hpr_metrics), ok. route_register_test() -> meck:new(hpr_metrics, [passthrough]), meck:expect(hpr_metrics, observe_grpc_connection, fun(_, _) -> ok end), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), meck:new(hpr_gateway_location, [passthrough]), meck:expect(hpr_gateway_location, get, fun(_) -> ok end), application:ensure_all_started(gproc), @@ -383,6 +392,8 @@ handle_info_test() -> meck:new(hpr_metrics, [passthrough]), meck:expect(hpr_metrics, packet_down, fun(_) -> ok end), meck:expect(hpr_metrics, observe_multi_buy, fun(_, _) -> ok end), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), ?assertEqual(stream_state, ?MODULE:handle_info({packet_down, PacketDown}, stream_state)), ?assertEqual(stream_state, ?MODULE:handle_info(msg, stream_state)), @@ -398,6 +409,8 @@ send_packet_down_test() -> meck:new(hpr_metrics, [passthrough]), meck:expect(hpr_metrics, packet_down, fun(_) -> ok end), meck:expect(hpr_metrics, observe_multi_buy, fun(_, _) -> ok end), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), #{public := PubKey0} = libp2p_crypto:generate_keys(ed25519), PubKeyBin0 = libp2p_crypto:pubkey_to_bin(PubKey0), diff --git a/src/hpr_routing.erl b/src/hpr_routing.erl index 7be1d00..a70f25c 100644 --- a/src/hpr_routing.erl +++ b/src/hpr_routing.erl @@ -507,11 +507,15 @@ foreach_setup() -> application:set_env(hpr, multi_buy_enabled, true), meck:new(hpr_gateway_location, [passthrough]), meck:expect(hpr_gateway_location, get, fun(_) -> {error, not_implemented} end), + meck:new(hpr_metrics, [passthrough]), + meck:expect(hpr_metrics, devaddr_cache_hit, fun() -> ok end), + meck:expect(hpr_metrics, devaddr_cache_miss, fun() -> ok end), ok. foreach_cleanup(ok) -> true = ets:delete(hpr_multi_buy_ets), true = ets:delete(hpr_route_devaddr_ranges_ets), + true = ets:delete(hpr_devaddr_cache_ets), true = ets:delete(hpr_route_eui_pairs_ets), true = ets:delete(hpr_device_stats_ets), true = ets:delete(hpr_netid_stats_ets), @@ -525,6 +529,7 @@ foreach_cleanup(ok) -> true = ets:delete(hpr_routes_ets), true = erlang:unregister(hpr_sup), meck:unload(hpr_gateway_location), + meck:unload(hpr_metrics), ok. find_routes_for_uplink_no_skf() -> @@ -984,7 +989,6 @@ maybe_deliver_packet_to_route_multi_buy() -> meck:new(hpr_protocol_router, [passthrough]), meck:expect(hpr_protocol_router, send, fun(_, _, _, _) -> ok end), - meck:new(hpr_metrics, [passthrough]), meck:expect(hpr_metrics, observe_multi_buy, fun(_, _) -> ok end), meck:expect(hpr_metrics, packet_up_per_oui, fun(_, _) -> ok end), @@ -1039,7 +1043,6 @@ maybe_deliver_packet_to_route_multi_buy() -> ?assertEqual(2, meck:num_calls(hpr_protocol_router, send, 4)), meck:unload(hpr_protocol_router), - meck:unload(hpr_metrics), ok. -endif. diff --git a/src/metrics/hpr_metrics.erl b/src/metrics/hpr_metrics.erl index b73999d..b6d7177 100644 --- a/src/metrics/hpr_metrics.erl +++ b/src/metrics/hpr_metrics.erl @@ -17,13 +17,21 @@ observe_find_routes/1, observe_grpc_connection/2, ics_update/2, - observe_gateway_location/2 + observe_gateway_location/2, + devaddr_cache_hit/0, + devaddr_cache_miss/0 ]). -export([ counts/0 ]). +-ifdef(TEST). +-export([ + record_routes/0 +]). +-endif. + %% ------------------------------------------------------------------ %% gen_server Function Exports %% ------------------------------------------------------------------ @@ -142,6 +150,16 @@ observe_gateway_location(Start, Status) -> erlang:system_time(millisecond) - Start ). +-spec devaddr_cache_hit() -> ok. +devaddr_cache_hit() -> + _ = prometheus_counter:inc(?METRICS_DEVADDR_CACHE_HIT_COUNTER, []), + ok. + +-spec devaddr_cache_miss() -> ok. +devaddr_cache_miss() -> + _ = prometheus_counter:inc(?METRICS_DEVADDR_CACHE_MISS_COUNTER, []), + ok. + %% ------------------------------------------------------------------ %% CLI Function Definitions %% ------------------------------------------------------------------ @@ -262,7 +280,12 @@ record_routes() -> RouteID = hpr_route:id(Route), OUI = hpr_route:oui(Route), NewBrokenMap = - case SKFCount > 0 andalso not sets:is_element(RouteID, RouteIDsWithDevAddr) of + case + SKFCount > 0 andalso + not sets:is_element(RouteID, RouteIDsWithDevAddr) andalso + hpr_route:active(Route) andalso + not hpr_route:locked(Route) + of true -> lager:warning( [{route_id, RouteID}, {oui, OUI}], diff --git a/test/hpr_cli_config_SUITE.erl b/test/hpr_cli_config_SUITE.erl index 3f78341..ff94294 100644 --- a/test/hpr_cli_config_SUITE.erl +++ b/test/hpr_cli_config_SUITE.erl @@ -12,7 +12,11 @@ sync_new_route_test/1, sync_remove_route_test/1, sync_new_remove_route_test/1, - sync_oui_only_test/1 + sync_oui_only_test/1, + route_activate_test/1, + route_deactivate_test/1, + oui_activate_test/1, + oui_deactivate_test/1 ]). %%-------------------------------------------------------------------- @@ -30,7 +34,11 @@ all() -> sync_new_route_test, sync_remove_route_test, sync_new_remove_route_test, - sync_oui_only_test + sync_oui_only_test, + route_activate_test, + route_deactivate_test, + oui_activate_test, + oui_deactivate_test ]. %%-------------------------------------------------------------------- @@ -260,6 +268,308 @@ sync_remove_route_test(_Config) -> ok. +route_deactivate_test(_Config) -> + %% Setup: Create a route with SKFs, EUIs, and DevAddr ranges + #{ + route_id := RouteID, + route := Route, + eui_pair := EUIPair, + devaddr := DevAddr, + devaddr_range := DevAddrRange, + skf := SessionKeyFilter + } = test_data("route-deactivate-test-001"), + + %% Insert route and all its data + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {route, Route}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {eui_pair, EUIPair}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {devaddr_range, DevAddrRange}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {skf, SessionKeyFilter}}) + ), + + %% Verify data exists + ok = check_config_counts(RouteID, 1, 1, 1, 1), + {ok, RouteETSBefore} = hpr_route_storage:lookup(RouteID), + ?assert(hpr_route:active(hpr_route_ets:route(RouteETSBefore))), + + %% Deactivate the route + [{text, [_Text]}] = hpr_cli_config:config_route_deactivate( + ["config", "route", "deactivate", RouteID], + [], + [] + ), + + %% Verify route is deactivated and data is removed + {ok, RouteETSAfter} = hpr_route_storage:lookup(RouteID), + RouteAfter = hpr_route_ets:route(RouteETSAfter), + ?assertNot(hpr_route:active(RouteAfter)), + + %% Verify all EUIs and DevAddr ranges are removed + %% Note: We can't check SKFs directly as the ETS table is deleted + ?assertEqual([], hpr_devaddr_range_storage:lookup(DevAddr)), + ?assertEqual([], hpr_eui_pair_storage:lookup(1, 0)), + + ok. + +route_activate_test(_Config) -> + %% Setup: Create a deactivated route + RouteID = "route-activate-test-001", + Route = hpr_route:test_new(#{ + id => RouteID, + net_id => 0, + oui => 1, + server => #{ + host => "localhost", + port => 8080, + protocol => {packet_router, #{}} + }, + max_copies => 10, + active => false + }), + + %% Insert deactivated route + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {route, Route}}) + ), + + %% Verify route exists and is deactivated + ok = test_utils:wait_until(fun() -> + case hpr_route_storage:lookup(RouteID) of + {ok, RouteETS} -> + not hpr_route:active(hpr_route_ets:route(RouteETS)); + _ -> + false + end + end), + + %% Setup test data to be returned by refresh + EUIPair = hpr_eui_pair:test_new(#{route_id => RouteID, app_eui => 1, dev_eui => 0}), + DevAddrRange = hpr_devaddr_range:test_new(#{ + route_id => RouteID, start_addr => 16#00000001, end_addr => 16#0000000A + }), + DevAddr = 16#00000001, + SessionKey = hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + SessionKeyFilter = hpr_skf:new(#{ + route_id => RouteID, devaddr => DevAddr, session_key => SessionKey, max_copies => 1 + }), + + application:set_env(hpr, test_route_get_euis, [EUIPair]), + application:set_env(hpr, test_route_get_devaddr_ranges, [DevAddrRange]), + application:set_env(hpr, test_route_list_skfs, [SessionKeyFilter]), + + %% Activate the route + [{text, [_Text]}] = hpr_cli_config:config_route_activate( + ["config", "route", "activate", RouteID], + [], + [] + ), + + %% Verify route is activated + ok = test_utils:wait_until(fun() -> + case hpr_route_storage:lookup(RouteID) of + {ok, RouteETS} -> + hpr_route:active(hpr_route_ets:route(RouteETS)); + _ -> + false + end + end), + + %% Verify data was refreshed (SKFs, EUIs, DevAddr ranges added) + ok = check_config_counts(RouteID, 1, 1, 1, 1), + + ok. + +oui_deactivate_test(_Config) -> + %% Setup: Create multiple routes for the same OUI + OUI = 100, + #{ + route_id := RouteID1, + route := Route1, + eui_pair := EUIPair1, + devaddr_range := DevAddrRange1, + skf := SKF1 + } = test_data("oui-deactivate-test-001", OUI), + + #{ + route_id := RouteID2, + route := Route2, + eui_pair := EUIPair2, + devaddr_range := DevAddrRange2, + skf := SKF2 + } = test_data("oui-deactivate-test-002", OUI), + + %% Insert both routes and their data + lists:foreach( + fun({Route, EUIPair, DevAddrRange, SKF}) -> + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {route, Route}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {eui_pair, EUIPair}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{ + action => add, data => {devaddr_range, DevAddrRange} + }) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {skf, SKF}}) + ) + end, + [{Route1, EUIPair1, DevAddrRange1, SKF1}, {Route2, EUIPair2, DevAddrRange2, SKF2}] + ), + + %% Verify both routes exist with data + ok = check_config_counts(RouteID1, 2, 2, 2, 1), + ok = check_config_counts(RouteID2, 2, 2, 2, 1), + + %% Deactivate all routes for the OUI + [{text, [_Text]}] = hpr_cli_config:config_oui_deactivate( + ["config", "oui", "deactivate", integer_to_list(OUI)], + [], + [] + ), + + %% Verify both routes are deactivated and data is removed + ok = test_utils:wait_until(fun() -> + case {hpr_route_storage:lookup(RouteID1), hpr_route_storage:lookup(RouteID2)} of + {{ok, ETS1}, {ok, ETS2}} -> + Route1After = hpr_route_ets:route(ETS1), + Route2After = hpr_route_ets:route(ETS2), + not hpr_route:active(Route1After) andalso not hpr_route:active(Route2After); + _ -> + false + end + end), + + %% Verify all EUIs and DevAddr ranges are removed + %% Note: We can't check SKFs directly as the ETS tables are deleted + ?assertEqual(0, ets:info(hpr_route_eui_pairs_ets, size)), + ?assertEqual(0, ets:info(hpr_route_devaddr_ranges_ets, size)), + + ok. + +oui_activate_test(_Config) -> + %% Setup: Create multiple deactivated routes for the same OUI + OUI = 200, + RouteID1 = "oui-activate-test-001", + Route1 = hpr_route:test_new(#{ + id => RouteID1, + net_id => 0, + oui => OUI, + server => #{ + host => "localhost", + port => 8080, + protocol => {packet_router, #{}} + }, + max_copies => 10, + active => false + }), + + RouteID2 = "oui-activate-test-002", + Route2 = hpr_route:test_new(#{ + id => RouteID2, + net_id => 0, + oui => OUI, + server => #{ + host => "localhost", + port => 8081, + protocol => {packet_router, #{}} + }, + max_copies => 10, + active => false + }), + + %% Insert both deactivated routes + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {route, Route1}}) + ), + ok = hpr_test_ics_route_service:stream_resp( + hpr_route_stream_res:test_new(#{action => add, data => {route, Route2}}) + ), + + %% Verify routes exist and are deactivated + ok = test_utils:wait_until(fun() -> + case {hpr_route_storage:lookup(RouteID1), hpr_route_storage:lookup(RouteID2)} of + {{ok, ETS1}, {ok, ETS2}} -> + Route1Check = hpr_route_ets:route(ETS1), + Route2Check = hpr_route_ets:route(ETS2), + not hpr_route:active(Route1Check) andalso not hpr_route:active(Route2Check); + _ -> + false + end + end), + + %% Setup test data to be returned by refresh for both routes + EUIPair1 = hpr_eui_pair:test_new(#{route_id => RouteID1, app_eui => 1, dev_eui => 0}), + DevAddrRange1 = hpr_devaddr_range:test_new(#{ + route_id => RouteID1, start_addr => 16#00000001, end_addr => 16#0000000A + }), + SKF1 = hpr_skf:new(#{ + route_id => RouteID1, + devaddr => 16#00000001, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + + EUIPair2 = hpr_eui_pair:test_new(#{route_id => RouteID2, app_eui => 2, dev_eui => 1}), + DevAddrRange2 = hpr_devaddr_range:test_new(#{ + route_id => RouteID2, start_addr => 16#0000000B, end_addr => 16#00000014 + }), + SKF2 = hpr_skf:new(#{ + route_id => RouteID2, + devaddr => 16#0000000B, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + + application:set_env(hpr, test_route_get_euis, [EUIPair1, EUIPair2]), + application:set_env(hpr, test_route_get_devaddr_ranges, [DevAddrRange1, DevAddrRange2]), + application:set_env(hpr, test_route_list_skfs, [SKF1, SKF2]), + + %% Activate all routes for the OUI + [{text, [_Text]}] = hpr_cli_config:config_oui_activate( + ["config", "oui", "activate", integer_to_list(OUI)], + [], + [] + ), + + %% Verify both routes are activated + ok = test_utils:wait_until(fun() -> + case {hpr_route_storage:lookup(RouteID1), hpr_route_storage:lookup(RouteID2)} of + {{ok, ETS1}, {ok, ETS2}} -> + Route1After = hpr_route_ets:route(ETS1), + Route2After = hpr_route_ets:route(ETS2), + hpr_route:active(Route1After) andalso hpr_route:active(Route2After); + _ -> + false + end + end), + + %% Verify data was refreshed for both routes + %% Note: Both routes will get all the test data since we set it globally + ok = test_utils:wait_until(fun() -> + case {hpr_route_storage:lookup(RouteID1), hpr_route_storage:lookup(RouteID2)} of + {{ok, ETS1}, {ok, ETS2}} -> + %% Just verify that some data was added + SKFCount1 = ets:info(hpr_route_ets:skf_ets(ETS1), size), + SKFCount2 = ets:info(hpr_route_ets:skf_ets(ETS2), size), + EUICount = ets:info(hpr_route_eui_pairs_ets, size), + DevAddrCount = ets:info(hpr_route_devaddr_ranges_ets, size), + SKFCount1 > 0 andalso SKFCount2 > 0 andalso EUICount > 0 andalso DevAddrCount > 0; + _ -> + false + end + end), + + ok. + %% =================================================================== %% Helpers %% =================================================================== diff --git a/test/hpr_metrics_SUITE.erl b/test/hpr_metrics_SUITE.erl index 91bc78a..e6bba1e 100644 --- a/test/hpr_metrics_SUITE.erl +++ b/test/hpr_metrics_SUITE.erl @@ -8,7 +8,8 @@ -export([ elli_started_test/1, - main_test/1 + main_test/1, + record_routes_test/1 ]). -include_lib("common_test/include/ct.hrl"). @@ -29,7 +30,8 @@ all() -> [ elli_started_test, - main_test + main_test, + record_routes_test ]. %%-------------------------------------------------------------------- @@ -197,3 +199,219 @@ main_test(_Config) -> ), ok. + +record_routes_test(_Config) -> + %% Setup Route 1: Normal route with DevAddr range and SKFs + RouteID1 = "RouteID1", + OUI1 = 1001, + Route1 = hpr_route:test_new(#{ + id => RouteID1, + net_id => 0, + oui => OUI1, + server => #{ + host => "127.0.0.1", + port => 8080, + protocol => {packet_router, #{}} + }, + max_copies => 1 + }), + + %% Setup Route 2: Broken route with SKFs but no DevAddr range (active, not locked) + RouteID2 = "RouteID2", + OUI2 = 2002, + Route2 = hpr_route:test_new(#{ + id => RouteID2, + net_id => 0, + oui => OUI2, + server => #{ + host => "127.0.0.1", + port => 8081, + protocol => {packet_router, #{}} + }, + max_copies => 1 + }), + + %% Setup Route 3: Route with DevAddr range but no SKFs + RouteID3 = "RouteID3", + OUI3 = 3003, + Route3 = hpr_route:test_new(#{ + id => RouteID3, + net_id => 0, + oui => OUI3, + server => #{ + host => "127.0.0.1", + port => 8082, + protocol => {packet_router, #{}} + }, + max_copies => 1 + }), + + %% Setup Route 4: Has SKFs but no DevAddr, but is NOT active (should not be broken) + RouteID4 = "RouteID4", + OUI4 = 4004, + Route4 = hpr_route:test_new(#{ + id => RouteID4, + net_id => 0, + oui => OUI4, + server => #{ + host => "127.0.0.1", + port => 8083, + protocol => {packet_router, #{}} + }, + max_copies => 1, + active => false + }), + + %% Setup Route 5: Has SKFs but no DevAddr, but IS locked (should not be broken) + RouteID5 = "RouteID5", + OUI5 = 5005, + Route5 = hpr_route:test_new(#{ + id => RouteID5, + net_id => 0, + oui => OUI5, + server => #{ + host => "127.0.0.1", + port => 8084, + protocol => {packet_router, #{}} + }, + max_copies => 1, + locked => true + }), + + %% Add routes to storage + ok = hpr_route_storage:insert(Route1), + ok = hpr_route_storage:insert(Route2), + ok = hpr_route_storage:insert(Route3), + ok = hpr_route_storage:insert(Route4), + ok = hpr_route_storage:insert(Route5), + + %% Wait for all routes to be inserted + ok = test_utils:wait_until(fun() -> + 5 =:= ets:info(hpr_routes_ets, size) + end), + + %% Add DevAddr ranges for Route 1 and Route 3 + DevAddrRange1 = hpr_devaddr_range:test_new(#{ + route_id => RouteID1, + start_addr => 16#00000000, + end_addr => 16#000000FF + }), + DevAddrRange3 = hpr_devaddr_range:test_new(#{ + route_id => RouteID3, + start_addr => 16#00000100, + end_addr => 16#000001FF + }), + ok = hpr_devaddr_range_storage:insert(DevAddrRange1), + ok = hpr_devaddr_range_storage:insert(DevAddrRange3), + + %% Add SKFs to Route 1 and Route 2 + %% Route 1 will have 2 SKFs and has DevAddr range (not broken) + SKF1_1 = hpr_skf:new(#{ + route_id => RouteID1, + devaddr => 16#00000001, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + SKF1_2 = hpr_skf:new(#{ + route_id => RouteID1, + devaddr => 16#00000002, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + ok = hpr_skf_storage:insert(SKF1_1), + ok = hpr_skf_storage:insert(SKF1_2), + + %% Route 2 will have 3 SKFs but no DevAddr range (broken because active and not locked) + SKF2_1 = hpr_skf:new(#{ + route_id => RouteID2, + devaddr => 16#00000010, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + SKF2_2 = hpr_skf:new(#{ + route_id => RouteID2, + devaddr => 16#00000011, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + SKF2_3 = hpr_skf:new(#{ + route_id => RouteID2, + devaddr => 16#00000012, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + ok = hpr_skf_storage:insert(SKF2_1), + ok = hpr_skf_storage:insert(SKF2_2), + ok = hpr_skf_storage:insert(SKF2_3), + + %% Route 4 will have 2 SKFs but no DevAddr range (NOT broken because inactive) + SKF4_1 = hpr_skf:new(#{ + route_id => RouteID4, + devaddr => 16#00000020, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + SKF4_2 = hpr_skf:new(#{ + route_id => RouteID4, + devaddr => 16#00000021, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + ok = hpr_skf_storage:insert(SKF4_1), + ok = hpr_skf_storage:insert(SKF4_2), + + %% Route 5 will have 1 SKF but no DevAddr range (NOT broken because locked) + SKF5_1 = hpr_skf:new(#{ + route_id => RouteID5, + devaddr => 16#00000030, + session_key => hpr_utils:bin_to_hex_string(crypto:strong_rand_bytes(16)), + max_copies => 1 + }), + ok = hpr_skf_storage:insert(SKF5_1), + + %% Call record_routes + ok = hpr_metrics:record_routes(), + + %% Verify metrics + %% Should have 5 routes total + ?assertEqual( + 5, + prometheus_gauge:value(?METRICS_ROUTES_GAUGE) + ), + + %% Should have 8 SKFs total (2 from Route1 + 3 from Route2 + 0 from Route3 + 2 from Route4 + 1 from Route5) + ?assertEqual( + 8, + prometheus_gauge:value(?METRICS_SKFS_GAUGE) + ), + + %% Should have 1 broken route (Route2) for OUI2 + %% Route2 is broken because it has SKFs, no DevAddr range, is active, and is not locked + ?assertEqual( + 1, + prometheus_gauge:value(?METRICS_BROKEN_ROUTES_GAUGE, [OUI2]) + ), + + %% OUI1 and OUI3 should not have broken routes (they have DevAddr ranges) + ?assertEqual( + undefined, + prometheus_gauge:value(?METRICS_BROKEN_ROUTES_GAUGE, [OUI1]) + ), + ?assertEqual( + undefined, + prometheus_gauge:value(?METRICS_BROKEN_ROUTES_GAUGE, [OUI3]) + ), + + %% OUI4 should not have broken routes (route is inactive) + ?assertEqual( + undefined, + prometheus_gauge:value(?METRICS_BROKEN_ROUTES_GAUGE, [OUI4]) + ), + + %% OUI5 should not have broken routes (route is locked) + ?assertEqual( + undefined, + prometheus_gauge:value(?METRICS_BROKEN_ROUTES_GAUGE, [OUI5]) + ), + + ok.