From b5c9a977b1a6792c09018f30c059bb08bc580e5e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 22 Aug 2026 02:46:38 -0400 Subject: [PATCH 01/10] feat(cache): honor result max age --- src/core/resolver/hb_cache_control.erl | 79 +++++++++++++++++++++----- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index ed22e208a..57c718b69 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -37,6 +37,8 @@ maybe_store(Base, Req, Res, Opts) -> %% a 504 `Status'. %% `no_cache': If set, the cached values are never used. Returns %% `continue' to the caller. +%% `max-age': If set, cached results must have a sufficiently +%% recent `created-at' timestamp. maybe_lookup(Base, Req, Opts) -> case exec_likely_faster_heuristic(Base, Req, Opts) of true -> @@ -60,27 +62,51 @@ lookup(Base, Req, Opts) -> {hit, not_found} -> {error, not_found}; {hit, {ok, Res}} -> - ?event(caching, - {cache_hit, - {base, Base}, - {req, Req}, - {res, Res} - } - ), - {ok, Res}; + case fresh(Res, Req, Opts) of + true -> + ?event(caching, + {cache_hit, + {base, Base}, + {req, Req}, + {res, Res} + } + ), + {ok, cached_result(Res)}; + false -> + ?event(caching, {stale_cache_result, Base, Req}), + cache_miss(Base, Req, Settings, Opts) + end; _ -> ?event(caching, {result_cache_miss, Base, Req}), - case Settings of - #{ <<"only-if-cached">> := true } -> - only_if_cached_not_found_error(Base, Req, Opts); - _ -> - maybe_load_base(Base, Req, Opts) - end + cache_miss(Base, Req, Settings, Opts) end end. %%% Internal functions +%% @doc Return whether a cached result satisfies the requested `max-age'. +fresh(Res, Req, Opts) -> + case hb_maps:get(<<"max-age">>, Req, infinity, Opts) of + infinity -> true; + <<"infinity">> -> true; + RawMaxAge -> + try + CreatedAt = hb_maps:get(<<"created-at">>, Res, undefined, Opts), + os:system_time(second) =< + hb_util:int(CreatedAt) + max(0, hb_util:int(RawMaxAge)) + catch + _:_ -> false + end + end. + +cache_miss(Base, Req, #{ <<"only-if-cached">> := true }, Opts) -> + only_if_cached_not_found_error(Base, Req, Opts); +cache_miss(Base, Req, _Settings, Opts) -> + maybe_load_base(Base, Req, Opts). + +cached_result(Res) when is_map(Res) -> maps:remove(<<"created-at">>, Res); +cached_result(Res) -> Res. + %% @doc Load an ID base required to execute the request. maybe_load_base(Base, Req, _Opts) when not ?IS_ID(Base) -> {continue, Base, Req}; @@ -140,7 +166,10 @@ perform_cache_write(Base, Req, Res, Opts) -> Opts ); Map when is_map(Map) -> - hb_cache:write(Res, Opts); + hb_cache:write( + Map#{ <<"created-at">> => os:system_time(second) }, + Opts + ); _ -> ?event({cannot_write_result, Res}), skip_caching @@ -422,3 +451,23 @@ cache_message_result_test() -> ?event({res2, Res2}), ?event({res3, Res3}), ?assertEqual(Res2, Res3). + +cache_result_max_age_test() -> + Opts = #{ + <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] + }, + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 1 }, + Result = #{ <<"value">> => <<"cached">> }, + {ok, ResultPath} = perform_cache_write(Base, Req, Result, Opts), + {ok, Cached} = hb_cache:read(ResultPath, Opts), + CreatedAt = hb_maps:get(<<"created-at">>, Cached, Opts), + ?assert(is_integer(CreatedAt)), + hb_cache:link(ResultPath, hb_path:hashpath(Base, Req, Opts), Opts), + {ok, Served} = hb_ao:resolve(Base, Req, Opts), + ?assertEqual(not_found, hb_maps:get(<<"created-at">>, Served, not_found, Opts)), + timer:sleep(2100), + {ok, Refreshed} = hb_ao:resolve(Base, Req, Opts), + ?assertEqual(not_found, hb_maps:get(<<"created-at">>, Refreshed, not_found, Opts)), + ?assertNotEqual(Served, Refreshed). From 5bba11708127a53c3db8bbb52a26b07cf2dabb2c Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 22 Aug 2026 15:56:50 -0400 Subject: [PATCH 02/10] test(lua): demonstrate cached top-holder query --- scripts/lua-top-holders.lua | 111 +++++++++++++++++++++++++++++++++++ src/preloaded/vm/dev_lua.erl | 63 ++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 scripts/lua-top-holders.lua diff --git a/scripts/lua-top-holders.lua b/scripts/lua-top-holders.lua new file mode 100644 index 000000000..d1bc42c82 --- /dev/null +++ b/scripts/lua-top-holders.lua @@ -0,0 +1,111 @@ +local function decimal(value) + local result = tostring(value):match("^0*(%d+)$") + return result or "0" +end + +local function better(left, right) + if #left.balance ~= #right.balance then + return #left.balance > #right.balance + end + if left.balance ~= right.balance then + return left.balance > right.balance + end + return left.address < right.address +end + +local function weaker(left, right) + return better(right, left) +end + +local function push(heap, holder, limit) + if #heap < limit then + heap[#heap + 1] = holder + local index = #heap + while index > 1 do + local parent = math.floor(index / 2) + if not weaker(heap[index], heap[parent]) then break end + heap[index], heap[parent] = heap[parent], heap[index] + index = parent + end + return + end + if not better(holder, heap[1]) then return end + heap[1] = holder + local index = 1 + while true do + local child = index * 2 + if child > #heap then break end + if child < #heap and weaker(heap[child + 1], heap[child]) then + child = child + 1 + end + if not weaker(heap[child], heap[index]) then break end + heap[index], heap[child] = heap[child], heap[index] + index = child + end +end + +local reserved = { + ["ao-types"] = true, + ["commitments"] = true, + ["device"] = true, + ["hashpath"] = true, + ["priv"] = true, +} + +local function walk(node, prefix, heap, limit) + if node["node-value"] ~= nil then + push(heap, { address = prefix, balance = decimal(node["node-value"]) }, limit) + end + for edge, child in pairs(node) do + if not reserved[edge] and edge ~= "node-value" then + local address = prefix .. edge + if type(child) == "table" then + walk(child, address, heap, limit) + else + push(heap, { address = address, balance = decimal(child) }, limit) + end + end + end +end + +local escapes = { + ['"'] = '\\"', + ['\\'] = '\\\\', + ['\b'] = '\\b', + ['\f'] = '\\f', + ['\n'] = '\\n', + ['\r'] = '\\r', + ['\t'] = '\\t', +} + +local function quote(value) + return '"' .. tostring(value):gsub('[%z\1-\31\\"]', function(char) + return escapes[char] or string.format('\\u%04x', string.byte(char)) + end) .. '"' +end + +function top(_, req) + local process = req["process-id"] + local limit = math.floor(tonumber(req.top) or 100) + if not process or limit < 1 then + return "error", { status = 400, body = "process-id and a positive top are required" } + end + + local status, balances = ao.resolve({ + path = "/" .. process .. "~process@1.0/compute/balances" + }) + if status ~= "ok" then return status, balances end + local heap = {} + walk(balances, "", heap, limit) + table.sort(heap, better) + local rows = {} + for index, holder in ipairs(heap) do + rows[index] = '{"address":' .. quote(holder.address) + .. ',"balance":' .. quote(holder.balance) .. '}' + end + return { + status = 200, + ["content-type"] = "application/json", + body = '[' .. table.concat(rows, ',') .. ']' + } +end diff --git a/src/preloaded/vm/dev_lua.erl b/src/preloaded/vm/dev_lua.erl index ca8baf299..0e8fc08b2 100644 --- a/src/preloaded/vm/dev_lua.erl +++ b/src/preloaded/vm/dev_lua.erl @@ -888,6 +888,69 @@ invoke_non_compute_key_test() -> ?event({result2, Result2}), ?assertEqual(<<"Alice">>, hb_ao:get(<<"hello">>, Result2, #{})). +top_holders_cache_test() -> + Store = hb_test_utils:test_store(), + Wallet = ar_wallet:new(), + Opts = #{ + <<"store">> => [Store], + <<"priv-wallet">> => Wallet, + <<"cache-control">> => [<<"always">>] + }, + Balances = #{ <<"alice">> => <<"1">>, <<"bob">> => <<"3">> }, + Trie = maps:fold( + fun(Address, Balance, Acc) -> + {ok, Updated} = hb_ao:resolve( + Acc, + #{ <<"path">> => <<"set">>, Address => Balance }, + Opts + ), + Updated + end, + #{ <<"device">> => <<"trie@1.0">> }, + Balances + ), + Process = hb_message:commit( + #{ + <<"device">> => <<"process@1.0">>, + <<"scheduler-device">> => <<"scheduler@1.0">>, + <<"scheduler-location">> => + hb_util:human_id(ar_wallet:to_address(Wallet)), + <<"execution-device">> => <<"message@1.0">>, + <<"type">> => <<"Process">> + }, + Opts + ), + ProcessID = hb_util:human_id(hb_message:id(Process, all, Opts)), + {ok, TrieID} = hb_cache:write(Trie, Opts), + {ok, StateID} = hb_cache:write( + (hb_message:uncommitted(Process, Opts))#{ + <<"at-slot">> => 0, + <<"balances">> => {link, TrieID, #{ <<"type">> => <<"link">> }}, + <<"process">> => Process + }, + Opts + ), + ok = hb_cache:link( + StateID, + <<"computed/", ProcessID/binary, "/slot/0">>, + Opts + ), + {ok, Script} = file:read_file("scripts/lua-top-holders.lua"), + Function = #{ + <<"device">> => <<"lua@5.3a">>, + <<"content-type">> => <<"application/lua">>, + <<"body">> => Script + }, + Req = #{ + <<"path">> => <<"top">>, + <<"top">> => 2, + <<"process-id">> => ProcessID, + <<"max-age">> => 60 + }, + {ok, Result} = hb_ao:resolve(Function, Req, Opts), + Holders = hb_json:decode(hb_maps:get(<<"body">>, Result, Opts)), + ?assertEqual(<<"bob">>, hb_maps:get(<<"address">>, hd(Holders), Opts)). + %% @doc Use a Lua module as a hook on the HTTP server via `~meta@1.0'. lua_http_hook_test() -> {ok, Module} = file:read_file("test/test.lua"), From b6c774384ae5145a0efe338570131a67733e2d2f Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Thu, 27 Aug 2026 11:13:06 -0400 Subject: [PATCH 03/10] feat: implement priv created at --- src/core/resolver/hb_cache.erl | 219 +++++++++++++++- src/core/resolver/hb_cache_control.erl | 329 +++++++++++++++++++++++-- src/preloaded/vm/dev_lua.erl | 50 +++- 3 files changed, 565 insertions(+), 33 deletions(-) diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index 59e972a23..a12ac5c93 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -40,7 +40,9 @@ -module(hb_cache). -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). --export([read/2, read_resolved/3, write/2, write_binary/3, write_hashpath/2, link/3]). +-export([read/2, read_resolved/3, resolved_address/3]). +-export([write/2, write_binary/3, write_hashpath/2, write_hashpath/3, write_resolved/4]). +-export([link/3]). -export([match/2, list/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). -include("include/hb.hrl"). @@ -266,9 +268,21 @@ generate_binary_path(Bin, Opts) -> %% the commitments of the inner messages. We do not, however, store the IDs from %% commitments on signed _inner_ messages. We may wish to revisit this. write(RawMsg, Opts) when is_map(RawMsg) -> - hb_message:paranoid_verify(cache_write, RawMsg, Opts), - {ok, Msg} = hb_message:with_only_committed(RawMsg, Opts), - TABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), + % Reattach only cache-owned freshness after private keys are filtered. + PrivCreatedAt = + case maps:find(<<"priv-created-at">>, RawMsg) of + {ok, CreatedAt} + when is_binary(CreatedAt), + byte_size(CreatedAt) < ?DIRECT_VALUE_LENGTH -> + #{ <<"priv-created-at">> => CreatedAt }; + _ -> + #{} + end, + PublicRawMsg = maps:remove(<<"priv-created-at">>, RawMsg), + hb_message:paranoid_verify(cache_write, PublicRawMsg, Opts), + {ok, Msg} = hb_message:with_only_committed(PublicRawMsg, Opts), + PublicTABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), + TABM = maps:merge(PublicTABM, PrivCreatedAt), ?event_debug(debug_cache, {writing_full_message, {msg, TABM}}), try do_write_message( @@ -666,6 +680,17 @@ write_binary(Hashpath, Bin, Store, Opts) -> hb_store:link(Store, #{ hb_path:to_binary(Hashpath) => Path }, Opts), {ok, Path}. +%% @doc Write a resolved message with its private cache creation timestamp. +write_resolved(CacheAddress, Msg, CreatedAt, Opts) -> + write_hashpath( + CacheAddress, + Msg#{ + <<"priv-created-at">> => + integer_to_binary(hb_util:int(CreatedAt)) + }, + Opts + ). + %% @doc Read the message at a path. Returns in `structured@1.0' format: Either %% a richly typed map or a direct binary. If `cache-read-mode' is `raw', %% composite reads return lazy links without decoding `ao-types'. @@ -1096,14 +1121,29 @@ read_in_memory_key(BaseMsg, NormKey, _Opts) -> %% @doc Read the output of a prior computation, given BaseMsg and Req. read_hashpath(BaseMsgID, ReqID, Opts) when ?IS_ID(BaseMsgID) and ?IS_ID(ReqID) -> ?event_debug({cache_lookup, {base, BaseMsgID}, {req, ReqID}, {opts, Opts}}), - hashpath_read_result(read(<>, Opts)); + hashpath_read_result(read(resolved_address(BaseMsgID, ReqID, Opts), Opts)); read_hashpath(BaseMsgID, Req, Opts) when ?IS_ID(BaseMsgID) and is_map(Req) -> - ReqID = hb_message:id(Req, all, Opts), - hashpath_read_result(read(<>, Opts)); + hashpath_read_result(read(resolved_address(BaseMsgID, Req, Opts), Opts)); read_hashpath(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> - hashpath_read_result(read(hb_path:hashpath(BaseMsg, Req, Opts), Opts)); + hashpath_read_result(read(resolved_address(BaseMsg, Req, Opts), Opts)); read_hashpath(_, _, _) -> miss. +%% @doc Return the cache address for a resolved Base and request. `max-age' +%% controls cache reuse, but does not identify the computation itself. +resolved_address(BaseMsgID, ReqID, _Opts) + when ?IS_ID(BaseMsgID) and ?IS_ID(ReqID) -> + <>; +resolved_address(BaseMsgID, Req, Opts) + when ?IS_ID(BaseMsgID) and is_map(Req) -> + ReqID = hb_message:id(cache_request(Req, Opts), all, Opts), + <>; +resolved_address(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> + hb_path:hashpath(BaseMsg, cache_request(Req, Opts), Opts). + +%% @doc Remove cache policy fields that do not identify a computation. +cache_request(Req, Opts) -> + hb_maps:without([<<"max-age">>], Req, Opts). + hashpath_read_result({ok, Msg}) -> {hit, {ok, Msg}}; hashpath_read_result({error, not_found}) -> miss; hashpath_read_result(Other) -> {hit, Other}. @@ -1434,6 +1474,169 @@ test_immediate_marker_values(Store) -> ok end. +%% @doc Resolved freshness is stored on the ordinary canonical message record. +resolved_entry_canonical_record_test() -> + Opts = #{ + <<"store">> => [ + hb_test_utils:test_store(hb_store_lmdb, <<"resolved-canonical">>) + ], + <<"priv-wallet">> => ar_wallet:new() + }, + Address1 = <<"resolved/canonical-1">>, + Address2 = <<"resolved/canonical-2">>, + Public = hb_message:commit( + #{ + <<"body">> => <<"same-result">>, + <<"created-at">> => <<"application-value">> + }, + Opts + ), + Msg = Public#{ <<"priv-other">> => <<"must-not-be-stored">> }, + ResultID = hb_message:id(Public, none, Opts), + ?assertEqual({ok, ResultID}, write_resolved(Address1, Msg, 10, Opts)), + {ok, Entry1} = read(Address1, Opts), + ?assertEqual(<<"same-result">>, hb_maps:get(<<"body">>, Entry1, Opts)), + ?assertEqual( + <<"application-value">>, + hb_maps:get(<<"created-at">>, Entry1, Opts) + ), + ?assertEqual( + <<"10">>, + hb_maps:get(<<"priv-created-at">>, Entry1, Opts) + ), + ?assertEqual(false, maps:is_key(<<"priv-other">>, Entry1)), + PublicEntry1 = hb_private:reset(Entry1), + Restored1 = read_all_commitments(PublicEntry1, Opts), + ?assertEqual( + maps:get(<<"commitments">>, Public), + maps:get(<<"commitments">>, Restored1) + ), + ?assertEqual( + hb_message:id(Public, none, Opts), + hb_message:id(Restored1, none, Opts) + ), + ?assertEqual( + hb_message:id(Public, all, Opts), + hb_message:id(Restored1, all, Opts) + ), + ?assert(hb_message:verify(Restored1, all, Opts)), + ?assertEqual({ok, ResultID}, write_resolved(Address2, Msg, 20, Opts)), + {ok, Updated1} = read(Address1, Opts), + {ok, Entry2} = read(Address2, Opts), + ?assertEqual( + <<"20">>, + hb_maps:get(<<"priv-created-at">>, Updated1, Opts) + ), + ?assertEqual( + <<"20">>, + hb_maps:get(<<"priv-created-at">>, Entry2, Opts) + ), + ?assertEqual( + {error, not_found}, + read(<<"created-at/", Address1/binary>>, Opts) + ), + {ok, InvalidID} = write( + #{ <<"body">> => <<"invalid-timestamp">>, <<"priv-created-at">> => 1 }, + Opts + ), + {ok, InvalidEntry} = read(InvalidID, Opts), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, InvalidEntry)), + {ok, MissingLinkID} = write( + #{ + <<"body">> => <<"missing-link-timestamp">>, + <<"priv-created-at">> => {link, <<"missing">>} + }, + Opts + ), + {ok, MissingLinkEntry} = read(MissingLinkID, Opts), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, MissingLinkEntry)), + {ok, OversizedID} = write( + #{ + <<"body">> => <<"oversized-timestamp">>, + <<"priv-created-at">> => + binary:copy(<<"1">>, ?DIRECT_VALUE_LENGTH) + }, + Opts + ), + {ok, OversizedEntry} = read(OversizedID, Opts), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, OversizedEntry)). + +%% @doc One LMDB root read returns the result and co-located freshness value. +resolved_entry_lmdb_single_root_read_test() -> + Store = hb_test_utils:test_store(hb_store_lmdb, <<"resolved-single-read">>), + Opts = #{ <<"store">> => [Store] }, + Address = <<"resolved/single-read">>, + {ok, _} = write_resolved( + Address, + #{ <<"body">> => <<"cached">> }, + 12345, + Opts + ), + Parent = self(), + Tracer = spawn(fun() -> cache_trace_forwarder(Parent) end), + erlang:trace(self(), true, [call, {tracer, Tracer}]), + erlang:trace_pattern({hb_store, read, 3}, true, []), + Entry = + try + {ok, ReadEntry} = read(Address, Opts), + ReadEntry + after + stop_cache_trace(Tracer) + end, + ?assertEqual(1, store_read_trace_count(0)), + ?assertEqual(<<"cached">>, hb_maps:get(<<"body">>, Entry, Opts)), + ?assertEqual( + <<"12345">>, + hb_maps:get(<<"priv-created-at">>, Entry, Opts) + ). + +%% @doc Disable store-read tracing and stop its forwarding process. +stop_cache_trace(Tracer) -> + erlang:trace_pattern({hb_store, read, 3}, false, []), + erlang:trace(self(), false, [call]), + TraceRef = erlang:trace_delivered(self()), + Delivered = + receive + {trace_delivered, _, TraceRef} -> true + after 1000 -> + false + end, + Tracer ! {flush, self()}, + Flushed = + receive + trace_flushed -> true + after 1000 -> + false + end, + Tracer ! stop, + case {Delivered, Flushed} of + {true, true} -> ok; + {false, _} -> error(trace_delivery_timeout); + {_, false} -> error(trace_flush_timeout) + end. + +%% @doc Forward cache trace events and provide an ordered flush barrier. +cache_trace_forwarder(Parent) -> + receive + {flush, ReplyTo} -> + ReplyTo ! trace_flushed, + cache_trace_forwarder(Parent); + stop -> + ok; + TraceEvent -> + Parent ! {trace_event, TraceEvent}, + cache_trace_forwarder(Parent) + end. + +%% @doc Count traced calls to the store read boundary. +store_read_trace_count(Count) -> + receive + {trace_event, {trace, _, call, {hb_store, read, _}}} -> + store_read_trace_count(Count + 1) + after 0 -> + Count + end. + match_store_control_test() -> Store = hb_test_utils:test_store(hb_store_volatile, <<"match-control">>), ?assertEqual([Store], match_store(#{ <<"store">> => Store })), diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 57c718b69..6e5b3f45c 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -38,7 +38,7 @@ maybe_store(Base, Req, Res, Opts) -> %% `no_cache': If set, the cached values are never used. Returns %% `continue' to the caller. %% `max-age': If set, cached results must have a sufficiently -%% recent `created-at' timestamp. +%% recent `priv-created-at' timestamp. maybe_lookup(Base, Req, Opts) -> case exec_likely_faster_heuristic(Base, Req, Opts) of true -> @@ -64,14 +64,15 @@ lookup(Base, Req, Opts) -> {hit, {ok, Res}} -> case fresh(Res, Req, Opts) of true -> + CachedRes = cached_result(Res, OutputScopedOpts), ?event(caching, {cache_hit, {base, Base}, {req, Req}, - {res, Res} + {res, CachedRes} } ), - {ok, cached_result(Res)}; + {ok, CachedRes}; false -> ?event(caching, {stale_cache_result, Base, Req}), cache_miss(Base, Req, Settings, Opts) @@ -91,9 +92,15 @@ fresh(Res, Req, Opts) -> <<"infinity">> -> true; RawMaxAge -> try - CreatedAt = hb_maps:get(<<"created-at">>, Res, undefined, Opts), - os:system_time(second) =< - hb_util:int(CreatedAt) + max(0, hb_util:int(RawMaxAge)) + CreatedAt = hb_util:int( + hb_maps:get(<<"priv-created-at">>, Res, Opts) + ), + MaxAge = hb_util:int(RawMaxAge), + Now = os:system_time(second), + true = CreatedAt >= 0, + true = CreatedAt =< Now, + true = MaxAge >= 0, + Now - CreatedAt < MaxAge catch _:_ -> false end @@ -104,8 +111,16 @@ cache_miss(Base, Req, #{ <<"only-if-cached">> := true }, Opts) -> cache_miss(Base, Req, _Settings, Opts) -> maybe_load_base(Base, Req, Opts). -cached_result(Res) when is_map(Res) -> maps:remove(<<"created-at">>, Res); -cached_result(Res) -> Res. +%% @doc Remove private freshness and restore separately stored commitments. +cached_result(Res, Opts) when is_map(Res) -> + PublicRes = maps:remove(<<"priv-created-at">>, Res), + WithCommitments = hb_cache:read_all_commitments(PublicRes, Opts), + case maps:get(<<"commitments">>, WithCommitments, #{}) of + Commitments when map_size(Commitments) > 0 -> WithCommitments; + _ -> PublicRes + end; +cached_result(Res, _Opts) -> + Res. %% @doc Load an ID base required to execute the request. maybe_load_base(Base, Req, _Opts) when not ?IS_ID(Base) -> @@ -158,16 +173,26 @@ async_writer() -> perform_cache_write(Base, Req, Res, Opts) -> hb_cache:write(Base, Opts), hb_cache:write(Req, Opts), - case Res of - <<_/binary>> -> + TracksFreshness = hb_maps:is_key(<<"max-age">>, Req, Opts), + CacheAddress = hb_cache:resolved_address(Base, Req, Opts), + case {Res, TracksFreshness} of + {<<_/binary>>, _} -> hb_cache:write_binary( - hb_path:hashpath(Base, Req, Opts), + CacheAddress, Res, Opts ); - Map when is_map(Map) -> - hb_cache:write( - Map#{ <<"created-at">> => os:system_time(second) }, + {Map, true} when is_map(Map) -> + hb_cache:write_resolved( + CacheAddress, + Map, + os:system_time(second), + Opts + ); + {Map, false} when is_map(Map) -> + hb_cache:write_hashpath( + CacheAddress, + maps:remove(<<"priv-created-at">>, Map), Opts ); _ -> @@ -458,16 +483,274 @@ cache_result_max_age_test() -> <<"cache-control">> => [<<"always">>] }, Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, - Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 1 }, - Result = #{ <<"value">> => <<"cached">> }, - {ok, ResultPath} = perform_cache_write(Base, Req, Result, Opts), - {ok, Cached} = hb_cache:read(ResultPath, Opts), - CreatedAt = hb_maps:get(<<"created-at">>, Cached, Opts), + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60 }, + Result = #{ + <<"value">> => <<"cached">>, + <<"priv-created-at">> => + integer_to_binary(os:system_time(second) + 3600) + }, + {ok, _} = perform_cache_write(Base, Req, Result, Opts), + {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, Opts), + CreatedAt = hb_util:int(hb_maps:get(<<"priv-created-at">>, Cached, Opts)), ?assert(is_integer(CreatedAt)), - hb_cache:link(ResultPath, hb_path:hashpath(Base, Req, Opts), Opts), {ok, Served} = hb_ao:resolve(Base, Req, Opts), - ?assertEqual(not_found, hb_maps:get(<<"created-at">>, Served, not_found, Opts)), - timer:sleep(2100), + ?assertEqual( + not_found, + hb_maps:get(<<"priv-created-at">>, Served, not_found, Opts) + ), + CacheAddress = hb_cache:resolved_address(Base, Req, Opts), + {ok, _} = hb_cache:write_resolved( + CacheAddress, + Result, + os:system_time(second) - 61, + Opts + ), {ok, Refreshed} = hb_ao:resolve(Base, Req, Opts), - ?assertEqual(not_found, hb_maps:get(<<"created-at">>, Refreshed, not_found, Opts)), + ?assertEqual( + not_found, + hb_maps:get(<<"priv-created-at">>, Refreshed, not_found, Opts) + ), ?assertNotEqual(Served, Refreshed). + +cache_result_priv_created_at_regression_test() -> + Wallet = ar_wallet:new(), + Opts = #{ + <<"store">> => [hb_test_utils:test_store()], + <<"priv-wallet">> => Wallet, + <<"cache-control">> => [<<"always">>] + }, + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 3600 }, + Result = hb_message:commit( + #{ + <<"body">> => <<"cached">>, + <<"content-type">> => <<"text/plain">>, + <<"created-at">> => <<"application-value">>, + <<"status">> => 200 + }, + Opts + ), + {ok, _} = perform_cache_write(Base, Req, Result, Opts), + CacheAddress = hb_cache:resolved_address(Base, Req, Opts), + {ok, _} = hb_cache:write_resolved( + CacheAddress, + Result, + os:system_time(second) - 1000, + Opts + ), + {hit, {ok, CachedEntry}} = hb_cache:read_resolved(Base, Req, Opts), + CachedTimestamp = hb_maps:get(<<"priv-created-at">>, CachedEntry, Opts), + ?assert(is_integer(hb_util:int(CachedTimestamp))), + Parent = self(), + EventHandler = #{ + <<"device">> => #{ + event => + fun(_, EventReq, _) -> + case { + maps:get(<<"module">>, EventReq), + maps:get(<<"body">>, EventReq) + } of + {?MODULE, {cache_hit, _, _, _} = Event} -> + Parent ! {cache_hit_event, Event}; + _ -> ok + end, + {ok, EventReq} + end + } + }, + CacheOnlyOpts = Opts#{ + <<"cache-control">> => [<<"only-if-cached">>], + <<"on">> => #{ <<"event">> => EventHandler } + }, + OldEventOpts = erlang:get({hb_event, event_opts}), + erlang:put({hb_event, event_opts}, CacheOnlyOpts), + Served = + try + {ok, CachedResult} = maybe_lookup(Base, Req, CacheOnlyOpts), + CachedResult + after + case OldEventOpts of + undefined -> erlang:erase({hb_event, event_opts}); + _ -> erlang:put({hb_event, event_opts}, OldEventOpts) + end + end, + CacheHitEvent = + receive + {cache_hit_event, Event} -> Event + after 1000 -> + error(cache_hit_event_not_observed) + end, + EncodedEvent = term_to_binary(CacheHitEvent), + ?assertEqual(nomatch, binary:match(EncodedEvent, <<"priv-created-at">>)), + ?assertEqual(nomatch, binary:match(EncodedEvent, CachedTimestamp)), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)), + ?assertEqual( + <<"application-value">>, + hb_maps:get(<<"created-at">>, Served, Opts) + ), + ?assertEqual( + hb_message:id(Result, none, Opts), + hb_message:id(Served, none, Opts) + ), + ?assertEqual( + hb_message:id(Result, all, Opts), + hb_message:id(Served, all, Opts) + ), + ?assertEqual( + hb_maps:get(<<"commitments">>, Result, Opts), + hb_maps:get(<<"commitments">>, Served, Opts) + ), + ?assert(hb_message:verify(Served, all, Opts)), + {ok, _} = hb_cache:write_resolved( + CacheAddress, + Result, + os:system_time(second) - 3601, + Opts + ), + ?assertMatch( + {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, + maybe_lookup(Base, Req, CacheOnlyOpts) + ), + ?assertEqual( + {error, not_found}, + hb_cache:read(<<"created-at/", CacheAddress/binary>>, Opts) + ). + +cache_binary_max_age_remains_available_without_age_test() -> + Opts = #{ + <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] + }, + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60 }, + ReqWithoutMaxAge = maps:remove(<<"max-age">>, Req), + ?assertEqual( + hb_cache:resolved_address(Base, ReqWithoutMaxAge, Opts), + hb_cache:resolved_address(Base, Req, Opts) + ), + {ok, _} = perform_cache_write(Base, Req, <<"cached">>, Opts), + CacheOnlyOpts = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, + ?assertMatch( + {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, + maybe_lookup(Base, Req, CacheOnlyOpts) + ), + ?assertEqual( + {ok, <<"cached">>}, + maybe_lookup(Base, ReqWithoutMaxAge, CacheOnlyOpts) + ). + +cache_map_without_max_age_is_reusable_test() -> + Opts = #{ + <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] + }, + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">> }, + ReqWithMaxAge = Req#{ <<"max-age">> => 60 }, + Result = #{ + <<"value">> => <<"cached">>, + <<"priv-created-at">> => + integer_to_binary(os:system_time(second) + 3600) + }, + ?assertEqual( + hb_cache:resolved_address(Base, Req, Opts), + hb_cache:resolved_address(Base, ReqWithMaxAge, Opts) + ), + {ok, _} = perform_cache_write(Base, Req, Result, Opts), + {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, Opts), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Cached)), + CacheOnlyOpts = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, + ?assertMatch( + {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, + maybe_lookup(Base, ReqWithMaxAge, CacheOnlyOpts) + ), + {ok, Served} = maybe_lookup(Base, Req, CacheOnlyOpts), + ?assertEqual(<<"cached">>, hb_maps:get(<<"value">>, Served, Opts)), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)). + +fresh_uses_loaded_metadata_without_store_read_test() -> + Parent = self(), + Tracer = spawn(fun() -> cache_control_trace_forwarder(Parent) end), + erlang:trace(self(), true, [call, {tracer, Tracer}]), + erlang:trace_pattern({hb_store, read, 3}, true, []), + IsFresh = + try + fresh( + #{ + <<"priv-created-at">> => + integer_to_binary(os:system_time(second)) + }, + #{ <<"max-age">> => 60 }, + #{} + ) + after + stop_cache_control_trace(Tracer) + end, + ?assert(IsFresh), + receive + {trace_event, {trace, _, call, {hb_store, read, _}}} -> + ?assert(false) + after 0 -> + ok + end. + +%% @doc Disable store-read tracing and stop its forwarding process. +stop_cache_control_trace(Tracer) -> + erlang:trace_pattern({hb_store, read, 3}, false, []), + erlang:trace(self(), false, [call]), + TraceRef = erlang:trace_delivered(self()), + Delivered = + receive + {trace_delivered, _, TraceRef} -> true + after 1000 -> + false + end, + Tracer ! {flush, self()}, + Flushed = + receive + trace_flushed -> true + after 1000 -> + false + end, + Tracer ! stop, + case {Delivered, Flushed} of + {true, true} -> ok; + {false, _} -> error(trace_delivery_timeout); + {_, false} -> error(trace_flush_timeout) + end. + +%% @doc Missing or invalid private timestamps fail closed when max-age applies. +freshness_private_metadata_validation_test() -> + Now = os:system_time(second), + Req = #{ <<"max-age">> => 60 }, + ?assert(fresh(#{}, #{}, #{})), + ?assertNot(fresh(#{}, Req, #{})), + ?assertNot(fresh(#{ <<"priv-created-at">> => <<"invalid">> }, Req, #{})), + ?assertNot(fresh(#{ <<"priv-created-at">> => <<"-1">> }, Req, #{})), + ?assertNot( + fresh( + #{ <<"priv-created-at">> => integer_to_binary(Now + 1) }, + Req, + #{} + ) + ), + ?assert( + fresh( + #{ <<"priv-created-at">> => integer_to_binary(Now) }, + Req, + #{} + ) + ). + +%% @doc Forward test trace events and provide an ordered flush barrier. +cache_control_trace_forwarder(Parent) -> + receive + {flush, ReplyTo} -> + ReplyTo ! trace_flushed, + cache_control_trace_forwarder(Parent); + stop -> + ok; + TraceEvent -> + Parent ! {trace_event, TraceEvent}, + cache_control_trace_forwarder(Parent) + end. diff --git a/src/preloaded/vm/dev_lua.erl b/src/preloaded/vm/dev_lua.erl index 0e8fc08b2..c34f58d29 100644 --- a/src/preloaded/vm/dev_lua.erl +++ b/src/preloaded/vm/dev_lua.erl @@ -888,7 +888,7 @@ invoke_non_compute_key_test() -> ?event({result2, Result2}), ?assertEqual(<<"Alice">>, hb_ao:get(<<"hello">>, Result2, #{})). -top_holders_cache_test() -> +top_holders_fixture(MaxAge) -> Store = hb_test_utils:test_store(), Wallet = ar_wallet:new(), Opts = #{ @@ -945,12 +945,58 @@ top_holders_cache_test() -> <<"path">> => <<"top">>, <<"top">> => 2, <<"process-id">> => ProcessID, - <<"max-age">> => 60 + <<"max-age">> => MaxAge }, + {Function, Req, Opts}. + +top_holders_cache_test() -> + {Function, Req, Opts} = top_holders_fixture(60), {ok, Result} = hb_ao:resolve(Function, Req, Opts), Holders = hb_json:decode(hb_maps:get(<<"body">>, Result, Opts)), ?assertEqual(<<"bob">>, hb_maps:get(<<"address">>, hd(Holders), Opts)). +top_holders_max_age_cache_test() -> + {Function, Req, Opts} = top_holders_fixture(60), + assert_top_holders_max_age_cache(Function, Req, Opts). + +%% @doc Cache and expire a Lua map when the function is addressed by its ID. +top_holders_max_age_cache_by_id_test() -> + {Function, Req, Opts} = top_holders_fixture(60), + {ok, FunctionID} = hb_cache:write(Function, Opts), + assert_top_holders_max_age_cache(FunctionID, Req, Opts). + +%% @doc Assert the fresh and stale cache behavior for a Lua function reference. +assert_top_holders_max_age_cache(Function, Req, Opts) -> + {ok, First} = hb_ao:resolve(Function, Req, Opts), + CacheOnlyOpts = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, + {ok, Fresh} = hb_ao:resolve(Function, Req, CacheOnlyOpts), + ?assertEqual( + hb_message:id(First, none, Opts), + hb_message:id(Fresh, none, Opts) + ), + ?assertEqual( + hb_message:id(First, all, Opts), + hb_message:id(Fresh, all, Opts) + ), + ?assertEqual( + hb_maps:get(<<"commitments">>, First, Opts), + hb_maps:get(<<"commitments">>, Fresh, Opts) + ), + ?assert(hb_message:verify(Fresh, all, Opts)), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Fresh)), + ?assertEqual(false, maps:is_key(<<"created-at">>, Fresh)), + CacheAddress = hb_cache:resolved_address(Function, Req, Opts), + {ok, _} = hb_cache:write_resolved( + CacheAddress, + First, + os:system_time(second) - 61, + Opts + ), + ?assertMatch( + {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, + hb_ao:resolve(Function, Req, CacheOnlyOpts) + ). + %% @doc Use a Lua module as a hook on the HTTP server via `~meta@1.0'. lua_http_hook_test() -> {ok, Module} = file:read_file("test/test.lua"), From 16f3c6471446b30c69cd30209c9ef6c37f9d1166 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Thu, 27 Aug 2026 12:02:07 -0400 Subject: [PATCH 04/10] feat: add cache freshness policy foundation --- src/core/resolver/hb_cache_control.erl | 497 ++++++++++++++++++++++++- 1 file changed, 484 insertions(+), 13 deletions(-) diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 6e5b3f45c..42938a484 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -11,6 +11,7 @@ %%% following settings. -define(DEFAULT_STORE_OPT, false). -define(DEFAULT_LOOKUP_OPT, true). +-define(MAX_DELTA_SECONDS, 2147483647). %%% Public API @@ -305,41 +306,337 @@ cache_source_to_cache_settings(Msg, Opts) -> _ -> #{} end. -%% @doc Convert a cache control list as received via HTTP headers into a -%% normalized map of simply whether we should store and/or lookup the result. -specifiers_to_cache_settings(CCSpecifier) when not is_list(CCSpecifier) -> - specifiers_to_cache_settings([CCSpecifier]); -specifiers_to_cache_settings(RawCCList) -> - CCList = lists:map(fun hb_ao:normalize_key/1, RawCCList), +%% @doc Parse Cache-Control values while preserving each original directive. +parse_cache_control(undefined) -> []; +parse_cache_control(RawList) when is_list(RawList) -> + lists:append(lists:map(fun parse_cache_control/1, RawList)); +parse_cache_control(Raw) -> + try + Bin = hb_util:bin(Raw), + lists:filtermap( + fun parse_cache_directive/1, + hb_util:split_depth_string_aware($,, Bin) + ) + catch + _:_ -> [] + end. + +%% @doc Parse one Cache-Control directive with Cowlib and a tolerant fallback. +parse_cache_directive(RawDirective) -> + Directive = string:trim(RawDirective), + case Directive of + <<>> -> false; + _ -> + try cow_http_hd:parse_cache_control(Directive) of + [Parsed] -> {true, normalize_cache_directive(Parsed, Directive)}; + _ -> parse_legacy_cache_directive(Directive) + catch + _:_ -> parse_legacy_cache_directive(Directive) + end + end. + +%% @doc Normalize a Cowlib directive without creating input-derived atoms. +normalize_cache_directive({RawName, RawValue}, Raw) -> + Name = normalize_cache_directive_name(RawName), + { + Name, + normalize_cache_directive_value(Name, RawValue), + Raw + }; +normalize_cache_directive(RawName, Raw) -> + {normalize_cache_directive_name(RawName), true, Raw}. + +%% @doc Normalize only the case-insensitive directive name. +normalize_cache_directive_name(Name) -> + hb_ao:normalize_key(string:lowercase(Name)). + +%% @doc Bound numeric parser output while preserving extension values. +normalize_cache_directive_value(_Name, Value) when is_integer(Value) -> + min(max(0, Value), ?MAX_DELTA_SECONDS); +normalize_cache_directive_value(Name, Value) -> + Unquoted = hb_util:unquote(Value), + case lists:member( + Name, + [ + <<"max-age">>, + <<"max-stale">>, + <<"min-fresh">>, + <<"stale-while-revalidate">>, + <<"stale-if-error">> + ] + ) of + false -> Unquoted; + true -> + case parse_delta_seconds(Unquoted) of + invalid -> Unquoted; + Delta -> Delta + end + end. + +%% @doc Parse OWS around `=' and quoted commas rejected by strict Cowlib. +parse_legacy_cache_directive(Directive) -> + case hb_util:split_depth_string_aware_single($=, Directive) of + {no_match, RawName, <<>>} -> + { + true, + { + normalize_cache_directive_name(string:trim(RawName)), + true, + Directive + } + }; + {_Match, RawName, RawValue} -> + Name = normalize_cache_directive_name(string:trim(RawName)), + Value = normalize_cache_directive_value( + Name, + string:trim(RawValue) + ), + {true, {Name, Value, Directive}} + end. + +%% @doc Return all normalized values for one directive name. +directive_values(Name, Parsed) -> + [Value || {Directive, Value, _Raw} <- Parsed, Directive =:= Name]. + +%% @doc Return whether a directive is present, regardless of its argument. +has_directive(Name, Parsed) -> + directive_values(Name, Parsed) =/= []. + +%% @doc Parse one non-negative delta-seconds value conservatively. +parse_delta_seconds(Value) when is_integer(Value), Value >= 0 -> + min(Value, ?MAX_DELTA_SECONDS); +parse_delta_seconds(Value) when is_binary(Value) -> + Trimmed = string:trim(hb_util:unquote(Value)), + case is_decimal(Trimmed) of + false -> invalid; + true when byte_size(Trimmed) > 10 -> ?MAX_DELTA_SECONDS; + true -> min(binary_to_integer(Trimmed), ?MAX_DELTA_SECONDS) + end; +parse_delta_seconds(_) -> + invalid. + +%% @doc Return whether a binary is a non-empty ASCII decimal number. +is_decimal(<<>>) -> false; +is_decimal(Bin) -> + lists:all( + fun(Char) -> Char >= $0 andalso Char =< $9 end, + binary_to_list(Bin) + ). + +%% @doc Parse a directive that must occur once with a delta-seconds value. +numeric_directive(Name, Parsed) -> + case directive_values(Name, Parsed) of + [] -> undefined; + [Value] -> parse_delta_seconds(Value); + _ -> invalid + end. + +%% @doc Parse request max-stale, including its valueless form. +max_stale_directive(Parsed) -> + case directive_values(<<"max-stale">>, Parsed) of + [] -> absent; + [true] -> any; + [Value] -> + case parse_delta_seconds(Value) of + invalid -> absent; + Delta -> Delta + end; + _ -> absent + end. + +%% @doc Combine header and compatibility max-age using the stricter value. +combine_max_age(undefined, undefined) -> undefined; +combine_max_age(invalid, _) -> invalid; +combine_max_age(_, invalid) -> invalid; +combine_max_age(undefined, Legacy) -> Legacy; +combine_max_age(Header, undefined) -> Header; +combine_max_age(Header, Legacy) -> min(Header, Legacy). + +%% @doc Parse the legacy top-level request max-age compatibility field. +legacy_max_age(Msg, Opts) -> + case hb_maps:find(<<"max-age">>, Msg, Opts) of + error -> undefined; + {ok, <<"infinity">>} -> undefined; + {ok, infinity} -> undefined; + {ok, Value} -> parse_delta_seconds(Value) + end. + +%% @doc Build the request-side cache policy without response directives. +request_policy(Msg, Opts) -> + Parsed = message_cache_control(Msg, Opts), + HeaderMaxAge = numeric_directive(<<"max-age">>, Parsed), + #{ + <<"max-age">> => + combine_max_age(HeaderMaxAge, legacy_max_age(Msg, Opts)), + <<"max-stale">> => max_stale_directive(Parsed), + <<"min-fresh">> => numeric_directive(<<"min-fresh">>, Parsed), + <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), + <<"no-store">> => has_directive(<<"no-store">>, Parsed), + <<"only-if-cached">> => + has_directive(<<"only-if-cached">>, Parsed) + }. + +%% @doc Build the response-side cache policy without request directives. +response_policy(Msg, Opts) -> + Parsed = message_cache_control(Msg, Opts), + #{ + <<"max-age">> => numeric_directive(<<"max-age">>, Parsed), + <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), + <<"no-store">> => has_directive(<<"no-store">>, Parsed), + <<"private">> => has_directive(<<"private">>, Parsed), + <<"must-revalidate">> => + has_directive(<<"must-revalidate">>, Parsed) + }. + +%% @doc Parse the Cache-Control field of an AO message when present. +message_cache_control(Msg, Opts) -> + case hb_maps:find(<<"cache-control">>, Msg, Opts) of + {ok, Raw} -> parse_cache_control(Raw); + _ -> [] + end. + +%% @doc Classify a cached result using already loaded timing metadata. +classify_cached(Res, ReqPolicy, ResPolicy, Now) -> + case cache_policy_prohibition(ReqPolicy, ResPolicy) of + none -> classify_cached_age(Res, ReqPolicy, ResPolicy, Now); + Reason -> {unacceptable, Reason} + end. + +%% @doc Return the first rule that requires validation or prohibits reuse. +cache_policy_prohibition(ReqPolicy, ResPolicy) -> + case { + maps:get(<<"no-cache">>, ReqPolicy), + maps:get(<<"no-store">>, ResPolicy), + maps:get(<<"private">>, ResPolicy), + maps:get(<<"no-cache">>, ResPolicy), + maps:get(<<"max-age">>, ReqPolicy), + maps:get(<<"min-fresh">>, ReqPolicy), + maps:get(<<"max-age">>, ResPolicy) + } of + {true, _, _, _, _, _, _} -> request_no_cache; + {_, true, _, _, _, _, _} -> response_no_store; + {_, _, true, _, _, _, _} -> response_private; + {_, _, _, true, _, _, _} -> response_no_cache; + {_, _, _, _, invalid, _, _} -> invalid_request_max_age; + {_, _, _, _, _, invalid, _} -> invalid_request_min_fresh; + {_, _, _, _, _, _, invalid} -> invalid_response_max_age; + _ -> none + end. + +%% @doc Calculate age only when some request or response rule needs it. +classify_cached_age(Res, ReqPolicy, ResPolicy, Now) -> + case age_required(ReqPolicy, ResPolicy) of + false -> {fresh, undefined}; + true -> + case cached_age(Res, Now) of + {ok, Age} -> classify_cached_lifetime(Age, ReqPolicy, ResPolicy); + error -> {unacceptable, invalid_created_at} + end + end. + +%% @doc Return whether the decision depends upon a stored timestamp. +age_required(ReqPolicy, ResPolicy) -> + maps:get(<<"max-age">>, ReqPolicy) =/= undefined orelse + maps:get(<<"min-fresh">>, ReqPolicy) =/= undefined orelse + maps:get(<<"max-stale">>, ReqPolicy) =/= absent orelse + maps:get(<<"max-age">>, ResPolicy) =/= undefined orelse + maps:get(<<"must-revalidate">>, ResPolicy). + +%% @doc Read a bounded immediate timestamp without invoking the cache/store. +cached_age(Res, Now) when is_map(Res), is_integer(Now), Now >= 0 -> + try hb_util:int(maps:get(<<"priv-created-at">>, Res)) of + CreatedAt when CreatedAt >= 0, CreatedAt =< Now -> {ok, Now - CreatedAt}; + _ -> error + catch + _:_ -> error + end; +cached_age(_, _) -> + error. + +%% @doc Apply request ceilings before the response freshness lifetime. +classify_cached_lifetime(Age, ReqPolicy, ResPolicy) -> + case maps:get(<<"max-age">>, ReqPolicy) of + MaxAge when is_integer(MaxAge), Age > MaxAge -> + {unacceptable, request_max_age}; + _ -> + classify_response_lifetime( + Age, + ReqPolicy, + ResPolicy, + maps:get(<<"max-age">>, ResPolicy) + ) + end. + +%% @doc Preserve legacy hits when no response lifetime-dependent rule exists. +classify_response_lifetime(Age, ReqPolicy, ResPolicy, undefined) -> + case { + maps:get(<<"min-fresh">>, ReqPolicy), + maps:get(<<"max-stale">>, ReqPolicy), + maps:get(<<"must-revalidate">>, ResPolicy) + } of + {undefined, absent, false} -> {fresh, Age}; + _ -> {unacceptable, response_lifetime_missing} + end; +classify_response_lifetime(Age, ReqPolicy, ResPolicy, Lifetime) -> + case min_fresh_satisfied(Age, Lifetime, ReqPolicy) of + false -> {unacceptable, min_fresh}; + true when Age < Lifetime -> {fresh, Age}; + true -> classify_stale(Age, Lifetime, ReqPolicy, ResPolicy) + end. + +%% @doc Require both freshness and the requested remaining lifetime. +min_fresh_satisfied(Age, Lifetime, ReqPolicy) -> + case maps:get(<<"min-fresh">>, ReqPolicy) of + undefined -> true; + MinFresh -> Age < Lifetime andalso Lifetime - Age >= MinFresh + end. + +%% @doc Permit stale reuse only through request max-stale. +classify_stale(Age, Lifetime, ReqPolicy, ResPolicy) -> + case maps:get(<<"must-revalidate">>, ResPolicy) of + true -> {unacceptable, must_revalidate}; + false -> + StaleBy = Age - Lifetime, + case maps:get(<<"max-stale">>, ReqPolicy) of + any -> {stale_allowed, Age, StaleBy, max_stale}; + MaxStale when is_integer(MaxStale), StaleBy =< MaxStale -> + {stale_allowed, Age, StaleBy, max_stale}; + _ -> {unacceptable, stale} + end + end. + +%% @doc Convert Cache-Control input into legacy store and lookup settings. +specifiers_to_cache_settings(CCSpecifier) -> + Parsed = parse_cache_control(CCSpecifier), #{ <<"store">> => - case lists:member(<<"always">>, CCList) of + case has_directive(<<"always">>, Parsed) of true -> true; false -> - case lists:member(<<"no-store">>, CCList) of + case has_directive(<<"no-store">>, Parsed) of true -> false; false -> - case lists:member(<<"store">>, CCList) of + case has_directive(<<"store">>, Parsed) of true -> true; false -> undefined end end end, <<"lookup">> => - case lists:member(<<"always">>, CCList) of + case has_directive(<<"always">>, Parsed) of true -> true; false -> - case lists:member(<<"no-cache">>, CCList) of + case has_directive(<<"no-cache">>, Parsed) of true -> false; false -> - case lists:member(<<"cache">>, CCList) of + case has_directive(<<"cache">>, Parsed) of true -> true; false -> undefined end end end, <<"only-if-cached">> => - case lists:member(<<"only-if-cached">>, CCList) of + case has_directive(<<"only-if-cached">>, Parsed) of true -> true; false -> undefined end @@ -754,3 +1051,177 @@ cache_control_trace_forwarder(Parent) -> Parent ! {trace_event, TraceEvent}, cache_control_trace_forwarder(Parent) end. + +%% @doc Parse wire and list Cache-Control forms without losing quoted commas. +cache_control_parser_regression_test() -> + Parsed = parse_cache_control( + <<"MaX-aGe = \"60\", only-if-cached, community = \"A,B\"">> + ), + ?assertEqual([60], directive_values(<<"max-age">>, Parsed)), + ?assertEqual([true], directive_values(<<"only-if-cached">>, Parsed)), + ?assertEqual([<<"A,B">>], directive_values(<<"community">>, Parsed)), + ListParsed = parse_cache_control([<<"always">>, <<"NO-STORE">>]), + ?assertEqual([true], directive_values(<<"always">>, ListParsed)), + ?assertEqual([true], directive_values(<<"no-store">>, ListParsed)). + +%% @doc Request policy handles numeric, bare, duplicate, and invalid forms. +request_cache_policy_regression_test() -> + Policy = request_policy( + #{ + <<"cache-control">> => + <<"max-age=60, max-stale=10, min-fresh=5, only-if-cached">> + }, + #{} + ), + ?assertEqual(60, maps:get(<<"max-age">>, Policy)), + ?assertEqual(10, maps:get(<<"max-stale">>, Policy)), + ?assertEqual(5, maps:get(<<"min-fresh">>, Policy)), + ?assert(maps:get(<<"only-if-cached">>, Policy)), + Bare = request_policy( + #{ <<"cache-control">> => <<"max-stale">> }, + #{} + ), + ?assertEqual(any, maps:get(<<"max-stale">>, Bare)), + Duplicate = request_policy( + #{ <<"cache-control">> => <<"max-age=60, max-age=30">> }, + #{} + ), + ?assertEqual(invalid, maps:get(<<"max-age">>, Duplicate)), + Invalid = request_policy( + #{ <<"cache-control">> => <<"min-fresh=-1, max-stale=nope">> }, + #{} + ), + ?assertEqual(invalid, maps:get(<<"min-fresh">>, Invalid)), + ?assertEqual(absent, maps:get(<<"max-stale">>, Invalid)), + Overflow = request_policy( + #{ <<"cache-control">> => <<"max-age=999999999999999999999">> }, + #{} + ), + ?assertEqual(2147483647, maps:get(<<"max-age">>, Overflow)). + +%% @doc Response policy remains independent from request-only directives. +response_cache_policy_regression_test() -> + Policy = response_policy( + #{ + <<"cache-control">> => + <<"max-age=60, no-cache=\"body\", private=\"secret\", " + "must-revalidate, max-stale=99">> + }, + #{} + ), + ?assertEqual(60, maps:get(<<"max-age">>, Policy)), + ?assert(maps:get(<<"no-cache">>, Policy)), + ?assert(maps:get(<<"private">>, Policy)), + ?assert(maps:get(<<"must-revalidate">>, Policy)), + ?assertEqual(false, maps:is_key(<<"max-stale">>, Policy)), + Duplicate = response_policy( + #{ <<"cache-control">> => <<"max-age=60, max-age=30">> }, + #{} + ), + ?assertEqual(invalid, maps:get(<<"max-age">>, Duplicate)). + +%% @doc Freshness boundaries use explicit time and the correct policy side. +cache_freshness_classifier_boundaries_test() -> + Now = 1000, + Response60 = response_policy( + #{ <<"cache-control">> => <<"max-age=60">> }, + #{} + ), + EmptyRequest = request_policy(#{}, #{}), + ?assertEqual( + {fresh, 59}, + classify_cached(cache_entry(Now - 59), EmptyRequest, Response60, Now) + ), + ?assertEqual( + {unacceptable, stale}, + classify_cached(cache_entry(Now - 60), EmptyRequest, Response60, Now) + ), + Request60 = request_policy(#{ <<"max-age">> => 60 }, #{}), + ?assertEqual( + {fresh, 60}, + classify_cached( + cache_entry(Now - 60), + Request60, + response_policy( + #{ <<"cache-control">> => <<"max-age=120">> }, + #{} + ), + Now + ) + ), + Request0 = request_policy(#{ <<"max-age">> => 0 }, #{}), + ?assertEqual( + {fresh, 0}, + classify_cached(cache_entry(Now), Request0, Response60, Now) + ), + MinFresh = request_policy( + #{ <<"cache-control">> => <<"min-fresh=10">> }, + #{} + ), + ?assertEqual( + {fresh, 50}, + classify_cached(cache_entry(Now - 50), MinFresh, Response60, Now) + ). + +%% @doc Max-stale is measured from the response freshness lifetime. +cache_max_stale_classifier_regression_test() -> + Now = 1000, + Response60 = response_policy( + #{ <<"cache-control">> => <<"max-age=60">> }, + #{} + ), + MaxStale10 = request_policy( + #{ <<"cache-control">> => <<"max-stale=10">> }, + #{} + ), + ?assertEqual( + {stale_allowed, 70, 10, max_stale}, + classify_cached(cache_entry(Now - 70), MaxStale10, Response60, Now) + ), + ?assertEqual( + {unacceptable, stale}, + classify_cached(cache_entry(Now - 71), MaxStale10, Response60, Now) + ), + Bare = request_policy( + #{ <<"cache-control">> => <<"max-stale">> }, + #{} + ), + ?assertEqual( + {stale_allowed, 500, 440, max_stale}, + classify_cached(cache_entry(Now - 500), Bare, Response60, Now) + ), + ?assertEqual( + {unacceptable, response_lifetime_missing}, + classify_cached( + cache_entry(Now - 1), + Bare, + response_policy(#{}, #{}), + Now + ) + ). + +%% @doc Age-dependent decisions reject unusable co-located metadata. +cache_freshness_metadata_regression_test() -> + Now = 1000, + Req = request_policy(#{ <<"max-age">> => 60 }, #{}), + Res = response_policy(#{}, #{}), + lists:foreach( + fun(Candidate) -> + ?assertEqual( + {unacceptable, invalid_created_at}, + classify_cached(Candidate, Req, Res, Now) + ) + end, + [ + #{}, + #{ <<"priv-created-at">> => <<"invalid">> }, + #{ <<"priv-created-at">> => <<"-1">> }, + #{ <<"priv-created-at">> => <<"1001">> }, + #{ <<"priv-created-at">> => {link, <<"missing">>} }, + <<"binary-candidate">> + ] + ). + +%% @doc Build a deterministic cached map for classifier tests. +cache_entry(CreatedAt) -> + #{ <<"priv-created-at">> => integer_to_binary(CreatedAt) }. From dd9c25ad4582b26b379223c59f5b76a830357167 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 03:30:52 -0400 Subject: [PATCH 05/10] impr: narrow cache freshness semantics --- src/core/resolver/hb_ao.erl | 4 +- src/core/resolver/hb_cache.erl | 14 +- src/core/resolver/hb_cache_control.erl | 985 ++++++++++--------------- 3 files changed, 410 insertions(+), 593 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 6969eaae6..fc7061c5f 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -111,6 +111,7 @@ <<"add-key">>, <<"force-message">>, <<"cache-control">>, + <<"priv-cache-request-policy">>, <<"spawn-worker">>, <<"only">>, <<"prefer">> @@ -415,7 +416,8 @@ resolve_stage(1, RawBase, RawReq, Opts) -> ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), - resolve_stage(2, Base, Req, Opts); + {SemanticReq, PreparedOpts} = hb_cache_control:prepare(Req, Opts), + resolve_stage(2, Base, SemanticReq, PreparedOpts); resolve_stage(2, Base, Req, Opts) -> ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index a12ac5c93..f1ee17ce5 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -1141,8 +1141,12 @@ resolved_address(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> hb_path:hashpath(BaseMsg, cache_request(Req, Opts), Opts). %% @doc Remove cache policy fields that do not identify a computation. +cache_request(Req, _Opts) when is_map(Req), + not is_map_key(<<"max-age">>, Req), + not is_map_key(<<"max-stale">>, Req) -> + Req; cache_request(Req, Opts) -> - hb_maps:without([<<"max-age">>], Req, Opts). + hb_message:without_unless_signed([<<"max-age">>, <<"max-stale">>], Req, Opts). hashpath_read_result({ok, Msg}) -> {hit, {ok, Msg}}; hashpath_read_result({error, not_found}) -> miss; @@ -1579,16 +1583,14 @@ resolved_entry_lmdb_single_root_read_test() -> Entry = try {ok, ReadEntry} = read(Address, Opts), + ?assertEqual(<<"12345">>, + hb_maps:get(<<"priv-created-at">>, ReadEntry, Opts)), ReadEntry after stop_cache_trace(Tracer) end, ?assertEqual(1, store_read_trace_count(0)), - ?assertEqual(<<"cached">>, hb_maps:get(<<"body">>, Entry, Opts)), - ?assertEqual( - <<"12345">>, - hb_maps:get(<<"priv-created-at">>, Entry, Opts) - ). + ?assertEqual(<<"cached">>, hb_maps:get(<<"body">>, Entry, Opts)). %% @doc Disable store-read tracing and stop its forwarding process. stop_cache_trace(Tracer) -> diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 42938a484..1d098266d 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/4, maybe_lookup/3]). +-export([maybe_store/4, maybe_lookup/3, prepare/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -11,10 +11,33 @@ %%% following settings. -define(DEFAULT_STORE_OPT, false). -define(DEFAULT_LOOKUP_OPT, true). +-define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). -define(MAX_DELTA_SECONDS, 2147483647). %%% Public API +%% @doc Remove uncommitted request freshness policy before AO execution. +prepare(Req, Opts) when is_map(Req), + not is_map_key(<<"max-age">>, Req), + not is_map_key(<<"max-stale">>, Req) -> + {Req, Opts}; +prepare(Req, Opts) -> + SemanticReq = hb_message:without_unless_signed( + [<<"max-age">>, <<"max-stale">>], + Req, + Opts + ), + Policy = #{ + <<"max-age">> => removed_max_age(Req, SemanticReq), + <<"max-stale">> => removed_max_stale(Req, SemanticReq) + }, + case Policy of + #{ <<"max-age">> := undefined, <<"max-stale">> := absent } -> + {SemanticReq, Opts}; + _ -> + {SemanticReq, Opts#{ ?REQUEST_POLICY_OPT => Policy }} + end. + %% @doc Write a resulting M3 message to the cache if requested. The precedence %% order of cache control sources is as follows: %% 1. The `Opts' map (letting the node operator have the final say). @@ -50,9 +73,15 @@ maybe_lookup(Base, Req, Opts) -> lookup(Base, Req, Opts) -> case derive_cache_settings([Base, Req], Opts) of - #{ <<"lookup">> := false } -> + Settings = #{ <<"lookup">> := false } -> ?event({skip_cache_check, lookup_disabled}), - maybe_load_base(Base, Req, Opts); + case derive_cache_settings( + [Base, Req], Opts#{ <<"only">> => local } + ) of + #{ <<"lookup">> := false, <<"only-if-cached">> := true } -> + cache_miss(Base, Req, Settings, Opts); + _ -> maybe_load_base(Base, Req, Opts) + end; Settings = #{ <<"lookup">> := true } -> OutputScopedOpts = hb_store:scope( @@ -63,8 +92,17 @@ lookup(Base, Req, Opts) -> {hit, not_found} -> {error, not_found}; {hit, {ok, Res}} -> - case fresh(Res, Req, Opts) of - true -> + ReqPolicy = request_policy(Req, Opts), + ResPolicy = response_policy(Res, OutputScopedOpts), + case classify_cached( + Res, + ReqPolicy, + ResPolicy, + os:system_time(second), + OutputScopedOpts + ) of + Decision when element(1, Decision) =:= fresh; + element(1, Decision) =:= stale_allowed -> CachedRes = cached_result(Res, OutputScopedOpts), ?event(caching, {cache_hit, @@ -74,7 +112,7 @@ lookup(Base, Req, Opts) -> } ), {ok, CachedRes}; - false -> + {unacceptable, _Reason} -> ?event(caching, {stale_cache_result, Base, Req}), cache_miss(Base, Req, Settings, Opts) end; @@ -86,27 +124,116 @@ lookup(Base, Req, Opts) -> %%% Internal functions -%% @doc Return whether a cached result satisfies the requested `max-age'. -fresh(Res, Req, Opts) -> - case hb_maps:get(<<"max-age">>, Req, infinity, Opts) of - infinity -> true; - <<"infinity">> -> true; - RawMaxAge -> - try - CreatedAt = hb_util:int( - hb_maps:get(<<"priv-created-at">>, Res, Opts) - ), - MaxAge = hb_util:int(RawMaxAge), - Now = os:system_time(second), - true = CreatedAt >= 0, - true = CreatedAt =< Now, - true = MaxAge >= 0, - Now - CreatedAt < MaxAge - catch - _:_ -> false +%% @doc Classify a loaded result using request and response age policy. +classify_cached(Res, ReqPolicy, ResPolicy, Now, Opts) -> + case { + maps:get(<<"max-age">>, ReqPolicy), + maps:get(<<"max-stale">>, ReqPolicy), + maps:get(<<"max-age">>, ResPolicy), + maps:get(<<"no-cache">>, ResPolicy) + } of + {_, _, _, true} -> {unacceptable, response_no_cache}; + {invalid, _, _, _} -> {unacceptable, invalid_request_max_age}; + {_, invalid, _, _} -> {unacceptable, invalid_request_max_stale}; + {_, _, invalid, _} -> {unacceptable, invalid_response_max_age}; + {undefined, absent, undefined, false} -> {fresh, undefined}; + _ -> classify_cached_age(Res, ReqPolicy, ResPolicy, Now, Opts) + end. + +%% @doc Apply the request ceiling and response freshness lifetime. +classify_cached_age(Res, ReqPolicy, ResPolicy, Now, Opts) -> + case cached_age(Res, Now, Opts) of + error -> {unacceptable, invalid_created_at}; + {ok, Age} -> + case maps:get(<<"max-age">>, ReqPolicy) of + MaxAge when is_integer(MaxAge), Age > MaxAge -> + {unacceptable, request_max_age}; + _ -> + classify_response_age(Age, ReqPolicy, ResPolicy) end end. +%% @doc Classify freshness and any request-permitted staleness. +classify_response_age(Age, ReqPolicy, ResPolicy) -> + case maps:get(<<"max-age">>, ResPolicy) of + undefined -> + case maps:get(<<"max-stale">>, ReqPolicy) of + absent -> {fresh, Age}; + _ -> {unacceptable, response_lifetime_missing} + end; + Lifetime when Age < Lifetime -> + {fresh, Age}; + Lifetime -> + case maps:get(<<"must-revalidate">>, ResPolicy) of + true -> {unacceptable, must_revalidate}; + false -> classify_stale(Age, Lifetime, ReqPolicy) + end + end. + +%% @doc Permit stale reuse only when max-stale allows the excess age. +classify_stale(Age, Lifetime, ReqPolicy) -> + StaleBy = Age - Lifetime, + case maps:get(<<"max-stale">>, ReqPolicy) of + any -> {stale_allowed, Age, StaleBy}; + MaxStale when is_integer(MaxStale), StaleBy =< MaxStale -> + {stale_allowed, Age, StaleBy}; + _ -> {unacceptable, stale} + end. + +%% @doc Calculate age from a cache-owned timestamp. +cached_age(Res, Now, Opts) when is_map(Res), is_integer(Now), Now >= 0 -> + try hb_util:int(hb_maps:get(<<"priv-created-at">>, Res, Opts)) of + CreatedAt when CreatedAt >= 0, CreatedAt =< Now -> {ok, Now - CreatedAt}; + _ -> error + catch + _:_ -> error + end; +cached_age(_, _, _) -> error. + +%% @doc Return a removed, uncommitted request max-age value. +removed_max_age(Req, SemanticReq) -> + case removed_value(<<"max-age">>, Req, SemanticReq) of + undefined -> undefined; + infinity -> undefined; + <<"infinity">> -> undefined; + Value -> parse_delta_seconds(Value) + end. + +%% @doc Return a removed, uncommitted request max-stale value. +removed_max_stale(Req, SemanticReq) -> + case removed_value(<<"max-stale">>, Req, SemanticReq) of + undefined -> absent; + true -> any; + Value -> parse_delta_seconds(Value) + end. + +%% @doc Return a field only when preparation removed it from the request. +removed_value(Key, Req, SemanticReq) -> + case {maps:find(Key, Req), maps:is_key(Key, SemanticReq)} of + {{ok, Value}, false} -> Value; + _ -> undefined + end. + +%% @doc Read request policy prepared by AO stage 1, with a direct-call fallback. +request_policy(Req, Opts) -> + case maps:find(?REQUEST_POLICY_OPT, Opts) of + {ok, Policy} -> Policy; + error -> + {_SemanticReq, PreparedOpts} = prepare(Req, Opts), + maps:get(?REQUEST_POLICY_OPT, PreparedOpts, + #{ <<"max-age">> => undefined, <<"max-stale">> => absent }) + end. + +%% @doc Read the response lifetime and rules that prohibit reuse. +response_policy(Res, Opts) -> + Parsed = message_cache_control(Res, Opts), + #{ + <<"max-age">> => numeric_directive(<<"max-age">>, Parsed), + <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), + <<"must-revalidate">> => + has_directive(<<"must-revalidate">>, Parsed) + }. + cache_miss(Base, Req, #{ <<"only-if-cached">> := true }, Opts) -> only_if_cached_not_found_error(Base, Req, Opts); cache_miss(Base, Req, _Settings, Opts) -> @@ -174,9 +301,18 @@ async_writer() -> perform_cache_write(Base, Req, Res, Opts) -> hb_cache:write(Base, Opts), hb_cache:write(Req, Opts), - TracksFreshness = hb_maps:is_key(<<"max-age">>, Req, Opts), + StorableRes = case Res of + Candidate when is_map(Candidate) -> + {ok, StorableMap} = hb_message:with_only_committed(Candidate, Opts), + StorableMap; + _ -> Res + end, + TracksFreshness = tracks_freshness( + request_policy(Req, Opts), + response_policy(StorableRes, Opts) + ), CacheAddress = hb_cache:resolved_address(Base, Req, Opts), - case {Res, TracksFreshness} of + WriteResult = case {Res, TracksFreshness} of {<<_/binary>>, _} -> hb_cache:write_binary( CacheAddress, @@ -191,15 +327,45 @@ perform_cache_write(Base, Req, Res, Opts) -> Opts ); {Map, false} when is_map(Map) -> - hb_cache:write_hashpath( - CacheAddress, + hb_cache:write( maps:remove(<<"priv-created-at">>, Map), Opts ); _ -> ?event({cannot_write_result, Res}), skip_caching - end. + end, + mirror_base_id_address( + Base, + Req, + CacheAddress, + WriteResult, + TracksFreshness, + Opts + ). + +%% @doc Mirror freshness-tracked results only; untracked maps stay canonical. +mirror_base_id_address( + Base, Req, CacheAddress, {ok, Path} = Result, true, Opts + ) when is_map(Base) -> + IDAddress = hb_cache:resolved_address( + hb_message:id(Base, all, Opts), + Req, + Opts + ), + case IDAddress =:= CacheAddress of + true -> Result; + false -> + hb_cache:link(Path, IDAddress, Opts), + Result + end; +mirror_base_id_address(_Base, _Req, _CacheAddress, Result, _Tracked, _Opts) -> + Result. + +%% @doc Return whether a cached map needs co-located age metadata. +tracks_freshness(ReqPolicy, ResPolicy) -> + is_integer(maps:get(<<"max-age">>, ReqPolicy)) orelse + is_integer(maps:get(<<"max-age">>, ResPolicy)). %% @doc Generate a message to return when `only_if_cached' was specified, and %% we don't have a cached result. @@ -306,337 +472,78 @@ cache_source_to_cache_settings(Msg, Opts) -> _ -> #{} end. -%% @doc Parse Cache-Control values while preserving each original directive. -parse_cache_control(undefined) -> []; +%% @doc Parse an AO list or a raw Cache-Control field. parse_cache_control(RawList) when is_list(RawList) -> - lists:append(lists:map(fun parse_cache_control/1, RawList)); + Parsed = [parse_cache_control(Raw) || Raw <- RawList], + case lists:member(invalid, Parsed) of + true -> invalid; + false -> lists:append(Parsed) + end; parse_cache_control(Raw) -> - try - Bin = hb_util:bin(Raw), - lists:filtermap( - fun parse_cache_directive/1, - hb_util:split_depth_string_aware($,, Bin) - ) - catch - _:_ -> [] - end. - -%% @doc Parse one Cache-Control directive with Cowlib and a tolerant fallback. -parse_cache_directive(RawDirective) -> - Directive = string:trim(RawDirective), - case Directive of - <<>> -> false; - _ -> - try cow_http_hd:parse_cache_control(Directive) of - [Parsed] -> {true, normalize_cache_directive(Parsed, Directive)}; - _ -> parse_legacy_cache_directive(Directive) - catch - _:_ -> parse_legacy_cache_directive(Directive) - end + try cow_http_hd:parse_cache_control(hb_util:bin(Raw)) + catch _:_ -> invalid end. -%% @doc Normalize a Cowlib directive without creating input-derived atoms. -normalize_cache_directive({RawName, RawValue}, Raw) -> - Name = normalize_cache_directive_name(RawName), - { - Name, - normalize_cache_directive_value(Name, RawValue), - Raw - }; -normalize_cache_directive(RawName, Raw) -> - {normalize_cache_directive_name(RawName), true, Raw}. - -%% @doc Normalize only the case-insensitive directive name. -normalize_cache_directive_name(Name) -> - hb_ao:normalize_key(string:lowercase(Name)). - -%% @doc Bound numeric parser output while preserving extension values. -normalize_cache_directive_value(_Name, Value) when is_integer(Value) -> - min(max(0, Value), ?MAX_DELTA_SECONDS); -normalize_cache_directive_value(Name, Value) -> - Unquoted = hb_util:unquote(Value), - case lists:member( - Name, - [ - <<"max-age">>, - <<"max-stale">>, - <<"min-fresh">>, - <<"stale-while-revalidate">>, - <<"stale-if-error">> - ] - ) of - false -> Unquoted; - true -> - case parse_delta_seconds(Unquoted) of - invalid -> Unquoted; - Delta -> Delta - end - end. - -%% @doc Parse OWS around `=' and quoted commas rejected by strict Cowlib. -parse_legacy_cache_directive(Directive) -> - case hb_util:split_depth_string_aware_single($=, Directive) of - {no_match, RawName, <<>>} -> - { - true, - { - normalize_cache_directive_name(string:trim(RawName)), - true, - Directive - } - }; - {_Match, RawName, RawValue} -> - Name = normalize_cache_directive_name(string:trim(RawName)), - Value = normalize_cache_directive_value( - Name, - string:trim(RawValue) - ), - {true, {Name, Value, Directive}} - end. - -%% @doc Return all normalized values for one directive name. -directive_values(Name, Parsed) -> - [Value || {Directive, Value, _Raw} <- Parsed, Directive =:= Name]. - -%% @doc Return whether a directive is present, regardless of its argument. -has_directive(Name, Parsed) -> - directive_values(Name, Parsed) =/= []. - -%% @doc Parse one non-negative delta-seconds value conservatively. -parse_delta_seconds(Value) when is_integer(Value), Value >= 0 -> - min(Value, ?MAX_DELTA_SECONDS); +%% @doc Parse one bounded non-negative delta-seconds value. +parse_delta_seconds(Value) + when is_integer(Value), Value >= 0, Value =< ?MAX_DELTA_SECONDS -> + Value; parse_delta_seconds(Value) when is_binary(Value) -> - Trimmed = string:trim(hb_util:unquote(Value)), - case is_decimal(Trimmed) of - false -> invalid; - true when byte_size(Trimmed) > 10 -> ?MAX_DELTA_SECONDS; - true -> min(binary_to_integer(Trimmed), ?MAX_DELTA_SECONDS) - end; -parse_delta_seconds(_) -> - invalid. - -%% @doc Return whether a binary is a non-empty ASCII decimal number. -is_decimal(<<>>) -> false; -is_decimal(Bin) -> - lists:all( - fun(Char) -> Char >= $0 andalso Char =< $9 end, - binary_to_list(Bin) - ). + try parse_delta_seconds(binary_to_integer(string:trim(Value))) + catch _:_ -> invalid end; +parse_delta_seconds(_) -> invalid. -%% @doc Parse a directive that must occur once with a delta-seconds value. +%% @doc Return one numeric directive, rejecting malformed duplicates. +numeric_directive(_Name, invalid) -> invalid; numeric_directive(Name, Parsed) -> - case directive_values(Name, Parsed) of - [] -> undefined; - [Value] -> parse_delta_seconds(Value); + case {lists:member(Name, Parsed), [Value || {Key, Value} <- Parsed, Key =:= Name]} of + {false, []} -> undefined; + {false, [Value]} -> parse_delta_seconds(Value); _ -> invalid end. -%% @doc Parse request max-stale, including its valueless form. -max_stale_directive(Parsed) -> - case directive_values(<<"max-stale">>, Parsed) of - [] -> absent; - [true] -> any; - [Value] -> - case parse_delta_seconds(Value) of - invalid -> absent; - Delta -> Delta - end; - _ -> absent - end. - -%% @doc Combine header and compatibility max-age using the stricter value. -combine_max_age(undefined, undefined) -> undefined; -combine_max_age(invalid, _) -> invalid; -combine_max_age(_, invalid) -> invalid; -combine_max_age(undefined, Legacy) -> Legacy; -combine_max_age(Header, undefined) -> Header; -combine_max_age(Header, Legacy) -> min(Header, Legacy). - -%% @doc Parse the legacy top-level request max-age compatibility field. -legacy_max_age(Msg, Opts) -> - case hb_maps:find(<<"max-age">>, Msg, Opts) of - error -> undefined; - {ok, <<"infinity">>} -> undefined; - {ok, infinity} -> undefined; - {ok, Value} -> parse_delta_seconds(Value) - end. - -%% @doc Build the request-side cache policy without response directives. -request_policy(Msg, Opts) -> - Parsed = message_cache_control(Msg, Opts), - HeaderMaxAge = numeric_directive(<<"max-age">>, Parsed), - #{ - <<"max-age">> => - combine_max_age(HeaderMaxAge, legacy_max_age(Msg, Opts)), - <<"max-stale">> => max_stale_directive(Parsed), - <<"min-fresh">> => numeric_directive(<<"min-fresh">>, Parsed), - <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), - <<"no-store">> => has_directive(<<"no-store">>, Parsed), - <<"only-if-cached">> => - has_directive(<<"only-if-cached">>, Parsed) - }. - -%% @doc Build the response-side cache policy without request directives. -response_policy(Msg, Opts) -> - Parsed = message_cache_control(Msg, Opts), - #{ - <<"max-age">> => numeric_directive(<<"max-age">>, Parsed), - <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), - <<"no-store">> => has_directive(<<"no-store">>, Parsed), - <<"private">> => has_directive(<<"private">>, Parsed), - <<"must-revalidate">> => - has_directive(<<"must-revalidate">>, Parsed) - }. - -%% @doc Parse the Cache-Control field of an AO message when present. -message_cache_control(Msg, Opts) -> - case hb_maps:find(<<"cache-control">>, Msg, Opts) of - {ok, Raw} -> parse_cache_control(Raw); - _ -> [] - end. - -%% @doc Classify a cached result using already loaded timing metadata. -classify_cached(Res, ReqPolicy, ResPolicy, Now) -> - case cache_policy_prohibition(ReqPolicy, ResPolicy) of - none -> classify_cached_age(Res, ReqPolicy, ResPolicy, Now); - Reason -> {unacceptable, Reason} - end. +%% @doc Return whether one argumentless directive is present. +has_directive(_Name, invalid) -> false; +has_directive(Name, Parsed) -> lists:member(Name, Parsed). -%% @doc Return the first rule that requires validation or prohibits reuse. -cache_policy_prohibition(ReqPolicy, ResPolicy) -> - case { - maps:get(<<"no-cache">>, ReqPolicy), - maps:get(<<"no-store">>, ResPolicy), - maps:get(<<"private">>, ResPolicy), - maps:get(<<"no-cache">>, ResPolicy), - maps:get(<<"max-age">>, ReqPolicy), - maps:get(<<"min-fresh">>, ReqPolicy), - maps:get(<<"max-age">>, ResPolicy) - } of - {true, _, _, _, _, _, _} -> request_no_cache; - {_, true, _, _, _, _, _} -> response_no_store; - {_, _, true, _, _, _, _} -> response_private; - {_, _, _, true, _, _, _} -> response_no_cache; - {_, _, _, _, invalid, _, _} -> invalid_request_max_age; - {_, _, _, _, _, invalid, _} -> invalid_request_min_fresh; - {_, _, _, _, _, _, invalid} -> invalid_response_max_age; - _ -> none - end. +%% @doc Parse Cache-Control directly from a map-shaped message. +message_cache_control(Msg, Opts) when is_map(Msg) -> + parse_cache_control(hb_maps:get(<<"cache-control">>, Msg, [], Opts)); +message_cache_control(_, _) -> []. -%% @doc Calculate age only when some request or response rule needs it. -classify_cached_age(Res, ReqPolicy, ResPolicy, Now) -> - case age_required(ReqPolicy, ResPolicy) of - false -> {fresh, undefined}; - true -> - case cached_age(Res, Now) of - {ok, Age} -> classify_cached_lifetime(Age, ReqPolicy, ResPolicy); - error -> {unacceptable, invalid_created_at} - end - end. - -%% @doc Return whether the decision depends upon a stored timestamp. -age_required(ReqPolicy, ResPolicy) -> - maps:get(<<"max-age">>, ReqPolicy) =/= undefined orelse - maps:get(<<"min-fresh">>, ReqPolicy) =/= undefined orelse - maps:get(<<"max-stale">>, ReqPolicy) =/= absent orelse - maps:get(<<"max-age">>, ResPolicy) =/= undefined orelse - maps:get(<<"must-revalidate">>, ResPolicy). - -%% @doc Read a bounded immediate timestamp without invoking the cache/store. -cached_age(Res, Now) when is_map(Res), is_integer(Now), Now >= 0 -> - try hb_util:int(maps:get(<<"priv-created-at">>, Res)) of - CreatedAt when CreatedAt >= 0, CreatedAt =< Now -> {ok, Now - CreatedAt}; - _ -> error - catch - _:_ -> error - end; -cached_age(_, _) -> - error. - -%% @doc Apply request ceilings before the response freshness lifetime. -classify_cached_lifetime(Age, ReqPolicy, ResPolicy) -> - case maps:get(<<"max-age">>, ReqPolicy) of - MaxAge when is_integer(MaxAge), Age > MaxAge -> - {unacceptable, request_max_age}; - _ -> - classify_response_lifetime( - Age, - ReqPolicy, - ResPolicy, - maps:get(<<"max-age">>, ResPolicy) - ) - end. - -%% @doc Preserve legacy hits when no response lifetime-dependent rule exists. -classify_response_lifetime(Age, ReqPolicy, ResPolicy, undefined) -> - case { - maps:get(<<"min-fresh">>, ReqPolicy), - maps:get(<<"max-stale">>, ReqPolicy), - maps:get(<<"must-revalidate">>, ResPolicy) - } of - {undefined, absent, false} -> {fresh, Age}; - _ -> {unacceptable, response_lifetime_missing} - end; -classify_response_lifetime(Age, ReqPolicy, ResPolicy, Lifetime) -> - case min_fresh_satisfied(Age, Lifetime, ReqPolicy) of - false -> {unacceptable, min_fresh}; - true when Age < Lifetime -> {fresh, Age}; - true -> classify_stale(Age, Lifetime, ReqPolicy, ResPolicy) - end. - -%% @doc Require both freshness and the requested remaining lifetime. -min_fresh_satisfied(Age, Lifetime, ReqPolicy) -> - case maps:get(<<"min-fresh">>, ReqPolicy) of - undefined -> true; - MinFresh -> Age < Lifetime andalso Lifetime - Age >= MinFresh - end. - -%% @doc Permit stale reuse only through request max-stale. -classify_stale(Age, Lifetime, ReqPolicy, ResPolicy) -> - case maps:get(<<"must-revalidate">>, ResPolicy) of - true -> {unacceptable, must_revalidate}; - false -> - StaleBy = Age - Lifetime, - case maps:get(<<"max-stale">>, ReqPolicy) of - any -> {stale_allowed, Age, StaleBy, max_stale}; - MaxStale when is_integer(MaxStale), StaleBy =< MaxStale -> - {stale_allowed, Age, StaleBy, max_stale}; - _ -> {unacceptable, stale} - end - end. - -%% @doc Convert Cache-Control input into legacy store and lookup settings. +%% @doc Convert a cache control list as received via HTTP headers into a +%% normalized map of simply whether we should store and/or lookup the result. specifiers_to_cache_settings(CCSpecifier) -> - Parsed = parse_cache_control(CCSpecifier), + CCList = parse_cache_control(CCSpecifier), #{ <<"store">> => - case has_directive(<<"always">>, Parsed) of + case has_directive(<<"always">>, CCList) of true -> true; false -> - case has_directive(<<"no-store">>, Parsed) of + case has_directive(<<"no-store">>, CCList) of true -> false; false -> - case has_directive(<<"store">>, Parsed) of + case has_directive(<<"store">>, CCList) of true -> true; false -> undefined end end end, <<"lookup">> => - case has_directive(<<"always">>, Parsed) of + case has_directive(<<"always">>, CCList) of true -> true; false -> - case has_directive(<<"no-cache">>, Parsed) of + case has_directive(<<"no-cache">>, CCList) of true -> false; false -> - case has_directive(<<"cache">>, Parsed) of + case has_directive(<<"cache">>, CCList) of true -> true; false -> undefined end end end, <<"only-if-cached">> => - case has_directive(<<"only-if-cached">>, Parsed) of + case has_directive(<<"only-if-cached">>, CCList) of true -> true; false -> undefined end @@ -796,18 +703,34 @@ cache_result_max_age_test() -> hb_maps:get(<<"priv-created-at">>, Served, not_found, Opts) ), CacheAddress = hb_cache:resolved_address(Base, Req, Opts), + StaleCreatedAt = os:system_time(second) - 61, {ok, _} = hb_cache:write_resolved( CacheAddress, Result, - os:system_time(second) - 61, + StaleCreatedAt, Opts ), + ?assertMatch( + {error, #{ <<"status">> := 504 }}, + hb_ao:resolve( + Base, + Req, + Opts#{ <<"cache-control">> => [<<"only-if-cached">>] } + ) + ), {ok, Refreshed} = hb_ao:resolve(Base, Req, Opts), ?assertEqual( not_found, hb_maps:get(<<"priv-created-at">>, Refreshed, not_found, Opts) ), - ?assertNotEqual(Served, Refreshed). + {hit, {ok, RefreshedEntry}} = hb_cache:read_resolved(Base, Req, Opts), + ?assert( + hb_util:int(hb_maps:get( + <<"priv-created-at">>, + RefreshedEntry, + Opts + )) > StaleCreatedAt + ). cache_result_priv_created_at_regression_test() -> Wallet = ar_wallet:new(), @@ -863,7 +786,8 @@ cache_result_priv_created_at_regression_test() -> erlang:put({hb_event, event_opts}, CacheOnlyOpts), Served = try - {ok, CachedResult} = maybe_lookup(Base, Req, CacheOnlyOpts), + {PreparedReq, PreparedOpts} = prepare(Req, CacheOnlyOpts), + {ok, CachedResult} = maybe_lookup(Base, PreparedReq, PreparedOpts), CachedResult after case OldEventOpts of @@ -936,7 +860,7 @@ cache_binary_max_age_remains_available_without_age_test() -> maybe_lookup(Base, ReqWithoutMaxAge, CacheOnlyOpts) ). -cache_map_without_max_age_is_reusable_test() -> +cache_map_tracked_then_reusable_without_max_age_test() -> Opts = #{ <<"store">> => [hb_test_utils:test_store()], <<"cache-control">> => [<<"always">>] @@ -954,274 +878,163 @@ cache_map_without_max_age_is_reusable_test() -> hb_cache:resolved_address(Base, ReqWithMaxAge, Opts) ), {ok, _} = perform_cache_write(Base, Req, Result, Opts), + ResultID = hb_message:id( + maps:remove(<<"priv-created-at">>, Result), + all, + Opts + ), + {ok, Canonical} = hb_cache:read(ResultID, Opts), + ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Canonical)), + ?assertEqual(miss, hb_cache:read_resolved(Base, Req, Opts)), + {ok, _} = perform_cache_write(Base, ReqWithMaxAge, Result, Opts), {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, Opts), - ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Cached)), + ?assert(maps:is_key(<<"priv-created-at">>, Cached)), CacheOnlyOpts = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, - ?assertMatch( - {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, - maybe_lookup(Base, ReqWithMaxAge, CacheOnlyOpts) - ), {ok, Served} = maybe_lookup(Base, Req, CacheOnlyOpts), ?assertEqual(<<"cached">>, hb_maps:get(<<"value">>, Served, Opts)), ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)). -fresh_uses_loaded_metadata_without_store_read_test() -> - Parent = self(), - Tracer = spawn(fun() -> cache_control_trace_forwarder(Parent) end), - erlang:trace(self(), true, [call, {tracer, Tracer}]), - erlang:trace_pattern({hb_store, read, 3}, true, []), - IsFresh = - try - fresh( - #{ - <<"priv-created-at">> => - integer_to_binary(os:system_time(second)) - }, - #{ <<"max-age">> => 60 }, - #{} - ) - after - stop_cache_control_trace(Tracer) - end, - ?assert(IsFresh), - receive - {trace_event, {trace, _, call, {hb_store, read, _}}} -> - ?assert(false) - after 0 -> - ok - end. - -%% @doc Disable store-read tracing and stop its forwarding process. -stop_cache_control_trace(Tracer) -> - erlang:trace_pattern({hb_store, read, 3}, false, []), - erlang:trace(self(), false, [call]), - TraceRef = erlang:trace_delivered(self()), - Delivered = - receive - {trace_delivered, _, TraceRef} -> true - after 1000 -> - false - end, - Tracer ! {flush, self()}, - Flushed = - receive - trace_flushed -> true - after 1000 -> - false - end, - Tracer ! stop, - case {Delivered, Flushed} of - {true, true} -> ok; - {false, _} -> error(trace_delivery_timeout); - {_, false} -> error(trace_flush_timeout) - end. +%% @doc Cover the exact freshness, stale, and prohibition boundaries. +minimal_freshness_policy_test() -> + Req0 = #{ <<"max-age">> => undefined, <<"max-stale">> => absent }, + Res0 = #{ <<"max-age">> => undefined, <<"no-cache">> => false, + <<"must-revalidate">> => false }, + Res60 = Res0#{ <<"max-age">> => 60 }, + Cases = [ + {{fresh, 59}, cached_at(41), Req0, Res60}, + {{unacceptable, stale}, cached_at(40), Req0, Res60}, + {{fresh, 60}, cached_at(40), Req0#{ <<"max-age">> => 60 }, Res0}, + {{unacceptable, request_max_age}, cached_at(39), Req0#{ <<"max-age">> => 60 }, Res0}, + {{stale_allowed, 70, 10}, cached_at(30), Req0#{ <<"max-stale">> => 10 }, Res60}, + {{unacceptable, stale}, cached_at(29), Req0#{ <<"max-stale">> => 10 }, Res60}, + {{stale_allowed, 90, 30}, cached_at(10), Req0#{ <<"max-stale">> => any }, Res60}, + {{unacceptable, response_lifetime_missing}, cached_at(30), Req0#{ <<"max-stale">> => 10 }, Res0}, + {{unacceptable, response_no_cache}, cached_at(90), Req0, Res60#{ <<"no-cache">> => true }}, + {{unacceptable, must_revalidate}, cached_at(30), Req0#{ <<"max-stale">> => any }, + Res60#{ <<"must-revalidate">> => true }}, + {{unacceptable, invalid_created_at}, #{}, Req0, Res60}, + {{unacceptable, invalid_created_at}, #{ <<"priv-created-at">> => <<"invalid">> }, Req0, Res60}, + {{unacceptable, invalid_created_at}, cached_at(-1), Req0, Res60}, + {{unacceptable, invalid_created_at}, cached_at(101), Req0, Res60} + ], + lists:foreach(fun({Expected, Res, Req, Policy}) -> + ?assertEqual(Expected, classify_cached(Res, Req, Policy, 100, #{})) + end, Cases), + ?assertEqual( + {fresh, undefined}, + classify_cached(#{}, Req0, Res0, 100, #{}) + ). -%% @doc Missing or invalid private timestamps fail closed when max-age applies. -freshness_private_metadata_validation_test() -> - Now = os:system_time(second), - Req = #{ <<"max-age">> => 60 }, - ?assert(fresh(#{}, #{}, #{})), - ?assertNot(fresh(#{}, Req, #{})), - ?assertNot(fresh(#{ <<"priv-created-at">> => <<"invalid">> }, Req, #{})), - ?assertNot(fresh(#{ <<"priv-created-at">> => <<"-1">> }, Req, #{})), - ?assertNot( - fresh( - #{ <<"priv-created-at">> => integer_to_binary(Now + 1) }, - Req, - #{} - ) +%% @doc Consume only uncommitted policy and reject malformed lifetimes. +minimal_policy_parsing_and_identity_test() -> + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, + <<"max-stale">> => true }, + {SemanticReq, PreparedOpts} = prepare(Req, #{}), + ?assertEqual(#{ <<"path">> => <<"index">> }, SemanticReq), + ?assertEqual(#{ <<"max-age">> => 60, <<"max-stale">> => any }, + maps:get(?REQUEST_POLICY_OPT, PreparedOpts)), + ?assertEqual(response_policy(#{ <<"cache-control">> => <<"max-age=60">> }, #{}), + response_policy(#{ <<"cache-control">> => [<<"max-age=60">>] }, #{})), + lists:foreach(fun(CC) -> + Policy = response_policy(#{ <<"cache-control">> => CC }, #{}), + ?assertEqual(invalid, maps:get(<<"max-age">>, Policy)) + end, [<<"max-age">>, <<"max-age=-1">>, <<"max-age=nope">>, + <<"max-age=999999999999999999999">>, <<"max-age=60, max-age=30">>]), + WalletOpts = #{ <<"priv-wallet">> => ar_wallet:new(), + <<"commitment-device">> => <<"httpsig@1.0">> }, + Commit = fun(MaxAge) -> + hb_message:commit(Req#{ <<"max-age">> => MaxAge }, WalletOpts, + #{ <<"committed">> => [<<"path">>, <<"max-age">>, <<"max-stale">>] }) + end, + {Committed60, _} = prepare(Commit(60), WalletOpts), + {Committed30, _} = prepare(Commit(30), WalletOpts), + ?assert(hb_message:verify(Committed60, all, WalletOpts)), + Base = #{ <<"device">> => <<"test-device@1.0">> }, + ?assertNotEqual(hb_cache:resolved_address(Base, Committed60, WalletOpts), + hb_cache:resolved_address(Base, Committed30, WalletOpts)). + +%% @doc Resolver policy is absent from grouping and device inputs. +minimal_request_policy_is_resolver_private_test() -> + Parent = self(), + Grouper = fun(_, Req, Opts) -> + Parent ! {policy_seen, Req, Opts}, ungrouped_exec + end, + Handler = fun(_, Req, Opts) -> + Parent ! {policy_seen, Req, Opts}, {ok, #{ <<"body">> => <<"ok">> }} + end, + Base = #{ <<"device">> => #{ + info => fun(_, _) -> #{ grouper => Grouper } end, + index => Handler + } }, + {ok, _} = hb_ao:resolve( + Base, + #{ <<"path">> => <<"index">>, <<"max-age">> => 60, + <<"max-stale">> => true }, + #{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } ), - ?assert( - fresh( - #{ <<"priv-created-at">> => integer_to_binary(Now) }, - Req, - #{} - ) - ). + assert_policy_hidden(), + assert_policy_hidden(), + CacheOnlyReq = #{ <<"path">> => <<"index">>, <<"cache-control">> => + [<<"no-cache">>, <<"only-if-cached">>] }, + ?assertMatch({error, #{ <<"status">> := 504 }}, + hb_ao:resolve(Base, CacheOnlyReq, #{})). + +inherited_no_cache_does_not_preempt_device_cache_only_test() -> + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">>, <<"cache-control">> => [<<"only-if-cached">>] }, + ?assertMatch({ok, _}, hb_ao:resolve(Base, Req, #{})). -%% @doc Forward test trace events and provide an ordered flush barrier. -cache_control_trace_forwarder(Parent) -> +assert_policy_hidden() -> receive - {flush, ReplyTo} -> - ReplyTo ! trace_flushed, - cache_control_trace_forwarder(Parent); - stop -> - ok; - TraceEvent -> - Parent ! {trace_event, TraceEvent}, - cache_control_trace_forwarder(Parent) + {policy_seen, Req, Opts} -> + ?assertEqual(false, maps:is_key(<<"max-age">>, Req)), + ?assertEqual(false, maps:is_key(<<"max-stale">>, Req)), + ?assertEqual(false, maps:is_key(?REQUEST_POLICY_OPT, Opts)) + after 1000 -> + error(policy_boundary_not_observed) end. -%% @doc Parse wire and list Cache-Control forms without losing quoted commas. -cache_control_parser_regression_test() -> - Parsed = parse_cache_control( - <<"MaX-aGe = \"60\", only-if-cached, community = \"A,B\"">> - ), - ?assertEqual([60], directive_values(<<"max-age">>, Parsed)), - ?assertEqual([true], directive_values(<<"only-if-cached">>, Parsed)), - ?assertEqual([<<"A,B">>], directive_values(<<"community">>, Parsed)), - ListParsed = parse_cache_control([<<"always">>, <<"NO-STORE">>]), - ?assertEqual([true], directive_values(<<"always">>, ListParsed)), - ?assertEqual([true], directive_values(<<"no-store">>, ListParsed)). - -%% @doc Request policy handles numeric, bare, duplicate, and invalid forms. -request_cache_policy_regression_test() -> - Policy = request_policy( - #{ - <<"cache-control">> => - <<"max-age=60, max-stale=10, min-fresh=5, only-if-cached">> - }, - #{} - ), - ?assertEqual(60, maps:get(<<"max-age">>, Policy)), - ?assertEqual(10, maps:get(<<"max-stale">>, Policy)), - ?assertEqual(5, maps:get(<<"min-fresh">>, Policy)), - ?assert(maps:get(<<"only-if-cached">>, Policy)), - Bare = request_policy( - #{ <<"cache-control">> => <<"max-stale">> }, - #{} - ), - ?assertEqual(any, maps:get(<<"max-stale">>, Bare)), - Duplicate = request_policy( - #{ <<"cache-control">> => <<"max-age=60, max-age=30">> }, - #{} - ), - ?assertEqual(invalid, maps:get(<<"max-age">>, Duplicate)), - Invalid = request_policy( - #{ <<"cache-control">> => <<"min-fresh=-1, max-stale=nope">> }, - #{} - ), - ?assertEqual(invalid, maps:get(<<"min-fresh">>, Invalid)), - ?assertEqual(absent, maps:get(<<"max-stale">>, Invalid)), - Overflow = request_policy( - #{ <<"cache-control">> => <<"max-age=999999999999999999999">> }, - #{} - ), - ?assertEqual(2147483647, maps:get(<<"max-age">>, Overflow)). - -%% @doc Response policy remains independent from request-only directives. -response_cache_policy_regression_test() -> - Policy = response_policy( - #{ - <<"cache-control">> => - <<"max-age=60, no-cache=\"body\", private=\"secret\", " - "must-revalidate, max-stale=99">> - }, - #{} - ), - ?assertEqual(60, maps:get(<<"max-age">>, Policy)), - ?assert(maps:get(<<"no-cache">>, Policy)), - ?assert(maps:get(<<"private">>, Policy)), - ?assert(maps:get(<<"must-revalidate">>, Policy)), - ?assertEqual(false, maps:is_key(<<"max-stale">>, Policy)), - Duplicate = response_policy( - #{ <<"cache-control">> => <<"max-age=60, max-age=30">> }, - #{} - ), - ?assertEqual(invalid, maps:get(<<"max-age">>, Duplicate)). - -%% @doc Freshness boundaries use explicit time and the correct policy side. -cache_freshness_classifier_boundaries_test() -> - Now = 1000, - Response60 = response_policy( - #{ <<"cache-control">> => <<"max-age=60">> }, - #{} - ), - EmptyRequest = request_policy(#{}, #{}), - ?assertEqual( - {fresh, 59}, - classify_cached(cache_entry(Now - 59), EmptyRequest, Response60, Now) - ), - ?assertEqual( - {unacceptable, stale}, - classify_cached(cache_entry(Now - 60), EmptyRequest, Response60, Now) - ), - Request60 = request_policy(#{ <<"max-age">> => 60 }, #{}), - ?assertEqual( - {fresh, 60}, - classify_cached( - cache_entry(Now - 60), - Request60, - response_policy( - #{ <<"cache-control">> => <<"max-age=120">> }, - #{} - ), - Now - ) - ), - Request0 = request_policy(#{ <<"max-age">> => 0 }, #{}), - ?assertEqual( - {fresh, 0}, - classify_cached(cache_entry(Now), Request0, Response60, Now) - ), - MinFresh = request_policy( - #{ <<"cache-control">> => <<"min-fresh=10">> }, - #{} - ), - ?assertEqual( - {fresh, 50}, - classify_cached(cache_entry(Now - 50), MinFresh, Response60, Now) - ). - -%% @doc Max-stale is measured from the response freshness lifetime. -cache_max_stale_classifier_regression_test() -> - Now = 1000, - Response60 = response_policy( - #{ <<"cache-control">> => <<"max-age=60">> }, - #{} - ), - MaxStale10 = request_policy( - #{ <<"cache-control">> => <<"max-stale=10">> }, - #{} - ), - ?assertEqual( - {stale_allowed, 70, 10, max_stale}, - classify_cached(cache_entry(Now - 70), MaxStale10, Response60, Now) - ), - ?assertEqual( - {unacceptable, stale}, - classify_cached(cache_entry(Now - 71), MaxStale10, Response60, Now) - ), - Bare = request_policy( - #{ <<"cache-control">> => <<"max-stale">> }, - #{} - ), - ?assertEqual( - {stale_allowed, 500, 440, max_stale}, - classify_cached(cache_entry(Now - 500), Bare, Response60, Now) +%% @doc Response max-age stores time and max-stale controls live cache reuse. +cache_response_max_age_and_request_max_stale_test() -> + Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, + Req = #{ <<"path">> => <<"index">> }, + Result = #{ <<"body">> => <<"cached">>, + <<"cache-control">> => <<"max-age=60">> }, + Opts = #{ <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] }, + {ok, _} = perform_cache_write(Base, Req, Result, Opts), + Address = hb_cache:resolved_address(Base, Req, Opts), + {hit, {ok, Stored}} = hb_cache:read_resolved(Base, Req, Opts), + ?assert(is_integer(hb_util:int( + hb_maps:get(<<"priv-created-at">>, Stored, Opts)))), + {ok, _} = hb_cache:write_resolved( + Address, Result, os:system_time(second) - 70, Opts), + CacheOnly = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, + {AllowedReq, AllowedOpts} = prepare(Req#{ <<"max-stale">> => 100 }, CacheOnly), + {ok, _Stale} = maybe_lookup(Base, AllowedReq, AllowedOpts), + {DeniedReq, DeniedOpts} = prepare(Req#{ <<"max-stale">> => 0 }, CacheOnly), + ?assertMatch( + {error, #{ <<"status">> := 504 }}, + maybe_lookup(Base, DeniedReq, DeniedOpts) ), - ?assertEqual( - {unacceptable, response_lifetime_missing}, - classify_cached( - cache_entry(Now - 1), - Bare, - response_policy(#{}, #{}), - Now - ) - ). - -%% @doc Age-dependent decisions reject unusable co-located metadata. -cache_freshness_metadata_regression_test() -> - Now = 1000, - Req = request_policy(#{ <<"max-age">> => 60 }, #{}), - Res = response_policy(#{}, #{}), - lists:foreach( - fun(Candidate) -> - ?assertEqual( - {unacceptable, invalid_created_at}, - classify_cached(Candidate, Req, Res, Now) - ) - end, - [ - #{}, - #{ <<"priv-created-at">> => <<"invalid">> }, - #{ <<"priv-created-at">> => <<"-1">> }, - #{ <<"priv-created-at">> => <<"1001">> }, - #{ <<"priv-created-at">> => {link, <<"missing">>} }, - <<"binary-candidate">> - ] - ). - -%% @doc Build a deterministic cached map for classifier tests. -cache_entry(CreatedAt) -> + ?assertMatch({continue, _, _}, maybe_lookup(Base, Req, Opts)). + +%% @doc Uncommitted response policy cannot outlive signed cache storage. +signed_uncommitted_response_policy_is_not_tracked_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new(), + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"store">> => [hb_test_utils:test_store()] }, + Base = #{ <<"device">> => <<"test-device@1.0">> }, + Req = #{ <<"path">> => <<"index">> }, + Result = hb_message:commit( + #{ <<"body">> => <<"cached">>, <<"cache-control">> => <<"max-age=60">> }, + Opts, #{ <<"committed">> => [<<"body">>] }), + {ok, ResultID} = perform_cache_write(Base, Req, Result, Opts), + {ok, Stored} = hb_cache:read(ResultID, Opts), + ?assertNot(maps:is_key(<<"cache-control">>, Stored)), + ?assertNot(maps:is_key(<<"priv-created-at">>, Stored)), + ?assertEqual(miss, hb_cache:read_resolved(Base, Req, Opts)). + +%% @doc Build an already-loaded cache record with a private creation time. +cached_at(CreatedAt) -> #{ <<"priv-created-at">> => integer_to_binary(CreatedAt) }. From cd9c380417ae20b7a782a80afd0f461ddb8fab7e Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 10:35:21 -0400 Subject: [PATCH 06/10] fix: remove hb_cache_control:prepare --- src/core/resolver/hb_ao.erl | 4 +- src/core/resolver/hb_cache_control.erl | 129 ++++++++----------------- 2 files changed, 42 insertions(+), 91 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index fc7061c5f..6969eaae6 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -111,7 +111,6 @@ <<"add-key">>, <<"force-message">>, <<"cache-control">>, - <<"priv-cache-request-policy">>, <<"spawn-worker">>, <<"only">>, <<"prefer">> @@ -416,8 +415,7 @@ resolve_stage(1, RawBase, RawReq, Opts) -> ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), - {SemanticReq, PreparedOpts} = hb_cache_control:prepare(Req, Opts), - resolve_stage(2, Base, SemanticReq, PreparedOpts); + resolve_stage(2, Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 1d098266d..9abe390d0 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/4, maybe_lookup/3, prepare/2]). +-export([maybe_store/4, maybe_lookup/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -11,33 +11,10 @@ %%% following settings. -define(DEFAULT_STORE_OPT, false). -define(DEFAULT_LOOKUP_OPT, true). --define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). -define(MAX_DELTA_SECONDS, 2147483647). %%% Public API -%% @doc Remove uncommitted request freshness policy before AO execution. -prepare(Req, Opts) when is_map(Req), - not is_map_key(<<"max-age">>, Req), - not is_map_key(<<"max-stale">>, Req) -> - {Req, Opts}; -prepare(Req, Opts) -> - SemanticReq = hb_message:without_unless_signed( - [<<"max-age">>, <<"max-stale">>], - Req, - Opts - ), - Policy = #{ - <<"max-age">> => removed_max_age(Req, SemanticReq), - <<"max-stale">> => removed_max_stale(Req, SemanticReq) - }, - case Policy of - #{ <<"max-age">> := undefined, <<"max-stale">> := absent } -> - {SemanticReq, Opts}; - _ -> - {SemanticReq, Opts#{ ?REQUEST_POLICY_OPT => Policy }} - end. - %% @doc Write a resulting M3 message to the cache if requested. The precedence %% order of cache control sources is as follows: %% 1. The `Opts' map (letting the node operator have the final say). @@ -190,39 +167,29 @@ cached_age(Res, Now, Opts) when is_map(Res), is_integer(Now), Now >= 0 -> end; cached_age(_, _, _) -> error. -%% @doc Return a removed, uncommitted request max-age value. -removed_max_age(Req, SemanticReq) -> - case removed_value(<<"max-age">>, Req, SemanticReq) of +%% @doc Return the request max-age policy. +request_max_age(Req, Opts) -> + case hb_maps:get(<<"max-age">>, Req, undefined, Opts) of undefined -> undefined; infinity -> undefined; <<"infinity">> -> undefined; Value -> parse_delta_seconds(Value) end. -%% @doc Return a removed, uncommitted request max-stale value. -removed_max_stale(Req, SemanticReq) -> - case removed_value(<<"max-stale">>, Req, SemanticReq) of +%% @doc Return the request max-stale policy. +request_max_stale(Req, Opts) -> + case hb_maps:get(<<"max-stale">>, Req, undefined, Opts) of undefined -> absent; true -> any; Value -> parse_delta_seconds(Value) end. -%% @doc Return a field only when preparation removed it from the request. -removed_value(Key, Req, SemanticReq) -> - case {maps:find(Key, Req), maps:is_key(Key, SemanticReq)} of - {{ok, Value}, false} -> Value; - _ -> undefined - end. - -%% @doc Read request policy prepared by AO stage 1, with a direct-call fallback. +%% @doc Read freshness policy directly from the request. request_policy(Req, Opts) -> - case maps:find(?REQUEST_POLICY_OPT, Opts) of - {ok, Policy} -> Policy; - error -> - {_SemanticReq, PreparedOpts} = prepare(Req, Opts), - maps:get(?REQUEST_POLICY_OPT, PreparedOpts, - #{ <<"max-age">> => undefined, <<"max-stale">> => absent }) - end. + #{ + <<"max-age">> => request_max_age(Req, Opts), + <<"max-stale">> => request_max_stale(Req, Opts) + }. %% @doc Read the response lifetime and rules that prohibit reuse. response_policy(Res, Opts) -> @@ -786,8 +753,7 @@ cache_result_priv_created_at_regression_test() -> erlang:put({hb_event, event_opts}, CacheOnlyOpts), Served = try - {PreparedReq, PreparedOpts} = prepare(Req, CacheOnlyOpts), - {ok, CachedResult} = maybe_lookup(Base, PreparedReq, PreparedOpts), + {ok, CachedResult} = maybe_lookup(Base, Req, CacheOnlyOpts), CachedResult after case OldEventOpts of @@ -925,14 +891,12 @@ minimal_freshness_policy_test() -> classify_cached(#{}, Req0, Res0, 100, #{}) ). -%% @doc Consume only uncommitted policy and reject malformed lifetimes. +%% @doc Parse request policy and keep only uncommitted policy out of identity. minimal_policy_parsing_and_identity_test() -> Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, <<"max-stale">> => true }, - {SemanticReq, PreparedOpts} = prepare(Req, #{}), - ?assertEqual(#{ <<"path">> => <<"index">> }, SemanticReq), ?assertEqual(#{ <<"max-age">> => 60, <<"max-stale">> => any }, - maps:get(?REQUEST_POLICY_OPT, PreparedOpts)), + request_policy(Req, #{})), ?assertEqual(response_policy(#{ <<"cache-control">> => <<"max-age=60">> }, #{}), response_policy(#{ <<"cache-control">> => [<<"max-age=60">>] }, #{})), lists:foreach(fun(CC) -> @@ -946,34 +910,33 @@ minimal_policy_parsing_and_identity_test() -> hb_message:commit(Req#{ <<"max-age">> => MaxAge }, WalletOpts, #{ <<"committed">> => [<<"path">>, <<"max-age">>, <<"max-stale">>] }) end, - {Committed60, _} = prepare(Commit(60), WalletOpts), - {Committed30, _} = prepare(Commit(30), WalletOpts), + Committed60 = Commit(60), + Committed30 = Commit(30), ?assert(hb_message:verify(Committed60, all, WalletOpts)), Base = #{ <<"device">> => <<"test-device@1.0">> }, + ?assertEqual( + hb_cache:resolved_address(Base, Req, #{}), + hb_cache:resolved_address(Base, Req#{ <<"max-age">> => 30 }, #{}) + ), + ?assertEqual(60, maps:get(<<"max-age">>, + request_policy(Committed60, WalletOpts))), ?assertNotEqual(hb_cache:resolved_address(Base, Committed60, WalletOpts), hb_cache:resolved_address(Base, Committed30, WalletOpts)). -%% @doc Resolver policy is absent from grouping and device inputs. -minimal_request_policy_is_resolver_private_test() -> - Parent = self(), - Grouper = fun(_, Req, Opts) -> - Parent ! {policy_seen, Req, Opts}, ungrouped_exec - end, - Handler = fun(_, Req, Opts) -> - Parent ! {policy_seen, Req, Opts}, {ok, #{ <<"body">> => <<"ok">> }} - end, - Base = #{ <<"device">> => #{ - info => fun(_, _) -> #{ grouper => Grouper } end, - index => Handler - } }, - {ok, _} = hb_ao:resolve( - Base, - #{ <<"path">> => <<"index">>, <<"max-age">> => 60, - <<"max-stale">> => true }, - #{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } - ), - assert_policy_hidden(), - assert_policy_hidden(), +%% @doc Request freshness policy remains visible to the executing device. +request_policy_remains_in_device_input_test() -> + Handler = fun(_, Req, _) -> {ok, Req} end, + Base = #{ <<"device">> => #{ index => Handler } }, + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, + <<"max-stale">> => true }, + {ok, Result} = hb_ao:resolve(Base, Req, + #{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }), + ?assertEqual(60, hb_maps:get(<<"max-age">>, Result, #{})), + ?assertEqual(true, hb_maps:get(<<"max-stale">>, Result, #{})). + +%% @doc An explicit cache-only request never executes with lookup disabled. +no_cache_only_if_cached_test() -> + Base = #{ <<"device">> => <<"test-device@1.0">> }, CacheOnlyReq = #{ <<"path">> => <<"index">>, <<"cache-control">> => [<<"no-cache">>, <<"only-if-cached">>] }, ?assertMatch({error, #{ <<"status">> := 504 }}, @@ -984,16 +947,6 @@ inherited_no_cache_does_not_preempt_device_cache_only_test() -> Req = #{ <<"path">> => <<"index">>, <<"cache-control">> => [<<"only-if-cached">>] }, ?assertMatch({ok, _}, hb_ao:resolve(Base, Req, #{})). -assert_policy_hidden() -> - receive - {policy_seen, Req, Opts} -> - ?assertEqual(false, maps:is_key(<<"max-age">>, Req)), - ?assertEqual(false, maps:is_key(<<"max-stale">>, Req)), - ?assertEqual(false, maps:is_key(?REQUEST_POLICY_OPT, Opts)) - after 1000 -> - error(policy_boundary_not_observed) - end. - %% @doc Response max-age stores time and max-stale controls live cache reuse. cache_response_max_age_and_request_max_stale_test() -> Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, @@ -1010,12 +963,12 @@ cache_response_max_age_and_request_max_stale_test() -> {ok, _} = hb_cache:write_resolved( Address, Result, os:system_time(second) - 70, Opts), CacheOnly = Opts#{ <<"cache-control">> => [<<"only-if-cached">>] }, - {AllowedReq, AllowedOpts} = prepare(Req#{ <<"max-stale">> => 100 }, CacheOnly), - {ok, _Stale} = maybe_lookup(Base, AllowedReq, AllowedOpts), - {DeniedReq, DeniedOpts} = prepare(Req#{ <<"max-stale">> => 0 }, CacheOnly), + AllowedReq = Req#{ <<"max-stale">> => 100 }, + {ok, _Stale} = maybe_lookup(Base, AllowedReq, CacheOnly), + DeniedReq = Req#{ <<"max-stale">> => 0 }, ?assertMatch( {error, #{ <<"status">> := 504 }}, - maybe_lookup(Base, DeniedReq, DeniedOpts) + maybe_lookup(Base, DeniedReq, CacheOnly) ), ?assertMatch({continue, _, _}, maybe_lookup(Base, Req, Opts)). From 5ce548cd62d74ebda8536d534870f970a8505ade Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 13:09:04 -0400 Subject: [PATCH 07/10] fix: restore scoped cache freshness policy Keep unsigned max-age and max-stale out of computation identity and device input while retaining them as resolver-only cache policy. Preserve co-located freshness metadata and commitments without adding a commitment scan to unsigned cache hits. --- scripts/lua-top-holders.lua | 3 + src/core/resolver/hb_ao.erl | 21 +- src/core/resolver/hb_cache.erl | 66 +++-- src/core/resolver/hb_cache_control.erl | 340 +++++++++++++++++++++---- src/preloaded/util/dev_test.erl | 13 +- 5 files changed, 373 insertions(+), 70 deletions(-) diff --git a/scripts/lua-top-holders.lua b/scripts/lua-top-holders.lua index d1bc42c82..4305ee93e 100644 --- a/scripts/lua-top-holders.lua +++ b/scripts/lua-top-holders.lua @@ -1,3 +1,6 @@ +-- Walk an AO balance trie while retaining only a bounded min-heap of the +-- strongest holders. Balances remain decimal strings so ranking is exact even +-- above Lua's numeric precision; length is compared before lexical value. local function decimal(value) local result = tostring(value):match("^0*(%d+)$") return result or "0" diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 6969eaae6..935f35cda 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -105,12 +105,15 @@ -export([deep_set/4]). -include("include/hb.hrl"). +-define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). + -define( TEMP_OPTS, [ <<"add-key">>, <<"force-message">>, <<"cache-control">>, + ?REQUEST_POLICY_OPT, <<"spawn-worker">>, <<"only">>, <<"prefer">> @@ -415,7 +418,8 @@ resolve_stage(1, RawBase, RawReq, Opts) -> ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), - resolve_stage(2, Base, Req, Opts); + {SemanticReq, PreparedOpts} = hb_cache_control:prepare(Req, Opts), + resolve_stage(2, Base, SemanticReq, PreparedOpts); resolve_stage(2, Base, Req, Opts) -> ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. @@ -470,7 +474,9 @@ resolve_stage(4, Base, Req, Opts) -> % There is another executor of this resolution in-flight. % Bail execution, register to receive the response, then % wait. - case hb_persistent:await(Leader, Base, Req, Opts) of + case hb_persistent:await( + Leader, Base, Req, without_request_policy(Opts) + ) of {error, leader_died} -> ?event( ao_core, @@ -760,8 +766,11 @@ resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> Res; {_, _} -> % Spawn a worker for the current execution - WorkerPID = hb_persistent:start_worker(ExecName, Res, Opts), - hb_persistent:forward_work(WorkerPID, Opts), + PersistentOpts = without_request_policy(Opts), + WorkerPID = hb_persistent:start_worker( + ExecName, Res, PersistentOpts + ), + hb_persistent:forward_work(WorkerPID, PersistentOpts), Res end; resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> @@ -1308,3 +1317,7 @@ execution_opts(Opts) -> recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 end. + +%% @doc Keep parsed request cache policy inside the resolver. +without_request_policy(Opts) -> + hb_maps:remove(?REQUEST_POLICY_OPT, Opts, Opts). diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index f1ee17ce5..091883168 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -41,7 +41,7 @@ -export([read_all_commitments/2]). -export([ensure_loaded/1, ensure_loaded/2, ensure_all_loaded/1, ensure_all_loaded/2]). -export([read/2, read_resolved/3, resolved_address/3]). --export([write/2, write_binary/3, write_hashpath/2, write_hashpath/3, write_resolved/4]). +-export([write/2, write_binary/3, write_hashpath/2, write_resolved/4]). -export([link/3]). -export([match/2, list/2, list_numbered/2]). -export([test_unsigned/1, test_signed/1]). @@ -268,21 +268,26 @@ generate_binary_path(Bin, Opts) -> %% the commitments of the inner messages. We do not, however, store the IDs from %% commitments on signed _inner_ messages. We may wish to revisit this. write(RawMsg, Opts) when is_map(RawMsg) -> - % Reattach only cache-owned freshness after private keys are filtered. - PrivCreatedAt = - case maps:find(<<"priv-created-at">>, RawMsg) of - {ok, CreatedAt} - when is_binary(CreatedAt), - byte_size(CreatedAt) < ?DIRECT_VALUE_LENGTH -> - #{ <<"priv-created-at">> => CreatedAt }; - _ -> - #{} + % Reattach only bounded cache-owned metadata after private keys are filtered. + PrivCacheMetadata = maps:filter( + fun(<<"priv-created-at">>, Value) -> + is_immediate_value(<<"priv-created-at">>, Value); + (<<"priv-has-commitments">>, Value) -> + Value =:= <<"true">> orelse Value =:= <<"false">> end, - PublicRawMsg = maps:remove(<<"priv-created-at">>, RawMsg), + maps:with( + [<<"priv-created-at">>, <<"priv-has-commitments">>], + RawMsg + ) + ), + PublicRawMsg = maps:without( + [<<"priv-created-at">>, <<"priv-has-commitments">>], + RawMsg + ), hb_message:paranoid_verify(cache_write, PublicRawMsg, Opts), {ok, Msg} = hb_message:with_only_committed(PublicRawMsg, Opts), PublicTABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), - TABM = maps:merge(PublicTABM, PrivCreatedAt), + TABM = maps:merge(PublicTABM, PrivCacheMetadata), ?event_debug(debug_cache, {writing_full_message, {msg, TABM}}), try do_write_message( @@ -682,12 +687,16 @@ write_binary(Hashpath, Bin, Store, Opts) -> %% @doc Write a resolved message with its private cache creation timestamp. write_resolved(CacheAddress, Msg, CreatedAt, Opts) -> + CacheMsg = (maps:remove(<<"priv-has-commitments">>, Msg))#{ + <<"priv-created-at">> => integer_to_binary(hb_util:int(CreatedAt)) + }, + WithCommitmentMarker = case hb_maps:is_key(<<"commitments">>, Msg, Opts) of + true -> CacheMsg#{ <<"priv-has-commitments">> => <<"true">> }; + false -> CacheMsg + end, write_hashpath( CacheAddress, - Msg#{ - <<"priv-created-at">> => - integer_to_binary(hb_util:int(CreatedAt)) - }, + WithCommitmentMarker, Opts ). @@ -1128,8 +1137,9 @@ read_hashpath(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> hashpath_read_result(read(resolved_address(BaseMsg, Req, Opts), Opts)); read_hashpath(_, _, _) -> miss. -%% @doc Return the cache address for a resolved Base and request. `max-age' -%% controls cache reuse, but does not identify the computation itself. +%% @doc Return the cache address for a resolved Base and request. Uncommitted +%% freshness policy controls reuse but does not identify the computation; +%% committed policy remains part of request identity. resolved_address(BaseMsgID, ReqID, _Opts) when ?IS_ID(BaseMsgID) and ?IS_ID(ReqID) -> <>; @@ -1140,7 +1150,7 @@ resolved_address(BaseMsgID, Req, Opts) resolved_address(BaseMsg, Req, Opts) when is_map(BaseMsg) and is_map(Req) -> hb_path:hashpath(BaseMsg, cache_request(Req, Opts), Opts). -%% @doc Remove cache policy fields that do not identify a computation. +%% @doc Remove only uncommitted cache policy from computation identity. cache_request(Req, _Opts) when is_map(Req), not is_map_key(<<"max-age">>, Req), not is_map_key(<<"max-stale">>, Req) -> @@ -1508,6 +1518,10 @@ resolved_entry_canonical_record_test() -> <<"10">>, hb_maps:get(<<"priv-created-at">>, Entry1, Opts) ), + ?assertEqual( + <<"true">>, + hb_maps:get(<<"priv-has-commitments">>, Entry1, Opts) + ), ?assertEqual(false, maps:is_key(<<"priv-other">>, Entry1)), PublicEntry1 = hb_private:reset(Entry1), Restored1 = read_all_commitments(PublicEntry1, Opts), @@ -1524,7 +1538,8 @@ resolved_entry_canonical_record_test() -> hb_message:id(Restored1, all, Opts) ), ?assert(hb_message:verify(Restored1, all, Opts)), - ?assertEqual({ok, ResultID}, write_resolved(Address2, Msg, 20, Opts)), + Unsigned = maps:remove(<<"commitments">>, Msg), + ?assertEqual({ok, ResultID}, write_resolved(Address2, Unsigned, 20, Opts)), {ok, Updated1} = read(Address1, Opts), {ok, Entry2} = read(Address2, Opts), ?assertEqual( @@ -1535,6 +1550,17 @@ resolved_entry_canonical_record_test() -> <<"20">>, hb_maps:get(<<"priv-created-at">>, Entry2, Opts) ), + ?assertEqual( + <<"true">>, + hb_maps:get(<<"priv-has-commitments">>, Updated1, Opts) + ), + RestoredAfterUnsigned = read_all_commitments( + hb_private:reset(Updated1), Opts), + ?assertEqual( + hb_message:id(Public, all, Opts), + hb_message:id(RestoredAfterUnsigned, all, Opts) + ), + ?assert(hb_message:verify(RestoredAfterUnsigned, all, Opts)), ?assertEqual( {error, not_found}, read(<<"created-at/", Address1/binary>>, Opts) diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 9abe390d0..760b74f4a 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/4, maybe_lookup/3]). +-export([maybe_store/4, maybe_lookup/3, prepare/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -11,10 +11,31 @@ %%% following settings. -define(DEFAULT_STORE_OPT, false). -define(DEFAULT_LOOKUP_OPT, true). +-define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). -define(MAX_DELTA_SECONDS, 2147483647). %%% Public API +%% @doc Remove uncommitted request freshness policy before AO execution. +prepare(Req, Opts) when is_map(Req), + not is_map_key(<<"max-age">>, Req), + not is_map_key(<<"max-stale">>, Req) -> + {Req, maps:remove(?REQUEST_POLICY_OPT, Opts)}; +prepare(Req, Opts) -> + CleanOpts = maps:remove(?REQUEST_POLICY_OPT, Opts), + SemanticReq = hb_message:without_unless_signed( + [<<"max-age">>, <<"max-stale">>], + Req, + CleanOpts + ), + Policy = request_policy_from_message(Req, CleanOpts), + case Policy of + #{ <<"max-age">> := undefined, <<"max-stale">> := absent } -> + {SemanticReq, CleanOpts}; + _ -> + {SemanticReq, CleanOpts#{ ?REQUEST_POLICY_OPT => Policy }} + end. + %% @doc Write a resulting M3 message to the cache if requested. The precedence %% order of cache control sources is as follows: %% 1. The `Opts' map (letting the node operator have the final say). @@ -31,15 +52,14 @@ maybe_store(Base, Req, Res, Opts) -> not_caching end. -%% @doc Handles cache lookup, modulated by the caching options requested by -%% the user. Honors the following `Opts' cache keys: -%% `only_if_cached': If set and we do not find a result in the cache, +%% @doc Handles cache lookup, modulated by cache-control and request freshness. +%% Honors these cache-control directives: +%% `only-if-cached': If set and we do not find a result in the cache, %% return an error with a `Cache-Status' of `miss' and %% a 504 `Status'. -%% `no_cache': If set, the cached values are never used. Returns +%% `no-cache': If set, the cached values are never used. Returns %% `continue' to the caller. -%% `max-age': If set, cached results must have a sufficiently -%% recent `priv-created-at' timestamp. +%% Top-level request `max-age' and `max-stale' fields constrain acceptable age. maybe_lookup(Base, Req, Opts) -> case exec_likely_faster_heuristic(Base, Req, Opts) of true -> @@ -167,7 +187,14 @@ cached_age(Res, Now, Opts) when is_map(Res), is_integer(Now), Now >= 0 -> end; cached_age(_, _, _) -> error. -%% @doc Return the request max-age policy. +%% @doc Parse freshness policy from the original request. Preparation decides +%% independently whether each field remains visible in the semantic request. +request_policy_from_message(Req, Opts) -> + #{ + <<"max-age">> => request_max_age(Req, Opts), + <<"max-stale">> => request_max_stale(Req, Opts) + }. + request_max_age(Req, Opts) -> case hb_maps:get(<<"max-age">>, Req, undefined, Opts) of undefined -> undefined; @@ -176,20 +203,19 @@ request_max_age(Req, Opts) -> Value -> parse_delta_seconds(Value) end. -%% @doc Return the request max-stale policy. request_max_stale(Req, Opts) -> - case hb_maps:get(<<"max-stale">>, Req, undefined, Opts) of - undefined -> absent; + case hb_maps:get(<<"max-stale">>, Req, absent, Opts) of + absent -> absent; true -> any; Value -> parse_delta_seconds(Value) end. -%% @doc Read freshness policy directly from the request. +%% @doc Read request policy prepared by AO stage 1, with a direct-call fallback. request_policy(Req, Opts) -> - #{ - <<"max-age">> => request_max_age(Req, Opts), - <<"max-stale">> => request_max_stale(Req, Opts) - }. + case maps:find(?REQUEST_POLICY_OPT, Opts) of + {ok, Policy} -> Policy; + error -> request_policy_from_message(Req, Opts) + end. %% @doc Read the response lifetime and rules that prohibit reuse. response_policy(Res, Opts) -> @@ -208,11 +234,23 @@ cache_miss(Base, Req, _Settings, Opts) -> %% @doc Remove private freshness and restore separately stored commitments. cached_result(Res, Opts) when is_map(Res) -> - PublicRes = maps:remove(<<"priv-created-at">>, Res), - WithCommitments = hb_cache:read_all_commitments(PublicRes, Opts), - case maps:get(<<"commitments">>, WithCommitments, #{}) of - Commitments when map_size(Commitments) > 0 -> WithCommitments; - _ -> PublicRes + HasCommitments = + hb_maps:get( + <<"priv-has-commitments">>, Res, <<"false">>, Opts + ) =:= <<"true">>, + PublicRes = maps:without( + [<<"priv-created-at">>, <<"priv-has-commitments">>], + Res + ), + case HasCommitments of + false -> + PublicRes; + true -> + WithCommitments = hb_cache:read_all_commitments(PublicRes, Opts), + case maps:get(<<"commitments">>, WithCommitments, #{}) of + Commitments when map_size(Commitments) > 0 -> WithCommitments; + _ -> PublicRes + end end; cached_result(Res, _Opts) -> Res. @@ -268,15 +306,15 @@ async_writer() -> perform_cache_write(Base, Req, Res, Opts) -> hb_cache:write(Base, Opts), hb_cache:write(Req, Opts), - StorableRes = case Res of + PolicyRes = case Res of Candidate when is_map(Candidate) -> - {ok, StorableMap} = hb_message:with_only_committed(Candidate, Opts), - StorableMap; + {ok, CommittedPolicyRes} = hb_message:with_only_committed(Candidate, Opts), + CommittedPolicyRes; _ -> Res end, TracksFreshness = tracks_freshness( request_policy(Req, Opts), - response_policy(StorableRes, Opts) + response_policy(PolicyRes, Opts) ), CacheAddress = hb_cache:resolved_address(Base, Req, Opts), WriteResult = case {Res, TracksFreshness} of @@ -295,7 +333,10 @@ perform_cache_write(Base, Req, Res, Opts) -> ); {Map, false} when is_map(Map) -> hb_cache:write( - maps:remove(<<"priv-created-at">>, Map), + maps:without( + [<<"priv-created-at">>, <<"priv-has-commitments">>], + Map + ), Opts ); _ -> @@ -728,6 +769,10 @@ cache_result_priv_created_at_regression_test() -> {hit, {ok, CachedEntry}} = hb_cache:read_resolved(Base, Req, Opts), CachedTimestamp = hb_maps:get(<<"priv-created-at">>, CachedEntry, Opts), ?assert(is_integer(hb_util:int(CachedTimestamp))), + ?assertEqual( + <<"true">>, + hb_maps:get(<<"priv-has-commitments">>, CachedEntry, Opts) + ), Parent = self(), EventHandler = #{ <<"device">> => #{ @@ -771,6 +816,7 @@ cache_result_priv_created_at_regression_test() -> ?assertEqual(nomatch, binary:match(EncodedEvent, <<"priv-created-at">>)), ?assertEqual(nomatch, binary:match(EncodedEvent, CachedTimestamp)), ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)), + ?assertEqual(false, maps:is_key(<<"priv-has-commitments">>, Served)), ?assertEqual( <<"application-value">>, hb_maps:get(<<"created-at">>, Served, Opts) @@ -860,6 +906,56 @@ cache_map_tracked_then_reusable_without_max_age_test() -> ?assertEqual(<<"cached">>, hb_maps:get(<<"value">>, Served, Opts)), ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)). +%% @doc An unsigned cache hit needs no commitment scan after its root read. +unsigned_hit_avoids_commitment_lookup_test() -> + Store = hb_test_utils:test_store(hb_store_lmdb, <<"unsigned-hit-read">>), + Opts = #{ <<"store">> => [Store], + <<"cache-control">> => [<<"always">>] }, + Base = #{ <<"device">> => <<"test-device@1.0">> }, + Req = #{ <<"path">> => <<"index">> }, + Result = #{ <<"body">> => <<"cached">>, + <<"cache-control">> => <<"max-age=60">> }, + {ok, _} = perform_cache_write(Base, Req, Result, Opts), + Parent = self(), + Tracer = spawn(fun() -> cache_store_trace_forwarder(Parent, 0) end), + erlang:trace(self(), true, [call, {tracer, Tracer}]), + erlang:trace_pattern({hb_store, list, 2}, true, []), + try + ?assertMatch( + {ok, #{ <<"body">> := <<"cached">> }}, + maybe_lookup( + Base, + Req, + Opts#{ <<"cache-control">> => [<<"only-if-cached">>] } + ) + ) + after + erlang:trace_pattern({hb_store, list, 2}, false, []), + erlang:trace(self(), false, [call]) + end, + TraceRef = erlang:trace_delivered(self()), + receive {trace_delivered, _, TraceRef} -> ok after 1000 -> + error(cache_trace_delivery_timeout) + end, + Tracer ! {flush, self()}, + Lists = receive {cache_store_trace_lists, Value} -> Value after 1000 -> + error(cache_trace_flush_timeout) + end, + Tracer ! stop, + ?assertEqual(0, Lists). + +%% @doc Count commitment lists for the traced cache hit. +cache_store_trace_forwarder(Parent, Lists) -> + receive + {trace, _, call, {hb_store, list, _}} -> + cache_store_trace_forwarder(Parent, Lists + 1); + {flush, Parent} -> + Parent ! {cache_store_trace_lists, Lists}, + cache_store_trace_forwarder(Parent, Lists); + stop -> + ok + end. + %% @doc Cover the exact freshness, stale, and prohibition boundaries. minimal_freshness_policy_test() -> Req0 = #{ <<"max-age">> => undefined, <<"max-stale">> => absent }, @@ -891,12 +987,14 @@ minimal_freshness_policy_test() -> classify_cached(#{}, Req0, Res0, 100, #{}) ). -%% @doc Parse request policy and keep only uncommitted policy out of identity. +%% @doc Consume only uncommitted policy and reject malformed lifetimes. minimal_policy_parsing_and_identity_test() -> Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, <<"max-stale">> => true }, + {SemanticReq, PreparedOpts} = prepare(Req, #{}), + ?assertEqual(#{ <<"path">> => <<"index">> }, SemanticReq), ?assertEqual(#{ <<"max-age">> => 60, <<"max-stale">> => any }, - request_policy(Req, #{})), + maps:get(?REQUEST_POLICY_OPT, PreparedOpts)), ?assertEqual(response_policy(#{ <<"cache-control">> => <<"max-age=60">> }, #{}), response_policy(#{ <<"cache-control">> => [<<"max-age=60">>] }, #{})), lists:foreach(fun(CC) -> @@ -910,29 +1008,156 @@ minimal_policy_parsing_and_identity_test() -> hb_message:commit(Req#{ <<"max-age">> => MaxAge }, WalletOpts, #{ <<"committed">> => [<<"path">>, <<"max-age">>, <<"max-stale">>] }) end, - Committed60 = Commit(60), - Committed30 = Commit(30), + {Committed60, _} = prepare(Commit(60), WalletOpts), + {Committed30, _} = prepare(Commit(30), WalletOpts), ?assert(hb_message:verify(Committed60, all, WalletOpts)), Base = #{ <<"device">> => <<"test-device@1.0">> }, - ?assertEqual( - hb_cache:resolved_address(Base, Req, #{}), - hb_cache:resolved_address(Base, Req#{ <<"max-age">> => 30 }, #{}) - ), - ?assertEqual(60, maps:get(<<"max-age">>, - request_policy(Committed60, WalletOpts))), ?assertNotEqual(hb_cache:resolved_address(Base, Committed60, WalletOpts), hb_cache:resolved_address(Base, Committed30, WalletOpts)). -%% @doc Request freshness policy remains visible to the executing device. -request_policy_remains_in_device_input_test() -> - Handler = fun(_, Req, _) -> {ok, Req} end, +%% @doc Committing request policy preserves its identity and still applies it. +committed_request_policy_remains_active_test() -> + WalletOpts = #{ + <<"priv-wallet">> => ar_wallet:new(), + <<"commitment-device">> => <<"httpsig@1.0">>, + <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] + }, + Req = hb_message:commit( + #{ <<"path">> => <<"max-age">>, <<"max-age">> => 1 }, + WalletOpts, + #{ <<"committed">> => [<<"path">>, <<"max-age">>] } + ), + {PreparedReq, PreparedOpts} = prepare(Req, WalletOpts), + ?assert(hb_message:verify(PreparedReq, all, PreparedOpts)), + ?assertEqual(1, hb_maps:get(<<"max-age">>, PreparedReq, PreparedOpts)), + ?assertEqual(1, maps:get(<<"max-age">>, + maps:get(?REQUEST_POLICY_OPT, PreparedOpts))), + Base = #{ <<"device">> => <<"test-device@1.0">> }, + {ok, _} = hb_ao:resolve(Base, Req, WalletOpts), + {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, WalletOpts), + {ok, _} = hb_cache:write_resolved( + hb_cache:resolved_address(Base, Req, WalletOpts), + Cached, + os:system_time(second) - 2, + WalletOpts + ), + CacheOnlyOpts = WalletOpts#{ + <<"cache-control">> => [<<"only-if-cached">>] + }, + ?assertMatch( + {error, #{ <<"status">> := 504 }}, + hb_ao:resolve(Base, Req, CacheOnlyOpts) + ). + +%% @doc Prepared policy belongs only to the request that supplied it. +prepared_policy_does_not_reach_the_next_request_test() -> + {_, OuterOpts} = prepare( + #{ <<"path">> => <<"outer">>, <<"max-age">> => 60 }, + #{} + ), + ?assert(maps:is_key(?REQUEST_POLICY_OPT, OuterOpts)), + {InnerReq, InnerOpts} = prepare(#{ <<"path">> => <<"inner">> }, OuterOpts), + ?assertEqual(#{ <<"path">> => <<"inner">> }, InnerReq), + ?assertEqual(false, maps:is_key(?REQUEST_POLICY_OPT, InnerOpts)). + +%% @doc Reserved freshness policy cannot affect a shared computation. +reserved_request_policy_is_not_device_input_test() -> + Parent = self(), + Ref = make_ref(), + Handler = fun(_, Req, Opts) -> + SeenPolicy = { + hb_maps:get(<<"max-age">>, Req, absent, Opts), + hb_maps:get(<<"max-stale">>, Req, absent, Opts) + }, + SeenPrivateOpt = maps:is_key(<<"priv-cache-request-policy">>, Opts), + Parent ! {executed, Ref, SeenPolicy, SeenPrivateOpt}, + {ok, #{ + <<"body">> => <<"cached">>, + <<"cache-control">> => <<"max-age=60">> + }} + end, Base = #{ <<"device">> => #{ index => Handler } }, - Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, - <<"max-stale">> => true }, - {ok, Result} = hb_ao:resolve(Base, Req, - #{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }), - ?assertEqual(60, hb_maps:get(<<"max-age">>, Result, #{})), - ?assertEqual(true, hb_maps:get(<<"max-stale">>, Result, #{})). + Req60 = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, + <<"max-stale">> => 5 }, + Req30 = Req60#{ <<"max-age">> => 30, <<"max-stale">> => 0 }, + Opts = #{ + <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] + }, + % Freshness policy does not identify a computation, so these requests + % intentionally share one cache address. + ?assertEqual( + hb_cache:resolved_address(Base, Req60, Opts), + hb_cache:resolved_address(Base, Req30, Opts) + ), + {ok, First} = hb_ao:resolve(Base, Req60, Opts), + {ok, Second} = hb_ao:resolve(Base, Req30, Opts), + ?assertEqual(hb_message:id(First, all, Opts), + hb_message:id(Second, all, Opts)), + receive + {executed, Ref, {absent, absent}, false} -> ok; + {executed, Ref, Seen, SeenPrivate} -> + error({reserved_policy_reached_device, Seen, SeenPrivate}) + after 1000 -> + error(device_not_executed) + end, + receive + {executed, Ref, _, _} -> error(device_recomputed) + after 0 -> + ok + end. + +%% @doc Resolver-private policy is not exposed while awaiting shared work. +reserved_policy_is_not_persistent_callback_input_test() -> + Parent = self(), + Ref = make_ref(), + Grouper = fun(_, _, _) -> {cache_policy_test, Ref} end, + Await = fun(Worker, Group, Base, Req, Opts) -> + Parent ! {await_policy, Ref, + maps:is_key(?REQUEST_POLICY_OPT, Opts)}, + hb_persistent:default_await(Worker, Group, Base, Req, Opts) + end, + Handler = fun(_, _, _) -> + Parent ! {execution_started, Ref, self()}, + receive {finish_execution, Ref} -> + {ok, #{ <<"body">> => <<"done">> }} + end + end, + Base = #{ <<"device">> => #{ + info => fun() -> #{ grouper => Grouper, await => Await } end, + index => Handler + }}, + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60 }, + Opts = #{ + <<"await-inprogress">> => true, + <<"cache-control">> => [<<"no-store">>] + }, + FirstPID = spawn(fun() -> Parent ! {first_result, Ref, + hb_ao:resolve(Base, Req, Opts)} end), + Leader = receive + {execution_started, Ref, PID} -> PID + after 1000 -> + error(first_execution_not_started) + end, + SecondPID = spawn(fun() -> Parent ! {second_result, Ref, + hb_ao:resolve(Base, Req, Opts)} end), + try + PolicyVisible = receive + {await_policy, Ref, Visible} -> Visible + after 1000 -> + error(second_execution_did_not_await) + end, + Leader ! {finish_execution, Ref}, + First = receive {first_result, Ref, Res1} -> Res1 after 1000 -> timeout end, + Second = receive {second_result, Ref, Res2} -> Res2 after 1000 -> timeout end, + ?assertEqual(First, Second), + ?assertEqual(false, PolicyVisible) + after + Leader ! {finish_execution, Ref}, + exit(FirstPID, kill), + exit(SecondPID, kill) + end. %% @doc An explicit cache-only request never executes with lookup disabled. no_cache_only_if_cached_test() -> @@ -942,6 +1167,31 @@ no_cache_only_if_cached_test() -> ?assertMatch({error, #{ <<"status">> := 504 }}, hb_ao:resolve(Base, CacheOnlyReq, #{})). +%% @doc An expired device response cannot satisfy a cache-only request. +device_response_max_age_cache_only_expiry_test() -> + Base = #{ <<"device">> => <<"test-device@1.0">> }, + Req = #{ <<"path">> => <<"max-age">> }, + Opts = #{ <<"store">> => [hb_test_utils:test_store()], + <<"cache-control">> => [<<"always">>] }, + {ok, First} = hb_ao:resolve(Base, Req, Opts), + ?assertEqual(<<"max-age=1">>, + hb_maps:get(<<"cache-control">>, First, Opts)), + CacheAddress = hb_cache:resolved_address(Base, Req, Opts), + {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, Opts), + {ok, _} = hb_cache:write_resolved( + CacheAddress, + Cached, + os:system_time(second) - 1, + Opts + ), + CacheOnlyOpts = Opts#{ + <<"cache-control">> => [<<"only-if-cached">>] + }, + ?assertMatch( + {error, #{ <<"status">> := 504, <<"cache-status">> := <<"miss">> }}, + hb_ao:resolve(Base, Req, CacheOnlyOpts) + ). + inherited_no_cache_does_not_preempt_device_cache_only_test() -> Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, Req = #{ <<"path">> => <<"index">>, <<"cache-control">> => [<<"only-if-cached">>] }, diff --git a/src/preloaded/util/dev_test.erl b/src/preloaded/util/dev_test.erl index 8851ecd88..d3d2f531f 100644 --- a/src/preloaded/util/dev_test.erl +++ b/src/preloaded/util/dev_test.erl @@ -3,7 +3,7 @@ -export([info/3]). -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). -export([mangle/3, update_state/3, increment_counter/3, delay/3, append/3]). --export([index/3, postprocess/3, load/3]). +-export([index/3, max_age/3, postprocess/3, load/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). @@ -42,6 +42,7 @@ info(_Base, _Req, _Opts) -> <<"mul">> => <<"Multiply function">>, <<"snapshot">> => <<"Snapshot function">>, <<"response">> => <<"Response function">>, + <<"max-age">> => <<"Return a one-second cache response">>, <<"append">> => <<"Append a test value">>, <<"update_state">> => <<"Update state function">> } @@ -58,6 +59,16 @@ index(Msg, _Req, Opts) -> } }. +%% @doc Return a response with a one-second freshness lifetime. +max_age(_Base, _Req, _Opts) -> + {ok, + #{ + <<"status">> => 200, + <<"body">> => <<"cached">>, + <<"cache-control">> => <<"max-age=1">> + } + }. + %% @doc Return a message with the device set to this module. load(Base, _, _Opts) -> {ok, Base#{ <<"device">> => <<"test-device@1.0">> }}. From bde8e1089a9815cd9e16638da78a1a0ea5dcd4c6 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 15:47:06 -0400 Subject: [PATCH 08/10] feat: remove cache control prepare --- src/core/resolver/hb_ao.erl | 21 +-- src/core/resolver/hb_cache_control.erl | 214 ++++--------------------- 2 files changed, 33 insertions(+), 202 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 935f35cda..6969eaae6 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -105,15 +105,12 @@ -export([deep_set/4]). -include("include/hb.hrl"). --define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). - -define( TEMP_OPTS, [ <<"add-key">>, <<"force-message">>, <<"cache-control">>, - ?REQUEST_POLICY_OPT, <<"spawn-worker">>, <<"only">>, <<"prefer">> @@ -418,8 +415,7 @@ resolve_stage(1, RawBase, RawReq, Opts) -> ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), - {SemanticReq, PreparedOpts} = hb_cache_control:prepare(Req, Opts), - resolve_stage(2, Base, SemanticReq, PreparedOpts); + resolve_stage(2, Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. @@ -474,9 +470,7 @@ resolve_stage(4, Base, Req, Opts) -> % There is another executor of this resolution in-flight. % Bail execution, register to receive the response, then % wait. - case hb_persistent:await( - Leader, Base, Req, without_request_policy(Opts) - ) of + case hb_persistent:await(Leader, Base, Req, Opts) of {error, leader_died} -> ?event( ao_core, @@ -766,11 +760,8 @@ resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> Res; {_, _} -> % Spawn a worker for the current execution - PersistentOpts = without_request_policy(Opts), - WorkerPID = hb_persistent:start_worker( - ExecName, Res, PersistentOpts - ), - hb_persistent:forward_work(WorkerPID, PersistentOpts), + WorkerPID = hb_persistent:start_worker(ExecName, Res, Opts), + hb_persistent:forward_work(WorkerPID, Opts), Res end; resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> @@ -1317,7 +1308,3 @@ execution_opts(Opts) -> recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 end. - -%% @doc Keep parsed request cache policy inside the resolver. -without_request_policy(Opts) -> - hb_maps:remove(?REQUEST_POLICY_OPT, Opts, Opts). diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 760b74f4a..0445a971c 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/4, maybe_lookup/3, prepare/2]). +-export([maybe_store/4, maybe_lookup/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -11,31 +11,10 @@ %%% following settings. -define(DEFAULT_STORE_OPT, false). -define(DEFAULT_LOOKUP_OPT, true). --define(REQUEST_POLICY_OPT, <<"priv-cache-request-policy">>). -define(MAX_DELTA_SECONDS, 2147483647). %%% Public API -%% @doc Remove uncommitted request freshness policy before AO execution. -prepare(Req, Opts) when is_map(Req), - not is_map_key(<<"max-age">>, Req), - not is_map_key(<<"max-stale">>, Req) -> - {Req, maps:remove(?REQUEST_POLICY_OPT, Opts)}; -prepare(Req, Opts) -> - CleanOpts = maps:remove(?REQUEST_POLICY_OPT, Opts), - SemanticReq = hb_message:without_unless_signed( - [<<"max-age">>, <<"max-stale">>], - Req, - CleanOpts - ), - Policy = request_policy_from_message(Req, CleanOpts), - case Policy of - #{ <<"max-age">> := undefined, <<"max-stale">> := absent } -> - {SemanticReq, CleanOpts}; - _ -> - {SemanticReq, CleanOpts#{ ?REQUEST_POLICY_OPT => Policy }} - end. - %% @doc Write a resulting M3 message to the cache if requested. The precedence %% order of cache control sources is as follows: %% 1. The `Opts' map (letting the node operator have the final say). @@ -187,14 +166,7 @@ cached_age(Res, Now, Opts) when is_map(Res), is_integer(Now), Now >= 0 -> end; cached_age(_, _, _) -> error. -%% @doc Parse freshness policy from the original request. Preparation decides -%% independently whether each field remains visible in the semantic request. -request_policy_from_message(Req, Opts) -> - #{ - <<"max-age">> => request_max_age(Req, Opts), - <<"max-stale">> => request_max_stale(Req, Opts) - }. - +%% @doc Return the request max-age policy. request_max_age(Req, Opts) -> case hb_maps:get(<<"max-age">>, Req, undefined, Opts) of undefined -> undefined; @@ -203,19 +175,20 @@ request_max_age(Req, Opts) -> Value -> parse_delta_seconds(Value) end. +%% @doc Return the request max-stale policy. request_max_stale(Req, Opts) -> - case hb_maps:get(<<"max-stale">>, Req, absent, Opts) of - absent -> absent; + case hb_maps:get(<<"max-stale">>, Req, undefined, Opts) of + undefined -> absent; true -> any; Value -> parse_delta_seconds(Value) end. -%% @doc Read request policy prepared by AO stage 1, with a direct-call fallback. +%% @doc Read freshness policy directly from the request. request_policy(Req, Opts) -> - case maps:find(?REQUEST_POLICY_OPT, Opts) of - {ok, Policy} -> Policy; - error -> request_policy_from_message(Req, Opts) - end. + #{ + <<"max-age">> => request_max_age(Req, Opts), + <<"max-stale">> => request_max_stale(Req, Opts) + }. %% @doc Read the response lifetime and rules that prohibit reuse. response_policy(Res, Opts) -> @@ -987,14 +960,12 @@ minimal_freshness_policy_test() -> classify_cached(#{}, Req0, Res0, 100, #{}) ). -%% @doc Consume only uncommitted policy and reject malformed lifetimes. +%% @doc Parse request policy and keep only uncommitted policy out of identity. minimal_policy_parsing_and_identity_test() -> Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, <<"max-stale">> => true }, - {SemanticReq, PreparedOpts} = prepare(Req, #{}), - ?assertEqual(#{ <<"path">> => <<"index">> }, SemanticReq), ?assertEqual(#{ <<"max-age">> => 60, <<"max-stale">> => any }, - maps:get(?REQUEST_POLICY_OPT, PreparedOpts)), + request_policy(Req, #{})), ?assertEqual(response_policy(#{ <<"cache-control">> => <<"max-age=60">> }, #{}), response_policy(#{ <<"cache-control">> => [<<"max-age=60">>] }, #{})), lists:foreach(fun(CC) -> @@ -1008,156 +979,29 @@ minimal_policy_parsing_and_identity_test() -> hb_message:commit(Req#{ <<"max-age">> => MaxAge }, WalletOpts, #{ <<"committed">> => [<<"path">>, <<"max-age">>, <<"max-stale">>] }) end, - {Committed60, _} = prepare(Commit(60), WalletOpts), - {Committed30, _} = prepare(Commit(30), WalletOpts), + Committed60 = Commit(60), + Committed30 = Commit(30), ?assert(hb_message:verify(Committed60, all, WalletOpts)), Base = #{ <<"device">> => <<"test-device@1.0">> }, + ?assertEqual( + hb_cache:resolved_address(Base, Req, #{}), + hb_cache:resolved_address(Base, Req#{ <<"max-age">> => 30 }, #{}) + ), + ?assertEqual(60, maps:get(<<"max-age">>, + request_policy(Committed60, WalletOpts))), ?assertNotEqual(hb_cache:resolved_address(Base, Committed60, WalletOpts), hb_cache:resolved_address(Base, Committed30, WalletOpts)). -%% @doc Committing request policy preserves its identity and still applies it. -committed_request_policy_remains_active_test() -> - WalletOpts = #{ - <<"priv-wallet">> => ar_wallet:new(), - <<"commitment-device">> => <<"httpsig@1.0">>, - <<"store">> => [hb_test_utils:test_store()], - <<"cache-control">> => [<<"always">>] - }, - Req = hb_message:commit( - #{ <<"path">> => <<"max-age">>, <<"max-age">> => 1 }, - WalletOpts, - #{ <<"committed">> => [<<"path">>, <<"max-age">>] } - ), - {PreparedReq, PreparedOpts} = prepare(Req, WalletOpts), - ?assert(hb_message:verify(PreparedReq, all, PreparedOpts)), - ?assertEqual(1, hb_maps:get(<<"max-age">>, PreparedReq, PreparedOpts)), - ?assertEqual(1, maps:get(<<"max-age">>, - maps:get(?REQUEST_POLICY_OPT, PreparedOpts))), - Base = #{ <<"device">> => <<"test-device@1.0">> }, - {ok, _} = hb_ao:resolve(Base, Req, WalletOpts), - {hit, {ok, Cached}} = hb_cache:read_resolved(Base, Req, WalletOpts), - {ok, _} = hb_cache:write_resolved( - hb_cache:resolved_address(Base, Req, WalletOpts), - Cached, - os:system_time(second) - 2, - WalletOpts - ), - CacheOnlyOpts = WalletOpts#{ - <<"cache-control">> => [<<"only-if-cached">>] - }, - ?assertMatch( - {error, #{ <<"status">> := 504 }}, - hb_ao:resolve(Base, Req, CacheOnlyOpts) - ). - -%% @doc Prepared policy belongs only to the request that supplied it. -prepared_policy_does_not_reach_the_next_request_test() -> - {_, OuterOpts} = prepare( - #{ <<"path">> => <<"outer">>, <<"max-age">> => 60 }, - #{} - ), - ?assert(maps:is_key(?REQUEST_POLICY_OPT, OuterOpts)), - {InnerReq, InnerOpts} = prepare(#{ <<"path">> => <<"inner">> }, OuterOpts), - ?assertEqual(#{ <<"path">> => <<"inner">> }, InnerReq), - ?assertEqual(false, maps:is_key(?REQUEST_POLICY_OPT, InnerOpts)). - -%% @doc Reserved freshness policy cannot affect a shared computation. -reserved_request_policy_is_not_device_input_test() -> - Parent = self(), - Ref = make_ref(), - Handler = fun(_, Req, Opts) -> - SeenPolicy = { - hb_maps:get(<<"max-age">>, Req, absent, Opts), - hb_maps:get(<<"max-stale">>, Req, absent, Opts) - }, - SeenPrivateOpt = maps:is_key(<<"priv-cache-request-policy">>, Opts), - Parent ! {executed, Ref, SeenPolicy, SeenPrivateOpt}, - {ok, #{ - <<"body">> => <<"cached">>, - <<"cache-control">> => <<"max-age=60">> - }} - end, +%% @doc Request freshness policy remains visible to the executing device. +request_policy_remains_in_device_input_test() -> + Handler = fun(_, Req, _) -> {ok, Req} end, Base = #{ <<"device">> => #{ index => Handler } }, - Req60 = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, - <<"max-stale">> => 5 }, - Req30 = Req60#{ <<"max-age">> => 30, <<"max-stale">> => 0 }, - Opts = #{ - <<"store">> => [hb_test_utils:test_store()], - <<"cache-control">> => [<<"always">>] - }, - % Freshness policy does not identify a computation, so these requests - % intentionally share one cache address. - ?assertEqual( - hb_cache:resolved_address(Base, Req60, Opts), - hb_cache:resolved_address(Base, Req30, Opts) - ), - {ok, First} = hb_ao:resolve(Base, Req60, Opts), - {ok, Second} = hb_ao:resolve(Base, Req30, Opts), - ?assertEqual(hb_message:id(First, all, Opts), - hb_message:id(Second, all, Opts)), - receive - {executed, Ref, {absent, absent}, false} -> ok; - {executed, Ref, Seen, SeenPrivate} -> - error({reserved_policy_reached_device, Seen, SeenPrivate}) - after 1000 -> - error(device_not_executed) - end, - receive - {executed, Ref, _, _} -> error(device_recomputed) - after 0 -> - ok - end. - -%% @doc Resolver-private policy is not exposed while awaiting shared work. -reserved_policy_is_not_persistent_callback_input_test() -> - Parent = self(), - Ref = make_ref(), - Grouper = fun(_, _, _) -> {cache_policy_test, Ref} end, - Await = fun(Worker, Group, Base, Req, Opts) -> - Parent ! {await_policy, Ref, - maps:is_key(?REQUEST_POLICY_OPT, Opts)}, - hb_persistent:default_await(Worker, Group, Base, Req, Opts) - end, - Handler = fun(_, _, _) -> - Parent ! {execution_started, Ref, self()}, - receive {finish_execution, Ref} -> - {ok, #{ <<"body">> => <<"done">> }} - end - end, - Base = #{ <<"device">> => #{ - info => fun() -> #{ grouper => Grouper, await => Await } end, - index => Handler - }}, - Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60 }, - Opts = #{ - <<"await-inprogress">> => true, - <<"cache-control">> => [<<"no-store">>] - }, - FirstPID = spawn(fun() -> Parent ! {first_result, Ref, - hb_ao:resolve(Base, Req, Opts)} end), - Leader = receive - {execution_started, Ref, PID} -> PID - after 1000 -> - error(first_execution_not_started) - end, - SecondPID = spawn(fun() -> Parent ! {second_result, Ref, - hb_ao:resolve(Base, Req, Opts)} end), - try - PolicyVisible = receive - {await_policy, Ref, Visible} -> Visible - after 1000 -> - error(second_execution_did_not_await) - end, - Leader ! {finish_execution, Ref}, - First = receive {first_result, Ref, Res1} -> Res1 after 1000 -> timeout end, - Second = receive {second_result, Ref, Res2} -> Res2 after 1000 -> timeout end, - ?assertEqual(First, Second), - ?assertEqual(false, PolicyVisible) - after - Leader ! {finish_execution, Ref}, - exit(FirstPID, kill), - exit(SecondPID, kill) - end. + Req = #{ <<"path">> => <<"index">>, <<"max-age">> => 60, + <<"max-stale">> => true }, + {ok, Result} = hb_ao:resolve(Base, Req, + #{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }), + ?assertEqual(60, hb_maps:get(<<"max-age">>, Result, #{})), + ?assertEqual(true, hb_maps:get(<<"max-stale">>, Result, #{})). %% @doc An explicit cache-only request never executes with lookup disabled. no_cache_only_if_cached_test() -> From 9f9d967ca7cf9814b1e0b266270d3f6837d64a63 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 16:13:31 -0400 Subject: [PATCH 09/10] fix: remove priv-has-commitments --- src/core/resolver/hb_cache.erl | 58 ++++---------- src/core/resolver/hb_cache_control.erl | 101 +++---------------------- 2 files changed, 26 insertions(+), 133 deletions(-) diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index 091883168..aa4062557 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -268,26 +268,20 @@ generate_binary_path(Bin, Opts) -> %% the commitments of the inner messages. We do not, however, store the IDs from %% commitments on signed _inner_ messages. We may wish to revisit this. write(RawMsg, Opts) when is_map(RawMsg) -> - % Reattach only bounded cache-owned metadata after private keys are filtered. - PrivCacheMetadata = maps:filter( - fun(<<"priv-created-at">>, Value) -> - is_immediate_value(<<"priv-created-at">>, Value); - (<<"priv-has-commitments">>, Value) -> - Value =:= <<"true">> orelse Value =:= <<"false">> - end, - maps:with( - [<<"priv-created-at">>, <<"priv-has-commitments">>], - RawMsg - ) - ), - PublicRawMsg = maps:without( - [<<"priv-created-at">>, <<"priv-has-commitments">>], - RawMsg - ), + % Reattach only cache-owned freshness after private keys are filtered. + PrivCreatedAt = case maps:find(<<"priv-created-at">>, RawMsg) of + {ok, CreatedAt} -> + case is_immediate_value(<<"priv-created-at">>, CreatedAt) of + true -> #{ <<"priv-created-at">> => CreatedAt }; + false -> #{} + end; + error -> #{} + end, + PublicRawMsg = maps:remove(<<"priv-created-at">>, RawMsg), hb_message:paranoid_verify(cache_write, PublicRawMsg, Opts), {ok, Msg} = hb_message:with_only_committed(PublicRawMsg, Opts), PublicTABM = hb_message:convert(Msg, tabm, <<"structured@1.0">>, Opts), - TABM = maps:merge(PublicTABM, PrivCacheMetadata), + TABM = maps:merge(PublicTABM, PrivCreatedAt), ?event_debug(debug_cache, {writing_full_message, {msg, TABM}}), try do_write_message( @@ -687,16 +681,12 @@ write_binary(Hashpath, Bin, Store, Opts) -> %% @doc Write a resolved message with its private cache creation timestamp. write_resolved(CacheAddress, Msg, CreatedAt, Opts) -> - CacheMsg = (maps:remove(<<"priv-has-commitments">>, Msg))#{ - <<"priv-created-at">> => integer_to_binary(hb_util:int(CreatedAt)) - }, - WithCommitmentMarker = case hb_maps:is_key(<<"commitments">>, Msg, Opts) of - true -> CacheMsg#{ <<"priv-has-commitments">> => <<"true">> }; - false -> CacheMsg - end, write_hashpath( CacheAddress, - WithCommitmentMarker, + Msg#{ + <<"priv-created-at">> => + integer_to_binary(hb_util:int(CreatedAt)) + }, Opts ). @@ -1518,10 +1508,6 @@ resolved_entry_canonical_record_test() -> <<"10">>, hb_maps:get(<<"priv-created-at">>, Entry1, Opts) ), - ?assertEqual( - <<"true">>, - hb_maps:get(<<"priv-has-commitments">>, Entry1, Opts) - ), ?assertEqual(false, maps:is_key(<<"priv-other">>, Entry1)), PublicEntry1 = hb_private:reset(Entry1), Restored1 = read_all_commitments(PublicEntry1, Opts), @@ -1538,8 +1524,7 @@ resolved_entry_canonical_record_test() -> hb_message:id(Restored1, all, Opts) ), ?assert(hb_message:verify(Restored1, all, Opts)), - Unsigned = maps:remove(<<"commitments">>, Msg), - ?assertEqual({ok, ResultID}, write_resolved(Address2, Unsigned, 20, Opts)), + ?assertEqual({ok, ResultID}, write_resolved(Address2, Msg, 20, Opts)), {ok, Updated1} = read(Address1, Opts), {ok, Entry2} = read(Address2, Opts), ?assertEqual( @@ -1550,17 +1535,6 @@ resolved_entry_canonical_record_test() -> <<"20">>, hb_maps:get(<<"priv-created-at">>, Entry2, Opts) ), - ?assertEqual( - <<"true">>, - hb_maps:get(<<"priv-has-commitments">>, Updated1, Opts) - ), - RestoredAfterUnsigned = read_all_commitments( - hb_private:reset(Updated1), Opts), - ?assertEqual( - hb_message:id(Public, all, Opts), - hb_message:id(RestoredAfterUnsigned, all, Opts) - ), - ?assert(hb_message:verify(RestoredAfterUnsigned, all, Opts)), ?assertEqual( {error, not_found}, read(<<"created-at/", Address1/binary>>, Opts) diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 0445a971c..d847692d2 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -51,13 +51,7 @@ lookup(Base, Req, Opts) -> case derive_cache_settings([Base, Req], Opts) of Settings = #{ <<"lookup">> := false } -> ?event({skip_cache_check, lookup_disabled}), - case derive_cache_settings( - [Base, Req], Opts#{ <<"only">> => local } - ) of - #{ <<"lookup">> := false, <<"only-if-cached">> := true } -> - cache_miss(Base, Req, Settings, Opts); - _ -> maybe_load_base(Base, Req, Opts) - end; + cache_miss(Base, Req, Settings, Opts); Settings = #{ <<"lookup">> := true } -> OutputScopedOpts = hb_store:scope( @@ -192,7 +186,7 @@ request_policy(Req, Opts) -> %% @doc Read the response lifetime and rules that prohibit reuse. response_policy(Res, Opts) -> - Parsed = message_cache_control(Res, Opts), + Parsed = parse_message_cache_control(Res, Opts), #{ <<"max-age">> => numeric_directive(<<"max-age">>, Parsed), <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), @@ -207,23 +201,11 @@ cache_miss(Base, Req, _Settings, Opts) -> %% @doc Remove private freshness and restore separately stored commitments. cached_result(Res, Opts) when is_map(Res) -> - HasCommitments = - hb_maps:get( - <<"priv-has-commitments">>, Res, <<"false">>, Opts - ) =:= <<"true">>, - PublicRes = maps:without( - [<<"priv-created-at">>, <<"priv-has-commitments">>], - Res - ), - case HasCommitments of - false -> - PublicRes; - true -> - WithCommitments = hb_cache:read_all_commitments(PublicRes, Opts), - case maps:get(<<"commitments">>, WithCommitments, #{}) of - Commitments when map_size(Commitments) > 0 -> WithCommitments; - _ -> PublicRes - end + PublicRes = maps:remove(<<"priv-created-at">>, Res), + WithCommitments = hb_cache:read_all_commitments(PublicRes, Opts), + case maps:get(<<"commitments">>, WithCommitments, #{}) of + Commitments when map_size(Commitments) > 0 -> WithCommitments; + _ -> PublicRes end; cached_result(Res, _Opts) -> Res. @@ -306,10 +288,7 @@ perform_cache_write(Base, Req, Res, Opts) -> ); {Map, false} when is_map(Map) -> hb_cache:write( - maps:without( - [<<"priv-created-at">>, <<"priv-has-commitments">>], - Map - ), + maps:remove(<<"priv-created-at">>, Map), Opts ); _ -> @@ -488,9 +467,9 @@ has_directive(_Name, invalid) -> false; has_directive(Name, Parsed) -> lists:member(Name, Parsed). %% @doc Parse Cache-Control directly from a map-shaped message. -message_cache_control(Msg, Opts) when is_map(Msg) -> +parse_message_cache_control(Msg, Opts) when is_map(Msg) -> parse_cache_control(hb_maps:get(<<"cache-control">>, Msg, [], Opts)); -message_cache_control(_, _) -> []. +parse_message_cache_control(_, _) -> []. %% @doc Convert a cache control list as received via HTTP headers into a %% normalized map of simply whether we should store and/or lookup the result. @@ -742,10 +721,6 @@ cache_result_priv_created_at_regression_test() -> {hit, {ok, CachedEntry}} = hb_cache:read_resolved(Base, Req, Opts), CachedTimestamp = hb_maps:get(<<"priv-created-at">>, CachedEntry, Opts), ?assert(is_integer(hb_util:int(CachedTimestamp))), - ?assertEqual( - <<"true">>, - hb_maps:get(<<"priv-has-commitments">>, CachedEntry, Opts) - ), Parent = self(), EventHandler = #{ <<"device">> => #{ @@ -789,7 +764,6 @@ cache_result_priv_created_at_regression_test() -> ?assertEqual(nomatch, binary:match(EncodedEvent, <<"priv-created-at">>)), ?assertEqual(nomatch, binary:match(EncodedEvent, CachedTimestamp)), ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)), - ?assertEqual(false, maps:is_key(<<"priv-has-commitments">>, Served)), ?assertEqual( <<"application-value">>, hb_maps:get(<<"created-at">>, Served, Opts) @@ -879,56 +853,6 @@ cache_map_tracked_then_reusable_without_max_age_test() -> ?assertEqual(<<"cached">>, hb_maps:get(<<"value">>, Served, Opts)), ?assertEqual(false, maps:is_key(<<"priv-created-at">>, Served)). -%% @doc An unsigned cache hit needs no commitment scan after its root read. -unsigned_hit_avoids_commitment_lookup_test() -> - Store = hb_test_utils:test_store(hb_store_lmdb, <<"unsigned-hit-read">>), - Opts = #{ <<"store">> => [Store], - <<"cache-control">> => [<<"always">>] }, - Base = #{ <<"device">> => <<"test-device@1.0">> }, - Req = #{ <<"path">> => <<"index">> }, - Result = #{ <<"body">> => <<"cached">>, - <<"cache-control">> => <<"max-age=60">> }, - {ok, _} = perform_cache_write(Base, Req, Result, Opts), - Parent = self(), - Tracer = spawn(fun() -> cache_store_trace_forwarder(Parent, 0) end), - erlang:trace(self(), true, [call, {tracer, Tracer}]), - erlang:trace_pattern({hb_store, list, 2}, true, []), - try - ?assertMatch( - {ok, #{ <<"body">> := <<"cached">> }}, - maybe_lookup( - Base, - Req, - Opts#{ <<"cache-control">> => [<<"only-if-cached">>] } - ) - ) - after - erlang:trace_pattern({hb_store, list, 2}, false, []), - erlang:trace(self(), false, [call]) - end, - TraceRef = erlang:trace_delivered(self()), - receive {trace_delivered, _, TraceRef} -> ok after 1000 -> - error(cache_trace_delivery_timeout) - end, - Tracer ! {flush, self()}, - Lists = receive {cache_store_trace_lists, Value} -> Value after 1000 -> - error(cache_trace_flush_timeout) - end, - Tracer ! stop, - ?assertEqual(0, Lists). - -%% @doc Count commitment lists for the traced cache hit. -cache_store_trace_forwarder(Parent, Lists) -> - receive - {trace, _, call, {hb_store, list, _}} -> - cache_store_trace_forwarder(Parent, Lists + 1); - {flush, Parent} -> - Parent ! {cache_store_trace_lists, Lists}, - cache_store_trace_forwarder(Parent, Lists); - stop -> - ok - end. - %% @doc Cover the exact freshness, stale, and prohibition boundaries. minimal_freshness_policy_test() -> Req0 = #{ <<"max-age">> => undefined, <<"max-stale">> => absent }, @@ -1036,11 +960,6 @@ device_response_max_age_cache_only_expiry_test() -> hb_ao:resolve(Base, Req, CacheOnlyOpts) ). -inherited_no_cache_does_not_preempt_device_cache_only_test() -> - Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, - Req = #{ <<"path">> => <<"index">>, <<"cache-control">> => [<<"only-if-cached">>] }, - ?assertMatch({ok, _}, hb_ao:resolve(Base, Req, #{})). - %% @doc Response max-age stores time and max-stale controls live cache reuse. cache_response_max_age_and_request_max_stale_test() -> Base = #{ <<"device">> => <<"test-device@1.0">>, <<"name">> => <<"HB">> }, From 9facc35016047275687244cafaa84410eea5a161 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Fri, 28 Aug 2026 16:30:24 -0400 Subject: [PATCH 10/10] fix: remove mirror base id --- src/core/resolver/hb_cache_control.erl | 58 +++++++------------------- 1 file changed, 16 insertions(+), 42 deletions(-) diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index d847692d2..4c951dbc1 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -188,10 +188,10 @@ request_policy(Req, Opts) -> response_policy(Res, Opts) -> Parsed = parse_message_cache_control(Res, Opts), #{ - <<"max-age">> => numeric_directive(<<"max-age">>, Parsed), - <<"no-cache">> => has_directive(<<"no-cache">>, Parsed), + <<"max-age">> => parse_numeric_setting(<<"max-age">>, Parsed), + <<"no-cache">> => has_setting(<<"no-cache">>, Parsed), <<"must-revalidate">> => - has_directive(<<"must-revalidate">>, Parsed) + has_setting(<<"must-revalidate">>, Parsed) }. cache_miss(Base, Req, #{ <<"only-if-cached">> := true }, Opts) -> @@ -272,7 +272,7 @@ perform_cache_write(Base, Req, Res, Opts) -> response_policy(PolicyRes, Opts) ), CacheAddress = hb_cache:resolved_address(Base, Req, Opts), - WriteResult = case {Res, TracksFreshness} of + case {Res, TracksFreshness} of {<<_/binary>>, _} -> hb_cache:write_binary( CacheAddress, @@ -294,33 +294,7 @@ perform_cache_write(Base, Req, Res, Opts) -> _ -> ?event({cannot_write_result, Res}), skip_caching - end, - mirror_base_id_address( - Base, - Req, - CacheAddress, - WriteResult, - TracksFreshness, - Opts - ). - -%% @doc Mirror freshness-tracked results only; untracked maps stay canonical. -mirror_base_id_address( - Base, Req, CacheAddress, {ok, Path} = Result, true, Opts - ) when is_map(Base) -> - IDAddress = hb_cache:resolved_address( - hb_message:id(Base, all, Opts), - Req, - Opts - ), - case IDAddress =:= CacheAddress of - true -> Result; - false -> - hb_cache:link(Path, IDAddress, Opts), - Result - end; -mirror_base_id_address(_Base, _Req, _CacheAddress, Result, _Tracked, _Opts) -> - Result. + end. %% @doc Return whether a cached map needs co-located age metadata. tracks_freshness(ReqPolicy, ResPolicy) -> @@ -454,8 +428,8 @@ parse_delta_seconds(Value) when is_binary(Value) -> parse_delta_seconds(_) -> invalid. %% @doc Return one numeric directive, rejecting malformed duplicates. -numeric_directive(_Name, invalid) -> invalid; -numeric_directive(Name, Parsed) -> +parse_numeric_setting(_Name, invalid) -> invalid; +parse_numeric_setting(Name, Parsed) -> case {lists:member(Name, Parsed), [Value || {Key, Value} <- Parsed, Key =:= Name]} of {false, []} -> undefined; {false, [Value]} -> parse_delta_seconds(Value); @@ -463,8 +437,8 @@ numeric_directive(Name, Parsed) -> end. %% @doc Return whether one argumentless directive is present. -has_directive(_Name, invalid) -> false; -has_directive(Name, Parsed) -> lists:member(Name, Parsed). +has_setting(_Name, invalid) -> false; +has_setting(Name, Parsed) -> lists:member(Name, Parsed). %% @doc Parse Cache-Control directly from a map-shaped message. parse_message_cache_control(Msg, Opts) when is_map(Msg) -> @@ -477,33 +451,33 @@ specifiers_to_cache_settings(CCSpecifier) -> CCList = parse_cache_control(CCSpecifier), #{ <<"store">> => - case has_directive(<<"always">>, CCList) of + case has_setting(<<"always">>, CCList) of true -> true; false -> - case has_directive(<<"no-store">>, CCList) of + case has_setting(<<"no-store">>, CCList) of true -> false; false -> - case has_directive(<<"store">>, CCList) of + case has_setting(<<"store">>, CCList) of true -> true; false -> undefined end end end, <<"lookup">> => - case has_directive(<<"always">>, CCList) of + case has_setting(<<"always">>, CCList) of true -> true; false -> - case has_directive(<<"no-cache">>, CCList) of + case has_setting(<<"no-cache">>, CCList) of true -> false; false -> - case has_directive(<<"cache">>, CCList) of + case has_setting(<<"cache">>, CCList) of true -> true; false -> undefined end end end, <<"only-if-cached">> => - case has_directive(<<"only-if-cached">>, CCList) of + case has_setting(<<"only-if-cached">>, CCList) of true -> true; false -> undefined end