diff --git a/src/core/http/hb_http.erl b/src/core/http/hb_http.erl index 67a8e7870..950367814 100644 --- a/src/core/http/hb_http.erl +++ b/src/core/http/hb_http.erl @@ -87,6 +87,10 @@ request(Method, #{ <<"opts">> := ReqOpts, <<"uri">> := URI }, _Path, Message, Op MergedOpts ), request(NewMethod, Node, NewPath, NewMsg, NewOpts); +request(Method, #{ <<"prefix">> := Prefix }, Path, RawMessage, Opts) -> + % A configured remote node carrying a `prefix' URL (its per-node `opts' are + % read elsewhere). Use the prefix as the peer, preserving the request path. + request(Method, Prefix, Path, RawMessage, Opts); request(Method, Peer, Path, RawMessage, Opts) -> ?event({request, {method, Method}, {peer, Peer}, {path, Path}, {priv_message, RawMessage}}), Req = @@ -329,9 +333,10 @@ http_response_to_httpsig(Status, HeaderMap, Body, Opts) -> 0 -> #{}; _ -> #{ <<"body">> => Body } end, - ConvertFrom = + NormalizedHeaders = lowercase_header_keys(HeaderMap), + ConvertFrom = hb_maps:merge( - HeaderMap#{ <<"status">> => BinStatus }, + NormalizedHeaders#{ <<"status">> => BinStatus }, BodyMap, Opts ), @@ -342,6 +347,18 @@ http_response_to_httpsig(Status, HeaderMap, Body, Opts) -> Opts ))#{ <<"status">> => hb_util:int(Status) }. +%% @doc Lowercase binary header keys from peer responses before the httpsig +%% codec runs, so committed component names (signed lowercase per RFC 9421) +%% match regardless of the peer's HTTP/1.1 header casing. Values are untouched. +lowercase_header_keys(Headers) when is_map(Headers) -> + maps:fold( + fun(K, V, Acc) when is_binary(K) -> Acc#{ string:lowercase(K) => V }; + (K, V, Acc) -> Acc#{ K => V } + end, + #{}, + Headers + ). + %% @doc Given a message, return the information needed to make the request. message_to_request(M, Opts) -> % Get the route for the message. @@ -1145,10 +1162,9 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> not_found -> WithPrivIP; Device -> WithPrivIP#{<<"device">> => Device} end, - Host = cowboy_req:host(Req), + Host = cowboy_req:header(<<"host">>, Req, cowboy_req:host(Req)), WithDevice#{<<"host">> => Host}. - %% @doc Determine the caller, honoring the `x-real-ip' header if present. real_ip(Req = #{ headers := RawHeaders }, Opts) -> case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of diff --git a/src/core/http/hb_http_client.erl b/src/core/http/hb_http_client.erl index a73cadd29..02d8343d1 100644 --- a/src/core/http/hb_http_client.erl +++ b/src/core/http/hb_http_client.erl @@ -5,6 +5,9 @@ -include("include/hb.hrl"). -include("include/hb_opts.hrl"). -include("include/hb_http_client.hrl"). +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. %% Public API -export([request/2, response_status_to_atom/1, setup_conn/1]). %% GenServer @@ -790,6 +793,12 @@ get_status_class({error, {down, noproc}}) -> <<"noproc">>; get_status_class({error, {stream_error, {closed, normal}}}) -> <<"stream-closed">>; +get_status_class({error, {tls_alert, {internal_error, _}}}) -> + <<"tls-alert-internal-error">>; +get_status_class({error, {tls_alert, {handshake_failure, _}}}) -> + <<"tls-alert-handshake-failure">>; +get_status_class({error, {tls_alert, {certificate_expired, _}}}) -> + <<"tls-alert-certificate-expired">>; get_status_class({error, Error}) when is_atom(Error) -> hb_util:atom_to_dashed_binary(Error); get_status_class(208) -> @@ -829,3 +838,21 @@ path_to_category(Path) -> undefined -> <<"unknown">>; _ -> <<"unknown">> end. + +-ifdef(TEST). + +tls_alert_status_class_test() -> + ?assertEqual( + <<"tls-alert-internal-error">>, + get_status_class({error, {tls_alert, {internal_error, test}}}) + ), + ?assertEqual( + <<"tls-alert-handshake-failure">>, + get_status_class({error, {tls_alert, {handshake_failure, test}}}) + ), + ?assertEqual( + <<"tls-alert-certificate-expired">>, + get_status_class({error, {tls_alert, {certificate_expired, test}}}) + ). + +-endif. diff --git a/src/core/http/hb_http_multi.erl b/src/core/http/hb_http_multi.erl index 204b6da5d..81dca463c 100644 --- a/src/core/http/hb_http_multi.erl +++ b/src/core/http/hb_http_multi.erl @@ -5,19 +5,33 @@ %%% The `Config' message for a call to `request/5' may contain the following %%% fields: %%% -%%% - `multirequest-nodes': A list of nodes to request from. -%%% - `multirequest-responses': The number of responses to gather. -%%% - `multirequest-stop-after': Whether to stop after the required number of -%%% responses. -%%% - `multirequest-parallel': Whether to run the requests in parallel. -%%% - `multirequest-admissible': A message to resolve against the response. -%%% - `multirequest-admissible-status': The statuses that are admissible. -%%% -%%% The `admissible' message is executed as a `base' message, with its `path' -%%% field moved to the request (or set to `is-admissible' if not present): +%%% - `nodes': A list of nodes to request from. +%%% - `responses': The number of responses to gather. +%%% - `stop-after': Whether to stop after the required number of responses. +%%% - `parallel': Whether to run the requests in parallel. +%%% - `admissible': A predicate message to resolve against each response. +%%% - `admissible-status': The response statuses that are admissible. +%%% +%%% A request message may override these fields with their `multirequest-' +%%% prefixed equivalents. +%%% +%%% The optional `admissible' predicate is resolved with its `path' moved to the +%%% response request. The path defaults to `is-admissible'. The request also +%%% carries the responding node as private `admissibility/node' context. %%% ``` %%% resolve(Base, Response#{ <<"path">> => Base/path OR /is-admissible }, Opts) -%%% ''' +%%% ``` +%%% +%%% An admissibility message may contain an optional `decorator' message. The +%%% decorator is resolved first with the predicate base in the request `body'. +%%% Its `path' defaults to `decorator' and may be set explicitly. The decorated +%%% result becomes the predicate base. A decorator error or exception makes the +%%% response inadmissible. +%%% +%%% A predicate admits a response by returning `true' as an atom or binary. It +%%% may instead return a message containing `ao-result', whose value identifies +%%% the result key to read from that message. Missing, false, invalid, or failed +%%% predicate results make the response inadmissible. -module(hb_http_multi). -export([request/5]). -include("include/hb.hrl"). @@ -120,7 +134,8 @@ multirequest_opt(Key, Config, Message, Default, Opts) -> {Config, Key} ], Default, - Opts#{ <<"hashpath">> => ignore } + %% force-message allow us to keep the response Erlang friendly + Opts#{ <<"hashpath">> => ignore, <<"force-message">> => false } ). %% @doc Check if a response is admissible, according to the configuration. First, @@ -129,7 +144,7 @@ multirequest_opt(Key, Config, Message, Default, Opts) -> %% %% If the response is `ok', we check the status and the response message against %% the configuration. -is_admissible(ok, Res, Admissible, Statuses, Opts) -> +is_admissible(ok, Res, Admissible, Statuses, Node, Opts) -> ?event(debug_multi, {is_admissible, {response, Res}, @@ -139,10 +154,10 @@ is_admissible(ok, Res, Admissible, Statuses, Opts) -> ), AdmissibleStatus = admissible_status(Res, Statuses), ?event(debug_multi, {admissible_status, {result, AdmissibleStatus}}), - AdmissibleResponse = admissible_response(Res, Admissible, Opts), + AdmissibleResponse = admissible_response(Res, Admissible, Node, Opts), ?event(debug_multi, {admissible_response, {result, AdmissibleResponse}}), AdmissibleStatus andalso AdmissibleResponse; -is_admissible(_, _, _, _, _) -> false. +is_admissible(_, _, _, _, _, _) -> false. %% @doc Serially request a message, collecting responses until the required %% number of responses have been gathered. Ensure that the statuses are @@ -153,7 +168,7 @@ serial_multirequest(_Nodes, 0, _Method, _Path, _Message, _Admissible, _Statuses, serial_multirequest([], _, _Method, _Path, _Message, _Admissible, _Statuses, _Opts) -> {[], []}; serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, Statuses, Opts) -> {ErlStatus, Res} = hb_http:request(Method, Node, Path, Message, Opts), - case is_admissible(ErlStatus, Res, Admissible, Statuses, Opts) of + case is_admissible(ErlStatus, Res, Admissible, Statuses, Node, Opts) of true -> ?event(debug_http, {admissible_status, {response, Res}}), {AdmissibleAcc, AllAcc} = serial_multirequest( @@ -200,12 +215,12 @@ start_workers(Count, Ref, Nodes, Method, Path, Message, Opts) -> fun(Node) -> spawn( fun() -> - Res = + {Status, NewRes} = try hb_http:request(Method, Node, Path, Message, Opts) catch C:R -> {error, {worker_crash, C, R}} end, receive no_reply -> stopping - after 0 -> Parent ! {Ref, self(), Res} + after 0 -> Parent ! {Ref, self(), {Status, NewRes, Node}} end end ) @@ -236,18 +251,42 @@ admissible_status(Status, Statuses) when is_list(Statuses) -> %% @doc If an `admissable` message is set for the request, check if the response %% adheres to it. Else, return `true'. -admissible_response(_Response, undefined, _Opts) -> true; -admissible_response(Response, Msg, Opts) -> +admissible_response(_Response, undefined, _Node, _Opts) -> true; +admissible_response(Response, Msg, Node, Opts) -> Path = hb_maps:get(<<"path">>, Msg, <<"is-admissible">>, Opts), - Req = Response#{ <<"path">> => Path }, - Base = hb_message:without_unless_signed([<<"path">>], Msg, Opts), - ?event(debug_multi, - {executing_admissible_message, {message, Base}, {req, Req}} - ), - try hb_ao:resolve(Base, Req, Opts) of + Decorator = hb_maps:get(<<"decorator">>, Msg, undefined, Opts), + Req = + hb_private:set( + Response#{ <<"path">> => Path }, + <<"admissibility/node">>, + Node, + Opts + ), + AdmissibleOpts = + Opts#{ <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }, + try + Base = + apply_decorator( + hb_message:without_unless_signed( + [<<"path">>, <<"decorator">>], + Msg, + Opts + ), + Req, + Decorator, + AdmissibleOpts + ), + ?event(debug_multi, + {executing_admissible_message, {message, Base}, {req, Req}} + ), + hb_ao:resolve(Base, Req, AdmissibleOpts) + of {ok, Res} when is_atom(Res) or is_binary(Res) -> ?event(debug_multi, {admissible_result, {result, Res}}), hb_util:atom(Res) == true; + {ok, #{<<"ao-result">> := Key} = Res} -> + ?event(debug_multi, {admissible_result, {result, Res}}), + hb_util:atom(hb_maps:get(Key, Res, false, Opts)) == true; {error, Reason} -> ?event(debug_multi, {admissible_error, {reason, Reason}}), false @@ -263,6 +302,29 @@ admissible_response(Response, Msg, Opts) -> false end. +%% @doc Apply the optional decorator before resolving the admissibility +%% predicate. +apply_decorator(Base, _Req, undefined, _Opts) -> Base; +apply_decorator(Base, Req, Decorator, Opts) -> + DecoratorPath = + hb_maps:get(<<"path">>, Decorator, <<"decorator">>, Opts), + DecoratorBase = + hb_maps:without([<<"path">>], Decorator, Opts), + case hb_ao:resolve( + DecoratorBase, + Req#{ <<"path">> => DecoratorPath, <<"body">> => Base }, + Opts + ) of + {ok, Decorated} -> Decorated; + Error -> + ?event(error, + {invalid_decorator_response, + {path, DecoratorPath}, + {error, Error} + }), + throw({invalid_decorator_response, Error}) + end. + %% @doc Collect the necessary number of responses, and stop workers if %% configured to do so. %% Returns {AdmissibleList, AllList} where AdmissibleList contains only @@ -280,10 +342,10 @@ parallel_responses(AdmissibleRes, AllRes, Procs, _, _, Ref, 0, true, _Admissible {AdmissibleRes, AllRes}; parallel_responses(AdmissibleRes, AllRes, Procs, Queue, {Method, Path, Message}, Ref, Awaiting, StopAfter, Admissible, Statuses, Opts) -> receive - {Ref, Pid, {Status, NewRes}} -> + {Ref, Pid, {Status, NewRes, Node}} -> WorkersWithoutPid = lists:delete(Pid, Procs), NewAllRes = [{Status, NewRes} | AllRes], - case is_admissible(Status, NewRes, Admissible, Statuses, Opts) of + case is_admissible(Status, NewRes, Admissible, Statuses, Node, Opts) of true -> NewAwaiting = Awaiting - 1, {NewProcs, NewQueue} = @@ -349,6 +411,102 @@ ao_node(URL) -> #{<<"uri">> => <>, <<"opts">> => #{ <<"http-client">> => httpc }}. +with_admissibility_value(Node, Value) -> + Node#{ + <<"opts">> => + (maps:get(<<"opts">>, Node))#{ + <<"admissibility-value">> => Value + } + }. + +node_context_predicate() -> + #{ + <<"device">> => <<"message@1.0">>, + <<"path">> => <<"admitted">>, + <<"decorator">> => + #{ + <<"device">> => + #{ + decorator => + fun(_Base, Req, Opts) -> + Target = hb_maps:get(<<"body">>, Req, #{}, Opts), + Node = + hb_private:get( + <<"admissibility/node">>, + Req, + #{}, + Opts + ), + NodeOpts = + hb_maps:get(<<"opts">>, Node, #{}, Opts), + Value = + hb_maps:get( + <<"admissibility-value">>, + NodeOpts, + false, + Opts + ), + {ok, Target#{ <<"admitted">> => Value }} + end + } + } + }. + +ao_result_predicate(Value) -> + #{ + <<"device">> => + #{ + predicate => + fun(_Base, _Req, _Opts) -> + {ok, + #{ + <<"ao-result">> => <<"admitted">>, + <<"admitted">> => Value + }} + end + }, + <<"path">> => <<"predicate">> + }. + +custom_decorator_predicate() -> + #{ + <<"device">> => <<"message@1.0">>, + <<"path">> => <<"decorated">>, + <<"decorator">> => + #{ + <<"device">> => + #{ + custom_decorator => + fun(_Base, Req, Opts) -> + Target = hb_maps:get(<<"body">>, Req, #{}, Opts), + {ok, Target#{ <<"decorated">> => true }} + end + }, + <<"path">> => <<"custom-decorator">> + } + }. + +failing_decorator_predicate() -> + #{ + <<"device">> => + #{ + predicate => + fun(_Base, _Req, _Opts) -> {ok, true} end + }, + <<"path">> => <<"predicate">>, + <<"decorator">> => + #{ + <<"device">> => + #{ + failing_decorator => + fun(_Base, _Req, _Opts) -> + {error, decoration_failed} + end + }, + <<"path">> => <<"failing-decorator">> + } + }. + dead_node() -> {ok, S} = gen_tcp:listen(0, []), {ok, Port} = inet:port(S), @@ -395,6 +553,55 @@ multirequest_test_() -> ?assertMatch({error, {no_viable_responses, _}}, multi([crash(), crash()], #{<<"parallel">> => true, <<"stop-after">> => true})) + end}, + {"node context decorator", fun() -> + Node = maps:get(fast, N), + Extra = #{ <<"admissible">> => node_context_predicate() }, + ?assertMatch( + {ok, _}, + multi([with_admissibility_value(Node, true)], Extra) + ), + ?assertMatch( + {error, {no_viable_responses, _}}, + multi([with_admissibility_value(Node, false)], Extra) + ) + end}, + {"ao-result predicate", fun() -> + Node = maps:get(fast, N), + ?assertMatch( + {ok, _}, + multi( + [Node], + #{ <<"admissible">> => ao_result_predicate(true) } + ) + ), + ?assertMatch( + {error, {no_viable_responses, _}}, + multi( + [Node], + #{ <<"admissible">> => ao_result_predicate(false) } + ) + ) + end}, + {"custom decorator path", fun() -> + Node = maps:get(fast, N), + ?assertMatch( + {ok, _}, + multi( + [Node], + #{ <<"admissible">> => custom_decorator_predicate() } + ) + ) + end}, + {"decorator failure is inadmissible", fun() -> + Node = maps:get(fast, N), + ?assertMatch( + {error, {no_viable_responses, _}}, + multi( + [Node], + #{ <<"admissible">> => failing_decorator_predicate() } + ) + ) end} ]} end}. diff --git a/src/core/store/hb_store_remote_node.erl b/src/core/store/hb_store_remote_node.erl index 8eba2a100..7501903ea 100644 --- a/src/core/store/hb_store_remote_node.erl +++ b/src/core/store/hb_store_remote_node.erl @@ -4,7 +4,7 @@ %%% been written to the remote node. In that case, the node would probably want %%% to upload it to an Arweave bundler to ensure persistence, too. -module(hb_store_remote_node). --export([scope/1, type/3, read/3, write/3, link/3, group/3, resolve/3]). +-export([scope/1, type/3, read/3, write/3, link/3, group/3, resolve/3, start/1]). %%% Public utilities. -export([maybe_cache/2, maybe_cache/3, read_local_cache/3]). -include("include/hb.hrl"). @@ -19,6 +19,15 @@ scope(_StoreOpts) -> remote. +%% @doc Start local store +%% +%% @param StoreOpts A message with the store options. +%% @returns `ok` or `{failure, _}`. +start(#{<<"local-store">> := LocalStore}) -> + hb_store:start(LocalStore); +start(_StoreOpts) -> + ok. + %% @doc Resolve a key path in the remote store. %% %% For the remote node store, the key is returned as-is. @@ -26,22 +35,25 @@ scope(_StoreOpts) -> %% @param Data A map containing node configuration. %% @param Key The key to resolve. %% @returns The resolved key. +resolve(#{ <<"nodes">> := Nodes }, #{ <<"resolve">> := Key }, _NodeOpts) + when is_list(Nodes) -> + ?event({remote_resolve, {nodes, length(Nodes)}, {key, Key}}), + {ok, Key}; resolve(#{ <<"node">> := Node }, #{ <<"resolve">> := Key }, _NodeOpts) -> ?event({remote_resolve, {node, Node}, {key, Key}}), {ok, Key}. %% @doc Determine the type of value at a given key. %% -%% Remote nodes support `simple', `composite', or `{error, not_found}'. +%% Remote nodes support `simple', or `{error, not_found}'. %% %% @param Opts A map of options (including node configuration). %% @param Key The key whose value type is determined. -%% @returns `{ok, simple}' or `{ok, composite}' if found, or +%% @returns `{ok, simple}' if found, or %% `{error, not_found}' otherwise. -type(Opts = #{ <<"node">> := Node }, #{ <<"type">> := Key }, _NodeOpts) -> - ?event({remote_type, {node, Node}, {key, Key}}), - case read_request(Opts, Key) of - {composite, _} -> {ok, composite}; +type(StoreOpts, #{ <<"type">> := Key }, NodeOpts) -> + ?event({remote_type, {key, Key}}), + case read_request(StoreOpts, Key, NodeOpts) of {ok, _} -> {ok, simple}; Other -> Other end. @@ -54,34 +66,71 @@ type(Opts = #{ <<"node">> := Node }, #{ <<"type">> := Key }, _NodeOpts) -> %% @param Opts A map of options (including node configuration). %% @param Key The key to read. %% @returns `{ok, Msg}' on success or `{error, not_found}' if the key is missing. -read_request(#{ <<"only-ids">> := true }, Key) when not ?IS_ID(Key) -> +read_request(#{ <<"only-ids">> := true }, Key, _NodeOpts) when not ?IS_ID(Key) -> {error, not_found}; -read_request(Opts = #{ <<"node">> := Node }, Key) -> +read_request(#{<<"node">> := Node} = StoreOpts, Key, Opts) -> ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), - HTTPRes = - hb_http:get( - Node, - #{ <<"path">> => <<"/~cache@1.0/read">>, <<"read">> => Key }, - Opts - ), - case HTTPRes of - {ok, Res} -> - % returning the whole response to get the test-key - {ok, Msg} = - hb_message:with_only_committed( - without_transport_commitment(Res, Opts), + %% admissible is only executed in hb_http_multi + StoreOpts2 = (hb_maps:without([<<"node">>], StoreOpts, Opts))#{<<"nodes">> => [Node]}, + read_request(StoreOpts2, Key, Opts); +read_request(#{ <<"nodes">> := Nodes } = StoreOpts, Key, Opts) when is_list(Nodes) -> + case read_local_cache(StoreOpts, Key, Opts) of + {ok, Value} -> + ?event(store_remote_node, {retrieve_from_cache, {key, Key}}), + {ok, Value}; + _ -> + ?event(store_remote_node, {executing_read, {nodes, length(Nodes)}, {key, Key}}), + Config = request_config(Nodes, Key, StoreOpts, Opts), + HTTPRes = + hb_http:request( + <<"GET">>, + Config, + <<"/~cache@1.0/read">>, + #{ <<"read">> => Key }, Opts ), - ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(Opts, Msg, [Key]), - {ok, Msg}; - {error, _Err} -> - ?event(store_remote_node, {read_not_found, {key, Key}}), - {error, not_found} + case HTTPRes of + {ok, Res} -> + % returning the whole response to get the test-key + {ok, Msg} = + hb_message:with_only_committed( + without_transport_commitment(Res, Opts), + Opts + ), + ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), + maybe_cache(StoreOpts, Msg, [Key]), + {ok, Msg}; + {error, Err} -> + ?event(store_remote_node, + {read_not_found, + {key, {string, Key}}, + {error, Err}}), + {error, not_found} + end end; -read_request(_, _) -> {error, not_found}. -read(Opts, #{ <<"read">> := Key }, _NodeOpts) -> - read_request(Opts, Key). +read_request(StoreOpts, _, Opts) -> + ?event(error, + {missing_node_config, + {name, hb_maps:get(<<"name">>, StoreOpts, no_name_store, Opts)}}), + {error, not_found}. + +%% @doc Override nodes and admissible configuration for custom behaviour. +request_config(Nodes, Key, StoreOpts, Opts) -> + Admissible2 = + case hb_maps:get(<<"admissible">>, StoreOpts, undefined, Opts) of + #{} = Admissible -> + Admissible#{<<"requested-key">> => Key}; + Admissible -> + Admissible + end, + ?event(store_remote_node, {request_config, {admissible, Admissible2}}), + StoreOpts#{ + <<"nodes">> => [ node_request(N) || N <- Nodes ], + <<"admissible">> => Admissible2 + }. + +read(StoreOpts, #{ <<"read">> := Key }, NodeOpts) -> + read_request(StoreOpts, Key, NodeOpts). %% @doc Remove the transport commitments from the response. without_transport_commitment(Msg, Opts) when is_map(Msg) -> @@ -95,6 +144,20 @@ without_transport_commitment(Msg, Opts) when is_map(Msg) -> without_transport_commitment(Res, _Opts) -> Res. +%% @doc Extract the base URL of a configured remote node. Accepts a bare URL +%% binary, or a node map carrying a `prefix' or `uri' key. +node_url(#{ <<"prefix">> := Prefix }) -> Prefix; +node_url(#{ <<"uri">> := URI }) -> URI; +node_url(URL) when is_binary(URL) -> URL. + +%% @doc Wrap each configured node so its URL and its per-node `opts' (e.g. +%% `http-reference') both reach the fan-out, letting the admissibility check +%% report which node served the content. +node_request(N) when is_map(N) -> + #{ <<"prefix">> => node_url(N), <<"opts">> => hb_maps:get(<<"opts">>, N, #{}, #{}) }; +node_request(N) -> + #{ <<"prefix">> => node_url(N), <<"opts">> => #{} }. + %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional %% paths may be provided that should be linked to the data. @@ -325,3 +388,65 @@ read_only_ids_test() -> <<"only-ids">> => true } ], ?assertEqual({error, not_found}, hb_cache:read(ID, #{ <<"store">> => RemoteStore })). + +%% @doc Return the admissibility predicate used by remote cache tests. +cache_admissibility() -> + #{ + <<"device">> => <<"cache-admissibility@1.0">>, + <<"path">> => <<"expected-response">> + }. + +%% @doc Read a committed message back through a multi-node remote store and +%% confirm the admissibility check passes and returns the verified message. +multiread_admissible_test() -> + LocalStore = + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-mainnet">> }, + hb_store:reset(LocalStore), + SignOpts = + #{ <<"store">> => LocalStore, <<"priv-wallet">> => ar_wallet:new() }, + Msg = hb_message:commit(#{ <<"test-key">> => <<"router-v1">> }, SignOpts), + SignedID = hb_message:id(Msg, signed, SignOpts), + {ok, _UnsignedID} = hb_cache:write(Msg, #{ <<"store">> => LocalStore }), + Node1 = hb_http_server:start_node(#{ <<"store">> => LocalStore }), + Node2 = hb_http_server:start_node(#{ <<"store">> => LocalStore }), + RemoteStore = + [ #{ <<"store-module">> => hb_store_remote_node, + <<"nodes">> => [Node1, Node2], + <<"admissible">> => cache_admissibility() } ], + {ok, Got} = hb_cache:read(SignedID, #{ <<"store">> => RemoteStore }), + ?assertMatch( + #{ <<"test-key">> := <<"router-v1">> }, + hb_cache:ensure_all_loaded(Got) + ). + +%% @doc A node that serves a different (but validly-signed) message under the +%% requested id must be rejected by the admissibility gate. We prove the gate is +%% what does the rejecting: a single-node store (no gate) returns the wrong +%% message, while the multi-node store (gate) returns not_found. +multiread_rejects_wrong_content_test() -> + LocalStore = + #{ <<"store-module">> => hb_store_fs, <<"name">> => <<"cache-badnode">> }, + hb_store:reset(LocalStore), + SignOpts = + #{ <<"store">> => LocalStore, <<"priv-wallet">> => ar_wallet:new() }, + MsgA = hb_message:commit(#{ <<"k">> => <<"A">> }, SignOpts), + MsgB = hb_message:commit(#{ <<"k">> => <<"B">> }, SignOpts), + IDA = hb_message:id(MsgA, signed, SignOpts), + {ok, IDB} = hb_cache:write(MsgB, #{ <<"store">> => LocalStore }), + %% Poison the node so that a request for IDA resolves to MsgB. + ok = hb_store:link(LocalStore, #{ IDA => IDB }, #{}), + Node = hb_http_server:start_node(#{ <<"store">> => LocalStore }), + %% Without the gate (single node) the poisoned node serves the wrong message. + SingleStore = + [ #{ <<"store-module">> => hb_store_remote_node, <<"node">> => Node } ], + {ok, Wrong} = hb_cache:read(IDA, #{ <<"store">> => SingleStore }), + ?assertMatch(#{ <<"k">> := <<"B">> }, hb_cache:ensure_all_loaded(Wrong)), + %% With the gate (multi-node) the wrong message is rejected. + MultiStore = + [ #{ <<"store-module">> => hb_store_remote_node, + <<"nodes">> => [Node], + <<"admissible">> => cache_admissibility() } ], + ?assertEqual( + {error, not_found}, + hb_cache:read(IDA, #{ <<"store">> => MultiStore }) + ). diff --git a/src/preloaded/codec/dev_httpsig.erl b/src/preloaded/codec/dev_httpsig.erl index fd0cbcb2a..0d1bf45d7 100644 --- a/src/preloaded/codec/dev_httpsig.erl +++ b/src/preloaded/codec/dev_httpsig.erl @@ -602,6 +602,32 @@ validate_large_message_from_http_test() -> ), ?assertEqual([<<"hashpath">>], HashpathCommitted). +%% @doc Ensure that a signed response that contains both `status' and another +%% top-level locally-typed key (e.g. an integer) round-trips through HTTP and +%% verifies on the receiving side. This exercises the case where `ao-types' +%% must agree between signer and verifier for multiple locally-typed fields. +validate_sibling_typed_key_over_http_test() -> + Node = hb_http_server:start_node(Opts = #{ + force_signed => true, + commitment_device => <<"httpsig@1.0">>, + % Top-level locally-typed siblings to `status' in the signed response: + % one integer that sorts before `status' alphabetically, one that sorts + % after, and an atom. All become entries in `ao-types' alongside + % `status'. + <<"alpha-count">> => 7, + <<"test-count">> => 42, + <<"zebra-flag">> => true + }), + {ok, Res} = hb_http:get(Node, <<"/~meta@1.0/info">>, Opts), + ?event({received_with_sibling_typed, Res}), + ?assertEqual(7, hb_ao:get(<<"alpha-count">>, Res, undefined, Opts)), + ?assertEqual(42, hb_ao:get(<<"test-count">>, Res, undefined, Opts)), + ?assertEqual(true, hb_ao:get(<<"zebra-flag">>, Res, undefined, Opts)), + Signers = hb_message:signers(Res, Opts), + ?assert(length(Signers) == 1), + ?assert(hb_message:verify(Res, Signers, Opts)), + ?assert(hb_message:verify(Res, all, Opts)). + committed_id_test() -> Msg = #{ <<"basic">> => <<"value">> }, Opts = #{ <<"priv-wallet">> => hb:wallet() }, diff --git a/src/preloaded/codec/dev_httpsig_siginfo.erl b/src/preloaded/codec/dev_httpsig_siginfo.erl index 4cbd41ca3..995d9ec0b 100644 --- a/src/preloaded/codec/dev_httpsig_siginfo.erl +++ b/src/preloaded/codec/dev_httpsig_siginfo.erl @@ -310,9 +310,15 @@ decoding_nested_map_binary(Bin) -> lists:foldl( fun (X, Acc) -> case binary:split(X, <<":">>, [global]) of - [ID, Key, Value] -> + [ID | Rest] when length(Rest) >= 2 -> + {KeyParts, [Value]} = + lists:split(length(Rest) - 1, Rest), + Key = + iolist_to_binary( + lists:join(<<":">>, KeyParts) + ), Acc#{ - ID => #{ + ID => #{ <<"name">> => Key, <<"value">> => hb_util:decode(Value) } @@ -563,6 +569,10 @@ escaped_value_test() -> <<"2">> => #{ <<"name">> => <<"Quotes">>, <<"value">> => <<"{\"function\":\"mint\"}">> + }, + <<"3">> => #{ + <<"name">> => <<"Data:Protocol">>, + <<"value">> => <<"ao">> } }, <<"signature">> => hb_util:encode(Signature), diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 30549f3e0..8d1175392 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -164,29 +164,38 @@ permissive_id(Msg, Opts) when is_map(Msg) -> hb_message:id(Msg, signed, Opts). %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. name_from_host(Host, no_host) -> + HostName = host_name(Host), % Handle the case where no host key is present in the node message. This % logic is also used when parsing of the host key from the node message % fails, or the node message host is not found in the client provided value % (node claims to be `x.com`, but the user request is for `abc.y.com`). - case binary:split(Host, <<".">>, [global, trim_all]) of + case binary:split(HostName, <<".">>, [global, trim_all]) of [_Host] -> {skip, <<"No subdomain found in `Host: ", Host/binary, "`.">>}; [Name|_] -> - case inet:parse_address(hb_util:list(Host)) of + case inet:parse_address(hb_util:list(HostName)) of {ok, _} -> {skip, <<"No subdomain found in `Host: ", Host/binary, "`.">>}; _ -> {ok, Name} end end; name_from_host(ReqHost, RawNodeHost) -> + ReqHostName = host_name(ReqHost), NodeHostName = maps:get(host, uri_string:parse(RawNodeHost), RawNodeHost), - case binary:split(ReqHost, <<".", NodeHostName/binary>>) of + case binary:split(ReqHostName, <<".", NodeHostName/binary>>) of [Subdomain, <<>>] -> {ok, Subdomain}; - _ when ReqHost =:= NodeHostName -> + _ when ReqHostName =:= NodeHostName -> {skip, <<"No subdomain found in `Host: ", ReqHost/binary, "`.">>}; _ -> name_from_host(ReqHost, no_host) end. +%% @doc Extract the hostname from an HTTP authority. +host_name(RawHost) -> + case uri_string:parse(<<"//", RawHost/binary>>) of + #{host := Host} -> Host; + _ -> RawHost + end. + %% @doc Reject host-derived names that the node has reserved for gateway %% routing. Direct protocol lookups through `~name@1.0/' are intentionally %% left to `resolve/4'. @@ -420,8 +429,16 @@ reserved_host_labels_do_not_block_direct_resolution_test_parallel() -> name_from_host_test_parallel() -> ?assertMatch({skip, _}, name_from_host(<<"127.0.0.1">>, no_host)), + ?assertMatch({skip, _}, name_from_host(<<"localhost:8734">>, no_host)), + ?assertMatch( + {skip, _}, + name_from_host(<<"example.com:8734">>, <<"example.com">>) + ), ?assertEqual({ok, <<"abc">>}, name_from_host(<<"abc.127.0.0.1">>, no_host)), ?assertEqual( {ok, <<"sub3.sub2">>}, - name_from_host(<<"sub3.sub2.sub1.abc.xyz">>, <<"sub1.abc.xyz">>) + name_from_host( + <<"sub3.sub2.sub1.abc.xyz:8734">>, + <<"sub1.abc.xyz">> + ) ). diff --git a/src/preloaded/node/dev_meta.erl b/src/preloaded/node/dev_meta.erl index 48a3d922a..f7af35403 100644 --- a/src/preloaded/node/dev_meta.erl +++ b/src/preloaded/node/dev_meta.erl @@ -86,15 +86,10 @@ build_info() -> handle(NodeMsg, RawRequest) -> ?event({singleton_tabm_request, RawRequest}), NormRequest = hb_singleton:from(RawRequest, NodeMsg), - ?event( - http, - {request, - hb_cache:ensure_all_loaded( - hb_ao:normalize_keys(NormRequest, NodeMsg), - NodeMsg - ) - } - ), + %% Log the raw singleton. Building an `ensure_all_loaded' form here forced + %% store reads on every request even when the `http' topic is off, because + %% the `?event' macro evaluates its argument eagerly. + ?event(http, {request, NormRequest}), case hb_opts:get(initialized, false, NodeMsg) of false -> Res = diff --git a/src/preloaded/test/hb_process_test_vectors.erl b/src/preloaded/test/hb_process_test_vectors.erl index 9fb6b9285..d249b39dd 100644 --- a/src/preloaded/test/hb_process_test_vectors.erl +++ b/src/preloaded/test/hb_process_test_vectors.erl @@ -98,13 +98,21 @@ test_process() -> test_process(#{}). test_process(Opts) -> Wallet = hb:wallet(), + % Strip the base commitment before merging in the execution stack. A + % re-commit over a message that still carries a prior commitment will + % replicate that commitment's `committed' key list verbatim (see the + % "stacked commitments" branch of `keys_to_commit/3' in + % `dev_codec_httpsig'), leaving the newly-added `execution-device' + % and `device-stack' keys out of the signed set — and therefore out + % of the `with_only_committed' view used in compute. The + % `wasm_process'/`aos_process' helpers already follow this pattern. hb_message:commit( hb_maps:merge( - base_process(Opts), + hb_message:uncommitted(base_process(Opts), Opts), #{ <<"execution-device">> => <<"stack@1.0">>, <<"device-stack">> => [<<"test-device@1.0">>, <<"test-device@1.0">>] - }, + }, Opts ), Opts#{ <<"priv-wallet">> => Wallet } diff --git a/src/preloaded/util/dev_relay.erl b/src/preloaded/util/dev_relay.erl index e589e24f8..57246cb7e 100644 --- a/src/preloaded/util/dev_relay.erl +++ b/src/preloaded/util/dev_relay.erl @@ -36,10 +36,10 @@ call(M1, RawM2, Opts) -> RelayPath = hb_ao:get_first( [ - {M1, <<"path">>}, - {{as, <<"message@1.0">>, BaseTarget}, <<"path">>}, {RawM2, <<"relay-path">>}, - {M1, <<"relay-path">>} + {M1, <<"relay-path">>}, + {M1, <<"path">>}, + {{as, <<"message@1.0">>, BaseTarget}, <<"path">>} ], undefined, Opts