From 92fafc1cb8316e8605493688924273947f35d8d8 Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Thu, 10 Sep 2026 12:09:10 +0200 Subject: [PATCH 1/4] refactor: use scheduler quotes for signing --- src/core/util/hb_mock_server.erl | 1 + src/preloaded/codec/dev_tx.erl | 94 ++++- .../process/dev_arweave_scheduler.erl | 111 +++++- src/preloaded/process/dev_push.erl | 332 +++++++++++------- 4 files changed, 400 insertions(+), 138 deletions(-) diff --git a/src/core/util/hb_mock_server.erl b/src/core/util/hb_mock_server.erl index eab4e43f6b..f0ff18279d 100644 --- a/src/core/util/hb_mock_server.erl +++ b/src/core/util/hb_mock_server.erl @@ -65,6 +65,7 @@ start_arweave_gateway(Responses) -> {"/chunk", chunk, maps:get(chunk, Responses, DefaultResponse)}, {"/tx", tx, maps:get(tx, Responses, DefaultResponse)}, {"/price/:size", price, maps:get(price, Responses, DefaultResponse)}, + {"/price/:size/:target", price, maps:get(price, Responses, DefaultResponse)}, {"/tx_anchor", tx_anchor, maps:get(tx_anchor, Responses, DefaultResponse)} ], {ok, MockServer, ServerHandle} = start(Endpoints), diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index a8527b8e18..c29916f6b0 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -17,6 +17,10 @@ commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> commit(Msg, Req#{ <<"type">> => ?RSA_SIGN_TYPE }, Opts); +commit(Msg, Req = #{ <<"type">> := ?RSA_SIGN_TYPE, + <<"tx-header">> := Header }, Opts) -> + {ok, TX} = header(Msg, Header, Opts), + commit(TX, maps:remove(<<"tx-header">>, Req), Opts); commit(Msg, Req = #{ <<"type">> := ?RSA_SIGN_TYPE }, Opts) -> ?event({committing, {msg, Msg}, {req, Req}}), % Convert the given message to an L1 TX record, sign it, and convert @@ -139,7 +143,49 @@ to(TABM, Req, Opts) when is_map(TABM) -> {ok, TX}; to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). - + +%% @doc Encode a data-free scheduling TX with a native quantity of one winston +%% and the quoted reward and anchor. +header(TABM, Header, Opts) -> + TagMsg = + hb_maps:without( + [ + <<"anchor">>, <<"ao-data-key">>, <<"ao-types">>, + <<"commitments">>, <<"data">>, <<"data_root">>, + <<"data_size">>, <<"format">>, <<"reward">>, + <<"tags">>, <<"target">> + ], + hb_private:reset(TABM), + Opts + ), + % Re-signing a committed header preserves its original tag list. + TX0 = + case hb_message:commitment( + #{ <<"commitment-device">> => <<"tx@1.0">> }, TABM, Opts) of + not_found -> + #tx{ tags = lib_arweave_common:tags( + #tx{}, not_found, TagMsg, [], Opts) }; + _ -> hb_util:ok(to(TABM, #{}, Opts)) + end, + TX = TX0#tx{ + format = 2, + target = hb_util:decode(hb_maps:get(<<"target">>, TABM, not_found, Opts)), + quantity = 1, + reward = hb_util:int(hb_maps:get(<<"reward">>, Header, not_found, Opts)), + anchor = hb_util:decode(hb_maps:get(<<"anchor">>, Header, not_found, Opts)), + data = <<>>, + data_size = 0, + data_root = <<>> + }, + enforce_valid_tx(TX), + % L1 tags have a 2048-byte budget and must fit the codec's tag count limit. + case length(TX#tx.tags) =< ?MAX_TAG_COUNT andalso + iolist_size([[Key, Value] || {Key, Value} <- TX#tx.tags]) =< 2048 of + true -> ok; + false -> throw({tx_header_too_large, TX#tx.tags}) + end, + {ok, ar_tx:normalize(TX)}. + %% @doc Verifies that the given transaction is a minimally valid signed or %% unsigned transaction. %% @@ -1456,6 +1502,52 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> FinalTABM = hb_util:ok(from(SignedTX, Req, #{})), ?assertEqual(SignedTABM, FinalTABM, signed_tabm_roundtrip). +header_commitment_test() -> + Opts = #{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => [hb_test_utils:test_store()] + }, + Target = hb_util:human_id(crypto:strong_rand_bytes(32)), + Anchor = hb_util:human_id(crypto:strong_rand_bytes(32)), + Msg = #{ + <<"target">> => Target, + <<"quantity">> => 42, + <<"action">> => <<"Test">> + }, + Header = #{ <<"reward">> => 7, <<"anchor">> => Anchor }, + {ok, HeaderID} = hb_cache:write(Header, Opts), + Spec = #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"tx-header">> => Header + }, + lists:foreach( + fun(QuoteHeader) -> + Signed = hb_message:commit( + Msg, Opts, Spec#{ <<"tx-header">> => QuoteHeader }), + TX = hb_message:convert(Signed, <<"tx@1.0">>, Opts), + ?assertEqual(hb_util:decode(Target), TX#tx.target), + ?assertEqual(1, TX#tx.quantity), + ?assertEqual(7, TX#tx.reward), + ?assertEqual(hb_util:decode(Anchor), TX#tx.anchor), + ?assertEqual(<<>>, TX#tx.data), + ?assertEqual(0, TX#tx.data_size), + ?assertEqual( + [{<<"action">>, <<"Test">>}, {<<"quantity">>, <<"42">>}], + TX#tx.tags + ), + Decoded = hb_message:convert( + TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ?assert(hb_message:verify(Decoded, signers, Opts)), + ?assertEqual(TX, hb_message:convert(Decoded, <<"tx@1.0">>, Opts)) + end, + [Header, {link, HeaderID, #{}}] + ), + ?assertThrow( + {tx_header_too_large, _}, + hb_message:commit( + Msg#{ <<"action">> => binary:copy(<<"x">>, 2048) }, Opts, Spec) + ). + bundle_commitment_test() -> test_bundle_commitment(unbundled, unbundled, unbundled), test_bundle_commitment(unbundled, bundled, unbundled), diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index bab630caad..33c64cee21 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -7,7 +7,7 @@ -implements(<<"arweave-scheduler@1.0">>). -device_libraries([lib_process]). -export([info/0, router/4]). --export([schedule/3, next/3, slot/3, status/3, sync/3, checkpoint/1]). +-export([schedule/3, quote/3, next/3, slot/3, status/3, sync/3, checkpoint/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -21,6 +21,7 @@ info() -> <<"sync">>, <<"next">>, <<"schedule">>, + <<"quote">>, <<"slot">>, <<"init">>, <<"checkpoint">> @@ -70,6 +71,50 @@ schedule(Base, Req, Opts) -> get -> get_schedule(Base, Req, Opts) end. +%% @doc Quote the reward and anchor for a data-free transaction. The quote is +%% a commitment specification; the sender supplies its own signing identity. +quote(Base, Req, Opts) -> + RawTarget = + hb_ao:get_first( + [{Req, <<"body/target">>}, {Req, <<"target">>}, {Base, <<"target">>}], + Opts + ), + [Target | _] = binary:split(RawTarget, [<<"?">>, <<"&">>]), + FreshOpts = Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] + }, + maybe + {ok, Reward} ?= + hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"price">>, + <<"size">> => 0, + <<"target">> => Target + }, + FreshOpts + ), + {ok, Anchor} ?= + hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ <<"path">> => <<"tx-anchor">> }, + FreshOpts + ), + {ok, + #{ + <<"commitment-spec">> => #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"tx-header">> => #{ + <<"reward">> => Reward, + <<"anchor">> => hb_util:encode(Anchor) + } + }, + <<"cache-control">> => <<"no-store">> + } + } + end. + get_schedule(Base, Req, Opts) -> ProcessID = find_process_id(Base, Req, Opts), {From, RequestedTo} = slot_range(Req, Opts), @@ -131,6 +176,11 @@ post_schedule(Base, Req, Opts) -> <<"process">> => ProcessID } }; + {error, #{ <<"require-codec">> := _ } = Rejection} -> + maybe + {ok, Quote} ?= quote(Base, Req, Opts), + {error, hb_maps:merge(Quote, Rejection, Opts)} + end; Error -> Error end. @@ -295,26 +345,59 @@ slot_range_test() -> invalid_commitment_requires_tx_codec_test() -> ProcessID = hb_util:human_id(crypto:strong_rand_bytes(32)), + Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), + {Server, Routes} = hb_mock_server:start_arweave_gateway(#{ + price => {200, <<"7">>}, + tx_anchor => {200, Anchor} + }), Opts = - #{ + Routes#{ <<"priv-wallet">> => ar_wallet:new(), <<"store">> => [hb_test_utils:test_store()] }, Message = hb_message:commit(#{ <<"target">> => ProcessID }, Opts), - ?assertMatch( - {error, - #{ - <<"status">> := 422, - <<"require-codec">> := <<"tx@1.0">> - }}, - hb_ao:resolve( - #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + try + ?assertMatch( + {error, + #{ + <<"status">> := 422, + <<"require-codec">> := <<"tx@1.0">>, + <<"commitment-spec">> := #{ + <<"commitment-device">> := <<"tx@1.0">>, + <<"tx-header">> := #{ + <<"reward">> := 7, + <<"anchor">> := Anchor + } + } + }}, + hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"target">> => ProcessID, + <<"body">> => Message + }, + Opts + ) + ), + Node = hb_http_server:start_node(Opts), + {ok, Quote} = hb_http:post( + Node, #{ - <<"path">> => <<"schedule">>, - <<"method">> => <<"POST">>, - <<"target">> => ProcessID, + <<"path">> => <<"/~arweave-scheduler@1.0/quote">>, <<"body">> => Message }, Opts + ), + ?assertEqual(7, + hb_ao:get(<<"commitment-spec/tx-header/reward">>, Quote, Opts)), + ?assertEqual([], hb_mock_server:get_requests(Server, tx)), + [PriceReq, _] = hb_mock_server:get_requests(Server, price), + ?assertEqual( + <<"/price/0/", ProcessID/binary>>, + hb_ao:get(<<"path">>, PriceReq, Opts) ) - ). + after + hb_mock_server:stop(Server) + end. diff --git a/src/preloaded/process/dev_push.erl b/src/preloaded/process/dev_push.erl index 3f54556dbe..af30352f00 100644 --- a/src/preloaded/process/dev_push.erl +++ b/src/preloaded/process/dev_push.erl @@ -2,7 +2,7 @@ %%% pushes the resulting messages to other processes. The `push'ing mechanism %%% continues until the there are no remaining messages to push. -module(dev_push). --device_libraries([lib_arweave_common, lib_process]). +-device_libraries([lib_process]). %%% Public API -export([push/3]). -include("include/hb.hrl"). @@ -563,37 +563,40 @@ calculate_base_id(GivenProcess, Opts) -> BaseID. %% @doc Add the necessary keys to the message to be scheduled, then schedule it. -%% If the remote scheduler does not support the given codec, it will be -%% downgraded and re-signed. -%% Schedulers can also request another codec explicitly. +%% Use a scheduler quote when available, otherwise negotiate the codec on +%% rejection. The recipient's security policy selects the local signing wallet. schedule_result(TargetProcess, MsgToPush, Origin, Opts) -> - schedule_result( - TargetProcess, - MsgToPush, - hb_opts:get( - scheduler_default_commitment_spec, - <<"httpsig@1.0">>, - Opts - ), - Origin, - Opts - ). -schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> + Augmented = augment_message(Origin, MsgToPush, Opts), + Prepared = normalize_message(Augmented, Opts), + case scheduler_quote(TargetProcess, Prepared, Opts) of + {ok, Spec} -> + schedule_result(TargetProcess, Prepared, Spec, Origin, Opts); + not_found -> + schedule_result( + TargetProcess, + Augmented, + hb_opts:get( + scheduler_default_commitment_spec, <<"httpsig@1.0">>, Opts), + Origin, + Opts + ); + Error -> Error + end. +schedule_result(TargetProcess, MsgToPush, CommitSpec, Origin, Opts) -> Target = hb_ao:get(<<"target">>, MsgToPush, Opts), ?event(push, {push_scheduling_result, {target, {string, Target}}, {target_process, TargetProcess}, {msg, MsgToPush}, - {codec, Codec}, + {codec, CommitSpec}, {origin, Origin} }, Opts ), - AugmentedMsg = augment_message(Origin, MsgToPush, Opts), - ?event(push, {prepared_msg, {msg, AugmentedMsg}}, Opts), + ?event(push, {prepared_msg, {msg, MsgToPush}}, Opts), % Load the `accept-id`'d wallet into the `Opts` map, if requested. - SignedMsg = sign_result(AugmentedMsg, TargetProcess, Codec, Opts), + SignedMsg = apply_security(MsgToPush, TargetProcess, CommitSpec, Opts), % Verify the signed message before writing to cache true = hb_message:verify(SignedMsg, signers, Opts), % Write the signed message to cache before including it in the schedule request @@ -638,22 +641,23 @@ schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> % already-augmented message to the target's policy -- preserving the % `from-*' provenance and honoring the policy, rather than % re-committing the raw message with the default wallet. - NormMsg = normalize_message(AugmentedMsg, Opts), - SignedNormMsg = sign_result(NormMsg, TargetProcess, Codec, Opts), + NormMsg = normalize_message(MsgToPush, Opts), + SignedNormMsg = apply_security( + NormMsg, TargetProcess, CommitSpec, Opts), retry_required_codec( TargetProcess, MsgToPush, - Codec, + CommitSpec, Origin, remote_schedule_result(Location, SignedNormMsg, Opts), Opts ); {error, 422} -> - ?event(push, {wrong_format, {422, Res}, {codec, Codec}}, Opts), + ?event(push, {wrong_format, {422, Res}, {codec, CommitSpec}}, Opts), retry_required_codec( TargetProcess, MsgToPush, - Codec, + CommitSpec, Origin, {error, Res}, Opts @@ -662,31 +666,64 @@ schedule_result(TargetProcess, MsgToPush, Codec, Origin, Opts) -> {error, Res} end. +%% @doc Ask an explicitly advertised scheduler quote key for a signing spec. +scheduler_quote(TargetProcess, Msg, Opts) -> + QuoteOpts = Opts#{ + <<"hashpath">> => ignore, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] + }, + maybe + {ok, Scheduler} ?= + hb_ao:resolve( + {as, <<"process@1.0">>, TargetProcess}, + #{ <<"path">> => <<"as">>, <<"as">> => <<"scheduler">> }, + QuoteOpts + ), + Info = hb_device:info(Scheduler, QuoteOpts), + true ?= lists:member(<<"quote">>, maps:get(exports, Info, [])), + {ok, Quote} ?= + hb_ao:resolve( + Scheduler, + #{ <<"path">> => <<"quote">>, <<"body">> => Msg }, + QuoteOpts + ), + Spec = hb_ao:get(<<"commitment-spec">>, Quote, QuoteOpts), + true ?= is_map(Spec) orelse {error, invalid_commitment_spec}, + {ok, quoted_commitment_spec(Spec)} + else + false -> not_found; + Error -> Error + end. + +%% @doc Apply a quote to the outgoing message using a signed commitment. +quoted_commitment_spec(Spec) -> + Spec#{ <<"target">> => <<"self">>, <<"type">> => <<"signed">> }. + %% @doc Retry an encoding rejection with the scheduler's required codec. retry_required_codec( - TargetProcess, Msg, Codec, Origin, Result = {error, Res}, Opts) -> + TargetProcess, Msg, CurrentSpec, Origin, Result = {error, Res}, Opts) -> DefaultCodec = hb_opts:get( scheduler_default_commitment_spec, <<"httpsig@1.0">>, Opts), RequiredCodec = case hb_maps:get(<<"require-codec">>, Res, not_found, Opts) of - not_found when Codec =:= <<"httpsig@1.0">> -> <<"ans104@1.0">>; - not_found -> Codec; + not_found when CurrentSpec =:= <<"httpsig@1.0">> -> <<"ans104@1.0">>; + not_found -> CurrentSpec; Required -> Required end, case { hb_ao:get(<<"status">>, Res, 500, Opts), - Codec, + CurrentSpec, RequiredCodec } of {422, DefaultCodec, RequiredCodec} when is_binary(RequiredCodec), RequiredCodec =/= DefaultCodec -> - case {Codec, RequiredCodec} of + case {CurrentSpec, RequiredCodec} of {<<"httpsig@1.0">>, <<"ans104@1.0">>} -> ?event(push, {downgrading_to_ans104, {422, Res}, - {codec, Codec}, + {codec, CurrentSpec}, {origin, Origin} }, Opts @@ -694,102 +731,37 @@ retry_required_codec( _ -> ?event(push, {retrying_schedule_codec, - {from, Codec}, + {from, CurrentSpec}, {to, RequiredCodec}, {origin, Origin} }, Opts ) end, + Spec = + hb_maps:get(<<"commitment-spec">>, Res, RequiredCodec, Opts), + {ToSign, CommitSpec} = + case Spec of + _ when is_map(Spec) -> + {normalize_message(Msg, Opts), + quoted_commitment_spec( + Spec#{ <<"commitment-device">> => RequiredCodec } + )}; + _ -> {Msg, RequiredCodec} + end, schedule_result( TargetProcess, - Msg, - RequiredCodec, + ToSign, + CommitSpec, Origin, Opts ); _ -> Result end; -retry_required_codec(_TargetProcess, _Msg, _Codec, _Origin, Result, _Opts) -> +retry_required_codec(_TargetProcess, _Msg, _Spec, _Origin, Result, _Opts) -> Result. -%% @doc Sign a result using the scheduler's required codec. -sign_result(Msg, TargetProcess, <<"tx@1.0">>, Opts) -> - Selected = apply_security( - Msg, TargetProcess, <<"httpsig@1.0">>, Opts), - case hb_message:signers(Selected, Opts) of - [Signer | _] -> sign_l1(Msg, Signer, Opts); - [] -> Selected - end; -sign_result(Msg, TargetProcess, Codec, Opts) -> - apply_security(Msg, TargetProcess, Codec, Opts). - -%% @doc Sign a data-free L1 transaction with an already-selected identity. -sign_l1(Msg, Signer, Opts) -> - Normalized = normalize_message(Msg, Opts), - {ok, SignerOpts} = hb_opts:as(Signer, Opts), - {ok, Price} = - hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - #{ - <<"path">> => <<"/price">>, - <<"size">> => 0, - <<"target">> => hb_ao:get(<<"target">>, Normalized, Opts) - }, - Opts - ), - {ok, Anchor} = - hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - #{ <<"path">> => <<"/tx_anchor">> }, - Opts - ), - commit_l1( - Normalized, - SignerOpts, - Price, - Anchor - ). - -%% @doc Build and sign a data-free L1 transaction from process intent. -commit_l1(Msg, SignerOpts, Price, Anchor) -> - TagMsg = - hb_maps:map( - fun(_, Value) -> hb_util:bin(Value) end, - hb_maps:without( - [ - <<"anchor">>, <<"ao-data-key">>, <<"ao-types">>, - <<"commitments">>, - <<"data">>, <<"data_root">>, <<"data_size">>, - <<"format">>, <<"reward">>, <<"tags">>, <<"target">> - ], - hb_private:reset(Msg), - SignerOpts - ), - SignerOpts - ), - TX0 = #tx{ - format = 2, - target = hb_util:decode(hb_ao:get(<<"target">>, Msg, SignerOpts)), - quantity = 1, - anchor = Anchor, - reward = Price - }, - TX = TX0#tx{ - tags = lib_arweave_common:tags( - TX0, not_found, TagMsg, [], SignerOpts) - }, - {ok, Signed} = - hb_ao:raw( - <<"tx@1.0">>, - <<"commit">>, - TX, - #{ <<"type">> => <<"signed">> }, - SignerOpts - ), - Signed. - %% @doc Set the necessary keys in order for the recipient to know where the %% message came from. augment_message(Origin, ToSched, Opts) -> @@ -1010,6 +982,120 @@ max_depth_test_cases() -> fun test_parse_max_depth/0 ]. +%% @doc Exercise quoted scheduling through the selected scheduler and TX codec. +%% Arweave submission is captured to avoid spending from a test wallet. +scheduler_quote_test_() -> + [ + {atom_to_list(Mode), + {timeout, 30, fun() -> test_push_scheduler_quote(Mode) end}} + || Mode <- [default, tx, required_codec, unavailable, multiple_authorities] + ]. + +test_push_scheduler_quote(Mode) -> + Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), + {Server, Routes} = hb_mock_server:start_arweave_gateway(#{ + price => + case Mode of + unavailable -> {503, <<"Unavailable">>}; + _ -> {200, <<"7">>} + end, + tx_anchor => {200, Anchor}, + tx => {200, <<"OK">>} + }), + Wallet = ar_wallet:new(), + Authority = hb_util:human_id(Wallet), + {Authorities, Identities} = + case Mode of + multiple_authorities -> + OtherWallet = ar_wallet:new(), + OtherAuthority = hb_util:human_id(OtherWallet), + {[OtherAuthority, Authority], + #{ + OtherAuthority => #{ <<"priv-wallet">> => OtherWallet }, + Authority => #{ <<"priv-wallet">> => Wallet } + }}; + _ -> + {Authority, #{ Authority => #{ <<"priv-wallet">> => Wallet } }} + end, + Opts = Routes#{ + <<"priv-wallet">> => ar_wallet:new(), + <<"identities">> => Identities, + <<"scheduler-default-commitment-spec">> => + case Mode of + tx -> <<"tx@1.0">>; + _ -> <<"httpsig@1.0">> + end, + <<"store">> => [hb_test_utils:test_store()] + }, + Process = hb_message:commit( + #{ + <<"device">> => <<"process@1.0">>, + <<"scheduler-device">> => <<"arweave-scheduler@1.0">>, + <<"authority">> => Authorities + }, + Opts + ), + Target = hb_util:human_id(hb_message:id(Process, all, Opts)), + Source = hb_util:human_id(crypto:strong_rand_bytes(32)), + Origin = #{ + <<"process">> => Source, + <<"from-uncommitted">> => Source, + <<"from-base">> => Source, + <<"from-scheduler">> => Authority, + <<"from-authority">> => Authority + }, + Msg = #{ + <<"target">> => Target, + <<"quantity">> => 42, + <<"action">> => <<"Test">> + }, + try + Outcome = + case Mode of + required_codec -> + schedule_result( + Process, + augment_message(Origin, Msg, Opts), + <<"httpsig@1.0">>, + Origin, + Opts + ); + _ -> schedule_result(Process, Msg, Origin, Opts) + end, + case Mode of + unavailable -> + ?assertMatch({error, _}, Outcome), + ?assertEqual([], hb_mock_server:get_requests(Server, tx)), + ?assertEqual([], hb_mock_server:get_requests(Server, tx_anchor)); + _ -> + {ok, Result} = Outcome, + ?assertEqual(202, hb_ao:get(<<"status">>, Result, Opts)), + [Submitted] = hb_mock_server:get_requests(tx, 1, Server), + TX = ar_tx:json_struct_to_tx( + hb_json:decode(hb_ao:get(<<"body">>, Submitted, Opts))), + ?assertEqual(hb_util:human_id(TX#tx.id), + hb_ao:get(<<"txid">>, Result, Opts)), + ?assertEqual(hb_util:decode(Target), TX#tx.target), + ?assertEqual(1, TX#tx.quantity), + ?assertEqual(7, TX#tx.reward), + ?assertEqual(hb_util:decode(Anchor), TX#tx.anchor), + ?assertEqual(0, TX#tx.data_size), + Decoded = hb_message:convert( + TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ?assert(hb_message:verify(Decoded, signers, Opts)), + ?assertEqual([Authority], hb_message:signers(Decoded, Opts)), + ?assertEqual({<<"quantity">>, <<"42">>}, + lists:keyfind(<<"quantity">>, 1, TX#tx.tags)), + ?assertEqual(<<"Test">>, hb_ao:get(<<"action">>, Decoded, Opts)), + ?assertEqual(Source, hb_ao:get(<<"from-process">>, Decoded, Opts)), + ?assertEqual(1, + length(hb_mock_server:get_requests(Server, tx_anchor))) + end, + ?assertEqual(1, length(hb_mock_server:get_requests(Server, price))) + after + hb_mock_server:stop(Server) + end. + test_tx_codec_uses_compute_authority() -> DefaultWallet = ar_wallet:new(), ComputeWallet = ar_wallet:new(), @@ -1032,22 +1118,22 @@ test_tx_codec_uses_compute_authority() -> <<"quantity">> => 42, <<"from-process">> => FromProcess }, - Selected = + Reward = 7, + Anchor = crypto:strong_rand_bytes(32), + Spec = #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"tx-header">> => #{ + <<"reward">> => Reward, + <<"anchor">> => hb_util:human_id(Anchor) + } + }, + Signed = apply_security( - Msg, + normalize_message(Msg, Opts), #{ <<"authority">> => ComputeID }, - <<"httpsig@1.0">>, + Spec, Opts ), - ?assertEqual([ComputeID], hb_message:signers(Selected, Opts)), - Reward = 7, - Anchor = crypto:strong_rand_bytes(32), - Signed = commit_l1( - normalize_message(Msg, Opts), - Opts#{ <<"priv-wallet">> => ComputeWallet }, - Reward, - Anchor - ), TX = hb_message:convert(Signed, <<"tx@1.0">>, Opts), ?assertEqual([<<"tx@1.0">>], hb_message:commitment_devices(Signed, Opts)), ?assertEqual([ComputeID], hb_message:signers(Signed, Opts)), From e06971f7e687c574742578d23f1cf7860aacb26f Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Thu, 10 Sep 2026 18:48:39 +0200 Subject: [PATCH 2/4] chore: fmt --- src/preloaded/process/dev_push.erl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/preloaded/process/dev_push.erl b/src/preloaded/process/dev_push.erl index af30352f00..98736f6877 100644 --- a/src/preloaded/process/dev_push.erl +++ b/src/preloaded/process/dev_push.erl @@ -642,8 +642,8 @@ schedule_result(TargetProcess, MsgToPush, CommitSpec, Origin, Opts) -> % `from-*' provenance and honoring the policy, rather than % re-committing the raw message with the default wallet. NormMsg = normalize_message(MsgToPush, Opts), - SignedNormMsg = apply_security( - NormMsg, TargetProcess, CommitSpec, Opts), + SignedNormMsg = + apply_security(NormMsg, TargetProcess, CommitSpec, Opts), retry_required_codec( TargetProcess, MsgToPush, @@ -743,10 +743,12 @@ retry_required_codec( {ToSign, CommitSpec} = case Spec of _ when is_map(Spec) -> - {normalize_message(Msg, Opts), + { + normalize_message(Msg, Opts), quoted_commitment_spec( Spec#{ <<"commitment-device">> => RequiredCodec } - )}; + ) + }; _ -> {Msg, RequiredCodec} end, schedule_result( From 2686f4d84f17547a92dca548ea8289cff8aeb149 Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Fri, 11 Sep 2026 11:58:42 +0200 Subject: [PATCH 3/4] refactor: use HEAD scheduling with flat commitment specs --- src/preloaded/codec/dev_tx.erl | 87 ++++++--- .../process/dev_arweave_scheduler.erl | 108 ++++++----- src/preloaded/process/dev_push.erl | 174 ++++++++++-------- src/preloaded/process/dev_scheduler.erl | 5 +- 4 files changed, 216 insertions(+), 158 deletions(-) diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index c29916f6b0..42b65d5e18 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -17,15 +17,18 @@ commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> commit(Msg, Req#{ <<"type">> => ?RSA_SIGN_TYPE }, Opts); -commit(Msg, Req = #{ <<"type">> := ?RSA_SIGN_TYPE, - <<"tx-header">> := Header }, Opts) -> - {ok, TX} = header(Msg, Header, Opts), - commit(TX, maps:remove(<<"tx-header">>, Req), Opts); commit(Msg, Req = #{ <<"type">> := ?RSA_SIGN_TYPE }, Opts) -> ?event({committing, {msg, Msg}, {req, Req}}), % Convert the given message to an L1 TX record, sign it, and convert % it back to a structured message. - {ok, TX} = to(hb_private:reset(Msg), Req, Opts), + TABM = hb_private:reset(Msg), + TX0 = + case hb_util:int(hb_maps:get(<<"field-data_size">>, Req, -1, Opts)) of + 0 when is_map(TABM) -> header(TABM, Opts); + _ -> hb_util:ok(to(TABM, Req, Opts)) + end, + TX = ar_tx:normalize(commit_fields(TX0, Req, Opts)), + enforce_valid_tx(TX), Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), Signed = ar_tx:sign(TX, Wallet), SignedStructured = @@ -144,9 +147,22 @@ to(TABM, Req, Opts) when is_map(TABM) -> to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). -%% @doc Encode a data-free scheduling TX with a native quantity of one winston -%% and the quoted reward and anchor. -header(TABM, Header, Opts) -> +%% @doc Apply explicit native fields from the signing request. +commit_fields(TX, Req, Opts) -> + FieldKeys = [<<"field-", Key/binary>> || Key <- ?BASE_FIELDS], + Fields = hb_maps:merge( + dev_tx_from:fields(TX, ?FIELD_PREFIX, Opts), + hb_maps:map( + fun(_, Value) -> hb_util:bin(Value) end, + hb_maps:with(FieldKeys, Req, Opts), + Opts + ), + Opts + ), + dev_tx_to:fields_to_tx(TX, ?FIELD_PREFIX, Fields, Opts). + +%% @doc Encode message tags when the signing request specifies zero data size. +header(TABM, Opts) -> TagMsg = hb_maps:without( [ @@ -163,28 +179,27 @@ header(TABM, Header, Opts) -> case hb_message:commitment( #{ <<"commitment-device">> => <<"tx@1.0">> }, TABM, Opts) of not_found -> - #tx{ tags = lib_arweave_common:tags( - #tx{}, not_found, TagMsg, [], Opts) }; + dev_tx_to:fields_to_tx( + #tx{ tags = lib_arweave_common:tags( + #tx{}, not_found, TagMsg, [], Opts) }, + <<>>, + TABM, + Opts + ); _ -> hb_util:ok(to(TABM, #{}, Opts)) end, TX = TX0#tx{ - format = 2, - target = hb_util:decode(hb_maps:get(<<"target">>, TABM, not_found, Opts)), - quantity = 1, - reward = hb_util:int(hb_maps:get(<<"reward">>, Header, not_found, Opts)), - anchor = hb_util:decode(hb_maps:get(<<"anchor">>, Header, not_found, Opts)), data = <<>>, data_size = 0, data_root = <<>> }, - enforce_valid_tx(TX), % L1 tags have a 2048-byte budget and must fit the codec's tag count limit. case length(TX#tx.tags) =< ?MAX_TAG_COUNT andalso iolist_size([[Key, Value] || {Key, Value} <- TX#tx.tags]) =< 2048 of true -> ok; false -> throw({tx_header_too_large, TX#tx.tags}) end, - {ok, ar_tx:normalize(TX)}. + TX. %% @doc Verifies that the given transaction is a minimally valid signed or %% unsigned transaction. @@ -1508,22 +1523,24 @@ header_commitment_test() -> <<"store">> => [hb_test_utils:test_store()] }, Target = hb_util:human_id(crypto:strong_rand_bytes(32)), - Anchor = hb_util:human_id(crypto:strong_rand_bytes(32)), + Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), Msg = #{ <<"target">> => Target, <<"quantity">> => 42, <<"action">> => <<"Test">> }, - Header = #{ <<"reward">> => 7, <<"anchor">> => Anchor }, - {ok, HeaderID} = hb_cache:write(Header, Opts), + {ok, AnchorID} = hb_cache:write(Anchor, Opts), Spec = #{ <<"commitment-device">> => <<"tx@1.0">>, - <<"tx-header">> => Header + <<"field-quantity">> => 1, + <<"field-data_size">> => 0, + <<"field-reward">> => 7, + <<"field-anchor">> => Anchor }, lists:foreach( - fun(QuoteHeader) -> + fun(QuoteAnchor) -> Signed = hb_message:commit( - Msg, Opts, Spec#{ <<"tx-header">> => QuoteHeader }), + Msg, Opts, Spec#{ <<"field-anchor">> => QuoteAnchor }), TX = hb_message:convert(Signed, <<"tx@1.0">>, Opts), ?assertEqual(hb_util:decode(Target), TX#tx.target), ?assertEqual(1, TX#tx.quantity), @@ -1540,7 +1557,7 @@ header_commitment_test() -> ?assert(hb_message:verify(Decoded, signers, Opts)), ?assertEqual(TX, hb_message:convert(Decoded, <<"tx@1.0">>, Opts)) end, - [Header, {link, HeaderID, #{}}] + [Anchor, {link, AnchorID, #{}}] ), ?assertThrow( {tx_header_too_large, _}, @@ -1548,6 +1565,28 @@ header_commitment_test() -> Msg#{ <<"action">> => binary:copy(<<"x">>, 2048) }, Opts, Spec) ). +commit_fields_test() -> + Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, + Target = hb_util:encode(crypto:strong_rand_bytes(32)), + Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), + Msg = #{ + <<"target">> => Target, + <<"quantity">> => 42, + <<"data">> => <<"Payload">> + }, + Signed = hb_message:commit(Msg, Opts, #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"field-reward">> => 7, + <<"field-anchor">> => Anchor + }), + TX = hb_message:convert(Signed, <<"tx@1.0">>, Opts), + ?assertEqual(hb_util:decode(Target), TX#tx.target), + ?assertEqual(42, TX#tx.quantity), + ?assertEqual(7, TX#tx.reward), + ?assertEqual(hb_util:decode(Anchor), TX#tx.anchor), + ?assertEqual(<<"Payload">>, TX#tx.data), + ?assert(hb_message:verify(Signed, signers, Opts)). + bundle_commitment_test() -> test_bundle_commitment(unbundled, unbundled, unbundled), test_bundle_commitment(unbundled, bundled, unbundled), diff --git a/src/preloaded/process/dev_arweave_scheduler.erl b/src/preloaded/process/dev_arweave_scheduler.erl index 33c64cee21..602e6b08fa 100644 --- a/src/preloaded/process/dev_arweave_scheduler.erl +++ b/src/preloaded/process/dev_arweave_scheduler.erl @@ -7,7 +7,7 @@ -implements(<<"arweave-scheduler@1.0">>). -device_libraries([lib_process]). -export([info/0, router/4]). --export([schedule/3, quote/3, next/3, slot/3, status/3, sync/3, checkpoint/1]). +-export([schedule/3, next/3, slot/3, status/3, sync/3, checkpoint/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -21,7 +21,6 @@ info() -> <<"sync">>, <<"next">>, <<"schedule">>, - <<"quote">>, <<"slot">>, <<"init">>, <<"checkpoint">> @@ -64,22 +63,18 @@ find_assignment(ProcessID, Slot, Opts) -> } end. -%% @doc Read a schedule or relay a presigned data-free L1 transaction. +%% @doc Read a schedule, prepare a commitment, or relay a presigned L1 TX. schedule(Base, Req, Opts) -> case hb_util:key_to_atom(hb_maps:get(<<"method">>, Req, <<"GET">>, Opts)) of + head -> head_schedule(Base, Req, Opts); post -> post_schedule(Base, Req, Opts); get -> get_schedule(Base, Req, Opts) end. %% @doc Quote the reward and anchor for a data-free transaction. The quote is %% a commitment specification; the sender supplies its own signing identity. -quote(Base, Req, Opts) -> - RawTarget = - hb_ao:get_first( - [{Req, <<"body/target">>}, {Req, <<"target">>}, {Base, <<"target">>}], - Opts - ), - [Target | _] = binary:split(RawTarget, [<<"?">>, <<"&">>]), +head_schedule(Base, Req, Opts) -> + Target = find_process_id(Base, Req, Opts), FreshOpts = Opts#{ <<"hashpath">> => ignore, <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] @@ -103,13 +98,14 @@ quote(Base, Req, Opts) -> ), {ok, #{ - <<"commitment-spec">> => #{ - <<"commitment-device">> => <<"tx@1.0">>, - <<"tx-header">> => #{ - <<"reward">> => Reward, - <<"anchor">> => hb_util:encode(Anchor) - } - }, + <<"commitment-device">> => <<"tx@1.0">>, + <<"field-format">> => 2, + <<"field-target">> => Target, + <<"field-quantity">> => 1, + <<"field-reward">> => Reward, + <<"field-anchor">> => hb_util:encode(Anchor), + <<"field-data_size">> => 0, + <<"field-data_root">> => <<>>, <<"cache-control">> => <<"no-store">> } } @@ -176,9 +172,9 @@ post_schedule(Base, Req, Opts) -> <<"process">> => ProcessID } }; - {error, #{ <<"require-codec">> := _ } = Rejection} -> + {error, #{ <<"commitment-device">> := _ } = Rejection} -> maybe - {ok, Quote} ?= quote(Base, Req, Opts), + {ok, Quote} ?= head_schedule(Base, Req, Opts), {error, hb_maps:merge(Quote, Rejection, Opts)} end; Error -> Error @@ -328,7 +324,7 @@ invalid_transaction() -> {error, #{ <<"status">> => 422, - <<"require-codec">> => <<"tx@1.0">>, + <<"commitment-device">> => <<"tx@1.0">>, <<"reason">> => <<"Message must have a valid signed tx@1.0 commitment.">> } @@ -343,8 +339,7 @@ slot_range_test() -> slot_range(#{ <<"from">> => -5, <<"to">> => 42 }, #{}) ). -invalid_commitment_requires_tx_codec_test() -> - ProcessID = hb_util:human_id(crypto:strong_rand_bytes(32)), +schedule_commitment_spec_test() -> Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), {Server, Routes} = hb_mock_server:start_arweave_gateway(#{ price => {200, <<"7">>}, @@ -355,43 +350,54 @@ invalid_commitment_requires_tx_codec_test() -> <<"priv-wallet">> => ar_wallet:new(), <<"store">> => [hb_test_utils:test_store()] }, + Process = hb_message:commit(#{ + <<"device">> => <<"process@1.0">>, + <<"scheduler-device">> => <<"arweave-scheduler@1.0">> + }, Opts), + {ok, _} = hb_cache:write(Process, Opts), + ProcessID = hb_message:id(Process, all, Opts), Message = hb_message:commit(#{ <<"target">> => ProcessID }, Opts), try + {error, Rejection} = hb_ao:resolve( + #{ <<"device">> => <<"arweave-scheduler@1.0">> }, + #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"target">> => ProcessID, + <<"body">> => Message + }, + Opts + ), ?assertMatch( - {error, - #{ - <<"status">> := 422, - <<"require-codec">> := <<"tx@1.0">>, - <<"commitment-spec">> := #{ - <<"commitment-device">> := <<"tx@1.0">>, - <<"tx-header">> := #{ - <<"reward">> := 7, - <<"anchor">> := Anchor - } - } - }}, - hb_ao:resolve( - #{ <<"device">> => <<"arweave-scheduler@1.0">> }, - #{ - <<"path">> => <<"schedule">>, - <<"method">> => <<"POST">>, - <<"target">> => ProcessID, - <<"body">> => Message - }, - Opts - ) + #{ + <<"status">> := 422, + <<"commitment-device">> := <<"tx@1.0">>, + <<"field-reward">> := 7, + <<"field-anchor">> := Anchor + }, + Rejection ), + Signed = hb_message:commit(Message, Opts, Rejection), + ?assert(hb_message:verify(Signed, signers, Opts)), Node = hb_http_server:start_node(Opts), - {ok, Quote} = hb_http:post( + {ok, Quote} = hb_http:request( + <<"HEAD">>, Node, - #{ - <<"path">> => <<"/~arweave-scheduler@1.0/quote">>, - <<"body">> => Message - }, + <<"/", ProcessID/binary, "/schedule">>, + #{ <<"accept">> => <<"application/httpsig">> }, Opts ), - ?assertEqual(7, - hb_ao:get(<<"commitment-spec/tx-header/reward">>, Quote, Opts)), + ?assertEqual(7, hb_ao:get(<<"field-reward">>, Quote, Opts)), + ?assertEqual(Anchor, hb_ao:get(<<"field-anchor">>, Quote, Opts)), + ?assertEqual(<<"tx@1.0">>, + hb_ao:get(<<"commitment-device">>, Quote, Opts)), + HeadSigned = hb_message:commit(Message, Opts, Quote), + TX = hb_message:convert(HeadSigned, <<"tx@1.0">>, Opts), + ?assertEqual(hb_util:decode(ProcessID), TX#tx.target), + ?assertEqual(7, TX#tx.reward), + ?assertEqual(hb_util:decode(Anchor), TX#tx.anchor), + ?assertEqual(1, TX#tx.quantity), + ?assert(hb_message:verify(HeadSigned, signers, Opts)), ?assertEqual([], hb_mock_server:get_requests(Server, tx)), [PriceReq, _] = hb_mock_server:get_requests(Server, price), ?assertEqual( diff --git a/src/preloaded/process/dev_push.erl b/src/preloaded/process/dev_push.erl index 98736f6877..711da01b4b 100644 --- a/src/preloaded/process/dev_push.erl +++ b/src/preloaded/process/dev_push.erl @@ -563,12 +563,12 @@ calculate_base_id(GivenProcess, Opts) -> BaseID. %% @doc Add the necessary keys to the message to be scheduled, then schedule it. -%% Use a scheduler quote when available, otherwise negotiate the codec on -%% rejection. The recipient's security policy selects the local signing wallet. +%% HEAD the schedule for a commitment spec, otherwise negotiate on rejection. +%% The recipient's security policy selects the local signing wallet. schedule_result(TargetProcess, MsgToPush, Origin, Opts) -> Augmented = augment_message(Origin, MsgToPush, Opts), Prepared = normalize_message(Augmented, Opts), - case scheduler_quote(TargetProcess, Prepared, Opts) of + case scheduler_commitment_spec(TargetProcess, Opts) of {ok, Spec} -> schedule_result(TargetProcess, Prepared, Spec, Origin, Opts); not_found -> @@ -644,7 +644,7 @@ schedule_result(TargetProcess, MsgToPush, CommitSpec, Origin, Opts) -> NormMsg = normalize_message(MsgToPush, Opts), SignedNormMsg = apply_security(NormMsg, TargetProcess, CommitSpec, Opts), - retry_required_codec( + retry_schedule( TargetProcess, MsgToPush, CommitSpec, @@ -654,7 +654,7 @@ schedule_result(TargetProcess, MsgToPush, CommitSpec, Origin, Opts) -> ); {error, 422} -> ?event(push, {wrong_format, {422, Res}, {codec, CommitSpec}}, Opts), - retry_required_codec( + retry_schedule( TargetProcess, MsgToPush, CommitSpec, @@ -666,102 +666,65 @@ schedule_result(TargetProcess, MsgToPush, CommitSpec, Origin, Opts) -> {error, Res} end. -%% @doc Ask an explicitly advertised scheduler quote key for a signing spec. -scheduler_quote(TargetProcess, Msg, Opts) -> +%% @doc Ask the target process's schedule for a commitment spec using HEAD. +scheduler_commitment_spec(TargetProcess, Opts) -> QuoteOpts = Opts#{ <<"hashpath">> => ignore, <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] }, maybe - {ok, Scheduler} ?= + {ok, Res} ?= hb_ao:resolve( {as, <<"process@1.0">>, TargetProcess}, - #{ <<"path">> => <<"as">>, <<"as">> => <<"scheduler">> }, + #{ <<"path">> => <<"schedule">>, <<"method">> => <<"HEAD">> }, QuoteOpts ), - Info = hb_device:info(Scheduler, QuoteOpts), - true ?= lists:member(<<"quote">>, maps:get(exports, Info, [])), - {ok, Quote} ?= - hb_ao:resolve( - Scheduler, - #{ <<"path">> => <<"quote">>, <<"body">> => Msg }, - QuoteOpts - ), - Spec = hb_ao:get(<<"commitment-spec">>, Quote, QuoteOpts), - true ?= is_map(Spec) orelse {error, invalid_commitment_spec}, - {ok, quoted_commitment_spec(Spec)} + Status = hb_ao:get(<<"status">>, Res, 200, QuoteOpts), + true ?= (Status >= 200 andalso Status < 300) orelse {error, Res}, + case hb_ao:get(<<"commitment-device">>, Res, not_found, QuoteOpts) of + not_found -> not_found; + _ -> {ok, Res} + end else - false -> not_found; Error -> Error end. -%% @doc Apply a quote to the outgoing message using a signed commitment. -quoted_commitment_spec(Spec) -> - Spec#{ <<"target">> => <<"self">>, <<"type">> => <<"signed">> }. - -%% @doc Retry an encoding rejection with the scheduler's required codec. -retry_required_codec( +%% @doc Retry a rejection with its commitment spec, or the legacy ANS-104 fallback. +retry_schedule( TargetProcess, Msg, CurrentSpec, Origin, Result = {error, Res}, Opts) -> - DefaultCodec = hb_opts:get( + DefaultSpec = hb_opts:get( scheduler_default_commitment_spec, <<"httpsig@1.0">>, Opts), - RequiredCodec = - case hb_maps:get(<<"require-codec">>, Res, not_found, Opts) of + Spec = + case hb_ao:get(<<"commitment-device">>, Res, not_found, Opts) of not_found when CurrentSpec =:= <<"httpsig@1.0">> -> <<"ans104@1.0">>; not_found -> CurrentSpec; - Required -> Required + _ -> Res end, case { hb_ao:get(<<"status">>, Res, 500, Opts), CurrentSpec, - RequiredCodec + Spec } of - {422, DefaultCodec, RequiredCodec} - when is_binary(RequiredCodec), - RequiredCodec =/= DefaultCodec -> - case {CurrentSpec, RequiredCodec} of - {<<"httpsig@1.0">>, <<"ans104@1.0">>} -> - ?event(push, - {downgrading_to_ans104, - {422, Res}, - {codec, CurrentSpec}, - {origin, Origin} - }, - Opts - ); - _ -> - ?event(push, - {retrying_schedule_codec, - {from, CurrentSpec}, - {to, RequiredCodec}, - {origin, Origin} - }, - Opts - ) - end, - Spec = - hb_maps:get(<<"commitment-spec">>, Res, RequiredCodec, Opts), - {ToSign, CommitSpec} = - case Spec of - _ when is_map(Spec) -> - { - normalize_message(Msg, Opts), - quoted_commitment_spec( - Spec#{ <<"commitment-device">> => RequiredCodec } - ) - }; - _ -> {Msg, RequiredCodec} - end, + {422, DefaultSpec, Spec} when Spec =/= CurrentSpec -> + ?event(push, + {retrying_schedule_commitment, + {from, CurrentSpec}, + {to, Spec}, + {origin, Origin} + }, + Opts + ), schedule_result( TargetProcess, - ToSign, - CommitSpec, + Msg, + Spec, Origin, Opts ); _ -> Result end; -retry_required_codec(_TargetProcess, _Msg, _Spec, _Origin, Result, _Opts) -> +retry_schedule(_TargetProcess, _Msg, _Spec, _Origin, Result, _Opts) -> Result. %% @doc Set the necessary keys in order for the recipient to know where the @@ -984,16 +947,65 @@ max_depth_test_cases() -> fun test_parse_max_depth/0 ]. -%% @doc Exercise quoted scheduling through the selected scheduler and TX codec. +%% @doc Exercise HEAD scheduling through the selected scheduler and TX codec. %% Arweave submission is captured to avoid spending from a test wallet. -scheduler_quote_test_() -> +scheduler_commitment_spec_test_() -> [ {atom_to_list(Mode), - {timeout, 30, fun() -> test_push_scheduler_quote(Mode) end}} - || Mode <- [default, tx, required_codec, unavailable, multiple_authorities] + {timeout, 30, fun() -> test_push_scheduler_spec(Mode) end}} + || Mode <- [default, tx, rejection, unavailable, multiple_authorities] ]. -test_push_scheduler_quote(Mode) -> +legacy_scheduler_spec_test() -> + Opts = #{ + <<"priv-wallet">> => Wallet = ar_wallet:new(), + <<"store">> => [hb_test_utils:test_store()] + }, + Authority = hb_util:human_id(Wallet), + Process = hb_message:commit(#{ + <<"device">> => <<"process@1.0">>, + <<"type">> => <<"Process">>, + <<"scheduler">> => Authority, + <<"authority">> => Authority + }, Opts), + {ok, _} = hb_cache:write(Process, Opts), + ?assertEqual(not_found, scheduler_commitment_spec(Process, Opts)), + {ok, _} = hb_ao:resolve(Process, #{ + <<"path">> => <<"schedule">>, + <<"method">> => <<"POST">>, + <<"body">> => Process + }, Opts), + Msg = #{ + <<"target">> => hb_message:id(Process, all, Opts), + <<"action">> => <<"Test">> + }, + lists:foreach( + fun(Rejection) -> + {ok, Assignment} = retry_schedule( + Process, Msg, <<"httpsig@1.0">>, #{}, {error, Rejection}, Opts), + Scheduled = hb_ao:get(<<"body">>, Assignment, Opts), + ?assertEqual([<<"ans104@1.0">>], + hb_message:commitment_devices(Scheduled, Opts)), + ?assertEqual([Authority], hb_message:signers(Scheduled, Opts)), + ?assert(hb_message:verify(Scheduled, signers, Opts)), + RetrySpec = + case hb_maps:is_key(<<"commitment-device">>, Rejection, Opts) of + true -> Rejection; + false -> <<"ans104@1.0">> + end, + ?assertEqual({error, Rejection}, retry_schedule( + Process, Msg, RetrySpec, #{}, {error, Rejection}, Opts)) + end, + [ + #{ <<"status">> => 422 }, + #{ + <<"status">> => 422, + <<"commitment-device">> => <<"ans104@1.0">> + } + ] + ). + +test_push_scheduler_spec(Mode) -> Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), {Server, Routes} = hb_mock_server:start_arweave_gateway(#{ price => @@ -1054,7 +1066,7 @@ test_push_scheduler_quote(Mode) -> try Outcome = case Mode of - required_codec -> + rejection -> schedule_result( Process, augment_message(Origin, Msg, Opts), @@ -1124,10 +1136,10 @@ test_tx_codec_uses_compute_authority() -> Anchor = crypto:strong_rand_bytes(32), Spec = #{ <<"commitment-device">> => <<"tx@1.0">>, - <<"tx-header">> => #{ - <<"reward">> => Reward, - <<"anchor">> => hb_util:human_id(Anchor) - } + <<"field-quantity">> => 1, + <<"field-data_size">> => 0, + <<"field-reward">> => Reward, + <<"field-anchor">> => hb_util:encode(Anchor) }, Signed = apply_security( diff --git a/src/preloaded/process/dev_scheduler.erl b/src/preloaded/process/dev_scheduler.erl index abe4ff0d7c..0ea3459b2c 100644 --- a/src/preloaded/process/dev_scheduler.erl +++ b/src/preloaded/process/dev_scheduler.erl @@ -366,6 +366,7 @@ status(_M1, _M2, _Opts) -> schedule(Base, Req, Opts) -> ?event({resolving_schedule_request, {req, Req}, {state_msg, Base}}), case hb_util:key_to_atom(hb_ao:get(<<"method">>, Req, <<"GET">>, Opts)) of + head -> {ok, #{}}; post -> post_schedule(Base, Req, Opts); get -> get_schedule(Base, Req, Opts) end. @@ -1182,7 +1183,7 @@ post_remote_schedule(RawProcID, Redirect, OnlyCommitted, Opts) -> [] -> {error, #{ <<"status">> => 422, - <<"require-codec">> => <<"ans104@1.0">>, + <<"commitment-device">> => <<"ans104@1.0">>, <<"body">> => << "Process resides on legacy scheduler. ", @@ -1235,7 +1236,7 @@ post_legacy_schedule(ProcID, OnlyCommitted, Node, Opts) -> ?event({could_not_encode_for_legacy_scheduler, {error, EncodingErr}}), {error, #{ <<"status">> => 422, - <<"require-codec">> => <<"ans104@1.0">>, + <<"commitment-device">> => <<"ans104@1.0">>, <<"body">> => <<"Incorrect encoding. Scheduler has variant: ao.TN.1">> } From 280deb585344e931914606935f6cd02248b697ed Mon Sep 17 00:00:00 2001 From: Rani Elhusseini Date: Fri, 11 Sep 2026 19:00:36 +0200 Subject: [PATCH 4/4] chore: address tx codec comments --- src/preloaded/codec/dev_tx.erl | 203 +++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 50 deletions(-) diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index 42b65d5e18..479e264f13 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -22,12 +22,12 @@ commit(Msg, Req = #{ <<"type">> := ?RSA_SIGN_TYPE }, Opts) -> % Convert the given message to an L1 TX record, sign it, and convert % it back to a structured message. TABM = hb_private:reset(Msg), - TX0 = - case hb_util:int(hb_maps:get(<<"field-data_size">>, Req, -1, Opts)) of - 0 when is_map(TABM) -> header(TABM, Opts); - _ -> hb_util:ok(to(TABM, Req, Opts)) - end, + ZeroData = hb_util:int(hb_maps:get(<<"field-data_size">>, Req, -1, Opts)) =:= 0, + {ok, TX0} = to(TABM, Req, ZeroData, Opts), + % Validate before overrides can replace an unloaded payload's metadata. + enforce_commit_spec(TX0, ZeroData), TX = ar_tx:normalize(commit_fields(TX0, Req, Opts)), + enforce_commit_spec(TX, ZeroData), enforce_valid_tx(TX), Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), Signed = ar_tx:sign(TX, Wallet), @@ -134,18 +134,36 @@ to(Binary, _Req, _Opts) when is_binary(Binary) -> }; to(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, TX}; to(TABM, Req, Opts) when is_map(TABM) -> + to(TABM, Req, false, Opts); +to(Other, _Req, _Opts) -> + throw({invalid_tx, Other}). + +%% @doc Encode a message, retaining intent tags for zero-data commitments. +to(TABM, Req, ZeroData, Opts) when is_map(TABM) -> ?event({to, {inbound, TABM}, {req, Req}}), + % Ignore an inline hint only when its data key is absent. + ToEncode = + maybe + true ?= ZeroData, + {ok, DataKey} ?= hb_maps:find(<<"ao-data-key">>, TABM, Opts), + false ?= hb_maps:is_key(DataKey, TABM, Opts), + hb_maps:remove(<<"ao-data-key">>, TABM, Opts) + else + _ -> TABM + end, TX = lib_arweave_common:to( - <<"tx@1.0">>, TABM, Req, + <<"tx@1.0">>, ToEncode, Req, fun dev_tx_to:fields_to_tx/4, - fun dev_tx_to:excluded_tags/3, + fun(TX0, TagMsg, TagOpts) -> + commit_tag_exclusions(TX0, TagMsg, ZeroData, TagOpts) + end, Opts ), enforce_valid_tx(TX), ?event({to_result, TX}), {ok, TX}; -to(Other, _Req, _Opts) -> - throw({invalid_tx, Other}). +to(Other, Req, _ZeroData, Opts) -> + to(Other, Req, Opts). %% @doc Apply explicit native fields from the signing request. commit_fields(TX, Req, Opts) -> @@ -161,45 +179,32 @@ commit_fields(TX, Req, Opts) -> ), dev_tx_to:fields_to_tx(TX, ?FIELD_PREFIX, Fields, Opts). -%% @doc Encode message tags when the signing request specifies zero data size. -header(TABM, Opts) -> - TagMsg = - hb_maps:without( - [ - <<"anchor">>, <<"ao-data-key">>, <<"ao-types">>, - <<"commitments">>, <<"data">>, <<"data_root">>, - <<"data_size">>, <<"format">>, <<"reward">>, - <<"tags">>, <<"target">> - ], - hb_private:reset(TABM), - Opts - ), - % Re-signing a committed header preserves its original tag list. - TX0 = - case hb_message:commitment( - #{ <<"commitment-device">> => <<"tx@1.0">> }, TABM, Opts) of - not_found -> - dev_tx_to:fields_to_tx( - #tx{ tags = lib_arweave_common:tags( - #tx{}, not_found, TagMsg, [], Opts) }, - <<>>, - TABM, - Opts - ); - _ -> hb_util:ok(to(TABM, #{}, Opts)) - end, - TX = TX0#tx{ - data = <<>>, - data_size = 0, - data_root = <<>> - }, +%% @doc Keep intent tags on fresh data-free messages and preserve committed tags. +commit_tag_exclusions(TX, TABM, true, Opts) -> + case hb_message:commitment( + #{ <<"commitment-device">> => <<"tx@1.0">> }, TABM, Opts) of + not_found -> + (?BASE_FIELDS -- [<<"quantity">>]) ++ + [<<"ao-data-key">>, <<"ao-types">>, <<"data">>, <<"tags">>]; + _ -> dev_tx_to:excluded_tags(TX, TABM, Opts) + end; +commit_tag_exclusions(TX, TABM, false, Opts) -> + dev_tx_to:excluded_tags(TX, TABM, Opts). + +%% @doc Enforce the requested zero-data constraint without discarding content. +enforce_commit_spec(_TX, false) -> ok; +enforce_commit_spec(TX, true) -> + case TX#tx.data =:= <<>> andalso TX#tx.data_size =:= 0 andalso + TX#tx.data_root =:= <<>> of + true -> ok; + false -> throw(tx_data_not_allowed) + end, % L1 tags have a 2048-byte budget and must fit the codec's tag count limit. case length(TX#tx.tags) =< ?MAX_TAG_COUNT andalso iolist_size([[Key, Value] || {Key, Value} <- TX#tx.tags]) =< 2048 of true -> ok; false -> throw({tx_header_too_large, TX#tx.tags}) - end, - TX. + end. %% @doc Verifies that the given transaction is a minimally valid signed or %% unsigned transaction. @@ -1517,7 +1522,7 @@ do_signed_tabm_roundtrip(UnsignedTX, UnsignedTABM, Commitment, Device, Req) -> FinalTABM = hb_util:ok(from(SignedTX, Req, #{})), ?assertEqual(SignedTABM, FinalTABM, signed_tabm_roundtrip). -header_commitment_test() -> +zero_data_commitment_test() -> Opts = #{ <<"priv-wallet">> => ar_wallet:new(), <<"store">> => [hb_test_utils:test_store()] @@ -1526,6 +1531,7 @@ header_commitment_test() -> Anchor = hb_util:encode(crypto:strong_rand_bytes(48)), Msg = #{ <<"target">> => Target, + <<"ao-data-key">> => <<"body">>, <<"quantity">> => 42, <<"action">> => <<"Test">> }, @@ -1538,9 +1544,14 @@ header_commitment_test() -> <<"field-anchor">> => Anchor }, lists:foreach( - fun(QuoteAnchor) -> + fun({Quantity, QuoteAnchor}) -> + Message = + case Quantity of + not_found -> maps:remove(<<"quantity">>, Msg); + _ -> Msg#{ <<"quantity">> => Quantity } + end, Signed = hb_message:commit( - Msg, Opts, Spec#{ <<"field-anchor">> => QuoteAnchor }), + Message, Opts, Spec#{ <<"field-anchor">> => QuoteAnchor }), TX = hb_message:convert(Signed, <<"tx@1.0">>, Opts), ?assertEqual(hb_util:decode(Target), TX#tx.target), ?assertEqual(1, TX#tx.quantity), @@ -1548,23 +1559,115 @@ header_commitment_test() -> ?assertEqual(hb_util:decode(Anchor), TX#tx.anchor), ?assertEqual(<<>>, TX#tx.data), ?assertEqual(0, TX#tx.data_size), + ?assertEqual(<<>>, TX#tx.data_root), + QuantityTags = + case Quantity of + not_found -> []; + _ -> [{<<"quantity">>, hb_util:bin(Quantity)}] + end, ?assertEqual( - [{<<"action">>, <<"Test">>}, {<<"quantity">>, <<"42">>}], + [{<<"action">>, <<"Test">>} | QuantityTags], TX#tx.tags ), Decoded = hb_message:convert( TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ?assert(hb_message:verify(Decoded, signers, Opts)), - ?assertEqual(TX, hb_message:convert(Decoded, <<"tx@1.0">>, Opts)) + ?assertEqual(TX, hb_message:convert(Decoded, <<"tx@1.0">>, Opts)), + ?assertEqual(Target, hb_ao:get(<<"target">>, Decoded, Opts)), + ?assertEqual(Anchor, hb_ao:get(<<"anchor">>, Decoded, Opts)), + ?assertEqual(1, hb_util:int(hb_ao:get(<<"quantity">>, Decoded, Opts))), + ?assertEqual(7, hb_util:int(hb_ao:get(<<"reward">>, Decoded, Opts))), + Resigned = hb_message:commit(Decoded, Opts, Spec#{ + <<"field-reward">> => <<"8">>, + <<"field-data_size">> => <<"0">> + }), + ResignedTX = hb_message:convert(Resigned, <<"tx@1.0">>, Opts), + ?assertEqual(TX#tx.tags, ResignedTX#tx.tags), + ?assertEqual(8, ResignedTX#tx.reward), + ?assert(hb_message:verify(Resigned, signers, Opts)) end, - [Anchor, {link, AnchorID, #{}}] + [{not_found, Anchor}, {1, Anchor}, {42, Anchor}, + {42, {link, AnchorID, #{}}}] ), + % A native-only quantity must not become a tag when re-signing. + Native = hb_message:commit(Msg, Opts, <<"tx@1.0">>), + OriginalTX = hb_message:convert(Native, <<"tx@1.0">>, Opts), + ResignedNative = hb_message:commit(Native, Opts, Spec), + NativeTX = hb_message:convert(ResignedNative, <<"tx@1.0">>, Opts), + ?assertEqual(1, NativeTX#tx.quantity), + ?assertEqual(OriginalTX#tx.tags, NativeTX#tx.tags), + ?assertNot(lists:keymember(<<"quantity">>, 1, NativeTX#tx.tags)), + ?assert(hb_message:verify(ResignedNative, signers, Opts)), + % The L1 tag budget is aggregate, even when individual tags fit. + BudgetMsg = #{ + <<"target">> => Target, + <<"a">> => binary:copy(<<"x">>, 1023), + <<"b">> => binary:copy(<<"x">>, 1023) + }, + BudgetSigned = hb_message:commit(BudgetMsg, Opts, Spec), + ?assert(hb_message:verify(BudgetSigned, signers, Opts)), ?assertThrow( {tx_header_too_large, _}, hb_message:commit( - Msg#{ <<"action">> => binary:copy(<<"x">>, 2048) }, Opts, Spec) + BudgetMsg#{ <<"b">> => binary:copy(<<"x">>, 1024) }, Opts, Spec) ). +zero_data_rejects_payload_test() -> + Opts = #{ + <<"priv-wallet">> => ar_wallet:new(), + <<"store">> => [hb_test_utils:test_store()] + }, + Msg = #{ <<"target">> => hb_util:encode(crypto:strong_rand_bytes(32)) }, + Root = ar_tx:data_root(arweavejs, <<"Hello">>), + Spec = #{ + <<"commitment-device">> => <<"tx@1.0">>, + <<"field-quantity">> => 1, + <<"field-data_size">> => 0, + <<"field-data_root">> => <<>> + }, + Detached = hb_message:commit(Msg#{ + <<"data_size">> => 5, + <<"data_root">> => hb_util:encode(Root) + }, Opts, <<"tx@1.0">>), + lists:foreach( + fun(Payload) -> + ?assertThrow(tx_data_not_allowed, + hb_message:commit(Payload, Opts, Spec)) + end, + [ + Msg#{ <<"data">> => <<"Hello">> }, + Msg#{ <<"body">> => <<"Hello">> }, + Msg#{ <<"ao-data-key">> => <<"content">>, <<"content">> => <<"Hello">> }, + Msg#{ <<"data_size">> => 5, <<"data_root">> => hb_util:encode(Root) }, + Msg#{ <<"data_root">> => hb_util:encode(Root) }, + Detached + ] + ), + % Bundled content must also satisfy the zero-data requirement. + lists:foreach( + fun(Payload) -> + ?assertThrow(tx_data_not_allowed, + hb_message:commit(Payload, Opts, Spec#{ <<"bundle">> => true })) + end, + [ + Msg#{ <<"nested">> => #{ <<"value">> => <<"Hello">> } }, + Msg#{ <<"large">> => binary:copy(<<"x">>, ?MAX_TAG_VALUE_SIZE + 1) } + ] + ), + ?assertThrow(tx_data_not_allowed, hb_message:commit(Msg, Opts, + Spec#{ <<"field-data_root">> => hb_util:encode(Root) })), + ?assertThrow(tx_data_not_allowed, hb_message:commit( + Msg#{ <<"body">> => <<"Hello">> }, + Opts#{ <<"priv-wallet">> => no_viable_wallet }, + Spec + )), + ?assertThrow(tx_data_not_allowed, hb_ao:raw( + <<"tx@1.0">>, <<"commit">>, + #tx{ format = 2, data_size = 5, data_root = Root }, + Spec#{ <<"type">> => <<"signed">> }, + Opts + )). + commit_fields_test() -> Opts = #{ <<"priv-wallet">> => ar_wallet:new() }, Target = hb_util:encode(crypto:strong_rand_bytes(32)),