diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index df9ea5aa9..2b6583f58 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -145,8 +145,7 @@ info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> {ok, Exclude} -> case lists:member(Key, Exclude) of true -> - MsgWithoutDevice = - hb_maps:without([<<"device">>], Msg, Opts), + MsgWithoutDevice = hb_maps:without([<<"device">>], Msg, Opts), message_to_fun( MsgWithoutDevice#{ <<"device">> => ?DEFAULT_DEVICE }, Key, diff --git a/src/core/device/hb_device_load.erl b/src/core/device/hb_device_load.erl index b132df0f6..44fccdd4f 100644 --- a/src/core/device/hb_device_load.erl +++ b/src/core/device/hb_device_load.erl @@ -23,7 +23,7 @@ %%% signature. %%% -module(hb_device_load). --export([reference/2]). +-export([reference/2, schema/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -297,13 +297,64 @@ load_archive(ID, Opts) -> end. load_archive_message(Msg, Opts) -> - hb_device_archive:load( - hb_maps:get(<<"module-name">>, Msg, undefined, Opts), - hb_maps:get(<<"body">>, Msg, undefined, Opts), - Msg, + Archive = hb_maps:get(<<"body">>, Msg, undefined, Opts), + maybe + {ok, Module} ?= + hb_device_archive:load( + hb_maps:get(<<"module-name">>, Msg, undefined, Opts), + Archive, + Msg, + Opts + ), + put_schemas(Archive, Opts), + {ok, Module} + end. + +%% @doc The function schemas of a loaded module (see `hb_types'): the +%% process dictionary first, then the shared `loaded-device-store', then +%% the module's own object code. A generated module name carries the hash +%% of the source that built it, so the atom alone identifies its schemas. +schema(Module, Opts) -> + case erlang:get({?MODULE, schema, Module}) of + undefined -> + Schemas = + maybe + {error, not_found} ?= + hb_store:read( + loaded_device_store(Opts), + schema_key(Module), + Opts + ), + hb_types:extract(Module) + end, + erlang:put({?MODULE, schema, Module}, Schemas), + Schemas; + Schemas -> + Schemas + end. + +%% @doc Memoise the function schemas of every module in a loaded archive in +%% the shared `loaded-device-store'. Archive modules are loaded from memory +%% rather than the code path, so their BEAMs are in hand here alone. The +%% schemas are Erlang terms: the store must be an `hb_store_volatile'. +put_schemas(Archive, Opts) -> + {ok, Modules, _Resources} = hb_device_archive:contents(Archive), + hb_store:write( + loaded_device_store(Opts), + maps:from_list( + [ + {schema_key(Module), Schemas} + || + {Module, _Path, Beam} <- Modules, + {ok, Schemas} <- [hb_types:extract(Beam)] + ] + ), Opts ). +schema_key(Module) -> + <<"~meta@1.0/devices/schemas/", (hb_util:bin(Module))/binary>>. + implementation_query(SpecID) -> #{ <<"data-protocol">> => <<"ao">>, @@ -432,6 +483,24 @@ compatible(Msg, Opts) -> _ -> {error, {failed_requirements, Failed}} end. +%% @doc Loading a device memoises its modules' schemas in the shared store, +%% where a process that never loaded the archive finds them. +schema_memoised_test() -> + erlang:erase({?MODULE, <<"test-device@1.0">>}), + Opts = + #{ + <<"loaded-device-store">> => + [hb_test_utils:test_store(hb_store_volatile)] + }, + {ok, Module} = reference(<<"test-device@1.0">>, Opts), + Parent = self(), + Ref = make_ref(), + spawn(fun() -> Parent ! {Ref, schema(Module, Opts)} end), + receive + {Ref, Schemas} -> + ?assertMatch({ok, #{ <<"snapshot">> := #{ 3 := _ } }}, Schemas) + end. + %% @doc Resolution against a preloaded store holding no devices must fail %% promptly with a circular-resolution error, not recurse through `name@1.0'. circular_name_resolution_test() -> diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index e3ad5819d..3081cf0be 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -113,7 +113,9 @@ <<"cache-control">>, <<"spawn-worker">>, <<"only">>, - <<"prefer">> + <<"prefer">>, + <<"resolved-func">>, + <<"varied">> ] ). @@ -123,16 +125,16 @@ %% `{ok | error, NewMessage}.' %% The resolver is composed of a series of discrete phases: %% 1: Normalization. -%% 2: Cache lookup. +%% 2: Vary inputs; lookup cached results. %% 3: Validation check. %% 4: Persistent-resolver lookup. -%% 5: Device lookup. -%% 6: Execution. -%% 7: Execution of the `step' hook. -%% 8: Subresolution. -%% 9: Cryptographic linking. -%% 10: Result caching. -%% 11: Notify waiters. +%% 5: Execution. +%% 6: Execution of the `step' hook. +%% 7: Subresolution. +%% 8: Cryptographic linking. +%% 9: Result caching. +%% 10: Notify waiters. +%% 11: Apply message extension. %% 12: Fork worker. %% 13: Recurse or terminate. resolve(Path, Opts) when is_binary(Path) -> @@ -429,25 +431,34 @@ resolve_stage(2, Base, Req, Opts = #{ <<"resolve-mode">> := raw }) -> % validation, persistence, linking, and worker stages. raw(Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), - % Lookup request in the cache. If we find a result, return it. - % If we do not find a result, we continue to the next stage, - % unless the cache lookup returns `halt' (the user has requested that we - % only return a result if it is already in the cache). - case hb_cache_control:maybe_lookup(Base, Req, Opts) of - {ok, Res} -> - ?event_debug(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), - {ok, Res}; - {continue, NewBase, NewReq} -> - resolve_stage(3, NewBase, NewReq, Opts); - {error, CacheResp} -> {error, CacheResp} - end; -resolve_stage(3, Base, Req, _Opts) when not is_map(Base) or not is_map(Req) -> - % Validation check: If the messages are not maps, we cannot find a key - % in them, so return not_found. - ?event_debug(debug_ao_core, {stage, 3, validation_check_type_error}, _Opts), - {error, not_found}; -resolve_stage(3, Base, Req, Opts) -> + ?event_debug(debug_ao_core, {stage, 2, vary_and_cache_lookup}, Opts), + % Vary the inputs by the schema of the function that will execute them + % before the cache lookup, such that every execution the schema deems + % equivalent shares one hashpath. If the function's schema declares it to be + % a patch, the `VariedResult` of the execution is overlaid on top of the + % appropriate input message before return. + case hb_device:is_direct_key_access(Base, Req, Opts) of + true -> + % TODO: Just look it up/return it? Through cache control perhaps? + todo; + false -> + {Func, VariedBase, VariedReq, MaybeOverlay, VariedOpts} = + vary_loaded(ensure_message_loaded(Base, Opts), Req, Opts), + case hb_cache_control:maybe_lookup(VariedBase, VariedReq, VariedOpts) of + {ok, Res} -> + ?event_debug( + debug_ao_core, + {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, + Opts + ), + {ok, Res}; + {continue, NewBase, NewReq} -> + resolve_stage(3, Func, NewBase, NewReq, MaybeOverlay, Opts); + {error, CacheResp} -> + {error, CacheResp} + end + end. +resolve_stage(3, Func, Base, Req, MaybeOverlay, Opts) -> ?event_debug(debug_ao_core, {stage, 3, validation_check}, Opts), % Validation checks: If `paranoid_message_verification' is enabled, we should % verify the base and request messages prior to execution. @@ -460,8 +471,8 @@ resolve_stage(3, Base, Req, Opts) -> }, Opts ), - resolve_stage(4, Base, Req, Opts); -resolve_stage(4, Base, Req, Opts) -> + resolve_stage(4, Func, Base, Req, MaybeOverlay, Opts); +resolve_stage(4, Func, Base, Req, MaybeOverlay, Opts) -> ?event_debug(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. @@ -477,7 +488,7 @@ resolve_stage(4, Base, Req, Opts) -> true -> ?event(worker_spawns, {will_become, ExecName}); _ -> ok end, - resolve_stage(5, Base, Req, ExecName, Opts); + resolve_stage(5, Func, Base, Req, MaybeOverlay, ExecName, Opts); {wait, Leader} -> % There is another executor of this resolution in-flight. % Bail execution, register to receive the response, then @@ -495,7 +506,7 @@ resolve_stage(4, Base, Req, Opts) -> Opts ), % Re-try again if the group leader has died. - resolve_stage(4, Base, Req, Opts); + resolve_stage(4, Func, Base, Req, MaybeOverlay, Opts); Res -> % Now that we have the result, we can skip right to potential % recursion (step 11) in the outer-wrapper. @@ -518,81 +529,14 @@ resolve_stage(4, Base, Req, Opts) -> case hb_opts:get(allow_infinite, false, Opts) of true -> % We are OK with infinite loops, so we just continue. - resolve_stage(5, Base, Req, GroupName, Opts); + resolve_stage(5, Func, Base, Req, MaybeOverlay, GroupName, Opts); false -> % We are not OK with infinite loops, so we raise an error. error_infinite(Base, Req, Opts) end end. -resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 5, device_lookup}, Opts), - % Device lookup: Find the Erlang function that should be utilized to - % execute Req on Base. - {ResolvedFunc, NewOpts} = - try - UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), - Key = hb_path:hd(Req, UserOpts), - % Try to load the device and get the function to call. - ?event( - { - resolving_key, - {key, Key}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), - ?event( - {found_func_for_exec, - {key, Key}, - {device, Device}, - {func, Func}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. - { - Func, - Opts#{ - <<"add-key">> => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - ?event( - ao_result, - { - load_device_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {exec_class, Class}, - {exec_exception, Exception}, - {exec_stacktrace, Stacktrace}, - {opts, Opts} - }, - Opts - ), - % If the device cannot be loaded, we alert the caller. - error_execution( - ExecName, - Req, - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). -resolve_stage(6, Func, Base, Req, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 6, ExecName, execution}, Opts), +resolve_stage(5, Func, Base, Req, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 5, ExecName, execution}, Opts), % Execution. ExecOpts = execution_opts(Opts), Args = @@ -659,16 +603,17 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> }, Opts ), - resolve_stage(7, Base, Req, Res, ExecName, Opts); + resolve_stage(6, Base, Req, Res, MaybeOverlay, ExecName, Opts); resolve_stage( - 7, + 6, Base, Req, {St, Res}, + MaybeOverlay, ExecName, Opts = #{ <<"on">> := _On = #{ <<"step">> := _ }} ) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, _On}}, Opts), + ?event_debug(debug_ao_core, {stage, 6, ExecName, executing_step_hook, {on, _On}}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every @@ -681,7 +626,7 @@ resolve_stage( }, case hb_hook:on(<<"step">>, HookReq, Opts) of {ok, #{ <<"status">> := NewStatus, <<"body">> := NewRes }} -> - resolve_stage(8, Base, Req, {NewStatus, NewRes}, ExecName, Opts); + resolve_stage(7, Base, Req, {NewStatus, NewRes}, MaybeOverlay, ExecName, Opts); Error -> ?event( ao_core, @@ -693,22 +638,22 @@ resolve_stage( ), Error end; -resolve_stage(7, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), - resolve_stage(8, Base, Req, Res, ExecName, Opts); -resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), +resolve_stage(6, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 6, ExecName, no_step_hook}, Opts), + resolve_stage(7, Base, Req, Res, MaybeOverlay, ExecName, Opts); +resolve_stage(7, Base, Req, {ok, {resolve, Sublist}}, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 7, ExecName, subresolve_result}, Opts), % If the result is a `{resolve, Sublist}' tuple, we need to execute it % as a sub-resolution. - resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); -resolve_stage(8, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), - resolve_stage(9, Base, Req, Res, ExecName, Opts); -resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), + resolve_stage(8, Base, Req, resolve_many(Sublist, Opts), MaybeOverlay, ExecName, Opts); +resolve_stage(7, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 7, ExecName, no_subresolution_necessary}, Opts), + resolve_stage(8, Base, Req, Res, MaybeOverlay, ExecName, Opts); +resolve_stage(8, Base, Req, {ok, Res}, MaybeOverlay, ExecName, Opts) when is_map(Res) -> + ?event_debug(debug_ao_core, {stage, 8, ExecName, generate_hashpath}, Opts), % Cryptographic linking. Now that we have generated the result, we % need to cryptographically link the output to its input via a hashpath. - resolve_stage(10, Base, Req, + resolve_stage(9, Base, Req, case hb_opts:get(hashpath, update, Opts#{ <<"only">> => local }) of update -> NormRes = Res, @@ -730,39 +675,56 @@ resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> {ok, Res} end end, + MaybeOverlay, ExecName, Opts ); -resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), +resolve_stage(8, Base, Req, {Status, Res}, MaybeOverlay, ExecName, Opts) when is_map(Res) -> + ?event_debug(debug_ao_core, {stage, 8, ExecName, abnormal_status_reset_hashpath}, Opts), ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), % Skip cryptographic linking and reset the hashpath if the result is abnormal. Priv = hb_private:from_message(Res), resolve_stage( - 10, Base, Req, + 9, Base, Req, {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, - ExecName, Opts); -resolve_stage(9, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), + MaybeOverlay, ExecName, Opts + ); +resolve_stage(8, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 8, ExecName, non_map_result_skipping_hash_path}, Opts), % Skip cryptographic linking and continue if we don't have a map that can have % a hashpath at all. - resolve_stage(10, Base, Req, Res, ExecName, Opts); -resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), + resolve_stage(9, Base, Req, Res, MaybeOverlay, ExecName, Opts); +resolve_stage(9, Base, Req, {ok, Res}, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 9, ExecName, result_caching}, Opts), % Result caching: Optionally, cache the result of the computation locally. hb_cache_control:maybe_store(Base, Req, Res, Opts), - resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); -resolve_stage(10, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), + resolve_stage(10, Base, Req, {ok, Res}, MaybeOverlay, ExecName, Opts); +resolve_stage(9, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 9, ExecName, abnormal_status_skip_caching}, Opts), % Skip result caching if the result is abnormal. - resolve_stage(11, Base, Req, Res, ExecName, Opts); -resolve_stage(11, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 11, ExecName}, Opts), + resolve_stage(10, Base, Req, Res, MaybeOverlay, ExecName, Opts); +resolve_stage(10, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 10, ExecName, notify_waiting}, Opts), % Notify processes that requested the resolution while we were executing and % unregister ourselves from the group. hb_persistent:unregister_notify(ExecName, Req, Res, Opts), - resolve_stage(12, Base, Req, Res, ExecName, Opts); -resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> + resolve_stage(11, Base, Req, Res, MaybeOverlay, ExecName, Opts); +resolve_stage(11, Base, Req, Res, MaybeOverlay, ExecName, Opts) -> + ?event_debug(debug_ao_core, {stage, 11, ExecName, apply_result_extension}, Opts), + % Set the result as an overlay upon either the base or the request messages + % if requested by the specification of the resolver function. + maybe_fork_worker( + apply_vary_overlay(Base, Req, Res, MaybeOverlay, Opts), + ExecName, + Opts + ). + +%% @doc Check if spawning a worker was requested after a completed computation. +%% If it is, start the worker and forward any work assigned to us downstream. If +%% not, ignore and return the result unmodified. +%% +%% This represents the 12th and final stage of the AO-Core resolution flow. +maybe_fork_worker({ok, Res} = Res, ExecName, Opts) -> ?event_debug(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case @@ -776,7 +738,7 @@ resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> hb_persistent:forward_work(WorkerPID, Opts), Res end; -resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> +maybe_fork_worker(OtherRes, _ExecName, _Opts) -> ?event_debug(debug_ao_core, {stage, 12, _ExecName, abnormal_status_skip_spawning}, _Opts), OtherRes. @@ -921,7 +883,7 @@ ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> LoadedMsg; failure -> failure; - not_found -> + {error, not_found} -> throw({necessary_message_not_found, <<"/">>, MsgID}) end; ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> @@ -929,6 +891,60 @@ ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> ensure_message_loaded(Msg, _Opts) -> Msg. +%% @doc Resolve the device function for a loaded base and vary the inputs +%% by its schema. The function is carried to stage 5 in the `resolved-func' +%% option rather than resolved twice, and a varied execution is marked as +%% such for the cache, which links its result under the varied hashpath rather +%% than the original. +vary_loaded(Base, Req, Opts) -> + UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), + Key = hb_path:hd(Req, UserOpts), + ?event( + { + resolving_key, + {key, Key}, + {base, Base}, + {req, Req}, + {opts, Opts} + } + ), + {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), + ?event( + {found_func_for_exec, + {key, Key}, + {device, Device}, + {func, Func}, + {base, Base}, + {req, Req}, + {opts, Opts} + } + ), + AddKey = + case Status of + add_key -> Key; + _ -> false + end, + ResolvedOpts = Opts#{ <<"resolved-func">> => {AddKey, Func} }, + case hb_types:vary(Key, Func, AddKey, Base, Req, UserOpts) of + {ok, Base, Req, none} -> + {Base, Req, none, ResolvedOpts}; + {ok, VariedBase, VariedReq, Overlay} -> + {VariedBase, VariedReq, Overlay, ResolvedOpts}; + no_spec -> + {Base, Req, none, ResolvedOpts} + end. + +%% @doc `set` a result that the schema declares to be an overlay on top of the +%% original `Base` or `Request` message as indicated, if and only if the result +%% is a message itself. Literal values are returned without extending either +%% input. +apply_vary_overlay(base, Base, _Req, {ok, Res}, Opts) when is_map(Res) -> + {ok, set(Base, Res, internal_opts(Opts))}; +apply_vary_overlay(request, _Base, Req, {ok, Res}, Opts) when is_map(Res) -> + {ok, set(Req, Res, internal_opts(Opts))}; +apply_vary_overlay(_IgnoredVaryState, _Base, _Req, {ok, Res}, _Opts) -> + {ok, Res}. + %% @doc Catch all return if we are in an infinite loop. error_infinite(Base, Req, Opts) -> ?event( diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index ed22e208a..01148ebc9 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -140,7 +140,12 @@ perform_cache_write(Base, Req, Res, Opts) -> Opts ); Map when is_map(Map) -> - hb_cache:write(Res, Opts); + % A varied execution is found again only under its varied + % hashpath, so its result is linked there. + case maps:is_key(<<"varied">>, Opts) of + true -> hb_cache:write_hashpath(Res, Opts); + false -> hb_cache:write(Res, Opts) + end; _ -> ?event({cannot_write_result, Res}), skip_caching diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl index c0fdc680a..23cdd3541 100644 --- a/src/core/resolver/hb_opts.erl +++ b/src/core/resolver/hb_opts.erl @@ -255,10 +255,12 @@ raw_default_message() -> <<"capacity">> => 1024 * 1024 * 1024, <<"read-only">> => true }, - % Store for resolved device reference -> loaded module atom, - % shared across processes so the first caller to resolve a - % device spares the rest the index read and archive - % extraction. Defaults to a `hb_store_volatile`. + % Store for resolved device reference -> loaded module atom and + % module -> function schemas, shared across processes so the first + % caller to resolve a device spares the rest the index read, archive + % extraction and schema extraction. The schemes are extremely hot, and + % so are written as fully loaded messages as a single Erlang term. As a + % consequence, the store must be a `hb_store_volatile`. <<"loaded-device-store">> => [ #{ diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl new file mode 100644 index 000000000..6248556e1 --- /dev/null +++ b/src/core/resolver/hb_types.erl @@ -0,0 +1,944 @@ +%%% @doc Vary the inputs of a device function by its `-spec'. +%%% +%%% A device function's Dialyzer spec describes the base and request messages +%%% it reads and the result it returns. AO-Core uses the spec to vary +%%% the messages before execution: the function receives the keys it declares, +%%% loaded and coerced to their declared types, and the execution's hashpath is +%%% derived from the varied messages alone. Every execution the spec deems +%%% equivalent thereby shares one cache entry, however its messages otherwise +%%% differ. A function without a spec executes upon its inputs as given. +%%% +%%% Specs are read from a module's BEAM by `extract/1' and compiled into a +%%% schema: a map from each spec'd function's normalized name and +%%% arity to the schemas of its arguments and result. `hb_device_load:schema/2' +%%% memoises them. The type syntax means, for a message argument: +%%% +%%% +%%% +%%% `device' is always kept in the base and `path' in the request, so a +%%% projection cannot detach an execution from its device or key. Scalar +%%% types coerce through the `hb_util' converters; lists, tuples, unions, +%%% ranges and literals apply recursively. A value that cannot be coerced to +%%% its type throws `{invalid_type, Schema, Value}'; a required key that is +%%% absent throws `{required_key_missing, Key}'. +%%% +%%% A result spec may declare `#{ '...' := base }' (or `request'): the +%%% result is then a patch that `hb_ao' lays over the unvaried +%%% message, so a function that reads only a projection of its base can still +%%% return the whole of it, updated. +-module(hb_types). +-export([extract/1, vary/6]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% The built-in types that coerce and check as scalars, named as their +%% normalized kinds are. +-define(SCALARS, + #{ + integer => true, non_neg_integer => true, pos_integer => true, + neg_integer => true, float => true, number => true, binary => true, + bitstring => true, atom => true, pid => true + } +). + +%%% -------------------------------------------------------------------- +%%% Varying an execution +%%% -------------------------------------------------------------------- + +%% @doc Vary an execution's base and request by the schema of the function +%% that will execute `Key'. `AddKey' is the key if the function takes it as +%% its first argument (a `handler' or `default'), or `false'. Returns the +%% varied messages and the overlay the result spec declares, or `no_spec'. +vary(Key, Func, AddKey, Base, Req, Opts) -> + case function_schema(Func, Key, Opts) of + {ok, Schema} -> + {BaseSchema, ReqSchema, ReturnSchema} = + execution_schemas(Schema, AddKey), + {ok, + apply_schema(implicit_base(BaseSchema), Base, Opts), + apply_schema( + implicit_request(ReqSchema), + request_with_key(Req, AddKey), + Opts + ), + overlay(ReturnSchema)}; + {error, _} -> + no_spec + end. + +%% @doc The schema of the function that will execute `Key': its spec, found +%% in its own module by its name and arity, or -- for a `handler' or +%% `default' that serves many keys -- by the key. +function_schema(Func, Key, Opts) -> + maybe + true ?= is_function(Func) orelse {error, not_found}, + {module, Module} = erlang:fun_info(Func, module), + {name, Name} = erlang:fun_info(Func, name), + {arity, Arity} = erlang:fun_info(Func, arity), + {ok, Schemas} ?= hb_device_load:schema(Module, Opts), + {error, not_found} ?= named_schema(normalize_name(Name), Arity, Schemas), + named_schema(Key, Arity, Schemas) + end. + +%% @doc The schema of the function of the given normalized name and arity. +named_schema(Name, Arity, Schemas) -> + case Schemas of + #{ Name := #{ Arity := Schema } } -> {ok, Schema}; + _ -> {error, not_found} + end. + +%% @doc The base, request and result schemas of an execution. A function that +%% takes the key as its first argument reads the base and request from its +%% second and third; an argument the spec does not cover is not varied. +execution_schemas(#{ <<"args">> := Args, <<"return">> := Return }, AddKey) -> + Offset = + case AddKey of + false -> 0; + _ -> 1 + end, + { + maybe_nth(1 + Offset, Args, any_type()), + maybe_nth(2 + Offset, Args, any_type()), + Return + }. + +%% @doc The `N'th element of a list, or the default if it is shorter. +maybe_nth(N, List, _Default) when N =< length(List) -> lists:nth(N, List); +maybe_nth(_N, _List, Default) -> Default. + +%% @doc A `handler' or `default' function serves many keys, so the request +%% it is varied by records the key it was chosen for as its path. +request_with_key(Req, false) -> Req; +request_with_key(Req, Key) -> Req#{ <<"path">> => Key }. + +%% @doc The base schema always admits the device, and the request schema +%% always requires the path: a projection keeps an execution attached to its +%% device and key. +implicit_base(Schema) -> + implicit_key(top_level_schema(Schema), <<"device">>, optional). +implicit_request(Schema) -> + implicit_key(top_level_schema(Schema), <<"path">>, required). + +%% @doc A bare `_' argument declares that the function reads nothing from +%% the message: it is projected to the implicit keys alone. +top_level_schema(#{ <<"kind">> := <<"wildcard">> }) -> message_type(#{}, none); +top_level_schema(Schema) -> Schema. + +%% @doc Add a key to a message schema, unless the schema declares it. +implicit_key( + Schema = #{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, + Key, + Presence +) when not is_map_key(Key, Keys) -> + Schema#{ + <<"keys">> => + Keys#{ + Key => #{ <<"presence">> => Presence, <<"type">> => any_type() } + } + }; +implicit_key(Schema, _Key, _Presence) -> + Schema. + +%% @doc The overlay a result spec declares: `base' or `request' when the +%% result message -- directly, or inside a tuple or union such as +%% `{ok, Result}' -- has a `...' key of that literal or alias type. +overlay(#{ <<"kind">> := <<"message">>, <<"keys">> := #{ <<"...">> := Field } }) -> + overlay_marker(maps:get(<<"type">>, Field)); +overlay(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }) -> + first_overlay(Items); +overlay(#{ <<"kind">> := <<"union">>, <<"members">> := Members }) -> + first_overlay(Members); +overlay(_Schema) -> + none. + +%% @doc The overlay of the first of the schemas that declares one. +first_overlay([]) -> + none; +first_overlay([Schema | Rest]) -> + case overlay(Schema) of + none -> first_overlay(Rest); + Overlay -> Overlay + end. + +%% @doc The overlay a `...' key's type names. +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := Marker }) -> + overlay_marker(Marker); +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := Marker }) -> + overlay_marker(Marker); +overlay_marker(<<"base">>) -> base; +overlay_marker(<<"request">>) -> request; +overlay_marker(_Type) -> none. + +%%% -------------------------------------------------------------------- +%%% Extracting schemas from a BEAM +%%% -------------------------------------------------------------------- + +%% @doc Extract the function schemas of a module from its BEAM: the given +%% bytes, or the object code of a module loaded from the code path. A module +%% compiled without `debug_info' has no abstract code, and so no schemas. +extract(Beam) when is_binary(Beam) -> + case beam_lib:chunks(Beam, [abstract_code]) of + {ok, {_Module, [{abstract_code, {_Version, Forms}}]}} -> + TypeEnv = build_type_env(Forms), + {ok, + lists:foldl( + fun(Spec, Acc) -> put_spec(Spec, TypeEnv, Acc) end, + #{}, + [ Attr || Attr = {attribute, _, spec, _} <- Forms ] + )}; + Other -> + {error, {abstract_code_unavailable, Other}} + end; +extract(Module) when is_atom(Module) -> + case code:get_object_code(Module) of + {Module, Beam, _Path} -> extract(Beam); + error -> {error, {object_code_unavailable, Module}} + end. + +%% @doc The module's own type declarations, by name, for expansion when a +%% spec refers to them. +build_type_env(Forms) -> + maps:from_list( + [ + {Name, #{ vars => [ var_name(Var) || Var <- Vars ], ast => Ast }} + || + {attribute, _, Tag, {Name, Ast, Vars}} <- Forms, + Tag =:= type orelse Tag =:= opaque + ] + ). + +%% @doc The name of a type's parameter. +var_name({var, _, Name}) -> Name; +var_name(Name) -> Name. + +%% @doc Add a spec to the module schema under its function's normalized name +%% and arity. A spec with several clauses describes no single shape of input +%% and is not varied. +put_spec({attribute, _, spec, {{Name, Arity}, [Clause]}}, TypeEnv, Schemas) -> + {Args, Return} = parse_fun_spec(Clause, TypeEnv), + NormName = normalize_name(Name), + Arities = maps:get(NormName, Schemas, #{}), + Schemas#{ + NormName => + Arities#{ + Arity => #{ <<"args">> => Args, <<"return">> => Return } + } + }; +put_spec(_Spec, _TypeEnv, Schemas) -> + Schemas. + +%% @doc The argument and result schemas of a spec clause, with any `when' +%% constraints dropped. +parse_fun_spec({type, _, bounded_fun, [FunSpec, _Constraints]}, TypeEnv) -> + parse_fun_spec(FunSpec, TypeEnv); +parse_fun_spec({type, _, 'fun', [{type, _, product, Args}, Return]}, TypeEnv) -> + { + [ parse_type(Arg, TypeEnv, #{}, []) || Arg <- Args ], + parse_type(Return, TypeEnv, #{}, []) + }; +parse_fun_spec(Other, _TypeEnv) -> + {[unknown_type(Other)], any_type()}. + +%% @doc Compile an abstract type into a schema. `TypeEnv' holds the module's +%% own types, `VarEnv' the bindings of the type variables of the one being +%% expanded, and `Seen' the types under expansion, so that a recursive type +%% becomes an alias rather than a loop. +parse_type({ann_type, _, [_Var, Type]}, TypeEnv, VarEnv, Seen) -> + parse_type(Type, TypeEnv, VarEnv, Seen); +parse_type({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + wildcard_type(); +parse_type({var, _, Name}, TypeEnv, VarEnv, Seen) -> + case maps:find(Name, VarEnv) of + {ok, Bound} -> parse_type(Bound, TypeEnv, VarEnv, Seen); + error -> variable_type(Name) + end; +parse_type({user_type, _, Name, Args}, TypeEnv, VarEnv, Seen) -> + case lists:member(Name, Seen) orelse maps:find(Name, TypeEnv) of + {ok, #{ vars := Vars, ast := Ast }} -> + parse_type( + Ast, + TypeEnv, + maps:merge(VarEnv, maps:from_list(lists:zip(Vars, Args))), + [Name | Seen] + ); + _ -> + alias_type(Name) + end; +parse_type( + {remote_type, _, [{atom, _, Mod}, {atom, _, Name}, Args]}, + TypeEnv, + VarEnv, + Seen +) -> + #{ + <<"kind">> => <<"remote">>, + <<"module">> => normalize_name(Mod), + <<"name">> => normalize_name(Name), + <<"args">> => [ parse_type(Arg, TypeEnv, VarEnv, Seen) || Arg <- Args ] + }; +parse_type({type, _, map, any}, _TypeEnv, _VarEnv, _Seen) -> + any_type(); +parse_type({type, _, map, Fields}, TypeEnv, VarEnv, Seen) -> + {Keys, Wildcard} = + lists:foldl( + fun({type, _, Assoc, [KeyAst, ValueAst]}, {KeyAcc, WildAcc}) -> + Field = + #{ + <<"presence">> => field_presence(Assoc), + <<"type">> => parse_type(ValueAst, TypeEnv, VarEnv, Seen) + }, + case key_name(KeyAst, TypeEnv, VarEnv, Seen) of + <<"_">> -> {KeyAcc, Field}; + Key -> {KeyAcc#{ Key => Field }, WildAcc} + end + end, + {#{}, none}, + Fields + ), + message_type(Keys, Wildcard); +parse_type({type, _, ListType, Items}, TypeEnv, VarEnv, Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ + <<"kind">> => <<"list">>, + <<"item">> => + case Items of + [] -> any_type(); + [Item] -> parse_type(Item, TypeEnv, VarEnv, Seen) + end + }; +parse_type({type, _, tuple, Items}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"tuple">>, + <<"items">> => [ parse_type(Item, TypeEnv, VarEnv, Seen) || Item <- Items ] + }; +parse_type({type, _, union, Members}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => + [ parse_type(Member, TypeEnv, VarEnv, Seen) || Member <- Members ] + }; +parse_type({type, _, range, [Min, Max]}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"range">>, + <<"min">> => literal_value(parse_type(Min, TypeEnv, VarEnv, Seen)), + <<"max">> => literal_value(parse_type(Max, TypeEnv, VarEnv, Seen)) + }; +parse_type({type, _, boolean, []}, _, _, _) -> boolean_type(); +parse_type({type, _, any, []}, _, _, _) -> any_type(); +parse_type({type, _, Scalar, _}, _, _, _) when is_map_key(Scalar, ?SCALARS) -> + scalar_type(normalize_name(Scalar)); +parse_type({atom, _, Atom}, _, _, _) -> literal_type(hb_util:bin(Atom)); +parse_type({integer, _, Int}, _, _, _) -> literal_type(Int); +parse_type({char, _, Char}, _, _, _) -> literal_type(<>); +parse_type({string, _, String}, _, _, _) -> literal_type(hb_util:bin(String)); +parse_type({nil, _}, _, _, _) -> literal_type([]); +parse_type(Other, _TypeEnv, _VarEnv, _Seen) -> unknown_type(Other). + +%% @doc `:=' declares a required key, `=>' an optional one. +field_presence(map_field_exact) -> required; +field_presence(map_field_assoc) -> optional. + +%% @doc The message key a map field's key type names. A literal key is the +%% key itself; `_' is the wildcard; any other type is named by its printed +%% form, so that it can never match a real key. +key_name({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> + normalize_name(Atom); +key_name({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> + hb_util:bin(String); +key_name({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + <<"_">>; +key_name(Other, TypeEnv, VarEnv, Seen) -> + case parse_type(Other, TypeEnv, VarEnv, Seen) of + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } + when is_binary(Value) -> + Value; + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } -> + hb_util:bin(io_lib:format("~tp", [Value])); + _ -> + hb_util:bin(io_lib:format("~tp", [Other])) + end. + +%%% -------------------------------------------------------------------- +%%% Applying a schema to a value +%%% -------------------------------------------------------------------- + +%% @doc Vary a value by its schema: pass it through if the schema does not +%% constrain it, else load it if it is a link, then project, coerce or +%% check it as the schema's kind requires. +apply_schema(#{ <<"kind">> := <<"any">> }, Value, _Opts) -> + Value; +apply_schema(#{ <<"kind">> := <<"wildcard">> }, Value, _Opts) -> + Value; +apply_schema(#{ <<"kind">> := Kind }, Value, _Opts) + when Kind =:= <<"remote">>; Kind =:= <<"alias">>; + Kind =:= <<"variable">>; Kind =:= <<"unknown">> -> + Value; +apply_schema(Schema, Link, Opts) when ?IS_LINK(Link) -> + apply_schema(Schema, hb_cache:ensure_loaded(Link, Opts), Opts); +apply_schema(Schema = #{ <<"kind">> := <<"message">> }, Value, Opts) + when not is_map(Value) -> + case coerce_type(Schema, Value, Opts) of + error -> throw({invalid_type, Schema, Value}); + Value -> throw({invalid_type, Schema, Value}); + Coerced -> apply_schema(Schema, Coerced, Opts) + end; +apply_schema( + #{ <<"kind">> := <<"message">>, <<"keys">> := Keys, <<"wildcard">> := Wildcard }, + Message, + Opts +) -> + % The declared keys are varied onto the undeclared ones the wildcard + % admits. A key kept as given is put back unchanged, so a message the + % schema does not alter stays the same term. + maps:fold( + fun(Key, Field, Acc) -> apply_key(Key, Field, Message, Acc, Opts) end, + apply_wildcard(Wildcard, Keys, Message, Opts), + Keys + ); +apply_schema( + Schema = #{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, + Value, + Opts +) -> + case try_coerce(fun hb_util:list/1, Value) of + List when is_list(List) -> + [ apply_schema(ItemType, Item, Opts) || Item <- List ]; + _ -> + throw({invalid_type, Schema, Value}) + end; +apply_schema( + Schema = #{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, + Value, + Opts +) -> + Values = + case Value of + Tuple when is_tuple(Tuple) -> tuple_to_list(Tuple); + List when is_list(List) -> List; + _ -> error + end, + case is_list(Values) andalso length(Values) =:= length(Items) of + true -> + list_to_tuple( + [ + apply_schema(Type, Item, Opts) + || + {Type, Item} <- lists:zip(Items, Values) + ] + ); + false -> + throw({invalid_type, Schema, Value}) + end; +apply_schema( + Schema = #{ <<"kind">> := <<"union">>, <<"members">> := Members }, + Value, + Opts +) -> + case apply_union(Members, Value, Opts) of + {ok, Varied} -> Varied; + error -> throw({invalid_type, Schema, Value}) + end; +apply_schema(Type, Value, Opts) -> + % A scalar, literal or range: keep a value of the type, else coerce it. + case check_type(Type, Value) of + true -> + Value; + false -> + Coerced = coerce_type(Type, Value, Opts), + case Coerced =/= error andalso check_type(Type, Coerced) of + true -> Coerced; + false -> throw({invalid_type, Type, Value}) + end + end. + +%% @doc The undeclared keys of a message, as its schema's wildcard admits +%% them: none for a projection, all of them as given for `_ => _', or each +%% coerced to the wildcard's type for `_ := type()'. +apply_wildcard(none, _Keys, _Message, _Opts) -> + #{}; +apply_wildcard(#{ <<"presence">> := optional }, _Keys, Message, _Opts) -> + Message; +apply_wildcard(#{ <<"type">> := Type }, Keys, Message, Opts) -> + maps:map( + fun(_Key, Value) -> apply_schema(Type, Value, Opts) end, + maps:without(maps:keys(Keys), Message) + ). + +%% @doc Vary one declared key of a message onto the accumulated result. +apply_key(Key, Field, Message, Acc, Opts) -> + #{ <<"presence">> := Presence, <<"type">> := Type } = Field, + case maps:find(Key, Message) of + {ok, Value} -> Acc#{ Key => apply_schema(Type, Value, Opts) }; + error when Presence =:= required -> throw({required_key_missing, Key}); + error -> Acc + end. + +%% @doc Vary a value by the first member of a union that admits it as it is, +%% else by the first that it can be coerced to. +apply_union(Members, Value, Opts) -> + case matching_union_member(Members, Value) of + {ok, Member} -> + try {ok, apply_schema(Member, Value, Opts)} + catch + throw:{invalid_type, _, _} -> + apply_coerced_union(Members, Value, Opts); + throw:{required_key_missing, _} -> + apply_coerced_union(Members, Value, Opts) + end; + error -> + apply_coerced_union(Members, Value, Opts) + end. + +%% @doc The first constraining member of a union that a value already +%% satisfies. Members that pass every value through never match, so that +%% they cannot shadow a constraining member. +matching_union_member([], _Value) -> + error; +matching_union_member([Member | Rest], Value) -> + case not is_passthrough_schema(Member) andalso check_type(Member, Value) of + true -> {ok, Member}; + false -> matching_union_member(Rest, Value) + end. + +%% @doc Vary a value by the first member of a union it can be coerced to, +%% trying the constraining members before those that pass it through. +apply_coerced_union(Members, Value, Opts) -> + {PassThrough, Constrained} = + lists:partition(fun is_passthrough_schema/1, Members), + apply_coerced_union_ordered(Constrained ++ PassThrough, Value, Opts). + +%% @doc Vary a value by the first of the members it can be coerced to. +apply_coerced_union_ordered([], _Value, _Opts) -> + error; +apply_coerced_union_ordered([Member | Rest], Value, Opts) -> + try {ok, apply_schema(Member, Value, Opts)} + catch + throw:{invalid_type, _, _} -> + apply_coerced_union_ordered(Rest, Value, Opts); + throw:{required_key_missing, _} -> + apply_coerced_union_ordered(Rest, Value, Opts) + end. + +%% @doc Whether a schema passes every value through unvaried. +is_passthrough_schema(#{ <<"kind">> := Kind }) -> + lists:member( + Kind, + [<<"any">>, <<"wildcard">>, <<"remote">>, <<"alias">>, <<"variable">>, + <<"unknown">>] + ); +is_passthrough_schema(_Schema) -> + false. + +%%% -------------------------------------------------------------------- +%%% Coercing and checking values +%%% -------------------------------------------------------------------- + +%% @doc Coerce a value to a schema's type through the `hb_util' converters, +%% or `error' if it cannot be. Compound types coerce their elements in turn. +coerce_type(_Type, undefined, _Opts) -> error; +coerce_type(#{ <<"kind">> := <<"integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"pos-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"range">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"float">> }, Value, _Opts) -> + try_coerce(fun hb_util:float/1, Value); +coerce_type(#{ <<"kind">> := <<"number">> }, Value, _Opts) -> + coerce_with([fun hb_util:int/1, fun hb_util:float/1], Value); +coerce_type(#{ <<"kind">> := <<"binary">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"bitstring">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"atom">> }, Value, _Opts) -> + try_coerce(fun hb_util:atom/1, Value); +coerce_type(#{ <<"kind">> := <<"message">> }, Value, _Opts) -> + try_coerce(fun hb_util:map/1, Value); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_tuple(Value) -> + coerce_type( + #{ <<"kind">> => <<"tuple">>, <<"items">> => Items }, + tuple_to_list(Value), + Opts + ); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_list(Value), length(Value) =:= length(Items) -> + case coerce_sequence(lists:zip(Items, Value), Opts) of + error -> error; + Coerced -> list_to_tuple(Coerced) + end; +coerce_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value, Opts) -> + case try_coerce(fun hb_util:list/1, Value) of + List when is_list(List) -> + coerce_sequence([ {ItemType, Item} || Item <- List ], Opts); + _ -> + error + end; +coerce_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value, Opts) -> + coerce_with( + [ fun(V) -> coerce_type(Member, V, Opts) end || Member <- Members ], + Value + ); +coerce_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Lit }, Value, _Opts) -> + coerce_literal(Lit, Value); +coerce_type(_Type, _Value, _Opts) -> + error. + +%% @doc Apply a converter to a value, or `error' if it rejects it. +try_coerce(Fun, Value) -> + try Fun(Value) + catch _:_ -> error + end. + +%% @doc The result of the first converter that accepts a value. +coerce_with([], _Value) -> + error; +coerce_with([Fun | Rest], Value) -> + case try_coerce(Fun, Value) of + error -> coerce_with(Rest, Value); + Coerced -> Coerced + end. + +%% @doc Coerce each value of a sequence to its type, or `error' if any cannot +%% be. +coerce_sequence([], _Opts) -> + []; +coerce_sequence([{Type, Value} | Rest], Opts) -> + case coerce_type(Type, Value, Opts) of + error -> + error; + Coerced -> + case coerce_sequence(Rest, Opts) of + error -> error; + CoercedRest -> [Coerced | CoercedRest] + end + end. + +%% @doc Coerce a value to a literal: the value must convert to the literal's +%% own type and then equal it. +coerce_literal(Expected, Value) when is_integer(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:int/1, Value)); +coerce_literal(Expected, Value) when is_float(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:float/1, Value)); +coerce_literal(Expected, Value) when is_binary(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:bin/1, Value)); +coerce_literal(Expected, Value) when is_boolean(Expected) -> + case is_boolean_coercible(Value) of + true -> coerce_exact(Expected, try_coerce(fun hb_util:bool/1, Value)); + false -> error + end; +coerce_literal(Expected, Value) when is_atom(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:atom/1, Value)); +coerce_literal(Expected, Value) when is_list(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:list/1, Value)); +coerce_literal(Expected, Expected) -> + Expected; +coerce_literal(_Expected, _Value) -> + error. + +%% @doc A coerced value, if it is the literal expected. +coerce_exact(Expected, Expected) -> Expected; +coerce_exact(_Expected, _Value) -> error. + +%% @doc The values `hb_util:bool/1' reads as a boolean. +is_boolean_coercible(Value) -> + lists:member( + Value, + [true, false, 1, 0, <<"true">>, <<"false">>, <<"1">>, <<"0">>] + ). + +%% @doc Whether a value is of a schema's type as it is. Types the varier does +%% not understand admit every value. +check_type(#{ <<"kind">> := <<"integer">> }, Value) -> is_integer(Value); +check_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value) -> + is_integer(Value) andalso Value >= 0; +check_type(#{ <<"kind">> := <<"pos-integer">> }, Value) -> + is_integer(Value) andalso Value > 0; +check_type(#{ <<"kind">> := <<"neg-integer">> }, Value) -> + is_integer(Value) andalso Value < 0; +check_type(#{ <<"kind">> := <<"float">> }, Value) -> is_float(Value); +check_type(#{ <<"kind">> := <<"number">> }, Value) -> is_number(Value); +check_type(#{ <<"kind">> := <<"binary">> }, Value) -> is_binary(Value); +check_type(#{ <<"kind">> := <<"bitstring">> }, Value) -> is_bitstring(Value); +check_type(#{ <<"kind">> := <<"atom">> }, Value) -> is_atom(Value); +check_type(#{ <<"kind">> := <<"pid">> }, Value) -> is_pid(Value); +check_type(#{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, Value) + when is_map(Value) -> + lists:all( + fun + ({Key, #{ <<"presence">> := required }}) -> is_map_key(Key, Value); + (_Field) -> true + end, + maps:to_list(Keys) + ); +check_type(#{ <<"kind">> := <<"message">> }, _Value) -> false; +check_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) -> + is_tuple(Value) + andalso tuple_size(Value) =:= length(Items) + andalso lists:all( + fun({Type, Item}) -> check_type(Type, Item) end, + lists:zip(Items, tuple_to_list(Value)) + ); +check_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value) -> + is_list(Value) + andalso lists:all(fun(Item) -> check_type(ItemType, Item) end, Value); +check_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value) -> + lists:any(fun(Member) -> check_type(Member, Value) end, Members); +check_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value) -> + Value =:= Expected; +check_type(#{ <<"kind">> := <<"range">>, <<"min">> := Min, <<"max">> := Max }, V) -> + is_integer(V) andalso V >= Min andalso V =< Max; +check_type(_Type, _Value) -> true. + +%%% -------------------------------------------------------------------- +%%% Schema constructors +%%% -------------------------------------------------------------------- + +%% @doc The normalized form of a function or key name: the dashed binary +%% that AO-Core keys are matched by. +normalize_name('_') -> <<"_">>; +normalize_name(Name) when is_atom(Name) -> hb_util:atom_to_dashed_binary(Name); +normalize_name(Name) -> hb_util:bin(Name). + +%% @doc The value of a literal schema, such as a range's bound. +literal_value(#{ <<"kind">> := <<"literal">>, <<"value">> := Value }) -> Value; +literal_value(_Type) -> undefined. + +message_type(Keys, Wildcard) -> + #{ <<"kind">> => <<"message">>, <<"keys">> => Keys, <<"wildcard">> => Wildcard }. +any_type() -> #{ <<"kind">> => <<"any">> }. +wildcard_type() -> #{ <<"kind">> => <<"wildcard">> }. +scalar_type(Name) -> #{ <<"kind">> => Name }. +literal_type(Value) -> #{ <<"kind">> => <<"literal">>, <<"value">> => Value }. +alias_type(Name) -> + #{ <<"kind">> => <<"alias">>, <<"name">> => normalize_name(Name) }. +variable_type(Name) -> + #{ <<"kind">> => <<"variable">>, <<"name">> => normalize_name(Name) }. +unknown_type(Ast) -> + #{ + <<"kind">> => <<"unknown">>, + <<"ast">> => hb_util:bin(io_lib:format("~tp", [Ast])) + }. +boolean_type() -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => [literal_type(true), literal_type(false)] + }. + +%%% Tests + +%% @doc A message schema requiring one key of a type, and nothing else. +required(Key, Type) -> + message_type( + #{ Key => #{ <<"presence">> => required, <<"type">> => Type } }, + none + ). + +parse_empty_projection_test() -> + ?assertEqual(wildcard_type(), parse_type({var, 1, '_'}, #{}, #{}, [])). + +map_wildcards_test() -> + Lazy = + parse_type( + {type, 1, map, + [ + {type, 1, map_field_exact, [{atom, 1, a}, {var, 1, '_'}]}, + {type, 1, map_field_assoc, [{var, 1, '_'}, {var, 1, '_'}]} + ]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"keys">> := #{ <<"a">> := _ }, + <<"wildcard">> := #{ <<"presence">> := optional } + }, + Lazy + ), + Force = + parse_type( + {type, 1, map, + [{type, 1, map_field_exact, [{var, 1, '_'}, {var, 1, '_'}]}]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"wildcard">> := #{ <<"presence">> := required } + }, + Force + ). + +apply_empty_projection_test() -> + ?assertEqual( + #{ <<"device">> => <<"test@1.0">> }, + apply_schema( + implicit_base(wildcard_type()), + #{ <<"device">> => <<"test@1.0">>, <<"extra">> => <<"drop">> }, + #{} + ) + ). + +%% @doc A message that a schema does not alter is returned as the same term. +unaltered_message_is_identical_test() -> + Message = #{ <<"device">> => <<"test@1.0">>, <<"a">> => 1, <<"b">> => <<"x">> }, + Schema = + message_type( + #{ + <<"a">> => + #{ + <<"presence">> => required, + <<"type">> => scalar_type(<<"integer">>) + } + }, + #{ <<"presence">> => optional, <<"type">> => wildcard_type() } + ), + Varied = apply_schema(implicit_base(Schema), Message, #{}), + ?assert(erts_debug:same(Message, Varied)). + +selected_links_are_materialized_without_loading_omitted_keys_test() -> + Store = hb_test_utils:test_store(), + Opts = #{ <<"store">> => Store }, + hb_store:reset(Store), + {ok, SlotPath} = hb_cache:write(<<"7">>, Opts), + Missing = {link, <<"data/not-present">>, #{}}, + Schema = required(<<"deep">>, required(<<"slot">>, scalar_type(<<"integer">>))), + ?assertEqual( + #{ <<"deep">> => #{ <<"slot">> => 7 } }, + apply_schema( + Schema, + #{ + <<"deep">> => + #{ + <<"slot">> => {link, SlotPath, #{}}, + <<"omitted">> => Missing + }, + <<"omitted">> => Missing + }, + Opts + ) + ). + +explicit_wildcard_preserves_lazy_links_test() -> + Missing = {link, <<"data/not-present">>, #{}}, + Schema = + message_type( + #{ + <<"scheduler">> => + #{ <<"presence">> => optional, <<"type">> => wildcard_type() } + }, + none + ), + ?assertEqual( + #{ <<"scheduler">> => Missing }, + apply_schema(Schema, #{ <<"scheduler">> => Missing }, #{}) + ). + +optional_wildcard_preserves_links_and_sequences_materialize_test() -> + Store = hb_test_utils:test_store(), + Opts = #{ <<"store">> => Store }, + hb_store:reset(Store), + {ok, ValuePath} = hb_cache:write(<<"8">>, Opts), + Link = {link, ValuePath, #{}}, + WildcardSchema = + message_type( + #{}, + #{ <<"presence">> => optional, <<"type">> => wildcard_type() } + ), + ?assertEqual( + #{ <<"extra">> => Link }, + apply_schema(WildcardSchema, #{ <<"extra">> => Link }, Opts) + ), + Integer = scalar_type(<<"integer">>), + ?assertEqual( + [8], + apply_schema( + #{ <<"kind">> => <<"list">>, <<"item">> => Integer }, + [Link], + Opts + ) + ), + ?assertEqual( + {8}, + apply_schema( + #{ <<"kind">> => <<"tuple">>, <<"items">> => [Integer] }, + {Link}, + Opts + ) + ). + +union_preserves_an_existing_member_type_test() -> + Binary = scalar_type(<<"binary">>), + List = #{ <<"kind">> => <<"list">>, <<"item">> => Binary }, + Value = [<<"one">>, <<"two">>], + ?assertEqual( + Value, + apply_schema( + #{ <<"kind">> => <<"union">>, <<"members">> => [Binary, List] }, + Value, + #{} + ) + ). + +union_passthrough_members_do_not_swallow_constrained_members_test() -> + Store = hb_test_utils:test_store(), + Opts = #{ <<"store">> => Store }, + hb_store:reset(Store), + {ok, SlotPath} = hb_cache:write(<<"9">>, Opts), + Integer = scalar_type(<<"integer">>), + Binary = scalar_type(<<"binary">>), + Union = + fun(Members) -> + #{ <<"kind">> => <<"union">>, <<"members">> => Members } + end, + ?assertEqual( + #{ <<"slot">> => 9 }, + apply_schema( + Union([unknown_type({record, tx}), required(<<"slot">>, Integer)]), + #{ <<"slot">> => {link, SlotPath, #{}} }, + Opts + ) + ), + ?assertEqual( + <<"value">>, + apply_schema(Union([any_type(), wildcard_type(), Binary]), <<"value">>, #{}) + ), + ?assertEqual( + #{ <<"second">> => 2 }, + apply_schema( + Union([required(<<"first">>, Integer), required(<<"second">>, Integer)]), + #{ <<"second">> => 2 }, + #{} + ) + ), + ?assertEqual( + #{ <<"value">> => <<"text">> }, + apply_schema( + Union([required(<<"value">>, Integer), required(<<"value">>, Binary)]), + #{ <<"value">> => <<"text">> }, + #{} + ) + ). diff --git a/src/core/test/hb_ao_test_vectors.erl b/src/core/test/hb_ao_test_vectors.erl index 3307780af..7061538ae 100644 --- a/src/core/test/hb_ao_test_vectors.erl +++ b/src/core/test/hb_ao_test_vectors.erl @@ -372,6 +372,21 @@ singleton_id_base_test() -> hb_ao:resolve(<<"/", MissingID/binary, "/keys">>, Opts) ). +direct_id_key_resolution_test() -> + Store = hb_test_utils:test_store(), + Opts = + #{ + <<"store">> => Store, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"spawn-worker">> => false + }, + hb_store:reset(Store), + {ok, ID} = hb_cache:write(#{ <<"value">> => <<"kept">> }, Opts), + ?assertEqual( + {ok, <<"kept">>}, + hb_ao:resolve(ID, <<"value">>, Opts) + ). + resolve_id_test(Opts) -> ?assertMatch( ID when byte_size(ID) == 43, diff --git a/src/preloaded/arweave/dev_arweave.erl b/src/preloaded/arweave/dev_arweave.erl index f5b73d97b..38917a924 100644 --- a/src/preloaded/arweave/dev_arweave.erl +++ b/src/preloaded/arweave/dev_arweave.erl @@ -26,12 +26,15 @@ info() -> }. %% @doc Proxy the `/info' endpoint from the Arweave node. +-spec status(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. status(_Base, _Request, Opts) -> request(<<"GET">>, <<"/info">>, Opts). %% @doc Returns the given transaction as an AO-Core message. By default, this %% embeds the `/raw` payload. Set `exclude-data` to true to return just the %% header. +-spec tx(#{ _ => _ }, #{ method => binary(), tx => binary(), target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. tx(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"POST">> -> post_tx(Base, Request, Opts); @@ -45,6 +48,8 @@ tx(Base, Request, Opts) -> %% Note: When uploading ans104 transactions, this function will use the %% node's default bundler. If instead you want to use this node as a bundler %% you should use the ~bundler@1.0 device. +-spec post_tx(#{ _ => _ }, #{ target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. post_tx(Base, RawRequest, Opts) -> {ok, Request} = extract_target(Base, RawRequest, Opts), case hb_maps:find(<<"commitment-device">>, Request, Opts) of @@ -157,6 +162,8 @@ get_tx(Base, Request, Opts) -> %% @doc A router for range requests by method. Both `HEAD` and `GET` requests %% are supported. +-spec raw(#{ raw => binary(), _ => _ }, #{ method => binary(), raw => binary(), range => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _}. raw(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"HEAD">> -> head_raw(Base, Request, Opts); @@ -403,6 +410,17 @@ list_find(Key, [{XKey, Value} | Rest], Default) -> %% offset and length. %% - `GET` with `txid`: `GET`s a chunk or range of bytes from the given offset, %% relative to the given transaction's data root. +-spec chunk( + #{ _ => _ }, + #{ + method => binary(), + offset => binary() | integer(), + length => binary() | integer(), + pending => binary(), + _ => _ + }, + #{ _ => _ } +) -> {ok, binary() | #{ _ => _ }} | {error, _}. chunk(Base, Request, Opts) -> case hb_maps:get(<<"method">>, Request, <<"GET">>, Opts) of <<"POST">> -> post_chunk(Base, Request, Opts); @@ -719,6 +737,11 @@ get_chunk(Offset, Opts) -> %% block hash length (43 characters), it is used as an ID. If it is parsable as %% an integer, it is used as a block height. If it is not present, the current %% block is used. +-spec block( + #{ block => binary(), _ => _ } | {id, binary()} | {height, integer()}, + #{ block => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. block(Base, Request, Opts) when is_map(Base) -> Block = hb_ao:get_first( @@ -796,9 +819,12 @@ only_if_cached(Req, Opts) -> ). %% @doc Retrieve the current block information from Arweave. +-spec current(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. current(_Base, _Request, Opts) -> request(<<"GET">>, <<"/block/current">>, Opts). +-spec price(#{ size => integer(), _ => _ }, #{ size => integer(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _}. price(Base, Request, Opts) -> Size = hb_ao:get_first( @@ -816,11 +842,14 @@ price(Base, Request, Opts) -> request(<<"GET">>, <<"/price/", (hb_util:bin(Size))/binary>>, Opts) end. +-spec tx_anchor(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }} | {error, _}. tx_anchor(_Base, _Request, Opts) -> request(<<"GET">>, <<"/tx_anchor">>, Opts). %% @doc Retrieve either a list of the pending TXIDs on the configured Arweave %% nodes, or a specific unconfirmed transaction header by its TXID. +-spec pending(#{ pending => binary(), _ => _ }, #{ pending => binary(), offset => integer(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | [binary()] | #{ _ => _ }} | {error, _}. pending(Base, Request, Opts) -> case find_key(<<"pending">>, Base, Request, Opts) of not_found -> request(<<"GET">>, <<"/tx/pending">>, Opts); diff --git a/src/preloaded/arweave/dev_arweave_offset.erl b/src/preloaded/arweave/dev_arweave_offset.erl index 9b0143418..dc8fda2c2 100644 --- a/src/preloaded/arweave/dev_arweave_offset.erl +++ b/src/preloaded/arweave/dev_arweave_offset.erl @@ -8,6 +8,7 @@ %% @doc Resolve either a message at an Arweave offset, or a direct key from the %% base message if the key is not an integer. +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). get(Key, Base, _Request, Opts) -> case parse(Key) of {ok, StartOffset, Length} -> diff --git a/src/preloaded/arweave/dev_bundler.erl b/src/preloaded/arweave/dev_bundler.erl index b7dc1d219..d6f9795b5 100644 --- a/src/preloaded/arweave/dev_bundler.erl +++ b/src/preloaded/arweave/dev_bundler.erl @@ -31,11 +31,15 @@ %%% Public interface. %% @doc An alias for `item/3'. +-spec tx(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ id := binary(), timestamp := integer(), _ => _ }} | {error, #{ _ => _ }}. tx(Base, Req, Opts) -> item(Base, Req, Opts). %% @doc Implements an `up.arweave.net'-compatible endpoint for %% bundling messages. +-spec item(#{ _ => _ }, #{ 'bundler-subject' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ id := binary(), timestamp := integer(), _ => _ }} | {error, #{ _ => _ }}. item(_Base, Req, Opts) -> ServerPID = ensure_server(Opts), ItemToProcess = diff --git a/src/preloaded/arweave/dev_manifest.erl b/src/preloaded/arweave/dev_manifest.erl index 8e339a604..942082c29 100644 --- a/src/preloaded/arweave/dev_manifest.erl +++ b/src/preloaded/arweave/dev_manifest.erl @@ -14,6 +14,10 @@ info() -> }. %% @doc Return the fallback index page when the manifest itself is requested. +-spec index(#{ index => #{ path => binary(), _ => _ }, paths => #{ _ => _ }, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, _} | {error, not_found}. index(M1, M2, Opts) -> ?event(debug_manifest, {index_request, {base, M1}, {request, M2}}, Opts), case route(<<"index">>, M1, M2, Opts) of @@ -25,6 +29,8 @@ index(M1, M2, Opts) -> end. %% @doc Route a request to the associated data via its manifest. +-spec route(binary(), #{ paths => #{ _ => _ }, index => #{ path => binary(), _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, not_found}. route(<<"index">>, M1, M2, Opts) -> ?event({manifest_index, M1, M2}), case manifest(M1, <<>>, Opts) of @@ -86,6 +92,8 @@ route(Key, M1, M2, Opts) -> %% @doc Implement the `on/request' hook for the `manifest@1.0' device, finding %% requests for legacy (non-device-tagged) manifests and casting them to %% `manifest@1.0' before execution. Allowing `/ID/path` style access for old data. +-spec request(#{ _ => _ }, #{ body := [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ body := [_], _ => _ }} | {error, #{ status := integer(), body := binary() }}. request(Base, Req, Opts) -> ?event({on_req_manifest_detector, {base, Base}, {req, Req}}), maybe diff --git a/src/preloaded/auth/dev_auth_hook.erl b/src/preloaded/auth/dev_auth_hook.erl index 08b1b62b9..c5e236bcd 100644 --- a/src/preloaded/auth/dev_auth_hook.erl +++ b/src/preloaded/auth/dev_auth_hook.erl @@ -103,6 +103,11 @@ %% by the user request). %% %% +-spec request( + #{ 'secret-provider' => _, 'generate-path' => binary(), 'finalize-path' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | {skip, _, _} | error. request(Base, HookReq, Opts) -> ?event({auth_hook_request, {base, Base}, {priv_hook_req, HookReq}}), maybe diff --git a/src/preloaded/auth/dev_cookie.erl b/src/preloaded/auth/dev_cookie.erl index 4e2f196fa..f522960bc 100644 --- a/src/preloaded/auth/dev_cookie.erl +++ b/src/preloaded/auth/dev_cookie.erl @@ -50,17 +50,22 @@ opts(Opts) -> hb_private:opts(Opts). %%% ~message@1.0 Commitments API keys. +-spec commit(#{ _ => _ }, #{ secret => binary(), _ => _ }, map()) -> term(). commit(Base, Req, RawOpts) -> dev_cookie_auth:commit(Base, Req, RawOpts). +-spec verify(#{ _ => _ }, #{ secret => binary(), _ => _ }, map()) -> term(). verify(Base, Req, RawOpts) -> dev_cookie_auth:verify(Base, Req, RawOpts). %% @doc Preprocessor keys that utilize cookies and the `~secret@1.0' device to %% sign inbound HTTP requests from users if they are not already signed. We use %% the hook authentication framework to implement this. +-spec generate(#{ _ => _ }, #{ committer => binary(), generator => _, _ => _ }, map()) -> term(). generate(Base, Req, Opts) -> dev_cookie_auth:generate(Base, Req, Opts). %% @doc Finalize an `on-request' hook by adding the `set-cookie' header to the %% end of the message sequence. +-spec finalize(#{ _ => _ }, #{ request := #{ _ => _ }, body := _, _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, no_request}. finalize(Base, Request, Opts) -> dev_cookie_auth:finalize(Base, Request, Opts). @@ -76,6 +81,8 @@ finalize(Base, Request, Opts) -> %% %% The `format' may be specified in the request message as the `req:format' key. %% If no `format' is specified, the default is `default'. +-spec get_cookie(#{ _ => _ }, #{ key := binary(), format => binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, not_found}. get_cookie(Base, Req, RawOpts) -> Opts = opts(RawOpts), {ok, Cookies} = extract(Base, Req, Opts), @@ -92,6 +99,7 @@ get_cookie(Base, Req, RawOpts) -> end. %% @doc Return the parsed and normalized cookies from a message. +-spec extract(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. extract(Msg, Req, Opts) -> {ok, MsgWithCookie} = from(Msg, Req, Opts), Cookies = hb_private:get(<<"cookie">>, MsgWithCookie, #{}, Opts), @@ -100,6 +108,7 @@ extract(Msg, Req, Opts) -> %% @doc Set the keys in the request message in the cookies of the caller. Removes %% a set of base keys from the request message before setting the remainder as %% cookies. +-spec store(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. store(Base, Req, RawOpts) -> Opts = opts(RawOpts), ?event({store, {priv_base, Base}, {priv_req, Req}}), @@ -158,6 +167,11 @@ reset(Base, _Req, Opts) -> %% %% Note that the `format: cookie' form is information lossy: All provided %% attributes and flags are discarded. +-spec to( + #{ cookie => binary() | [binary()], 'set-cookie' => binary() | [binary()], _ => _ }, + #{ format => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ cookie => binary(), 'set-cookie' => [binary()], _ => _ }}. to(Msg, Req, Opts) -> ?event({to, {priv_msg, Msg}, {priv_req, Req}}), CookieOpts = opts(Opts), @@ -258,6 +272,11 @@ to_cookie_line(Key, Cookie) -> %% @doc Normalize a message containing a `cookie', `set-cookie', and potentially %% a `priv/cookie' key into a message with only the `priv/cookie' key. +-spec from( + #{ cookie => binary() | [binary()], 'set-cookie' => binary() | [binary()], _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. from(Msg, Req, Opts) -> CookieOpts = opts(Opts), LoadedMsg = ensure_cookie_loaded(Msg, CookieOpts), diff --git a/src/preloaded/auth/dev_cookie_auth.erl b/src/preloaded/auth/dev_cookie_auth.erl index 7be436200..aee7167db 100644 --- a/src/preloaded/auth/dev_cookie_auth.erl +++ b/src/preloaded/auth/dev_cookie_auth.erl @@ -11,6 +11,7 @@ %% key for the `httpsig@1.0' commitment. If a `committer' is given, we search %% for it in the cookie message instead of generating a new secret. See the %% module documentation of `dev_cookie' for more details on its scheme. +-spec generate(#{ _ => _ }, #{ committer => binary(), generator => _, _ => _ }, map()) -> term(). generate(Base, Request, Opts) -> {WithCookie, Secrets} = case find_secrets(Request, Opts) of @@ -33,6 +34,7 @@ generate(Base, Request, Opts) -> %% messages. The inbound request has the same structure as a normal request %% hook: The message sequence is the body of the request, and the request is %% the request message. +-spec finalize(#{ _ => _ }, #{ request := #{ _ => _ }, body := list(), _ => _ }, map()) -> term(). finalize(Base, Request, Opts) -> ?event(debug_auth, {finalize, {base, Base}, {request, Request}}), maybe @@ -58,6 +60,7 @@ finalize(Base, Request, Opts) -> %% key for the `httpsig@1.0' commitment. If a `committer' is given, we search %% for it in the cookie message instead of generating a new secret. See the %% module documentation of `dev_cookie' for more details on its scheme. +-spec commit(#{ _ => _ }, #{ secret => binary(), committer => binary(), generator => _, _ => _ }, map()) -> term(). commit(Base, Request, RawOpts) when ?IS_LINK(Request) -> Opts = dev_cookie:opts(RawOpts), commit(Base, hb_cache:ensure_loaded(Request, Opts), Opts); @@ -111,6 +114,7 @@ store_secret(Secret, Msg, Opts) -> %% @doc Verify the HMAC commitment with the key being the secret from the %% request cookies. We find the appropriate cookie from the cookie message by %% the committer ID given in the request message. +-spec verify(#{ _ => _ }, #{ secret => binary(), committer => binary(), _ => _ }, map()) -> term(). verify(Base, ReqLink, RawOpts) when ?IS_LINK(ReqLink) -> Opts = dev_cookie:opts(RawOpts), verify(Base, hb_cache:ensure_loaded(ReqLink, Opts), Opts); diff --git a/src/preloaded/auth/dev_http_auth.erl b/src/preloaded/auth/dev_http_auth.erl index 18b009105..824b9cf7c 100644 --- a/src/preloaded/auth/dev_http_auth.erl +++ b/src/preloaded/auth/dev_http_auth.erl @@ -46,6 +46,19 @@ %% @doc Generate or extract a new secret and commit to the message with the %% `~httpsig@1.0/commit?type=hmac-sha256&scheme=secret' commitment mechanism. +-spec commit(#{ _ => _ }, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. commit(Base, Req, Opts) -> case generate(Base, Req, Opts) of {ok, Key} -> @@ -68,6 +81,19 @@ commit(Base, Req, Opts) -> %% @doc Verify a given `Base' message with a derived `Key' using the %% `~httpsig@1.0' secret key HMAC commitment scheme. +-spec verify(#{ _ => _ }, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, boolean()}. verify(Base, RawReq, Opts) -> ?event({verify_invoked, {priv_base, Base}, {priv_req, RawReq}}), {ok, Key} = generate(Base, RawReq, Opts), @@ -88,6 +114,19 @@ verify(Base, RawReq, Opts) -> %% @doc Collect authentication information from the client. If the `raw' flag %% is set to `true', return the raw authentication information. Otherwise, %% derive a key from the authentication information and return it. +-spec generate(#{ _ => _ }, + #{ + secret => binary(), + authorization => binary(), + raw => boolean(), + alg => atom(), + salt => binary(), + iterations => integer(), + 'key-length' => integer(), + _ => _ + }, + #{ _ => _ } +) -> {ok, binary()} | {error, #{ status := integer(), _ => _ }}. generate(_Msg, ReqLink, Opts) when ?IS_LINK(ReqLink) -> generate(_Msg, hb_cache:ensure_loaded(ReqLink, Opts), Opts); generate(_Msg, #{ <<"secret">> := Secret }, _Opts) -> diff --git a/src/preloaded/auth/dev_secret.erl b/src/preloaded/auth/dev_secret.erl index d0b8a35af..5dd89d766 100644 --- a/src/preloaded/auth/dev_secret.erl +++ b/src/preloaded/auth/dev_secret.erl @@ -174,6 +174,8 @@ %% @doc Generate a new wallet for a user and register it on the node. If the %% `committer' field is provided, we first check whether there is a wallet %% already registered for it. If there is, we return the wallet details. +-spec generate(#{ _ => _ }, #{ committer => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, _}. generate(Base, Request, Opts) -> case request_to_wallets(Base, Request, Opts) of [] -> @@ -199,6 +201,8 @@ generate(Base, Request, Opts) -> %% @doc Import a wallet for hosting on the node. Expects the keys to be either %% provided as a list of keys, or a single key in the `key' field. If neither %% are provided, the keys are extracted from the cookie. +-spec import(#{ _ => _ }, #{ key => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. import(Base, Request, Opts) -> Wallets = case hb_maps:find(<<"key">>, Request, Opts) of @@ -388,10 +392,12 @@ persist_registered_wallet(WalletDetails, RespBase, Opts) -> end. %% @doc List all hosted wallets +-spec list(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]}. list(_Base, _Request, Opts) -> {ok, list_wallets(Opts)}. %% @doc Sign a message with a wallet. +-spec commit(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, binary()}. commit(Base, Request, Opts) -> ?event({commit_invoked, {priv_base, Base}, {priv_request, Request}}), case request_to_wallets(Base, Request, Opts) of @@ -577,6 +583,8 @@ commit_message(Message, #{ <<"wallet">> := Key }, Opts) -> %% @doc Export wallets from a request. The request should contain a source of %% wallets (cookies, keys, or wallet names), or a specific list/name of a %% wallet to authenticate and export. +-spec export(#{ _ => _ }, #{ keyids => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, binary()}. export(Base, Request, Opts) -> PrivOpts = priv_store_opts(Opts), ModReq = @@ -604,6 +612,8 @@ export(Base, Request, Opts) -> end. %% @doc Sync wallets from a remote node +-spec sync(#{ _ => _ }, #{ node => binary(), as => binary(), keyids => binary() | [binary()], _ => _ }, #{ _ => _ }) -> + {ok, [_]} | {error, _}. sync(_Base, Request, Opts) -> case hb_ao:get(<<"node">>, Request, undefined, Opts) of undefined -> diff --git a/src/preloaded/codec/dev_ans104.erl b/src/preloaded/codec/dev_ans104.erl index 58584d345..58d8d8448 100644 --- a/src/preloaded/codec/dev_ans104.erl +++ b/src/preloaded/codec/dev_ans104.erl @@ -13,12 +13,15 @@ content_type(_) -> {ok, <<"application/ans104">>}. %% @doc Serialize a message or TX to a binary. +-spec serialize(binary() | #tx{} | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()}. serialize(Msg, Req, Opts) when is_map(Msg) -> serialize(to(Msg, Req, Opts), Req, Opts); serialize(TX, _Req, _Opts) when is_record(TX, tx) -> {ok, ar_bundles:serialize(TX)}. %% @doc Deserialize a binary ans104 message to a TABM. +-spec deserialize(binary() | #tx{} | #{ body := binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }}. deserialize(#{ <<"body">> := Binary }, Req, Opts) -> deserialize(Binary, Req, Opts); deserialize(Binary, Req, Opts) when is_binary(Binary) -> @@ -29,6 +32,7 @@ deserialize(TX, Req, Opts) when is_record(TX, tx) -> %% @doc Sign a message using the `priv-wallet' key in the options. Supports both %% the `hmac-sha256' and `rsa-pss-sha256' algorithms, offering unsigned and %% signed commitments. +-spec commit(#{ _ => _ }, #{ type := binary(), _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -77,6 +81,7 @@ sign_tx(TX, Wallet, Opts) -> {ok, SignedStructured}. %% @doc Verify an ANS-104 commitment. +-spec verify(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. verify(Msg, Req, Opts) -> ?event({verify, {base, Msg}, {req, Req}}), OnlyWithCommitment = @@ -94,6 +99,7 @@ verify(Msg, Req, Opts) -> {ok, Res}. %% @doc Convert a #tx record into a message map recursively. +-spec from(binary() | #tx{}, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; from(TX, Req, Opts) when is_record(TX, tx) -> case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of @@ -141,6 +147,8 @@ to_hint(Msg, Req, Opts) -> %% message's device in order to get the keys that we will be checkpointing. We %% do this recursively to handle nested messages. The base case is that we hit %% a binary, which we return as is. +-spec to(binary() | #tx{} | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #tx{}}. to(Binary, _Req, _Opts) when is_binary(Binary) -> % ar_bundles cannot serialize just a simple binary or get an ID for it, so % we turn it into a TX record with a special tag, tx_to_message will diff --git a/src/preloaded/codec/dev_base64url.erl b/src/preloaded/codec/dev_base64url.erl index e1be22c2c..9cc4ea4d3 100644 --- a/src/preloaded/codec/dev_base64url.erl +++ b/src/preloaded/codec/dev_base64url.erl @@ -7,10 +7,14 @@ -include_lib("eunit/include/eunit.hrl"). %% @doc Replace the base message's base64url `body' with its decoded bytes. +-spec decode(#{ body => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, _}. decode(Base, _Req, Opts) -> with_body(Base, Opts, fun hb_util:decode/1). %% @doc Replace the base message's binary `body' with its base64url encoding. +-spec encode(#{ body => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, _}. encode(Base, _Req, Opts) -> with_body(Base, Opts, fun hb_util:encode/1). diff --git a/src/preloaded/codec/dev_bits.erl b/src/preloaded/codec/dev_bits.erl index 5fd7b06f5..4f7f5c6ef 100644 --- a/src/preloaded/codec/dev_bits.erl +++ b/src/preloaded/codec/dev_bits.erl @@ -48,6 +48,11 @@ %% @doc Decode the base message's body into a message of the fields named by %% the format given as the key's argument, or into a list of such messages %% for a `repeat(...)' format. +-spec from( + #{ body => bitstring() }, + #{ from => binary() }, + #{ _ => _ } +) -> {ok, #{ _ => _ } | [#{ _ => _ }]} | {error, _}. from(Base, Req, Opts) -> maybe {ok, Body} ?= find_body(Base, Opts), @@ -90,6 +95,8 @@ decode_records(Fields, Record, Body, Records) -> end. %% @doc Return the leading N bits of the base message's body. +-spec take(#{ body => bitstring() }, #{ take => pos_integer() }, #{ _ => _ }) -> + {ok, bitstring()} | {error, _}. take(Base, Req, Opts) -> maybe {ok, Body} ?= find_body(Base, Opts), diff --git a/src/preloaded/codec/dev_flat.erl b/src/preloaded/codec/dev_flat.erl index ec0da1117..fc9f6cfba 100644 --- a/src/preloaded/codec/dev_flat.erl +++ b/src/preloaded/codec/dev_flat.erl @@ -9,6 +9,7 @@ -include("include/hb.hrl"). %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -19,6 +20,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -29,6 +31,7 @@ verify(Msg, Req, Opts) -> }. %% @doc Convert a flat map to a TABM. +-spec from(binary() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; from(Map, Req, Opts) when is_map(Map) -> {ok, @@ -59,6 +62,8 @@ from(Map, Req, Opts) when is_map(Map) -> }. %% @doc Convert a TABM to a flat map. +-spec to(binary() | [_] | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }}. to(Bin, _, _Opts) when is_binary(Bin) -> {ok, Bin}; to(List, Req, Opts) when is_list(List) -> to( diff --git a/src/preloaded/codec/dev_gzip.erl b/src/preloaded/codec/dev_gzip.erl index 298125dcf..315def140 100644 --- a/src/preloaded/codec/dev_gzip.erl +++ b/src/preloaded/codec/dev_gzip.erl @@ -8,6 +8,8 @@ %% containting a gzip-encoded payload. Returns the rest of the base message %% unchanged, with the `content-encoding' key unset. %% +-spec unzip(#{ body => binary(), 'content-encoding' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body => binary(), _ => _ }}. unzip(Base, _Req, Opts) -> case hb_maps:get(<<"content-encoding">>, Base, <<"gzip">>, Opts) of <<"gzip">> -> @@ -48,6 +50,8 @@ unzip(Base, _Req, Opts) -> %% @doc Take a base message with a `body' key and return it zipped, in-place. %% Add a `content-encoding' key with the value `gzip'. +-spec zip(#{ body => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), 'content-encoding' := binary(), _ => _ }} | {error, binary()}. zip(Base, _Req, Opts) -> case hb_maps:find(<<"body">>, Base, Opts) of {ok, Body} -> diff --git a/src/preloaded/codec/dev_httpsig.erl b/src/preloaded/codec/dev_httpsig.erl index fd0cbcb2a..aae6e788a 100644 --- a/src/preloaded/codec/dev_httpsig.erl +++ b/src/preloaded/codec/dev_httpsig.erl @@ -21,7 +21,9 @@ -include_lib("eunit/include/eunit.hrl"). %%% Routing functions for the `dev_httpsig_conv' module +-spec to(#{ _ => _ }, #{ _ => _ }, map()) -> term(). to(Msg, Req, Opts) -> dev_httpsig_conv:to(Msg, Req, Opts). +-spec from(#{ _ => _ }, #{ _ => _ }, map()) -> term(). from(Msg, Req, Opts) -> dev_httpsig_conv:from(Msg, Req, Opts). %% @doc Generate the `Opts' to use during AO-Core operations in the codec. @@ -63,6 +65,8 @@ proxy_verify(_Base, Req, Opts) -> %% Optionally, the `index` key can be set to override resolution of the default %% index page into HTTP responses that do not contain their own `body` field. serialize(Msg, Opts) -> serialize(Msg, #{}, Opts). +-spec serialize(#{ _ => _ }, #{ format => binary(), index => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ headers := #{ _ => _ }, body := _, _ => _ }}. serialize(Msg, #{ <<"format">> := <<"components">> }, Opts) -> % Convert to HTTPSig via TABM through calling `hb_message:convert` rather % than executing `to/3` directly. This ensures that our responses are @@ -80,6 +84,11 @@ serialize(Msg, _Req, Opts) -> HTTPSig = hb_message:convert(Msg, <<"httpsig@1.0">>, Opts), {ok, dev_httpsig_conv:encode_http_msg(HTTPSig, Opts) }. +-spec verify( + #{ _ => _ }, + #{ signature := binary(), type := binary(), _ => _ }, + #{ _ => _ } +) -> {ok, boolean()} | {failure, _}. verify(Base, Req, RawOpts) -> % A rsa-pss-sha512 commitment is verified by regenerating the signature % base and validating against the signature. @@ -138,6 +147,11 @@ verify(Base, Req, RawOpts) -> %% parameter to determine the type of commitment to use. If the `type' parameter %% is `signed', we default to the rsa-pss-sha512 algorithm. If the `type' %% parameter is `unsigned', we default to the hmac-sha256 algorithm. +-spec commit( + #{ _ => _ }, + #{ type := binary(), bundle => boolean(), committed => [_], _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"hmac-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -351,6 +365,8 @@ add_content_digest(Msg, _Opts) -> %% @doc Given a base message and a commitment, derive the message and commitment %% normalized for encoding. +-spec normalize_for_encoding(#{ _ => _ }, #{ committed => [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }, #{ committed := [_], _ => _ }, [_]}. normalize_for_encoding(Msg, Commitment, Opts) -> % Extract the requested keys to include in the signature base. RawInputs = diff --git a/src/preloaded/codec/dev_json.erl b/src/preloaded/codec/dev_json.erl index cbbcb0103..b150143dc 100644 --- a/src/preloaded/codec/dev_json.erl +++ b/src/preloaded/codec/dev_json.erl @@ -19,6 +19,7 @@ to_hint(Msg, Req, Opts) -> end. %% @doc Encode a message to a JSON string, using JSON-native typing. +-spec to(binary() | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> {ok, binary()}. to(Msg, _Req, _Opts) when is_binary(Msg) -> {ok, hb_util:bin(json:encode(Msg))}; to(Msg, Req, Opts) -> @@ -56,6 +57,8 @@ to(Msg, Req, Opts) -> {ok, hb_json:encode(JSONStructured)}. %% @doc Decode a JSON string to a message. +-spec from(binary() | #{ _ => _ }, #{ 'accept-codec' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. from(Map, _Req, _Opts) when is_map(Map) -> {ok, Map}; from(JSON, Req, Opts) -> ConvOpts = Opts#{ <<"hashpath">> => ignore }, @@ -89,6 +92,7 @@ from(JSON, Req, Opts) -> end. %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -99,6 +103,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -108,12 +113,15 @@ verify(Msg, Req, Opts) -> ) }. +-spec committed(binary() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> [binary()]. committed(Msg, Req, Opts) when is_binary(Msg) -> committed(hb_util:ok(from(Msg, Req, Opts)), Req, Opts); committed(Msg, _Req, Opts) -> hb_message:committed(Msg, all, Opts). %% @doc Deserialize the JSON string found at the given path. +-spec deserialize(#{ _ => _ }, #{ target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. deserialize(Base, Req, Opts) -> Payload = hb_ao:get( @@ -141,6 +149,8 @@ deserialize(Base, Req, Opts) -> end. %% @doc Serialize a message to a JSON string. +-spec serialize(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary(), _ => _ }}. serialize(Base, Msg, Opts) -> {ok, #{ diff --git a/src/preloaded/codec/dev_json_iface.erl b/src/preloaded/codec/dev_json_iface.erl index b37621d6c..5f9f9f5b3 100644 --- a/src/preloaded/codec/dev_json_iface.erl +++ b/src/preloaded/codec/dev_json_iface.erl @@ -42,10 +42,17 @@ -include("include/hb.hrl"). %% @doc Initialize the device. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ function := binary(), _ => _ }}. init(M1, _M2, Opts) -> {ok, hb_ao:set(M1, #{<<"function">> => <<"handle">>}, Opts)}. %% @doc On first pass prepare the call, on second pass get the results. +-spec compute( + #{ pass => integer(), process => #{ _ => _ }, _ => _ }, + #{ body => #{ _ => _ }, 'block-height' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. compute(M1, M2, Opts) -> case hb_ao:get(<<"pass">>, M1, Opts) of 1 -> prep_call(M1, M2, Opts); @@ -512,6 +519,8 @@ generate_stack(File) -> generate_stack(File, <<"WASM">>). generate_stack(File, Mode) -> generate_stack(File, Mode, #{}). +-spec generate_stack(binary() | list(), binary(), #{ _ => _ }) -> + #{ _ => _ }. generate_stack(File, _Mode, RawOpts) -> Opts = normalize_test_opts(RawOpts), test_init(), diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index c3955b48b..4edc5908d 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -26,6 +26,7 @@ -define(SUPPORTED_TYPES, [<<"integer">>, <<"float">>, <<"atom">>, <<"list">>]). %% @doc Route commitments through `httpsig@1.0'. +-spec commit(#{ _ => _ }, #{ _ => _ }, map()) -> term(). commit(Msg, Req, Opts) -> {ok, hb_message:commit( @@ -36,6 +37,7 @@ commit(Msg, Req, Opts) -> }. %% @doc Route verification through `httpsig@1.0'. +-spec verify(#{ _ => _ }, #{ _ => _ }, map()) -> term(). verify(Msg, Req, Opts) -> {ok, hb_message:verify( @@ -46,6 +48,10 @@ verify(Msg, Req, Opts) -> }. %% @doc Convert a rich message into a 'Type-Annotated-Binary-Message' (TABM). +-spec from(binary() | [_] | #{ _ => _ }, + #{ 'encode-types' => [binary()], bundle => boolean(), _ => _ }, + #{ _ => _ } +) -> {ok, binary() | [_] | #{ _ => _ }}. from(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; from(List, Req, Opts) when is_list(List) -> % Encode the list as a map, then -- if our request indicates that we are @@ -224,6 +230,8 @@ linkify_mode(Req, Opts) -> end. %% @doc Convert a TABM into a native HyperBEAM message. +-spec to(binary() | [_] | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary() | [_] | #{ _ => _ }}. to(Bin, _Req, _Opts) when is_binary(Bin) -> {ok, Bin}; to(TABM0, Req, Opts) when is_list(TABM0) -> % If we receive a list, we convert it to a message and run `to/3' on it. diff --git a/src/preloaded/codec/dev_tx.erl b/src/preloaded/codec/dev_tx.erl index a8527b8e1..883daa568 100644 --- a/src/preloaded/codec/dev_tx.erl +++ b/src/preloaded/codec/dev_tx.erl @@ -13,6 +13,7 @@ %% @doc Sign a message using the `priv-wallet' key in the options. Supports both %% the `hmac-sha256' and `rsa-pss-sha256' algorithms, offering unsigned and %% signed commitments. +-spec commit(#{ _ => _ }, #{ type := binary(), _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. commit(Msg, Req = #{ <<"type">> := <<"unsigned">> }, Opts) -> commit(Msg, Req#{ <<"type">> => <<"unsigned-sha256">> }, Opts); commit(Msg, Req = #{ <<"type">> := <<"signed">> }, Opts) -> @@ -47,6 +48,7 @@ commit(Msg, #{ <<"type">> := <<"unsigned-sha256">> }, Opts) -> }. %% @doc Verify an L1 TX commitment. +-spec verify(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. verify(Msg, Req, Opts) -> ?event({verify, {base, Msg}, {req, Req}}), OnlyWithCommitment = @@ -64,6 +66,7 @@ verify(Msg, Req, Opts) -> {ok, Res}. %% @doc Convert a #tx record into a message map recursively. +-spec from(binary() | #tx{}, #{ _ => _ }, #{ _ => _ }) -> {ok, binary() | #{ _ => _ }}. from(Binary, _Req, _Opts) when is_binary(Binary) -> {ok, Binary}; from(TX, Req, Opts) when is_record(TX, tx) -> case lists:keyfind(<<"ao-type">>, 1, TX#tx.tags) of @@ -114,6 +117,8 @@ to_hint(Msg, Req, Opts) -> %% message's device in order to get the keys that we will be checkpointing. We %% do this recursively to handle nested messages. The base case is that we hit %% a binary, which we return as is. +-spec to(binary() | #tx{} | #{ _ => _ }, #{ bundle => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #tx{}}. to(Binary, _Req, _Opts) when is_binary(Binary) -> % ar_tx cannot serialize just a simple binary or get an ID for it, so % we turn it into a TX record with a special tag, tx_to_message will diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 90a9a0ff4..bdd19d422 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -47,6 +47,8 @@ info() -> %% was a device name. %% 3. Execute the `default_index_path` (base: `index') upon the message, %% giving the rest of the request unchanged. +-spec index(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. index(Msg, Req, Opts) -> case hb_opts:get(default_index, not_found, Opts) of not_found -> @@ -84,6 +86,14 @@ index(Msg, Req, Opts) -> %% if/when non-map message structures are created. id(Base) -> id(Base, #{}). id(Base, Req) -> id(Base, Req, #{}). +-spec id(binary() | [#{ _ => _ }] | #{ commitments => #{ _ => _ }, _ => _ }, + #{ committers => _, + 'commitment-ids' => _, + 'id-device' => binary(), + _ => _ + }, + #{ _ => _ } +) -> {ok, binary()}. id(Base, _, NodeOpts) when is_binary(Base) -> % Return the hashpath of the message in native format, to match the native % format of the message ID return. @@ -227,6 +237,8 @@ id_device(_, _) -> %% @doc Return the committers of a message that are present in the given request. committers(Base) -> committers(Base, #{}). committers(Base, Req) -> committers(Base, Req, #{}). +-spec committers(#{ commitments => #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [_]}. committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> {ok, hb_maps:values( @@ -249,6 +261,10 @@ committers(_, _, _) -> %% @doc Commit to a message, using the `commitment-device' key to specify the %% device that should be used to commit to the message. If the key is not set, %% the default device (`httpsig@1.0') is used. +-spec commit(#{ _ => _ }, + #{ 'commitment-device' => binary(), type => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ commitments := #{ _ => _ }, _ => _ }}. commit(Self, Req, Opts) -> {ok, Base} = hb_message:find_target(Self, Req, Opts), AttDev = @@ -296,6 +312,10 @@ commit(Self, Req, Opts) -> %% `committers' key in the request can be used to specify that only the %% commitments from specific committers should be verified. Similarly, specific %% commitments can be specified using the `commitments' key. +-spec verify(#{ _ => _ }, + #{ committers => _, 'commitment-ids' => _, commitments => _, _ => _ }, + #{ _ => _ } +) -> {ok, boolean()}. verify(Self, Req, Opts) -> % Get the target message of the verification request. {ok, RawBase} = hb_message:find_target(Self, Req, Opts), @@ -385,6 +405,10 @@ verify_commitment(Base, Commitment, Opts) -> hb_ao:raw(AttDev, <<"verify">>, Base, Commitment, Opts). %% @doc Return the list of committed keys from a message. +-spec committed(#{ _ => _ }, + #{ raw => boolean(), committers => _, 'commitment-ids' => _, _ => _ }, + #{ _ => _ } +) -> {ok, [binary()]}. committed(Self, Req, Opts) -> % Get the target message of the verification request and ensure its % commitments are loaded. @@ -611,6 +635,8 @@ commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) -> %% @doc Deep merge keys in a message. Takes a map of key-value pairs and sets %% them in the message, overwriting any existing values. +-spec set(#{ _ => _ }, #{ 'set-mode' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. set(Base, NewValuesMsg, Opts) -> OriginalPriv = hb_private:from_message(Base), % Filter keys that are in the default device (this one). @@ -663,6 +689,11 @@ set(Base, NewValuesMsg, Opts) -> KeysToSet ) ), + AddedKeys = + lists:filter( + fun(Key) -> not hb_maps:is_key(Key, Base, Opts) end, + maps:keys(NewValues) + ), % Calculate if the keys to be set conflict with any committed keys. {ok, CommittedKeys} = committed( @@ -702,11 +733,13 @@ set(Base, NewValuesMsg, Opts) -> end, OriginalPriv ), - case OverwrittenCommittedKeys of - [] -> + case {AddedKeys, OverwrittenCommittedKeys} of + {[_ | _], _} -> + {ok, hb_maps:without([<<"commitments">>], Merged, Opts)}; + {[], []} -> ?event_debug(message_set, {no_overwritten_committed_keys, {merged, Merged}}), {ok, Merged}; - _ -> + {[], _} -> % We did overwrite some keys, but do their values match the original? % If not, we must remove the commitments. ChangedBaseKeys = hb_maps:with(OverwrittenCommittedKeys, Base, Opts), @@ -799,6 +832,8 @@ do_deep_merge(BaseValues, NewValues, Opts) -> %% transmit the present key that is being executed. Subsequently, to call `path' %% we would need to set `path' to `set', removing the ability to specify its %% new value. +-spec set_path(#{ path => _, _ => _ }, #{ value => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. set_path(Base, #{ <<"value">> := Value }, Opts) -> set_path(Base, Value, Opts); set_path(Base, Value, Opts) when not is_map(Value) -> @@ -825,6 +860,8 @@ set_path(Base, Value, Opts) when not is_map(Value) -> end. %% @doc Remove a key or keys from a message. +-spec remove(#{ _ => _ }, #{ item => _, items => [_], _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. remove(Base, #{ <<"item">> := Key }, Opts) -> remove(Base, #{ <<"items">> => [Key] }, Opts); remove(Base, #{ <<"items">> := Keys }, Opts) -> @@ -968,6 +1005,21 @@ set_conflicting_keys_test() -> ?assertMatch({ok, #{ <<"dangerous">> := <<"Value2">> }}, hb_ao:resolve(Base, Req, #{})). +set_new_key_drops_commitments_test() -> + Opts = #{ + <<"store">> => hb_test_utils:test_store(), + <<"priv-wallet">> => hb:wallet() + }, + Signed = hb_message:commit(#{ <<"a">> => <<"1">> }, Opts, <<"httpsig@1.0">>), + {ok, Updated} = hb_ao:resolve( + Signed, + #{ <<"path">> => <<"set">>, <<"b">> => <<"2">> }, + Opts + ), + ?assertNot(maps:is_key(<<"commitments">>, Updated)), + {ok, Canonical} = hb_message:with_only_committed(Updated, Opts), + ?assertEqual(<<"2">>, maps:get(<<"b">>, Canonical)). + unset_with_set_test() -> Base = #{ <<"dangerous">> => <<"Value1">> }, Req = #{ <<"path">> => <<"set">>, <<"dangerous">> => unset }, diff --git a/src/preloaded/message/dev_trie.erl b/src/preloaded/message/dev_trie.erl index 2dae71520..4524c4571 100644 --- a/src/preloaded/message/dev_trie.erl +++ b/src/preloaded/message/dev_trie.erl @@ -75,8 +75,12 @@ collect_keys(TrieNode, Prefix, Opts, Acc) -> %% @doc Get the value associated with a key from a trie represented in a base %% message. +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, binary()}. get(Key, Trie, Req, Opts) -> get(Trie, Req#{<<"key">> => Key}, Opts). +-spec get(#{ _ => _ }, #{ key := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, binary()}. get(TrieNode, Req, Opts) -> case hb_maps:find(<<"key">>, Req, Opts) of error -> {error, <<"'key' parameter is required for trie lookup.">>}; @@ -84,6 +88,8 @@ get(TrieNode, Req, Opts) -> end. %% @doc Set keys and their values in the trie. +-spec set(#{ _ => _ }, #{ path => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. set(Trie, Req, Opts) -> Insertable = hb_maps:without([<<"path">>], Req, Opts), KeyVals = hb_maps:to_list(Insertable, Opts), diff --git a/src/preloaded/name/dev_b32_name.erl b/src/preloaded/name/dev_b32_name.erl index 5856ac22c..08cff0533 100644 --- a/src/preloaded/name/dev_b32_name.erl +++ b/src/preloaded/name/dev_b32_name.erl @@ -12,6 +12,8 @@ info(_Opts) -> }. %% @doc Try to resolve 52char subdomain back to its original TX ID +-spec get(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, not_found}. get(Key, _, _HookMsg, _Opts) -> ?event({resolve_52char, {key, Key}}), case decode(Key) of diff --git a/src/preloaded/name/dev_local_name.erl b/src/preloaded/name/dev_local_name.erl index efcf15246..e987a8e7a 100644 --- a/src/preloaded/name/dev_local_name.erl +++ b/src/preloaded/name/dev_local_name.erl @@ -17,6 +17,8 @@ info(_Opts) -> }. %% @doc Takes a `key' argument and returns the value of the name, if it exists. +-spec lookup(#{ _ => _ }, #{ key := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. lookup(_, Req, Opts) -> Key = hb_ao:get(<<"key">>, Req, no_key_specified, Opts), ?event(local_name, {lookup, Key}), @@ -27,11 +29,15 @@ lookup(_, Req, Opts) -> ). %% @doc Handle all other requests by delegating to the lookup function. +-spec default_lookup(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. default_lookup(Key, _, Req, Opts) -> lookup(Key, Req#{ <<"key">> => Key }, Opts). %% @doc Takes a `key' and `value' argument and registers the name. The caller %% must be the node operator in order to register a name. +-spec register(#{ _ => _ }, #{ key := binary(), value := _, _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, #{ status := integer(), message := binary() }} | not_found. register(_, Req, Opts) -> case hb_ao:resolve( #{ <<"device">> => <<"meta@1.0">> }, diff --git a/src/preloaded/name/dev_name.erl b/src/preloaded/name/dev_name.erl index 30549f3e0..85ea76e1f 100644 --- a/src/preloaded/name/dev_name.erl +++ b/src/preloaded/name/dev_name.erl @@ -25,6 +25,8 @@ info(_) -> %% pointer and its contents is loaded from the cache. For example, %% `GET /~name@1.0/reference' yields the message at the path specified by the %% `reference' key. +-spec resolve(binary(), #{ _ => _ }, #{ load => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, _} | not_found. resolve(Key, _, Req, Opts) -> Resolvers = hb_opts:get(name_resolvers, [], Opts), ?event({resolvers, Resolvers}), @@ -78,6 +80,11 @@ execute_resolver(Key, Resolver, Opts) when is_map(Resolver) -> %% @doc Implements an `on/request' compatible hook that resolves names given in %% the `host` key to their corresponding ID and prepends it to the execution path. +-spec request( + #{ _ => _ }, + #{ request := #{ host := binary(), _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. request(HookMsg, HookReq, Opts) -> ?event({request_hook, {hook_msg, HookMsg}, {hook_req, HookReq}}), maybe diff --git a/src/preloaded/node/dev_blacklist.erl b/src/preloaded/node/dev_blacklist.erl index b4fe7f15a..547027ee1 100644 --- a/src/preloaded/node/dev_blacklist.erl +++ b/src/preloaded/node/dev_blacklist.erl @@ -44,6 +44,8 @@ <<"/~hyperbuddy@1.0/bundle.js">>]). %% @doc Hook handler: block requests that involve blacklisted IDs. +-spec request(#{ _ => _ }, #{ request := #{ path => binary(), _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ _ => _ }} | {blocked_txid, binary()}. request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), case hb_opts:get(blacklist_providers, false, Opts) of diff --git a/src/preloaded/node/dev_cache.erl b/src/preloaded/node/dev_cache.erl index 2d90dac56..45d3440ac 100644 --- a/src/preloaded/node/dev_cache.erl +++ b/src/preloaded/node/dev_cache.erl @@ -21,6 +21,8 @@ %% @returns {ok, Data} on success, %% {error, not_found} if the key does not exist, %% {error, Reason} or {failure, Reason} on failure. +-spec read(#{ _ => _ }, #{ read := binary(), accept => binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _} | {failure, _}. read(_M1, M2, Opts) -> Location = hb_ao:get(<<"read">>, M2, Opts), ?event({read, {key_extracted, Location}}), @@ -80,6 +82,8 @@ read(_M1, M2, Opts) -> %% @param Opts A map of configuration options. %% @returns {ok, Path} on success, where Path indicates where the data was %% stored, {error, Reason} or {failure, Reason} on failure. +-spec write(#{ _ => _ }, #{ body => binary() | #{ _ => _ }, type => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, _} | {failure, _} | #{ _ => _ }. write(_M1, M2, Opts) -> case is_trusted_writer(M2, Opts) of true -> @@ -135,6 +139,8 @@ write(_M1, M2, Opts) -> end. %% @doc Link a source to a destination in the cache. +-spec link(#{ _ => _ }, #{ destination := binary(), source := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. link(_Base, Req, Opts) -> case is_trusted_writer(Req, Opts) of true -> @@ -145,6 +151,8 @@ link(_Base, Req, Opts) -> {error, not_authorized} end. +-spec group(#{ _ => _ }, #{ group := binary(), _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. group(_Base, Req, Opts) -> case is_trusted_writer(Req, Opts) of true -> diff --git a/src/preloaded/node/dev_cacheviz.erl b/src/preloaded/node/dev_cacheviz.erl index 64a79e658..7a33ebf73 100644 --- a/src/preloaded/node/dev_cacheviz.erl +++ b/src/preloaded/node/dev_cacheviz.erl @@ -6,6 +6,8 @@ %% @doc Output the dot representation of the cache, or a specific path within %% the cache set by the `target' key in the request. +-spec dot(#{ _ => _ }, #{ target => binary(), 'render-data' => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary() }}. dot(_, Req, Opts) -> Target = hb_ao:get(<<"target">>, Req, all, Opts), Dot = @@ -23,6 +25,8 @@ dot(_, Req, Opts) -> %% @doc Output the SVG representation of the cache, or a specific path within %% the cache set by the `target' key in the request. +-spec svg(#{ _ => _ }, #{ target => binary(), 'render-data' => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ 'content-type' := binary(), body := binary() }}. svg(Base, Req, Opts) -> {ok, #{ <<"body">> := Dot }} = dot(Base, Req, Opts), ?event(cacheviz, {dot, Dot}), @@ -33,6 +37,8 @@ svg(Base, Req, Opts) -> %% the `graph.js' library. If the request specifies a `target' key, we use that %% target. Otherwise, we generate a new target by writing the message to the %% cache and using the ID of the written message. +-spec json(#{ _ => _ }, #{ target => binary(), 'max-size' => integer(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. json(Base, Req, Opts) -> ?event({json, {base, Base}, {req, Req}}), Target = @@ -60,10 +66,12 @@ json(Base, Req, Opts) -> Res. %% @doc Return a renderer in HTML form for the JSON format. +-spec index(#{ _ => _ }, #{ _ => _ }, map()) -> term(). index(Base, _, Opts) -> ?event({cacheviz_index, {base, Base}}), hb_http_server:static(<<"cacheviz@1.0">>, <<"graph.html">>, Opts). %% @doc Return a JS library that can be used to render the JSON format. +-spec js(#{ _ => _ }, #{ _ => _ }, map()) -> term(). js(_, _, Opts) -> hb_http_server:static(<<"cacheviz@1.0">>, <<"graph.js">>, Opts). diff --git a/src/preloaded/node/dev_cron.erl b/src/preloaded/node/dev_cron.erl index 3129a4a06..a1d6dae3b 100644 --- a/src/preloaded/node/dev_cron.erl +++ b/src/preloaded/node/dev_cron.erl @@ -9,6 +9,8 @@ info(_) -> #{ default => fun handler/4 }. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := #{ _ => _ }, _ => _ }}. info(_Base, _Req, _Opts) -> InfoBody = #{ <<"description">> => <<"Cron device for scheduling messages">>, @@ -24,6 +26,7 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Default handler: Assume that the key is an interval descriptor. +-spec handler(term(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). handler(<<"set">>, Base, Req, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"set">>, Base, Req, Opts); handler(<<"keys">>, Base, _Req, _Opts) -> @@ -32,6 +35,8 @@ handler(Interval, Base, Req, Opts) -> every(Base, Req#{ <<"interval">> => Interval }, Opts). %% @doc Exported function for scheduling a one-time message. +-spec once(#{ _ => _ }, #{ 'cron-path' => binary(), once => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := binary(), _ => _ }} | {error, _}. once(_Base, Req, Opts) -> case extract_path(<<"once">>, Req, Opts) of not_found -> @@ -77,6 +82,8 @@ once_worker(Path, Req, Opts) -> %% @doc Exported function for scheduling a recurring message. +-spec every(#{ _ => _ }, #{ interval := binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := binary(), _ => _ }} | {error, _}. every(_Base, Req, Opts) -> case { extract_path(Req, Opts), @@ -137,6 +144,8 @@ every(_Base, Req, Opts) -> end. %% @doc Exported function for stopping a scheduled task. +-spec stop(#{ _ => _ }, #{ stop := binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := _, _ => _ }} | {error, _}. stop(_Base, Req, Opts) -> case hb_maps:get(<<"stop">>, Req, not_found, Opts) of not_found -> @@ -165,6 +174,8 @@ stop(_Base, Req, Opts) -> end. %% @doc Exported function for getting a scheduled task status report. +-spec report(#{ _ => _ }, #{ report := binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ 'task-id' := binary(), active := boolean(), _ => _ }} | {error, _}. report(_Base, Req, Opts) -> case hb_maps:get(<<"report">>, Req, not_found, Opts) of not_found -> diff --git a/src/preloaded/node/dev_hyperbuddy.erl b/src/preloaded/node/dev_hyperbuddy.erl index b68c9927e..03fc23908 100644 --- a/src/preloaded/node/dev_hyperbuddy.erl +++ b/src/preloaded/node/dev_hyperbuddy.erl @@ -34,6 +34,7 @@ info(Opts) -> }. %% @doc The main HTML page for the REPL device. +-spec metrics(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ body := binary(), _ => _ }}. metrics(_, Req, Opts) -> case hb_opts:get(prometheus, not hb_features:test(), Opts) of true -> @@ -71,6 +72,7 @@ metrics(_, Req, Opts) -> end. %% @doc Return the current event counters as a message. +-spec events(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. events(_, _Req, _Opts) -> {ok, hb_event:counters()}. @@ -96,6 +98,11 @@ events(_, _Req, _Opts) -> %% ``` %% `resolve-links' sets the link depth to resolve. It defaults to the node's %% `debug-resolve-links' option; `true' resolves all levels. +-spec format( + #{ _ => _ }, + #{ format => binary() | [binary()], 'truncate-keys' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := binary(), _ => _ }}. format(Base, Req, Opts) -> % Find the scope of the environment that should be printed. Scope = @@ -160,6 +167,7 @@ format(Base, Req, Opts) -> }. %% @doc Test key for validating the behavior of the `500` HTTP response. +-spec throw(#{ _ => _ }, #{ _ => _ }, #{ mode => atom(), _ => _ }) -> {error, binary()}. throw(_Msg, _Req, Opts) -> case hb_opts:get(mode, prod, Opts) of prod -> {error, <<"Forced-throw unavailable in `prod` mode.">>}; @@ -168,6 +176,7 @@ throw(_Msg, _Req, Opts) -> %% @doc Serve a file from the priv directory. Only serves files that are explicitly %% listed in the `routes' field of the `info/1' return value. +-spec serve(term(), #{ _ => _ }, #{ _ => _ }, map()) -> term(). serve(<<"keys">>, M1, _M2, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, Opts); serve(<<"set">>, M1, M2, Opts) -> diff --git a/src/preloaded/node/dev_location.erl b/src/preloaded/node/dev_location.erl index dfbc0a2f8..040e6f196 100644 --- a/src/preloaded/node/dev_location.erl +++ b/src/preloaded/node/dev_location.erl @@ -33,6 +33,8 @@ info() -> %% @doc Route either `POST' or `GET' requests to the correct handler for known %% location records. +-spec known(#{ _ => _ }, #{ method => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. known(Base, Req, Opts) -> case hb_ao:get(<<"method">>, Req, <<"GET">>, Opts) of <<"POST">> -> write_foreign(Base, Req, Opts); @@ -40,6 +42,7 @@ known(Base, Req, Opts) -> end. %% @doc List all known location records. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]} | {error, _}. all(_Base, _Req, Opts) -> dev_location_cache:list(Opts). @@ -47,6 +50,8 @@ all(_Base, _Req, Opts) -> %% cache. If an address is provided, we search for the location of that %% specific scheduler. Otherwise, we return the location record for the current %% node's scheduler, if it has been established. +-spec read(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), body := binary(), _ => _ }}. read(Address, _Base, _Req, Opts) -> read(Address, Opts). read(Address, Opts) -> @@ -100,6 +105,7 @@ find_target(Base, RawReq, Opts) -> %% @doc Generate a new scheduler location record and register it. We both send %% the new scheduler-location to the given registry, and return it to the caller. +-spec node(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. node(Base, RawReq, RawOpts) -> Opts = case hb_ao:resolve( diff --git a/src/preloaded/node/dev_meta.erl b/src/preloaded/node/dev_meta.erl index 48a3d922a..d26853492 100644 --- a/src/preloaded/node/dev_meta.erl +++ b/src/preloaded/node/dev_meta.erl @@ -56,6 +56,7 @@ is_operator(_Base, Req, NodeMsg) -> %% Subsequently, rather than embedding the `git-short-hash-length', for the %% avoidance of doubt, we include the short hash separately, as well as its long %% hash. +-spec build(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. build(_, _, _NodeMsg) -> BuildInfo = build_info(), {ok, @@ -125,6 +126,7 @@ handle_initialize([], _NodeMsg) -> %% @doc Get/set the node message. If the request is a `POST', we check that the %% request is signed by the owner of the node. If not, we return the node message %% as-is, aside all keys that are private (according to `hb_private'). +-spec info(#{ _ => _ }, #{ _ => _ }, map()) -> term(). info(_, Request, NodeMsg) -> case hb_ao:get(<<"method">>, Request, NodeMsg) of <<"POST">> -> @@ -444,6 +446,8 @@ maybe_sign(Res, NodeMsg) -> %% The `role' can be one of `operator' or `initiator'. is(Request, NodeMsg) -> is(operator, Request, NodeMsg). +-spec is(atom(), #{ _ => _ }, #{ _ => _ }) -> + boolean() | {ok, boolean()} | {error, #{ status := integer(), _ => _ }}. is(admin, Request, NodeMsg) -> % Does the caller have the right to change the node message? RequestSigners = hb_message:signers(Request, NodeMsg), diff --git a/src/preloaded/node/dev_node_process.erl b/src/preloaded/node/dev_node_process.erl index 58a4b7934..dc42d7e3f 100644 --- a/src/preloaded/node/dev_node_process.erl +++ b/src/preloaded/node/dev_node_process.erl @@ -18,6 +18,8 @@ info(_Opts) -> }. %% @doc Lookup a process by name. +-spec lookup(binary(), #{ _ => _ }, #{ spawn => boolean(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. lookup(Name, _Base, Req, Opts) -> ?event(node_process, {lookup, {name, Name}}), LookupRes = diff --git a/src/preloaded/node/dev_profile.erl b/src/preloaded/node/dev_profile.erl index 2353b5781..95647b89c 100644 --- a/src/preloaded/node/dev_profile.erl +++ b/src/preloaded/node/dev_profile.erl @@ -36,6 +36,7 @@ info(_) -> %% output from the engine formatted as an AO-Core message. eval(Fun) -> eval(Fun, #{}). eval(Fun, Opts) -> eval(Fun, #{}, Opts). +-spec eval(function() | #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _} | {_, _}. eval(Fun, Req, Opts) when is_function(Fun) -> do_eval( Fun, @@ -47,6 +48,7 @@ eval(Fun, Req, Opts) when is_function(Fun) -> ); eval(Base, Request, Opts) -> eval(<<"eval">>, Base, Request, Opts). +-spec eval(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _} | {_, _}. eval(PathKey, Base, Req, Opts) when not is_function(Base) -> case hb_ao:get(PathKey, Req, undefined, Opts) of undefined -> diff --git a/src/preloaded/node/dev_rate_limit.erl b/src/preloaded/node/dev_rate_limit.erl index f30c05198..fea45f684 100644 --- a/src/preloaded/node/dev_rate_limit.erl +++ b/src/preloaded/node/dev_rate_limit.erl @@ -40,6 +40,8 @@ %% 429 status code and response if the limit is exceeded. The response includes %% a `retry-after' header that indicates the number of seconds the client should %% wait before making the next request. +-spec request(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, #{ status := integer(), reason := binary(), body := binary(), _ => _ }}. request(_, Msg, Opts) -> ?event(rate_limit, {request, {msg, Msg}}), Reference = request_reference(hb_maps:get(<<"request">>, Msg, #{}, Opts), Opts), diff --git a/src/preloaded/node/dev_router.erl b/src/preloaded/node/dev_router.erl index b94b21cbf..35a3d3385 100644 --- a/src/preloaded/node/dev_router.erl +++ b/src/preloaded/node/dev_router.erl @@ -46,6 +46,7 @@ info(_) -> }. %% @doc HTTP info response providing information about this device +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. info(_Base, _Req, _Opts) -> InfoBody = #{ <<"description">> => <<"Router device for handling outbound message routing">>, @@ -91,6 +92,7 @@ info(_Base, _Req, _Opts) -> %% @doc Register function that allows telling the current node to register %% a new route with a remote router node. This function should also be idempotent. %% so that it can be called only once. +-spec register(#{ _ => _ }, #{ as => binary(), _ => _ }, #{ _ => _ }) -> {ok, binary()}. register(_M1, M2, Opts) -> %% Extract all required parameters from options %% These values will be used to construct the registration message @@ -138,6 +140,8 @@ register(_M1, M2, Opts) -> {ok, <<"Routes registered.">>}. %% @doc Device function that returns all known routes. +-spec routes(#{ _ => _ }, #{ method => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | [_] | #{ _ => _ }} | {error, _}. routes(M1, M2, Opts) -> ?event({routes_msg, M1, M2}), Routes = load_routes(Opts), @@ -236,6 +240,8 @@ routes(M1, M2, Opts) -> %% routing based on the Opts and request message provided, or as a standalone %% function, taking only the request message and the `Opts' map. route(Msg, Opts) -> route(undefined, Msg, Opts). +-spec route(#{ _ => _ }, #{ path => binary(), 'route-path' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary() | #{ _ => _ }} | {error, no_matches}. route(_, Msg, Opts) -> Routes = load_routes(Opts), MatchedRoute = match_routes(Msg, Routes, Opts), @@ -405,6 +411,11 @@ do_apply_route( %% @doc Find the first matching template in a list of known routes. Allows the %% path to be specified by either the explicit `path' (for internal use by this %% module), or `route-path' for use by external devices and users. +-spec match(#{ routes => [_] | #{ _ => _ }, _ => _ }, + #{ path => binary(), 'route-path' => binary(), _ => _ }, + #{ _ => _ } +) -> + {ok, #{ _ => _ }} | {error, no_matching_route}. match(Base, Req, Opts) -> ?event(debug_preprocess, {matching_routes, @@ -724,6 +735,11 @@ binary_to_bignum(Bin) when ?IS_ID(Bin) -> Num. %% @doc Preprocess a request to check if it should be relayed to a different node. +-spec preprocess( + #{ 'commit-request' => boolean(), _ => _ }, + #{ request := #{ path := binary(), _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := [_], _ => _ }}. preprocess(Base, RawReq, Opts) -> Req = hb_ao:get(<<"request">>, RawReq, Opts#{ <<"hashpath">> => ignore }), ?event(debug_preprocess, {called_preprocess,Req}), diff --git a/src/preloaded/node/dev_whois.erl b/src/preloaded/node/dev_whois.erl index 19a174d22..00ffb2173 100644 --- a/src/preloaded/node/dev_whois.erl +++ b/src/preloaded/node/dev_whois.erl @@ -9,11 +9,15 @@ -include_lib("eunit/include/eunit.hrl"). %% @doc Return the calculated host information for the requester. +-spec echo(#{ _ => _ }, #{ 'ao-peer' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, binary()}. echo(_, Req, Opts) -> {ok, hb_maps:get(<<"ao-peer">>, Req, <<"unknown">>, Opts)}. %% @doc Return the host information for the node. Sets the `host' key in the %% node message if it is not already set. +-spec node(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, binary()} | {error, _}. node(_, _, Opts) -> case ensure_host(Opts) of {ok, NewOpts} -> diff --git a/src/preloaded/payment/dev_faff.erl b/src/preloaded/payment/dev_faff.erl index e047262e4..f21f9c121 100644 --- a/src/preloaded/payment/dev_faff.erl +++ b/src/preloaded/payment/dev_faff.erl @@ -21,6 +21,8 @@ -include("include/hb.hrl"). %% @doc Decide whether or not to service a request from a given address. +-spec estimate(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, integer() | binary()}. estimate(_, Msg, NodeMsg) -> ?event(payment, {estimate, {msg, Msg}}), % Check if the address is in the allow-list. @@ -41,6 +43,7 @@ is_admissible(Msg, NodeMsg) -> ). %% @doc Charge the user's account if the request is allowed. +-spec charge(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, boolean()}. charge(_, Req, _NodeMsg) -> ?event(payment, {charge, Req}), {ok, true}. diff --git a/src/preloaded/payment/dev_p4.erl b/src/preloaded/payment/dev_p4.erl index 4260609ca..1c72f8a0c 100644 --- a/src/preloaded/payment/dev_p4.erl +++ b/src/preloaded/payment/dev_p4.erl @@ -47,6 +47,11 @@ %% @doc Estimate the cost of a transaction and decide whether to proceed with %% a request. The default behavior if `pricing-device' or `p4_balances' are %% not set is to proceed, so it is important that a user initialize them. +-spec request( + #{ 'pricing-device' => binary(), 'ledger-device' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := [_], _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := [_], _ => _ }} | {error, _}. request(State, Raw, NodeMsg) -> PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), @@ -169,6 +174,11 @@ request(State, Raw, NodeMsg) -> end. %% @doc Postprocess the request after it has been fulfilled. +-spec response( + #{ 'pricing-device' => binary(), 'ledger-device' => binary(), _ => _ }, + #{ request := #{ _ => _ }, body := _, _ => _ }, + #{ _ => _ } +) -> {ok, #{ body := _, _ => _ }} | {error, _}. response(State, RawResponse, NodeMsg) -> PricingDevice = hb_ao:get(<<"pricing-device">>, State, false, NodeMsg), LedgerDevice = hb_ao:get(<<"ledger-device">>, State, false, NodeMsg), @@ -269,6 +279,7 @@ response(State, RawResponse, NodeMsg) -> %% A node may run several request hooks, so the P4 handler has to be selected %% rather than assumed to be the only entry. P4 handlers carry a %% `ledger-device'. +-spec balance(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. balance(_, Req, NodeMsg) -> case p4_handler(hb_hook:find(<<"request">>, NodeMsg), NodeMsg) of not_found -> diff --git a/src/preloaded/payment/dev_simple_pay.erl b/src/preloaded/payment/dev_simple_pay.erl index d42cd6f14..44bd14f71 100644 --- a/src/preloaded/payment/dev_simple_pay.erl +++ b/src/preloaded/payment/dev_simple_pay.erl @@ -26,6 +26,8 @@ %% @doc Estimate the cost of the request, using the rules outlined in the %% moduledoc. +-spec estimate(#{ _ => _ }, #{ request => #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, non_neg_integer()}. estimate(_Base, EstimateReq, NodeMsg) -> Req = hb_ao:get( @@ -136,6 +138,8 @@ price_from_count(Messages, NodeMsg) -> %% @doc Preprocess a request by checking the ledger and charging the user. We %% can charge the user at this stage because we know statically what the price %% will be +-spec charge(#{ _ => _ }, #{ request => #{ _ => _ }, quantity => integer(), _ => _ }, #{ _ => _ }) -> + {ok, boolean()} | {error, #{ status := integer(), body := binary(), _ => _ }}. charge(_, RawReq, NodeMsg) -> ?event(payment, {charge, RawReq}), Req = @@ -195,6 +199,8 @@ charge(_, RawReq, NodeMsg) -> end. %% @doc Get the balance of a user in the ledger. +-spec balance(#{ _ => _ }, #{ request => #{ _ => _ }, target => binary(), _ => _ }, #{ _ => _ }) -> + {ok, integer()}. balance(_, RawReq, NodeMsg) -> Target = case @@ -260,6 +266,8 @@ get_balance(Signer, NodeMsg) -> hb_ao:get(NormSigner, Ledger, 0, NodeMsg). %% @doc Top up the user's balance in the ledger. +-spec topup(#{ _ => _ }, #{ amount => integer(), recipient => binary(), _ => _ }, #{ _ => _ }) -> + {ok, integer()} | {error, binary()}. topup(_, Req, NodeMsg) -> ?event({topup, {req, Req}, {node_msg, NodeMsg}}), case is_operator(Req, NodeMsg) of diff --git a/src/preloaded/process/dev_process.erl b/src/preloaded/process/dev_process.erl index 3f017fcd2..c4356c083 100644 --- a/src/preloaded/process/dev_process.erl +++ b/src/preloaded/process/dev_process.erl @@ -79,6 +79,9 @@ info(_Base) -> %% @doc Return the process state with the device swapped out for the device %% of the given key. +-spec as(#{ 'input-prefix' => binary(), _ => _ }, + #{ as => binary(), 'as-device' => binary(), _ => _ }, + #{ _ => _ }) -> {ok, #{ device := binary(), _ => _ }}. as(RawBase, Req, Opts) -> {ok, Base} = ensure_loaded(RawBase, Req, Opts), Key = @@ -126,13 +129,19 @@ as(RawBase, Req, Opts) -> %% _must_ be set in all processes aside those marked with `ao.TN.1' variant. %% This is in order to ensure that post-mainnet processes do not default to %% using infrastructure that should not be present on nodes in the future. +-spec default_device(#{ 'process/variant' => binary(), _ => _ }, binary(), #{ _ => _ }) -> + binary(). default_device(Base, Key, Opts) -> lib_process:default_device(Base, Key, Opts). %% @doc Wraps functions in the Scheduler device. +-spec schedule(#{ scheduler => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. schedule(Base, Req, Opts) -> lib_process:run_as(<<"scheduler">>, Base, Req, Opts). +-spec slot(#{ scheduler => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. slot(Base, Req, Opts) -> ?event({slot_called, {base, Base}, {req, Req}}), lib_process:run_as(<<"scheduler">>, Base, Req, Opts). @@ -140,6 +149,8 @@ slot(Base, Req, Opts) -> next(Base, _Req, Opts) -> lib_process:run_as(<<"scheduler">>, Base, next, Opts). +-spec snapshot(#{ execution => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. snapshot(RawBase, _Req, Opts) -> Base = lib_process:ensure_process_key(RawBase, Opts), {ok, SnapshotMsg} = @@ -190,6 +201,19 @@ init(Base, Req, Opts) -> %% handlers and previewing results. The POST method is the key entry point %% for the dryrun functionality that allows external clients to test %% message processing without side effects. +-spec compute( + #{ initialized => binary(), 'at-slot' => integer(), _ => _ }, + #{ + compute => integer(), + slot => integer(), + init => binary(), + push => _, + 'result-depth' => _, + async => _, + 'max-depth' => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | {failure, _}. compute(Base, Req, Opts) -> ProcBase = lib_process:ensure_process_key(Base, Opts), ProcID = lib_process:process_id(ProcBase, #{}, Opts), @@ -623,6 +647,8 @@ should_snapshot_time(Res, Opts) -> %% @doc Returns the known state of the process at either the current slot, or %% the latest slot in the cache depending on the `process-now-from-cache' option. +-spec now(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {failure, _} | {error, _}. now(RawBase, Req, Opts) -> Base = lib_process:ensure_process_key(RawBase, Opts), ProcessID = lib_process:process_id(Base, #{}, Opts), @@ -680,6 +706,8 @@ now(RawBase, Req, Opts) -> %% @doc Recursively push messages to the scheduler until we find a message %% that does not lead to any further messages being scheduled. +-spec push(#{ push => _, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. push(Base, Req, Opts) -> lib_process:run_as( <<"push">>, diff --git a/src/preloaded/process/dev_push.erl b/src/preloaded/process/dev_push.erl index cf6b4dd46..990357a07 100644 --- a/src/preloaded/process/dev_push.erl +++ b/src/preloaded/process/dev_push.erl @@ -37,6 +37,11 @@ %% `N > 0' - recurse, with the inner `/push' %% inheriting `max-depth = N - 1'. %% Unwinds at most `N' levels deep. +-spec push( + #{ _ => _ }, + #{ slot => integer(), body => #{ _ => _ }, async => boolean(), 'max-depth' => integer(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _} | pid(). push(Base, Req, Opts) -> Process = lib_process:as_process(Base, Opts), ?event(push, {push_base, {base, Process}, {req, Req}}, Opts), diff --git a/src/preloaded/process/dev_scheduler.erl b/src/preloaded/process/dev_scheduler.erl index fe8dff675..49d51a068 100644 --- a/src/preloaded/process/dev_scheduler.erl +++ b/src/preloaded/process/dev_scheduler.erl @@ -71,6 +71,8 @@ parse_schedulers(SchedLoc) when is_binary(SchedLoc) -> ). %% @doc The default handler for the scheduler device. +-spec router(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. router(_, Base, Req, Opts) -> ?event({scheduler_router_called, {req, Req}, {opts, Opts}}), schedule(Base, Req, Opts). @@ -79,6 +81,8 @@ router(_, Base, Req, Opts) -> %% assignment. Assumes that Base is a `dev_process' or similar message, having %% a `Current-Slot' key. It stores a local cache of the schedule in the %% `priv/To-Process' key. +-spec next(#{ 'at-slot' := integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := #{ _ => _ }, state := #{ _ => _ }, _ => _ }} | {error, _}. next(Base, Req, Opts) -> ?event(debug_next, {scheduler_next_called, {base, Base}, {req, Req}}), ?event(next, started_next), @@ -346,6 +350,8 @@ check_lookahead_and_local_cache(undefined, ProcID, TargetSlot, Opts) -> end. %% @doc Returns information about the entire scheduler. +-spec status(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ address := binary(), processes := [binary()], 'cache-control' := binary(), _ => _ }}. status(_M1, _M2, _Opts) -> ?event(getting_scheduler_status), Wallet = dev_scheduler_registry:get_wallet(), @@ -363,6 +369,11 @@ status(_M1, _M2, _Opts) -> %% @doc A router for choosing between getting the existing schedule, or %% scheduling a new message. +-spec schedule( + #{ _ => _ }, + #{ method => binary(), from => integer(), to => integer(), accept => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ } | binary()} | {error, _}. schedule(Base, Req, Opts) -> ?event({resolving_schedule_request, {req, Req}, {state_msg, Base}}), case hb_util:key_to_atom(hb_ao:get(<<"method">>, Req, <<"GET">>, Opts)) of @@ -715,6 +726,7 @@ find_remote_scheduler(ProcID, Scheduler, Opts) -> end. %% @doc Returns information about the current slot for a process. +-spec slot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. slot(M1, M2, Opts) -> ?event({getting_current_slot, {msg, M1}}), ProcID = find_target_id(M1, M2, Opts), diff --git a/src/preloaded/query/dev_copycat.erl b/src/preloaded/query/dev_copycat.erl index 61126a93d..303ddfa2f 100644 --- a/src/preloaded/query/dev_copycat.erl +++ b/src/preloaded/query/dev_copycat.erl @@ -11,10 +11,12 @@ %% @doc Fetch data from a GraphQL endpoint for replication. See %% `dev_copycat_graphql' for implementation details. +-spec graphql(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. graphql(Base, Request, Opts) -> dev_copycat_graphql:graphql(Base, Request, Opts). %% @doc Fetch data from an Arweave node for replication. See `dev_copycat_arweave' %% for implementation details. +-spec arweave(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. arweave(Base, Request, Opts) -> dev_copycat_arweave:arweave(Base, Request, Opts). \ No newline at end of file diff --git a/src/preloaded/query/dev_match.erl b/src/preloaded/query/dev_match.erl index 2095aa98f..455a6c3f8 100644 --- a/src/preloaded/query/dev_match.erl +++ b/src/preloaded/query/dev_match.erl @@ -80,6 +80,8 @@ value_path(Other, Opts) -> %% @doc Match a single key-value pair in the index, returning all message IDs that %% contain the key-value pair. +-spec match(binary() | atom(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [binary()]} | {error, not_found}. match(Key, Base, _Req, Opts) -> match(Key, Base, Opts). match(Key, Base, Opts) -> Store = store(Opts), @@ -98,6 +100,8 @@ match(Key, Base, Opts) -> %% @doc Match the full base message against the index, returning the intersection %% of all matches for each key. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, [binary()]} | {error, not_found}. all(Base, _Req, Opts) -> IndexBase = hb_message:uncommitted(hb_private:reset(Base)), Keys = diff --git a/src/preloaded/query/dev_query.erl b/src/preloaded/query/dev_query.erl index 9e6b676bb..77470b570 100644 --- a/src/preloaded/query/dev_query.erl +++ b/src/preloaded/query/dev_query.erl @@ -44,12 +44,15 @@ info(_Opts) -> }. %% @doc Execute the query via GraphQL. +-spec graphql(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. graphql(Req, Base, Opts) -> dev_query_graphql:handle(Req, Base, Opts). %% @doc Return whether a GraphQL esponse in a message has transaction results. %% This key is used in HB's gateway client multirequest configuration to %% determine if the response from the node should be considered admissible. +-spec has_results(#{ body => binary(), _ => _ }, #{ body => binary(), _ => _ }, #{ _ => _ }) -> + {ok, boolean()}. has_results(Base, Req, Opts) -> JSON = hb_ao:get_first( @@ -70,22 +73,30 @@ has_results(Base, Req, Opts) -> end. %% @doc Search for the keys specified in the request message. +-spec default(_, #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. default(_, Base, Req, Opts) -> all(Base, Req, Opts). %% @doc Search the node's store for all of the keys and values in the request, %% aside from the `commitments' and `path' keys. +-spec all(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. all(Base, Req, Opts) -> match(Req, Base, Req, Opts). %% @doc Search the node's store for all of the keys and values in the base %% message, aside from the `commitments' and `path' keys. +-spec base(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, _} | {error, _}. base(Base, Req, Opts) -> match(Base, Base, Req, Opts). %% @doc Search only for the (list of) key(s) specified in `only' in the request. %% The `only' key can be a binary, a map, or a list of keys. See the moduledoc %% for semantics. +-spec only( + #{ _ => _ }, + #{ only => binary() | [binary()] | #{ _ => _ }, exclude => [binary()], return => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, _} | {error, _}. only(Base, Req, Opts) -> case hb_maps:get(<<"only">>, Req, not_found, Opts) of KeyBin when is_binary(KeyBin) -> diff --git a/src/preloaded/util/dev_apply.erl b/src/preloaded/util/dev_apply.erl index c1d7e813e..225bcfa3a 100644 --- a/src/preloaded/util/dev_apply.erl +++ b/src/preloaded/util/dev_apply.erl @@ -29,6 +29,8 @@ info(_) -> %% @doc The default handler. If the `base' and `request' keys are present in %% the given request, then the `pair' function is called. Otherwise, the `eval' %% key is used to resolve the request. +-spec default(binary(), #{ _ => _ }, #{ base => _, request => _, source => _, _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. default(Key, Base, Request, Opts) -> ?event(debug_apply, {req, {key, Key}, {base, Base}, {request, Request}}), FoundBase = hb_maps:get(<<"base">>, Request, not_found, Opts), @@ -85,6 +87,8 @@ eval(Base, Request, Opts) -> end. %% @doc Apply the message found at `request' to the message found at `base'. +-spec pair(#{ _ => _ }, #{ base => _, request => _, _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. pair(Base, Request, Opts) -> pair(<<"undefined">>, Base, Request, Opts). pair(PathToSet, Base, Request, Opts) -> diff --git a/src/preloaded/util/dev_dedup.erl b/src/preloaded/util/dev_dedup.erl index eb9459cc1..692829687 100644 --- a/src/preloaded/util/dev_dedup.erl +++ b/src/preloaded/util/dev_dedup.erl @@ -31,6 +31,8 @@ info(_M1) -> %% @doc Forward the keys and `set' functions to the message device, handle all %% others with deduplication. This allows the device to be used in any context %% where a key is called. If the `dedup-key +-spec handle(binary(), #{ 'dedup-subject' => binary(), pass => integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {skip, #{ _ => _ }}. handle(<<"keys">>, M1, _M2, _Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, #{}); handle(<<"set">>, M1, M2, Opts) -> diff --git a/src/preloaded/util/dev_multipass.erl b/src/preloaded/util/dev_multipass.erl index 49695d9ef..fcc3e235e 100644 --- a/src/preloaded/util/dev_multipass.erl +++ b/src/preloaded/util/dev_multipass.erl @@ -13,6 +13,8 @@ info(_M1) -> %% @doc Forward the keys function to the message device, handle all others %% with deduplication. We only act on the first pass. +-spec handle(binary(), #{ passes => integer(), pass => integer(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {pass, #{ _ => _ }}. handle(<<"keys">>, M1, _M2, Opts) -> hb_ao:raw(<<"message@1.0">>, <<"keys">>, M1, #{}, Opts); handle(<<"set">>, M1, M2, Opts) -> diff --git a/src/preloaded/util/dev_patch.erl b/src/preloaded/util/dev_patch.erl index 4e288ab9e..b49708f63 100644 --- a/src/preloaded/util/dev_patch.erl +++ b/src/preloaded/util/dev_patch.erl @@ -28,20 +28,28 @@ -include_lib("include/hb.hrl"). %% @doc Necessary hooks for compliance with the `execution-device' standard. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Base, _Req, _Opts) -> {ok, Base}. +-spec normalize(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. normalize(Base, _Req, _Opts) -> {ok, Base}. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. snapshot(Base, _Req, _Opts) -> {ok, Base}. +-spec compute(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }} | {error, _}. compute(Base, Req, Opts) -> patches(Base, Req, Opts). %% @doc Get the value found at the `patch-from' key of the message, or the %% `from' key if the former is not present. Remove it from the message and set %% the new source to the value found. +-spec all(#{ _ => _ }, #{ from => binary(), to => binary(), 'patch-from' => binary(), 'patch-to' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. all(Base, Req, Opts) -> move(all, Base, Req, Opts). %% @doc Find relevant `PATCH' messages in the given source key of the execution %% and request messages, and apply them to the given destination key of the %% request. +-spec patches(#{ _ => _ }, #{ from => binary(), to => binary(), 'patch-from' => binary(), 'patch-to' => binary(), _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. patches(Base, Req, Opts) -> move(patches, Base, Req, Opts). diff --git a/src/preloaded/util/dev_relay.erl b/src/preloaded/util/dev_relay.erl index e589e24f8..252fa2515 100644 --- a/src/preloaded/util/dev_relay.erl +++ b/src/preloaded/util/dev_relay.erl @@ -19,6 +19,7 @@ %%% to the node's routing table. -export([request/3]). -include("include/hb.hrl"). +-include("include/hb_opts.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Execute a `call' request using a node's routes. @@ -29,6 +30,11 @@ %% - `method': The method to use for the request. Defaults to the original method. %% - `commit-request': Whether the request should be committed before dispatching. %% Defaults to `false'. +-spec call( + #{ _ => _ }, + #{ target => binary(), 'relay-path' => binary(), method => binary(), peer => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. call(M1, RawM2, Opts) -> ?event({relay_call, {m1, M1}, {raw_m2, RawM2}}), {ok, BaseTarget} = hb_message:find_target(M1, RawM2, Opts), @@ -136,7 +142,11 @@ do_call(RelayPath, BaseTarget, M1, RawM2, Opts) -> ?event(debug_relay, {relay_call, {with_http_params, TargetMod5}}), true = hb_message:verify(TargetMod5), ?event(debug_relay, {relay_call, {verified, true}}), - Client = hb_opts:get(relay_http_client, Opts), + Client = hb_opts:get( + relay_http_client, + hb_opts:get(http_client, ?DEFAULT_HTTP_CLIENT, Opts), + Opts + ), % Let `hb_http:request/2' handle finding the peer and dispatching the % request, unless the peer is explicitly given. HTTPOpts = Opts#{ <<"http-client">> => Client, <<"http-only-result">> => false }, @@ -212,11 +222,14 @@ host_matches(Host, Entry) -> %% @doc Execute a request in the same way as `call/3', but asynchronously. Always %% returns `<<"OK">>'. +-spec cast(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, binary()}. cast(M1, M2, Opts) -> spawn(fun() -> call(M1, M2, Opts) end), {ok, <<"OK">>}. %% @doc Preprocess a request to check if it should be relayed to a different node. +-spec request(#{ _ => _ }, #{ request := #{ _ => _ }, _ => _ }, #{ _ => _ }) -> + {ok, #{ body := [#{ _ => _ }], _ => _ }}. request(_Base, Req, Opts) -> {ok, #{ diff --git a/src/preloaded/util/dev_stack.erl b/src/preloaded/util/dev_stack.erl index a818c82f4..0799cc075 100644 --- a/src/preloaded/util/dev_stack.erl +++ b/src/preloaded/util/dev_stack.erl @@ -117,20 +117,28 @@ info(Msg, Opts) -> ). %% @doc Return the default prefix for the stack. +-spec prefix(#{ 'output-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). prefix(Base, _Req, Opts) -> hb_ao:get(<<"output-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc Return the input prefix for the stack. +-spec input_prefix(#{ 'input-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). input_prefix(Base, _Req, Opts) -> hb_ao:get(<<"input-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc Return the output prefix for the stack. +-spec output_prefix(#{ 'output-prefix' => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + binary(). output_prefix(Base, _Req, Opts) -> hb_ao:get(<<"output-prefix">>, {as, <<"message@1.0">>, Base}, <<"">>, Opts). %% @doc The device stack key router. Sends the request to `resolve_stack', %% except for `set/2' which is handled by the default implementation in %% `dev_message'. +-spec router(binary(), #{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, _} | {error, _}. router(<<"keys">>, Base, Request, Opts) -> ?event({keys_called, {base, Base}, {req, Request}}), hb_ao:raw(<<"message@1.0">>, <<"keys">>, Base, #{}, Opts); diff --git a/src/preloaded/util/dev_test.erl b/src/preloaded/util/dev_test.erl index 8851ecd88..e1ceefa76 100644 --- a/src/preloaded/util/dev_test.erl +++ b/src/preloaded/util/dev_test.erl @@ -4,6 +4,8 @@ -export([info/1, test_func/1, compute/3, init/3, restore/3, snapshot/3, mul/2]). -export([mangle/3, update_state/3, increment_counter/3, delay/3, append/3]). -export([index/3, postprocess/3, load/3]). +-export([vary_projection/3, vary_wildcard/3, vary_unspecified/3]). +-export([vary_overlay/3]). -include_lib("eunit/include/eunit.hrl"). -include("include/hb.hrl"). @@ -29,6 +31,8 @@ info(_) -> %% @doc Exports a default_handler function that can be used to test the %% handler resolution mechanism. +-spec info(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ status := integer(), body := #{ _ => _ } }}. info(_Base, _Req, _Opts) -> InfoBody = #{ <<"description">> => <<"Test device for testing the AO-Core framework">>, @@ -49,6 +53,8 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Example index handler. +-spec index(#{ name => binary(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), 'content-type' := binary(), _ => _ }}. index(Msg, _Req, Opts) -> Name = hb_ao:get(<<"name">>, Msg, <<"turtles">>, Opts), {ok, @@ -59,6 +65,8 @@ index(Msg, _Req, Opts) -> }. %% @doc Return a message with the device set to this module. +-spec load(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ device := binary(), _ => _ }}. load(Base, _, _Opts) -> {ok, Base#{ <<"device">> => <<"test-device@1.0">> }}. @@ -68,6 +76,8 @@ test_func(_) -> %% @doc Example implementation of a `compute' handler. Makes a running list of %% the slots that have been computed in the state message and places the new %% slot number in the results key. +-spec compute(#{ 'already-seen' => [integer()], _ => _ }, #{ slot := integer() }, #{ _ => _ }) -> + {ok, #{ 'already-seen' := [integer()], results := #{ 'assignment-slot' := integer() }, _ => _ }}. compute(Base, Req, Opts) -> AssignmentSlot = hb_ao:get(<<"slot">>, Req, Opts), Seen = hb_ao:get(<<"already-seen">>, Base, Opts), @@ -86,12 +96,16 @@ compute(Base, Req, Opts) -> }. %% @doc Example `init/3' handler. Sets the `Already-Seen' key to an empty list. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ 'already-seen' := list(), _ => _ }}. init(Msg, _Req, Opts) -> ?event({init_called_on_dev_test, Msg}), {ok, hb_ao:set(Msg, #{ <<"already-seen">> => [] }, Opts)}. %% @doc Example `restore/3' handler. Sets the hidden key `Test/Started' to the %% value of `Current-Slot' and checks whether the `Already-Seen' key is valid. +-spec restore(#{ 'already-seen' => list(), _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, binary()}. restore(Msg, _Req, Opts) -> ?event({restore_called_on_dev_test, Msg}), case hb_ao:get(<<"already-seen">>, Msg, Opts) of @@ -119,6 +133,7 @@ mul(Base, Req) -> {ok, #{ <<"state">> => State, <<"results">> => [Arg1 * Arg2] }}. %% @doc Do nothing when asked to snapshot. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{}}. snapshot(Base, Req, _Opts) -> ?event({snapshot_called, {base, Base}, {req, Req}}), {ok, #{}}. @@ -133,12 +148,16 @@ append(Base, Req, Opts) -> {ok, Base#{ <<"result">> => <> }}. %% @doc Set the `postprocessor-called' key to true in the HTTP server. +-spec postprocess(#{ _ => _ }, #{ body := _, _ => _ }, #{ _ => _ }) -> + {ok, _}. postprocess(_Msg, #{ <<"body">> := Msgs }, Opts) -> ?event({postprocess_called, Opts}), hb_http_server:set_opts(Opts#{ <<"postprocessor-called">> => true }), {ok, Msgs}. %% @doc Find a test worker's PID and send it an update message. +-spec update_state(#{ _ => _ }, #{ 'test-id' => _, _ => _ }, #{ _ => _ }) -> + {ok, ok} | {error, binary()}. update_state(_Msg, Req, _Opts) -> case hb_ao:get(<<"test-id">>, Req) of not_found -> @@ -155,6 +174,8 @@ update_state(_Msg, Req, _Opts) -> end. %% @doc Find a test worker's PID and send it an increment message. +-spec increment_counter(#{ _ => _ }, #{ 'test-id' => _, _ => _ }, #{ _ => _ }) -> + {ok, ok} | {error, binary()}. increment_counter(_Base, Req, _Opts) -> case hb_ao:get(<<"test-id">>, Req) of not_found -> @@ -174,6 +195,8 @@ increment_counter(_Base, Req, _Opts) -> %% @doc Does nothing, just sleeps `Req/duration or 750' ms and returns the %% appropriate form in order to be used as a hook. +-spec delay(#{ _ => _ }, #{ duration => integer(), result => _, body => _, _ => _ }, #{ _ => _ }) -> + {ok, _}. delay(Base, Req, Opts) -> Duration = hb_ao:get_first( @@ -203,6 +226,8 @@ delay(Base, Req, Opts) -> %% %% Caution: This function is not safe to use in production, as it may cause %% state inconsistencies. +-spec mangle(#{ commitments => #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, binary()}. mangle(Base, _Req, Opts) -> case hb_opts:get(mode, prod, Opts) of prod -> {error, <<"`mangle' unavailable in `prod` mode.">>}; @@ -220,6 +245,35 @@ mangle(Base, _Req, Opts) -> end end. +%% @doc Return the inputs selected by the function's schema. +-spec vary_projection( + #{ required := integer(), optional => binary(), deep := #{ slot := integer() } }, + #{ path := binary(), deep_request := #{ slot := integer() } }, + #{ _ => _ } +) -> {ok, #{ base := #{ _ => _ }, request := #{ _ => _ } }}. +vary_projection(Base, Req, _Opts) -> + {ok, #{ <<"base">> => Base, <<"request">> => Req }}. + +%% @doc Return schema-selected inputs while retaining wildcard keys. +-spec vary_wildcard( + #{ required := integer(), _ => _ }, + #{ path := binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ base := #{ _ => _ }, request := #{ _ => _ } }}. +vary_wildcard(Base, Req, _Opts) -> + {ok, #{ <<"base">> => Base, <<"request">> => Req }}. + +%% @doc Return an input without declaring a Vary schema. +vary_unspecified(Base, _Req, _Opts) -> + {ok, maps:get(<<"noise">>, Base)}. + +%% @doc Increment a counter in a projection of the base, returning a patch +%% that the resolver lays over the whole base. +-spec vary_overlay(#{ counter := integer() }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ '...' := base, counter := integer() }}. +vary_overlay(Base = #{ <<"counter">> := Counter }, _Req, _Opts) -> + {ok, Base#{ <<"counter">> => Counter + 1 }}. + %%% Tests %% @doc Tests the resolution of a default function. @@ -264,3 +318,152 @@ restore_test() -> Base = #{ <<"device">> => <<"test-device@1.0">>, <<"already-seen">> => [1] }, {ok, Res} = hb_ao:resolve(Base, <<"restore">>, #{}), ?assertEqual([1], hb_private:get(<<"test-key/started-state">>, Res, #{})). + +vary_projection_and_coercion_test() -> + Opts = vary_opts(), + {ok, RequiredPath} = hb_cache:write(<<"7">>, Opts), + {ok, DeepPath} = + hb_cache:write( + #{ <<"slot">> => <<"8">>, <<"noise">> => <<"drop">> }, + Opts + ), + {ok, DeepRequestPath} = + hb_cache:write(#{ <<"slot">> => <<"9">> }, Opts), + {ok, Res} = + hb_ao:resolve( + #{ + <<"device">> => <<"test-device@1.0">>, + <<"required">> => {link, RequiredPath, #{}}, + <<"deep">> => {link, DeepPath, #{}}, + <<"noise">> => <<"drop">> + }, + #{ + <<"path">> => <<"vary-projection">>, + <<"deep-request">> => {link, DeepRequestPath, #{}}, + <<"noise">> => <<"drop">> + }, + Opts + ), + VariedBase = maps:get(<<"base">>, Res), + VariedReq = maps:get(<<"request">>, Res), + ?assertEqual(7, maps:get(<<"required">>, VariedBase)), + ?assertEqual(#{ <<"slot">> => 8 }, maps:get(<<"deep">>, VariedBase)), + ?assertNot(maps:is_key(<<"optional">>, VariedBase)), + ?assertNot(maps:is_key(<<"noise">>, VariedBase)), + ?assertEqual( + #{ <<"slot">> => 9 }, + maps:get(<<"deep-request">>, VariedReq) + ), + ?assertNot(maps:is_key(<<"noise">>, VariedReq)). + +vary_required_key_missing_test() -> + Opts = vary_opts(), + ?assertThrow( + {required_key_missing, <<"required">>}, + hb_ao:resolve( + #{ + <<"device">> => <<"test-device@1.0">>, + <<"deep">> => #{ <<"slot">> => 1 } + }, + #{ + <<"path">> => <<"vary-projection">>, + <<"deep-request">> => #{ <<"slot">> => 1 } + }, + Opts + ) + ). + +vary_wildcard_preserves_other_keys_test() -> + Opts = vary_opts(), + {ok, BaseExtraPath} = hb_cache:write(<<"base">>, Opts), + {ok, RequestExtraPath} = hb_cache:write(<<"request">>, Opts), + BaseExtra = {link, BaseExtraPath, #{}}, + RequestExtra = {link, RequestExtraPath, #{}}, + {ok, Res} = + hb_ao:resolve( + #{ + <<"device">> => <<"test-device@1.0">>, + <<"required">> => <<"1">>, + <<"extra">> => BaseExtra + }, + #{ + <<"path">> => <<"vary-wildcard">>, + <<"extra">> => RequestExtra + }, + Opts + ), + ?assertEqual(1, maps:get(<<"required">>, maps:get(<<"base">>, Res))), + ?assertEqual(BaseExtra, maps:get(<<"extra">>, maps:get(<<"base">>, Res))), + ?assertEqual( + RequestExtra, + maps:get(<<"extra">>, maps:get(<<"request">>, Res)) + ). + +vary_unspecified_function_is_identity_test() -> + Opts = vary_opts(), + ?assertEqual( + {ok, <<"kept">>}, + hb_ao:resolve( + #{ + <<"device">> => <<"test-device@1.0">>, + <<"noise">> => <<"kept">> + }, + <<"vary-unspecified">>, + Opts + ) + ). + +vary_overlay_patches_unvaried_base_test() -> + Opts = vary_opts(), + {ok, Res} = + hb_ao:resolve( + #{ + <<"device">> => <<"test-device@1.0">>, + <<"counter">> => <<"1">>, + <<"noise">> => <<"kept">> + }, + <<"vary-overlay">>, + Opts + ), + ?assertEqual(2, hb_ao:get(<<"counter">>, Res, Opts)), + ?assertEqual(<<"kept">>, hb_ao:get(<<"noise">>, Res, Opts)). + +vary_projection_uses_projected_cache_key_test() -> + Store = hb_test_utils:test_store(), + Opts = + #{ + <<"store">> => Store, + <<"cache-control">> => [<<"always">>], + <<"spawn-worker">> => false + }, + Base = + #{ + <<"device">> => <<"test-device@1.0">>, + <<"required">> => <<"7">>, + <<"deep">> => #{ <<"slot">> => <<"8">> }, + <<"noise">> => <<"first">> + }, + Req = + #{ + <<"path">> => <<"vary-projection">>, + <<"deep-request">> => #{ <<"slot">> => <<"9">> } + }, + {ok, First} = hb_ao:resolve(Base, Req, Opts), + {ok, Second} = + hb_ao:resolve( + Base#{ <<"noise">> => <<"second">> }, + Req, + Opts#{ <<"cache-control">> => [<<"only-if-cached">>] } + ), + ?assertEqual(7, hb_ao:get(<<"base/required">>, Second, Opts)), + ?assertEqual(8, hb_ao:get(<<"base/deep/slot">>, Second, Opts)), + ?assertEqual(9, hb_ao:get(<<"request/deep-request/slot">>, Second, Opts)). + +vary_opts() -> + Store = hb_test_utils:test_store(), + hb_store:reset(Store), + #{ + <<"store">> => Store, + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"spawn-worker">> => false + }. diff --git a/src/preloaded/vm/dev_delegated_compute.erl b/src/preloaded/vm/dev_delegated_compute.erl index 64daea028..0350aff38 100644 --- a/src/preloaded/vm/dev_delegated_compute.erl +++ b/src/preloaded/vm/dev_delegated_compute.erl @@ -10,12 +10,15 @@ %% @doc Initialize or normalize the compute-lite device. For now, we don't %% need to do anything special here. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Base, _Req, _Opts) -> {ok, Base}. %% @doc We assume that the compute engine stores its own internal state, %% with snapshots triggered only when HyperBEAM requests them. Subsequently, %% to load a snapshot, we just need to return the original message. +-spec normalize(#{ snapshot => #{ type => binary(), data => _, _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | #{ _ => _ }. normalize(Base, _Req, Opts) -> case hb_maps:find(<<"snapshot">>, Base, Opts) of error -> {ok, Base}; @@ -50,6 +53,11 @@ load_state(Snapshot, Opts) -> %% @doc Call the delegated server to compute the result. The endpoint is %% `POST /compute' and the body is the JSON-encoded message that we want to %% evaluate. +-spec compute( + #{ _ => _ }, + #{ type => binary(), slot => integer(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. compute(Base, Req, Opts) -> OutputPrefix = hb_ao:get( @@ -222,6 +230,7 @@ handle_relay_response(Base, Req, Opts, Response, OutputPrefix, ProcessID, Slot) %% @doc Generate a snapshot of a running computation by calling the %% `GET /snapshot' endpoint. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. snapshot(Msg, Req, Opts) -> ?event({snapshotting, {req, Req}}), ProcID = lib_process:process_id(Msg, #{}, Opts), diff --git a/src/preloaded/vm/dev_genesis_wasm.erl b/src/preloaded/vm/dev_genesis_wasm.erl index 8d765fe47..47ffba573 100644 --- a/src/preloaded/vm/dev_genesis_wasm.erl +++ b/src/preloaded/vm/dev_genesis_wasm.erl @@ -12,9 +12,14 @@ -define(STATUS_TIMEOUT, 100). %% @doc Initialize the device. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(Msg, _Req, _Opts) -> {ok, Msg}. %% @doc Normalize the device. +-spec normalize(#{ snapshot => #{ type => binary(), data => _, _ => _ }, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, #{ status := integer(), message := binary(), _ => _ }}. normalize(Msg, Req, Opts) -> case ensure_started(Opts) of true -> @@ -36,6 +41,11 @@ normalize(Msg, Req, Opts) -> %% @doc Genesis-wasm device compute handler. %% Normal compute execution through external CU with state persistence +-spec compute( + #{ _ => _ }, + #{ slot => integer(), type => binary(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. compute(Msg, Req, Opts) -> % Validate whether the genesis-wasm feature is enabled. case delegate_request(Msg, Req, Opts) of @@ -63,6 +73,8 @@ compute(Msg, Req, Opts) -> end. %% @doc Snapshot the state of the process via the `delegated-compute@1.0' device. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }} | {error, _}. snapshot(Msg, Req, Opts) -> delegate_request(Msg, Req, Opts). @@ -353,6 +365,11 @@ ensure_started(Opts) -> %% @doc Find either a specific checkpoint by its ID, or find the most recent %% checkpoint via GraphQL. +-spec import( + #{ _ => _ }, + #{ import => binary(), 'process-id' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. import(Base, Req, Opts) -> PassedProcID = hb_maps:find(<<"process-id">>, Req, Opts), ProcMsg = diff --git a/src/preloaded/vm/dev_lua.erl b/src/preloaded/vm/dev_lua.erl index ca8baf299..850e9f95d 100644 --- a/src/preloaded/vm/dev_lua.erl +++ b/src/preloaded/vm/dev_lua.erl @@ -52,6 +52,11 @@ info(Base) -> %% @doc Initialize the device state, loading the script into memory if it is %% a reference. +-spec init( + #{ module => _ , 'content-type' => binary(), body => binary(), sandbox => _, _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }} | {error, _}. init(Base, Req, Opts) -> ensure_initialized(Base, Req, Opts). @@ -236,6 +241,7 @@ initialize(Base, Modules, Opts) -> {ok, hb_private:set(Base, <<"state">>, State3, Opts)}. %%% @doc Return a list of all functions in the Lua environment. +-spec functions(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, [_]} | {error, not_found}. functions(Base, _Req, Opts) -> case hb_private:get(<<"state">>, Base, Opts) of not_found -> @@ -276,6 +282,12 @@ sandbox(State, [Path | Rest], Opts) -> sandbox(NextState, Rest, Opts). %% @doc Call the Lua script with the given arguments. +-spec compute( + binary(), + #{ _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, _} | {error, #{ status := integer(), _ => _ }}. compute(Key, RawBase, RawReq, Opts) -> ?event(debug_lua, compute_called), Req = @@ -382,6 +394,8 @@ process_response({error, Reason, Trace}, _Priv, _Opts) -> %% @doc Snapshot the Lua state from a live computation. Normalizes its `priv' %% state element, then serializes the state to a binary. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary(), _ => _ }} | {error, binary()}. snapshot(Base, _Req, Opts) -> case hb_private:get(<<"state">>, Base, Opts) of not_found -> @@ -391,6 +405,8 @@ snapshot(Base, _Req, Opts) -> end. %% @doc Restore the Lua state from a snapshot, if it exists. +-spec normalize(#{ snapshot => #{ body => binary(), _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. normalize(Base, _Req, RawOpts) -> Opts = RawOpts#{ <<"hashpath">> => ignore }, case hb_private:get(<<"state">>, Base, Opts) of diff --git a/src/preloaded/vm/dev_wasi.erl b/src/preloaded/vm/dev_wasi.erl index e8433a80c..3c1373622 100644 --- a/src/preloaded/vm/dev_wasi.erl +++ b/src/preloaded/vm/dev_wasi.erl @@ -41,6 +41,7 @@ %% - Empty stdio files %% - WASI-preview-1 compatible functions for accessing the filesystem %% - File descriptors for those files. +-spec init(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> {ok, #{ _ => _ }}. init(M1, _M2, Opts) -> ?event(running_init), MsgWithLib = @@ -77,6 +78,8 @@ stdout(M) -> %% @doc Adds a file descriptor to the state message. %path_open(M, Instance, [FDPtr, LookupFlag, PathPtr|_]) -> +-spec path_open(#{ _ => _ }, #{ args := [integer()], _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. path_open(Base, Req, Opts) -> FDs = hb_ao:get(<<"file-descriptors">>, Base, Opts), Instance = hb_private:get(<<"instance">>, Base, Opts), @@ -111,6 +114,8 @@ path_open(Base, Req, Opts) -> %% @doc WASM stdlib implementation of `fd_write', using the WASI-p1 standard %% interface. +-spec fd_write(#{ state := #{ _ => _ }, _ => _ }, #{ args := [integer()], 'func-sig' => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. fd_write(Base, Req, Opts) -> State = hb_ao:get(<<"state">>, Base, Opts), Instance = hb_private:get(<<"wasm/instance">>, State, Opts), @@ -165,6 +170,8 @@ fd_write(S, Instance, [FDnum, Ptr, Vecs, RetPtr], BytesWritten, Opts) -> ). %% @doc Read from a file using the WASI-p1 standard interface. +-spec fd_read(#{ state := #{ _ => _ }, _ => _ }, #{ args := [integer()], 'func-sig' => _, _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. fd_read(Base, Req, Opts) -> State = hb_ao:get(<<"state">>, Base, Opts), Instance = hb_private:get(<<"wasm/instance">>, State, Opts), @@ -218,6 +225,8 @@ parse_iovec(Instance, Ptr) -> {BinPtr, Len}. %%% Misc WASI-preview-1 handlers. +-spec clock_time_get(#{ state := #{ _ => _ }, _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ state := #{ _ => _ }, results := [integer()], _ => _ }}. clock_time_get(Base, _Req, Opts) -> ?event({clock_time_get, {returning, 1}}), State = hb_ao:get(<<"state">>, Base, Opts), diff --git a/src/preloaded/vm/dev_wasm.erl b/src/preloaded/vm/dev_wasm.erl index 182891871..7580327ba 100644 --- a/src/preloaded/vm/dev_wasm.erl +++ b/src/preloaded/vm/dev_wasm.erl @@ -49,6 +49,11 @@ info(_Base, _Opts) -> %% @doc Boot a WASM image on the image stated in the `process/image' field of %% the message. +-spec init( + #{ body => binary(), image => binary() | #{ _ => _ }, 'input-prefix' => binary(), _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. init(M1, _M2, Opts) -> ?event(running_init), % Where we should read initial parameters from. @@ -171,6 +176,15 @@ default_import_resolver(Base, Req, Opts) -> %% @doc Call the WASM executor with a message that has been prepared by a prior %% pass. +-spec compute( + #{ pass => integer(), function => binary(), parameters => [_], _ => _ }, + #{ body => binary() | #{ function => binary(), parameters => [_], _ => _ }, + function => binary(), + parameters => [_], + _ => _ + }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. compute(RawM1, M2, Opts) -> % Normalize the message to have an open WASM instance, but no literal `State'. % The hashpath is not updated during this process. This allows us to take @@ -259,6 +273,11 @@ compute(RawM1, M2, Opts) -> %% @doc Normalize the message to have an open WASM instance, but no literal %% `State' key. Ensure that we do not change the hashpath during this process. +-spec normalize( + #{ body => binary(), snapshot => #{ body => binary(), _ => _ }, 'device-key' => binary(), _ => _ }, + #{ _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. normalize(RawM1, M2, Opts) -> ?event({normalize_raw_m1, RawM1}), M3 = @@ -295,6 +314,8 @@ normalize(RawM1, M2, Opts) -> {ok, hb_ao:set(M3, #{ <<"snapshot">> => unset }, Opts)}. %% @doc Serialize the WASM state to a binary. +-spec snapshot(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ body := binary() }}. snapshot(M1, M2, Opts) -> ?event(snapshot, generating_snapshot), Instance = instance(M1, M2, Opts), @@ -306,6 +327,8 @@ snapshot(M1, M2, Opts) -> }. %% @doc Tear down the WASM executor. +-spec terminate(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> + {ok, #{ _ => _ }}. terminate(M1, M2, Opts) -> ?event(terminate_called_on_dev_wasm), Prefix = @@ -327,6 +350,7 @@ terminate(M1, M2, Opts) -> %% @doc Get the WASM instance from the message. Note that this function is exported %% such that other devices can use it, but it is excluded from calls from AO-Core %% resolution directly. +-spec instance(#{ _ => _ }, #{ _ => _ }, #{ _ => _ }) -> pid() | not_found | _. instance(M1, _M2, Opts) -> Prefix = hb_ao:get( @@ -345,6 +369,11 @@ instance(M1, _M2, Opts) -> %% 3. Resolving the adjusted-path-Req against the added-state-Base. %% 4. If it succeeds, return the new state from the message. %% 5. If it fails with `not_found', call the stub handler. +-spec import( + #{ _ => _ }, + #{ module := binary(), func := binary(), args => [_], 'func-sig' => binary(), _ => _ }, + #{ _ => _ } +) -> {ok, #{ _ => _ }}. import(Base, Req, Opts) -> % 1. Adjust the path to the stdlib. ModName = hb_ao:get(<<"module">>, Req, Opts),