From d32ee9f22ef52025239a27edfb125e6dd96eaad7 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 25 Jun 2026 19:03:46 -0400 Subject: [PATCH 01/35] wip: 1.0 --- src/core/resolver/hb_ao.erl | 540 ++++------------------ src/core/resolver/hb_singleton.erl | 112 ++--- src/core/resolver/hb_types.erl | 715 +++++++++++++++++++++++++++++ 3 files changed, 839 insertions(+), 528 deletions(-) create mode 100644 src/core/resolver/hb_types.erl diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 6969eaae6..05b6f096f 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -93,23 +93,20 @@ %%% -module(hb_ao). %%% Main AO-Core API: --export([resolve/2, resolve/3, resolve_many/2]). +-export([resolve/2, resolve/3]). -export([raw/3, raw/4, raw/5]). -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). --export([force_message/2]). %%% Shortcuts and tools: --export([keys/1, keys/2, keys/3]). --export([get/2, get/3, get/4, get_first/2, get_first/3]). --export([set/3, set/4, remove/2, remove/3]). +-export([keys/2]). +-export([get/3, get/4, get_first/2, get_first/3]). +-export([set/3, set/4, deep_set/3, remove/3]). %%% Exports for tests in hb_ao_test_vectors.erl: --export([deep_set/4]). -include("include/hb.hrl"). -define( TEMP_OPTS, [ <<"add-key">>, - <<"force-message">>, <<"cache-control">>, <<"spawn-worker">>, <<"only">>, @@ -117,41 +114,46 @@ ] ). -%% @doc Get the value of a message's key by running its associated device +%% @doc Get the value of an AO-Core message's key by running its associated device %% function. Optionally, takes options that control the runtime environment. %% This function returns the raw result of the device function call: %% `{ok | error, NewMessage}.' %% The resolver is composed of a series of discrete phases: -%% 1: Normalization. -%% 2: Cache lookup. -%% 3: Validation check. -%% 4: Persistent-resolver lookup. -%% 5: Device lookup. +%% 1: Loading IDs/links. +%% 2: Device or direct key lookup. +%% 3: Vary `Base` and `Request`. +%% 4: Cache lookup +%% 5: Persistent resolver lookup. %% 6: Execution. %% 7: Execution of the `step' hook. -%% 8: Subresolution. -%% 9: Cryptographic linking. -%% 10: Result caching. -%% 11: Notify waiters. -%% 12: Fork worker. -%% 13: Recurse or terminate. +%% 8: Result caching. +%% 9: Extension of result upon original base. +%% 10: Notify waiters. +%% 11: Fork worker. +%% 12: Recurse or terminate. +resolve(MsgSeq, Opts) when is_list(MsgSeq) -> + resolve_many(MsgSeq, Opts); resolve(Path, Opts) when is_binary(Path) -> resolve(#{ <<"path">> => Path }, Opts); -resolve(SingletonMsg, _Opts) - when is_map(SingletonMsg), not is_map_key(<<"path">>, SingletonMsg) -> - {error, <<"Attempted to resolve a message without a path.">>}; +resolve(SingleResult, _Opts) + when is_map(SingleResult), not is_map_key(<<"path">>, SingleResult) -> + % Nothing to resolve; return as-is. + {ok, SingleResult}; resolve(SingletonMsg, Opts) -> resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). -resolve(Base, Path, Opts) when not is_map(Path) -> +%% @doc Resolve a single AO-Core base and request pair, If the `path` in the +%% request has multiple parts, each is executed in sequence, in the context of +%% the other request keys, over the original base message. +resolve(Base, Path, Opts) when is_binary(Path) -> resolve(Base, #{ <<"path">> => Path }, Opts); resolve(Base, Req, Opts) -> - PathParts = hb_path:from_message(request, Req, Opts), - ?event( - ao_core, - {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}} - ), - MessagesToExec = [ Req#{ <<"path">> => Path } || Path <- PathParts ], + MessagesToExec = + case hb_path:from_message(request, Req, Opts) of + undefined -> [Req]; + PathParts -> + [ #{ <<"path">> => Path, <<"...">> => Req } || Path <- PathParts ] + end, ?event_debug(debug_ao_core, {stage, 1, @@ -185,8 +187,10 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> BaseWithDevice = case ForcedDevice of undefined -> Base; - _ when is_map(Base) -> Base#{ <<"device">> => ForcedDevice }; - _ -> #{ <<"device">> => ForcedDevice } + _ when is_map(Base) -> + #{ <<"device">> => ForcedDevice, <<"...">> => Base }; + _ -> + #{ <<"device">> => ForcedDevice } end, {ExecFun, PrefixArgs} = case hb_device:message_to_fun(BaseWithDevice, Key, ExecOpts) of @@ -216,10 +220,8 @@ resolve_many([ID], Opts) when ?IS_ID(ID) -> % 2. The main AO-Core logic looks for linkages between message input % pairs and outputs. With only a single ID, there is not a valid pairing % to use in looking up a cached result. - ?event_debug(debug_ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), - try {ok, ensure_message_loaded(ID, Opts)} - catch _:_:_ -> {error, not_found} - end; + ?event_debug(debug_ao_core, {stage, na, resolve_directly_to_id, ID}, Opts), + hb_cache:read(ID, Opts); resolve_many(ListMsg, Opts) when is_map(ListMsg) -> % We have been given a message rather than a list of messages, so we should % convert it to a list, assuming that the message is monotonically numbered. @@ -237,10 +239,6 @@ resolve_many(ListMsg, Opts) when is_map(ListMsg) -> ) end, resolve_many(ListOfMessages, Opts); -resolve_many({as, DevID, Msg}, Opts) -> - subresolve(#{}, DevID, Msg, Opts); -resolve_many([{resolve, Subres}], Opts) -> - resolve_many(Subres, Opts); resolve_many(MsgList, Opts) -> ?event_debug(debug_ao_core, {resolve_many, MsgList}, Opts), Res = do_resolve_many(MsgList, Opts), @@ -250,7 +248,7 @@ do_resolve_many([], _Opts) -> {failure, <<"Attempted to resolve an empty message sequence.">>}; do_resolve_many([Res], Opts) -> ?event_debug(debug_ao_core, {stage, 11, resolve_complete, Res}), - hb_cache:ensure_loaded(maybe_force_message(Res, Opts), Opts); + hb_cache:ensure_loaded(Res, Opts); do_resolve_many([Base, Req | MsgList], Opts) -> ?event_debug(debug_ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), case resolve_stage(1, Base, Req, Opts) of @@ -269,9 +267,15 @@ do_resolve_many([Base, Req | MsgList], Opts) -> Res -> % The result is not a resolvable message. Return it. ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), - maybe_force_message(Res, Opts) + Res end. +resolve_stage(1, Path, Req, Opts) when is_binary(Path) -> + % The base has been granted to us as a binary. Execute it in and resurse. + case resolve(Path, Opts) of + {ok, ResolvedBase} -> resolve_stage(1, ResolvedBase, Req, Opts); + Other -> Other + end; resolve_stage(1, Link, Req, Opts) when ?IS_LINK(Link) -> % If the first message is a link, we should load the message and % continue with the resolution. @@ -282,122 +286,6 @@ resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> % continue with the resolution. ?event_debug(debug_ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), resolve_stage(1, Base, hb_cache:ensure_loaded(Link, Opts), Opts); -resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> - % Normalize `as' requests with a raw ID or link as the path. Links will be - % loaded in following stages. - resolve_stage(1, {as, DevID, #{ <<"path">> => Ref }}, Req, Opts); -resolve_stage(1, {as, DevID, Link}, Req, Opts) when ?IS_LINK(Link) -> - % If the first message is an `as' with a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), - resolve_stage(1, {as, DevID, hb_cache:ensure_loaded(Link, Opts)}, Req, Opts); -resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Req, Opts) when ?IS_ID(ID) -> - % If the first message is an `as' with an ID, we should load the message and - % apply the non-path elements of the sub-request to it. - ?event_debug(debug_ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), - RemBase = hb_maps:without([<<"path">>], Raw, Opts), - ?event_debug(debug_subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), - Baseb = ensure_message_loaded(ID, Opts), - ?event_debug(debug_subresolution, {loaded_message, {msg, Baseb}}, Opts), - Basec = hb_maps:merge(Baseb, RemBase, Opts), - ?event_debug(debug_subresolution, {merged_message, {msg, Basec}}, Opts), - Based = set(Basec, <<"device">>, DevID, Opts), - ?event_debug(debug_subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), - resolve_stage(1, Based, Req, Opts); -resolve_stage(1, Raw = {as, DevID, SubReq}, Req, Opts) -> - % Set the device of the message to the specified one and resolve the sub-path. - % As this is the first message, we will then continue to execute the request - % on the result. - ?event_debug(debug_ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), - ?event_debug(debug_subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), - case subresolve(SubReq, DevID, SubReq, Opts) of - {ok, SubRes} -> - % The subresolution has returned a new message. Continue with it. - ?event(subresolution, - {continuing_with_subresolved_message, {base, SubRes}} - ), - resolve_stage(1, SubRes, Req, Opts); - OtherRes -> - % The subresolution has returned an error. Return it. - ?event(subresolution, - {subresolution_error, {base, Raw}, {res, OtherRes}} - ), - OtherRes - end; -resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, Opts) -> - % Set the device to the specified `DevID' and resolve the message. Merging - % the `ReqInner' into the `ReqOuter' message first. We return the result - % of the sub-resolution directly. - ?event_debug(debug_ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), - LoadedInner = ensure_message_loaded(ReqInner, Opts), - Req = - hb_maps:merge( - set(ReqOuter, <<"path">>, unset, Opts), - if is_binary(LoadedInner) -> #{ <<"path">> => LoadedInner }; - true -> LoadedInner - end, - Opts - ), - ?event(subresolution, - {subresolving_request_before_execution, - {dev, DevID}, - {req, Req} - } - ), - subresolve(RawBase, DevID, Req, Opts); -resolve_stage(1, {resolve, Subres}, Req, Opts) -> - % If the first message is a `{resolve, Subres}' tuple, we should execute it - % directly, then apply the request to the result. - ?event_debug(debug_ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), - % Unlike the `request' case for pre-subresolutions, we do not need to unset - % the `force-message' option, because the result should be a message, anyway. - % If it is not, it is more helpful to have the message placed into the `body' - % of a result, which can then be executed upon. - case resolve_many(Subres, Opts) of - {ok, Base} -> - ?event_debug(debug_ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), - resolve_stage(1, Base, Req, Opts); - OtherRes -> - ?event_debug(debug_ao_core, - {stage, - 1, - subresolve_failed, - {subres, Subres}, - {res, OtherRes}}, - Opts - ), - OtherRes - end; -resolve_stage(1, Base, {resolve, Subres}, Opts) -> - % If the second message is a `{resolve, Subresolution}' tuple, we should - % execute the subresolution directly to gain the underlying `Req' for - % our execution. We assume that the subresolution is already in a normalized, - % executable form, so we pass it to `resolve_many' for execution. - ?event_debug(debug_ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), - % We make sure to unset the `force-message' option so that if the subresolution - % returns a literal, the rest of `resolve' will normalize it to a path. - case resolve_many(Subres, maps:without([<<"force-message">>], Opts)) of - {ok, Req} -> - ?event( - ao_core, - {stage, 1, request_subresolve_success, {req, Req}}, - Opts - ), - resolve_stage(1, Base, Req, Opts); - OtherRes -> - ?event( - ao_core, - { - stage, - 1, - request_subresolve_failed, - {subres, Subres}, - {res, OtherRes} - }, - Opts - ), - OtherRes - end; resolve_stage(1, Base, Req, Opts) when is_list(Base) -> % Normalize lists to numbered maps (base=1) if necessary. ?event_debug(debug_ao_core, {stage, 1, list_normalize}, Opts), @@ -406,15 +294,8 @@ resolve_stage(1, Base, Req, Opts) when is_list(Base) -> Req, Opts ); -resolve_stage(1, Base, NonMapReq, Opts) when not is_map(NonMapReq) -> - ?event_debug(debug_ao_core, {stage, 1, path_normalize}), - resolve_stage(1, Base, #{ <<"path">> => NonMapReq }, Opts); -resolve_stage(1, RawBase, RawReq, Opts) -> - % Normalize the path to a private key containing the list of remaining - % keys to resolve. - ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), - Base = normalize_keys(RawBase, Opts), - Req = normalize_keys(RawReq, Opts), +resolve_stage(1, Base, Req, Opts) -> + ?event_debug(debug_ao_core, {stage, 1, normalize_complete}, Opts), resolve_stage(2, Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), @@ -424,7 +305,7 @@ resolve_stage(2, Base, Req, Opts) -> % only return a result if it is already in the cache). case hb_cache_control:maybe_lookup(Base, Req, Opts) of {ok, Res} -> - ?event_debug(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), + ?event_debug(debug_ao_core, {stage, 2, cache_hit, {res, Res}}, Opts), {ok, Res}; {continue, NewBase, NewReq} -> resolve_stage(3, NewBase, NewReq, Opts); @@ -503,7 +384,7 @@ resolve_stage(4, Base, Req, Opts) -> }, Opts ), - case hb_opts:get(allow_infinite, false, Opts) of + case hb_opts:get(<<"allow-infinite">>, false, Opts) of true -> % We are OK with infinite loops, so we just continue. resolve_stage(5, Base, Req, GroupName, Opts); @@ -584,7 +465,7 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> % Execution. ExecOpts = execution_opts(Opts), Args = - case hb_opts:get(add_key, false, Opts) of + case hb_opts:get(<<"add-key">>, false, Opts) of false -> [Base, Req, ExecOpts]; Key -> [Key, Base, Req, ExecOpts] end, @@ -654,9 +535,9 @@ resolve_stage( Req, {St, Res}, ExecName, - Opts = #{ <<"on">> := _On = #{ <<"step">> := _ }} + Opts = #{ <<"on">> := On = #{ <<"step">> := _ }} ) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, _On}}, Opts), + ?event_debug(debug_ao_core, {stage, 7, ExecName, executing_step_hook, On}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every @@ -697,7 +578,7 @@ resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> % Cryptographic linking. Now that we have generated the result, we % need to cryptographically link the output to its input via a hashpath. resolve_stage(10, Base, Req, - case hb_opts:get(hashpath, update, Opts#{ <<"only">> => local }) of + case hb_opts:get(<<"hashpath">>, update, Opts#{ <<"only">> => local }) of update -> NormRes = Res, Priv = hb_private:from_message(NormRes), @@ -768,61 +649,6 @@ resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> ?event_debug(debug_ao_core, {stage, 12, _ExecName, abnormal_status_skip_spawning}, _Opts), OtherRes. -%% @doc Execute a sub-resolution. -subresolve(RawBase, DevID, ReqPath, Opts) when is_binary(ReqPath) -> - % If the request is a binary, we assume that it is a path. - subresolve(RawBase, DevID, #{ <<"path">> => ReqPath }, Opts); -subresolve(RawBase, DevID, Req, Opts) -> - % First, ensure that the message is loaded from the cache. - Base = ensure_message_loaded(RawBase, Opts), - ?event(subresolution, - {subresolving, {base, Base}, {dev, DevID}, {req, Req}} - ), - % Next, set the device ID if it is given. - Base2 = - case DevID of - undefined -> Base; - _ -> - set( - Base, - <<"device">>, - DevID, - hb_maps:without(?TEMP_OPTS, Opts, Opts) - ) - end, - % If there is no path but there are elements to the request, we set these on - % the base message. If there is a path, we do not modify the base message - % and instead apply the request message directly. - case hb_path:from_message(request, Req, Opts) of - undefined -> - Base3 = - case map_size(hb_maps:without([<<"path">>], Req, Opts)) of - 0 -> Base2; - _ -> - set( - Base2, - set(Req, <<"path">>, unset, Opts), - Opts#{ <<"force-message">> => false } - ) - end, - ?event(subresolution, - {subresolve_modified_base, Base3}, - Opts - ), - {ok, Base3}; - Path -> - ?event(subresolution, - {exec_subrequest_on_base, - {mod_base, Base2}, - {req, Path}, - {req, Req} - } - ), - Res = resolve(Base2, Req, Opts), - ?event(subresolution, {subresolved_with_new_device, {res, Res}}), - Res - end. - %% @doc If the `AO_PROFILING' macro is defined (set by building/launching with %% `rebar3 as ao_profiling') we record statistics about the execution of the %% function. This is a costly operation, so if it is not defined, we simply @@ -901,22 +727,6 @@ maybe_profiled_apply(Func, Args, Base, Req, Opts) -> Res. -endif. -%% @doc Ensure that a message is loaded from the cache if it is an ID, or -%% a link, such that it is ready for execution. -ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> - case hb_cache:read(MsgID, Opts) of - {ok, LoadedMsg} -> - LoadedMsg; - failure -> - failure; - not_found -> - throw({necessary_message_not_found, <<"/">>, MsgID}) - end; -ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> - hb_cache:ensure_loaded(MsgLink, Opts); -ensure_message_loaded(Msg, _Opts) -> - Msg. - %% @doc Catch all return if we are in an infinite loop. error_infinite(Base, Req, Opts) -> ?event( @@ -947,41 +757,6 @@ error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> _ -> Error end. -%% @doc Force the result of a device call into a message if the result is not -%% requested by the `Opts'. If the result is a literal, we wrap it in a message -%% and signal the location of the result inside. We also similarly handle ao-result -%% when the result is a single value and an explicit status code. -maybe_force_message({Status, Res}, Opts) -> - case hb_opts:get(force_message, false, Opts) of - true -> force_message({Status, Res}, Opts); - false -> {Status, Res} - end; -maybe_force_message(Res, Opts) -> - maybe_force_message({ok, Res}, Opts). - -%% @doc Force a resolution result into a message, suitable for transmission -%% via HTTP. -force_message({Status, ResLink}, Opts) when ?IS_LINK(ResLink) -> - force_message({Status, hb_cache:ensure_loaded(ResLink, Opts)}, Opts); -force_message({Status, Res}, Opts) when is_list(Res) -> - force_message({Status, normalize_keys(Res, Opts)}, Opts); -force_message({Status, Subres = {resolve, _}}, _Opts) -> - {Status, Subres}; -force_message({Status, Literal}, _Opts) when not is_map(Literal) -> - ?event(encode_result, {force_message_from_literal, Literal}), - {Status, #{ <<"ao-result">> => <<"body">>, <<"body">> => Literal }}; -force_message({Status, M = #{ <<"status">> := Status, <<"body">> := Body }}, _Opts) - when map_size(M) == 2 -> - ?event(encode_result, {force_message_from_literal_with_status, M}), - {Status, #{ - <<"status">> => Status, - <<"ao-result">> => <<"body">>, - <<"body">> => Body - }}; -force_message({Status, Map}, _Opts) -> - ?event(encode_result, {force_message_from_map, Map}), - {Status, Map}. - %% @doc Shortcut for resolving a key in a message without its status if it is %% `ok'. This makes it easier to write complex logic on top of messages while %% maintaining a functional style. @@ -992,8 +767,6 @@ force_message({Status, Map}, _Opts) -> %% %% Returns the value of the key if it is found, otherwise returns the default %% provided by the user, or `not_found' if no default is provided. -get(Path, Msg) -> - get(Path, Msg, #{}). get(Path, Msg, Opts) -> get(Path, Msg, not_found, Opts). get(Path, {as, Device, Msg}, Default, Opts) -> @@ -1024,194 +797,43 @@ get_first([{Base, Path}|Msgs], Default, Opts) -> end. %% @doc Shortcut to get the list of keys from a message. -keys(Msg) -> keys(Msg, #{}). -keys(Msg, Opts) -> keys(Msg, Opts, keep). -keys(Msg, Opts, keep) -> - % There is quite a lot of AO-Core-specific machinery here. We: - % 1. `get' the keys from the message, via AO-Core in order to trigger the - % `keys' function on its device. - % 2. Ensure that the result is normalized to a message (not just a list) - % with `normalize_keys'. - % 3. Now we have a map of the original keys, so we can use `hb_maps:values' to - % get a list of them. - % 4. Normalize each of those keys in turn. - try - lists:map( - fun normalize_key/1, - hb_maps:values( - normalize_keys( - hb_private:reset(get(<<"keys">>, Msg, Opts)) - ), - Opts - ) - ) - catch - A:B:St -> - throw( - {cannot_get_keys, - {msg, Msg}, - {opts, Opts}, - {error, {A, B}}, - {stacktrace, St} - } - ) - end; -keys(Msg, Opts, remove) -> - lists:filter( - fun(Key) -> not lists:member(Key, ?AO_CORE_KEYS) end, - keys(Msg, Opts, keep) - ). +keys(Msg, Opts) -> + get(<<"keys">>, Msg, Opts). -%% @doc Shortcut for setting a key in the message using its underlying device. -%% Like the `get/3' function, this function honors the `error_strategy' option. -%% `set' works with maps and recursive paths while maintaining the appropriate -%% `HashPath' for each step. -set(RawBase, RawReq, Opts) when is_map(RawReq) -> - Base = normalize_keys(RawBase, Opts), - Req = - hb_maps:without( - [<<"hashpath">>, <<"priv">>], - normalize_keys(RawReq, Opts), - Opts - ), +%% @doc Extend a message using its underlying device's `set' key. +set(Base, Req, Opts) -> ?event_debug(ao_internal, {set_called, {base, Base}, {req, Req}}, Opts), - % Get the next key to set. - case keys(Req, internal_opts(Opts)) of - [] -> Base; - [Key|_] -> - % Get the value to set. Use AO-Core by default, but fall back to - % getting via `maps' if it is not found. - Val = - case get(Key, Req, internal_opts(Opts)) of - not_found -> hb_maps:get(Key, Req, undefined, Opts); - Body -> Body - end, - ?event_debug({got_val_to_set, {key, Key}, {val, Val}, {req, Req}}), - % Next, set the key and recurse, removing the key from the Req. - set( - set(Base, Key, Val, internal_opts(Opts)), - remove(Req, Key, internal_opts(Opts)), - Opts - ) - end. + hb_util:ok( + raw(undefined, <<"set">>, Base, Req, internal_opts(Opts)), + Opts + ). + +%% @doc Set an individual (potentially deep) key in a message to a new using +%% the message's set device. set(Base, Key, Value, Opts) -> - % For an individual key, we run deep_set with the key as the path. - % This handles both the case that the key is a path as well as the case - % that it is a single key. - Path = hb_path:term_to_path_parts(Key, Opts), - % ?event( - % {setting_individual_key, - % {base, Base}, - % {key, Key}, - % {path, Path}, - % {value, Value} - % } - % ), - deep_set(Base, Path, Value, Opts). + deep_set(Base, deep_base(Key, Value), Opts). -%% @doc Recursively search a map, resolving keys, and set the value of the key -%% at the given path. This function has special cases for handling `set' calls -%% where the path is an empty list (`/'). In this case, if the value is an -%% immediate, non-complex term, we can set it directly. Otherwise, we use the -%% device's `set' function to set the value. -deep_set(Msg, [], Value, Opts) when is_map(Msg) or is_list(Msg) -> - device_set(Msg, <<"/">>, Value, Opts); -deep_set(_Msg, [], Value, _Opts) -> - Value; -deep_set(Msg, [Key], Value, Opts) -> - device_set(Msg, Key, Value, Opts); -deep_set(Msg, [Key|Rest], Value, Opts) -> - case resolve(Msg, Key, Opts) of - {ok, SubMsg} -> - ?event_debug(debug_set, - {traversing_deeper_to_set, - {current_key, Key}, - {current_value, SubMsg}, - {rest, Rest} - }, - Opts - ), - Res = - device_set( - Msg, - Key, - deep_set(SubMsg, Rest, Value, Opts), - <<"explicit">>, - Opts - ), - ?event_debug(debug_set, {deep_set, {msg, Msg}, {key, Key}, {res, Res}}, Opts), - Res; - _ -> - ?event_debug(debug_set, - {creating_new_map, - {current_key, Key}, - {rest, Rest} - }, - Opts - ), - Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } - end. +%% @doc Create a deeply nested message with a layer for each part of the given +%% key. For example: +%% `deep_base(<<"a/b">>, Value) -> #{ <<"a">> => #{ <<"b">> => Value } }` +deep_base([Key], Value) -> #{ Key => Value }; +deep_base([NextKey|Rest], Value) -> #{ NextKey => deep_base(Rest, Value) }. -%% @doc Call the device's `set' function. -device_set(Msg, Key, Value, Opts) -> - device_set(Msg, Key, Value, <<"deep">>, Opts). -device_set(Msg, Key, Value, Mode, Opts) -> - ReqWithoutMode = - case Key of - <<"path">> -> - #{ <<"path">> => <<"set_path">>, <<"value">> => Value }; - <<"/">> when is_map(Value) -> - % The value is a map and it is to be `set' at the root of the - % message. Subsequently, we call the device's `set' function - % with all of the keys found in the message, leading it to be - % merged into the message. - Value#{ <<"path">> => <<"set">> }; - _ -> - #{ <<"path">> => <<"set">>, Key => Value } - end, - Req = - case Mode of - <<"deep">> -> ReqWithoutMode; - <<"explicit">> -> ReqWithoutMode#{ <<"set-mode">> => Mode } - end, - ?event( - debug_set, - { - calling_device_set, - {base, Msg}, - {key, Key}, - {value, Value}, - {full_req, Req} - }, - Opts - ), - Res = - hb_util:ok( - resolve( - Msg, - Req, - internal_opts(Opts) - ), +%% @doc Recursively extend nested message values. +deep_set(Base, Req, Opts) when is_map(Req) -> + hb_util:ok( + raw( + undefined, + <<"set">>, + Base, + Req#{ <<"set">> => <<"deep">> }, internal_opts(Opts) ), - ?event( - debug_set, - {device_set_result, Res}, Opts - ), - Res. + ). %% @doc Remove a key from a message, using its underlying device. -remove(Msg, Key) -> remove(Msg, Key, #{}). -remove(Msg, Key, Opts) -> - hb_util:ok( - resolve( - Msg, - #{ <<"path">> => <<"remove">>, <<"item">> => Key }, - internal_opts(Opts) - ), - Opts - ). +remove(Msg, Key, Opts) -> set(Msg, Key, unset, Opts). %% @doc Convert a key to a binary in normalized form. normalize_key(Key) -> normalize_key(Key, #{}). @@ -1304,7 +926,7 @@ execution_opts(Opts) -> % Unless the user has explicitly requested recursive spawning, we % unset the spawn-worker option so that we do not spawn a new worker % for every resulting execution. - case hb_opts:get(spawn_worker, false, Opts) of + case hb_opts:get(<<"spawn-worker">>, false, Opts) of recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 end. diff --git a/src/core/resolver/hb_singleton.erl b/src/core/resolver/hb_singleton.erl index 04de744f8..161fd80d7 100644 --- a/src/core/resolver/hb_singleton.erl +++ b/src/core/resolver/hb_singleton.erl @@ -135,15 +135,10 @@ type(_Value) -> unknown. from(RawMsg, Opts) when is_binary(RawMsg) -> from(#{ <<"path">> => RawMsg }, Opts); from(RawMsg, Opts) -> - RawPath = hb_maps:get(<<"path">>, RawMsg, <<>>), + RawPath = hb_maps:get(<<"path">>, RawMsg, <<>>, Opts), ?event_debug(parsing, {raw_path, RawPath}), {ok, Path, Query} = from_path(RawPath), ?event_debug(parsing, {parsed_path, Path, Query}), - MsgWithoutBasePath = - hb_maps:merge( - hb_maps:remove(<<"path">>, RawMsg), - Query - ), % 2. Decode, split, and sanitize path segments. Each yields one step message. RawMsgs = lists:flatten( @@ -155,11 +150,11 @@ from(RawMsg, Opts) -> ?event_debug(parsing, {raw_messages, RawMsgs}), Msgs = normalize_base(RawMsgs), ?event_debug(parsing, {normalized_messages, Msgs}), - % 3. Type keys and values - Typed = apply_types(MsgWithoutBasePath, Opts), + % 3. Type keys and values over base + Typed = apply_types(Query#{ <<"...">> => RawMsg }, Opts), ?event_debug(parsing, {typed_messages, Typed}), % 4. Group keys by N-scope and global scope - ScopedModifications = group_scoped(Typed, Msgs), + ScopedModifications = group_scoped(Typed, Msgs, Opts), ?event_debug(parsing, {scoped_modifications, ScopedModifications}), % 5. Generate the list of messages (plus-notation, device, typed keys). Result = build_messages(Msgs, ScopedModifications, Opts), @@ -168,8 +163,6 @@ from(RawMsg, Opts) -> %% @doc Parse the relative reference into path, query, and fragment. from_path(RelativeRef) -> - %?event_debug(parsing, {raw_relative_ref, RawRelativeRef}), - %RelativeRef = hb_escape:decode(RawRelativeRef), Decoded = decode_string(RelativeRef), ?event_debug(parsing, {parsed_relative_ref, Decoded}), {Path, QKVList} = @@ -218,49 +211,56 @@ path_parts(Sep, PathBin) when is_binary(PathBin) -> %% @doc Extract all of the parts from the binary, given (a list of) separators. all_path_parts(_Sep, <<>>) -> []; -all_path_parts(Sep, Bin) -> - hb_util:split_depth_string_aware(Sep, Bin). +all_path_parts(Sep, Bin) -> hb_util:split_depth_string_aware(Sep, Bin). %% @doc Extract the characters from the binary until a separator is found. %% The first argument of the function is an explicit separator character, or %% a list of separator characters. Returns a tuple with the separator, the %% accumulated characters, and the rest of the binary. -part(Sep, Bin) when not is_list(Sep) -> - part([Sep], Bin); -part(Seps, Bin) -> - hb_util:split_depth_string_aware_single(Seps, Bin). +part(Sep, Bin) when not is_list(Sep) -> part([Sep], Bin); +part(Seps, Bin) -> hb_util:split_depth_string_aware_single(Seps, Bin). %% @doc Step 3: Apply types to values and remove specifiers. apply_types(Msg, Opts) -> - hb_maps:fold( - fun(Key, Val, Acc) -> - {_, K, V} = maybe_typed(Key, Val, Opts), - hb_maps:put(K, V, Acc, Opts) - end, - #{}, + hb_ao:set( Msg, + hb_maps:fold( + fun(Key, Val, Acc) -> + {_, K, V} = maybe_typed(Key, Val, Opts), + Acc#{ K => V } + end, + #{}, + Msg, + Opts + ), Opts ). %% @doc Step 4: Group headers/query by N-scope. %% `N.Key' => applies to Nth step. Otherwise => `global' -group_scoped(Map, Msgs) -> +group_scoped(Map, Msgs, Opts) -> {NScope, Global} = hb_maps:fold( fun(KeyBin, Val, {Ns, Gs}) -> case parse_scope(KeyBin) of {OkN, RealKey} when OkN > 0 -> - Curr = hb_maps:get(OkN, Ns, #{}), - Ns2 = hb_maps:put(OkN, hb_maps:put(RealKey, Val, Curr), Ns), + Curr = hb_maps:get(OkN, Ns, #{}, Opts), + Ns2 = + hb_maps:put( + OkN, + hb_maps:put(RealKey, Val, Curr, Opts), + Ns, + Opts + ), {Ns2, Gs}; - global -> {Ns, hb_maps:put(KeyBin, Val, Gs)} + global -> {Ns, hb_maps:put(KeyBin, Val, Gs, Opts)} end end, {#{}, #{}}, Map ), [ - hb_maps:merge(Global, hb_maps:get(N, NScope, #{})) + hb_ao:set(Global, hb_maps:get(N, NScope, #{}, Opts), Opts) || N <- lists:seq(1, length(Msgs)) ]. @@ -269,7 +269,7 @@ group_scoped(Map, Msgs) -> parse_scope(KeyBin) -> case binary:split(KeyBin, <<".">>, [global]) of [Front, Remainder] -> - case catch erlang:binary_to_integer(Front) of + case catch hb_util:int(Front) of NInt when is_integer(NInt) -> {NInt + 1, Remainder}; _ -> throw({error, invalid_scope, KeyBin}) end; @@ -281,46 +281,29 @@ build_messages(Msgs, ScopedModifications, Opts) -> do_build(1, Msgs, ScopedModifications, Opts). do_build(_, [], _, _) -> []; -do_build(I, [{as, DevID, RawMsg} | Rest], ScopedKeys, Opts) when is_map(RawMsg) -> +do_build(I, [{as, DevID, Direct} | Rest], ScopedKeys, Opts) when is_map(Direct) -> % We are processing an `as' message. If the path is empty, we need to % remove it from the message and the additional message, such that AO-Core % returns only the message with the device specifier changed. If the message % does have a path, AO-Core will subresolve it. - RawAdditional = lists:nth(I, ScopedKeys), - {Msg, Additional} = - case hb_maps:get(<<"path">>, RawMsg, <<"">>, Opts) of - ID when ?IS_ID(ID) -> - % When we have an ID, we do not merge the globally scoped elements. - { - RawMsg, - #{} - }; - <<"">> -> - % When we have an empty path, we remove the path from both - % messages. AO-Core will then simply set the device specifier - % and not execute a subresolve. - { - hb_ao:set(RawMsg, <<"path">>, unset, Opts), - hb_ao:set(RawAdditional, <<"path">>, unset, Opts) - }; - _BasePath -> - % When we have a non-empty path, we merge the messages in - % totality. The path-part's path will be subresolved. - {RawMsg, RawAdditional} - end, - Merged = hb_maps:merge(Additional, Msg, Opts), - StepMsg = hb_message:convert( - Merged, - <<"structured@1.0">>, - Opts#{ <<"topic">> => ao_internal } - ), + StepMsg = + hb_message:convert( + lists:nth(I, ScopedKeys), + <<"structured@1.0">>, + Opts#{ <<"topic">> => ao_internal } + ), ?event_debug(parsing, {build_messages, {base, Msg}, {additional, Additional}}), - [{as, DevID, StepMsg} | do_build(I + 1, Rest, ScopedKeys, Opts)]; + [ + #{ <<"device">> => DevID }, + Direct#{ <<"...">> => StepMsg } + | + do_build(I + 1, Rest, ScopedKeys, Opts) + ]; do_build(I, [Msg | Rest], ScopedKeys, Opts) when not is_map(Msg) -> [Msg | do_build(I + 1, Rest, ScopedKeys, Opts)]; do_build(I, [Msg | Rest], ScopedKeys, Opts) -> Additional = lists:nth(I, ScopedKeys), - Merged = hb_maps:merge(Additional, Msg, Opts), + Merged = hb_ao:set(Additional, Msg, Opts), StepMsg = hb_message:convert( Merged, <<"structured@1.0">>, @@ -443,15 +426,6 @@ maybe_typed(Key, Value, Opts) -> end end. -%% @doc Join a list of items with a separator, or return the first item if there -%% is only one item. If there are no items, return an empty binary. -maybe_join(Items, Sep) -> - case length(Items) of - 0 -> <<>>; - 1 -> hd(Items); - _ -> iolist_to_binary(lists:join(Sep, Items)) - end. - %%% Tests parse_explicit_message_test() -> diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl new file mode 100644 index 000000000..5f6746c91 --- /dev/null +++ b/src/core/resolver/hb_types.erl @@ -0,0 +1,715 @@ +%%% @doc Extract device specs and vary AO-Core execution inputs. +-module(hb_types). +-export([extract/2, vary/7]). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Apply the resolved function's base/request schemas to one execution. +vary(Device, Key, Func, AddKey, Base, Req, Opts) -> + case function_schema(Device, Func, Key, Opts) of + undefined -> + {ok, Base, Req, none}; + Schema -> + {BaseSchema, ReqSchema, ReturnSchema} = + execution_schemas(Schema, AddKey), + ReqWithKey = + case AddKey of + false -> Req; + _ -> Req#{ <<"path">> => Key } + end, + {ok, + apply_schema(implicit_base(BaseSchema), Base, Opts), + apply_schema(implicit_request(ReqSchema), ReqWithKey, Opts), + overlay(ReturnSchema)} + end. + +%% @doc Extract a device module's function schemas. This intentionally has no +%% cache; callers can add one once the algorithm is settled. +extract(Device, _Opts) when is_map(Device) -> + {error, {unsupported_device_type, Device}}; +extract(Module, _Opts) when is_atom(Module) -> + case code:ensure_loaded(Module) of + {module, Module} -> do_extract(Module); + {error, Reason} -> {error, {module_not_loaded, Module, Reason}} + end; +extract(Device, Opts) when is_binary(Device) -> + case hb_device_load:reference(Device, Opts) of + {ok, Module} -> extract(Module, Opts); + Error -> Error + end; +extract(Device, _Opts) -> + {error, {unsupported_device_type, Device}}. + +function_schema(Device, Func, Key, Opts) -> + case schema_from_device(Device, Func, Key, Opts) of + undefined -> schema_from_fun_module(Device, Func, Key, Opts); + Schema -> Schema + end. + +schema_from_device(Device, Func, Key, Opts) -> + case extract(Device, Opts) of + {ok, #{ <<"keys">> := Schemas }} -> + select_schema(Func, Key, Schemas); + {error, _Reason} -> + undefined + end. + +schema_from_fun_module(Device, Func, Key, Opts) when is_atom(Device) -> + case erlang:fun_info(Func, module) of + {module, Device} -> + undefined; + {module, Module} -> + schema_from_device(Module, Func, Key, Opts); + _ -> + undefined + end; +schema_from_fun_module(_Device, _Func, _Key, _Opts) -> + undefined. + +select_schema(Func, _Key, Schemas) -> + case {erlang:fun_info(Func, name), erlang:fun_info(Func, arity)} of + {{name, Name}, {arity, Arity}} -> + maps:get({normalize_name(Name), Arity}, Schemas, undefined); + _ -> + undefined + end. + +execution_schemas(Schema, AddKey) -> + Args = maps:get(<<"args">>, Schema, []), + Offset = + case AddKey of + false -> 0; + _ -> 1 + end, + { + nth_or(1 + Offset, Args, any_type()), + nth_or(2 + Offset, Args, any_type()), + maps:get(<<"return">>, Schema, any_type()) + }. + +nth_or(N, List, _Default) when length(List) >= N -> + lists:nth(N, List); +nth_or(_N, _List, Default) -> + Default. + +implicit_base(Schema) -> + implicit_key(top_level_schema(Schema), <<"device">>, optional). + +implicit_request(Schema) -> + implicit_key(top_level_schema(Schema), <<"path">>, required). + +top_level_schema(#{ <<"kind">> := <<"empty">> }) -> + message_type({#{}, none}); +top_level_schema(Schema) -> + Schema. + +implicit_key(Schema = #{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, + Key, + Presence) -> + case maps:is_key(Key, Keys) of + true -> + Schema; + false -> + Schema#{ + <<"keys">> => + Keys#{ + Key => + #{ + <<"presence">> => Presence, + <<"type">> => any_type() + } + } + } + end; +implicit_key(Schema, _Key, _Presence) -> + Schema. + +do_extract(Module) -> + case module_beam(Module) of + unavailable -> + {error, {abstract_code_unavailable, Module, unavailable}}; + Beam -> + case beam_lib:chunks(Beam, [abstract_code]) of + {ok, {_, [{abstract_code, {_, Forms}}]}} -> + TypeEnv = build_type_env(Forms), + {ok, + #{ + <<"keys">> => + lists:foldl( + fun(Spec, Acc) -> + maps:merge( + Acc, + spec_schemas(Spec, TypeEnv) + ) + end, + #{}, + [Attr || Attr = {attribute, _, spec, _} <- Forms] + ) + }}; + Error -> + {error, {abstract_code_unavailable, Module, Error}} + end + end. + +module_beam(Module) -> + case code:get_object_code(Module) of + {Module, Beam, _Filename} -> + Beam; + _ -> + case code:which(Module) of + Path when is_list(Path); is_binary(Path) -> Path; + _ -> unavailable + end + end. + +build_type_env(Forms) -> + lists:foldl( + fun + ({attribute, _, Tag, {Name, Ast, Vars}}, Acc) + when Tag =:= type; Tag =:= opaque -> + Acc#{ + Name => + #{ + vars => [var_name(Var) || Var <- Vars], + ast => Ast + } + }; + (_, Acc) -> + Acc + end, + #{}, + Forms + ). + +spec_schemas({attribute, _, spec, {{Name, Arity}, [Spec]}}, TypeEnv) -> + #{ + {normalize_name(Name), Arity} => fun_schema(Spec, TypeEnv) + }; +spec_schemas({attribute, _, spec, {{Name, Arity}, [Spec | _Rest]}}, TypeEnv) -> + #{ + {normalize_name(Name), Arity} => fun_schema(Spec, TypeEnv) + }; +spec_schemas({attribute, _, spec, {{Name, Arity}, Spec}}, TypeEnv) -> + maps:from_list( + [{{normalize_name(Name), Arity}, fun_schema(Spec, TypeEnv)}] + ); +spec_schemas(_Spec, _TypeEnv) -> + #{}. + +fun_schema(Spec, TypeEnv) -> + {Args, Return} = parse_fun_spec(Spec, TypeEnv), + #{ + <<"args">> => Args, + <<"return">> => Return + }. + +parse_fun_spec({type, _, bounded_fun, [FunSpec, _Constraints]}, TypeEnv) -> + parse_fun_spec(FunSpec, TypeEnv); +parse_fun_spec({type, _, 'fun', [{type, _, product, Args}, Return]}, TypeEnv) -> + { + [parse_type(Arg, TypeEnv, #{}, []) || Arg <- Args], + parse_type(Return, TypeEnv, #{}, []) + }; +parse_fun_spec(Other, _TypeEnv) -> + {[unknown_type(Other)], any_type()}. + +parse_type({ann_type, _, [_Var, Type]}, TypeEnv, VarEnv, Seen) -> + parse_type(Type, TypeEnv, VarEnv, Seen); +parse_type({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + empty_type(); +parse_type({var, _, Name}, TypeEnv, VarEnv, Seen) -> + case maps:get(Name, VarEnv, undefined) of + undefined -> variable_type(Name); + Bound -> parse_type(Bound, TypeEnv, VarEnv, Seen) + end; +parse_type({user_type, _, Name, Args}, TypeEnv, VarEnv, Seen) -> + case lists:member(Name, Seen) of + true -> + alias_type(Name); + false -> + case maps:get(Name, TypeEnv, undefined) of + undefined -> + alias_type(Name); + #{ vars := Vars, ast := Ast } -> + parse_type( + Ast, + TypeEnv, + maps:merge(VarEnv, maps:from_list(lists:zip(Vars, Args))), + [Name | Seen] + ) + end + end; +parse_type({remote_type, _, [{atom, _, Mod}, {atom, _, Name}, Args]}, + TypeEnv, + VarEnv, + Seen) -> + #{ + <<"kind">> => <<"remote">>, + <<"module">> => normalize_name(Mod), + <<"name">> => normalize_name(Name), + <<"args">> => [parse_type(Arg, TypeEnv, VarEnv, Seen) || Arg <- Args] + }; +parse_type({type, _, map, any}, _TypeEnv, _VarEnv, _Seen) -> + any_type(); +parse_type({type, _, map, Fields}, TypeEnv, VarEnv, Seen) -> + message_type(parse_fields(Fields, TypeEnv, VarEnv, Seen)); +parse_type({type, _, ListType, [Item]}, TypeEnv, VarEnv, Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ <<"kind">> => <<"list">>, <<"item">> => parse_type(Item, TypeEnv, VarEnv, Seen) }; +parse_type({type, _, ListType, []}, _TypeEnv, _VarEnv, _Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ <<"kind">> => <<"list">>, <<"item">> => any_type() }; +parse_type({type, _, tuple, Items}, TypeEnv, VarEnv, Seen) -> + #{ <<"kind">> => <<"tuple">>, <<"items">> => + [parse_type(Item, TypeEnv, VarEnv, Seen) || Item <- Items] }; +parse_type({type, _, union, Members}, TypeEnv, VarEnv, Seen) -> + #{ <<"kind">> => <<"union">>, <<"members">> => + [parse_type(Member, TypeEnv, VarEnv, Seen) || Member <- Members] }; +parse_type({type, _, range, [Min, Max]}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"range">>, + <<"min">> => literal_value(parse_type(Min, TypeEnv, VarEnv, Seen)), + <<"max">> => literal_value(parse_type(Max, TypeEnv, VarEnv, Seen)) + }; +parse_type({type, _, integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"integer">>); +parse_type({type, _, non_neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"non-neg-integer">>); +parse_type({type, _, pos_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pos-integer">>); +parse_type({type, _, neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"neg-integer">>); +parse_type({type, _, float, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"float">>); +parse_type({type, _, number, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"number">>); +parse_type({type, _, binary, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"binary">>); +parse_type({type, _, bitstring, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"bitstring">>); +parse_type({type, _, boolean, []}, _TypeEnv, _VarEnv, _Seen) -> boolean_type(); +parse_type({type, _, atom, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"atom">>); +parse_type({type, _, pid, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pid">>); +parse_type({type, _, any, []}, _TypeEnv, _VarEnv, _Seen) -> any_type(); +parse_type({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(Atom)); +parse_type({integer, _, Int}, _TypeEnv, _VarEnv, _Seen) -> literal_type(Int); +parse_type({char, _, Char}, _TypeEnv, _VarEnv, _Seen) -> literal_type(<>); +parse_type({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(String)); +parse_type({nil, _}, _TypeEnv, _VarEnv, _Seen) -> literal_type([]); +parse_type(Other, _TypeEnv, _VarEnv, _Seen) -> + unknown_type(Other). + +parse_fields(Fields, TypeEnv, VarEnv, Seen) -> + lists:foldl( + fun({type, _, Assoc, [KeyAst, ValueAst]}, {Keys, Wildcard}) -> + Field = + #{ + <<"presence">> => field_presence(Assoc), + <<"type">> => parse_type(ValueAst, TypeEnv, VarEnv, Seen) + }, + case key_name(KeyAst, TypeEnv, VarEnv, Seen) of + <<"_">> -> {Keys, Field}; + Key -> {Keys#{ Key => Field }, Wildcard} + end + end, + {#{}, none}, + Fields + ). + +field_presence(map_field_exact) -> required; +field_presence(map_field_assoc) -> optional; +field_presence(Other) -> normalize_name(Other). + +key_name({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> + normalize_name(Atom); +key_name({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> + hb_util:bin(String); +key_name({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + <<"_">>; +key_name(Other, TypeEnv, VarEnv, Seen) -> + case parse_type(Other, TypeEnv, VarEnv, Seen) of + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } when is_binary(Value) -> + Value; + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } -> + hb_util:bin(io_lib:format("~tp", [Value])); + _ -> + hb_util:bin(io_lib:format("~tp", [Other])) + end. + +apply_schema(#{ <<"kind">> := <<"any">> }, Value, _Opts) -> + Value; +apply_schema(#{ <<"kind">> := <<"empty">> }, Value, _Opts) -> + Value; +apply_schema(Schema = #{ <<"kind">> := <<"message">> }, Value, Opts) + when not is_map(Value) -> + apply_schema(Schema, hb_cache:ensure_loaded(Value, Opts), Opts); +apply_schema( + #{ <<"kind">> := <<"message">>, <<"keys">> := Keys, <<"wildcard">> := Wildcard }, + Message, + Opts +) when is_map(Message) -> + Explicit = explicit_keys(Keys, Message, Opts), + maps:merge(wildcard_keys(Wildcard, Keys, Message, Opts), Explicit); +apply_schema(Type, Value, Opts) -> + case check_type(Type, Value) of + true -> + Value; + false -> + case coerce_type(Type, Value, Opts) of + error -> throw({invalid_type, Type, Value}); + Coerced -> + case check_type(Type, Coerced) of + true -> Coerced; + false -> throw({invalid_type, Type, Value}) + end + end + end. + +explicit_keys(Keys, Message, Opts) -> + maps:fold( + fun(Key, #{ <<"presence">> := Presence, <<"type">> := Type }, Acc) -> + case hb_message:find_active(Key, Message, Opts) of + {ok, Value} -> + Acc#{ Key => apply_schema(Type, Value, Opts) }; + error when Presence =:= required -> + throw({required_key_missing, Key}); + error -> + Acc + end + end, + #{}, + Keys + ). + +wildcard_keys(none, _Keys, _Message, _Opts) -> + #{}; +wildcard_keys(#{ <<"presence">> := optional }, Keys, Message, _Opts) -> + maps:without(maps:keys(Keys), Message); +wildcard_keys(#{ <<"presence">> := required, <<"type">> := Type }, Keys, Message, Opts) -> + maps:map( + fun(_Key, Value) -> + apply_schema(Type, hb_cache:ensure_all_loaded(Value, Opts), Opts) + end, + maps:without(maps:keys(Keys), hb_message:active(Message, Opts)) + ). + +overlay(ReturnSchema) -> + case overlay_type(ReturnSchema) of + base -> base; + request -> request; + _ -> none + end. + +overlay_type(#{ <<"kind">> := <<"message">>, <<"keys">> := Keys }) -> + overlay_marker( + maps:get( + <<"type">>, + maps:get(<<"...">>, Keys, #{}), + #{} + ) + ); +overlay_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }) -> + first_overlay(Items); +overlay_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }) -> + first_overlay(Members); +overlay_type(_) -> + none. + +first_overlay([]) -> + none; +first_overlay([Schema | Rest]) -> + case overlay_type(Schema) of + none -> first_overlay(Rest); + Overlay -> Overlay + end. + +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"request">> }) -> request; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"request">> }) -> request; +overlay_marker(_) -> none. + +coerce_type(_, undefined, _Opts) -> error; +coerce_type(#{ <<"kind">> := <<"integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"pos-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"float">> }, Value, _Opts) -> + try_coerce(fun hb_util:float/1, Value); +coerce_type(#{ <<"kind">> := <<"number">> }, Value, _Opts) -> + coerce_with([fun hb_util:int/1, fun hb_util:float/1], Value); +coerce_type(#{ <<"kind">> := <<"binary">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"bitstring">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"boolean">> }, Value, _Opts) -> + case lists:member(Value, [true, false, 1, 0, <<"true">>, <<"false">>, <<"1">>, <<"0">>]) of + true -> try_coerce(fun hb_util:bool/1, Value); + false -> error + end; +coerce_type(#{ <<"kind">> := <<"atom">> }, Value, _Opts) -> + try_coerce(fun hb_util:atom/1, Value); +coerce_type(#{ <<"kind">> := <<"message">> }, Value, _Opts) -> + try_coerce(fun hb_util:map/1, Value); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_tuple(Value) -> + coerce_type(#{ <<"kind">> => <<"tuple">>, <<"items">> => Items }, tuple_to_list(Value), Opts); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_list(Value) -> + case length(Value) =:= length(Items) of + true -> coerce_tuple(Items, Value, Opts); + false -> error + end; +coerce_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value, Opts) -> + case try_coerce(fun hb_util:list/1, Value) of + error -> error; + Coerced -> coerce_list(ItemType, Coerced, Opts) + end; +coerce_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value, Opts) -> + coerce_union(Members, Value, Opts); +coerce_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value, _Opts) -> + coerce_literal(Expected, Value); +coerce_type(#{ <<"kind">> := <<"range">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(_, _, _Opts) -> + error. + +coerce_tuple(Items, Values, Opts) -> + case coerce_sequence(lists:zip(Items, Values), Opts) of + error -> error; + Coerced -> list_to_tuple(Coerced) + end. + +coerce_list(ItemType, Value, Opts) when is_list(Value) -> + coerce_sequence([{ItemType, Item} || Item <- Value], Opts); +coerce_list(_ItemType, _Value, _Opts) -> + error. + +coerce_sequence([], _Opts) -> + []; +coerce_sequence([{Type, Value} | Rest], Opts) -> + case coerce_type(Type, Value, Opts) of + error -> + error; + Coerced -> + case coerce_sequence(Rest, Opts) of + error -> error; + CoercedRest -> [Coerced | CoercedRest] + end + end. + +coerce_union([], _Value, _Opts) -> + error; +coerce_union([Member | Rest], Value, Opts) -> + case coerce_type(Member, Value, Opts) of + error -> coerce_union(Rest, Value, Opts); + Coerced -> Coerced + end. + +coerce_literal(Expected, Value) when is_integer(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:int/1, Value)); +coerce_literal(Expected, Value) when is_float(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:float/1, Value)); +coerce_literal(Expected, Value) when is_binary(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:bin/1, Value)); +coerce_literal(Expected, Value) when is_atom(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:atom/1, Value)); +coerce_literal(Expected, Value) when is_list(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:list/1, Value)); +coerce_literal(Expected, Expected) -> + Expected; +coerce_literal(_Expected, _Value) -> + error. + +coerce_exact(Expected, Expected) -> + Expected; +coerce_exact(_Expected, _Value) -> + error. + +try_coerce(Fun, Value) -> + try Fun(Value) of + Coerced -> Coerced + catch + _:_ -> error + end. + +coerce_with([], _Value) -> + error; +coerce_with([Fun | Rest], Value) -> + case try_coerce(Fun, Value) of + error -> coerce_with(Rest, Value); + Coerced -> Coerced + end. + +check_type(#{ <<"kind">> := <<"integer">> }, Value) -> is_integer(Value); +check_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value) -> is_integer(Value) andalso Value >= 0; +check_type(#{ <<"kind">> := <<"pos-integer">> }, Value) -> is_integer(Value) andalso Value > 0; +check_type(#{ <<"kind">> := <<"neg-integer">> }, Value) -> is_integer(Value) andalso Value < 0; +check_type(#{ <<"kind">> := <<"float">> }, Value) -> is_float(Value); +check_type(#{ <<"kind">> := <<"number">> }, Value) -> is_number(Value); +check_type(#{ <<"kind">> := <<"binary">> }, Value) -> is_binary(Value); +check_type(#{ <<"kind">> := <<"bitstring">> }, Value) -> is_bitstring(Value); +check_type(#{ <<"kind">> := <<"boolean">> }, Value) -> is_boolean(Value); +check_type(#{ <<"kind">> := <<"atom">> }, Value) -> is_atom(Value); +check_type(#{ <<"kind">> := <<"pid">> }, Value) -> is_pid(Value); +check_type(#{ <<"kind">> := <<"message">> }, Value) -> is_map(Value); +check_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) -> + is_tuple(Value) + andalso tuple_size(Value) =:= length(Items) + andalso lists:all( + fun({Index, ItemType}) -> check_type(ItemType, element(Index, Value)) end, + lists:zip(lists:seq(1, length(Items)), Items) + ); +check_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value) -> + is_list(Value) andalso lists:all(fun(Item) -> check_type(ItemType, Item) end, Value); +check_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value) -> + lists:any(fun(Member) -> check_type(Member, Value) end, Members); +check_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value) -> + Value =:= Expected; +check_type(#{ <<"kind">> := <<"range">>, <<"min">> := Min, <<"max">> := Max }, Value) -> + is_integer(Value) andalso Value >= Min andalso Value =< Max; +check_type(#{ <<"kind">> := <<"remote">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"alias">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"variable">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"unknown">> }, _Value) -> true; +check_type(_, _Value) -> true. + +normalize_name('_') -> <<"_">>; +normalize_name(Name) when is_atom(Name) -> hb_util:atom_to_dashed_binary(Name); +normalize_name(Name) -> hb_util:bin(Name). + +literal_value(#{ <<"kind">> := <<"literal">>, <<"value">> := Value }) -> Value; +literal_value(_) -> undefined. + +var_name({var, _, Name}) -> Name; +var_name(Name) -> Name. + +message_type({Keys, Wildcard}) -> + #{ + <<"kind">> => <<"message">>, + <<"keys">> => Keys, + <<"wildcard">> => Wildcard + }. + +any_type() -> #{ <<"kind">> => <<"any">> }. +empty_type() -> #{ <<"kind">> => <<"empty">> }. +scalar_type(Name) -> #{ <<"kind">> => Name }. +literal_type(Value) -> #{ <<"kind">> => <<"literal">>, <<"value">> => Value }. +alias_type(Name) -> #{ <<"kind">> => <<"alias">>, <<"name">> => normalize_name(Name) }. +variable_type(Name) -> #{ <<"kind">> => <<"variable">>, <<"name">> => normalize_name(Name) }. +unknown_type(Other) -> + #{ <<"kind">> => <<"unknown">>, <<"ast">> => hb_util:bin(io_lib:format("~tp", [Other])) }. +boolean_type() -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => [literal_type(true), literal_type(false)] + }. + +%%% Tests + +empty_projection_test() -> + ?assertEqual( + #{ <<"device">> => <<"test@1.0">> }, + apply_schema( + implicit_base(empty_type()), + #{ <<"device">> => <<"test@1.0">>, <<"extra">> => <<"drop">> }, + #{} + ) + ). + +map_wildcards_test() -> + Lazy = + parse_type( + {type, 1, map, + [ + {type, 1, map_field_exact, [{atom, 1, a}, {var, 1, '_'}]}, + {type, 1, map_field_assoc, [{var, 1, '_'}, {var, 1, '_'}]} + ]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"keys">> := #{ <<"a">> := _ }, + <<"wildcard">> := #{ <<"presence">> := optional } + }, + Lazy + ), + Force = + parse_type( + {type, 1, map, + [ + {type, 1, map_field_exact, [{var, 1, '_'}, {var, 1, '_'}]} + ]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"wildcard">> := #{ <<"presence">> := required } + }, + Force + ). + +default_handler_uses_resolved_function_schema_test() -> + Func = fun default_schema_fun/4, + ?assertEqual( + default_schema, + select_schema( + Func, + <<"requested-key">>, + #{ + {<<"default-schema-fun">>, 4} => default_schema, + {<<"requested-key">>, 3} => requested_key_schema + } + ) + ), + ?assertEqual( + undefined, + select_schema( + Func, + <<"requested-key">>, + #{{<<"requested-key">>, 3} => requested_key_schema} + ) + ). + +default_schema_fun(_Key, _Base, _Req, _Opts) -> + {ok, unused}. + +extension_projection_test() -> + Schema = + message_type( + { + #{ + <<"a">> => #{ <<"presence">> => required, <<"type">> => empty_type() }, + <<"c">> => #{ <<"presence">> => optional, <<"type">> => empty_type() } + }, + none + } + ), + Msg = + #{ + <<"b">> => 2, + <<"...">> => + #{ + <<"a">> => 1, + <<"b">> => 1 + } + }, + ?assertEqual(#{ <<"a">> => 1 }, apply_schema(Schema, Msg, #{})). + +extension_wildcard_carry_test() -> + Schema = + message_type( + { + #{ + <<"a">> => #{ <<"presence">> => optional, <<"type">> => empty_type() } + }, + #{ <<"presence">> => optional, <<"type">> => empty_type() } + } + ), + Parent = #{ <<"a">> => 1, <<"b">> => 1, <<"c">> => 1 }, + Msg = #{ <<"b">> => 2, <<"...">> => Parent }, + ?assertEqual( + #{ <<"a">> => 1, <<"b">> => 2, <<"...">> => Parent }, + apply_schema(Schema, Msg, #{}) + ). From d30997878929d30de1b56e13eab560920cbee210 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 26 Jun 2026 13:26:45 -0400 Subject: [PATCH 02/35] wip: `~message@1.0` --- src/preloaded/message/dev_message.erl | 325 +++++--------------------- 1 file changed, 55 insertions(+), 270 deletions(-) diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 90a9a0ff4..83c50077f 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -5,12 +5,10 @@ %%% `set', `remove', `get', and `verify'. Their function comments describe the %%% behaviour of the device when these keys are set. -module(dev_message). -%%% Base AO-Core reserved keys: --export([info/0, keys/1, keys/2]). --export([set/3, set_path/3, remove/3, get/3, get/4]). +%%% Base AO-Core state manipulation functions. +-export([info/0, keys/3, set/3, id/3]). %%% Commitment-specific keys: --export([id/1, id/2, id/3]). --export([commit/3, committed/3, committers/1, committers/2, committers/3, verify/3]). +-export([commit/3, committed/3, committers/3, verify/3]). %%% Non-protocol enforced keys: -export([index/3]). -include_lib("eunit/include/eunit.hrl"). @@ -18,22 +16,10 @@ -define(DEFAULT_ID_DEVICE, <<"httpsig@1.0">>). -define(DEFAULT_ATT_DEVICE, <<"httpsig@1.0">>). -%% The list of keys that are exported by this device. --define(DEVICE_KEYS, [ - <<"id">>, - <<"commitments">>, - <<"committers">>, - <<"keys">>, - <<"path">>, - <<"set">>, - <<"remove">>, - <<"verify">> -]). - %% @doc Return the info for the identity device. info() -> #{ - default => fun dev_message:get/4 + default => fun default_accessor/4 }. %% @doc Generate an index page for a message, in the event that the `body' and @@ -82,8 +68,6 @@ index(Msg, Req, Opts) -> %% Note: This function _does not_ use AO-Core's `get/3' function, as it %% would require significant computation. We may want to change this %% if/when non-map message structures are created. -id(Base) -> id(Base, #{}). -id(Base, Req) -> id(Base, Req, #{}). id(Base, _, NodeOpts) when is_binary(Base) -> % Return the hashpath of the message in native format, to match the native % format of the message ID return. @@ -116,7 +100,11 @@ id(RawBase, Req, NodeOpts) -> [] -> % If there are no commitments, we must (re)calculate the ID. ?event_debug(debug_id, regenerating_id), - calculate_id(hb_maps:without([<<"commitments">>], Base), Req, IDOpts); + calculate_id( + hb_maps:without([<<"commitments">>], Base, IDOpts), + Req, + IDOpts + ); IDs -> % Accumulate the relevant IDs into a single value. This is performed % by module arithmetic of each of the IDs. The effect of this is that: @@ -225,8 +213,7 @@ id_device(_, _) -> {ok, ?DEFAULT_ID_DEVICE}. %% @doc Return the committers of a message that are present in the given request. -committers(Base) -> committers(Base, #{}). -committers(Base, Req) -> committers(Base, Req, #{}). +-spec committers(#{ commitments => _ }, _, _) -> {ok, [binary()]}. committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> {ok, hb_maps:values( @@ -611,239 +598,42 @@ commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) -> %% @doc Deep merge keys in a message. Takes a map of key-value pairs and sets %% them in the message, overwriting any existing values. -set(Base, NewValuesMsg, Opts) -> - OriginalPriv = hb_private:from_message(Base), - % Filter keys that are in the default device (this one). - {ok, NewValuesKeys} = keys(NewValuesMsg, Opts), - KeysToSet = - lists:filter( - fun(Key) -> - not lists:member(Key, ?DEVICE_KEYS ++ [<<"set-mode">>]) andalso - (hb_maps:get(Key, NewValuesMsg, undefined, Opts) =/= undefined) - end, - NewValuesKeys - ), - % Find keys in the message that are already set (case-insensitive), and - % note them for removal. - _ConflictingKeys = - lists:filter( - fun(Key) -> lists:member(Key, KeysToSet) end, - hb_maps:keys(Base, Opts) - ), - UnsetKeys = - lists:filter( - fun(Key) -> - case hb_maps:get(Key, NewValuesMsg, not_found, Opts) of - unset -> true; - _ -> false - end - end, - hb_maps:keys(Base, Opts) - ), - % Base message with keys-to-unset removed - BaseValues = hb_maps:without(UnsetKeys, Base, Opts), - ?event_debug(debug_message_set, - {performing_set, - {conflicting_keys, _ConflictingKeys}, - {keys_to_unset, UnsetKeys}, - {new_values, NewValuesMsg}, - {original_message, Base} - } - ), - % Create the map of new values - NewValues = hb_maps:from_list( - lists:filtermap( - fun(Key) -> - case hb_maps:get(Key, NewValuesMsg, undefined, Opts) of - undefined -> false; - unset -> false; - Value -> {true, {Key, Value}} - end - end, - KeysToSet - ) - ), - % Calculate if the keys to be set conflict with any committed keys. - {ok, CommittedKeys} = - committed( - Base, - #{ - <<"committers">> => <<"all">> - }, - Opts - ), - ?event_debug(message_set, - {setting, - {committed_keys, CommittedKeys}, - {keys_to_set, KeysToSet}, - {message, Base} - } - ), - OverwrittenCommittedKeys = - lists:filtermap( - fun(Key) -> - NormKey = hb_ao:normalize_key(Key), - ?event_debug({checking_committed_key, {key, Key}, {norm_key, NormKey}}), - Res = case lists:member(NormKey, KeysToSet) of - true -> {true, NormKey}; - false -> false - end, - Res - end, - CommittedKeys - ), - ?event_debug({setting, {overwritten_committed_keys, OverwrittenCommittedKeys}}), - % Combine with deep merge or if `set-mode` is `explicit' then just merge. - Merged = - hb_private:set_priv( - case maps:get(<<"set-mode">>, NewValuesMsg, <<"deep">>) of - <<"explicit">> -> maps:merge(BaseValues, NewValues); - _ -> do_deep_merge(BaseValues, NewValues, Opts) - end, - OriginalPriv - ), - case OverwrittenCommittedKeys of - [] -> - ?event_debug(message_set, {no_overwritten_committed_keys, {merged, Merged}}), - {ok, Merged}; - _ -> - % We did overwrite some keys, but do their values match the original? - % If not, we must remove the commitments. - ChangedBaseKeys = hb_maps:with(OverwrittenCommittedKeys, Base, Opts), - ChangedMergedKeys = hb_maps:with(OverwrittenCommittedKeys, Merged, Opts), - Matches = hb_message:match(ChangedMergedKeys, ChangedBaseKeys, strict, Opts), - case Matches of - true -> - ?event_debug(message_set, {set_keys_matched, {merged, Merged}}), - {ok, Merged}; - % {error, {Details, {trace, Stacktrace}}} -> - % erlang:raise(error, Details, Stacktrace); - % {mismatch, Type, Path, Val1, Val2} -> - % ?event( - % set_conflict, - % {set_conflict_removing_commitments, - % {merged, Merged}, - % {mismatch, Type}, - % {path, Path}, - % {expected, Val1}, - % {received, Val2} - % } - % ), - _ -> - {ok, hb_maps:without([<<"commitments">>], Merged, Opts)} - end - end. - -%% @doc Deep merge keys in a message, utilizing the set device of any child -%% keys that are themselves messages. -do_deep_merge(BaseValues, NewValues, Opts) -> - {WithNestedMerges, StillToDeepMerge} = - maps:fold( - fun(Key, NewValue, {Acc, ToDeepMerge}) - when is_map(NewValue) - andalso is_map(map_get(Key, Acc)) -> - BaseValue = map_get(Key, Acc), - NewValueSet = NewValue#{ <<"path">> => <<"set">> }, - { - Acc#{ - Key => - hb_util:ok( - hb_ao:resolve( - BaseValue, - NewValueSet, - Opts - ), - Opts - ) - }, - ToDeepMerge - }; - (Key, NewValue, {Acc, ToDeepMerge}) - when is_map(NewValue) - andalso ?IS_LINK(map_get(Key, Acc)) -> - LoadedBaseValue = hb_cache:ensure_loaded(map_get(Key, Acc), Opts), - case is_map(LoadedBaseValue) of - true -> - NewValueSet = NewValue#{ <<"path">> => <<"set">> }, - { - Acc#{ - Key => - hb_util:ok( - hb_ao:resolve( - LoadedBaseValue, - NewValueSet, - Opts - ), - Opts - ) - }, - ToDeepMerge - }; - false -> - {Acc, [Key | ToDeepMerge]} +set(Base, Req = #{ <<"set">> := <<"deep">> }, Opts) -> + NewValues = + maps:map( + fun(Key, Nested) when is_map(Nested) or ?IS_LINK(Nested) -> + case hb_maps:find(Key, Base, Opts) of + {ok, NestedBase} -> + % We use `hb_ao:deep_set` rather than just recursing + % such that the correct device is used for the downstream + % set operation. + hb_util:ok(hb_ao:deep_set(NestedBase, Nested, Opts)); + error -> Nested end; - (Key, _, {Acc, ToDeepMerge}) -> - {Acc, [Key | ToDeepMerge]} - end, - {BaseValues, []}, - NewValues - ), - hb_util:deep_merge( - WithNestedMerges, - maps:with(StillToDeepMerge, NewValues), - Opts - ). - -%% @doc Special case of `set/3' for setting the `path' key. This cannot be set -%% using the normal `set' function, as the `path' is a reserved key, used to -%% transmit the present key that is being executed. Subsequently, to call `path' -%% we would need to set `path' to `set', removing the ability to specify its -%% new value. -set_path(Base, #{ <<"value">> := Value }, Opts) -> - set_path(Base, Value, Opts); -set_path(Base, Value, Opts) when not is_map(Value) -> - % Determine whether the `path' key is committed. If it is, we remove the - % commitment if the new value is different. We try to minimize work by - % doing the `hb_maps:get` first, as it is far cheaper than calculating - % the committed keys. - BaseWithCorrectedComms = - case hb_maps:get(<<"path">>, Base, undefined, Opts) of - Value -> Base; - _ -> - % The new value is different, but is it committed? If so, we - % must remove the commitments. - case hb_message:is_signed_key(<<"path">>, Base, Opts) of - true -> hb_message:uncommitted(Base, Opts); - false -> Base - end - end, - case Value of - unset -> - {ok, hb_maps:without([<<"path">>], BaseWithCorrectedComms, Opts)}; - _ -> - BaseWithCorrectedComms#{ <<"path">> => Value } + (_Key, NewValue) -> NewValue + end, + hb_message:uncommitted(Req, Opts) + ), + set(Base, NewValues, Opts); +set(Base, NewValues, _Opts) -> + case lists:partition(fun hb_private:is_private/1, maps:keys(Base)) of + {[], _} -> {ok, NewValues#{ <<"...">> => Base }}; + {PrivKeys, _} -> + { + ok, + (maps:merge(NewValues, maps:with(PrivKeys, Base)))#{ + <<"...">> => maps:without(PrivKeys, Base) + } + } end. -%% @doc Remove a key or keys from a message. -remove(Base, #{ <<"item">> := Key }, Opts) -> - remove(Base, #{ <<"items">> => [Key] }, Opts); -remove(Base, #{ <<"items">> := Keys }, Opts) -> - set( - Base, - #{ Key => unset || Key <- Keys }, - Opts - ). - %% @doc Get the public keys of a message. -keys(Msg) -> - keys(Msg, #{}). - -keys(Msg, Opts) when not is_map(Msg) -> +keys(Msg, Req, Opts) when not is_map(Msg) -> case hb_ao:normalize_keys(Msg, Opts) of - NormMsg when is_map(NormMsg) -> keys(NormMsg, Opts); + NormMsg when is_map(NormMsg) -> keys(NormMsg, Req, Opts); _ -> throw(badarg) end; -keys(Msg, Opts) -> +keys(Msg, _Req, Opts) -> { ok, lists:filter( @@ -855,27 +645,28 @@ keys(Msg, Opts) -> %% @doc Return the value associated with the key as it exists in the message's %% underlying Erlang map. First check the public keys, then check case- %% insensitively if the key is a binary. -get(Key, Msg, Opts) -> get(Key, Msg, #{ <<"path">> => <<"get">> }, Opts). -get(Key, Msg, _Req, Opts) -> +default_accessor(Key, Msg, Opts) -> + default_accessor(Key, Msg, #{ <<"path">> => <<"get">> }, Opts). +default_accessor(Key, Msg, Req, Opts) -> case hb_private:is_private(Key) of true -> {error, not_found}; false -> - case hb_maps:get(Key, Msg, not_found, Opts) of - not_found -> case_insensitive_get(Key, Msg, Opts); - Value -> {ok, Value} + case hb_maps:find(Key, Msg, Opts) of + {ok, Value} -> {ok, Value}; + error -> + case hb_maps:find(<<"...">>, Msg, Opts) of + {ok, Nested} -> hb_ao:raw(Nested, Req, Opts); + error -> {error, not_found} + end end end. -%% @doc Key matching should be case insensitive, following RFC-9110, so we -%% implement a case-insensitive key lookup rather than delegating to -%% `hb_maps:get/2'. Encode the key to a binary if it is not already. -case_insensitive_get(Key, Msg, Opts) -> - NormKey = hb_util:to_lower(hb_util:bin(Key)), - NormMsg = hb_ao:normalize_keys(Msg, Opts), - case hb_maps:get(NormKey, NormMsg, not_found, Opts) of - not_found -> {error, not_found}; - Value -> {ok, Value} - end. +%% @doc Determines the schema for the resolution of a `Base/Request` pair and +%% applies it to the inputs. Returns `base` and `request` submessages, as well +%% as the resolvable function Erlang function at `priv/function` if it was found +%% during the process of `vary`ing. +vary(Base, Req, Opts) -> + todo. %%% Tests @@ -916,12 +707,6 @@ is_private_mod_test() -> keys_from_device_test() -> ?assertEqual({ok, [<<"a">>]}, hb_ao:resolve(#{ <<"a">> => 1 }, keys, #{})). -case_insensitive_get_test() -> - ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"a">> => 1 }, #{})), -% ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"A">> => 1 }, #{})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"a">> => 1 }, #{})). - %?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"A">> => 1 }, #{})). - private_keys_are_filtered_test() -> ?assertEqual( {ok, [<<"a">>]}, From 8b27fcd5c0399a57623c8dc3149fdceed975e4c5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 26 Jun 2026 16:56:14 -0400 Subject: [PATCH 03/35] wip: `hb_types` --- AO-CORE.md | 203 ++++++++++++++++++++++++++ src/core/resolver/hb_ao.erl | 12 +- src/core/resolver/hb_private.erl | 12 +- src/core/resolver/hb_types.erl | 63 +++----- src/preloaded/message/dev_message.erl | 39 ++++- 5 files changed, 265 insertions(+), 64 deletions(-) create mode 100644 AO-CORE.md diff --git a/AO-CORE.md b/AO-CORE.md new file mode 100644 index 000000000..4bc53af64 --- /dev/null +++ b/AO-CORE.md @@ -0,0 +1,203 @@ +# AO-Core 1.0 Minimal Semantics + +AO-Core is a protocol for attestable computation over messages. + +The only primitive relation is membership: + +```text +Base / Request -> Result +``` + +Everything else is syntax, storage, caching, transport, or implementation. + +## Required Properties + +1. **All computations must be atomically attested**: Upon receipt of a succinct committed string, we must be able to identify a set of computation result claims, and then _individually_ verify if each is true, without dependence on executing any other computations in the results. + +2. **All computations must be succinctly portable**: Any user wishing to move a computation from one machine to another should be able to take a hashpath and post a single message to another node, effectively duplicating the state and allowing for further computation elsewhere. By the nature of extension, complex collections of keys and values must be efficiently loadable from smaller, more common subsets. + +3. **All computations must be fully traceable**: Given any result, the full set of computations that gave rise to each of its values must be enumeratable. Receipt of a single value should allow you to efficiently isolate every strand of other computations, no matter how distant or tangentially related, whose results were dependencies of the given value. + +## Values + +A value is a literal, message, or link. + +A message is a device-interpreted value with public keys, optional commitments, +optional private runtime state, and optional ancestry. + +`priv` is runtime-local. It is not part of IDs, public commitments, signatures, +or protocol equality. + +A link names another value. Loading a link must preserve the same protocol +value. + +## Extension + +`...` is ancestry. + +```text +{ a: 2, ...: { a: 1, b: 3 } } +``` + +has active `a = 2` and active `b = 3`. + +Outer layers shadow inner layers. `unset` masks an inherited key. + +`set(Base, Patch)` constructs extension: + +```text +{ PatchKeys..., ...: Base } +``` + +It does not mutate `Base`. + +Pathless composition is set: + +```text +AO(Base, PatchWithoutPath) = set(Base, PatchWithoutPath) +``` + +A sequence is a fold: + +```text +AO([M0, M1, M2]) = AO(AO(M0, M1), M2) +``` + +## Devices + +A device defines memberships for request keys. + +If no device applies, the default device is `message@1.0`. + +Resolution through extension is layer-ordered. An outer layer interpreted by +`message@1.0` can return its own direct key before an inherited device is +reached. A layer that declares a device is interpreted by that device. + +Implementation artifacts such as Erlang modules and function pointers are +node-local and must not be public protocol facts. + +## `message@1.0` + +`message@1.0` is the default message device. + +Reserved keys include: + +```text +get, set, remove, *, keys, id, commit, verify, +commitments, committed, committers, vary, schema +``` + +For non-reserved keys, `message@1.0` resolves: + +```text +local public key -> value +local unset -> not_found +otherwise -> resolve through ... +``` + +`*` materializes active keys: + +```text +M/* -> { ActiveKeys..., ...: M } +``` + +`remove(K)` is `set({ K: unset })`. + +`id` commits to the selected public surface, never to `priv`. + +## Vary + +Execution is prepared before it runs. + +Preparation is itself an AO-Core membership. For a transition: + +```text +Base / Req > VarBase + VarReq = Res +``` + +the `>` clause means: + +```text +Base / vary(Req) -> { base: VarBase, request: VarReq } +VarBase / VarReq -> Res +``` + +`vary` is device-owned. It determines the exact base and request witnesses used +for execution. Execution must observe only those witnesses. + +A varied message may materialize keys, preserve links, or preserve extension, +but it must not hide dependencies. Any value it exposes must remain traceable +to the membership claim that produced it. + +## Hashpaths + +A hashpath is a succinct executable claim string. + +```text +Base/Req>VarBase+VarReq=Patch +``` + +means the execution produced an extension patch. The accumulated state becomes: + +```text +set(Base, Patch) +``` + +```text +Base/Req>VarBase+VarReq.Result +``` + +means the execution produced a full replacement. The accumulated state becomes: + +```text +Result +``` + +```text +HP/*=ID +``` + +means materializing the active state at `HP` has unsigned ID `ID`. + +A multi-step hashpath composes these transitions left to right. + +Each transition must be independently verifiable from its witnesses. Verifying +step `N` must not require executing steps `0..N-1`; the prior state is +reconstructed from the hashpath and loaded witnesses. + +## Storage And Portability + +Reusable result patches should be stored without caller-specific ancestry. + +Loading a hashpath reconstructs live extension semantics: + +```text +load(HP/Req>VB+VR=Patch) + -> { PatchKeys..., ...: load(HP), priv: { hashpath: HP/... } } +``` + +A portable computation package is one message containing a hashpath plus the +witness messages needed to resolve its IDs. Posting that package to another +node gives the node enough data to reconstruct the state and continue +computation. + +## Commitments + +A commitment attests public membership claims and public values. + +Computed-result commitments attest the hashpath context. + +Commitments never attest `priv`. + +Signed ancestors remain valid under extension because extension does not mutate +them. A signed request used to construct a new extension layer does not +automatically sign that new layer. + +## Core Rule + +No hidden inputs. + +Every value observed by execution must be present in `VarBase` or `VarReq`. + +Every value in `VarBase`, `VarReq`, or `Result` must be traceable to ordinary +AO-Core memberships. diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 05b6f096f..be610341d 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -811,13 +811,11 @@ set(Base, Req, Opts) -> %% @doc Set an individual (potentially deep) key in a message to a new using %% the message's set device. set(Base, Key, Value, Opts) -> - deep_set(Base, deep_base(Key, Value), Opts). - -%% @doc Create a deeply nested message with a layer for each part of the given -%% key. For example: -%% `deep_base(<<"a/b">>, Value) -> #{ <<"a">> => #{ <<"b">> => Value } }` -deep_base([Key], Value) -> #{ Key => Value }; -deep_base([NextKey|Rest], Value) -> #{ NextKey => deep_base(Rest, Value) }. + DeepBase = + fun Deep([LeafKey]) -> #{ LeafKey => Value }; + Deep([NextKey|Rest]) -> #{ NextKey => Deep(Rest, Value) } + end, + deep_set(Base, DeepBase(Key, Value), Opts). %% @doc Recursively extend nested message values. deep_set(Base, Req, Opts) when is_map(Req) -> diff --git a/src/core/resolver/hb_private.erl b/src/core/resolver/hb_private.erl index 0f886445f..300706864 100644 --- a/src/core/resolver/hb_private.erl +++ b/src/core/resolver/hb_private.erl @@ -85,12 +85,12 @@ set_priv(Msg, PrivMap) -> Msg#{ <<"priv">> => PrivMap }. %% @doc Check if a key is private. -is_private(Key) -> - try hb_util:bin(Key) of - <<"priv", _/binary>> -> true; - _ -> false - catch _:_ -> false - end. +is_private(ListKey) when is_list(ListKey) -> + % Strings should always be lists, but in case for some reason the caller + % ignores that... + try is_private(hb_util:bin(ListKey)) catch _ -> false end; +is_private(<<"priv", _>>) -> true; +is_private(_) -> false. %% @doc Remove the first key from the path if it is a private specifier. remove_private_specifier(InputPath, Opts) -> diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 5f6746c91..6545f725c 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -39,12 +39,6 @@ extract(Device, Opts) when is_binary(Device) -> extract(Device, _Opts) -> {error, {unsupported_device_type, Device}}. -function_schema(Device, Func, Key, Opts) -> - case schema_from_device(Device, Func, Key, Opts) of - undefined -> schema_from_fun_module(Device, Func, Key, Opts); - Schema -> Schema - end. - schema_from_device(Device, Func, Key, Opts) -> case extract(Device, Opts) of {ok, #{ <<"keys">> := Schemas }} -> @@ -53,18 +47,6 @@ schema_from_device(Device, Func, Key, Opts) -> undefined end. -schema_from_fun_module(Device, Func, Key, Opts) when is_atom(Device) -> - case erlang:fun_info(Func, module) of - {module, Device} -> - undefined; - {module, Module} -> - schema_from_device(Module, Func, Key, Opts); - _ -> - undefined - end; -schema_from_fun_module(_Device, _Func, _Key, _Opts) -> - undefined. - select_schema(Func, _Key, Schemas) -> case {erlang:fun_info(Func, name), erlang:fun_info(Func, arity)} of {{name, Name}, {arity, Arity}} -> @@ -86,10 +68,8 @@ execution_schemas(Schema, AddKey) -> maps:get(<<"return">>, Schema, any_type()) }. -nth_or(N, List, _Default) when length(List) >= N -> - lists:nth(N, List); -nth_or(_N, _List, Default) -> - Default. +nth_or(N, List, _Default) when length(List) >= N -> lists:nth(N, List); +nth_or(_N, _List, Default) -> Default. implicit_base(Schema) -> implicit_key(top_level_schema(Schema), <<"device">>, optional). @@ -134,15 +114,23 @@ do_extract(Module) -> {ok, #{ <<"keys">> => - lists:foldl( - fun(Spec, Acc) -> - maps:merge( - Acc, - spec_schemas(Spec, TypeEnv) - ) + lists:filtermap( + fun(Attr = {attribute, _, spec, {{Name, _}, Heads}}) -> + { + true, + { + Name, + [ + fun_schema(Head, TypeEnv) + || + Head <- Heads + ] + } + }; + (_) -> false + end, end, - #{}, - [Attr || Attr = {attribute, _, spec, _} <- Forms] + Forms ) }}; Error -> @@ -180,21 +168,6 @@ build_type_env(Forms) -> Forms ). -spec_schemas({attribute, _, spec, {{Name, Arity}, [Spec]}}, TypeEnv) -> - #{ - {normalize_name(Name), Arity} => fun_schema(Spec, TypeEnv) - }; -spec_schemas({attribute, _, spec, {{Name, Arity}, [Spec | _Rest]}}, TypeEnv) -> - #{ - {normalize_name(Name), Arity} => fun_schema(Spec, TypeEnv) - }; -spec_schemas({attribute, _, spec, {{Name, Arity}, Spec}}, TypeEnv) -> - maps:from_list( - [{{normalize_name(Name), Arity}, fun_schema(Spec, TypeEnv)}] - ); -spec_schemas(_Spec, _TypeEnv) -> - #{}. - fun_schema(Spec, TypeEnv) -> {Args, Return} = parse_fun_spec(Spec, TypeEnv), #{ diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 83c50077f..bb33d07ca 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -6,8 +6,8 @@ %%% behaviour of the device when these keys are set. -module(dev_message). %%% Base AO-Core state manipulation functions. --export([info/0, keys/3, set/3, id/3]). -%%% Commitment-specific keys: +-export([info/0, keys/3, set/3, id/3, vary/3, schema/3]). +%%% Commitments API keys: -export([commit/3, committed/3, committers/3, verify/3]). %%% Non-protocol enforced keys: -export([index/3]). @@ -645,8 +645,6 @@ keys(Msg, _Req, Opts) -> %% @doc Return the value associated with the key as it exists in the message's %% underlying Erlang map. First check the public keys, then check case- %% insensitively if the key is a binary. -default_accessor(Key, Msg, Opts) -> - default_accessor(Key, Msg, #{ <<"path">> => <<"get">> }, Opts). default_accessor(Key, Msg, Req, Opts) -> case hb_private:is_private(Key) of true -> {error, not_found}; @@ -655,7 +653,16 @@ default_accessor(Key, Msg, Req, Opts) -> {ok, Value} -> {ok, Value}; error -> case hb_maps:find(<<"...">>, Msg, Opts) of - {ok, Nested} -> hb_ao:raw(Nested, Req, Opts); + {ok, AncestorMsg} when is_map(AncestorMsg) -> + hb_ao:raw(AncestorMsg, Req, Opts); + {ok, Ancestor} + when is_binary(Ancestor) + orelse ?IS_LINK(Ancestor) -> + case hb_cache:read(Ancestor, Opts) of + {ok, AncestorMsg} -> + hb_ao:raw(AncestorMsg, Req, Opts); + OtherStatus -> OtherStatus + end; error -> {error, not_found} end end @@ -666,7 +673,27 @@ default_accessor(Key, Msg, Req, Opts) -> %% as the resolvable function Erlang function at `priv/function` if it was found %% during the process of `vary`ing. vary(Base, Req, Opts) -> - todo. + Path = hb_maps:get(<<"path">>, Req, undefined, Opts), + case schema(Base, Req, Opts) of + {ok, #{ <<"keys">> := #{ Path := Schema }}} -> + + {ok, Schema}; + _ -> {error, not_found} + end. + +%% @doc Returns the device schema for a `Base` message. +-spec schema(#{ device => binary(), _ => _ }, #{}, #{}) -> #{}. +schema(Base = #{ <<"device">> := Device }, _, Opts) -> + case hb_types:extract(Device, Opts) of + {ok, Schema} -> {ok, Schema}; + _ -> generate_default_schema(Base, Opts) + end. + +%% @doc Calculate the default schema for a message given its explicit keys and +%% the keys of the message device. +generate_default_schema(Base, Opts) -> + % TODO: What form is necessary for + {ok, keys(Base, #{}, Opts)}. %%% Tests From dab9c53232db04fd6e0aa58f9214d4096a10038f Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 27 Jun 2026 11:01:15 -0400 Subject: [PATCH 04/35] wip: `hb_ao:raw` varied resolver impl --- src/core/device/hb_device.erl | 75 +++++++++++++++++++++++---- src/core/device/hb_device_load.erl | 8 +-- src/core/resolver/hb_ao.erl | 48 +++++++++++------ src/preloaded/message/dev_message.erl | 23 +++++++- 4 files changed, 123 insertions(+), 31 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index df9ea5aa9..54291cf5b 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -2,7 +2,7 @@ %%% Offers services for loading, verifying executability, and extracting Erlang %%% functions from a device. -module(hb_device). --export([truncate_args/2, message_to_fun/3, message_to_device/2]). +-export([truncate_args/2, add_resolver/2, message_to_fun/3, module/2]). -export([is_direct_key_access/3, is_direct_key_access/4]). -export([find_exported_function/5, is_exported/4, info/2, info/3]). -include("include/hb.hrl"). @@ -29,6 +29,36 @@ truncate_args(Fun, Args) -> {arity, Arity} = erlang:fun_info(Fun, arity), lists:sublist(Args, Arity). +%% @doc Generates an execution context for a given key and device pair. +%% Returns `{ok, ExecCtx}`, where `ExecCtx` is a message containing: +%% #{ +%% base: The base message, unvaried. +%% key: The key to be resolved in the execution. +%% base-device: The device derived from the message itself. +%% priv/exec-device: The device from which the execution function originates. +%% If the `base-device` resolvers from another device, +%% this key will differ from the `base-device`. +%% priv/resolver: The function to execute to resolve the `base-device`. +%% priv/add-key: Whether the execution function expects that we should +%% add the `key` as an additional argument to the start of +%% the argument list. +%% } +add_resolver(Context = #{<<"base">> := Base, <<"key">> := Key }, Opts) -> + DeviceID = message_device_id(Base, Opts), + InitialDevice = module(DeviceID, Opts), + {Type, ExecDev, Fun} = message_to_fun(InitialDevice, Base, Key, Opts), + {ok, + Context#{ + <<"base-device">> => DeviceID, + <<"priv">> => + #{ + <<"exec-device">> => ExecDev, + <<"add-key">> => Type == add_key, + <<"resolver">> => Fun + } + } + }. + %% @doc Calculate the Erlang function that should be called to get a value for %% a given key from a device. %% @@ -99,7 +129,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> % rules. ?event_debug({found_default_device, {mod, DefaultDevice}}), message_to_fun( - Msg#{ <<"device">> => DefaultDevice }, + with_device(Msg, DefaultDevice), Key, Opts ); @@ -107,7 +137,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> % Case 6: The device has no default handler. % We retry with the default unless the message % already names it (loop guard). - case hb_maps:get(<<"device">>, Msg, undefined, Opts) of + case message_device_id(Msg, Opts) of ?DEFAULT_DEVICE -> throw({ error, @@ -117,7 +147,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> _ -> ?event_debug({using_default_device, ?DEFAULT_DEVICE}), message_to_fun( - Msg#{ <<"device">> => ?DEFAULT_DEVICE }, + with_device(Msg, ?DEFAULT_DEVICE), Key, Opts ) @@ -130,13 +160,36 @@ message_to_fun(Dev, Msg, Key, Opts) -> %% message has no `<<"device">>' key, we resolve the default %% (`message@1.0') just like any other device: There is no privileged %% internal module-loading path. -message_to_device(Msg, Opts) -> - DevID = hb_maps:get(<<"device">>, Msg, ?DEFAULT_DEVICE, Opts), +module(DevID, Opts) -> case hb_device_load:reference(DevID, Opts) of {error, Reason} -> throw({error, {device_not_loadable, DevID, Reason}}); {ok, DevMod} -> DevMod end. +%% @doc Return the device ID from a message, resolving through any ancestors +%% as necessary. +message_device_id(Msg, Opts) -> + case hb_maps:find(<<"device">>, Msg, Opts) of + {ok, Device} -> Device; + error -> + case hb_maps:find(<<"...">>, Msg, Opts) of + {ok, Ancestor} -> message_device_id(Ancestor, Opts); + error -> ?DEFAULT_DEVICE + end + end. + +%% @doc Small helper to extend a message with a device. +with_device(Msg, Device) when is_map(Msg) -> + hb_private:set_priv( + #{ + <<"device">> => Device, + <<"...">> => Msg + }, + hb_private:from_message(Msg) + ); +with_device(_Msg, Device) -> + #{ <<"device">> => Device }. + %% @doc Parse a handler key given by a device's `info'. info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> {add_key, Handler}; @@ -144,11 +197,11 @@ info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> case hb_maps:find(excludes, HandlerMap, Opts) of {ok, Exclude} -> case lists:member(Key, Exclude) of - true -> - MsgWithoutDevice = - hb_maps:without([<<"device">>], Msg, Opts), + true -> + MsgWithoutDevice = + hb_maps:without([<<"device">>], Msg, Opts), message_to_fun( - MsgWithoutDevice#{ <<"device">> => ?DEFAULT_DEVICE }, + with_device(MsgWithoutDevice, ?DEFAULT_DEVICE), Key, Opts ); @@ -240,7 +293,7 @@ maybe_normalize_device_key(Key, Mode) -> %% @doc Get the info map for a device, optionally giving it a message if the %% device's info function is parameterized by one. -info(Msg, Opts) -> info(message_to_device(Msg, Opts), Msg, Opts). +info(Msg, Opts) -> info(module(Msg, Opts), Msg, Opts). info(DevMod, Msg, Opts) -> case find_exported_function(Msg, DevMod, info, 2, Opts) of {ok, Fun} -> apply(Fun, truncate_args(Fun, [Msg, Opts])); diff --git a/src/core/device/hb_device_load.erl b/src/core/device/hb_device_load.erl index 8d84cc6a4..806f39e03 100644 --- a/src/core/device/hb_device_load.erl +++ b/src/core/device/hb_device_load.erl @@ -27,9 +27,11 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc A message is already a device. A binary reference is resolved, -%% then memoised in the process cache unless it is a forge seed. +%% @doc Resolve a device referece to an executable device representation +%% (either a module atom or a binary reference). reference(Loaded, _Opts) when is_map(Loaded) -> + % A message is already a viable device. A binary reference is resolved, + % then memoised in the process cache unless it is a forge seed. {ok, Loaded}; reference(Ref, Opts) when is_binary(Ref) -> NormRef = hb_ao:normalize_key(Ref), @@ -156,7 +158,7 @@ preloaded_spec(Ref, Store, Opts) -> hb_store:read(Store, <>, Opts). %% @doc The preloaded store, with request-local cache keys stripped so it is -%% visible inside a request-scoped resolution. +%% active inside a request-scoped resolution. preloaded(Opts) -> Node = maps:without([<<"cache-control">>, <<"only">>, <<"prefer">>], Opts), hb_opts:get(preloaded_store, undefined, Node). diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index be610341d..b52b2227f 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -187,25 +187,44 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> BaseWithDevice = case ForcedDevice of undefined -> Base; - _ when is_map(Base) -> - #{ <<"device">> => ForcedDevice, <<"...">> => Base }; - _ -> - #{ <<"device">> => ForcedDevice } - end, - {ExecFun, PrefixArgs} = - case hb_device:message_to_fun(BaseWithDevice, Key, ExecOpts) of - {add_key, _DevMod, Fun} -> {Fun, [Key]}; - {_Status, _DevMod, Fun} -> {Fun, []} + _ when is_map(Base) -> set(Base, <<"device">>, ForcedDevice, Opts) end, + Ctx0 = #{ <<"base">> => BaseWithDevice, <<"key">> => Key }, + {ok, Ctx1} = hb_device:add_resolver(Ctx0, ExecOpts), + {ok, Ctx2} = hb_types:add_schema(Ctx1, Opts), + {ok, Ctx3} = hb_types:vary(Ctx2, Opts), % Apply the function and return the result directly, without any further % processing. We add the `PrefixArgs` to the list of arguments to be passed % to the function to accomodate default handlers, which take the key that % was invoked on the device as the first argument (ahead of `Base`, `Req`, % and `ExecOpts`). - apply( - ExecFun, - hb_device:truncate_args(ExecFun, PrefixArgs ++ [Base, Req, ExecOpts]) - ). + #{ + <<"varied-base">> := VariedBase, + <<"varied-req">> := VariedReq, + <<"return-extends">> := Extend, + <<"priv">> := + #{ + <<"resolver">> := Fun, + <<"add-key">> := AddKey + } + } = Ctx3, + ResWithStatus = + apply( + Fun, + hb_device:truncate_args( + Fun, + if AddKey -> [Key]; true -> [] end + ++ [VariedBase, VariedReq, ExecOpts] + ) + ), + % Extend with the unvaried base/req values if specified by the execution + % schema. + case ResWithStatus of + {ok, Res} when Extend == none -> {ok, Res}; + {ok, Res} when Extend == base -> {ok, Res#{ <<"...">> => Base } }; + {ok, Res} when Extend == request -> {ok, Res#{ <<"...">> => Req } }; + OtherRet -> OtherRet + end. %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, @@ -797,8 +816,7 @@ get_first([{Base, Path}|Msgs], Default, Opts) -> end. %% @doc Shortcut to get the list of keys from a message. -keys(Msg, Opts) -> - get(<<"keys">>, Msg, Opts). +keys(Msg, Opts) -> get(<<"keys">>, Msg, Opts). %% @doc Extend a message using its underlying device's `set' key. set(Base, Req, Opts) -> diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index bb33d07ca..52b878f78 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -156,7 +156,7 @@ calculate_id(RawBase, Req, NodeOpts) -> % by checking whether the resolved device module is this module itself. % `hb_ao:raw/5' expects a device name, not a resolved module. CommitDev = - case hb_device:message_to_device(#{ <<"device">> => IDDev }, NodeOpts) of + case hb_device:module(#{ <<"device">> => IDDev }, NodeOpts) of ?MODULE -> ?DEFAULT_ID_DEVICE; _ -> IDDev end, @@ -628,11 +628,30 @@ set(Base, NewValues, _Opts) -> end. %% @doc Get the public keys of a message. -keys(Msg, Req, Opts) when not is_map(Msg) -> +-spec keys(#{ _ => _ }, #{ keys => _, _ => _ }, _) -> {ok, [binary()]}. +keys(Msg, Req, Opts) when is_list(Msg) -> case hb_ao:normalize_keys(Msg, Opts) of NormMsg when is_map(NormMsg) -> keys(NormMsg, Req, Opts); _ -> throw(badarg) end; +keys(Msg, #{ <<"keys">> := <<"deep">> }, Opts) when is_map(Msg) -> + Inherited = + case maps:find(<<"...">>, Msg) of + {ok, Extension} -> hb_ao:keys(Extension, Opts); + error -> [] + end, + InheritedPublic = Inherited -- [<<"commitments">>], + DirectKeys = + maps:fold( + fun + (_Key, unset, Acc) -> Acc; + (<<"...">>, _Value, Acc) -> Acc; + (Key, _Value, Acc) -> [Key | Acc] + end, + [], + Msg + ), + {ok, DirectKeys ++ InheritedPublic}; keys(Msg, _Req, Opts) -> { ok, From 3262130b051e20fdb85cee0aa561627a6b403047 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 27 Jun 2026 11:45:51 -0400 Subject: [PATCH 05/35] wip: allow device forcing without setting `Base/device` in `hb_ao:raw` --- src/core/device/hb_device.erl | 8 ++++++-- src/core/resolver/hb_ao.erl | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index 54291cf5b..ab88240ee 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -43,8 +43,12 @@ truncate_args(Fun, Args) -> %% add the `key` as an additional argument to the start of %% the argument list. %% } -add_resolver(Context = #{<<"base">> := Base, <<"key">> := Key }, Opts) -> - DeviceID = message_device_id(Base, Opts), +add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> + DeviceID = + case maps:find(<<"base-device">>, Base) of + {ok, ForcedBaseDevice} -> ForcedBaseDevice; + error -> message_device_id(Base, Opts) + end, InitialDevice = module(DeviceID, Opts), {Type, ExecDev, Fun} = message_to_fun(InitialDevice, Base, Key, Opts), {ok, diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index b52b2227f..0830d6fa7 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -183,16 +183,16 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> true -> hb_path:hd(Req, ExecOpts) end, % If an explicit device is provided we use it _only on the lookup_ -- not - % during execution. - BaseWithDevice = - case ForcedDevice of - undefined -> Base; - _ when is_map(Base) -> set(Base, <<"device">>, ForcedDevice, Opts) + % during execution. We do this by shortcutting the `base-device` field in + % the initial context, if `ForcedDevice` is provided. + Ctx0 = + if ForcedDevice =/= undefined -> #{ <<"base-device">> => ForcedDevice }; + true -> #{} end, - Ctx0 = #{ <<"base">> => BaseWithDevice, <<"key">> => Key }, - {ok, Ctx1} = hb_device:add_resolver(Ctx0, ExecOpts), - {ok, Ctx2} = hb_types:add_schema(Ctx1, Opts), - {ok, Ctx3} = hb_types:vary(Ctx2, Opts), + Ctx1 = Ctx0#{ <<"base">> => Base, <<"req">> => Req, <<"key">> => Key }, + {ok, Ctx2} = hb_device:add_resolver(Ctx1, ExecOpts), + {ok, Ctx3} = hb_types:add_schema(Ctx2, Opts), + {ok, Ctx4} = hb_types:vary(Ctx3, Opts), % Apply the function and return the result directly, without any further % processing. We add the `PrefixArgs` to the list of arguments to be passed % to the function to accomodate default handlers, which take the key that @@ -207,7 +207,7 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> <<"resolver">> := Fun, <<"add-key">> := AddKey } - } = Ctx3, + } = Ctx4, ResWithStatus = apply( Fun, @@ -833,7 +833,7 @@ set(Base, Key, Value, Opts) -> fun Deep([LeafKey]) -> #{ LeafKey => Value }; Deep([NextKey|Rest]) -> #{ NextKey => Deep(Rest, Value) } end, - deep_set(Base, DeepBase(Key, Value), Opts). + deep_set(Base, DeepBase(hb_path:term_to_path_parts(Key, Opts)), Opts). %% @doc Recursively extend nested message values. deep_set(Base, Req, Opts) when is_map(Req) -> From 52ccf7f4e79f1513fca967697c6cbde2c34aea35 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 27 Jun 2026 21:30:44 -0400 Subject: [PATCH 06/35] wip: AO-Core 1.0 spec draft-3 --- AO-CORE-DRAFT-3.md | 468 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 AO-CORE-DRAFT-3.md diff --git a/AO-CORE-DRAFT-3.md b/AO-CORE-DRAFT-3.md new file mode 100644 index 000000000..eeca1aa92 --- /dev/null +++ b/AO-CORE-DRAFT-3.md @@ -0,0 +1,468 @@ +# AO-Core 1.0 Draft 3 + +AO-Core is a protocol for attestable computation over protocol-addressed +messages. It is not itself a virtual machine. It is a neutral meta-VM: a common +language for describing, transporting, attesting, challenging, and tracing the +computations performed by many VMs, called devices. + +AO-Core offers three core properties: + +1. **Atomic attestation**: any transition claim can be verified or challenged + independently from unrelated computation history. +2. **Succinct portability**: a state can be moved as a hashpath plus the values + needed by that hashpath, without enumerating every value that could later be + resolved. +3. **Traceability**: every computation result can be traced through every state + transformation to the origins of the values on which it depends. + +Through these properties in a decentralized protocol, AO-Core intends to give +rise to a globally distributed supercomputer presenting a single system image +across a machine of arbitrary size. + +## Values And Messages + +A value is any protocol-addressable content. + +A message is a value that contains keys and values. Values may identify other +values by protocol address; such links are literals from the point of view of the +core protocol. + +All values can be named by ID. All messages can be represented by extension of +other messages through ancestry: + +```text +set(Base, Patch) = { Patch..., ...: Base } +``` + +`...` is ancestry. `unset` masks an inherited key. + +Extension does not mutate `Base`. It constructs a new value that shares +structure with `Base`. This is the basis of message deduplication: large states +can be represented as small patches over existing states. + +## Resolution + +The primitive relation is: + +```text +Base / Request -> Result +``` + +For a request whose key is `K`, resolution walks the ancestry of `Base` from the +outermost layer inward. + +At each layer: + +1. Look locally for `device`. +2. If `device = Dev` is found, execute `Dev` against the original outermost + `Base` and the original `Request`. +3. Otherwise, look locally for `K`. +4. If `K = Value` is found, return `Value`. +5. If `K = unset` is found, return `not_found`. +6. Otherwise, look locally for `...`. +7. If `... = Ancestor` is found, continue at `Ancestor`. +8. Otherwise, return `not_found`. + +Pseudocode: + +```text +resolve(Outer, Layer, Request): + K = key(Request) + + case local(Layer, "device") of + Dev -> return execute(Dev, Outer, Request) + not_found -> continue + end + + case local(Layer, K) of + Value -> return Value + unset -> return not_found + not_found -> continue + end + + case local(Layer, "...") of + Ancestor -> return resolve(Outer, Ancestor, Request) + not_found -> return not_found + end +``` + +This local inspection is `message@1.0` behavior. It asks only about the current +layer's decoded members; it does not recursively resolve while inspecting that +layer. + +If a device is inherited from an ancestor, it executes over the outermost state: + +```text +{ x: 5, ...: { device: dev1, x: 1 } } / x-is-5 +``` + +selects `dev1` from the ancestor, but `dev1` sees `x = 5`. + +## Devices + +A device defines how requests are computed for a state. + +Examples of devices include message interpreters, process VMs, WASM VMs, +Lua VMs, codecs, stores, payment devices, and application-specific evaluators. +AO-Core does not privilege one compute model over another. It standardizes how +their transitions are named, witnessed, attested, transported, and traced. + +If no device is found, the default device is `message@1.0`. + +Device loading, Erlang modules, function pointers, local caches, and runtime +workers are implementation facts. The protocol fact is the device value that +the computation commits to using. + +## Vary + +Before execution, a transition is varied: + +```text +Base / Request / vary -> VariedBase + VariedRequest @ Depends +``` + +`VariedBase` and `VariedRequest` are ordinary messages containing exactly the +values required by the execution. They use canonical nested structure, not path +strings: + +```text +VariedBase = { + device: process@1.0, + balance: { + OUR_ADDRESS: 7, + SENDER: 93 + } +} + +VariedRequest = { + path: transfer, + from: SENDER, + to: OUR_ADDRESS, + quantity: 3 +} +``` + +`Depends` records where each varied value originated. It has the same shape as +the varied messages, rooted under `base` and `request`. Each leaf is a hashpath +whose terminal value is the corresponding varied value: + +```text +Depends = { + base: { + device: HP_for_process_device, + balance: { + OUR_ADDRESS: HP_for_prior_our_balance, + SENDER: HP_for_prior_sender_balance + } + }, + request: { + path: HP_for_request_path, + from: HP_for_request_from, + to: HP_for_request_to, + quantity: HP_for_quantity + } +} +``` + +The dependency leaf is a single value: the origin hashpath. The value itself +does not need to be duplicated inside `Depends`, because the hashpath binds the +origin to the value it yields. + +If no exact vary specification is available, the conservative valid vary is +identity: + +```text +VariedBase = Base +VariedRequest = Request +``` + +The core rule is: + +```text +No hidden inputs. +``` + +Everything observed by execution must be present in `VariedBase` or +`VariedRequest`, and every varied value must have an origin in `Depends`. + +## Shared Computation + +Varying creates a reusable computation point. + +Many concrete states may vary to the same pair: + +```text +BaseA / Request -> VariedBase + VariedRequest +BaseB / Request -> VariedBase + VariedRequest +``` + +The execution: + +```text +VariedBase / VariedRequest -> Patch +``` + +is shared by all sufficiently alike concrete states for that request. The final +states may still differ because the patch is equivalent to applying the original +transition to each original base: + +```text +BaseA / Request == set(BaseA, Patch) +BaseB / Request == set(BaseB, Patch) +``` + +This is the default mode of AO-Core computation: do a computation once for the +material inputs that matter, then reuse it across every state/request pair that +varies to those inputs. + +All AO-Core states and transition results are cacheable by address. Prior +computations from many different execution traces can therefore be reused +seamlessly. Because AO-Core is expressed naturally through HTTP semantics, the +same `Vary`-style caching and routing ideas that already power web +infrastructure can be applied to decentralized computation. + +## Transition Equivalence + +A transition asserts an equivalence between resolving a request and extending +the base with the patch produced by varied execution: + +```text +Base / Request + == set(Base, Patch) +``` + +where: + +```text +Base / Request / vary -> VariedBase + VariedRequest @ Depends +VariedBase / VariedRequest -> Patch +``` + +Extension is just `set`. There is no special state update operation beyond +constructing a new message with ancestry. + +## Hashpaths + +A hashpath is a succinct executable claim and an addressable protocol value. +Like any other value, it may be encountered as a link and resolved through +AO-Core, or encountered as serialized content and decoded into its in-memory +form. + +The full transition forms are: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +``` + +`=` means the execution produced a patch that extends the prior state: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID +``` + +is equivalent to: + +```text +set(Base, Patch) +``` + +`.` means the execution produced a replacement value: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +``` + +The accumulated state becomes `ResultID`, dropping the prior state's keys rather +than extending them. + +A compact form may omit fields when they are derivable or supplied elsewhere, +but the full receipt must be recoverable for challenge and trace. + +A hashpath is a sequence of transition claims. Later segments operate on the +result established by earlier segments. + +Segments without explicit vary syntax are not special. They are compact +transition claims. For example: + +```text +HP/*=FinalResultID +``` + +is simply a claim that resolving `*` at `HP` yields `FinalResultID`. HTTP +gateways commonly append such a segment so the response body is tied to the +specific keys and values returned to the client. + +## Hashpath Loading And Portability + +Hashpaths are not a storage system outside AO-Core. They are addressable values +whose loaded form reconstructs ordinary message semantics. + +Reusable patches can be stored by their generic IDs, without caller-specific +ancestry. The hashpath records how that generic value is reached from a prior +state. For an extension segment: + +```text +PriorHP/Req>VariedBase+VariedReq@Depends=Patch +``` + +the cacheable value may be the hashpath link itself: + +```text +link:PriorHP/Req>VariedBase+VariedReq@Depends=Patch +``` + +loading the segment loads `Patch` and presents it as an extension whose `...` +is the transition context before the patch result: + +```text +load(PriorHP/Req>VB+VR@Deps=Patch) + -> { PatchKeys..., ...: PriorHP/Req>VB+VR@Deps } +``` + +The `...` value is itself a hashpath value. Loading it reconstructs the prior +state for inherited-key resolution and retains the request, varied witnesses, +and dependency message needed to challenge the transition. Implementations may +cache this decoded context in runtime metadata, but the protocol-visible +ancestry remains the hashpath value. + +For replacement segments: + +```text +PriorHP/Req>VB+VR@Deps.Result +``` + +loading the segment yields `Result`. The prior state is not inherited as active +message keys, but the hashpath still carries the transition context needed for +challenge and trace. + +This gives portability as a single addressable value: posting a hashpath plus +the values needed to resolve the IDs it names gives another node enough data to +reconstruct the live state, challenge any segment, and continue computation. + +## Challenge And Audit + +A transition can be challenged locally, without verifying its entire dependency +tree. The usual practical operation is to pick one claim and verify only the +facts needed for that claim. A full provenance audit is the recursive version of +the same process. + +To challenge a transition claim: + +1. Verify that `BaseID` and `ReqID` identify the claimed values. +2. Verify the vary claim: + + ```text + Base / Request / vary -> VariedBase + VariedRequest @ Depends + ``` + +3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf + in `Depends` and verify that it yields that value. +4. Verify execution: + + ```text + VariedBase / VariedRequest -> Patch + ``` + +5. Verify transition equivalence: + + ```text + "=" means the accumulated state is set(Base, Patch) + "." means the accumulated state is Result + ``` + +Any one of these checks can be challenged independently. To audit the full +provenance tree, recursively challenge the dependency hashpaths named in +`Depends`. + +## Traceability + +To trace a value in a result: + +1. Locate the transition that produced the state containing the value. +2. If the value was introduced by the patch, trace it to that transition's + varied witness. +3. Follow every corresponding leaf in `Depends`. +4. If the value was inherited through `...`, continue tracing in the ancestor + state. +5. Repeat recursively until reaching literals, signed inputs, codec inputs, or + externally attested transition claims. + +For example, a process state may claim: + +```text +ProcessStateN.balance.OUR_ADDRESS = 10 +``` + +The trace may show that this came from: + +```text +ProcessStateN-1 / TransferRequest + > VariedBase + VariedRequest @ Depends + = ProcessStateN +``` + +with: + +```text +VariedBase.balance.OUR_ADDRESS = 7 +VariedBase.balance.SENDER = 93 +VariedRequest.quantity = 3 +``` + +`Depends` then points to the hashpaths that produced `7`, `93`, and `3`. The +quantity may trace to an inbound message from a swap process, whose sale-price +transition may itself be attested by another node. + +The trace is not a narrative. It is a recursive chain of AO-Core claims. + +## HTTP Expression + +HTTP is an expression of AO-Core, not the foundation of AO-Core. + +An HTTP request is decoded into an AO-Core request message. URL path segments, +query parameters, method, headers, and body are message members. Content +negotiation selects codecs for values and messages. The server resolves: + +```text +Base / Request -> Result +``` + +and returns an HTTP response containing the encoded result plus enough hashpath +and commitment information for the receiver to verify, port, and continue the +computation. + +Thus HTTP gives AO-Core a universal transport and user-facing syntax while the +protocol remains independent of HTTP itself. + +HTTP computations should append a terminal materialization claim by default: + +```text +HP/*=MaterializedID +``` + +This claim is not special in the hashpath calculus. It is the ordinary +materialization request for `*`, included so that the transport response is tied +to the concrete enumerated keys and values returned to the client. + +The default HTTPSig response carries two independently useful commitments: + +1. A commitment over the enumerated response keys and values, excluding the + `...` hashpath field. This commitment is typically unsigned unless the + response is already a pre-existing signed message. Its ID is + `MaterializedID`. +2. A signature over just the excluded `...` field, whose value is the hashpath + terminating in `=MaterializedID`. + +The first commitment lets the returned value stand alone by ID. The second lets +the recipient port or challenge the computation through the hashpath. They can +be used together or independently. + +## Summary + +AO-Core turns computation into portable, challengeable, traceable message +transitions. + +Devices provide the compute. Vary provides exact witnesses. Depends provides +origin trace. Hashpaths provide portable claims. Extension provides deduplicated +state. HTTP provides a practical expression layer. From d24e60d65d469b409b1f5945a3f5878bb547e87b Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 10:44:32 -0400 Subject: [PATCH 07/35] impr: `hb_types:add_schema` to execution context --- src/core/resolver/hb_types.erl | 71 +++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 6545f725c..c5beac760 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -1,27 +1,55 @@ %%% @doc Extract device specs and vary AO-Core execution inputs. -module(hb_types). --export([extract/2, vary/7]). +-export([extract/2, vary/2, add_schema/2]). -include_lib("eunit/include/eunit.hrl"). -%% @doc Apply the resolved function's base/request schemas to one execution. -vary(Device, Key, Func, AddKey, Base, Req, Opts) -> - case function_schema(Device, Func, Key, Opts) of - undefined -> - {ok, Base, Req, none}; +%% @doc Add the schema for a resolution to an execution context, given a +%% resolved key, function, and execution module (e.g, from +%% `hb_device:add_resolver`). +add_schema( + Ctx = + #{ + <<"key">> := Key, + <<"priv">> := + #{ + <<"resolver">> := Func, + <<"add-key">> := AddKey, + <<"exec-device">> := Device + } + }, + Opts) -> + case schema_from_device(Device, Func, Key, Opts) of + undefined -> {ok, Ctx#{ <<"schema">> => undefined } }; Schema -> - {BaseSchema, ReqSchema, ReturnSchema} = - execution_schemas(Schema, AddKey), - ReqWithKey = - case AddKey of - false -> Req; - _ -> Req#{ <<"path">> => Key } - end, {ok, - apply_schema(implicit_base(BaseSchema), Base, Opts), - apply_schema(implicit_request(ReqSchema), ReqWithKey, Opts), - overlay(ReturnSchema)} + Ctx#{ + <<"schema">> => execution_schema(Schema, AddKey) + } + } end. +%% @doc Apply the resolved function's base/request schemas to one execution. +vary(Ctx = #{ <<"schema">> := undefined, <<"base">> := Base, <<"req">> := Req }, _Opts) -> + {ok, + Ctx#{ + <<"varied-base">> => Base, + <<"varied-req">> => Req, + <<"return-extends">> => none + } + }; +vary(Ctx = #{ + <<"schema">> := [BaseSchema, ReqSchema, ReturnSchema], + <<"base">> := Base, + <<"req">> := Req +}, Opts) -> + {ok, + Ctx#{ + <<"varied-base">> => apply_schema(implicit_base(BaseSchema), Base, Opts), + <<"varied-req">> => apply_schema(implicit_request(ReqSchema), Req, Opts), + <<"return-extends">> => overlay(ReturnSchema) + } + }. + %% @doc Extract a device module's function schemas. This intentionally has no %% cache; callers can add one once the algorithm is settled. extract(Device, _Opts) when is_map(Device) -> @@ -55,18 +83,18 @@ select_schema(Func, _Key, Schemas) -> undefined end. -execution_schemas(Schema, AddKey) -> +execution_schema(Schema, AddKey) -> Args = maps:get(<<"args">>, Schema, []), Offset = case AddKey of false -> 0; _ -> 1 end, - { + [ nth_or(1 + Offset, Args, any_type()), nth_or(2 + Offset, Args, any_type()), maps:get(<<"return">>, Schema, any_type()) - }. + ]. nth_or(N, List, _Default) when length(List) >= N -> lists:nth(N, List); nth_or(_N, _List, Default) -> Default. @@ -129,7 +157,6 @@ do_extract(Module) -> }; (_) -> false end, - end, Forms ) }}; @@ -332,7 +359,7 @@ apply_schema(Type, Value, Opts) -> explicit_keys(Keys, Message, Opts) -> maps:fold( fun(Key, #{ <<"presence">> := Presence, <<"type">> := Type }, Acc) -> - case hb_message:find_active(Key, Message, Opts) of + case hb_ao:raw(Message, #{ <<"path">> => Key }, Opts) of {ok, Value} -> Acc#{ Key => apply_schema(Type, Value, Opts) }; error when Presence =:= required -> @@ -354,7 +381,7 @@ wildcard_keys(#{ <<"presence">> := required, <<"type">> := Type }, Keys, Message fun(_Key, Value) -> apply_schema(Type, hb_cache:ensure_all_loaded(Value, Opts), Opts) end, - maps:without(maps:keys(Keys), hb_message:active(Message, Opts)) + hb_util:list_without(maps:keys(Keys), hb_ao:keys(Message, Opts)) ). overlay(ReturnSchema) -> From 61e0180f25caf619f8aa193f718d57f86f992efd Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 12:36:21 -0400 Subject: [PATCH 08/35] fix: message-to-device key resolver flow --- src/core/device/hb_device.erl | 19 ++++++++++++------- src/core/resolver/hb_message.erl | 8 +------- src/core/resolver/hb_singleton.erl | 1 - src/core/resolver/hb_types.erl | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index ab88240ee..abfc2bbe1 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -45,7 +45,7 @@ truncate_args(Fun, Args) -> %% } add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> DeviceID = - case maps:find(<<"base-device">>, Base) of + case maps:find(<<"base-device">>, Context) of {ok, ForcedBaseDevice} -> ForcedBaseDevice; error -> message_device_id(Base, Opts) end, @@ -89,7 +89,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key, Opts) -> % Get the device module from the message and recurse. - message_to_fun(message_to_device(Msg, Opts), Msg, Key, Opts). + message_to_fun(module(Msg, Opts), Msg, Key, Opts). message_to_fun(Dev, Msg, Key, Opts) -> Info = info(Dev, Msg, Opts), % Is the key exported by the device? @@ -141,7 +141,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> % Case 6: The device has no default handler. % We retry with the default unless the message % already names it (loop guard). - case message_device_id(Msg, Opts) of + case message_device_id(Msg, undefined, Opts) of ?DEFAULT_DEVICE -> throw({ error, @@ -149,9 +149,10 @@ message_to_fun(Dev, Msg, Key, Opts) -> {key, Key} }); _ -> - ?event_debug({using_default_device, ?DEFAULT_DEVICE}), + WithDev = with_device(Msg, ?DEFAULT_DEVICE), message_to_fun( - with_device(Msg, ?DEFAULT_DEVICE), + module(?DEFAULT_DEVICE, Opts), + WithDev, Key, Opts ) @@ -173,12 +174,16 @@ module(DevID, Opts) -> %% @doc Return the device ID from a message, resolving through any ancestors %% as necessary. message_device_id(Msg, Opts) -> + message_device_id(Msg, ?DEFAULT_DEVICE, Opts). +message_device_id(Msg, Default, Opts) -> + ?event({finding_device_id, Msg}), case hb_maps:find(<<"device">>, Msg, Opts) of {ok, Device} -> Device; error -> case hb_maps:find(<<"...">>, Msg, Opts) of - {ok, Ancestor} -> message_device_id(Ancestor, Opts); - error -> ?DEFAULT_DEVICE + {ok, Ancestor} -> message_device_id(Ancestor, Default, Opts); + error -> + Default end end. diff --git a/src/core/resolver/hb_message.erl b/src/core/resolver/hb_message.erl index a903b9607..e2e6dc57d 100644 --- a/src/core/resolver/hb_message.erl +++ b/src/core/resolver/hb_message.erl @@ -207,13 +207,7 @@ conversion_spec_to_req(Spec, Opts) -> { case Device of tabm -> tabm; - _ -> - hb_device:message_to_device( - #{ - <<"device">> => Device - }, - Opts - ) + _ -> hb_device:module(Device, Opts) end, hb_maps:without([<<"device">>], Spec, Opts) } diff --git a/src/core/resolver/hb_singleton.erl b/src/core/resolver/hb_singleton.erl index 161fd80d7..925f4ade4 100644 --- a/src/core/resolver/hb_singleton.erl +++ b/src/core/resolver/hb_singleton.erl @@ -292,7 +292,6 @@ do_build(I, [{as, DevID, Direct} | Rest], ScopedKeys, Opts) when is_map(Direct) <<"structured@1.0">>, Opts#{ <<"topic">> => ao_internal } ), - ?event_debug(parsing, {build_messages, {base, Msg}, {additional, Additional}}), [ #{ <<"device">> => DevID }, Direct#{ <<"...">> => StepMsg } diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index c5beac760..94c62e6f4 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -103,7 +103,7 @@ implicit_base(Schema) -> implicit_key(top_level_schema(Schema), <<"device">>, optional). implicit_request(Schema) -> - implicit_key(top_level_schema(Schema), <<"path">>, required). + implicit_key(top_level_schema(Schema), <<"path">>, optional). top_level_schema(#{ <<"kind">> := <<"empty">> }) -> message_type({#{}, none}); From ef0c0a1464e8839efaf47f2e16dfdfe63017be96 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 12:43:58 -0400 Subject: [PATCH 09/35] impr: add primitive debugging macro --- src/core/include/hb.hrl | 12 ++++++++---- src/core/resolver/hb_ao.erl | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/core/include/hb.hrl b/src/core/include/hb.hrl index 538716040..e398474f4 100644 --- a/src/core/include/hb.hrl +++ b/src/core/include/hb.hrl @@ -51,13 +51,17 @@ -define(debug_print(X), hb_event:debug_print(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). -define(no_prod(X), hb:no_prod(X, ?MODULE, ?LINE)). +-ifdef(HB_PRIMITIVE_DEBUG). +%% @doc Primitive debugging macro for low-level printing when testing AO-Core +%% changes. +-define(prim_dbg(X), io:format(standard_error, "PRIM ~s: ~p~n", [?trace_short(), X])). +-else. +-define(prim_dbg(X), ignored_primitive_debug). +-endif. + %%% Macro shortcuts for debugging. %% @doc A macro for marking that you got 'here'. -define(h(), hb_event:log("[Debug point reached.]", ?MODULE, ?FUNCTION_NAME, ?LINE)). -%% @doc Quickly print a value in the logs. Currently uses the event -%% function, but should be moved to a debug-specific function once we -%% build out better logging infrastructure. --define(p(X), hb_event:log(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). %% @doc Print the trace of the current stack, up to the first non-hyperbeam %% module. -define(trace(), hb_format:trace_macro_helper(fun hb_format:print_trace/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 0830d6fa7..0941eabba 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -177,6 +177,13 @@ raw(Device, Base, Req, Opts) -> raw(Device, undefined, Base, Req, Opts). raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> ExecOpts = execution_opts(Opts), + ?prim_dbg( + {executing, + {forced_device, ForcedDevice}, + {forced_key, ForcedKey}, + {req, Req} + } + ), % If a forced key is provided, use it; otherwise, extract from the request. Key = if ForcedKey =/= undefined -> ForcedKey; From e108ef795564d3d5320007d8c547cfa1605ba068 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 12:56:19 -0400 Subject: [PATCH 10/35] chore: add math test device with specs --- src/preloaded/util/dev_math.erl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/preloaded/util/dev_math.erl diff --git a/src/preloaded/util/dev_math.erl b/src/preloaded/util/dev_math.erl new file mode 100644 index 000000000..772cd7941 --- /dev/null +++ b/src/preloaded/util/dev_math.erl @@ -0,0 +1,24 @@ +%%% @doc Simple test device with math utilities. Not intended for production use. +-module(dev_math). +-export([inc_x/3, dec_x/3, add_x/3, sum/3, with_sum/3]). + +-spec inc_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. +inc_x(#{ <<"x">> := X }, _, _) -> {ok, #{ <<"x">> => X + 1 }}. + +-spec dec_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. +dec_x(#{ <<"x">> := X }, _, _) -> {ok, #{ <<"x">> => X - 1 }}. + +-spec add_x(#{ x := integer() }, #{ add := integer() }, _) -> + {ok, #{ '...' => base }}. +add_x(#{ <<"x">> := X }, #{ <<"add">> := Add }, _) -> + {ok, #{ <<"x">> => X + Add }}. + +-spec sum(#{ x := integer(), y := integer() }, _, _) -> + {ok, #{ '...' => base }}. +sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> + {ok, X + Y}. + +-spec with_sum(#{ x := integer(), y := integer() }, _, _) -> + {ok, #{ sum => integer(), '...' => base }}. +with_sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> + {ok, #{ <<"sum">> => X + Y }}. \ No newline at end of file From d8fafa8d690bdb58910e85cfb4523467a3a1daf0 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 18:11:19 -0400 Subject: [PATCH 11/35] wip: Cache device schema on load --- src/core/device/hb_device_load.erl | 69 +++++++++++++++++---- src/core/resolver/hb_types.erl | 86 ++++++++++++++------------- src/preloaded/message/dev_message.erl | 17 ++---- src/preloaded/util/dev_test.erl | 6 +- 4 files changed, 112 insertions(+), 66 deletions(-) diff --git a/src/core/device/hb_device_load.erl b/src/core/device/hb_device_load.erl index 806f39e03..ff28db885 100644 --- a/src/core/device/hb_device_load.erl +++ b/src/core/device/hb_device_load.erl @@ -23,7 +23,7 @@ %%% signature. %%% -module(hb_device_load). --export([reference/2]). +-export([reference/2, schema/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -72,7 +72,7 @@ get_resolved_device(Ref, Opts) -> {ok, Bin} ?= hb_store:read( loaded_device_store(Opts), - store_key(Ref), + module_cache_key(Ref), Opts ), Mod = hb_util:atom(Bin), @@ -89,13 +89,15 @@ put_resolved_device(Ref, Mod, Opts) -> erlang:put({?MODULE, Ref}, Mod), hb_store:write( loaded_device_store(Opts), - #{ store_key(Ref) => hb_util:bin(Mod) }, + #{ module_cache_key(Ref) => hb_util:bin(Mod) }, Opts ). loaded_device_store(Opts) -> hb_opts:get(loaded_device_store, [], Opts). -store_key(Ref) -> <<"~meta@1.0/devices/", Ref/binary>>. +loaded_device_opts(Opts) -> Opts#{ <<"store">> => loaded_device_store(Opts) }. + +module_cache_key(Ref) -> <<"~meta@1.0/devices/modules/", Ref/binary>>. %%% -------------------------------------------------------------------- %%% High trust @@ -135,7 +137,10 @@ from_preloaded(Ref, Opts) -> {error, not_found}; Store -> PreOpts = - Opts#{ <<"store">> => [Store], <<"cache-read-mode">> => raw }, + Opts#{ + <<"store">> => [Store], + <<"cache-read-mode">> => raw + }, maybe {ok, SpecID} ?= preloaded_spec(Ref, Store, PreOpts), lazy_first( @@ -274,13 +279,55 @@ load_archive(ID, Opts) -> load_archive_message(Msg, Opts) end. +%% @doc Load an archive and cache the schema of its device. load_archive_message(Msg, Opts) -> - hb_device_archive:load( - hb_maps:get(<<"module-name">>, Msg, undefined, Opts), - hb_maps:get(<<"body">>, Msg, undefined, Opts), - Msg, - Opts - ). + Archive = hb_maps:get(<<"body">>, Msg, undefined, Opts), + maybe + {ok, Module} ?= + hb_device_archive:load( + hb_maps:get(<<"module-name">>, Msg, undefined, Opts), + Archive, + Msg, + Opts + ), + {ok, Schema} ?= archive_to_schema(Module, Archive, Opts), + cache_schema(Module, Schema, Opts), + {ok, Module} + end. + +%% @doc Extract the schema for a newly loaded device module from a packaged +%% device archive. +archive_to_schema(Module, Archive, _Opts) -> + case hb_device_archive:contents(Archive) of + {ok, Modules, _Resources} -> + case lists:keyfind(Module, 1, Modules) of + {Module, _Path, Beam} -> hb_types:beam_to_schema(Module, Beam); + false -> ok + end; + {error, _} -> + ok + end. + +%% @doc Read the cached schema for a device module if known. +schema(Module, Opts) -> + hb_cache:read(schema_key(Module), loaded_device_opts(Opts)). + +%% @doc Write the schema for a device to the cache, under its reference. +%% Avoids caching the schema for devices during the write of another schema +%% through the `cache-schemas' option in the node message. +cache_schema(_Module, _Schema, #{ <<"cache-schemas">> := false }) -> + skip; +cache_schema(Module, Schema, Opts) -> + SchemaOpts = (loaded_device_opts(Opts))#{ <<"cache-schemas">> => false }, + case hb_cache:write(Schema, SchemaOpts) of + {ok, SchemaID} -> + hb_store:link(#{ SchemaID => schema_key(Module) }, Opts), + {ok, SchemaID}; + Other -> Other + end. + +schema_key(Module) -> + <<"~meta@1.0/devices/schemas/", (atom_to_binary(Module, utf8))/binary>>. implementation_query(SpecID) -> #{ diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 94c62e6f4..24249f5ff 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -1,6 +1,7 @@ %%% @doc Extract device specs and vary AO-Core execution inputs. -module(hb_types). --export([extract/2, vary/2, add_schema/2]). +-export([extract/2, vary/2, add_schema/2, beam_to_schema/2]). +-include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Add the schema for a resolution to an execution context, given a @@ -50,14 +51,19 @@ vary(Ctx = #{ } }. -%% @doc Extract a device module's function schemas. This intentionally has no -%% cache; callers can add one once the algorithm is settled. +%% @doc Extract a device module's function schemas. We first check the +%% `loaded-device-store` cache, then fall back to loading the module manually +%% if it isn't already available. extract(Device, _Opts) when is_map(Device) -> {error, {unsupported_device_type, Device}}; -extract(Module, _Opts) when is_atom(Module) -> - case code:ensure_loaded(Module) of - {module, Module} -> do_extract(Module); - {error, Reason} -> {error, {module_not_loaded, Module, Reason}} +extract(Module, Opts) when is_atom(Module) -> + case hb_device_load:schema(Module, Opts) of + {ok, Schema} -> {ok, Schema}; + _ -> + case code:ensure_loaded(Module) of + {module, Module} -> beam_to_schema(Module); + {error, Reason} -> {error, {module_not_loaded, Module, Reason}} + end end; extract(Device, Opts) when is_binary(Device) -> case hb_device_load:reference(Device, Opts) of @@ -76,9 +82,9 @@ schema_from_device(Device, Func, Key, Opts) -> end. select_schema(Func, _Key, Schemas) -> - case {erlang:fun_info(Func, name), erlang:fun_info(Func, arity)} of - {{name, Name}, {arity, Arity}} -> - maps:get({normalize_name(Name), Arity}, Schemas, undefined); + case erlang:fun_info(Func, name) of + {name, Name} -> + maps:get(normalize_name(Name), Schemas, undefined); _ -> undefined end. @@ -131,38 +137,36 @@ implicit_key(Schema = #{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, implicit_key(Schema, _Key, _Presence) -> Schema. -do_extract(Module) -> +beam_to_schema(Module) -> case module_beam(Module) of unavailable -> {error, {abstract_code_unavailable, Module, unavailable}}; - Beam -> - case beam_lib:chunks(Beam, [abstract_code]) of - {ok, {_, [{abstract_code, {_, Forms}}]}} -> - TypeEnv = build_type_env(Forms), - {ok, - #{ - <<"keys">> => - lists:filtermap( - fun(Attr = {attribute, _, spec, {{Name, _}, Heads}}) -> - { - true, - { - Name, - [ - fun_schema(Head, TypeEnv) - || - Head <- Heads - ] - } - }; - (_) -> false - end, - Forms - ) - }}; - Error -> - {error, {abstract_code_unavailable, Module, Error}} - end + Beam -> beam_to_schema(Module, Beam) + end. + +beam_to_schema(Module, Beam) -> + case beam_lib:chunks(Beam, [abstract_code]) of + {ok, {_, [{abstract_code, {_, Forms}}]}} -> + TypeEnv = build_type_env(Forms), + {ok, + #{ + <<"keys">> => + lists:foldl( + fun + ({attribute, _, spec, {{Name, _}, [Head | _]}}, Acc) -> + Acc#{ + normalize_name(Name) => + fun_schema(Head, TypeEnv) + }; + (_, Acc) -> + Acc + end, + #{}, + Forms + ) + }}; + Error -> + {error, {abstract_code_unavailable, Module, Error}} end. module_beam(Module) -> @@ -362,9 +366,9 @@ explicit_keys(Keys, Message, Opts) -> case hb_ao:raw(Message, #{ <<"path">> => Key }, Opts) of {ok, Value} -> Acc#{ Key => apply_schema(Type, Value, Opts) }; - error when Presence =:= required -> + {error, not_found} when Presence =:= required -> throw({required_key_missing, Key}); - error -> + {error, not_found} -> Acc end end, diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 52b878f78..543c16038 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -691,29 +691,22 @@ default_accessor(Key, Msg, Req, Opts) -> %% applies it to the inputs. Returns `base` and `request` submessages, as well %% as the resolvable function Erlang function at `priv/function` if it was found %% during the process of `vary`ing. +-spec vary(#{ _ => _ }, #{ vary => binary() | [binary()]}, #{}) -> {ok, #{}}. vary(Base, Req, Opts) -> Path = hb_maps:get(<<"path">>, Req, undefined, Opts), case schema(Base, Req, Opts) of - {ok, #{ <<"keys">> := #{ Path := Schema }}} -> - - {ok, Schema}; + {ok, #{ <<"keys">> := #{ Path := Schema }}} -> {ok, Schema}; _ -> {error, not_found} end. %% @doc Returns the device schema for a `Base` message. --spec schema(#{ device => binary(), _ => _ }, #{}, #{}) -> #{}. -schema(Base = #{ <<"device">> := Device }, _, Opts) -> +-spec schema(_, _, _) -> {ok, undefined | #{}}. +schema(#{ <<"device">> := Device }, _, Opts) -> case hb_types:extract(Device, Opts) of {ok, Schema} -> {ok, Schema}; - _ -> generate_default_schema(Base, Opts) + _ -> {ok, undefined} end. -%% @doc Calculate the default schema for a message given its explicit keys and -%% the keys of the message device. -generate_default_schema(Base, Opts) -> - % TODO: What form is necessary for - {ok, keys(Base, #{}, Opts)}. - %%% Tests %%% Internal module functionality tests: diff --git a/src/preloaded/util/dev_test.erl b/src/preloaded/util/dev_test.erl index 8851ecd88..2fbfcf885 100644 --- a/src/preloaded/util/dev_test.erl +++ b/src/preloaded/util/dev_test.erl @@ -49,8 +49,10 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Example index handler. -index(Msg, _Req, Opts) -> - Name = hb_ao:get(<<"name">>, Msg, <<"turtles">>, Opts), +-spec index(#{ name => binary() }, _, _) -> + {ok, #{ content_type => binary(), body => binary() }}. +index(Msg, _Req, _Opts) -> + Name = maps:get(<<"name">>, Msg, <<"turtles">>), {ok, #{ <<"content-type">> => <<"text/html">>, From 75b68d02adc8981d9505a7acfb254652f5b92e6d Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 30 Jun 2026 20:07:17 -0400 Subject: [PATCH 12/35] wip: schema loading --- src/core/device/hb_device.erl | 6 ++-- src/core/device/hb_device_load.erl | 58 +++++++++++++++++++----------- src/core/include/hb.hrl | 2 +- src/core/resolver/hb_ao.erl | 7 +++- src/core/resolver/hb_types.erl | 45 ++++++++++++++++------- 5 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index abfc2bbe1..a9144561a 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -49,14 +49,14 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> {ok, ForcedBaseDevice} -> ForcedBaseDevice; error -> message_device_id(Base, Opts) end, - InitialDevice = module(DeviceID, Opts), - {Type, ExecDev, Fun} = message_to_fun(InitialDevice, Base, Key, Opts), + {Type, ExecDev, ExecMod, Fun} = message_to_fun(DeviceID, Base, Key, Opts), {ok, Context#{ <<"base-device">> => DeviceID, + <<"resolver-device">> => ExecDev, <<"priv">> => #{ - <<"exec-device">> => ExecDev, + <<"resolver-module">> => ExecMod, <<"add-key">> => Type == add_key, <<"resolver">> => Fun } diff --git a/src/core/device/hb_device_load.erl b/src/core/device/hb_device_load.erl index ff28db885..3f36deb38 100644 --- a/src/core/device/hb_device_load.erl +++ b/src/core/device/hb_device_load.erl @@ -290,40 +290,58 @@ load_archive_message(Msg, Opts) -> Msg, Opts ), - {ok, Schema} ?= archive_to_schema(Module, Archive, Opts), - cache_schema(Module, Schema, Opts), + maybe_cache_schema(Module, Archive, Opts), {ok, Module} end. - -%% @doc Extract the schema for a newly loaded device module from a packaged -%% device archive. + +%% @doc Extract and cache a packaged device schema when available. +maybe_cache_schema(Module, Archive, Opts) -> + case archive_to_schema(Module, Archive, Opts) of + {ok, Schema} -> + cache_schema(Module, Schema, Opts), + ok; + _ -> ok + end. + +%% @doc Extract the schema for a loaded device module from a packaged archive. archive_to_schema(Module, Archive, _Opts) -> case hb_device_archive:contents(Archive) of {ok, Modules, _Resources} -> case lists:keyfind(Module, 1, Modules) of {Module, _Path, Beam} -> hb_types:beam_to_schema(Module, Beam); - false -> ok + false -> {error, schema_not_found} end; - {error, _} -> - ok + {error, _} = Error -> + Error end. - + %% @doc Read the cached schema for a device module if known. schema(Module, Opts) -> - hb_cache:read(schema_key(Module), loaded_device_opts(Opts)). + SchemaOpts = loaded_device_opts(Opts), + case hb_cache:read(schema_key(Module), SchemaOpts) of + {ok, Schema} -> {ok, hb_cache:ensure_all_loaded(Schema, SchemaOpts)}; + Other -> Other + end. %% @doc Write the schema for a device to the cache, under its reference. -%% Avoids caching the schema for devices during the write of another schema -%% through the `cache-schemas' option in the node message. -cache_schema(_Module, _Schema, #{ <<"cache-schemas">> := false }) -> - skip; cache_schema(Module, Schema, Opts) -> - SchemaOpts = (loaded_device_opts(Opts))#{ <<"cache-schemas">> => false }, - case hb_cache:write(Schema, SchemaOpts) of - {ok, SchemaID} -> - hb_store:link(#{ SchemaID => schema_key(Module) }, Opts), - {ok, SchemaID}; - Other -> Other + case hb_opts:get(<<"caching-schema">>, false, Opts) of + true -> skipping_recursive_cache; + false -> + SchemaOpts = + (loaded_device_opts(Opts))#{ + <<"caching-schema">> => true + }, + case hb_cache:write(Schema, SchemaOpts) of + {ok, SchemaID} -> + hb_store:link( + loaded_device_store(Opts), + #{ schema_key(Module) => SchemaID }, + SchemaOpts + ); + _ -> + ok + end end. schema_key(Module) -> diff --git a/src/core/include/hb.hrl b/src/core/include/hb.hrl index e398474f4..a16974669 100644 --- a/src/core/include/hb.hrl +++ b/src/core/include/hb.hrl @@ -56,7 +56,7 @@ %% changes. -define(prim_dbg(X), io:format(standard_error, "PRIM ~s: ~p~n", [?trace_short(), X])). -else. --define(prim_dbg(X), ignored_primitive_debug). +-define(prim_dbg(X), io:format(standard_error, "PRIM ~s: ~p~n", [?trace_short(), X])). -endif. %%% Macro shortcuts for debugging. diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 0941eabba..e6c5b3fc8 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -437,7 +437,12 @@ resolve_stage(5, Base, Req, ExecName, Opts) -> {opts, Opts} } ), - {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), + {Status, _Dev, Device, Func} = + hb_device:message_to_fun( + Base, + Key, + UserOpts + ), ?event( {found_func_for_exec, {key, Key}, diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 24249f5ff..dd7387d8f 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -15,18 +15,30 @@ add_schema( #{ <<"resolver">> := Func, <<"add-key">> := AddKey, - <<"exec-device">> := Device + <<"exec-module">> := DevMod } }, Opts) -> - case schema_from_device(Device, Func, Key, Opts) of - undefined -> {ok, Ctx#{ <<"schema">> => undefined } }; + case schema_from_device(DevMod, Func, Key, Opts) of + undefined -> + ?prim_dbg( + {schema_not_found, + {device, DevMod}, + {func, Func}, + {key, Key} + } + ), + {ok, Ctx#{ <<"schema">> => undefined } }; Schema -> - {ok, - Ctx#{ - <<"schema">> => execution_schema(Schema, AddKey) + ?prim_dbg( + {schema_found, + {device, DevMod}, + {func, Func}, + {key, Key}, + {schema, Schema} } - } + ), + {ok, Ctx#{ <<"schema">> => execution_schema(Schema, AddKey) } } end. %% @doc Apply the resolved function's base/request schemas to one execution. @@ -57,12 +69,19 @@ vary(Ctx = #{ extract(Device, _Opts) when is_map(Device) -> {error, {unsupported_device_type, Device}}; extract(Module, Opts) when is_atom(Module) -> - case hb_device_load:schema(Module, Opts) of - {ok, Schema} -> {ok, Schema}; - _ -> - case code:ensure_loaded(Module) of - {module, Module} -> beam_to_schema(Module); - {error, Reason} -> {error, {module_not_loaded, Module, Reason}} + % If we are already caching a schema at the moment, skip the recursive cache + % call and error early. + case hb_opts:get(<<"caching-schema">>, false, Opts) of + true -> + {error, caching_schema}; + false -> + case hb_device_load:schema(Module, Opts) of + {ok, Schema} -> {ok, Schema}; + _ -> + case code:ensure_loaded(Module) of + {module, Module} -> beam_to_schema(Module); + {error, Reason} -> {error, {module_not_loaded, Module, Reason}} + end end end; extract(Device, Opts) when is_binary(Device) -> From e6411df2a3aec2af9511a40585982ce6036b574e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 1 Jul 2026 09:59:58 -0400 Subject: [PATCH 13/35] wip: resolver impl --- src/core/device/hb_device.erl | 47 ++++++----- src/core/resolver/hb_ao.erl | 107 ++++++++++++++++++++++---- src/core/resolver/hb_types.erl | 10 +-- src/preloaded/message/dev_message.erl | 20 +++-- 4 files changed, 139 insertions(+), 45 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index a9144561a..985cdd9fe 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -88,16 +88,19 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> %% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key, Opts) -> - % Get the device module from the message and recurse. - message_to_fun(module(Msg, Opts), Msg, Key, Opts). -message_to_fun(Dev, Msg, Key, Opts) -> - Info = info(Dev, Msg, Opts), + DeviceID = message_device_id(Msg, Opts), + message_to_fun(DeviceID, Msg, Key, Opts). +message_to_fun(DevID, Msg, Key, Opts) -> + message_to_fun(DevID, module(DevID, Opts), Msg, Key, Opts). +message_to_fun(DevID, DevMod, Msg, Key, Opts) -> + Info = info(DevMod, Msg, Opts), % Is the key exported by the device? Exported = is_exported(Info, Key, Opts), ?event( ao_devices, {message_to_fun, - {dev, Dev}, + {device, DevID}, + {module, DevMod}, {key, Key}, {is_exported, Exported}, {opts, Opts} @@ -110,22 +113,21 @@ message_to_fun(Dev, Msg, Key, Opts) -> % Case 2: The device has an explicit handler function. ?event( ao_devices, - {handler_found, {dev, Dev}, {key, Key}, {handler, Handler}} + {handler_found, {dev, DevID}, {key, Key}, {handler, Handler}} ), - {Status, Func} = info_handler_to_fun(Handler, Msg, Key, Opts), - {Status, Dev, Func}; + info_handler_to_fun(Handler, DevID, DevMod, Msg, Key, Opts); _ -> - ?event_debug(ao_devices, {no_override_handler, {dev, Dev}, {key, Key}}), - case {find_exported_function(Msg, Dev, Key, 3, 1, Opts), Exported} of + ?event_debug(ao_devices, {no_override_handler, {dev, DevID}, {key, Key}}), + case {find_exported_function(Msg, DevMod, Key, 3, 1, Opts), Exported} of {{ok, Func}, true} -> % Case 3: The device has a function of the name `Key'. - {ok, Dev, Func}; + {ok, DevID, DevMod, Func}; _ -> case {hb_maps:find(default, Info, Opts), Exported} of {{ok, DefaultFunc}, true} when is_function(DefaultFunc) -> % Case 4: The device has a default handler. ?event_debug({found_default_handler, {func, DefaultFunc}}), - {add_key, Dev, DefaultFunc}; + {add_key, DevID, DevMod, DefaultFunc}; {{ok, DefaultDevice}, true} when is_binary(DefaultDevice) orelse is_atom(DefaultDevice) -> % Case 5: The device gives a specific further device @@ -133,6 +135,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> % rules. ?event_debug({found_default_device, {mod, DefaultDevice}}), message_to_fun( + DefaultDevice, with_device(Msg, DefaultDevice), Key, Opts @@ -151,7 +154,7 @@ message_to_fun(Dev, Msg, Key, Opts) -> _ -> WithDev = with_device(Msg, ?DEFAULT_DEVICE), message_to_fun( - module(?DEFAULT_DEVICE, Opts), + ?DEFAULT_DEVICE, WithDev, Key, Opts @@ -165,6 +168,10 @@ message_to_fun(Dev, Msg, Key, Opts) -> %% message has no `<<"device">>' key, we resolve the default %% (`message@1.0') just like any other device: There is no privileged %% internal module-loading path. +module(DevMod, _Opts) when is_atom(DevMod) -> + DevMod; +module(Dev, _Opts) when is_map(Dev) -> + Dev; module(DevID, Opts) -> case hb_device_load:reference(DevID, Opts) of {error, Reason} -> throw({error, {device_not_loadable, DevID, Reason}}); @@ -200,9 +207,10 @@ with_device(_Msg, Device) -> #{ <<"device">> => Device }. %% @doc Parse a handler key given by a device's `info'. -info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> - {add_key, Handler}; -info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> +info_handler_to_fun(Handler, DevID, DevMod, _Msg, _Key, _Opts) + when is_function(Handler) -> + {add_key, DevID, DevMod, Handler}; +info_handler_to_fun(HandlerMap, DevID, DevMod, Msg, Key, Opts) -> case hb_maps:find(excludes, HandlerMap, Opts) of {ok, Exclude} -> case lists:member(Key, Exclude) of @@ -210,13 +218,16 @@ info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> MsgWithoutDevice = hb_maps:without([<<"device">>], Msg, Opts), message_to_fun( + ?DEFAULT_DEVICE, with_device(MsgWithoutDevice, ?DEFAULT_DEVICE), Key, Opts ); - false -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + false -> + {add_key, DevID, DevMod, hb_maps:get(func, HandlerMap, undefined, Opts)} end; - error -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + error -> + {add_key, DevID, DevMod, hb_maps:get(func, HandlerMap, undefined, Opts)} end. %% @doc Find the function with the highest arity that has the given name, if it diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index e6c5b3fc8..93e3d0ae7 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -119,7 +119,7 @@ %% This function returns the raw result of the device function call: %% `{ok | error, NewMessage}.' %% The resolver is composed of a series of discrete phases: -%% 1: Loading IDs/links. +%% 1: Normalization and loading of IDs/links. %% 2: Device or direct key lookup. %% 3: Vary `Base` and `Request`. %% 4: Cache lookup @@ -196,26 +196,73 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> if ForcedDevice =/= undefined -> #{ <<"base-device">> => ForcedDevice }; true -> #{} end, - Ctx1 = Ctx0#{ <<"base">> => Base, <<"req">> => Req, <<"key">> => Key }, - {ok, Ctx2} = hb_device:add_resolver(Ctx1, ExecOpts), - {ok, Ctx3} = hb_types:add_schema(Ctx2, Opts), - {ok, Ctx4} = hb_types:vary(Ctx3, Opts), + vary_context( + Ctx0#{ <<"base">> => Base, <<"req">> => Req, <<"key">> => Key }, + Opts + ). + +%% @doc Perform the `vary` step of an AO-Core execution. This step requires +%% using the device (or the fallback device) to materialize the appropriate +%% protocol values from the `Base` and `Request` messages, as well as the +%% Erlang function to be executed, in order to evaluate an AO-Core resolution. +vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> + VaryRes = + case maps:find(<<"primitive">>, Ctx) of + {ok, true} -> + % Skip the varying step during `primitive` execution, but still + % find the resolver and add it to the context. + hb_device:add_resolver(Ctx, Opts); + _ -> + % First resolve the `vary` operation for the device and key + % upon the base and request, using the `Base` message's + % `device` function. We do this by performing a `primitive` + % execution as a recursive call. + execute_context( + #{ + <<"varied-req">> => Req#{ + <<"path">> => <<"vary">>, + <<"vary">> => + hb_maps:get( + <<"path">>, + Req, + undefined, + Opts + ) + }, + <<"base">> => Base, + <<"primitive">> => true + }, + Opts + ) + end, + ?prim_dbg({vary_result, VaryRes}), + case VaryRes of + {ok, CtxWithRes = #{ <<"result">> := _Result }} -> + post_execution(CtxWithRes, Opts); + {ok, PreparedCtx} -> + execute_context(PreparedCtx, Opts); + Other -> + Other + end. + +execute_context(Ctx, Opts) -> % Apply the function and return the result directly, without any further % processing. We add the `PrefixArgs` to the list of arguments to be passed % to the function to accomodate default handlers, which take the key that % was invoked on the device as the first argument (ahead of `Base`, `Req`, % and `ExecOpts`). #{ + <<"key">> := Key, <<"varied-base">> := VariedBase, <<"varied-req">> := VariedReq, - <<"return-extends">> := Extend, <<"priv">> := #{ <<"resolver">> := Fun, - <<"add-key">> := AddKey + <<"add-key">> := AddKey, + <<"exec-opt">> := ExecOpts } - } = Ctx4, - ResWithStatus = + } = Ctx, + {Status, Res} = apply( Fun, hb_device:truncate_args( @@ -224,14 +271,40 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> ++ [VariedBase, VariedReq, ExecOpts] ) ), - % Extend with the unvaried base/req values if specified by the execution - % schema. - case ResWithStatus of - {ok, Res} when Extend == none -> {ok, Res}; - {ok, Res} when Extend == base -> {ok, Res#{ <<"...">> => Base } }; - {ok, Res} when Extend == request -> {ok, Res#{ <<"...">> => Req } }; - OtherRet -> OtherRet - end. + post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). + +%% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return +%% the result directly; else, apply the `return-extends` schema to the result. +post_execution( + #{ + <<"primitive">> := true, + <<"result">> := Res, + <<"status">> := Status + }, + _Opts) -> + {Status, Res}; +post_execution( + Context = #{ + <<"base">> := Base, + <<"request">> := Req, + <<"result">> := RawResult, + <<"status">> := Status, + <<"return-extends">> := Extend + }, + Opts) -> + maybe_cache(Context, Opts), + Res = + case Extend of + none -> RawResult; + base -> RawResult#{ <<"...">> => Base }; + request -> RawResult#{ <<"...">> => Req } + end, + {Status, Res}. + +maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> + skipped_caching; +maybe_cache(_Context, _Opts) -> + todo. %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index dd7387d8f..9a22bcef4 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -11,19 +11,19 @@ add_schema( Ctx = #{ <<"key">> := Key, + <<"resolver-device">> := Device, <<"priv">> := #{ <<"resolver">> := Func, - <<"add-key">> := AddKey, - <<"exec-module">> := DevMod + <<"add-key">> := AddKey } }, Opts) -> - case schema_from_device(DevMod, Func, Key, Opts) of + case schema_from_device(Device, Func, Key, Opts) of undefined -> ?prim_dbg( {schema_not_found, - {device, DevMod}, + {device, Device}, {func, Func}, {key, Key} } @@ -32,7 +32,7 @@ add_schema( Schema -> ?prim_dbg( {schema_found, - {device, DevMod}, + {device, Device}, {func, Func}, {key, Key}, {schema, Schema} diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 543c16038..f74faa2ea 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -691,12 +691,22 @@ default_accessor(Key, Msg, Req, Opts) -> %% applies it to the inputs. Returns `base` and `request` submessages, as well %% as the resolvable function Erlang function at `priv/function` if it was found %% during the process of `vary`ing. --spec vary(#{ _ => _ }, #{ vary => binary() | [binary()]}, #{}) -> {ok, #{}}. +-spec vary(#{ _ => _ }, #{ vary => binary(), _ => _ }, #{}) -> {ok, #{}}. vary(Base, Req, Opts) -> - Path = hb_maps:get(<<"path">>, Req, undefined, Opts), - case schema(Base, Req, Opts) of - {ok, #{ <<"keys">> := #{ Path := Schema }}} -> {ok, Schema}; - _ -> {error, not_found} + maybe + {ok, Key} ?= hb_maps:find(<<"vary">>, Req, Opts), + Ctx0 = #{ <<"base">> => Base, <<"key">> => Key }, + case hb_device:add_resolver(Ctx0, Opts) of + {ok, Ctx1 = #{ <<"result">> := _ }} -> + % Finding the resolver led us to first find the complete + % solution. We return the context as-is. + {ok, Ctx1}; + {ok, Ctx1} -> + % The resolver has been found. We must vary the `Base` and + % `Req` using its schema, then return the updated context. + hb_types:vary(Ctx1, Opts); + Err -> Err + end end. %% @doc Returns the device schema for a `Base` message. From 40d9def9cb5325d1dc1e690ca8afb322f1b317f9 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 1 Jul 2026 12:44:30 -0400 Subject: [PATCH 14/35] wip: resolver flow --- src/core/device/hb_device.erl | 64 +++++++++++++++++----------- src/core/resolver/hb_ao.erl | 53 +++++++++++++++-------- src/core/test/hb_ao_test_vectors.erl | 2 +- 3 files changed, 76 insertions(+), 43 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index 985cdd9fe..6714dd237 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -44,24 +44,35 @@ truncate_args(Fun, Args) -> %% the argument list. %% } add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> - DeviceID = + % TODO: What if we force a device _and_ have a direct key hit? + DevRes = case maps:find(<<"base-device">>, Context) of - {ok, ForcedBaseDevice} -> ForcedBaseDevice; - error -> message_device_id(Base, Opts) + {ok, ForcedBaseDevice} -> {ok, ForcedBaseDevice}; + error -> message_device_id(Base, Key, Opts) end, - {Type, ExecDev, ExecMod, Fun} = message_to_fun(DeviceID, Base, Key, Opts), - {ok, - Context#{ - <<"base-device">> => DeviceID, - <<"resolver-device">> => ExecDev, - <<"priv">> => - #{ - <<"resolver-module">> => ExecMod, - <<"add-key">> => Type == add_key, - <<"resolver">> => Fun + case DevRes of + {hit, Value} -> + {ok, + Context#{ + <<"result">> => Value } - } - }. + }; + {ok, DeviceID} -> + {Type, ExecDev, ExecMod, Fun} = + message_to_fun(DeviceID, Base, Key, Opts), + {ok, + Context#{ + <<"base-device">> => DeviceID, + <<"resolver-device">> => ExecDev, + <<"priv">> => + #{ + <<"resolver-module">> => ExecMod, + <<"add-key">> => Type == add_key, + <<"resolver">> => Fun + } + } + } + end. %% @doc Calculate the Erlang function that should be called to get a value for %% a given key from a device. @@ -88,7 +99,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> %% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key, Opts) -> - DeviceID = message_device_id(Msg, Opts), + DeviceID = message_device_id(Msg, Key, Opts), message_to_fun(DeviceID, Msg, Key, Opts). message_to_fun(DevID, Msg, Key, Opts) -> message_to_fun(DevID, module(DevID, Opts), Msg, Key, Opts). @@ -144,8 +155,8 @@ message_to_fun(DevID, DevMod, Msg, Key, Opts) -> % Case 6: The device has no default handler. % We retry with the default unless the message % already names it (loop guard). - case message_device_id(Msg, undefined, Opts) of - ?DEFAULT_DEVICE -> + case message_device_id(Msg, Key, Opts) of + {ok, ?DEFAULT_DEVICE} -> throw({ error, default_device_could_not_resolve_key, @@ -180,17 +191,20 @@ module(DevID, Opts) -> %% @doc Return the device ID from a message, resolving through any ancestors %% as necessary. -message_device_id(Msg, Opts) -> - message_device_id(Msg, ?DEFAULT_DEVICE, Opts). -message_device_id(Msg, Default, Opts) -> +message_device_id(Msg, Key, Opts) -> ?event({finding_device_id, Msg}), case hb_maps:find(<<"device">>, Msg, Opts) of - {ok, Device} -> Device; + {ok, Device} -> {ok, Device}; error -> - case hb_maps:find(<<"...">>, Msg, Opts) of - {ok, Ancestor} -> message_device_id(Ancestor, Default, Opts); + case hb_maps:find(Key, Msg, Opts) of + {ok, Value} -> {hit, Value}; error -> - Default + case hb_maps:find(<<"...">>, Msg, Opts) of + {ok, Ancestor} -> + message_device_id(Ancestor, Key, Opts); + error -> + {ok, Default} + end end end. diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 93e3d0ae7..cc3898e48 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -217,20 +217,29 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> % upon the base and request, using the `Base` message's % `device` function. We do this by performing a `primitive` % execution as a recursive call. - execute_context( - #{ - <<"varied-req">> => Req#{ - <<"path">> => <<"vary">>, - <<"vary">> => - hb_maps:get( - <<"path">>, - Req, - undefined, - Opts - ) + {ok, VaryCtxWithResolver} = + hb_device:add_resolver( + #{ + <<"key">> => <<"vary">>, + <<"req">> => Req#{ + <<"path">> => <<"vary">>, + <<"vary">> => + hb_maps:get( + <<"path">>, + Req, + undefined, + Opts + ) + }, + <<"base">> => Base, + <<"primitive">> => true }, - <<"base">> => Base, - <<"primitive">> => true + Opts + ), + execute_context( + VaryCtxWithResolver#{ + <<"varied-base">> => Base, + <<"varied-req">> => Req }, Opts ) @@ -240,11 +249,21 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> {ok, CtxWithRes = #{ <<"result">> := _Result }} -> post_execution(CtxWithRes, Opts); {ok, PreparedCtx} -> - execute_context(PreparedCtx, Opts); + lookup_or_exec(PreparedCtx, Opts); Other -> Other end. +%% @doc Look up a result from the cache and if not found, execute the context. +lookup_or_exec(Ctx = #{ <<"varied-base">> := VBase, <<"varied-req">> := VReq }, Opts) -> + case hb_cache_control:maybe_lookup(VBase, VReq, Opts) of + {hit, Res} -> Res; + _ -> execute_context(Ctx, Opts) + end. + +%% @doc Takes a fully prepated context (with `varied-base` and `varied-req` +%% set, along with `priv/resolver` etc.) and executes it. Does not cache, +%% extend, or otherwise post-process the result. execute_context(Ctx, Opts) -> % Apply the function and return the result directly, without any further % processing. We add the `PrefixArgs` to the list of arguments to be passed @@ -258,8 +277,7 @@ execute_context(Ctx, Opts) -> <<"priv">> := #{ <<"resolver">> := Fun, - <<"add-key">> := AddKey, - <<"exec-opt">> := ExecOpts + <<"add-key">> := AddKey } } = Ctx, {Status, Res} = @@ -268,7 +286,7 @@ execute_context(Ctx, Opts) -> hb_device:truncate_args( Fun, if AddKey -> [Key]; true -> [] end - ++ [VariedBase, VariedReq, ExecOpts] + ++ [VariedBase, VariedReq, Opts] ) ), post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). @@ -514,6 +532,7 @@ resolve_stage(5, Base, Req, ExecName, Opts) -> hb_device:message_to_fun( Base, Key, + <<"message@1.0">>, UserOpts ), ?event( diff --git a/src/core/test/hb_ao_test_vectors.erl b/src/core/test/hb_ao_test_vectors.erl index 1f2e3877f..7bdfc11b8 100644 --- a/src/core/test/hb_ao_test_vectors.erl +++ b/src/core/test/hb_ao_test_vectors.erl @@ -840,7 +840,7 @@ denormalized_device_name_test(Opts) -> ), ?assertEqual( {ok, Dev, maps:get(test_func, Dev)}, - hb_device:message_to_fun(Msg, test_func, Opts) + hb_device:message_to_fun(Msg, test_func, <<"message@1.0">>, Opts) ). denormalized_key_test(Opts) -> From 264d32ba1558103bc578e28bdaeb1ad27cefe72a Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 1 Jul 2026 16:49:16 -0400 Subject: [PATCH 15/35] wip: resolver --- src/core/device/hb_device.erl | 12 +-- src/core/resolver/hb_ao.erl | 121 ++++++++++++++++---------- src/core/resolver/hb_opts.erl | 2 +- src/core/resolver/hb_types.erl | 3 +- src/preloaded/message/dev_message.erl | 8 +- 5 files changed, 93 insertions(+), 53 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index 6714dd237..30306bd98 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -50,6 +50,8 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> {ok, ForcedBaseDevice} -> {ok, ForcedBaseDevice}; error -> message_device_id(Base, Key, Opts) end, + ?prim_dbg({adding_resolver_for, DevRes, Key}), + OldPriv = maps:get(<<"priv">>, Context, #{}), case DevRes of {hit, Value} -> {ok, @@ -65,7 +67,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> <<"base-device">> => DeviceID, <<"resolver-device">> => ExecDev, <<"priv">> => - #{ + OldPriv#{ <<"resolver-module">> => ExecMod, <<"add-key">> => Type == add_key, <<"resolver">> => Fun @@ -191,6 +193,8 @@ module(DevID, Opts) -> %% @doc Return the device ID from a message, resolving through any ancestors %% as necessary. +message_device_id(List, _, _) when is_list(List) -> + {ok, <<"message@1.0">>}; message_device_id(Msg, Key, Opts) -> ?event({finding_device_id, Msg}), case hb_maps:find(<<"device">>, Msg, Opts) of @@ -200,10 +204,8 @@ message_device_id(Msg, Key, Opts) -> {ok, Value} -> {hit, Value}; error -> case hb_maps:find(<<"...">>, Msg, Opts) of - {ok, Ancestor} -> - message_device_id(Ancestor, Key, Opts); - error -> - {ok, Default} + {ok, Ancestor} -> message_device_id(Ancestor, Key, Opts); + error -> {ok, ?DEFAULT_DEVICE} end end end. diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index cc3898e48..a4bef5e60 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -180,15 +180,9 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> ?prim_dbg( {executing, {forced_device, ForcedDevice}, - {forced_key, ForcedKey}, - {req, Req} + {forced_key, ForcedKey} } ), - % If a forced key is provided, use it; otherwise, extract from the request. - Key = - if ForcedKey =/= undefined -> ForcedKey; - true -> hb_path:hd(Req, ExecOpts) - end, % If an explicit device is provided we use it _only on the lookup_ -- not % during execution. We do this by shortcutting the `base-device` field in % the initial context, if `ForcedDevice` is provided. @@ -196,16 +190,31 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> if ForcedDevice =/= undefined -> #{ <<"base-device">> => ForcedDevice }; true -> #{} end, - vary_context( - Ctx0#{ <<"base">> => Base, <<"req">> => Req, <<"key">> => Key }, + % If a forced key is provided, use it; otherwise, extract from the request. + Key = + if ForcedKey =/= undefined -> ForcedKey; + true -> hb_path:hd(Req, ExecOpts) + end, + Res = vary_context( + Ctx0#{ + <<"base">> => Base, + <<"req">> => Req, + <<"key">> => Key, + <<"priv">> => #{ <<"mode">> => <<"raw">> } + }, Opts - ). + ), + ?prim_dbg({execution_complete, + {forced_device, ForcedDevice}, + {forced_key, ForcedKey} + }), + Res. %% @doc Perform the `vary` step of an AO-Core execution. This step requires %% using the device (or the fallback device) to materialize the appropriate %% protocol values from the `Base` and `Request` messages, as well as the %% Erlang function to be executed, in order to evaluate an AO-Core resolution. -vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> +vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> VaryRes = case maps:find(<<"primitive">>, Ctx) of {ok, true} -> @@ -217,34 +226,44 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> % upon the base and request, using the `Base` message's % `device` function. We do this by performing a `primitive` % execution as a recursive call. + MaybeCtxBaseDev = maps:with([<<"base-device">>], Ctx), + MaybeDeviceMsg = + case maps:get(<<"base-device">>, Ctx, undefined) of + undefined -> #{}; + Device -> #{ <<"device">> => Device } + end, + BaseWithDevice = + if is_map(Base) -> maps:merge(Base, MaybeDeviceMsg); + true -> Base + end, {ok, VaryCtxWithResolver} = hb_device:add_resolver( #{ <<"key">> => <<"vary">>, - <<"req">> => Req#{ - <<"path">> => <<"vary">>, - <<"vary">> => - hb_maps:get( - <<"path">>, - Req, - undefined, - Opts - ) - }, - <<"base">> => Base, + <<"req">> => + VaryReq = #{ + <<"path">> => <<"vary">>, + <<"vary">> => Key, + <<"...">> => Req + }, + <<"base">> => BaseWithDevice, <<"primitive">> => true }, Opts ), - execute_context( - VaryCtxWithResolver#{ - <<"varied-base">> => Base, - <<"varied-req">> => Req - }, - Opts - ) + % Call vary using key `vary=Key` + {ok, VariedCtx = #{ <<"priv">> := NewPriv }} = + execute_context( + (maps:merge(VaryCtxWithResolver, MaybeCtxBaseDev))#{ + <<"varied-base">> => BaseWithDevice, + <<"varied-req">> => VaryReq, + <<"return-extends">> => none + }, + Opts + ), + {ok, VariedCtx#{ <<"priv">> => maps:merge(Priv, NewPriv) }} end, - ?prim_dbg({vary_result, VaryRes}), + ?prim_dbg({vary_result, erlang:external_size(VaryRes)}), case VaryRes of {ok, CtxWithRes = #{ <<"result">> := _Result }} -> post_execution(CtxWithRes, Opts); @@ -255,7 +274,16 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> end. %% @doc Look up a result from the cache and if not found, execute the context. -lookup_or_exec(Ctx = #{ <<"varied-base">> := VBase, <<"varied-req">> := VReq }, Opts) -> +lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, Opts) -> + ?prim_dbg({raw_skip_lookup, erlang:external_size(Ctx)}), + execute_context(Ctx, Opts); +lookup_or_exec( + Ctx = #{ + <<"varied-base">> := VBase, + <<"varied-req">> := VReq, + <<"key">> := Key + }, Opts) -> + ?prim_dbg({lookup_or_exec, Key}), case hb_cache_control:maybe_lookup(VBase, VReq, Opts) of {hit, Res} -> Res; _ -> execute_context(Ctx, Opts) @@ -264,22 +292,24 @@ lookup_or_exec(Ctx = #{ <<"varied-base">> := VBase, <<"varied-req">> := VReq }, %% @doc Takes a fully prepated context (with `varied-base` and `varied-req` %% set, along with `priv/resolver` etc.) and executes it. Does not cache, %% extend, or otherwise post-process the result. -execute_context(Ctx, Opts) -> +execute_context( + Ctx = #{ + <<"key">> := Key, + <<"varied-base">> := VariedBase, + <<"varied-req">> := VariedReq, + <<"priv">> := + #{ + <<"resolver">> := Fun, + <<"add-key">> := AddKey + } + }, + Opts) -> % Apply the function and return the result directly, without any further % processing. We add the `PrefixArgs` to the list of arguments to be passed % to the function to accomodate default handlers, which take the key that % was invoked on the device as the first argument (ahead of `Base`, `Req`, % and `ExecOpts`). - #{ - <<"key">> := Key, - <<"varied-base">> := VariedBase, - <<"varied-req">> := VariedReq, - <<"priv">> := - #{ - <<"resolver">> := Fun, - <<"add-key">> := AddKey - } - } = Ctx, + ?prim_dbg({execute_context, Fun}), {Status, Res} = apply( Fun, @@ -297,19 +327,22 @@ post_execution( #{ <<"primitive">> := true, <<"result">> := Res, - <<"status">> := Status + <<"status">> := Status, + <<"key">> := Key }, _Opts) -> + ?prim_dbg({post_exec_primitive, {status, Status}, {key, Key}}), {Status, Res}; post_execution( Context = #{ <<"base">> := Base, - <<"request">> := Req, + <<"req">> := Req, <<"result">> := RawResult, <<"status">> := Status, <<"return-extends">> := Extend }, Opts) -> + ?prim_dbg({post_execution, {extend, Extend}}), maybe_cache(Context, Opts), Res = case Extend of diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl index b1acd6be8..12bc10269 100644 --- a/src/core/resolver/hb_opts.erl +++ b/src/core/resolver/hb_opts.erl @@ -291,7 +291,7 @@ raw_default_message() -> % Every modification to `Opts' called directly by the node operator % should be recorded here. <<"node-history">> => [], - <<"debug-stack-depth">> => 8, + <<"debug-stack-depth">> => 40, <<"debug-print">> => false, <<"debug-log">> => false, <<"log-dir">> => <<"logs">>, diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 9a22bcef4..9e8fe512f 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -42,7 +42,8 @@ add_schema( end. %% @doc Apply the resolved function's base/request schemas to one execution. -vary(Ctx = #{ <<"schema">> := undefined, <<"base">> := Base, <<"req">> := Req }, _Opts) -> +vary(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, _Opts) + when not is_map_key(<<"schema">>, Ctx) -> {ok, Ctx#{ <<"varied-base">> => Base, diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index f74faa2ea..af087f94b 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -691,11 +691,15 @@ default_accessor(Key, Msg, Req, Opts) -> %% applies it to the inputs. Returns `base` and `request` submessages, as well %% as the resolvable function Erlang function at `priv/function` if it was found %% during the process of `vary`ing. --spec vary(#{ _ => _ }, #{ vary => binary(), _ => _ }, #{}) -> {ok, #{}}. +-spec vary( + #{ _ => _ }, + #{ vary => binary(), '...' => #{ _ => _ }, _ => _ }, + #{}) -> {ok, #{}}. vary(Base, Req, Opts) -> maybe {ok, Key} ?= hb_maps:find(<<"vary">>, Req, Opts), - Ctx0 = #{ <<"base">> => Base, <<"key">> => Key }, + {ok, VaryReq} ?= hb_maps:find(<<"...">>, Req, Opts), + Ctx0 = #{ <<"base">> => Base, <<"key">> => Key, <<"req">> => VaryReq }, case hb_device:add_resolver(Ctx0, Opts) of {ok, Ctx1 = #{ <<"result">> := _ }} -> % Finding the resolver led us to first find the complete From 7262ba7d85213e7d828aec0e40dbed8b3d6ca27c Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 1 Jul 2026 19:16:18 -0400 Subject: [PATCH 16/35] wip: resolution caching --- src/core/resolver/hb_ao.erl | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index a4bef5e60..c3b49b093 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -285,7 +285,11 @@ lookup_or_exec( }, Opts) -> ?prim_dbg({lookup_or_exec, Key}), case hb_cache_control:maybe_lookup(VBase, VReq, Opts) of - {hit, Res} -> Res; + {hit, {Status, Msg}} -> + post_execution( + Ctx#{ <<"result">> => Msg, <<"status">> => Status }, + Opts + ); _ -> execute_context(Ctx, Opts) end. @@ -323,6 +327,11 @@ execute_context( %% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return %% the result directly; else, apply the `return-extends` schema to the result. +%% Handles the three core variants via two separate function heads: +%% 1. `primitive` mode: return the result directly +%% 2a. `raw` mode: Apply the `return-extends` schema to the result, don't store. +%% 2b. `normal` mode: Apply the `return-extends` schema to the result, store in +%% the cache if `cache-control` permits. post_execution( #{ <<"primitive">> := true, @@ -354,8 +363,13 @@ post_execution( maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> skipped_caching; -maybe_cache(_Context, _Opts) -> - todo. +maybe_cache( + #{ + <<"varied-base">> := Base, + <<"request">> := Req, + <<"result">> := Res + }, Opts) -> + hb_cache_control:maybe_store(Base, Req, Res, Opts). %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, From aaec66c447d3b76120ace311aa9454bf14a397f2 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 2 Jul 2026 11:59:44 -0400 Subject: [PATCH 17/35] wip: AO-Core 1.0 resolver implementation --- src/core/resolver/hb_ao.erl | 119 +++++++++++++++------ src/core/resolver/hb_cache.erl | 64 ++--------- src/core/resolver/hb_cache_control.erl | 50 ++++----- src/preloaded/codec/dev_structured.erl | 7 +- src/preloaded/codec/lib_arweave_common.erl | 8 +- src/preloaded/util/dev_math.erl | 6 +- 6 files changed, 132 insertions(+), 122 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index c3b49b093..8278fba78 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -162,6 +162,16 @@ resolve(Base, Req, Opts) -> } ), resolve_many([Base | MessagesToExec], Opts). + +resolve_single(Base, Req, Opts) -> + vary_context( + #{ + <<"base">> => Base, + <<"req">> => Req, + <<"key">> => hb_path:hd(Req, Opts) + }, + Opts + ). %% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring %% normalization, cache, hashpath, worker, and other management components. @@ -195,29 +205,44 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> if ForcedKey =/= undefined -> ForcedKey; true -> hb_path:hd(Req, ExecOpts) end, - Res = vary_context( - Ctx0#{ - <<"base">> => Base, - <<"req">> => Req, - <<"key">> => Key, - <<"priv">> => #{ <<"mode">> => <<"raw">> } - }, - Opts - ), + Res = + vary_context( + Ctx0#{ + <<"base">> => Base, + <<"req">> => Req, + <<"key">> => Key, + <<"priv">> => #{ <<"mode">> => <<"raw">> } + }, + Opts + ), ?prim_dbg({execution_complete, {forced_device, ForcedDevice}, {forced_key, ForcedKey} }), Res. +primitive(Base, Req, Opts) -> + vary_context( + #{ + <<"base">> => Base, + <<"req">> => Req, + <<"varied-base">> => Base, + <<"varied-req">> => Req, + <<"key">> => hb_path:hd(Req, Opts), + <<"priv">> => #{ <<"mode">> => <<"primitive">> } + }, + Opts + ). + %% @doc Perform the `vary` step of an AO-Core execution. This step requires %% using the device (or the fallback device) to materialize the appropriate %% protocol values from the `Base` and `Request` messages, as well as the %% Erlang function to be executed, in order to evaluate an AO-Core resolution. -vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> +vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key }, Opts) -> + Priv = hb_private:from_message(Ctx), VaryRes = - case maps:find(<<"primitive">>, Ctx) of - {ok, true} -> + case maps:find(<<"mode">>, Priv) of + {ok, <<"primitive">>} -> % Skip the varying step during `primitive` execution, but still % find the resolver and add it to the context. hb_device:add_resolver(Ctx, Opts); @@ -247,7 +272,7 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, << <<"...">> => Req }, <<"base">> => BaseWithDevice, - <<"primitive">> => true + <<"priv">> => #{ <<"mode">> => <<"primitive">> } }, Opts ), @@ -270,12 +295,12 @@ vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, << {ok, PreparedCtx} -> lookup_or_exec(PreparedCtx, Opts); Other -> - Other + throw({unexpected_vary_result, Other}) end. %% @doc Look up a result from the cache and if not found, execute the context. -lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, Opts) -> - ?prim_dbg({raw_skip_lookup, erlang:external_size(Ctx)}), +lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := SkipMode } }, Opts) + when SkipMode =:= <<"raw">>; SkipMode =:= <<"primitive">> -> execute_context(Ctx, Opts); lookup_or_exec( Ctx = #{ @@ -284,13 +309,28 @@ lookup_or_exec( <<"key">> := Key }, Opts) -> ?prim_dbg({lookup_or_exec, Key}), - case hb_cache_control:maybe_lookup(VBase, VReq, Opts) of - {hit, {Status, Msg}} -> + {ok, VBaseID} = primitive(VBase, #{ <<"path">> => <<"id">> }, Opts), + {ok, VReqID} = primitive(VReq, #{ <<"path">> => <<"id">> }, Opts), + CtxWithIDs = + Ctx#{ + <<"varied-base-id">> => VBaseID, + <<"varied-req-id">> => VReqID + }, + Priv = maps:get(<<"priv">>, Ctx, #{}), + case hb_cache_control:maybe_lookup(VBaseID, VReqID, Opts) of + {Status = ok, Msg} -> + ?prim_dbg({cache_hit, Key, {Status, Msg}}), post_execution( - Ctx#{ <<"result">> => Msg, <<"status">> => Status }, + CtxWithIDs#{ + <<"result">> => Msg, + <<"status">> => Status, + <<"priv">> => Priv#{ <<"source">> => <<"cached">> } + }, Opts ); - _ -> execute_context(Ctx, Opts) + Miss -> + ?prim_dbg({cache_miss, Key, Miss}), + execute_context(CtxWithIDs, Opts) end. %% @doc Takes a fully prepated context (with `varied-base` and `varied-req` @@ -302,7 +342,7 @@ execute_context( <<"varied-base">> := VariedBase, <<"varied-req">> := VariedReq, <<"priv">> := - #{ + Priv = #{ <<"resolver">> := Fun, <<"add-key">> := AddKey } @@ -323,7 +363,14 @@ execute_context( ++ [VariedBase, VariedReq, Opts] ) ), - post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). + post_execution( + Ctx#{ + <<"result">> => Res, + <<"status">> => Status, + <<"priv">> => Priv#{ <<"source">> => <<"executed">> } + }, + Opts + ). %% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return %% the result directly; else, apply the `return-extends` schema to the result. @@ -334,16 +381,17 @@ execute_context( %% the cache if `cache-control` permits. post_execution( #{ - <<"primitive">> := true, + <<"priv">> := #{ <<"mode">> := <<"primitive">> }, <<"result">> := Res, <<"status">> := Status, <<"key">> := Key }, _Opts) -> - ?prim_dbg({post_exec_primitive, {status, Status}, {key, Key}}), + ?prim_dbg({post_exec, primitive, {key, Key}}), {Status, Res}; post_execution( Context = #{ + <<"key">> := Key, <<"base">> := Base, <<"req">> := Req, <<"result">> := RawResult, @@ -351,7 +399,7 @@ post_execution( <<"return-extends">> := Extend }, Opts) -> - ?prim_dbg({post_execution, {extend, Extend}}), + ?prim_dbg({post_execution, non_primitive, {key, Key}, {extend, Extend}}), maybe_cache(Context, Opts), Res = case Extend of @@ -361,15 +409,21 @@ post_execution( end, {Status, Res}. +maybe_cache(#{ <<"priv">> := #{ <<"source">> := <<"cached">> } }, _Opts) -> + skipped_caching; maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> skipped_caching; maybe_cache( #{ - <<"varied-base">> := Base, - <<"request">> := Req, + <<"key">> := Key, + <<"varied-base">> := VBase, + <<"varied-req">> := VReq, + <<"varied-base-id">> := VBaseID, + <<"varied-req-id">> := VReqID, <<"result">> := Res }, Opts) -> - hb_cache_control:maybe_store(Base, Req, Res, Opts). + ?prim_dbg({maybe_cache, Key}), + hb_cache_control:maybe_store(VBaseID, VReqID, VBase, VReq, Res, Opts). %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, @@ -410,12 +464,12 @@ resolve_many(MsgList, Opts) -> Res. do_resolve_many([], _Opts) -> {failure, <<"Attempted to resolve an empty message sequence.">>}; -do_resolve_many([Res], Opts) -> - ?event_debug(debug_ao_core, {stage, 11, resolve_complete, Res}), - hb_cache:ensure_loaded(Res, Opts); +do_resolve_many([Res], _Opts) -> + ?event_debug(debug_ao_core, {stage, 11, resolve_complete, Res}, _Opts), + {ok, Res}; do_resolve_many([Base, Req | MsgList], Opts) -> ?event_debug(debug_ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), - case resolve_stage(1, Base, Req, Opts) of + case resolve_single(Base, Req, Opts) of {ok, Res} -> ?event_debug(debug_ao_core, { @@ -429,6 +483,7 @@ do_resolve_many([Base, Req | MsgList], Opts) -> ), do_resolve_many([Res | MsgList], Opts); Res -> + throw({unexpected_resolve_many_response, Res}), % The result is not a resolvable message. Return it. ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), Res diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index 59e972a23..f4ba51d4c 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -1010,15 +1010,12 @@ types_to_implicit(Types) -> structured_decode_types(Types, Opts) -> hb_util:ok( hb_ao:raw( - <<"structured@1.0">>, - <<"decode-types">>, - Types, - #{}, + #{ <<"device">> => <<"structured@1.0">>, <<"ao-types">> => Types }, + #{ <<"path">> => <<"decode-types">> }, Opts ) ). - %% @doc Read the result of a computation, using heuristics. The supported %% heuristics are as follows: %% 1. If the base message is an ID, we try to determine if the message has an @@ -1029,56 +1026,13 @@ structured_decode_types(Types, Opts) -> %% message and return it if it exists. %% 3. If the message has an explicit device, we attempt to read the hashpath to %% see if it has already been computed. -read_resolved(BaseMsg, Key, Opts) when is_binary(Key) -> - read_resolved(BaseMsg, #{ <<"path">> => Key }, Opts); -read_resolved({link, ID, LinkOpts}, Req, Opts) -> - read_resolved(ID, Req, maps:merge(LinkOpts, Opts)); -read_resolved(BaseMsgID, Req = #{ <<"path">> := Key }, Opts) when ?IS_ID(BaseMsgID) -> - Store = hb_opts:get(store, no_viable_store, Opts), - _NormKey = hb_ao:normalize_key(Key, Opts), - case hb_device:is_direct_key_access(BaseMsgID, Req, Opts, Store) of - unknown -> miss; - false -> - ?event_debug(read_cached, - {found_non_message_device, - {key, _NormKey} - } - ), - read_hashpath(BaseMsgID, Req, Opts); - true -> - % Either the message does not exist in the store, or there is no - % explicit device in the message. If the message exists this implies - % that the default (`message@1.0`) device will be used to execute - % the key. Subsequently, we can simply read the key and return it if - % it exists. - ?event_debug(read_cached, - {skipping_execution_store_lookup, - {base_msg, BaseMsgID}, - {key, _NormKey} - } - ), - case hb_store:resolve(Store, [BaseMsgID, Key], Opts) of - {ok, KeyPath} -> hashpath_read_result(read(KeyPath, Opts)); - {error, not_found} -> miss; - Other -> {hit, Other} - end - end; -read_resolved(BaseMsg, Req = #{ <<"path">> := Key }, Opts) when is_map(BaseMsg) -> - % The base message is loaded, so we determine if it has an explicit device - % and perform a direct lookup if it does not. - NormKey = hb_ao:normalize_key(Key, Opts), - case hb_device:is_direct_key_access(BaseMsg, Req, Opts) of - false -> read_hashpath(BaseMsg, Req, Opts); - true -> - ?event_debug(read_cached, - {skip_execution_memory_lookup, - {path, NormKey} - } - ), - {hit, read_in_memory_key(BaseMsg, NormKey, Opts)} - end; -read_resolved(Base, Req, Opts) -> - read_hashpath(Base, Req, Opts). +read_resolved(BaseID, ReqID, Opts) -> + ExpectedPath = <>, + ?prim_dbg({read_resolved, ExpectedPath}), + case read(ExpectedPath, Opts) of + {error, not_found} -> miss; + Other -> {hit, Other} + end. %% @doc Return a key from an in-memory message, returning the same form as %% a store read (`{Status, Value}'). diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 85746bf0d..baa10b983 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/4, maybe_lookup/3]). +-export([maybe_store/6, maybe_lookup/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -21,11 +21,13 @@ %% 3. The `Req' message (the user's request). %% Base is not used, such that it can specify cache control information about %% itself, without affecting its outputs. -maybe_store(Base, Req, Res, Opts) -> +maybe_store(BaseID, ReqID, Base, Req, Res, Opts) -> case derive_cache_settings([Res, Req], Opts) of #{ <<"store">> := true } -> - ?event(caching, {caching_result, {base, Base}, {req, Req}, {res, Res}}), - dispatch_cache_write(Base, Req, Res, Opts); + ?prim_dbg({caching_result, {base, Base}, {req, Req}, {res, Res}}), + dispatch_cache_write(Base, Req, Res, Opts), + ?prim_dbg({cached_result, {res, Res}}), + ok; _ -> not_caching end. @@ -74,27 +76,8 @@ lookup(Base, Req, Opts) -> #{ <<"only-if-cached">> := true } -> only_if_cached_not_found_error(Base, Req, Opts); _ -> - case ?IS_ID(Base) of - false -> {continue, Base, Req}; - true -> - case hb_cache:read(Base, Opts) of - {ok, FullBase} -> - ?event(load_message, - {cache_hit_base_message_load, - {base_id, Base}, - {base_loaded, FullBase} - } - ), - {continue, FullBase, Req}; - not_found -> - necessary_messages_not_found_error( - Base, - Req, - Opts - ) - end - end - end + {error, not_found} + end end end. @@ -132,8 +115,8 @@ async_writer() -> %% @doc Internal function to write a compute result to the cache. perform_cache_write(Base, Req, Res, Opts) -> - hb_cache:write(Base, Opts), - hb_cache:write(Req, Opts), + {ok, BaseID} = hb_cache:write(Base, Opts), + {ok, ReqID} = hb_cache:write(Req, Opts), case Res of <<_/binary>> -> hb_cache:write_binary( @@ -141,8 +124,17 @@ perform_cache_write(Base, Req, Res, Opts) -> Res, Opts ); - Map when is_map(Map) -> - hb_cache:write(Res, Opts); + Msg when is_map(Msg) -> + ?prim_dbg(caching_result_msg), + {ok, ResID} = hb_cache:write(Res, Opts), + LinkPath = <>, + hb_cache:link( + ResID, + LinkPath, + Opts + ), + ?prim_dbg({linked, LinkPath, ResID}), + ok; _ -> ?event({cannot_write_result, Res}), skip_caching diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index c3955b48b..61807b3d8 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -285,8 +285,11 @@ encode_ao_types(Types, _Opts) -> )). %% @doc Device key for parsing an `ao-types' field. -decode_types(Base, Req, Opts) -> - {ok, decode_ao_types(hb_maps:get(<<"body">>, Req, Base, Opts), Opts)}. +-spec decode_types(#{ ao_types => binary() | map() }, _, _) -> {ok, #{}}. +decode_types(#{ <<"ao-types">> := Types }, _Req, Opts) -> + {ok, decode_ao_types(Types, Opts)}; +decode_types(_BaseWithoutTypes, _Req, _Opts) -> + {ok, #{}}. %% @doc Parse the `ao-types' field of a TABM if present, and return a map of %% keys and their types. If the given value is a list, we return an empty map diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index 755b9492c..f1859cd72 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -124,10 +124,12 @@ ao_types(#{ <<"ao-types">> := AoTypes } = Tags, Opts) -> ConvOpts = Opts#{ <<"hashpath">> => ignore }, {ok, AOTypes} = hb_ao:resolve( - #{ <<"device">> => <<"structured@1.0">> }, #{ - <<"path">> => <<"decode-types">>, - <<"body">> => AoTypes + <<"device">> => <<"structured@1.0">>, + <<"ao-types">> => AoTypes + }, + #{ + <<"path">> => <<"decode-types">> }, ConvOpts ), diff --git a/src/preloaded/util/dev_math.erl b/src/preloaded/util/dev_math.erl index 772cd7941..c51eef708 100644 --- a/src/preloaded/util/dev_math.erl +++ b/src/preloaded/util/dev_math.erl @@ -1,9 +1,12 @@ %%% @doc Simple test device with math utilities. Not intended for production use. -module(dev_math). +-include("include/hb.hrl"). -export([inc_x/3, dec_x/3, add_x/3, sum/3, with_sum/3]). -spec inc_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. -inc_x(#{ <<"x">> := X }, _, _) -> {ok, #{ <<"x">> => X + 1 }}. +inc_x(#{ <<"x">> := X }, _, _) -> + ?prim_dbg({inc_x_called, X}), + {ok, #{ <<"x">> => X + 1 }}. -spec dec_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. dec_x(#{ <<"x">> := X }, _, _) -> {ok, #{ <<"x">> => X - 1 }}. @@ -21,4 +24,5 @@ sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> -spec with_sum(#{ x := integer(), y := integer() }, _, _) -> {ok, #{ sum => integer(), '...' => base }}. with_sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> + ?prim_dbg({doing_sum, {x, X}, {y, Y}}), {ok, #{ <<"sum">> => X + Y }}. \ No newline at end of file From 2e762a6635a6e8eba99f7161a76d78ddb98da405 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 8 Jul 2026 16:59:13 -0400 Subject: [PATCH 18/35] wip: AO-Core 1.0 resolver re-org --- erlang_ls.config | 2 - src/core/device/hb_device.erl | 27 ++-- src/core/resolver/hb_ao.erl | 182 +++++++++++-------------- src/core/resolver/hb_cache_control.erl | 4 - src/preloaded/codec/dev_structured.erl | 2 +- src/preloaded/message/dev_message.erl | 12 ++ 6 files changed, 105 insertions(+), 124 deletions(-) diff --git a/erlang_ls.config b/erlang_ls.config index f5621bee0..781836ef9 100644 --- a/erlang_ls.config +++ b/erlang_ls.config @@ -6,8 +6,6 @@ diagnostics: apps_dirs: - "src" - "src/*" -include_dirs: - - "src/include" include_dirs: - "src" - "src/include" diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index 30306bd98..f7d2bafd7 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -34,11 +34,11 @@ truncate_args(Fun, Args) -> %% #{ %% base: The base message, unvaried. %% key: The key to be resolved in the execution. -%% base-device: The device derived from the message itself. +%% forced-device: The device derived from the message itself. %% priv/exec-device: The device from which the execution function originates. -%% If the `base-device` resolvers from another device, -%% this key will differ from the `base-device`. -%% priv/resolver: The function to execute to resolve the `base-device`. +%% If the `forced-device` resolvers from another device, +%% this key will differ from the `forced-device`. +%% priv/resolver: The function to execute to resolve the `forced-device`. %% priv/add-key: Whether the execution function expects that we should %% add the `key` as an additional argument to the start of %% the argument list. @@ -46,11 +46,10 @@ truncate_args(Fun, Args) -> add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> % TODO: What if we force a device _and_ have a direct key hit? DevRes = - case maps:find(<<"base-device">>, Context) of + case maps:find(<<"forced-device">>, Context) of {ok, ForcedBaseDevice} -> {ok, ForcedBaseDevice}; error -> message_device_id(Base, Key, Opts) end, - ?prim_dbg({adding_resolver_for, DevRes, Key}), OldPriv = maps:get(<<"priv">>, Context, #{}), case DevRes of {hit, Value} -> @@ -64,7 +63,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> message_to_fun(DeviceID, Base, Key, Opts), {ok, Context#{ - <<"base-device">> => DeviceID, + <<"forced-device">> => DeviceID, <<"resolver-device">> => ExecDev, <<"priv">> => OldPriv#{ @@ -157,13 +156,13 @@ message_to_fun(DevID, DevMod, Msg, Key, Opts) -> % Case 6: The device has no default handler. % We retry with the default unless the message % already names it (loop guard). - case message_device_id(Msg, Key, Opts) of - {ok, ?DEFAULT_DEVICE} -> - throw({ - error, - default_device_could_not_resolve_key, - {key, Key} - }); + case DevID of + ?DEFAULT_DEVICE -> + throw({ + error, + default_device_could_not_resolve_key, + {key, Key} + }); _ -> WithDev = with_device(Msg, ?DEFAULT_DEVICE), message_to_fun( diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 8278fba78..6cbe2fc3a 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -94,7 +94,7 @@ -module(hb_ao). %%% Main AO-Core API: -export([resolve/2, resolve/3]). --export([raw/3, raw/4, raw/5]). +-export([raw/3, raw/4, raw/5, primitive/5]). -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). %%% Shortcuts and tools: -export([keys/2]). @@ -170,7 +170,7 @@ resolve_single(Base, Req, Opts) -> <<"req">> => Req, <<"key">> => hb_path:hd(Req, Opts) }, - Opts + Opts#{ <<"mode">> => <<"normal">> } ). %% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring @@ -186,109 +186,99 @@ raw(Base, Req, Opts) -> raw(Device, Base, Req, Opts) -> raw(Device, undefined, Base, Req, Opts). raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> - ExecOpts = execution_opts(Opts), ?prim_dbg( {executing, {forced_device, ForcedDevice}, {forced_key, ForcedKey} } ), - % If an explicit device is provided we use it _only on the lookup_ -- not - % during execution. We do this by shortcutting the `base-device` field in - % the initial context, if `ForcedDevice` is provided. - Ctx0 = - if ForcedDevice =/= undefined -> #{ <<"base-device">> => ForcedDevice }; - true -> #{} - end, - % If a forced key is provided, use it; otherwise, extract from the request. - Key = - if ForcedKey =/= undefined -> ForcedKey; - true -> hb_path:hd(Req, ExecOpts) - end, - Res = - vary_context( - Ctx0#{ + vary_context( + forced_context( + ForcedDevice, + ForcedKey, + #{ <<"base">> => Base, <<"req">> => Req, - <<"key">> => Key, <<"priv">> => #{ <<"mode">> => <<"raw">> } }, Opts ), - ?prim_dbg({execution_complete, - {forced_device, ForcedDevice}, - {forced_key, ForcedKey} - }), - Res. + Opts#{ <<"mode">> => <<"raw">> } + ). + +%% @doc Executes a context in primitive mode: Skipping cache lookup and +%% varying. Caution: Does not produce cryptographically valid results in all +%% cases! Should not be used as a normal AO-Core resolver. +primitive(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> + {ok, CtxWithResolver} = + hb_device:add_resolver( + Ctx#{ + <<"varied-base">> => Base, + <<"varied-req">> => Req, + <<"priv">> => #{ <<"mode">> => <<"primitive">> } + }, + Opts + ), + execute_context(CtxWithResolver, Opts). +primitive(ForcedDevice, ForceKey, Ctx, Opts) -> + primitive(forced_context(ForcedDevice, ForceKey, Ctx, Opts), Opts). primitive(Base, Req, Opts) -> - vary_context( + primitive(undefined, undefined, Base, Req, Opts). +primitive(ForcedDevice, ForcedKey, Base, Req, Opts) -> + primitive( + ForcedDevice, + ForcedKey, #{ <<"base">> => Base, <<"req">> => Req, - <<"varied-base">> => Base, - <<"varied-req">> => Req, - <<"key">> => hb_path:hd(Req, Opts), - <<"priv">> => #{ <<"mode">> => <<"primitive">> } + <<"key">> => hb_path:hd(Req, Opts) }, Opts ). +%% @doc Apply `forced` device and key elements to the context when they are +%% provided, otherwise return the context as-is. For use only in `primitive` +%% or `raw` execution modes, where we are not attesting to the result of the +%% computation. +forced_context(undefined, undefined, Ctx, _) -> Ctx; +forced_context(ForcedDevice, Key, Ctx, Opts) when ForcedDevice =/= undefined -> + forced_context( + undefined, + Key, + Ctx#{ <<"forced-device">> => ForcedDevice }, + Opts + ); +forced_context(undefined, ForcedKey, Ctx, _Opts) when ForcedKey =/= undefined -> + Ctx#{ <<"key">> => ForcedKey }. + %% @doc Perform the `vary` step of an AO-Core execution. This step requires %% using the device (or the fallback device) to materialize the appropriate %% protocol values from the `Base` and `Request` messages, as well as the %% Erlang function to be executed, in order to evaluate an AO-Core resolution. -vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key }, Opts) -> - Priv = hb_private:from_message(Ctx), - VaryRes = - case maps:find(<<"mode">>, Priv) of - {ok, <<"primitive">>} -> - % Skip the varying step during `primitive` execution, but still - % find the resolver and add it to the context. - hb_device:add_resolver(Ctx, Opts); - _ -> - % First resolve the `vary` operation for the device and key - % upon the base and request, using the `Base` message's - % `device` function. We do this by performing a `primitive` - % execution as a recursive call. - MaybeCtxBaseDev = maps:with([<<"base-device">>], Ctx), - MaybeDeviceMsg = - case maps:get(<<"base-device">>, Ctx, undefined) of - undefined -> #{}; - Device -> #{ <<"device">> => Device } - end, - BaseWithDevice = - if is_map(Base) -> maps:merge(Base, MaybeDeviceMsg); - true -> Base - end, - {ok, VaryCtxWithResolver} = - hb_device:add_resolver( - #{ - <<"key">> => <<"vary">>, - <<"req">> => - VaryReq = #{ - <<"path">> => <<"vary">>, - <<"vary">> => Key, - <<"...">> => Req - }, - <<"base">> => BaseWithDevice, - <<"priv">> => #{ <<"mode">> => <<"primitive">> } - }, - Opts - ), - % Call vary using key `vary=Key` - {ok, VariedCtx = #{ <<"priv">> := NewPriv }} = - execute_context( - (maps:merge(VaryCtxWithResolver, MaybeCtxBaseDev))#{ - <<"varied-base">> => BaseWithDevice, - <<"varied-req">> => VaryReq, - <<"return-extends">> => none - }, - Opts - ), - {ok, VariedCtx#{ <<"priv">> => maps:merge(Priv, NewPriv) }} - end, +vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> + VaryRes = + primitive( + Ctx#{ + <<"key">> => <<"vary">>, + <<"base">> => Base, + <<"req">> => + #{ + <<"path">> => <<"vary">>, + <<"vary">> => Key, + <<"...">> => Req + } + }, + Opts#{ <<"mode">> => hb_opts:get(<<"mode">>, <<"raw">>, Opts) } + ), ?prim_dbg({vary_result, erlang:external_size(VaryRes)}), + ?prim_dbg( + { + priv_vary, + {before, hb_private:from_message(Ctx)}, + {'after', hb_private:from_message(VaryRes)} + } + ), case VaryRes of {ok, CtxWithRes = #{ <<"result">> := _Result }} -> post_execution(CtxWithRes, Opts); @@ -306,9 +296,10 @@ lookup_or_exec( Ctx = #{ <<"varied-base">> := VBase, <<"varied-req">> := VReq, - <<"key">> := Key + <<"key">> := Key, + <<"priv">> := Priv }, Opts) -> - ?prim_dbg({lookup_or_exec, Key}), + ?prim_dbg({priv_during_lookup, Priv}), {ok, VBaseID} = primitive(VBase, #{ <<"path">> => <<"id">> }, Opts), {ok, VReqID} = primitive(VReq, #{ <<"path">> => <<"id">> }, Opts), CtxWithIDs = @@ -316,16 +307,11 @@ lookup_or_exec( <<"varied-base-id">> => VBaseID, <<"varied-req-id">> => VReqID }, - Priv = maps:get(<<"priv">>, Ctx, #{}), case hb_cache_control:maybe_lookup(VBaseID, VReqID, Opts) of - {Status = ok, Msg} -> + {hit, {Status, Msg}} -> ?prim_dbg({cache_hit, Key, {Status, Msg}}), post_execution( - CtxWithIDs#{ - <<"result">> => Msg, - <<"status">> => Status, - <<"priv">> => Priv#{ <<"source">> => <<"cached">> } - }, + CtxWithIDs#{ <<"result">> => Msg, <<"status">> => Status }, Opts ); Miss -> @@ -342,7 +328,7 @@ execute_context( <<"varied-base">> := VariedBase, <<"varied-req">> := VariedReq, <<"priv">> := - Priv = #{ + #{ <<"resolver">> := Fun, <<"add-key">> := AddKey } @@ -353,7 +339,7 @@ execute_context( % to the function to accomodate default handlers, which take the key that % was invoked on the device as the first argument (ahead of `Base`, `Req`, % and `ExecOpts`). - ?prim_dbg({execute_context, Fun}), + ?prim_dbg({execute, Fun}), {Status, Res} = apply( Fun, @@ -363,14 +349,7 @@ execute_context( ++ [VariedBase, VariedReq, Opts] ) ), - post_execution( - Ctx#{ - <<"result">> => Res, - <<"status">> => Status, - <<"priv">> => Priv#{ <<"source">> => <<"executed">> } - }, - Opts - ). + post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). %% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return %% the result directly; else, apply the `return-extends` schema to the result. @@ -387,7 +366,7 @@ post_execution( <<"key">> := Key }, _Opts) -> - ?prim_dbg({post_exec, primitive, {key, Key}}), + ?prim_dbg({completed_exec, primitive, {key, Key}}), {Status, Res}; post_execution( Context = #{ @@ -399,7 +378,7 @@ post_execution( <<"return-extends">> := Extend }, Opts) -> - ?prim_dbg({post_execution, non_primitive, {key, Key}, {extend, Extend}}), + ?prim_dbg({completed_exec, normal, {key, Key}, {extend, Extend}}), maybe_cache(Context, Opts), Res = case Extend of @@ -409,8 +388,6 @@ post_execution( end, {Status, Res}. -maybe_cache(#{ <<"priv">> := #{ <<"source">> := <<"cached">> } }, _Opts) -> - skipped_caching; maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> skipped_caching; maybe_cache( @@ -422,7 +399,6 @@ maybe_cache( <<"varied-req-id">> := VReqID, <<"result">> := Res }, Opts) -> - ?prim_dbg({maybe_cache, Key}), hb_cache_control:maybe_store(VBaseID, VReqID, VBase, VReq, Res, Opts). %% @doc Resolve a list of messages in sequence. Take the output of the first diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index baa10b983..9bad9988b 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -24,9 +24,7 @@ maybe_store(BaseID, ReqID, Base, Req, Res, Opts) -> case derive_cache_settings([Res, Req], Opts) of #{ <<"store">> := true } -> - ?prim_dbg({caching_result, {base, Base}, {req, Req}, {res, Res}}), dispatch_cache_write(Base, Req, Res, Opts), - ?prim_dbg({cached_result, {res, Res}}), ok; _ -> not_caching @@ -125,7 +123,6 @@ perform_cache_write(Base, Req, Res, Opts) -> Opts ); Msg when is_map(Msg) -> - ?prim_dbg(caching_result_msg), {ok, ResID} = hb_cache:write(Res, Opts), LinkPath = <>, hb_cache:link( @@ -133,7 +130,6 @@ perform_cache_write(Base, Req, Res, Opts) -> LinkPath, Opts ), - ?prim_dbg({linked, LinkPath, ResID}), ok; _ -> ?event({cannot_write_result, Res}), diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index 61807b3d8..61177a5ae 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -202,7 +202,7 @@ apply_bundle_hint(Msg, Req, Opts) -> case hb_maps:get(<<"hint-device">>, Req, undefined, Opts) of undefined -> Req; DeviceBin -> - case hb_ao:raw(DeviceBin, <<"to-hint">>, Msg, Req, Opts) of + case hb_ao:primitive(DeviceBin, <<"to-hint">>, Msg, Req, Opts) of {ok, HintedReq} -> % May add a `bundle` key to the request HintedReq; diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index af087f94b..dfbae6515 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -22,6 +22,14 @@ info() -> default => fun default_accessor/4 }. +% router(#{ <<"path">> := Key }, #{ <<"path">> := <<"path">> }, _Opts) -> {ok, Key}; +% router(_, #{ <<"path">> := <<"path">> }, _Opts) -> {error, not_found}; +% router(Base, Req, Opts) -> +% case hb_ao:raw(Req, #{ <<"path">> => <<"path">> }, Opts) of +% {error, not_found} -> set(Base, Req, Opts); +% {ok, Key} -> default_accessor(Key, Base, Req, Opts) +% end. + %% @doc Generate an index page for a message, in the event that the `body' and %% `content-type' of a message returned to the client are both empty. We do this %% as follows: @@ -711,6 +719,10 @@ vary(Base, Req, Opts) -> hb_types:vary(Ctx1, Opts); Err -> Err end + else + error -> + ?prim_dbg({vary_error, {base, Base}, {req, Req}}), + throw({invalid_vary_call, {base, Base}, {req, Req}}) end. %% @doc Returns the device schema for a `Base` message. From 56cc23d6cf09f06e09cbc4fad95d7c60b007557f Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 8 Jul 2026 22:27:59 -0400 Subject: [PATCH 19/35] wip: resolver --- src/core/resolver/hb_ao.erl | 616 ++++++++++++++++++++---------------- 1 file changed, 345 insertions(+), 271 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 6cbe2fc3a..dbbd069dc 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -114,23 +114,41 @@ ] ). -%% @doc Get the value of an AO-Core message's key by running its associated device -%% function. Optionally, takes options that control the runtime environment. -%% This function returns the raw result of the device function call: -%% `{ok | error, NewMessage}.' -%% The resolver is composed of a series of discrete phases: -%% 1: Normalization and loading of IDs/links. -%% 2: Device or direct key lookup. -%% 3: Vary `Base` and `Request`. -%% 4: Cache lookup -%% 5: Persistent resolver lookup. -%% 6: Execution. -%% 7: Execution of the `step' hook. -%% 8: Result caching. -%% 9: Extension of result upon original base. -%% 10: Notify waiters. -%% 11: Fork worker. -%% 12: Recurse or terminate. +%% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring +%% normalization, cache, hashpath, worker, and other management components. +%% This function comes in `/3`-`/5` variants, allowing the caller to optionally +%% specify that a specific device must be used for the execution (as if +%% `Base/device: Device` was set), or to force a specific key to be used. +%% Critically, the modifiers do _not_ affect the message that the call is +%% executed upon: `Base`, `Req`, and `Opts` are passed directly to the execution +%% function as-is, but the chosen function is altered by these modifiers. +raw(Base, Req, Opts) -> + raw(undefined, Base, Req, Opts). +raw(Device, Base, Req, Opts) -> + raw(Device, undefined, Base, Req, Opts). +raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> + ?prim_dbg( + {executing, + {forced_device, ForcedDevice}, + {forced_key, ForcedKey} + } + ), + do( + execution_context( + ForcedDevice, + ForcedKey, + Base, + Req, + Opts#{ + <<"cache-control">> => <<"none">>, + <<"hashpath">> => ignore + } + ) + ). + +%% @doc Resolve a single AO-Core base and request pair or singleton message. +%% If the `path` in the request has multiple parts, each is executed in sequence, +%% over the original base message. resolve(MsgSeq, Opts) when is_list(MsgSeq) -> resolve_many(MsgSeq, Opts); resolve(Path, Opts) when is_binary(Path) -> @@ -141,10 +159,6 @@ resolve(SingleResult, _Opts) {ok, SingleResult}; resolve(SingletonMsg, Opts) -> resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). - -%% @doc Resolve a single AO-Core base and request pair, If the `path` in the -%% request has multiple parts, each is executed in sequence, in the context of -%% the other request keys, over the original base message. resolve(Base, Path, Opts) when is_binary(Path) -> resolve(Base, #{ <<"path">> => Path }, Opts); resolve(Base, Req, Opts) -> @@ -162,244 +176,6 @@ resolve(Base, Req, Opts) -> } ), resolve_many([Base | MessagesToExec], Opts). - -resolve_single(Base, Req, Opts) -> - vary_context( - #{ - <<"base">> => Base, - <<"req">> => Req, - <<"key">> => hb_path:hd(Req, Opts) - }, - Opts#{ <<"mode">> => <<"normal">> } - ). - -%% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring -%% normalization, cache, hashpath, worker, and other management components. -%% This function comes in `/3`-`/5` variants, allowing the caller to optionally -%% specify that a specific device must be used for the execution (as if -%% `Base/device: Device` was set), or to force a specific key to be used. -%% Critically, the modifiers do _not_ affect the message that the call is -%% executed upon: `Base`, `Req`, and `Opts` are passed directly to the execution -%% function as-is, but the chosen function is altered by these modifiers. -raw(Base, Req, Opts) -> - raw(undefined, Base, Req, Opts). -raw(Device, Base, Req, Opts) -> - raw(Device, undefined, Base, Req, Opts). -raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> - ?prim_dbg( - {executing, - {forced_device, ForcedDevice}, - {forced_key, ForcedKey} - } - ), - vary_context( - forced_context( - ForcedDevice, - ForcedKey, - #{ - <<"base">> => Base, - <<"req">> => Req, - <<"priv">> => #{ <<"mode">> => <<"raw">> } - }, - Opts - ), - Opts#{ <<"mode">> => <<"raw">> } - ). - -%% @doc Executes a context in primitive mode: Skipping cache lookup and -%% varying. Caution: Does not produce cryptographically valid results in all -%% cases! Should not be used as a normal AO-Core resolver. -primitive(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> - {ok, CtxWithResolver} = - hb_device:add_resolver( - Ctx#{ - <<"varied-base">> => Base, - <<"varied-req">> => Req, - <<"priv">> => #{ <<"mode">> => <<"primitive">> } - }, - Opts - ), - execute_context(CtxWithResolver, Opts). -primitive(ForcedDevice, ForceKey, Ctx, Opts) -> - primitive(forced_context(ForcedDevice, ForceKey, Ctx, Opts), Opts). - -primitive(Base, Req, Opts) -> - primitive(undefined, undefined, Base, Req, Opts). -primitive(ForcedDevice, ForcedKey, Base, Req, Opts) -> - primitive( - ForcedDevice, - ForcedKey, - #{ - <<"base">> => Base, - <<"req">> => Req, - <<"key">> => hb_path:hd(Req, Opts) - }, - Opts - ). - -%% @doc Apply `forced` device and key elements to the context when they are -%% provided, otherwise return the context as-is. For use only in `primitive` -%% or `raw` execution modes, where we are not attesting to the result of the -%% computation. -forced_context(undefined, undefined, Ctx, _) -> Ctx; -forced_context(ForcedDevice, Key, Ctx, Opts) when ForcedDevice =/= undefined -> - forced_context( - undefined, - Key, - Ctx#{ <<"forced-device">> => ForcedDevice }, - Opts - ); -forced_context(undefined, ForcedKey, Ctx, _Opts) when ForcedKey =/= undefined -> - Ctx#{ <<"key">> => ForcedKey }. - -%% @doc Perform the `vary` step of an AO-Core execution. This step requires -%% using the device (or the fallback device) to materialize the appropriate -%% protocol values from the `Base` and `Request` messages, as well as the -%% Erlang function to be executed, in order to evaluate an AO-Core resolution. -vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> - VaryRes = - primitive( - Ctx#{ - <<"key">> => <<"vary">>, - <<"base">> => Base, - <<"req">> => - #{ - <<"path">> => <<"vary">>, - <<"vary">> => Key, - <<"...">> => Req - } - }, - Opts#{ <<"mode">> => hb_opts:get(<<"mode">>, <<"raw">>, Opts) } - ), - ?prim_dbg({vary_result, erlang:external_size(VaryRes)}), - ?prim_dbg( - { - priv_vary, - {before, hb_private:from_message(Ctx)}, - {'after', hb_private:from_message(VaryRes)} - } - ), - case VaryRes of - {ok, CtxWithRes = #{ <<"result">> := _Result }} -> - post_execution(CtxWithRes, Opts); - {ok, PreparedCtx} -> - lookup_or_exec(PreparedCtx, Opts); - Other -> - throw({unexpected_vary_result, Other}) - end. - -%% @doc Look up a result from the cache and if not found, execute the context. -lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := SkipMode } }, Opts) - when SkipMode =:= <<"raw">>; SkipMode =:= <<"primitive">> -> - execute_context(Ctx, Opts); -lookup_or_exec( - Ctx = #{ - <<"varied-base">> := VBase, - <<"varied-req">> := VReq, - <<"key">> := Key, - <<"priv">> := Priv - }, Opts) -> - ?prim_dbg({priv_during_lookup, Priv}), - {ok, VBaseID} = primitive(VBase, #{ <<"path">> => <<"id">> }, Opts), - {ok, VReqID} = primitive(VReq, #{ <<"path">> => <<"id">> }, Opts), - CtxWithIDs = - Ctx#{ - <<"varied-base-id">> => VBaseID, - <<"varied-req-id">> => VReqID - }, - case hb_cache_control:maybe_lookup(VBaseID, VReqID, Opts) of - {hit, {Status, Msg}} -> - ?prim_dbg({cache_hit, Key, {Status, Msg}}), - post_execution( - CtxWithIDs#{ <<"result">> => Msg, <<"status">> => Status }, - Opts - ); - Miss -> - ?prim_dbg({cache_miss, Key, Miss}), - execute_context(CtxWithIDs, Opts) - end. - -%% @doc Takes a fully prepated context (with `varied-base` and `varied-req` -%% set, along with `priv/resolver` etc.) and executes it. Does not cache, -%% extend, or otherwise post-process the result. -execute_context( - Ctx = #{ - <<"key">> := Key, - <<"varied-base">> := VariedBase, - <<"varied-req">> := VariedReq, - <<"priv">> := - #{ - <<"resolver">> := Fun, - <<"add-key">> := AddKey - } - }, - Opts) -> - % Apply the function and return the result directly, without any further - % processing. We add the `PrefixArgs` to the list of arguments to be passed - % to the function to accomodate default handlers, which take the key that - % was invoked on the device as the first argument (ahead of `Base`, `Req`, - % and `ExecOpts`). - ?prim_dbg({execute, Fun}), - {Status, Res} = - apply( - Fun, - hb_device:truncate_args( - Fun, - if AddKey -> [Key]; true -> [] end - ++ [VariedBase, VariedReq, Opts] - ) - ), - post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). - -%% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return -%% the result directly; else, apply the `return-extends` schema to the result. -%% Handles the three core variants via two separate function heads: -%% 1. `primitive` mode: return the result directly -%% 2a. `raw` mode: Apply the `return-extends` schema to the result, don't store. -%% 2b. `normal` mode: Apply the `return-extends` schema to the result, store in -%% the cache if `cache-control` permits. -post_execution( - #{ - <<"priv">> := #{ <<"mode">> := <<"primitive">> }, - <<"result">> := Res, - <<"status">> := Status, - <<"key">> := Key - }, - _Opts) -> - ?prim_dbg({completed_exec, primitive, {key, Key}}), - {Status, Res}; -post_execution( - Context = #{ - <<"key">> := Key, - <<"base">> := Base, - <<"req">> := Req, - <<"result">> := RawResult, - <<"status">> := Status, - <<"return-extends">> := Extend - }, - Opts) -> - ?prim_dbg({completed_exec, normal, {key, Key}, {extend, Extend}}), - maybe_cache(Context, Opts), - Res = - case Extend of - none -> RawResult; - base -> RawResult#{ <<"...">> => Base }; - request -> RawResult#{ <<"...">> => Req } - end, - {Status, Res}. - -maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> - skipped_caching; -maybe_cache( - #{ - <<"key">> := Key, - <<"varied-base">> := VBase, - <<"varied-req">> := VReq, - <<"varied-base-id">> := VBaseID, - <<"varied-req-id">> := VReqID, - <<"result">> := Res - }, Opts) -> - hb_cache_control:maybe_store(VBaseID, VReqID, VBase, VReq, Res, Opts). %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, @@ -422,7 +198,7 @@ resolve_many(ListMsg, Opts) when is_map(ListMsg) -> ListOfMessages = try hb_util:message_to_ordered_list(ListMsg, internal_opts(Opts)) catch - Type:Exception:Stacktrace -> + Type:Exception:Stacktrace -> throw( {resolve_many_error, {given_message_not_ordered_list, ListMsg}, @@ -455,7 +231,7 @@ do_resolve_many([Base, Req | MsgList], Opts) -> {res, Res}, {opts, Opts} }, - Opts + Opts ), do_resolve_many([Res | MsgList], Opts); Res -> @@ -465,17 +241,133 @@ do_resolve_many([Base, Req | MsgList], Opts) -> Res end. -resolve_stage(1, Path, Req, Opts) when is_binary(Path) -> - % The base has been granted to us as a binary. Execute it in and resurse. - case resolve(Path, Opts) of - {ok, ResolvedBase} -> resolve_stage(1, ResolvedBase, Req, Opts); - Other -> Other +%% @doc Resolve only a single, explicit, computation step over a path-normalized +%% (single part) `Base` and `Req` pair. +resolve_single(Base, Req, Opts) -> + case do(context(undefined, undefined, Base, Req, Opts)) of + {ok, #{ <<"result">> := Res}} -> {ok, Res}; + {error, Reason} -> {error, Reason} + end. + +%% @doc Create an execution context from a set of core parameters: +%% `ForcedDevice` (if relevant), `ForcedKey` (if relevant), `Base`, `Req`, +%% and `Opts`. +context(FDevice, FKey, Base, Req, Opts) -> + maps:filter( + fun(_K, V) -> V =/= undefined end, + #{ + <<"forced-device">> => FDevice, + <<"forcedkey">> => FKey, + <<"base">> => Base, + <<"req">> => Req, + <<"opts">> => Opts + } + ). + +%% @doc Resolves a fully normalized execution context, returning early if any +%% stage fails. +do(Ctx0) -> + maybe + % Stage 1: Normalization and loading of IDs/links. + {ok, Ctx1} ?= stage_1(Ctx0), + % Stage 2: Device or direct key lookup. + {ok, Ctx2} ?= stage_2(Ctx1), + % Stage 3: Vary `Base` and `Request`. + {ok, Ctx3} ?= stage_3(Ctx2), + % Stage 4: Persistent resolver lookup. + {ok, Ctx4} ?= stage_4(Ctx3), + % Stage 5: Cache lookup. + {ok, Ctx5} ?= stage_5(Ctx4), + % Stage 6: Execution. + {ok, Ctx6} ?= stage_6(Ctx5), + % Stage 7: Result caching. + {ok, Ctx7} ?= stage_7(Ctx6), + % Stage 8: Notify waiters. + {ok, Ctx8} ?= stage_8(Ctx7), + % Stage 9: Extension of result upon original base. + {ok, Ctx9} ?= stage_9(Ctx8), + % Stage 10: Execution of the `step' hook. + {ok, Ctx10} ?= stage_10(Ctx9), + % Stage 11: Fork worker. + {ok, Ctx11} ?= stage_11(Ctx10) + else + {hit, Ctx = #{ <<"must-cache">> := false } } -> + % If we hit the cache early (e.g., a direct key access), return + % immediately without the `must-cache' flag. + {ok, maps:without([<<"must-cache">>], Ctx)} + end. + +%% @doc Normalize the context of an execution request. Ensures that a device and +%% path are available for execution. +stage_1(Ctx = #{<<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; +stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> + case hb_device:message_device_id(Base, Path, Opts) of + {hit, Res} -> + {hit, + Ctx#{ + <<"result">> => Res, + <<"cache-hit">> => false + } + }; + {ok, Device} -> + end. + +%% @doc Normalize the context of an execution request. Ensures that a device and +%% path are available for execution. +stage_1(Ctx = #{<<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; +stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> + case hb_device:message_device_id(Base, Path, Opts) of + {hit, Res} -> + {hit, + Ctx#{ + <<"result">> => Res, + <<"store-hit">> => false + } + }; + {ok, Device} -> + stage_1(Ctx#{ <<"device">> => Device }) end; -resolve_stage(1, Link, Req, Opts) when ?IS_LINK(Link) -> - % If the first message is a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), - resolve_stage(1, hb_cache:ensure_loaded(Link, Opts), Req, Opts); +stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) -> + stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }). + +%% @doc Lookup the device and function to use during an execution. +stage_2( + Ctx = #{ + <<"device">> := Device, + <<"base">> := Base, + <<"path">> := Path, + <<"priv">> := Priv, + <<"opts">> := Opts + } +) -> + case hb_device:message_to_fun(Device, Base, Path, Opts) of + {ok, Function} -> + {ok, Ctx#{ <<"priv">> => Priv#{ <<"function">> => Function }}}; + {error, Reason} -> + {error, Reason} + end. + +%% @doc Vary the `Base` and `Req` of the resolution. To do so, assuming that +%% the request is not itself a `vary` call, we recursively resolve the +%% computation as a sub-request. This allows the device to offer its own `vary` +%% logic, based upon the device's own understanding of the request. +stage_3(Ctx = #{ <<"path">> := <<"vary">>, <<"base">> := Base, <<"request">> := Req }) -> + % We are already varying. Do not recurse. + {ok, Ctx#{ <<"varied-base">> => Base, <<"varied-req">> => Req }}; +stage_3( + Ctx = + #{ + <<"base">> := Base, + <<"request">> := Req, + <<"priv">> := #{ <<"function">> := Function }, + <<"opts">> := Opts + } +) -> + case raw(undefined, <<"vary">>, Base, Req, Opts) of + {ok, VariedCtx} -> {ok, maps:merge(Ctx, VariedCtx)}; + {error, Reason} -> {error, Reason} + end. + resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> % If the second message is a link, we should load the message and % continue with the resolution. @@ -1128,3 +1020,185 @@ execution_opts(Opts) -> recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 end. + +%%% ============= 1.0 POC IMPLEMENTATION ============= + +%% @doc Executes a context in primitive mode: Skipping cache lookup and +%% varying. Caution: Does not produce cryptographically valid results in all +%% cases! Should not be used as a normal AO-Core resolver. +primitive(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> + {ok, CtxWithResolver} = + hb_device:add_resolver( + Ctx#{ + <<"varied-base">> => Base, + <<"varied-req">> => Req, + <<"priv">> => #{ <<"mode">> => <<"primitive">> } + }, + Opts + ), + execute_context(CtxWithResolver, Opts). +primitive(ForcedDevice, ForceKey, Ctx, Opts) -> + primitive(forced_context(ForcedDevice, ForceKey, Ctx, Opts), Opts). + +primitive(Base, Req, Opts) -> + primitive(undefined, undefined, Base, Req, Opts). +primitive(ForcedDevice, ForcedKey, Base, Req, Opts) -> + primitive( + ForcedDevice, + ForcedKey, + #{ + <<"base">> => Base, + <<"req">> => Req, + <<"key">> => hb_path:hd(Req, Opts) + }, + Opts + ). + +%% @doc Perform the `vary` step of an AO-Core execution. This step requires +%% using the device (or the fallback device) to materialize the appropriate +%% protocol values from the `Base` and `Request` messages, as well as the +%% Erlang function to be executed, in order to evaluate an AO-Core resolution. +vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> + VaryRes = + primitive( + Ctx#{ + <<"key">> => <<"vary">>, + <<"base">> => Base, + <<"req">> => + #{ + <<"path">> => <<"vary">>, + <<"vary">> => Key, + <<"...">> => Req + } + }, + Opts#{ <<"mode">> => hb_opts:get(<<"mode">>, <<"raw">>, Opts) } + ), + ?prim_dbg({vary_result, erlang:external_size(VaryRes)}), + ?prim_dbg( + { + priv_vary, + {before, hb_private:from_message(Ctx)}, + {'after', hb_private:from_message(VaryRes)} + } + ), + case VaryRes of + {ok, CtxWithRes = #{ <<"result">> := _Result }} -> + post_execution(CtxWithRes, Opts); + {ok, PreparedCtx} -> + lookup_or_exec(PreparedCtx, Opts); + Other -> + throw({unexpected_vary_result, Other}) + end. + +%% @doc Look up a result from the cache and if not found, execute the context. +lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := SkipMode } }, Opts) + when SkipMode =:= <<"raw">>; SkipMode =:= <<"primitive">> -> + execute_context(Ctx, Opts); +lookup_or_exec( + Ctx = #{ + <<"varied-base">> := VBase, + <<"varied-req">> := VReq, + <<"key">> := Key, + <<"priv">> := Priv + }, Opts) -> + ?prim_dbg({priv_during_lookup, Priv}), + {ok, VBaseID} = primitive(VBase, #{ <<"path">> => <<"id">> }, Opts), + {ok, VReqID} = primitive(VReq, #{ <<"path">> => <<"id">> }, Opts), + CtxWithIDs = + Ctx#{ + <<"varied-base-id">> => VBaseID, + <<"varied-req-id">> => VReqID + }, + case hb_cache_control:maybe_lookup(VBaseID, VReqID, Opts) of + {hit, {Status, Msg}} -> + ?prim_dbg({cache_hit, Key, {Status, Msg}}), + post_execution( + CtxWithIDs#{ <<"result">> => Msg, <<"status">> => Status }, + Opts + ); + Miss -> + ?prim_dbg({cache_miss, Key, Miss}), + execute_context(CtxWithIDs, Opts) + end. + +%% @doc Takes a fully prepated context (with `varied-base` and `varied-req` +%% set, along with `priv/resolver` etc.) and executes it. Does not cache, +%% extend, or otherwise post-process the result. +execute_context( + Ctx = #{ + <<"key">> := Key, + <<"varied-base">> := VariedBase, + <<"varied-req">> := VariedReq, + <<"priv">> := + #{ + <<"resolver">> := Fun, + <<"add-key">> := AddKey + } + }, + Opts) -> + % Apply the function and return the result directly, without any further + % processing. We add the `PrefixArgs` to the list of arguments to be passed + % to the function to accomodate default handlers, which take the key that + % was invoked on the device as the first argument (ahead of `Base`, `Req`, + % and `ExecOpts`). + ?prim_dbg({execute, Fun}), + {Status, Res} = + apply( + Fun, + hb_device:truncate_args( + Fun, + if AddKey -> [Key]; true -> [] end + ++ [VariedBase, VariedReq, Opts] + ) + ), + post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). + +%% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return +%% the result directly; else, apply the `return-extends` schema to the result. +%% Handles the three core variants via two separate function heads: +%% 1. `primitive` mode: return the result directly +%% 2a. `raw` mode: Apply the `return-extends` schema to the result, don't store. +%% 2b. `normal` mode: Apply the `return-extends` schema to the result, store in +%% the cache if `cache-control` permits. +post_execution( + #{ + <<"priv">> := #{ <<"mode">> := <<"primitive">> }, + <<"result">> := Res, + <<"status">> := Status, + <<"key">> := Key + }, + _Opts) -> + ?prim_dbg({completed_exec, primitive, {key, Key}}), + {Status, Res}; +post_execution( + Context = #{ + <<"key">> := Key, + <<"base">> := Base, + <<"req">> := Req, + <<"result">> := RawResult, + <<"status">> := Status, + <<"return-extends">> := Extend + }, + Opts) -> + ?prim_dbg({completed_exec, normal, {key, Key}, {extend, Extend}}), + maybe_cache(Context, Opts), + Res = + case Extend of + none -> RawResult; + base -> RawResult#{ <<"...">> => Base }; + request -> RawResult#{ <<"...">> => Req } + end, + {Status, Res}. + +maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> + skipped_caching; +maybe_cache( + #{ + <<"key">> := Key, + <<"varied-base">> := VBase, + <<"varied-req">> := VReq, + <<"varied-base-id">> := VBaseID, + <<"varied-req-id">> := VReqID, + <<"result">> := Res + }, Opts) -> + hb_cache_control:maybe_store(VBaseID, VReqID, VBase, VReq, Res, Opts). \ No newline at end of file From bee48c1ce01d697b8dc133336ada71d50695a811 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 9 Jul 2026 16:02:36 -0400 Subject: [PATCH 20/35] wip: add persistent resolver and cache lookup to resolver --- src/core/resolver/hb_ao.erl | 229 ++++++++++++++++++++++---- src/preloaded/message/dev_message.erl | 21 +-- 2 files changed, 204 insertions(+), 46 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index dbbd069dc..e32a954b7 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -302,30 +302,8 @@ do(Ctx0) -> stage_1(Ctx = #{<<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> case hb_device:message_device_id(Base, Path, Opts) of - {hit, Res} -> - {hit, - Ctx#{ - <<"result">> => Res, - <<"cache-hit">> => false - } - }; - {ok, Device} -> - end. - -%% @doc Normalize the context of an execution request. Ensures that a device and -%% path are available for execution. -stage_1(Ctx = #{<<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; -stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> - case hb_device:message_device_id(Base, Path, Opts) of - {hit, Res} -> - {hit, - Ctx#{ - <<"result">> => Res, - <<"store-hit">> => false - } - }; - {ok, Device} -> - stage_1(Ctx#{ <<"device">> => Device }) + {hit, Res} -> {hit, Ctx#{ <<"result">> => Res }}; + {ok, Device} -> stage_1(Ctx#{ <<"device">> => Device }) end; stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) -> stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }). @@ -363,11 +341,201 @@ stage_3( <<"opts">> := Opts } ) -> - case raw(undefined, <<"vary">>, Base, Req, Opts) of - {ok, VariedCtx} -> {ok, maps:merge(Ctx, VariedCtx)}; - {error, Reason} -> {error, Reason} + maybe + % Raw call `vary` to find the varied base/req, passing the `priv/function` + % such that varying does not require us to repeat the associated work. + {ok, VariedCtx} ?= + raw( + undefined, + <<"vary">>, + Base, + hb_private:set(Req, <<"function">>, Function, Opts), + Opts + ), + {ok, maps:merge(Ctx, VariedCtx)} + end. + +%% @doc Determine if the request is already being resolved right now. If so, +%% await the result rather than resolving again. If not, register as the +%% resolver for the request in question and proceed. If we are executing with +%% `no-cache`, skip the lookup and proceed with the current context. +stage_4( + Ctx = #{ + <<"varied-base">> := Base, + <<"varied-req">> := Req, + <<"opts">> := Opts + }) -> + ?event_debug(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), + % Persistent-resolver lookup: Search for local (or Distributed + % Erlang cluster) processes that are already performing the execution. + % Before we search for a live executor, we check if the device specifies + % a function that tailors the 'group' name of the execution. For example, + % the `~process@1.0' device 'groups' all calls to the same process onto + % calls to a single executor. By default, `{Base, Req}' is used as the + % group name. + case hb_persistent:find_or_register(Base, Req, Opts) of + {leader, ExecName} -> + % We are the leader for this resolution. Continue while noting + % this in our context. + {ok, Ctx#{ <<"leader">> => ExecName }}; + {wait, Leader} -> + % There is another executor of this resolution in-flight. + % register to receive the response, then + % wait. + case hb_persistent:await(Leader, Base, Req, Opts) of + {error, leader_died} -> + ?event( + ao_core, + {leader_died_during_wait, + {leader, Leader}, + {base, Base}, + {req, Req}, + {opts, Opts} + }, + Opts + ), + % Re-try again if the group leader has died. + stage_4(Ctx); + {ok, Result} -> + % We received a successful result from another worker. They + % will store the result if appropriate, so we skip the store + % write. + {hit, Ctx#{ <<"result">> := Result }} + end; + {infinite_recursion, GroupName} -> + % We are the leader for this resolution, but we executing the + % computation again. This may plausibly be OK in _some_ cases, + % but in general it is the sign of a bug. + ?event( + ao_core, + {infinite_recursion, + {exec_group, GroupName}, + {base, Base}, + {req, Req}, + {opts, Opts} + }, + Opts + ), + case hb_opts:get(<<"allow-infinite">>, false, Opts) of + true -> + % We are OK with infinite loops, so we just continue. + % No need to register, as we are the leader. + {ok, Ctx}; + false -> + % We are not OK with infinite loops, so we raise an error. + { + error, + #{ + <<"status">> => 508, + <<"body">> => <<"Request creates infinite recursion.">> + } + }. + end end. +%% @doc Look up whether the varied base and request pair already has a known +%% result in the cache. If so, we skip the resolution step and proceed using +%% the cached result. We signal that further execution is not needed by +%% returning the context +stage_5(Ctx = #{ + <<"varied-base">> := Base, + <<"varied-req">> := Req, + <<"opts">> := Opts +}) -> + case hb_cache_control:maybe_lookup(Base, Req, Opts) of + {ok, Result} -> + % The result was found, so we return a `hit`, but we also unregister + % and notify if + case maps:get(<<"leader">>, Ctx, false) of + false -> skip; + ExecName -> + hb_persistent:unregister_notify(ExecName, Req, Result, Opts) + end, + {hit, Ctx#{ <<"result">> := Result }}; + _ -> {ok, Ctx} + end. + +%% @doc Perform the actual resolution of an AO-Core request. +stage_6(Ctx = #{ + <<"varied-base">> := Base, + <<"varied-req">> := Req, + <<"opts">> := Opts, + <<"priv">> := #{ <<"function">> := Func } +}) -> + ?event_debug(debug_ao_core, {stage, 6, ExecName, execution}, Opts), + ExecOpts = execution_opts(Opts), + ExecName = maps:get(<<"leader">>, Ctx, unnamed), + Args = + case maps:get(<<"add-key">>, Ctx, false) of + false -> [Base, Req, ExecOpts]; + Key -> [Key, Base, Req, ExecOpts] + end, + % Try to execute the function. + Res = + try + TruncatedArgs = hb_device:truncate_args(Func, Args), + MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), + ?event( + debug_ao_result, + { + ao_result, + {exec_name, ExecName}, + {base, Base}, + {req, Req}, + {res, MsgRes} + }, + Opts + ), + MsgRes + catch + ExecClass:ExecException:ExecStacktrace -> + ?event( + ao_core, + {device_call_failed, ExecName, {func, Func}}, + Opts + ), + ?event( + ao_result, + { + exec_failed, + {base, Base}, + {req, Req}, + {exec_name, ExecName}, + {func, Func}, + {exec_class, ExecClass}, + {exec_exception, ExecException}, + {exec_stacktrace, + {trace, erlang:process_info(self(), backtrace)} + }, + {opts, Opts} + }, + Opts + ), + % If the function call fails, we raise an error in the manner + % indicated by caller's `#Opts'. + error_execution( + ExecName, + Req, + device_call, + {ExecClass, ExecException, ExecStacktrace}, + Opts + ) + end, + hb_message:paranoid_verify( + post_resolve, + #{ + <<"reason">> => <<"AO-Core Post-Execution Validation">>, + <<"base">> => Base, + <<"request">> => Req, + <<"result">> => Res + }, + Opts + ), + case Result of + -> + Body. + + resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> % If the second message is a link, we should load the message and % continue with the resolution. @@ -832,13 +1000,6 @@ error_infinite(Base, Req, Opts) -> Opts ), ?trace(), - { - error, - #{ - <<"status">> => 508, - <<"body">> => <<"Request creates infinite recursion.">> - } - }. %% @doc Handle an error in a device call. error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index dfbae6515..300d724b0 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -707,18 +707,15 @@ vary(Base, Req, Opts) -> maybe {ok, Key} ?= hb_maps:find(<<"vary">>, Req, Opts), {ok, VaryReq} ?= hb_maps:find(<<"...">>, Req, Opts), - Ctx0 = #{ <<"base">> => Base, <<"key">> => Key, <<"req">> => VaryReq }, - case hb_device:add_resolver(Ctx0, Opts) of - {ok, Ctx1 = #{ <<"result">> := _ }} -> - % Finding the resolver led us to first find the complete - % solution. We return the context as-is. - {ok, Ctx1}; - {ok, Ctx1} -> - % The resolver has been found. We must vary the `Base` and - % `Req` using its schema, then return the updated context. - hb_types:vary(Ctx1, Opts); - Err -> Err - end + Ctx1 = #{ <<"base">> => Base, <<"key">> => Key, <<"req">> => VaryReq }, + {ok, Ctx2} ?= + case hb_private:get(<<"function">>, Req, not_found, Opts) of + not_found -> + hb_device:add_resolver(Ctx1, Opts); + Fun -> + {ok, Ctx1#{ <<"priv">> => #{ <<"function">> => Fun } }} + end, + hb_types:vary(Ctx2, Opts) else error -> ?prim_dbg({vary_error, {base, Base}, {req, Req}}), From 013db25e54df427faf80f2eb0cb79173a2860074 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 9 Jul 2026 17:47:16 -0400 Subject: [PATCH 21/35] wip: v0 draft of the new AO-Core resolver logic --- src/core/device/hb_device.erl | 1 + src/core/resolver/hb_ao.erl | 769 +++++-------------------- src/core/resolver/hb_cache_control.erl | 2 +- src/core/resolver/hb_persistent.erl | 4 +- 4 files changed, 147 insertions(+), 629 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index f7d2bafd7..ef788e3b2 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -3,6 +3,7 @@ %%% functions from a device. -module(hb_device). -export([truncate_args/2, add_resolver/2, message_to_fun/3, module/2]). +-export([message_device_id/3]). -export([is_direct_key_access/3, is_direct_key_access/4]). -export([find_exported_function/5, is_exported/4, info/2, info/3]). -include("include/hb.hrl"). diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index e32a954b7..b3ecec79b 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -429,7 +429,7 @@ stage_4( <<"status">> => 508, <<"body">> => <<"Request creates infinite recursion.">> } - }. + } end end. @@ -449,7 +449,12 @@ stage_5(Ctx = #{ case maps:get(<<"leader">>, Ctx, false) of false -> skip; ExecName -> - hb_persistent:unregister_notify(ExecName, Req, Result, Opts) + hb_persistent:unregister_notify( + ExecName, + Req, + {ok, Result}, + Opts + ) end, {hit, Ctx#{ <<"result">> := Result }}; _ -> {ok, Ctx} @@ -471,56 +476,7 @@ stage_6(Ctx = #{ Key -> [Key, Base, Req, ExecOpts] end, % Try to execute the function. - Res = - try - TruncatedArgs = hb_device:truncate_args(Func, Args), - MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), - ?event( - debug_ao_result, - { - ao_result, - {exec_name, ExecName}, - {base, Base}, - {req, Req}, - {res, MsgRes} - }, - Opts - ), - MsgRes - catch - ExecClass:ExecException:ExecStacktrace -> - ?event( - ao_core, - {device_call_failed, ExecName, {func, Func}}, - Opts - ), - ?event( - ao_result, - { - exec_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {func, Func}, - {exec_class, ExecClass}, - {exec_exception, ExecException}, - {exec_stacktrace, - {trace, erlang:process_info(self(), backtrace)} - }, - {opts, Opts} - }, - Opts - ), - % If the function call fails, we raise an error in the manner - % indicated by caller's `#Opts'. - error_execution( - ExecName, - Req, - device_call, - {ExecClass, ExecException, ExecStacktrace}, - Opts - ) - end, + {Status, Res} = maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts), hb_message:paranoid_verify( post_resolve, #{ @@ -531,394 +487,100 @@ stage_6(Ctx = #{ }, Opts ), - case Result of - -> - Body. - - -resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> - % If the second message is a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), - resolve_stage(1, Base, hb_cache:ensure_loaded(Link, Opts), Opts); -resolve_stage(1, Base, Req, Opts) when is_list(Base) -> - % Normalize lists to numbered maps (base=1) if necessary. - ?event_debug(debug_ao_core, {stage, 1, list_normalize}, Opts), - resolve_stage(1, - normalize_keys(Base, Opts), - Req, - Opts - ); -resolve_stage(1, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 1, normalize_complete}, Opts), - resolve_stage(2, Base, Req, Opts); -resolve_stage(2, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), - % Lookup request in the cache. If we find a result, return it. - % If we do not find a result, we continue to the next stage, - % unless the cache lookup returns `halt' (the user has requested that we - % only return a result if it is already in the cache). - case hb_cache_control:maybe_lookup(Base, Req, Opts) of - {ok, Res} -> - ?event_debug(debug_ao_core, {stage, 2, cache_hit, {res, Res}}, Opts), - {ok, Res}; - {continue, NewBase, NewReq} -> - resolve_stage(3, NewBase, NewReq, Opts); - {error, CacheResp} -> {error, CacheResp} - end; -resolve_stage(3, Base, Req, _Opts) when not is_map(Base) or not is_map(Req) -> - % Validation check: If the messages are not maps, we cannot find a key - % in them, so return not_found. - ?event_debug(debug_ao_core, {stage, 3, validation_check_type_error}, _Opts), - {error, not_found}; -resolve_stage(3, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 3, validation_check}, Opts), - % Validation checks: If `paranoid_message_verification' is enabled, we should - % verify the base and request messages prior to execution. - hb_message:paranoid_verify( - pre_resolve, - #{ - <<"reason">> => <<"AO-Core Pre-Execution Validation">>, - <<"base">> => Base, - <<"request">> => Req - }, - Opts - ), - resolve_stage(4, Base, Req, Opts); -resolve_stage(4, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), - % Persistent-resolver lookup: Search for local (or Distributed - % Erlang cluster) processes that are already performing the execution. - % Before we search for a live executor, we check if the device specifies - % a function that tailors the 'group' name of the execution. For example, - % the `dev_process' device 'groups' all calls to the same process onto - % calls to a single executor. By default, `{Base, Req}' is used as the - % group name. - case hb_persistent:find_or_register(Base, Req, hb_maps:without(?TEMP_OPTS, Opts, Opts)) of - {leader, ExecName} -> - % We are the leader for this resolution. Continue to the next stage. - case hb_opts:get(spawn_worker, false, Opts) of - true -> ?event(worker_spawns, {will_become, ExecName}); - _ -> ok - end, - resolve_stage(5, Base, Req, ExecName, Opts); - {wait, Leader} -> - % There is another executor of this resolution in-flight. - % Bail execution, register to receive the response, then - % wait. - case hb_persistent:await(Leader, Base, Req, Opts) of - {error, leader_died} -> - ?event( - ao_core, - {leader_died_during_wait, - {leader, Leader}, - {base, Base}, - {req, Req}, - {opts, Opts} - }, - Opts - ), - % Re-try again if the group leader has died. - resolve_stage(4, Base, Req, Opts); - Res -> - % Now that we have the result, we can skip right to potential - % recursion (step 11) in the outer-wrapper. - Res - end; - {infinite_recursion, GroupName} -> - % We are the leader for this resolution, but we executing the - % computation again. This may plausibly be OK in _some_ cases, - % but in general it is the sign of a bug. - ?event( - ao_core, - {infinite_recursion, - {exec_group, GroupName}, - {base, Base}, - {req, Req}, - {opts, Opts} - }, - Opts - ), - case hb_opts:get(<<"allow-infinite">>, false, Opts) of - true -> - % We are OK with infinite loops, so we just continue. - resolve_stage(5, Base, Req, GroupName, Opts); - false -> - % We are not OK with infinite loops, so we raise an error. - error_infinite(Base, Req, Opts) - end - end. -resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 5, device_lookup}, Opts), - % Device lookup: Find the Erlang function that should be utilized to - % execute Req on Base. - {ResolvedFunc, NewOpts} = - try - UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), - Key = hb_path:hd(Req, UserOpts), - % Try to load the device and get the function to call. - ?event( - { - resolving_key, - {key, Key}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - {Status, _Dev, Device, Func} = - hb_device:message_to_fun( - Base, - Key, - <<"message@1.0">>, - UserOpts - ), - ?event( - {found_func_for_exec, - {key, Key}, - {device, Device}, - {func, Func}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. - { - Func, - Opts#{ - <<"add-key">> => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - ?event( - ao_result, - { - load_device_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {exec_class, Class}, - {exec_exception, Exception}, - {exec_stacktrace, Stacktrace}, - {opts, Opts} - }, - Opts - ), - % If the device cannot be loaded, we alert the caller. - error_execution( - ExecName, - Req, - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). -resolve_stage(6, Func, Base, Req, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 6, ExecName, execution}, Opts), - % Execution. - ExecOpts = execution_opts(Opts), - Args = - case hb_opts:get(<<"add-key">>, false, Opts) of - false -> [Base, Req, ExecOpts]; - Key -> [Key, Base, Req, ExecOpts] - end, - % Try to execute the function. - Res = - try - TruncatedArgs = hb_device:truncate_args(Func, Args), - MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), - ?event( - debug_ao_result, - { - ao_result, - {exec_name, ExecName}, - {base, Base}, - {req, Req}, - {res, MsgRes} - }, - Opts - ), - MsgRes - catch - ExecClass:ExecException:ExecStacktrace -> - ?event( - ao_core, - {device_call_failed, ExecName, {func, Func}}, - Opts - ), - ?event( - ao_result, - { - exec_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {func, Func}, - {exec_class, ExecClass}, - {exec_exception, ExecException}, - {exec_stacktrace, erlang:process_info(self(), backtrace)}, - {opts, Opts} - }, - Opts - ), - % If the function call fails, we raise an error in the manner - % indicated by caller's `#Opts'. - error_execution( - ExecName, - Req, - device_call, - {ExecClass, ExecException, ExecStacktrace}, - Opts - ) - end, - hb_message:paranoid_verify( - post_resolve, - #{ - <<"reason">> => <<"AO-Core Post-Execution Validation">>, - <<"base">> => Base, - <<"request">> => Req, - <<"result">> => Res - }, - Opts - ), - resolve_stage(7, Base, Req, Res, ExecName, Opts); -resolve_stage( - 7, - Base, - Req, - {St, Res}, - ExecName, - Opts = #{ <<"on">> := On = #{ <<"step">> := _ }} + {Status, Ctx#{ <<"result">> => Res, <<"status">> => Status }}. + +%% @doc Cache the result of an execution if appropriate for the context. +stage_7( + Ctx = #{ + <<"varied-base">> := VariedBase, + <<"varied-req">> := VariedReq, + <<"result">> := Res, + <<"opts">> := Opts + } ) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, executing_step_hook, On}, Opts), + hb_cache_control:maybe_store(VariedBase, VariedReq, Res, Opts), + {ok, Ctx}. + +%% @doc Return the resolved response to any waiting callers. +stage_8( + Ctx = #{ + <<"leader">> := ExecName, + <<"varied-request">> := Req, + <<"result">> := Res, + <<"status">> := Status, + <<"opts">> := Opts + } +) -> + hb_persistent:unregister_notify(ExecName, Req, {Status, Res}, Opts), + {ok, Ctx}; +stage_8(Ctx) -> + {ok, Ctx}. + +%% @doc When specified in the schema for the device call, normalize the execution's +%% result on top of the `Base` or `Request` message. +stage_9( + Ctx = #{ + <<"normalizer">> := Normalizer, + <<"base">> := Base, + <<"request">> := Req, + <<"result">> := Result, + <<"opts">> := Opts + } +) -> + { + ok, + Ctx#{ + <<"result">> := + case Normalizer of + base -> Result#{ <<"...">> => Base }; + request -> Result#{ <<"...">> => Req }; + Fun when is_function(Fun) -> Fun(Base, Result, Opts) + end + } + }. + +%% @doc If a hook has been specified for the `step` action, we call it with our +%% context including the result. +stage_10(Ctx = #{ <<"opts">> := Opts = #{ <<"on">> := #{ <<"step">> := _ }}}) -> + ?event_debug(debug_ao_core, {stage, 7, executing_step_hook}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every % execution. - HookReq = #{ - <<"base">> => Base, - <<"request">> => Req, - <<"status">> => St, - <<"body">> => Res - }, - case hb_hook:on(<<"step">>, HookReq, Opts) of - {ok, #{ <<"status">> := NewStatus, <<"body">> := NewRes }} -> - resolve_stage(8, Base, Req, {NewStatus, NewRes}, ExecName, Opts); - Error -> - ?event( - ao_core, - {step_hook_error, - {error, Error}, - {hook_req, HookReq} - }, - Opts - ), - Error - end; -resolve_stage(7, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), - resolve_stage(8, Base, Req, Res, ExecName, Opts); -resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), - % If the result is a `{resolve, Sublist}' tuple, we need to execute it - % as a sub-resolution. - resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); -resolve_stage(8, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), - resolve_stage(9, Base, Req, Res, ExecName, Opts); -resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), - % Cryptographic linking. Now that we have generated the result, we - % need to cryptographically link the output to its input via a hashpath. - resolve_stage(10, Base, Req, - case hb_opts:get(<<"hashpath">>, update, Opts#{ <<"only">> => local }) of - update -> - NormRes = Res, - Priv = hb_private:from_message(NormRes), - HP = hb_path:hashpath(Base, Req, Opts), - if not is_binary(HP) or not is_map(Priv) -> - throw({invalid_hashpath, {hp, HP}, {res, NormRes}}); - true -> - {ok, NormRes#{ <<"priv">> => Priv#{ <<"hashpath">> => HP } }} - end; - reset -> - Priv = hb_private:from_message(Res), - {ok, Res#{ <<"priv">> => hb_maps:without([<<"hashpath">>], Priv, Opts) }}; - ignore -> - Priv = hb_private:from_message(Res), - if not is_map(Priv) -> - throw({invalid_private_message, {res, Res}}); - true -> - {ok, Res} - end - end, - ExecName, - Opts - ); -resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), - ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), - % Skip cryptographic linking and reset the hashpath if the result is abnormal. - Priv = hb_private:from_message(Res), - resolve_stage( - 10, Base, Req, - {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, - ExecName, Opts); -resolve_stage(9, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), - % Skip cryptographic linking and continue if we don't have a map that can have - % a hashpath at all. - resolve_stage(10, Base, Req, Res, ExecName, Opts); -resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), - % Result caching: Optionally, cache the result of the computation locally. - hb_cache_control:maybe_store(Base, Req, Res, Opts), - resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); -resolve_stage(10, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), - % Skip result caching if the result is abnormal. - resolve_stage(11, Base, Req, Res, ExecName, Opts); -resolve_stage(11, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 11, ExecName}, Opts), - % Notify processes that requested the resolution while we were executing and - % unregister ourselves from the group. - hb_persistent:unregister_notify(ExecName, Req, Res, Opts), - resolve_stage(12, Base, Req, Res, ExecName, Opts); -resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + hb_hook:on(<<"step">>, Ctx, Opts); +stage_10(Ctx) -> {ok, Ctx}. + +%% @doc If we have been requested to spawn a worker process to remain active +%% at this stage, with this context in memory, we do so. +stage_11( + Ctx = #{ + <<"exec_name">> := ExecName, + <<"result">> := Res, + <<"opts">> := Opts + }) -> + ?event_debug(debug_ao_core, {stage, 12, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case - {is_map(Res), hb_opts:get(spawn_worker, false, Opts#{ <<"prefer">> => local })} + {is_map(Res), hb_opts:get(<<"spawn-worker">>, false, Opts#{ <<"prefer">> => local })} of {A, B} when (A == false) or (B == false) -> - Res; + {ok, Ctx}; {_, _} -> % Spawn a worker for the current execution WorkerPID = hb_persistent:start_worker(ExecName, Res, Opts), hb_persistent:forward_work(WorkerPID, Opts), - Res - end; -resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> - ?event_debug(debug_ao_core, {stage, 12, _ExecName, abnormal_status_skip_spawning}, _Opts), - OtherRes. + {ok, Ctx} + end. %% @doc If the `AO_PROFILING' macro is defined (set by building/launching with %% `rebar3 as ao_profiling') we record statistics about the execution of the %% function. This is a costly operation, so if it is not defined, we simply %% apply the function and return the result. -ifndef(AO_PROFILING). -maybe_profiled_apply(Func, Args, _Base, _Req, _Opts) -> - apply(Func, Args). +maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts) -> + do_apply(ExecName, Func, Args, Base, Req, Opts). -else. -maybe_profiled_apply(Func, Args, Base, Req, Opts) -> +maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts) -> CallStack = erlang:get(ao_stack), ?event(ao_trace, {profiling_apply, @@ -957,7 +619,12 @@ maybe_profiled_apply(Func, Args, Base, Req, Opts) -> Stack -> [Key | Stack] end ), - {ExecMicroSecs, Res} = timer:tc(fun() -> apply(Func, Args) end), + {ExecMicroSecs, Res} = + timer:tc( + fun() -> + do_apply(ExecName, Func, Args, Base, Req, Opts) + end + ), put(ao_stack, CallStack), hb_event:record(<<"ao-call-counts">>, Key, Opts), hb_event:record(<<"ao-total-durations">>, Key, Opts, ExecMicroSecs), @@ -988,27 +655,61 @@ maybe_profiled_apply(Func, Args, Base, Req, Opts) -> Res. -endif. -%% @doc Catch all return if we are in an infinite loop. -error_infinite(Base, Req, Opts) -> - ?event( - ao_core, - {error, {type, infinite_recursion}, - {base, Base}, - {req, Req}, - {opts, Opts} - }, - Opts - ), - ?trace(), - -%% @doc Handle an error in a device call. -error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> - Error = {error, Whence, {Class, Exception, Stacktrace}}, - hb_persistent:unregister_notify(ExecGroup, Req, Error, Opts), - ?event_debug(debug_ao_core, {handle_error, Error, {opts, Opts}}, Opts), - case hb_opts:get(error_strategy, throw, Opts) of - throw -> erlang:raise(Class, Exception, Stacktrace); - _ -> Error +%% @doc Execute a device call, wrapped with failure handling. +do_apply(ExecName, Func, Args, Base, Req, Opts) -> + try + case apply(Func, Truncated = hb_device:truncate_args(Func, Args)) of + {Status, Res} -> {Status, Res}; + Other -> + throw( + {unexpected_device_call_response, + #{ + <<"exec-name">> => ExecName, + <<"args">> => Truncated, + <<"result">> => Other, + <<"func">> => Func + } + } + ) + end + catch + Class:Exception:Stacktrace -> + ?event( + ao_core, + {device_call_failed, ExecName, {func, Func}}, + Opts + ), + ?event( + ao_result, + { + exec_failed, + {base, Base}, + {req, Req}, + {exec_name, ExecName}, + {func, Func}, + {exec_class, Class}, + {exec_exception, Exception}, + {exec_stacktrace, erlang:process_info(self(), backtrace)}, + {opts, Opts} + }, + Opts + ), + % If the function call fails, we raise an error in the manner + % indicated by caller's `#Opts', as well as returning the error to + % any registered listeners. + Error = + {failure, + #{ + <<"class">> => Class, + <<"exception">> => Exception, + <<"stacktrace">> => Stacktrace + } + }, + hb_persistent:unregister_notify(ExecName, Req, Error, Opts), + case hb_opts:get(error_strategy, throw, Opts) of + throw -> erlang:raise(Class, Exception, Stacktrace); + _ -> Error + end end. %% @doc Shortcut for resolving a key in a message without its status if it is @@ -1180,186 +881,4 @@ execution_opts(Opts) -> case hb_opts:get(<<"spawn-worker">>, false, Opts) of recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 - end. - -%%% ============= 1.0 POC IMPLEMENTATION ============= - -%% @doc Executes a context in primitive mode: Skipping cache lookup and -%% varying. Caution: Does not produce cryptographically valid results in all -%% cases! Should not be used as a normal AO-Core resolver. -primitive(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, Opts) -> - {ok, CtxWithResolver} = - hb_device:add_resolver( - Ctx#{ - <<"varied-base">> => Base, - <<"varied-req">> => Req, - <<"priv">> => #{ <<"mode">> => <<"primitive">> } - }, - Opts - ), - execute_context(CtxWithResolver, Opts). -primitive(ForcedDevice, ForceKey, Ctx, Opts) -> - primitive(forced_context(ForcedDevice, ForceKey, Ctx, Opts), Opts). - -primitive(Base, Req, Opts) -> - primitive(undefined, undefined, Base, Req, Opts). -primitive(ForcedDevice, ForcedKey, Base, Req, Opts) -> - primitive( - ForcedDevice, - ForcedKey, - #{ - <<"base">> => Base, - <<"req">> => Req, - <<"key">> => hb_path:hd(Req, Opts) - }, - Opts - ). - -%% @doc Perform the `vary` step of an AO-Core execution. This step requires -%% using the device (or the fallback device) to materialize the appropriate -%% protocol values from the `Base` and `Request` messages, as well as the -%% Erlang function to be executed, in order to evaluate an AO-Core resolution. -vary_context(Ctx = #{ <<"base">> := Base, <<"req">> := Req, <<"key">> := Key, <<"priv">> := Priv }, Opts) -> - VaryRes = - primitive( - Ctx#{ - <<"key">> => <<"vary">>, - <<"base">> => Base, - <<"req">> => - #{ - <<"path">> => <<"vary">>, - <<"vary">> => Key, - <<"...">> => Req - } - }, - Opts#{ <<"mode">> => hb_opts:get(<<"mode">>, <<"raw">>, Opts) } - ), - ?prim_dbg({vary_result, erlang:external_size(VaryRes)}), - ?prim_dbg( - { - priv_vary, - {before, hb_private:from_message(Ctx)}, - {'after', hb_private:from_message(VaryRes)} - } - ), - case VaryRes of - {ok, CtxWithRes = #{ <<"result">> := _Result }} -> - post_execution(CtxWithRes, Opts); - {ok, PreparedCtx} -> - lookup_or_exec(PreparedCtx, Opts); - Other -> - throw({unexpected_vary_result, Other}) - end. - -%% @doc Look up a result from the cache and if not found, execute the context. -lookup_or_exec(Ctx = #{ <<"priv">> := #{ <<"mode">> := SkipMode } }, Opts) - when SkipMode =:= <<"raw">>; SkipMode =:= <<"primitive">> -> - execute_context(Ctx, Opts); -lookup_or_exec( - Ctx = #{ - <<"varied-base">> := VBase, - <<"varied-req">> := VReq, - <<"key">> := Key, - <<"priv">> := Priv - }, Opts) -> - ?prim_dbg({priv_during_lookup, Priv}), - {ok, VBaseID} = primitive(VBase, #{ <<"path">> => <<"id">> }, Opts), - {ok, VReqID} = primitive(VReq, #{ <<"path">> => <<"id">> }, Opts), - CtxWithIDs = - Ctx#{ - <<"varied-base-id">> => VBaseID, - <<"varied-req-id">> => VReqID - }, - case hb_cache_control:maybe_lookup(VBaseID, VReqID, Opts) of - {hit, {Status, Msg}} -> - ?prim_dbg({cache_hit, Key, {Status, Msg}}), - post_execution( - CtxWithIDs#{ <<"result">> => Msg, <<"status">> => Status }, - Opts - ); - Miss -> - ?prim_dbg({cache_miss, Key, Miss}), - execute_context(CtxWithIDs, Opts) - end. - -%% @doc Takes a fully prepated context (with `varied-base` and `varied-req` -%% set, along with `priv/resolver` etc.) and executes it. Does not cache, -%% extend, or otherwise post-process the result. -execute_context( - Ctx = #{ - <<"key">> := Key, - <<"varied-base">> := VariedBase, - <<"varied-req">> := VariedReq, - <<"priv">> := - #{ - <<"resolver">> := Fun, - <<"add-key">> := AddKey - } - }, - Opts) -> - % Apply the function and return the result directly, without any further - % processing. We add the `PrefixArgs` to the list of arguments to be passed - % to the function to accomodate default handlers, which take the key that - % was invoked on the device as the first argument (ahead of `Base`, `Req`, - % and `ExecOpts`). - ?prim_dbg({execute, Fun}), - {Status, Res} = - apply( - Fun, - hb_device:truncate_args( - Fun, - if AddKey -> [Key]; true -> [] end - ++ [VariedBase, VariedReq, Opts] - ) - ), - post_execution(Ctx#{ <<"result">> => Res, <<"status">> => Status }, Opts). - -%% @doc Handle post-execution bookkeeping. If the mode is `primitive`, return -%% the result directly; else, apply the `return-extends` schema to the result. -%% Handles the three core variants via two separate function heads: -%% 1. `primitive` mode: return the result directly -%% 2a. `raw` mode: Apply the `return-extends` schema to the result, don't store. -%% 2b. `normal` mode: Apply the `return-extends` schema to the result, store in -%% the cache if `cache-control` permits. -post_execution( - #{ - <<"priv">> := #{ <<"mode">> := <<"primitive">> }, - <<"result">> := Res, - <<"status">> := Status, - <<"key">> := Key - }, - _Opts) -> - ?prim_dbg({completed_exec, primitive, {key, Key}}), - {Status, Res}; -post_execution( - Context = #{ - <<"key">> := Key, - <<"base">> := Base, - <<"req">> := Req, - <<"result">> := RawResult, - <<"status">> := Status, - <<"return-extends">> := Extend - }, - Opts) -> - ?prim_dbg({completed_exec, normal, {key, Key}, {extend, Extend}}), - maybe_cache(Context, Opts), - Res = - case Extend of - none -> RawResult; - base -> RawResult#{ <<"...">> => Base }; - request -> RawResult#{ <<"...">> => Req } - end, - {Status, Res}. - -maybe_cache(#{ <<"priv">> := #{ <<"mode">> := <<"raw">> } }, _Opts) -> - skipped_caching; -maybe_cache( - #{ - <<"key">> := Key, - <<"varied-base">> := VBase, - <<"varied-req">> := VReq, - <<"varied-base-id">> := VBaseID, - <<"varied-req-id">> := VReqID, - <<"result">> := Res - }, Opts) -> - hb_cache_control:maybe_store(VBaseID, VReqID, VBase, VReq, Res, Opts). \ No newline at end of file + end. \ No newline at end of file diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 9bad9988b..4e7fa7dce 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -21,7 +21,7 @@ %% 3. The `Req' message (the user's request). %% Base is not used, such that it can specify cache control information about %% itself, without affecting its outputs. -maybe_store(BaseID, ReqID, Base, Req, Res, Opts) -> +maybe_store(Base, Req, Res, Opts) -> case derive_cache_settings([Res, Req], Opts) of #{ <<"store">> := true } -> dispatch_cache_write(Base, Req, Res, Opts), diff --git a/src/core/resolver/hb_persistent.erl b/src/core/resolver/hb_persistent.erl index 17f575962..626aea84a 100644 --- a/src/core/resolver/hb_persistent.erl +++ b/src/core/resolver/hb_persistent.erl @@ -103,12 +103,10 @@ do_monitor(Group, Last, Opts) -> %% explicitly disabled (the common case for internal/recursive resolves) we %% skip the grouper dispatch entirely: the leader short-circuit below would %% discard the computed group name anyway. -find_or_register(_Base, _Req, #{ <<"await-inprogress">> := false }) -> - {leader, ungrouped_exec}; find_or_register(Base, Req, Opts) -> find_or_register(group(Base, Req, Opts), Base, Req, Opts). find_or_register(ungrouped_exec, _Base, _Req, _Opts) -> - {leader, ungrouped_exec}; + {skip, ungrouped_exec}; find_or_register(GroupName, _Base, _Req, Opts) -> case hb_opts:get(await_inprogress, false, Opts) of false -> {leader, GroupName}; From 7e0aaff0cce979a95a451c80f5540eed9ca88f76 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 10 Jul 2026 14:47:22 -0400 Subject: [PATCH 22/35] wip: More AO-Core 1.0 resolver work --- src/core/device/hb_device.erl | 6 +- src/core/resolver/hb_ao.erl | 219 +++++++++++++------------ src/core/resolver/hb_cache.erl | 8 +- src/core/resolver/hb_cache_control.erl | 2 +- src/core/resolver/hb_persistent.erl | 4 +- src/core/resolver/hb_private.erl | 2 +- src/core/resolver/hb_types.erl | 26 +-- src/preloaded/codec/dev_structured.erl | 2 +- src/preloaded/message/dev_message.erl | 20 ++- src/preloaded/util/dev_apply.erl | 2 +- 10 files changed, 160 insertions(+), 131 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index ef788e3b2..7b6efea44 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -2,7 +2,7 @@ %%% Offers services for loading, verifying executability, and extracting Erlang %%% functions from a device. -module(hb_device). --export([truncate_args/2, add_resolver/2, message_to_fun/3, module/2]). +-export([truncate_args/2, add_resolver/2, message_to_fun/3, message_to_fun/4, module/2]). -export([message_device_id/3]). -export([is_direct_key_access/3, is_direct_key_access/4]). -export([find_exported_function/5, is_exported/4, info/2, info/3]). @@ -39,7 +39,7 @@ truncate_args(Fun, Args) -> %% priv/exec-device: The device from which the execution function originates. %% If the `forced-device` resolvers from another device, %% this key will differ from the `forced-device`. -%% priv/resolver: The function to execute to resolve the `forced-device`. +%% priv/function: The function to execute to resolve the `forced-device`. %% priv/add-key: Whether the execution function expects that we should %% add the `key` as an additional argument to the start of %% the argument list. @@ -70,7 +70,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> OldPriv#{ <<"resolver-module">> => ExecMod, <<"add-key">> => Type == add_key, - <<"resolver">> => Fun + <<"function">> => Fun } } } diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index b3ecec79b..bc4cacdd3 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -82,19 +82,11 @@ %%% %%% info/worker : A function that should be run as the 'server' loop of %%% the executor for interactions using the device. -%%% -%%% The HyperBEAM resolver also takes a number of runtime options that change -%%% the way that the environment operates: -%%% -%%% `update_hashpath': Whether to add the `Req' to `HashPath' for the `Res'. -%%% Default: true. -%%% `add_key': Whether to add the key to the start of the arguments. -%%% Default: `'. %%% -module(hb_ao). %%% Main AO-Core API: -export([resolve/2, resolve/3]). --export([raw/3, raw/4, raw/5, primitive/5]). +-export([raw/3, raw/4, raw/5]). -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). %%% Shortcuts and tools: -export([keys/2]). @@ -106,7 +98,6 @@ -define( TEMP_OPTS, [ - <<"add-key">>, <<"cache-control">>, <<"spawn-worker">>, <<"only">>, @@ -133,16 +124,19 @@ raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> {forced_key, ForcedKey} } ), - do( - execution_context( - ForcedDevice, - ForcedKey, - Base, - Req, - Opts#{ - <<"cache-control">> => <<"none">>, - <<"hashpath">> => ignore - } + from_context( + do( + to_context( + ForcedDevice, + ForcedKey, + Base, + Req, + Opts#{ + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"await-inprogress">> => false, + <<"hashpath">> => ignore + } + ) ) ). @@ -235,44 +229,48 @@ do_resolve_many([Base, Req | MsgList], Opts) -> ), do_resolve_many([Res | MsgList], Opts); Res -> - throw({unexpected_resolve_many_response, Res}), - % The result is not a resolvable message. Return it. - ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), + % The result is not a continuable message. Return it. + ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating, Res}), Res end. %% @doc Resolve only a single, explicit, computation step over a path-normalized %% (single part) `Base` and `Req` pair. resolve_single(Base, Req, Opts) -> - case do(context(undefined, undefined, Base, Req, Opts)) of - {ok, #{ <<"result">> := Res}} -> {ok, Res}; - {error, Reason} -> {error, Reason} - end. + from_context(do(to_context(undefined, undefined, Base, Req, Opts))). %% @doc Create an execution context from a set of core parameters: %% `ForcedDevice` (if relevant), `ForcedKey` (if relevant), `Base`, `Req`, %% and `Opts`. -context(FDevice, FKey, Base, Req, Opts) -> +to_context(FDevice, FKey, Base, Req, Opts) -> maps:filter( fun(_K, V) -> V =/= undefined end, #{ - <<"forced-device">> => FDevice, - <<"forcedkey">> => FKey, + <<"device">> => FDevice, + <<"path">> => FKey, <<"base">> => Base, - <<"req">> => Req, + <<"request">> => Req, <<"opts">> => Opts } ). - -%% @doc Resolves a fully normalized execution context, returning early if any -%% stage fails. + +%% @doc Convert a completed execution context into the caller-facing +%% `{ExecStatus, Result}` form from its stage-bound `context`. +from_context({ok, Ctx = #{ <<"result">> := Res }}) -> + {maps:get(<<"status">>, Ctx, ok), Res}; +from_context(Other) -> + throw({unexpected_context_after_exec, Other}). + +%% @doc Resolves a fully normalized execution context. Stages that find the +%% result early set it in the context; later stages skip themselves when they +%% see it. Any non-`{ok, Ctx}` return propagates as the result. do(Ctx0) -> maybe - % Stage 1: Normalization and loading of IDs/links. + % Stage 1: Normalization; device or direct key lookup. {ok, Ctx1} ?= stage_1(Ctx0), - % Stage 2: Device or direct key lookup. + % Stage 2: Function lookup. {ok, Ctx2} ?= stage_2(Ctx1), - % Stage 3: Vary `Base` and `Request`. + % Stage 3: Vary `Base` and `Req`. {ok, Ctx3} ?= stage_3(Ctx2), % Stage 4: Persistent resolver lookup. {ok, Ctx4} ?= stage_4(Ctx3), @@ -289,20 +287,22 @@ do(Ctx0) -> % Stage 10: Execution of the `step' hook. {ok, Ctx10} ?= stage_10(Ctx9), % Stage 11: Fork worker. - {ok, Ctx11} ?= stage_11(Ctx10) + stage_11(Ctx10) else - {hit, Ctx = #{ <<"must-cache">> := false } } -> - % If we hit the cache early (e.g., a direct key access), return - % immediately without the `must-cache' flag. - {ok, maps:without([<<"must-cache">>], Ctx)} + {hit, Ctx} -> + % A stage may choose to return early. If it does, return it in + % `ok`-form. We assume that the stage in question will have + % appropriately cleaned up. + {ok, Ctx} end. %% @doc Normalize the context of an execution request. Ensures that a device and -%% path are available for execution. -stage_1(Ctx = #{<<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; +%% path are available for execution, or that a direct key hit resolves the +%% request without one. +stage_1(Ctx = #{ <<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> case hb_device:message_device_id(Base, Path, Opts) of - {hit, Res} -> {hit, Ctx#{ <<"result">> => Res }}; + {hit, Res} -> {hit, Ctx#{ <<"status">> => ok, <<"result">> => Res }}; {ok, Device} -> stage_1(Ctx#{ <<"device">> => Device }) end; stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) -> @@ -314,24 +314,25 @@ stage_2( <<"device">> := Device, <<"base">> := Base, <<"path">> := Path, - <<"priv">> := Priv, <<"opts">> := Opts } ) -> - case hb_device:message_to_fun(Device, Base, Path, Opts) of - {ok, Function} -> - {ok, Ctx#{ <<"priv">> => Priv#{ <<"function">> => Function }}}; - {error, Reason} -> - {error, Reason} - end. + {Status, _ExecDev, _ExecMod, Function} = + hb_device:message_to_fun(Device, Base, Path, Opts), + {ok, + Ctx#{ + <<"add-key">> => case Status of add_key -> Path; _ -> false end, + <<"priv">> => #{ <<"function">> => Function } + } + }. -%% @doc Vary the `Base` and `Req` of the resolution. To do so, assuming that +%% @doc Vary the `Base` and `Req` of the resolution. To do so, assuming that %% the request is not itself a `vary` call, we recursively resolve the %% computation as a sub-request. This allows the device to offer its own `vary` %% logic, based upon the device's own understanding of the request. stage_3(Ctx = #{ <<"path">> := <<"vary">>, <<"base">> := Base, <<"request">> := Req }) -> % We are already varying. Do not recurse. - {ok, Ctx#{ <<"varied-base">> => Base, <<"varied-req">> => Req }}; + {ok, Ctx#{ <<"varied-base">> => Base, <<"varied-request">> => Req }}; stage_3( Ctx = #{ @@ -342,8 +343,10 @@ stage_3( } ) -> maybe - % Raw call `vary` to find the varied base/req, passing the `priv/function` - % such that varying does not require us to repeat the associated work. + % Raw call `vary` to find the varied base/req, passing the + % `priv/function` such that varying does not require us to repeat the + % associated work. The returned context fragment (varied messages, + % result normalization) is merged into our own. {ok, VariedCtx} ?= raw( undefined, @@ -362,22 +365,20 @@ stage_3( stage_4( Ctx = #{ <<"varied-base">> := Base, - <<"varied-req">> := Req, + <<"varied-request">> := Req, <<"opts">> := Opts }) -> ?event_debug(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. - % Before we search for a live executor, we check if the device specifies - % a function that tailors the 'group' name of the execution. For example, + % Before we search for a live executor, we check if the device specifies + % a function that tailors the 'group' name of the execution. For example, % the `~process@1.0' device 'groups' all calls to the same process onto % calls to a single executor. By default, `{Base, Req}' is used as the % group name. case hb_persistent:find_or_register(Base, Req, Opts) of - {leader, ExecName} -> - % We are the leader for this resolution. Continue while noting - % this in our context. - {ok, Ctx#{ <<"leader">> => ExecName }}; + {skip, ungrouped_exec} -> {ok, Ctx}; + {leader, ExecName} -> {ok, Ctx#{ <<"leader">> => ExecName }}; {wait, Leader} -> % There is another executor of this resolution in-flight. % register to receive the response, then @@ -396,14 +397,18 @@ stage_4( ), % Re-try again if the group leader has died. stage_4(Ctx); - {ok, Result} -> + {Status, Result} -> % We received a successful result from another worker. They % will store the result if appropriate, so we skip the store % write. - {hit, Ctx#{ <<"result">> := Result }} + {ok, Ctx#{ <<"status">> => Status, <<"result">> => Result }}; + Other -> + % The leader resolved to a non-`ok` result. Propagate it. + ?event_debug(debug_ao_core, {unexpected_worker_result, Other}), + Other end; {infinite_recursion, GroupName} -> - % We are the leader for this resolution, but we executing the + % We are the leader for this resolution, but we executing the % computation again. This may plausibly be OK in _some_ cases, % but in general it is the sign of a bug. ?event( @@ -434,36 +439,30 @@ stage_4( end. %% @doc Look up whether the varied base and request pair already has a known -%% result in the cache. If so, we skip the resolution step and proceed using -%% the cached result. We signal that further execution is not needed by -%% returning the context +%% result in the cache. If so, set it in the context: subsequent stages skip +%% execution, and stage 8 notifies any waiting callers. stage_5(Ctx = #{ <<"varied-base">> := Base, - <<"varied-req">> := Req, + <<"varied-request">> := Req, <<"opts">> := Opts }) -> case hb_cache_control:maybe_lookup(Base, Req, Opts) of - {ok, Result} -> - % The result was found, so we return a `hit`, but we also unregister - % and notify if - case maps:get(<<"leader">>, Ctx, false) of - false -> skip; - ExecName -> - hb_persistent:unregister_notify( - ExecName, - Req, - {ok, Result}, - Opts - ) - end, - {hit, Ctx#{ <<"result">> := Result }}; - _ -> {ok, Ctx} + {error, not_found} -> {ok, Ctx}; + {Status, Result} -> + { + ok, + Ctx#{ + <<"result">> => Result, + <<"status">> => Status + } + } end. %% @doc Perform the actual resolution of an AO-Core request. +stage_6(Ctx = #{ <<"result">> := _ }) -> {ok, Ctx}; stage_6(Ctx = #{ <<"varied-base">> := Base, - <<"varied-req">> := Req, + <<"varied-request">> := Req, <<"opts">> := Opts, <<"priv">> := #{ <<"function">> := Func } }) -> @@ -487,19 +486,31 @@ stage_6(Ctx = #{ }, Opts ), - {Status, Ctx#{ <<"result">> => Res, <<"status">> => Status }}. + { + ok, + Ctx#{ + <<"status">> => Status, + <<"result">> => Res, + <<"fresh">> => true + } + }. %% @doc Cache the result of an execution if appropriate for the context. +%% Results that were not freshly and successfully executed (cache hits, direct +%% key hits, awaited results, non-`ok` statuses) are not stored. stage_7( Ctx = #{ + <<"status">> := ok, + <<"fresh">> := true, <<"varied-base">> := VariedBase, - <<"varied-req">> := VariedReq, + <<"varied-request">> := VariedReq, <<"result">> := Res, <<"opts">> := Opts } ) -> hb_cache_control:maybe_store(VariedBase, VariedReq, Res, Opts), - {ok, Ctx}. + {ok, Ctx}; +stage_7(Ctx) -> {ok, Ctx}. %% @doc Return the resolved response to any waiting callers. stage_8( @@ -511,15 +522,22 @@ stage_8( <<"opts">> := Opts } ) -> - hb_persistent:unregister_notify(ExecName, Req, {Status, Res}, Opts), + hb_persistent:unregister_notify( + ExecName, + Req, + {Status, Res}, + Opts + ), {ok, Ctx}; stage_8(Ctx) -> + % If we are not the leader, we can ignore the unregister step. {ok, Ctx}. -%% @doc When specified in the schema for the device call, normalize the execution's -%% result on top of the `Base` or `Request` message. +%% @doc When specified in the schema for the device call, normalize the +%% execution's result on top of the `Base` or `Req` message. stage_9( Ctx = #{ + <<"status">> := ok, <<"normalizer">> := Normalizer, <<"base">> := Base, <<"request">> := Req, @@ -550,15 +568,15 @@ stage_10(Ctx = #{ <<"opts">> := Opts = #{ <<"on">> := #{ <<"step">> := _ }}}) -> hb_hook:on(<<"step">>, Ctx, Opts); stage_10(Ctx) -> {ok, Ctx}. -%% @doc If we have been requested to spawn a worker process to remain active +%% @doc If we have been requested to spawn a worker process to remain active %% at this stage, with this context in memory, we do so. stage_11( Ctx = #{ - <<"exec_name">> := ExecName, + <<"leader">> := ExecName, <<"result">> := Res, <<"opts">> := Opts }) -> - ?event_debug(debug_ao_core, {stage, 12, maybe_spawn_worker}, Opts), + ?event_debug(debug_ao_core, {stage, 11, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case {is_map(Res), hb_opts:get(<<"spawn-worker">>, false, Opts#{ <<"prefer">> => local })} @@ -570,7 +588,8 @@ stage_11( WorkerPID = hb_persistent:start_worker(ExecName, Res, Opts), hb_persistent:forward_work(WorkerPID, Opts), {ok, Ctx} - end. + end; +stage_11(Ctx) -> {ok, Ctx}. %% @doc If the `AO_PROFILING' macro is defined (set by building/launching with %% `rebar3 as ao_profiling') we record statistics about the execution of the @@ -767,7 +786,7 @@ set(Base, Req, Opts) -> set(Base, Key, Value, Opts) -> DeepBase = fun Deep([LeafKey]) -> #{ LeafKey => Value }; - Deep([NextKey|Rest]) -> #{ NextKey => Deep(Rest, Value) } + Deep([NextKey|Rest]) -> #{ NextKey => Deep(Rest) } end, deep_set(Base, DeepBase(hb_path:term_to_path_parts(Key, Opts)), Opts). @@ -807,10 +826,8 @@ normalize_key(Key, _Opts) when is_list(Key) -> %% @doc Ensure that a message is processable by the AO-Core resolver: No lists. %% Fast path: when every key is already a binary, `normalize_key/2' would %% pass them through unchanged (values are never recursed here), so return -%% the map as-is. This is the overwhelming majority case on the resolve hot -%% path -- `resolve_stage(1, ...)' normalises both `Base' and `Req' on every -%% resolution -- and skips the `hb_maps:to_list'/`from_list' round-trip and -%% per-key dispatch. +%% the map as-is. This is the overwhelming majority case, and skips the +%% `hb_maps:to_list'/`from_list' round-trip and per-key dispatch. normalize_keys(Msg) -> normalize_keys(Msg, #{}). normalize_keys(Base, Opts) when is_list(Base) -> normalize_keys( diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index f4ba51d4c..13db27a13 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -1026,13 +1026,17 @@ structured_decode_types(Types, Opts) -> %% message and return it if it exists. %% 3. If the message has an explicit device, we attempt to read the hashpath to %% see if it has already been computed. -read_resolved(BaseID, ReqID, Opts) -> +read_resolved(BaseID, ReqID, Opts) when is_binary(BaseID), is_binary(ReqID) -> ExpectedPath = <>, ?prim_dbg({read_resolved, ExpectedPath}), case read(ExpectedPath, Opts) of {error, not_found} -> miss; Other -> {hit, Other} - end. + end; +read_resolved(_Base, _Req, _Opts) -> + % The varied pair's IDs have not been generated, so there is no address to + % consult. + miss. %% @doc Return a key from an in-memory message, returning the same form as %% a store read (`{Status, Value}'). diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 4e7fa7dce..08a5586f6 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -3,7 +3,7 @@ %%% node Opts. It applies these settings when asked to maybe store/lookup in %%% response to a request. -module(hb_cache_control). --export([maybe_store/6, maybe_lookup/3]). +-export([maybe_store/4, maybe_lookup/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/core/resolver/hb_persistent.erl b/src/core/resolver/hb_persistent.erl index 626aea84a..36cf57fa8 100644 --- a/src/core/resolver/hb_persistent.erl +++ b/src/core/resolver/hb_persistent.erl @@ -103,12 +103,14 @@ do_monitor(Group, Last, Opts) -> %% explicitly disabled (the common case for internal/recursive resolves) we %% skip the grouper dispatch entirely: the leader short-circuit below would %% discard the computed group name anyway. +find_or_register(_Base, _Req, #{ <<"await-inprogress">> := false }) -> + {skip, ungrouped_exec}; find_or_register(Base, Req, Opts) -> find_or_register(group(Base, Req, Opts), Base, Req, Opts). find_or_register(ungrouped_exec, _Base, _Req, _Opts) -> {skip, ungrouped_exec}; find_or_register(GroupName, _Base, _Req, Opts) -> - case hb_opts:get(await_inprogress, false, Opts) of + case hb_opts:get(<<"await-inprogress">>, false, Opts) of false -> {leader, GroupName}; _ -> Self = self(), diff --git a/src/core/resolver/hb_private.erl b/src/core/resolver/hb_private.erl index 300706864..71fd881ea 100644 --- a/src/core/resolver/hb_private.erl +++ b/src/core/resolver/hb_private.erl @@ -89,7 +89,7 @@ is_private(ListKey) when is_list(ListKey) -> % Strings should always be lists, but in case for some reason the caller % ignores that... try is_private(hb_util:bin(ListKey)) catch _ -> false end; -is_private(<<"priv", _>>) -> true; +is_private(<<"priv", _/binary>>) -> true; is_private(_) -> false. %% @doc Remove the first key from the path if it is a private specifier. diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl index 9e8fe512f..249e667f3 100644 --- a/src/core/resolver/hb_types.erl +++ b/src/core/resolver/hb_types.erl @@ -14,7 +14,7 @@ add_schema( <<"resolver-device">> := Device, <<"priv">> := #{ - <<"resolver">> := Func, + <<"function">> := Func, <<"add-key">> := AddKey } }, @@ -28,7 +28,7 @@ add_schema( {key, Key} } ), - {ok, Ctx#{ <<"schema">> => undefined } }; + {ok, Ctx}; Schema -> ?prim_dbg( {schema_found, @@ -42,25 +42,25 @@ add_schema( end. %% @doc Apply the resolved function's base/request schemas to one execution. -vary(Ctx = #{ <<"base">> := Base, <<"req">> := Req }, _Opts) +vary(Ctx = #{ <<"base">> := Base, <<"request">> := Req }, _Opts) when not is_map_key(<<"schema">>, Ctx) -> {ok, Ctx#{ <<"varied-base">> => Base, - <<"varied-req">> => Req, - <<"return-extends">> => none + <<"varied-request">> => Req, + <<"normalizer">> => none } }; vary(Ctx = #{ <<"schema">> := [BaseSchema, ReqSchema, ReturnSchema], <<"base">> := Base, - <<"req">> := Req + <<"request">> := Req }, Opts) -> {ok, Ctx#{ <<"varied-base">> => apply_schema(implicit_base(BaseSchema), Base, Opts), - <<"varied-req">> => apply_schema(implicit_request(ReqSchema), Req, Opts), - <<"return-extends">> => overlay(ReturnSchema) + <<"varied-request">> => apply_schema(implicit_request(ReqSchema), Req, Opts), + <<"normalizer">> => overlay(ReturnSchema) } }. @@ -628,10 +628,10 @@ boolean_type() -> empty_projection_test() -> ?assertEqual( - #{ <<"device">> => <<"test@1.0">> }, + #{ <<"device">> => <<"test-device@1.0">> }, apply_schema( implicit_base(empty_type()), - #{ <<"device">> => <<"test@1.0">>, <<"extra">> => <<"drop">> }, + #{ <<"device">> => <<"test-device@1.0">>, <<"extra">> => <<"drop">> }, #{} ) ). @@ -682,8 +682,8 @@ default_handler_uses_resolved_function_schema_test() -> Func, <<"requested-key">>, #{ - {<<"default-schema-fun">>, 4} => default_schema, - {<<"requested-key">>, 3} => requested_key_schema + <<"default-schema-fun">> => default_schema, + <<"requested-key">> => requested_key_schema } ) ), @@ -692,7 +692,7 @@ default_handler_uses_resolved_function_schema_test() -> select_schema( Func, <<"requested-key">>, - #{{<<"requested-key">>, 3} => requested_key_schema} + #{ <<"other-key">> => other_schema } ) ). diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index 61177a5ae..61807b3d8 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -202,7 +202,7 @@ apply_bundle_hint(Msg, Req, Opts) -> case hb_maps:get(<<"hint-device">>, Req, undefined, Opts) of undefined -> Req; DeviceBin -> - case hb_ao:primitive(DeviceBin, <<"to-hint">>, Msg, Req, Opts) of + case hb_ao:raw(DeviceBin, <<"to-hint">>, Msg, Req, Opts) of {ok, HintedReq} -> % May add a `bundle` key to the request HintedReq; diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 300d724b0..9a207e58b 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -701,13 +701,21 @@ default_accessor(Key, Msg, Req, Opts) -> %% during the process of `vary`ing. -spec vary( #{ _ => _ }, - #{ vary => binary(), '...' => #{ _ => _ }, _ => _ }, + #{ vary => binary(), _ => _ }, #{}) -> {ok, #{}}. vary(Base, Req, Opts) -> maybe - {ok, Key} ?= hb_maps:find(<<"vary">>, Req, Opts), - {ok, VaryReq} ?= hb_maps:find(<<"...">>, Req, Opts), - Ctx1 = #{ <<"base">> => Base, <<"key">> => Key, <<"req">> => VaryReq }, + {ok, Key} ?= + case maps:find(<<"vary">>, Req) of + {ok, KeyToVaryOn} -> KeyToVaryOn; + error -> + case maps:find(<<"path">>, Req) of + {ok, <<"vary">>} -> + {error, <<"Cannot vary the `vary` path.">>}; + Other -> Other + end + end, + Ctx1 = #{ <<"base">> => Base, <<"key">> => Key, <<"request">> => Req }, {ok, Ctx2} ?= case hb_private:get(<<"function">>, Req, not_found, Opts) of not_found -> @@ -717,9 +725,7 @@ vary(Base, Req, Opts) -> end, hb_types:vary(Ctx2, Opts) else - error -> - ?prim_dbg({vary_error, {base, Base}, {req, Req}}), - throw({invalid_vary_call, {base, Base}, {req, Req}}) + error -> {error, <<"invalid-vary-request">>} end. %% @doc Returns the device schema for a `Base` message. diff --git a/src/preloaded/util/dev_apply.erl b/src/preloaded/util/dev_apply.erl index c1d7e813e..99659a083 100644 --- a/src/preloaded/util/dev_apply.erl +++ b/src/preloaded/util/dev_apply.erl @@ -136,7 +136,7 @@ find_key(Path, Base, Request, Opts) -> {message, Request}; [<<"base">>, Key] -> {resolve, [{BaseAs, normalize_path([Key|RestKeys])}]}; - [Req, Key] when Req == <<"request">> orelse Req == <<"req">> -> + [Req, Key] when Req == <<"request">> orelse Req == <<"request">> -> {resolve, [{RequestAs, normalize_path([Key|RestKeys])}]}; [_] -> {resolve, [ From f1352d6d3cf56856b2095eec85d6bb4cd2425232 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 11 Jul 2026 11:57:20 -0400 Subject: [PATCH 23/35] impr: AO-Core 1.0 spec DRAFT-4 --- AO-CORE-DRAFT-3.md => AO-CORE-DRAFT-4.md | 39 +++++++++++++++++------- 1 file changed, 28 insertions(+), 11 deletions(-) rename AO-CORE-DRAFT-3.md => AO-CORE-DRAFT-4.md (93%) diff --git a/AO-CORE-DRAFT-3.md b/AO-CORE-DRAFT-4.md similarity index 93% rename from AO-CORE-DRAFT-3.md rename to AO-CORE-DRAFT-4.md index eeca1aa92..6048f32ba 100644 --- a/AO-CORE-DRAFT-3.md +++ b/AO-CORE-DRAFT-4.md @@ -1,4 +1,4 @@ -# AO-Core 1.0 Draft 3 +# AO-Core 1.0 Draft 4 AO-Core is a protocol for attestable computation over protocol-addressed messages. It is not itself a virtual machine. It is a neutral meta-VM: a common @@ -53,12 +53,12 @@ outermost layer inward. At each layer: -1. Look locally for `device`. -2. If `device = Dev` is found, execute `Dev` against the original outermost +1. Look locally for `K`. +2. If `K = Value` is found, return `Value`. +3. If `K = unset` is found, return `not_found`. +4. Otherwise, look locally for `device`. +5. If `device = Dev` is found, execute `Dev` against the original outermost `Base` and the original `Request`. -3. Otherwise, look locally for `K`. -4. If `K = Value` is found, return `Value`. -5. If `K = unset` is found, return `not_found`. 6. Otherwise, look locally for `...`. 7. If `... = Ancestor` is found, continue at `Ancestor`. 8. Otherwise, return `not_found`. @@ -69,17 +69,17 @@ Pseudocode: resolve(Outer, Layer, Request): K = key(Request) - case local(Layer, "device") of - Dev -> return execute(Dev, Outer, Request) - not_found -> continue - end - case local(Layer, K) of Value -> return Value unset -> return not_found not_found -> continue end + case local(Layer, "device") of + Dev -> return execute(Dev, Outer, Request) + not_found -> continue + end + case local(Layer, "...") of Ancestor -> return resolve(Outer, Ancestor, Request) not_found -> return not_found @@ -98,6 +98,10 @@ If a device is inherited from an ancestor, it executes over the outermost state: selects `dev1` from the ancestor, but `dev1` sees `x = 5`. +Layer order determines which value answers a key; it does not restrict what a +device sees. A local key answers before its layer's device, but a device that +answers always executes over the complete outermost state. + ## Devices A device defines how requests are computed for a state. @@ -109,6 +113,19 @@ their transitions are named, witnessed, attested, transported, and traced. If no device is found, the default device is `message@1.0`. +A device is itself a value, and may be named by link. When the device value is +a message, execution is itself resolution: the request is resolved against the +device message extended over the outermost state: + +```text +execute(Dev, Outer, Request) = resolve(set(Outer, Dev), Request) +``` + +The device message's own keys answer first, its own `device` computes the +remainder, and the outermost state remains visible as ancestry. The recursion +terminates at a device the host implements directly; which devices those are is +an implementation fact. + Device loading, Erlang modules, function pointers, local caches, and runtime workers are implementation facts. The protocol fact is the device value that the computation commits to using. From 1bd811f7db77c70eb02c86e1265b78702c631ab9 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 11 Jul 2026 11:58:00 -0400 Subject: [PATCH 24/35] impr: implements DRAFT-4 message-key precedence ordering --- src/core/device/hb_device.erl | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index 7b6efea44..fcd4de15f 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -3,7 +3,7 @@ %%% functions from a device. -module(hb_device). -export([truncate_args/2, add_resolver/2, message_to_fun/3, message_to_fun/4, module/2]). --export([message_device_id/3]). +-export([id_or_direct_key/3]). -export([is_direct_key_access/3, is_direct_key_access/4]). -export([find_exported_function/5, is_exported/4, info/2, info/3]). -include("include/hb.hrl"). @@ -49,7 +49,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> DevRes = case maps:find(<<"forced-device">>, Context) of {ok, ForcedBaseDevice} -> {ok, ForcedBaseDevice}; - error -> message_device_id(Base, Key, Opts) + error -> id_or_direct_key(Base, Key, Opts) end, OldPriv = maps:get(<<"priv">>, Context, #{}), case DevRes of @@ -101,7 +101,7 @@ add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> %% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key, Opts) -> - DeviceID = message_device_id(Msg, Key, Opts), + DeviceID = id_or_direct_key(Msg, Key, Opts), message_to_fun(DeviceID, Msg, Key, Opts). message_to_fun(DevID, Msg, Key, Opts) -> message_to_fun(DevID, module(DevID, Opts), Msg, Key, Opts). @@ -193,18 +193,23 @@ module(DevID, Opts) -> %% @doc Return the device ID from a message, resolving through any ancestors %% as necessary. -message_device_id(List, _, _) when is_list(List) -> +id_or_direct_key(_, <<"priv", _/binary>>, _) -> + {error, <<"`priv*` keys not resolvable.">>}; +id_or_direct_key(Msg, Key, Opts) -> + do_id_or_direct_key(Msg, Key, Opts). +do_id_or_direct_key(List, _, _) when is_list(List) -> {ok, <<"message@1.0">>}; -message_device_id(Msg, Key, Opts) -> +do_id_or_direct_key(Msg, Key, Opts) -> ?event({finding_device_id, Msg}), - case hb_maps:find(<<"device">>, Msg, Opts) of - {ok, Device} -> {ok, Device}; + case hb_maps:find(Key, Msg, Opts) of + {ok, <<"unset">>} -> {error, not_found}; + {ok, Value} -> {hit, Value}; error -> - case hb_maps:find(Key, Msg, Opts) of - {ok, Value} -> {hit, Value}; + case hb_maps:find(<<"device">>, Msg, Opts) of + {ok, Device} -> {ok, Device}; error -> case hb_maps:find(<<"...">>, Msg, Opts) of - {ok, Ancestor} -> message_device_id(Ancestor, Key, Opts); + {ok, Ancestor} -> do_id_or_direct_key(Ancestor, Key, Opts); error -> {ok, ?DEFAULT_DEVICE} end end From c9a706940c0e7ab044d2ef33a8d8ac79e835ce77 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 11 Jul 2026 11:58:25 -0400 Subject: [PATCH 25/35] fix: smaller resolver draft bugs --- src/core/resolver/hb_ao.erl | 24 ++++++++++++++++++++---- src/preloaded/message/dev_message.erl | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index bc4cacdd3..e68b1e3bf 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -293,7 +293,13 @@ do(Ctx0) -> % A stage may choose to return early. If it does, return it in % `ok`-form. We assume that the stage in question will have % appropriately cleaned up. - {ok, Ctx} + {ok, Ctx}; + {error, Ctx} -> + % The stage returned an error. Return to the caller early after + % unregistering as the active worker, if appropriate. We can call + % this twice, even if we erred after stage 8, as it is idempotent. + stage_8(Ctx), + {error, Ctx} end. %% @doc Normalize the context of an execution request. Ensures that a device and @@ -301,9 +307,11 @@ do(Ctx0) -> %% request without one. stage_1(Ctx = #{ <<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> - case hb_device:message_device_id(Base, Path, Opts) of + case hb_device:id_or_direct_key(Base, Path, Opts) of {hit, Res} -> {hit, Ctx#{ <<"status">> => ok, <<"result">> => Res }}; - {ok, Device} -> stage_1(Ctx#{ <<"device">> => Device }) + {ok, Device} -> stage_1(Ctx#{ <<"device">> => Device }); + {error, Reason} -> + {error, Ctx#{ <<"status">> => error, <<"reason">> => Reason }} end; stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) -> stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }). @@ -447,6 +455,13 @@ stage_5(Ctx = #{ <<"opts">> := Opts }) -> case hb_cache_control:maybe_lookup(Base, Req, Opts) of + {continue, NewBase, NewReq} -> + {ok, + #{ + <<"varied-base">> => NewBase, + <<"varied-request">> => NewReq + } + }; {error, not_found} -> {ok, Ctx}; {Status, Result} -> { @@ -555,7 +570,8 @@ stage_9( Fun when is_function(Fun) -> Fun(Base, Result, Opts) end } - }. + }; +stage_9(Ctx) -> {ok, Ctx}. %% @doc If a hook has been specified for the `step` action, we call it with our %% context including the result. diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 9a207e58b..e08f49fd5 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -707,7 +707,7 @@ vary(Base, Req, Opts) -> maybe {ok, Key} ?= case maps:find(<<"vary">>, Req) of - {ok, KeyToVaryOn} -> KeyToVaryOn; + {ok, KeyToVaryOn} -> {ok, KeyToVaryOn}; error -> case maps:find(<<"path">>, Req) of {ok, <<"vary">>} -> From 14a5c078bc137d8fb4060f786370279f0fa6b89d Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 12 Jul 2026 13:47:51 -0400 Subject: [PATCH 26/35] wip: more AO-Core 1.0 resolver work. ~50% of `hb_ao_test_vectors` pass. --- src/core/device/hb_device.erl | 48 ++-- src/core/resolver/hb_ao.erl | 41 +++- src/core/resolver/hb_message.erl | 7 +- src/core/test/hb_ao_test_vectors.erl | 322 +++++++++++++------------- src/preloaded/message/dev_message.erl | 60 +++-- 5 files changed, 253 insertions(+), 225 deletions(-) diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index fcd4de15f..2a87f7e71 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -105,6 +105,20 @@ message_to_fun(Msg, Key, Opts) -> message_to_fun(DeviceID, Msg, Key, Opts). message_to_fun(DevID, Msg, Key, Opts) -> message_to_fun(DevID, module(DevID, Opts), Msg, Key, Opts). +message_to_fun(DevID, DevMsg = #{ <<"device">> := _ }, _Msg, Key, _Opts) -> + % A message-valued device: execution is itself resolution. The request is + % resolved against the device message extended over the outermost state: + % `execute(Dev, Outer, Request) = resolve(set(Outer, Dev), Request)'. + % The key is forced upon the inner resolution: for forced-key executions + % (`vary', `set', ...) the request's own path differs from the key that + % this function was extracted to compute. + {ok, + DevID, + DevMsg, + fun(Base, Req, ExecOpts) -> + hb_ao:raw(undefined, Key, DevMsg#{ <<"...">> => Base }, Req, ExecOpts) + end + }; message_to_fun(DevID, DevMod, Msg, Key, Opts) -> Info = info(DevMod, Msg, Opts), % Is the key exported by the device? @@ -149,7 +163,7 @@ message_to_fun(DevID, DevMod, Msg, Key, Opts) -> ?event_debug({found_default_device, {mod, DefaultDevice}}), message_to_fun( DefaultDevice, - with_device(Msg, DefaultDevice), + hb_ao:as(DefaultDevice, Msg, Opts), Key, Opts ); @@ -165,10 +179,9 @@ message_to_fun(DevID, DevMod, Msg, Key, Opts) -> {key, Key} }); _ -> - WithDev = with_device(Msg, ?DEFAULT_DEVICE), message_to_fun( ?DEFAULT_DEVICE, - WithDev, + hb_ao:as(?DEFAULT_DEVICE, Msg, Opts), Key, Opts ) @@ -181,10 +194,8 @@ message_to_fun(DevID, DevMod, Msg, Key, Opts) -> %% message has no `<<"device">>' key, we resolve the default %% (`message@1.0') just like any other device: There is no privileged %% internal module-loading path. -module(DevMod, _Opts) when is_atom(DevMod) -> - DevMod; -module(Dev, _Opts) when is_map(Dev) -> - Dev; +module(DevMod, _Opts) when is_atom(DevMod) -> DevMod; +module(Dev, _Opts) when is_map(Dev) -> Dev; module(DevID, Opts) -> case hb_device_load:reference(DevID, Opts) of {error, Reason} -> throw({error, {device_not_loadable, DevID, Reason}}); @@ -215,18 +226,6 @@ do_id_or_direct_key(Msg, Key, Opts) -> end end. -%% @doc Small helper to extend a message with a device. -with_device(Msg, Device) when is_map(Msg) -> - hb_private:set_priv( - #{ - <<"device">> => Device, - <<"...">> => Msg - }, - hb_private:from_message(Msg) - ); -with_device(_Msg, Device) -> - #{ <<"device">> => Device }. - %% @doc Parse a handler key given by a device's `info'. info_handler_to_fun(Handler, DevID, DevMod, _Msg, _Key, _Opts) when is_function(Handler) -> @@ -240,7 +239,7 @@ info_handler_to_fun(HandlerMap, DevID, DevMod, Msg, Key, Opts) -> hb_maps:without([<<"device">>], Msg, Opts), message_to_fun( ?DEFAULT_DEVICE, - with_device(MsgWithoutDevice, ?DEFAULT_DEVICE), + hb_ao:as(?DEFAULT_DEVICE, MsgWithoutDevice, Opts), Key, Opts ); @@ -334,7 +333,14 @@ maybe_normalize_device_key(Key, Mode) -> %% @doc Get the info map for a device, optionally giving it a message if the %% device's info function is parameterized by one. -info(Msg, Opts) -> info(module(Msg, Opts), Msg, Opts). +info(Msg, Opts) -> + Device = + case id_or_direct_key(Msg, <<"device">>, Opts) of + {hit, Dev} -> Dev; + {ok, Dev} -> Dev; + {error, _} -> ?DEFAULT_DEVICE % `device => unset' masks: default. + end, + info(module(Device, Opts), Msg, Opts). info(DevMod, Msg, Opts) -> case find_exported_function(Msg, DevMod, info, 2, Opts) of {ok, Fun} -> apply(Fun, truncate_args(Fun, [Msg, Opts])); diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index e68b1e3bf..72a748be2 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -90,7 +90,7 @@ -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). %%% Shortcuts and tools: -export([keys/2]). --export([get/3, get/4, get_first/2, get_first/3]). +-export([as/3, get/3, get/4, get_first/2, get_first/3]). -export([set/3, set/4, deep_set/3, remove/3]). %%% Exports for tests in hb_ao_test_vectors.erl: -include("include/hb.hrl"). @@ -105,6 +105,18 @@ ] ). +%% @doc Small helper to extend a message with a device. +as(Device, Link, Opts) when ?IS_LINK(Link) -> + as(Device, hb_cache:ensure_loaded(Link, Opts), Opts); +as(Device, Msg, _Opts) when is_map(Msg) -> + hb_private:set_priv( + #{ + <<"device">> => Device, + <<"...">> => Msg + }, + hb_private:from_message(Msg) + ). + %% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring %% normalization, cache, hashpath, worker, and other management components. %% This function comes in `/3`-`/5` variants, allowing the caller to optionally @@ -118,12 +130,12 @@ raw(Base, Req, Opts) -> raw(Device, Base, Req, Opts) -> raw(Device, undefined, Base, Req, Opts). raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> - ?prim_dbg( - {executing, - {forced_device, ForcedDevice}, - {forced_key, ForcedKey} - } - ), + % ?prim_dbg( + % {executing, + % {forced_device, ForcedDevice}, + % {forced_key, ForcedKey} + % } + % ), from_context( do( to_context( @@ -313,8 +325,12 @@ stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) - {error, Reason} -> {error, Ctx#{ <<"status">> => error, <<"reason">> => Reason }} end; -stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) -> - stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }). +stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) when is_map(Req) -> + stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }); +stage_1(Ctx = #{ <<"request">> := Key }) -> + % If the request is for a direct key we normalize it to a message with + % only a path of that key. + stage_1(Ctx#{ <<"request">> => #{ <<"path">> => normalize_key(Key) } }). %% @doc Lookup the device and function to use during an execution. stage_2( @@ -456,8 +472,9 @@ stage_5(Ctx = #{ }) -> case hb_cache_control:maybe_lookup(Base, Req, Opts) of {continue, NewBase, NewReq} -> - {ok, - #{ + { + ok, + Ctx#{ <<"varied-base">> => NewBase, <<"varied-request">> => NewReq } @@ -559,7 +576,7 @@ stage_9( <<"result">> := Result, <<"opts">> := Opts } -) -> +) when Normalizer =/= none -> { ok, Ctx#{ diff --git a/src/core/resolver/hb_message.erl b/src/core/resolver/hb_message.erl index e2e6dc57d..eeb198be6 100644 --- a/src/core/resolver/hb_message.erl +++ b/src/core/resolver/hb_message.erl @@ -569,7 +569,7 @@ paranoid_verify(Topic, Msg, Opts) -> % without emitting an event or walking a topic list. This path fires % twice per `hb_ao:resolve/3', so the event/`lists:member' overhead is % noticeable on the hot path. - case hb_opts:get(paranoid_verify, false, Opts) of + case hb_opts:get(<<"paranoid-verify">>, false, Opts) of false -> true; [] -> true; true -> @@ -586,7 +586,10 @@ paranoid_verify(Topic, Msg, Opts) -> do_paranoid_verify(Topic, Msg, Opts) -> try - do_paranoid_verify(Topic, [], Msg, Opts), + % We disable paranoid verification on _downstream_ (only!) execution + % here to avoid infinite recursion. We must avoid disabling it anywhere + % else. + do_paranoid_verify(Topic, [], Msg, Opts#{ <<"paranoid-verify">> => false }), ?event_debug(debug_paranoia, {paranoid_verify_complete, ok}, Opts), true catch diff --git a/src/core/test/hb_ao_test_vectors.erl b/src/core/test/hb_ao_test_vectors.erl index 7bdfc11b8..e0f2e1b04 100644 --- a/src/core/test/hb_ao_test_vectors.erl +++ b/src/core/test/hb_ao_test_vectors.erl @@ -49,8 +49,6 @@ test_suite() -> fun resolve_from_multiple_keys_test/1}, {resolve_path_element, "resolve path element", fun resolve_path_element_test/1}, - {resolve_binary_key, "resolve binary key", - fun resolve_binary_key_test/1}, {key_to_binary, "key to binary", fun key_to_binary_test/1}, {key_from_id_device_with_args, "key from id device with args", @@ -124,82 +122,82 @@ test_opts() -> desc => "Default opts", opts => #{ <<"store">> => hb_test_utils:test_store() }, skip => [] - }, - #{ - name => without_hashpath, - desc => "Default without hashpath", - opts => #{ - <<"hashpath">> => ignore, - <<"store">> => hb_test_utils:test_store() - }, - skip => [] - }, - #{ - name => no_cache, - desc => "No cache read or write", - opts => #{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], - <<"spawn-worker">> => false, - <<"store">> => hb_test_utils:test_store() - }, - skip => [load_as] - }, - #{ - name => only_store, - desc => "Store, don't read", - opts => #{ - <<"hashpath">> => update, - <<"cache-control">> => [<<"no-cache">>], - <<"spawn-worker">> => false, - <<"store">> => CachedExecStore - }, - skip => [ - denormalized_device_name, - deep_set_with_device, - load_as - ], - reset => false - }, - #{ - name => only_if_cached, - desc => "Only read, don't exec", - opts => #{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"only-if-cached">>], - <<"spawn-worker">> => false, - <<"store">> => CachedExecStore - }, - skip => [ - % Skip test with locally defined device, amongst others. - resolve_id, - start_as, - start_as_with_parameters, - as_path, - multiple_as_subresolutions, - key_from_id_device_with_args, - get_with_denormalized_key, - set_new_messages, - resolve_from_multiple_keys, - resolve_path_element, - device_with_default_handler_function, - device_with_handler_function, - denormalized_device_name, - get_with_device, - get_as_with_device, - set_with_device, - device_exports, - device_excludes, - device_inheritance, - deep_set_with_device, - as, - as_commitments, - step_hook, - paranoid_message_verification, - paranoid_input_verification, - paranoid_result_verification - ] } + % #{ + % name => without_hashpath, + % desc => "Default without hashpath", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"store">> => hb_test_utils:test_store() + % }, + % skip => [] + % }, + % #{ + % name => no_cache, + % desc => "No cache read or write", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + % <<"spawn-worker">> => false, + % <<"store">> => hb_test_utils:test_store() + % }, + % skip => [load_as] + % }, + % #{ + % name => only_store, + % desc => "Store, don't read", + % opts => #{ + % <<"hashpath">> => update, + % <<"cache-control">> => [<<"no-cache">>], + % <<"spawn-worker">> => false, + % <<"store">> => CachedExecStore + % }, + % skip => [ + % denormalized_device_name, + % deep_set_with_device, + % load_as + % ], + % reset => false + % }, + % #{ + % name => only_if_cached, + % desc => "Only read, don't exec", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"cache-control">> => [<<"only-if-cached">>], + % <<"spawn-worker">> => false, + % <<"store">> => CachedExecStore + % }, + % skip => [ + % % Skip test with locally defined device, amongst others. + % resolve_id, + % start_as, + % start_as_with_parameters, + % as_path, + % multiple_as_subresolutions, + % key_from_id_device_with_args, + % get_with_denormalized_key, + % set_new_messages, + % resolve_from_multiple_keys, + % resolve_path_element, + % device_with_default_handler_function, + % device_with_handler_function, + % denormalized_device_name, + % get_with_device, + % get_as_with_device, + % set_with_device, + % device_exports, + % device_excludes, + % device_inheritance, + % deep_set_with_device, + % as, + % as_commitments, + % step_hook, + % paranoid_message_verification, + % paranoid_input_verification, + % paranoid_result_verification + % ] + % } ]. %%% Standalone test vectors @@ -343,7 +341,7 @@ resolve_simple_test(Opts) -> resolve_id_test(Opts) -> ?assertMatch( - ID when byte_size(ID) == 43, + ID when ?IS_ID(ID), hb_ao:get(<<"id">>, #{ <<"test_key">> => <<"1">> }, Opts) ). @@ -375,22 +373,6 @@ key_to_binary_test(Opts) -> ?assertEqual(<<"a">>, hb_ao:normalize_key(<<"a">>, Opts)), ?assertEqual(<<"a">>, hb_ao:normalize_key("a", Opts)). -resolve_binary_key_test(Opts) -> - ?assertEqual( - {ok, <<"RESULT">>}, - hb_ao:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) - ), - ?assertEqual( - {ok, <<"1">>}, - hb_ao:resolve( - #{ - <<"Test-Header">> => <<"1">> - }, - <<"Test-Header">>, - Opts - ) - ). - %% @doc Generates a test device with three keys, each of which uses %% progressively more of the arguments that can be passed to a device key. generate_device_with_keys_using_args() -> @@ -398,15 +380,15 @@ generate_device_with_keys_using_args() -> key_using_only_state => fun(State) -> {ok, - <<(hb_maps:get(<<"state_key">>, State))/binary>> + <<(hb_maps:get(<<"state-key">>, State))/binary>> } end, key_using_state_and_msg => fun(State, Msg) -> {ok, << - (hb_maps:get(<<"state_key">>, State))/binary, - (hb_maps:get(<<"msg_key">>, Msg))/binary + (hb_maps:get(<<"state-key">>, State))/binary, + (hb_maps:get(<<"msg-key">>, Msg))/binary >> } end, @@ -414,9 +396,9 @@ generate_device_with_keys_using_args() -> fun(State, Msg, Opts) -> {ok, << - (hb_maps:get(<<"state_key">>, State, undefined, Opts))/binary, - (hb_maps:get(<<"msg_key">>, Msg, undefined, Opts))/binary, - (hb_maps:get(<<"opts_key">>, Opts, undefined, Opts))/binary + (hb_maps:get(<<"state-key">>, State, undefined, Opts))/binary, + (hb_maps:get(<<"msg-key">>, Msg, undefined, Opts))/binary, + (hb_maps:get(<<"opts-key">>, Opts, undefined, Opts))/binary >> } end @@ -434,7 +416,7 @@ gen_default_device() -> end } end, - <<"state_key">> => + <<"state-key">> => fun(_) -> {ok, <<"STATE">>} end @@ -455,6 +437,13 @@ gen_handler_device() -> M2, Opts ); + (<<"vary">>, Base, Req, Opts) -> + {ok, + #{ + <<"varied-base">> => Base, + <<"varied-request">> => Req + } + }; (_, _, _, _) -> {ok, <<"HANDLER VALUE">>} end @@ -468,8 +457,8 @@ gen_handler_device() -> key_from_id_device_with_args_test(Opts) -> Msg = #{ - device => generate_device_with_keys_using_args(), - state_key => <<"1">> + <<"device">> => generate_device_with_keys_using_args(), + <<"state-key">> => <<"1">> }, ?assertEqual( {ok, <<"1">>}, @@ -477,7 +466,7 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_only_state">>, - <<"msg_key">> => <<"2">> % Param message, which is ignored + <<"msg-key">> => <<"2">> % Param message, which is ignored }, Opts ) @@ -488,7 +477,7 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_state_and_msg">>, - <<"msg_key">> => <<"3">> % Param message, with value to add + <<"msg-key">> => <<"3">> % Param message, with value to add }, Opts ) @@ -499,46 +488,45 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_all">>, - <<"msg_key">> => <<"3">> % Param message + <<"msg-key">> => <<"3">> % Param message }, Opts#{ - <<"opts_key">> => <<"37">>, + <<"opts-key">> => <<"37">>, <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } ) ). device_with_handler_function_test(Opts) -> - Msg = - #{ - device => gen_handler_device(), - test_key => <<"BAD">> - }, ?assertEqual( {ok, <<"HANDLER VALUE">>}, - hb_ao:resolve(Msg, <<"test_key">>, Opts) + hb_ao:resolve( + hb_ao:as(gen_handler_device(), #{ <<"test-key">> => <<"BAD">> }, Opts), + #{ <<"path">> => <<"test-key">> }, + Opts + ) ). device_with_default_handler_function_test(Opts) -> Msg = #{ - device => gen_default_device() + <<"device">> => gen_default_device() }, ?assertEqual( {ok, <<"STATE">>}, - hb_ao:resolve(Msg, <<"state_key">>, Opts) + hb_ao:resolve(Msg, #{<<"path">> => <<"state-key">>}, Opts) ), ?assertEqual( {ok, <<"DEFAULT">>}, - hb_ao:resolve(Msg, <<"any_random_key">>, Opts) + hb_ao:resolve(Msg, #{<<"path">> => <<"any-random-key">>}, Opts) ). basic_get_test(Opts) -> Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get([<<"key2">>], Msg, Opts)). + % Repeat to ensure consistency in caching setups. + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)). recursive_get_test(Opts) -> Msg = #{ @@ -591,24 +579,24 @@ get_with_device_test(Opts) -> Msg = #{ <<"device">> => generate_device_with_keys_using_args(), - <<"state_key">> => <<"STATE">> + <<"state-key">> => <<"STATE">> }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"key_using_only_state">>, Msg, Opts)). + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, Msg, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"key-using-only-state">>, Msg, Opts)). get_as_with_device_test(Opts) -> Msg = #{ <<"device">> => gen_handler_device(), - <<"test_key">> => <<"ACTUAL VALUE">> + <<"test-key">> => <<"ACTUAL VALUE">> }, ?assertEqual( <<"HANDLER VALUE">>, - hb_ao:get(test_key, Msg, Opts) + hb_ao:get(<<"test-key">>, Msg, Opts) ), ?assertEqual( <<"ACTUAL VALUE">>, - hb_ao:get(test_key, {as, <<"message@1.0">>, Msg}, Opts) + hb_ao:get(<<"test-key">>, hb_ao:as(<<"message@1.0">>, Msg, Opts), Opts) ). set_with_device_test(Opts) -> @@ -618,22 +606,22 @@ set_with_device_test(Opts) -> #{ <<"set">> => fun(State, _Msg) -> - Acc = hb_maps:get(<<"set_count">>, State, <<"">>, Opts), + Acc = hb_maps:get(<<"set-count">>, State, <<"">>, Opts), {ok, State#{ - <<"set_count">> => << Acc/binary, "." >> + <<"set-count">> => << Acc/binary, "." >> } } end }, - <<"state_key">> => <<"STATE">> + <<"state-key">> => <<"STATE">> }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - SetOnce = hb_ao:set(Msg, #{ <<"state_key">> => <<"SET_ONCE">> }, Opts), - ?assertEqual(<<".">>, hb_ao:get(<<"set_count">>, SetOnce, Opts)), - SetTwice = hb_ao:set(SetOnce, #{ <<"state_key">> => <<"SET_TWICE">> }, Opts), - ?assertEqual(<<"..">>, hb_ao:get(<<"set_count">>, SetTwice, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, SetTwice, Opts)). + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, Msg, Opts)), + SetOnce = hb_ao:set(Msg, #{ <<"state-key">> => <<"SET_ONCE">> }, Opts), + ?assertEqual(<<".">>, hb_ao:get(<<"set-count">>, SetOnce, Opts)), + SetTwice = hb_ao:set(SetOnce, #{ <<"state-key">> => <<"SET_TWICE">> }, Opts), + ?assertEqual(<<"..">>, hb_ao:get(<<"set-count">>, SetTwice, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, SetTwice, Opts)). deep_set_test(Opts) -> % First validate second layer changes are handled correctly. @@ -726,9 +714,9 @@ deep_set_with_device_test(Opts) -> B = hb_ao:get(<<"b">>, A, Opts), C = hb_ao:get(<<"c">>, B, Opts), ?assertEqual(<<"2">>, C), - ?assertEqual(true, hb_ao:get(<<"modified">>, Outer)), - ?assertEqual(false, hb_ao:get(<<"modified">>, A)), - ?assertEqual(true, hb_ao:get(<<"modified">>, B)). + ?assertEqual(true, hb_ao:get(<<"modified">>, Outer, Opts)), + ?assertEqual(false, hb_ao:get(<<"modified">>, A, Opts)), + ?assertEqual(true, hb_ao:get(<<"modified">>, B, Opts)). device_exports_test(Opts) -> Msg = #{ <<"device">> => dev_message }, @@ -862,11 +850,11 @@ denormalized_key_test(Opts) -> }, ?assertEqual( {ok, <<"TEST VALUE">>}, - hb_ao:resolve(Msg, <<"test_key">>, Opts) + hb_ao:get(<<"test_key">>, Msg, Opts) ), ?assertEqual( {ok, <<"TEST VALUE">>}, - hb_ao:resolve(Msg, <<"test-key">>, Opts) + hb_ao:get(<<"test-key">>, Msg, Opts) ). list_transform_test(Opts) -> @@ -880,10 +868,10 @@ list_transform_test(Opts) -> start_as_test(Opts) -> ?assertEqual( {ok, <<"GOOD FUNCTION">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"test-device@1.0">>, #{ <<"path">> => <<>> }}, - #{ <<"path">> => <<"test_func">> } + hb_ao:as(<<"test-device@1.0">>, #{}, Opts), + #{ <<"path">> => <<"test-func">> } ], Opts ) @@ -892,14 +880,14 @@ start_as_with_parameters_test(Opts) -> % Resolve a key on a message that has its device set with `as'. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, ?assertEqual( {ok, <<"GOOD FUNCTION">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, - #{ <<"path">> => <<"test_func">> } + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> } ], Opts ) @@ -909,7 +897,7 @@ load_as_test(Opts) -> % Load a message as a device with the `as' keyword. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, % There is a race condition where we write to the store and a % reset happens making not read the written value. @@ -920,30 +908,31 @@ load_as_test(Opts) -> ?assert(hb_message:match(Msg, ReadMsg, primary, Opts)), ?assertEqual( {ok, <<"MESSAGE">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, #{ <<"path">> => <> }}, - <<"test_func">>, - <<"test_key">> + {link, ID, #{}}, + #{ <<"device">> => <<"test-device@1.0">> }, + <<"test-func">>, + <<"test-key">> ], Opts ) ). as_path_test(Opts) -> - % Create a message with the test device, which implements the test_func + % Create a message with the test device, which implements the test-func % function. It normally returns `GOOD FUNCTION'. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, - ?assertEqual(<<"GOOD FUNCTION">>, hb_ao:get(<<"test_func">>, Msg, Opts)), + ?assertEqual(<<"GOOD FUNCTION">>, hb_ao:get(<<"test-func">>, Msg, Opts)), % Now use the `as' keyword to subresolve a key with the message device. ?assertMatch( - {ok, #{ <<"test_key">> := <<"MESSAGE">> }}, + {ok, #{ <<"test-key">> := <<"MESSAGE">> }}, hb_ao:resolve( - Msg, - {as, <<"message@1.0">>, #{ <<"path">> => <<"test_func">> }}, + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> }, Opts ) ). @@ -952,16 +941,15 @@ continue_as_test(Opts) -> % Resolve a list of messages in sequence, swapping the device in the middle. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, ?assertEqual( {ok, <<"MESSAGE">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - Msg, - {as, <<"message@1.0">>, <<>>}, - #{ <<"path">> => <<"test_func">> }, - #{ <<"path">> => <<"test_key">> } + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> }, + #{ <<"path">> => <<"test-key">> } ], Opts ) @@ -982,16 +970,16 @@ as_commitments_test(RawOpts) -> InitialComms = hb_ao:get(<<"commitments">>, Msg, OptsWithWallet), {ok, ResolvedMsg} = hb_ao:resolve( - {as, <<"test-device@1.0">>, Msg}, + hb_ao:as(<<"test-device@1.0">>, Msg, OptsWithWallet), <<"commitments">>, OptsWithWallet ), ?assertEqual(InitialComms, ResolvedMsg), ?assertEqual( {ok, []}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, + hb_ao:as(<<"message@1.0">>, Msg, OptsWithWallet), <<"committers">> ], OptsWithWallet @@ -1009,9 +997,9 @@ multiple_as_subresolutions_test(Opts) -> #{ <<"test-key-2">> => <<"MESSAGE-2">> } } }, - Res = hb_ao:resolve_many( + Res = hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, + hb_ao:as(<<"message@1.0">>, Msg, Opts), #{ <<"path">> => <<"test-message">> }, #{ <<"path">> => <<"test-message-2">>, <<"extraneous">> => <<"1">> }, <<"test-key-2">> diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index e08f49fd5..bf14648a5 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -221,7 +221,6 @@ id_device(_, _) -> {ok, ?DEFAULT_ID_DEVICE}. %% @doc Return the committers of a message that are present in the given request. --spec committers(#{ commitments => _ }, _, _) -> {ok, [binary()]}. committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> {ok, hb_maps:values( @@ -611,16 +610,16 @@ set(Base, Req = #{ <<"set">> := <<"deep">> }, Opts) -> maps:map( fun(Key, Nested) when is_map(Nested) or ?IS_LINK(Nested) -> case hb_maps:find(Key, Base, Opts) of - {ok, NestedBase} -> + {ok, NestedBase} when is_map(NestedBase) or ?IS_LINK(NestedBase) -> % We use `hb_ao:deep_set` rather than just recursing % such that the correct device is used for the downstream % set operation. - hb_util:ok(hb_ao:deep_set(NestedBase, Nested, Opts)); + hb_ao:deep_set(NestedBase, Nested, Opts); error -> Nested end; (_Key, NewValue) -> NewValue end, - hb_message:uncommitted(Req, Opts) + maps:without([<<"set">>], hb_message:uncommitted(Req, Opts)) ), set(Base, NewValues, Opts); set(Base, NewValues, _Opts) -> @@ -701,31 +700,46 @@ default_accessor(Key, Msg, Req, Opts) -> %% during the process of `vary`ing. -spec vary( #{ _ => _ }, - #{ vary => binary(), _ => _ }, - #{}) -> {ok, #{}}. + #{ _ => _ }, + #{}) -> {ok, #{}} | {error, binary()}. vary(Base, Req, Opts) -> maybe - {ok, Key} ?= - case maps:find(<<"vary">>, Req) of - {ok, KeyToVaryOn} -> {ok, KeyToVaryOn}; - error -> - case maps:find(<<"path">>, Req) of - {ok, <<"vary">>} -> - {error, <<"Cannot vary the `vary` path.">>}; - Other -> Other - end - end, - Ctx1 = #{ <<"base">> => Base, <<"key">> => Key, <<"request">> => Req }, - {ok, Ctx2} ?= + {ok, Ctx} ?= case hb_private:get(<<"function">>, Req, not_found, Opts) of not_found -> - hb_device:add_resolver(Ctx1, Opts); + % No function given: derive the key to resolve one. + maybe + {ok, Key} ?= + case Req of + #{ <<"vary">> := KeyToVaryOn } -> + {ok, KeyToVaryOn}; + #{ <<"path">> := <<"vary">> } -> + {error, <<"Cannot vary the `vary` path.">>}; + #{ <<"path">> := PathKey } -> + {ok, PathKey}; + _ -> + {error, <<"invalid-vary-request">>} + end, + hb_device:add_resolver( + #{ + <<"base">> => Base, + <<"key">> => Key, + <<"request">> => Req + }, + Opts + ) + end; Fun -> - {ok, Ctx1#{ <<"priv">> => #{ <<"function">> => Fun } }} + % The executor is already known: no key is required. + {ok, + #{ + <<"base">> => Base, + <<"request">> => Req, + <<"priv">> => #{ <<"function">> => Fun } + } + } end, - hb_types:vary(Ctx2, Opts) - else - error -> {error, <<"invalid-vary-request">>} + hb_types:vary(Ctx, Opts) end. %% @doc Returns the device schema for a `Base` message. From 986ed07503777f4b240d7b1e9e99dd9a12e5741e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 12 Jul 2026 18:03:45 -0400 Subject: [PATCH 27/35] wip: Add AO-Core 1.0 hashpath parser and basic challenge logic --- src/core/resolver/hb_ao.erl | 15 +- src/core/resolver/hb_hashpath.erl | 320 ++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 7 deletions(-) create mode 100644 src/core/resolver/hb_hashpath.erl diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 72a748be2..cf61da16f 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -514,7 +514,7 @@ stage_6(Ctx = #{ <<"reason">> => <<"AO-Core Post-Execution Validation">>, <<"base">> => Base, <<"request">> => Req, - <<"result">> => Res + <<"varied-result">> => Res }, Opts ), @@ -522,7 +522,7 @@ stage_6(Ctx = #{ ok, Ctx#{ <<"status">> => Status, - <<"result">> => Res, + <<"varied-result">> => Res, <<"fresh">> => true } }. @@ -536,11 +536,11 @@ stage_7( <<"fresh">> := true, <<"varied-base">> := VariedBase, <<"varied-request">> := VariedReq, - <<"result">> := Res, + <<"varied-result">> := Res, <<"opts">> := Opts } ) -> - hb_cache_control:maybe_store(VariedBase, VariedReq, Res, Opts), + hb_cache_control:maybe_store(VariedBase, VariedReq, VariedRes, Opts), {ok, Ctx}; stage_7(Ctx) -> {ok, Ctx}. @@ -549,7 +549,7 @@ stage_8( Ctx = #{ <<"leader">> := ExecName, <<"varied-request">> := Req, - <<"result">> := Res, + <<"varied-result">> := Res, <<"status">> := Status, <<"opts">> := Opts } @@ -573,7 +573,7 @@ stage_9( <<"normalizer">> := Normalizer, <<"base">> := Base, <<"request">> := Req, - <<"result">> := Result, + <<"varied-result">> := Result, <<"opts">> := Opts } ) when Normalizer =/= none -> @@ -588,7 +588,8 @@ stage_9( end } }; -stage_9(Ctx) -> {ok, Ctx}. +stage_9(Ctx#{ <<"varied-result">> := Result}) -> + {ok, Ctx#{ <<"result">> := Result }}. %% @doc If a hook has been specified for the `step` action, we call it with our %% context including the result. diff --git a/src/core/resolver/hb_hashpath.erl b/src/core/resolver/hb_hashpath.erl new file mode 100644 index 000000000..4c4b36af2 --- /dev/null +++ b/src/core/resolver/hb_hashpath.erl @@ -0,0 +1,320 @@ +%%% @doc Hashpaths: Succinct executable claims of AO-Core transitions: Their +%%% results and dependencies, as addressable protocol values. +%%% +%%% Hashpaths abide by the following grammar: +%%% +%%% 
+%%%     Hashpath     :: Base "/" Request Variance? Dependencies? Equivalence?
+%%%     Base         :: MessageID | Hashpath
+%%%     Request      :: MessageID | PathString
+%%%     Variance     :: "" | ">" VariedBaseID "+" VariedReqID
+%%%     Dependencies :: "" | "@" DependenceMessageID
+%%%     Equivalence  :: Normalizer ResultMessageID
+%%%     Normalizer   :: "." | "="
+%%%
+%%% +%%% A compact form may omit fields when they are derivable: the vary pair is +%%% omitted when it is the identity vary, `Dependencies' when there are none, and +%%% the terminal before a result exists. Segments without explicit vary +%%% syntax are not special: `HP/*=FinalResultID' is an ordinary claim that +%%% resolving `*' at `HP' yields `FinalResultID'. +%%% +%%% Every separator of the syntax (`/', `>', `+', `@', `=', `.') is outside +%%% the base64url alphabet, so the grammar is unambiguous without escaping. +%%% The request position holds an ID when the request is addressed, or a +%%% literal key when it is self-describing (e.g. `*'). +-module(hb_hashpath). +%%% Create and parse hashpaths. +-export([format/1, parse/1, context/1]). +%%% Verify hashpath claims. +-export([verify/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Encode a hashpath from an execution context used/returned by `hb_ao:do/1` +%% and its internal stages. +%% +%% The first stage of format extracts the first of two universal hashpath elements +%% -- the `Base` ID or existing hashpath. We then recurse with this value and the +%% remaining context. +format(Ctx, Opts) -> + maybe + {ok, BasePart} ?= format_base(Ctx, Opts), + {ok, RequestPart} ?= format_request(Ctx, Opts), + << + BasePart/binary, "/", RequestPart/binary, + (format_varied(Ctx, Opts))/binary, + (format_dependencies(Ctx, Opts))/binary, + (format_equivalence(Ctx, Opts))/binary + >> + else + {not_found, Name} -> + throw({context_not_viable, unavailable_field, Name}) + end. + +%% @doc Utilize the `hashpath` of the prior resolution, if it is available, +%% falling back to the `BaseID` if known, and recomputing it only if necessary. +format_base(Ctx = #{ <<"base">> := #{ <<"...">> := PriorHashpath } }, _) -> + {ok, PriorHashpath}; +format_base(Ctx, Opts) -> + find_id(<<"base">>, Ctx, Opts). + +%% @doc General utility for extracting the ID of a message by its name from a +%% context if it is already known, recomputing only if necessary. +find_id(Name, #{ << Name/binary, "-id">> := ID }, _Opts) -> + {ok, ID}; +find_id(Name, #{ Name := Msg }, Opts) -> + {ok, hb_message:id(Msg, signers, Opts)}; +find_id(Name, _Ctx, _Opts) -> + {not_found, Name}. + +%% @doc Format the varied base and requests, if given, into their hashpath +%% components. +format_varied(Ctx, Opts) -> + maybe + {ok, VBase} ?= find_id(<<"varied-base">>, Ctx, Opts), + {ok, VReq} ?= find_id(<<"varied-request">>, Ctx, Opts), + <<">", VBase/binary, "+", VReq/binary>> + else + {not_found, _} -> + % Either the base or request is not found, so we omit the varied + % component of the hashpath. + <<>> + end. + +%% @doc If the dependencies of a resolution are known, format them into the +%% hashpath depends component. If not, return an empty string. Honors already +%% calculated dependency IDs if provided in the context. +format_dependencies(Ctx, Opts) -> + case find_id(<<"dependencies">>, Ctx, Opts) of + {ok, Depends} -> <<"@", Depends/binary>>; + {not_found, _} -> <<>> + end. + +%% @doc If the result of the execution has already been calculated, format it +%% into the hashpath equivalence component. If not, return an empty string. +format_equivalence(Ctx = #{ <<"result">> := Result }, Opts) -> + <<(format_normalizer(Ctx, Opts)), Result/binary>>; +format_equivalence(_) -> <<>>. + +%% @doc Format the normalizer component of the hashpath. +format_normalizer(#{ <<"normalizer">> := base }, _Opts) -> <<"=">>; +format_normalizer(_, _) -> <<".">>. + +%% @doc Decode a hashpath into a list of context segments. The first segment will +%% have both a base and a request part, while the latter segments will only have +%% the request part -- the base being inferred from the result of the prior +%% segments. +parse(Hashpath, Opts) when is_binary(Hashpath) -> + [Base, Req1 | Reqs] = binary:split(Hashpath, <<"/">>, [global]), + [ + parse_part(Base, Req1, Opts) + | + lists:map( + fun(ReqPart) -> parse_part(undefined, ReqPart, Opts) end, + Reqs + ) + ]. + +%% @doc Parse the last segment of a hashpath into an executable context that +%% can be additionally executed upon. +context(Hashpath, Opts) -> + case lists:reverse(binary:split(Hashpath, <<"/">>, [global])) of + [ LoneBase ] -> #{ <<"base-id">> => LoneBase }; + [ LastReq, Rest ] -> + ReconstitutedBase = binary:join(lists:reverse(Rest), <<"/">>), + parse_part(ReconstitutedBase, LastReq, Opts); + _ -> #{ <<"base-id">> => Hashpath } + end. + +%% @doc Calculate the context for a hashpath segment. If the base is known +%% explicitly, add it to the result from parsing the request part. If not, +%% parse the request part and return as-is. +parse_part(undefined, ReqPart, Opts) -> + parse_request(ReqPart, Opts); +parse_part(Base, ReqPart, Opts) -> + (parse_request(ReqPart, Opts, Base)#{ <<"base-id">> => Base }). + +%% @doc Parse a single segment of the hashpath into a context segment. +parse_request(Part, Opts) -> + maybe + {next, NextDelim, Ctx1, Part2} ?= + parse_request_id(Part, Opts), + {ok, NextDelim2, Part3, Ctx2} ?= + parse_varied(NextDelim, Part2, Ctx1, Opts), + {ok, NextDelim3, Part4, Ctx3} ?= + parse_dependencies(NextDelim2, Part3, Ctx2, Opts), + {ok, Ctx4} ?= + parse_equivalence(NextDelim3, Part4, Ctx3, Opts), + {ok, Ctx4} + end. + +%% @doc Parse the request ID part of a hashpath segment. If the request ID is +%% the only part, it is returned as-is and the remainder of the part parsing is +%% skipped. +parse_request_id(Part, Opts) -> + case next(Part, Opts) of + {no_match, Part, <<>>} -> {ok, #{ <<"request-id">> => Part } }; + {Sep, ReqID, Part2} -> {next, Sep, Part2, #{ <<"request-id">> => ReqID }} + end. + +%% @doc If the delimiter that starts our segment is `>` we handle the inner +%% segment as a `VariedBase` and `VariedRequest` pair and get the next delimited +%% component. If the delimiter is not `>`, we pass the segment forward as-is. +parse_varied($>, Part2, Ctx0, Opts) -> + {NextDelim, Next, After} = next([$@, $., $=], Part), + case binary:split(Next, <<"+">>) of + [VBase, VReq] -> + { + next, + NextDelim, + After, + Ctx0#{ <<"varied-base">> => VBase, <<"varied-request">> => VReq } + }; + Malformed -> + {error, {invalid_variance_parts, Malformed}} + end; +parse_varied(NextDelim, Part2, Ctx0, _Opts) -> + {next, NextDelim, Part2, Ctx0}. + +%% @doc Parse the dependencies if present. We short-curcuit the parser and +%% return the context early if we have already hit the end of the string. +parse_dependencies(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; +parse_dependencies($@, DepID, Ctx0, Opts) -> + {NextDelim, Next, After} = next([$@, $., $=], Part), + {next, NextDelim, Next, After, Ctx0#{ <<"dependencies-id">> => DepID }}; +parse_dependencies(Delim, Part, Ctx0, _Opts) -> + {next, Delim, Part, <<>>, Ctx0}. + +%% @doc Parse the equivalent relationship if stated in the hashpath. +parse_equivalence(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; +parse_equivalence($=, ResultID, Ctx0, _Opts) -> + {ok, #{ <<"normalizer">> => base, <<"result-id">> => ResultID }}; +parse_equivalence($., ResultID, Ctx0, _Opts) -> + {ok, #{ <<"result-id">> => ResultID }}. + +%% @doc Utility to split at the next syntax delimiter (e.g. `=`, `.`, `>`, `@`). +%% Returns the syntax element matched, and the rest of the string. Notably, this +%% utility does not break apart `VBase+VReq` pairs. They are treated as a single +%% unit and parsed internally in `parse_varied/4`. +next(S) -> next([$=, $., $>, $@], S). +next(Symbols, S) -> hb_util:split_depth_string_aware_single(Symbols, S). + +%% @doc Challenge a complete hashpath, verifying each part's claims. +verify(Bin, Opts) when is_binary(Bin) -> + case parse(Bin) of + [] -> + % We treat an empty hashpath as failing verification. + false; + [Init | Parts] -> + case verify(Init, Parts, Opts) of + {true, #{ <<"result">> := ComputedState }} -> + verify(ComputedState, Parts, Opts); + false -> + false + end; + _ -> + false + end. +verify(_FinalBase, [], _Opts) -> + % The full hashpath has resolved and we have no more parts to verify. Each + % passed verification. + true; +verify(State, [Part | Rest], Opts) -> + % Add the currently computed state to the part's context and verify it. + case verify(Part#{ <<"base">> => State }, Opts) of + {true, ComputedState} -> verify(ComputedState, Rest, Opts); + false -> false + end. + +%% @doc Verify a full single context, parsed from a binary hashpath. The context +%% must contain a `Base' representation. We remove all of the non-`Base` and +%% `Request` fields, then utilize `hb_ao:do` to re-execute the context. Assuming +%% successful computation, we then verify the `VariedBase` and `VariedRequest` +%% fields against the parsed context, the `DependenciesID` if given, the +%% `Normalizer` type, and finally the `Result` message itself. If all of these +%% verify, the context is considered valid. +verify_context(Ctx, Opts) -> + StrippedCtx = + maps:with( + [<<"base">>, <<"request">>, <<"base-id">>, <<"request-id">>], + Ctx#{ <<"opts">> => Opts } + ), + maybe + {ok, ExecutedCtx} = hb_ao:do(StrippedCtx), + true ?= verify_varied(Ctx, ExecutedCtx), + true ?= verify_dependencies(Ctx, ExecutedCtx), + true ?= verify_equivalence(Ctx, ExecutedCtx), + {true, ExecutedCtx} + else + {error, Type} -> + ?event_debug( + hashpath_debug, + {hashpath_verify_context_failed, {type, Type}, {ctx, Ctx}}, + Opts + ), + false + end. + +%% @doc If varied `Req` and `Base` statements were present in the hashpath, +%% we verify that they match the executed context. +verify_varied(HPCtx, ExecutedCtx) -> + maybe + {ok, HPVBase} ?= find_id(<<"varied-base">>, HPCtx, Opts), + {ok, ExecVBase} ?= find_id(<<"varied-base">>, ExecutedCtx, Opts), + true ?= HPVBase =:= ExecVBase + orelse {error, <<"Varied `Base`s do not match">>}, + {ok, HPVReq} ?= find_id(<<"varied-request">>, HPCtx, Opts), + {ok, ExecVReq} ?= find_id(<<"varied-request">>, ExecutedCtx, Opts), + true ?= HPVReq =:= ExecVReq + orelse {error, <<"Varied `Request`s do not match">>}, + end. + +%% @doc Verify that the dependencies in the hashpath claim match those in the +%% executed context, if both are present. +verify_dependencies(HPCtx, ExecutedCtx) -> + maybe + {ok, HPDeps} ?= find_id(<<"dependencies">>, HPCtx, Opts), + {ok, ExecDeps} ?= find_id(<<"dependencies">>, ExecutedCtx, Opts), + true ?= HPDeps =:= ExecDeps + orelse {error, <<"Dependencies do not match">>} + end. + +%% @doc Verify that the results of the execution match those in the claim. +verify_equivalence(HPCtx, ExecutedCtx) -> + maybe + HPNormalizer = maps:get(<<"normalizer">>, HPCtx, replace), + ExecNormalizer = maps:get(<<"normalizer">>, ExecutedCtx, replace), + true ?= HPNormalizer =:= ExecNormalizer + orelse {error, <<"Normalizers do not match">>}, + {ok, HPResult} ?= find_id(<<"result">>, HPCtx, Opts), + {ok, ExecResult} ?= find_id(<<"result">>, ExecutedCtx, Opts), + true ?= HPResult =:= ExecResult + orelse {error, <<"Results do not match">>} + end. + +%%% Tests + +full_form_round_trip_test() -> + HP = + << + "BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4" + "/tYRVDkT2X7wjYYVaZWuBBWWzZatEsMoR2NBjcJ8CmZk" + ">a2Fub25pY2FsLXZhcmllZC1iYXNlLWlkLTAwMDAwMDAw" + "+a2Fub25pY2FsLXZhcmllZC1yZXEtaWQtMDAwMDAwMDAw" + "@ZGVwZW5kcy1tZXNzYWdlLWlkLTAwMDAwMDAwMDAwMDAw" + "=cGF0Y2gtaWQtMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw" + >>, + ?assertEqual(HP, format(parse(HP))). + +compact_form_round_trip_test() -> + lists:foreach( + fun(HP) -> ?assertEqual(HP, format(parse(HP))) end, + [ + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/balance">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/transfer/balance">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/*" + "=cGF0Y2gtaWQtMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw">> + ] + ). \ No newline at end of file From e7b36fb8ec8b5d4207e5d4e0d7a3027ce17d0107 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 12 Jul 2026 18:21:51 -0400 Subject: [PATCH 28/35] wip: allow selective part verification --- src/core/resolver/hb_hashpath.erl | 76 ++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/src/core/resolver/hb_hashpath.erl b/src/core/resolver/hb_hashpath.erl index 4c4b36af2..1d08ab0ba 100644 --- a/src/core/resolver/hb_hashpath.erl +++ b/src/core/resolver/hb_hashpath.erl @@ -27,7 +27,7 @@ %%% Create and parse hashpaths. -export([format/1, parse/1, context/1]). %%% Verify hashpath claims. --export([verify/2]). +-export([verify_all/2, verify_part/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -189,9 +189,9 @@ parse_dependencies(Delim, Part, Ctx0, _Opts) -> %% @doc Parse the equivalent relationship if stated in the hashpath. parse_equivalence(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; parse_equivalence($=, ResultID, Ctx0, _Opts) -> - {ok, #{ <<"normalizer">> => base, <<"result-id">> => ResultID }}; + {ok, #{ <<"normalizer">> => base, <<"varied-result-id">> => ResultID }}; parse_equivalence($., ResultID, Ctx0, _Opts) -> - {ok, #{ <<"result-id">> => ResultID }}. + {ok, #{ <<"varied-result-id">> => ResultID }}. %% @doc Utility to split at the next syntax delimiter (e.g. `=`, `.`, `>`, `@`). %% Returns the syntax element matched, and the rest of the string. Notably, this @@ -201,32 +201,66 @@ next(S) -> next([$=, $., $>, $@], S). next(Symbols, S) -> hb_util:split_depth_string_aware_single(Symbols, S). %% @doc Challenge a complete hashpath, verifying each part's claims. -verify(Bin, Opts) when is_binary(Bin) -> - case parse(Bin) of - [] -> - % We treat an empty hashpath as failing verification. - false; - [Init | Parts] -> - case verify(Init, Parts, Opts) of - {true, #{ <<"result">> := ComputedState }} -> - verify(ComputedState, Parts, Opts); - false -> - false - end; - _ -> - false - end. -verify(_FinalBase, [], _Opts) -> +verify_all(Bin, Opts) when is_binary(Bin) -> + verify_all(parse(Bin), Opts); +verify_all([], _Opts) -> + % We treat an empty hashpath as failing verification. + false; +verify_all([Init | Parts], Opts) -> + verify_all(Init, Parts, Opts). + +verify_all(_FinalBase, [], _Opts) -> % The full hashpath has resolved and we have no more parts to verify. Each % passed verification. true; -verify(State, [Part | Rest], Opts) -> +verify_all(State, [Part | Rest], Opts) -> % Add the currently computed state to the part's context and verify it. case verify(Part#{ <<"base">> => State }, Opts) of - {true, ComputedState} -> verify(ComputedState, Rest, Opts); + {true, ComputedState} -> verify_all(ComputedState, Rest, Opts); false -> false end. +%% @doc Verify a single hashpath execution contained inside a larger hashpath +%% sequence. +verify_part(Hashpath, PartNum, Opts) when is_binary(Hashpath) -> + verify_part(parse(Hashpath), PartNum, Opts); +verify_part([InitialCtx|Rest], PartNum, Opts) -> + verify_part(InitialCtx, Rest, PartNum, Opts). + +verify_part(State, [], _PartNum, _Opts) -> + {error, <<"Part number not found.">>}; +verify_part(State, [Part | Rest], 0, Opts) -> + maybe + {true, _} ?= + verify_context( + Part#{ <<"base">> => State, <<"opts">> => Opts} + ), + true + end; +verify_part(State, [Next | Rest], PartNum, Opts) -> + % Extract the varied-result from the next context and apply it to the current + % state. + case maps:get(<<"normalizer">>, Next, replace) of + replace -> + % Replace the full accumulated state with the varied result and + % recurse to the next part. + verify_part( + maps:get(<<"varied-result">>, Next), + Rest, + PartNum - 1, + Opts + ); + base -> + % The base normalizer means we should extend the result over the + % base state. + verify_part( + maps:get(<<"varied-result">>, Next)#{ <<"...">> => State }, + Rest, + PartNum - 1, + Opts + ); + end. + %% @doc Verify a full single context, parsed from a binary hashpath. The context %% must contain a `Base' representation. We remove all of the non-`Base` and %% `Request` fields, then utilize `hb_ao:do` to re-execute the context. Assuming From 7c7dbb75e73e86b404c6bea31ae9e1e3d0216650 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 13 Jul 2026 19:10:34 -0400 Subject: [PATCH 29/35] wip: hashpath generation ordering, small fixes --- src/core/resolver/hb_ao.erl | 50 +++---- src/core/resolver/hb_hashpath.erl | 209 +++++++++++++++++++++--------- 2 files changed, 162 insertions(+), 97 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index cf61da16f..602d39f7d 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -290,11 +290,11 @@ do(Ctx0) -> {ok, Ctx5} ?= stage_5(Ctx4), % Stage 6: Execution. {ok, Ctx6} ?= stage_6(Ctx5), - % Stage 7: Result caching. + % Stage 7: `Normalizer` application and hashpath generation {ok, Ctx7} ?= stage_7(Ctx6), - % Stage 8: Notify waiters. + % Stage 8: Result caching. {ok, Ctx8} ?= stage_8(Ctx7), - % Stage 9: Extension of result upon original base. + % Stage 9: Notify waiters. {ok, Ctx9} ?= stage_9(Ctx8), % Stage 10: Execution of the `step' hook. {ok, Ctx10} ?= stage_10(Ctx9), @@ -527,10 +527,20 @@ stage_6(Ctx = #{ } }. +%% @doc When specified in the schema for the device call, normalize the +%% execution's result on top of the `Base` or `Req` message. In cases where +%% hashpath calculation is disabled (`hashpath => ignore` in the node message, +%% etc.), we +stage_7(Ctx = #{ <<"base">> => Base, <<"opts">> := Opts }) -> + maybe + {ok, Result} ?= hb_hashpath:result_from_context(Base, Ctx, Opts), + {ok, Ctx#{ <<"result">> => Result }} + end. + %% @doc Cache the result of an execution if appropriate for the context. %% Results that were not freshly and successfully executed (cache hits, direct %% key hits, awaited results, non-`ok` statuses) are not stored. -stage_7( +stage_8( Ctx = #{ <<"status">> := ok, <<"fresh">> := true, @@ -542,10 +552,10 @@ stage_7( ) -> hb_cache_control:maybe_store(VariedBase, VariedReq, VariedRes, Opts), {ok, Ctx}; -stage_7(Ctx) -> {ok, Ctx}. +stage_8(Ctx) -> {ok, Ctx}. %% @doc Return the resolved response to any waiting callers. -stage_8( +stage_9( Ctx = #{ <<"leader">> := ExecName, <<"varied-request">> := Req, @@ -561,36 +571,10 @@ stage_8( Opts ), {ok, Ctx}; -stage_8(Ctx) -> +stage_9(Ctx) -> % If we are not the leader, we can ignore the unregister step. {ok, Ctx}. -%% @doc When specified in the schema for the device call, normalize the -%% execution's result on top of the `Base` or `Req` message. -stage_9( - Ctx = #{ - <<"status">> := ok, - <<"normalizer">> := Normalizer, - <<"base">> := Base, - <<"request">> := Req, - <<"varied-result">> := Result, - <<"opts">> := Opts - } -) when Normalizer =/= none -> - { - ok, - Ctx#{ - <<"result">> := - case Normalizer of - base -> Result#{ <<"...">> => Base }; - request -> Result#{ <<"...">> => Req }; - Fun when is_function(Fun) -> Fun(Base, Result, Opts) - end - } - }; -stage_9(Ctx#{ <<"varied-result">> := Result}) -> - {ok, Ctx#{ <<"result">> := Result }}. - %% @doc If a hook has been specified for the `step` action, we call it with our %% context including the result. stage_10(Ctx = #{ <<"opts">> := Opts = #{ <<"on">> := #{ <<"step">> := _ }}}) -> diff --git a/src/core/resolver/hb_hashpath.erl b/src/core/resolver/hb_hashpath.erl index 1d08ab0ba..2df4b7415 100644 --- a/src/core/resolver/hb_hashpath.erl +++ b/src/core/resolver/hb_hashpath.erl @@ -25,7 +25,7 @@ %%% literal key when it is self-describing (e.g. `*'). -module(hb_hashpath). %%% Create and parse hashpaths. --export([format/1, parse/1, context/1]). +-export([format/2, parse/2, context/2]). %%% Verify hashpath claims. -export([verify_all/2, verify_part/3]). -include("include/hb.hrl"). @@ -61,10 +61,15 @@ format_base(Ctx, Opts) -> %% @doc General utility for extracting the ID of a message by its name from a %% context if it is already known, recomputing only if necessary. -find_id(Name, #{ << Name/binary, "-id">> := ID }, _Opts) -> - {ok, ID}; -find_id(Name, #{ Name := Msg }, Opts) -> - {ok, hb_message:id(Msg, signers, Opts)}; +find_id(Name, Ctx, _Opts) when is_map_key(<>, Ctx) -> + {ok, maps:get(Name, Ctx)}; +find_id(Name, Ctx, Opts) when is_map_key(Name, Ctx) -> + case hb_opts:get(<<"hashpath">>, enabled, Opts) of + enabled -> + {ok, hb_message:id(maps:get(Name, Ctx), signers, Opts)}; + _ -> + {not_found, Name} + end; find_id(Name, _Ctx, _Opts) -> {not_found, Name}. @@ -93,9 +98,9 @@ format_dependencies(Ctx, Opts) -> %% @doc If the result of the execution has already been calculated, format it %% into the hashpath equivalence component. If not, return an empty string. -format_equivalence(Ctx = #{ <<"result">> := Result }, Opts) -> +format_equivalence(Ctx = #{ <<"varied-result">> := Result }, Opts) -> <<(format_normalizer(Ctx, Opts)), Result/binary>>; -format_equivalence(_) -> <<>>. +format_equivalence(_, _) -> <<>>. %% @doc Format the normalizer component of the hashpath. format_normalizer(#{ <<"normalizer">> := base }, _Opts) -> <<"=">>; @@ -133,7 +138,7 @@ context(Hashpath, Opts) -> parse_part(undefined, ReqPart, Opts) -> parse_request(ReqPart, Opts); parse_part(Base, ReqPart, Opts) -> - (parse_request(ReqPart, Opts, Base)#{ <<"base-id">> => Base }). + (parse_request(ReqPart, Opts, Base))#{ <<"base-id">> => Base }. %% @doc Parse a single segment of the hashpath into a context segment. parse_request(Part, Opts) -> @@ -161,7 +166,7 @@ parse_request_id(Part, Opts) -> %% @doc If the delimiter that starts our segment is `>` we handle the inner %% segment as a `VariedBase` and `VariedRequest` pair and get the next delimited %% component. If the delimiter is not `>`, we pass the segment forward as-is. -parse_varied($>, Part2, Ctx0, Opts) -> +parse_varied($>, Part, Ctx0, _Opts) -> {NextDelim, Next, After} = next([$@, $., $=], Part), case binary:split(Next, <<"+">>) of [VBase, VReq] -> @@ -174,8 +179,8 @@ parse_varied($>, Part2, Ctx0, Opts) -> Malformed -> {error, {invalid_variance_parts, Malformed}} end; -parse_varied(NextDelim, Part2, Ctx0, _Opts) -> - {next, NextDelim, Part2, Ctx0}. +parse_varied(NextDelim, Part, Ctx0, _Opts) -> + {next, NextDelim, Part, Ctx0}. %% @doc Parse the dependencies if present. We short-curcuit the parser and %% return the context early if we have already hit the end of the string. @@ -188,10 +193,10 @@ parse_dependencies(Delim, Part, Ctx0, _Opts) -> %% @doc Parse the equivalent relationship if stated in the hashpath. parse_equivalence(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; -parse_equivalence($=, ResultID, Ctx0, _Opts) -> - {ok, #{ <<"normalizer">> => base, <<"varied-result-id">> => ResultID }}; -parse_equivalence($., ResultID, Ctx0, _Opts) -> - {ok, #{ <<"varied-result-id">> => ResultID }}. +parse_equivalence($=, ResultID, Ctx, _Opts) -> + {ok, Ctx#{ <<"normalizer">> => base, <<"varied-result-id">> => ResultID }}; +parse_equivalence($., ResultID, Ctx, _Opts) -> + {ok, Ctx#{ <<"varied-result-id">> => ResultID }}. %% @doc Utility to split at the next syntax delimiter (e.g. `=`, `.`, `>`, `@`). %% Returns the syntax element matched, and the rest of the string. Notably, this @@ -202,7 +207,7 @@ next(Symbols, S) -> hb_util:split_depth_string_aware_single(Symbols, S). %% @doc Challenge a complete hashpath, verifying each part's claims. verify_all(Bin, Opts) when is_binary(Bin) -> - verify_all(parse(Bin), Opts); + verify_all(parse(Bin, Opts), Opts); verify_all([], _Opts) -> % We treat an empty hashpath as failing verification. false; @@ -215,7 +220,7 @@ verify_all(_FinalBase, [], _Opts) -> true; verify_all(State, [Part | Rest], Opts) -> % Add the currently computed state to the part's context and verify it. - case verify(Part#{ <<"base">> => State }, Opts) of + case verify_context(Part#{ <<"base">> => State }, Opts) of {true, ComputedState} -> verify_all(ComputedState, Rest, Opts); false -> false end. @@ -223,42 +228,11 @@ verify_all(State, [Part | Rest], Opts) -> %% @doc Verify a single hashpath execution contained inside a larger hashpath %% sequence. verify_part(Hashpath, PartNum, Opts) when is_binary(Hashpath) -> - verify_part(parse(Hashpath), PartNum, Opts); -verify_part([InitialCtx|Rest], PartNum, Opts) -> - verify_part(InitialCtx, Rest, PartNum, Opts). - -verify_part(State, [], _PartNum, _Opts) -> - {error, <<"Part number not found.">>}; -verify_part(State, [Part | Rest], 0, Opts) -> - maybe - {true, _} ?= - verify_context( - Part#{ <<"base">> => State, <<"opts">> => Opts} - ), - true - end; -verify_part(State, [Next | Rest], PartNum, Opts) -> - % Extract the varied-result from the next context and apply it to the current - % state. - case maps:get(<<"normalizer">>, Next, replace) of - replace -> - % Replace the full accumulated state with the varied result and - % recurse to the next part. - verify_part( - maps:get(<<"varied-result">>, Next), - Rest, - PartNum - 1, - Opts - ); - base -> - % The base normalizer means we should extend the result over the - % base state. - verify_part( - maps:get(<<"varied-result">>, Next)#{ <<"...">> => State }, - Rest, - PartNum - 1, - Opts - ); + verify_part(parse(Hashpath, Opts), PartNum, Opts); +verify_part(Hashpath, PartNum, Opts) -> + maybe + {ok, Base} ?= load(Hashpath, PartNum, Opts), + verify_context(Part#{ <<"base">> => State }, Opts) end. %% @doc Verify a full single context, parsed from a binary hashpath. The context @@ -276,9 +250,9 @@ verify_context(Ctx, Opts) -> ), maybe {ok, ExecutedCtx} = hb_ao:do(StrippedCtx), - true ?= verify_varied(Ctx, ExecutedCtx), - true ?= verify_dependencies(Ctx, ExecutedCtx), - true ?= verify_equivalence(Ctx, ExecutedCtx), + true ?= verify_varied(Ctx, ExecutedCtx, Opts), + true ?= verify_dependencies(Ctx, ExecutedCtx, Opts), + true ?= verify_equivalence(Ctx, ExecutedCtx, Opts), {true, ExecutedCtx} else {error, Type} -> @@ -292,7 +266,7 @@ verify_context(Ctx, Opts) -> %% @doc If varied `Req` and `Base` statements were present in the hashpath, %% we verify that they match the executed context. -verify_varied(HPCtx, ExecutedCtx) -> +verify_varied(HPCtx, ExecutedCtx, Opts) -> maybe {ok, HPVBase} ?= find_id(<<"varied-base">>, HPCtx, Opts), {ok, ExecVBase} ?= find_id(<<"varied-base">>, ExecutedCtx, Opts), @@ -301,35 +275,141 @@ verify_varied(HPCtx, ExecutedCtx) -> {ok, HPVReq} ?= find_id(<<"varied-request">>, HPCtx, Opts), {ok, ExecVReq} ?= find_id(<<"varied-request">>, ExecutedCtx, Opts), true ?= HPVReq =:= ExecVReq - orelse {error, <<"Varied `Request`s do not match">>}, + orelse {error, <<"Varied `Request`s do not match">>} + else + {not_found, _} -> + % Skip validation if one or more required components are not + % provided. + true end. %% @doc Verify that the dependencies in the hashpath claim match those in the %% executed context, if both are present. -verify_dependencies(HPCtx, ExecutedCtx) -> +verify_dependencies(HPCtx, ExecutedCtx, Opts) -> maybe {ok, HPDeps} ?= find_id(<<"dependencies">>, HPCtx, Opts), {ok, ExecDeps} ?= find_id(<<"dependencies">>, ExecutedCtx, Opts), true ?= HPDeps =:= ExecDeps orelse {error, <<"Dependencies do not match">>} + else + {not_found, <<"dependencies">>} -> + % Skip verification if dependencies not provided + true end. %% @doc Verify that the results of the execution match those in the claim. -verify_equivalence(HPCtx, ExecutedCtx) -> +verify_equivalence(HPCtx, ExecutedCtx, Opts) -> maybe HPNormalizer = maps:get(<<"normalizer">>, HPCtx, replace), ExecNormalizer = maps:get(<<"normalizer">>, ExecutedCtx, replace), true ?= HPNormalizer =:= ExecNormalizer orelse {error, <<"Normalizers do not match">>}, - {ok, HPResult} ?= find_id(<<"result">>, HPCtx, Opts), - {ok, ExecResult} ?= find_id(<<"result">>, ExecutedCtx, Opts), + {ok, HPResult} ?= find_id(<<"varied-result">>, HPCtx, Opts), + {ok, ExecResult} ?= find_id(<<"varied-result">>, ExecutedCtx, Opts), true ?= HPResult =:= ExecResult orelse {error, <<"Results do not match">>} end. +%% @doc Load the minimal executable base for a hashpath or a given part number +%% within it. +load(Hashpath, Opts) when is_binary(Hashpath) -> + load(parse(Hashpath, Opts), Opts); +load(Parts, Opts) -> + load(Parts, length(Parts), Opts). +load(Hashpath, PartNum, Opts) when is_binary(Hashpath) -> + load(parse(Hashpath, Opts), PartNum, Opts); +load(Parts, PartNum, Opts) -> + maybe + [Init|Rest] ?= load_sequence(Parts, PartNum, Opts), + {ok, InitState} ?= result_from_context(Init, Opts), + lists:foldl( + fun(Ctx, {error, X}) -> {error, X}; + (Ctx, {ok, State}) -> result_from_context(State, Ctx, Opts) + end, + {ok, InitState} + ) + else + [] -> + {error, <<"Cannot load empty hashpath.">>}; + {error, X} -> + {error, <<"Initial state not loadable: ", (hb_util:bin(X))/binary>>} + end. + +%% @doc Find the minimal sequence of hashpath contexts to load/apply such that +%% a valid base can be constructed at position `PartNum`. +load_sequence(Parts, OutOfBounds, Opts) when length(Parts) > OutOfBounds -> + { + error, + << + "Hashpath part number `", + (hb_util:bin(OutOfBounds))/binary, + "` not found. Hashpath length: ", + (hb_util:bin(length(Parts))), + "." + >> + }; +load_sequence(Parts, PartNum, Opts) -> + { + ok, + lists:reverse( + lists:takewhile( + fun(Ctx) -> + case result_from_context(Ctx, Opts) of + {error, _} -> true; + _ -> false + end, + lists:reverse(lists:sublist(Parts, PartNum)) + ) + ) + }. + +%% @doc Extract, if we can, a workable post-exec `message` from a context either +%% via the fully qualified result if possible or layering of the `varied-result` +%% atop the `base` if provided explicitly. +result_from_context(Ctx, Opts) -> result_from_context(undefined, Ctx, Opts). +result_from_context( + undefined, + #{ <<"normalizer">> => replace, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, VRes}; +result_from_context( + S, + #{ <<"normalizer">> => replace, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, Vres}; +result_from_context( + undefined, + Ctx = #{ <<"normalizer">> => base, <<"varied-result">> => VRes }, + _Opts +) -> + case find_id(<<"base">>, Ctx, Opts) of + {ok, Base} -> {ok, VRes#{ <<"...">> => S }}; + {not_found, _} -> + { + error, + << + "Context with `base` extension normalizer", + " without accessible `base`." + >> + } + end; +result_from_context( + S, + #{ <<"normalizer">> => base, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, VRes#{ <<"...">> => S }}; +result_from_context(S, #{ <<"normalizer">> => X }, _Opts) -> + {error, <<"Unsupported normalizer `", (hb_util:bin(X))/binary, "`.">>}; +result_from_context(S, Ctx, Opts) -> + result_from_context(S, Ctx#{ <<"normalizer">> => replace }, Opts). + %%% Tests full_form_round_trip_test() -> + Opts = #{}, HP = << "BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4" @@ -339,11 +419,12 @@ full_form_round_trip_test() -> "@ZGVwZW5kcy1tZXNzYWdlLWlkLTAwMDAwMDAwMDAwMDAw" "=cGF0Y2gtaWQtMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw" >>, - ?assertEqual(HP, format(parse(HP))). + ?assertEqual(HP, format(parse(HP, Opts), Opts)). compact_form_round_trip_test() -> + Opts = #{}, lists:foreach( - fun(HP) -> ?assertEqual(HP, format(parse(HP))) end, + fun(HP) -> ?assertEqual(HP, format(parse(HP, Opts), Opts)) end, [ <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4">>, <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/balance">>, From 65d1c90385e5bc7e3d7ce9099f295b22138fd0f3 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 17 Jul 2026 16:24:39 -0400 Subject: [PATCH 30/35] wip: add execution context component derivation --- src/core/resolver/hb_ao.erl | 53 ++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 602d39f7d..e6454d321 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -87,6 +87,7 @@ %%% Main AO-Core API: -export([resolve/2, resolve/3]). -export([raw/3, raw/4, raw/5]). +-export([with/3]). -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). %%% Shortcuts and tools: -export([keys/2]). @@ -273,6 +274,43 @@ from_context({ok, Ctx = #{ <<"result">> := Res }}) -> from_context(Other) -> throw({unexpected_context_after_exec, Other}). +%% @doc Ensure that each of the given fields is present in an execution +%% context, deriving those that are absent from the components already known. +%% Derived fields are added to the context, such that repeated demands do not +%% repeat work. A field that is neither present nor derivable yields +%% `{not_found, Field}`. +with([], Ctx, _Opts) -> {ok, Ctx}; +with([Field | Fields], Ctx, Opts) when is_map_key(Field, Ctx) -> + with(Fields, Ctx, Opts); +with([Field | Fields], Ctx, Opts) -> + case derive(Field, Ctx, Opts) of + {ok, Value} -> with(Fields, Ctx#{ Field => Value }, Opts); + _ -> {not_found, Field} + end. + +%% @doc Derive a single context field from its complementary components: +%% messages are read from the store by their IDs, while IDs are calculated +%% from their messages. A request may also be derived from a literal path +%% (e.g. `*`), which names itself. Messages read from the store are lazily +%% loaded: their keys are only read when they are themselves demanded. +derive(<<"base">>, #{ <<"base-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(<<"request">>, #{ <<"request-id">> := ID }, Opts) when ?IS_ID(ID) -> + hb_cache:read(ID, Opts); +derive(<<"request">>, #{ <<"request-id">> := Path }, _Opts) -> + {ok, #{ <<"path">> => Path }}; +derive(<<"varied-base">>, #{ <<"varied-base-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(<<"varied-request">>, #{ <<"varied-request-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(Field, Ctx, Opts) -> + % Every `x-id` field is derivable from its `x` message, if present. + case binary:split(Field, <<"-id">>) of + [Name, <<>>] when is_map_key(Name, Ctx) -> + {ok, hb_message:id(maps:get(Name, Ctx), all, Opts)}; + _ -> {not_found, Field} + end. + %% @doc Resolves a fully normalized execution context. Stages that find the %% result early set it in the context; later stages skip themselves when they %% see it. Any non-`{ok, Ctx}` return propagates as the result. @@ -330,7 +368,16 @@ stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) when is_map(Req) -> stage_1(Ctx = #{ <<"request">> := Key }) -> % If the request is for a direct key we normalize it to a message with % only a path of that key. - stage_1(Ctx#{ <<"request">> => #{ <<"path">> => normalize_key(Key) } }). + stage_1(Ctx#{ <<"request">> => #{ <<"path">> => normalize_key(Key) } }); +stage_1(Ctx = #{ <<"opts">> := Opts }) -> + % The context does not carry its messages directly. Derive the `base` and + % `request` from the components that are given (e.g. their IDs), then + % normalize as usual. + case with([<<"base">>, <<"request">>], Ctx, Opts) of + {ok, Loaded} -> stage_1(Loaded); + {not_found, Field} -> + {error, Ctx#{ <<"status">> => error, <<"reason">> => {not_found, Field} }} + end. %% @doc Lookup the device and function to use during an execution. stage_2( @@ -531,7 +578,7 @@ stage_6(Ctx = #{ %% execution's result on top of the `Base` or `Req` message. In cases where %% hashpath calculation is disabled (`hashpath => ignore` in the node message, %% etc.), we -stage_7(Ctx = #{ <<"base">> => Base, <<"opts">> := Opts }) -> +stage_7(Ctx = #{ <<"base">> := Base, <<"opts">> := Opts }) -> maybe {ok, Result} ?= hb_hashpath:result_from_context(Base, Ctx, Opts), {ok, Ctx#{ <<"result">> => Result }} @@ -546,7 +593,7 @@ stage_8( <<"fresh">> := true, <<"varied-base">> := VariedBase, <<"varied-request">> := VariedReq, - <<"varied-result">> := Res, + <<"varied-result">> := VariedRes, <<"opts">> := Opts } ) -> From 8644ae258318df04ab7c3e84ff1e9ec4879f4d75 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 21 Jul 2026 14:22:06 -0400 Subject: [PATCH 31/35] wip: Spec `DRAFT-5` edits, to `Vary` section --- AO-CORE-DRAFT-4.md | 485 ----------------------------------------- AO-CORE.md | 523 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 426 insertions(+), 582 deletions(-) delete mode 100644 AO-CORE-DRAFT-4.md diff --git a/AO-CORE-DRAFT-4.md b/AO-CORE-DRAFT-4.md deleted file mode 100644 index 6048f32ba..000000000 --- a/AO-CORE-DRAFT-4.md +++ /dev/null @@ -1,485 +0,0 @@ -# AO-Core 1.0 Draft 4 - -AO-Core is a protocol for attestable computation over protocol-addressed -messages. It is not itself a virtual machine. It is a neutral meta-VM: a common -language for describing, transporting, attesting, challenging, and tracing the -computations performed by many VMs, called devices. - -AO-Core offers three core properties: - -1. **Atomic attestation**: any transition claim can be verified or challenged - independently from unrelated computation history. -2. **Succinct portability**: a state can be moved as a hashpath plus the values - needed by that hashpath, without enumerating every value that could later be - resolved. -3. **Traceability**: every computation result can be traced through every state - transformation to the origins of the values on which it depends. - -Through these properties in a decentralized protocol, AO-Core intends to give -rise to a globally distributed supercomputer presenting a single system image -across a machine of arbitrary size. - -## Values And Messages - -A value is any protocol-addressable content. - -A message is a value that contains keys and values. Values may identify other -values by protocol address; such links are literals from the point of view of the -core protocol. - -All values can be named by ID. All messages can be represented by extension of -other messages through ancestry: - -```text -set(Base, Patch) = { Patch..., ...: Base } -``` - -`...` is ancestry. `unset` masks an inherited key. - -Extension does not mutate `Base`. It constructs a new value that shares -structure with `Base`. This is the basis of message deduplication: large states -can be represented as small patches over existing states. - -## Resolution - -The primitive relation is: - -```text -Base / Request -> Result -``` - -For a request whose key is `K`, resolution walks the ancestry of `Base` from the -outermost layer inward. - -At each layer: - -1. Look locally for `K`. -2. If `K = Value` is found, return `Value`. -3. If `K = unset` is found, return `not_found`. -4. Otherwise, look locally for `device`. -5. If `device = Dev` is found, execute `Dev` against the original outermost - `Base` and the original `Request`. -6. Otherwise, look locally for `...`. -7. If `... = Ancestor` is found, continue at `Ancestor`. -8. Otherwise, return `not_found`. - -Pseudocode: - -```text -resolve(Outer, Layer, Request): - K = key(Request) - - case local(Layer, K) of - Value -> return Value - unset -> return not_found - not_found -> continue - end - - case local(Layer, "device") of - Dev -> return execute(Dev, Outer, Request) - not_found -> continue - end - - case local(Layer, "...") of - Ancestor -> return resolve(Outer, Ancestor, Request) - not_found -> return not_found - end -``` - -This local inspection is `message@1.0` behavior. It asks only about the current -layer's decoded members; it does not recursively resolve while inspecting that -layer. - -If a device is inherited from an ancestor, it executes over the outermost state: - -```text -{ x: 5, ...: { device: dev1, x: 1 } } / x-is-5 -``` - -selects `dev1` from the ancestor, but `dev1` sees `x = 5`. - -Layer order determines which value answers a key; it does not restrict what a -device sees. A local key answers before its layer's device, but a device that -answers always executes over the complete outermost state. - -## Devices - -A device defines how requests are computed for a state. - -Examples of devices include message interpreters, process VMs, WASM VMs, -Lua VMs, codecs, stores, payment devices, and application-specific evaluators. -AO-Core does not privilege one compute model over another. It standardizes how -their transitions are named, witnessed, attested, transported, and traced. - -If no device is found, the default device is `message@1.0`. - -A device is itself a value, and may be named by link. When the device value is -a message, execution is itself resolution: the request is resolved against the -device message extended over the outermost state: - -```text -execute(Dev, Outer, Request) = resolve(set(Outer, Dev), Request) -``` - -The device message's own keys answer first, its own `device` computes the -remainder, and the outermost state remains visible as ancestry. The recursion -terminates at a device the host implements directly; which devices those are is -an implementation fact. - -Device loading, Erlang modules, function pointers, local caches, and runtime -workers are implementation facts. The protocol fact is the device value that -the computation commits to using. - -## Vary - -Before execution, a transition is varied: - -```text -Base / Request / vary -> VariedBase + VariedRequest @ Depends -``` - -`VariedBase` and `VariedRequest` are ordinary messages containing exactly the -values required by the execution. They use canonical nested structure, not path -strings: - -```text -VariedBase = { - device: process@1.0, - balance: { - OUR_ADDRESS: 7, - SENDER: 93 - } -} - -VariedRequest = { - path: transfer, - from: SENDER, - to: OUR_ADDRESS, - quantity: 3 -} -``` - -`Depends` records where each varied value originated. It has the same shape as -the varied messages, rooted under `base` and `request`. Each leaf is a hashpath -whose terminal value is the corresponding varied value: - -```text -Depends = { - base: { - device: HP_for_process_device, - balance: { - OUR_ADDRESS: HP_for_prior_our_balance, - SENDER: HP_for_prior_sender_balance - } - }, - request: { - path: HP_for_request_path, - from: HP_for_request_from, - to: HP_for_request_to, - quantity: HP_for_quantity - } -} -``` - -The dependency leaf is a single value: the origin hashpath. The value itself -does not need to be duplicated inside `Depends`, because the hashpath binds the -origin to the value it yields. - -If no exact vary specification is available, the conservative valid vary is -identity: - -```text -VariedBase = Base -VariedRequest = Request -``` - -The core rule is: - -```text -No hidden inputs. -``` - -Everything observed by execution must be present in `VariedBase` or -`VariedRequest`, and every varied value must have an origin in `Depends`. - -## Shared Computation - -Varying creates a reusable computation point. - -Many concrete states may vary to the same pair: - -```text -BaseA / Request -> VariedBase + VariedRequest -BaseB / Request -> VariedBase + VariedRequest -``` - -The execution: - -```text -VariedBase / VariedRequest -> Patch -``` - -is shared by all sufficiently alike concrete states for that request. The final -states may still differ because the patch is equivalent to applying the original -transition to each original base: - -```text -BaseA / Request == set(BaseA, Patch) -BaseB / Request == set(BaseB, Patch) -``` - -This is the default mode of AO-Core computation: do a computation once for the -material inputs that matter, then reuse it across every state/request pair that -varies to those inputs. - -All AO-Core states and transition results are cacheable by address. Prior -computations from many different execution traces can therefore be reused -seamlessly. Because AO-Core is expressed naturally through HTTP semantics, the -same `Vary`-style caching and routing ideas that already power web -infrastructure can be applied to decentralized computation. - -## Transition Equivalence - -A transition asserts an equivalence between resolving a request and extending -the base with the patch produced by varied execution: - -```text -Base / Request - == set(Base, Patch) -``` - -where: - -```text -Base / Request / vary -> VariedBase + VariedRequest @ Depends -VariedBase / VariedRequest -> Patch -``` - -Extension is just `set`. There is no special state update operation beyond -constructing a new message with ancestry. - -## Hashpaths - -A hashpath is a succinct executable claim and an addressable protocol value. -Like any other value, it may be encountered as a link and resolved through -AO-Core, or encountered as serialized content and decoded into its in-memory -form. - -The full transition forms are: - -```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID -``` - -`=` means the execution produced a patch that extends the prior state: - -```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID -``` - -is equivalent to: - -```text -set(Base, Patch) -``` - -`.` means the execution produced a replacement value: - -```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID -``` - -The accumulated state becomes `ResultID`, dropping the prior state's keys rather -than extending them. - -A compact form may omit fields when they are derivable or supplied elsewhere, -but the full receipt must be recoverable for challenge and trace. - -A hashpath is a sequence of transition claims. Later segments operate on the -result established by earlier segments. - -Segments without explicit vary syntax are not special. They are compact -transition claims. For example: - -```text -HP/*=FinalResultID -``` - -is simply a claim that resolving `*` at `HP` yields `FinalResultID`. HTTP -gateways commonly append such a segment so the response body is tied to the -specific keys and values returned to the client. - -## Hashpath Loading And Portability - -Hashpaths are not a storage system outside AO-Core. They are addressable values -whose loaded form reconstructs ordinary message semantics. - -Reusable patches can be stored by their generic IDs, without caller-specific -ancestry. The hashpath records how that generic value is reached from a prior -state. For an extension segment: - -```text -PriorHP/Req>VariedBase+VariedReq@Depends=Patch -``` - -the cacheable value may be the hashpath link itself: - -```text -link:PriorHP/Req>VariedBase+VariedReq@Depends=Patch -``` - -loading the segment loads `Patch` and presents it as an extension whose `...` -is the transition context before the patch result: - -```text -load(PriorHP/Req>VB+VR@Deps=Patch) - -> { PatchKeys..., ...: PriorHP/Req>VB+VR@Deps } -``` - -The `...` value is itself a hashpath value. Loading it reconstructs the prior -state for inherited-key resolution and retains the request, varied witnesses, -and dependency message needed to challenge the transition. Implementations may -cache this decoded context in runtime metadata, but the protocol-visible -ancestry remains the hashpath value. - -For replacement segments: - -```text -PriorHP/Req>VB+VR@Deps.Result -``` - -loading the segment yields `Result`. The prior state is not inherited as active -message keys, but the hashpath still carries the transition context needed for -challenge and trace. - -This gives portability as a single addressable value: posting a hashpath plus -the values needed to resolve the IDs it names gives another node enough data to -reconstruct the live state, challenge any segment, and continue computation. - -## Challenge And Audit - -A transition can be challenged locally, without verifying its entire dependency -tree. The usual practical operation is to pick one claim and verify only the -facts needed for that claim. A full provenance audit is the recursive version of -the same process. - -To challenge a transition claim: - -1. Verify that `BaseID` and `ReqID` identify the claimed values. -2. Verify the vary claim: - - ```text - Base / Request / vary -> VariedBase + VariedRequest @ Depends - ``` - -3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf - in `Depends` and verify that it yields that value. -4. Verify execution: - - ```text - VariedBase / VariedRequest -> Patch - ``` - -5. Verify transition equivalence: - - ```text - "=" means the accumulated state is set(Base, Patch) - "." means the accumulated state is Result - ``` - -Any one of these checks can be challenged independently. To audit the full -provenance tree, recursively challenge the dependency hashpaths named in -`Depends`. - -## Traceability - -To trace a value in a result: - -1. Locate the transition that produced the state containing the value. -2. If the value was introduced by the patch, trace it to that transition's - varied witness. -3. Follow every corresponding leaf in `Depends`. -4. If the value was inherited through `...`, continue tracing in the ancestor - state. -5. Repeat recursively until reaching literals, signed inputs, codec inputs, or - externally attested transition claims. - -For example, a process state may claim: - -```text -ProcessStateN.balance.OUR_ADDRESS = 10 -``` - -The trace may show that this came from: - -```text -ProcessStateN-1 / TransferRequest - > VariedBase + VariedRequest @ Depends - = ProcessStateN -``` - -with: - -```text -VariedBase.balance.OUR_ADDRESS = 7 -VariedBase.balance.SENDER = 93 -VariedRequest.quantity = 3 -``` - -`Depends` then points to the hashpaths that produced `7`, `93`, and `3`. The -quantity may trace to an inbound message from a swap process, whose sale-price -transition may itself be attested by another node. - -The trace is not a narrative. It is a recursive chain of AO-Core claims. - -## HTTP Expression - -HTTP is an expression of AO-Core, not the foundation of AO-Core. - -An HTTP request is decoded into an AO-Core request message. URL path segments, -query parameters, method, headers, and body are message members. Content -negotiation selects codecs for values and messages. The server resolves: - -```text -Base / Request -> Result -``` - -and returns an HTTP response containing the encoded result plus enough hashpath -and commitment information for the receiver to verify, port, and continue the -computation. - -Thus HTTP gives AO-Core a universal transport and user-facing syntax while the -protocol remains independent of HTTP itself. - -HTTP computations should append a terminal materialization claim by default: - -```text -HP/*=MaterializedID -``` - -This claim is not special in the hashpath calculus. It is the ordinary -materialization request for `*`, included so that the transport response is tied -to the concrete enumerated keys and values returned to the client. - -The default HTTPSig response carries two independently useful commitments: - -1. A commitment over the enumerated response keys and values, excluding the - `...` hashpath field. This commitment is typically unsigned unless the - response is already a pre-existing signed message. Its ID is - `MaterializedID`. -2. A signature over just the excluded `...` field, whose value is the hashpath - terminating in `=MaterializedID`. - -The first commitment lets the returned value stand alone by ID. The second lets -the recipient port or challenge the computation through the hashpath. They can -be used together or independently. - -## Summary - -AO-Core turns computation into portable, challengeable, traceable message -transitions. - -Devices provide the compute. Vary provides exact witnesses. Depends provides -origin trace. Hashpaths provide portable claims. Extension provides deduplicated -state. HTTP provides a practical expression layer. diff --git a/AO-CORE.md b/AO-CORE.md index 4bc53af64..5e156b2b8 100644 --- a/AO-CORE.md +++ b/AO-CORE.md @@ -1,203 +1,532 @@ -# AO-Core 1.0 Minimal Semantics +# AO 1.0: A universal protocol for computation over URIs. + +AO is a language for describing a method of combining -- computing over -- URI-named +resourced, yielding further URI-addressable values. AO forms a computing environment, +but it is not itself a virtual machine. It is a neutral meta-VM: a common +language for describing, transporting, attesting, challenging, and tracing the +computations performed by many VMs, called devices. + +AO offers three core properties: + +1. **Atomic attestation**: any computation claim can be verified or challenged + independently from all other history. +2. **Succinct portability**: any computation may be continued on any other + participating machine by invoking its accrued hashpath upon the recipient. + Each element of the state of the computation may be moved eagerly or recalled + from untrusted peers on-demand. +3. **Traceability**: every computation result can be traced through every state + transformation to the origins of the values on which it depends. + +Through these properties in a decentralized protocol, AO-Core intends to give +rise to a globally distributed supercomputer presenting a single system image +across a machine of arbitrary size. + +## Resources, Collections, and Values. + +A resource is any protocol-addressable content, referenced by URIs ([RFC 3986]). + +As stated in the specification, URIs may come in hierarchical and non-hierarchical +forms. Their values are produced by resolvers that yield a representation of the +bytes that the named resource refers to. Resources may identify other values by +protocol address; creating a graph of connected computed elements. + +AO runtime implementations use a system of caches that associated resources with +their associated values, then make these available for computations to access. +When a URI is not known in a node's existing stores, it may be requested for +resolution using the host environments native mechanisms, or through custom +providers built-in to the runtime. + +Resources may share a common prefix, resembling a directory or other composite +structure via their path-parts. AO resolutions may both consume and produce such +collections of associated URIs by referring to their shared prefix in either +inputs or outputs to computations. + +## Hashpaths and Execution + +In addition to consuming URIs as inputs the AO protocol defines the `ao://` +URI-scheme, which allows its universal computation language to name state +transitions, attest to their dependencies, and their results. We call the +namespace of URIs under this scheme collectively `hashpaths`. + +Hashpaths refer to a merklized log of computation contexts, each of which +contains at minimum a `Base` URI and a representation of the request. +The simplest form of a hashpath is resolving a `data:` `request` value against +a resource collection, which yields the standard resolution of the concatenated +values (`BaseURI/RequestBinary`). + +If the request is instead of a composite form, we attempt to resolve `/path` +element upon the second ('request') URI for the computation, and search the +primary ('base') with its value. In the event that the resource is resolved, its +value is returned. If the direct URI cannot be resolved, we attempt to resolve +`BaseURI/device`, yielding a specification -- or a binary, whose value can be +resolved to a specification -- of a `device` (a virtual machine) to utilize for +the computation. If a device is not found in the `BaseURI` we recurse backwards +through prior computed results of the hashpath, repeating the process until +either a direct key or `device` is found, or the hashpath reaches a 'replacement' +element (implying that the computation's result does not 'extend' the elements +found in the hashpath prior to it, but instead nullifies each). Through this +recursive mechanism, new messages can be composed via the combination of prior +message IDs, and values in a present state may lazily include those from prior +states -- allowing their history (when each was set, what the inputs to that +computation were, etc) to be traced individually. + +Each new Extension does not mutate the `Base`. Instead, it constructs a new value +that shares elements from the `Base`, overlayed with additional resolvable +resources and prefixed names. This grants the basis of message deduplication: +large states can be represented as small patches over existing states, maintaining +provenance. + +Hashpaths represent a sequence of execution frames, each of which may contain a +number of elements: +1. A `Base` URI upon which the request is being made. Must always be present on + the first context of the hashpath, but may be ommitted for the second + and further elements, instead having their `Base` inferred from the rolling + context of the hashpath itself. +2. A raw binary or `Request` URI, holding the metadata of the request upon the + `Base`. +3. A `Result`, optionally, if computed, the result of the computation as a + further URI. +4. `Varied-Base` and `Varied-Request` emelements, each containing URIs for + collections of _only_ the necessary components that were utilized in the + state transition. +5. `Dependencies`, an optional collection hosting the hashpaths of each utilized + state component from the `[Varied-][Base|Request]` elements. +5. An `Attestation`, cryptographically linking an identity with each of the + constituent claims made in the hashpath context. + +A hashpath may convey any number of connected frames of execution attested by a +single commitment. Hashpath may be extended by Merklizing the ordered list of +contexts, such that only a single root is provided as the `Base` to the final +frame. Alternatively, any number of `Base`/`Request` pairs may be provided, +with the `Base` of each frame being the ancestry (ordered prior results) of the +`Request` at that layer. + +## Resolution Semantics + +Concretely, the primitive AO relation is: -AO-Core is a protocol for attestable computation over messages. +```text +BaseURI / RequestURI | Value -> ResultURI | Value +``` + +For a request whose `path` is `P`, resolution walks the ancestry of `Base` from +the outermost layer inward until either a direct result, a `device`, or a +non-extending hashpath context is found. + +At each layer: + +1. Resolve `BaseURI/path`. Return if found. +2. Resolve `BaseURI/device`. +3. If `Device` is found, lookup runtime-compliant implementation and execute + against the original outermost `Base` and the original `Request`. +4. Else, if a prior extending element of the hashpath context is found, recursively + resolve at that element. -The only primitive relation is membership: +Pseudocode: ```text -Base / Request -> Result +resolve(Outer, BaseURI, Request): + P = path(Request) + + case lookup(BaseURI/P) of + Value -> return Value + unset -> return not_found + not_found -> continue + end + + case lookup(BaseURI/device) of + Dev -> return execute(Dev, Outer, Request) + not_found -> continue + end + + case local(BaseURI/...) of + Ancestor -> return resolve(Outer, Ancestor, Request) + not_found -> return not_found + end ``` -Everything else is syntax, storage, caching, transport, or implementation. - -## Required Properties +If a device is inherited from an ancestor, it executes over the outermost state: -1. **All computations must be atomically attested**: Upon receipt of a succinct committed string, we must be able to identify a set of computation result claims, and then _individually_ verify if each is true, without dependence on executing any other computations in the results. +```text +{ x: 5, ...: { device: dev1, x: 1 } } / x-is-5 +``` -2. **All computations must be succinctly portable**: Any user wishing to move a computation from one machine to another should be able to take a hashpath and post a single message to another node, effectively duplicating the state and allowing for further computation elsewhere. By the nature of extension, complex collections of keys and values must be efficiently loadable from smaller, more common subsets. +selects `dev1` from the ancestor, but `dev1` sees `x = 5`. -3. **All computations must be fully traceable**: Given any result, the full set of computations that gave rise to each of its values must be enumeratable. Receipt of a single value should allow you to efficiently isolate every strand of other computations, no matter how distant or tangentially related, whose results were dependencies of the given value. +## Devices -## Values +A device defines how requests are computed for a state. -A value is a literal, message, or link. +Examples of devices include message interpreters, process VMs, WASM VMs, +Lua VMs, codecs, stores, payment devices, and application-specific evaluators. +AO-Core does not privilege one compute model over another. It standardizes how +their transitions are named, witnessed, attested, transported, and traced. -A message is a device-interpreted value with public keys, optional commitments, -optional private runtime state, and optional ancestry. +`Device` references may be stated in three forms: -`priv` is runtime-local. It is not part of IDs, public commitments, signatures, -or protocol equality. +1. **Specification URIs**: fully-qualified URIs that yield the intended device's + full specification when resolved. +2. **Binaries, resolved as Permaweb Names**: Binary literals that imply a + request to resolve the name against `ao://~name@1.0/[Name]` in the resolver. +3. **`ao://` Resource Prefixes**: Recursively resolved devices whos functionality + is defined by extending the `Base` resource with its values and calling the + resolution upon its new form. -A link names another value. Loading a link must preserve the same protocol -value. +The third form allows for recursive resolution of devices, in which a new device +is constructed using the same underlying virtual machine implementations inside +the AO runtime, but with the values found in the extension resource taking +precedence over the base resource. -## Extension +## Vary -`...` is ancestry. +Before execution, a transition is varied: ```text -{ a: 2, ...: { a: 1, b: 3 } } +Base / Request / vary -> VariedBase + VariedRequest @ Depends ``` -has active `a = 2` and active `b = 3`. +`VariedBase` and `VariedRequest` are ordinary messages containing exactly the +values required by the execution. They use canonical nested structure, not path +strings: -Outer layers shadow inner layers. `unset` masks an inherited key. +```text +VariedBase = { + device: process@1.0, + balance: { + OUR_ADDRESS: 7, + SENDER: 93 + } +} + +VariedRequest = { + path: transfer, + from: SENDER, + to: OUR_ADDRESS, + quantity: 3 +} +``` -`set(Base, Patch)` constructs extension: +`Depends` records where each varied value originated. It has the same shape as +the varied messages, rooted under `base` and `request`. Each leaf is a hashpath +whose terminal value is the corresponding varied value: ```text -{ PatchKeys..., ...: Base } +Depends = { + base: { + device: HP_for_process_device, + balance: { + OUR_ADDRESS: HP_for_prior_our_balance, + SENDER: HP_for_prior_sender_balance + } + }, + request: { + path: HP_for_request_path, + from: HP_for_request_from, + to: HP_for_request_to, + quantity: HP_for_quantity + } +} ``` -It does not mutate `Base`. +The dependency leaf is a single value: the origin hashpath. The value itself +does not need to be duplicated inside `Depends`, because the hashpath binds the +origin to the value it yields. -Pathless composition is set: +If no exact vary specification is available, the conservative valid vary is +identity: ```text -AO(Base, PatchWithoutPath) = set(Base, PatchWithoutPath) +VariedBase = Base +VariedRequest = Request ``` -A sequence is a fold: +The core rule is: ```text -AO([M0, M1, M2]) = AO(AO(M0, M1), M2) +No hidden inputs. ``` -## Devices +Everything observed by execution must be present in `VariedBase` or +`VariedRequest`, and every varied value must have an origin in `Depends`. -A device defines memberships for request keys. +## Shared Computation -If no device applies, the default device is `message@1.0`. +Varying creates a reusable computation point. -Resolution through extension is layer-ordered. An outer layer interpreted by -`message@1.0` can return its own direct key before an inherited device is -reached. A layer that declares a device is interpreted by that device. +Many concrete states may vary to the same pair: -Implementation artifacts such as Erlang modules and function pointers are -node-local and must not be public protocol facts. +```text +BaseA / Request -> VariedBase + VariedRequest +BaseB / Request -> VariedBase + VariedRequest +``` -## `message@1.0` +The execution: -`message@1.0` is the default message device. +```text +VariedBase / VariedRequest -> Patch +``` -Reserved keys include: +is shared by all sufficiently alike concrete states for that request. The final +states may still differ because the patch is equivalent to applying the original +transition to each original base: ```text -get, set, remove, *, keys, id, commit, verify, -commitments, committed, committers, vary, schema +BaseA / Request == set(BaseA, Patch) +BaseB / Request == set(BaseB, Patch) ``` -For non-reserved keys, `message@1.0` resolves: +This is the default mode of AO-Core computation: do a computation once for the +material inputs that matter, then reuse it across every state/request pair that +varies to those inputs. + +All AO-Core states and transition results are cacheable by address. Prior +computations from many different execution traces can therefore be reused +seamlessly. Because AO-Core is expressed naturally through HTTP semantics, the +same `Vary`-style caching and routing ideas that already power web +infrastructure can be applied to decentralized computation. + +## Transition Equivalence + +A transition asserts an equivalence between resolving a request and extending +the base with the patch produced by varied execution: ```text -local public key -> value -local unset -> not_found -otherwise -> resolve through ... +Base / Request + == set(Base, Patch) ``` -`*` materializes active keys: +where: ```text -M/* -> { ActiveKeys..., ...: M } +Base / Request / vary -> VariedBase + VariedRequest @ Depends +VariedBase / VariedRequest -> Patch ``` -`remove(K)` is `set({ K: unset })`. +Extension is just `set`. There is no special state update operation beyond +constructing a new message with ancestry. -`id` commits to the selected public surface, never to `priv`. +## Hashpath Attestations -## Vary +A hashpath is a succinct executable claim and an addressable protocol value. +Like any other value, it may be encountered as a link and resolved through +AO-Core, or encountered as serialized content and decoded into its in-memory +form. -Execution is prepared before it runs. +The full transition forms are: -Preparation is itself an AO-Core membership. For a transition: +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +``` + +`=` means the execution produced a patch that extends the prior state: ```text -Base / Req > VarBase + VarReq = Res +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID ``` -the `>` clause means: +is equivalent to: ```text -Base / vary(Req) -> { base: VarBase, request: VarReq } -VarBase / VarReq -> Res +set(Base, Patch) ``` -`vary` is device-owned. It determines the exact base and request witnesses used -for execution. Execution must observe only those witnesses. +`.` means the execution produced a replacement value: -A varied message may materialize keys, preserve links, or preserve extension, -but it must not hide dependencies. Any value it exposes must remain traceable -to the membership claim that produced it. +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +``` + +The accumulated state becomes `ResultID`, dropping the prior state's keys rather +than extending them. + +A compact form may omit fields when they are derivable or supplied elsewhere, +but the full receipt must be recoverable for challenge and trace. -## Hashpaths +A hashpath is a sequence of transition claims. Later segments operate on the +result established by earlier segments. -A hashpath is a succinct executable claim string. +Segments without explicit vary syntax are not special. They are compact +transition claims. For example: ```text -Base/Req>VarBase+VarReq=Patch +HP/*=FinalResultID ``` -means the execution produced an extension patch. The accumulated state becomes: +is simply a claim that resolving `*` at `HP` yields `FinalResultID`. HTTP +gateways commonly append such a segment so the response body is tied to the +specific keys and values returned to the client. + +## Hashpath Loading And Portability + +Hashpaths are not a storage system outside AO-Core. They are addressable values +whose loaded form reconstructs ordinary message semantics. + +Reusable patches can be stored by their generic IDs, without caller-specific +ancestry. The hashpath records how that generic value is reached from a prior +state. For an extension segment: ```text -set(Base, Patch) +PriorHP/Req>VariedBase+VariedReq@Depends=Patch ``` +the cacheable value may be the hashpath link itself: + +```text +link:PriorHP/Req>VariedBase+VariedReq@Depends=Patch +``` + +loading the segment loads `Patch` and presents it as an extension whose `...` +is the transition context before the patch result: + ```text -Base/Req>VarBase+VarReq.Result +load(PriorHP/Req>VB+VR@Deps=Patch) + -> { PatchKeys..., ...: PriorHP/Req>VB+VR@Deps } ``` -means the execution produced a full replacement. The accumulated state becomes: +The `...` value is itself a hashpath value. Loading it reconstructs the prior +state for inherited-key resolution and retains the request, varied witnesses, +and dependency message needed to challenge the transition. Implementations may +cache this decoded context in runtime metadata, but the protocol-visible +ancestry remains the hashpath value. + +For replacement segments: + +```text +PriorHP/Req>VB+VR@Deps.Result +``` + +loading the segment yields `Result`. The prior state is not inherited as active +message keys, but the hashpath still carries the transition context needed for +challenge and trace. + +This gives portability as a single addressable value: posting a hashpath plus +the values needed to resolve the IDs it names gives another node enough data to +reconstruct the live state, challenge any segment, and continue computation. + +## Challenge And Audit + +A transition can be challenged locally, without verifying its entire dependency +tree. The usual practical operation is to pick one claim and verify only the +facts needed for that claim. A full provenance audit is the recursive version of +the same process. + +To challenge a transition claim: + +1. Verify that `BaseID` and `ReqID` identify the claimed values. +2. Verify the vary claim: + + ```text + Base / Request / vary -> VariedBase + VariedRequest @ Depends + ``` + +3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf + in `Depends` and verify that it yields that value. +4. Verify execution: + + ```text + VariedBase / VariedRequest -> Patch + ``` + +5. Verify transition equivalence: + + ```text + "=" means the accumulated state is set(Base, Patch) + "." means the accumulated state is Result + ``` + +Any one of these checks can be challenged independently. To audit the full +provenance tree, recursively challenge the dependency hashpaths named in +`Depends`. + +## Traceability + +To trace a value in a result: + +1. Locate the transition that produced the state containing the value. +2. If the value was introduced by the patch, trace it to that transition's + varied witness. +3. Follow every corresponding leaf in `Depends`. +4. If the value was inherited through `...`, continue tracing in the ancestor + state. +5. Repeat recursively until reaching literals, signed inputs, codec inputs, or + externally attested transition claims. + +For example, a process state may claim: ```text -Result +ProcessStateN.balance.OUR_ADDRESS = 10 ``` +The trace may show that this came from: + ```text -HP/*=ID +ProcessStateN-1 / TransferRequest + > VariedBase + VariedRequest @ Depends + = ProcessStateN ``` -means materializing the active state at `HP` has unsigned ID `ID`. +with: -A multi-step hashpath composes these transitions left to right. +```text +VariedBase.balance.OUR_ADDRESS = 7 +VariedBase.balance.SENDER = 93 +VariedRequest.quantity = 3 +``` -Each transition must be independently verifiable from its witnesses. Verifying -step `N` must not require executing steps `0..N-1`; the prior state is -reconstructed from the hashpath and loaded witnesses. +`Depends` then points to the hashpaths that produced `7`, `93`, and `3`. The +quantity may trace to an inbound message from a swap process, whose sale-price +transition may itself be attested by another node. -## Storage And Portability +The trace is not a narrative. It is a recursive chain of AO-Core claims. -Reusable result patches should be stored without caller-specific ancestry. +## HTTP Expression -Loading a hashpath reconstructs live extension semantics: +HTTP is an expression of AO-Core, not the foundation of AO-Core. + +An HTTP request is decoded into an AO-Core request message. URL path segments, +query parameters, method, headers, and body are message members. Content +negotiation selects codecs for values and messages. The server resolves: ```text -load(HP/Req>VB+VR=Patch) - -> { PatchKeys..., ...: load(HP), priv: { hashpath: HP/... } } +Base / Request -> Result ``` -A portable computation package is one message containing a hashpath plus the -witness messages needed to resolve its IDs. Posting that package to another -node gives the node enough data to reconstruct the state and continue +and returns an HTTP response containing the encoded result plus enough hashpath +and commitment information for the receiver to verify, port, and continue the computation. -## Commitments +Thus HTTP gives AO-Core a universal transport and user-facing syntax while the +protocol remains independent of HTTP itself. -A commitment attests public membership claims and public values. +HTTP computations should append a terminal materialization claim by default: -Computed-result commitments attest the hashpath context. +```text +HP/*=MaterializedID +``` -Commitments never attest `priv`. +This claim is not special in the hashpath calculus. It is the ordinary +materialization request for `*`, included so that the transport response is tied +to the concrete enumerated keys and values returned to the client. -Signed ancestors remain valid under extension because extension does not mutate -them. A signed request used to construct a new extension layer does not -automatically sign that new layer. +The default HTTPSig response carries two independently useful commitments: -## Core Rule +1. A commitment over the enumerated response keys and values, excluding the + `...` hashpath field. This commitment is typically unsigned unless the + response is already a pre-existing signed message. Its ID is + `MaterializedID`. +2. A signature over just the excluded `...` field, whose value is the hashpath + terminating in `=MaterializedID`. -No hidden inputs. +The first commitment lets the returned value stand alone by ID. The second lets +the recipient port or challenge the computation through the hashpath. They can +be used together or independently. + +## Summary -Every value observed by execution must be present in `VarBase` or `VarReq`. +AO-Core turns computation into portable, challengeable, traceable message +transitions. -Every value in `VarBase`, `VarReq`, or `Result` must be traceable to ordinary -AO-Core memberships. +Devices provide the compute. Vary provides exact witnesses. Depends provides +origin trace. Hashpaths provide portable claims. Extension provides deduplicated +state. HTTP provides a practical expression layer. From 84688779c14f974e90ac0c4c3bce09ed08ff2279 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 21 Jul 2026 17:09:06 -0400 Subject: [PATCH 32/35] wip: spec DRAFT-5 --- AO-CORE.md | 174 ++++++++++++++++++++++++++--------------------------- 1 file changed, 86 insertions(+), 88 deletions(-) diff --git a/AO-CORE.md b/AO-CORE.md index 5e156b2b8..38a461c86 100644 --- a/AO-CORE.md +++ b/AO-CORE.md @@ -179,13 +179,13 @@ precedence over the base resource. ## Vary -Before execution, a transition is varied: +Before execution, the selected device varies the transition: ```text -Base / Request / vary -> VariedBase + VariedRequest @ Depends +Base / Request / vary -> VariedBase + VariedRequest @ Dependencies ``` -`VariedBase` and `VariedRequest` are ordinary messages containing exactly the +`VariedBase` and `VariedRequest` are resource collections containing exactly the values required by the execution. They use canonical nested structure, not path strings: @@ -206,12 +206,12 @@ VariedRequest = { } ``` -`Depends` records where each varied value originated. It has the same shape as -the varied messages, rooted under `base` and `request`. Each leaf is a hashpath -whose terminal value is the corresponding varied value: +`Dependencies` records where each varied value originated. It has the same shape +as the varied collections, rooted under `base` and `request`. Each leaf is a +hashpath whose terminal value is the corresponding varied value: ```text -Depends = { +Dependencies = { base: { device: HP_for_process_device, balance: { @@ -229,7 +229,7 @@ Depends = { ``` The dependency leaf is a single value: the origin hashpath. The value itself -does not need to be duplicated inside `Depends`, because the hashpath binds the +does not need to be duplicated inside `Dependencies`, because the hashpath binds the origin to the value it yields. If no exact vary specification is available, the conservative valid vary is @@ -240,6 +240,8 @@ VariedBase = Base VariedRequest = Request ``` +Identity variation does not waive dependency recording. + The core rule is: ```text @@ -247,17 +249,17 @@ No hidden inputs. ``` Everything observed by execution must be present in `VariedBase` or -`VariedRequest`, and every varied value must have an origin in `Depends`. +`VariedRequest`, and every varied value must have an origin in `Dependencies`. ## Shared Computation Varying creates a reusable computation point. -Many concrete states may vary to the same pair: +Many concrete bases may vary to the same pair: ```text -BaseA / Request -> VariedBase + VariedRequest -BaseB / Request -> VariedBase + VariedRequest +BaseA / Request / vary -> VariedBase + VariedRequest @ DependenciesA +BaseB / Request / vary -> VariedBase + VariedRequest @ DependenciesB ``` The execution: @@ -266,8 +268,8 @@ The execution: VariedBase / VariedRequest -> Patch ``` -is shared by all sufficiently alike concrete states for that request. The final -states may still differ because the patch is equivalent to applying the original +is shared by all sufficiently alike concrete bases for that request. The final +results may still differ because the patch is equivalent to applying the original transition to each original base: ```text @@ -276,14 +278,12 @@ BaseB / Request == set(BaseB, Patch) ``` This is the default mode of AO-Core computation: do a computation once for the -material inputs that matter, then reuse it across every state/request pair that +material inputs that matter, then reuse it across every base/request pair that varies to those inputs. -All AO-Core states and transition results are cacheable by address. Prior -computations from many different execution traces can therefore be reused -seamlessly. Because AO-Core is expressed naturally through HTTP semantics, the -same `Vary`-style caching and routing ideas that already power web -infrastructure can be applied to decentralized computation. +Both the varied computation and complete transition results are cacheable by +address, allowing prior computations from different execution traces to be +reused. ## Transition Equivalence @@ -298,31 +298,32 @@ Base / Request where: ```text -Base / Request / vary -> VariedBase + VariedRequest @ Depends +Base / Request / vary -> VariedBase + VariedRequest @ Dependencies VariedBase / VariedRequest -> Patch ``` -Extension is just `set`. There is no special state update operation beyond -constructing a new message with ancestry. +Extension is just `set`. There is no special update operation beyond +constructing an extension whose ancestry is the original base. -## Hashpath Attestations +## Hashpath Assertions And Claims -A hashpath is a succinct executable claim and an addressable protocol value. -Like any other value, it may be encountered as a link and resolved through -AO-Core, or encountered as serialized content and decoded into its in-memory -form. +A hashpath without a terminal result is an address naming a computation. A +hashpath containing its result is an assertion. A signature over one or more +hashpaths is a claim. Like any other value, a hashpath may be encountered as a +link and resolved through AO-Core, or encountered as serialized content and +decoded into its in-memory form. The full transition forms are: ```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID=PatchID +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID.ResultID ``` -`=` means the execution produced a patch that extends the prior state: +`=` means the execution produced a patch that extends the prior result: ```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID=PatchID +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID=PatchID ``` is equivalent to: @@ -334,94 +335,88 @@ set(Base, Patch) `.` means the execution produced a replacement value: ```text -BaseID/ReqID>VariedBaseID+VariedReqID@DependsID.ResultID +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID.ResultID ``` -The accumulated state becomes `ResultID`, dropping the prior state's keys rather -than extending them. +The accumulated result becomes `ResultID`, dropping the prior result's resources +rather than extending them. A compact form may omit fields when they are derivable or supplied elsewhere, -but the full receipt must be recoverable for challenge and trace. +but the full assertion must be recoverable for challenge and trace. -A hashpath is a sequence of transition claims. Later segments operate on the +A hashpath is a sequence of transition assertions. Later segments operate on the result established by earlier segments. Segments without explicit vary syntax are not special. They are compact -transition claims. For example: +transition assertions. For example: ```text HP/*=FinalResultID ``` -is simply a claim that resolving `*` at `HP` yields `FinalResultID`. HTTP +is simply an assertion that resolving `*` at `HP` yields `FinalResultID`. HTTP gateways commonly append such a segment so the response body is tied to the specific keys and values returned to the client. ## Hashpath Loading And Portability Hashpaths are not a storage system outside AO-Core. They are addressable values -whose loaded form reconstructs ordinary message semantics. +whose loaded form reconstructs resource-prefix and extension semantics. Reusable patches can be stored by their generic IDs, without caller-specific ancestry. The hashpath records how that generic value is reached from a prior -state. For an extension segment: - -```text -PriorHP/Req>VariedBase+VariedReq@Depends=Patch -``` - -the cacheable value may be the hashpath link itself: +result. For an extension segment: ```text -link:PriorHP/Req>VariedBase+VariedReq@Depends=Patch +PriorHP/Req>VariedBase+VariedReq@Dependencies=Patch ``` -loading the segment loads `Patch` and presents it as an extension whose `...` -is the transition context before the patch result: +Stores may cache the asserted result at that hashpath. Loading the segment loads +`Patch` and presents it as an extension whose `...` is the previous accumulated +result: ```text -load(PriorHP/Req>VB+VR@Deps=Patch) - -> { PatchKeys..., ...: PriorHP/Req>VB+VR@Deps } +load(PriorHP/Req>VB+VR@Dependencies=Patch) + -> { PatchKeys..., ...: PriorHP } ``` -The `...` value is itself a hashpath value. Loading it reconstructs the prior -state for inherited-key resolution and retains the request, varied witnesses, -and dependency message needed to challenge the transition. Implementations may -cache this decoded context in runtime metadata, but the protocol-visible -ancestry remains the hashpath value. +The `...` value is itself a hashpath. Loading it reconstructs the prior result +for inherited-key resolution. The complete result remains addressed by the full +hashpath, which retains the request, varied witnesses, and dependencies needed +to challenge the transition. For replacement segments: ```text -PriorHP/Req>VB+VR@Deps.Result +PriorHP/Req>VB+VR@Dependencies.Result ``` -loading the segment yields `Result`. The prior state is not inherited as active -message keys, but the hashpath still carries the transition context needed for +loading the segment yields `Result`. The prior result is not inherited as active +keys, but the hashpath still carries the transition context needed for challenge and trace. This gives portability as a single addressable value: posting a hashpath plus the values needed to resolve the IDs it names gives another node enough data to -reconstruct the live state, challenge any segment, and continue computation. +reconstruct the result collection, challenge any segment, and continue computation. ## Challenge And Audit A transition can be challenged locally, without verifying its entire dependency -tree. The usual practical operation is to pick one claim and verify only the -facts needed for that claim. A full provenance audit is the recursive version of -the same process. +tree. The usual practical operation is to pick one assertion and verify only the +values needed to reproduce it. A full provenance audit is the recursive version +of the same process. -To challenge a transition claim: +To challenge a transition assertion: -1. Verify that `BaseID` and `ReqID` identify the claimed values. -2. Verify the vary claim: +1. Verify that `BaseID` and `ReqID` identify the asserted values. +2. Verify the vary assertion: ```text - Base / Request / vary -> VariedBase + VariedRequest @ Depends + Base / Request / vary -> VariedBase + VariedRequest @ Dependencies ``` 3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf - in `Depends` and verify that it yields that value. + in `Dependencies` and verify that it yields that value. 4. Verify execution: ```text @@ -431,28 +426,29 @@ To challenge a transition claim: 5. Verify transition equivalence: ```text - "=" means the accumulated state is set(Base, Patch) - "." means the accumulated state is Result + "=" means the accumulated result is set(Base, Patch) + "." means the accumulated result is Result ``` Any one of these checks can be challenged independently. To audit the full provenance tree, recursively challenge the dependency hashpaths named in -`Depends`. +`Dependencies`. ## Traceability To trace a value in a result: -1. Locate the transition that produced the state containing the value. +1. Locate the transition that produced the result collection containing the + value. 2. If the value was introduced by the patch, trace it to that transition's varied witness. -3. Follow every corresponding leaf in `Depends`. +3. Follow every corresponding leaf in `Dependencies`. 4. If the value was inherited through `...`, continue tracing in the ancestor - state. + result. 5. Repeat recursively until reaching literals, signed inputs, codec inputs, or - externally attested transition claims. + externally supplied claims. -For example, a process state may claim: +For example, a process result may assert: ```text ProcessStateN.balance.OUR_ADDRESS = 10 @@ -462,7 +458,7 @@ The trace may show that this came from: ```text ProcessStateN-1 / TransferRequest - > VariedBase + VariedRequest @ Depends + > VariedBase + VariedRequest @ Dependencies = ProcessStateN ``` @@ -474,38 +470,40 @@ VariedBase.balance.SENDER = 93 VariedRequest.quantity = 3 ``` -`Depends` then points to the hashpaths that produced `7`, `93`, and `3`. The +`Dependencies` then points to the hashpaths that produced `7`, `93`, and `3`. The quantity may trace to an inbound message from a swap process, whose sale-price transition may itself be attested by another node. -The trace is not a narrative. It is a recursive chain of AO-Core claims. +The trace is not a narrative. It is a recursive chain of AO-Core assertions. +Any signed assertion in the chain is a claim. ## HTTP Expression HTTP is an expression of AO-Core, not the foundation of AO-Core. -An HTTP request is decoded into an AO-Core request message. URL path segments, -query parameters, method, headers, and body are message members. Content -negotiation selects codecs for values and messages. The server resolves: +An HTTP request is decoded into an AO-Core request resource. URL path segments, +query parameters, method, headers, and body are associated resources. Content +negotiation selects codecs for values and resource collections. The server +resolves: ```text Base / Request -> Result ``` -and returns an HTTP response containing the encoded result plus enough hashpath -and commitment information for the receiver to verify, port, and continue the -computation. +and returns an HTTP response containing the encoded result, its result assertion, +supporting values, and any signed claims needed for the receiver to verify, port, +and continue the computation. Thus HTTP gives AO-Core a universal transport and user-facing syntax while the protocol remains independent of HTTP itself. -HTTP computations should append a terminal materialization claim by default: +HTTP computations should append a terminal materialization assertion by default: ```text HP/*=MaterializedID ``` -This claim is not special in the hashpath calculus. It is the ordinary +This assertion is not special in the hashpath calculus. It is the ordinary materialization request for `*`, included so that the transport response is tied to the concrete enumerated keys and values returned to the client. From b8a4e582808de916f43ede260376e661f88b673a Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 21 Jul 2026 18:59:14 -0400 Subject: [PATCH 33/35] wip: more spec edits --- AO-CORE.md | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/AO-CORE.md b/AO-CORE.md index 38a461c86..6305b5943 100644 --- a/AO-CORE.md +++ b/AO-CORE.md @@ -56,8 +56,8 @@ values (`BaseURI/RequestBinary`). If the request is instead of a composite form, we attempt to resolve `/path` element upon the second ('request') URI for the computation, and search the -primary ('base') with its value. In the event that the resource is resolved, its -value is returned. If the direct URI cannot be resolved, we attempt to resolve +primary ('base') with its value. In the event that the resource is found, its +value is returned. If the direct URI is not found, we look for a directly asserted `BaseURI/device`, yielding a specification -- or a binary, whose value can be resolved to a specification -- of a `device` (a virtual machine) to utilize for the computation. If a device is not found in the `BaseURI` we recurse backwards @@ -89,17 +89,17 @@ number of elements: 4. `Varied-Base` and `Varied-Request` emelements, each containing URIs for collections of _only_ the necessary components that were utilized in the state transition. -5. `Dependencies`, an optional collection hosting the hashpaths of each utilized - state component from the `[Varied-][Base|Request]` elements. -5. An `Attestation`, cryptographically linking an identity with each of the - constituent claims made in the hashpath context. - -A hashpath may convey any number of connected frames of execution attested by a -single commitment. Hashpath may be extended by Merklizing the ordered list of -contexts, such that only a single root is provided as the `Base` to the final -frame. Alternatively, any number of `Base`/`Request` pairs may be provided, -with the `Base` of each frame being the ancestry (ordered prior results) of the -`Request` at that layer. +5. `Dependencies`, a collection hosting the origin hashpath of each utilized + component from the `[Varied-][Base|Request]` elements. It may be omitted only + when empty or fully derivable. + +Claims are signatures over one or more complete hashpath assertions rather than +fields within their frames. A hashpath may convey any number of connected frames +of execution covered by a single claim. Hashpath may be extended by Merklizing +the ordered list of contexts, such that only a single root is provided as the +`Base` to the final frame. Alternatively, any number of `Base`/`Request` pairs may +be provided, with the `Base` of each frame being the ancestry (ordered prior +results) of the `Request` at that layer. ## Resolution Semantics @@ -115,14 +115,16 @@ non-extending hashpath context is found. At each layer: -1. Resolve `BaseURI/path`. Return if found. -2. Resolve `BaseURI/device`. -3. If `Device` is found, lookup runtime-compliant implementation and execute +1. Look for `BaseURI/P` among the resources asserted directly at this layer. + Return if found. +2. Look directly for `BaseURI/device` at this layer. +3. If `Device` is found, look up a runtime-compliant implementation and execute against the original outermost `Base` and the original `Request`. 4. Else, if a prior extending element of the hashpath context is found, recursively resolve at that element. -Pseudocode: +In the pseudocode below, `lookup` inspects direct assertions in the loaded layer. +It does not recursively perform AO execution. ```text resolve(Outer, BaseURI, Request): @@ -179,7 +181,10 @@ precedence over the base resource. ## Vary -Before execution, the selected device varies the transition: +Device application has two phases. First, the selected device prepares an +executable context containing the local function to invoke, the minimized +`Varied-Base` and `Varied-Request`, their `Dependencies`, and whether the result +extends or replaces the `Base`. The protocol-visible portion of preparation is: ```text Base / Request / vary -> VariedBase + VariedRequest @ Dependencies @@ -302,8 +307,10 @@ Base / Request / vary -> VariedBase + VariedRequest @ Dependencies VariedBase / VariedRequest -> Patch ``` -Extension is just `set`. There is no special update operation beyond -constructing an extension whose ancestry is the original base. +The device's preparation selects extension or replacement as the result mode. +AO-Core applies that mode: extension constructs a new layer whose ancestry is +the original `Base`, while replacement terminates active inheritance. Devices do +not implement ancestry traversal themselves. ## Hashpath Assertions And Claims @@ -409,7 +416,8 @@ of the same process. To challenge a transition assertion: 1. Verify that `BaseID` and `ReqID` identify the asserted values. -2. Verify the vary assertion: +2. Repeat the selected device's preparation phase and verify its asserted + `VariedBase`, `VariedRequest`, `Dependencies`, and result mode: ```text Base / Request / vary -> VariedBase + VariedRequest @ Dependencies @@ -417,7 +425,9 @@ To challenge a transition assertion: 3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf in `Dependencies` and verify that it yields that value. -4. Verify execution: +4. Invoke the prepared function and verify its result. An existing result may + substitute for execution only if it was previously verified under the + verifier's trust policy: ```text VariedBase / VariedRequest -> Patch From ba0aa3d3054d803e06b6ea2989d5a27c8741ba59 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 21 Jul 2026 19:21:36 -0400 Subject: [PATCH 34/35] wip: DRAFT-5 copy errors --- AO-CORE.md | 57 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/AO-CORE.md b/AO-CORE.md index 6305b5943..2958c290b 100644 --- a/AO-CORE.md +++ b/AO-CORE.md @@ -1,7 +1,7 @@ # AO 1.0: A universal protocol for computation over URIs. AO is a language for describing a method of combining -- computing over -- URI-named -resourced, yielding further URI-addressable values. AO forms a computing environment, +resources, yielding further URI-addressable values. AO forms a computing environment, but it is not itself a virtual machine. It is a neutral meta-VM: a common language for describing, transporting, attesting, challenging, and tracing the computations performed by many VMs, called devices. @@ -30,10 +30,10 @@ forms. Their values are produced by resolvers that yield a representation of the bytes that the named resource refers to. Resources may identify other values by protocol address; creating a graph of connected computed elements. -AO runtime implementations use a system of caches that associated resources with +AO runtime implementations use a system of caches that associate resources with their associated values, then make these available for computations to access. When a URI is not known in a node's existing stores, it may be requested for -resolution using the host environments native mechanisms, or through custom +resolution using the host environment's native mechanisms, or through custom providers built-in to the runtime. Resources may share a common prefix, resembling a directory or other composite @@ -54,8 +54,8 @@ The simplest form of a hashpath is resolving a `data:` `request` value against a resource collection, which yields the standard resolution of the concatenated values (`BaseURI/RequestBinary`). -If the request is instead of a composite form, we attempt to resolve `/path` -element upon the second ('request') URI for the computation, and search the +If the request is a composite resource, we attempt to resolve the `/path` +element upon the second ('request') URI for the computation, and search the primary ('base') with its value. In the event that the resource is found, its value is returned. If the direct URI is not found, we look for a directly asserted `BaseURI/device`, yielding a specification -- or a binary, whose value can be @@ -79,14 +79,14 @@ provenance. Hashpaths represent a sequence of execution frames, each of which may contain a number of elements: 1. A `Base` URI upon which the request is being made. Must always be present on - the first context of the hashpath, but may be ommitted for the second + the first context of the hashpath, but may be omitted for the second and further elements, instead having their `Base` inferred from the rolling context of the hashpath itself. 2. A raw binary or `Request` URI, holding the metadata of the request upon the `Base`. 3. A `Result`, optionally, if computed, the result of the computation as a further URI. -4. `Varied-Base` and `Varied-Request` emelements, each containing URIs for +4. `Varied-Base` and `Varied-Request` elements, each containing URIs for collections of _only_ the necessary components that were utilized in the state transition. 5. `Dependencies`, a collection hosting the origin hashpath of each utilized @@ -118,7 +118,7 @@ At each layer: 1. Look for `BaseURI/P` among the resources asserted directly at this layer. Return if found. 2. Look directly for `BaseURI/device` at this layer. -3. If `Device` is found, look up a runtime-compliant implementation and execute +3. If `Device` is found, look up a runtime-compliant implementation and apply it against the original outermost `Base` and the original `Request`. 4. Else, if a prior extending element of the hashpath context is found, recursively resolve at that element. @@ -160,7 +160,7 @@ selects `dev1` from the ancestor, but `dev1` sees `x = 5`. A device defines how requests are computed for a state. Examples of devices include message interpreters, process VMs, WASM VMs, -Lua VMs, codecs, stores, payment devices, and application-specific evaluators. +Lua VMs, codecs, payment devices, and application-specific evaluators. AO-Core does not privilege one compute model over another. It standardizes how their transitions are named, witnessed, attested, transported, and traced. @@ -170,7 +170,7 @@ their transitions are named, witnessed, attested, transported, and traced. full specification when resolved. 2. **Binaries, resolved as Permaweb Names**: Binary literals that imply a request to resolve the name against `ao://~name@1.0/[Name]` in the resolver. -3. **`ao://` Resource Prefixes**: Recursively resolved devices whos functionality +3. **`ao://` Resource Prefixes**: Recursively resolved devices whose functionality is defined by extending the `Base` resource with its values and calling the resolution upon its new form. @@ -258,7 +258,8 @@ Everything observed by execution must be present in `VariedBase` or ## Shared Computation -Varying creates a reusable computation point. +The second phase of device application begins at the reusable computation point +created by varying. Many concrete bases may vary to the same pair: @@ -270,18 +271,24 @@ BaseB / Request / vary -> VariedBase + VariedRequest @ DependenciesB The execution: ```text -VariedBase / VariedRequest -> Patch +VariedBase / VariedRequest -> VariedResult ``` -is shared by all sufficiently alike concrete bases for that request. The final -results may still differ because the patch is equivalent to applying the original -transition to each original base: +is shared by all sufficiently alike concrete bases for that request. AO-Core +first looks for a result at this address. On a miss, the kernel invokes the +prepared function and caches its result; on a hit, it reuses the result without +invoking the function. + +For extension, `VariedResult` is a patch. Final results may still differ because +the patch is applied to each original base: ```text BaseA / Request == set(BaseA, Patch) BaseB / Request == set(BaseB, Patch) ``` +For replacement, `VariedResult` is the final result and is shared directly. + This is the default mode of AO-Core computation: do a computation once for the material inputs that matter, then reuse it across every base/request pair that varies to those inputs. @@ -292,25 +299,27 @@ reused. ## Transition Equivalence -A transition asserts an equivalence between resolving a request and extending -the base with the patch produced by varied execution: +A transition asserts an equivalence between resolving a request and applying the +selected result mode to the varied result: ```text -Base / Request - == set(Base, Patch) +Base / Request == + extension: set(Base, VariedResult) + replacement: VariedResult ``` where: ```text Base / Request / vary -> VariedBase + VariedRequest @ Dependencies -VariedBase / VariedRequest -> Patch +VariedBase / VariedRequest -> VariedResult ``` -The device's preparation selects extension or replacement as the result mode. -AO-Core applies that mode: extension constructs a new layer whose ancestry is -the original `Base`, while replacement terminates active inheritance. Devices do -not implement ancestry traversal themselves. +The device's preparation selects extension or replacement as `ResultMode`. +Extension applies `set(Base, VariedResult)`, constructing a new layer whose +ancestry is the original `Base`. Replacement returns `VariedResult` and terminates +active inheritance. AO-Core applies the mode; devices do not implement ancestry +traversal themselves. ## Hashpath Assertions And Claims From e2115697c2cdbb06f1e53343df404e951d5e95f0 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 21 Jul 2026 20:15:36 -0400 Subject: [PATCH 35/35] wip: DRAFT-5 `no-content` | `not-found` semantics --- AO-CORE.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/AO-CORE.md b/AO-CORE.md index 2958c290b..7ec69c69d 100644 --- a/AO-CORE.md +++ b/AO-CORE.md @@ -124,26 +124,26 @@ At each layer: resolve at that element. In the pseudocode below, `lookup` inspects direct assertions in the loaded layer. -It does not recursively perform AO execution. +It does not recursively perform AO execution. A `not_found` error is scoped to +the resource searched. ```text resolve(Outer, BaseURI, Request): P = path(Request) case lookup(BaseURI/P) of - Value -> return Value - unset -> return not_found - not_found -> continue + {ok, Response} -> return {ok, Response} + {error, not_found} -> continue end case lookup(BaseURI/device) of - Dev -> return execute(Dev, Outer, Request) - not_found -> continue + {ok, Dev} -> return execute(Dev, Outer, Request) + {error, not_found} -> continue end case local(BaseURI/...) of - Ancestor -> return resolve(Outer, Ancestor, Request) - not_found -> return not_found + {ok, Ancestor} -> return resolve(Outer, Ancestor, Request) + {error, not_found} -> return {error, not_found} end ```