diff --git a/AO-CORE.md b/AO-CORE.md new file mode 100644 index 000000000..7ec69c69d --- /dev/null +++ b/AO-CORE.md @@ -0,0 +1,549 @@ +# AO 1.0: A universal protocol for computation over URIs. + +AO is a language for describing a method of combining -- computing over -- URI-named +resources, yielding further URI-addressable values. AO forms a computing environment, +but it is not itself a virtual machine. It is a neutral meta-VM: a common +language for describing, transporting, attesting, challenging, and tracing the +computations performed by many VMs, called devices. + +AO offers three core properties: + +1. **Atomic attestation**: any computation claim can be verified or challenged + independently from all other history. +2. **Succinct portability**: any computation may be continued on any other + participating machine by invoking its accrued hashpath upon the recipient. + Each element of the state of the computation may be moved eagerly or recalled + from untrusted peers on-demand. +3. **Traceability**: every computation result can be traced through every state + transformation to the origins of the values on which it depends. + +Through these properties in a decentralized protocol, AO-Core intends to give +rise to a globally distributed supercomputer presenting a single system image +across a machine of arbitrary size. + +## Resources, Collections, and Values. + +A resource is any protocol-addressable content, referenced by URIs ([RFC 3986]). + +As stated in the specification, URIs may come in hierarchical and non-hierarchical +forms. Their values are produced by resolvers that yield a representation of the +bytes that the named resource refers to. Resources may identify other values by +protocol address; creating a graph of connected computed elements. + +AO runtime implementations use a system of caches that associate resources with +their associated values, then make these available for computations to access. +When a URI is not known in a node's existing stores, it may be requested for +resolution using the host environment's native mechanisms, or through custom +providers built-in to the runtime. + +Resources may share a common prefix, resembling a directory or other composite +structure via their path-parts. AO resolutions may both consume and produce such +collections of associated URIs by referring to their shared prefix in either +inputs or outputs to computations. + +## Hashpaths and Execution + +In addition to consuming URIs as inputs the AO protocol defines the `ao://` +URI-scheme, which allows its universal computation language to name state +transitions, attest to their dependencies, and their results. We call the +namespace of URIs under this scheme collectively `hashpaths`. + +Hashpaths refer to a merklized log of computation contexts, each of which +contains at minimum a `Base` URI and a representation of the request. +The simplest form of a hashpath is resolving a `data:` `request` value against +a resource collection, which yields the standard resolution of the concatenated +values (`BaseURI/RequestBinary`). + +If the request is a composite resource, we attempt to resolve the `/path` +element upon the second ('request') URI for the computation, and search the +primary ('base') with its value. In the event that the resource is found, its +value is returned. If the direct URI is not found, we look for a directly asserted +`BaseURI/device`, yielding a specification -- or a binary, whose value can be +resolved to a specification -- of a `device` (a virtual machine) to utilize for +the computation. If a device is not found in the `BaseURI` we recurse backwards +through prior computed results of the hashpath, repeating the process until +either a direct key or `device` is found, or the hashpath reaches a 'replacement' +element (implying that the computation's result does not 'extend' the elements +found in the hashpath prior to it, but instead nullifies each). Through this +recursive mechanism, new messages can be composed via the combination of prior +message IDs, and values in a present state may lazily include those from prior +states -- allowing their history (when each was set, what the inputs to that +computation were, etc) to be traced individually. + +Each new Extension does not mutate the `Base`. Instead, it constructs a new value +that shares elements from the `Base`, overlayed with additional resolvable +resources and prefixed names. This grants the basis of message deduplication: +large states can be represented as small patches over existing states, maintaining +provenance. + +Hashpaths represent a sequence of execution frames, each of which may contain a +number of elements: +1. A `Base` URI upon which the request is being made. Must always be present on + the first context of the hashpath, but may be omitted for the second + and further elements, instead having their `Base` inferred from the rolling + context of the hashpath itself. +2. A raw binary or `Request` URI, holding the metadata of the request upon the + `Base`. +3. A `Result`, optionally, if computed, the result of the computation as a + further URI. +4. `Varied-Base` and `Varied-Request` elements, each containing URIs for + collections of _only_ the necessary components that were utilized in the + state transition. +5. `Dependencies`, a collection hosting the origin hashpath of each utilized + component from the `[Varied-][Base|Request]` elements. It may be omitted only + when empty or fully derivable. + +Claims are signatures over one or more complete hashpath assertions rather than +fields within their frames. A hashpath may convey any number of connected frames +of execution covered by a single claim. Hashpath may be extended by Merklizing +the ordered list of contexts, such that only a single root is provided as the +`Base` to the final frame. Alternatively, any number of `Base`/`Request` pairs may +be provided, with the `Base` of each frame being the ancestry (ordered prior +results) of the `Request` at that layer. + +## Resolution Semantics + +Concretely, the primitive AO relation is: + +```text +BaseURI / RequestURI | Value -> ResultURI | Value +``` + +For a request whose `path` is `P`, resolution walks the ancestry of `Base` from +the outermost layer inward until either a direct result, a `device`, or a +non-extending hashpath context is found. + +At each layer: + +1. Look for `BaseURI/P` among the resources asserted directly at this layer. + Return if found. +2. Look directly for `BaseURI/device` at this layer. +3. If `Device` is found, look up a runtime-compliant implementation and apply it + against the original outermost `Base` and the original `Request`. +4. Else, if a prior extending element of the hashpath context is found, recursively + resolve at that element. + +In the pseudocode below, `lookup` inspects direct assertions in the loaded layer. +It does not recursively perform AO execution. A `not_found` error is scoped to +the resource searched. + +```text +resolve(Outer, BaseURI, Request): + P = path(Request) + + case lookup(BaseURI/P) of + {ok, Response} -> return {ok, Response} + {error, not_found} -> continue + end + + case lookup(BaseURI/device) of + {ok, Dev} -> return execute(Dev, Outer, Request) + {error, not_found} -> continue + end + + case local(BaseURI/...) of + {ok, Ancestor} -> return resolve(Outer, Ancestor, Request) + {error, not_found} -> return {error, not_found} + end +``` + +If a device is inherited from an ancestor, it executes over the outermost state: + +```text +{ x: 5, ...: { device: dev1, x: 1 } } / x-is-5 +``` + +selects `dev1` from the ancestor, but `dev1` sees `x = 5`. + +## Devices + +A device defines how requests are computed for a state. + +Examples of devices include message interpreters, process VMs, WASM VMs, +Lua VMs, codecs, payment devices, and application-specific evaluators. +AO-Core does not privilege one compute model over another. It standardizes how +their transitions are named, witnessed, attested, transported, and traced. + +`Device` references may be stated in three forms: + +1. **Specification URIs**: fully-qualified URIs that yield the intended device's + full specification when resolved. +2. **Binaries, resolved as Permaweb Names**: Binary literals that imply a + request to resolve the name against `ao://~name@1.0/[Name]` in the resolver. +3. **`ao://` Resource Prefixes**: Recursively resolved devices whose functionality + is defined by extending the `Base` resource with its values and calling the + resolution upon its new form. + +The third form allows for recursive resolution of devices, in which a new device +is constructed using the same underlying virtual machine implementations inside +the AO runtime, but with the values found in the extension resource taking +precedence over the base resource. + +## Vary + +Device application has two phases. First, the selected device prepares an +executable context containing the local function to invoke, the minimized +`Varied-Base` and `Varied-Request`, their `Dependencies`, and whether the result +extends or replaces the `Base`. The protocol-visible portion of preparation is: + +```text +Base / Request / vary -> VariedBase + VariedRequest @ Dependencies +``` + +`VariedBase` and `VariedRequest` are resource collections containing exactly the +values required by the execution. They use canonical nested structure, not path +strings: + +```text +VariedBase = { + device: process@1.0, + balance: { + OUR_ADDRESS: 7, + SENDER: 93 + } +} + +VariedRequest = { + path: transfer, + from: SENDER, + to: OUR_ADDRESS, + quantity: 3 +} +``` + +`Dependencies` records where each varied value originated. It has the same shape +as the varied collections, rooted under `base` and `request`. Each leaf is a +hashpath whose terminal value is the corresponding varied value: + +```text +Dependencies = { + base: { + device: HP_for_process_device, + balance: { + OUR_ADDRESS: HP_for_prior_our_balance, + SENDER: HP_for_prior_sender_balance + } + }, + request: { + path: HP_for_request_path, + from: HP_for_request_from, + to: HP_for_request_to, + quantity: HP_for_quantity + } +} +``` + +The dependency leaf is a single value: the origin hashpath. The value itself +does not need to be duplicated inside `Dependencies`, because the hashpath binds the +origin to the value it yields. + +If no exact vary specification is available, the conservative valid vary is +identity: + +```text +VariedBase = Base +VariedRequest = Request +``` + +Identity variation does not waive dependency recording. + +The core rule is: + +```text +No hidden inputs. +``` + +Everything observed by execution must be present in `VariedBase` or +`VariedRequest`, and every varied value must have an origin in `Dependencies`. + +## Shared Computation + +The second phase of device application begins at the reusable computation point +created by varying. + +Many concrete bases may vary to the same pair: + +```text +BaseA / Request / vary -> VariedBase + VariedRequest @ DependenciesA +BaseB / Request / vary -> VariedBase + VariedRequest @ DependenciesB +``` + +The execution: + +```text +VariedBase / VariedRequest -> VariedResult +``` + +is shared by all sufficiently alike concrete bases for that request. AO-Core +first looks for a result at this address. On a miss, the kernel invokes the +prepared function and caches its result; on a hit, it reuses the result without +invoking the function. + +For extension, `VariedResult` is a patch. Final results may still differ because +the patch is applied to each original base: + +```text +BaseA / Request == set(BaseA, Patch) +BaseB / Request == set(BaseB, Patch) +``` + +For replacement, `VariedResult` is the final result and is shared directly. + +This is the default mode of AO-Core computation: do a computation once for the +material inputs that matter, then reuse it across every base/request pair that +varies to those inputs. + +Both the varied computation and complete transition results are cacheable by +address, allowing prior computations from different execution traces to be +reused. + +## Transition Equivalence + +A transition asserts an equivalence between resolving a request and applying the +selected result mode to the varied result: + +```text +Base / Request == + extension: set(Base, VariedResult) + replacement: VariedResult +``` + +where: + +```text +Base / Request / vary -> VariedBase + VariedRequest @ Dependencies +VariedBase / VariedRequest -> VariedResult +``` + +The device's preparation selects extension or replacement as `ResultMode`. +Extension applies `set(Base, VariedResult)`, constructing a new layer whose +ancestry is the original `Base`. Replacement returns `VariedResult` and terminates +active inheritance. AO-Core applies the mode; devices do not implement ancestry +traversal themselves. + +## Hashpath Assertions And Claims + +A hashpath without a terminal result is an address naming a computation. A +hashpath containing its result is an assertion. A signature over one or more +hashpaths is a claim. Like any other value, a hashpath may be encountered as a +link and resolved through AO-Core, or encountered as serialized content and +decoded into its in-memory form. + +The full transition forms are: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID=PatchID +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID.ResultID +``` + +`=` means the execution produced a patch that extends the prior result: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID=PatchID +``` + +is equivalent to: + +```text +set(Base, Patch) +``` + +`.` means the execution produced a replacement value: + +```text +BaseID/ReqID>VariedBaseID+VariedReqID@DependenciesID.ResultID +``` + +The accumulated result becomes `ResultID`, dropping the prior result's resources +rather than extending them. + +A compact form may omit fields when they are derivable or supplied elsewhere, +but the full assertion must be recoverable for challenge and trace. + +A hashpath is a sequence of transition assertions. Later segments operate on the +result established by earlier segments. + +Segments without explicit vary syntax are not special. They are compact +transition assertions. For example: + +```text +HP/*=FinalResultID +``` + +is simply an assertion that resolving `*` at `HP` yields `FinalResultID`. HTTP +gateways commonly append such a segment so the response body is tied to the +specific keys and values returned to the client. + +## Hashpath Loading And Portability + +Hashpaths are not a storage system outside AO-Core. They are addressable values +whose loaded form reconstructs resource-prefix and extension semantics. + +Reusable patches can be stored by their generic IDs, without caller-specific +ancestry. The hashpath records how that generic value is reached from a prior +result. For an extension segment: + +```text +PriorHP/Req>VariedBase+VariedReq@Dependencies=Patch +``` + +Stores may cache the asserted result at that hashpath. Loading the segment loads +`Patch` and presents it as an extension whose `...` is the previous accumulated +result: + +```text +load(PriorHP/Req>VB+VR@Dependencies=Patch) + -> { PatchKeys..., ...: PriorHP } +``` + +The `...` value is itself a hashpath. Loading it reconstructs the prior result +for inherited-key resolution. The complete result remains addressed by the full +hashpath, which retains the request, varied witnesses, and dependencies needed +to challenge the transition. + +For replacement segments: + +```text +PriorHP/Req>VB+VR@Dependencies.Result +``` + +loading the segment yields `Result`. The prior result is not inherited as active +keys, but the hashpath still carries the transition context needed for +challenge and trace. + +This gives portability as a single addressable value: posting a hashpath plus +the values needed to resolve the IDs it names gives another node enough data to +reconstruct the result collection, challenge any segment, and continue computation. + +## Challenge And Audit + +A transition can be challenged locally, without verifying its entire dependency +tree. The usual practical operation is to pick one assertion and verify only the +values needed to reproduce it. A full provenance audit is the recursive version +of the same process. + +To challenge a transition assertion: + +1. Verify that `BaseID` and `ReqID` identify the asserted values. +2. Repeat the selected device's preparation phase and verify its asserted + `VariedBase`, `VariedRequest`, `Dependencies`, and result mode: + + ```text + Base / Request / vary -> VariedBase + VariedRequest @ Dependencies + ``` + +3. For every leaf in `VariedBase` and `VariedRequest`, follow the matching leaf + in `Dependencies` and verify that it yields that value. +4. Invoke the prepared function and verify its result. An existing result may + substitute for execution only if it was previously verified under the + verifier's trust policy: + + ```text + VariedBase / VariedRequest -> Patch + ``` + +5. Verify transition equivalence: + + ```text + "=" means the accumulated result is set(Base, Patch) + "." means the accumulated result is Result + ``` + +Any one of these checks can be challenged independently. To audit the full +provenance tree, recursively challenge the dependency hashpaths named in +`Dependencies`. + +## Traceability + +To trace a value in a result: + +1. Locate the transition that produced the result collection containing the + value. +2. If the value was introduced by the patch, trace it to that transition's + varied witness. +3. Follow every corresponding leaf in `Dependencies`. +4. If the value was inherited through `...`, continue tracing in the ancestor + result. +5. Repeat recursively until reaching literals, signed inputs, codec inputs, or + externally supplied claims. + +For example, a process result may assert: + +```text +ProcessStateN.balance.OUR_ADDRESS = 10 +``` + +The trace may show that this came from: + +```text +ProcessStateN-1 / TransferRequest + > VariedBase + VariedRequest @ Dependencies + = ProcessStateN +``` + +with: + +```text +VariedBase.balance.OUR_ADDRESS = 7 +VariedBase.balance.SENDER = 93 +VariedRequest.quantity = 3 +``` + +`Dependencies` then points to the hashpaths that produced `7`, `93`, and `3`. The +quantity may trace to an inbound message from a swap process, whose sale-price +transition may itself be attested by another node. + +The trace is not a narrative. It is a recursive chain of AO-Core assertions. +Any signed assertion in the chain is a claim. + +## HTTP Expression + +HTTP is an expression of AO-Core, not the foundation of AO-Core. + +An HTTP request is decoded into an AO-Core request resource. URL path segments, +query parameters, method, headers, and body are associated resources. Content +negotiation selects codecs for values and resource collections. The server +resolves: + +```text +Base / Request -> Result +``` + +and returns an HTTP response containing the encoded result, its result assertion, +supporting values, and any signed claims needed for the receiver to verify, port, +and continue the computation. + +Thus HTTP gives AO-Core a universal transport and user-facing syntax while the +protocol remains independent of HTTP itself. + +HTTP computations should append a terminal materialization assertion by default: + +```text +HP/*=MaterializedID +``` + +This assertion is not special in the hashpath calculus. It is the ordinary +materialization request for `*`, included so that the transport response is tied +to the concrete enumerated keys and values returned to the client. + +The default HTTPSig response carries two independently useful commitments: + +1. A commitment over the enumerated response keys and values, excluding the + `...` hashpath field. This commitment is typically unsigned unless the + response is already a pre-existing signed message. Its ID is + `MaterializedID`. +2. A signature over just the excluded `...` field, whose value is the hashpath + terminating in `=MaterializedID`. + +The first commitment lets the returned value stand alone by ID. The second lets +the recipient port or challenge the computation through the hashpath. They can +be used together or independently. + +## Summary + +AO-Core turns computation into portable, challengeable, traceable message +transitions. + +Devices provide the compute. Vary provides exact witnesses. Depends provides +origin trace. Hashpaths provide portable claims. Extension provides deduplicated +state. HTTP provides a practical expression layer. diff --git a/erlang_ls.config b/erlang_ls.config index f5621bee0..781836ef9 100644 --- a/erlang_ls.config +++ b/erlang_ls.config @@ -6,8 +6,6 @@ diagnostics: apps_dirs: - "src" - "src/*" -include_dirs: - - "src/include" include_dirs: - "src" - "src/include" diff --git a/src/core/device/hb_device.erl b/src/core/device/hb_device.erl index df9ea5aa9..2a87f7e71 100644 --- a/src/core/device/hb_device.erl +++ b/src/core/device/hb_device.erl @@ -2,7 +2,8 @@ %%% Offers services for loading, verifying executability, and extracting Erlang %%% functions from a device. -module(hb_device). --export([truncate_args/2, message_to_fun/3, message_to_device/2]). +-export([truncate_args/2, add_resolver/2, message_to_fun/3, message_to_fun/4, module/2]). +-export([id_or_direct_key/3]). -export([is_direct_key_access/3, is_direct_key_access/4]). -export([find_exported_function/5, is_exported/4, info/2, info/3]). -include("include/hb.hrl"). @@ -29,6 +30,52 @@ truncate_args(Fun, Args) -> {arity, Arity} = erlang:fun_info(Fun, arity), lists:sublist(Args, Arity). +%% @doc Generates an execution context for a given key and device pair. +%% Returns `{ok, ExecCtx}`, where `ExecCtx` is a message containing: +%% #{ +%% base: The base message, unvaried. +%% key: The key to be resolved in the execution. +%% forced-device: The device derived from the message itself. +%% priv/exec-device: The device from which the execution function originates. +%% If the `forced-device` resolvers from another device, +%% this key will differ from the `forced-device`. +%% priv/function: The function to execute to resolve the `forced-device`. +%% priv/add-key: Whether the execution function expects that we should +%% add the `key` as an additional argument to the start of +%% the argument list. +%% } +add_resolver(Context = #{ <<"base">> := Base, <<"key">> := Key }, Opts) -> + % TODO: What if we force a device _and_ have a direct key hit? + DevRes = + case maps:find(<<"forced-device">>, Context) of + {ok, ForcedBaseDevice} -> {ok, ForcedBaseDevice}; + error -> id_or_direct_key(Base, Key, Opts) + end, + OldPriv = maps:get(<<"priv">>, Context, #{}), + case DevRes of + {hit, Value} -> + {ok, + Context#{ + <<"result">> => Value + } + }; + {ok, DeviceID} -> + {Type, ExecDev, ExecMod, Fun} = + message_to_fun(DeviceID, Base, Key, Opts), + {ok, + Context#{ + <<"forced-device">> => DeviceID, + <<"resolver-device">> => ExecDev, + <<"priv">> => + OldPriv#{ + <<"resolver-module">> => ExecMod, + <<"add-key">> => Type == add_key, + <<"function">> => Fun + } + } + } + end. + %% @doc Calculate the Erlang function that should be called to get a value for %% a given key from a device. %% @@ -54,16 +101,33 @@ truncate_args(Fun, Args) -> %% Returns {ok | add_key, Fun} where Fun is the function to call, and add_key %% indicates that the key should be added to the start of the call's arguments. message_to_fun(Msg, Key, Opts) -> - % Get the device module from the message and recurse. - message_to_fun(message_to_device(Msg, Opts), Msg, Key, Opts). -message_to_fun(Dev, Msg, Key, Opts) -> - Info = info(Dev, Msg, Opts), + DeviceID = id_or_direct_key(Msg, Key, Opts), + message_to_fun(DeviceID, Msg, Key, Opts). +message_to_fun(DevID, Msg, Key, Opts) -> + message_to_fun(DevID, module(DevID, Opts), Msg, Key, Opts). +message_to_fun(DevID, DevMsg = #{ <<"device">> := _ }, _Msg, Key, _Opts) -> + % A message-valued device: execution is itself resolution. The request is + % resolved against the device message extended over the outermost state: + % `execute(Dev, Outer, Request) = resolve(set(Outer, Dev), Request)'. + % The key is forced upon the inner resolution: for forced-key executions + % (`vary', `set', ...) the request's own path differs from the key that + % this function was extracted to compute. + {ok, + DevID, + DevMsg, + fun(Base, Req, ExecOpts) -> + hb_ao:raw(undefined, Key, DevMsg#{ <<"...">> => Base }, Req, ExecOpts) + end + }; +message_to_fun(DevID, DevMod, Msg, Key, Opts) -> + Info = info(DevMod, Msg, Opts), % Is the key exported by the device? Exported = is_exported(Info, Key, Opts), ?event( ao_devices, {message_to_fun, - {dev, Dev}, + {device, DevID}, + {module, DevMod}, {key, Key}, {is_exported, Exported}, {opts, Opts} @@ -76,22 +140,21 @@ message_to_fun(Dev, Msg, Key, Opts) -> % Case 2: The device has an explicit handler function. ?event( ao_devices, - {handler_found, {dev, Dev}, {key, Key}, {handler, Handler}} + {handler_found, {dev, DevID}, {key, Key}, {handler, Handler}} ), - {Status, Func} = info_handler_to_fun(Handler, Msg, Key, Opts), - {Status, Dev, Func}; + info_handler_to_fun(Handler, DevID, DevMod, Msg, Key, Opts); _ -> - ?event_debug(ao_devices, {no_override_handler, {dev, Dev}, {key, Key}}), - case {find_exported_function(Msg, Dev, Key, 3, 1, Opts), Exported} of + ?event_debug(ao_devices, {no_override_handler, {dev, DevID}, {key, Key}}), + case {find_exported_function(Msg, DevMod, Key, 3, 1, Opts), Exported} of {{ok, Func}, true} -> % Case 3: The device has a function of the name `Key'. - {ok, Dev, Func}; + {ok, DevID, DevMod, Func}; _ -> case {hb_maps:find(default, Info, Opts), Exported} of {{ok, DefaultFunc}, true} when is_function(DefaultFunc) -> % Case 4: The device has a default handler. ?event_debug({found_default_handler, {func, DefaultFunc}}), - {add_key, Dev, DefaultFunc}; + {add_key, DevID, DevMod, DefaultFunc}; {{ok, DefaultDevice}, true} when is_binary(DefaultDevice) orelse is_atom(DefaultDevice) -> % Case 5: The device gives a specific further device @@ -99,7 +162,8 @@ message_to_fun(Dev, Msg, Key, Opts) -> % rules. ?event_debug({found_default_device, {mod, DefaultDevice}}), message_to_fun( - Msg#{ <<"device">> => DefaultDevice }, + DefaultDevice, + hb_ao:as(DefaultDevice, Msg, Opts), Key, Opts ); @@ -107,17 +171,17 @@ message_to_fun(Dev, Msg, Key, Opts) -> % Case 6: The device has no default handler. % We retry with the default unless the message % already names it (loop guard). - case hb_maps:get(<<"device">>, Msg, undefined, Opts) of - ?DEFAULT_DEVICE -> - throw({ - error, - default_device_could_not_resolve_key, - {key, Key} - }); + case DevID of + ?DEFAULT_DEVICE -> + throw({ + error, + default_device_could_not_resolve_key, + {key, Key} + }); _ -> - ?event_debug({using_default_device, ?DEFAULT_DEVICE}), message_to_fun( - Msg#{ <<"device">> => ?DEFAULT_DEVICE }, + ?DEFAULT_DEVICE, + hb_ao:as(?DEFAULT_DEVICE, Msg, Opts), Key, Opts ) @@ -130,31 +194,60 @@ message_to_fun(Dev, Msg, Key, Opts) -> %% message has no `<<"device">>' key, we resolve the default %% (`message@1.0') just like any other device: There is no privileged %% internal module-loading path. -message_to_device(Msg, Opts) -> - DevID = hb_maps:get(<<"device">>, Msg, ?DEFAULT_DEVICE, Opts), +module(DevMod, _Opts) when is_atom(DevMod) -> DevMod; +module(Dev, _Opts) when is_map(Dev) -> Dev; +module(DevID, Opts) -> case hb_device_load:reference(DevID, Opts) of {error, Reason} -> throw({error, {device_not_loadable, DevID, Reason}}); {ok, DevMod} -> DevMod end. +%% @doc Return the device ID from a message, resolving through any ancestors +%% as necessary. +id_or_direct_key(_, <<"priv", _/binary>>, _) -> + {error, <<"`priv*` keys not resolvable.">>}; +id_or_direct_key(Msg, Key, Opts) -> + do_id_or_direct_key(Msg, Key, Opts). +do_id_or_direct_key(List, _, _) when is_list(List) -> + {ok, <<"message@1.0">>}; +do_id_or_direct_key(Msg, Key, Opts) -> + ?event({finding_device_id, Msg}), + case hb_maps:find(Key, Msg, Opts) of + {ok, <<"unset">>} -> {error, not_found}; + {ok, Value} -> {hit, Value}; + error -> + case hb_maps:find(<<"device">>, Msg, Opts) of + {ok, Device} -> {ok, Device}; + error -> + case hb_maps:find(<<"...">>, Msg, Opts) of + {ok, Ancestor} -> do_id_or_direct_key(Ancestor, Key, Opts); + error -> {ok, ?DEFAULT_DEVICE} + end + end + end. + %% @doc Parse a handler key given by a device's `info'. -info_handler_to_fun(Handler, _Msg, _Key, _Opts) when is_function(Handler) -> - {add_key, Handler}; -info_handler_to_fun(HandlerMap, Msg, Key, Opts) -> +info_handler_to_fun(Handler, DevID, DevMod, _Msg, _Key, _Opts) + when is_function(Handler) -> + {add_key, DevID, DevMod, Handler}; +info_handler_to_fun(HandlerMap, DevID, DevMod, Msg, Key, Opts) -> case hb_maps:find(excludes, HandlerMap, Opts) of {ok, Exclude} -> case lists:member(Key, Exclude) of - true -> - MsgWithoutDevice = - hb_maps:without([<<"device">>], Msg, Opts), + true -> + MsgWithoutDevice = + hb_maps:without([<<"device">>], Msg, Opts), message_to_fun( - MsgWithoutDevice#{ <<"device">> => ?DEFAULT_DEVICE }, + ?DEFAULT_DEVICE, + hb_ao:as(?DEFAULT_DEVICE, MsgWithoutDevice, Opts), Key, Opts ); - false -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + false -> + {add_key, DevID, DevMod, hb_maps:get(func, HandlerMap, undefined, Opts)} end; - error -> {add_key, hb_maps:get(func, HandlerMap, undefined, Opts)} + error -> + {add_key, DevID, DevMod, hb_maps:get(func, HandlerMap, undefined, Opts)} end. %% @doc Find the function with the highest arity that has the given name, if it @@ -240,7 +333,14 @@ maybe_normalize_device_key(Key, Mode) -> %% @doc Get the info map for a device, optionally giving it a message if the %% device's info function is parameterized by one. -info(Msg, Opts) -> info(message_to_device(Msg, Opts), Msg, Opts). +info(Msg, Opts) -> + Device = + case id_or_direct_key(Msg, <<"device">>, Opts) of + {hit, Dev} -> Dev; + {ok, Dev} -> Dev; + {error, _} -> ?DEFAULT_DEVICE % `device => unset' masks: default. + end, + info(module(Device, Opts), Msg, Opts). info(DevMod, Msg, Opts) -> case find_exported_function(Msg, DevMod, info, 2, Opts) of {ok, Fun} -> apply(Fun, truncate_args(Fun, [Msg, Opts])); diff --git a/src/core/device/hb_device_load.erl b/src/core/device/hb_device_load.erl index 8d84cc6a4..3f36deb38 100644 --- a/src/core/device/hb_device_load.erl +++ b/src/core/device/hb_device_load.erl @@ -23,13 +23,15 @@ %%% signature. %%% -module(hb_device_load). --export([reference/2]). +-export([reference/2, schema/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc A message is already a device. A binary reference is resolved, -%% then memoised in the process cache unless it is a forge seed. +%% @doc Resolve a device referece to an executable device representation +%% (either a module atom or a binary reference). reference(Loaded, _Opts) when is_map(Loaded) -> + % A message is already a viable device. A binary reference is resolved, + % then memoised in the process cache unless it is a forge seed. {ok, Loaded}; reference(Ref, Opts) when is_binary(Ref) -> NormRef = hb_ao:normalize_key(Ref), @@ -70,7 +72,7 @@ get_resolved_device(Ref, Opts) -> {ok, Bin} ?= hb_store:read( loaded_device_store(Opts), - store_key(Ref), + module_cache_key(Ref), Opts ), Mod = hb_util:atom(Bin), @@ -87,13 +89,15 @@ put_resolved_device(Ref, Mod, Opts) -> erlang:put({?MODULE, Ref}, Mod), hb_store:write( loaded_device_store(Opts), - #{ store_key(Ref) => hb_util:bin(Mod) }, + #{ module_cache_key(Ref) => hb_util:bin(Mod) }, Opts ). loaded_device_store(Opts) -> hb_opts:get(loaded_device_store, [], Opts). -store_key(Ref) -> <<"~meta@1.0/devices/", Ref/binary>>. +loaded_device_opts(Opts) -> Opts#{ <<"store">> => loaded_device_store(Opts) }. + +module_cache_key(Ref) -> <<"~meta@1.0/devices/modules/", Ref/binary>>. %%% -------------------------------------------------------------------- %%% High trust @@ -133,7 +137,10 @@ from_preloaded(Ref, Opts) -> {error, not_found}; Store -> PreOpts = - Opts#{ <<"store">> => [Store], <<"cache-read-mode">> => raw }, + Opts#{ + <<"store">> => [Store], + <<"cache-read-mode">> => raw + }, maybe {ok, SpecID} ?= preloaded_spec(Ref, Store, PreOpts), lazy_first( @@ -156,7 +163,7 @@ preloaded_spec(Ref, Store, Opts) -> hb_store:read(Store, <>, Opts). %% @doc The preloaded store, with request-local cache keys stripped so it is -%% visible inside a request-scoped resolution. +%% active inside a request-scoped resolution. preloaded(Opts) -> Node = maps:without([<<"cache-control">>, <<"only">>, <<"prefer">>], Opts), hb_opts:get(preloaded_store, undefined, Node). @@ -272,13 +279,73 @@ load_archive(ID, Opts) -> load_archive_message(Msg, Opts) end. +%% @doc Load an archive and cache the schema of its device. load_archive_message(Msg, Opts) -> - hb_device_archive:load( - hb_maps:get(<<"module-name">>, Msg, undefined, Opts), - hb_maps:get(<<"body">>, Msg, undefined, Opts), - Msg, - Opts - ). + Archive = hb_maps:get(<<"body">>, Msg, undefined, Opts), + maybe + {ok, Module} ?= + hb_device_archive:load( + hb_maps:get(<<"module-name">>, Msg, undefined, Opts), + Archive, + Msg, + Opts + ), + maybe_cache_schema(Module, Archive, Opts), + {ok, Module} + end. + +%% @doc Extract and cache a packaged device schema when available. +maybe_cache_schema(Module, Archive, Opts) -> + case archive_to_schema(Module, Archive, Opts) of + {ok, Schema} -> + cache_schema(Module, Schema, Opts), + ok; + _ -> ok + end. + +%% @doc Extract the schema for a loaded device module from a packaged archive. +archive_to_schema(Module, Archive, _Opts) -> + case hb_device_archive:contents(Archive) of + {ok, Modules, _Resources} -> + case lists:keyfind(Module, 1, Modules) of + {Module, _Path, Beam} -> hb_types:beam_to_schema(Module, Beam); + false -> {error, schema_not_found} + end; + {error, _} = Error -> + Error + end. + +%% @doc Read the cached schema for a device module if known. +schema(Module, Opts) -> + SchemaOpts = loaded_device_opts(Opts), + case hb_cache:read(schema_key(Module), SchemaOpts) of + {ok, Schema} -> {ok, hb_cache:ensure_all_loaded(Schema, SchemaOpts)}; + Other -> Other + end. + +%% @doc Write the schema for a device to the cache, under its reference. +cache_schema(Module, Schema, Opts) -> + case hb_opts:get(<<"caching-schema">>, false, Opts) of + true -> skipping_recursive_cache; + false -> + SchemaOpts = + (loaded_device_opts(Opts))#{ + <<"caching-schema">> => true + }, + case hb_cache:write(Schema, SchemaOpts) of + {ok, SchemaID} -> + hb_store:link( + loaded_device_store(Opts), + #{ schema_key(Module) => SchemaID }, + SchemaOpts + ); + _ -> + ok + end + end. + +schema_key(Module) -> + <<"~meta@1.0/devices/schemas/", (atom_to_binary(Module, utf8))/binary>>. implementation_query(SpecID) -> #{ diff --git a/src/core/include/hb.hrl b/src/core/include/hb.hrl index 538716040..a16974669 100644 --- a/src/core/include/hb.hrl +++ b/src/core/include/hb.hrl @@ -51,13 +51,17 @@ -define(debug_print(X), hb_event:debug_print(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). -define(no_prod(X), hb:no_prod(X, ?MODULE, ?LINE)). +-ifdef(HB_PRIMITIVE_DEBUG). +%% @doc Primitive debugging macro for low-level printing when testing AO-Core +%% changes. +-define(prim_dbg(X), io:format(standard_error, "PRIM ~s: ~p~n", [?trace_short(), X])). +-else. +-define(prim_dbg(X), io:format(standard_error, "PRIM ~s: ~p~n", [?trace_short(), X])). +-endif. + %%% Macro shortcuts for debugging. %% @doc A macro for marking that you got 'here'. -define(h(), hb_event:log("[Debug point reached.]", ?MODULE, ?FUNCTION_NAME, ?LINE)). -%% @doc Quickly print a value in the logs. Currently uses the event -%% function, but should be moved to a debug-specific function once we -%% build out better logging infrastructure. --define(p(X), hb_event:log(X, ?MODULE, ?FUNCTION_NAME, ?LINE)). %% @doc Print the trace of the current stack, up to the first non-hyperbeam %% module. -define(trace(), hb_format:trace_macro_helper(fun hb_format:print_trace/4, catch error(test), ?MODULE, ?FUNCTION_NAME, ?LINE)). diff --git a/src/core/resolver/hb_ao.erl b/src/core/resolver/hb_ao.erl index 6969eaae6..e6454d321 100644 --- a/src/core/resolver/hb_ao.erl +++ b/src/core/resolver/hb_ao.erl @@ -82,34 +82,23 @@ %%% %%% info/worker : A function that should be run as the 'server' loop of %%% the executor for interactions using the device. -%%% -%%% The HyperBEAM resolver also takes a number of runtime options that change -%%% the way that the environment operates: -%%% -%%% `update_hashpath': Whether to add the `Req' to `HashPath' for the `Res'. -%%% Default: true. -%%% `add_key': Whether to add the key to the start of the arguments. -%%% Default: `'. %%% -module(hb_ao). %%% Main AO-Core API: --export([resolve/2, resolve/3, resolve_many/2]). +-export([resolve/2, resolve/3]). -export([raw/3, raw/4, raw/5]). +-export([with/3]). -export([normalize_key/1, normalize_key/2, normalize_keys/1, normalize_keys/2]). --export([force_message/2]). %%% Shortcuts and tools: --export([keys/1, keys/2, keys/3]). --export([get/2, get/3, get/4, get_first/2, get_first/3]). --export([set/3, set/4, remove/2, remove/3]). +-export([keys/2]). +-export([as/3, get/3, get/4, get_first/2, get_first/3]). +-export([set/3, set/4, deep_set/3, remove/3]). %%% Exports for tests in hb_ao_test_vectors.erl: --export([deep_set/4]). -include("include/hb.hrl"). -define( TEMP_OPTS, [ - <<"add-key">>, - <<"force-message">>, <<"cache-control">>, <<"spawn-worker">>, <<"only">>, @@ -117,49 +106,17 @@ ] ). -%% @doc Get the value of a message's key by running its associated device -%% function. Optionally, takes options that control the runtime environment. -%% This function returns the raw result of the device function call: -%% `{ok | error, NewMessage}.' -%% The resolver is composed of a series of discrete phases: -%% 1: Normalization. -%% 2: Cache lookup. -%% 3: Validation check. -%% 4: Persistent-resolver lookup. -%% 5: Device lookup. -%% 6: Execution. -%% 7: Execution of the `step' hook. -%% 8: Subresolution. -%% 9: Cryptographic linking. -%% 10: Result caching. -%% 11: Notify waiters. -%% 12: Fork worker. -%% 13: Recurse or terminate. -resolve(Path, Opts) when is_binary(Path) -> - resolve(#{ <<"path">> => Path }, Opts); -resolve(SingletonMsg, _Opts) - when is_map(SingletonMsg), not is_map_key(<<"path">>, SingletonMsg) -> - {error, <<"Attempted to resolve a message without a path.">>}; -resolve(SingletonMsg, Opts) -> - resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). - -resolve(Base, Path, Opts) when not is_map(Path) -> - resolve(Base, #{ <<"path">> => Path }, Opts); -resolve(Base, Req, Opts) -> - PathParts = hb_path:from_message(request, Req, Opts), - ?event( - ao_core, - {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}} - ), - MessagesToExec = [ Req#{ <<"path">> => Path } || Path <- PathParts ], - ?event_debug(debug_ao_core, - {stage, - 1, - prepare_multimessage_resolution, - {messages_to_exec, MessagesToExec} - } - ), - resolve_many([Base | MessagesToExec], Opts). +%% @doc Small helper to extend a message with a device. +as(Device, Link, Opts) when ?IS_LINK(Link) -> + as(Device, hb_cache:ensure_loaded(Link, Opts), Opts); +as(Device, Msg, _Opts) when is_map(Msg) -> + hb_private:set_priv( + #{ + <<"device">> => Device, + <<"...">> => Msg + }, + hb_private:from_message(Msg) + ). %% @doc Invoke only the raw execution of the AO-Core resolution flow, ignoring %% normalization, cache, hashpath, worker, and other management components. @@ -174,35 +131,59 @@ raw(Base, Req, Opts) -> raw(Device, Base, Req, Opts) -> raw(Device, undefined, Base, Req, Opts). raw(ForcedDevice, ForcedKey, Base, Req, Opts) -> - ExecOpts = execution_opts(Opts), - % If a forced key is provided, use it; otherwise, extract from the request. - Key = - if ForcedKey =/= undefined -> ForcedKey; - true -> hb_path:hd(Req, ExecOpts) - end, - % If an explicit device is provided we use it _only on the lookup_ -- not - % during execution. - BaseWithDevice = - case ForcedDevice of - undefined -> Base; - _ when is_map(Base) -> Base#{ <<"device">> => ForcedDevice }; - _ -> #{ <<"device">> => ForcedDevice } - end, - {ExecFun, PrefixArgs} = - case hb_device:message_to_fun(BaseWithDevice, Key, ExecOpts) of - {add_key, _DevMod, Fun} -> {Fun, [Key]}; - {_Status, _DevMod, Fun} -> {Fun, []} - end, - % Apply the function and return the result directly, without any further - % processing. We add the `PrefixArgs` to the list of arguments to be passed - % to the function to accomodate default handlers, which take the key that - % was invoked on the device as the first argument (ahead of `Base`, `Req`, - % and `ExecOpts`). - apply( - ExecFun, - hb_device:truncate_args(ExecFun, PrefixArgs ++ [Base, Req, ExecOpts]) + % ?prim_dbg( + % {executing, + % {forced_device, ForcedDevice}, + % {forced_key, ForcedKey} + % } + % ), + from_context( + do( + to_context( + ForcedDevice, + ForcedKey, + Base, + Req, + Opts#{ + <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + <<"await-inprogress">> => false, + <<"hashpath">> => ignore + } + ) + ) ). +%% @doc Resolve a single AO-Core base and request pair or singleton message. +%% If the `path` in the request has multiple parts, each is executed in sequence, +%% over the original base message. +resolve(MsgSeq, Opts) when is_list(MsgSeq) -> + resolve_many(MsgSeq, Opts); +resolve(Path, Opts) when is_binary(Path) -> + resolve(#{ <<"path">> => Path }, Opts); +resolve(SingleResult, _Opts) + when is_map(SingleResult), not is_map_key(<<"path">>, SingleResult) -> + % Nothing to resolve; return as-is. + {ok, SingleResult}; +resolve(SingletonMsg, Opts) -> + resolve_many(hb_singleton:from(SingletonMsg, Opts), Opts). +resolve(Base, Path, Opts) when is_binary(Path) -> + resolve(Base, #{ <<"path">> => Path }, Opts); +resolve(Base, Req, Opts) -> + MessagesToExec = + case hb_path:from_message(request, Req, Opts) of + undefined -> [Req]; + PathParts -> + [ #{ <<"path">> => Path, <<"...">> => Req } || Path <- PathParts ] + end, + ?event_debug(debug_ao_core, + {stage, + 1, + prepare_multimessage_resolution, + {messages_to_exec, MessagesToExec} + } + ), + resolve_many([Base | MessagesToExec], Opts). + %% @doc Resolve a list of messages in sequence. Take the output of the first %% message as the input for the next message. Once the last message is resolved, %% return the result. @@ -216,17 +197,15 @@ resolve_many([ID], Opts) when ?IS_ID(ID) -> % 2. The main AO-Core logic looks for linkages between message input % pairs and outputs. With only a single ID, there is not a valid pairing % to use in looking up a cached result. - ?event_debug(debug_ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), - try {ok, ensure_message_loaded(ID, Opts)} - catch _:_:_ -> {error, not_found} - end; + ?event_debug(debug_ao_core, {stage, na, resolve_directly_to_id, ID}, Opts), + hb_cache:read(ID, Opts); resolve_many(ListMsg, Opts) when is_map(ListMsg) -> % We have been given a message rather than a list of messages, so we should % convert it to a list, assuming that the message is monotonically numbered. ListOfMessages = try hb_util:message_to_ordered_list(ListMsg, internal_opts(Opts)) catch - Type:Exception:Stacktrace -> + Type:Exception:Stacktrace -> throw( {resolve_many_error, {given_message_not_ordered_list, ListMsg}, @@ -237,10 +216,6 @@ resolve_many(ListMsg, Opts) when is_map(ListMsg) -> ) end, resolve_many(ListOfMessages, Opts); -resolve_many({as, DevID, Msg}, Opts) -> - subresolve(#{}, DevID, Msg, Opts); -resolve_many([{resolve, Subres}], Opts) -> - resolve_many(Subres, Opts); resolve_many(MsgList, Opts) -> ?event_debug(debug_ao_core, {resolve_many, MsgList}, Opts), Res = do_resolve_many(MsgList, Opts), @@ -248,12 +223,12 @@ resolve_many(MsgList, Opts) -> Res. do_resolve_many([], _Opts) -> {failure, <<"Attempted to resolve an empty message sequence.">>}; -do_resolve_many([Res], Opts) -> - ?event_debug(debug_ao_core, {stage, 11, resolve_complete, Res}), - hb_cache:ensure_loaded(maybe_force_message(Res, Opts), Opts); +do_resolve_many([Res], _Opts) -> + ?event_debug(debug_ao_core, {stage, 11, resolve_complete, Res}, _Opts), + {ok, Res}; do_resolve_many([Base, Req | MsgList], Opts) -> ?event_debug(debug_ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), - case resolve_stage(1, Base, Req, Opts) of + case resolve_single(Base, Req, Opts) of {ok, Res} -> ?event_debug(debug_ao_core, { @@ -263,212 +238,221 @@ do_resolve_many([Base, Req | MsgList], Opts) -> {res, Res}, {opts, Opts} }, - Opts + Opts ), do_resolve_many([Res | MsgList], Opts); Res -> - % The result is not a resolvable message. Return it. - ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), - maybe_force_message(Res, Opts) + % The result is not a continuable message. Return it. + ?event_debug(debug_ao_core, {stage, 13, resolve_many_terminating, Res}), + Res end. -resolve_stage(1, Link, Req, Opts) when ?IS_LINK(Link) -> - % If the first message is a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), - resolve_stage(1, hb_cache:ensure_loaded(Link, Opts), Req, Opts); -resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> - % If the second message is a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), - resolve_stage(1, Base, hb_cache:ensure_loaded(Link, Opts), Opts); -resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> - % Normalize `as' requests with a raw ID or link as the path. Links will be - % loaded in following stages. - resolve_stage(1, {as, DevID, #{ <<"path">> => Ref }}, Req, Opts); -resolve_stage(1, {as, DevID, Link}, Req, Opts) when ?IS_LINK(Link) -> - % If the first message is an `as' with a link, we should load the message and - % continue with the resolution. - ?event_debug(debug_ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), - resolve_stage(1, {as, DevID, hb_cache:ensure_loaded(Link, Opts)}, Req, Opts); -resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Req, Opts) when ?IS_ID(ID) -> - % If the first message is an `as' with an ID, we should load the message and - % apply the non-path elements of the sub-request to it. - ?event_debug(debug_ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), - RemBase = hb_maps:without([<<"path">>], Raw, Opts), - ?event_debug(debug_subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), - Baseb = ensure_message_loaded(ID, Opts), - ?event_debug(debug_subresolution, {loaded_message, {msg, Baseb}}, Opts), - Basec = hb_maps:merge(Baseb, RemBase, Opts), - ?event_debug(debug_subresolution, {merged_message, {msg, Basec}}, Opts), - Based = set(Basec, <<"device">>, DevID, Opts), - ?event_debug(debug_subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), - resolve_stage(1, Based, Req, Opts); -resolve_stage(1, Raw = {as, DevID, SubReq}, Req, Opts) -> - % Set the device of the message to the specified one and resolve the sub-path. - % As this is the first message, we will then continue to execute the request - % on the result. - ?event_debug(debug_ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), - ?event_debug(debug_subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), - case subresolve(SubReq, DevID, SubReq, Opts) of - {ok, SubRes} -> - % The subresolution has returned a new message. Continue with it. - ?event(subresolution, - {continuing_with_subresolved_message, {base, SubRes}} - ), - resolve_stage(1, SubRes, Req, Opts); - OtherRes -> - % The subresolution has returned an error. Return it. - ?event(subresolution, - {subresolution_error, {base, Raw}, {res, OtherRes}} - ), - OtherRes - end; -resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, Opts) -> - % Set the device to the specified `DevID' and resolve the message. Merging - % the `ReqInner' into the `ReqOuter' message first. We return the result - % of the sub-resolution directly. - ?event_debug(debug_ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), - LoadedInner = ensure_message_loaded(ReqInner, Opts), - Req = - hb_maps:merge( - set(ReqOuter, <<"path">>, unset, Opts), - if is_binary(LoadedInner) -> #{ <<"path">> => LoadedInner }; - true -> LoadedInner - end, - Opts - ), - ?event(subresolution, - {subresolving_request_before_execution, - {dev, DevID}, - {req, Req} +%% @doc Resolve only a single, explicit, computation step over a path-normalized +%% (single part) `Base` and `Req` pair. +resolve_single(Base, Req, Opts) -> + from_context(do(to_context(undefined, undefined, Base, Req, Opts))). + +%% @doc Create an execution context from a set of core parameters: +%% `ForcedDevice` (if relevant), `ForcedKey` (if relevant), `Base`, `Req`, +%% and `Opts`. +to_context(FDevice, FKey, Base, Req, Opts) -> + maps:filter( + fun(_K, V) -> V =/= undefined end, + #{ + <<"device">> => FDevice, + <<"path">> => FKey, + <<"base">> => Base, + <<"request">> => Req, + <<"opts">> => Opts } - ), - subresolve(RawBase, DevID, Req, Opts); -resolve_stage(1, {resolve, Subres}, Req, Opts) -> - % If the first message is a `{resolve, Subres}' tuple, we should execute it - % directly, then apply the request to the result. - ?event_debug(debug_ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), - % Unlike the `request' case for pre-subresolutions, we do not need to unset - % the `force-message' option, because the result should be a message, anyway. - % If it is not, it is more helpful to have the message placed into the `body' - % of a result, which can then be executed upon. - case resolve_many(Subres, Opts) of - {ok, Base} -> - ?event_debug(debug_ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), - resolve_stage(1, Base, Req, Opts); - OtherRes -> - ?event_debug(debug_ao_core, - {stage, - 1, - subresolve_failed, - {subres, Subres}, - {res, OtherRes}}, - Opts - ), - OtherRes + ). + +%% @doc Convert a completed execution context into the caller-facing +%% `{ExecStatus, Result}` form from its stage-bound `context`. +from_context({ok, Ctx = #{ <<"result">> := Res }}) -> + {maps:get(<<"status">>, Ctx, ok), Res}; +from_context(Other) -> + throw({unexpected_context_after_exec, Other}). + +%% @doc Ensure that each of the given fields is present in an execution +%% context, deriving those that are absent from the components already known. +%% Derived fields are added to the context, such that repeated demands do not +%% repeat work. A field that is neither present nor derivable yields +%% `{not_found, Field}`. +with([], Ctx, _Opts) -> {ok, Ctx}; +with([Field | Fields], Ctx, Opts) when is_map_key(Field, Ctx) -> + with(Fields, Ctx, Opts); +with([Field | Fields], Ctx, Opts) -> + case derive(Field, Ctx, Opts) of + {ok, Value} -> with(Fields, Ctx#{ Field => Value }, Opts); + _ -> {not_found, Field} + end. + +%% @doc Derive a single context field from its complementary components: +%% messages are read from the store by their IDs, while IDs are calculated +%% from their messages. A request may also be derived from a literal path +%% (e.g. `*`), which names itself. Messages read from the store are lazily +%% loaded: their keys are only read when they are themselves demanded. +derive(<<"base">>, #{ <<"base-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(<<"request">>, #{ <<"request-id">> := ID }, Opts) when ?IS_ID(ID) -> + hb_cache:read(ID, Opts); +derive(<<"request">>, #{ <<"request-id">> := Path }, _Opts) -> + {ok, #{ <<"path">> => Path }}; +derive(<<"varied-base">>, #{ <<"varied-base-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(<<"varied-request">>, #{ <<"varied-request-id">> := ID }, Opts) -> + hb_cache:read(ID, Opts); +derive(Field, Ctx, Opts) -> + % Every `x-id` field is derivable from its `x` message, if present. + case binary:split(Field, <<"-id">>) of + [Name, <<>>] when is_map_key(Name, Ctx) -> + {ok, hb_message:id(maps:get(Name, Ctx), all, Opts)}; + _ -> {not_found, Field} + end. + +%% @doc Resolves a fully normalized execution context. Stages that find the +%% result early set it in the context; later stages skip themselves when they +%% see it. Any non-`{ok, Ctx}` return propagates as the result. +do(Ctx0) -> + maybe + % Stage 1: Normalization; device or direct key lookup. + {ok, Ctx1} ?= stage_1(Ctx0), + % Stage 2: Function lookup. + {ok, Ctx2} ?= stage_2(Ctx1), + % Stage 3: Vary `Base` and `Req`. + {ok, Ctx3} ?= stage_3(Ctx2), + % Stage 4: Persistent resolver lookup. + {ok, Ctx4} ?= stage_4(Ctx3), + % Stage 5: Cache lookup. + {ok, Ctx5} ?= stage_5(Ctx4), + % Stage 6: Execution. + {ok, Ctx6} ?= stage_6(Ctx5), + % Stage 7: `Normalizer` application and hashpath generation + {ok, Ctx7} ?= stage_7(Ctx6), + % Stage 8: Result caching. + {ok, Ctx8} ?= stage_8(Ctx7), + % Stage 9: Notify waiters. + {ok, Ctx9} ?= stage_9(Ctx8), + % Stage 10: Execution of the `step' hook. + {ok, Ctx10} ?= stage_10(Ctx9), + % Stage 11: Fork worker. + stage_11(Ctx10) + else + {hit, Ctx} -> + % A stage may choose to return early. If it does, return it in + % `ok`-form. We assume that the stage in question will have + % appropriately cleaned up. + {ok, Ctx}; + {error, Ctx} -> + % The stage returned an error. Return to the caller early after + % unregistering as the active worker, if appropriate. We can call + % this twice, even if we erred after stage 8, as it is idempotent. + stage_8(Ctx), + {error, Ctx} + end. + +%% @doc Normalize the context of an execution request. Ensures that a device and +%% path are available for execution, or that a direct key hit resolves the +%% request without one. +stage_1(Ctx = #{ <<"device">> := _, <<"path">> := _ }) -> {ok, Ctx}; +stage_1(Ctx = #{ <<"base">> := Base, <<"path">> := Path, <<"opts">> := Opts }) -> + case hb_device:id_or_direct_key(Base, Path, Opts) of + {hit, Res} -> {hit, Ctx#{ <<"status">> => ok, <<"result">> => Res }}; + {ok, Device} -> stage_1(Ctx#{ <<"device">> => Device }); + {error, Reason} -> + {error, Ctx#{ <<"status">> => error, <<"reason">> => Reason }} end; -resolve_stage(1, Base, {resolve, Subres}, Opts) -> - % If the second message is a `{resolve, Subresolution}' tuple, we should - % execute the subresolution directly to gain the underlying `Req' for - % our execution. We assume that the subresolution is already in a normalized, - % executable form, so we pass it to `resolve_many' for execution. - ?event_debug(debug_ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), - % We make sure to unset the `force-message' option so that if the subresolution - % returns a literal, the rest of `resolve' will normalize it to a path. - case resolve_many(Subres, maps:without([<<"force-message">>], Opts)) of - {ok, Req} -> - ?event( - ao_core, - {stage, 1, request_subresolve_success, {req, Req}}, - Opts - ), - resolve_stage(1, Base, Req, Opts); - OtherRes -> - ?event( - ao_core, - { - stage, - 1, - request_subresolve_failed, - {subres, Subres}, - {res, OtherRes} - }, +stage_1(Ctx = #{ <<"request">> := Req, <<"opts">> := Opts }) when is_map(Req) -> + stage_1(Ctx#{ <<"path">> => hb_path:hd(Req, Opts) }); +stage_1(Ctx = #{ <<"request">> := Key }) -> + % If the request is for a direct key we normalize it to a message with + % only a path of that key. + stage_1(Ctx#{ <<"request">> => #{ <<"path">> => normalize_key(Key) } }); +stage_1(Ctx = #{ <<"opts">> := Opts }) -> + % The context does not carry its messages directly. Derive the `base` and + % `request` from the components that are given (e.g. their IDs), then + % normalize as usual. + case with([<<"base">>, <<"request">>], Ctx, Opts) of + {ok, Loaded} -> stage_1(Loaded); + {not_found, Field} -> + {error, Ctx#{ <<"status">> => error, <<"reason">> => {not_found, Field} }} + end. + +%% @doc Lookup the device and function to use during an execution. +stage_2( + Ctx = #{ + <<"device">> := Device, + <<"base">> := Base, + <<"path">> := Path, + <<"opts">> := Opts + } +) -> + {Status, _ExecDev, _ExecMod, Function} = + hb_device:message_to_fun(Device, Base, Path, Opts), + {ok, + Ctx#{ + <<"add-key">> => case Status of add_key -> Path; _ -> false end, + <<"priv">> => #{ <<"function">> => Function } + } + }. + +%% @doc Vary the `Base` and `Req` of the resolution. To do so, assuming that +%% the request is not itself a `vary` call, we recursively resolve the +%% computation as a sub-request. This allows the device to offer its own `vary` +%% logic, based upon the device's own understanding of the request. +stage_3(Ctx = #{ <<"path">> := <<"vary">>, <<"base">> := Base, <<"request">> := Req }) -> + % We are already varying. Do not recurse. + {ok, Ctx#{ <<"varied-base">> => Base, <<"varied-request">> => Req }}; +stage_3( + Ctx = + #{ + <<"base">> := Base, + <<"request">> := Req, + <<"priv">> := #{ <<"function">> := Function }, + <<"opts">> := Opts + } +) -> + maybe + % Raw call `vary` to find the varied base/req, passing the + % `priv/function` such that varying does not require us to repeat the + % associated work. The returned context fragment (varied messages, + % result normalization) is merged into our own. + {ok, VariedCtx} ?= + raw( + undefined, + <<"vary">>, + Base, + hb_private:set(Req, <<"function">>, Function, Opts), Opts ), - OtherRes - end; -resolve_stage(1, Base, Req, Opts) when is_list(Base) -> - % Normalize lists to numbered maps (base=1) if necessary. - ?event_debug(debug_ao_core, {stage, 1, list_normalize}, Opts), - resolve_stage(1, - normalize_keys(Base, Opts), - Req, - Opts - ); -resolve_stage(1, Base, NonMapReq, Opts) when not is_map(NonMapReq) -> - ?event_debug(debug_ao_core, {stage, 1, path_normalize}), - resolve_stage(1, Base, #{ <<"path">> => NonMapReq }, Opts); -resolve_stage(1, RawBase, RawReq, Opts) -> - % Normalize the path to a private key containing the list of remaining - % keys to resolve. - ?event_debug(debug_ao_core, {stage, 1, normalize}, Opts), - Base = normalize_keys(RawBase, Opts), - Req = normalize_keys(RawReq, Opts), - resolve_stage(2, Base, Req, Opts); -resolve_stage(2, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 2, cache_lookup}, Opts), - % Lookup request in the cache. If we find a result, return it. - % If we do not find a result, we continue to the next stage, - % unless the cache lookup returns `halt' (the user has requested that we - % only return a result if it is already in the cache). - case hb_cache_control:maybe_lookup(Base, Req, Opts) of - {ok, Res} -> - ?event_debug(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), - {ok, Res}; - {continue, NewBase, NewReq} -> - resolve_stage(3, NewBase, NewReq, Opts); - {error, CacheResp} -> {error, CacheResp} - end; -resolve_stage(3, Base, Req, _Opts) when not is_map(Base) or not is_map(Req) -> - % Validation check: If the messages are not maps, we cannot find a key - % in them, so return not_found. - ?event_debug(debug_ao_core, {stage, 3, validation_check_type_error}, _Opts), - {error, not_found}; -resolve_stage(3, Base, Req, Opts) -> - ?event_debug(debug_ao_core, {stage, 3, validation_check}, Opts), - % Validation checks: If `paranoid_message_verification' is enabled, we should - % verify the base and request messages prior to execution. - hb_message:paranoid_verify( - pre_resolve, - #{ - <<"reason">> => <<"AO-Core Pre-Execution Validation">>, - <<"base">> => Base, - <<"request">> => Req - }, - Opts - ), - resolve_stage(4, Base, Req, Opts); -resolve_stage(4, Base, Req, Opts) -> + {ok, maps:merge(Ctx, VariedCtx)} + end. + +%% @doc Determine if the request is already being resolved right now. If so, +%% await the result rather than resolving again. If not, register as the +%% resolver for the request in question and proceed. If we are executing with +%% `no-cache`, skip the lookup and proceed with the current context. +stage_4( + Ctx = #{ + <<"varied-base">> := Base, + <<"varied-request">> := Req, + <<"opts">> := Opts + }) -> ?event_debug(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. - % Before we search for a live executor, we check if the device specifies - % a function that tailors the 'group' name of the execution. For example, - % the `dev_process' device 'groups' all calls to the same process onto + % Before we search for a live executor, we check if the device specifies + % a function that tailors the 'group' name of the execution. For example, + % the `~process@1.0' device 'groups' all calls to the same process onto % calls to a single executor. By default, `{Base, Req}' is used as the % group name. - case hb_persistent:find_or_register(Base, Req, hb_maps:without(?TEMP_OPTS, Opts, Opts)) of - {leader, ExecName} -> - % We are the leader for this resolution. Continue to the next stage. - case hb_opts:get(spawn_worker, false, Opts) of - true -> ?event(worker_spawns, {will_become, ExecName}); - _ -> ok - end, - resolve_stage(5, Base, Req, ExecName, Opts); + case hb_persistent:find_or_register(Base, Req, Opts) of + {skip, ungrouped_exec} -> {ok, Ctx}; + {leader, ExecName} -> {ok, Ctx#{ <<"leader">> => ExecName }}; {wait, Leader} -> % There is another executor of this resolution in-flight. - % Bail execution, register to receive the response, then + % register to receive the response, then % wait. case hb_persistent:await(Leader, Base, Req, Opts) of {error, leader_died} -> @@ -483,14 +467,19 @@ resolve_stage(4, Base, Req, Opts) -> Opts ), % Re-try again if the group leader has died. - resolve_stage(4, Base, Req, Opts); - Res -> - % Now that we have the result, we can skip right to potential - % recursion (step 11) in the outer-wrapper. - Res + stage_4(Ctx); + {Status, Result} -> + % We received a successful result from another worker. They + % will store the result if appropriate, so we skip the store + % write. + {ok, Ctx#{ <<"status">> => Status, <<"result">> => Result }}; + Other -> + % The leader resolved to a non-`ok` result. Propagate it. + ?event_debug(debug_ao_core, {unexpected_worker_result, Other}), + Other end; {infinite_recursion, GroupName} -> - % We are the leader for this resolution, but we executing the + % We are the leader for this resolution, but we executing the % computation again. This may plausibly be OK in _some_ cases, % but in general it is the sign of a bug. ?event( @@ -503,335 +492,179 @@ resolve_stage(4, Base, Req, Opts) -> }, Opts ), - case hb_opts:get(allow_infinite, false, Opts) of + case hb_opts:get(<<"allow-infinite">>, false, Opts) of true -> % We are OK with infinite loops, so we just continue. - resolve_stage(5, Base, Req, GroupName, Opts); + % No need to register, as we are the leader. + {ok, Ctx}; false -> % We are not OK with infinite loops, so we raise an error. - error_infinite(Base, Req, Opts) + { + error, + #{ + <<"status">> => 508, + <<"body">> => <<"Request creates infinite recursion.">> + } + } end end. -resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 5, device_lookup}, Opts), - % Device lookup: Find the Erlang function that should be utilized to - % execute Req on Base. - {ResolvedFunc, NewOpts} = - try - UserOpts = hb_maps:without(?TEMP_OPTS, Opts, Opts), - Key = hb_path:hd(Req, UserOpts), - % Try to load the device and get the function to call. - ?event( - { - resolving_key, - {key, Key}, - {base, Base}, - {req, Req}, - {opts, Opts} + +%% @doc Look up whether the varied base and request pair already has a known +%% result in the cache. If so, set it in the context: subsequent stages skip +%% execution, and stage 8 notifies any waiting callers. +stage_5(Ctx = #{ + <<"varied-base">> := Base, + <<"varied-request">> := Req, + <<"opts">> := Opts +}) -> + case hb_cache_control:maybe_lookup(Base, Req, Opts) of + {continue, NewBase, NewReq} -> + { + ok, + Ctx#{ + <<"varied-base">> => NewBase, + <<"varied-request">> => NewReq } - ), - {Status, Device, Func} = hb_device:message_to_fun(Base, Key, UserOpts), - ?event( - {found_func_for_exec, - {key, Key}, - {device, Device}, - {func, Func}, - {base, Base}, - {req, Req}, - {opts, Opts} - } - ), - % Next, add an option to the Opts map to indicate if we should - % add the key to the start of the arguments. - { - Func, - Opts#{ - <<"add-key">> => - case Status of - add_key -> Key; - _ -> false - end - } - } - catch - Class:Exception:Stacktrace -> - ?event( - ao_result, - { - load_device_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {exec_class, Class}, - {exec_exception, Exception}, - {exec_stacktrace, Stacktrace}, - {opts, Opts} - }, - Opts - ), - % If the device cannot be loaded, we alert the caller. - error_execution( - ExecName, - Req, - loading_device, - {Class, Exception, Stacktrace}, - Opts - ) - end, - resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). -resolve_stage(6, Func, Base, Req, ExecName, Opts) -> + }; + {error, not_found} -> {ok, Ctx}; + {Status, Result} -> + { + ok, + Ctx#{ + <<"result">> => Result, + <<"status">> => Status + } + } + end. + +%% @doc Perform the actual resolution of an AO-Core request. +stage_6(Ctx = #{ <<"result">> := _ }) -> {ok, Ctx}; +stage_6(Ctx = #{ + <<"varied-base">> := Base, + <<"varied-request">> := Req, + <<"opts">> := Opts, + <<"priv">> := #{ <<"function">> := Func } +}) -> ?event_debug(debug_ao_core, {stage, 6, ExecName, execution}, Opts), - % Execution. ExecOpts = execution_opts(Opts), - Args = - case hb_opts:get(add_key, false, Opts) of - false -> [Base, Req, ExecOpts]; - Key -> [Key, Base, Req, ExecOpts] - end, - % Try to execute the function. - Res = - try - TruncatedArgs = hb_device:truncate_args(Func, Args), - MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), - ?event( - debug_ao_result, - { - ao_result, - {exec_name, ExecName}, - {base, Base}, - {req, Req}, - {res, MsgRes} - }, - Opts - ), - MsgRes - catch - ExecClass:ExecException:ExecStacktrace -> - ?event( - ao_core, - {device_call_failed, ExecName, {func, Func}}, - Opts - ), - ?event( - ao_result, - { - exec_failed, - {base, Base}, - {req, Req}, - {exec_name, ExecName}, - {func, Func}, - {exec_class, ExecClass}, - {exec_exception, ExecException}, - {exec_stacktrace, erlang:process_info(self(), backtrace)}, - {opts, Opts} - }, - Opts - ), - % If the function call fails, we raise an error in the manner - % indicated by caller's `#Opts'. - error_execution( - ExecName, - Req, - device_call, - {ExecClass, ExecException, ExecStacktrace}, - Opts - ) + ExecName = maps:get(<<"leader">>, Ctx, unnamed), + Args = + case maps:get(<<"add-key">>, Ctx, false) of + false -> [Base, Req, ExecOpts]; + Key -> [Key, Base, Req, ExecOpts] end, + % Try to execute the function. + {Status, Res} = maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts), hb_message:paranoid_verify( post_resolve, #{ <<"reason">> => <<"AO-Core Post-Execution Validation">>, <<"base">> => Base, <<"request">> => Req, - <<"result">> => Res + <<"varied-result">> => Res }, Opts ), - resolve_stage(7, Base, Req, Res, ExecName, Opts); -resolve_stage( - 7, - Base, - Req, - {St, Res}, - ExecName, - Opts = #{ <<"on">> := _On = #{ <<"step">> := _ }} + { + ok, + Ctx#{ + <<"status">> => Status, + <<"varied-result">> => Res, + <<"fresh">> => true + } + }. + +%% @doc When specified in the schema for the device call, normalize the +%% execution's result on top of the `Base` or `Req` message. In cases where +%% hashpath calculation is disabled (`hashpath => ignore` in the node message, +%% etc.), we +stage_7(Ctx = #{ <<"base">> := Base, <<"opts">> := Opts }) -> + maybe + {ok, Result} ?= hb_hashpath:result_from_context(Base, Ctx, Opts), + {ok, Ctx#{ <<"result">> => Result }} + end. + +%% @doc Cache the result of an execution if appropriate for the context. +%% Results that were not freshly and successfully executed (cache hits, direct +%% key hits, awaited results, non-`ok` statuses) are not stored. +stage_8( + Ctx = #{ + <<"status">> := ok, + <<"fresh">> := true, + <<"varied-base">> := VariedBase, + <<"varied-request">> := VariedReq, + <<"varied-result">> := VariedRes, + <<"opts">> := Opts + } ) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, _On}}, Opts), + hb_cache_control:maybe_store(VariedBase, VariedReq, VariedRes, Opts), + {ok, Ctx}; +stage_8(Ctx) -> {ok, Ctx}. + +%% @doc Return the resolved response to any waiting callers. +stage_9( + Ctx = #{ + <<"leader">> := ExecName, + <<"varied-request">> := Req, + <<"varied-result">> := Res, + <<"status">> := Status, + <<"opts">> := Opts + } +) -> + hb_persistent:unregister_notify( + ExecName, + Req, + {Status, Res}, + Opts + ), + {ok, Ctx}; +stage_9(Ctx) -> + % If we are not the leader, we can ignore the unregister step. + {ok, Ctx}. + +%% @doc If a hook has been specified for the `step` action, we call it with our +%% context including the result. +stage_10(Ctx = #{ <<"opts">> := Opts = #{ <<"on">> := #{ <<"step">> := _ }}}) -> + ?event_debug(debug_ao_core, {stage, 7, executing_step_hook}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every % execution. - HookReq = #{ - <<"base">> => Base, - <<"request">> => Req, - <<"status">> => St, - <<"body">> => Res - }, - case hb_hook:on(<<"step">>, HookReq, Opts) of - {ok, #{ <<"status">> := NewStatus, <<"body">> := NewRes }} -> - resolve_stage(8, Base, Req, {NewStatus, NewRes}, ExecName, Opts); - Error -> - ?event( - ao_core, - {step_hook_error, - {error, Error}, - {hook_req, HookReq} - }, - Opts - ), - Error - end; -resolve_stage(7, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), - resolve_stage(8, Base, Req, Res, ExecName, Opts); -resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), - % If the result is a `{resolve, Sublist}' tuple, we need to execute it - % as a sub-resolution. - resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); -resolve_stage(8, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), - resolve_stage(9, Base, Req, Res, ExecName, Opts); -resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), - % Cryptographic linking. Now that we have generated the result, we - % need to cryptographically link the output to its input via a hashpath. - resolve_stage(10, Base, Req, - case hb_opts:get(hashpath, update, Opts#{ <<"only">> => local }) of - update -> - NormRes = Res, - Priv = hb_private:from_message(NormRes), - HP = hb_path:hashpath(Base, Req, Opts), - if not is_binary(HP) or not is_map(Priv) -> - throw({invalid_hashpath, {hp, HP}, {res, NormRes}}); - true -> - {ok, NormRes#{ <<"priv">> => Priv#{ <<"hashpath">> => HP } }} - end; - reset -> - Priv = hb_private:from_message(Res), - {ok, Res#{ <<"priv">> => hb_maps:without([<<"hashpath">>], Priv, Opts) }}; - ignore -> - Priv = hb_private:from_message(Res), - if not is_map(Priv) -> - throw({invalid_private_message, {res, Res}}); - true -> - {ok, Res} - end - end, - ExecName, - Opts - ); -resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), - ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), - % Skip cryptographic linking and reset the hashpath if the result is abnormal. - Priv = hb_private:from_message(Res), - resolve_stage( - 10, Base, Req, - {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, - ExecName, Opts); -resolve_stage(9, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), - % Skip cryptographic linking and continue if we don't have a map that can have - % a hashpath at all. - resolve_stage(10, Base, Req, Res, ExecName, Opts); -resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), - % Result caching: Optionally, cache the result of the computation locally. - hb_cache_control:maybe_store(Base, Req, Res, Opts), - resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); -resolve_stage(10, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), - % Skip result caching if the result is abnormal. - resolve_stage(11, Base, Req, Res, ExecName, Opts); -resolve_stage(11, Base, Req, Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 11, ExecName}, Opts), - % Notify processes that requested the resolution while we were executing and - % unregister ourselves from the group. - hb_persistent:unregister_notify(ExecName, Req, Res, Opts), - resolve_stage(12, Base, Req, Res, ExecName, Opts); -resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> - ?event_debug(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + hb_hook:on(<<"step">>, Ctx, Opts); +stage_10(Ctx) -> {ok, Ctx}. + +%% @doc If we have been requested to spawn a worker process to remain active +%% at this stage, with this context in memory, we do so. +stage_11( + Ctx = #{ + <<"leader">> := ExecName, + <<"result">> := Res, + <<"opts">> := Opts + }) -> + ?event_debug(debug_ao_core, {stage, 11, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case - {is_map(Res), hb_opts:get(spawn_worker, false, Opts#{ <<"prefer">> => local })} + {is_map(Res), hb_opts:get(<<"spawn-worker">>, false, Opts#{ <<"prefer">> => local })} of {A, B} when (A == false) or (B == false) -> - Res; + {ok, Ctx}; {_, _} -> % Spawn a worker for the current execution WorkerPID = hb_persistent:start_worker(ExecName, Res, Opts), hb_persistent:forward_work(WorkerPID, Opts), - Res + {ok, Ctx} end; -resolve_stage(12, _Base, _Req, OtherRes, _ExecName, _Opts) -> - ?event_debug(debug_ao_core, {stage, 12, _ExecName, abnormal_status_skip_spawning}, _Opts), - OtherRes. - -%% @doc Execute a sub-resolution. -subresolve(RawBase, DevID, ReqPath, Opts) when is_binary(ReqPath) -> - % If the request is a binary, we assume that it is a path. - subresolve(RawBase, DevID, #{ <<"path">> => ReqPath }, Opts); -subresolve(RawBase, DevID, Req, Opts) -> - % First, ensure that the message is loaded from the cache. - Base = ensure_message_loaded(RawBase, Opts), - ?event(subresolution, - {subresolving, {base, Base}, {dev, DevID}, {req, Req}} - ), - % Next, set the device ID if it is given. - Base2 = - case DevID of - undefined -> Base; - _ -> - set( - Base, - <<"device">>, - DevID, - hb_maps:without(?TEMP_OPTS, Opts, Opts) - ) - end, - % If there is no path but there are elements to the request, we set these on - % the base message. If there is a path, we do not modify the base message - % and instead apply the request message directly. - case hb_path:from_message(request, Req, Opts) of - undefined -> - Base3 = - case map_size(hb_maps:without([<<"path">>], Req, Opts)) of - 0 -> Base2; - _ -> - set( - Base2, - set(Req, <<"path">>, unset, Opts), - Opts#{ <<"force-message">> => false } - ) - end, - ?event(subresolution, - {subresolve_modified_base, Base3}, - Opts - ), - {ok, Base3}; - Path -> - ?event(subresolution, - {exec_subrequest_on_base, - {mod_base, Base2}, - {req, Path}, - {req, Req} - } - ), - Res = resolve(Base2, Req, Opts), - ?event(subresolution, {subresolved_with_new_device, {res, Res}}), - Res - end. +stage_11(Ctx) -> {ok, Ctx}. %% @doc If the `AO_PROFILING' macro is defined (set by building/launching with %% `rebar3 as ao_profiling') we record statistics about the execution of the %% function. This is a costly operation, so if it is not defined, we simply %% apply the function and return the result. -ifndef(AO_PROFILING). -maybe_profiled_apply(Func, Args, _Base, _Req, _Opts) -> - apply(Func, Args). +maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts) -> + do_apply(ExecName, Func, Args, Base, Req, Opts). -else. -maybe_profiled_apply(Func, Args, Base, Req, Opts) -> +maybe_profiled_apply(ExecName, Func, Args, Base, Req, Opts) -> CallStack = erlang:get(ao_stack), ?event(ao_trace, {profiling_apply, @@ -870,7 +703,12 @@ maybe_profiled_apply(Func, Args, Base, Req, Opts) -> Stack -> [Key | Stack] end ), - {ExecMicroSecs, Res} = timer:tc(fun() -> apply(Func, Args) end), + {ExecMicroSecs, Res} = + timer:tc( + fun() -> + do_apply(ExecName, Func, Args, Base, Req, Opts) + end + ), put(ao_stack, CallStack), hb_event:record(<<"ao-call-counts">>, Key, Opts), hb_event:record(<<"ao-total-durations">>, Key, Opts, ExecMicroSecs), @@ -901,87 +739,63 @@ maybe_profiled_apply(Func, Args, Base, Req, Opts) -> Res. -endif. -%% @doc Ensure that a message is loaded from the cache if it is an ID, or -%% a link, such that it is ready for execution. -ensure_message_loaded(MsgID, Opts) when ?IS_ID(MsgID) -> - case hb_cache:read(MsgID, Opts) of - {ok, LoadedMsg} -> - LoadedMsg; - failure -> - failure; - not_found -> - throw({necessary_message_not_found, <<"/">>, MsgID}) - end; -ensure_message_loaded(MsgLink, Opts) when ?IS_LINK(MsgLink) -> - hb_cache:ensure_loaded(MsgLink, Opts); -ensure_message_loaded(Msg, _Opts) -> - Msg. - -%% @doc Catch all return if we are in an infinite loop. -error_infinite(Base, Req, Opts) -> - ?event( - ao_core, - {error, {type, infinite_recursion}, - {base, Base}, - {req, Req}, - {opts, Opts} - }, - Opts - ), - ?trace(), - { - error, - #{ - <<"status">> => 508, - <<"body">> => <<"Request creates infinite recursion.">> - } - }. - -%% @doc Handle an error in a device call. -error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> - Error = {error, Whence, {Class, Exception, Stacktrace}}, - hb_persistent:unregister_notify(ExecGroup, Req, Error, Opts), - ?event_debug(debug_ao_core, {handle_error, Error, {opts, Opts}}, Opts), - case hb_opts:get(error_strategy, throw, Opts) of - throw -> erlang:raise(Class, Exception, Stacktrace); - _ -> Error +%% @doc Execute a device call, wrapped with failure handling. +do_apply(ExecName, Func, Args, Base, Req, Opts) -> + try + case apply(Func, Truncated = hb_device:truncate_args(Func, Args)) of + {Status, Res} -> {Status, Res}; + Other -> + throw( + {unexpected_device_call_response, + #{ + <<"exec-name">> => ExecName, + <<"args">> => Truncated, + <<"result">> => Other, + <<"func">> => Func + } + } + ) + end + catch + Class:Exception:Stacktrace -> + ?event( + ao_core, + {device_call_failed, ExecName, {func, Func}}, + Opts + ), + ?event( + ao_result, + { + exec_failed, + {base, Base}, + {req, Req}, + {exec_name, ExecName}, + {func, Func}, + {exec_class, Class}, + {exec_exception, Exception}, + {exec_stacktrace, erlang:process_info(self(), backtrace)}, + {opts, Opts} + }, + Opts + ), + % If the function call fails, we raise an error in the manner + % indicated by caller's `#Opts', as well as returning the error to + % any registered listeners. + Error = + {failure, + #{ + <<"class">> => Class, + <<"exception">> => Exception, + <<"stacktrace">> => Stacktrace + } + }, + hb_persistent:unregister_notify(ExecName, Req, Error, Opts), + case hb_opts:get(error_strategy, throw, Opts) of + throw -> erlang:raise(Class, Exception, Stacktrace); + _ -> Error + end end. -%% @doc Force the result of a device call into a message if the result is not -%% requested by the `Opts'. If the result is a literal, we wrap it in a message -%% and signal the location of the result inside. We also similarly handle ao-result -%% when the result is a single value and an explicit status code. -maybe_force_message({Status, Res}, Opts) -> - case hb_opts:get(force_message, false, Opts) of - true -> force_message({Status, Res}, Opts); - false -> {Status, Res} - end; -maybe_force_message(Res, Opts) -> - maybe_force_message({ok, Res}, Opts). - -%% @doc Force a resolution result into a message, suitable for transmission -%% via HTTP. -force_message({Status, ResLink}, Opts) when ?IS_LINK(ResLink) -> - force_message({Status, hb_cache:ensure_loaded(ResLink, Opts)}, Opts); -force_message({Status, Res}, Opts) when is_list(Res) -> - force_message({Status, normalize_keys(Res, Opts)}, Opts); -force_message({Status, Subres = {resolve, _}}, _Opts) -> - {Status, Subres}; -force_message({Status, Literal}, _Opts) when not is_map(Literal) -> - ?event(encode_result, {force_message_from_literal, Literal}), - {Status, #{ <<"ao-result">> => <<"body">>, <<"body">> => Literal }}; -force_message({Status, M = #{ <<"status">> := Status, <<"body">> := Body }}, _Opts) - when map_size(M) == 2 -> - ?event(encode_result, {force_message_from_literal_with_status, M}), - {Status, #{ - <<"status">> => Status, - <<"ao-result">> => <<"body">>, - <<"body">> => Body - }}; -force_message({Status, Map}, _Opts) -> - ?event(encode_result, {force_message_from_map, Map}), - {Status, Map}. - %% @doc Shortcut for resolving a key in a message without its status if it is %% `ok'. This makes it easier to write complex logic on top of messages while %% maintaining a functional style. @@ -992,8 +806,6 @@ force_message({Status, Map}, _Opts) -> %% %% Returns the value of the key if it is found, otherwise returns the default %% provided by the user, or `not_found' if no default is provided. -get(Path, Msg) -> - get(Path, Msg, #{}). get(Path, Msg, Opts) -> get(Path, Msg, not_found, Opts). get(Path, {as, Device, Msg}, Default, Opts) -> @@ -1024,195 +836,41 @@ get_first([{Base, Path}|Msgs], Default, Opts) -> end. %% @doc Shortcut to get the list of keys from a message. -keys(Msg) -> keys(Msg, #{}). -keys(Msg, Opts) -> keys(Msg, Opts, keep). -keys(Msg, Opts, keep) -> - % There is quite a lot of AO-Core-specific machinery here. We: - % 1. `get' the keys from the message, via AO-Core in order to trigger the - % `keys' function on its device. - % 2. Ensure that the result is normalized to a message (not just a list) - % with `normalize_keys'. - % 3. Now we have a map of the original keys, so we can use `hb_maps:values' to - % get a list of them. - % 4. Normalize each of those keys in turn. - try - lists:map( - fun normalize_key/1, - hb_maps:values( - normalize_keys( - hb_private:reset(get(<<"keys">>, Msg, Opts)) - ), - Opts - ) - ) - catch - A:B:St -> - throw( - {cannot_get_keys, - {msg, Msg}, - {opts, Opts}, - {error, {A, B}}, - {stacktrace, St} - } - ) - end; -keys(Msg, Opts, remove) -> - lists:filter( - fun(Key) -> not lists:member(Key, ?AO_CORE_KEYS) end, - keys(Msg, Opts, keep) - ). +keys(Msg, Opts) -> get(<<"keys">>, Msg, Opts). -%% @doc Shortcut for setting a key in the message using its underlying device. -%% Like the `get/3' function, this function honors the `error_strategy' option. -%% `set' works with maps and recursive paths while maintaining the appropriate -%% `HashPath' for each step. -set(RawBase, RawReq, Opts) when is_map(RawReq) -> - Base = normalize_keys(RawBase, Opts), - Req = - hb_maps:without( - [<<"hashpath">>, <<"priv">>], - normalize_keys(RawReq, Opts), - Opts - ), +%% @doc Extend a message using its underlying device's `set' key. +set(Base, Req, Opts) -> ?event_debug(ao_internal, {set_called, {base, Base}, {req, Req}}, Opts), - % Get the next key to set. - case keys(Req, internal_opts(Opts)) of - [] -> Base; - [Key|_] -> - % Get the value to set. Use AO-Core by default, but fall back to - % getting via `maps' if it is not found. - Val = - case get(Key, Req, internal_opts(Opts)) of - not_found -> hb_maps:get(Key, Req, undefined, Opts); - Body -> Body - end, - ?event_debug({got_val_to_set, {key, Key}, {val, Val}, {req, Req}}), - % Next, set the key and recurse, removing the key from the Req. - set( - set(Base, Key, Val, internal_opts(Opts)), - remove(Req, Key, internal_opts(Opts)), - Opts - ) - end. -set(Base, Key, Value, Opts) -> - % For an individual key, we run deep_set with the key as the path. - % This handles both the case that the key is a path as well as the case - % that it is a single key. - Path = hb_path:term_to_path_parts(Key, Opts), - % ?event( - % {setting_individual_key, - % {base, Base}, - % {key, Key}, - % {path, Path}, - % {value, Value} - % } - % ), - deep_set(Base, Path, Value, Opts). - -%% @doc Recursively search a map, resolving keys, and set the value of the key -%% at the given path. This function has special cases for handling `set' calls -%% where the path is an empty list (`/'). In this case, if the value is an -%% immediate, non-complex term, we can set it directly. Otherwise, we use the -%% device's `set' function to set the value. -deep_set(Msg, [], Value, Opts) when is_map(Msg) or is_list(Msg) -> - device_set(Msg, <<"/">>, Value, Opts); -deep_set(_Msg, [], Value, _Opts) -> - Value; -deep_set(Msg, [Key], Value, Opts) -> - device_set(Msg, Key, Value, Opts); -deep_set(Msg, [Key|Rest], Value, Opts) -> - case resolve(Msg, Key, Opts) of - {ok, SubMsg} -> - ?event_debug(debug_set, - {traversing_deeper_to_set, - {current_key, Key}, - {current_value, SubMsg}, - {rest, Rest} - }, - Opts - ), - Res = - device_set( - Msg, - Key, - deep_set(SubMsg, Rest, Value, Opts), - <<"explicit">>, - Opts - ), - ?event_debug(debug_set, {deep_set, {msg, Msg}, {key, Key}, {res, Res}}, Opts), - Res; - _ -> - ?event_debug(debug_set, - {creating_new_map, - {current_key, Key}, - {rest, Rest} - }, - Opts - ), - Msg#{ Key => deep_set(#{}, Rest, Value, Opts) } - end. + hb_util:ok( + raw(undefined, <<"set">>, Base, Req, internal_opts(Opts)), + Opts + ). -%% @doc Call the device's `set' function. -device_set(Msg, Key, Value, Opts) -> - device_set(Msg, Key, Value, <<"deep">>, Opts). -device_set(Msg, Key, Value, Mode, Opts) -> - ReqWithoutMode = - case Key of - <<"path">> -> - #{ <<"path">> => <<"set_path">>, <<"value">> => Value }; - <<"/">> when is_map(Value) -> - % The value is a map and it is to be `set' at the root of the - % message. Subsequently, we call the device's `set' function - % with all of the keys found in the message, leading it to be - % merged into the message. - Value#{ <<"path">> => <<"set">> }; - _ -> - #{ <<"path">> => <<"set">>, Key => Value } - end, - Req = - case Mode of - <<"deep">> -> ReqWithoutMode; - <<"explicit">> -> ReqWithoutMode#{ <<"set-mode">> => Mode } +%% @doc Set an individual (potentially deep) key in a message to a new using +%% the message's set device. +set(Base, Key, Value, Opts) -> + DeepBase = + fun Deep([LeafKey]) -> #{ LeafKey => Value }; + Deep([NextKey|Rest]) -> #{ NextKey => Deep(Rest) } end, - ?event( - debug_set, - { - calling_device_set, - {base, Msg}, - {key, Key}, - {value, Value}, - {full_req, Req} - }, - Opts - ), - Res = - hb_util:ok( - resolve( - Msg, - Req, - internal_opts(Opts) - ), - internal_opts(Opts) - ), - ?event( - debug_set, - {device_set_result, Res}, - Opts - ), - Res. + deep_set(Base, DeepBase(hb_path:term_to_path_parts(Key, Opts)), Opts). -%% @doc Remove a key from a message, using its underlying device. -remove(Msg, Key) -> remove(Msg, Key, #{}). -remove(Msg, Key, Opts) -> - hb_util:ok( - resolve( - Msg, - #{ <<"path">> => <<"remove">>, <<"item">> => Key }, +%% @doc Recursively extend nested message values. +deep_set(Base, Req, Opts) when is_map(Req) -> + hb_util:ok( + raw( + undefined, + <<"set">>, + Base, + Req#{ <<"set">> => <<"deep">> }, internal_opts(Opts) ), Opts ). +%% @doc Remove a key from a message, using its underlying device. +remove(Msg, Key, Opts) -> set(Msg, Key, unset, Opts). + %% @doc Convert a key to a binary in normalized form. normalize_key(Key) -> normalize_key(Key, #{}). normalize_key(Key, _Opts) when is_binary(Key) -> Key; @@ -1233,10 +891,8 @@ normalize_key(Key, _Opts) when is_list(Key) -> %% @doc Ensure that a message is processable by the AO-Core resolver: No lists. %% Fast path: when every key is already a binary, `normalize_key/2' would %% pass them through unchanged (values are never recursed here), so return -%% the map as-is. This is the overwhelming majority case on the resolve hot -%% path -- `resolve_stage(1, ...)' normalises both `Base' and `Req' on every -%% resolution -- and skips the `hb_maps:to_list'/`from_list' round-trip and -%% per-key dispatch. +%% the map as-is. This is the overwhelming majority case, and skips the +%% `hb_maps:to_list'/`from_list' round-trip and per-key dispatch. normalize_keys(Msg) -> normalize_keys(Msg, #{}). normalize_keys(Base, Opts) when is_list(Base) -> normalize_keys( @@ -1304,7 +960,7 @@ execution_opts(Opts) -> % Unless the user has explicitly requested recursive spawning, we % unset the spawn-worker option so that we do not spawn a new worker % for every resulting execution. - case hb_opts:get(spawn_worker, false, Opts) of + case hb_opts:get(<<"spawn-worker">>, false, Opts) of recursive -> Opts1#{ <<"spawn-worker">> => recursive }; _ -> Opts1 - end. + end. \ No newline at end of file diff --git a/src/core/resolver/hb_cache.erl b/src/core/resolver/hb_cache.erl index 59e972a23..13db27a13 100644 --- a/src/core/resolver/hb_cache.erl +++ b/src/core/resolver/hb_cache.erl @@ -1010,15 +1010,12 @@ types_to_implicit(Types) -> structured_decode_types(Types, Opts) -> hb_util:ok( hb_ao:raw( - <<"structured@1.0">>, - <<"decode-types">>, - Types, - #{}, + #{ <<"device">> => <<"structured@1.0">>, <<"ao-types">> => Types }, + #{ <<"path">> => <<"decode-types">> }, Opts ) ). - %% @doc Read the result of a computation, using heuristics. The supported %% heuristics are as follows: %% 1. If the base message is an ID, we try to determine if the message has an @@ -1029,56 +1026,17 @@ structured_decode_types(Types, Opts) -> %% message and return it if it exists. %% 3. If the message has an explicit device, we attempt to read the hashpath to %% see if it has already been computed. -read_resolved(BaseMsg, Key, Opts) when is_binary(Key) -> - read_resolved(BaseMsg, #{ <<"path">> => Key }, Opts); -read_resolved({link, ID, LinkOpts}, Req, Opts) -> - read_resolved(ID, Req, maps:merge(LinkOpts, Opts)); -read_resolved(BaseMsgID, Req = #{ <<"path">> := Key }, Opts) when ?IS_ID(BaseMsgID) -> - Store = hb_opts:get(store, no_viable_store, Opts), - _NormKey = hb_ao:normalize_key(Key, Opts), - case hb_device:is_direct_key_access(BaseMsgID, Req, Opts, Store) of - unknown -> miss; - false -> - ?event_debug(read_cached, - {found_non_message_device, - {key, _NormKey} - } - ), - read_hashpath(BaseMsgID, Req, Opts); - true -> - % Either the message does not exist in the store, or there is no - % explicit device in the message. If the message exists this implies - % that the default (`message@1.0`) device will be used to execute - % the key. Subsequently, we can simply read the key and return it if - % it exists. - ?event_debug(read_cached, - {skipping_execution_store_lookup, - {base_msg, BaseMsgID}, - {key, _NormKey} - } - ), - case hb_store:resolve(Store, [BaseMsgID, Key], Opts) of - {ok, KeyPath} -> hashpath_read_result(read(KeyPath, Opts)); - {error, not_found} -> miss; - Other -> {hit, Other} - end - end; -read_resolved(BaseMsg, Req = #{ <<"path">> := Key }, Opts) when is_map(BaseMsg) -> - % The base message is loaded, so we determine if it has an explicit device - % and perform a direct lookup if it does not. - NormKey = hb_ao:normalize_key(Key, Opts), - case hb_device:is_direct_key_access(BaseMsg, Req, Opts) of - false -> read_hashpath(BaseMsg, Req, Opts); - true -> - ?event_debug(read_cached, - {skip_execution_memory_lookup, - {path, NormKey} - } - ), - {hit, read_in_memory_key(BaseMsg, NormKey, Opts)} +read_resolved(BaseID, ReqID, Opts) when is_binary(BaseID), is_binary(ReqID) -> + ExpectedPath = <>, + ?prim_dbg({read_resolved, ExpectedPath}), + case read(ExpectedPath, Opts) of + {error, not_found} -> miss; + Other -> {hit, Other} end; -read_resolved(Base, Req, Opts) -> - read_hashpath(Base, Req, Opts). +read_resolved(_Base, _Req, _Opts) -> + % The varied pair's IDs have not been generated, so there is no address to + % consult. + miss. %% @doc Return a key from an in-memory message, returning the same form as %% a store read (`{Status, Value}'). diff --git a/src/core/resolver/hb_cache_control.erl b/src/core/resolver/hb_cache_control.erl index 85746bf0d..08a5586f6 100644 --- a/src/core/resolver/hb_cache_control.erl +++ b/src/core/resolver/hb_cache_control.erl @@ -24,8 +24,8 @@ maybe_store(Base, Req, Res, Opts) -> case derive_cache_settings([Res, Req], Opts) of #{ <<"store">> := true } -> - ?event(caching, {caching_result, {base, Base}, {req, Req}, {res, Res}}), - dispatch_cache_write(Base, Req, Res, Opts); + dispatch_cache_write(Base, Req, Res, Opts), + ok; _ -> not_caching end. @@ -74,27 +74,8 @@ lookup(Base, Req, Opts) -> #{ <<"only-if-cached">> := true } -> only_if_cached_not_found_error(Base, Req, Opts); _ -> - case ?IS_ID(Base) of - false -> {continue, Base, Req}; - true -> - case hb_cache:read(Base, Opts) of - {ok, FullBase} -> - ?event(load_message, - {cache_hit_base_message_load, - {base_id, Base}, - {base_loaded, FullBase} - } - ), - {continue, FullBase, Req}; - not_found -> - necessary_messages_not_found_error( - Base, - Req, - Opts - ) - end - end - end + {error, not_found} + end end end. @@ -132,8 +113,8 @@ async_writer() -> %% @doc Internal function to write a compute result to the cache. perform_cache_write(Base, Req, Res, Opts) -> - hb_cache:write(Base, Opts), - hb_cache:write(Req, Opts), + {ok, BaseID} = hb_cache:write(Base, Opts), + {ok, ReqID} = hb_cache:write(Req, Opts), case Res of <<_/binary>> -> hb_cache:write_binary( @@ -141,8 +122,15 @@ perform_cache_write(Base, Req, Res, Opts) -> Res, Opts ); - Map when is_map(Map) -> - hb_cache:write(Res, Opts); + Msg when is_map(Msg) -> + {ok, ResID} = hb_cache:write(Res, Opts), + LinkPath = <>, + hb_cache:link( + ResID, + LinkPath, + Opts + ), + ok; _ -> ?event({cannot_write_result, Res}), skip_caching diff --git a/src/core/resolver/hb_hashpath.erl b/src/core/resolver/hb_hashpath.erl new file mode 100644 index 000000000..2df4b7415 --- /dev/null +++ b/src/core/resolver/hb_hashpath.erl @@ -0,0 +1,435 @@ +%%% @doc Hashpaths: Succinct executable claims of AO-Core transitions: Their +%%% results and dependencies, as addressable protocol values. +%%% +%%% Hashpaths abide by the following grammar: +%%% +%%% 
+%%%     Hashpath     :: Base "/" Request Variance? Dependencies? Equivalence?
+%%%     Base         :: MessageID | Hashpath
+%%%     Request      :: MessageID | PathString
+%%%     Variance     :: "" | ">" VariedBaseID "+" VariedReqID
+%%%     Dependencies :: "" | "@" DependenceMessageID
+%%%     Equivalence  :: Normalizer ResultMessageID
+%%%     Normalizer   :: "." | "="
+%%%
+%%% +%%% A compact form may omit fields when they are derivable: the vary pair is +%%% omitted when it is the identity vary, `Dependencies' when there are none, and +%%% the terminal before a result exists. Segments without explicit vary +%%% syntax are not special: `HP/*=FinalResultID' is an ordinary claim that +%%% resolving `*' at `HP' yields `FinalResultID'. +%%% +%%% Every separator of the syntax (`/', `>', `+', `@', `=', `.') is outside +%%% the base64url alphabet, so the grammar is unambiguous without escaping. +%%% The request position holds an ID when the request is addressed, or a +%%% literal key when it is self-describing (e.g. `*'). +-module(hb_hashpath). +%%% Create and parse hashpaths. +-export([format/2, parse/2, context/2]). +%%% Verify hashpath claims. +-export([verify_all/2, verify_part/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Encode a hashpath from an execution context used/returned by `hb_ao:do/1` +%% and its internal stages. +%% +%% The first stage of format extracts the first of two universal hashpath elements +%% -- the `Base` ID or existing hashpath. We then recurse with this value and the +%% remaining context. +format(Ctx, Opts) -> + maybe + {ok, BasePart} ?= format_base(Ctx, Opts), + {ok, RequestPart} ?= format_request(Ctx, Opts), + << + BasePart/binary, "/", RequestPart/binary, + (format_varied(Ctx, Opts))/binary, + (format_dependencies(Ctx, Opts))/binary, + (format_equivalence(Ctx, Opts))/binary + >> + else + {not_found, Name} -> + throw({context_not_viable, unavailable_field, Name}) + end. + +%% @doc Utilize the `hashpath` of the prior resolution, if it is available, +%% falling back to the `BaseID` if known, and recomputing it only if necessary. +format_base(Ctx = #{ <<"base">> := #{ <<"...">> := PriorHashpath } }, _) -> + {ok, PriorHashpath}; +format_base(Ctx, Opts) -> + find_id(<<"base">>, Ctx, Opts). + +%% @doc General utility for extracting the ID of a message by its name from a +%% context if it is already known, recomputing only if necessary. +find_id(Name, Ctx, _Opts) when is_map_key(<>, Ctx) -> + {ok, maps:get(Name, Ctx)}; +find_id(Name, Ctx, Opts) when is_map_key(Name, Ctx) -> + case hb_opts:get(<<"hashpath">>, enabled, Opts) of + enabled -> + {ok, hb_message:id(maps:get(Name, Ctx), signers, Opts)}; + _ -> + {not_found, Name} + end; +find_id(Name, _Ctx, _Opts) -> + {not_found, Name}. + +%% @doc Format the varied base and requests, if given, into their hashpath +%% components. +format_varied(Ctx, Opts) -> + maybe + {ok, VBase} ?= find_id(<<"varied-base">>, Ctx, Opts), + {ok, VReq} ?= find_id(<<"varied-request">>, Ctx, Opts), + <<">", VBase/binary, "+", VReq/binary>> + else + {not_found, _} -> + % Either the base or request is not found, so we omit the varied + % component of the hashpath. + <<>> + end. + +%% @doc If the dependencies of a resolution are known, format them into the +%% hashpath depends component. If not, return an empty string. Honors already +%% calculated dependency IDs if provided in the context. +format_dependencies(Ctx, Opts) -> + case find_id(<<"dependencies">>, Ctx, Opts) of + {ok, Depends} -> <<"@", Depends/binary>>; + {not_found, _} -> <<>> + end. + +%% @doc If the result of the execution has already been calculated, format it +%% into the hashpath equivalence component. If not, return an empty string. +format_equivalence(Ctx = #{ <<"varied-result">> := Result }, Opts) -> + <<(format_normalizer(Ctx, Opts)), Result/binary>>; +format_equivalence(_, _) -> <<>>. + +%% @doc Format the normalizer component of the hashpath. +format_normalizer(#{ <<"normalizer">> := base }, _Opts) -> <<"=">>; +format_normalizer(_, _) -> <<".">>. + +%% @doc Decode a hashpath into a list of context segments. The first segment will +%% have both a base and a request part, while the latter segments will only have +%% the request part -- the base being inferred from the result of the prior +%% segments. +parse(Hashpath, Opts) when is_binary(Hashpath) -> + [Base, Req1 | Reqs] = binary:split(Hashpath, <<"/">>, [global]), + [ + parse_part(Base, Req1, Opts) + | + lists:map( + fun(ReqPart) -> parse_part(undefined, ReqPart, Opts) end, + Reqs + ) + ]. + +%% @doc Parse the last segment of a hashpath into an executable context that +%% can be additionally executed upon. +context(Hashpath, Opts) -> + case lists:reverse(binary:split(Hashpath, <<"/">>, [global])) of + [ LoneBase ] -> #{ <<"base-id">> => LoneBase }; + [ LastReq, Rest ] -> + ReconstitutedBase = binary:join(lists:reverse(Rest), <<"/">>), + parse_part(ReconstitutedBase, LastReq, Opts); + _ -> #{ <<"base-id">> => Hashpath } + end. + +%% @doc Calculate the context for a hashpath segment. If the base is known +%% explicitly, add it to the result from parsing the request part. If not, +%% parse the request part and return as-is. +parse_part(undefined, ReqPart, Opts) -> + parse_request(ReqPart, Opts); +parse_part(Base, ReqPart, Opts) -> + (parse_request(ReqPart, Opts, Base))#{ <<"base-id">> => Base }. + +%% @doc Parse a single segment of the hashpath into a context segment. +parse_request(Part, Opts) -> + maybe + {next, NextDelim, Ctx1, Part2} ?= + parse_request_id(Part, Opts), + {ok, NextDelim2, Part3, Ctx2} ?= + parse_varied(NextDelim, Part2, Ctx1, Opts), + {ok, NextDelim3, Part4, Ctx3} ?= + parse_dependencies(NextDelim2, Part3, Ctx2, Opts), + {ok, Ctx4} ?= + parse_equivalence(NextDelim3, Part4, Ctx3, Opts), + {ok, Ctx4} + end. + +%% @doc Parse the request ID part of a hashpath segment. If the request ID is +%% the only part, it is returned as-is and the remainder of the part parsing is +%% skipped. +parse_request_id(Part, Opts) -> + case next(Part, Opts) of + {no_match, Part, <<>>} -> {ok, #{ <<"request-id">> => Part } }; + {Sep, ReqID, Part2} -> {next, Sep, Part2, #{ <<"request-id">> => ReqID }} + end. + +%% @doc If the delimiter that starts our segment is `>` we handle the inner +%% segment as a `VariedBase` and `VariedRequest` pair and get the next delimited +%% component. If the delimiter is not `>`, we pass the segment forward as-is. +parse_varied($>, Part, Ctx0, _Opts) -> + {NextDelim, Next, After} = next([$@, $., $=], Part), + case binary:split(Next, <<"+">>) of + [VBase, VReq] -> + { + next, + NextDelim, + After, + Ctx0#{ <<"varied-base">> => VBase, <<"varied-request">> => VReq } + }; + Malformed -> + {error, {invalid_variance_parts, Malformed}} + end; +parse_varied(NextDelim, Part, Ctx0, _Opts) -> + {next, NextDelim, Part, Ctx0}. + +%% @doc Parse the dependencies if present. We short-curcuit the parser and +%% return the context early if we have already hit the end of the string. +parse_dependencies(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; +parse_dependencies($@, DepID, Ctx0, Opts) -> + {NextDelim, Next, After} = next([$@, $., $=], Part), + {next, NextDelim, Next, After, Ctx0#{ <<"dependencies-id">> => DepID }}; +parse_dependencies(Delim, Part, Ctx0, _Opts) -> + {next, Delim, Part, <<>>, Ctx0}. + +%% @doc Parse the equivalent relationship if stated in the hashpath. +parse_equivalence(no_match, <<>>, Ctx, _Opts) -> {ok, Ctx}; +parse_equivalence($=, ResultID, Ctx, _Opts) -> + {ok, Ctx#{ <<"normalizer">> => base, <<"varied-result-id">> => ResultID }}; +parse_equivalence($., ResultID, Ctx, _Opts) -> + {ok, Ctx#{ <<"varied-result-id">> => ResultID }}. + +%% @doc Utility to split at the next syntax delimiter (e.g. `=`, `.`, `>`, `@`). +%% Returns the syntax element matched, and the rest of the string. Notably, this +%% utility does not break apart `VBase+VReq` pairs. They are treated as a single +%% unit and parsed internally in `parse_varied/4`. +next(S) -> next([$=, $., $>, $@], S). +next(Symbols, S) -> hb_util:split_depth_string_aware_single(Symbols, S). + +%% @doc Challenge a complete hashpath, verifying each part's claims. +verify_all(Bin, Opts) when is_binary(Bin) -> + verify_all(parse(Bin, Opts), Opts); +verify_all([], _Opts) -> + % We treat an empty hashpath as failing verification. + false; +verify_all([Init | Parts], Opts) -> + verify_all(Init, Parts, Opts). + +verify_all(_FinalBase, [], _Opts) -> + % The full hashpath has resolved and we have no more parts to verify. Each + % passed verification. + true; +verify_all(State, [Part | Rest], Opts) -> + % Add the currently computed state to the part's context and verify it. + case verify_context(Part#{ <<"base">> => State }, Opts) of + {true, ComputedState} -> verify_all(ComputedState, Rest, Opts); + false -> false + end. + +%% @doc Verify a single hashpath execution contained inside a larger hashpath +%% sequence. +verify_part(Hashpath, PartNum, Opts) when is_binary(Hashpath) -> + verify_part(parse(Hashpath, Opts), PartNum, Opts); +verify_part(Hashpath, PartNum, Opts) -> + maybe + {ok, Base} ?= load(Hashpath, PartNum, Opts), + verify_context(Part#{ <<"base">> => State }, Opts) + end. + +%% @doc Verify a full single context, parsed from a binary hashpath. The context +%% must contain a `Base' representation. We remove all of the non-`Base` and +%% `Request` fields, then utilize `hb_ao:do` to re-execute the context. Assuming +%% successful computation, we then verify the `VariedBase` and `VariedRequest` +%% fields against the parsed context, the `DependenciesID` if given, the +%% `Normalizer` type, and finally the `Result` message itself. If all of these +%% verify, the context is considered valid. +verify_context(Ctx, Opts) -> + StrippedCtx = + maps:with( + [<<"base">>, <<"request">>, <<"base-id">>, <<"request-id">>], + Ctx#{ <<"opts">> => Opts } + ), + maybe + {ok, ExecutedCtx} = hb_ao:do(StrippedCtx), + true ?= verify_varied(Ctx, ExecutedCtx, Opts), + true ?= verify_dependencies(Ctx, ExecutedCtx, Opts), + true ?= verify_equivalence(Ctx, ExecutedCtx, Opts), + {true, ExecutedCtx} + else + {error, Type} -> + ?event_debug( + hashpath_debug, + {hashpath_verify_context_failed, {type, Type}, {ctx, Ctx}}, + Opts + ), + false + end. + +%% @doc If varied `Req` and `Base` statements were present in the hashpath, +%% we verify that they match the executed context. +verify_varied(HPCtx, ExecutedCtx, Opts) -> + maybe + {ok, HPVBase} ?= find_id(<<"varied-base">>, HPCtx, Opts), + {ok, ExecVBase} ?= find_id(<<"varied-base">>, ExecutedCtx, Opts), + true ?= HPVBase =:= ExecVBase + orelse {error, <<"Varied `Base`s do not match">>}, + {ok, HPVReq} ?= find_id(<<"varied-request">>, HPCtx, Opts), + {ok, ExecVReq} ?= find_id(<<"varied-request">>, ExecutedCtx, Opts), + true ?= HPVReq =:= ExecVReq + orelse {error, <<"Varied `Request`s do not match">>} + else + {not_found, _} -> + % Skip validation if one or more required components are not + % provided. + true + end. + +%% @doc Verify that the dependencies in the hashpath claim match those in the +%% executed context, if both are present. +verify_dependencies(HPCtx, ExecutedCtx, Opts) -> + maybe + {ok, HPDeps} ?= find_id(<<"dependencies">>, HPCtx, Opts), + {ok, ExecDeps} ?= find_id(<<"dependencies">>, ExecutedCtx, Opts), + true ?= HPDeps =:= ExecDeps + orelse {error, <<"Dependencies do not match">>} + else + {not_found, <<"dependencies">>} -> + % Skip verification if dependencies not provided + true + end. + +%% @doc Verify that the results of the execution match those in the claim. +verify_equivalence(HPCtx, ExecutedCtx, Opts) -> + maybe + HPNormalizer = maps:get(<<"normalizer">>, HPCtx, replace), + ExecNormalizer = maps:get(<<"normalizer">>, ExecutedCtx, replace), + true ?= HPNormalizer =:= ExecNormalizer + orelse {error, <<"Normalizers do not match">>}, + {ok, HPResult} ?= find_id(<<"varied-result">>, HPCtx, Opts), + {ok, ExecResult} ?= find_id(<<"varied-result">>, ExecutedCtx, Opts), + true ?= HPResult =:= ExecResult + orelse {error, <<"Results do not match">>} + end. + +%% @doc Load the minimal executable base for a hashpath or a given part number +%% within it. +load(Hashpath, Opts) when is_binary(Hashpath) -> + load(parse(Hashpath, Opts), Opts); +load(Parts, Opts) -> + load(Parts, length(Parts), Opts). +load(Hashpath, PartNum, Opts) when is_binary(Hashpath) -> + load(parse(Hashpath, Opts), PartNum, Opts); +load(Parts, PartNum, Opts) -> + maybe + [Init|Rest] ?= load_sequence(Parts, PartNum, Opts), + {ok, InitState} ?= result_from_context(Init, Opts), + lists:foldl( + fun(Ctx, {error, X}) -> {error, X}; + (Ctx, {ok, State}) -> result_from_context(State, Ctx, Opts) + end, + {ok, InitState} + ) + else + [] -> + {error, <<"Cannot load empty hashpath.">>}; + {error, X} -> + {error, <<"Initial state not loadable: ", (hb_util:bin(X))/binary>>} + end. + +%% @doc Find the minimal sequence of hashpath contexts to load/apply such that +%% a valid base can be constructed at position `PartNum`. +load_sequence(Parts, OutOfBounds, Opts) when length(Parts) > OutOfBounds -> + { + error, + << + "Hashpath part number `", + (hb_util:bin(OutOfBounds))/binary, + "` not found. Hashpath length: ", + (hb_util:bin(length(Parts))), + "." + >> + }; +load_sequence(Parts, PartNum, Opts) -> + { + ok, + lists:reverse( + lists:takewhile( + fun(Ctx) -> + case result_from_context(Ctx, Opts) of + {error, _} -> true; + _ -> false + end, + lists:reverse(lists:sublist(Parts, PartNum)) + ) + ) + }. + +%% @doc Extract, if we can, a workable post-exec `message` from a context either +%% via the fully qualified result if possible or layering of the `varied-result` +%% atop the `base` if provided explicitly. +result_from_context(Ctx, Opts) -> result_from_context(undefined, Ctx, Opts). +result_from_context( + undefined, + #{ <<"normalizer">> => replace, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, VRes}; +result_from_context( + S, + #{ <<"normalizer">> => replace, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, Vres}; +result_from_context( + undefined, + Ctx = #{ <<"normalizer">> => base, <<"varied-result">> => VRes }, + _Opts +) -> + case find_id(<<"base">>, Ctx, Opts) of + {ok, Base} -> {ok, VRes#{ <<"...">> => S }}; + {not_found, _} -> + { + error, + << + "Context with `base` extension normalizer", + " without accessible `base`." + >> + } + end; +result_from_context( + S, + #{ <<"normalizer">> => base, <<"varied-result">> => VRes }, + _Opts +) -> + {ok, VRes#{ <<"...">> => S }}; +result_from_context(S, #{ <<"normalizer">> => X }, _Opts) -> + {error, <<"Unsupported normalizer `", (hb_util:bin(X))/binary, "`.">>}; +result_from_context(S, Ctx, Opts) -> + result_from_context(S, Ctx#{ <<"normalizer">> => replace }, Opts). + +%%% Tests + +full_form_round_trip_test() -> + Opts = #{}, + HP = + << + "BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4" + "/tYRVDkT2X7wjYYVaZWuBBWWzZatEsMoR2NBjcJ8CmZk" + ">a2Fub25pY2FsLXZhcmllZC1iYXNlLWlkLTAwMDAwMDAw" + "+a2Fub25pY2FsLXZhcmllZC1yZXEtaWQtMDAwMDAwMDAw" + "@ZGVwZW5kcy1tZXNzYWdlLWlkLTAwMDAwMDAwMDAwMDAw" + "=cGF0Y2gtaWQtMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw" + >>, + ?assertEqual(HP, format(parse(HP, Opts), Opts)). + +compact_form_round_trip_test() -> + Opts = #{}, + lists:foreach( + fun(HP) -> ?assertEqual(HP, format(parse(HP, Opts), Opts)) end, + [ + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/balance">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/transfer/balance">>, + <<"BQQF7TjcHTPT57eIcABDeIbfHkkOTDPKAQ9tJqScTV4/*" + "=cGF0Y2gtaWQtMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw">> + ] + ). \ No newline at end of file diff --git a/src/core/resolver/hb_message.erl b/src/core/resolver/hb_message.erl index a903b9607..eeb198be6 100644 --- a/src/core/resolver/hb_message.erl +++ b/src/core/resolver/hb_message.erl @@ -207,13 +207,7 @@ conversion_spec_to_req(Spec, Opts) -> { case Device of tabm -> tabm; - _ -> - hb_device:message_to_device( - #{ - <<"device">> => Device - }, - Opts - ) + _ -> hb_device:module(Device, Opts) end, hb_maps:without([<<"device">>], Spec, Opts) } @@ -575,7 +569,7 @@ paranoid_verify(Topic, Msg, Opts) -> % without emitting an event or walking a topic list. This path fires % twice per `hb_ao:resolve/3', so the event/`lists:member' overhead is % noticeable on the hot path. - case hb_opts:get(paranoid_verify, false, Opts) of + case hb_opts:get(<<"paranoid-verify">>, false, Opts) of false -> true; [] -> true; true -> @@ -592,7 +586,10 @@ paranoid_verify(Topic, Msg, Opts) -> do_paranoid_verify(Topic, Msg, Opts) -> try - do_paranoid_verify(Topic, [], Msg, Opts), + % We disable paranoid verification on _downstream_ (only!) execution + % here to avoid infinite recursion. We must avoid disabling it anywhere + % else. + do_paranoid_verify(Topic, [], Msg, Opts#{ <<"paranoid-verify">> => false }), ?event_debug(debug_paranoia, {paranoid_verify_complete, ok}, Opts), true catch diff --git a/src/core/resolver/hb_opts.erl b/src/core/resolver/hb_opts.erl index b1acd6be8..12bc10269 100644 --- a/src/core/resolver/hb_opts.erl +++ b/src/core/resolver/hb_opts.erl @@ -291,7 +291,7 @@ raw_default_message() -> % Every modification to `Opts' called directly by the node operator % should be recorded here. <<"node-history">> => [], - <<"debug-stack-depth">> => 8, + <<"debug-stack-depth">> => 40, <<"debug-print">> => false, <<"debug-log">> => false, <<"log-dir">> => <<"logs">>, diff --git a/src/core/resolver/hb_persistent.erl b/src/core/resolver/hb_persistent.erl index 17f575962..36cf57fa8 100644 --- a/src/core/resolver/hb_persistent.erl +++ b/src/core/resolver/hb_persistent.erl @@ -104,13 +104,13 @@ do_monitor(Group, Last, Opts) -> %% skip the grouper dispatch entirely: the leader short-circuit below would %% discard the computed group name anyway. find_or_register(_Base, _Req, #{ <<"await-inprogress">> := false }) -> - {leader, ungrouped_exec}; + {skip, ungrouped_exec}; find_or_register(Base, Req, Opts) -> find_or_register(group(Base, Req, Opts), Base, Req, Opts). find_or_register(ungrouped_exec, _Base, _Req, _Opts) -> - {leader, ungrouped_exec}; + {skip, ungrouped_exec}; find_or_register(GroupName, _Base, _Req, Opts) -> - case hb_opts:get(await_inprogress, false, Opts) of + case hb_opts:get(<<"await-inprogress">>, false, Opts) of false -> {leader, GroupName}; _ -> Self = self(), diff --git a/src/core/resolver/hb_private.erl b/src/core/resolver/hb_private.erl index 0f886445f..71fd881ea 100644 --- a/src/core/resolver/hb_private.erl +++ b/src/core/resolver/hb_private.erl @@ -85,12 +85,12 @@ set_priv(Msg, PrivMap) -> Msg#{ <<"priv">> => PrivMap }. %% @doc Check if a key is private. -is_private(Key) -> - try hb_util:bin(Key) of - <<"priv", _/binary>> -> true; - _ -> false - catch _:_ -> false - end. +is_private(ListKey) when is_list(ListKey) -> + % Strings should always be lists, but in case for some reason the caller + % ignores that... + try is_private(hb_util:bin(ListKey)) catch _ -> false end; +is_private(<<"priv", _/binary>>) -> true; +is_private(_) -> false. %% @doc Remove the first key from the path if it is a private specifier. remove_private_specifier(InputPath, Opts) -> diff --git a/src/core/resolver/hb_singleton.erl b/src/core/resolver/hb_singleton.erl index 04de744f8..925f4ade4 100644 --- a/src/core/resolver/hb_singleton.erl +++ b/src/core/resolver/hb_singleton.erl @@ -135,15 +135,10 @@ type(_Value) -> unknown. from(RawMsg, Opts) when is_binary(RawMsg) -> from(#{ <<"path">> => RawMsg }, Opts); from(RawMsg, Opts) -> - RawPath = hb_maps:get(<<"path">>, RawMsg, <<>>), + RawPath = hb_maps:get(<<"path">>, RawMsg, <<>>, Opts), ?event_debug(parsing, {raw_path, RawPath}), {ok, Path, Query} = from_path(RawPath), ?event_debug(parsing, {parsed_path, Path, Query}), - MsgWithoutBasePath = - hb_maps:merge( - hb_maps:remove(<<"path">>, RawMsg), - Query - ), % 2. Decode, split, and sanitize path segments. Each yields one step message. RawMsgs = lists:flatten( @@ -155,11 +150,11 @@ from(RawMsg, Opts) -> ?event_debug(parsing, {raw_messages, RawMsgs}), Msgs = normalize_base(RawMsgs), ?event_debug(parsing, {normalized_messages, Msgs}), - % 3. Type keys and values - Typed = apply_types(MsgWithoutBasePath, Opts), + % 3. Type keys and values over base + Typed = apply_types(Query#{ <<"...">> => RawMsg }, Opts), ?event_debug(parsing, {typed_messages, Typed}), % 4. Group keys by N-scope and global scope - ScopedModifications = group_scoped(Typed, Msgs), + ScopedModifications = group_scoped(Typed, Msgs, Opts), ?event_debug(parsing, {scoped_modifications, ScopedModifications}), % 5. Generate the list of messages (plus-notation, device, typed keys). Result = build_messages(Msgs, ScopedModifications, Opts), @@ -168,8 +163,6 @@ from(RawMsg, Opts) -> %% @doc Parse the relative reference into path, query, and fragment. from_path(RelativeRef) -> - %?event_debug(parsing, {raw_relative_ref, RawRelativeRef}), - %RelativeRef = hb_escape:decode(RawRelativeRef), Decoded = decode_string(RelativeRef), ?event_debug(parsing, {parsed_relative_ref, Decoded}), {Path, QKVList} = @@ -218,49 +211,56 @@ path_parts(Sep, PathBin) when is_binary(PathBin) -> %% @doc Extract all of the parts from the binary, given (a list of) separators. all_path_parts(_Sep, <<>>) -> []; -all_path_parts(Sep, Bin) -> - hb_util:split_depth_string_aware(Sep, Bin). +all_path_parts(Sep, Bin) -> hb_util:split_depth_string_aware(Sep, Bin). %% @doc Extract the characters from the binary until a separator is found. %% The first argument of the function is an explicit separator character, or %% a list of separator characters. Returns a tuple with the separator, the %% accumulated characters, and the rest of the binary. -part(Sep, Bin) when not is_list(Sep) -> - part([Sep], Bin); -part(Seps, Bin) -> - hb_util:split_depth_string_aware_single(Seps, Bin). +part(Sep, Bin) when not is_list(Sep) -> part([Sep], Bin); +part(Seps, Bin) -> hb_util:split_depth_string_aware_single(Seps, Bin). %% @doc Step 3: Apply types to values and remove specifiers. apply_types(Msg, Opts) -> - hb_maps:fold( - fun(Key, Val, Acc) -> - {_, K, V} = maybe_typed(Key, Val, Opts), - hb_maps:put(K, V, Acc, Opts) - end, - #{}, + hb_ao:set( Msg, + hb_maps:fold( + fun(Key, Val, Acc) -> + {_, K, V} = maybe_typed(Key, Val, Opts), + Acc#{ K => V } + end, + #{}, + Msg, + Opts + ), Opts ). %% @doc Step 4: Group headers/query by N-scope. %% `N.Key' => applies to Nth step. Otherwise => `global' -group_scoped(Map, Msgs) -> +group_scoped(Map, Msgs, Opts) -> {NScope, Global} = hb_maps:fold( fun(KeyBin, Val, {Ns, Gs}) -> case parse_scope(KeyBin) of {OkN, RealKey} when OkN > 0 -> - Curr = hb_maps:get(OkN, Ns, #{}), - Ns2 = hb_maps:put(OkN, hb_maps:put(RealKey, Val, Curr), Ns), + Curr = hb_maps:get(OkN, Ns, #{}, Opts), + Ns2 = + hb_maps:put( + OkN, + hb_maps:put(RealKey, Val, Curr, Opts), + Ns, + Opts + ), {Ns2, Gs}; - global -> {Ns, hb_maps:put(KeyBin, Val, Gs)} + global -> {Ns, hb_maps:put(KeyBin, Val, Gs, Opts)} end end, {#{}, #{}}, Map ), [ - hb_maps:merge(Global, hb_maps:get(N, NScope, #{})) + hb_ao:set(Global, hb_maps:get(N, NScope, #{}, Opts), Opts) || N <- lists:seq(1, length(Msgs)) ]. @@ -269,7 +269,7 @@ group_scoped(Map, Msgs) -> parse_scope(KeyBin) -> case binary:split(KeyBin, <<".">>, [global]) of [Front, Remainder] -> - case catch erlang:binary_to_integer(Front) of + case catch hb_util:int(Front) of NInt when is_integer(NInt) -> {NInt + 1, Remainder}; _ -> throw({error, invalid_scope, KeyBin}) end; @@ -281,46 +281,28 @@ build_messages(Msgs, ScopedModifications, Opts) -> do_build(1, Msgs, ScopedModifications, Opts). do_build(_, [], _, _) -> []; -do_build(I, [{as, DevID, RawMsg} | Rest], ScopedKeys, Opts) when is_map(RawMsg) -> +do_build(I, [{as, DevID, Direct} | Rest], ScopedKeys, Opts) when is_map(Direct) -> % We are processing an `as' message. If the path is empty, we need to % remove it from the message and the additional message, such that AO-Core % returns only the message with the device specifier changed. If the message % does have a path, AO-Core will subresolve it. - RawAdditional = lists:nth(I, ScopedKeys), - {Msg, Additional} = - case hb_maps:get(<<"path">>, RawMsg, <<"">>, Opts) of - ID when ?IS_ID(ID) -> - % When we have an ID, we do not merge the globally scoped elements. - { - RawMsg, - #{} - }; - <<"">> -> - % When we have an empty path, we remove the path from both - % messages. AO-Core will then simply set the device specifier - % and not execute a subresolve. - { - hb_ao:set(RawMsg, <<"path">>, unset, Opts), - hb_ao:set(RawAdditional, <<"path">>, unset, Opts) - }; - _BasePath -> - % When we have a non-empty path, we merge the messages in - % totality. The path-part's path will be subresolved. - {RawMsg, RawAdditional} - end, - Merged = hb_maps:merge(Additional, Msg, Opts), - StepMsg = hb_message:convert( - Merged, - <<"structured@1.0">>, - Opts#{ <<"topic">> => ao_internal } - ), - ?event_debug(parsing, {build_messages, {base, Msg}, {additional, Additional}}), - [{as, DevID, StepMsg} | do_build(I + 1, Rest, ScopedKeys, Opts)]; + StepMsg = + hb_message:convert( + lists:nth(I, ScopedKeys), + <<"structured@1.0">>, + Opts#{ <<"topic">> => ao_internal } + ), + [ + #{ <<"device">> => DevID }, + Direct#{ <<"...">> => StepMsg } + | + do_build(I + 1, Rest, ScopedKeys, Opts) + ]; do_build(I, [Msg | Rest], ScopedKeys, Opts) when not is_map(Msg) -> [Msg | do_build(I + 1, Rest, ScopedKeys, Opts)]; do_build(I, [Msg | Rest], ScopedKeys, Opts) -> Additional = lists:nth(I, ScopedKeys), - Merged = hb_maps:merge(Additional, Msg, Opts), + Merged = hb_ao:set(Additional, Msg, Opts), StepMsg = hb_message:convert( Merged, <<"structured@1.0">>, @@ -443,15 +425,6 @@ maybe_typed(Key, Value, Opts) -> end end. -%% @doc Join a list of items with a separator, or return the first item if there -%% is only one item. If there are no items, return an empty binary. -maybe_join(Items, Sep) -> - case length(Items) of - 0 -> <<>>; - 1 -> hd(Items); - _ -> iolist_to_binary(lists:join(Sep, Items)) - end. - %%% Tests parse_explicit_message_test() -> diff --git a/src/core/resolver/hb_types.erl b/src/core/resolver/hb_types.erl new file mode 100644 index 000000000..249e667f3 --- /dev/null +++ b/src/core/resolver/hb_types.erl @@ -0,0 +1,739 @@ +%%% @doc Extract device specs and vary AO-Core execution inputs. +-module(hb_types). +-export([extract/2, vary/2, add_schema/2, beam_to_schema/2]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Add the schema for a resolution to an execution context, given a +%% resolved key, function, and execution module (e.g, from +%% `hb_device:add_resolver`). +add_schema( + Ctx = + #{ + <<"key">> := Key, + <<"resolver-device">> := Device, + <<"priv">> := + #{ + <<"function">> := Func, + <<"add-key">> := AddKey + } + }, + Opts) -> + case schema_from_device(Device, Func, Key, Opts) of + undefined -> + ?prim_dbg( + {schema_not_found, + {device, Device}, + {func, Func}, + {key, Key} + } + ), + {ok, Ctx}; + Schema -> + ?prim_dbg( + {schema_found, + {device, Device}, + {func, Func}, + {key, Key}, + {schema, Schema} + } + ), + {ok, Ctx#{ <<"schema">> => execution_schema(Schema, AddKey) } } + end. + +%% @doc Apply the resolved function's base/request schemas to one execution. +vary(Ctx = #{ <<"base">> := Base, <<"request">> := Req }, _Opts) + when not is_map_key(<<"schema">>, Ctx) -> + {ok, + Ctx#{ + <<"varied-base">> => Base, + <<"varied-request">> => Req, + <<"normalizer">> => none + } + }; +vary(Ctx = #{ + <<"schema">> := [BaseSchema, ReqSchema, ReturnSchema], + <<"base">> := Base, + <<"request">> := Req +}, Opts) -> + {ok, + Ctx#{ + <<"varied-base">> => apply_schema(implicit_base(BaseSchema), Base, Opts), + <<"varied-request">> => apply_schema(implicit_request(ReqSchema), Req, Opts), + <<"normalizer">> => overlay(ReturnSchema) + } + }. + +%% @doc Extract a device module's function schemas. We first check the +%% `loaded-device-store` cache, then fall back to loading the module manually +%% if it isn't already available. +extract(Device, _Opts) when is_map(Device) -> + {error, {unsupported_device_type, Device}}; +extract(Module, Opts) when is_atom(Module) -> + % If we are already caching a schema at the moment, skip the recursive cache + % call and error early. + case hb_opts:get(<<"caching-schema">>, false, Opts) of + true -> + {error, caching_schema}; + false -> + case hb_device_load:schema(Module, Opts) of + {ok, Schema} -> {ok, Schema}; + _ -> + case code:ensure_loaded(Module) of + {module, Module} -> beam_to_schema(Module); + {error, Reason} -> {error, {module_not_loaded, Module, Reason}} + end + end + end; +extract(Device, Opts) when is_binary(Device) -> + case hb_device_load:reference(Device, Opts) of + {ok, Module} -> extract(Module, Opts); + Error -> Error + end; +extract(Device, _Opts) -> + {error, {unsupported_device_type, Device}}. + +schema_from_device(Device, Func, Key, Opts) -> + case extract(Device, Opts) of + {ok, #{ <<"keys">> := Schemas }} -> + select_schema(Func, Key, Schemas); + {error, _Reason} -> + undefined + end. + +select_schema(Func, _Key, Schemas) -> + case erlang:fun_info(Func, name) of + {name, Name} -> + maps:get(normalize_name(Name), Schemas, undefined); + _ -> + undefined + end. + +execution_schema(Schema, AddKey) -> + Args = maps:get(<<"args">>, Schema, []), + Offset = + case AddKey of + false -> 0; + _ -> 1 + end, + [ + nth_or(1 + Offset, Args, any_type()), + nth_or(2 + Offset, Args, any_type()), + maps:get(<<"return">>, Schema, any_type()) + ]. + +nth_or(N, List, _Default) when length(List) >= N -> lists:nth(N, List); +nth_or(_N, _List, Default) -> Default. + +implicit_base(Schema) -> + implicit_key(top_level_schema(Schema), <<"device">>, optional). + +implicit_request(Schema) -> + implicit_key(top_level_schema(Schema), <<"path">>, optional). + +top_level_schema(#{ <<"kind">> := <<"empty">> }) -> + message_type({#{}, none}); +top_level_schema(Schema) -> + Schema. + +implicit_key(Schema = #{ <<"kind">> := <<"message">>, <<"keys">> := Keys }, + Key, + Presence) -> + case maps:is_key(Key, Keys) of + true -> + Schema; + false -> + Schema#{ + <<"keys">> => + Keys#{ + Key => + #{ + <<"presence">> => Presence, + <<"type">> => any_type() + } + } + } + end; +implicit_key(Schema, _Key, _Presence) -> + Schema. + +beam_to_schema(Module) -> + case module_beam(Module) of + unavailable -> + {error, {abstract_code_unavailable, Module, unavailable}}; + Beam -> beam_to_schema(Module, Beam) + end. + +beam_to_schema(Module, Beam) -> + case beam_lib:chunks(Beam, [abstract_code]) of + {ok, {_, [{abstract_code, {_, Forms}}]}} -> + TypeEnv = build_type_env(Forms), + {ok, + #{ + <<"keys">> => + lists:foldl( + fun + ({attribute, _, spec, {{Name, _}, [Head | _]}}, Acc) -> + Acc#{ + normalize_name(Name) => + fun_schema(Head, TypeEnv) + }; + (_, Acc) -> + Acc + end, + #{}, + Forms + ) + }}; + Error -> + {error, {abstract_code_unavailable, Module, Error}} + end. + +module_beam(Module) -> + case code:get_object_code(Module) of + {Module, Beam, _Filename} -> + Beam; + _ -> + case code:which(Module) of + Path when is_list(Path); is_binary(Path) -> Path; + _ -> unavailable + end + end. + +build_type_env(Forms) -> + lists:foldl( + fun + ({attribute, _, Tag, {Name, Ast, Vars}}, Acc) + when Tag =:= type; Tag =:= opaque -> + Acc#{ + Name => + #{ + vars => [var_name(Var) || Var <- Vars], + ast => Ast + } + }; + (_, Acc) -> + Acc + end, + #{}, + Forms + ). + +fun_schema(Spec, TypeEnv) -> + {Args, Return} = parse_fun_spec(Spec, TypeEnv), + #{ + <<"args">> => Args, + <<"return">> => Return + }. + +parse_fun_spec({type, _, bounded_fun, [FunSpec, _Constraints]}, TypeEnv) -> + parse_fun_spec(FunSpec, TypeEnv); +parse_fun_spec({type, _, 'fun', [{type, _, product, Args}, Return]}, TypeEnv) -> + { + [parse_type(Arg, TypeEnv, #{}, []) || Arg <- Args], + parse_type(Return, TypeEnv, #{}, []) + }; +parse_fun_spec(Other, _TypeEnv) -> + {[unknown_type(Other)], any_type()}. + +parse_type({ann_type, _, [_Var, Type]}, TypeEnv, VarEnv, Seen) -> + parse_type(Type, TypeEnv, VarEnv, Seen); +parse_type({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + empty_type(); +parse_type({var, _, Name}, TypeEnv, VarEnv, Seen) -> + case maps:get(Name, VarEnv, undefined) of + undefined -> variable_type(Name); + Bound -> parse_type(Bound, TypeEnv, VarEnv, Seen) + end; +parse_type({user_type, _, Name, Args}, TypeEnv, VarEnv, Seen) -> + case lists:member(Name, Seen) of + true -> + alias_type(Name); + false -> + case maps:get(Name, TypeEnv, undefined) of + undefined -> + alias_type(Name); + #{ vars := Vars, ast := Ast } -> + parse_type( + Ast, + TypeEnv, + maps:merge(VarEnv, maps:from_list(lists:zip(Vars, Args))), + [Name | Seen] + ) + end + end; +parse_type({remote_type, _, [{atom, _, Mod}, {atom, _, Name}, Args]}, + TypeEnv, + VarEnv, + Seen) -> + #{ + <<"kind">> => <<"remote">>, + <<"module">> => normalize_name(Mod), + <<"name">> => normalize_name(Name), + <<"args">> => [parse_type(Arg, TypeEnv, VarEnv, Seen) || Arg <- Args] + }; +parse_type({type, _, map, any}, _TypeEnv, _VarEnv, _Seen) -> + any_type(); +parse_type({type, _, map, Fields}, TypeEnv, VarEnv, Seen) -> + message_type(parse_fields(Fields, TypeEnv, VarEnv, Seen)); +parse_type({type, _, ListType, [Item]}, TypeEnv, VarEnv, Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ <<"kind">> => <<"list">>, <<"item">> => parse_type(Item, TypeEnv, VarEnv, Seen) }; +parse_type({type, _, ListType, []}, _TypeEnv, _VarEnv, _Seen) + when ListType =:= list; ListType =:= nonempty_list -> + #{ <<"kind">> => <<"list">>, <<"item">> => any_type() }; +parse_type({type, _, tuple, Items}, TypeEnv, VarEnv, Seen) -> + #{ <<"kind">> => <<"tuple">>, <<"items">> => + [parse_type(Item, TypeEnv, VarEnv, Seen) || Item <- Items] }; +parse_type({type, _, union, Members}, TypeEnv, VarEnv, Seen) -> + #{ <<"kind">> => <<"union">>, <<"members">> => + [parse_type(Member, TypeEnv, VarEnv, Seen) || Member <- Members] }; +parse_type({type, _, range, [Min, Max]}, TypeEnv, VarEnv, Seen) -> + #{ + <<"kind">> => <<"range">>, + <<"min">> => literal_value(parse_type(Min, TypeEnv, VarEnv, Seen)), + <<"max">> => literal_value(parse_type(Max, TypeEnv, VarEnv, Seen)) + }; +parse_type({type, _, integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"integer">>); +parse_type({type, _, non_neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"non-neg-integer">>); +parse_type({type, _, pos_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pos-integer">>); +parse_type({type, _, neg_integer, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"neg-integer">>); +parse_type({type, _, float, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"float">>); +parse_type({type, _, number, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"number">>); +parse_type({type, _, binary, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"binary">>); +parse_type({type, _, bitstring, _}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"bitstring">>); +parse_type({type, _, boolean, []}, _TypeEnv, _VarEnv, _Seen) -> boolean_type(); +parse_type({type, _, atom, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"atom">>); +parse_type({type, _, pid, []}, _TypeEnv, _VarEnv, _Seen) -> scalar_type(<<"pid">>); +parse_type({type, _, any, []}, _TypeEnv, _VarEnv, _Seen) -> any_type(); +parse_type({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(Atom)); +parse_type({integer, _, Int}, _TypeEnv, _VarEnv, _Seen) -> literal_type(Int); +parse_type({char, _, Char}, _TypeEnv, _VarEnv, _Seen) -> literal_type(<>); +parse_type({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> literal_type(hb_util:bin(String)); +parse_type({nil, _}, _TypeEnv, _VarEnv, _Seen) -> literal_type([]); +parse_type(Other, _TypeEnv, _VarEnv, _Seen) -> + unknown_type(Other). + +parse_fields(Fields, TypeEnv, VarEnv, Seen) -> + lists:foldl( + fun({type, _, Assoc, [KeyAst, ValueAst]}, {Keys, Wildcard}) -> + Field = + #{ + <<"presence">> => field_presence(Assoc), + <<"type">> => parse_type(ValueAst, TypeEnv, VarEnv, Seen) + }, + case key_name(KeyAst, TypeEnv, VarEnv, Seen) of + <<"_">> -> {Keys, Field}; + Key -> {Keys#{ Key => Field }, Wildcard} + end + end, + {#{}, none}, + Fields + ). + +field_presence(map_field_exact) -> required; +field_presence(map_field_assoc) -> optional; +field_presence(Other) -> normalize_name(Other). + +key_name({atom, _, Atom}, _TypeEnv, _VarEnv, _Seen) -> + normalize_name(Atom); +key_name({string, _, String}, _TypeEnv, _VarEnv, _Seen) -> + hb_util:bin(String); +key_name({var, _, '_'}, _TypeEnv, _VarEnv, _Seen) -> + <<"_">>; +key_name(Other, TypeEnv, VarEnv, Seen) -> + case parse_type(Other, TypeEnv, VarEnv, Seen) of + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } when is_binary(Value) -> + Value; + #{ <<"kind">> := <<"literal">>, <<"value">> := Value } -> + hb_util:bin(io_lib:format("~tp", [Value])); + _ -> + hb_util:bin(io_lib:format("~tp", [Other])) + end. + +apply_schema(#{ <<"kind">> := <<"any">> }, Value, _Opts) -> + Value; +apply_schema(#{ <<"kind">> := <<"empty">> }, Value, _Opts) -> + Value; +apply_schema(Schema = #{ <<"kind">> := <<"message">> }, Value, Opts) + when not is_map(Value) -> + apply_schema(Schema, hb_cache:ensure_loaded(Value, Opts), Opts); +apply_schema( + #{ <<"kind">> := <<"message">>, <<"keys">> := Keys, <<"wildcard">> := Wildcard }, + Message, + Opts +) when is_map(Message) -> + Explicit = explicit_keys(Keys, Message, Opts), + maps:merge(wildcard_keys(Wildcard, Keys, Message, Opts), Explicit); +apply_schema(Type, Value, Opts) -> + case check_type(Type, Value) of + true -> + Value; + false -> + case coerce_type(Type, Value, Opts) of + error -> throw({invalid_type, Type, Value}); + Coerced -> + case check_type(Type, Coerced) of + true -> Coerced; + false -> throw({invalid_type, Type, Value}) + end + end + end. + +explicit_keys(Keys, Message, Opts) -> + maps:fold( + fun(Key, #{ <<"presence">> := Presence, <<"type">> := Type }, Acc) -> + case hb_ao:raw(Message, #{ <<"path">> => Key }, Opts) of + {ok, Value} -> + Acc#{ Key => apply_schema(Type, Value, Opts) }; + {error, not_found} when Presence =:= required -> + throw({required_key_missing, Key}); + {error, not_found} -> + Acc + end + end, + #{}, + Keys + ). + +wildcard_keys(none, _Keys, _Message, _Opts) -> + #{}; +wildcard_keys(#{ <<"presence">> := optional }, Keys, Message, _Opts) -> + maps:without(maps:keys(Keys), Message); +wildcard_keys(#{ <<"presence">> := required, <<"type">> := Type }, Keys, Message, Opts) -> + maps:map( + fun(_Key, Value) -> + apply_schema(Type, hb_cache:ensure_all_loaded(Value, Opts), Opts) + end, + hb_util:list_without(maps:keys(Keys), hb_ao:keys(Message, Opts)) + ). + +overlay(ReturnSchema) -> + case overlay_type(ReturnSchema) of + base -> base; + request -> request; + _ -> none + end. + +overlay_type(#{ <<"kind">> := <<"message">>, <<"keys">> := Keys }) -> + overlay_marker( + maps:get( + <<"type">>, + maps:get(<<"...">>, Keys, #{}), + #{} + ) + ); +overlay_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }) -> + first_overlay(Items); +overlay_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }) -> + first_overlay(Members); +overlay_type(_) -> + none. + +first_overlay([]) -> + none; +first_overlay([Schema | Rest]) -> + case overlay_type(Schema) of + none -> first_overlay(Rest); + Overlay -> Overlay + end. + +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"literal">>, <<"value">> := <<"request">> }) -> request; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"base">> }) -> base; +overlay_marker(#{ <<"kind">> := <<"alias">>, <<"name">> := <<"request">> }) -> request; +overlay_marker(_) -> none. + +coerce_type(_, undefined, _Opts) -> error; +coerce_type(#{ <<"kind">> := <<"integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"pos-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"neg-integer">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(#{ <<"kind">> := <<"float">> }, Value, _Opts) -> + try_coerce(fun hb_util:float/1, Value); +coerce_type(#{ <<"kind">> := <<"number">> }, Value, _Opts) -> + coerce_with([fun hb_util:int/1, fun hb_util:float/1], Value); +coerce_type(#{ <<"kind">> := <<"binary">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"bitstring">> }, Value, _Opts) -> + try_coerce(fun hb_util:bin/1, Value); +coerce_type(#{ <<"kind">> := <<"boolean">> }, Value, _Opts) -> + case lists:member(Value, [true, false, 1, 0, <<"true">>, <<"false">>, <<"1">>, <<"0">>]) of + true -> try_coerce(fun hb_util:bool/1, Value); + false -> error + end; +coerce_type(#{ <<"kind">> := <<"atom">> }, Value, _Opts) -> + try_coerce(fun hb_util:atom/1, Value); +coerce_type(#{ <<"kind">> := <<"message">> }, Value, _Opts) -> + try_coerce(fun hb_util:map/1, Value); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_tuple(Value) -> + coerce_type(#{ <<"kind">> => <<"tuple">>, <<"items">> => Items }, tuple_to_list(Value), Opts); +coerce_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value, Opts) + when is_list(Value) -> + case length(Value) =:= length(Items) of + true -> coerce_tuple(Items, Value, Opts); + false -> error + end; +coerce_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value, Opts) -> + case try_coerce(fun hb_util:list/1, Value) of + error -> error; + Coerced -> coerce_list(ItemType, Coerced, Opts) + end; +coerce_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value, Opts) -> + coerce_union(Members, Value, Opts); +coerce_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value, _Opts) -> + coerce_literal(Expected, Value); +coerce_type(#{ <<"kind">> := <<"range">> }, Value, _Opts) -> + try_coerce(fun hb_util:int/1, Value); +coerce_type(_, _, _Opts) -> + error. + +coerce_tuple(Items, Values, Opts) -> + case coerce_sequence(lists:zip(Items, Values), Opts) of + error -> error; + Coerced -> list_to_tuple(Coerced) + end. + +coerce_list(ItemType, Value, Opts) when is_list(Value) -> + coerce_sequence([{ItemType, Item} || Item <- Value], Opts); +coerce_list(_ItemType, _Value, _Opts) -> + error. + +coerce_sequence([], _Opts) -> + []; +coerce_sequence([{Type, Value} | Rest], Opts) -> + case coerce_type(Type, Value, Opts) of + error -> + error; + Coerced -> + case coerce_sequence(Rest, Opts) of + error -> error; + CoercedRest -> [Coerced | CoercedRest] + end + end. + +coerce_union([], _Value, _Opts) -> + error; +coerce_union([Member | Rest], Value, Opts) -> + case coerce_type(Member, Value, Opts) of + error -> coerce_union(Rest, Value, Opts); + Coerced -> Coerced + end. + +coerce_literal(Expected, Value) when is_integer(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:int/1, Value)); +coerce_literal(Expected, Value) when is_float(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:float/1, Value)); +coerce_literal(Expected, Value) when is_binary(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:bin/1, Value)); +coerce_literal(Expected, Value) when is_atom(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:atom/1, Value)); +coerce_literal(Expected, Value) when is_list(Expected) -> + coerce_exact(Expected, try_coerce(fun hb_util:list/1, Value)); +coerce_literal(Expected, Expected) -> + Expected; +coerce_literal(_Expected, _Value) -> + error. + +coerce_exact(Expected, Expected) -> + Expected; +coerce_exact(_Expected, _Value) -> + error. + +try_coerce(Fun, Value) -> + try Fun(Value) of + Coerced -> Coerced + catch + _:_ -> error + end. + +coerce_with([], _Value) -> + error; +coerce_with([Fun | Rest], Value) -> + case try_coerce(Fun, Value) of + error -> coerce_with(Rest, Value); + Coerced -> Coerced + end. + +check_type(#{ <<"kind">> := <<"integer">> }, Value) -> is_integer(Value); +check_type(#{ <<"kind">> := <<"non-neg-integer">> }, Value) -> is_integer(Value) andalso Value >= 0; +check_type(#{ <<"kind">> := <<"pos-integer">> }, Value) -> is_integer(Value) andalso Value > 0; +check_type(#{ <<"kind">> := <<"neg-integer">> }, Value) -> is_integer(Value) andalso Value < 0; +check_type(#{ <<"kind">> := <<"float">> }, Value) -> is_float(Value); +check_type(#{ <<"kind">> := <<"number">> }, Value) -> is_number(Value); +check_type(#{ <<"kind">> := <<"binary">> }, Value) -> is_binary(Value); +check_type(#{ <<"kind">> := <<"bitstring">> }, Value) -> is_bitstring(Value); +check_type(#{ <<"kind">> := <<"boolean">> }, Value) -> is_boolean(Value); +check_type(#{ <<"kind">> := <<"atom">> }, Value) -> is_atom(Value); +check_type(#{ <<"kind">> := <<"pid">> }, Value) -> is_pid(Value); +check_type(#{ <<"kind">> := <<"message">> }, Value) -> is_map(Value); +check_type(#{ <<"kind">> := <<"tuple">>, <<"items">> := Items }, Value) -> + is_tuple(Value) + andalso tuple_size(Value) =:= length(Items) + andalso lists:all( + fun({Index, ItemType}) -> check_type(ItemType, element(Index, Value)) end, + lists:zip(lists:seq(1, length(Items)), Items) + ); +check_type(#{ <<"kind">> := <<"list">>, <<"item">> := ItemType }, Value) -> + is_list(Value) andalso lists:all(fun(Item) -> check_type(ItemType, Item) end, Value); +check_type(#{ <<"kind">> := <<"union">>, <<"members">> := Members }, Value) -> + lists:any(fun(Member) -> check_type(Member, Value) end, Members); +check_type(#{ <<"kind">> := <<"literal">>, <<"value">> := Expected }, Value) -> + Value =:= Expected; +check_type(#{ <<"kind">> := <<"range">>, <<"min">> := Min, <<"max">> := Max }, Value) -> + is_integer(Value) andalso Value >= Min andalso Value =< Max; +check_type(#{ <<"kind">> := <<"remote">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"alias">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"variable">> }, _Value) -> true; +check_type(#{ <<"kind">> := <<"unknown">> }, _Value) -> true; +check_type(_, _Value) -> true. + +normalize_name('_') -> <<"_">>; +normalize_name(Name) when is_atom(Name) -> hb_util:atom_to_dashed_binary(Name); +normalize_name(Name) -> hb_util:bin(Name). + +literal_value(#{ <<"kind">> := <<"literal">>, <<"value">> := Value }) -> Value; +literal_value(_) -> undefined. + +var_name({var, _, Name}) -> Name; +var_name(Name) -> Name. + +message_type({Keys, Wildcard}) -> + #{ + <<"kind">> => <<"message">>, + <<"keys">> => Keys, + <<"wildcard">> => Wildcard + }. + +any_type() -> #{ <<"kind">> => <<"any">> }. +empty_type() -> #{ <<"kind">> => <<"empty">> }. +scalar_type(Name) -> #{ <<"kind">> => Name }. +literal_type(Value) -> #{ <<"kind">> => <<"literal">>, <<"value">> => Value }. +alias_type(Name) -> #{ <<"kind">> => <<"alias">>, <<"name">> => normalize_name(Name) }. +variable_type(Name) -> #{ <<"kind">> => <<"variable">>, <<"name">> => normalize_name(Name) }. +unknown_type(Other) -> + #{ <<"kind">> => <<"unknown">>, <<"ast">> => hb_util:bin(io_lib:format("~tp", [Other])) }. +boolean_type() -> + #{ + <<"kind">> => <<"union">>, + <<"members">> => [literal_type(true), literal_type(false)] + }. + +%%% Tests + +empty_projection_test() -> + ?assertEqual( + #{ <<"device">> => <<"test-device@1.0">> }, + apply_schema( + implicit_base(empty_type()), + #{ <<"device">> => <<"test-device@1.0">>, <<"extra">> => <<"drop">> }, + #{} + ) + ). + +map_wildcards_test() -> + Lazy = + parse_type( + {type, 1, map, + [ + {type, 1, map_field_exact, [{atom, 1, a}, {var, 1, '_'}]}, + {type, 1, map_field_assoc, [{var, 1, '_'}, {var, 1, '_'}]} + ]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"keys">> := #{ <<"a">> := _ }, + <<"wildcard">> := #{ <<"presence">> := optional } + }, + Lazy + ), + Force = + parse_type( + {type, 1, map, + [ + {type, 1, map_field_exact, [{var, 1, '_'}, {var, 1, '_'}]} + ]}, + #{}, + #{}, + [] + ), + ?assertMatch( + #{ + <<"kind">> := <<"message">>, + <<"wildcard">> := #{ <<"presence">> := required } + }, + Force + ). + +default_handler_uses_resolved_function_schema_test() -> + Func = fun default_schema_fun/4, + ?assertEqual( + default_schema, + select_schema( + Func, + <<"requested-key">>, + #{ + <<"default-schema-fun">> => default_schema, + <<"requested-key">> => requested_key_schema + } + ) + ), + ?assertEqual( + undefined, + select_schema( + Func, + <<"requested-key">>, + #{ <<"other-key">> => other_schema } + ) + ). + +default_schema_fun(_Key, _Base, _Req, _Opts) -> + {ok, unused}. + +extension_projection_test() -> + Schema = + message_type( + { + #{ + <<"a">> => #{ <<"presence">> => required, <<"type">> => empty_type() }, + <<"c">> => #{ <<"presence">> => optional, <<"type">> => empty_type() } + }, + none + } + ), + Msg = + #{ + <<"b">> => 2, + <<"...">> => + #{ + <<"a">> => 1, + <<"b">> => 1 + } + }, + ?assertEqual(#{ <<"a">> => 1 }, apply_schema(Schema, Msg, #{})). + +extension_wildcard_carry_test() -> + Schema = + message_type( + { + #{ + <<"a">> => #{ <<"presence">> => optional, <<"type">> => empty_type() } + }, + #{ <<"presence">> => optional, <<"type">> => empty_type() } + } + ), + Parent = #{ <<"a">> => 1, <<"b">> => 1, <<"c">> => 1 }, + Msg = #{ <<"b">> => 2, <<"...">> => Parent }, + ?assertEqual( + #{ <<"a">> => 1, <<"b">> => 2, <<"...">> => Parent }, + apply_schema(Schema, Msg, #{}) + ). diff --git a/src/core/test/hb_ao_test_vectors.erl b/src/core/test/hb_ao_test_vectors.erl index 1f2e3877f..e0f2e1b04 100644 --- a/src/core/test/hb_ao_test_vectors.erl +++ b/src/core/test/hb_ao_test_vectors.erl @@ -49,8 +49,6 @@ test_suite() -> fun resolve_from_multiple_keys_test/1}, {resolve_path_element, "resolve path element", fun resolve_path_element_test/1}, - {resolve_binary_key, "resolve binary key", - fun resolve_binary_key_test/1}, {key_to_binary, "key to binary", fun key_to_binary_test/1}, {key_from_id_device_with_args, "key from id device with args", @@ -124,82 +122,82 @@ test_opts() -> desc => "Default opts", opts => #{ <<"store">> => hb_test_utils:test_store() }, skip => [] - }, - #{ - name => without_hashpath, - desc => "Default without hashpath", - opts => #{ - <<"hashpath">> => ignore, - <<"store">> => hb_test_utils:test_store() - }, - skip => [] - }, - #{ - name => no_cache, - desc => "No cache read or write", - opts => #{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], - <<"spawn-worker">> => false, - <<"store">> => hb_test_utils:test_store() - }, - skip => [load_as] - }, - #{ - name => only_store, - desc => "Store, don't read", - opts => #{ - <<"hashpath">> => update, - <<"cache-control">> => [<<"no-cache">>], - <<"spawn-worker">> => false, - <<"store">> => CachedExecStore - }, - skip => [ - denormalized_device_name, - deep_set_with_device, - load_as - ], - reset => false - }, - #{ - name => only_if_cached, - desc => "Only read, don't exec", - opts => #{ - <<"hashpath">> => ignore, - <<"cache-control">> => [<<"only-if-cached">>], - <<"spawn-worker">> => false, - <<"store">> => CachedExecStore - }, - skip => [ - % Skip test with locally defined device, amongst others. - resolve_id, - start_as, - start_as_with_parameters, - as_path, - multiple_as_subresolutions, - key_from_id_device_with_args, - get_with_denormalized_key, - set_new_messages, - resolve_from_multiple_keys, - resolve_path_element, - device_with_default_handler_function, - device_with_handler_function, - denormalized_device_name, - get_with_device, - get_as_with_device, - set_with_device, - device_exports, - device_excludes, - device_inheritance, - deep_set_with_device, - as, - as_commitments, - step_hook, - paranoid_message_verification, - paranoid_input_verification, - paranoid_result_verification - ] } + % #{ + % name => without_hashpath, + % desc => "Default without hashpath", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"store">> => hb_test_utils:test_store() + % }, + % skip => [] + % }, + % #{ + % name => no_cache, + % desc => "No cache read or write", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"cache-control">> => [<<"no-cache">>, <<"no-store">>], + % <<"spawn-worker">> => false, + % <<"store">> => hb_test_utils:test_store() + % }, + % skip => [load_as] + % }, + % #{ + % name => only_store, + % desc => "Store, don't read", + % opts => #{ + % <<"hashpath">> => update, + % <<"cache-control">> => [<<"no-cache">>], + % <<"spawn-worker">> => false, + % <<"store">> => CachedExecStore + % }, + % skip => [ + % denormalized_device_name, + % deep_set_with_device, + % load_as + % ], + % reset => false + % }, + % #{ + % name => only_if_cached, + % desc => "Only read, don't exec", + % opts => #{ + % <<"hashpath">> => ignore, + % <<"cache-control">> => [<<"only-if-cached">>], + % <<"spawn-worker">> => false, + % <<"store">> => CachedExecStore + % }, + % skip => [ + % % Skip test with locally defined device, amongst others. + % resolve_id, + % start_as, + % start_as_with_parameters, + % as_path, + % multiple_as_subresolutions, + % key_from_id_device_with_args, + % get_with_denormalized_key, + % set_new_messages, + % resolve_from_multiple_keys, + % resolve_path_element, + % device_with_default_handler_function, + % device_with_handler_function, + % denormalized_device_name, + % get_with_device, + % get_as_with_device, + % set_with_device, + % device_exports, + % device_excludes, + % device_inheritance, + % deep_set_with_device, + % as, + % as_commitments, + % step_hook, + % paranoid_message_verification, + % paranoid_input_verification, + % paranoid_result_verification + % ] + % } ]. %%% Standalone test vectors @@ -343,7 +341,7 @@ resolve_simple_test(Opts) -> resolve_id_test(Opts) -> ?assertMatch( - ID when byte_size(ID) == 43, + ID when ?IS_ID(ID), hb_ao:get(<<"id">>, #{ <<"test_key">> => <<"1">> }, Opts) ). @@ -375,22 +373,6 @@ key_to_binary_test(Opts) -> ?assertEqual(<<"a">>, hb_ao:normalize_key(<<"a">>, Opts)), ?assertEqual(<<"a">>, hb_ao:normalize_key("a", Opts)). -resolve_binary_key_test(Opts) -> - ?assertEqual( - {ok, <<"RESULT">>}, - hb_ao:resolve(#{ a => <<"RESULT">> }, <<"a">>, Opts) - ), - ?assertEqual( - {ok, <<"1">>}, - hb_ao:resolve( - #{ - <<"Test-Header">> => <<"1">> - }, - <<"Test-Header">>, - Opts - ) - ). - %% @doc Generates a test device with three keys, each of which uses %% progressively more of the arguments that can be passed to a device key. generate_device_with_keys_using_args() -> @@ -398,15 +380,15 @@ generate_device_with_keys_using_args() -> key_using_only_state => fun(State) -> {ok, - <<(hb_maps:get(<<"state_key">>, State))/binary>> + <<(hb_maps:get(<<"state-key">>, State))/binary>> } end, key_using_state_and_msg => fun(State, Msg) -> {ok, << - (hb_maps:get(<<"state_key">>, State))/binary, - (hb_maps:get(<<"msg_key">>, Msg))/binary + (hb_maps:get(<<"state-key">>, State))/binary, + (hb_maps:get(<<"msg-key">>, Msg))/binary >> } end, @@ -414,9 +396,9 @@ generate_device_with_keys_using_args() -> fun(State, Msg, Opts) -> {ok, << - (hb_maps:get(<<"state_key">>, State, undefined, Opts))/binary, - (hb_maps:get(<<"msg_key">>, Msg, undefined, Opts))/binary, - (hb_maps:get(<<"opts_key">>, Opts, undefined, Opts))/binary + (hb_maps:get(<<"state-key">>, State, undefined, Opts))/binary, + (hb_maps:get(<<"msg-key">>, Msg, undefined, Opts))/binary, + (hb_maps:get(<<"opts-key">>, Opts, undefined, Opts))/binary >> } end @@ -434,7 +416,7 @@ gen_default_device() -> end } end, - <<"state_key">> => + <<"state-key">> => fun(_) -> {ok, <<"STATE">>} end @@ -455,6 +437,13 @@ gen_handler_device() -> M2, Opts ); + (<<"vary">>, Base, Req, Opts) -> + {ok, + #{ + <<"varied-base">> => Base, + <<"varied-request">> => Req + } + }; (_, _, _, _) -> {ok, <<"HANDLER VALUE">>} end @@ -468,8 +457,8 @@ gen_handler_device() -> key_from_id_device_with_args_test(Opts) -> Msg = #{ - device => generate_device_with_keys_using_args(), - state_key => <<"1">> + <<"device">> => generate_device_with_keys_using_args(), + <<"state-key">> => <<"1">> }, ?assertEqual( {ok, <<"1">>}, @@ -477,7 +466,7 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_only_state">>, - <<"msg_key">> => <<"2">> % Param message, which is ignored + <<"msg-key">> => <<"2">> % Param message, which is ignored }, Opts ) @@ -488,7 +477,7 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_state_and_msg">>, - <<"msg_key">> => <<"3">> % Param message, with value to add + <<"msg-key">> => <<"3">> % Param message, with value to add }, Opts ) @@ -499,46 +488,45 @@ key_from_id_device_with_args_test(Opts) -> Msg, #{ <<"path">> => <<"key_using_all">>, - <<"msg_key">> => <<"3">> % Param message + <<"msg-key">> => <<"3">> % Param message }, Opts#{ - <<"opts_key">> => <<"37">>, + <<"opts-key">> => <<"37">>, <<"cache-control">> => [<<"no-cache">>, <<"no-store">>] } ) ). device_with_handler_function_test(Opts) -> - Msg = - #{ - device => gen_handler_device(), - test_key => <<"BAD">> - }, ?assertEqual( {ok, <<"HANDLER VALUE">>}, - hb_ao:resolve(Msg, <<"test_key">>, Opts) + hb_ao:resolve( + hb_ao:as(gen_handler_device(), #{ <<"test-key">> => <<"BAD">> }, Opts), + #{ <<"path">> => <<"test-key">> }, + Opts + ) ). device_with_default_handler_function_test(Opts) -> Msg = #{ - device => gen_default_device() + <<"device">> => gen_default_device() }, ?assertEqual( {ok, <<"STATE">>}, - hb_ao:resolve(Msg, <<"state_key">>, Opts) + hb_ao:resolve(Msg, #{<<"path">> => <<"state-key">>}, Opts) ), ?assertEqual( {ok, <<"DEFAULT">>}, - hb_ao:resolve(Msg, <<"any_random_key">>, Opts) + hb_ao:resolve(Msg, #{<<"path">> => <<"any-random-key">>}, Opts) ). basic_get_test(Opts) -> Msg = #{ <<"key1">> => <<"value1">>, <<"key2">> => <<"value2">> }, ?assertEqual(<<"value1">>, hb_ao:get(<<"key1">>, Msg, Opts)), ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)), - ?assertEqual(<<"value2">>, hb_ao:get([<<"key2">>], Msg, Opts)). + % Repeat to ensure consistency in caching setups. + ?assertEqual(<<"value2">>, hb_ao:get(<<"key2">>, Msg, Opts)). recursive_get_test(Opts) -> Msg = #{ @@ -591,24 +579,24 @@ get_with_device_test(Opts) -> Msg = #{ <<"device">> => generate_device_with_keys_using_args(), - <<"state_key">> => <<"STATE">> + <<"state-key">> => <<"STATE">> }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"key_using_only_state">>, Msg, Opts)). + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, Msg, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"key-using-only-state">>, Msg, Opts)). get_as_with_device_test(Opts) -> Msg = #{ <<"device">> => gen_handler_device(), - <<"test_key">> => <<"ACTUAL VALUE">> + <<"test-key">> => <<"ACTUAL VALUE">> }, ?assertEqual( <<"HANDLER VALUE">>, - hb_ao:get(test_key, Msg, Opts) + hb_ao:get(<<"test-key">>, Msg, Opts) ), ?assertEqual( <<"ACTUAL VALUE">>, - hb_ao:get(test_key, {as, <<"message@1.0">>, Msg}, Opts) + hb_ao:get(<<"test-key">>, hb_ao:as(<<"message@1.0">>, Msg, Opts), Opts) ). set_with_device_test(Opts) -> @@ -618,22 +606,22 @@ set_with_device_test(Opts) -> #{ <<"set">> => fun(State, _Msg) -> - Acc = hb_maps:get(<<"set_count">>, State, <<"">>, Opts), + Acc = hb_maps:get(<<"set-count">>, State, <<"">>, Opts), {ok, State#{ - <<"set_count">> => << Acc/binary, "." >> + <<"set-count">> => << Acc/binary, "." >> } } end }, - <<"state_key">> => <<"STATE">> + <<"state-key">> => <<"STATE">> }, - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, Msg, Opts)), - SetOnce = hb_ao:set(Msg, #{ <<"state_key">> => <<"SET_ONCE">> }, Opts), - ?assertEqual(<<".">>, hb_ao:get(<<"set_count">>, SetOnce, Opts)), - SetTwice = hb_ao:set(SetOnce, #{ <<"state_key">> => <<"SET_TWICE">> }, Opts), - ?assertEqual(<<"..">>, hb_ao:get(<<"set_count">>, SetTwice, Opts)), - ?assertEqual(<<"STATE">>, hb_ao:get(<<"state_key">>, SetTwice, Opts)). + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, Msg, Opts)), + SetOnce = hb_ao:set(Msg, #{ <<"state-key">> => <<"SET_ONCE">> }, Opts), + ?assertEqual(<<".">>, hb_ao:get(<<"set-count">>, SetOnce, Opts)), + SetTwice = hb_ao:set(SetOnce, #{ <<"state-key">> => <<"SET_TWICE">> }, Opts), + ?assertEqual(<<"..">>, hb_ao:get(<<"set-count">>, SetTwice, Opts)), + ?assertEqual(<<"STATE">>, hb_ao:get(<<"state-key">>, SetTwice, Opts)). deep_set_test(Opts) -> % First validate second layer changes are handled correctly. @@ -726,9 +714,9 @@ deep_set_with_device_test(Opts) -> B = hb_ao:get(<<"b">>, A, Opts), C = hb_ao:get(<<"c">>, B, Opts), ?assertEqual(<<"2">>, C), - ?assertEqual(true, hb_ao:get(<<"modified">>, Outer)), - ?assertEqual(false, hb_ao:get(<<"modified">>, A)), - ?assertEqual(true, hb_ao:get(<<"modified">>, B)). + ?assertEqual(true, hb_ao:get(<<"modified">>, Outer, Opts)), + ?assertEqual(false, hb_ao:get(<<"modified">>, A, Opts)), + ?assertEqual(true, hb_ao:get(<<"modified">>, B, Opts)). device_exports_test(Opts) -> Msg = #{ <<"device">> => dev_message }, @@ -840,7 +828,7 @@ denormalized_device_name_test(Opts) -> ), ?assertEqual( {ok, Dev, maps:get(test_func, Dev)}, - hb_device:message_to_fun(Msg, test_func, Opts) + hb_device:message_to_fun(Msg, test_func, <<"message@1.0">>, Opts) ). denormalized_key_test(Opts) -> @@ -862,11 +850,11 @@ denormalized_key_test(Opts) -> }, ?assertEqual( {ok, <<"TEST VALUE">>}, - hb_ao:resolve(Msg, <<"test_key">>, Opts) + hb_ao:get(<<"test_key">>, Msg, Opts) ), ?assertEqual( {ok, <<"TEST VALUE">>}, - hb_ao:resolve(Msg, <<"test-key">>, Opts) + hb_ao:get(<<"test-key">>, Msg, Opts) ). list_transform_test(Opts) -> @@ -880,10 +868,10 @@ list_transform_test(Opts) -> start_as_test(Opts) -> ?assertEqual( {ok, <<"GOOD FUNCTION">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"test-device@1.0">>, #{ <<"path">> => <<>> }}, - #{ <<"path">> => <<"test_func">> } + hb_ao:as(<<"test-device@1.0">>, #{}, Opts), + #{ <<"path">> => <<"test-func">> } ], Opts ) @@ -892,14 +880,14 @@ start_as_with_parameters_test(Opts) -> % Resolve a key on a message that has its device set with `as'. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, ?assertEqual( {ok, <<"GOOD FUNCTION">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, - #{ <<"path">> => <<"test_func">> } + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> } ], Opts ) @@ -909,7 +897,7 @@ load_as_test(Opts) -> % Load a message as a device with the `as' keyword. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, % There is a race condition where we write to the store and a % reset happens making not read the written value. @@ -920,30 +908,31 @@ load_as_test(Opts) -> ?assert(hb_message:match(Msg, ReadMsg, primary, Opts)), ?assertEqual( {ok, <<"MESSAGE">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, #{ <<"path">> => <> }}, - <<"test_func">>, - <<"test_key">> + {link, ID, #{}}, + #{ <<"device">> => <<"test-device@1.0">> }, + <<"test-func">>, + <<"test-key">> ], Opts ) ). as_path_test(Opts) -> - % Create a message with the test device, which implements the test_func + % Create a message with the test device, which implements the test-func % function. It normally returns `GOOD FUNCTION'. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, - ?assertEqual(<<"GOOD FUNCTION">>, hb_ao:get(<<"test_func">>, Msg, Opts)), + ?assertEqual(<<"GOOD FUNCTION">>, hb_ao:get(<<"test-func">>, Msg, Opts)), % Now use the `as' keyword to subresolve a key with the message device. ?assertMatch( - {ok, #{ <<"test_key">> := <<"MESSAGE">> }}, + {ok, #{ <<"test-key">> := <<"MESSAGE">> }}, hb_ao:resolve( - Msg, - {as, <<"message@1.0">>, #{ <<"path">> => <<"test_func">> }}, + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> }, Opts ) ). @@ -952,16 +941,15 @@ continue_as_test(Opts) -> % Resolve a list of messages in sequence, swapping the device in the middle. Msg = #{ <<"device">> => <<"test-device@1.0">>, - <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } + <<"test-func">> => #{ <<"test-key">> => <<"MESSAGE">> } }, ?assertEqual( {ok, <<"MESSAGE">>}, - hb_ao:resolve_many( + hb_ao:resolve( [ - Msg, - {as, <<"message@1.0">>, <<>>}, - #{ <<"path">> => <<"test_func">> }, - #{ <<"path">> => <<"test_key">> } + hb_ao:as(<<"message@1.0">>, Msg, Opts), + #{ <<"path">> => <<"test-func">> }, + #{ <<"path">> => <<"test-key">> } ], Opts ) @@ -982,16 +970,16 @@ as_commitments_test(RawOpts) -> InitialComms = hb_ao:get(<<"commitments">>, Msg, OptsWithWallet), {ok, ResolvedMsg} = hb_ao:resolve( - {as, <<"test-device@1.0">>, Msg}, + hb_ao:as(<<"test-device@1.0">>, Msg, OptsWithWallet), <<"commitments">>, OptsWithWallet ), ?assertEqual(InitialComms, ResolvedMsg), ?assertEqual( {ok, []}, - hb_ao:resolve_many( + hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, + hb_ao:as(<<"message@1.0">>, Msg, OptsWithWallet), <<"committers">> ], OptsWithWallet @@ -1009,9 +997,9 @@ multiple_as_subresolutions_test(Opts) -> #{ <<"test-key-2">> => <<"MESSAGE-2">> } } }, - Res = hb_ao:resolve_many( + Res = hb_ao:resolve( [ - {as, <<"message@1.0">>, Msg}, + hb_ao:as(<<"message@1.0">>, Msg, Opts), #{ <<"path">> => <<"test-message">> }, #{ <<"path">> => <<"test-message-2">>, <<"extraneous">> => <<"1">> }, <<"test-key-2">> diff --git a/src/preloaded/codec/dev_structured.erl b/src/preloaded/codec/dev_structured.erl index c3955b48b..61807b3d8 100644 --- a/src/preloaded/codec/dev_structured.erl +++ b/src/preloaded/codec/dev_structured.erl @@ -285,8 +285,11 @@ encode_ao_types(Types, _Opts) -> )). %% @doc Device key for parsing an `ao-types' field. -decode_types(Base, Req, Opts) -> - {ok, decode_ao_types(hb_maps:get(<<"body">>, Req, Base, Opts), Opts)}. +-spec decode_types(#{ ao_types => binary() | map() }, _, _) -> {ok, #{}}. +decode_types(#{ <<"ao-types">> := Types }, _Req, Opts) -> + {ok, decode_ao_types(Types, Opts)}; +decode_types(_BaseWithoutTypes, _Req, _Opts) -> + {ok, #{}}. %% @doc Parse the `ao-types' field of a TABM if present, and return a map of %% keys and their types. If the given value is a list, we return an empty map diff --git a/src/preloaded/codec/lib_arweave_common.erl b/src/preloaded/codec/lib_arweave_common.erl index 755b9492c..f1859cd72 100644 --- a/src/preloaded/codec/lib_arweave_common.erl +++ b/src/preloaded/codec/lib_arweave_common.erl @@ -124,10 +124,12 @@ ao_types(#{ <<"ao-types">> := AoTypes } = Tags, Opts) -> ConvOpts = Opts#{ <<"hashpath">> => ignore }, {ok, AOTypes} = hb_ao:resolve( - #{ <<"device">> => <<"structured@1.0">> }, #{ - <<"path">> => <<"decode-types">>, - <<"body">> => AoTypes + <<"device">> => <<"structured@1.0">>, + <<"ao-types">> => AoTypes + }, + #{ + <<"path">> => <<"decode-types">> }, ConvOpts ), diff --git a/src/preloaded/message/dev_message.erl b/src/preloaded/message/dev_message.erl index 90a9a0ff4..bf14648a5 100644 --- a/src/preloaded/message/dev_message.erl +++ b/src/preloaded/message/dev_message.erl @@ -5,12 +5,10 @@ %%% `set', `remove', `get', and `verify'. Their function comments describe the %%% behaviour of the device when these keys are set. -module(dev_message). -%%% Base AO-Core reserved keys: --export([info/0, keys/1, keys/2]). --export([set/3, set_path/3, remove/3, get/3, get/4]). -%%% Commitment-specific keys: --export([id/1, id/2, id/3]). --export([commit/3, committed/3, committers/1, committers/2, committers/3, verify/3]). +%%% Base AO-Core state manipulation functions. +-export([info/0, keys/3, set/3, id/3, vary/3, schema/3]). +%%% Commitments API keys: +-export([commit/3, committed/3, committers/3, verify/3]). %%% Non-protocol enforced keys: -export([index/3]). -include_lib("eunit/include/eunit.hrl"). @@ -18,24 +16,20 @@ -define(DEFAULT_ID_DEVICE, <<"httpsig@1.0">>). -define(DEFAULT_ATT_DEVICE, <<"httpsig@1.0">>). -%% The list of keys that are exported by this device. --define(DEVICE_KEYS, [ - <<"id">>, - <<"commitments">>, - <<"committers">>, - <<"keys">>, - <<"path">>, - <<"set">>, - <<"remove">>, - <<"verify">> -]). - %% @doc Return the info for the identity device. info() -> #{ - default => fun dev_message:get/4 + default => fun default_accessor/4 }. +% router(#{ <<"path">> := Key }, #{ <<"path">> := <<"path">> }, _Opts) -> {ok, Key}; +% router(_, #{ <<"path">> := <<"path">> }, _Opts) -> {error, not_found}; +% router(Base, Req, Opts) -> +% case hb_ao:raw(Req, #{ <<"path">> => <<"path">> }, Opts) of +% {error, not_found} -> set(Base, Req, Opts); +% {ok, Key} -> default_accessor(Key, Base, Req, Opts) +% end. + %% @doc Generate an index page for a message, in the event that the `body' and %% `content-type' of a message returned to the client are both empty. We do this %% as follows: @@ -82,8 +76,6 @@ index(Msg, Req, Opts) -> %% Note: This function _does not_ use AO-Core's `get/3' function, as it %% would require significant computation. We may want to change this %% if/when non-map message structures are created. -id(Base) -> id(Base, #{}). -id(Base, Req) -> id(Base, Req, #{}). id(Base, _, NodeOpts) when is_binary(Base) -> % Return the hashpath of the message in native format, to match the native % format of the message ID return. @@ -116,7 +108,11 @@ id(RawBase, Req, NodeOpts) -> [] -> % If there are no commitments, we must (re)calculate the ID. ?event_debug(debug_id, regenerating_id), - calculate_id(hb_maps:without([<<"commitments">>], Base), Req, IDOpts); + calculate_id( + hb_maps:without([<<"commitments">>], Base, IDOpts), + Req, + IDOpts + ); IDs -> % Accumulate the relevant IDs into a single value. This is performed % by module arithmetic of each of the IDs. The effect of this is that: @@ -168,7 +164,7 @@ calculate_id(RawBase, Req, NodeOpts) -> % by checking whether the resolved device module is this module itself. % `hb_ao:raw/5' expects a device name, not a resolved module. CommitDev = - case hb_device:message_to_device(#{ <<"device">> => IDDev }, NodeOpts) of + case hb_device:module(#{ <<"device">> => IDDev }, NodeOpts) of ?MODULE -> ?DEFAULT_ID_DEVICE; _ -> IDDev end, @@ -225,8 +221,6 @@ id_device(_, _) -> {ok, ?DEFAULT_ID_DEVICE}. %% @doc Return the committers of a message that are present in the given request. -committers(Base) -> committers(Base, #{}). -committers(Base, Req) -> committers(Base, Req, #{}). committers(#{ <<"commitments">> := Commitments }, _, NodeOpts) -> {ok, hb_maps:values( @@ -611,239 +605,61 @@ commitment_ids_from_committers(CommitterAddrs, Commitments, Opts) -> %% @doc Deep merge keys in a message. Takes a map of key-value pairs and sets %% them in the message, overwriting any existing values. -set(Base, NewValuesMsg, Opts) -> - OriginalPriv = hb_private:from_message(Base), - % Filter keys that are in the default device (this one). - {ok, NewValuesKeys} = keys(NewValuesMsg, Opts), - KeysToSet = - lists:filter( - fun(Key) -> - not lists:member(Key, ?DEVICE_KEYS ++ [<<"set-mode">>]) andalso - (hb_maps:get(Key, NewValuesMsg, undefined, Opts) =/= undefined) - end, - NewValuesKeys - ), - % Find keys in the message that are already set (case-insensitive), and - % note them for removal. - _ConflictingKeys = - lists:filter( - fun(Key) -> lists:member(Key, KeysToSet) end, - hb_maps:keys(Base, Opts) - ), - UnsetKeys = - lists:filter( - fun(Key) -> - case hb_maps:get(Key, NewValuesMsg, not_found, Opts) of - unset -> true; - _ -> false - end - end, - hb_maps:keys(Base, Opts) - ), - % Base message with keys-to-unset removed - BaseValues = hb_maps:without(UnsetKeys, Base, Opts), - ?event_debug(debug_message_set, - {performing_set, - {conflicting_keys, _ConflictingKeys}, - {keys_to_unset, UnsetKeys}, - {new_values, NewValuesMsg}, - {original_message, Base} - } - ), - % Create the map of new values - NewValues = hb_maps:from_list( - lists:filtermap( - fun(Key) -> - case hb_maps:get(Key, NewValuesMsg, undefined, Opts) of - undefined -> false; - unset -> false; - Value -> {true, {Key, Value}} - end - end, - KeysToSet - ) - ), - % Calculate if the keys to be set conflict with any committed keys. - {ok, CommittedKeys} = - committed( - Base, - #{ - <<"committers">> => <<"all">> - }, - Opts - ), - ?event_debug(message_set, - {setting, - {committed_keys, CommittedKeys}, - {keys_to_set, KeysToSet}, - {message, Base} - } - ), - OverwrittenCommittedKeys = - lists:filtermap( - fun(Key) -> - NormKey = hb_ao:normalize_key(Key), - ?event_debug({checking_committed_key, {key, Key}, {norm_key, NormKey}}), - Res = case lists:member(NormKey, KeysToSet) of - true -> {true, NormKey}; - false -> false - end, - Res - end, - CommittedKeys - ), - ?event_debug({setting, {overwritten_committed_keys, OverwrittenCommittedKeys}}), - % Combine with deep merge or if `set-mode` is `explicit' then just merge. - Merged = - hb_private:set_priv( - case maps:get(<<"set-mode">>, NewValuesMsg, <<"deep">>) of - <<"explicit">> -> maps:merge(BaseValues, NewValues); - _ -> do_deep_merge(BaseValues, NewValues, Opts) - end, - OriginalPriv - ), - case OverwrittenCommittedKeys of - [] -> - ?event_debug(message_set, {no_overwritten_committed_keys, {merged, Merged}}), - {ok, Merged}; - _ -> - % We did overwrite some keys, but do their values match the original? - % If not, we must remove the commitments. - ChangedBaseKeys = hb_maps:with(OverwrittenCommittedKeys, Base, Opts), - ChangedMergedKeys = hb_maps:with(OverwrittenCommittedKeys, Merged, Opts), - Matches = hb_message:match(ChangedMergedKeys, ChangedBaseKeys, strict, Opts), - case Matches of - true -> - ?event_debug(message_set, {set_keys_matched, {merged, Merged}}), - {ok, Merged}; - % {error, {Details, {trace, Stacktrace}}} -> - % erlang:raise(error, Details, Stacktrace); - % {mismatch, Type, Path, Val1, Val2} -> - % ?event( - % set_conflict, - % {set_conflict_removing_commitments, - % {merged, Merged}, - % {mismatch, Type}, - % {path, Path}, - % {expected, Val1}, - % {received, Val2} - % } - % ), - _ -> - {ok, hb_maps:without([<<"commitments">>], Merged, Opts)} - end - end. - -%% @doc Deep merge keys in a message, utilizing the set device of any child -%% keys that are themselves messages. -do_deep_merge(BaseValues, NewValues, Opts) -> - {WithNestedMerges, StillToDeepMerge} = - maps:fold( - fun(Key, NewValue, {Acc, ToDeepMerge}) - when is_map(NewValue) - andalso is_map(map_get(Key, Acc)) -> - BaseValue = map_get(Key, Acc), - NewValueSet = NewValue#{ <<"path">> => <<"set">> }, - { - Acc#{ - Key => - hb_util:ok( - hb_ao:resolve( - BaseValue, - NewValueSet, - Opts - ), - Opts - ) - }, - ToDeepMerge - }; - (Key, NewValue, {Acc, ToDeepMerge}) - when is_map(NewValue) - andalso ?IS_LINK(map_get(Key, Acc)) -> - LoadedBaseValue = hb_cache:ensure_loaded(map_get(Key, Acc), Opts), - case is_map(LoadedBaseValue) of - true -> - NewValueSet = NewValue#{ <<"path">> => <<"set">> }, - { - Acc#{ - Key => - hb_util:ok( - hb_ao:resolve( - LoadedBaseValue, - NewValueSet, - Opts - ), - Opts - ) - }, - ToDeepMerge - }; - false -> - {Acc, [Key | ToDeepMerge]} +set(Base, Req = #{ <<"set">> := <<"deep">> }, Opts) -> + NewValues = + maps:map( + fun(Key, Nested) when is_map(Nested) or ?IS_LINK(Nested) -> + case hb_maps:find(Key, Base, Opts) of + {ok, NestedBase} when is_map(NestedBase) or ?IS_LINK(NestedBase) -> + % We use `hb_ao:deep_set` rather than just recursing + % such that the correct device is used for the downstream + % set operation. + hb_ao:deep_set(NestedBase, Nested, Opts); + error -> Nested end; - (Key, _, {Acc, ToDeepMerge}) -> - {Acc, [Key | ToDeepMerge]} - end, - {BaseValues, []}, - NewValues - ), - hb_util:deep_merge( - WithNestedMerges, - maps:with(StillToDeepMerge, NewValues), - Opts - ). - -%% @doc Special case of `set/3' for setting the `path' key. This cannot be set -%% using the normal `set' function, as the `path' is a reserved key, used to -%% transmit the present key that is being executed. Subsequently, to call `path' -%% we would need to set `path' to `set', removing the ability to specify its -%% new value. -set_path(Base, #{ <<"value">> := Value }, Opts) -> - set_path(Base, Value, Opts); -set_path(Base, Value, Opts) when not is_map(Value) -> - % Determine whether the `path' key is committed. If it is, we remove the - % commitment if the new value is different. We try to minimize work by - % doing the `hb_maps:get` first, as it is far cheaper than calculating - % the committed keys. - BaseWithCorrectedComms = - case hb_maps:get(<<"path">>, Base, undefined, Opts) of - Value -> Base; - _ -> - % The new value is different, but is it committed? If so, we - % must remove the commitments. - case hb_message:is_signed_key(<<"path">>, Base, Opts) of - true -> hb_message:uncommitted(Base, Opts); - false -> Base - end - end, - case Value of - unset -> - {ok, hb_maps:without([<<"path">>], BaseWithCorrectedComms, Opts)}; - _ -> - BaseWithCorrectedComms#{ <<"path">> => Value } + (_Key, NewValue) -> NewValue + end, + maps:without([<<"set">>], hb_message:uncommitted(Req, Opts)) + ), + set(Base, NewValues, Opts); +set(Base, NewValues, _Opts) -> + case lists:partition(fun hb_private:is_private/1, maps:keys(Base)) of + {[], _} -> {ok, NewValues#{ <<"...">> => Base }}; + {PrivKeys, _} -> + { + ok, + (maps:merge(NewValues, maps:with(PrivKeys, Base)))#{ + <<"...">> => maps:without(PrivKeys, Base) + } + } end. -%% @doc Remove a key or keys from a message. -remove(Base, #{ <<"item">> := Key }, Opts) -> - remove(Base, #{ <<"items">> => [Key] }, Opts); -remove(Base, #{ <<"items">> := Keys }, Opts) -> - set( - Base, - #{ Key => unset || Key <- Keys }, - Opts - ). - %% @doc Get the public keys of a message. -keys(Msg) -> - keys(Msg, #{}). - -keys(Msg, Opts) when not is_map(Msg) -> +-spec keys(#{ _ => _ }, #{ keys => _, _ => _ }, _) -> {ok, [binary()]}. +keys(Msg, Req, Opts) when is_list(Msg) -> case hb_ao:normalize_keys(Msg, Opts) of - NormMsg when is_map(NormMsg) -> keys(NormMsg, Opts); + NormMsg when is_map(NormMsg) -> keys(NormMsg, Req, Opts); _ -> throw(badarg) end; -keys(Msg, Opts) -> +keys(Msg, #{ <<"keys">> := <<"deep">> }, Opts) when is_map(Msg) -> + Inherited = + case maps:find(<<"...">>, Msg) of + {ok, Extension} -> hb_ao:keys(Extension, Opts); + error -> [] + end, + InheritedPublic = Inherited -- [<<"commitments">>], + DirectKeys = + maps:fold( + fun + (_Key, unset, Acc) -> Acc; + (<<"...">>, _Value, Acc) -> Acc; + (Key, _Value, Acc) -> [Key | Acc] + end, + [], + Msg + ), + {ok, DirectKeys ++ InheritedPublic}; +keys(Msg, _Req, Opts) -> { ok, lists:filter( @@ -855,26 +671,83 @@ keys(Msg, Opts) -> %% @doc Return the value associated with the key as it exists in the message's %% underlying Erlang map. First check the public keys, then check case- %% insensitively if the key is a binary. -get(Key, Msg, Opts) -> get(Key, Msg, #{ <<"path">> => <<"get">> }, Opts). -get(Key, Msg, _Req, Opts) -> +default_accessor(Key, Msg, Req, Opts) -> case hb_private:is_private(Key) of true -> {error, not_found}; false -> - case hb_maps:get(Key, Msg, not_found, Opts) of - not_found -> case_insensitive_get(Key, Msg, Opts); - Value -> {ok, Value} + case hb_maps:find(Key, Msg, Opts) of + {ok, Value} -> {ok, Value}; + error -> + case hb_maps:find(<<"...">>, Msg, Opts) of + {ok, AncestorMsg} when is_map(AncestorMsg) -> + hb_ao:raw(AncestorMsg, Req, Opts); + {ok, Ancestor} + when is_binary(Ancestor) + orelse ?IS_LINK(Ancestor) -> + case hb_cache:read(Ancestor, Opts) of + {ok, AncestorMsg} -> + hb_ao:raw(AncestorMsg, Req, Opts); + OtherStatus -> OtherStatus + end; + error -> {error, not_found} + end end end. -%% @doc Key matching should be case insensitive, following RFC-9110, so we -%% implement a case-insensitive key lookup rather than delegating to -%% `hb_maps:get/2'. Encode the key to a binary if it is not already. -case_insensitive_get(Key, Msg, Opts) -> - NormKey = hb_util:to_lower(hb_util:bin(Key)), - NormMsg = hb_ao:normalize_keys(Msg, Opts), - case hb_maps:get(NormKey, NormMsg, not_found, Opts) of - not_found -> {error, not_found}; - Value -> {ok, Value} +%% @doc Determines the schema for the resolution of a `Base/Request` pair and +%% applies it to the inputs. Returns `base` and `request` submessages, as well +%% as the resolvable function Erlang function at `priv/function` if it was found +%% during the process of `vary`ing. +-spec vary( + #{ _ => _ }, + #{ _ => _ }, + #{}) -> {ok, #{}} | {error, binary()}. +vary(Base, Req, Opts) -> + maybe + {ok, Ctx} ?= + case hb_private:get(<<"function">>, Req, not_found, Opts) of + not_found -> + % No function given: derive the key to resolve one. + maybe + {ok, Key} ?= + case Req of + #{ <<"vary">> := KeyToVaryOn } -> + {ok, KeyToVaryOn}; + #{ <<"path">> := <<"vary">> } -> + {error, <<"Cannot vary the `vary` path.">>}; + #{ <<"path">> := PathKey } -> + {ok, PathKey}; + _ -> + {error, <<"invalid-vary-request">>} + end, + hb_device:add_resolver( + #{ + <<"base">> => Base, + <<"key">> => Key, + <<"request">> => Req + }, + Opts + ) + end; + Fun -> + % The executor is already known: no key is required. + {ok, + #{ + <<"base">> => Base, + <<"request">> => Req, + <<"priv">> => #{ <<"function">> => Fun } + } + } + end, + hb_types:vary(Ctx, Opts) + end. + +%% @doc Returns the device schema for a `Base` message. +-spec schema(_, _, _) -> {ok, undefined | #{}}. +schema(#{ <<"device">> := Device }, _, Opts) -> + case hb_types:extract(Device, Opts) of + {ok, Schema} -> {ok, Schema}; + _ -> {ok, undefined} end. %%% Tests @@ -916,12 +789,6 @@ is_private_mod_test() -> keys_from_device_test() -> ?assertEqual({ok, [<<"a">>]}, hb_ao:resolve(#{ <<"a">> => 1 }, keys, #{})). -case_insensitive_get_test() -> - ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"a">> => 1 }, #{})), -% ?assertEqual({ok, 1}, case_insensitive_get(<<"a">>, #{ <<"A">> => 1 }, #{})), - ?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"a">> => 1 }, #{})). - %?assertEqual({ok, 1}, case_insensitive_get(<<"A">>, #{ <<"A">> => 1 }, #{})). - private_keys_are_filtered_test() -> ?assertEqual( {ok, [<<"a">>]}, diff --git a/src/preloaded/util/dev_apply.erl b/src/preloaded/util/dev_apply.erl index c1d7e813e..99659a083 100644 --- a/src/preloaded/util/dev_apply.erl +++ b/src/preloaded/util/dev_apply.erl @@ -136,7 +136,7 @@ find_key(Path, Base, Request, Opts) -> {message, Request}; [<<"base">>, Key] -> {resolve, [{BaseAs, normalize_path([Key|RestKeys])}]}; - [Req, Key] when Req == <<"request">> orelse Req == <<"req">> -> + [Req, Key] when Req == <<"request">> orelse Req == <<"request">> -> {resolve, [{RequestAs, normalize_path([Key|RestKeys])}]}; [_] -> {resolve, [ diff --git a/src/preloaded/util/dev_math.erl b/src/preloaded/util/dev_math.erl new file mode 100644 index 000000000..c51eef708 --- /dev/null +++ b/src/preloaded/util/dev_math.erl @@ -0,0 +1,28 @@ +%%% @doc Simple test device with math utilities. Not intended for production use. +-module(dev_math). +-include("include/hb.hrl"). +-export([inc_x/3, dec_x/3, add_x/3, sum/3, with_sum/3]). + +-spec inc_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. +inc_x(#{ <<"x">> := X }, _, _) -> + ?prim_dbg({inc_x_called, X}), + {ok, #{ <<"x">> => X + 1 }}. + +-spec dec_x(#{ x := integer() }, _, _) -> {ok, #{ '...' => base }}. +dec_x(#{ <<"x">> := X }, _, _) -> {ok, #{ <<"x">> => X - 1 }}. + +-spec add_x(#{ x := integer() }, #{ add := integer() }, _) -> + {ok, #{ '...' => base }}. +add_x(#{ <<"x">> := X }, #{ <<"add">> := Add }, _) -> + {ok, #{ <<"x">> => X + Add }}. + +-spec sum(#{ x := integer(), y := integer() }, _, _) -> + {ok, #{ '...' => base }}. +sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> + {ok, X + Y}. + +-spec with_sum(#{ x := integer(), y := integer() }, _, _) -> + {ok, #{ sum => integer(), '...' => base }}. +with_sum(#{ <<"x">> := X, <<"y">> := Y }, _, _) -> + ?prim_dbg({doing_sum, {x, X}, {y, Y}}), + {ok, #{ <<"sum">> => X + Y }}. \ No newline at end of file diff --git a/src/preloaded/util/dev_test.erl b/src/preloaded/util/dev_test.erl index 8851ecd88..2fbfcf885 100644 --- a/src/preloaded/util/dev_test.erl +++ b/src/preloaded/util/dev_test.erl @@ -49,8 +49,10 @@ info(_Base, _Req, _Opts) -> {ok, #{<<"status">> => 200, <<"body">> => InfoBody}}. %% @doc Example index handler. -index(Msg, _Req, Opts) -> - Name = hb_ao:get(<<"name">>, Msg, <<"turtles">>, Opts), +-spec index(#{ name => binary() }, _, _) -> + {ok, #{ content_type => binary(), body => binary() }}. +index(Msg, _Req, _Opts) -> + Name = maps:get(<<"name">>, Msg, <<"turtles">>), {ok, #{ <<"content-type">> => <<"text/html">>,