diff --git a/.github/scripts/convert_img.py b/.github/scripts/convert_img.py new file mode 100644 index 00000000..354315b6 --- /dev/null +++ b/.github/scripts/convert_img.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +Convert Markdown image syntax to raw HTML tags in Jekyll (kramdown) docs. + +kramdown/markdown renders + ![alt text](path/to/image.png "optional title") +fine on its own, but some doc pipelines need the equivalent explicit HTML so +attributes (classes, sizing, lazy-loading, etc.) can be added later, or so +markdown and HTML image usage in a repo is consistent. This script rewrites +every Markdown image reference in a file into: + alt text +preserving a "title" attribute when the source included one. + +Fenced code blocks are left untouched, since image syntax written as a +documentation example inside a code fence shouldn't be rewritten. + +Usage: + python3 convert_images.py /path/to/documentation # apply changes + python3 convert_images.py /path/to/documentation --dry-run # preview only + python3 convert_images.py /path/to/documentation --ext .md,.mdx +""" + +import argparse +import re +import sys +from pathlib import Path + +# ![alt](src) or ![alt](src "title") or ![alt](src 'title') +IMAGE_RE = re.compile( + r'!\[([^\]]*)\]\(' + r'(\S+?)' + r'(?:\s+(?:"([^"]*)"|\'([^\']*)\'))?' + r'\)' +) + +# Fenced code blocks shouldn't have their contents touched. +FENCE_RE = re.compile(r'^\s*```') + + +def escape_attr(value: str) -> str: + """Escape a value for safe placement inside a double-quoted HTML attribute.""" + return value.replace("&", "&").replace('"', """) + + +def convert_line(line: str, converted: list) -> str: + def repl(m): + alt, src, title_dq, title_sq = m.group(1), m.group(2), m.group(3), m.group(4) + title = title_dq if title_dq is not None else title_sq + attrs = f'src="{escape_attr(src)}" alt="{escape_attr(alt)}"' + if title: + attrs += f' title="{escape_attr(title)}"' + converted.append((m.group(0), f"")) + return f"" + + return IMAGE_RE.sub(repl, line) + + +def process_file(path: Path, dry_run: bool): + raw = path.read_bytes() + newline = b"\r\n" if b"\r\n" in raw else b"\n" + text = raw.decode("utf-8") + lines = text.splitlines() + + converted = [] + in_fence = False + new_lines = [] + for line in lines: + if FENCE_RE.match(line): + in_fence = not in_fence + new_lines.append(line) + continue + if in_fence: + new_lines.append(line) + continue + new_lines.append(convert_line(line, converted)) + + trailing = "\n" if text.endswith("\n") else "" + updated_text = "\n".join(new_lines) + trailing + + changed = updated_text != text + if changed: + if dry_run: + print(f"[DRY RUN] {path}") + else: + out_bytes = updated_text.replace("\n", newline.decode("ascii")).encode("utf-8") + path.write_bytes(out_bytes) + print(f"{path}") + for old, new in converted: + print(f" - {old} -> {new}") + + return len(converted) + + +def main(): + parser = argparse.ArgumentParser(description="Convert Markdown image syntax to HTML tags in Jekyll docs.") + parser.add_argument("root", help="Root directory (or single file) to process") + parser.add_argument("--ext", default=".md", help="Comma-separated file extensions (default: .md)") + parser.add_argument("--dry-run", action="store_true", help="Preview changes without writing files") + args = parser.parse_args() + + root = Path(args.root) + if not root.exists(): + print(f"Error: path does not exist: {root}", file=sys.stderr) + sys.exit(1) + + extensions = tuple(e.strip() if e.strip().startswith(".") else f".{e.strip()}" + for e in args.ext.split(",")) + + paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in extensions) + + total_files = 0 + total_fixes = 0 + for path in paths: + n = process_file(path, args.dry_run) + if n: + total_files += 1 + total_fixes += n + + verb = "would be made" if args.dry_run else "made" + print(f"\nDone: {total_fixes} conversion(s) across {total_files} file(s) {verb}.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/scripts/convert_url.py b/.github/scripts/convert_url.py new file mode 100644 index 00000000..ea7264de --- /dev/null +++ b/.github/scripts/convert_url.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +Clean up mixed HTML/Markdown anchor IDs in Jekyll (kramdown) docs. + +kramdown auto-generates an id for every heading by lowercasing the heading +text and replacing runs of non-alphanumeric characters with a single hyphen +(duplicate headings get -1, -2, ... appended). That means a manual + + ### Bucket File Upload +is redundant (and produces a duplicate id in the rendered HTML) whenever the +explicit id exactly matches the slug kramdown would generate anyway. + +This script, for every heading in a file: + 1. Computes the kramdown auto-slug (handling duplicate-heading suffixes). + 2. If a ``/`` tag sits directly above (or on) the + heading line and its id matches the auto-slug, the tag is removed as + redundant. + 3. If the explicit id does NOT match the auto-slug (a genuinely custom + anchor), it's left alone and reported, since removing it could break + links elsewhere. + 4. Rewrites in-file link references [text](#Some-Anchor) whose casing + doesn't match the real slug, so links that are silently broken (e.g. + #Get-All-Bucket-Names vs the real get-all-bucket-names) start working. + +Usage: + python3 fix_anchors.py /path/to/documentation # apply changes + python3 fix_anchors.py /path/to/documentation --dry-run # preview only + python3 fix_anchors.py /path/to/documentation --ext .md,.mdx +""" + +import argparse +import re +import sys +from pathlib import Path + +HEADING_RE = re.compile(r'^(#{1,6})\s+(.*?)\s*#*\s*$') +ANCHOR_TAG_RE = re.compile(r'^\s*\s*\s*$', re.IGNORECASE) +LINK_RE = re.compile(r'\[([^\]]*)\]\(#([^\s)]+)\)') + +# Fenced code blocks shouldn't have their contents touched. +FENCE_RE = re.compile(r'^\s*```') + + +def kramdown_slug(text: str) -> str: + """Approximate kramdown's auto-generated heading id algorithm.""" + # Strip markdown emphasis/code/link markup that wouldn't count toward the id text + stripped = re.sub(r'[`*_]', '', text) + stripped = re.sub(r'\[([^\]]*)\]\([^)]*\)', r'\1', stripped) + slug = stripped.strip().lower() + slug = re.sub(r'[^a-z0-9\s-]', '', slug) + slug = re.sub(r'[\s]+', '-', slug) + slug = re.sub(r'-{2,}', '-', slug) + slug = slug.strip('-') + return slug + + +def compute_heading_slugs(lines): + """ + Walk the file (skipping fenced code blocks) and return, for each heading + line index, its final kramdown slug (accounting for -1, -2 dedup suffixes). + Returns dict: line_index -> slug + """ + seen = {} + result = {} + in_fence = False + for i, line in enumerate(lines): + if FENCE_RE.match(line): + in_fence = not in_fence + continue + if in_fence: + continue + m = HEADING_RE.match(line) + if not m: + continue + base = kramdown_slug(m.group(2)) + if base in seen: + seen[base] += 1 + slug = f"{base}-{seen[base]}" + else: + seen[base] = 0 + slug = base + result[i] = slug + return result + + +def process_file(path: Path, dry_run: bool): + raw = path.read_bytes() + newline = b"\r\n" if b"\r\n" in raw else b"\n" + text = raw.decode("utf-8") + lines = text.splitlines() + + heading_slugs = compute_heading_slugs(lines) + removed_anchors = [] + custom_anchors = [] + + # Step 1: find lines immediately preceding a heading line + # (allowing a single blank line between), and drop them if redundant. + to_delete = set() + for i, line in enumerate(lines): + am = ANCHOR_TAG_RE.match(line) + if not am: + continue + anchor_id = am.group(1) + + # look ahead for the next non-blank line + j = i + 1 + while j < len(lines) and lines[j].strip() == "": + j += 1 + if j < len(lines) and j in heading_slugs: + if heading_slugs[j] == anchor_id: + to_delete.add(i) + removed_anchors.append(anchor_id) + continue + custom_anchors.append((i + 1, anchor_id)) + + new_lines = [line for idx, line in enumerate(lines) if idx not in to_delete] + + # Collapse a run of blank lines down to a single blank line (removing an + # anchor that had its own blank-line spacing above a heading can leave + # two consecutive blank lines behind). + collapsed = [] + for line in new_lines: + if line.strip() == "" and collapsed and collapsed[-1].strip() == "": + continue + collapsed.append(line) + new_lines = collapsed + + updated_text = "\n".join(new_lines) + trailing = "\n" if text.endswith("\n") else "" + + # Step 2: build the authoritative slug set (post-removal, slugs unchanged + # since we only removed redundant lines) for fixing link casing. + valid_slugs = set(heading_slugs.values()) + valid_slugs.update(anchor_id for _, anchor_id in custom_anchors) + lower_to_real = {s.lower(): s for s in valid_slugs} + + fixed_links = [] + + def fix_link(m): + link_text, anchor = m.group(1), m.group(2) + if anchor in valid_slugs: + return m.group(0) + real = lower_to_real.get(anchor.lower()) + if real and real != anchor: + fixed_links.append((anchor, real)) + return f"[{link_text}](#{real})" + return m.group(0) + + updated_text = LINK_RE.sub(fix_link, updated_text) + trailing + + changed = updated_text != text + if changed: + if dry_run: + print(f"[DRY RUN] {path}") + else: + out_bytes = updated_text.replace("\n", newline.decode("ascii")).encode("utf-8") + path.write_bytes(out_bytes) + print(f"{path}") + if removed_anchors: + print(f" - removed {len(removed_anchors)} redundant anchor(s): {', '.join(removed_anchors)}") + if fixed_links: + for old, new in fixed_links: + print(f" - fixed link casing: #{old} -> #{new}") + if custom_anchors: + for lineno, anchor_id in custom_anchors: + print(f" ! kept custom anchor '#{anchor_id}' at {path}:{lineno} (doesn't match any heading slug)") + + return len(removed_anchors) + len(fixed_links) + + +def main(): + parser = argparse.ArgumentParser(description="Fix mixed HTML/Markdown anchor IDs in Jekyll docs.") + parser.add_argument("root", help="Root directory (or single file) to process") + parser.add_argument("--ext", default=".md", help="Comma-separated file extensions (default: .md)") + parser.add_argument("--dry-run", action="store_true", help="Preview changes without writing files") + args = parser.parse_args() + + root = Path(args.root) + if not root.exists(): + print(f"Error: path does not exist: {root}", file=sys.stderr) + sys.exit(1) + + extensions = tuple(e.strip() if e.strip().startswith(".") else f".{e.strip()}" + for e in args.ext.split(",")) + + paths = [root] if root.is_file() else sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in extensions) + + total_files = 0 + total_fixes = 0 + for path in paths: + n = process_file(path, args.dry_run) + if n: + total_files += 1 + total_fixes += n + + verb = "would be made" if args.dry_run else "made" + print(f"\nDone: {total_fixes} fix(es) across {total_files} file(s) {verb}.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/06- Networking & Security/05- MQTT Message Broker.md b/06- Networking & Security/05- MQTT Message Broker.md index d68fd4bd..2de23a43 100644 --- a/06- Networking & Security/05- MQTT Message Broker.md +++ b/06- Networking & Security/05- MQTT Message Broker.md @@ -171,7 +171,6 @@ run msg client where [connection parameters] and [config parameters] and topic = A single `run msg client` command can include multiple `topic = (...)` blocks. - ### Connection options | Option | Description | @@ -212,8 +211,6 @@ A single `run msg client` command can include multiple `topic = (...)` blocks. | `dynamic` | `true` auto-generates tables and UNS policies from the topic and JSON payload. | | `policy` | Reusable mapping policy previously inserted into the blockchain. Replaces inline `dbms`, `table`, and `column...` mappings. | - - ## Topic matching and table naming MQTT topic matching is traditionally case-sensitive. A subscription to `C/#` matches `C/data`; it does not match `c/data`. @@ -231,7 +228,6 @@ This prevents logically equivalent objects from being created under names that d capitalization and provides consistent references across local storage, distributed queries, metadata, and applications, especially because databases are case insensitive. - > **Important:** MQTT subscription matching itself remains case-sensitive. `C/#` does not match `c/data`. To ingest both topic paths, configure subscriptions for both `C/#` and `c/#`. If they represent different data sources, ensure that their generated or explicitly configured AnyLog table names do not collide. For example, MQTT considers these two topics different: @@ -343,7 +339,6 @@ Data will be written to `dbms=mydb` and `table_name = my_new_table` (table overw )> ``` - ### QoS | Level | Meaning | @@ -526,7 +521,6 @@ mosquitto_pub \ For the full mTLS walkthrough (CA creation, `id sign certificate request`, user certs, MQTT Explorer, external org certs, and optional CA on the blockchain), see [Broker Setup TLS Example](./05-3%20Broker%20Setup%20TLS%20Example.md). - ## Debugging and Validation ```anylog @@ -614,7 +608,6 @@ AnyLog Co.|mydb|data |fed71895ee0161ffe92bb79f7e85791c|active |oper | | |6ca7df77cc8f4777cfd427dff870af5f|active |operator2| 212|192.168.0.138:32248| | + |active | ``` - This data hosted by two AnyLog operators on two physical machines or sites can then be queried: ```anylog AL operator2 > run client () sql mydb format=table and extend=(+ip, +node_name, @table_name) "select broker, value from data" @@ -837,4 +830,4 @@ The message inserts into `mydb.broker_data`, with no external MQTT broker hop. * [Connectors To Data Sources](./05-2%20Connectors%20To%20Data%20Sources.md) * [Network Processing](./02-%20Network%20Processing.md) * [Using REST](./04-%20Using%20REST.md) -* [Unified Namespace](../08-%20Blockchain%20&%20Metadata/05-%20Unitfied%20Namespace.md) +* [Unified Namespace](../08-%20Blockchain%20&%20Metadata/04-%20Unified%20Namespace.md) diff --git a/06- Networking & Security/05-3 Broker Setup TLS Example.md b/06- Networking & Security/05-3 Broker Setup TLS Example.md index 145242b4..dfbe05eb 100644 --- a/06- Networking & Security/05-3 Broker Setup TLS Example.md +++ b/06- Networking & Security/05-3 Broker Setup TLS Example.md @@ -183,7 +183,7 @@ In MQTT Explorer, configure the connection to `mqtt://192.168.1.60:8883/` with: - **CLIENT CERTIFICATE:** e.g. `user1.crt` / `AnyLogUser1.crt` - **CLIENT KEY:** e.g. `user1.key` / `AnyLogUser1.key` -![MQTT Explorer TLS setup](../imgs/mqtt_explorer_setup.png) +MQTT Explorer TLS setup ## Share CA Users in the blockchain @@ -394,5 +394,5 @@ Additional Operators repeat the CSR + remote-sign steps with their own `output_n ## Link to resources -- [mosquitto-tls man page](https://mosquitto.org/man/mosquitto-tls-7.html) -- [Mosquitto broker TLS config (Medium)](https://medium.com/@sonadorje/mosquitto-broker-tls-config-5f8bfaa5c047) +- mosquitto-tls man page +- Mosquitto broker TLS config (Medium) diff --git a/07- CLI/05- JSON Data Transformation.md b/07- CLI/05- JSON Data Transformation.md index ba72e453..5aa7b10a 100644 --- a/07- CLI/05- JSON Data Transformation.md +++ b/07- CLI/05- JSON Data Transformation.md @@ -111,8 +111,11 @@ values associated with the keys and the string values are added to the retrieved * ```bring.min``` - return the minimum value of an attribute. * ```bring.max``` - return the maximum value of an attribute. * ```bring.list``` - return the requested attributes as a list. - * ```bring.children``` - Return the immediate children of a policy by retrieving all policies whose 'parent' attribute matches the parent policy's 'id' attribute. - + * ```bring.children``` - returns the immediate children of a policy by retrieving all policies whose `parent` attribute matches the policy's `id`. + * ```bring.parents``` - returns the policy with a dynamic `parents` attribute containing the IDs of all policies that reference it as their immediate child. + * ```bring.paths``` - returns the policy with all paths that contain the policy. + * ```bring.extend``` - returns the policy extended with the policy identified by the value of its `object_id` attribute. + ### Special Bring Values * **Basic Usage:** If the **bring** command values are wrapped in square brackets, it designates keys into the policy, and the associated values are returned. @@ -236,4 +239,357 @@ blockchain get tag bring.table [tag][table] [tag][path(rfind(/))] blockchain get tag bring.table [tag][table] [tag][path(find(/))] blockchain get tag bring.table [tag][table] [tag][path(suffix(10))] blockchain get tag bring.table [tag][table] [tag][path(prefix(10))] -``` \ No newline at end of file +``` + +## Navigating Policy Relationships with `bring` + +Policies can reference other policies to create relationships and hierarchical structures. A common example is a Unified Namespace (UNS), where `uns` policies define paths and reference `object` policies containing the metadata associated with each object. + +The following `bring` extensions allow applications to navigate and resolve these relationships: + +| Command | Description | +|---|---| +| `bring.extend` | Extends a policy with the policy referenced by its `object_id` | +| `bring.children` | Returns the immediate child policies of the selected policy | +| `bring.parents` | Returns the UNS policies that reference the selected object | +| `bring.paths` | Returns the complete UNS path or paths associated with an object | + +Consider the following UNS hierarchy: + +```text +manufacturer + └── Caterpillar + └── Generator +``` + +The hierarchy is represented by `uns` policies, while the information associated with each element is maintained in separate `object` policies. + +For example: + +```json +{ + "object": { + "name": "Generator", + "id": "85b7bcddf4e0cf05ce32f002af42d6fb" + } +} +``` + +The corresponding UNS policy references the object using `object_id`: + +```json +{ + "uns": { + "namespace": "manufacturer/Caterpillar/Generator", + "object_id": "85b7bcddf4e0cf05ce32f002af42d6fb", + "parent": "0fa2a4130b6e132bf5d982c33b9a6204", + "id": "e23c9eadfe2182db44252ed7b0e30ff2" + } +} +``` + +This separation allows the same object to participate in different hierarchies or paths without duplicating the object metadata. + +--- + +### `bring.extend` + +`bring.extend` resolves the `object_id` referenced by a UNS policy and returns the corresponding `object` policy together with the UNS policy. + +```anylog +blockchain get uns bring.extend +``` + +Example result: + +```json +[ + { + "uns": { + "namespace": "manufacturer/Caterpillar", + "object_id": "257e7557a2206c8517e913606eb56cf7", + "parent": "06bd59ebecfa093a1015527055bed273", + "id": "0fa2a4130b6e132bf5d982c33b9a6204" + }, + "object": { + "name": "Caterpillar", + "id": "257e7557a2206c8517e913606eb56cf7" + } + } +] +``` + +The returned structure contains both: + +- the `uns` policy defining the position in the hierarchy; and +- the `object` policy containing the metadata associated with that element. + +For example, the root UNS policy: + +```json +{ + "uns": { + "namespace": "manufacturer", + "object_id": "e47882f7e3f920b05ddca15c9b3aa314" + } +} +``` + +can be extended with the referenced object: + +```json +{ + "object": { + "name": "manufacturer", + "description": "Asset/provenance view: organized by manufacturer, device type, model, customer type, and customer instance.", + "id": "e47882f7e3f920b05ddca15c9b3aa314" + } +} +``` + +`bring.extend` is useful when the UNS defines the relationship or hierarchy while the object policy maintains the descriptive metadata. + +--- + +### `bring.children` + +`bring.children` returns the **immediate child policies** of the selected policy. + +For example, the root of the hierarchy is: + +```text +manufacturer +``` + +with policy ID: + +```text +06bd59ebecfa093a1015527055bed273 +``` + +To return its children: + +```anylog +blockchain get * where id = 06bd59ebecfa093a1015527055bed273 bring.children +``` + +Result: + +```json +[ + { + "uns": { + "namespace": "manufacturer/Caterpillar", + "object_id": "257e7557a2206c8517e913606eb56cf7", + "parent": "06bd59ebecfa093a1015527055bed273", + "id": "0fa2a4130b6e132bf5d982c33b9a6204" + } + } +] +``` + +In this example: + +```text +manufacturer + └── Caterpillar +``` + +`Caterpillar` is returned because its UNS policy identifies the `manufacturer` policy as its parent. + +`bring.children` returns the next level of the hierarchy rather than recursively returning every descendant. + +--- + +### `bring.parents` + +An object can be referenced by one or more UNS policies. `bring.parents` identifies the UNS policy or policies that reference the selected object. + +For example: + +```anylog +blockchain get * where id = 85b7bcddf4e0cf05ce32f002af42d6fb bring.parents +``` + +Result: + +```json +[ + { + "object": { + "name": "Generator", + "id": "85b7bcddf4e0cf05ce32f002af42d6fb" + }, + "parents": [ + "e23c9eadfe2182db44252ed7b0e30ff2" + ] + } +] +``` + +The `parents` array contains the IDs of the UNS policies that reference the object. + +In this example: + +```text +85b7bcddf4e0cf05ce32f002af42d6fb +``` + +is the ID of the `Generator` object, while: + +```text +e23c9eadfe2182db44252ed7b0e30ff2 +``` + +is the ID of the UNS policy that places the object at: + +```text +manufacturer/Caterpillar/Generator +``` + +An object can participate in multiple UNS hierarchies. In that case, `parents` can contain multiple UNS policy IDs. + +--- + +### `bring.paths` + +`bring.paths` resolves the complete UNS hierarchy associated with an object. + +For example: + +```anylog +blockchain get object where id = 85b7bcddf4e0cf05ce32f002af42d6fb bring.paths +``` + +The selected object is: + +```text +Generator +``` + +and its UNS path is: + +```text +manufacturer → Caterpillar → Generator +``` + +The result contains the object together with a `paths` object: + +```json +[ + { + "object": { + "name": "Generator", + "id": "85b7bcddf4e0cf05ce32f002af42d6fb" + }, + "paths": { + "e23c9eadfe2182db44252ed7b0e30ff2": [ + { + "uns": { + "namespace": "manufacturer", + "object_id": "e47882f7e3f920b05ddca15c9b3aa314", + "id": "06bd59ebecfa093a1015527055bed273" + }, + "object": { + "name": "manufacturer", + "id": "e47882f7e3f920b05ddca15c9b3aa314" + } + }, + { + "uns": { + "namespace": "manufacturer/Caterpillar", + "object_id": "257e7557a2206c8517e913606eb56cf7", + "parent": "06bd59ebecfa093a1015527055bed273", + "id": "0fa2a4130b6e132bf5d982c33b9a6204" + }, + "object": { + "name": "Caterpillar", + "id": "257e7557a2206c8517e913606eb56cf7" + } + }, + { + "uns": { + "namespace": "manufacturer/Caterpillar/Generator", + "object_id": "85b7bcddf4e0cf05ce32f002af42d6fb", + "parent": "0fa2a4130b6e132bf5d982c33b9a6204", + "id": "e23c9eadfe2182db44252ed7b0e30ff2" + }, + "object": { + "name": "Generator", + "id": "85b7bcddf4e0cf05ce32f002af42d6fb" + } + } + ] + } + } +] +``` + +The key under `paths` is the ID of the UNS policy that references the selected object. + +The array describes the complete hierarchy from the root policy to the selected object: + +```text +manufacturer + ↓ +Caterpillar + ↓ +Generator +``` + +Each element contains both the UNS policy and its associated object policy. + +--- + +### Objects in Multiple Paths + +An important property of the AnyLog metadata model is that an object can participate in more than one hierarchy. + +For example, the same `Generator` object could appear in: + +```text +manufacturer → Caterpillar → Generator +``` + +and: + +```text +City → Plant → Electricity → Generator +``` + +The `Generator` object does not need to be duplicated. Each UNS hierarchy can reference the same object ID: + +```text +85b7bcddf4e0cf05ce32f002af42d6fb +``` + +In this case: + +```anylog +blockchain get object where id = 85b7bcddf4e0cf05ce32f002af42d6fb bring.parents +``` + +can return multiple UNS policy IDs, and: + +```anylog +blockchain get object where id = 85b7bcddf4e0cf05ce32f002af42d6fb bring.paths +``` + +can return the complete path associated with each of those UNS policies. + +This allows AnyLog to maintain **multiple logical views of the same physical or logical object** while keeping the object's metadata in a single object policy. + +--- + +### Summary + +The relationship-oriented `bring` commands provide different views of the same metadata graph: + +| Command | Starting Point | Returns | +|---|---|---| +| `bring.extend` | UNS policy | The UNS policy together with its referenced object policy | +| `bring.children` | Policy | Its immediate child policies | +| `bring.parents` | Object policy | UNS policy IDs that reference the object | +| `bring.paths` | Object policy | Complete root-to-object path(s), including UNS and object policies | + +Together, these commands allow applications to navigate the AnyLog metadata and UNS structure without manually resolving policy IDs and parent relationships. \ No newline at end of file diff --git a/08- Blockchain & Metadata/03- Blockchain Commands.md b/08- Blockchain & Metadata/03- Blockchain Commands.md index 495c0981..8679ad04 100644 --- a/08- Blockchain & Metadata/03- Blockchain Commands.md +++ b/08- Blockchain & Metadata/03- Blockchain Commands.md @@ -9,327 +9,908 @@ layout: page |------------|----------------|------------------|----------| | 2026-07-27 | Ori Shadmon | Removed a duplicated "help blockchain set account info" block; fixed `blockchain deploy contract` example to match its own Usage line; condensed `help` transcripts into command+description style; flagged several open questions between this reference and other docs (see inline notes): `run blockchain sync`'s `connection` vs `master_node` param, `dest` specified twice in one call, `blockchain wait for` vs `blockchain wait where`, and whether the `where`-style `blockchain update` form still exists; restored the narrower `add`/`push`/`commit` insert-variant table; typo fixes | | | 2026-07-27 | Ori Shadmon | New page — split out of "03 Blockchain & Metadata.md". Moved the master-vs-blockchain-platform comparison and all connect/sync/seed commands to the new standalone Blockchain Connectivity doc, since those are about wiring a node to a ledger source rather than managing policies once connected. Fixed `master_npode` typo. | | + | 2026-08-15 | Moshe Shadmon | Updated Page. | | + ---> +--- +title: Blockchain Commands +description: Insert, query, and remove AnyLog metadata policies using the blockchain ledger. +layout: page +source_path: blockchain commands.md +--- -In general the blockchain or metadata layer is the platform that informs all the nodes in the network where data resides -and which nodes have access to what. The previous section discussed the different types -of policies and metadata content that can be stored in the blockchain. This section covers how to interact with the -blockchain layer. +# Blockchain Commands -* Connect & Sync -* Publish & Drop Policy -* Query the Blockchain +AnyLog uses a distributed ledger to maintain the metadata that describes the network: where data resides, which nodes +and services are available, how data is organized, and which policies control access and operation. -## Connect & Sync +The metadata ledger can be maintained using a blockchain platform, or alternatively using AnyLog's blockchain emulator, +referred to as a master node. -### Real Blockchain +Most blockchain commands are compatible with both implementations, allowing users to switch between a blockchain platform and a master node with minimal changes to their applications or workflows. +The metadata is stored as **policies**. A policy is a JSON object with a single root key, called the **policy type**. +Examples of policy types include `operator`, `cluster`, `publisher`, and `uns`. -`blockchain set account info where platform = [platform name] and [platform parameters]` - associate parameters -(private key, public key, chain ID, etc.) with a blockchain platform. +For normal operation, most users interact with the metadata layer through three commands: ```anylog - +blockchain insert +blockchain get +blockchain update +blockchain drop +``` + +| Command | Purpose | +|---|---| +| `blockchain insert` | Add a policy to the metadata ledger | +| `blockchain get` | Query policies from the local metadata view | +| `blockchain update` | Update an existing policy while preserving its policy ID | +| `blockchain drop` | Remove or invalidate an existing policy | + +These commands provide a consistent interface whether the global metadata ledger is maintained by a **master / metadata node** or by a blockchain platform such as Ethereum. + +> **Note:** Every AnyLog node maintains a local view of the metadata it needs. Queries are executed against this local view, so `blockchain get` does not depend on the availability or latency of the global ledger. + +--- + +## Metadata Storage Model + +AnyLog metadata is managed by the blockchain platform or master node, which serves as the shared metadata ledger for the network. + +Nodes in the network periodically synchronize with this ledger to maintain a local copy of the metadata. The local copy can be maintained as a JSON file, in a local database, or both. + +AnyLog commands and services use this local copy during normal operation, allowing each node to access the metadata it needs without requiring continuous access to the blockchain or master node. + +A node operates in the same manner regardless of how the global ledger is implemented. The configuration determines whether updates are sent to a master node or to a blockchain platform. + +When a policy is inserted into the local ledger before it is confirmed by the global ledger, AnyLog marks it with: + +```json +"ledger": "local" +``` + +After synchronization confirms the policy on the global ledger, the value changes to: + +```json +"ledger": "global" ``` -`blockchain deploy contract where platform = [platform name] and public_key = [public key]` - deploy the AnyLog -contract on the blockchain platform. +--- + +# `blockchain insert` + +`blockchain insert` adds a policy to the metadata ledger. + +It is the primary command for publishing metadata because it can update the local ledger and the configured global ledger in one operation. + +## Syntax ```anylog -blockchain deploy contract where platform = ethereum and public_key = !public_key +blockchain insert where + policy = [policy] + and local = [true|false] + and master = [IP:Port] + and blockchain = [platform] ``` +The `local`, `master`, and `blockchain` parameters identify the **destination(s) for the policy**. Only the destinations that apply to the deployment need to be specified. -> A metadata manager (i.e. master node) does not need to define the steps above. Instead, it creates a logical -> database and table (`blockchain.ledger`) and syncs the content against it. +| Parameter | Policy Destination | +|---|---| +| `local` | The node's local metadata copy | +| `master` | The AnyLog master node / blockchain emulator | +| `blockchain` | The configured blockchain platform, such as Ethereum | -### Master / Metadata Node Blockchain +A policy can be written to one or multiple destinations in the same command. -Master nodes, and optionally any node, can maintain the ledger in a local database: +For example, to insert a policy locally and into the master node: ```anylog -blockchain create table # create the ledger table -blockchain pull to json [output-file] # export to JSON file -blockchain pull to sql [output-file] # export as INSERT statements -blockchain pull to stdout # print to console -blockchain update dbms [file] # load file into local DB -sql blockchain "select * from ledger" # query directly with SQL +blockchain insert where policy = !policy and local = true and master = !master_node ``` +## Parameters + +| Parameter | Description | +|---|---| +| `policy` | JSON policy to insert | +| `local` | If `true`, update the local JSON ledger. Default: `true` | +| `master` | IP and port of the master / metadata node | +| `blockchain` | Connected blockchain platform, for example `ethereum` | + +A typical deployment writes to the local ledger and to **one** global ledger: + +- local ledger + master node, or +- local ledger + blockchain platform. -### Pull from a master node +## Insert using a master node ```anylog -master_node = 127.45.35.12:32048 -run client (!master_node) blockchain pull to json -run client (!master_node) file get !!blockchain_file !blockchain_file -blockchain load metadata # force node to use updated file +blockchain insert where + policy = !policy + and local = true + and master = !master_node ``` -### Blockchain sync +## Insert using a blockchain platform -`blockchain seed from [ip:port]` - pull the metadata from a source node (typically used once, on startup). +```anylog +blockchain insert where + policy = !policy + and local = true + and blockchain = ethereum +``` + +## Policy ID and date + +When a policy is inserted, AnyLog validates the policy and associates metadata such as its unique ID and update date. + +A policy ID can be provided explicitly, but in most cases AnyLog should generate the ID automatically from the policy content. + +If a policy needs a short, stable identifier because it will be referenced manually and frequently, a user-defined ID may be appropriate. + +### Prepare a policy before insertion ```anylog -blockchain seed from 73.202.142.172:7848 +blockchain prepare policy !operator ``` -`run blockchain sync where [options]` - repeatedly update the local copy of the blockchain. +If the policy `id` or `date` is not provided by the user, `blockchain prepare policy` adds the `id` and `date` attributes before the policy is published. -| Option | Description | +## Lower-level insert commands + +`blockchain insert` is the recommended general command. The following lower-level commands target a specific storage layer. + +| Command | Target | |---|---| -| `source` | The source of the metadata (`blockchain` or `master`) | -| `dest` | Destination for the blockchain data — `file` (local file) and/or `dbms` (local database) | -| `connection` | Connection info needed to retrieve the data — for a master, its IP:Port | -| `time` | Frequency of updates | +| `blockchain add [policy]` | Local JSON ledger only | +| `blockchain push [policy]` | Local metadata database only | +| `blockchain commit [policy]` | Blockchain platform only | + +Examples: ```anylog -run blockchain sync where source = master and time = 60 seconds and dest = file and dest = dbms and connection = !ip_port -run blockchain sync where source = blockchain and time = !sync_time and dest = file and platform = ethereum +blockchain add !policy +blockchain push !policy +blockchain commit !policy ``` -> The sync logic should run on every node in the network (though frequency may differ based on node type). +Use these commands when working directly with a specific ledger layer. For normal application workflows, prefer `blockchain insert`. --- -## Publish & Drop Policy +# `blockchain get` + +`blockchain get` queries metadata policies. + +Queries are processed against the **local metadata view** maintained by the node. This allows applications and commands to use metadata without waiting for a remote blockchain platform or master node. -`blockchain prepare policy [policy]` - add `id` and `date` attributes to a policy. +## Syntax ```anylog -blockchain prepare policy !operator +blockchain get [policy-type] [where ...] [bring ...] +``` + +The command has three main parts: + +1. **Policy type** — selects the type of policy. +2. **`where`** — optionally filters the policies. +3. **`bring`** — optionally extracts and formats values from the returned policies. + +--- + +## Select policies by type + +Return all operator policies: + +```anylog +blockchain get operator ``` -`blockchain insert where policy = [policy] and blockchain = [platform] and local = [true/false] and master = [IP:Port]` - -add a JSON policy to the specified destination(s). +Return multiple policy types: ```anylog -blockchain insert where policy = !policy and local = true and master = !ledger_conn -blockchain insert where policy = !policy and local = true and blockchain = ethereum +blockchain get (operator, publisher) ``` -| Key | Description | -|---|---| -| `policy` | The JSON policy to add | -| `local` | `true` — also update the local JSON file | -| `master` | IP:Port of the master node | -| `blockchain` | Blockchain platform name (e.g. `ethereum`) | +Return all policies: -When inserted locally, the policy gets `"ledger": "local"`. Once confirmed on the global ledger, it changes to `"ledger": "global"`. +```anylog +blockchain get * +``` -### Narrower insert variants +--- -| Command | Target | -|---|---| -| `blockchain add [policy]` | Local JSON file only | -| `blockchain push [policy]` | Local database only | -| `blockchain commit [policy]` | Blockchain platform only | +## Filter policies with `where` -`blockchain wait where [condition]` - pause the process until the local copy of the blockchain is updated with the -policy. `[condition]` is specified as `[key] = [value]`. +A simple condition uses attribute/value pairs: ```anylog -blockchain wait where policy = !operator -blockchain wait where id = [id] -blockchain wait where command = "blockchain get cluster where name = cluster_1" +blockchain get operator where dbms = my_data ``` -`blockchain update to [blockchain name] [policy_id] [policy]` - update an existing JSON policy on the blockchain platform. +Multiple conditions can be combined with `and`: ```anylog -blockchain update to ethereum !policy_id !policy +blockchain get operator where + dbms = my_data + and ip = 24.23.250.144 ``` -`blockchain drop policy where id = [policy id]` / `blockchain drop policy [policy]` - on the master node, delete the -provided policy (or policies) from the local blockchain database. +Another example: ```anylog -blockchain drop policy where id = 4a0c16ff565c6dfc05eb5a1aca4bf825 -blockchain drop policy !operator +blockchain get cluster where company = my-company +``` + +### Conditional expressions + +Square-bracket paths can be used when more complex Boolean logic is needed. + +```anylog +blockchain get operator where + [name] == operator1 + or [name] == operator2 ``` -Blockchain drop is interesting because the actual ledger is immutable - i.e. cannot be changed. So when a user executes -`blockchain drop policy` when using the master / metadata manager node, the policy / row is actually removed. -However, when using an actual blockchain, `blockchain drop policy` instead adds an annotation telling the network to -ignore the policy with the given ID. +```anylog +blockchain get operator where + [country] == US + and ([city] == "San Francisco" or [city] == "San Jose") +``` -Each policy has a unique ID, which can either be manually defined or automatically assigned based on a hash of the -policy's content. Since the id must be unique, we recommend letting AnyLog assign it automatically, unless the policy -is one you'll reference manually and often — where a short, readable ID is easier to work with than an auto-generated -hash (e.g. a schedule policy used to monitor nodes). +A path can identify nested values inside a policy. For example: -### Securing Policies +```text +[operator][name] +``` -By default, each policy has a unique ID that's based on the hash of the policy, or is defined by the user (though -this is less recommended). The system is also intelligent enough to stop a user from registering multiple nodes of -the same type on the same IP and port. +The root element may be omitted when the policy type is already known: + +```text +[name] +``` + +--- + +## Path matching + +Metadata paths can be filtered using path operators. + +### `startwith` + +Return paths beginning with the specified value: + +```anylog +blockchain get tag where + [path] startwith 'Root/Objects/DeviceSet' +``` -Since the blockchain is intended as a tool for untrusted groups to view / share data (almost like an immutable -contract), it may also be worth validating who is writing to the blockchain, using public / private keys: +### `childfrom` -`id sign [JSON Policy] where key = [private key] and password = [password]` / `id sign [JSON Policy] where password = [password]` - -sign a policy, adding the public key and signature to it. +Return child paths below the specified path: ```anylog -id sign !json_script where password = !my_password -id sign !json_script where key = !my_key and password = !my_password +blockchain get tag where + [path] childfrom 'Root/Objects/DeviceSet' ``` --- -## Query the Blockchain +# Formatting results with `bring` -Queries run against the **local copy** and do not depend on global ledger availability. +By default, `blockchain get` returns matching policy objects. `bring` can extract specific fields or transform the result. -| Command | Description | -|---|---| -| `blockchain get` | Returns policies after runtime dynamic updates (use this normally) | -| `blockchain read` | Returns policies exactly as received from the global ledger, without dynamic updates | +A detailed explanation of the ```bring``` option is available in the [05- JSON Data Transformation.md](../07-%20CLI/05-%20JSON%20Data%20Transformation.md#the-bring-keyword) section. -### Basic get +## Return a single field ```anylog -blockchain get [policy-type] -blockchain get operator -blockchain get (operator, publisher) # multiple types -blockchain get * # all policies +blockchain get operator bring [name] ``` -### Where conditions +## Combine fields ```anylog -blockchain get operator where dbms = my_data -blockchain get operator where dbms = my_data and ip = 24.23.250.144 -blockchain get cluster where company = my-company +blockchain get operator bring [name] [ip]:[port] +``` + +## Return IP:Port values + +```anylog +blockchain get operator bring.ip_port +``` + +## Table output + +```anylog +blockchain get operator bring.table +``` + +## JSON output + +```anylog +blockchain get operator bring.json ``` -Using conditional expressions (square-bracket paths): +## Sorted table + ```anylog -blockchain get operator where [name] == operator1 or [name] == operator2 -blockchain get operator where [country] == US and ([city] == "San Francisco" or [city] == "San Jose") +blockchain get operator bring.table.sort [operator][name] ``` -Special path operators: +--- + +## Query examples + +### Find operators supporting a table + ```anylog -blockchain get tag where [path] startwith 'Root/Objects/DeviceSet' -blockchain get tag where [path] childfrom 'Root/Objects/DeviceSet' +blockchain get operator where + dbms = my_data + and table = ping_sensor + bring [name] [ip]:[port] ``` -### bring — format the output +### Get operator addresses by country ```anylog -blockchain get operator bring [name] # single field -blockchain get operator bring [name] [ip]:[port] # concatenate fields -blockchain get operator bring.ip_port # standard IP:Port list -blockchain get operator bring.table # tabular output -blockchain get operator bring.json # JSON output -blockchain get operator bring.table.sort [operator][name] # sorted table +blockchain get operator where + [country] == US + or [country] == UK + bring.ip_port ``` -### from — apply bring to a variable +### Get the cluster ID for a table + +```anylog +blockchain get cluster where + table[dbms] = my_data + and table[name] = ping_sensor + bring [cluster][id] separator = , +``` + +--- + +# Save results and apply `bring` later + +The result of `blockchain get` can be assigned to a variable and processed separately. ```anylog operators = blockchain get operator -from !operators bring [name] [ip]:[port] ``` -**Examples** +Then: ```anylog -# All operators supporting a specific table -blockchain get operator where dbms = my_data and table = ping_sensor bring [name] [ip]:[port] +from !operators bring [name] [ip]:[port] +``` + +This is useful when the same set of policies is reused for multiple operations or output formats. + +--- + +# Use metadata to determine command destinations -# Operators in specific countries -blockchain get operator where [country] == US or [country] == UK bring.ip_port +A common AnyLog pattern is to query metadata and use the result as the destination of another command. -# Cluster ID for a specific table -blockchain get cluster where table[dbms] = my_data and table[name] = ping_sensor bring [cluster][id] separator = , +For example: + +```anylog +destinations = blockchain get (operator, query) where + [country] == US + or [country] == IL + bring [*][ip]:[*][port] separator = , ``` -### JOIN — keep both objects separate +Then: ```anylog -blockchain get bucket where name = my_bucket join (blockchain get operator where name = [bucket][operator]) +run client (!destinations) get node info net_io_counters ``` -Output: -```json -[{"bucket": {...}, "operator": {...}}] +The query can also be embedded directly: + +```anylog +run client ( + blockchain get (operator, query) where + [country] == US + or [country] == IL + bring [*][ip]:[*][port] separator = , +) get node info net_io_counters ``` -- If no RHS match → record is omitted (inner join behavior) -- Path interpolation: `[bucket][operator]` is resolved from the LHS record +This allows policies to dynamically define which AnyLog nodes receive a command. -### MERGE — flatten RHS into LHS +--- + +# Join and Merge Policy Queries + +`blockchain get` can combine metadata from related policies using `join` or `merge`. + +## `join` + +`join` preserves both policy objects as separate objects in the result. ```anylog -blockchain get bucket where name = my_bucket merge (blockchain get operator where name = [bucket][operator]) +blockchain get bucket where + name = my_bucket + join ( + blockchain get operator where + name = [bucket][operator] + ) ``` -Output: +Example result: + ```json -[{"bucket": {"name": "my_bucket", "operator": "op1", "ip": "24.5.219.50", "port": 7848}}] +[ + { + "bucket": { + "name": "my_bucket", + "operator": "operator1" + }, + "operator": { + "name": "operator1", + "ip": "24.5.219.50", + "port": 7848 + } + } +] ``` -- LHS wins on key conflicts -- If no RHS match → LHS returned unchanged (left merge behavior) +Behavior: + +- the left-hand and right-hand policies remain separate; +- values from the left-hand policy can be referenced in the right-hand query; +- if the right-hand query has no match, the record is omitted. -### With bring formatting +### Format joined results ```anylog -# Table output with join -blockchain get bucket where name = my_bucket join (blockchain get operator where name = [bucket][operator]) bring.table [bucket][name] [operator][ip] [operator][port] +blockchain get bucket where + name = my_bucket + join ( + blockchain get operator where + name = [bucket][operator] + ) + bring.table + [bucket][name] + [operator][ip] + [operator][port] +``` + +--- -# JSON output with merge -blockchain get bucket where name = my_bucket merge (blockchain get operator where name = [bucket][operator]) bring.json [bucket][name] [bucket][ip] [bucket][port] +## `merge` + +`merge` adds fields returned by the second query directly into the first policy object. + +```anylog +blockchain get bucket where + name = my_bucket + merge ( + blockchain get operator where + name = [bucket][operator] + ) ``` -## Root and child policies (UNS hierarchy) +Example result: -Root policies have no `parent` attribute. Child policies reference a parent via the `parent` field, forming a hierarchy used by the Unified Namespace (UNS). +```json +[ + { + "bucket": { + "name": "my_bucket", + "operator": "operator1", + "ip": "24.5.219.50", + "port": 7848 + } + } +] +``` + +Behavior: + +- fields from the right-hand result are added to the left-hand object; +- the left-hand policy wins if both objects contain the same key; +- if no right-hand policy matches, the left-hand policy is returned unchanged. + +--- + +# Root and Child Policies + +AnyLog policies can form a hierarchy. This is used, among other things, to represent Unified Namespace (UNS) structures. + +A **root policy** does not contain a `parent` attribute. + +A **child policy** references another policy through its `parent` attribute. + +Return root policies: ```anylog blockchain get root policies ``` -Example output: +Example: + ```json [ - {"uns": {"name": "Enterprise_A", "namespace": "Enterprise_A", "id": "00ddf..."}}, - {"uns": {"name": "Sensors", "namespace": "Enterprise_A/Sensors", "parent": "00ddf...", "dbms": "my_data", "table": "ping_sensor"}} + { + "uns": { + "name": "Enterprise_A", + "namespace": "Enterprise_A", + "id": "00ddf..." + } + }, + { + "uns": { + "name": "Sensors", + "namespace": "Enterprise_A/Sensors", + "parent": "00ddf...", + "dbms": "my_data", + "table": "ping_sensor" + } + } ] ``` -Child policies inherit structure from their parent and carry `dbms`/`table` attributes used by the query engine. +In this example, `Enterprise_A` is a root policy and `Sensors` is a child policy. + +## Include selected root policy types + +```anylog +blockchain get root policies include cluster uns +``` + +## Exclude selected root policy types + +```anylog +blockchain get root policies exclude cluster +``` + +--- + +# `blockchain get` vs. `blockchain read` + +For normal metadata queries, use: + +```anylog +blockchain get +``` + +`blockchain get` returns the node's operational view of the policies after AnyLog has applied runtime and dynamic updates. + +`blockchain read` returns the policies as they were received from the global ledger, before those dynamic updates. + +```anylog +blockchain read operator +``` + +Use `blockchain read` primarily for troubleshooting or for examining the source representation of a policy. + +--- + +# `blockchain drop` + +`blockchain drop` removes or invalidates metadata policies. + +The exact behavior depends on the type of global ledger. + +## Drop by policy ID + +```anylog +blockchain drop policy where + id = 4a0c16ff565c6dfc05eb5a1aca4bf825 +``` + +A variable can also be used: + +```anylog +blockchain drop policy where id = !policy_id +``` + +## Drop using a policy object + +```anylog +blockchain drop policy !operator +``` + +--- + +## Master node vs. immutable blockchain + +The meaning of "drop" is important because a blockchain ledger is immutable. + +### Master / metadata node + +When the global ledger is maintained by a master / metadata node, the matching policy can be removed from the master's local metadata database. + +### Blockchain platform + +When the global ledger is an immutable blockchain platform, the original ledger entry cannot be physically deleted. + +Instead, AnyLog records metadata that identifies the policy as dropped so that the network ignores the policy. + +From the application's point of view, the policy is no longer active even though the historical blockchain record remains immutable. + +--- + +## Drop policies associated with a host + +A master metadata database can remove policies associated with a particular host: + +```anylog +blockchain drop by host +``` + +This is a lower-level administrative operation and should be used when cleaning metadata associated with a node or host rather than removing an individual policy. + +--- + +# Securing Policies + +Policies can be signed to verify the identity of the party writing metadata. + +## Sign using the configured key + +```anylog +id sign !json_script where + password = !my_password +``` + +## Sign using a specified private key + +```anylog +id sign !json_script where + key = !my_key + and password = !my_password +``` + +The signature information is added to the JSON policy and can be used to validate the source of metadata written to the ledger. + +--- + +# Connecting and Synchronizing the Ledger + +Most application users only need `blockchain insert`, `blockchain get`, and `blockchain drop`. + +The commands in this section are primarily used when configuring or administering the metadata infrastructure. + +--- + +## Seed a node from another AnyLog node + +When a new node joins an existing network, it can retrieve an initial metadata copy from another node: + +```anylog +blockchain seed from 73.202.142.172:7848 +``` + +General form: + +```anylog +blockchain seed from [IP:Port] +``` + +This operation is commonly used during startup. Ongoing updates should normally be handled by blockchain synchronization. + +--- + +# Blockchain Synchronization + +`run blockchain sync` continuously refreshes the local metadata representation. + +## Synchronize from a master node + +```anylog +run blockchain sync where + source = master + and time = 60 seconds + and dest = file + and dest = dbms + and connection = !ip_port +``` + +## Synchronize from a blockchain platform + +```anylog +run blockchain sync where + source = blockchain + and time = !sync_time + and dest = file + and platform = ethereum +``` + +## Synchronization options + +| Option | Description | +|---|---| +| `source` | Metadata source: `master` or `blockchain` | +| `dest` | Destination to update: `file` and/or `dbms` | +| `connection` | Connection information for a master node | +| `platform` | Blockchain platform when `source = blockchain` | +| `time` | Synchronization frequency | + +Every node that depends on changing metadata should maintain an appropriate synchronization process. The required frequency may vary by node role. + +--- + +# Master / Metadata Node Administration + +A master / metadata node maintains a complete metadata ledger in a local database. + +## Create the ledger table + +```anylog +blockchain create table +``` + +The ledger is stored in: + +```text +blockchain.ledger +``` -## Compare Policies +## Export the ledger as JSON + +```anylog +blockchain pull to json [output-file] +``` + +## Export the ledger as SQL + +```anylog +blockchain pull to sql [output-file] +``` + +## Print the ledger + +```anylog +blockchain pull to stdout +``` + +## Load a ledger file into the local database + +```anylog +blockchain update dbms [file] +``` + +## Query the master ledger directly + +```anylog +sql blockchain "select * from ledger" +``` + +Direct SQL access is primarily an administrative and troubleshooting capability. Application logic should normally query metadata using `blockchain get`. + +--- + +# Copy Metadata from a Master Node + +A node can explicitly retrieve the ledger from a master node. + +```anylog +master_node = 127.45.35.12:32048 +``` + +Retrieve the ledger: + +```anylog +run client (!master_node) blockchain pull to json +``` + +Copy the generated file: + +```anylog +run client (!master_node) file get !!blockchain_file !blockchain_file +``` + +If automatic synchronization is not running, force AnyLog to reload the updated local metadata: + +```anylog +blockchain load metadata +``` + +--- + +# Blockchain Platform Setup + +When the global ledger is maintained on a blockchain platform, the platform must be configured before policies can be published. + +## Configure account information + +```anylog +blockchain set account info where + platform = ethereum + and private_key = !private_key + and public_key = !public_key + and chain_id = 11155111 +``` + +## Deploy the AnyLog contract + +```anylog +blockchain deploy contract where + platform = ethereum + and public_key = !public_key +``` + +A master / metadata node does not require blockchain account or smart-contract configuration because its global ledger is maintained in the local metadata database. + +--- + +# Update an Existing Policy + +An existing policy can be updated while preserving its policy ID. + +The ID must already exist on the target ledger, and the ID in the policy must match the ID supplied to the update command. + +Example for Ethereum: + +```anylog +blockchain update to ethereum !policy_id !policy +``` + +An update should be used when the policy represents the same logical object. Use `blockchain insert` when publishing a new policy. + +--- + +# Compare Policies + +Policies can be compared to identify differences in attributes and values. + +## Syntax + +```anylog +get policies diff [object-1] [object-2] +``` -Policies can be compared to determine the different attribute and values. -The following command returns a report indicating the differences between the two policies, or between lists of policies. -Usage: -```anylog -get policies diff [object 1] [object 2] -``` -Object 1 and Object 2 are policies or lists of policies to compare. -When lists are compared, the number of policies in the lists needs to be equal with one exception: -If a policy is compared to a list with a single policy, the policy is assumed to be in a list, and the comparison is allowed. Example: -```anylog + +```anylog get policies diff !policy1 !policy2 ``` -## Other blockchain commands +The objects can be individual policies or lists of policies. + +When two lists are compared, the lists normally need to contain the same number of policies. A single policy may also be compared with a list containing one policy. + +--- + +# Additional Blockchain Commands + +The following commands are useful for validation, troubleshooting, administration, or direct manipulation of a specific metadata layer. | Command | Description | |---|---| -| `blockchain test` | Validate local JSON file structure | -| `blockchain test id` | Check if a policy ID exists locally | -| `blockchain get id [json]` | Return the hash of a JSON structure | -| `blockchain prepare policy [json]` | Add ID and date to a policy | -| `blockchain checkout` | Pull latest data from blockchain platform to local JSON | -| `blockchain update file [path]` | Replace local blockchain file (backs up `.old`) | -| `blockchain delete local file` | Delete the local JSON file | -| `blockchain query metadata` | Diagram view of the local metadata structure | -| `blockchain test cluster` | Analyse cluster policies | -| `blockchain state where platform = [name]` | State of the active contract | \ No newline at end of file +| `blockchain test` | Validate the local blockchain JSON file and its structure | +| `blockchain test id` | Test whether a policy ID exists locally | +| `blockchain get id [json]` | Return the hash / ID associated with a JSON structure | +| `blockchain prepare policy [json]` | Add an ID and date to a policy | +| `blockchain checkout` | Retrieve the latest ledger data from a blockchain platform | +| `blockchain update file [path]` | Replace the local blockchain file and preserve the prior version as `.old` | +| `blockchain delete local file` | Delete the local JSON ledger file | +| `blockchain query metadata` | Display a diagram view of the local metadata structure | +| `blockchain test cluster` | Analyze cluster policies | +| `blockchain state where platform = [name]` | Return the state of the active blockchain contract | + +--- + +# Recommended Usage + +For most AnyLog applications and integrations, the primary blockchain commands are `insert`, `get`, `update`, and `drop`: + +```anylog +# Publish metadata +blockchain insert where policy = !policy and local = true and master = !master_node + +# Query metadata +blockchain get operator where dbms = my_data bring [name] [ip]:[port] + +# Update existing metadata +blockchain update to ethereum !policy_id !policy + +# Remove metadata +blockchain drop policy where id = !policy_id +``` + +The remaining blockchain commands support deployment configuration, synchronization, administration, debugging, or direct access to an individual ledger layer. diff --git a/11- Extended Services/01- Performance.md b/11- Extended Services/01- Performance.md index 1e775668..cd2c5043 100644 --- a/11- Extended Services/01- Performance.md +++ b/11- Extended Services/01- Performance.md @@ -79,7 +79,6 @@ run helpers where type = [helper type] and count = [helpers count] | type | Helper type (e.g., psql). Defines what kind of task the helper will process. | | count | Number of helper processes to launch. Each runs independently in parallel. | - **Example:** ```anylog run helpers where type = psql and count = 2 @@ -134,13 +133,11 @@ helper * * exit node helper psql 1 exit node ``` - ## Dynamic monitoring of internal processes The `get dynamic stats` command retrieves **live execution metadata** about a specific operation running in the main or helper processes — such as timing, status, or active resource usage — by referencing its associated request or file name. - **Usage:** ```anylog get dynamic stats where name = [monitored topic] @@ -154,7 +151,6 @@ get dynamic stats where name = [monitored topic] | operator.sql | psql | The SQL processing time | | operator.jql | psql | The SQL processing time directly from JSON conversion | - **Examples:** ```anylog helper psql 1 get dynamic stats where name = operator.json @@ -168,7 +164,6 @@ supports multiple types of SQL-based physical databases. However, we do not reco performance-dependent insertion as it's a single-file store with concurrency limitations. Instead it is recommended to use a server-based physical database like Postgres. - However, even with Postgres, the default configuration can still be improved substantially. On a mid-size machine with configuration: | Parameter | Value | @@ -212,7 +207,6 @@ max_parallel_workers = 16 # Match to your logical cores > Disk and Indexing: Remember to drop indexes before bulk inserts and recreate afterward for better performance - ## Sample Results With the machine above, and the provided Postgres configurations, we configured AnyLog as follows: