diff --git a/apps/importer/.env.example b/apps/importer/.env.example new file mode 100644 index 0000000..14ff8bc --- /dev/null +++ b/apps/importer/.env.example @@ -0,0 +1,24 @@ +# Copy to .env, which is ignored, and keep it readable by nobody else: +# +# cp .env.example .env && chmod 600 .env +# +# Anything already set in the shell wins over this file, so a single run can be +# pointed at another relay without editing it. + +# The relays `publish:events` sends to, separated by commas, and what `--relay` +# overrides. There is no default and there will not be one: where a corpus +# signed by a key this project holds ends up is a decision, and one taken by +# leaving a variable empty is not one. +OPENSPECS_IMPORT_RELAYS=ws://127.0.0.1:7777 + +# One key per corpus, named after its manifest in ./manifests. An nsec, or the +# same key in hex. It signs the documents copied from that repository, and what +# it proves is who copied them, never who wrote them: every event carries the +# file and the commit it came from, and anyone can fetch the source and check +# the copy against it. +# +# The importer refuses to sign with a key that is not the one its manifest +# names, since a corpus published under the wrong key cannot be taken back. +OPENSPECS_IMPORT_KEY_NIPS= +OPENSPECS_IMPORT_KEY_BUDS= +OPENSPECS_IMPORT_KEY_NUTS= diff --git a/apps/importer/.gitignore b/apps/importer/.gitignore new file mode 100644 index 0000000..96f6762 --- /dev/null +++ b/apps/importer/.gitignore @@ -0,0 +1,3 @@ +.cache +events +.env diff --git a/apps/importer/README.md b/apps/importer/README.md new file mode 100644 index 0000000..2f86498 --- /dev/null +++ b/apps/importer/README.md @@ -0,0 +1,81 @@ +# Importer + +Publishes specifications that live in git as ordinary Nostr documents: the NIPs, the BUDs +and the NUTs, one addressable event each, signed by a key of their own per corpus. + +The key proves who copied a document, never who wrote it. What stands in for authorship +is that the copy can be checked: every event names the file it came from in a `proxy` tag, +pinned to a commit, and carries the sha256 of that file's bytes in an `x` tag. Fetch the +source, hash it, compare. + +That only means something if the import is reproducible, so it is a pure function of the +manifests and the source repository. Nothing is read from a clock, from the network, or +from a model. Two runs at the same commits write the same bytes, and `manifests/README.md` +is where the judgement went. + +## Commands + +```sh +pnpm --filter @openspecs/importer build:events # write the events, sign nothing +pnpm --filter @openspecs/importer publish:events # show what would be sent +pnpm --filter @openspecs/importer publish:events --yes +``` + +`build:events` clones each repository into `.cache`, reads every file the manifest lists +at the head of its branch, rewrites the links, and writes one unsigned event per document +into `events/`. Both directories are ignored: one is a checkout, the other is output. + +It reports what left the corpus, which is the part worth reading. A link pinned to GitHub +is either a file that is genuinely not a specification, or one the manifest forgot. + +## Links + +A document that only makes sense next to a website is not a document that lives on relays, +so cross references are rewritten: + +| Link | Becomes | +| --------------------------------------------- | -------------------------------- | +| To a document of any of the three manifests | `nostr:naddr1...`, fragment kept | +| To another file of the same repository | GitHub, pinned to the commit | +| To anywhere else | left alone | +| Into the document itself | left alone, `rehype-slug` resolves it | + +Fenced code is never touched: a link inside an example is part of the example. + +## Publishing + +`publish:events` reads what `build:events` wrote, signs it, and sends what the relays +do not already hold. It prints its plan and stops; `--yes` is what sends. + +A run that changes nothing sends nothing. Every document is compared against the +revision the relays serve, and only what differs is signed, which is what makes running +it again safe and what running it again is for. A document that changed in its manifest +rather than in git is stamped one second past the revision it replaces, since a relay +keeps the older of two events sharing a timestamp. + +Keys come from the environment, one per corpus, `OPENSPECS_IMPORT_KEY_`, an nsec +or the same key in hex. Every key is resolved before the first event is sent, and a key +that is not the one its manifest names is refused: a corpus published under the wrong key +cannot be taken back. + +Relays come from `--relay`, repeatable, or `OPENSPECS_IMPORT_RELAYS`. There is no default +and there will not be one. `infra/docker-compose.relay.yml` runs the relay this was +written for. + +Both are read from `.env` beside this file, which is ignored, and anything already set in +the shell wins over it: + +```sh +cp .env.example .env && chmod 600 .env +``` + +A file rather than a variable typed in front of the command, because a command is written +to a shell history and a key does not belong there. And a file of its own rather than +`infra/.env`, because that one is read by Docker Compose, and nothing here is a service: +publishing is a command somebody runs on purpose. + +## What it does not do yet + +Walk history. Every revision a document had before this import is still only in git, and +until it is written as kind 1349 snapshots, publishing a new revision replaces the last +one on the relays and nothing keeps what it said. diff --git a/apps/importer/manifests/README.md b/apps/importer/manifests/README.md new file mode 100644 index 0000000..242d9f8 --- /dev/null +++ b/apps/importer/manifests/README.md @@ -0,0 +1,57 @@ +# Manifests + +One file per corpus. Each lists every document to import, and carries everything the +importer cannot read from the document itself. + +The importer is a pure function of these files and the source repository. Nothing here is +computed at run time, so two runs at the same commits produce the same events, and a +third party can rebuild the corpus and compare it byte for byte. + +## Fields + +| Field | Meaning | +| --------- | ------------------------------------------------------------------ | +| `file` | Path in the source repository | +| `d` | Identifier of the addressable event, and the last part of its URL | +| `title` | `title` tag | +| `status` | `s` tag, or null when the document claims none | +| `summary` | `summary` tag | +| `kinds` | `k` tags, omitted when the document defines none | + +`topics` is declared once per corpus and applies to every document in it. + +## How these values were chosen + +**The title carries the number.** A listing row shows the title alone, and nobody looks +for "Basic protocol flow description". `NIP-01: Basic protocol flow description`. + +**Status uses each corpus's own vocabulary**, read from the badge line under the heading: +`draft`, `final`, `unrecommended` for NIPs, `draft` for BUDs, `mandatory` or `optional` +for NUTs. A document with no badge gets null rather than a guess. The NIPs marked +`unrecommended` are those struck through in the repository's own index, which is where +that judgement is made and the only place it is written down. + +The site colours `draft`, `final` and the closed states, but not `unrecommended`. Adding +it to `STATUS_TONE` in `spec-tags.tsx` is a one line change, and until then those +documents render neutral. + +**Kinds come from the tables the repositories maintain**, the event kinds table in the +NIPs index and the one in the Blossom readme. Only kinds a document actually defines are +listed: NUT-27 carries 30078 because it specifies how that kind is used, and the other +NUTs carry none. + +**Summaries are written here rather than derived.** The site can take a description from +the first paragraph, but that paragraph is written for a reader who already knows what +the document is. + +## Membership + +These files are the whole truth about what exists. `nips.json` holds 94 documents where +the repository has 98 files: `12.md`, `16.md`, `20.md` and `33.md` were folded into +NIP-01 and dropped from the index, so they are gone from here too. Files that are not +specifications, `error_codes.md` and `suppl/` in the NUTs repository among them, are +absent for the same reason. + +A new specification appears when a line is added. Removing one withdraws the document, +by publishing an empty revision over it. Neither happens on its own, which is the point: +both are a diff somebody read. diff --git a/apps/importer/manifests/buds.json b/apps/importer/manifests/buds.json new file mode 100644 index 0000000..f1983fd --- /dev/null +++ b/apps/importer/manifests/buds.json @@ -0,0 +1,114 @@ +{ + "name": "buds", + "title": "Blossom Upgrade Documents", + "npub": "npub1zvm2zlskr58g4u4k3m54454y087r30hedgtavyn75q4yp55dm9lqz4932r", + "repo": "https://github.com/hzrd149/blossom", + "branch": "master", + "license": "Unlicense", + "topics": ["blossom", "bud"], + "specs": [ + { + "file": "buds/00.md", + "d": "bud-00", + "title": "BUD-00: Blossom Upgrade Documents", + "status": "draft", + "summary": "Defines the language every other BUD is written in, and what a blob is: raw binary data addressed by the sha256 hash of that data." + }, + { + "file": "buds/01.md", + "d": "bud-01", + "title": "BUD-01: Server requirements and blob retrieval", + "status": "draft", + "summary": "The base every Blossom server implements: the CORS headers, the error reporting, and the GET and HEAD endpoints that serve a blob by its hash." + }, + { + "file": "buds/02.md", + "d": "bud-02", + "title": "BUD-02: Blob upload", + "status": "draft", + "summary": "Defines PUT /upload and the blob descriptor, the JSON object describing a stored blob: its url, hash, size, mime type and upload time." + }, + { + "file": "buds/03.md", + "d": "bud-03", + "title": "BUD-03: User Server List", + "status": "draft", + "summary": "Defines kind 10063, the replaceable event where a user lists the Blossom servers holding their blobs, most trusted first.", + "kinds": [ + { + "kind": 10063, + "name": "User server list" + } + ] + }, + { + "file": "buds/04.md", + "d": "bud-04", + "title": "BUD-04: Mirroring blobs", + "status": "draft", + "summary": "Defines PUT /mirror, which asks a server to fetch a blob from a URL itself instead of having the client upload the same bytes twice." + }, + { + "file": "buds/05.md", + "d": "bud-05", + "title": "BUD-05: Media optimization endpoints", + "status": "draft", + "summary": "Defines PUT /media, where a server transcodes and optimises what it is given rather than storing the bytes untouched." + }, + { + "file": "buds/06.md", + "d": "bud-06", + "title": "BUD-06: Upload requirements", + "status": "draft", + "summary": "Defines HEAD /upload, a preflight request telling a client whether an upload would be accepted before it sends the bytes." + }, + { + "file": "buds/07.md", + "d": "bud-07", + "title": "BUD-07: Paid upload and download", + "status": "draft", + "summary": "How a server charges for an endpoint: a 402 response carrying the payment terms, for uploads, downloads or media processing." + }, + { + "file": "buds/08.md", + "d": "bud-08", + "title": "BUD-08: Nostr File Metadata Tags", + "status": "draft", + "summary": "How a server returns NIP-94 file metadata alongside a blob descriptor, so a client can publish the file without recomputing it." + }, + { + "file": "buds/09.md", + "d": "bud-09", + "title": "BUD-09: Blob Report", + "status": "draft", + "summary": "Defines PUT /report, where a client submits a signed NIP-56 report naming the blobs it objects to." + }, + { + "file": "buds/10.md", + "d": "bud-10", + "title": "BUD-10: Blossom URI Schema", + "status": "draft", + "summary": "Defines the blossom: URI, which carries a blob hash, its extension, and hints about the servers where it can be found." + }, + { + "file": "buds/11.md", + "d": "bud-11", + "title": "BUD-11: Nostr Authorization", + "status": "draft", + "summary": "Defines the kind 24242 event a user signs to prove to a server that they allowed an application to act on their behalf.", + "kinds": [ + { + "kind": 24242, + "name": "Blossom authorization" + } + ] + }, + { + "file": "buds/12.md", + "d": "bud-12", + "title": "BUD-12: Blob management endpoints", + "status": "draft", + "summary": "Defines the management endpoints: listing what a pubkey uploaded, and deleting a blob by its hash." + } + ] +} diff --git a/apps/importer/manifests/nips.json b/apps/importer/manifests/nips.json new file mode 100644 index 0000000..726c49e --- /dev/null +++ b/apps/importer/manifests/nips.json @@ -0,0 +1,1443 @@ +{ + "name": "nips", + "title": "Nostr Implementation Possibilities", + "npub": "npub19vumfllx9yeal9cwrymxcgkpuzf0j5lc8l876a2wpuzdtga5t8usdm3xgr", + "repo": "https://github.com/nostr-protocol/nips", + "branch": "master", + "license": "public domain", + "topics": ["nostr", "nip"], + "specs": [ + { + "file": "01.md", + "d": "nip-01", + "title": "NIP-01: Basic protocol flow description", + "status": "draft", + "summary": "The core protocol: the event object, how its id and signature are computed, what makes a kind regular, replaceable or addressable, and the messages clients and relays exchange.", + "kinds": [ + { + "kind": 0, + "name": "User Metadata" + } + ] + }, + { + "file": "02.md", + "d": "nip-02", + "title": "NIP-02: Follow List", + "status": "final", + "summary": "Kind 3, the follow list: one p tag per followed key, each carrying an optional relay hint and petname.", + "kinds": [ + { + "kind": 3, + "name": "Follows" + } + ] + }, + { + "file": "03.md", + "d": "nip-03", + "title": "NIP-03: OpenTimestamps Attestations for Events", + "status": "unrecommended", + "summary": "Kind 1040, which carries an OpenTimestamps proof that another event existed before a given Bitcoin block.", + "kinds": [ + { + "kind": 1040, + "name": "OpenTimestamps" + } + ] + }, + { + "file": "04.md", + "d": "nip-04", + "title": "NIP-04: Encrypted Direct Message", + "status": "unrecommended", + "summary": "Kind 4, encrypted direct messages built on a shared secret and aes-256-cbc, superseded by NIP-17.", + "kinds": [ + { + "kind": 4, + "name": "Encrypted Direct Messages" + } + ] + }, + { + "file": "05.md", + "d": "nip-05", + "title": "NIP-05: Mapping Nostr keys to DNS-based internet identifiers", + "status": "final", + "summary": "How a key claims an email-like address, by serving a nostr.json document under the domain that maps a name to that pubkey." + }, + { + "file": "06.md", + "d": "nip-06", + "title": "NIP-06: Basic key derivation from mnemonic seed phrase", + "status": "unrecommended", + "summary": "Deriving a key from a BIP39 mnemonic along the m/44'/1237' path." + }, + { + "file": "07.md", + "d": "nip-07", + "title": "NIP-07: `window.nostr` capability for web browsers", + "status": "draft", + "summary": "The window.nostr object a browser extension exposes, letting a web page read a public key and have events signed without ever seeing the private key." + }, + { + "file": "08.md", + "d": "nip-08", + "title": "NIP-08: Handling Mentions", + "status": "unrecommended", + "summary": "The old #[index] notation for mentioning events and pubkeys inside a note, replaced by NIP-27." + }, + { + "file": "09.md", + "d": "nip-09", + "title": "NIP-09: Event Deletion Request", + "status": "draft", + "summary": "Kind 5, the deletion request, where an author asks relays to drop events they signed earlier.", + "kinds": [ + { + "kind": 5, + "name": "Event Deletion Request" + } + ] + }, + { + "file": "10.md", + "d": "nip-10", + "title": "NIP-10: Text Notes and Threads", + "status": "draft", + "summary": "How kind 1 notes thread: the markers on e tags that name the root and the parent, and the q tags that cite another event.", + "kinds": [ + { + "kind": 1, + "name": "Short Text Note" + } + ] + }, + { + "file": "11.md", + "d": "nip-11", + "title": "NIP-11: Relay Information Document", + "status": "draft", + "summary": "The JSON document a relay serves over HTTP to describe itself: its name, its contacts, the NIPs it supports, its limits and its policies." + }, + { + "file": "13.md", + "d": "nip-13", + "title": "NIP-13: Proof of Work", + "status": "draft", + "summary": "Proof of work over an event id, mined with a nonce tag and measured in leading zero bits." + }, + { + "file": "14.md", + "d": "nip-14", + "title": "NIP-14: Subject tag in text events", + "status": "draft", + "summary": "The subject tag, which gives a note a heading the way an email has one." + }, + { + "file": "15.md", + "d": "nip-15", + "title": "NIP-15: Nostr Marketplace (for resilient marketplaces)", + "status": "unrecommended", + "summary": "A marketplace protocol built from stalls, products and order messages, kept for the clients that implemented it.", + "kinds": [ + { + "kind": 1021, + "name": "Bid" + }, + { + "kind": 1022, + "name": "Bid confirmation" + }, + { + "kind": 30017, + "name": "Create or update a stall" + }, + { + "kind": 30018, + "name": "Create or update a product" + }, + { + "kind": 30019, + "name": "Marketplace UI/UX" + }, + { + "kind": 30020, + "name": "Product sold as an auction" + } + ] + }, + { + "file": "17.md", + "d": "nip-17", + "title": "NIP-17: Private Direct Messages", + "status": "draft", + "summary": "Private direct messages, built from NIP-44 encryption and NIP-59 gift wraps, where the set of participants is what defines a room.", + "kinds": [ + { + "kind": 14, + "name": "Direct Message" + }, + { + "kind": 15, + "name": "File Message" + }, + { + "kind": 10050, + "name": "Relay list to receive DMs" + } + ] + }, + { + "file": "18.md", + "d": "nip-18", + "title": "NIP-18: Reposts", + "status": "draft", + "summary": "Kind 6, the repost, which carries the reposted note in its content and points at it with an e tag.", + "kinds": [ + { + "kind": 6, + "name": "Repost" + }, + { + "kind": 16, + "name": "Generic Repost" + } + ] + }, + { + "file": "19.md", + "d": "nip-19", + "title": "NIP-19: bech32-encoded entities", + "status": "draft", + "summary": "The bech32 forms of keys, ids and pointers: npub, nsec, note, nprofile, nevent and naddr." + }, + { + "file": "21.md", + "d": "nip-21", + "title": "NIP-21: `nostr:` URI scheme", + "status": "draft", + "summary": "The nostr: URI scheme, which turns a NIP-19 entity into a link any client can open." + }, + { + "file": "22.md", + "d": "nip-22", + "title": "NIP-22: Comment", + "status": "draft", + "summary": "Kind 1111, the comment: a reply scoped to a root event or to an external identifier, with uppercase tags for the root and lowercase ones for the parent.", + "kinds": [ + { + "kind": 1111, + "name": "Comment" + } + ] + }, + { + "file": "23.md", + "d": "nip-23", + "title": "NIP-23: Long-form Content", + "status": "draft", + "summary": "Kind 30023, long-form articles written in Markdown, addressable so their author can revise them in place.", + "kinds": [ + { + "kind": 30023, + "name": "Long-form Content" + }, + { + "kind": 30024, + "name": "Draft Long-form Content" + } + ] + }, + { + "file": "24.md", + "d": "nip-24", + "title": "NIP-24: Extra metadata fields and tags", + "status": "draft", + "summary": "The fields and tags that became conventions without a NIP of their own, from display_name to the alt tag." + }, + { + "file": "25.md", + "d": "nip-25", + "title": "NIP-25: Reactions", + "status": "draft", + "summary": "Kind 7, the reaction, whose content is a plus, a minus or an emoji, pointing at the event it answers.", + "kinds": [ + { + "kind": 7, + "name": "Reaction" + }, + { + "kind": 17, + "name": "Reaction to a website" + } + ] + }, + { + "file": "26.md", + "d": "nip-26", + "title": "NIP-26: Delegated Event Signing", + "status": "unrecommended", + "summary": "The delegation tag, which let one key authorise another to sign events on its behalf." + }, + { + "file": "27.md", + "d": "nip-27", + "title": "NIP-27: Text Note References", + "status": "draft", + "summary": "How to write a mention inside content: a nostr: URI in the text, and nothing else demanded of the reader." + }, + { + "file": "28.md", + "d": "nip-28", + "title": "NIP-28: Public Chat", + "status": "unrecommended", + "summary": "Kinds 40 to 44, public chat channels with moderation left to the client.", + "kinds": [ + { + "kind": 40, + "name": "Channel Creation" + }, + { + "kind": 41, + "name": "Channel Metadata" + }, + { + "kind": 42, + "name": "Channel Message" + }, + { + "kind": 43, + "name": "Channel Hide Message" + }, + { + "kind": 44, + "name": "Channel Mute User" + } + ] + }, + { + "file": "29.md", + "d": "nip-29", + "title": "NIP-29: Relay-based Groups", + "status": "draft", + "summary": "Groups whose membership and rules are enforced by the relay hosting them, identified by an id scoped to that relay.", + "kinds": [ + { + "kind": 10009, + "name": "User groups" + } + ] + }, + { + "file": "30.md", + "d": "nip-30", + "title": "NIP-30: Custom Emoji", + "status": "draft", + "summary": "The emoji tag, which maps a shortcode written in the content to an image URL." + }, + { + "file": "31.md", + "d": "nip-31", + "title": "NIP-31: Dealing with Unknown Events", + "status": "unrecommended", + "summary": "The alt tag, a plain text summary so a client that does not know a kind can still show something for it." + }, + { + "file": "32.md", + "d": "nip-32", + "title": "NIP-32: Labeling", + "status": "draft", + "summary": "Kind 1985 and the L and l tags, which attach a namespaced label to an event, a key, or an external identifier.", + "kinds": [ + { + "kind": 1985, + "name": "Label" + } + ] + }, + { + "file": "34.md", + "d": "nip-34", + "title": "NIP-34: `git` stuff", + "status": "draft", + "summary": "Code collaboration on Nostr: repository announcements, repository state, patches, and issues.", + "kinds": [ + { + "kind": 1617, + "name": "Patches" + }, + { + "kind": 1618, + "name": "Pull Requests" + }, + { + "kind": 1619, + "name": "Pull Request Updates" + }, + { + "kind": 1621, + "name": "Issues" + }, + { + "kind": 1622, + "name": "Git Replies (deprecated)" + }, + { + "kind": 30617, + "name": "Repository announcements" + }, + { + "kind": 30618, + "name": "Repository state announcements" + } + ] + }, + { + "file": "35.md", + "d": "nip-35", + "title": "NIP-35: Torrents", + "status": "draft", + "summary": "Kind 2003, a torrent index carrying the info hash, the file list and the trackers needed to rebuild a magnet link.", + "kinds": [ + { + "kind": 2003, + "name": "Torrent" + }, + { + "kind": 2004, + "name": "Torrent Comment" + } + ] + }, + { + "file": "36.md", + "d": "nip-36", + "title": "NIP-36: Sensitive Content", + "status": "draft", + "summary": "The content-warning tag, which asks a client to hide an event until the reader chooses to see it." + }, + { + "file": "37.md", + "d": "nip-37", + "title": "NIP-37: Draft Events", + "status": "draft", + "summary": "Kind 31234, an encrypted draft of an unsigned event of any kind, stored on relays for its author alone.", + "kinds": [ + { + "kind": 1234, + "name": "Draft Checkpoint" + }, + { + "kind": 10013, + "name": "Private event relay list" + }, + { + "kind": 31234, + "name": "Draft Event" + } + ] + }, + { + "file": "38.md", + "d": "nip-38", + "title": "NIP-38: User Statuses", + "status": "draft", + "summary": "Kind 30315, a short-lived status: what someone is doing, or what they are listening to.", + "kinds": [ + { + "kind": 30315, + "name": "User Statuses" + } + ] + }, + { + "file": "39.md", + "d": "nip-39", + "title": "NIP-39: Linking Profiles to Other Platforms", + "status": "draft", + "summary": "The i tags that claim an identity on another platform, each pointing at a proof anyone can check.", + "kinds": [ + { + "kind": 10011, + "name": "External Identities" + } + ] + }, + { + "file": "40.md", + "d": "nip-40", + "title": "NIP-40: Expiration Timestamp", + "status": "draft", + "summary": "The expiration tag, a timestamp after which relays and clients should treat an event as gone." + }, + { + "file": "42.md", + "d": "nip-42", + "title": "NIP-42: Authentication of clients to relays", + "status": "draft", + "summary": "The AUTH flow, where a relay challenges a client to sign an ephemeral event before granting it access.", + "kinds": [ + { + "kind": 22242, + "name": "Client Authentication" + } + ] + }, + { + "file": "43.md", + "d": "nip-43", + "title": "NIP-43: Relay Access Metadata and Requests", + "status": "draft", + "summary": "How a relay advertises the roles it grants and the membership it enforces, and how a client asks to be admitted.", + "kinds": [ + { + "kind": 8000, + "name": "Add User" + }, + { + "kind": 8001, + "name": "Remove User" + }, + { + "kind": 13534, + "name": "Membership Lists" + }, + { + "kind": 28934, + "name": "Join Request" + }, + { + "kind": 28935, + "name": "Invite Request" + }, + { + "kind": 28936, + "name": "Leave Request" + } + ] + }, + { + "file": "44.md", + "d": "nip-44", + "title": "NIP-44: Encrypted Payloads (Versioned)", + "status": null, + "summary": "The versioned encryption format the rest of Nostr builds on: secp256k1 ECDH, HKDF, padding, ChaCha20 and HMAC-SHA256." + }, + { + "file": "45.md", + "d": "nip-45", + "title": "NIP-45: Counting results", + "status": "draft", + "summary": "The COUNT verb, which asks a relay how many events match a filter instead of asking for the events themselves." + }, + { + "file": "46.md", + "d": "nip-46", + "title": "NIP-46: Nostr Remote Signing", + "status": null, + "summary": "Remote signing: a client sends signing requests over an encrypted channel to a signer that holds the key and never surrenders it.", + "kinds": [ + { + "kind": 24133, + "name": "Nostr Connect" + } + ] + }, + { + "file": "47.md", + "d": "nip-47", + "title": "NIP-47: Nostr Wallet Connect", + "status": "draft", + "summary": "Nostr Wallet Connect, where a client drives a remote Lightning wallet through encrypted events.", + "kinds": [ + { + "kind": 13194, + "name": "Wallet Info" + }, + { + "kind": 23194, + "name": "Wallet Request" + }, + { + "kind": 23195, + "name": "Wallet Response" + } + ] + }, + { + "file": "48.md", + "d": "nip-48", + "title": "NIP-48: Bridged Events", + "status": "draft", + "summary": "The proxy tag, which links an event bridged from another protocol back to the object it came from." + }, + { + "file": "49.md", + "d": "nip-49", + "title": "NIP-49: Private Key Encryption (`ncryptsec`)", + "status": "draft", + "summary": "ncryptsec, a private key encrypted under a password, with scrypt deciding how expensive guessing it is." + }, + { + "file": "50.md", + "d": "nip-50", + "title": "NIP-50: Search Capability", + "status": "draft", + "summary": "The search field on a filter, and what a relay is expected to make of it." + }, + { + "file": "51.md", + "d": "nip-51", + "title": "NIP-51: Lists", + "status": "draft", + "summary": "The list kinds, with public entries in the tags and private ones encrypted in the content: mutes, bookmarks, relay sets, follow sets and the rest.", + "kinds": [ + { + "kind": 10000, + "name": "Mute list" + }, + { + "kind": 10001, + "name": "Pin list" + }, + { + "kind": 10002, + "name": "Relay List Metadata" + }, + { + "kind": 10003, + "name": "Bookmark list" + }, + { + "kind": 10004, + "name": "Communities list" + }, + { + "kind": 10005, + "name": "Public chats list" + }, + { + "kind": 10006, + "name": "Blocked relays list" + }, + { + "kind": 10007, + "name": "Search relays list" + }, + { + "kind": 10008, + "name": "Profile Badges" + }, + { + "kind": 10009, + "name": "User groups" + }, + { + "kind": 10012, + "name": "Favorite relays list" + }, + { + "kind": 10015, + "name": "Interests list" + }, + { + "kind": 10020, + "name": "Media follows" + }, + { + "kind": 10030, + "name": "User emoji list" + }, + { + "kind": 10050, + "name": "Relay list to receive DMs" + }, + { + "kind": 10054, + "name": "Favorite podcasts list" + }, + { + "kind": 10064, + "name": "Authored podcasts list" + }, + { + "kind": 30000, + "name": "Follow sets" + }, + { + "kind": 30002, + "name": "Relay sets" + }, + { + "kind": 30003, + "name": "Bookmark sets" + }, + { + "kind": 30004, + "name": "Curation sets" + }, + { + "kind": 30005, + "name": "Video sets" + }, + { + "kind": 30006, + "name": "Picture sets" + }, + { + "kind": 30007, + "name": "Kind mute sets" + }, + { + "kind": 30008, + "name": "Badge sets" + }, + { + "kind": 30015, + "name": "Interest sets" + }, + { + "kind": 30030, + "name": "Emoji sets" + }, + { + "kind": 30063, + "name": "Release artifact sets" + }, + { + "kind": 30267, + "name": "App curation sets" + }, + { + "kind": 39089, + "name": "Starter packs" + }, + { + "kind": 39092, + "name": "Media starter packs" + } + ] + }, + { + "file": "52.md", + "d": "nip-52", + "title": "NIP-52: Calendar Events", + "status": "draft", + "summary": "Calendar events, date-based and time-based, and the RSVPs that answer them.", + "kinds": [ + { + "kind": 31922, + "name": "Date-Based Calendar Event" + }, + { + "kind": 31923, + "name": "Time-Based Calendar Event" + }, + { + "kind": 31924, + "name": "Calendar" + }, + { + "kind": 31925, + "name": "Calendar Event RSVP" + } + ] + }, + { + "file": "53.md", + "d": "nip-53", + "title": "NIP-53: Live Streaming and Spaces", + "status": "draft", + "summary": "Kind 30311, a live stream or audio space, rewritten as participants join and leave it.", + "kinds": [ + { + "kind": 1311, + "name": "Live Chat Message" + }, + { + "kind": 10312, + "name": "Room Presence" + }, + { + "kind": 30311, + "name": "Live Event" + }, + { + "kind": 30312, + "name": "Interactive Room" + }, + { + "kind": 30313, + "name": "Conference Event" + } + ] + }, + { + "file": "54.md", + "d": "nip-54", + "title": "NIP-54: Wiki", + "status": "draft", + "summary": "Kind 30818, wiki articles addressed by a normalised subject, where several people are expected to write about the same one.", + "kinds": [ + { + "kind": 818, + "name": "Merge Requests" + }, + { + "kind": 30818, + "name": "Wiki article" + }, + { + "kind": 30819, + "name": "Redirects" + } + ] + }, + { + "file": "55.md", + "d": "nip-55", + "title": "NIP-55: Android Signer Application", + "status": "draft", + "summary": "How a client on Android asks a signer application on the same device to sign, so it never handles the private key." + }, + { + "file": "56.md", + "d": "nip-56", + "title": "NIP-56: Reporting", + "status": null, + "summary": "Kind 1984, a report naming content or a key as objectionable, with a type such as nudity, malware or impersonation.", + "kinds": [ + { + "kind": 1984, + "name": "Reporting" + } + ] + }, + { + "file": "57.md", + "d": "nip-57", + "title": "NIP-57: Lightning Zaps", + "status": "draft", + "summary": "Zaps: the kind 9734 request a payer signs, and the kind 9735 receipt the recipient's wallet publishes once the invoice is paid.", + "kinds": [ + { + "kind": 9734, + "name": "Zap Request" + }, + { + "kind": 9735, + "name": "Zap" + } + ] + }, + { + "file": "58.md", + "d": "nip-58", + "title": "NIP-58: Badges", + "status": "draft", + "summary": "Badges: the definition an issuer publishes, the award it grants to a key, and the list a user chooses to show on their profile.", + "kinds": [ + { + "kind": 8, + "name": "Badge Award" + }, + { + "kind": 10008, + "name": "Profile Badges" + }, + { + "kind": 30008, + "name": "Badge sets" + }, + { + "kind": 30009, + "name": "Badge Definition" + } + ] + }, + { + "file": "59.md", + "d": "nip-59", + "title": "NIP-59: Gift Wrap", + "status": null, + "summary": "Gift wraps: a rumor sealed to its recipient, then wrapped again under a throwaway key, so relays see only the wrapper.", + "kinds": [ + { + "kind": 13, + "name": "Seal" + }, + { + "kind": 1059, + "name": "Gift Wrap" + } + ] + }, + { + "file": "5A.md", + "d": "nip-5a", + "title": "NIP-5A: Static Websites (nsites)", + "status": "draft", + "summary": "Static websites whose files are Blossom blobs, listed by a manifest event: one root site per key, and named sites beside it.", + "kinds": [ + { + "kind": 5128, + "name": "nsite manifest snapshot" + }, + { + "kind": 15128, + "name": "Root nsite manifest" + }, + { + "kind": 34128, + "name": "Legacy nsite manifest" + }, + { + "kind": 35128, + "name": "Named nsite manifest" + } + ] + }, + { + "file": "60.md", + "d": "nip-60", + "title": "NIP-60: Cashu Wallet", + "status": "draft", + "summary": "A Cashu wallet whose proofs and spending history live encrypted on relays, so it follows its owner between clients.", + "kinds": [ + { + "kind": 7374, + "name": "Reserved Cashu Wallet Tokens" + }, + { + "kind": 7375, + "name": "Cashu Wallet Tokens" + }, + { + "kind": 7376, + "name": "Cashu Wallet History" + }, + { + "kind": 17375, + "name": "Cashu Wallet Event" + } + ] + }, + { + "file": "61.md", + "d": "nip-61", + "title": "NIP-61: Nutzaps", + "status": "draft", + "summary": "Nutzaps, where the payment is a Cashu token locked to the recipient's key and published to their relays, so the token itself is the receipt.", + "kinds": [ + { + "kind": 9321, + "name": "Nutzap" + }, + { + "kind": 10019, + "name": "Nutzap Mint Recommendation" + } + ] + }, + { + "file": "62.md", + "d": "nip-62", + "title": "NIP-62: Request to Vanish", + "status": "draft", + "summary": "Kind 62, a request that a relay delete everything a key ever published there.", + "kinds": [ + { + "kind": 62, + "name": "Request to Vanish" + } + ] + }, + { + "file": "64.md", + "d": "nip-64", + "title": "NIP-64: Chess (PGN)", + "status": "draft", + "summary": "Kind 64, a chess game written in PGN.", + "kinds": [ + { + "kind": 64, + "name": "Chess (PGN)" + } + ] + }, + { + "file": "65.md", + "d": "nip-65", + "title": "NIP-65: Relay List Metadata", + "status": "draft", + "summary": "Kind 10002, the list saying where a key writes and where it reads mentions, which is what the outbox model is built on.", + "kinds": [ + { + "kind": 10002, + "name": "Relay List Metadata" + } + ] + }, + { + "file": "66.md", + "d": "nip-66", + "title": "NIP-66: Relay Discovery and Liveness Monitoring", + "status": "draft", + "summary": "Kind 30166, relay discovery events published by monitors, describing what a relay claims to be and whether it answers.", + "kinds": [ + { + "kind": 10166, + "name": "Relay Monitor Announcement" + }, + { + "kind": 30166, + "name": "Relay Discovery" + } + ] + }, + { + "file": "67.md", + "d": "nip-67", + "title": "NIP-67: EOSE Completeness Hint", + "status": "draft", + "summary": "A third element on EOSE, telling a client whether the relay really sent every stored event matching the filter." + }, + { + "file": "68.md", + "d": "nip-68", + "title": "NIP-68: Picture-first feeds", + "status": "draft", + "summary": "Kind 20, picture-first posts where the images carry the event rather than illustrate it.", + "kinds": [ + { + "kind": 20, + "name": "Picture" + } + ] + }, + { + "file": "69.md", + "d": "nip-69", + "title": "NIP-69: Peer-to-peer Order events", + "status": "draft", + "summary": "Kind 38383, peer-to-peer trade orders shared across platforms so that their separate liquidity pools into one.", + "kinds": [ + { + "kind": 38383, + "name": "Peer-to-peer Order events" + } + ] + }, + { + "file": "70.md", + "d": "nip-70", + "title": "NIP-70: Protected Events", + "status": "draft", + "summary": "The dash tag, marking an event that only its author may publish, which relays enforce through NIP-42." + }, + { + "file": "71.md", + "d": "nip-71", + "title": "NIP-71: Video Events", + "status": "draft", + "summary": "Video events, long and short, carrying the metadata a video client needs around media hosted elsewhere.", + "kinds": [ + { + "kind": 21, + "name": "Video Event" + }, + { + "kind": 22, + "name": "Short-form Portrait Video Event" + }, + { + "kind": 34235, + "name": "Addressable Video Event" + }, + { + "kind": 34236, + "name": "Addressable Short Video Event" + } + ] + }, + { + "file": "72.md", + "d": "nip-72", + "title": "NIP-72: Moderated Communities", + "status": "unrecommended", + "summary": "Kind 34550, Reddit-style communities where moderators approve the posts addressed to them.", + "kinds": [ + { + "kind": 4550, + "name": "Community Post Approval" + }, + { + "kind": 34550, + "name": "Community Definition" + } + ] + }, + { + "file": "73.md", + "d": "nip-73", + "title": "NIP-73: External Content IDs", + "status": "draft", + "summary": "The i tags for identifiers that exist outside Nostr: URLs, ISBNs, podcast GUIDs, DOIs, hashtags and blockchain transactions." + }, + { + "file": "75.md", + "d": "nip-75", + "title": "NIP-75: Zap Goals", + "status": "draft", + "summary": "Kind 9041, a fundraising goal that zaps are tallied against.", + "kinds": [ + { + "kind": 9041, + "name": "Zap Goal" + } + ] + }, + { + "file": "77.md", + "d": "nip-77", + "title": "NIP-77: Negentropy Syncing", + "status": "draft", + "summary": "Negentropy over Nostr: range-based set reconciliation, so two sides transfer only the events one of them lacks." + }, + { + "file": "78.md", + "d": "nip-78", + "title": "NIP-78: Application-specific data", + "status": "draft", + "summary": "Kind 30078, where an application stores data it has no intention of sharing with any other.", + "kinds": [ + { + "kind": 78, + "name": "Application-specific Data" + }, + { + "kind": 30078, + "name": "Application-specific Data" + } + ] + }, + { + "file": "7D.md", + "d": "nip-7d", + "title": "NIP-7D: Forum Threads", + "status": "draft", + "summary": "Kind 11, a forum thread with a title, whose replies are NIP-22 comments rather than nested notes.", + "kinds": [ + { + "kind": 11, + "name": "Thread" + } + ] + }, + { + "file": "84.md", + "d": "nip-84", + "title": "NIP-84: Highlights", + "status": "draft", + "summary": "Kind 9802, a highlight: the passage someone found worth keeping, pointing back at the event or the URL it was taken from.", + "kinds": [ + { + "kind": 9802, + "name": "Highlights" + } + ] + }, + { + "file": "85.md", + "d": "nip-85", + "title": "NIP-85: Trusted Assertions", + "status": "draft", + "summary": "Assertions a trusted provider publishes about a key, an event or an external identifier, so a client does not have to compute a web of trust itself.", + "kinds": [ + { + "kind": 30382, + "name": "User Trusted Assertion" + }, + { + "kind": 30383, + "name": "Event Trusted Assertion" + }, + { + "kind": 30384, + "name": "Addressable Trusted Assertion" + } + ] + }, + { + "file": "86.md", + "d": "nip-86", + "title": "NIP-86: Relay Management API", + "status": "draft", + "summary": "A JSON-RPC API over HTTP for administering a relay: banning keys, allowing kinds, listing what it refuses." + }, + { + "file": "87.md", + "d": "nip-87", + "title": "NIP-87: Cashu and Fedimint Discoverability", + "status": "draft", + "summary": "How an ecash mint announces itself and its capabilities, and how users recommend the mints they trust.", + "kinds": [ + { + "kind": 38172, + "name": "Cashu Mint Announcement" + }, + { + "kind": 38173, + "name": "Fedimint Announcement" + } + ] + }, + { + "file": "88.md", + "d": "nip-88", + "title": "NIP-88: Polls", + "status": "draft", + "summary": "Kind 1068, a poll with its options, its type and its deadline, answered on the relays it names.", + "kinds": [ + { + "kind": 1018, + "name": "Poll Response" + }, + { + "kind": 1068, + "name": "Poll" + } + ] + }, + { + "file": "89.md", + "d": "nip-89", + "title": "NIP-89: Recommended Application Handlers", + "status": "draft", + "summary": "How an application declares the event kinds it can handle, and how users recommend one for a kind their client cannot open.", + "kinds": [ + { + "kind": 31989, + "name": "Handler recommendation" + }, + { + "kind": 31990, + "name": "Handler information" + } + ] + }, + { + "file": "90.md", + "d": "nip-90", + "title": "NIP-90: Data Vending Machines", + "status": "unrecommended", + "summary": "Data vending machines: paid jobs requested and delivered as events, reserving the kind range 5000 to 7000." + }, + { + "file": "92.md", + "d": "nip-92", + "title": "NIP-92: Media Attachments Metadata (`imeta`)", + "status": null, + "summary": "The imeta tag, which describes a media URL found in the content: its dimensions, its mime type, its hash and its fallbacks." + }, + { + "file": "94.md", + "d": "nip-94", + "title": "NIP-94: File Metadata", + "status": "draft", + "summary": "Kind 1063, file metadata: the url, the hash, the mime type and the size of a shared file.", + "kinds": [ + { + "kind": 1063, + "name": "File Metadata" + } + ] + }, + { + "file": "96.md", + "d": "nip-96", + "title": "NIP-96: HTTP File Storage Integration", + "status": "unrecommended", + "summary": "An HTTP file storage API discovered through a well-known document, replaced by Blossom.", + "kinds": [ + { + "kind": 10096, + "name": "File storage server list" + } + ] + }, + { + "file": "98.md", + "d": "nip-98", + "title": "NIP-98: HTTP Auth", + "status": "draft", + "summary": "Kind 27235, an ephemeral event signed to authorise a single HTTP request.", + "kinds": [ + { + "kind": 27235, + "name": "HTTP Auth" + } + ] + }, + { + "file": "99.md", + "d": "nip-99", + "title": "NIP-99: Classified Listings", + "status": "draft", + "summary": "Kind 30402, a classified listing: something for sale or on offer, with enough structure to be worth querying.", + "kinds": [ + { + "kind": 30402, + "name": "Classified Listing" + }, + { + "kind": 30403, + "name": "Draft Classified Listing" + } + ] + }, + { + "file": "A0.md", + "d": "nip-a0", + "title": "NIP-A0: Voice Messages", + "status": "draft", + "summary": "Kinds 1222 and 1244, short voice messages whose content is the URL of an audio file.", + "kinds": [ + { + "kind": 1222, + "name": "Voice Message" + }, + { + "kind": 1244, + "name": "Voice Message Comment" + } + ] + }, + { + "file": "A4.md", + "d": "nip-a4", + "title": "NIP-A4: Public Messages", + "status": "draft", + "summary": "Kind 24, a plain text message addressed to one or more keys, meant for a notification screen rather than a thread.", + "kinds": [ + { + "kind": 24, + "name": "Public Message" + } + ] + }, + { + "file": "B0.md", + "d": "nip-b0", + "title": "NIP-B0: Web Bookmarks", + "status": "draft", + "summary": "Kind 39701, a web bookmark addressed by its own URI, editable by whoever saved it.", + "kinds": [ + { + "kind": 39701, + "name": "Web bookmarks" + } + ] + }, + { + "file": "B7.md", + "d": "nip-b7", + "title": "NIP-B7: Blossom", + "status": "draft", + "summary": "How a Nostr client uses Blossom for media: reading a user's server list, and uploading to the servers it names.", + "kinds": [ + { + "kind": 10063, + "name": "User server list" + }, + { + "kind": 24242, + "name": "Blobs stored on mediaservers" + } + ] + }, + { + "file": "BE.md", + "d": "nip-be", + "title": "NIP-BE: Nostr BLE Communications Protocol", + "status": "unrecommended", + "summary": "How two devices speak Nostr over Bluetooth Low Energy, emulating the websocket protocol within its limits." + }, + { + "file": "C0.md", + "d": "nip-c0", + "title": "NIP-C0: Code Snippets", + "status": "draft", + "summary": "Kind 1337, a code snippet carrying its language, its filename, its extension and its runtime.", + "kinds": [ + { + "kind": 1337, + "name": "Code Snippet" + } + ] + }, + { + "file": "C7.md", + "d": "nip-c7", + "title": "NIP-C7: Chats", + "status": "draft", + "summary": "Kind 9, a chat message, whose replies are further kind 9 events quoting their parent.", + "kinds": [ + { + "kind": 9, + "name": "Chat Message" + } + ] + }, + { + "file": "CC.md", + "d": "nip-cc", + "title": "NIP-CC: Geocaching", + "status": "draft", + "summary": "Kind 37516 and its companions, geocache listings and the logs of the people who found them.", + "kinds": [ + { + "kind": 7516, + "name": "Geocache log" + }, + { + "kind": 7517, + "name": "Geocache proof of find" + }, + { + "kind": 37516, + "name": "Geocache Listing" + }, + { + "kind": 37517, + "name": "Geocache Curation List" + } + ] + }, + { + "file": "EE.md", + "d": "nip-ee", + "title": "NIP-EE: E2EE Messaging using MLS Protocol", + "status": "unrecommended", + "summary": "End-to-end encrypted group messaging built on the MLS protocol, superseded by Marmot." + }, + { + "file": "F4.md", + "d": "nip-f4", + "title": "NIP-F4: Podcasts", + "status": "draft", + "summary": "Podcast episodes published as events, so a feed stops depending on one URL staying alive.", + "kinds": [ + { + "kind": 54, + "name": "Podcast Episode" + }, + { + "kind": 10154, + "name": "Podcast Metadata" + } + ] + } + ] +} diff --git a/apps/importer/manifests/nuts.json b/apps/importer/manifests/nuts.json new file mode 100644 index 0000000..25c83f8 --- /dev/null +++ b/apps/importer/manifests/nuts.json @@ -0,0 +1,234 @@ +{ + "name": "nuts", + "title": "Cashu NUTs", + "npub": "npub169nculhev5m5h04rpzsjz4sf57phdhq4sfm604jhdq8e6hhatsuq0xuard", + "repo": "https://github.com/cashubtc/nuts", + "branch": "main", + "license": "MIT", + "topics": ["cashu", "ecash", "nut"], + "specs": [ + { + "file": "00.md", + "d": "nut-00", + "title": "NUT-00: Notation, Utilization, and Terminology", + "status": "mandatory", + "summary": "The notation, models and cryptography the rest of the specification is written in: blinding, signing, proofs, and how a token is serialised." + }, + { + "file": "01.md", + "d": "nut-01", + "title": "NUT-01: Mint public key exchange", + "status": "mandatory", + "summary": "How a wallet fetches a mint's active public keys, one per amount, and what a keyset is." + }, + { + "file": "02.md", + "d": "nut-02", + "title": "NUT-02: Keysets and fees", + "status": "mandatory", + "summary": "Keysets: how an id is derived from one, how a mint rotates them, and the input fee a wallet pays when spending ecash from one." + }, + { + "file": "03.md", + "d": "nut-03", + "title": "NUT-03: Swap tokens", + "status": "mandatory", + "summary": "The swap operation, where a wallet hands proofs to the mint and receives new blind signatures in return." + }, + { + "file": "04.md", + "d": "nut-04", + "title": "NUT-04: Mint tokens", + "status": "mandatory", + "summary": "Minting: the two-step flow where a wallet requests a quote, pays it, and then asks the mint to issue ecash." + }, + { + "file": "05.md", + "d": "nut-05", + "title": "NUT-05: Melting tokens", + "status": "mandatory", + "summary": "Melting: the two-step flow where a wallet asks the mint to pay a request and spends ecash to cover it." + }, + { + "file": "06.md", + "d": "nut-06", + "title": "NUT-06: Mint information", + "status": "mandatory", + "summary": "The info endpoint, where a mint declares its name, its keys, its contacts, and which NUTs it supports." + }, + { + "file": "07.md", + "d": "nut-07", + "title": "NUT-07: Token state check", + "status": "optional", + "summary": "How a wallet asks the mint whether a proof is unspent, pending, or already spent." + }, + { + "file": "08.md", + "d": "nut-08", + "title": "NUT-08: Lightning fee return", + "status": "optional", + "summary": "Blank outputs, which let a mint return the Lightning fees a wallet overpaid when melting." + }, + { + "file": "09.md", + "d": "nut-09", + "title": "NUT-09: Restore signatures", + "status": "optional", + "summary": "How a wallet asks the mint to reissue blind signatures it once received, so a lost wallet can be restored." + }, + { + "file": "10.md", + "d": "nut-10", + "title": "NUT-10: Spending conditions", + "status": "optional", + "summary": "The well-known secret format that turns a proof's secret into a spending condition the mint enforces." + }, + { + "file": "11.md", + "d": "nut-11", + "title": "NUT-11: Pay to Public Key (P2PK)", + "status": "optional", + "summary": "Locking ecash to a public key, redeemable only against a Schnorr signature from the key that owns it." + }, + { + "file": "12.md", + "d": "nut-12", + "title": "NUT-12: Offline ecash signature validation", + "status": "optional", + "summary": "DLEQ proofs, which let anyone verify a mint's signature offline with nothing but its public keys." + }, + { + "file": "13.md", + "d": "nut-13", + "title": "NUT-13: Deterministic Secrets", + "status": "optional", + "summary": "Secrets derived from a seed phrase, so a wallet can regenerate its ecash after losing the device that held it." + }, + { + "file": "14.md", + "d": "nut-14", + "title": "NUT-14: Hashed Timelock Contracts (HTLCs)", + "status": "optional", + "summary": "Ecash locked to the hash of a preimage and a deadline, which makes atomic swaps between users possible." + }, + { + "file": "15.md", + "d": "nut-15", + "title": "NUT-15: Partial multi-path payments", + "status": "optional", + "summary": "Paying one Lightning invoice from several mints at once, all of them or none." + }, + { + "file": "16.md", + "d": "nut-16", + "title": "NUT-16: Animated QR codes", + "status": "optional", + "summary": "How to show a token too large for a static QR code as an animated one, using the UR protocol." + }, + { + "file": "17.md", + "d": "nut-17", + "title": "NUT-17: WebSockets", + "status": "optional", + "summary": "A JSON-RPC WebSocket protocol for subscribing to quote and proof state changes instead of polling for them." + }, + { + "file": "18.md", + "d": "nut-18", + "title": "NUT-18: Payment Requests", + "status": "optional", + "summary": "A request format carrying everything a sending wallet needs, so a transaction can be started by whoever is being paid." + }, + { + "file": "19.md", + "d": "nut-19", + "title": "NUT-19: Cached Responses", + "status": "optional", + "summary": "Response caching on the critical endpoints, so a wallet can safely replay a request that a network error interrupted." + }, + { + "file": "20.md", + "d": "nut-20", + "title": "NUT-20: Signature on Mint Quote", + "status": "optional", + "summary": "Binding a mint quote to a public key, so only the holder of the matching secret key can redeem it." + }, + { + "file": "21.md", + "d": "nut-21", + "title": "NUT-21: Clear Authentication", + "status": "optional", + "summary": "How a mint restricts its endpoints to users holding a token from an OpenID Connect service it trusts." + }, + { + "file": "22.md", + "d": "nut-22", + "title": "NUT-22: Blind Authentication", + "status": "optional", + "summary": "Blind authentication tokens, which let a mint restrict access to registered users without learning which one is calling." + }, + { + "file": "23.md", + "d": "nut-23", + "title": "NUT-23: BOLT11", + "status": "optional", + "summary": "Minting and melting with bolt11 Lightning invoices, the method specifics on top of the shared mint and melt flows." + }, + { + "file": "24.md", + "d": "nut-24", + "title": "NUT-24: HTTP 402 Payment Required", + "status": "optional", + "summary": "How an HTTP server demands payment with a 402 response carrying a Cashu payment request." + }, + { + "file": "25.md", + "d": "nut-25", + "title": "NUT-25: BOLT12", + "status": "optional", + "summary": "Minting and melting with bolt12 Lightning offers." + }, + { + "file": "26.md", + "d": "nut-26", + "title": "NUT-26: Payment Request Bech32m Encoding", + "status": "optional", + "summary": "A bech32m encoding of payment requests, smaller than the CBOR form and friendlier to QR codes." + }, + { + "file": "27.md", + "d": "nut-27", + "title": "NUT-27: Nostr Mint Backup", + "status": "optional", + "summary": "How a wallet backs up its mint list to Nostr relays, encrypted under keys derived from its own seed phrase.", + "kinds": [ + { + "kind": 30078, + "name": "Application-specific data" + } + ] + }, + { + "file": "28.md", + "d": "nut-28", + "title": "NUT-28: Pay to Blinded Key (P2BK)", + "status": "optional", + "summary": "Blinding the receiver's public key on each payment, so the mint cannot link several of them to the same person." + }, + { + "file": "29.md", + "d": "nut-29", + "title": "NUT-29: Batched Minting", + "status": "optional", + "summary": "Minting several quotes in one atomic request." + }, + { + "file": "30.md", + "d": "nut-30", + "title": "NUT-30: Payment Method: Onchain", + "status": "optional", + "summary": "Minting and melting with onchain Bitcoin payments." + } + ] +} diff --git a/apps/importer/package.json b/apps/importer/package.json new file mode 100644 index 0000000..dd6f97c --- /dev/null +++ b/apps/importer/package.json @@ -0,0 +1,25 @@ +{ + "name": "@openspecs/importer", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "typecheck": "tsc", + "test": "vitest run", + "build:events": "vite-node src/cli.ts build", + "publish:events": "vite-node src/cli.ts publish", + "publish:identity": "vite-node src/cli.ts identity" + }, + "dependencies": { + "@openspecs/nostr": "workspace:*", + "nostr-tools": "2.24.1", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^24.13.3", + "nostr-mock-relay": "^0.1.1", + "typescript": "^7.0.2", + "vite-node": "^6.0.0", + "vitest": "^4.1.10" + } +} diff --git a/apps/importer/src/build.ts b/apps/importer/src/build.ts new file mode 100644 index 0000000..81c21fb --- /dev/null +++ b/apps/importer/src/build.ts @@ -0,0 +1,72 @@ +import { createHash } from "node:crypto"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { buildImport } from "./document.ts"; +import { ensureRepo, fileAt, fileHistory, headCommit } from "./git.ts"; +import { rewriteLinks, specIndex } from "./links.ts"; +import type { Corpus } from "./manifest.ts"; + +export type BuildOptions = { + corpora: Corpus[]; + cache: string; + out: string; +}; + +/** + * Every document as an event, from the manifests and the repositories and + * nothing else. Signs nothing and reaches no relay: what this writes is read + * before any of it is published. + */ +export const buildEvents = async ({ corpora, cache, out }: BuildOptions): Promise => { + const index = specIndex(corpora); + await rm(out, { recursive: true, force: true }); + + for (const corpus of corpora) { + const dir = `${cache}/${corpus.name}`; + await ensureRepo(dir, corpus.repo, corpus.branch); + const commit = await headCommit(dir, corpus.branch); + await mkdir(`${out}/${corpus.name}`, { recursive: true }); + + let internal = 0; + let external = 0; + let anchors = 0; + const absolutized: { target: string; url: string }[] = []; + + for (const entry of corpus.specs) { + const bytes = await fileAt(dir, commit, entry.file); + const { publishedAt, createdAt } = await fileHistory(dir, commit, entry.file); + const { content, report } = rewriteLinks(bytes.toString("utf8"), { + corpus, + file: entry.file, + commit, + index, + }); + + const event = buildImport(corpus, entry, content, { + commit, + publishedAt, + createdAt, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + + await writeFile( + `${out}/${corpus.name}/${entry.d}.json`, + `${JSON.stringify(event, null, 2)}\n`, + ); + + internal += report.internal; + external += report.external; + anchors += report.anchors; + absolutized.push(...report.absolutized); + } + + console.log( + `${corpus.name}\t${corpus.specs.length} documents at ${commit.slice(0, 7)}\t` + + `${internal} internal, ${absolutized.length} pinned, ${external} external, ${anchors} anchors`, + ); + for (const link of [...new Set(absolutized.map((link) => link.target))].sort()) { + console.log(`\tpinned ${link}`); + } + } + + console.log(`\nwritten to ${out}`); +}; diff --git a/apps/importer/src/cli.ts b/apps/importer/src/cli.ts new file mode 100644 index 0000000..b077e91 --- /dev/null +++ b/apps/importer/src/cli.ts @@ -0,0 +1,96 @@ +import { existsSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { closeRelayPool, INDEXER_RELAYS, relaySet, relayUrl } from "@openspecs/nostr"; +import { buildEvents } from "./build.ts"; +import { publishIdentity } from "./identity.ts"; +import { loadManifests, MANIFEST_DIR } from "./manifest.ts"; +import { publishEvents } from "./publish.ts"; + +const args = process.argv.slice(2); + +const flag = (name: string, fallback: string): string => { + const at = args.indexOf(`--${name}`); + return at === -1 ? fallback : (args[at + 1] ?? fallback); +}; + +const flags = (name: string): string[] => + args.flatMap((value, at) => (value === `--${name}` ? [args[at + 1] ?? ""] : [])); + +const here = (path: string) => fileURLToPath(new URL(path, import.meta.url)); + +/** + * The keys, if this operator keeps them in a file rather than typing them. A + * variable already set wins over the file, so one run can be pointed somewhere + * else without editing anything, and a key never has to be typed at a prompt + * that writes it to a history file. + */ +const loadEnv = (): void => { + const path = here("../.env"); + if (existsSync(path)) process.loadEnvFile(path); +}; + +/** + * No default: a relay to publish to is a decision, and one taken by leaving a + * flag out is not one. Nothing here knows where the corpus belongs. + */ +const relaysFrom = (): string[] => { + const named = [...flags("relay"), ...(process.env.OPENSPECS_IMPORT_RELAYS ?? "").split(",")]; + const relays = relaySet(named.map((url) => relayUrl(url) ?? "").filter((url) => url !== "")); + if (relays.length === 0) { + throw new Error("name a relay with --relay, or set OPENSPECS_IMPORT_RELAYS"); + } + return relays; +}; + +const main = async (): Promise => { + loadEnv(); + const command = args[0]; + const manifests = flag("manifests", ""); + const corpora = await loadManifests( + manifests === "" ? MANIFEST_DIR : pathToFileURL(`${manifests}/`), + ); + + if (command === "build") { + await buildEvents({ + corpora, + cache: flag("cache", here("../.cache")), + out: flag("out", here("../events")), + }); + return; + } + + if (command === "publish") { + try { + await publishEvents({ + corpora, + events: flag("events", here("../events")), + relays: relaysFrom(), + confirmed: args.includes("--yes"), + }); + } finally { + closeRelayPool(); + } + return; + } + + if (command === "identity") { + try { + const relays = relaysFrom(); + await publishIdentity({ + corpora, + relays, + // The indexers as well: a relay list nobody can find resolves nothing, + // and finding one is what every outbox lookup starts with. + targets: relaySet(relays, INDEXER_RELAYS), + confirmed: args.includes("--yes"), + }); + } finally { + closeRelayPool(); + } + return; + } + + throw new Error("usage: build | publish | identity"); +}; + +await main(); diff --git a/apps/importer/src/document.ts b/apps/importer/src/document.ts new file mode 100644 index 0000000..4404fa8 --- /dev/null +++ b/apps/importer/src/document.ts @@ -0,0 +1,61 @@ +import { buildSpec, type EventDraft } from "@openspecs/nostr"; +import { sourceUrl } from "./links.ts"; +import type { Corpus, SpecEntry } from "./manifest.ts"; + +/** Not the web app: an event built here was never written by anyone using it. */ +export const CLIENT = "openspecs-import"; + +export type ImportSource = { + commit: string; + /** Unix seconds of the commit that added the file, and the same on every revision. */ + publishedAt: number; + /** Unix seconds of the commit this revision is. */ + createdAt: number; + /** sha256 of the source bytes, before a single link was rewritten. */ + sha256: string; +}; + +/** + * An event with the moment it stands for. `EventDraft` leaves `created_at` to + * whoever publishes, which is right for something somebody just wrote and wrong + * here: this revision happened when the commit did. + */ +export type ImportedEvent = EventDraft & { created_at: number }; + +/** + * The document as an event, built from the manifest and the file, and from + * nothing else. Same inputs, same bytes: that is what `verify` rests on, and + * why no value here is read from the network or from a clock. + * + * The tags this adds to the schema's own are what make it a mirror rather than + * a claim of authorship: `proxy` names the file it copies, and `x` is the hash + * of that file's bytes, so anyone can fetch the source and check the copy. + */ +export const buildImport = ( + corpus: Corpus, + entry: SpecEntry, + content: string, + source: ImportSource, +): ImportedEvent => { + const draft = buildSpec({ + identifier: entry.d, + title: entry.title, + summary: entry.summary, + content, + status: entry.status ?? "", + topics: corpus.topics, + kinds: entry.kinds.map((entry) => ({ raw: String(entry.kind), name: entry.name })), + }); + + return { + kind: draft.kind, + content: draft.content, + created_at: source.createdAt, + tags: [ + ...draft.tags.map((tag) => (tag[0] === "client" ? ["client", CLIENT] : tag)), + ["published_at", String(source.publishedAt)], + ["proxy", sourceUrl(corpus.repo, source.commit, entry.file), "web"], + ["x", source.sha256], + ], + }; +}; diff --git a/apps/importer/src/git.ts b/apps/importer/src/git.ts new file mode 100644 index 0000000..c00d82e --- /dev/null +++ b/apps/importer/src/git.ts @@ -0,0 +1,59 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { promisify } from "node:util"; + +const run = promisify(execFile); + +const MAX_OUTPUT = 32 * 1024 * 1024; + +const git = async (args: string[], cwd?: string): Promise => { + const { stdout } = await run("git", args, { cwd, maxBuffer: MAX_OUTPUT }); + return stdout; +}; + +/** A clone with no blobs until they are asked for: history is what this needs, not checkouts. */ +export const ensureRepo = async (dir: string, url: string, branch: string): Promise => { + try { + await git(["-C", dir, "rev-parse", "--git-dir"]); + await git(["-C", dir, "fetch", "--quiet", "origin", branch]); + } catch { + await mkdir(dir, { recursive: true }); + await git(["clone", "--quiet", "--filter=blob:none", "--no-checkout", url, dir]); + } +}; + +export const headCommit = async (dir: string, branch: string): Promise => + (await git(["-C", dir, "rev-parse", `origin/${branch}`])).trim(); + +export const fileAt = async (dir: string, commit: string, path: string): Promise => { + const { stdout } = await run("git", ["-C", dir, "show", `${commit}:${path}`], { + maxBuffer: MAX_OUTPUT, + encoding: "buffer", + }); + return stdout; +}; + +const commitDates = async (dir: string, commit: string, path: string): Promise => { + // `--follow` carries a document through the renames its number survived, and + // takes exactly one path, which is why this asks per file rather than in bulk. + const stdout = await git(["-C", dir, "log", "--follow", "--format=%ct", commit, "--", path]); + return stdout + .split("\n") + .map((line) => Number(line.trim())) + .filter((seconds) => Number.isFinite(seconds) && seconds > 0); +}; + +/** When the file first appeared and when it last changed, newest first from `git log`. */ +export const fileHistory = async ( + dir: string, + commit: string, + path: string, +): Promise<{ publishedAt: number; createdAt: number }> => { + const dates = await commitDates(dir, commit, path); + const newest = dates[0]; + const oldest = dates.at(-1); + if (newest === undefined || oldest === undefined) { + throw new Error(`${path} has no history at ${commit.slice(0, 7)}`); + } + return { publishedAt: oldest, createdAt: newest }; +}; diff --git a/apps/importer/src/identity.ts b/apps/importer/src/identity.ts new file mode 100644 index 0000000..1cdb6c2 --- /dev/null +++ b/apps/importer/src/identity.ts @@ -0,0 +1,123 @@ +import type { EventDraft, NostrEvent } from "@openspecs/nostr"; +import { buildProfile, buildRelayList, PROFILE_KIND, RELAY_LIST_KIND } from "@openspecs/nostr"; +import type { SimplePool } from "nostr-tools/pool"; +import { finalizeEvent } from "nostr-tools/pure"; +import { secretFor } from "./keys.ts"; +import type { Corpus } from "./manifest.ts"; +import { liveReplaceable, publishEvent } from "./relays.ts"; + +/** + * What a corpus says about itself, so that a reader landing on one of its + * documents can tell a copy from the thing it copies. + * + * The name carries it too, not only the description: a name is what a client + * puts beside a document, and one reading `Nostr Implementation Possibilities` + * with nothing else would pass for the authors themselves. + */ +export const profileOf = (corpus: Corpus): EventDraft => + buildProfile({ + name: `${corpus.title} (mirror)`, + about: [ + `An unofficial mirror of ${corpus.repo}.`, + "", + "This key copied these documents, it did not write them. Each one names the file " + + "and the commit it was copied from and carries the sha256 of those bytes, so any " + + `copy can be checked against its source. The specifications are ${corpus.license}.`, + ].join("\n"), + }); + +/** + * Where this corpus is published, in the form every client already resolves. + * This is what lets a document be found from its address alone: the site reads + * a NIP-65 list before it goes looking, and so does anything else that follows + * the outbox model, which is why the relay it names needs no hardcoding. + */ +export const relayListOf = (relays: string[]): EventDraft => buildRelayList(relays); + +export const identityOf = (corpus: Corpus, relays: string[]): EventDraft[] => [ + profileOf(corpus), + relayListOf(relays), +]; + +export const IDENTITY_KINDS = [PROFILE_KIND, RELAY_LIST_KIND]; + +export type IdentityOptions = { + corpora: Corpus[]; + /** Where the corpus lives, which is what the relay list names. */ + relays: string[]; + /** Where the identity itself is published, indexers included. */ + targets: string[]; + confirmed: boolean; + env?: NodeJS.ProcessEnv; + pool?: SimplePool; + now?: number; +}; + +const sameEvent = (draft: EventDraft, live: NostrEvent): boolean => + draft.content === live.content && JSON.stringify(draft.tags) === JSON.stringify(live.tags); + +/** + * Publishes what each corpus says about itself. Nothing here is derived from a + * repository, so unlike a document these carry the moment they were signed, + * stamped past whatever they replace so a relay keeps the newer. + * + * A relay list nobody can find is a relay list that does nothing, which is why + * this is published to the indexers as well as to the corpus's own relay. + */ +export const publishIdentity = async ({ + corpora, + relays, + targets, + confirmed, + env, + pool, + now = Math.floor(Date.now() / 1000), +}: IdentityOptions): Promise => { + const secrets = confirmed + ? new Map(corpora.map((corpus) => [corpus.name, secretFor(corpus, env)])) + : new Map(); + + const live = await liveReplaceable( + corpora.map((corpus) => corpus.pubkey), + IDENTITY_KINDS, + targets, + pool, + ); + + for (const corpus of corpora) { + const drafts = identityOf(corpus, relays); + const stale = drafts.filter((draft) => { + const held = live.get(`${corpus.pubkey}:${draft.kind}`); + return held === undefined || !sameEvent(draft, held); + }); + + console.log( + `${corpus.name}\t${stale.length} of ${drafts.length} to send\t${corpus.npub.slice(0, 20)}...`, + ); + if (!confirmed || stale.length === 0) continue; + + const secret = secrets.get(corpus.name); + if (secret === undefined) throw new Error(`no key resolved for ${corpus.name}`); + + for (const draft of stale) { + const held = live.get(`${corpus.pubkey}:${draft.kind}`); + const signed = finalizeEvent( + { + kind: draft.kind, + created_at: held !== undefined && held.created_at >= now ? held.created_at + 1 : now, + tags: draft.tags, + content: draft.content, + }, + secret, + ); + const results = await publishEvent(signed, targets, pool); + const accepted = results.filter((result) => result.accepted).length; + console.log(`\tkind ${draft.kind}\taccepted by ${accepted} of ${results.length}`); + for (const refused of results.filter((result) => !result.accepted)) { + console.log(`\t\t${refused.relay}\t${refused.message}`); + } + } + } + + if (!confirmed) console.log("\nnothing was sent, pass --yes to publish this"); +}; diff --git a/apps/importer/src/keys.ts b/apps/importer/src/keys.ts new file mode 100644 index 0000000..1aa9c1e --- /dev/null +++ b/apps/importer/src/keys.ts @@ -0,0 +1,47 @@ +import { toNpub } from "@openspecs/nostr"; +import { decode } from "nostr-tools/nip19"; +import { getPublicKey } from "nostr-tools/pure"; +import { hexToBytes } from "nostr-tools/utils"; +import type { Corpus } from "./manifest.ts"; + +/** One variable per corpus, so a key can only ever sign the corpus it was made for. */ +export const keyVariable = (name: string): string => `OPENSPECS_IMPORT_KEY_${name.toUpperCase()}`; + +/** An nsec, or the same key in hex, which is what the crawler takes for its own key. */ +const toSecret = (value: string): Uint8Array | null => { + if (/^[0-9a-f]{64}$/i.test(value)) return hexToBytes(value.toLowerCase()); + try { + const decoded = decode(value); + return decoded.type === "nsec" ? decoded.data : null; + } catch { + return null; + } +}; + +/** + * The key a corpus is signed with, read from the environment and never from a + * file this repository holds. + * + * The check against the manifest is the one that matters. A corpus published + * under the wrong key is not a mistake anyone can take back: the coordinates + * belong to that key, the documents are already mirrored, and the only remedy + * is to withdraw a hundred documents and publish them again somewhere else. So + * the key proves it is the one the manifest names before it signs anything. + */ +export const secretFor = (corpus: Corpus, env: NodeJS.ProcessEnv = process.env): Uint8Array => { + const variable = keyVariable(corpus.name); + const value = (env[variable] ?? "").trim(); + if (value === "") throw new Error(`${variable} is not set, and ${corpus.name} cannot be signed`); + + const secret = toSecret(value); + if (secret === null) throw new Error(`${variable} is neither an nsec nor a key in hex`); + + const pubkey = getPublicKey(secret); + if (pubkey !== corpus.pubkey) { + throw new Error( + `${variable} signs as ${toNpub(pubkey)}, and ${corpus.name}.json names ${corpus.npub}`, + ); + } + + return secret; +}; diff --git a/apps/importer/src/links.ts b/apps/importer/src/links.ts new file mode 100644 index 0000000..539d107 --- /dev/null +++ b/apps/importer/src/links.ts @@ -0,0 +1,199 @@ +import { dirname, join, normalize } from "node:path/posix"; +import { toNaddr } from "@openspecs/nostr"; +import type { Corpus } from "./manifest.ts"; + +export type Destination = { pubkey: string; identifier: string }; + +export type SpecIndex = { + repos: string[]; + find: (repo: string, path: string) => Destination | null; +}; + +export type LinkReport = { + /** Rewritten to a `nostr:` reference, whether inside this corpus or another. */ + internal: number; + /** Left as they were: they point outside the three repositories. */ + external: number; + /** Into the document itself, which `rehype-slug` resolves on its own. */ + anchors: number; + /** + * Pinned to the source repository because the file they name is not imported. + * Listed rather than counted: this is what a reviewer reads to find a link + * that should have stayed inside the corpus. + */ + absolutized: { target: string; url: string }[]; +}; + +export type Rewrite = { content: string; report: LinkReport }; + +const GITHUB = "https://github.com/"; +const RAW = "https://raw.githubusercontent.com/"; + +const trimRepo = (repo: string): string => repo.replace(/\.git$/, "").replace(/\/+$/, ""); + +/** Where a file lives in its repository, pinned so the bytes behind it cannot change. */ +export const sourceUrl = (repo: string, commit: string, path: string): string => + `${trimRepo(repo)}/blob/${commit}/${path}`; + +export const specIndex = (corpora: Corpus[]): SpecIndex => { + const byRepo = new Map>(); + for (const corpus of corpora) { + const files = new Map(); + for (const spec of corpus.specs) { + files.set(spec.file, { pubkey: corpus.pubkey, identifier: spec.d }); + } + byRepo.set(trimRepo(corpus.repo), files); + } + + return { + repos: [...byRepo.keys()], + find: (repo, path) => byRepo.get(trimRepo(repo))?.get(path) ?? null, + }; +}; + +/** The repository and path a GitHub URL names, or null when it names something else. */ +const repoFile = (url: string, repos: string[]): { repo: string; path: string } | null => { + const afterRef = (rest: string): string | null => { + const slash = rest.indexOf("/"); + return slash > 0 ? rest.slice(slash + 1) : null; + }; + + for (const repo of repos) { + for (const view of ["blob", "tree"]) { + const prefix = `${repo}/${view}/`; + if (url.startsWith(prefix)) { + const path = afterRef(url.slice(prefix.length)); + if (path !== null) return { repo, path }; + } + } + + const prefix = repo.startsWith(GITHUB) ? `${RAW}${repo.slice(GITHUB.length)}/` : null; + if (prefix !== null && url.startsWith(prefix)) { + const path = afterRef(url.slice(prefix.length)); + if (path !== null) return { repo, path }; + } + } + + return null; +}; + +export type LinkContext = { + corpus: Pick; + /** Repo-relative path of the document being rewritten, which relative links resolve against. */ + file: string; + /** What every link leaving the corpus is pinned to. */ + commit: string; + index: SpecIndex; +}; + +/** + * A `nostr:` reference keeps its fragment. NIP-21 says nothing about one, but + * bech32's alphabet has no `#`, so a reader scanning for the identifier stops + * where the fragment starts and what follows is either used or ignored. + */ +const reference = (destination: Destination, fragment: string): string => + `nostr:${toNaddr(destination)}${fragment}`; + +const SCHEME = /^[a-z][a-z0-9+.-]*:/i; + +const rewriteTarget = (target: string, context: LinkContext, report: LinkReport): string => { + if (target === "" || target.startsWith("<")) return target; + + if (target.startsWith("#")) { + report.anchors++; + return target; + } + + const cut = target.indexOf("#"); + const path = cut === -1 ? target : target.slice(0, cut); + const fragment = cut === -1 ? "" : target.slice(cut); + + if (SCHEME.test(path)) { + if (!/^https?:\/\//i.test(path)) return target; + + const named = repoFile(path, context.index.repos); + const destination = named === null ? null : context.index.find(named.repo, named.path); + if (destination === null) { + report.external++; + return target; + } + report.internal++; + return reference(destination, fragment); + } + + const repo = trimRepo(context.corpus.repo); + const resolved = path.startsWith("/") + ? path.slice(1) + : normalize(join(dirname(context.file), path)); + if (resolved.startsWith("..")) return target; + + const destination = context.index.find(repo, resolved); + if (destination !== null) { + report.internal++; + return reference(destination, fragment); + } + + const url = sourceUrl(repo, context.commit, resolved) + fragment; + report.absolutized.push({ target, url }); + return url; +}; + +const FENCE = /^[ ]{0,3}(`{3,}|~{3,})/; + +/** Prose and fenced code, in order. Only the prose is rewritten. */ +const segments = (content: string): { text: string; code: boolean }[] => { + const parts: { text: string; code: boolean }[] = []; + let lines: string[] = []; + let fence: string | null = null; + + const flush = (code: boolean) => { + if (lines.length > 0) parts.push({ text: lines.join("\n"), code }); + lines = []; + }; + + for (const line of content.split("\n")) { + const opener = FENCE.exec(line)?.[1]; + if (fence === null && opener !== undefined) { + flush(false); + fence = opener[0] ?? null; + lines.push(line); + continue; + } + if (fence !== null && opener?.startsWith(fence)) { + lines.push(line); + flush(true); + fence = null; + continue; + } + lines.push(line); + } + flush(fence !== null); + + return parts; +}; + +/** The target of an inline link or image, anchored on `](` so a label may hold anything. */ +const INLINE = /(\]\()([^()\s]*)((?:[ \t]+"[^"]*")?\))/g; +/** The target of a reference definition, which is what the NUTs use throughout. */ +const REFERENCE = /^([ ]{0,3}\[[^\]]+\]:[ \t]*)(\S+)/gm; + +export const rewriteLinks = (content: string, context: LinkContext): Rewrite => { + const report: LinkReport = { internal: 0, external: 0, anchors: 0, absolutized: [] }; + + const rewritten = segments(content) + .map(({ text, code }) => + code + ? text + : text + .replace(INLINE, (_, open, target, close) => + [open, rewriteTarget(target, context, report), close].join(""), + ) + .replace( + REFERENCE, + (_, label, target) => label + rewriteTarget(target, context, report), + ), + ) + .join("\n"); + + return { content: rewritten, report }; +}; diff --git a/apps/importer/src/manifest.ts b/apps/importer/src/manifest.ts new file mode 100644 index 0000000..d03cdd5 --- /dev/null +++ b/apps/importer/src/manifest.ts @@ -0,0 +1,80 @@ +import { readdir, readFile } from "node:fs/promises"; +import { parsePubkey } from "@openspecs/nostr"; +import { z } from "zod"; + +const kindSchema = z.object({ + kind: z.number().int().nonnegative(), + name: z.string().min(1), +}); + +const specSchema = z.object({ + file: z.string().min(1), + d: z.string().min(1), + title: z.string().min(1), + /** Null where the document claims none, rather than a status invented here. */ + status: z.string().min(1).nullable(), + summary: z.string().min(1), + kinds: z.array(kindSchema).default([]), +}); + +const manifestSchema = z.object({ + name: z.string().min(1), + title: z.string().min(1), + npub: z.string().startsWith("npub1"), + repo: z.url(), + branch: z.string().min(1), + license: z.string().min(1), + topics: z.array(z.string().min(1)), + specs: z.array(specSchema).min(1), +}); + +export type SpecEntry = z.infer; +export type Manifest = z.infer; + +/** A manifest with its key resolved, which is what every naddr in the corpus is built on. */ +export type Corpus = Manifest & { pubkey: string }; + +export const MANIFEST_DIR = new URL("../manifests/", import.meta.url); + +const parseManifest = (name: string, raw: unknown): Corpus => { + const manifest = manifestSchema.parse(raw); + if (manifest.name !== name) { + throw new Error(`${name}.json declares the name ${manifest.name}`); + } + + const pubkey = parsePubkey(manifest.npub); + if (pubkey === null) throw new Error(`${name}.json has an unreadable npub`); + + const seen = new Set(); + for (const spec of manifest.specs) { + for (const value of [spec.d, spec.file]) { + if (seen.has(value)) throw new Error(`${name}.json lists ${value} twice`); + seen.add(value); + } + } + + return { ...manifest, pubkey }; +}; + +export const loadManifest = async (name: string, dir = MANIFEST_DIR): Promise => { + const raw = await readFile(new URL(`${name}.json`, dir), "utf8"); + return parseManifest(name, JSON.parse(raw)); +}; + +export const loadManifests = async (dir = MANIFEST_DIR): Promise => { + const names = (await readdir(dir)) + .filter((file) => file.endsWith(".json")) + .map((file) => file.slice(0, -".json".length)) + .sort(); + + const corpora = await Promise.all(names.map((name) => loadManifest(name, dir))); + + const keys = new Map(); + for (const corpus of corpora) { + const owner = keys.get(corpus.pubkey); + if (owner !== undefined) throw new Error(`${corpus.name} signs with the same key as ${owner}`); + keys.set(corpus.pubkey, corpus.name); + } + + return corpora; +}; diff --git a/apps/importer/src/publish.ts b/apps/importer/src/publish.ts new file mode 100644 index 0000000..512d710 --- /dev/null +++ b/apps/importer/src/publish.ts @@ -0,0 +1,166 @@ +import { readFile } from "node:fs/promises"; +import type { NostrEvent } from "@openspecs/nostr"; +import type { SimplePool } from "nostr-tools/pool"; +import { finalizeEvent } from "nostr-tools/pure"; +import type { ImportedEvent } from "./document.ts"; +import { secretFor } from "./keys.ts"; +import type { Corpus } from "./manifest.ts"; +import { liveEvents, publishEvent, type RelayResult } from "./relays.ts"; + +export type Planned = { + identifier: string; + event: ImportedEvent; + /** What the relays hold at this coordinate, and null where they hold nothing. */ + live: NostrEvent | null; +}; + +export type Plan = { + corpus: Corpus; + send: Planned[]; + unchanged: number; +}; + +const sameDocument = (draft: ImportedEvent, live: NostrEvent): boolean => + draft.content === live.content && JSON.stringify(draft.tags) === JSON.stringify(live.tags); + +/** + * A revision has to outrank the one it replaces, and a relay keeps the older of + * two events sharing a timestamp. So a document whose commit is older than what + * was published from it, which is any document changed in the manifest rather + * than in git, is stamped one second past the revision it replaces. + */ +export const stampOf = (draft: ImportedEvent, live: NostrEvent | null): number => + live !== null && draft.created_at <= live.created_at ? live.created_at + 1 : draft.created_at; + +export const planCorpus = ( + corpus: Corpus, + drafts: ImportedEvent[], + live: Map, +): Plan => { + const send: Planned[] = []; + let unchanged = 0; + + for (const event of drafts) { + const identifier = event.tags.find((tag) => tag[0] === "d")?.[1] ?? ""; + const held = live.get(identifier) ?? null; + if (held !== null && sameDocument(event, held)) { + unchanged++; + continue; + } + send.push({ identifier, event, live: held }); + } + + return { corpus, send, unchanged }; +}; + +export const readEvents = async (dir: string, corpus: Corpus): Promise => + Promise.all( + corpus.specs.map(async (spec) => + JSON.parse(await readFile(`${dir}/${corpus.name}/${spec.d}.json`, "utf8")), + ), + ); + +export type SendOptions = { + relays: string[]; + secret: Uint8Array; + pool?: SimplePool; + onSent?: (identifier: string, results: RelayResult[]) => void; +}; + +export const sendPlan = async ( + plan: Plan, + { relays, secret, pool, onSent }: SendOptions, +): Promise> => { + const sent = new Map(); + + for (const planned of plan.send) { + const signed = finalizeEvent( + { + kind: planned.event.kind, + created_at: stampOf(planned.event, planned.live), + tags: planned.event.tags, + content: planned.event.content, + }, + secret, + ); + const results = await publishEvent(signed, relays, pool); + sent.set(planned.identifier, results); + onSent?.(planned.identifier, results); + } + + return sent; +}; + +export type PublishOptions = { + corpora: Corpus[]; + events: string; + relays: string[]; + confirmed: boolean; + env?: NodeJS.ProcessEnv; + pool?: SimplePool; +}; + +/** + * Reads what `build` wrote, signs it, and sends what the relays do not already + * hold. A run that changes nothing sends nothing, which is what makes running + * it twice safe and what a second run is for. + * + * Every key is resolved before the first event is sent. A corpus whose key is + * missing must not be discovered halfway through publishing another one. + */ +export const publishEvents = async ({ + corpora, + events, + relays, + confirmed, + env, + pool, +}: PublishOptions): Promise => { + // A plan needs no key, and asking for one to print what would be sent would + // make the dry run the harder of the two things to do. Confirming resolves + // every key before the first event goes out, so a corpus whose key is missing + // is still not discovered halfway through publishing another one. + const secrets = confirmed + ? new Map(corpora.map((corpus) => [corpus.name, secretFor(corpus, env)])) + : new Map(); + + for (const corpus of corpora) { + const drafts = await readEvents(events, corpus); + const live = await liveEvents( + corpus.pubkey, + drafts.map((event) => event.tags.find((tag) => tag[0] === "d")?.[1] ?? ""), + relays, + pool, + ); + const plan = planCorpus(corpus, drafts, live); + const fresh = plan.send.filter((planned) => planned.live === null).length; + + console.log( + `${corpus.name}\t${plan.send.length} to send (${fresh} new, ${plan.send.length - fresh} revised), ` + + `${plan.unchanged} already published`, + ); + + if (!confirmed || plan.send.length === 0) continue; + + const secret = secrets.get(corpus.name); + if (secret === undefined) throw new Error(`no key resolved for ${corpus.name}`); + + let refused = 0; + await sendPlan(plan, { + relays, + secret, + pool, + onSent: (identifier, results) => { + const failed = results.filter((result) => !result.accepted); + if (failed.length === 0) return; + refused++; + for (const result of failed) { + console.log(`\t${identifier}\t${result.relay}\t${result.message}`); + } + }, + }); + console.log(`\t${plan.send.length - refused} of ${plan.send.length} accepted everywhere`); + } + + if (!confirmed) console.log("\nnothing was sent, pass --yes to publish this plan"); +}; diff --git a/apps/importer/src/relays.ts b/apps/importer/src/relays.ts new file mode 100644 index 0000000..3880a7c --- /dev/null +++ b/apps/importer/src/relays.ts @@ -0,0 +1,118 @@ +import { type NostrEvent, queryRelays, relayPool, SPEC_KIND } from "@openspecs/nostr"; +import type { SimplePool } from "nostr-tools/pool"; + +export type RelayResult = { relay: string; accepted: boolean; message: string }; + +const TIMEOUT_MS = 10_000; + +/** How many identifiers one filter carries: a relay that refuses a huge one would read as empty. */ +const PER_QUERY = 50; + +/** + * `SimplePool.publish` resolves rather than rejects when it never reached the + * relay at all, so an unreachable relay would otherwise be counted as one that + * accepted what was never sent. + */ +const UNREACHABLE = "connection failure:"; + +const said = (value: unknown, fallback: string): string => { + const text = value instanceof Error ? value.message : String(value ?? ""); + return text.trim() === "" ? fallback : text.trim(); +}; + +const answered = (relay: string, settled: PromiseSettledResult): RelayResult => { + if (settled.status === "rejected") { + return { relay, accepted: false, message: said(settled.reason, "refused") }; + } + const answer = said(settled.value, "accepted"); + return answer.startsWith(UNREACHABLE) + ? { relay, accepted: false, message: "not reached" } + : { relay, accepted: true, message: answer }; +}; + +const timed = (work: Promise): Promise => + Promise.race([ + work, + new Promise((_, reject) => { + setTimeout(() => reject(new Error("no answer")), TIMEOUT_MS).unref(); + }), + ]); + +export const publishEvent = async ( + event: NostrEvent, + relays: string[], + pool: SimplePool = relayPool(), +): Promise => { + const settled = await Promise.allSettled( + pool.publish(relays, event).map((answer) => timed(answer)), + ); + return settled.map((result, index) => answered(relays[index] ?? "", result)); +}; + +/** + * The revision each coordinate holds now, so a run can tell what it would + * change from what it would repeat. + * + * Relays keep serving revisions they have already replaced, so the newest of + * what comes back wins, and a tie goes to the lowest id as NIP-01 asks. + */ +export const liveEvents = async ( + pubkey: string, + identifiers: string[], + relays: string[], + pool?: SimplePool, +): Promise> => { + const live = new Map(); + if (relays.length === 0) return live; + + for (let at = 0; at < identifiers.length; at += PER_QUERY) { + const events = await queryRelays( + relays, + { + kinds: [SPEC_KIND], + authors: [pubkey], + "#d": identifiers.slice(at, at + PER_QUERY), + }, + { pool, timeoutMs: TIMEOUT_MS }, + ); + + for (const event of events) { + const identifier = event.tags.find((tag) => tag[0] === "d")?.[1] ?? ""; + const held = live.get(identifier); + const newer = + held === undefined || + event.created_at > held.created_at || + (event.created_at === held.created_at && event.id < held.id); + if (newer) live.set(identifier, event); + } + } + + return live; +}; + +/** + * The replaceable events a set of keys hold now, keyed by `pubkey:kind`. Same + * reason as above: what is already published is what a run does not repeat. + */ +export const liveReplaceable = async ( + authors: string[], + kinds: number[], + relays: string[], + pool?: SimplePool, +): Promise> => { + const live = new Map(); + if (relays.length === 0 || authors.length === 0) return live; + + const events = await queryRelays(relays, { kinds, authors }, { pool, timeoutMs: TIMEOUT_MS }); + for (const event of events) { + const key = `${event.pubkey}:${event.kind}`; + const held = live.get(key); + const newer = + held === undefined || + event.created_at > held.created_at || + (event.created_at === held.created_at && event.id < held.id); + if (newer) live.set(key, event); + } + + return live; +}; diff --git a/apps/importer/test/document.test.ts b/apps/importer/test/document.test.ts new file mode 100644 index 0000000..9acd81c --- /dev/null +++ b/apps/importer/test/document.test.ts @@ -0,0 +1,82 @@ +import { parseSpec, SPEC_KIND } from "@openspecs/nostr"; +import { describe, expect, it } from "vitest"; +import { buildImport, CLIENT, type ImportSource } from "../src/document.ts"; +import type { Corpus, SpecEntry } from "../src/manifest.ts"; + +const corpus: Corpus = { + name: "buds", + title: "Blossom Upgrade Documents", + npub: "npub1zvm2zlskr58g4u4k3m54454y087r30hedgtavyn75q4yp55dm9lqz4932r", + repo: "https://github.com/hzrd149/blossom", + branch: "master", + license: "Unlicense", + topics: ["blossom", "bud"], + pubkey: "1336a17e161d0e8af2b68ee95ad2a479fc38bef96a17d6127ea02a40d28dd97e", + specs: [], +}; + +const entry: SpecEntry = { + file: "buds/03.md", + d: "bud-03", + title: "BUD-03: User Server List", + status: "draft", + summary: "Where a user lists the servers holding their blobs.", + kinds: [{ kind: 10063, name: "User server list" }], +}; + +const source: ImportSource = { + commit: "b5bd2801d1763aa635fc8fea7a76597e0eb18990", + publishedAt: 1708455998, + createdAt: 1775927115, + sha256: "aac0c1c5b0364352494064e2a9da74147e8b1101bcf65f7136ee4d86d1fd4053", +}; + +const built = buildImport(corpus, entry, "# BUD-03\n", source); + +const tag = (name: string) => built.tags.find((tag) => tag[0] === name); + +/** parseSpec takes an event, not a draft, and reads no signature to do it. */ +const signed = { + ...built, + id: "a".repeat(64), + pubkey: corpus.pubkey, + sig: "b".repeat(128), +}; + +describe("buildImport", () => { + it("stands the revision at the moment of its commit", () => { + expect(built.kind).toBe(SPEC_KIND); + expect(built.created_at).toBe(source.createdAt); + expect(tag("published_at")).toEqual(["published_at", String(source.publishedAt)]); + }); + + it("names the file it copies, and the bytes it copied", () => { + expect(tag("proxy")).toEqual([ + "proxy", + "https://github.com/hzrd149/blossom/blob/b5bd2801d1763aa635fc8fea7a76597e0eb18990/buds/03.md", + "web", + ]); + expect(tag("x")).toEqual(["x", source.sha256]); + expect(tag("client")).toEqual(["client", CLIENT]); + }); + + it("reads back through the schema the site reads", () => { + const spec = parseSpec(signed); + expect(spec).not.toBeNull(); + expect(spec?.identifier).toBe(entry.d); + expect(spec?.title).toBe(entry.title); + expect(spec?.titleIsDerived).toBe(false); + expect(spec?.summary).toBe(entry.summary); + expect(spec?.summaryIsDerived).toBe(false); + expect(spec?.status).toBe("draft"); + expect(spec?.topics).toEqual(corpus.topics); + expect(spec?.kinds).toEqual([{ raw: "10063", kind: 10063, name: "User server list" }]); + expect(spec?.publishedAt).toBe(source.publishedAt); + expect(spec?.isEmpty).toBe(false); + }); + + it("leaves out a status the document does not claim", () => { + const without = buildImport(corpus, { ...entry, status: null }, "# BUD-03\n", source); + expect(without.tags.some((tag) => tag[0] === "s")).toBe(false); + }); +}); diff --git a/apps/importer/test/fixtures/dupe.json b/apps/importer/test/fixtures/dupe.json new file mode 100644 index 0000000..0d3e706 --- /dev/null +++ b/apps/importer/test/fixtures/dupe.json @@ -0,0 +1,13 @@ +{ + "name": "dupe", + "title": "Two documents, one identifier", + "npub": "npub1zvm2zlskr58g4u4k3m54454y087r30hedgtavyn75q4yp55dm9lqz4932r", + "repo": "https://github.com/example/dupe", + "branch": "main", + "license": "public domain", + "topics": ["example"], + "specs": [ + { "file": "01.md", "d": "x-01", "title": "One", "status": null, "summary": "First." }, + { "file": "02.md", "d": "x-01", "title": "Two", "status": null, "summary": "Second." } + ] +} diff --git a/apps/importer/test/identity.test.ts b/apps/importer/test/identity.test.ts new file mode 100644 index 0000000..7ad4cf1 --- /dev/null +++ b/apps/importer/test/identity.test.ts @@ -0,0 +1,125 @@ +import { + PROFILE_KIND, + parseProfile, + parseRelayList, + RELAY_LIST_KIND, + toNpub, +} from "@openspecs/nostr"; +import { createMockRelay, type MockRelay } from "nostr-mock-relay"; +import { nsecEncode } from "nostr-tools/nip19"; +import { SimplePool } from "nostr-tools/pool"; +import { generateSecretKey, getPublicKey } from "nostr-tools/pure"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { identityOf, profileOf, publishIdentity, relayListOf } from "../src/identity.ts"; +import { keyVariable } from "../src/keys.ts"; +import type { Corpus } from "../src/manifest.ts"; + +const corpus: Corpus = { + name: "nips", + title: "Nostr Implementation Possibilities", + npub: "npub19vumfllx9yeal9cwrymxcgkpuzf0j5lc8l876a2wpuzdtga5t8usdm3xgr", + repo: "https://github.com/nostr-protocol/nips", + branch: "master", + license: "public domain", + topics: ["nostr", "nip"], + pubkey: "2b39b4ffe62933df970e19366c22c1e092f953f83fcfed754e0f04d5a3b459f9", + specs: [], +}; + +const signed = (draft: { kind: number; content: string; tags: string[][] }) => ({ + ...draft, + id: "a".repeat(64), + pubkey: corpus.pubkey, + created_at: 1, + sig: "b".repeat(128), +}); + +describe("profileOf", () => { + it("says it is a mirror in the name, where a client will show it", () => { + const profile = parseProfile(signed(profileOf(corpus))); + expect(profile?.name).toBe("Nostr Implementation Possibilities (mirror)"); + }); + + it("names the repository it copies and the licence of what it copied", () => { + const { content } = profileOf(corpus); + expect(content).toContain("https://github.com/nostr-protocol/nips"); + expect(content).toContain("public domain"); + expect(content).toContain("did not write them"); + }); +}); + +describe("relayListOf", () => { + it("names where the corpus is published, for reading and for writing", () => { + const list = parseRelayList(signed(relayListOf(["wss://relay.openspecs.uid.ovh"]))); + expect(list?.write).toEqual(["wss://relay.openspecs.uid.ovh/"]); + expect(list?.read).toEqual(["wss://relay.openspecs.uid.ovh/"]); + }); +}); + +describe("identityOf", () => { + it("is a profile and a relay list, and nothing else", () => { + expect(identityOf(corpus, ["wss://relay.example"]).map((draft) => draft.kind)).toEqual([ + PROFILE_KIND, + RELAY_LIST_KIND, + ]); + }); +}); + +describe("publishIdentity", () => { + const secret = generateSecretKey(); + const pubkey = getPublicKey(secret); + const mine: Corpus = { ...corpus, pubkey, npub: toNpub(pubkey) }; + const env = { [keyVariable(corpus.name)]: nsecEncode(secret) }; + + let relay: MockRelay; + let pool: SimplePool; + + const publish = (confirmed = true) => + publishIdentity({ + corpora: [mine], + relays: ["wss://relay.openspecs.uid.ovh"], + targets: [relay.url ?? ""], + confirmed, + env, + pool, + }); + + beforeEach(async () => { + relay = createMockRelay(); + await relay.start(); + pool = new SimplePool(); + }); + + afterEach(async () => { + pool.close([relay.url ?? ""]); + await relay.stop(); + }); + + it("publishes a profile and a relay list", async () => { + await publish(); + expect( + relay + .getEvents() + .map((event) => event.kind) + .sort(), + ).toEqual([PROFILE_KIND, RELAY_LIST_KIND]); + }); + + it("says nothing twice", async () => { + await publish(); + await publish(); + expect(relay.getEvents()).toHaveLength(2); + }); + + it("sends nothing without a key, and nothing until it is told to", async () => { + await publishIdentity({ + corpora: [mine], + relays: ["wss://relay.openspecs.uid.ovh"], + targets: [relay.url ?? ""], + confirmed: false, + env: {}, + pool, + }); + expect(relay.getEvents()).toHaveLength(0); + }); +}); diff --git a/apps/importer/test/keys.test.ts b/apps/importer/test/keys.test.ts new file mode 100644 index 0000000..a5202d1 --- /dev/null +++ b/apps/importer/test/keys.test.ts @@ -0,0 +1,49 @@ +import { toNpub } from "@openspecs/nostr"; +import { nsecEncode } from "nostr-tools/nip19"; +import { generateSecretKey, getPublicKey } from "nostr-tools/pure"; +import { bytesToHex } from "nostr-tools/utils"; +import { describe, expect, it } from "vitest"; +import { keyVariable, secretFor } from "../src/keys.ts"; +import type { Corpus } from "../src/manifest.ts"; + +const secret = generateSecretKey(); +const pubkey = getPublicKey(secret); + +const corpus: Corpus = { + name: "buds", + title: "Blossom Upgrade Documents", + npub: toNpub(pubkey), + repo: "https://github.com/hzrd149/blossom", + branch: "master", + license: "Unlicense", + topics: ["blossom"], + pubkey, + specs: [], +}; + +describe("secretFor", () => { + it("takes an nsec", () => { + const env = { [keyVariable("buds")]: nsecEncode(secret) }; + expect(secretFor(corpus, env)).toEqual(secret); + }); + + it("takes the same key in hex", () => { + const env = { [keyVariable("buds")]: bytesToHex(secret) }; + expect(secretFor(corpus, env)).toEqual(secret); + }); + + it("names the variable it wanted", () => { + expect(() => secretFor(corpus, {})).toThrow(/OPENSPECS_IMPORT_KEY_BUDS/); + }); + + it("refuses to sign a corpus with a key that is not the one it names", () => { + const other = generateSecretKey(); + const env = { [keyVariable("buds")]: nsecEncode(other) }; + expect(() => secretFor(corpus, env)).toThrow(toNpub(getPublicKey(other))); + expect(() => secretFor(corpus, env)).toThrow(corpus.npub); + }); + + it("refuses anything that is not a key", () => { + expect(() => secretFor(corpus, { [keyVariable("buds")]: "hunter2" })).toThrow(/nsec/); + }); +}); diff --git a/apps/importer/test/links.test.ts b/apps/importer/test/links.test.ts new file mode 100644 index 0000000..9c33559 --- /dev/null +++ b/apps/importer/test/links.test.ts @@ -0,0 +1,120 @@ +import { parseSpecAddress } from "@openspecs/nostr"; +import { describe, expect, it } from "vitest"; +import { rewriteLinks, specIndex } from "../src/links.ts"; +import type { Corpus } from "../src/manifest.ts"; + +const BUDS_KEY = "1336a17e161d0e8af2b68ee95ad2a479fc38bef96a17d6127ea02a40d28dd97e"; +const NIPS_KEY = "2b39b4ffe62933df970e19366c22c1e092f953f83fcfed754e0f04d5a3b459f9"; + +const corpus = (over: Partial): Corpus => ({ + name: "buds", + title: "Blossom Upgrade Documents", + npub: "npub1", + repo: "https://github.com/hzrd149/blossom", + branch: "master", + license: "Unlicense", + topics: ["blossom"], + pubkey: BUDS_KEY, + specs: [ + { file: "buds/01.md", d: "bud-01", title: "One", status: null, summary: "s", kinds: [] }, + { file: "buds/02.md", d: "bud-02", title: "Two", status: null, summary: "s", kinds: [] }, + ], + ...over, +}); + +const nips = corpus({ + name: "nips", + repo: "https://github.com/nostr-protocol/nips", + pubkey: NIPS_KEY, + specs: [{ file: "94.md", d: "nip-94", title: "File", status: null, summary: "s", kinds: [] }], +}); + +const index = specIndex([corpus({}), nips]); +const context = { corpus: corpus({}), file: "buds/02.md", commit: "c0ffee", index }; + +const rewrite = (content: string) => rewriteLinks(content, context); + +const addressOf = (content: string) => { + const match = /nostr:(naddr1[a-z0-9]+)/.exec(content); + return match?.[1] === undefined ? null : parseSpecAddress(match[1]); +}; + +describe("rewriteLinks", () => { + it("turns a link to a document of the corpus into a nostr reference", () => { + const { content, report } = rewrite("see [BUD-01](./01.md) for that"); + expect(report.internal).toBe(1); + expect(addressOf(content)).toMatchObject({ pubkey: BUDS_KEY, identifier: "bud-01" }); + }); + + it("keeps the fragment beside the reference", () => { + const { content } = rewrite("[get](./01.md#get-sha256---get-blob)"); + expect(content).toMatch(/#get-sha256---get-blob\)$/); + expect(addressOf(content)).toMatchObject({ identifier: "bud-01" }); + }); + + it("reads a path from the root of the repository", () => { + const { content } = rewrite("[one](/buds/01.md)"); + expect(addressOf(content)).toMatchObject({ identifier: "bud-01" }); + }); + + it("crosses corpora, from a URL into another repository", () => { + const { content, report } = rewrite( + "[NIP-94](https://github.com/nostr-protocol/nips/blob/master/94.md)", + ); + expect(report.internal).toBe(1); + expect(addressOf(content)).toMatchObject({ pubkey: NIPS_KEY, identifier: "nip-94" }); + }); + + it("pins a file of the repository that is not imported", () => { + const { content, report } = rewrite("[the tests](../tests/02.md)"); + expect(content).toContain("https://github.com/hzrd149/blossom/blob/c0ffee/tests/02.md"); + expect(report.absolutized).toEqual([ + { + target: "../tests/02.md", + url: "https://github.com/hzrd149/blossom/blob/c0ffee/tests/02.md", + }, + ]); + }); + + it("pins an image the same way, so nothing points at a path that no longer exists", () => { + const { content } = rewrite("![a diagram](./flow.png)"); + expect(content).toBe( + "![a diagram](https://github.com/hzrd149/blossom/blob/c0ffee/buds/flow.png)", + ); + }); + + it("leaves a link out of the three repositories alone", () => { + const { content, report } = rewrite("[RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)"); + expect(content).toContain("https://www.rfc-editor.org/rfc/rfc2119"); + expect(report.external).toBe(1); + }); + + it("leaves an anchor into the document itself alone", () => { + const { content, report } = rewrite("[above](#blob-descriptor)"); + expect(content).toBe("[above](#blob-descriptor)"); + expect(report.anchors).toBe(1); + }); + + it("leaves a scheme it does not handle alone", () => { + const { content } = rewrite("[write us](mailto:nobody@example.com) and [b](blossom:abc.pdf)"); + expect(content).toBe("[write us](mailto:nobody@example.com) and [b](blossom:abc.pdf)"); + }); + + it("rewrites a reference definition, which is what carries the NUTs", () => { + const { content, report } = rewrite("as in [one][01].\n\n[01]: ./01.md"); + expect(report.internal).toBe(1); + expect(content).toMatch(/^\[01\]: nostr:naddr1[a-z0-9]+$/m); + }); + + it("leaves a fenced block untouched, example links included", () => { + const source = ["before [one](./01.md)", "```md", "[one](./01.md)", "```"].join("\n"); + const { content, report } = rewrite(source); + expect(report.internal).toBe(1); + expect(content.split("\n").at(2)).toBe("[one](./01.md)"); + }); + + it("returns a document it changed nothing in unchanged", () => { + const source = "# Title\n\nNothing to see, `01.md` in code only.\n"; + expect(rewrite(source).content).toBe(source); + }); +}); diff --git a/apps/importer/test/manifest.test.ts b/apps/importer/test/manifest.test.ts new file mode 100644 index 0000000..6822b46 --- /dev/null +++ b/apps/importer/test/manifest.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { loadManifest, loadManifests } from "../src/manifest.ts"; + +const FIXTURES = new URL("./fixtures/", import.meta.url); + +describe("loadManifests", () => { + it("reads the three corpora", async () => { + const corpora = await loadManifests(); + expect(corpora.map((corpus) => corpus.name)).toEqual(["buds", "nips", "nuts"]); + expect(corpora.map((corpus) => corpus.specs.length)).toEqual([13, 94, 31]); + }); + + it("resolves every npub to a key of its own", async () => { + const corpora = await loadManifests(); + const keys = corpora.map((corpus) => corpus.pubkey); + expect(keys.every((key) => /^[0-9a-f]{64}$/.test(key))).toBe(true); + expect(new Set(keys).size).toBe(keys.length); + }); + + it("names a document once, and a file once", async () => { + for (const corpus of await loadManifests()) { + const ids = corpus.specs.map((spec) => spec.d); + const files = corpus.specs.map((spec) => spec.file); + expect(new Set(ids).size, corpus.name).toBe(ids.length); + expect(new Set(files).size, corpus.name).toBe(files.length); + } + }); + + it("gives every document a title, a summary and an identifier that reads as a path", async () => { + for (const corpus of await loadManifests()) { + for (const spec of corpus.specs) { + expect(spec.d, spec.file).toMatch(/^[a-z0-9-]+$/); + expect(spec.title.length, spec.d).toBeGreaterThan(0); + expect(spec.summary.length, spec.d).toBeGreaterThan(0); + } + } + }); + + it("refuses a manifest that lists one identifier twice", async () => { + await expect(loadManifest("dupe", FIXTURES)).rejects.toThrow(/twice/); + }); +}); diff --git a/apps/importer/test/no-network.ts b/apps/importer/test/no-network.ts new file mode 100644 index 0000000..d542072 --- /dev/null +++ b/apps/importer/test/no-network.ts @@ -0,0 +1,14 @@ +const LOCAL = /^wss?:\/\/(127\.0\.0\.1|\[::1\]|localhost)([:/]|$)/i; + +/** + * Tests talk to mock relays on the loopback, never to a public one: a suite + * that publishes to a relay somebody reads is not a test. + */ +globalThis.WebSocket = new Proxy(globalThis.WebSocket, { + construct(target, args: [string]) { + if (!LOCAL.test(String(args[0]))) { + throw new Error(`a test tried to reach ${args[0]}, use a mock relay instead`); + } + return new target(...args); + }, +}); diff --git a/apps/importer/test/publish.test.ts b/apps/importer/test/publish.test.ts new file mode 100644 index 0000000..3379823 --- /dev/null +++ b/apps/importer/test/publish.test.ts @@ -0,0 +1,155 @@ +import { mkdirSync } from "node:fs"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { toNpub } from "@openspecs/nostr"; +import { createMockRelay, type MockRelay } from "nostr-mock-relay"; +import { nsecEncode } from "nostr-tools/nip19"; +import { SimplePool } from "nostr-tools/pool"; +import { generateSecretKey, getPublicKey } from "nostr-tools/pure"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { buildImport } from "../src/document.ts"; +import { keyVariable } from "../src/keys.ts"; +import type { Corpus, SpecEntry } from "../src/manifest.ts"; +import { publishEvents } from "../src/publish.ts"; + +const secret = generateSecretKey(); +const pubkey = getPublicKey(secret); +const env = { [keyVariable("buds")]: nsecEncode(secret) }; + +const entries: SpecEntry[] = [ + { file: "buds/01.md", d: "bud-01", title: "One", status: "draft", summary: "First.", kinds: [] }, + { file: "buds/02.md", d: "bud-02", title: "Two", status: "draft", summary: "Second.", kinds: [] }, +]; + +const corpus: Corpus = { + name: "buds", + title: "Blossom Upgrade Documents", + npub: toNpub(pubkey), + repo: "https://github.com/hzrd149/blossom", + branch: "master", + license: "Unlicense", + topics: ["blossom"], + pubkey, + specs: entries, +}; + +const source = { + commit: "b5bd2801d1763aa635fc8fea7a76597e0eb18990", + publishedAt: 1708455998, + createdAt: 1775927115, + sha256: "aac0c1c5b0364352494064e2a9da74147e8b1101bcf65f7136ee4d86d1fd4053", +}; + +let relay: MockRelay; +let pool: SimplePool; +let events: string; + +const publish = (confirmed = true) => + publishEvents({ + corpora: [corpus], + events, + relays: [relay.url ?? ""], + confirmed, + env, + pool, + }); + +const documents = () => relay.getEvents().filter((event) => event.kind === 30817); + +const identifierOf = (event: { tags: string[][] }) => + event.tags.find((tag) => tag[0] === "d")?.[1] ?? ""; + +beforeEach(async () => { + relay = createMockRelay(); + await relay.start(); + pool = new SimplePool(); + events = await mkdtemp(join(tmpdir(), "openspecs-")); + mkdirSync(join(events, corpus.name)); + for (const entry of entries) { + const event = buildImport(corpus, entry, `# ${entry.title}\n`, source); + await writeFile(join(events, corpus.name, `${entry.d}.json`), JSON.stringify(event)); + } +}); + +afterEach(async () => { + pool.close([relay.url ?? ""]); + await relay.stop(); +}); + +describe("publishEvents", () => { + it("sends nothing until it is told to", async () => { + await publish(false); + expect(documents()).toHaveLength(0); + }); + + it("plans without a key, so seeing what would be sent is the easy thing to do", async () => { + await publishEvents({ + corpora: [corpus], + events, + relays: [relay.url ?? ""], + confirmed: false, + env: {}, + pool, + }); + expect(documents()).toHaveLength(0); + }); + + it("signs every document with the key the manifest names", async () => { + await publish(); + const sent = documents(); + expect(sent).toHaveLength(2); + expect(sent.every((event) => event.pubkey === pubkey)).toBe(true); + expect(sent.map(identifierOf).sort()).toEqual(["bud-01", "bud-02"]); + }); + + it("sends nothing on a second run, which is what makes running it again safe", async () => { + await publish(); + await publish(); + expect(documents()).toHaveLength(2); + }); + + it("sends the one document that changed, and leaves the other where it was", async () => { + await publish(); + const before = new Map(documents().map((event) => [identifierOf(event), event.id])); + + const path = join(events, corpus.name, "bud-02.json"); + const draft = JSON.parse(await readFile(path, "utf8")); + draft.content = "# Two\n\nA sentence that was not there before.\n"; + await writeFile(path, JSON.stringify(draft)); + + await publish(); + const after = new Map(documents().map((event) => [identifierOf(event), event])); + expect(after.get("bud-01")?.id).toBe(before.get("bud-01")); + expect(after.get("bud-02")?.id).not.toBe(before.get("bud-02")); + expect(after.get("bud-02")?.content).toContain("was not there before"); + }); + + it("stamps a revision past the one it replaces, so a relay keeps the newer", async () => { + await publish(); + + const path = join(events, corpus.name, "bud-01.json"); + const draft = JSON.parse(await readFile(path, "utf8")); + draft.content = "# One\n\nRevised without a commit behind it.\n"; + await writeFile(path, JSON.stringify(draft)); + + await publish(); + const held = documents().find((event) => identifierOf(event) === "bud-01"); + expect(held?.created_at).toBe(source.createdAt + 1); + expect(held?.content).toContain("Revised without a commit"); + }); + + it("refuses the whole run when a key is missing, before it sends anything", async () => { + await expect( + publishEvents({ + corpora: [corpus], + events, + relays: [relay.url ?? ""], + confirmed: true, + env: {}, + pool, + }), + ).rejects.toThrow(/OPENSPECS_IMPORT_KEY_BUDS/); + expect(documents()).toHaveLength(0); + }); +}); diff --git a/apps/importer/tsconfig.json b/apps/importer/tsconfig.json new file mode 100644 index 0000000..7b4ee4b --- /dev/null +++ b/apps/importer/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*", "test/**/*"], + "compilerOptions": { + "types": ["node"], + "allowImportingTsExtensions": true + } +} diff --git a/apps/importer/vitest.config.ts b/apps/importer/vitest.config.ts new file mode 100644 index 0000000..c185ea8 --- /dev/null +++ b/apps/importer/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + name: "importer", + environment: "node", + include: ["test/**/*.test.ts"], + setupFiles: ["./test/no-network.ts"], + }, +}); diff --git a/apps/web/app/components/spec-row.tsx b/apps/web/app/components/spec-row.tsx index abf32ae..f831a20 100644 --- a/apps/web/app/components/spec-row.tsx +++ b/apps/web/app/components/spec-row.tsx @@ -43,10 +43,10 @@ export const SpecRow = ({ {line !== "" && ( diff --git a/apps/web/app/components/spec-tags.tsx b/apps/web/app/components/spec-tags.tsx index 8c63c2d..daaf3f9 100644 --- a/apps/web/app/components/spec-tags.tsx +++ b/apps/web/app/components/spec-tags.tsx @@ -17,6 +17,7 @@ export const STATUS_TONE: Record = { final: "text-signal-settled", stable: "text-signal-settled", deprecated: "text-signal-closed", + unrecommended: "text-signal-closed", obsolete: "text-signal-closed", rejected: "text-signal-closed", retired: "text-signal-closed", diff --git a/apps/web/app/lib/spec-page.test.ts b/apps/web/app/lib/spec-page.test.ts index 45ce9b2..9797fc5 100644 --- a/apps/web/app/lib/spec-page.test.ts +++ b/apps/web/app/lib/spec-page.test.ts @@ -1,91 +1,41 @@ -import { SPEC_KIND, type Spec } from "@openspecs/nostr"; +import { parseSpec, SPEC_KIND, toNaddr, toNpub } from "@openspecs/nostr"; import { describe, expect, it } from "vitest"; -import { isNewerRevision, toPage } from "./spec-page"; +import { toPage } from "./spec-page"; -const PUBKEY = "a".repeat(64); +const PUBKEY = "1336a17e161d0e8af2b68ee95ad2a479fc38bef96a17d6127ea02a40d28dd97e"; -const revision = ( - createdAt: number, - id: string, - content = "# A document\n\nA paragraph.\n\n## A section\n\nAnother.", -): Spec => ({ - event: { - id, +const specOf = (content: string) => { + const spec = parseSpec({ + id: "a".repeat(64), pubkey: PUBKEY, - created_at: createdAt, + created_at: 1, kind: SPEC_KIND, - tags: [["d", "a-document"]], + tags: [ + ["d", "bud-02"], + ["title", "BUD-02"], + ], content, - sig: "f".repeat(128), - }, - pubkey: PUBKEY, - identifier: "a-document", - title: "A document", - titleIsDerived: false, - summary: "A paragraph.", - summaryIsDerived: true, - content, - kinds: [], - topics: [], - status: null, - createdAt, - publishedAt: createdAt, - forks: [], - isEmpty: content.trim() === "", -}); - -const shown = toPage(revision(1_700_000_000, "b".repeat(64))); - -describe("toPage", () => { - it("renders the document rather than shipping it twice", () => { - const page = toPage(revision(1_700_000_000, "c".repeat(64))); - expect(page.html).toContain("

A paragraph.

"); - // The opening heading repeats the title and the renderer drops it, so what - // the contents rail is built from is what is left below it. - expect(page.headings.map((heading) => heading.text)).toEqual(["A section"]); + sig: "b".repeat(128), }); + if (spec === null) throw new Error("the fixture does not parse as a document"); + return spec; +}; - /** - * `isNewerRevision` reads the moment a revision was signed off this field, so - * the day it stops meaning `createdAt` the comparison silently stops working. - */ - it("dates the page by when its revision was signed", () => { - expect(toPage(revision(1_700_000_500, "c".repeat(64))).revisedAt).toBe(1_700_000_500); - }); - - it("carries the links a document cites, so their previews can be matched to it", () => { - const page = toPage(revision(1_700_000_000, "c".repeat(64), "See .")); - expect(page.links).toContain("https://example.com/spec"); - }); -}); - -describe("isNewerRevision", () => { - it("takes a revision signed after the one on screen", () => { - expect(isNewerRevision(revision(1_700_000_100, "d".repeat(64)), shown)).toBe(true); - }); - - /** A relay serving a superseded revision is answering honestly, and is ignored. */ - it("refuses one signed before it, however recently a relay served it", () => { - expect(isNewerRevision(revision(1_699_999_900, "d".repeat(64)), shown)).toBe(false); - }); - - it("has nothing to offer for the revision already on screen", () => { - expect(isNewerRevision(revision(1_700_000_000, "b".repeat(64)), shown)).toBe(false); - }); +describe("toPage", () => { + const naddr = toNaddr({ pubkey: PUBKEY, identifier: "bud-01" }); + const path = `/spec/${toNpub(PUBKEY)}/bud-01`; - /** The same tie NIP-01 and `latestByCoordinate` break, broken the same way. */ - it("breaks a tie on the lowest id, so this and the loader agree", () => { - expect(isNewerRevision(revision(1_700_000_000, "a".repeat(64)), shown)).toBe(true); - expect(isNewerRevision(revision(1_700_000_000, "f".repeat(64)), shown)).toBe(false); + it("points a reference at the document it addresses", () => { + expect(toPage(specOf(`see [BUD-01](nostr:${naddr})`)).html).toContain(`href="${path}"`); }); - it("never swaps in another author's document", () => { - const theirs = { ...revision(1_700_000_100, "d".repeat(64)), pubkey: "e".repeat(64) }; - expect(isNewerRevision(theirs, shown)).toBe(false); + it("keeps the section a reference names", () => { + expect(toPage(specOf(`[get](nostr:${naddr}#get-blob)`)).html).toContain( + `href="${path}#get-blob"`, + ); }); - it("never swaps in another document of the same author", () => { - const other = { ...revision(1_700_000_100, "d".repeat(64)), identifier: "another" }; - expect(isNewerRevision(other, shown)).toBe(false); + it("draws a reference standing in the text as a link too", () => { + expect(toPage(specOf(`as nostr:${naddr} says`)).html).toContain(`href="${path}"`); }); }); diff --git a/apps/web/app/lib/spec-page.ts b/apps/web/app/lib/spec-page.ts index f727022..da8c23a 100644 --- a/apps/web/app/lib/spec-page.ts +++ b/apps/web/app/lib/spec-page.ts @@ -1,6 +1,7 @@ import type { MarkdownHeading } from "@openspecs/markdown"; import { renderMarkdown } from "@openspecs/markdown"; import { type Spec, type SpecKindRef, toNaddr, toNpub } from "@openspecs/nostr"; +import { mentionResolver } from "./mention"; export type SpecPage = { kind: number; @@ -38,7 +39,13 @@ export type SpecPage = { * payload of every page for no reader. */ export const toPage = (spec: Spec): SpecPage => { - const { html, headings, links } = renderMarkdown(spec.content, { title: spec.title }); + const { html, headings, links } = renderMarkdown(spec.content, { + title: spec.title, + // Without names: a body is rendered here on the server and again in the + // browser, and neither has profiles to hand. A key still becomes a link, + // wearing the short form of itself, and a document becomes its address. + mention: mentionResolver({}), + }); return { kind: spec.event.kind, title: spec.title, diff --git a/infra/.env.example b/infra/.env.example index 8a39e44..6811132 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -33,6 +33,11 @@ OPENSPECS_CRAWLER_SOURCES= # it should be readable by nobody else. OPENSPECS_CRAWLER_ARCHIVIST_KEY= +# Where the relay holding the imported corpus listens on the host, used only +# with the relay overlay. Loopback by default: a relay reachable from outside is +# a relay whose address you decided to publish. +OPENSPECS_RELAY_BIND=127.0.0.1:7777 + # Only used with the Caddy overlay: the domain Caddy serves and requests a # certificate for. Use localhost for local runs. OPENSPECS_DOMAIN=localhost diff --git a/infra/docker-compose.relay.yml b/infra/docker-compose.relay.yml new file mode 100644 index 0000000..391a357 --- /dev/null +++ b/infra/docker-compose.relay.yml @@ -0,0 +1,41 @@ +# The relay the imported corpus is published to. +# +# docker compose -f docker-compose.relay.yml up -d +# +# A relay of its own, and a file of its own. The documents mirrored from git are +# signed by keys this project runs, so they are held where this project can take +# them back: emptying a relay you operate is a command, while asking the network +# to forget an event is a request most of it ignores. Until the corpus has been +# read and is worth keeping, that difference is the whole plan. +# +# Loopback by default, like the site: a relay reachable from outside is a relay +# whose address you decided to publish. +# +# strfry, because the crawler mirrors over NIP-77 and this is the relay that +# speaks it, in the image its own author builds. +services: + relay: + image: ghcr.io/hoytech/strfry:latest + restart: unless-stopped + # Two settings the shipped config gets wrong for this. It binds the loopback + # of the container, which no published port can reach, so Docker draws that + # boundary in `ports` instead. And it refuses events older than three years, + # which is a spam rule written for what people say, not for what they wrote + # once and left alone: a specification carries the date of the commit that + # last changed it, and one nobody has touched since 2023 is not stale, it is + # finished. + command: + - "--set" + - "relay.bind=0.0.0.0" + - "--set" + - "events.rejectEventsOlderThanSeconds=946080000" + - "relay" + ports: + - "${OPENSPECS_RELAY_BIND:-127.0.0.1:7777}:7777" + volumes: + - relay_data:/app/strfry-db + +volumes: + # The corpus itself. Losing it costs one run of the importer, which is what an + # import that reproduces its own output is for. + relay_data: diff --git a/packages/markdown/src/render.ts b/packages/markdown/src/render.ts index 272dc47..1c15596 100644 --- a/packages/markdown/src/render.ts +++ b/packages/markdown/src/render.ts @@ -37,9 +37,9 @@ export type RenderOptions = { */ headingOffset?: number; /** - * How to draw a `nostr:` reference. Left out, they stay the bech32 text they - * were: this package knows nothing about keys, and the caller that does can - * turn one into a name. + * How to draw a `nostr:` reference, and where to point a link already written + * as one. Left out, they stay the text they were: this package knows nothing + * about keys, and the caller that does can turn one into a name. */ mention?: MentionResolver; }; @@ -175,6 +175,32 @@ const drawMentions = (resolve: MentionResolver | undefined) => (tree: Root) => { }); }; +/** + * Where a `nostr:` link points. The text is the author's own, so only the + * destination is resolved, and a reference nothing can resolve keeps the URI it + * was: a reader whose browser handles the scheme still follows it. + * + * A fragment survives. NIP-21 defines none, but bech32's alphabet has no `#`, + * so an identifier ends where a fragment starts, and a link into a section of + * another document has no other way to be written. + */ +const pointLinks = (resolve: MentionResolver | undefined) => (tree: Root) => { + if (resolve === undefined) return; + + visit(tree, "element", (node: Element) => { + if (node.tagName !== "a") return; + const href = String(node.properties.href ?? ""); + if (!/^nostr:/i.test(href)) return; + + const rest = href.slice("nostr:".length); + const cut = rest.indexOf("#"); + const mention = resolve(cut === -1 ? rest : rest.slice(0, cut)); + if (mention?.href === undefined) return; + + node.properties.href = cut === -1 ? mention.href : mention.href + rest.slice(cut); + }); +}; + const collectHeadings = (headings: MarkdownHeading[]) => (tree: Root) => { visit(tree, "element", (node: Element) => { // The footnote label is generated, not written, and belongs to no section. @@ -211,7 +237,16 @@ const linkHeadings = (headings: MarkdownHeading[]): AutolinkOptions => ({ * clobbering are the ones this pipeline generates itself, and prefixing them * would only break the footnote links pointing at them. */ -const schema = { ...defaultSchema, clobberPrefix: "" }; +const schema = { + ...defaultSchema, + clobberPrefix: "", + // NIP-21, added to a list of protocols a browser might act on. This one it + // cannot: an unresolved `nostr:` link goes nowhere rather than somewhere. + protocols: { + ...defaultSchema.protocols, + href: [...(defaultSchema.protocols?.href ?? []), "nostr"], + }, +}; /** * Sanitizing before slugs and link hardening rather than last: those plugins @@ -234,6 +269,8 @@ export const renderMarkdown = (content: string, options: RenderOptions = {}): Re .use(collectLinks, links) // After the collection: a mention is a reference to a key, not a cited link. .use(drawMentions, options.mention) + // After the mentions: a reference already written as a link keeps its text. + .use(pointLinks, options.mention) .use(hardenLinks) .use(rehypeStringify) .processSync(content) diff --git a/packages/markdown/test/render.test.ts b/packages/markdown/test/render.test.ts index 1adc439..bad4056 100644 --- a/packages/markdown/test/render.test.ts +++ b/packages/markdown/test/render.test.ts @@ -167,6 +167,42 @@ describe("mentions", () => { }); }); +describe("references", () => { + const NADDR = + "naddr1qvzqqqrcvypzpzvm2zlskr58g4u4k3m54454y087r30hedgtavyn75q4yp55dm9lqqrxuat595ergwfsx56rgvfe"; + const point = (href?: string) => () => (href === undefined ? null : { label: "x", href }); + + it("points a link written as a reference at wherever the resolver says", () => { + const out = html(`see [BUD-01](nostr:${NADDR})`, { mention: point("/spec/npub1a/bud-01") }); + expect(out).toContain('BUD-01'); + }); + + it("keeps the fragment, so a link into a section still lands on it", () => { + const out = html(`[get](nostr:${NADDR}#get-blob)`, { mention: point("/spec/npub1a/bud-01") }); + expect(out).toContain('href="/spec/npub1a/bud-01#get-blob"'); + }); + + it("leaves the reference itself when nothing resolves it", () => { + const out = html(`see [BUD-01](nostr:${NADDR})`, { mention: point() }); + expect(out).toContain(`href="nostr:${NADDR}"`); + }); + + it("keeps the reference when no resolver was given, rather than dropping it", () => { + expect(html(`see [BUD-01](nostr:${NADDR})`)).toContain(`href="nostr:${NADDR}"`); + }); + + it("does not cite a reference as an outgoing link", () => { + const { links } = renderMarkdown(`[one](nostr:${NADDR}) and [two](https://example.com)`, { + mention: point("/spec/npub1a/bud-01"), + }); + expect(links).toEqual(["https://example.com"]); + }); + + it("still drops a scripting protocol", () => { + expect(html("[click](javascript:alert(1))")).toContain("click"); + }); +}); + it("renders an empty document to nothing", () => { expect(renderMarkdown("")).toEqual({ html: "", headings: [], links: [] }); }); diff --git a/packages/nostr/src/corpus.ts b/packages/nostr/src/corpus.ts index 2fae39d..446820b 100644 --- a/packages/nostr/src/corpus.ts +++ b/packages/nostr/src/corpus.ts @@ -1,7 +1,7 @@ import type { Filter } from "nostr-tools/filter"; import { type NostrEvent, SPEC_KIND } from "./event"; import { queryRelays, type RelayOptions, relaySet } from "./pool"; -import { DEFAULT_RELAYS, latestByCoordinate } from "./relay"; +import { latestByCoordinate, READ_RELAYS } from "./relay"; import { parseSpec, type Spec } from "./spec"; /** What is known about one relay since the last synchronisation. */ @@ -100,7 +100,7 @@ const syncRelay = async (relay: string, options: SyncOptions): Promise => { - const relays = relaySet(options.relays ?? DEFAULT_RELAYS); + const relays = relaySet(options.relays ?? READ_RELAYS); const settled = await Promise.allSettled(relays.map((relay) => syncRelay(relay, options))); const events: NostrEvent[] = []; diff --git a/packages/nostr/src/index.ts b/packages/nostr/src/index.ts index 61b8bee..600ac72 100644 --- a/packages/nostr/src/index.ts +++ b/packages/nostr/src/index.ts @@ -185,7 +185,9 @@ export { fetchSpec, fetchSpecEvent, fetchSpecs, + IMPORT_RELAYS, latestByCoordinate, + READ_RELAYS, type SpecQuery, } from "./relay"; export { diff --git a/packages/nostr/src/relay.ts b/packages/nostr/src/relay.ts index d5eec87..28de521 100644 --- a/packages/nostr/src/relay.ts +++ b/packages/nostr/src/relay.ts @@ -18,6 +18,19 @@ export const DEFAULT_RELAYS = [ "wss://nostr.oxtr.dev", ]; +/** + * Read alongside the ones above, and never written to. The specifications + * mirrored from git live here, and this stays a list of its own because + * `DEFAULT_RELAYS` is not only where this site reads: it is where a document + * signed here is published, what a key made here declares as its own, and where + * the rebroadcast button aims. A relay holding a copy of somebody else's + * specifications has no business collecting any of that. + */ +export const IMPORT_RELAYS = ["wss://relay.openspecs.uid.ovh"]; + +/** Everything a reader is shown comes from one of these. */ +export const READ_RELAYS = [...DEFAULT_RELAYS, ...IMPORT_RELAYS]; + /** Resolving relay lists for a whole listing would cost more than it can return. */ const MAX_OUTBOX_AUTHORS = 20; @@ -82,7 +95,7 @@ const querySpecs = async ( : writeRelaysOf(authors, options).then((relays) => queryRelays(relays, filter, options)); const [fromDefaults, fromOutbox] = await Promise.all([ - queryRelays(relaySet(options.relays ?? DEFAULT_RELAYS, hints), filter, options), + queryRelays(relaySet(options.relays ?? READ_RELAYS, hints), filter, options), outbox, ]); return latestByCoordinate(parseAll([...fromDefaults, ...fromOutbox])); @@ -126,8 +139,12 @@ export const fetchSpecs = async ( if (specQuery.until !== undefined) filter.until = specQuery.until; if (specQuery.limit !== undefined) filter.limit = specQuery.limit; + // On the revision rather than on the first publication, which is what a relay + // answering a `limit` selected: it returns its newest by `created_at`, so + // ordering the window on anything else would show a list neither the query nor + // the sort ever asked for. const specs = (await querySpecs(filter, specQuery.authors ?? [], options)).sort( - (a, b) => b.publishedAt - a.publishedAt, + (a, b) => b.createdAt - a.createdAt, ); // The filter limit is per relay, so it only caps what comes in. The caller asked // for a number of documents, not a number of documents per operator. diff --git a/packages/nostr/test/relay.test.ts b/packages/nostr/test/relay.test.ts index ff358c4..8074e7e 100644 --- a/packages/nostr/test/relay.test.ts +++ b/packages/nostr/test/relay.test.ts @@ -4,11 +4,14 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { SPEC_KIND } from "../src/event"; import { clearRelayListCache, RELAY_LIST_KIND } from "../src/nip65"; import { + DEFAULT_RELAYS, type FetchOptions, fetchSpec, fetchSpecEvent, fetchSpecs, + IMPORT_RELAYS, latestByCoordinate, + READ_RELAYS, } from "../src/relay"; import { parseSpec, type Spec } from "../src/spec"; import { caseEvents, events } from "./fixtures"; @@ -186,3 +189,60 @@ describe("fetchSpecs", () => { expect(await fetchSpecs({ authors: [stranger] }, options)).toEqual([]); }); }); + +describe("READ_RELAYS", () => { + it("is the defaults and the relays the imported corpus lives on", () => { + expect(READ_RELAYS).toEqual([...DEFAULT_RELAYS, ...IMPORT_RELAYS]); + }); + + /** + * `DEFAULT_RELAYS` is where a document signed here is published, what a key + * made here declares as its own, and where the rebroadcast button aims. This + * is what keeps a relay holding a copy of somebody else's specifications from + * quietly becoming all of that. + */ + it("never lets a corpus relay reach the list this project writes to", () => { + for (const relay of IMPORT_RELAYS) expect(DEFAULT_RELAYS).not.toContain(relay); + }); +}); + +describe("fetchSpecs", () => { + const dated = (id: string, createdAt: number, publishedAt: number) => + finalizeEvent( + { + kind: SPEC_KIND, + created_at: createdAt, + tags: [ + ["d", id], + ["title", id], + ["published_at", String(publishedAt)], + ], + content: `# ${id}\n\nA specification, in one paragraph.`, + }, + secretKey, + ); + + /** + * A relay answering a `limit` hands back its newest by `created_at`, so a + * listing ordered on anything else shows a window chosen one way and sorted + * another. This is what keeps the two in step. + */ + it("puts the newest revision first, whatever it says about its first publication", async () => { + const relay = await startRelay(); + relay.seed([ + dated("published-long-ago-revised-yesterday", 1_800_000_200, 1_500_000_000), + dated("published-recently-untouched-since", 1_800_000_100, 1_700_000_000), + ]); + + const specs = await fetchSpecs( + {}, + { relays: [relay.url ?? ""], outbox: false, timeoutMs: 2000 }, + ); + + expect(specs.map((spec) => spec.identifier)).toEqual([ + "published-long-ago-revised-yesterday", + "published-recently-untouched-since", + ]); + await relay.stop(); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ee8a6a..400bf71 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,34 @@ importers: specifier: ^4.1.10 version: 4.1.10(@types/node@24.13.3)(vite@8.2.1(@types/node@24.13.3)(jiti@2.7.0)) + apps/importer: + dependencies: + '@openspecs/nostr': + specifier: workspace:* + version: link:../../packages/nostr + nostr-tools: + specifier: 2.24.1 + version: 2.24.1(typescript@7.0.2) + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: + '@types/node': + specifier: ^24.13.3 + version: 24.13.3 + nostr-mock-relay: + specifier: ^0.1.1 + version: 0.1.1(typescript@7.0.2) + typescript: + specifier: ^7.0.2 + version: 7.0.2 + vite-node: + specifier: ^6.0.0 + version: 6.0.0(@types/node@24.13.3)(jiti@2.7.0) + vitest: + specifier: ^4.1.10 + version: 4.1.10(@types/node@24.13.3)(vite@8.2.1(@types/node@24.13.3)(jiti@2.7.0)) + apps/web: dependencies: '@fontsource/jetbrains-mono': @@ -874,6 +902,10 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + cac@7.0.0: + resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==} + engines: {node: '>=20.19.0'} + call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -1897,6 +1929,11 @@ packages: vfile@6.0.3: resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite-node@6.0.0: + resolution: {integrity: sha512-oj4PVrT+pDh6GYf5wfUXkcZyekYS8kKPfLPXVl8qe324Ec6l4K2DUKNadRbZ3LQl0qGcDz+PyOo7ZAh00Y+JjQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + vite@8.2.1: resolution: {integrity: sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2661,6 +2698,8 @@ snapshots: bytes@3.1.2: {} + cac@7.0.0: {} + call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -3871,6 +3910,27 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 + vite-node@6.0.0(@types/node@24.13.3)(jiti@2.7.0): + dependencies: + cac: 7.0.0 + es-module-lexer: 2.3.1 + obug: 2.1.4 + pathe: 2.0.3 + vite: 8.2.1(@types/node@24.13.3)(jiti@2.7.0) + transitivePeerDependencies: + - '@types/node' + - '@vitejs/devtools' + - esbuild + - jiti + - less + - sass + - sass-embedded + - stylus + - sugarss + - terser + - tsx + - yaml + vite@8.2.1(@types/node@24.13.3)(jiti@2.7.0): dependencies: lightningcss: 1.33.0