From ba7d675160a32704e348769789965030c6fa5a00 Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Wed, 15 Jul 2026 17:57:40 -0500 Subject: [PATCH] fix: extract text from tiptap bodies, and read the right description field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking the library against the live API disproved what its comments claimed. Comment and Description were handing back raw JSON blobs, and the parser this package has always carried could not have prevented it. DeviantArt no longer stores rich text as Draft.js. It uses tiptap: Draft.js, what we parse: {"blocks":[{"text":"..."}]} tiptap, what DA sends: {"version":1,"document":{"type":"doc",...}} Across 34 live comments: 33 tiptap, 0 Draft.js, and 33 whose .Comment was the raw JSON. flattenMarkup now walks the tiptap tree — text nodes joined per block, hardBreak as a newline, entities decoded — and keeps the Draft.js path for old bodies that still carry it. Nodes with no text of their own (images, galleries, emotes) drop out, as they always did. The description had a second, larger bug behind that one. It lives in Extended.DescriptionText, not TextContent, which GetDeviation read: of 24 deviations sampled, 23 populated the former and 1 the latter. So Post.Description was empty for nearly every deviation, and the txt[1] guard removed in 3dad0ea was never what stood in the way. Prefer Extended.DescriptionText and fall back, since either may be the one set. Verified against the live API rather than by reading: comments still raw JSON went 33/34 to 0/34, and descriptions that were "" now return prose. Also corrects the doc comments that asserted the Draft.js story on pkg.go.dev, and notes that flattening drops images, links, and emotes. --- comments.go | 108 ++++++++++++++++++++++++++++-------- comments_test.go | 139 +++++++++++++++++++++++++++++++++++++++-------- deviantion.go | 25 +++++++-- 3 files changed, 222 insertions(+), 50 deletions(-) diff --git a/comments.go b/comments.go index 6e41f38..c65364e 100644 --- a/comments.go +++ b/comments.go @@ -2,6 +2,7 @@ package devianter import ( "encoding/json" + "html" "net/url" "strconv" "strings" @@ -25,6 +26,10 @@ type Thread struct { // Comment is the comment's plain text, which [GetComments] extracts from // TextContent. Prefer it; TextContent is the unprocessed original. + // + // Text is all it holds: a comment is a rich document, and its images, + // emotes, mentions, and link targets are dropped in the flattening. Read + // TextContent for those. Comment string TextContent Text @@ -71,46 +76,105 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment cursor = cmmts.Cursor for i := 0; i < len(cmmts.Thread); i++ { - cmmts.Thread[i].Comment = flattenComment(cmmts.Thread[i].TextContent.Html.Markup) + cmmts.Thread[i].Comment = flattenMarkup(cmmts.Thread[i].TextContent.Html.Markup) } } return } -// flattenComment renders a body of user-written markup as plain text, be it a -// comment or a deviation's description. Bodies are JSON inside JSON: newer ones -// are a Draft.js document encoded into the markup string, older ones are plain -// HTML, which passes through unchanged. Markup that does not parse, and empty -// markup, also pass through. +// flattenMarkup renders a body of user-written markup as plain text, be it a +// comment or a deviation's description. Bodies are JSON inside JSON, and +// DeviantArt still serves all three formats it has used over the years: // -// A Draft.js document is a list of blocks, which are block-level elements -// (paragraphs, list items); they are joined with newlines, one block per line. -func flattenComment(m string) string { +// - tiptap, current: {"version":1,"document":{"type":"doc","content":[...]}} +// - Draft.js, legacy: {"blocks":[{"text":"..."}]} +// - plain HTML, oldest, which passes through unchanged +// +// Block-level elements (paragraphs, headings) are joined with newlines, one per +// line, and a hard break inside one becomes a newline too. HTML entities in the +// text are decoded, so a body reads as ’ on the wire but an apostrophe +// here. Markup matching no known format passes through unchanged rather than +// being replaced by an empty string. +func flattenMarkup(m string) string { l := len(m) if l == 0 || m[0] != '{' || m[l-1] != '}' { return m } + if text, ok := flattenTiptap(m); ok { + return text + } + if text, ok := flattenDraftJS(m); ok { + return text + } + return m +} + +// tiptapNode is one node of a tiptap (ProseMirror) document tree. +// +// The document's "version" field is deliberately not modelled: DeviantArt sends +// it as a number on some bodies and a string on others, so any typed field for +// it fails to unmarshal on half of them. +type tiptapNode struct { + Type string `json:"type"` + Text string `json:"text"` + Content []tiptapNode `json:"content"` +} + +// flattenTiptap renders a tiptap document, reporting false if the markup is not +// one. +func flattenTiptap(m string) (string, bool) { + var doc struct { + Document tiptapNode `json:"document"` + } + if json.Unmarshal([]byte(m), &doc) != nil || doc.Document.Type != "doc" { + return "", false + } + + lines := make([]string, 0, len(doc.Document.Content)) + for _, block := range doc.Document.Content { + var b strings.Builder + writeTiptapText(block, &b) + lines = append(lines, b.String()) + } + + return html.UnescapeString(strings.Join(lines, "\n")), true +} + +// writeTiptapText collects the text of a node and everything nested inside it. +// Nodes carrying no text of their own — images, galleries, emotes — contribute +// nothing. +func writeTiptapText(n tiptapNode, b *strings.Builder) { + switch n.Type { + case "text": + b.WriteString(n.Text) + return + case "hardBreak": + b.WriteString("\n") + return + } + for _, c := range n.Content { + writeTiptapText(c, b) + } +} + +// flattenDraftJS renders a legacy Draft.js document, reporting false if the +// markup is not one. +func flattenDraftJS(m string) (string, bool) { var content struct { Blocks []struct { Text string } } - - e := json.Unmarshal([]byte(m), &content) - try(e) - - if len(content.Blocks) == 0 { - return m + if json.Unmarshal([]byte(m), &content) != nil || len(content.Blocks) == 0 { + return "", false } - var b strings.Builder - for i, a := range content.Blocks { - if i > 0 { - b.WriteString("\n") - } - b.WriteString(a.Text) + lines := make([]string, 0, len(content.Blocks)) + for _, blk := range content.Blocks { + lines = append(lines, blk.Text) } - return b.String() + + return html.UnescapeString(strings.Join(lines, "\n")), true } diff --git a/comments_test.go b/comments_test.go index a6c5a47..60d38e4 100644 --- a/comments_test.go +++ b/comments_test.go @@ -2,50 +2,145 @@ package devianter import "testing" -// Regression: flattenComment's shape check used to read m[0] and m[len(m)-1] -// without a length check, so a comment with an empty markup body panicked with -// index out of range and killed the caller's process. -func TestFlattenCommentEmptyMarkup(t *testing.T) { - if got := flattenComment(""); got != "" { +// Regression: the shape check used to read m[0] and m[len(m)-1] without a length +// check, so a comment with an empty markup body panicked with index out of range +// and killed the caller's process. +func TestFlattenMarkupEmptyMarkup(t *testing.T) { + if got := flattenMarkup(""); got != "" { t.Errorf("want an empty comment for empty markup, got %q", got) } } -func TestFlattenComment(t *testing.T) { - // A newer, Draft.js-encoded body is flattened to its text. - draft := `{"blocks":[{"text":"hello there"}]}` - if got := flattenComment(draft); got != "hello there" { - t.Errorf("want the Draft.js block text, got %q", got) +// tiptap is what DeviantArt actually serves today; the fixtures below are shaped +// like responses captured from the live API. +func TestFlattenMarkupTiptap(t *testing.T) { + one := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","attrs":{"textAlign":"left"},"content":[` + + `{"type":"text","text":"Nice artwork!"}]}]}}` + if got := flattenMarkup(one); got != "Nice artwork!" { + t.Errorf("want the paragraph's text, got %q", got) } - // An older, plain-HTML body passes through untouched. - html := "hello there" - if got := flattenComment(html); got != html { - t.Errorf("want plain HTML passed through, got %q", got) + // Regression: DeviantArt sends "version" as a number on some bodies and a + // string on others. Modelling it as either type fails to unmarshal half of + // them, so it must not be modelled at all. + stringVersion := `{"version":"1","document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"hello"}]}]}}` + if got := flattenMarkup(stringVersion); got != "hello" { + t.Errorf(`want a string "version" handled the same as a numeric one, got %q`, got) + } + + // Each block is its own line. + two := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"first"}]},` + + `{"type":"paragraph","content":[{"type":"text","text":"second"}]}]}}` + if got, want := flattenMarkup(two), "first\nsecond"; got != want { + t.Errorf("want one line per block:\n got %q\nwant %q", got, want) + } + + // An empty paragraph is a blank line, not something to skip. + blank := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"a"}]},` + + `{"type":"paragraph"},` + + `{"type":"paragraph","content":[{"type":"text","text":"b"}]}]}}` + if got, want := flattenMarkup(blank), "a\n\nb"; got != want { + t.Errorf("want an empty paragraph preserved as a blank line:\n got %q\nwant %q", got, want) + } + + // A hard break is a newline within its block. + brk := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"up"},` + + `{"type":"hardBreak"},{"type":"text","text":"down"}]}]}}` + if got, want := flattenMarkup(brk), "up\ndown"; got != want { + t.Errorf("want a hardBreak as a newline:\n got %q\nwant %q", got, want) + } + + // Marked-up runs are separate text nodes and must be concatenated, not + // separated. + marks := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[` + + `{"type":"text","text":"plain "},` + + `{"type":"text","marks":[{"type":"bold"}],"text":"bold"},` + + `{"type":"text","text":" tail"}]}]}}` + if got, want := flattenMarkup(marks), "plain bold tail"; got != want { + t.Errorf("want marked runs concatenated:\n got %q\nwant %q", got, want) + } + + // Text carrying a link mark still surfaces; the href does not. + link := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","marks":[` + + `{"type":"link","attrs":{"href":"https://example.com"}}],"text":"click here"}]}]}}` + if got, want := flattenMarkup(link), "click here"; got != want { + t.Errorf("want the link text without the href:\n got %q\nwant %q", got, want) + } + + // Headings are blocks like any other. + heading := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"heading","attrs":{"level":2},"content":[{"type":"text","text":"Title"}]},` + + `{"type":"paragraph","content":[{"type":"text","text":"body"}]}]}}` + if got, want := flattenMarkup(heading), "Title\nbody"; got != want { + t.Errorf("want a heading flattened as a block:\n got %q\nwant %q", got, want) + } + + // Bodies carry HTML entities on the wire; plain text should not. + entity := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"I’d love & more"}]}]}}` + if got, want := flattenMarkup(entity), "I’d love & more"; got != want { + t.Errorf("want HTML entities decoded:\n got %q\nwant %q", got, want) + } + + // A node carrying no text of its own contributes nothing. + emote := `{"version":1,"document":{"type":"doc","content":[` + + `{"type":"paragraph","content":[{"type":"text","text":"hi "},` + + `{"type":"da-emote","attrs":{"name":":happy:"}}]}]}}` + if got, want := flattenMarkup(emote), "hi "; got != want { + t.Errorf("want a textless node to contribute nothing:\n got %q\nwant %q", got, want) + } +} + +// Draft.js is legacy but still served on old bodies, so the path stays. +func TestFlattenMarkupDraftJS(t *testing.T) { + draft := `{"blocks":[{"text":"hello there"}]}` + if got := flattenMarkup(draft); got != "hello there" { + t.Errorf("want the Draft.js block text, got %q", got) } // Regression: the block loop used to assign rather than accumulate, so every // block but the last was silently dropped and a multi-paragraph comment came // back as its closing line only. multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}` - if got, want := flattenComment(multi), "first\nsecond\nthird"; got != want { + if got, want := flattenMarkup(multi), "first\nsecond\nthird"; got != want { t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want) } - // An empty block is a blank line in the comment, not something to skip. + // An empty block is a blank line, not something to skip. blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}` - if got, want := flattenComment(blank), "first\n\nthird"; got != want { + if got, want := flattenMarkup(blank), "first\n\nthird"; got != want { t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want) } +} + +func TestFlattenMarkupPassthrough(t *testing.T) { + // An older, plain-HTML body passes through untouched. + html := "hello there" + if got := flattenMarkup(html); got != html { + t.Errorf("want plain HTML passed through, got %q", got) + } + + // Brace-shaped markup in no known format falls back to itself rather than to + // an empty string. + if got := flattenMarkup("{}"); got != "{}" { + t.Errorf("want the original markup when nothing parses, got %q", got) + } - // Brace-shaped markup that isn't a Draft.js document falls back to itself - // rather than to an empty string. - if got := flattenComment("{}"); got != "{}" { - t.Errorf("want the original markup when there are no blocks, got %q", got) + // Well-formed JSON that is neither format is still not silently eaten. + other := `{"something":"else"}` + if got := flattenMarkup(other); got != other { + t.Errorf("want unrecognised JSON passed through, got %q", got) } // A single brace satisfies neither end of the shape check. - if got := flattenComment("{"); got != "{" { + if got := flattenMarkup("{"); got != "{" { t.Errorf("want a lone brace passed through, got %q", got) } } diff --git a/deviantion.go b/deviantion.go index 5b6df39..f3f9103 100644 --- a/deviantion.go +++ b/deviantion.go @@ -79,9 +79,14 @@ type Media struct { } // Text is a block of user-written text — a description, a comment, a group's -// about page. Markup holds either HTML or a JSON-encoded Draft.js document, -// distinguished by Type; the functions that return a Text generally extract the -// plain text into a neighbouring field, which is easier to use. +// about page. +// +// Markup is a rich document rather than a string of prose, in whichever format +// DeviantArt stored it: tiptap JSON on anything recent (Type is "tiptap"), +// Draft.js JSON on older bodies, or plain HTML on the oldest. The functions +// returning a Text generally flatten it to plain text in a neighbouring field, +// which is what most callers want; read Markup itself for the formatting, +// images, and links that flattening discards. type Text struct { Excerpt string Html struct { @@ -91,8 +96,9 @@ type Text struct { // Post is a deviation together with its comment metadata, as returned by // [GetDeviation]. IMG and Description are conveniences that GetDeviation derives -// from the Deviation, so callers need not assemble a URL or decode Draft.js -// markup themselves. +// from the Deviation, so callers need not assemble a URL or flatten a rich-text +// document themselves. Description is empty for the many deviations that have +// none. // // Comments holds only a total and a cursor. To retrieve the comments, pass them // to [GetComments] with type 1. @@ -175,7 +181,14 @@ func GetDeviation(id string, user string) (st Post, err Error) { st.IMG, _ = UrlFromMedia(st.Deviation.Media) - st.Description = flattenComment(st.Deviation.TextContent.Html.Markup) + // The description lives in Extended.DescriptionText on the great majority of + // deviations; TextContent carries it on only a small minority. Prefer the + // former and fall back, since either may be the populated one. + desc := st.Deviation.Extended.DescriptionText.Html.Markup + if desc == "" { + desc = st.Deviation.TextContent.Html.Markup + } + st.Description = flattenMarkup(desc) return }