diff --git a/SCHEMA.md b/SCHEMA.md index 2b8a64c..053e28c 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -33,6 +33,30 @@ error: - `whoami` returns `data.user` - write commands also support explicit `--json` / `--yaml` +## Sensitive Media Fields + +Tweet objects in `--json` / `--yaml` include Twitter's sensitive-media labels so +frontends can skip inlining adult or graphic photos: + +```yaml +data: + - id: "1234567890" + possiblySensitive: true + media: + - type: photo + url: https://pbs.twimg.com/media/example.jpg + width: 1200 + height: 800 + adultContent: true + graphicViolence: false + otherWarning: false +``` + +- `possiblySensitive` is the tweet-level `legacy.possibly_sensitive` flag +- media warning fields come from `ext_sensitive_media_warning` / + `sensitive_media_warning` on the media entity (false when absent) +- quoted tweets include `possiblySensitive` on `quotedTweet` + ## Article Fields `twitter article --json` returns the standard tweet object plus: diff --git a/tests/conftest.py b/tests/conftest.py index d435289..008f660 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,6 +40,7 @@ def _make_tweet(tweet_id: str = "1", **overrides: Any) -> Tweet: article_text=overrides.pop("article_text", None), is_subscriber_only=overrides.pop("is_subscriber_only", False), is_promoted=overrides.pop("is_promoted", False), + possibly_sensitive=overrides.pop("possibly_sensitive", False), ) return _make_tweet diff --git a/tests/test_parser_fixtures.py b/tests/test_parser_fixtures.py index 0e37183..04f068d 100644 --- a/tests/test_parser_fixtures.py +++ b/tests/test_parser_fixtures.py @@ -1,7 +1,7 @@ from __future__ import annotations from twitter_cli.client import TwitterClient -from twitter_cli.parser import _deep_get, parse_timeline_response +from twitter_cli.parser import _deep_get, _extract_media, parse_timeline_response def _make_client() -> TwitterClient: @@ -26,6 +26,8 @@ def test_parse_home_timeline_fixture(fixture_loader) -> None: assert [tweet.id for tweet in tweets] == ["1", "20"] assert cursor == "cursor-bottom-1" assert tweets[0].media[0].type == "photo" + assert tweets[0].possibly_sensitive is False + assert tweets[0].media[0].adult_content is False # note_tweet full text should be preferred over legacy.full_text for long tweets assert "Show More" in tweets[0].text assert tweets[0].text.startswith("Hello\nworld\n") @@ -36,6 +38,51 @@ def test_parse_home_timeline_fixture(fixture_loader) -> None: assert tweets[1].quoted_tweet.id == "30" +def test_parse_home_timeline_fixture_sensitive_media(fixture_loader) -> None: + payload = fixture_loader("home_timeline.json") + result = payload["data"]["home"]["home_timeline_urt"]["instructions"][0]["entries"][0] + legacy = result["content"]["itemContent"]["tweet_results"]["result"]["legacy"] + legacy["possibly_sensitive"] = True + legacy["extended_entities"]["media"][0]["ext_sensitive_media_warning"] = { + "adult_content": True, + "graphic_violence": False, + "other": False, + } + + tweets, _ = parse_timeline_response( + payload, + lambda data: _deep_get(data, "data", "home", "home_timeline_urt", "instructions"), + ) + + assert tweets[0].possibly_sensitive is True + assert tweets[0].media[0].adult_content is True + assert tweets[0].media[0].graphic_violence is False + assert tweets[1].possibly_sensitive is False + + +def test_extract_media_reads_nested_sensitive_warning() -> None: + legacy = { + "extended_entities": { + "media": [ + { + "type": "photo", + "media_url_https": "https://pbs.twimg.com/media/fake.jpg", + "original_info": {"width": 10, "height": 10}, + "media_results": { + "result": { + "sensitive_media_warning": {"graphic_violence": True} + } + }, + } + ] + } + } + media = _extract_media(legacy) + assert len(media) == 1 + assert media[0].graphic_violence is True + assert media[0].adult_content is False + + def test_parse_home_timeline_fixture_marks_promoted_entries(fixture_loader) -> None: payload = fixture_loader("home_timeline.json") entry = payload["data"]["home"]["home_timeline_urt"]["instructions"][0]["entries"][0] diff --git a/tests/test_serialization.py b/tests/test_serialization.py index cd11545..15e2431 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,5 +1,6 @@ from __future__ import annotations +from twitter_cli.models import TweetMedia from twitter_cli.serialization import tweet_from_dict, tweet_to_dict, tweets_from_json, tweets_to_json @@ -11,6 +12,8 @@ def test_tweet_roundtrip_dict(tweet_factory) -> None: assert restored.id == tweet.id assert restored.author.screen_name == tweet.author.screen_name assert restored.metrics.likes == tweet.metrics.likes + assert payload["possiblySensitive"] is False + assert restored.possibly_sensitive is False def test_tweets_json_roundtrip(tweet_factory) -> None: @@ -93,3 +96,37 @@ def test_tweet_roundtrip_preserves_promoted_flag(tweet_factory) -> None: assert payload["isPromoted"] is True restored = tweet_from_dict(payload) assert restored.is_promoted is True + + +def test_tweet_roundtrip_preserves_sensitive_media(tweet_factory) -> None: + quoted = tweet_factory("101", possibly_sensitive=True) + tweet = tweet_factory( + "100", + possibly_sensitive=True, + quoted_tweet=quoted, + media=[ + TweetMedia( + type="photo", + url="https://pbs.twimg.com/media/example.jpg", + width=10, + height=10, + adult_content=True, + graphic_violence=True, + other_warning=False, + ) + ], + ) + payload = tweet_to_dict(tweet) + assert payload["possiblySensitive"] is True + assert payload["quotedTweet"]["possiblySensitive"] is True + assert payload["media"][0]["adultContent"] is True + assert payload["media"][0]["graphicViolence"] is True + assert payload["media"][0]["otherWarning"] is False + + restored = tweet_from_dict(payload) + assert restored.possibly_sensitive is True + assert restored.quoted_tweet is not None + assert restored.quoted_tweet.possibly_sensitive is True + assert restored.media[0].adult_content is True + assert restored.media[0].graphic_violence is True + assert restored.media[0].other_warning is False diff --git a/twitter_cli/models.py b/twitter_cli/models.py index ee1d314..cf977d8 100644 --- a/twitter_cli/models.py +++ b/twitter_cli/models.py @@ -34,6 +34,9 @@ class TweetMedia: url: str width: Optional[int] = None height: Optional[int] = None + adult_content: bool = False + graphic_violence: bool = False + other_warning: bool = False @dataclass @@ -54,6 +57,7 @@ class Tweet: article_text: Optional[str] = None is_subscriber_only: bool = False is_promoted: bool = False + possibly_sensitive: bool = False @dataclass diff --git a/twitter_cli/parser.py b/twitter_cli/parser.py index 58378be..6186bf9 100644 --- a/twitter_cli/parser.py +++ b/twitter_cli/parser.py @@ -60,6 +60,64 @@ def _extract_cursor(content): # ── Media / Author extraction ──────────────────────────────────────────── +def _truthy(value): + # type: (Any) -> bool + return value is True or value == 1 or value in ("true", "True") + + +def _media_warning(item): + # type: (Any) -> Dict[str, bool] + """Read adult/graphic/other labels from a GraphQL media entity.""" + out = {"adult_content": False, "graphic_violence": False, "other": False} + if not isinstance(item, dict): + return out + + sources = [ + item.get("ext_sensitive_media_warning"), + item.get("sensitive_media_warning"), + item.get("extSensitiveMediaWarning"), + item.get("sensitiveMediaWarning"), + ] + media_results = item.get("media_results") or item.get("mediaResults") + if isinstance(media_results, dict): + result = media_results.get("result") or media_results + if isinstance(result, dict): + sources.extend( + [ + result.get("ext_sensitive_media_warning"), + result.get("sensitive_media_warning"), + result.get("extSensitiveMediaWarning"), + result.get("sensitiveMediaWarning"), + ] + ) + for src in sources: + if not isinstance(src, dict): + continue + for key, value in src.items(): + lk = str(key).replace("-", "_").lower() + if lk in ("adult_content", "adultcontent") and _truthy(value): + out["adult_content"] = True + elif lk in ("graphic_violence", "graphicviolence") and _truthy(value): + out["graphic_violence"] = True + elif lk in ("other", "other_warning", "otherwarning") and _truthy(value): + out["other"] = True + return out + + +def _tweet_media(media_type, url, media_item): + # type: (str, str, Dict[str, Any]) -> TweetMedia + warning = _media_warning(media_item) + return TweetMedia( + type=media_type, + url=url, + width=_deep_get(media_item, "original_info", "width"), + height=_deep_get(media_item, "original_info", "height"), + adult_content=warning["adult_content"], + graphic_violence=warning["graphic_violence"], + other_warning=warning["other"], + ) + + def _extract_media(legacy): # type: (Dict[str, Any]) -> List[TweetMedia] """Extract media items from tweet legacy data.""" @@ -68,25 +126,18 @@ def _extract_media(legacy): media_type = media_item.get("type", "") if media_type == "photo": media.append( - TweetMedia( - type="photo", - url=media_item.get("media_url_https", ""), - width=_deep_get(media_item, "original_info", "width"), - height=_deep_get(media_item, "original_info", "height"), - ) + _tweet_media("photo", media_item.get("media_url_https", ""), media_item) ) elif media_type in {"video", "animated_gif"}: variants = media_item.get("video_info", {}).get("variants", []) mp4_variants = [v for v in variants if v.get("content_type") == "video/mp4"] mp4_variants.sort(key=lambda v: v.get("bitrate", 0), reverse=True) - media.append( - TweetMedia( - type=media_type, - url=mp4_variants[0]["url"] if mp4_variants else media_item.get("media_url_https", ""), - width=_deep_get(media_item, "original_info", "width"), - height=_deep_get(media_item, "original_info", "height"), - ) + url = ( + mp4_variants[0]["url"] + if mp4_variants + else media_item.get("media_url_https", "") ) + media.append(_tweet_media(media_type, url, media_item)) return media @@ -484,6 +535,9 @@ def parse_tweet_result(result, depth=0): quoted_tweet=quoted_tweet, lang=actual_legacy.get("lang", ""), is_subscriber_only=(is_subscriber_only or retweet_subscriber_only) if is_retweet else is_subscriber_only, + possibly_sensitive=bool( + actual_legacy.get("possibly_sensitive") or actual_legacy.get("possiblySensitive") + ), **_parse_article(actual_data), ) diff --git a/twitter_cli/serialization.py b/twitter_cli/serialization.py index 25ceede..43d04fe 100644 --- a/twitter_cli/serialization.py +++ b/twitter_cli/serialization.py @@ -32,15 +32,7 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]: "createdAt": tweet.created_at, "createdAtLocal": format_local_time(tweet.created_at), "createdAtISO": format_iso8601(tweet.created_at), - "media": [ - { - "type": media.type, - "url": media.url, - "width": media.width, - "height": media.height, - } - for media in tweet.media - ], + "media": [_media_to_dict(media) for media in tweet.media], "urls": list(tweet.urls), "isRetweet": tweet.is_retweet, "retweetedBy": tweet.retweeted_by, @@ -48,6 +40,7 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]: "score": tweet.score, "isSubscriberOnly": tweet.is_subscriber_only, "isPromoted": tweet.is_promoted, + "possiblySensitive": tweet.possibly_sensitive, } if tweet.article_title is not None: data["articleTitle"] = tweet.article_title @@ -61,10 +54,24 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]: "screenName": tweet.quoted_tweet.author.screen_name, "name": tweet.quoted_tweet.author.name, }, + "possiblySensitive": tweet.quoted_tweet.possibly_sensitive, } return data +def _media_to_dict(media: TweetMedia) -> Dict[str, Any]: + """Convert a TweetMedia dataclass into a JSON-safe dict.""" + return { + "type": media.type, + "url": media.url, + "width": media.width, + "height": media.height, + "adultContent": media.adult_content, + "graphicViolence": media.graphic_violence, + "otherWarning": media.other_warning, + } + + def tweet_from_dict(data: Dict[str, Any]) -> Tweet: """Convert a dict into a Tweet dataclass.""" author_data = data.get("author") or {} @@ -85,6 +92,7 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet: ), metrics=Metrics(), created_at="", + possibly_sensitive=bool(quoted_data.get("possiblySensitive", False)), ) return Tweet( @@ -112,6 +120,9 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet: url=str(item.get("url") or ""), width=_optional_int(item.get("width")), height=_optional_int(item.get("height")), + adult_content=bool(item.get("adultContent", False)), + graphic_violence=bool(item.get("graphicViolence", False)), + other_warning=bool(item.get("otherWarning", False)), ) for item in media_data if isinstance(item, dict) @@ -126,6 +137,7 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet: article_text=_optional_str(data.get("articleText")), is_subscriber_only=bool(data.get("isSubscriberOnly", False)), is_promoted=bool(data.get("isPromoted", False)), + possibly_sensitive=bool(data.get("possiblySensitive", False)), )