Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions SCHEMA.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,30 @@ error:
- `whoami` returns `data.user`
- write commands also support explicit `--json` / `--yaml`

## Sensitive Media Fields

Tweet objects in `--json` / `--yaml` include Twitter's sensitive-media labels so
frontends can skip inlining adult or graphic photos:

```yaml
data:
- id: "1234567890"
possiblySensitive: true
media:
- type: photo
url: https://pbs.twimg.com/media/example.jpg
width: 1200
height: 800
adultContent: true
graphicViolence: false
otherWarning: false
```

- `possiblySensitive` is the tweet-level `legacy.possibly_sensitive` flag
- media warning fields come from `ext_sensitive_media_warning` /
`sensitive_media_warning` on the media entity (false when absent)
- quoted tweets include `possiblySensitive` on `quotedTweet`

## Article Fields

`twitter article <id> --json` returns the standard tweet object plus:
Expand Down
1 change: 1 addition & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ def _make_tweet(tweet_id: str = "1", **overrides: Any) -> Tweet:
article_text=overrides.pop("article_text", None),
is_subscriber_only=overrides.pop("is_subscriber_only", False),
is_promoted=overrides.pop("is_promoted", False),
possibly_sensitive=overrides.pop("possibly_sensitive", False),
)

return _make_tweet
Expand Down
49 changes: 48 additions & 1 deletion tests/test_parser_fixtures.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from __future__ import annotations

from twitter_cli.client import TwitterClient
from twitter_cli.parser import _deep_get, parse_timeline_response
from twitter_cli.parser import _deep_get, _extract_media, parse_timeline_response


def _make_client() -> TwitterClient:
Expand All @@ -26,6 +26,8 @@ def test_parse_home_timeline_fixture(fixture_loader) -> None:
assert [tweet.id for tweet in tweets] == ["1", "20"]
assert cursor == "cursor-bottom-1"
assert tweets[0].media[0].type == "photo"
assert tweets[0].possibly_sensitive is False
assert tweets[0].media[0].adult_content is False
# note_tweet full text should be preferred over legacy.full_text for long tweets
assert "Show More" in tweets[0].text
assert tweets[0].text.startswith("Hello\nworld\n")
Expand All @@ -36,6 +38,51 @@ def test_parse_home_timeline_fixture(fixture_loader) -> None:
assert tweets[1].quoted_tweet.id == "30"


def test_parse_home_timeline_fixture_sensitive_media(fixture_loader) -> None:
payload = fixture_loader("home_timeline.json")
result = payload["data"]["home"]["home_timeline_urt"]["instructions"][0]["entries"][0]
legacy = result["content"]["itemContent"]["tweet_results"]["result"]["legacy"]
legacy["possibly_sensitive"] = True
legacy["extended_entities"]["media"][0]["ext_sensitive_media_warning"] = {
"adult_content": True,
"graphic_violence": False,
"other": False,
}

tweets, _ = parse_timeline_response(
payload,
lambda data: _deep_get(data, "data", "home", "home_timeline_urt", "instructions"),
)

assert tweets[0].possibly_sensitive is True
assert tweets[0].media[0].adult_content is True
assert tweets[0].media[0].graphic_violence is False
assert tweets[1].possibly_sensitive is False


def test_extract_media_reads_nested_sensitive_warning() -> None:
legacy = {
"extended_entities": {
"media": [
{
"type": "photo",
"media_url_https": "https://pbs.twimg.com/media/fake.jpg",
"original_info": {"width": 10, "height": 10},
"media_results": {
"result": {
"sensitive_media_warning": {"graphic_violence": True}
}
},
}
]
}
}
media = _extract_media(legacy)
assert len(media) == 1
assert media[0].graphic_violence is True
assert media[0].adult_content is False


def test_parse_home_timeline_fixture_marks_promoted_entries(fixture_loader) -> None:
payload = fixture_loader("home_timeline.json")
entry = payload["data"]["home"]["home_timeline_urt"]["instructions"][0]["entries"][0]
Expand Down
37 changes: 37 additions & 0 deletions tests/test_serialization.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

from twitter_cli.models import TweetMedia
from twitter_cli.serialization import tweet_from_dict, tweet_to_dict, tweets_from_json, tweets_to_json


Expand All @@ -11,6 +12,8 @@ def test_tweet_roundtrip_dict(tweet_factory) -> None:
assert restored.id == tweet.id
assert restored.author.screen_name == tweet.author.screen_name
assert restored.metrics.likes == tweet.metrics.likes
assert payload["possiblySensitive"] is False
assert restored.possibly_sensitive is False


def test_tweets_json_roundtrip(tweet_factory) -> None:
Expand Down Expand Up @@ -93,3 +96,37 @@ def test_tweet_roundtrip_preserves_promoted_flag(tweet_factory) -> None:
assert payload["isPromoted"] is True
restored = tweet_from_dict(payload)
assert restored.is_promoted is True


def test_tweet_roundtrip_preserves_sensitive_media(tweet_factory) -> None:
quoted = tweet_factory("101", possibly_sensitive=True)
tweet = tweet_factory(
"100",
possibly_sensitive=True,
quoted_tweet=quoted,
media=[
TweetMedia(
type="photo",
url="https://pbs.twimg.com/media/example.jpg",
width=10,
height=10,
adult_content=True,
graphic_violence=True,
other_warning=False,
)
],
)
payload = tweet_to_dict(tweet)
assert payload["possiblySensitive"] is True
assert payload["quotedTweet"]["possiblySensitive"] is True
assert payload["media"][0]["adultContent"] is True
assert payload["media"][0]["graphicViolence"] is True
assert payload["media"][0]["otherWarning"] is False

restored = tweet_from_dict(payload)
assert restored.possibly_sensitive is True
assert restored.quoted_tweet is not None
assert restored.quoted_tweet.possibly_sensitive is True
assert restored.media[0].adult_content is True
assert restored.media[0].graphic_violence is True
assert restored.media[0].other_warning is False
4 changes: 4 additions & 0 deletions twitter_cli/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ class TweetMedia:
url: str
width: Optional[int] = None
height: Optional[int] = None
adult_content: bool = False
graphic_violence: bool = False
other_warning: bool = False


@dataclass
Expand All @@ -54,6 +57,7 @@ class Tweet:
article_text: Optional[str] = None
is_subscriber_only: bool = False
is_promoted: bool = False
possibly_sensitive: bool = False


@dataclass
Expand Down
80 changes: 67 additions & 13 deletions twitter_cli/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,64 @@ def _extract_cursor(content):
# ── Media / Author extraction ────────────────────────────────────────────


def _truthy(value):
# type: (Any) -> bool
return value is True or value == 1 or value in ("true", "True")


def _media_warning(item):
# type: (Any) -> Dict[str, bool]
"""Read adult/graphic/other labels from a GraphQL media entity."""
out = {"adult_content": False, "graphic_violence": False, "other": False}
if not isinstance(item, dict):
return out

sources = [
item.get("ext_sensitive_media_warning"),
item.get("sensitive_media_warning"),
item.get("extSensitiveMediaWarning"),
item.get("sensitiveMediaWarning"),
]
media_results = item.get("media_results") or item.get("mediaResults")
if isinstance(media_results, dict):
result = media_results.get("result") or media_results
if isinstance(result, dict):
sources.extend(
[
result.get("ext_sensitive_media_warning"),
result.get("sensitive_media_warning"),
result.get("extSensitiveMediaWarning"),
result.get("sensitiveMediaWarning"),
]
)
for src in sources:
if not isinstance(src, dict):
continue
for key, value in src.items():
lk = str(key).replace("-", "_").lower()
if lk in ("adult_content", "adultcontent") and _truthy(value):
out["adult_content"] = True
elif lk in ("graphic_violence", "graphicviolence") and _truthy(value):
out["graphic_violence"] = True
elif lk in ("other", "other_warning", "otherwarning") and _truthy(value):
out["other"] = True
return out


def _tweet_media(media_type, url, media_item):
# type: (str, str, Dict[str, Any]) -> TweetMedia
warning = _media_warning(media_item)
return TweetMedia(
type=media_type,
url=url,
width=_deep_get(media_item, "original_info", "width"),
height=_deep_get(media_item, "original_info", "height"),
adult_content=warning["adult_content"],
graphic_violence=warning["graphic_violence"],
other_warning=warning["other"],
)


def _extract_media(legacy):
# type: (Dict[str, Any]) -> List[TweetMedia]
"""Extract media items from tweet legacy data."""
Expand All @@ -68,25 +126,18 @@ def _extract_media(legacy):
media_type = media_item.get("type", "")
if media_type == "photo":
media.append(
TweetMedia(
type="photo",
url=media_item.get("media_url_https", ""),
width=_deep_get(media_item, "original_info", "width"),
height=_deep_get(media_item, "original_info", "height"),
)
_tweet_media("photo", media_item.get("media_url_https", ""), media_item)
)
elif media_type in {"video", "animated_gif"}:
variants = media_item.get("video_info", {}).get("variants", [])
mp4_variants = [v for v in variants if v.get("content_type") == "video/mp4"]
mp4_variants.sort(key=lambda v: v.get("bitrate", 0), reverse=True)
media.append(
TweetMedia(
type=media_type,
url=mp4_variants[0]["url"] if mp4_variants else media_item.get("media_url_https", ""),
width=_deep_get(media_item, "original_info", "width"),
height=_deep_get(media_item, "original_info", "height"),
)
url = (
mp4_variants[0]["url"]
if mp4_variants
else media_item.get("media_url_https", "")
)
media.append(_tweet_media(media_type, url, media_item))
return media


Expand Down Expand Up @@ -484,6 +535,9 @@ def parse_tweet_result(result, depth=0):
quoted_tweet=quoted_tweet,
lang=actual_legacy.get("lang", ""),
is_subscriber_only=(is_subscriber_only or retweet_subscriber_only) if is_retweet else is_subscriber_only,
possibly_sensitive=bool(
actual_legacy.get("possibly_sensitive") or actual_legacy.get("possiblySensitive")
),
**_parse_article(actual_data),
)

Expand Down
30 changes: 21 additions & 9 deletions twitter_cli/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,15 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]:
"createdAt": tweet.created_at,
"createdAtLocal": format_local_time(tweet.created_at),
"createdAtISO": format_iso8601(tweet.created_at),
"media": [
{
"type": media.type,
"url": media.url,
"width": media.width,
"height": media.height,
}
for media in tweet.media
],
"media": [_media_to_dict(media) for media in tweet.media],
"urls": list(tweet.urls),
"isRetweet": tweet.is_retweet,
"retweetedBy": tweet.retweeted_by,
"lang": tweet.lang,
"score": tweet.score,
"isSubscriberOnly": tweet.is_subscriber_only,
"isPromoted": tweet.is_promoted,
"possiblySensitive": tweet.possibly_sensitive,
}
if tweet.article_title is not None:
data["articleTitle"] = tweet.article_title
Expand All @@ -61,10 +54,24 @@ def tweet_to_dict(tweet: Tweet) -> Dict[str, Any]:
"screenName": tweet.quoted_tweet.author.screen_name,
"name": tweet.quoted_tweet.author.name,
},
"possiblySensitive": tweet.quoted_tweet.possibly_sensitive,
}
return data


def _media_to_dict(media: TweetMedia) -> Dict[str, Any]:
"""Convert a TweetMedia dataclass into a JSON-safe dict."""
return {
"type": media.type,
"url": media.url,
"width": media.width,
"height": media.height,
"adultContent": media.adult_content,
"graphicViolence": media.graphic_violence,
"otherWarning": media.other_warning,
}


def tweet_from_dict(data: Dict[str, Any]) -> Tweet:
"""Convert a dict into a Tweet dataclass."""
author_data = data.get("author") or {}
Expand All @@ -85,6 +92,7 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet:
),
metrics=Metrics(),
created_at="",
possibly_sensitive=bool(quoted_data.get("possiblySensitive", False)),
)

return Tweet(
Expand Down Expand Up @@ -112,6 +120,9 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet:
url=str(item.get("url") or ""),
width=_optional_int(item.get("width")),
height=_optional_int(item.get("height")),
adult_content=bool(item.get("adultContent", False)),
graphic_violence=bool(item.get("graphicViolence", False)),
other_warning=bool(item.get("otherWarning", False)),
)
for item in media_data
if isinstance(item, dict)
Expand All @@ -126,6 +137,7 @@ def tweet_from_dict(data: Dict[str, Any]) -> Tweet:
article_text=_optional_str(data.get("articleText")),
is_subscriber_only=bool(data.get("isSubscriberOnly", False)),
is_promoted=bool(data.get("isPromoted", False)),
possibly_sensitive=bool(data.get("possiblySensitive", False)),
)


Expand Down