Skip to content

fix(gmail): make void inbox mirror Gmail's INBOX label #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MaximeGaudin wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(gmail): make void inbox mirror Gmail's INBOX label #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ void gmail draft create --reply-to <id> --subject "Re: Q3" --body "LGTM, approve
void gmail batch-modify <id1> <id2> --remove INBOX
```

`void inbox` is the Inbox Zero surface for Gmail's `INBOX` label: after every sync it reconciles its archive state with Gmail, so mail you archive in the Gmail web UI disappears from `void inbox` (and vice versa). It lists one item per **thread** — the thread's latest unarchived message — while `void gmail search 'in:inbox'` lists individual **messages**; archiving the shown item removes that message's `INBOX` label and the thread's next message surfaces until the thread is fully out of Gmail's inbox.

### Calendar

Docs: [commands](docs/commands.md#calendar)
Expand Down
30 changes: 18 additions & 12 deletions crates/void-gmail/src/api/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,15 @@ impl GmailApiClient {
if let Some(q) = query {
params.push(("q", q.to_string()));
}
let resp: MessageListResponse = self
let resp = self
.http
.get(format!("{}/gmail/v1/users/me/messages", self.base_url))
.bearer_auth(&self.access_token)
.query(&params)
.send()
.await?
.json()
.await?;
.error_for_status()?;
let resp: MessageListResponse = resp.json().await?;
let count = resp.messages.as_ref().map(|m| m.len()).unwrap_or(0);
debug!(
message_count = count,
Expand All @@ -109,7 +109,7 @@ impl GmailApiClient {

pub async fn get_message(&self, message_id: &str) -> Result<GmailMessage, GmailError> {
debug!(message_id, "gmail: get_message");
let resp: GmailMessage = self
let resp = self
.http
.get(format!(
"{}/gmail/v1/users/me/messages/{message_id}",
Expand All @@ -119,8 +119,8 @@ impl GmailApiClient {
.query(&[("format", "full")])
.send()
.await?
.json()
.await?;
.error_for_status()?;
let resp: GmailMessage = resp.json().await?;
Ok(resp)
}

Expand All @@ -143,23 +143,29 @@ impl GmailApiClient {
if let Some(pt) = &page_token {
params.push(("pageToken", pt.clone()));
}
let resp: HistoryListResponse = self
let resp = self
.http
.get(format!("{}/gmail/v1/users/me/history", self.base_url))
.bearer_auth(&self.access_token)
.query(&params)
.send()
.await?
.json()
.await?;
// Gmail returns 404 once the startHistoryId is too old (history is
// only kept for a limited window). Surface that distinctly so the
// sync loop can fall back to a full INBOX refresh.
if resp.status() == reqwest::StatusCode::NOT_FOUND {
return Err(GmailError::HistoryExpired);
}
let resp = resp.error_for_status()?;
let page_resp: HistoryListResponse = resp.json().await?;

if let Some(records) = resp.history {
if let Some(records) = page_resp.history {
let count = records.len();
all_records.extend(records);
debug!(page, record_count = count, "gmail: listed history page");
}
latest_history_id = resp.history_id.or(latest_history_id);
page_token = resp.next_page_token;
latest_history_id = page_resp.history_id.or(latest_history_id);
page_token = page_resp.next_page_token;
if page_token.is_none() {
break;
}
Expand Down
125 changes: 72 additions & 53 deletions crates/void-gmail/src/connector/sync.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,17 @@ use tracing::{debug, info, warn};
use void_core::db::Database;
use void_core::models::{Conversation, ConversationKind, Message};

use crate::api::GmailMessage;
use crate::api::{GmailApiClient, GmailMessage};
use crate::error::GmailError;

use super::compose::{html_to_markdown, looks_like_html, parse_email_address, parse_email_name};
use super::GmailConnector;

/// Page size for INBOX listings (Gmail API allows up to 500).
const INBOX_PAGE_SIZE: u32 = 500;
/// Safety cap on INBOX listing pages (20 × 500 = 10 000 messages).
const INBOX_MAX_PAGES: u32 = 20;

impl GmailConnector {
pub(crate) async fn initial_sync(&self, db: &Database) -> anyhow::Result<()> {
let api = self.get_client().await?;
Expand All @@ -23,7 +29,7 @@ impl GmailConnector {

if had_history {
debug!(config_id = %self.config_id, "history_id exists, refreshing inbox state");
self.refresh_inbox(db).await?;
self.refresh_inbox_with_api(db, &api).await?;
return Ok(());
}

Expand All @@ -33,71 +39,42 @@ impl GmailConnector {

info!(config_id = %self.config_id, "starting Gmail initial sync");

let mut page_token: Option<String> = None;
let max_pages: u64 = 5;

let mut progress = void_core::progress::BackfillProgress::new(
&format!("gmail:{}", self.config_id),
"messages",
);
progress.set_pages(max_pages);

loop {
let resp = api
.list_messages(
100,
page_token.as_deref(),
Some(&["INBOX"]),
Some("newer_than:7d"),
)
.await?;
progress.inc_page();

if let Some(msgs) = resp.messages {
for msg_ref in &msgs {
match api.get_message(&msg_ref.id).await {
Ok(msg) => {
self.store_message(db, &msg)?;
progress.inc(1);
}
Err(e) => {
warn!(message_id = %msg_ref.id, "failed to fetch message: {e}");
}
}
}
}

page_token = resp.next_page_token;
if page_token.is_none() || progress.pages_done >= max_pages {
break;
}
}
self.refresh_inbox_with_api(db, &api).await?;

progress.finish();
info!(config_id = %self.config_id, messages = progress.items, "Gmail initial sync complete");
info!(config_id = %self.config_id, "Gmail initial sync complete");
Ok(())
}

/// Refresh inbox state: fetch current INBOX message IDs from Gmail and
/// reconcile `is_archived` in the local DB so it mirrors Gmail exactly.
/// Also fetches any new INBOX messages not yet in the local DB.
/// Refresh inbox state: fetch the *complete* INBOX message ID list from
/// Gmail (no date filter) and reconcile `is_archived` in the local DB so
/// it mirrors Gmail exactly. Also fetches full bodies for any INBOX
/// messages not yet in the local DB, so `void inbox` matches
/// `gmail search 'in:inbox'`.
pub(crate) async fn refresh_inbox(&self, db: &Database) -> anyhow::Result<()> {
let api = self.get_client().await?;
self.refresh_inbox_with_api(db, &api).await
}

pub(crate) async fn refresh_inbox_with_api(
&self,
db: &Database,
api: &GmailApiClient,
) -> anyhow::Result<()> {
let connection_id = self.display_connection_id();

let mut inbox_ids: HashSet<String> = HashSet::new();
let mut new_msg_ids: Vec<String> = Vec::new();
let mut page_token: Option<String> = None;
let max_pages = 5u32;
let mut truncated = false;
let mut pages = 0u32;

loop {
let resp = api
.list_messages(
100,
INBOX_PAGE_SIZE,
page_token.as_deref(),
Some(&["INBOX"]),
Some("newer_than:7d"),
None,
)
.await?;
pages += 1;
Expand All @@ -112,7 +89,16 @@ impl GmailConnector {
}

page_token = resp.next_page_token;
if page_token.is_none() || pages >= max_pages {
if page_token.is_none() {
break;
}
if pages >= INBOX_MAX_PAGES {
warn!(
config_id = %self.config_id,
listed = inbox_ids.len(),
"Gmail INBOX listing hit page cap; older INBOX messages may be missed"
);
truncated = true;
break;
}
}
Expand All @@ -128,7 +114,13 @@ impl GmailConnector {
}
}

let (unarchived, archived) = db.reconcile_inbox(&connection_id, "gmail", &inbox_ids)?;
// Reconcile only when the INBOX listing is complete: a partial listing
// would wrongly archive messages that Gmail still keeps in INBOX.
let (unarchived, archived) = if truncated {
(0, 0)
} else {
db.reconcile_inbox(&connection_id, "gmail", &inbox_ids)?
};

if unarchived > 0 || archived > 0 || !new_msg_ids.is_empty() {
info!(
Expand All @@ -144,14 +136,41 @@ impl GmailConnector {
}

pub(crate) async fn incremental_sync(&self, db: &Database) -> anyhow::Result<()> {
let api = self.get_client().await?;
self.incremental_sync_with_api(db, &api).await
}

pub(crate) async fn incremental_sync_with_api(
&self,
db: &Database,
api: &GmailApiClient,
) -> anyhow::Result<()> {
let Some(history_id) = db.get_sync_state(&self.config_id, "history_id")? else {
debug!("no history_id, skipping incremental sync");
return Ok(());
};

let api = self.get_client().await?;
let connection_id = self.display_connection_id();
let resp = api.list_history(&history_id, Some("INBOX")).await?;
let resp = match api.list_history(&history_id, Some("INBOX")).await {
Ok(resp) => resp,
Err(GmailError::HistoryExpired) => {
// historyId is only valid for a limited window (e.g. daemon
// offline for a while). Reconcile against the full INBOX
// listing so local archive state mirrors Gmail again, then
// resume incremental sync from a fresh historyId.
warn!(
config_id = %self.config_id,
"gmail history expired, falling back to full inbox refresh"
);
self.refresh_inbox_with_api(db, api).await?;
let profile = api.get_profile().await?;
if let Some(new_id) = profile.history_id {
db.set_sync_state(&self.config_id, "history_id", &new_id)?;
}
return Ok(());
}
Err(e) => return Err(e.into()),
};

if let Some(records) = resp.history {
for record in &records {
Expand Down
Loading