From fb26972ba1310e36fbdb0dfbe212a08a43a9a8b4 Mon Sep 17 00:00:00 2001 From: Ryan Benson Date: Wed, 8 Jul 2026 14:58:54 -0700 Subject: [PATCH] Enhance extension functionality by adding detailed permission handling and support for request headers storage/display. --- pyhindsight/analysis.py | 59 ++++++++++++++++++++++++------ pyhindsight/browsers/chrome.py | 38 ++++++++++++------- pyhindsight/browsers/webbrowser.py | 14 +++++++ 3 files changed, 86 insertions(+), 25 deletions(-) diff --git a/pyhindsight/analysis.py b/pyhindsight/analysis.py index c6b475b..825c474 100644 --- a/pyhindsight/analysis.py +++ b/pyhindsight/analysis.py @@ -1335,6 +1335,9 @@ def source_item(source, profile): w.write_string(row_number, 25, open_friendly, green_value_format) # opened w.write(row_number, 29, item.etag, green_value_format) # ETag w.write(row_number, 30, item.last_modified, green_value_format) # Last Modified + request_headers = getattr(item, 'request_headers', None) + if request_headers: + w.write(row_number, 31, str(request_headers), green_value_format) # headers elif item.row_type.startswith("bookmark folder"): w.write_string(row_number, 0, item.row_type, red_type_format) # record_type @@ -2489,7 +2492,7 @@ def annotate_origin(origin): # Column headers ext_ws.write(1, 0, 'Extension', header_format) ext_ws.write(1, 1, 'Type', header_format) - ext_ws.write(1, 2, 'Matches / Host', header_format) + ext_ws.write(1, 2, 'Matches / Host / Permissions', header_format) ext_ws.write(1, 3, 'run_at', header_format) ext_ws.write(1, 4, 'all_frames', header_format) ext_ws.write(1, 5, 'world', header_format) @@ -2498,7 +2501,7 @@ def annotate_origin(origin): # Column widths ext_ws.set_column('A:A', 22) # Extension (merged headers; data column is empty) - ext_ws.set_column('B:B', 16) # Type + ext_ws.set_column('B:B', 30) # Type ext_ws.set_column('C:C', 55) # Matches / Host ext_ws.set_column('D:D', 16) # run_at ext_ws.set_column('E:E', 10) # all_frames @@ -2594,6 +2597,23 @@ def write_script_row(label, cs, fmt, center_fmt): profile = getattr(ext, 'profile', '') or '' wrote_child = False + # 0) Manifest-declared permissions (API + host patterns). Chrome stores + # the parsed manifest list; Firefox stores a JSON-string summary. + perms = getattr(ext, 'permissions', None) + if isinstance(perms, str): + perms_str = '' if perms in ('None', '[]') else perms + elif perms: + perms_str = fmt_patterns([str(p) for p in perms]) + else: + perms_str = '' + if perms_str: + ext_ws.write(row_number, 0, '', cs_format) + ext_ws.write_string(row_number, 1, 'permissions', cs_format) + ext_ws.write_string(row_number, 2, perms_str, cs_format) + ext_ws.write_string(row_number, 7, profile, cs_format) + row_number += 1 + wrote_child = True + # 1) Current capability: manifest content scripts, then live dynamically # registered scripts (chrome.scripting / chrome.userScripts). for cs in (getattr(ext, 'content_scripts', None) or []): @@ -2607,15 +2627,24 @@ def write_script_row(label, cs, fmt, center_fmt): write_script_row(label, cs, dyn_format, dyn_center_format) wrote_child = True - # 2) Current host scope (where the extension can actually inject). - for label, hosts in (('host (granted)', getattr(ext, 'granted_scriptable_host', None)), - ('host (withheld)', getattr(ext, 'withholding_scriptable_host', None)), - ('host (runtime grant)', getattr(ext, 'runtime_granted_scriptable_host', None))): - if not hosts: + # 2) Current effective capability from Secure Preferences: API + # permissions, scriptable hosts (content script injection), and + # explicit hosts (cross-origin fetch/XHR access). + for label, values in ( + ('api (granted)', getattr(ext, 'granted_api', None)), + ('api (withheld)', getattr(ext, 'withholding_api', None)), + ('api (runtime grant)', getattr(ext, 'runtime_granted_api', None)), + ('scriptable host (granted)', getattr(ext, 'granted_scriptable_host', None)), + ('scriptable host (withheld)', getattr(ext, 'withholding_scriptable_host', None)), + ('scriptable host (runtime grant)', getattr(ext, 'runtime_granted_scriptable_host', None)), + ('explicit host (granted)', getattr(ext, 'granted_explicit_host', None)), + ('explicit host (withheld)', getattr(ext, 'withholding_explicit_host', None)), + ('explicit host (runtime grant)', getattr(ext, 'runtime_granted_explicit_host', None))): + if not values: continue ext_ws.write(row_number, 0, '', host_format) ext_ws.write_string(row_number, 1, label, host_format) - ext_ws.write_string(row_number, 2, ', '.join(hosts), host_format) + ext_ws.write_string(row_number, 2, fmt_patterns([str(x) for x in values]), host_format) ext_ws.write_string(row_number, 7, profile, host_format) row_number += 1 wrote_child = True @@ -2765,14 +2794,15 @@ def is_empty(value): c.execute( 'INSERT INTO timeline (type, timestamp, url, title, value, interpretation, profile, source_item, ' 'interrupt_reason, danger_type, opened, etag, last_modified, ' - 'mime_type, referrer, tab_url, download_source, hash, guid) ' - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + 'mime_type, referrer, tab_url, download_source, hash, guid, http_headers) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (item.row_type, friendly_date(item.timestamp), item.url, item.status_friendly, item.value, item.interpretation, item.profile, item.source_item, item.interrupt_reason_friendly, item.danger_type_friendly, item.opened, item.etag, item.last_modified, getattr(item, 'mime_type', None), getattr(item, 'referrer', None), getattr(item, 'tab_url', None), getattr(item, 'download_source', None), - getattr(item, 'hash', None), getattr(item, 'guid', None))) + getattr(item, 'hash', None), getattr(item, 'guid', None), + str(item.request_headers) if getattr(item, 'request_headers', None) else None)) elif item.row_type.startswith('bookmark folder'): c.execute( @@ -2883,12 +2913,17 @@ def is_empty(value): if self.__dict__.get('installed_extensions'): for extension in self.installed_extensions['data']: + # permissions is a parsed list on Chrome records (a JSON string on + # Firefox ones); serialize to JSON here at the storage boundary. + permissions = extension.permissions + if permissions is not None and not isinstance(permissions, str): + permissions = json.dumps(permissions) c.execute( 'INSERT INTO installed_extensions (name, description, version, ext_id, profile, ' 'permissions, manifest) ' 'VALUES (?, ?, ?, ?, ?, ?, ?)', (extension.name, extension.description, extension.version, extension.ext_id, - extension.profile, extension.permissions, extension.manifest)) + extension.profile, permissions, extension.manifest)) for preference_group in self.preferences: title = preference_group.get('presentation', {}).get('title') diff --git a/pyhindsight/browsers/chrome.py b/pyhindsight/browsers/chrome.py index 817be11..a4efb69 100644 --- a/pyhindsight/browsers/chrome.py +++ b/pyhindsight/browsers/chrome.py @@ -540,8 +540,8 @@ def _download_interpretation(item): parts.append(f'By extension: {by_ext}') if getattr(item, 'by_web_app_id', None): parts.append(f'By web app: {item.by_web_app_id}') - # mime_type / referrer / tab_url have dedicated Timeline columns, so they are not - # repeated here. + # mime_type / referrer / tab_url / request_headers have dedicated Timeline columns, + # so they are not repeated here. if getattr(item, 'site_url', None): parts.append(f'Site: {item.site_url}') if getattr(item, 'http_method', None): @@ -553,8 +553,10 @@ def _download_interpretation(item): chain = getattr(item, 'url_chain', None) if chain and len(chain) > 1: parts.append('Redirect chain: ' + ' -> '.join(chain)) - if getattr(item, 'request_headers', None): - parts.append(f'{len(item.request_headers)} request header(s)') + # In practice only set for shared_proto_db downloads; the History downloads + # table has a hash column but Chrome never populates it. + if getattr(item, 'hash', None): + parts.append(f'SHA-256: {item.hash}') if getattr(item, 'transient', None): parts.append('Transient') return ' | '.join(parts) @@ -2083,14 +2085,14 @@ def get_extensions(self, profile, dir_name): if ext is None: ext = Chrome.BrowserExtension( profile=profile, ext_id=ext_id, name=name, description=description, - version=manifest.get('version'), permissions=str(manifest.get('permissions')), + version=manifest.get('version'), permissions=manifest.get('permissions'), manifest=json.dumps(manifest)) self._extensions_by_id[ext_id] = ext else: ext.name = name or ext.name ext.description = description or ext.description ext.version = manifest.get('version') or ext.version - ext.permissions = str(manifest.get('permissions')) + ext.permissions = manifest.get('permissions') or ext.permissions ext.manifest = json.dumps(manifest) ext.on_disk = True ext.profile = profile @@ -2177,16 +2179,18 @@ def get_extension_settings(self, path, preferences_file): manifest = v.get('manifest') if isinstance(v.get('manifest'), dict) else {} - # Granted/withheld host scope determines where content scripts can inject. - def _scriptable(perm_key): + # Each permission-set pref (granted / withholding / runtime_granted) holds + # 'api' (API permissions), 'explicit_host' (cross-origin API access), and + # 'scriptable_host' (content script injection scope) lists. + def _perm_list(perm_key, sub_key): perms = v.get(perm_key) if isinstance(perms, dict): - return perms.get('scriptable_host') or [] + return perms.get(sub_key) or [] return [] - granted_scriptable = _scriptable('granted_permissions') - withholding_scriptable = _scriptable('withholding_permissions') - runtime_scriptable = _scriptable('runtime_granted_permissions') + granted_scriptable = _perm_list('granted_permissions', 'scriptable_host') + withholding_scriptable = _perm_list('withholding_permissions', 'scriptable_host') + runtime_scriptable = _perm_list('runtime_granted_permissions', 'scriptable_host') content_scripts = manifest.get('content_scripts') or [] @@ -2209,7 +2213,7 @@ def _scriptable(perm_key): ext = Chrome.BrowserExtension( profile=path, ext_id=ext_id, name=name, description=manifest.get('description'), version=manifest.get('version'), - permissions=str(manifest.get('permissions')), manifest=json.dumps(manifest)) + permissions=manifest.get('permissions'), manifest=json.dumps(manifest)) self._extensions_by_id[ext_id] = ext else: # Disk data is authoritative for name/description/version/manifest; only @@ -2220,6 +2224,8 @@ def _scriptable(perm_key): ext.description = manifest.get('description') if not ext.version: ext.version = manifest.get('version') + if not ext.permissions: + ext.permissions = manifest.get('permissions') if not ext.manifest: ext.manifest = json.dumps(manifest) @@ -2235,6 +2241,12 @@ def _scriptable(perm_key): ext.granted_scriptable_host = granted_scriptable ext.withholding_scriptable_host = withholding_scriptable ext.runtime_granted_scriptable_host = runtime_scriptable + ext.granted_api = _perm_list('granted_permissions', 'api') + ext.withholding_api = _perm_list('withholding_permissions', 'api') + ext.runtime_granted_api = _perm_list('runtime_granted_permissions', 'api') + ext.granted_explicit_host = _perm_list('granted_permissions', 'explicit_host') + ext.withholding_explicit_host = _perm_list('withholding_permissions', 'explicit_host') + ext.runtime_granted_explicit_host = _perm_list('runtime_granted_permissions', 'explicit_host') # On-disk content scripts (actual files) win; otherwise use the cached copy here. if not ext.content_scripts and content_scripts: ext.content_scripts = content_scripts diff --git a/pyhindsight/browsers/webbrowser.py b/pyhindsight/browsers/webbrowser.py index 1405cd0..57c30b8 100644 --- a/pyhindsight/browsers/webbrowser.py +++ b/pyhindsight/browsers/webbrowser.py @@ -510,6 +510,9 @@ def __init__(self, profile, ext_id, name, description, version, permissions, man self.name = name self.description = description self.version = version + # Manifest-declared permissions. Chrome stores the parsed manifest list + # (or None); Firefox stores a compact JSON-string summary. Serialized to + # a string only at the SQLite output boundary. self.permissions = permissions self.manifest = manifest # Presence: whether the extension was found unpacked on disk (Extensions//), @@ -531,6 +534,17 @@ def __init__(self, profile, ext_id, name, description, version, permissions, man self.granted_scriptable_host = granted_scriptable_host or [] self.withholding_scriptable_host = withholding_scriptable_host or [] self.runtime_granted_scriptable_host = runtime_granted_scriptable_host or [] + # API permissions actually granted / withheld / granted at runtime + # (Secure Preferences granted_permissions.api etc.), which can differ + # from the manifest-declared set above. + self.granted_api = [] + self.withholding_api = [] + self.runtime_granted_api = [] + # Host scope for cross-origin API access (fetch/XHR from extension + # contexts) — 'explicit_host', distinct from scriptable_host (injection). + self.granted_explicit_host = [] + self.withholding_explicit_host = [] + self.runtime_granted_explicit_host = [] # List of declared content script blocks (from the manifest) self.content_scripts = content_scripts or [] # Dynamically registered scripts from the 'Extension Scripts' StateStore