diff --git a/README.md b/README.md index edb2f3e..4c564d5 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,11 @@ This package is for implementing the software needs of the E3SM Communications t ### Simple commands `e3sm-comms-e3sm-org-reviewer` -- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for -- output: txt file listing e3sm.org pages containing those phrases +- input: + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml user@host:/remote/path/wordpress_posts.xml`. + - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages + - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change) +- output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items `e3sm-comms-html-reviewer` - input: 1 txt file of html copied from WordPress that includes yellow highlights left over from Confluence. @@ -23,6 +26,13 @@ This package is for implementing the software needs of the E3SM Communications t - input: txt file of time intervals to cut from the video, txt file of initial timestamps - output: txt file of new timestamps after cutting the specified intervals +`e3sm-comms-exported-xml-reviewer` +- input: + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. NOTE: Only ever use the XML files downloaded directly from WordPress; the source must be trusted. + - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages + - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages, txt file of links known to be inaccessible for automated review +- output: 6 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`, (6) `wordpress_invalid_external_links_report.md` + ### Confluence API commands (require Confluence token) `e3sm-comms-newsletter-reviewer` diff --git a/e3sm_comms/e3sm_org_reviewer/README.md b/e3sm_comms/e3sm_org_reviewer/README.md new file mode 100644 index 0000000..1a0fecb --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/README.md @@ -0,0 +1,8 @@ +# Dependency hierarchy + +It is important to not introduce circular dependencies. +To avoid this, the dependency hierarchy is listed below: + +- Level 1: `main.py` +- Level 2: `parsers.py`, `reporters.py` +- Level 3: `classifiers.py`, `confluence.py`, `readers.py`, `record.py`, `utils.py` diff --git a/e3sm_comms/e3sm_org_reviewer/classifiers.py b/e3sm_comms/e3sm_org_reviewer/classifiers.py new file mode 100644 index 0000000..dc63e69 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/classifiers.py @@ -0,0 +1,81 @@ +import re +from typing import Dict, Optional, Set, Tuple + +CLASS_PUBLISHED = "Published" +CLASS_ARCHIVED = "Archived" +CLASS_SHOULD_BE_ARCHIVED = "Should be archived" +CLASS_NOT_PUBLISHED = "Not published" +CLASS_KNOWN_OK = "Known OK" +CLASS_KEEP_UNCHANGED = "Keep unchanged" +CLASS_NO_MAPPED_E3SM_URL = "No mapped e3sm.org URL" +CLASS_PREDICTED_URL_NOT_IN_EXPORT = "Predicted e3sm.org URL not in WordPress export" + +FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") + + +def classify_e3sm_url( + e3sm_url: Optional[str], + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Tuple[str, Optional[str]]: + if not e3sm_url: + return CLASS_NO_MAPPED_E3SM_URL, None + + if e3sm_url in known_ok_urls: + return CLASS_KNOWN_OK, url_to_status.get(e3sm_url) + + if e3sm_url in keep_unchanged_urls: + return CLASS_KEEP_UNCHANGED, url_to_status.get(e3sm_url) + + wordpress_status = url_to_status.get(e3sm_url) + + if wordpress_status is None: + return CLASS_PREDICTED_URL_NOT_IN_EXPORT, None + + if wordpress_status == "archive": + return CLASS_ARCHIVED, wordpress_status + + if e3sm_url in expected_archived_urls: + return CLASS_SHOULD_BE_ARCHIVED, wordpress_status + + if wordpress_status != "publish": + return CLASS_NOT_PUBLISHED, wordpress_status + + return CLASS_PUBLISHED, wordpress_status + + +def classification_sort_key(classification: str) -> Tuple[int, str]: + order = { + CLASS_PUBLISHED: 0, + CLASS_SHOULD_BE_ARCHIVED: 1, + CLASS_ARCHIVED: 2, + CLASS_NOT_PUBLISHED: 3, + CLASS_KNOWN_OK: 4, + CLASS_KEEP_UNCHANGED: 5, + CLASS_NO_MAPPED_E3SM_URL: 6, + CLASS_PREDICTED_URL_NOT_IN_EXPORT: 7, + } + return (order.get(classification, 999), classification) + + +def year_sort_key(year_str: str) -> Tuple[int, int]: + if year_str == "Unknown year": + return (1, 0) + if year_str == "N/A": + return (2, 0) + try: + return (0, -int(year_str)) + except ValueError: + return (3, 0) + + +def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: + match = FROM_PREFIX_RE.match(line) + if not match: + return None, line + + year = int(match.group(1)) + remainder = match.group(2).strip() + return year, remainder diff --git a/e3sm_comms/e3sm_org_reviewer/confluence.py b/e3sm_comms/e3sm_org_reviewer/confluence.py new file mode 100644 index 0000000..70e4d9c --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/confluence.py @@ -0,0 +1,34 @@ +from typing import List, Tuple + +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.utils import parse_confluence_hierarchy_file + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" + + +def get_confluence_predicted_e3sm_urls( + input_file: str, +) -> Tuple[List[str], List[str]]: + valid_predicted_urls: List[str] = [] + unmapped_confluence_pages: List[str] = [] + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + valid_predicted_urls.append(e3sm_url) + else: + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + + return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index ecb0739..0efb7bf 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,27 +1,294 @@ -from typing import Dict, List +from typing import Dict, List, Set -from e3sm_comms.page_reviewer.utils_base import LinkedURLs -from e3sm_comms.utils import IO_DIR +from e3sm_comms.e3sm_org_reviewer.classifiers import CLASS_PUBLISHED +from e3sm_comms.e3sm_org_reviewer.confluence import get_confluence_predicted_e3sm_urls +from e3sm_comms.e3sm_org_reviewer.parsers import ( + parse_confluence_record, + parse_wordpress_record, + parse_wordpress_sensitive_terms_lines, +) +from e3sm_comms.e3sm_org_reviewer.readers import print_status_counts +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.e3sm_org_reviewer.reporters import ( + write_action_items_report, + write_markdown_report, + write_sensitive_terms_report, +) +from e3sm_comms.e3sm_org_reviewer.utils import ( + build_url_to_status, + get_all_non_published_urls, + get_all_urls, + get_combined_urls_by_status, + get_list_difference, + get_total_count, +) +from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status +from e3sm_comms.utils import ( + IO_DIR, + expand_patterns_to_urls, + get_invalid_patterns, + get_wordpress_urls_by_status, + read_lines, +) -INPUT_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/e3sm_org_reviewer/web_pages.txt" +# From WordPress under Tools > Export: +INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" +INPUT_XML_POSTS: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_posts.xml" + +# From output of `e3sm-comms-website-reviewer`: +INPUT_CONFLUENCE_HIERARCHY: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt" +) +INPUT_CONFLUENCE_SENSITIVE_TERMS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt" +) + +# Other: +INPUT_WHITELIST: str = f"{IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt" +INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/shared/archived_web_pages.txt" +) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -OUTPUT: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" +INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/known_ok_e3sm_org_paths.txt" +) +INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/keep_unchanged_e3sm_org_paths.txt" +) + +OUTPUT_MARKDOWN_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/path_report.md" +OUTPUT_SENSITIVE_TERMS_REPORT: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" +) +OUTPUT_ACTION_ITEMS_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/action_items.md" + +RUN_CHECKS: bool = True # Set to False for faster debugging def main(): - with open(INPUT_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_e3sm_org_paths: List[str] = [line.strip() for line in f] - with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: - terms: List[str] = [line.rstrip("\n").lower() for line in f] - list_search_phrases: List[str] = sorted(terms) - - print(f"Checking {len(list_input_e3sm_org_paths)} e3sm.org pages") - links = LinkedURLs( - list_input_e3sm_org_paths, - scan_links_for_sensitive_terms=True, - list_sensitive_terms=list_search_phrases, - ) - relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms - with open(OUTPUT, "w", encoding="utf-8") as f: - for link in relevant_links: - f.write(f"{link}: {relevant_links[link]}\n") + pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_PAGES, "page" + ) + posts_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_POSTS, "post" + ) + num_pages: int = get_total_count(pages_by_status) + num_posts: int = get_total_count(posts_by_status) + print(f"Found {num_pages} pages, {num_posts} posts") + print( + f"Pages have status in {pages_by_status.keys()}; posts have status in {posts_by_status.keys()}" + ) + + all_urls_by_status: Dict[str, List[str]] = get_combined_urls_by_status( + pages_by_status, posts_by_status + ) + print_status_counts(all_urls_by_status) + + non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) + print(f"Total non-published URLs: {len(non_published_urls)}") + + list_whitelisted_paths: List[str] = read_lines(INPUT_WHITELIST) + list_expected_archived_paths: List[str] = read_lines( + INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS + ) + list_known_ok_paths: List[str] = read_lines(INPUT_KNOWN_OK_E3SM_ORG_PATHS) + list_keep_unchanged_paths: List[str] = read_lines( + INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS + ) + + all_urls: List[str] = get_all_urls(all_urls_by_status) + url_to_status: Dict[str, str] = build_url_to_status(all_urls_by_status) + + invalid_whitelisted_paths: List[str] = get_invalid_patterns( + list_whitelisted_paths, all_urls + ) + valid_whitelisted_paths: List[str] = [ + path for path in list_whitelisted_paths if path not in invalid_whitelisted_paths + ] + + invalid_expected_archived_paths: List[str] = get_invalid_patterns( + list_expected_archived_paths, all_urls + ) + valid_expected_archived_paths: List[str] = [ + path + for path in list_expected_archived_paths + if path not in invalid_expected_archived_paths + ] + + confluence_predicted_urls, confluence_unmapped_entries = ( + get_confluence_predicted_e3sm_urls(INPUT_CONFLUENCE_HIERARCHY) + ) + invalid_confluence_paths: List[str] = get_invalid_patterns( + confluence_predicted_urls, all_urls + ) + valid_confluence_paths: List[str] = [ + path + for path in confluence_predicted_urls + if path not in invalid_confluence_paths + ] + + whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( + valid_whitelisted_paths, all_urls + ) + print( + f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns. Expanding patterns, it's {len(whitelisted_urls_expanded)} valid URLs." + ) + print( + f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" + ) + print( + f"Of {len(confluence_predicted_urls)} predicted Confluence e3sm.org paths, " + f"{len(valid_confluence_paths)} are valid URLs" + ) + print( + f"Confluence pages with no predicted e3sm.org URL: {len(confluence_unmapped_entries)}" + ) + + published_urls: List[str] = all_urls_by_status.get("publish", []) + archived_urls: List[str] = all_urls_by_status.get("archive", []) + + expected_archived_urls_expanded: List[str] = expand_patterns_to_urls( + valid_expected_archived_paths, all_urls + ) + + whitelisted_but_not_published: List[str] = get_list_difference( + whitelisted_urls_expanded, published_urls + ) + published_but_not_whitelisted: List[str] = get_list_difference( + published_urls, whitelisted_urls_expanded + ) + should_be_archived: List[str] = get_list_difference( + expected_archived_urls_expanded, archived_urls + ) + published_but_not_in_confluence: List[str] = get_list_difference( + published_urls, valid_confluence_paths + ) + published_not_whitelisted_and_not_in_confluence: List[str] = get_list_difference( + published_but_not_in_confluence, whitelisted_urls_expanded + ) + + print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") + print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") + print(f"Not archived, but should be archived: {len(should_be_archived)}") + print( + f"Published, but no matching Confluence path found: {len(published_but_not_in_confluence)}" + ) + print( + "Published, but not whitelisted and no matching Confluence path found: " + f"{len(published_not_whitelisted_and_not_in_confluence)}" + ) + print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") + print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") + print( + f"Invalid Confluence-predicted e3sm.org paths: {len(invalid_confluence_paths)}" + ) + + incorrectly_accessible_non_published_urls: List[str] = [] + + e3sm_records: List[SensitiveTermRecord] = [] + confluence_records: List[SensitiveTermRecord] = [] + + if RUN_CHECKS: + print( + f"Checking {len(whitelisted_urls_expanded)} whitelisted e3sm.org pages for search phrases" + ) + with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: + terms: List[str] = [line.rstrip("\n").lower() for line in f] + list_search_phrases: List[str] = sorted(terms) + + links = LinkedURLs( + whitelisted_urls_expanded, + scan_links_for_sensitive_terms=True, + list_sensitive_terms=list_search_phrases, + ) + relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms + + expected_archived_urls_set: Set[str] = set(expected_archived_urls_expanded) + known_ok_urls_set: Set[str] = set(list_known_ok_paths) + keep_unchanged_urls_set: Set[str] = set(list_keep_unchanged_paths) + + wordpress_lines_input: List[str] = [ + f"{link}: {relevant_links[link]}" for link in relevant_links + ] + wordpress_lines = parse_wordpress_sensitive_terms_lines(wordpress_lines_input) + for _, line in wordpress_lines: + record = parse_wordpress_record( + line=line, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls_set, + known_ok_urls=known_ok_urls_set, + keep_unchanged_urls=keep_unchanged_urls_set, + ) + if record: + e3sm_records.append(record) + + if INPUT_CONFLUENCE_SENSITIVE_TERMS: + try: + with open(INPUT_CONFLUENCE_SENSITIVE_TERMS, "r", encoding="utf-8") as f: + for raw_line in f: + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + record = parse_confluence_record( + line=line, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls_set, + known_ok_urls=known_ok_urls_set, + keep_unchanged_urls=keep_unchanged_urls_set, + ) + if record: + confluence_records.append(record) + except FileNotFoundError: + print( + f"Confluence sensitive terms input not found: {INPUT_CONFLUENCE_SENSITIVE_TERMS}" + ) + + print( + f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" + ) + for e3sm_url in non_published_urls: + e3sm_url_status = get_e3sm_url_status(e3sm_url) + if e3sm_url_status == "link works not logged-in": + incorrectly_accessible_non_published_urls.append(e3sm_url) + + write_markdown_report( + output_path=OUTPUT_MARKDOWN_REPORT, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + whitelisted_but_not_published=whitelisted_but_not_published, + published_but_not_whitelisted=published_but_not_whitelisted, + should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, + published_not_whitelisted_and_not_in_confluence=published_not_whitelisted_and_not_in_confluence, + incorrectly_accessible_non_published_urls=incorrectly_accessible_non_published_urls, + invalid_whitelisted_paths=invalid_whitelisted_paths, + invalid_expected_archived_paths=invalid_expected_archived_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) + + write_sensitive_terms_report( + output_path=OUTPUT_SENSITIVE_TERMS_REPORT, + e3sm_records=e3sm_records, + confluence_records=confluence_records, + ) + + confluence_published_sensitive_records: List[SensitiveTermRecord] = [ + record + for record in confluence_records + if record.classification == CLASS_PUBLISHED + ] + + write_action_items_report( + output_path=OUTPUT_ACTION_ITEMS_REPORT, + should_be_archived=should_be_archived, + published_not_whitelisted_and_not_in_confluence=published_not_whitelisted_and_not_in_confluence, + confluence_published_sensitive_records=confluence_published_sensitive_records, + ) + + +if __name__ == "__main__": + main() diff --git a/e3sm_comms/e3sm_org_reviewer/parsers.py b/e3sm_comms/e3sm_org_reviewer/parsers.py new file mode 100644 index 0000000..f1abdf7 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/parsers.py @@ -0,0 +1,197 @@ +import ast +from typing import Dict, List, Optional, Set, Tuple + +from e3sm_comms.e3sm_org_reviewer.classifiers import ( + classify_e3sm_url, + extract_year_and_remainder, +) +from e3sm_comms.e3sm_org_reviewer.confluence import build_confluence_url +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm + + +def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: + try: + data = ast.literal_eval(dict_str) + except (SyntaxError, ValueError): + return None + + if not isinstance(data, dict): + return None + + try: + total = sum(data.values()) + except TypeError: + return None + + if not isinstance(total, (int, float)): + return None + + return data + + +def parse_wordpress_sensitive_terms_lines(lines: List[str]) -> List[Tuple[int, str]]: + parsed: List[Tuple[int, str]] = [] + + for raw_line in lines: + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + dict_start = line.find("{") + if dict_start == -1: + print(f"Skipping malformed WordPress sensitive-terms line: {line}") + continue + + dict_str = line[dict_start:].strip() + dict_data = parse_dict(dict_str) + if dict_data is None: + print(f"Skipping malformed dictionary in WordPress line: {line}") + continue + + total = int(sum(dict_data.values())) + parsed.append((total, line)) + + parsed.sort(key=lambda x: x[0], reverse=True) + return parsed + + +def parse_wordpress_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + dict_start = line.find("{") + if dict_start == -1: + return None + + url = extract_wordpress_url(line) + if not url: + return None + + dict_str = line[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + return None + + total_terms = int(sum(term_counts.values())) + classification, wordpress_status = classify_e3sm_url( + e3sm_url=url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + return SensitiveTermRecord( + source="e3sm.org", + raw_line=line, + year_label="N/A", + year_int=None, + total_terms=total_terms, + term_counts=term_counts, + source_url=url, + title=None, + confluence_url=None, + e3sm_url=url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def parse_confluence_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + components = extract_confluence_components(line) + if components is None: + return None + + year, page_id, title, term_counts = components + confluence_url = build_confluence_url(page_id) + + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + e3sm_url = None + + classification, wordpress_status = classify_e3sm_url( + e3sm_url=e3sm_url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + total_terms = int(sum(term_counts.values())) + year_label = str(year) if year is not None else "Unknown year" + + return SensitiveTermRecord( + source="confluence", + raw_line=line, + year_label=year_label, + year_int=year, + total_terms=total_terms, + term_counts=term_counts, + source_url=confluence_url, + title=title, + confluence_url=confluence_url, + e3sm_url=e3sm_url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def extract_wordpress_url(line: str) -> Optional[str]: + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + if prefix.endswith(":"): + prefix = prefix[:-1].rstrip() + + return prefix + + +def extract_confluence_components( + line: str, +) -> Optional[Tuple[Optional[int], str, str, Dict[str, int]]]: + year, remainder = extract_year_and_remainder(line) + + dict_start = remainder.find("{") + if dict_start == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + dict_str = remainder[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + print(f"Skipping malformed dictionary in Confluence line: {line}") + return None + + prefix = remainder[:dict_start].rstrip() + if prefix.endswith("--"): + prefix = prefix[:-2].rstrip() + + first_colon = prefix.find(":") + if first_colon == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + page_id = prefix[:first_colon].strip() + title = prefix[first_colon + 1 :].strip() + + if not page_id.isdigit(): + print(f"Skipping Confluence line with non-numeric page id: {line}") + return None + + return year, page_id, title, term_counts diff --git a/e3sm_comms/e3sm_org_reviewer/readers.py b/e3sm_comms/e3sm_org_reviewer/readers.py new file mode 100644 index 0000000..3214274 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/readers.py @@ -0,0 +1,7 @@ +from typing import Dict, List + + +def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: + for status in ["publish", "archive", "draft", "future", "pending", "private"]: + if status in all_urls_by_status: + print(f"Found {len(all_urls_by_status[status])} {status} URLs") diff --git a/e3sm_comms/e3sm_org_reviewer/record.py b/e3sm_comms/e3sm_org_reviewer/record.py new file mode 100644 index 0000000..d358a04 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/record.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass +from typing import Dict, Optional + + +@dataclass +class SensitiveTermRecord: + source: str # "e3sm.org" or "confluence" + raw_line: str + year_label: str + year_int: Optional[int] + total_terms: int + term_counts: Dict[str, int] + source_url: Optional[str] + title: Optional[str] + confluence_url: Optional[str] + e3sm_url: Optional[str] + classification: str + wordpress_status: Optional[str] diff --git a/e3sm_comms/e3sm_org_reviewer/reporters.py b/e3sm_comms/e3sm_org_reviewer/reporters.py new file mode 100644 index 0000000..acadbe2 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/reporters.py @@ -0,0 +1,425 @@ +from collections import defaultdict +from typing import Callable, DefaultDict, Dict, List, Set, TextIO + +from e3sm_comms.e3sm_org_reviewer.classifiers import ( + classification_sort_key, + year_sort_key, +) +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.e3sm_org_reviewer.utils import get_all_urls, get_status_counts_for_urls +from e3sm_comms.page_reviewer.utils_base import get_e3sm_url_status +from e3sm_comms.utils import expand_patterns_to_urls + + +def write_markdown_report( + output_path: str, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + whitelisted_but_not_published: List[str], + published_but_not_whitelisted: List[str], + should_be_archived: List[str], + published_but_not_in_confluence: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], + incorrectly_accessible_non_published_urls: List[str], + invalid_whitelisted_paths: List[str], + invalid_expected_archived_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + write_summary_table( + f, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) + + f.write("# Valid Paths\n\n") + write_markdown_section( + f, + "Whitelisted but not published", + whitelisted_but_not_published, + ) + write_markdown_section( + f, + "Published but not whitelisted", + published_but_not_whitelisted, + ) + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + write_markdown_section( + f, + "Published but no matching Confluence path found", + published_but_not_in_confluence, + ) + write_markdown_section( + f, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, + ) + write_markdown_section( + f, + "Non-published e3sm.org pages that are still accessible without login", + incorrectly_accessible_non_published_urls, + ) + + f.write("# Invalid Paths\n\n") + write_markdown_section( + f, + "Identified in whitelist input", + invalid_whitelisted_paths, + ) + write_markdown_section( + f, + "Identified in archive input", + invalid_expected_archived_paths, + ) + write_markdown_section( + f, + "Identified in Confluence input", + invalid_confluence_paths, + ) + write_markdown_section( + f, + "Confluence pages with no mappable e3sm.org URL", + confluence_unmapped_entries, + ) + + +def write_summary_table( + file_obj: TextIO, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], +) -> None: + statuses: List[str] = sorted(all_urls_by_status.keys()) + all_urls: List[str] = get_all_urls(all_urls_by_status) + + whitelist_set: Set[str] = set( + expand_patterns_to_urls(valid_whitelisted_paths, all_urls) + ) + expected_archived_set: Set[str] = set( + expand_patterns_to_urls(valid_expected_archived_paths, all_urls) + ) + both_set: Set[str] = whitelist_set.intersection(expected_archived_set) + neither_set: Set[str] = set(all_urls) - whitelist_set.union(expected_archived_set) + + rows = [ + ("Whitelisted URLs", whitelist_set), + ("Expected archived", expected_archived_set), + ("Both whitelisted and expected archived", both_set), + ("Neither whitelisted nor expected archived", neither_set), + ("TOTAL", set(all_urls)), + ] + + file_obj.write("# Summary\n\n") + file_obj.write("| Type | " + " | ".join(statuses) + " | Total |\n") + file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " | --- |\n") + + for row_name, row_urls in rows: + counts = get_status_counts_for_urls( + urls=list(row_urls), + all_urls_by_status=all_urls_by_status, + statuses=statuses, + ) + total_count = sum(counts.values()) + file_obj.write( + f"| {row_name} | " + + " | ".join(str(counts[status]) for status in statuses) + + f" | {total_count} |\n" + ) + + confluence_valid_set: Set[str] = set(valid_confluence_paths) + all_urls_set: Set[str] = set(all_urls) + + e3sm_with_confluence: Set[str] = all_urls_set.intersection(confluence_valid_set) + e3sm_without_confluence: Set[str] = all_urls_set - confluence_valid_set + + confluence_not_valid_count: int = len(invalid_confluence_paths) + len( + confluence_unmapped_entries + ) + total_confluence_urls: int = len(confluence_valid_set) + confluence_not_valid_count + total_e3sm_urls: int = len(all_urls_set) + + confluence_counts_match: bool = ( + confluence_not_valid_count + len(e3sm_with_confluence) == total_confluence_urls + ) + e3sm_counts_match: bool = ( + len(e3sm_without_confluence) + len(e3sm_with_confluence) == total_e3sm_urls + ) + + file_obj.write("\n## Confluence Mapping Summary\n\n") + file_obj.write("| Type | Count |\n") + file_obj.write("| --- | --- |\n") + file_obj.write( + f"| Confluence paths that do not map to a valid e3sm.org path | {confluence_not_valid_count} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do not have a Confluence path associated with them | {len(e3sm_without_confluence)} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do have a Confluence counterpart | {len(e3sm_with_confluence)} |\n" + ) + file_obj.write(f"| Total Confluence-derived paths | {total_confluence_urls} |\n") + file_obj.write(f"| Total e3sm.org paths | {total_e3sm_urls} |\n") + file_obj.write("\n") + + file_obj.write("Validation:\n\n") + file_obj.write( + f"- Confluence counts match: " + f"{confluence_not_valid_count} + {len(e3sm_with_confluence)} = {total_confluence_urls} " + f"({'yes' if confluence_counts_match else 'no'})\n" + ) + file_obj.write( + f"- e3sm.org counts match: " + f"{len(e3sm_without_confluence)} + {len(e3sm_with_confluence)} = {total_e3sm_urls} " + f"({'yes' if e3sm_counts_match else 'no'})\n\n" + ) + + +def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: + file_obj.write(f"## {title}\n\n") + if not items: + file_obj.write("_None._\n\n") + return + + for i, item in enumerate(items, start=1): + file_obj.write(f"{i}. {item}\n") + file_obj.write("\n") + + +def write_action_items_report( + output_path: str, + should_be_archived: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], + confluence_published_sensitive_records: List[SensitiveTermRecord], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Action Items Report\n\n") + + f.write("## Summary\n\n") + f.write("| Action Area | Count |\n") + f.write("| --- | ---: |\n") + f.write( + f"| Expecting to be archived, but not yet archived | {len(should_be_archived)} |\n" + ) + f.write( + f"| Published but not whitelisted and no matching Confluence path found | {len(published_not_whitelisted_and_not_in_confluence)} |\n" + ) + f.write( + f"| Confluence pages with sensitive terms mapped to published e3sm.org pages | {len(confluence_published_sensitive_records)} |\n" + ) + f.write("\n") + + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + + write_markdown_section( + f, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, + ) + + write_action_items_confluence_section(f, confluence_published_sensitive_records) + + +def write_action_items_confluence_section( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + f.write( + "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" + ) + + if not records: + f.write("_None._\n\n") + return + + grouped = group_records_by_classification_and_year(records) + + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") + + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {format_confluence_record(record)}\n") + f.write("\n") + + +def write_sensitive_terms_report( + output_path: str, + e3sm_records: List[SensitiveTermRecord], + confluence_records: List[SensitiveTermRecord], +) -> None: + description_e3sm_org = ( + "These are the currently reviewed e3sm.org pages that include sensitive terms. " + "Classification is derived from WordPress status, expected archived inputs, and the manual exception lists for known-ok and keep-unchanged paths." + ) + description_confluence = ( + "These are the Confluence pages that include sensitive terms. The confluence links are what the website reviewer scanned. " + "The e3sm.org links are predicted from Confluence mapping and then classified against the WordPress export." + ) + + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Sensitive Terms Report\n\n") + + write_sensitive_terms_section( + f, + "e3sm.org", + description_e3sm_org, + e3sm_records, + format_e3sm_record, + ) + write_sensitive_terms_section( + f, + "Confluence", + description_confluence, + confluence_records, + format_confluence_record, + ) + + +def write_sensitive_terms_section( + f: TextIO, + section_title: str, + description: str, + records: List[SensitiveTermRecord], + formatter: Callable[[SensitiveTermRecord], str], +) -> None: + f.write(f"## {section_title}\n\n") + f.write(f"{description}\n\n") + + write_sensitive_terms_summary_table(f, records) + + grouped = group_records_by_classification_and_year(records) + + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") + + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {formatter(record)}\n") + f.write("\n") + + +def write_sensitive_terms_summary_table( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + summary = build_classification_summary(records) + + f.write("### Summary Table\n\n") + f.write("| Classification | Total | 1 | 2 | 3 | 4 | 5+ |\n") + f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") + + for classification in sorted(summary.keys(), key=classification_sort_key): + counts = summary[classification] + f.write( + f"| {classification} | {counts['total']} | {counts['1']} | {counts['2']} | " + f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" + ) + + f.write("\n") + + +def format_e3sm_record(record: SensitiveTermRecord) -> str: + url = record.e3sm_url or record.source_url or "UNKNOWN" + md = f"[{url}]({url})" + if record.wordpress_status: + md += f" (status: {record.wordpress_status})" + md += f" -- {format_term_counts(record.term_counts)}" + return md + + +def format_confluence_record(record: SensitiveTermRecord) -> str: + title = record.title or "Untitled" + confluence_url = record.confluence_url or record.source_url or "" + md = f"{title}: [confluence]({confluence_url})" + + if record.e3sm_url: + md += f" [e3sm.org]({record.e3sm_url})" + + try: + e3sm_url_status = get_e3sm_url_status(record.e3sm_url) + except Exception as exc: + print(f"Could not get e3sm.org URL status for {record.e3sm_url}: {exc}") + e3sm_url_status = None + + if e3sm_url_status: + md += f" (Note: {e3sm_url_status})" + + if record.wordpress_status: + md += f" (WordPress status: {record.wordpress_status})" + + md += f" -- {format_term_counts(record.term_counts)}" + return md + + +def format_term_counts(term_counts: Dict[str, int]) -> str: + return str(term_counts) + + +def build_classification_summary( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, int]]: + summary: Dict[str, Dict[str, int]] = {} + + for record in records: + classification = record.classification + if classification not in summary: + summary[classification] = { + "total": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5+": 0, + } + + summary[classification]["total"] += 1 + if record.total_terms == 1: + summary[classification]["1"] += 1 + elif record.total_terms == 2: + summary[classification]["2"] += 1 + elif record.total_terms == 3: + summary[classification]["3"] += 1 + elif record.total_terms == 4: + summary[classification]["4"] += 1 + elif record.total_terms >= 5: + summary[classification]["5+"] += 1 + + return summary + + +def group_records_by_classification_and_year( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, List[SensitiveTermRecord]]]: + grouped: DefaultDict[str, DefaultDict[str, List[SensitiveTermRecord]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + + for record in records: + grouped[record.classification][record.year_label].append(record) + + for classification in grouped: + for year_label in grouped[classification]: + grouped[classification][year_label].sort( + key=lambda r: (r.total_terms, r.e3sm_url or "", r.title or ""), + reverse=True, + ) + + return {k: dict(v) for k, v in grouped.items()} diff --git a/e3sm_comms/e3sm_org_reviewer/utils.py b/e3sm_comms/e3sm_org_reviewer/utils.py new file mode 100644 index 0000000..3c72a75 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/utils.py @@ -0,0 +1,53 @@ +from collections import defaultdict +from typing import Dict, List + + +def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: + return sorted(set(list1) - set(list2)) + + +def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + all_urls: List[str] = [] + for urls in urls_by_status.values(): + all_urls.extend(urls) + return sorted(all_urls) + + +def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + non_published_urls: List[str] = [] + for status, urls in urls_by_status.items(): + if status != "publish": + non_published_urls.extend(urls) + return sorted(non_published_urls) + + +def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: + return sum(len(urls) for urls in urls_by_status.values()) + + +def get_combined_urls_by_status( + pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] +) -> Dict[str, List[str]]: + merged: Dict[str, List[str]] = defaultdict(list) + for source in (pages_by_status, posts_by_status): + for status, urls in source.items(): + merged[status].extend(urls) + return {status: sorted(urls) for status, urls in sorted(merged.items())} + + +def get_status_counts_for_urls( + urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] +) -> Dict[str, int]: + url_set = set(urls) + counts: Dict[str, int] = {} + for status in statuses: + counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) + return counts + + +def build_url_to_status(all_urls_by_status: Dict[str, List[str]]) -> Dict[str, str]: + result: Dict[str, str] = {} + for status, urls in all_urls_by_status.items(): + for url in urls: + result[url] = status + return result diff --git a/e3sm_comms/exported_xml_reviewer/README.md b/e3sm_comms/exported_xml_reviewer/README.md new file mode 100644 index 0000000..f14df1f --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/README.md @@ -0,0 +1,9 @@ +# Dependency hierarchy + +It is important to not introduce circular dependencies. +To avoid this, the dependency hierarchy is listed below: + +- Level 1: `main.py` +- Level 2: `builders.py` +- Level 3: `confluence.py`, `link_analysis.py`, `readers.py` +- Level 4: `utils.py` diff --git a/e3sm_comms/exported_xml_reviewer/__init__.py b/e3sm_comms/exported_xml_reviewer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py new file mode 100644 index 0000000..9f69963 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, Dict, List, Optional, Set, Tuple + +from e3sm_comms.exported_xml_reviewer.confluence import get_confluence_mapping +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + check_external_link, + check_redirect_target, + extract_external_links, + extract_internal_e3sm_links, +) +from e3sm_comms.exported_xml_reviewer.readers import ( + read_expected_archived_patterns, + read_keep_unchanged_links, + read_known_ok_links, + read_requested_links, + read_sensitive_terms, + read_whitelist_patterns, +) +from e3sm_comms.exported_xml_reviewer.utils import ( + count_sensitive_terms, + display_status, + normalize_status, + strip_html, +) +from e3sm_comms.page_reviewer.utils_base import get_e3sm_url_status +from e3sm_comms.utils import ( + WordpressItem, + expand_patterns_to_urls, + normalize_url, + parse_wordpress_xml_items, +) + + +@dataclass +class ReportRecord: + title: str + e3sm_url: str + status: str + sensitive_terms: Dict[str, int] + confluence_draft_url: Optional[str] + + +@dataclass +class RequestedLinkRecord: + e3sm_url: str + included_later: bool + current_status: str + currently_whitelisted: bool + requesting_urls: str + + +@dataclass +class TopLevelPageIssue: + title: str + url: str + status: str + + +@dataclass +class ArchivedParentPublishedChildIssue: + parent_title: str + parent_url: str + parent_status: str + child_title: str + child_url: str + child_status: str + + +@dataclass +class PublishedContentLinkSummary: + title: str + url: str + archived_links: List[str] + redirected_links: List[str] + broken_links: List[str] + valid_links: List[str] + + +@dataclass +class ExternalContentLinkSummary: + title: str + url: str + not_found_links: List[str] + timed_out_links: List[str] + security_error_links: List[str] + inaccessible_links: List[str] + valid_links: List[str] + + +@dataclass +class AccessibleNonPublishedIssue: + title: str + url: str + status: str + e3sm_url_status: str + + +def build_records( + xml_pages: str, + xml_posts: str, + confluence_hierarchy: str, + sensitive_terms_file: str, + whitelist_file: str, + requested_links_file: str, + known_ok_links_file: str, + expected_archived_file: str = "", + keep_unchanged_links_file: str = "", +) -> Tuple[ + List[ReportRecord], + Dict[str, int], + List[RequestedLinkRecord], + List[WordpressItem], + List[Tuple[str, str]], + List[Tuple[str, str]], +]: + sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) + + confluence_map = {} + if confluence_hierarchy: + confluence_map = get_confluence_mapping(confluence_hierarchy) + + known_ok_urls = read_known_ok_links(known_ok_links_file) + keep_unchanged_urls = ( + read_keep_unchanged_links(keep_unchanged_links_file) + if keep_unchanged_links_file + else set() + ) + + raw_items: List[WordpressItem] = [] + raw_items.extend(parse_wordpress_xml_items(xml_pages, "page")) + raw_items.extend(parse_wordpress_xml_items(xml_posts, "post")) + + all_urls = [item.url for item in raw_items if item.url] + if whitelist_file: + whitelist_patterns = read_whitelist_patterns(whitelist_file) + whitelisted_urls = set(expand_patterns_to_urls(whitelist_patterns, all_urls)) + else: + whitelisted_urls = set(all_urls) + + records: List[ReportRecord] = [] + status_totals: DefaultDict[str, int] = defaultdict(int) + + for item in raw_items: + base_status = normalize_status(item.status) + report_status = base_status + + if base_status == "published": + if item.url in whitelisted_urls: + if item.url in known_ok_urls: + report_status = "published & whitelisted, known ok" + else: + report_status = "published & whitelisted" + else: + report_status = "published & not whitelisted" + + if item.url in keep_unchanged_urls: + report_status = f"{report_status}, keep unchanged" + + status_totals[report_status] += 1 + + plain_text = strip_html(item.body) + term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) + + if not term_counts: + continue + + records.append( + ReportRecord( + title=item.title, + e3sm_url=item.url, + status=report_status, + sensitive_terms=term_counts, + confluence_draft_url=confluence_map.get(item.url), + ) + ) + + flagged_urls = {record.e3sm_url for record in records} + requested_link_records = build_requested_link_records( + requested_links_file=requested_links_file, + raw_items=raw_items, + whitelisted_urls=whitelisted_urls, + flagged_urls=flagged_urls, + ) + + should_be_archived = build_should_be_archived( + raw_items=raw_items, + all_urls=all_urls, + expected_archived_file=expected_archived_file, + ) + + published_not_in_confluence = build_published_not_in_confluence( + raw_items=raw_items, + confluence_map=confluence_map, + whitelisted_urls=whitelisted_urls, + ) + + return ( + records, + dict(status_totals), + requested_link_records, + raw_items, + should_be_archived, + published_not_in_confluence, + ) + + +def build_should_be_archived( + raw_items: List[WordpressItem], + all_urls: List[str], + expected_archived_file: str, +) -> List[Tuple[str, str]]: + """ + Ported from e3sm_org_reviewer.classifiers: cross-reference a manually + curated list of e3sm.org paths that are expected to be archived against + each page/post's actual WordPress status, and flag any that have not + actually been archived yet. Returns (title, url) pairs, sorted by title. + """ + if not expected_archived_file: + return [] + + expected_archived_patterns = read_expected_archived_patterns(expected_archived_file) + expected_archived_urls = set( + expand_patterns_to_urls(expected_archived_patterns, all_urls) + ) + + if not expected_archived_urls: + return [] + + flagged: List[Tuple[str, str]] = [] + for item in raw_items: + if not item.url or item.url not in expected_archived_urls: + continue + if normalize_status(item.status) != "archived": + flagged.append((item.title, item.url)) + + return sorted(flagged, key=lambda x: x[0].lower()) + + +def build_published_not_in_confluence( + raw_items: List[WordpressItem], + confluence_map: Dict[str, str], + whitelisted_urls: Set[str], +) -> List[Tuple[str, str]]: + """ + Ported from e3sm_org_reviewer.main: published pages/posts with no + corresponding entry in the Confluence-predicted URL map, i.e. content + that's live on e3sm.org but has no source-of-truth Confluence page. + Restricted to whitelisted URLs, mirroring the original tool's + `published_not_whitelisted_and_not_in_confluence` check. Returns + (title, url) pairs, sorted by title. Empty if no Confluence hierarchy + file was supplied. + """ + if not confluence_map: + return [] + + flagged: List[Tuple[str, str]] = [] + for item in raw_items: + if not item.url or normalize_status(item.status) != "published": + continue + if item.url not in whitelisted_urls: + continue + if normalize_url(item.url) not in confluence_map: + flagged.append((item.title, item.url)) + + return sorted(flagged, key=lambda x: x[0].lower()) + + +def build_accessible_non_published_issues( + raw_items: List[WordpressItem], +) -> List[AccessibleNonPublishedIssue]: + """ + Ported from e3sm_org_reviewer.main: for every non-published page/post, + make a live, logged-out HTTP request to its e3sm.org URL and flag it if + the page is actually reachable. Draft/private/pending/future content + should 404 or redirect to a login when fetched without credentials; if + it returns 200, it's effectively public despite its WordPress status. + + This makes one live network request per non-published URL, so it's + opt-in (see `--check-non-published-access` in main.py) and can be slow + on a site with many drafts. + """ + flagged: List[AccessibleNonPublishedIssue] = [] + + for item in raw_items: + if not item.url or normalize_status(item.status) == "published": + continue + + e3sm_url_status = get_e3sm_url_status(item.url) + if e3sm_url_status == "link works not logged-in": + flagged.append( + AccessibleNonPublishedIssue( + title=item.title, + url=item.url, + status=display_status(normalize_status(item.status)), + e3sm_url_status=e3sm_url_status, + ) + ) + + return sorted(flagged, key=lambda x: x.title.lower()) + + +def build_navigation_issue_records( + items: List[WordpressItem], +) -> Tuple[List[TopLevelPageIssue], List[ArchivedParentPublishedChildIssue]]: + allowed_top_level_titles = { + "about", + "news", + "resources", + "tools", + "policies", + "home page", + } + + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + top_level_issues: List[TopLevelPageIssue] = [] + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue] = [] + + for page in pages: + normalized_status = normalize_status(page.status) + + is_top_level = ( + not page.post_parent + or page.post_parent == "0" + or page.post_parent not in item_by_id + ) + + if is_top_level and page.title.strip().lower() not in allowed_top_level_titles: + top_level_issues.append( + TopLevelPageIssue( + title=page.title, + url=page.url, + status=display_status(normalized_status), + ) + ) + + if ( + page.post_parent + and page.post_parent != "0" + and page.post_parent in item_by_id + and normalized_status == "published" + ): + parent = item_by_id[page.post_parent] + parent_status = normalize_status(parent.status) + + if parent_status == "archived": + archived_parent_published_child_issues.append( + ArchivedParentPublishedChildIssue( + parent_title=parent.title, + parent_url=parent.url, + parent_status=display_status(parent_status), + child_title=page.title, + child_url=page.url, + child_status=display_status(normalized_status), + ) + ) + + status_order = { + "Published": 0, + "Draft": 1, + "Pending": 2, + "Future": 3, + "Private": 4, + "Archived": 5, + "Unknown": 6, + } + + top_level_issues.sort( + key=lambda x: ( + status_order.get(x.status, 99), + x.title.lower(), + x.url.lower(), + ) + ) + + archived_parent_published_child_issues.sort( + key=lambda x: ( + x.parent_title.lower(), + x.child_title.lower(), + x.child_url.lower(), + ) + ) + + return top_level_issues, archived_parent_published_child_issues + + +def build_published_content_link_summaries( + items: List[WordpressItem], + post_type: str, +) -> List[PublishedContentLinkSummary]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + actual_urls = set(item_by_url.keys()) + + if post_type == "page": + ordered_items = get_page_hierarchy_order(items) + else: + ordered_items = sorted( + [item for item in items if item.post_type == post_type and item.post_id], + key=lambda x: x.title.lower(), + ) + + summaries: List[PublishedContentLinkSummary] = [] + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + + archived_links: Set[str] = set() + redirected_links: Set[str] = set() + broken_links: Set[str] = set() + valid_links: Set[str] = set() + + for linked_url in extract_internal_e3sm_links(item.body): + linked_norm = normalize_url(linked_url) + target_item = item_by_url.get(linked_norm) + + if target_item is not None: + target_status = normalize_status(target_item.status) + if target_status == "archived": + archived_links.add(linked_norm) + else: + valid_links.add(linked_norm) + continue + + redirect_target, _redirect_status = check_redirect_target(linked_norm) + if redirect_target and normalize_url(redirect_target) in actual_urls: + redirected_links.add(linked_norm) + valid_links.add(linked_norm) + else: + broken_links.add(linked_norm) + + summaries.append( + PublishedContentLinkSummary( + title=item.title, + url=item.url, + archived_links=sorted(archived_links), + redirected_links=sorted(redirected_links), + broken_links=sorted(broken_links), + valid_links=sorted(valid_links), + ) + ) + + return summaries + + +def build_external_content_link_summaries( + items: List[WordpressItem], + post_type: str, + inaccessible_prefixes: Tuple[str, ...] = (), +) -> List[ExternalContentLinkSummary]: + """ + Mirror of build_published_content_link_summaries, but for external links. + + Deduplicates URLs across all items before fetching so each external URL + is checked exactly once. + + URLs whose prefix matches any entry in `inaccessible_prefixes` are skipped + entirely (no network request) and reported in the "inaccessible" column. + """ + if post_type == "page": + ordered_items = get_page_hierarchy_order(items) + else: + ordered_items = sorted( + [item for item in items if item.post_type == post_type and item.post_id], + key=lambda x: x.title.lower(), + ) + + # Collect all unique external URLs first to avoid redundant fetches. + all_external_urls: Set[str] = set() + item_to_external_urls: Dict[str, Set[str]] = {} + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + found = extract_external_links(item.body) + item_to_external_urls[item.url] = found + all_external_urls.update(found) + + # Check each unique URL once, skipping known-inaccessible prefixes. + url_results: Dict[str, str] = {} + for ext_url in sorted(all_external_urls): + if inaccessible_prefixes and any( + ext_url.startswith(prefix) for prefix in inaccessible_prefixes + ): + url_results[ext_url] = "inaccessible" + else: + url_results[ext_url] = check_external_link(ext_url).status + + summaries: List[ExternalContentLinkSummary] = [] + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + + ext_urls = item_to_external_urls.get(item.url, set()) + if not ext_urls: + continue + + not_found: Set[str] = set() + timed_out: Set[str] = set() + security_error: Set[str] = set() + inaccessible: Set[str] = set() + valid: Set[str] = set() + + for ext_url in ext_urls: + status = url_results.get(ext_url, "not_found") + if status == "valid": + valid.add(ext_url) + elif status == "timed_out": + timed_out.add(ext_url) + elif status == "security_error": + security_error.add(ext_url) + elif status == "inaccessible": + inaccessible.add(ext_url) + else: + not_found.add(ext_url) + + summaries.append( + ExternalContentLinkSummary( + title=item.title, + url=item.url, + not_found_links=sorted(not_found), + timed_out_links=sorted(timed_out), + security_error_links=sorted(security_error), + inaccessible_links=sorted(inaccessible), + valid_links=sorted(valid), + ) + ) + + return summaries + + +def build_requested_link_records( + requested_links_file: str, + raw_items: List[WordpressItem], + whitelisted_urls: Set[str], + flagged_urls: Set[str], +) -> List[RequestedLinkRecord]: + requested_rows = read_requested_links(requested_links_file) + item_by_url = {item.url: item for item in raw_items if item.url} + + records: List[RequestedLinkRecord] = [] + for e3sm_url, requesting_urls in requested_rows: + item = item_by_url.get(e3sm_url) + + if item is None: + current_status = "Not found" + currently_whitelisted = False + else: + current_status = display_status(normalize_status(item.status)) + currently_whitelisted = e3sm_url in whitelisted_urls + + records.append( + RequestedLinkRecord( + e3sm_url=e3sm_url, + included_later=e3sm_url in flagged_urls, + current_status=current_status, + currently_whitelisted=currently_whitelisted, + requesting_urls=requesting_urls, + ) + ) + + return records + + +def get_page_hierarchy_order(items: List[WordpressItem]) -> List[WordpressItem]: + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + for item in pages: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in pages + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + ordered: List[WordpressItem] = [] + seen: Set[str] = set() + + def walk(node: WordpressItem) -> None: + if node.post_id in seen: + return + seen.add(node.post_id) + ordered.append(node) + for child in children_by_parent.get(node.post_id, []): + walk(child) + + for root in roots: + walk(root) + + return ordered + + +def sort_requested_link_records( + requested_link_records: List[RequestedLinkRecord], +) -> List[RequestedLinkRecord]: + status_order = { + "Published": 0, + "Archived": 1, + "Not found": 3, + } + + return sorted( + requested_link_records, + key=lambda r: ( + 0 if r.included_later else 1, + status_order.get(r.current_status, 2), + 0 if r.currently_whitelisted else 1, + r.e3sm_url.lower(), + ), + ) diff --git a/e3sm_comms/exported_xml_reviewer/confluence.py b/e3sm_comms/exported_xml_reviewer/confluence.py new file mode 100644 index 0000000..215e616 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/confluence.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Dict + +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.utils import normalize_url, parse_confluence_hierarchy_file + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def get_confluence_mapping(input_file: str) -> Dict[str, str]: + mapping: Dict[str, str] = {} + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + mapping[normalize_url(e3sm_url)] = confluence_url + except Exception as exc: + print(f"Could not map {confluence_url}: {exc}") + + return mapping + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" diff --git a/e3sm_comms/exported_xml_reviewer/link_analysis.py b/e3sm_comms/exported_xml_reviewer/link_analysis.py new file mode 100644 index 0000000..2b11832 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/link_analysis.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, List, Set, Tuple +from urllib.parse import urlsplit + +import requests # type: ignore + +from e3sm_comms.exported_xml_reviewer.utils import ( + display_status, + is_legacy_content_url, + normalize_status, +) +from e3sm_comms.utils import WordpressItem, normalize_url + +_EXTERNAL_TIMEOUT = 15 # seconds + + +@dataclass +class InvalidInternalLinkGroup: + linked_url: str + redirect_target: str + redirect_status: str + inferred_link: str + found_under_different_prefix: str + linked_target_status: str + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] + + +@dataclass +class NonPublishedInternalLinkGroup: + linked_url: str + target_status: str + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] + + +@dataclass +class ExternalLinkResult: + """Outcome of checking a single external URL.""" + + url: str + status: str # "valid" | "not_found" | "timed_out" | "security_error" + + +def extract_internal_e3sm_links(html_text: str) -> Set[str]: + links: Set[str] = set() + + for match in re.finditer( + r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE + ): + href = match.group(1).strip() + if not href: + continue + + try: + parts = urlsplit(href) + except ValueError: + continue + + host = parts.netloc.lower() + path = parts.path.lower() + + if host == "docs.e3sm.org": + continue + + if host.endswith("e3sm.org"): + if path.startswith("/wp-content"): + continue + + normalized = normalize_url(href) + if is_legacy_content_url(normalized): + continue + + links.add(normalized) + elif not parts.scheme and not parts.netloc and href.startswith("/"): + normalized_relative = normalize_url(f"https://e3sm.org{href}") + if "/wp-content" in normalized_relative.lower(): + continue + if is_legacy_content_url(normalized_relative): + continue + + links.add(normalized_relative) + + return links + + +def check_redirect_target(link_url: str) -> Tuple[str, str]: + try: + response = requests.get(link_url, timeout=10, allow_redirects=True) + redirect_status = "" + + if response.history: + first_response = response.history[0] + if first_response.status_code in {301, 302, 303, 307, 308} and response.ok: + final_url = normalize_url(response.url) + redirect_status = str(first_response.status_code) + return final_url, redirect_status + + return "", "" + + except (requests.exceptions.Timeout, requests.exceptions.RequestException): + return "", "" + + +def extract_external_links(html_text: str) -> Set[str]: + links: Set[str] = set() + for match in re.finditer( + r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE + ): + href = match.group(1).strip() + if not href or href.startswith("#") or href.startswith("mailto:"): + continue + try: + parts = urlsplit(href) + except ValueError: + continue + if parts.scheme not in ("http", "https"): # changed from `if not parts.scheme` + continue + if not parts.netloc: + continue + if " " in parts.netloc: + continue + if parts.netloc.lower().endswith("e3sm.org"): + continue + links.add(href) + return links + + +def check_external_link(url: str) -> ExternalLinkResult: + """ + HEAD-then-GET a URL and return an ExternalLinkResult. + + Outcomes + -------- + "valid" – 2xx response (or safe redirect to one) + "not_found" – 4xx / 5xx response, or connection error + "timed_out" – requests.Timeout + "security_error" – SSL error (mirrors Firefox "potential security risk ahead") + """ + headers = {"User-Agent": "Mozilla/5.0 (compatible; e3sm-link-checker/1.0)"} + try: + try: + resp = requests.head( + url, + timeout=_EXTERNAL_TIMEOUT, + allow_redirects=True, + headers=headers, + ) + if resp.status_code == 405: + raise requests.exceptions.HTTPError("HEAD not allowed") + except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError): + resp = requests.get( + url, + timeout=_EXTERNAL_TIMEOUT, + allow_redirects=True, + headers=headers, + stream=True, + ) + resp.close() + + if resp.ok: + return ExternalLinkResult(url=url, status="valid") + return ExternalLinkResult(url=url, status="not_found") + + except requests.exceptions.Timeout: + return ExternalLinkResult(url=url, status="timed_out") + except requests.exceptions.SSLError: + return ExternalLinkResult(url=url, status="security_error") + except requests.exceptions.RequestException: + return ExternalLinkResult(url=url, status="not_found") + + +def infer_likely_new_link(linked_url: str) -> str: + parts = urlsplit(linked_url) + path = parts.path.rstrip("/").lower() + + if path.startswith("/model"): + suffix = parts.path[len("/model") :].lstrip("/") + return f"https://e3sm.org/resources/model/{suffix}".rstrip("/") + + if path.startswith("/data"): + suffix = parts.path[len("/data") :].lstrip("/") + return f"https://e3sm.org/resources/data/{suffix}".rstrip("/") + + if path.startswith("/about/news"): + suffix = parts.path[len("/about/news") :].lstrip("/") + return f"https://e3sm.org/news/{suffix}".rstrip("/") + + if path.startswith("/resources/policies"): + suffix = parts.path[len("/resources/policies") :].lstrip("/") + return f"https://e3sm.org/policies/{suffix}".rstrip("/") + + if path.startswith("/resources/tools"): + suffix = parts.path[len("/resources/tools") :].lstrip("/") + return f"https://e3sm.org/tools/{suffix}".rstrip("/") + + return "" + + +def guess_redirect_target(linked_url: str, actual_urls: Set[str]) -> str: + parts = urlsplit(linked_url) + path = parts.path.strip("/").lower() + + if not path: + return "" + + slug = path.split("/")[-1] + candidates = [] + + for actual_url in actual_urls: + actual_parts = urlsplit(actual_url) + actual_path = actual_parts.path.strip("/").lower() + + if actual_path.endswith("/" + slug) or actual_path == slug: + candidates.append(normalize_url(actual_url)) + + if len(candidates) == 1: + return candidates[0] + + return "" + + +def build_invalid_internal_link_groups( + items: List[WordpressItem], +) -> List[InvalidInternalLinkGroup]: + actual_urls = {normalize_url(item.url) for item in items if item.url} + item_by_url = {normalize_url(item.url): item for item in items if item.url} + + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) + + for item in items: + if not item.url or not item.body: + continue + + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + + for linked_url in extract_internal_e3sm_links(item.body): + if linked_url not in actual_urls: + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) + + groups: List[InvalidInternalLinkGroup] = [] + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: + redirect_target, redirect_status = check_redirect_target(linked_url) + + inferred_candidate = infer_likely_new_link(linked_url) + inferred_link = ( + inferred_candidate + if inferred_candidate and normalize_url(inferred_candidate) in actual_urls + else "" + ) + + found_under_different_prefix = "" + if not inferred_link: + guessed_candidate = guess_redirect_target(linked_url, actual_urls) + if guessed_candidate and normalize_url(guessed_candidate) in actual_urls: + found_under_different_prefix = guessed_candidate + + status_target = inferred_link or found_under_different_prefix + linked_target_status = "" + if status_target: + matched_item = item_by_url.get(normalize_url(status_target)) + if matched_item is not None: + linked_target_status = display_status( + normalize_status(matched_item.status) + ) + + groups.append( + InvalidInternalLinkGroup( + linked_url=linked_url, + redirect_target=redirect_target, + redirect_status=redirect_status, + inferred_link=inferred_link, + found_under_different_prefix=found_under_different_prefix, + linked_target_status=linked_target_status, + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + ) + ) + + def status_rank(status: str) -> int: + status = status.lower() + if status == "published": + return 0 + if status == "archived": + return 1 + return 2 + + def inference_rank(group: InvalidInternalLinkGroup) -> int: + if group.inferred_link: + return 0 + if group.found_under_different_prefix: + return 1 + return 2 + + def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, int, str]: + return ( + 0 if group.redirect_target else 1, + inference_rank(group), + status_rank(group.linked_target_status), + group.linked_url.lower(), + ) + + groups.sort(key=sort_key) + return groups + + +def build_non_published_internal_link_groups( + items: List[WordpressItem], +) -> List[NonPublishedInternalLinkGroup]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) + + for item in items: + if not item.url or not item.body: + continue + + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + + for linked_url in extract_internal_e3sm_links(item.body): + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + normalized_target_status = normalize_status(matched_item.status) + if normalized_target_status == "published": + continue + + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) + + groups: List[NonPublishedInternalLinkGroup] = [] + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + groups.append( + NonPublishedInternalLinkGroup( + linked_url=linked_url, + target_status=display_status(normalize_status(matched_item.status)), + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + ) + ) + + groups.sort( + key=lambda group: (group.target_status.lower(), group.linked_url.lower()) + ) + return groups diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py new file mode 100644 index 0000000..5585502 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import argparse + +from e3sm_comms.exported_xml_reviewer.builders import ( + build_accessible_non_published_issues, + build_external_content_link_summaries, + build_navigation_issue_records, + build_published_content_link_summaries, + build_records, +) +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + build_invalid_internal_link_groups, + build_non_published_internal_link_groups, +) +from e3sm_comms.exported_xml_reviewer.readers import read_inaccessible_prefixes +from e3sm_comms.exported_xml_reviewer.reporters import ( + write_external_links_report, + write_hierarchical_outline, + write_invalid_internal_links_report, + write_navigation_issues_report, + write_non_published_accessibility_report, + write_published_pages_link_report, + write_terms_report, +) +from e3sm_comms.utils import IO_DIR + +# ----------------------------------------------------------------------------- +# Configuration +# ----------------------------------------------------------------------------- + +# Required inputs: +INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" +INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" +# These 3 inputs are only used for wordpress_sensitive_terms_report.md: +INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" +INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" +INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" +INPUT_INACCESSIBLE_PREFIXES: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/inaccessible_prefixes.txt" +) + +# Optional inputs: +DEFAULT_CONFLUENCE_HIERARCHY: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" +) +DEFAULT_WHITELIST: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +) +# Ported from e3sm_org_reviewer: same shared file it used for this input. +DEFAULT_EXPECTED_ARCHIVED: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" +DEFAULT_KEEP_UNCHANGED: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/keep_unchanged_web_pages.txt" +) + +# Outputs: +OUTPUT_TERMS_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" +) +OUTPUT_HIERARCHICAL_OUTLINE: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.txt" +) +OUTPUT_NAVIGATION_ISSUES_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" +) +OUTPUT_INVALID_INTERNAL_LINKS_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" +) +OUTPUT_PUBLISHED_PAGES_LINK_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" +) +OUTPUT_EXTERNAL_LINKS_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_external_links_report.md" +) +OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_non_published_accessibility_report.md" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Review exported WordPress XML files.") + parser.add_argument( + "--use-confluence", + action="store_true", + help="Use the Confluence hierarchy file for hierarchical outline generation.", + ) + parser.add_argument( + "--use-whitelist", + action="store_true", + help="Use the whitelisted web pages file to filter results.", + ) + parser.add_argument( + "--use-expected-archived", + action="store_true", + help=( + "Flag pages/posts that are expected to be archived (per the shared " + "archived_web_pages.txt list) but are not yet archived in WordPress." + ), + ) + parser.add_argument( + "--use-keep-unchanged", + action="store_true", + help="Annotate pages/posts that are on the keep-unchanged exception list.", + ) + parser.add_argument( + "--check-non-published-access", + action="store_true", + help=( + "Make a live, logged-out HTTP request to every non-published " + "page/post and flag any that are actually reachable. Makes one " + "network request per non-published URL, so this is slow and " + "off by default." + ), + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + input_confluence_hierarchy = ( + DEFAULT_CONFLUENCE_HIERARCHY if args.use_confluence else "" + ) + input_whitelist = DEFAULT_WHITELIST if args.use_whitelist else "" + input_expected_archived = ( + DEFAULT_EXPECTED_ARCHIVED if args.use_expected_archived else "" + ) + input_keep_unchanged = DEFAULT_KEEP_UNCHANGED if args.use_keep_unchanged else "" + + inaccessible_prefixes = read_inaccessible_prefixes(INPUT_INACCESSIBLE_PREFIXES) + + ( + records, + status_totals, + requested_link_records, + raw_items, + should_be_archived, + published_not_in_confluence, + ) = build_records( + xml_pages=INPUT_XML_PAGES, + xml_posts=INPUT_XML_POSTS, + confluence_hierarchy=input_confluence_hierarchy, + sensitive_terms_file=INPUT_SEARCH_PHRASES, + whitelist_file=input_whitelist, + requested_links_file=INPUT_REQUESTED_LINKS, + known_ok_links_file=INPUT_KNOWN_OK_LINKS, + expected_archived_file=input_expected_archived, + keep_unchanged_links_file=input_keep_unchanged, + ) + + write_terms_report( + OUTPUT_TERMS_REPORT, + records, + status_totals, + requested_link_records, + ) + + write_hierarchical_outline( + OUTPUT_HIERARCHICAL_OUTLINE, + raw_items, + ) + + top_level_issues, archived_parent_published_child_issues = ( + build_navigation_issue_records(raw_items) + ) + + write_navigation_issues_report( + OUTPUT_NAVIGATION_ISSUES_REPORT, + top_level_issues, + archived_parent_published_child_issues, + should_be_archived=should_be_archived, + published_not_in_confluence=published_not_in_confluence, + ) + + invalid_link_groups = build_invalid_internal_link_groups(raw_items) + non_published_link_groups = build_non_published_internal_link_groups(raw_items) + write_invalid_internal_links_report( + OUTPUT_INVALID_INTERNAL_LINKS_REPORT, + invalid_link_groups, + non_published_link_groups, + ) + + published_page_link_summaries = build_published_content_link_summaries( + raw_items, + "page", + ) + published_post_link_summaries = build_published_content_link_summaries( + raw_items, + "post", + ) + write_published_pages_link_report( + OUTPUT_PUBLISHED_PAGES_LINK_REPORT, + published_page_link_summaries, + published_post_link_summaries, + ) + + external_page_summaries = build_external_content_link_summaries( + raw_items, "page", inaccessible_prefixes + ) + external_post_summaries = build_external_content_link_summaries( + raw_items, "post", inaccessible_prefixes + ) + write_external_links_report( + OUTPUT_EXTERNAL_LINKS_REPORT, + external_page_summaries, + external_post_summaries, + ) + + if args.check_non_published_access: + accessibility_issues = build_accessible_non_published_issues(raw_items) + write_non_published_accessibility_report( + OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT, + accessibility_issues, + ) + print( + "Wrote non-published accessibility report to " + f"{OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT}" + ) + + print(f"Wrote report to {OUTPUT_TERMS_REPORT}") + print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") + print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") + print( + f"Wrote invalid internal links report to {OUTPUT_INVALID_INTERNAL_LINKS_REPORT}" + ) + print(f"Wrote published pages link report to {OUTPUT_PUBLISHED_PAGES_LINK_REPORT}") + print(f"Wrote external links report to {OUTPUT_EXTERNAL_LINKS_REPORT}") + + +if __name__ == "__main__": + main() diff --git a/e3sm_comms/exported_xml_reviewer/readers.py b/e3sm_comms/exported_xml_reviewer/readers.py new file mode 100644 index 0000000..d9e1fea --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/readers.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import csv +from typing import List, Set, Tuple + +from e3sm_comms.utils import normalize_url, read_lines + + +def read_sensitive_terms(file_path: str) -> List[str]: + return sorted(set(read_lines(file_path, lowercase=True))) + + +def read_known_ok_links(file_path: str) -> Set[str]: + with open(file_path, "r", encoding="utf-8") as f: + return {normalize_url(line.strip()) for line in f if line.strip()} + + +def read_inaccessible_prefixes(file_path: str) -> Tuple[str, ...]: + """ + Read a plain-text file of URL prefixes (one per line) that are known to + block automated access. Blank lines and lines starting with '#' are + ignored. Returns a tuple suitable for use with str.startswith(). + """ + with open(file_path, "r", encoding="utf-8") as f: + prefixes = [ + line.strip() + for line in f + if line.strip() and not line.strip().startswith("#") + ] + return tuple(prefixes) + + +def read_requested_links(file_path: str) -> List[Tuple[str, str]]: + rows: List[Tuple[str, str]] = [] + + with open(file_path, "r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + + if not reader.fieldnames: + print(f"Requested links CSV has no headers: {file_path}") + return rows + + normalized_to_actual = { + header.strip().lower(): header for header in reader.fieldnames if header + } + + e3sm_header = normalized_to_actual.get("e3sm.org link") + requesting_header = normalized_to_actual.get( + "list of urls that wants to link to it" + ) + + if requesting_header is None: + for candidate in [ + "list of urls that want to link to it", + "requesting urls", + "requesting url", + "list of urls", + ]: + requesting_header = normalized_to_actual.get(candidate) + if requesting_header: + break + + if e3sm_header is None: + print( + f"Requested links CSV is missing required column 'e3sm.org link'. " + f"Found headers: {reader.fieldnames}" + ) + return rows + + if requesting_header is None: + print( + "Requested links CSV could not find the requesting URLs column. " + f"Found headers: {reader.fieldnames}" + ) + + for row in reader: + e3sm_url = normalize_url(row.get(e3sm_header, "")) + requesting_urls = ( + row.get(requesting_header, "").strip() if requesting_header else "" + ) + + if e3sm_url: + rows.append((e3sm_url, requesting_urls)) + + return rows + + +def read_whitelist_patterns(file_path: str) -> List[str]: + return read_lines(file_path) + + +def read_expected_archived_patterns(file_path: str) -> List[str]: + return read_lines(file_path) + + +def read_keep_unchanged_links(file_path: str) -> Set[str]: + with open(file_path, "r", encoding="utf-8") as f: + return {normalize_url(line.strip()) for line in f if line.strip()} diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py new file mode 100644 index 0000000..a2cd031 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -0,0 +1,680 @@ +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import DefaultDict, Dict, List, Set + +from e3sm_comms.exported_xml_reviewer.builders import ( + AccessibleNonPublishedIssue, + ArchivedParentPublishedChildIssue, + ExternalContentLinkSummary, + PublishedContentLinkSummary, + ReportRecord, + RequestedLinkRecord, + TopLevelPageIssue, + sort_requested_link_records, +) +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + InvalidInternalLinkGroup, + NonPublishedInternalLinkGroup, +) +from e3sm_comms.exported_xml_reviewer.utils import display_status, normalize_status +from e3sm_comms.utils import WordpressItem + + +def write_terms_report( + output_path: str, + records: List[ReportRecord], + status_totals: Dict[str, int], + requested_link_records: List[RequestedLinkRecord], +) -> None: + grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) + for record in records: + grouped[record.status].append(record) + + for status in grouped: + grouped[status].sort( + key=lambda r: (-sum(r.sensitive_terms.values()), r.title.lower()) + ) + + ordered_statuses = [ + "published & whitelisted, known ok", + "published & whitelisted", + "published & not whitelisted", + "archived", + "draft", + "future", + "pending", + "private", + "unknown", + ] + + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Sensitive Terms Report\n\n") + f.write( + "The detailed sections below include only e3sm.org pages/posts where one or more sensitive terms were found. " + "The summary table includes counts for both flagged and unflagged items.\n\n" + ) + + f.write("| Status | With sensitive terms | Without sensitive terms | Total |\n") + f.write("| --- | ---: | ---: | ---: |\n") + + total_with_terms = 0 + total_without_terms = 0 + + all_summary_statuses = set(status_totals) | set(grouped) + extra_statuses = sorted( + s for s in all_summary_statuses if s not in ordered_statuses + ) + + for status in ordered_statuses + extra_statuses: + total_in_status = status_totals.get(status, 0) + with_terms = len(grouped.get(status, [])) + without_terms = total_in_status - with_terms + + if total_in_status == 0 and with_terms == 0: + continue + + total_with_terms += with_terms + total_without_terms += without_terms + f.write( + f"| {status} | {with_terms} | {without_terms} | {total_in_status} |\n" + ) + + grand_total = total_with_terms + total_without_terms + f.write( + f"| TOTAL | {total_with_terms} | {total_without_terms} | {grand_total} |\n" + ) + f.write("\n") + + if requested_link_records: + requested_link_records = sort_requested_link_records(requested_link_records) + + f.write("## Requested Links\n\n") + f.write( + "| e3sm.org link | Included later on this page? | Current status | Currently whitelisted? | Requesting URLs |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + for requested_record in requested_link_records: + included_later = ( + "Yes" + if requested_record.included_later + else "No (i.e., contains no sensitive terms)" + ) + currently_whitelisted = ( + "Yes" if requested_record.currently_whitelisted else "No" + ) + f.write( + f"| {requested_record.e3sm_url} | {included_later} | {requested_record.current_status} | " + f"{currently_whitelisted} | {requested_record.requesting_urls} |\n" + ) + + f.write("\n") + + all_statuses = ordered_statuses + extra_statuses + seen = set() + + for status in all_statuses: + if status not in grouped or status in seen: + continue + seen.add(status) + + f.write(f"## {status.capitalize()} ({len(grouped[status])})\n\n") + + for idx, record in enumerate(grouped[status], start=1): + e3sm_md = f"[e3sm.org]({record.e3sm_url})" + confluence_md = ( + f" [(confluence draft)]({record.confluence_draft_url})" + if record.confluence_draft_url + else "" + ) + + f.write( + f"{idx}. {record.title}: {e3sm_md}{confluence_md} -- {record.sensitive_terms}\n" + ) + + f.write("\n") + + +def write_invalid_internal_links_report( + output_path: str, + groups: List[InvalidInternalLinkGroup], + non_published_groups: List[NonPublishedInternalLinkGroup], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: + f.write( + "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on these published pages | Referenced on these non-published pages |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- | --- |\n") + + for group in table_groups: + redirect_md = ( + f"[{group.redirect_target}]({group.redirect_target})" + if group.redirect_target + else "" + ) + if redirect_md and group.redirect_status: + redirect_md = ( + f"{redirect_md} (redirect status: {group.redirect_status})" + ) + + inferred_md = ( + f"[{group.inferred_link}]({group.inferred_link})" + if group.inferred_link + else "" + ) + prefix_md = ( + f"[{group.found_under_different_prefix}]({group.found_under_different_prefix})" + if group.found_under_different_prefix + else "" + ) + + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published + ) + + f.write( + f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_published} | {referenced_non_published} |\n" + ) + + def render_non_published_table( + f, table_groups: List[NonPublishedInternalLinkGroup] + ) -> None: + f.write( + "| Valid linked URL | Target status | Referenced on these published pages | Referenced on these non-published pages |\n" + ) + f.write("| --- | --- | --- | --- |\n") + + for group in table_groups: + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published + ) + f.write( + f"| {group.linked_url} | {group.target_status} | {referenced_published} | {referenced_non_published} |\n" + ) + + working_redirects: List[InvalidInternalLinkGroup] = [] + published_targets: List[InvalidInternalLinkGroup] = [] + archived_targets: List[InvalidInternalLinkGroup] = [] + no_candidate: List[InvalidInternalLinkGroup] = [] + + for group in groups: + if group.redirect_target: + working_redirects.append(group) + elif group.linked_target_status == "Published": + published_targets.append(group) + elif group.linked_target_status == "Archived": + archived_targets.append(group) + else: + no_candidate.append(group) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Invalid Internal e3sm.org Links\n\n") + + if not groups and not non_published_groups: + f.write("No invalid internal links found.\n") + return + + f.write( + f"## 1. These have working redirections already ({len(working_redirects)})\n\n" + ) + if working_redirects: + render_table(f, working_redirects) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 2. The target pages are published, we just need to set up the redirections ({len(published_targets)})\n\n" + ) + if published_targets: + render_table(f, published_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write(f"## 3. The target pages are archived ({len(archived_targets)})\n\n") + if archived_targets: + render_table(f, archived_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 4. Couldn't find a redirection candidate ({len(no_candidate)})\n\n" + ) + if no_candidate: + render_table(f, no_candidate) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 5. Technically valid links that point to non-published targets ({len(non_published_groups)})\n\n" + ) + if non_published_groups: + render_non_published_table(f, non_published_groups) + else: + f.write("None found.\n") + + +def write_published_pages_link_report( + output_path: str, + page_summaries: List[PublishedContentLinkSummary], + post_summaries: List[PublishedContentLinkSummary], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_link_list(urls: List[str]) -> str: + return ", ".join(f"[{url}]({url})" for url in urls) + + def write_section( + f, + section_title: str, + summaries: List[PublishedContentLinkSummary], + ) -> None: + invalid_summaries = [ + s + for s in summaries + if s.archived_links or s.redirected_links or s.broken_links + ] + valid_only_summaries = [ + s + for s in summaries + if not (s.archived_links or s.redirected_links or s.broken_links) + ] + + f.write(f"## {section_title}\n\n") + + if not summaries: + f.write("No published items with links found.\n\n") + return + + f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") + if invalid_summaries: + f.write( + "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + archived_total = 0 + redirected_total = 0 + broken_total = 0 + valid_total = 0 + + archived_unique: Set[str] = set() + redirected_unique: Set[str] = set() + broken_unique: Set[str] = set() + valid_unique: Set[str] = set() + + for summary in invalid_summaries: + item_md = f"[{summary.title}]({summary.url})" + archived_md = render_link_list(summary.archived_links) + redirected_md = render_link_list(summary.redirected_links) + broken_md = render_link_list(summary.broken_links) + valid_md = render_link_list(summary.valid_links) + + archived_total += len(summary.archived_links) + redirected_total += len(summary.redirected_links) + broken_total += len(summary.broken_links) + valid_total += len(summary.valid_links) + + archived_unique.update(summary.archived_links) + redirected_unique.update(summary.redirected_links) + broken_unique.update(summary.broken_links) + valid_unique.update(summary.valid_links) + + f.write( + f"| {item_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" + ) + + f.write( + f"| Total link count | {archived_total} | {redirected_total} | {broken_total} | {valid_total} |\n" + ) + f.write( + f"| Unique link count | {len(archived_unique)} | {len(redirected_unique)} | {len(broken_unique)} | {len(valid_unique)} |\n" + ) + else: + f.write("No items with invalid links found.\n") + + f.write("\n") + f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") + + if valid_only_summaries: + f.write("| Published item | valid e3sm.org link count |\n") + f.write("| --- | ---: |\n") + + total_links = 0 + unique_links: Set[str] = set() + + for summary in valid_only_summaries: + item_md = f"[{summary.title}]({summary.url})" + f.write(f"| {item_md} | {len(summary.valid_links)} |\n") + total_links += len(summary.valid_links) + unique_links.update(summary.valid_links) + + f.write(f"| Total link count | {total_links} |\n") + f.write(f"| Unique link count | {len(unique_links)} |\n") + else: + f.write("No items with only valid links found.\n") + + f.write("\n") + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Content Invalid Link Report\n\n") + write_section(f, "Published Pages", page_summaries) + write_section(f, "Published Posts", post_summaries) + + +def write_external_links_report( + output_path: str, + page_summaries: List[ExternalContentLinkSummary], + post_summaries: List[ExternalContentLinkSummary], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_link_list(urls: List[str]) -> str: + return ", ".join(f"[{url}]({url})" for url in urls) + + def write_section( + f, + section_title: str, + summaries: List[ExternalContentLinkSummary], + ) -> None: + invalid_summaries = [ + s + for s in summaries + if s.not_found_links + or s.timed_out_links + or s.security_error_links + or s.inaccessible_links + ] + valid_only_summaries = [ + s + for s in summaries + if not ( + s.not_found_links + or s.timed_out_links + or s.security_error_links + or s.inaccessible_links + ) + ] + + f.write(f"## {section_title}\n\n") + + if not summaries: + f.write("No published items with external links found.\n\n") + return + + f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") + + if invalid_summaries: + f.write( + "Known inaccessible to script: these are likely accessible manually, or are fake Lorem Ipsum links\n" + ) + f.write( + "| Published item | Link not found | Link timed out | Security error | Known inaccessible to script | Valid link |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- |\n") + + not_found_total = 0 + timed_out_total = 0 + security_total = 0 + inaccessible_total = 0 + valid_total = 0 + + not_found_unique: Set[str] = set() + timed_out_unique: Set[str] = set() + security_unique: Set[str] = set() + inaccessible_unique: Set[str] = set() + valid_unique: Set[str] = set() + + for s in invalid_summaries: + item_md = f"[{s.title}]({s.url})" + not_found_md = render_link_list(s.not_found_links) + timed_out_md = render_link_list(s.timed_out_links) + security_md = render_link_list(s.security_error_links) + inaccessible_md = render_link_list(s.inaccessible_links) + valid_md = render_link_list(s.valid_links) + + not_found_total += len(s.not_found_links) + timed_out_total += len(s.timed_out_links) + security_total += len(s.security_error_links) + inaccessible_total += len(s.inaccessible_links) + valid_total += len(s.valid_links) + + not_found_unique.update(s.not_found_links) + timed_out_unique.update(s.timed_out_links) + security_unique.update(s.security_error_links) + inaccessible_unique.update(s.inaccessible_links) + valid_unique.update(s.valid_links) + + f.write( + f"| {item_md} | {not_found_md} | {timed_out_md} | {security_md} | {inaccessible_md} | {valid_md} |\n" + ) + + f.write( + f"| Total link count | {not_found_total} | {timed_out_total} | {security_total} | {inaccessible_total} | {valid_total} |\n" + ) + f.write( + f"| Unique link count | {len(not_found_unique)} | {len(timed_out_unique)} | {len(security_unique)} | {len(inaccessible_unique)} | {len(valid_unique)} |\n" + ) + else: + f.write("No items with invalid external links found.\n") + + f.write("\n") + f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") + + if valid_only_summaries: + f.write("| Published item | Valid external link count |\n") + f.write("| --- | ---: |\n") + + total_links = 0 + unique_links: Set[str] = set() + + for s in valid_only_summaries: + item_md = f"[{s.title}]({s.url})" + f.write(f"| {item_md} | {len(s.valid_links)} |\n") + total_links += len(s.valid_links) + unique_links.update(s.valid_links) + + f.write(f"| Total link count | {total_links} |\n") + f.write(f"| Unique link count | {len(unique_links)} |\n") + else: + f.write("No items with only valid external links found.\n") + + f.write("\n") + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Content External Links Report\n\n") + write_section(f, "Published Pages", page_summaries) + write_section(f, "Published Posts", post_summaries) + + +def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def write_section(f, section_items: List[WordpressItem], heading: str) -> None: + from collections import defaultdict as _defaultdict + + section_items = [item for item in section_items if item.post_id] + + children_by_parent: DefaultDict[str, List[WordpressItem]] = _defaultdict(list) + item_by_id: Dict[str, WordpressItem] = {} + + for item in section_items: + item_by_id[item.post_id] = item + + for item in section_items: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in section_items + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + f.write(f"{heading}\n") + + seen: Set[str] = set() + + def walk(node: WordpressItem, depth: int) -> None: + indent = " " * depth + status_label = display_status(normalize_status(node.status)) + line = f"{indent}{node.title} [{status_label}]" + if node.url: + line += f" [{node.url}]" + f.write(line + "\n") + + if node.post_id in seen: + return + + seen.add(node.post_id) + + for child in children_by_parent.get(node.post_id, []): + walk(child, depth + 1) + + for root in roots: + walk(root, 0) + + f.write("\n") + + pages = [item for item in items if item.post_type == "page"] + posts = [item for item in items if item.post_type == "post"] + + with open(output_path, "w", encoding="utf-8") as f: + write_section(f, pages, "Pages") + write_section(f, posts, "Posts") + + +def write_non_published_accessibility_report( + output_path: str, + issues: List[AccessibleNonPublishedIssue], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Non-Published Pages That Are Still Accessible\n\n") + f.write( + "For every page/post whose WordPress status is not \"publish\", this " + "checks the live e3sm.org URL (logged out) and flags it here if it's " + "actually reachable without logging in.\n\n" + ) + + if not issues: + f.write("No incorrectly accessible non-published pages found.\n") + return + + f.write(f"## Found {len(issues)} issue(s)\n\n") + f.write("| Title | WordPress status | URL | Live check result |\n") + f.write("| --- | --- | --- | --- |\n") + for issue in issues: + f.write( + f"| {issue.title} | {issue.status} | {issue.url} | {issue.e3sm_url_status} |\n" + ) + + +def write_navigation_issues_report( + output_path: str, + top_level_issues: List[TopLevelPageIssue], + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue], + should_be_archived: List[tuple] = (), + published_not_in_confluence: List[tuple] = (), +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Navigation Issues Report\n\n") + + f.write("## 1. Top-level pages that are not expected top-level tabs\n\n") + f.write( + "Expected top-level tabs are: About, News, Resources, Tools, Policies, Home Page.\n\n" + ) + + if top_level_issues: + f.write("| Title | Status | URL |\n") + f.write("| --- | --- | --- |\n") + for top_level_issue in top_level_issues: + f.write( + f"| {top_level_issue.title} | {top_level_issue.status} | {top_level_issue.url} |\n" + ) + else: + f.write("No unexpected top-level pages found.\n") + + f.write("\n") + + f.write("## 2. Published child pages under archived parent pages\n\n") + + if archived_parent_published_child_issues: + f.write( + "| Parent title | Parent status | Parent URL | Child title | Child status | Child URL |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- |\n") + for archived_child_issue in archived_parent_published_child_issues: + f.write( + f"| {archived_child_issue.parent_title} | {archived_child_issue.parent_status} | {archived_child_issue.parent_url} | " + f"{archived_child_issue.child_title} | {archived_child_issue.child_status} | {archived_child_issue.child_url} |\n" + ) + else: + f.write( + "No published child pages were found under archived parent pages.\n" + ) + + f.write("\n") + + f.write( + "## 3. Expecting to be archived, but not yet archived " + f"({len(should_be_archived)})\n\n" + ) + if should_be_archived: + f.write("| Title | URL |\n") + f.write("| --- | --- |\n") + for title, url in should_be_archived: + f.write(f"| {title} | {url} |\n") + else: + f.write( + "No pages/posts found (or `--use-expected-archived` was not passed).\n" + ) + + f.write("\n") + + f.write( + "## 4. Published & whitelisted, but no matching Confluence page found " + f"({len(published_not_in_confluence)})\n\n" + ) + if published_not_in_confluence: + f.write("| Title | URL |\n") + f.write("| --- | --- |\n") + for title, url in published_not_in_confluence: + f.write(f"| {title} | {url} |\n") + else: + f.write( + "No pages/posts found (or `--use-confluence` was not passed).\n" + ) diff --git a/e3sm_comms/exported_xml_reviewer/utils.py b/e3sm_comms/exported_xml_reviewer/utils.py new file mode 100644 index 0000000..9f34350 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/utils.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import re +from typing import Dict, List, Optional +from urllib.parse import urlsplit + + +def normalize_status(raw_status: Optional[str]) -> str: + if not raw_status: + return "unknown" + + mapping = { + "publish": "published", + "archive": "archived", + "draft": "draft", + "future": "future", + "pending": "pending", + "private": "private", + } + return mapping.get(raw_status.strip().lower(), raw_status.strip().lower()) + + +def display_status(status: str) -> str: + mapping = { + "published": "Published", + "archived": "Archived", + "draft": "Draft", + "future": "Future", + "pending": "Pending", + "private": "Private", + "unknown": "Unknown", + } + return mapping.get(status, status.title()) + + +def strip_html(text: str) -> str: + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r" ", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def is_legacy_content_url(url: str) -> bool: + parts = urlsplit(url) + slug = parts.path.strip("/").lower() + return bool(re.fullmatch(r"\d{6,8}[_-].+", slug)) + + +def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: + counts: Dict[str, int] = {} + lowered = text.lower() + + for term in terms: + escaped = re.escape(term) + pattern = rf"\b{escaped}\b" + matches = re.findall(pattern, lowered) + if matches: + counts[term] = len(matches) + + return counts diff --git a/e3sm_comms/page_reviewer/confluence_page_reviewer.py b/e3sm_comms/page_reviewer/confluence_page_reviewer.py index 4ffe5d8..7e7ccff 100644 --- a/e3sm_comms/page_reviewer/confluence_page_reviewer.py +++ b/e3sm_comms/page_reviewer/confluence_page_reviewer.py @@ -1,5 +1,5 @@ import re -from typing import Dict, List +from typing import Dict, List, Optional from e3sm_comms.page_reviewer.utils_base import ( Config, @@ -35,15 +35,21 @@ # Main functionality ########################################################## def run(config: Config): remove_output_files(config) + credentials: Optional[ConfluenceCredentials] = None try: credentials = ConfluenceCredentials() if config.mode in ["resource", "website"]: for tab in config.list_input_confluence_paths: walk_page_and_child_pages(config, credentials, tab) + if config.mode == "newsletter": newsletter_page_list: List[ConfluencePage] = read_page_list(config) for page in newsletter_page_list: - extract_data_from_page(config, credentials, page) + try: + extract_data_from_page(config, credentials, page) + except Exception as e: + print(f"ERROR processing newsletter page {page.url}: {e}") + newsletter_dict: Dict[str, str] if config.newsletter_test_link: newsletter_dict = process_newsletter( @@ -53,7 +59,8 @@ def run(config: Config): newsletter_dict = {} construct_markdown_table(config, newsletter_page_list, newsletter_dict) finally: - del credentials.api_token # Clear the API token from memory, for added security + if credentials is not None and hasattr(credentials, "api_token"): + del credentials.api_token # Clear the API token from memory # Recurse through pages ####################################################### @@ -63,54 +70,87 @@ def walk_page_and_child_pages( page_url: str, current_depth: int = 0, ): - current_page = ConfluencePage(page_url, current_depth) - extract_data_from_page(config, credentials, current_page) - if config.mode == "resource": - process_resource(config, current_page) - for child_page_id in current_page.child_page_ids: - child_page_url = ( - f"https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/{child_page_id}/" - ) - walk_page_and_child_pages( - config, credentials, child_page_url, current_depth=current_depth + 1 - ) + indent = " " * current_depth + current_page: Optional[ConfluencePage] = None + + try: + current_page = ConfluencePage(page_url, current_depth) + print(f"{indent}Visiting page_id={current_page.page_id}, url={page_url}") + extract_data_from_page(config, credentials, current_page) -# Per page analysis ############################################################### + if config.mode == "resource": + process_resource(config, current_page) + + for child_page_id in current_page.child_page_ids: + child_page_url = ( + f"https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/{child_page_id}/" + ) + walk_page_and_child_pages( + config, credentials, child_page_url, current_depth=current_depth + 1 + ) + + except Exception as e: + page_id = current_page.page_id if current_page else "unknown" + print(f"{indent}ERROR on page_id={page_id}, url={page_url}: {e}") + # Continue traversal for sibling branches by not re-raising + + +# Per page analysis ########################################################## def extract_data_from_page( config: Config, credentials: ConfluenceCredentials, page: ConfluencePage ): extract_data_from_content_url(credentials, page) + if config.mode in ["newsletter", "website"]: extract_data_from_content_url_body(config, credentials, page) + if config.mode == "newsletter": extract_data_from_comments_url(credentials, page) + if config.mode in ["resource", "website"]: extract_data_from_child_pages_url(credentials, page) + if config.mode == "website": write_results(config, page) -# Functions used by all modes ################################################# +# Functions used by all modes ################################################ def extract_data_from_content_url( credentials: ConfluenceCredentials, page: ConfluencePage ): - data = get_json(credentials, page.page_id, page.content_url) + data = get_json( + credentials, + page.page_id, + page.content_url, + params={"expand": "version,history"}, + ) + if "title" not in data: raise RuntimeError( f"Response for page_id={page.page_id} does not contain 'title'. Full response: {data}" ) + page.title = re.sub(r"[\r\n]+", "", data["title"]) print(f"Extracting data from page_id={page.page_id}, title={page.title}") + if "version" not in data or "number" not in data["version"]: raise RuntimeError( f"Response for page_id={page.page_id} does not contain 'version' or 'version > number'. Full response: {data}" ) - current_version: str = data["version"]["number"] + + current_version = data["version"]["number"] page.current_version = int(current_version) + if "history" not in data or "createdDate" not in data["history"]: + raise RuntimeError( + f"Response for page_id={page.page_id} does not contain 'history' or 'history > createdDate'. Full response: {data}" + ) + + page.created_date = data["history"]["createdDate"] + -# Functions used by newsletter, website modes ################################### +# Functions used by newsletter, website modes ############################### def extract_data_from_content_url_body( config: Config, credentials: ConfluenceCredentials, page: ConfluencePage ): @@ -120,17 +160,20 @@ def extract_data_from_content_url_body( page.content_url, params={"expand": "body.view.value"}, ) - # print_json(data) # For debugging + raw_html = data.get("body", {}).get("view", {}).get("value", "") if config.mode == "newsletter": raw_html = skip_newsletter_metadata_in_header(raw_html) + page.main_html, page.metadata_html = split_html(raw_html) + if config.check_links_work: page.main_html.linked_urls = LinkedURLs( page.main_html.links, config.scan_links_for_sensitive_terms, config.list_sensitive_terms, ) + if ("sensitive_terms" in config.requested_output) or ( "newsletter_review_table" in config.requested_output ): @@ -150,9 +193,11 @@ def extract_data_from_content_url_body( ) else: print(" Skipping first-person review. Page URL is in the approved list.") + page.main_html.double_spaces_after_periods = find_double_spaces_after_periods( page.main_html.paragraphs ) + lowercase_text: str = page.main_html.text.lower() page.main_html.img_mentions = get_image_mention_frequencies( lowercase_text, page.main_html.num_imgs @@ -160,11 +205,13 @@ def extract_data_from_content_url_body( page.main_html.img_resolutions = get_image_resolutions( page.main_html.img_srcs, "https://e3sm.atlassian.net/wiki", credentials ) + acronyms = get_acronyms( page.main_html.text ) # Use original text, not lowercase_text!! page.main_html.acronyms = filter_acronyms(page.url, acronyms) set_wordpress_keys(page) + if "need_to_sync_wordpress" in config.requested_output: if page.metadata_html: table = extract_confluence_table_to_dict(page.metadata_html) @@ -173,12 +220,52 @@ def extract_data_from_content_url_body( page.need_to_sync_wordpress = True -# Functions used by resource, website modes ################################### +# Functions used by resource, website modes ################################# def extract_data_from_child_pages_url( credentials: ConfluenceCredentials, page: ConfluencePage ): - data = get_json(credentials, page.page_id, page.child_pages_url) - page.child_page_ids = [page["id"] for page in data.get("results", [])] + child_page_ids: List[str] = [] + start = 0 + limit = 100 + + while True: + data = get_json( + credentials, + page.page_id, + page.child_pages_url, + params={"start": str(start), "limit": str(limit)}, + ) + + results = data.get("results", []) + child_page_ids.extend([child["id"] for child in results if "id" in child]) + + batch_count = len(results) + if batch_count == 0: + break + + size = data.get("size") + returned_limit = data.get("limit", limit) + next_link = data.get("_links", {}).get("next") + + print( + f" Retrieved {batch_count} child pages for page_id={page.page_id} " + f"(start={start}, limit={returned_limit})" + ) + + if next_link: + start += batch_count + continue + + if size is not None and batch_count < returned_limit: + break + + if batch_count < limit: + break + + start += batch_count + + page.child_page_ids = child_page_ids + count = len(page.child_page_ids) if count: print(f" Found {count} child pages: {page.child_page_ids}") diff --git a/e3sm_comms/page_reviewer/utils_base.py b/e3sm_comms/page_reviewer/utils_base.py index 752d854..3c9d8a8 100644 --- a/e3sm_comms/page_reviewer/utils_base.py +++ b/e3sm_comms/page_reviewer/utils_base.py @@ -10,6 +10,8 @@ from bs4 import BeautifulSoup from requests.auth import HTTPBasicAuth # type: ignore +from e3sm_comms.utils import count_sensitive_terms + # Classes ##################################################################### # Set these values in newsletter_review/main.py, resource_reviewer/main.py, website_reviewer/main.py @@ -100,6 +102,7 @@ def __init__(self, url: str, depth: int = 0): # Set by extract_data_from_content_url self.title: str = "" self.current_version: int = 0 + self.created_date: str = "" # Set by extract_data_from_content_url_body self.main_html: Optional[ParsedHTML] = None @@ -261,14 +264,9 @@ def split_html(raw_html: str) -> Tuple[ParsedHTML, Optional[ParsedHTML]]: def find_sensitive_terms( list_sensitive_terms: List[str], lowercase_text: str ) -> Dict[str, int]: - result = {} - for term in list_sensitive_terms: - pattern = re.escape(term) - matches = re.findall(pattern, lowercase_text) - count = len(matches) - if count > 0: - result[term] = count - return result + return count_sensitive_terms( + lowercase_text, list_sensitive_terms, whole_words_only=False + ) def remove_output_files(config: Config): @@ -294,7 +292,7 @@ def remove_output_files(config: Config): print(f"Could not remove {filename}: {e}") -# Functions used by newsletter, resource modes ################################ +# Functions used by newsletter, resource, term modes ########################## def map_confluence_to_e3sm(url: str, page_title: str = "") -> str: if page_title: # Map: @@ -348,6 +346,27 @@ def map_confluence_to_e3sm(url: str, page_title: str = "") -> str: return new_url +# Functions used by e3sm_org, term modes #################################### +def get_e3sm_url_status(e3sm_url: str) -> str: + try: + response = requests.get(e3sm_url, timeout=10) + response.raise_for_status() # Raises HTTPError for 4xx/5xx responses + return "link works not logged-in" + except requests.exceptions.Timeout: + return "link times out" + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code if e.response is not None else None + if status_code == 503 and ( + e.response.url if e.response is not None else "" + ).startswith("https://e3sm.org"): + return "link not whitelisted" + return "link raises RequestException" + except requests.exceptions.RequestException: + return "link raises RequestException" + except Exception: + return "link raises Exception" + + # Debugging ################################################################### def print_json(data: Dict): print(json.dumps(data, indent=4)) diff --git a/e3sm_comms/page_reviewer/utils_website_reviewer.py b/e3sm_comms/page_reviewer/utils_website_reviewer.py index c1fb134..62ba61a 100644 --- a/e3sm_comms/page_reviewer/utils_website_reviewer.py +++ b/e3sm_comms/page_reviewer/utils_website_reviewer.py @@ -26,7 +26,7 @@ def write_results(config: Config, page: ConfluencePage): with open( f"{config.output_dir}hierarchical_outline.txt", "a", encoding="utf-8" ) as f: - f.write(f"{page.depth * " "}{line_id}\n") + f.write(f"{page.depth * ' '}{line_id}\n") if "sensitive_terms" in config.requested_output: # Append if sensitive terms were found. @@ -34,7 +34,9 @@ def write_results(config: Config, page: ConfluencePage): with open( f"{config.output_dir}sensitive_terms.txt", "a", encoding="utf-8" ) as f: - f.write(f"{line_id} -- {page.main_html.sensitive_terms}\n") + f.write( + f"[From {page.created_date}] {line_id} -- {page.main_html.sensitive_terms}\n" + ) if "missing_metadata" in config.requested_output: # Append if there is no metadata table. diff --git a/e3sm_comms/utils.py b/e3sm_comms/utils.py index 817a879..385d984 100644 --- a/e3sm_comms/utils.py +++ b/e3sm_comms/utils.py @@ -1,5 +1,251 @@ +import re +import xml.etree.ElementTree as ET +from collections import defaultdict +from dataclasses import dataclass +from typing import Dict, List, Set, Tuple +from urllib.parse import urlsplit, urlunsplit + +# Note: to use a more hardened XML library, instead use: +# import defusedxml.ElementTree as ET +# And add defusedxml to conda/dev.yml. +# However, we are only using XML files downloaded directly from WordPress. + +# IO DIR ###################################################################### # Chrysalis # IO_DIR="/home/ac.forsyth2/ez/e3sm-comms-io" # Perlmutter IO_DIR = "/global/homes/f/forsyth/ez/e3sm-comms-io" + +# Confluence ################################################################## + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +# Pattern matching ############################################################ + + +def normalize_url(url: str) -> str: + url = url.strip() + if not url: + return "" + parts = urlsplit(url) + scheme = parts.scheme.lower() + netloc = parts.netloc.lower() + path = parts.path.rstrip("/") + return urlunsplit((scheme, netloc, path, "", "")) + + +def matches_pattern(pattern: str, url: str, normalize: bool = True) -> bool: + if normalize: + pattern = normalize_url(pattern) + url = normalize_url(url) + + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + return url.startswith(pattern[:-1]) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls( + patterns: List[str], all_urls: List[str], normalize: bool = True +) -> List[str]: + matched: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url, normalize=normalize): + matched.add(url) + return sorted(matched) + + +def get_invalid_patterns( + patterns: List[str], all_urls: List[str], normalize: bool = True +) -> List[str]: + return sorted( + pattern + for pattern in patterns + if not any( + matches_pattern(pattern, url, normalize=normalize) for url in all_urls + ) + ) + + +# Sensitive term counting ##################################################### + + +def count_sensitive_terms( + text: str, terms: List[str], whole_words_only: bool = True +) -> Dict[str, int]: + counts: Dict[str, int] = {} + for term in terms: + escaped = re.escape(term) + pattern = rf"\b{escaped}\b" if whole_words_only else escaped + matches = re.findall(pattern, text) + if matches: + counts[term] = len(matches) + return counts + + +# WordPress XML parsing ####################################################### + +WORDPRESS_NS = {"wp": "http://wordpress.org/export/1.2/"} + + +@dataclass +class WordpressItem: + post_id: str + post_parent: str + post_type: str + title: str + url: str + status: str + body: str + + +def _get_item_text(item: ET.Element, tag: str, ns: bool = True) -> str: + elem = item.find(tag, WORDPRESS_NS) if ns else item.find(tag) + return elem.text.strip() if elem is not None and elem.text else "" + + +def parse_wordpress_xml_items( + xml_file_path: str, post_type: str +) -> List[WordpressItem]: + tree = ET.parse(xml_file_path) + root = tree.getroot() + channel = root.find("channel") + if channel is None: + return [] + + items: List[WordpressItem] = [] + for item in channel.findall("item"): + post_type_elem = item.find("wp:post_type", WORDPRESS_NS) + if post_type_elem is None or (post_type_elem.text or "").strip() != post_type: + continue + + items.append( + WordpressItem( + post_id=_get_item_text(item, "wp:post_id"), + post_parent=_get_item_text(item, "wp:post_parent") or "0", + post_type=post_type, + title=_get_item_text(item, "title", ns=False) or "Untitled", + url=normalize_url(_get_item_text(item, "link", ns=False)), + status=_get_item_text(item, "wp:status") or "unknown", + body=_extract_item_body(item), + ) + ) + + return items + + +def get_wordpress_urls_by_status( + xml_file_path: str, post_type: str +) -> Dict[str, List[str]]: + grouped: Dict[str, List[str]] = defaultdict(list) + for item in parse_wordpress_xml_items(xml_file_path, post_type): + if item.url: + grouped[item.status].append(item.url) + return {status: sorted(urls) for status, urls in sorted(grouped.items())} + + +def _extract_item_body(item: ET.Element) -> str: + ns = { + "wp": "http://wordpress.org/export/1.2/", + "content": "http://purl.org/rss/1.0/modules/content/", + "excerpt": "http://wordpress.org/export/1.2/excerpt/", + } + body_parts: List[str] = [] + + content_elem = item.find("content:encoded", ns) + if content_elem is not None and content_elem.text and content_elem.text.strip(): + body_parts.append(content_elem.text.strip()) + + excerpt_elem = item.find("excerpt:encoded", ns) + if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): + body_parts.append(excerpt_elem.text.strip()) + + for postmeta in item.findall("wp:postmeta", ns): + meta_key_elem = postmeta.find("wp:meta_key", ns) + meta_value_elem = postmeta.find("wp:meta_value", ns) + meta_key = ( + meta_key_elem.text.strip() + if meta_key_elem is not None and meta_key_elem.text + else "" + ) + meta_value = ( + meta_value_elem.text.strip() + if meta_value_elem is not None and meta_value_elem.text + else "" + ) + if ( + meta_value + and meta_key.endswith("_free_form_content") + and not meta_key.startswith("_") + ): + body_parts.append(meta_value) + + return "\n".join(body_parts) + + +# File reading ################################################################ + + +def read_lines( + file_path: str, + strip: bool = True, + skip_empty: bool = True, + lowercase: bool = False, +) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + lines = [line.rstrip("\n") for line in f] + if strip: + lines = [line.strip() for line in lines] + if skip_empty: + lines = [line for line in lines if line] + if lowercase: + lines = [line.lower() for line in lines] + return lines diff --git a/e3sm_comms/website_reviewer/main.py b/e3sm_comms/website_reviewer/main.py index a1479e7..0813ebf 100644 --- a/e3sm_comms/website_reviewer/main.py +++ b/e3sm_comms/website_reviewer/main.py @@ -5,10 +5,16 @@ def main(): c = Config("website") + # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" c.file_input_confluence_paths = ( - f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" + f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" ) - # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" + + # _partial excludes these tabs: RESEARCH, MODEL, DATA + # c.file_input_confluence_paths = ( + # f"{IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt" + # ) + c.sensitive_terms_file = f"{IO_DIR}/input/shared/sensitive_terms.txt" c.output_dir = f"{IO_DIR}/output/website_reviewer/" # Must end with "/" c.requested_output = [ diff --git a/examples/review_terms.bash b/examples/review_terms.bash new file mode 100755 index 0000000..acf6b35 --- /dev/null +++ b/examples/review_terms.bash @@ -0,0 +1,25 @@ +# Before running: +# WordPress: Tools > Export > export pages (wordpress_pages.xml) and posts (wordpress_posts.xml) +# Copy those XMLs into ${IO_DIR}/input/e3sm_org_reviewer/ +# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt + +# Replace this with your IO dir! +IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io + +echo "Count of top-level Confluence pages:" +wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt +echo "Count of whitelisted e3sm.org pages:" +wc -l ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt + +echo "Step 1. Review Confluence" +echo "Note: this will require a Confluence login" +e3sm-comms-website-reviewer + +echo "Step 2. Review e3sm.org" +cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt +cp ${IO_DIR}/output/website_reviewer/sensitive_terms.txt ${IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt +e3sm-comms-e3sm-org-reviewer +echo "Output reports:" +echo "1. ${IO_DIR}/output/e3sm_org_reviewer/path_report.md" +echo "2. ${IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" +echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md" diff --git a/examples/review_xml.bash b/examples/review_xml.bash new file mode 100755 index 0000000..0b1507b --- /dev/null +++ b/examples/review_xml.bash @@ -0,0 +1,34 @@ +# Before running: + +# WordPress: Tools > Export > export pages +# WordPress: Tools > Export > export posts +# scp wordpress_pages.xml user@host:/path/to/e3sm-comms-io/input/exported_xml_reviewer/wordpress_pages.xml +# scp wordpress_posts.xml user@host:/path/to/e3sm-comms-io/input/exported_xml_reviewer/wordpress_posts.xml + +# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to ${IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt + +# Replace this with your IO dir! +IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io + +echo "Count of top-level Confluence pages:" +wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt + +echo "Count of requested links:" +wc -l ${IO_DIR}/input/exported_xml_reviewer/requested_links.csv +echo "Count of whitelisted pages:" +wc -l ${IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt + +echo "Step 1. Review Confluence pages" +echo "Note: this will require a Confluence login" +e3sm-comms-website-reviewer + +echo "Step 2. Review xml exported from e3sm.org" +cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt +e3sm-comms-exported-xml-reviewer --use-confluence --use-whitelist +echo "Output reports:" +echo "1. ${IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" +echo "2. ${IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.txt" +echo "3. ${IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" +echo "4. ${IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" +echo "5. ${IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" +echo "6. ${IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_external_links_report.md" diff --git a/pyproject.toml b/pyproject.toml index 8795d9c..1d24d7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ dependencies = [ "beautifulsoup4", + "requests", ] [project.optional-dependencies] @@ -116,6 +117,7 @@ e3sm-comms-resource-reviewer = "e3sm_comms.resource_reviewer.main:main" e3sm-comms-website-reviewer = "e3sm_comms.website_reviewer.main:main" # These do not: e3sm-comms-e3sm-org-reviewer = "e3sm_comms.e3sm_org_reviewer.main:main" +e3sm-comms-exported-xml-reviewer = "e3sm_comms.exported_xml_reviewer.main:main" e3sm-comms-html-reviewer = "e3sm_comms.html_reviewer.main:main" e3sm-comms-tree-reviewer = "e3sm_comms.tree_reviewer.main:main" e3sm-comms-video-reviewer = "e3sm_comms.video_reviewer.main:main"