From 9059c99c2e73aace2a9dde78600801f24588480b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Apr 2026 14:59:29 -0700 Subject: [PATCH 01/85] Sort by number of matching terms --- README.md | 4 +++ e3sm_comms/term_reviewer/__init__.py | 0 e3sm_comms/term_reviewer/main.py | 52 ++++++++++++++++++++++++++++ e3sm_comms/website_reviewer/main.py | 4 +-- pyproject.toml | 1 + 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 e3sm_comms/term_reviewer/__init__.py create mode 100644 e3sm_comms/term_reviewer/main.py diff --git a/README.md b/README.md index edb2f3e..4f9a9e4 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ This package is for implementing the software needs of the E3SM Communications t - output: 1 txt file of html with those highlights removed. - Known issues: more than just `` tags are changed (presumably no other semantic changes though) +`e3sm-comms-term-reviewer` +- input: txt file of sensitive terms (e.g., output from `e3sm-comms-e3sm-org-reviewer` or `e3sm-comms-website-reviewer`) +- output: sorted version of that txt file + `e3sm-comms-tree-reviewer` - input: 2 txt files showing the website structure in hierarchical form (via indents) -- i.e. in tree form - output: txt file listing the steps of moving subtrees to get from one tree to the other diff --git a/e3sm_comms/term_reviewer/__init__.py b/e3sm_comms/term_reviewer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py new file mode 100644 index 0000000..15c5367 --- /dev/null +++ b/e3sm_comms/term_reviewer/main.py @@ -0,0 +1,52 @@ +import ast + +from e3sm_comms.utils import IO_DIR + +INPUT: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" +# INPUT: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" +OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.txt" + + +def sort_by_match_sum(input_file, output_file): + entries = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line in f: + line = line.rstrip("\n") + if not line.strip(): + continue + + dict_start = line.find("{") + if dict_start == -1: + print(f"Skipping malformed line: {line}") + continue + + dict_str = line[dict_start:].strip() + + try: + dict_data = ast.literal_eval(dict_str) + except (SyntaxError, ValueError): + print(f"Skipping malformed dictionary: {line}") + continue + + if not isinstance(dict_data, dict): + print(f"Skipping non-dictionary line: {line}") + continue + + try: + total = sum(dict_data.values()) + except TypeError: + print(f"Skipping line with non-numeric values: {line}") + continue + + entries.append((total, line)) + + entries.sort(key=lambda x: x[0], reverse=True) + + with open(output_file, "w", encoding="utf-8") as f: + for _, line in entries: + f.write(line + "\n") + + +def main(): + sort_by_match_sum(INPUT, OUTPUT) diff --git a/e3sm_comms/website_reviewer/main.py b/e3sm_comms/website_reviewer/main.py index a1479e7..9f76a88 100644 --- a/e3sm_comms/website_reviewer/main.py +++ b/e3sm_comms/website_reviewer/main.py @@ -5,10 +5,10 @@ def main(): c = Config("website") + # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" c.file_input_confluence_paths = ( - f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" + f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" ) - # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" c.sensitive_terms_file = f"{IO_DIR}/input/shared/sensitive_terms.txt" c.output_dir = f"{IO_DIR}/output/website_reviewer/" # Must end with "/" c.requested_output = [ diff --git a/pyproject.toml b/pyproject.toml index 8795d9c..c108d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,7 @@ e3sm-comms-website-reviewer = "e3sm_comms.website_reviewer.main:main" # These do not: e3sm-comms-e3sm-org-reviewer = "e3sm_comms.e3sm_org_reviewer.main:main" e3sm-comms-html-reviewer = "e3sm_comms.html_reviewer.main:main" +e3sm-comms-term-reviewer = "e3sm_comms.term_reviewer.main:main" e3sm-comms-tree-reviewer = "e3sm_comms.tree_reviewer.main:main" e3sm-comms-video-reviewer = "e3sm_comms.video_reviewer.main:main" From 39d6056b7cee7bd3596da74525f8c7e15e47ce36 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 1 Apr 2026 15:14:19 -0700 Subject: [PATCH 02/85] Combine output for e3sm.org and Confluence --- e3sm_comms/term_reviewer/main.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 15c5367..a50fc27 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -1,14 +1,15 @@ import ast +from typing import List, Tuple from e3sm_comms.utils import IO_DIR -INPUT: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" -# INPUT: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" +INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" +INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.txt" -def sort_by_match_sum(input_file, output_file): - entries = [] +def sort_by_match_sum(input_file: str) -> List[Tuple[int, str]]: + entries: List[Tuple[int, str]] = [] with open(input_file, "r", encoding="utf-8") as f: for line in f: @@ -34,7 +35,7 @@ def sort_by_match_sum(input_file, output_file): continue try: - total = sum(dict_data.values()) + total: int = sum(dict_data.values()) except TypeError: print(f"Skipping line with non-numeric values: {line}") continue @@ -42,11 +43,14 @@ def sort_by_match_sum(input_file, output_file): entries.append((total, line)) entries.sort(key=lambda x: x[0], reverse=True) - - with open(output_file, "w", encoding="utf-8") as f: - for _, line in entries: - f.write(line + "\n") + return entries def main(): - sort_by_match_sum(INPUT, OUTPUT) + entries_e3sm_org: List[Tuple[int, str]] = sort_by_match_sum(INPUT_E3SM_ORG) + entries_confluence: List[Tuple[int, str]] = sort_by_match_sum(INPUT_CONFLUENCE) + with open(OUTPUT, "w", encoding="utf-8") as f: + for _, line in entries_e3sm_org: + f.write(line + "\n") + for _, line in entries_confluence: + f.write(line + "\n") From 358eef21460d15628116236d5658ade8b86ef2b3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Wed, 8 Apr 2026 14:28:56 -0700 Subject: [PATCH 03/85] Improvements to term-reviewer --- e3sm_comms/term_reviewer/main.py | 142 ++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 21 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index a50fc27..3d7b10f 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -1,19 +1,48 @@ import ast -from typing import List, Tuple +from typing import Dict, List, Optional, Tuple +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" -OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.txt" +OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" + + +def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: + try: + data = ast.literal_eval(dict_str) + except (SyntaxError, ValueError): + return None + + if not isinstance(data, dict): + return None + + try: + # Force numeric validation + total = sum(data.values()) + except TypeError: + return None + + if not isinstance(total, (int, float)): + return None + + return data def sort_by_match_sum(input_file: str) -> List[Tuple[int, str]]: entries: List[Tuple[int, str]] = [] with open(input_file, "r", encoding="utf-8") as f: - for line in f: - line = line.rstrip("\n") + for raw_line in f: + line = raw_line.rstrip("\n") if not line.strip(): continue @@ -23,34 +52,105 @@ def sort_by_match_sum(input_file: str) -> List[Tuple[int, str]]: continue dict_str = line[dict_start:].strip() - - try: - dict_data = ast.literal_eval(dict_str) - except (SyntaxError, ValueError): + dict_data = parse_dict(dict_str) + if dict_data is None: print(f"Skipping malformed dictionary: {line}") continue - if not isinstance(dict_data, dict): - print(f"Skipping non-dictionary line: {line}") - continue - - try: - total: int = sum(dict_data.values()) - except TypeError: - print(f"Skipping line with non-numeric values: {line}") - continue - + total = int(sum(dict_data.values())) entries.append((total, line)) entries.sort(key=lambda x: x[0], reverse=True) return entries -def main(): +def format_wordpress_line(line: str) -> Optional[str]: + """ + Input example: + https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/: {'str1': 1} + + Output example: + [https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/](https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/) -- {'str1': 1} + """ + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + counts = line[dict_start:].strip() + + if prefix.endswith(":"): + prefix = prefix[:-1].rstrip() + + url = prefix + return f"[{url}]({url}) -- {counts}" + + +def format_confluence_line(line: str) -> Optional[str]: + """ + Input example: + 3841294373: E3SM Publicity -- {'str1': 85, 'str2': 20, 'str3': 2} + + Output example: + E3SM Publicity: [confluence](https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/3841294373) [e3sm.org](https://e3sm.org/e3sm-publicity) -- {'str1': 85, 'str2': 20, 'str3': 2} + """ + dict_start = line.find("{") + if dict_start == -1: + return None + + counts = line[dict_start:].strip() + prefix = line[:dict_start].rstrip() + + # Expected prefix format: + # ": --" + if prefix.endswith("--"): + prefix = prefix[:-2].rstrip() + + first_colon = prefix.find(":") + if first_colon == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + page_id = prefix[:first_colon].strip() + title = prefix[first_colon + 1 :].strip() + + if not page_id.isdigit(): + print(f"Skipping Confluence line with non-numeric page id: {line}") + return None + + confluence_url = build_confluence_url(page_id) + + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + e3sm_url = None + + md = f"{title}: [confluence]({confluence_url})" + if e3sm_url: + md += f" [e3sm.org]({e3sm_url})" + md += f" -- {counts}" + + return md + + +def main() -> None: entries_e3sm_org: List[Tuple[int, str]] = sort_by_match_sum(INPUT_E3SM_ORG) entries_confluence: List[Tuple[int, str]] = sort_by_match_sum(INPUT_CONFLUENCE) + with open(OUTPUT, "w", encoding="utf-8") as f: + f.write("# Sensitive Terms Report\n\n") + + f.write("## e3sm.org\n\n") for _, line in entries_e3sm_org: - f.write(line + "\n") + formatted = format_wordpress_line(line) + if formatted: + f.write(f"- {formatted}\n") + + f.write("\n## Confluence\n\n") for _, line in entries_confluence: - f.write(line + "\n") + formatted = format_confluence_line(line) + if formatted: + f.write(f"- {formatted}\n") From a0547da8a184f0e809379fb528aec9719a163bc9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 8 Apr 2026 14:52:41 -0700 Subject: [PATCH 04/85] Add created dates to website-reviewer output --- e3sm_comms/page_reviewer/confluence_page_reviewer.py | 5 +++++ e3sm_comms/page_reviewer/utils_base.py | 1 + e3sm_comms/page_reviewer/utils_website_reviewer.py | 4 +++- e3sm_comms/website_reviewer/main.py | 8 +++++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/page_reviewer/confluence_page_reviewer.py b/e3sm_comms/page_reviewer/confluence_page_reviewer.py index 4ffe5d8..618ce5a 100644 --- a/e3sm_comms/page_reviewer/confluence_page_reviewer.py +++ b/e3sm_comms/page_reviewer/confluence_page_reviewer.py @@ -108,6 +108,11 @@ def extract_data_from_content_url( ) current_version: str = data["version"]["number"] page.current_version = int(current_version) + if "history" not in data or "createdDate" not in data["history"]: + raise RuntimeError( + f"Response for page_id={page.page_id} does not contain 'history' or 'history > createdDate'. Full response: {data}" + ) + page.created_date = data["history"]["createdDate"] # Functions used by newsletter, website modes ################################### diff --git a/e3sm_comms/page_reviewer/utils_base.py b/e3sm_comms/page_reviewer/utils_base.py index 752d854..03a9ffc 100644 --- a/e3sm_comms/page_reviewer/utils_base.py +++ b/e3sm_comms/page_reviewer/utils_base.py @@ -100,6 +100,7 @@ def __init__(self, url: str, depth: int = 0): # Set by extract_data_from_content_url self.title: str = "" self.current_version: int = 0 + self.created_date: str = "" # Set by extract_data_from_content_url_body self.main_html: Optional[ParsedHTML] = None diff --git a/e3sm_comms/page_reviewer/utils_website_reviewer.py b/e3sm_comms/page_reviewer/utils_website_reviewer.py index c1fb134..785ea96 100644 --- a/e3sm_comms/page_reviewer/utils_website_reviewer.py +++ b/e3sm_comms/page_reviewer/utils_website_reviewer.py @@ -34,7 +34,9 @@ def write_results(config: Config, page: ConfluencePage): with open( f"{config.output_dir}sensitive_terms.txt", "a", encoding="utf-8" ) as f: - f.write(f"{line_id} -- {page.main_html.sensitive_terms}\n") + f.write( + f"[From {page.created_date}] {line_id} -- {page.main_html.sensitive_terms}\n" + ) if "missing_metadata" in config.requested_output: # Append if there is no metadata table. diff --git a/e3sm_comms/website_reviewer/main.py b/e3sm_comms/website_reviewer/main.py index 9f76a88..33b314a 100644 --- a/e3sm_comms/website_reviewer/main.py +++ b/e3sm_comms/website_reviewer/main.py @@ -6,9 +6,15 @@ def main(): c = Config("website") # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" + # c.file_input_confluence_paths = ( + # f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" + # ) + + # _partial excludes these tabs: RESEARCH, MODEL, DATA c.file_input_confluence_paths = ( - f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" + f"{IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt" ) + c.sensitive_terms_file = f"{IO_DIR}/input/shared/sensitive_terms.txt" c.output_dir = f"{IO_DIR}/output/website_reviewer/" # Must end with "/" c.requested_output = [ From 21cba16051753594fbc56727a591f570442f3c6e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 8 Apr 2026 15:04:17 -0700 Subject: [PATCH 05/85] Have term-reviewer sort by year --- e3sm_comms/term_reviewer/main.py | 97 ++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 3d7b10f..826bc9f 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -1,5 +1,7 @@ import ast -from typing import Dict, List, Optional, Tuple +import re +from collections import defaultdict +from typing import DefaultDict, Dict, List, Optional, Tuple from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR @@ -11,6 +13,8 @@ CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" +FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") + def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" @@ -26,7 +30,6 @@ def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: return None try: - # Force numeric validation total = sum(data.values()) except TypeError: return None @@ -37,8 +40,22 @@ def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: return data -def sort_by_match_sum(input_file: str) -> List[Tuple[int, str]]: - entries: List[Tuple[int, str]] = [] +def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: + """ + Supports lines like: + [From 2023-04-12T21:05:24.198Z] 3746136122: Title -- {'str1': 3} + """ + match = FROM_PREFIX_RE.match(line) + if not match: + return None, line + + year = int(match.group(1)) + remainder = match.group(2).strip() + return year, remainder + + +def sort_and_group_by_year(input_file: str) -> Dict[str, List[Tuple[int, str]]]: + grouped_entries: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list) with open(input_file, "r", encoding="utf-8") as f: for raw_line in f: @@ -46,32 +63,30 @@ def sort_by_match_sum(input_file: str) -> List[Tuple[int, str]]: if not line.strip(): continue - dict_start = line.find("{") + year, remainder = extract_year_and_remainder(line) + year_key = str(year) if year is not None else "Unknown" + + dict_start = remainder.find("{") if dict_start == -1: print(f"Skipping malformed line: {line}") continue - dict_str = line[dict_start:].strip() + dict_str = remainder[dict_start:].strip() dict_data = parse_dict(dict_str) if dict_data is None: print(f"Skipping malformed dictionary: {line}") continue total = int(sum(dict_data.values())) - entries.append((total, line)) + grouped_entries[year_key].append((total, remainder)) - entries.sort(key=lambda x: x[0], reverse=True) - return entries + for year_key in grouped_entries: + grouped_entries[year_key].sort(key=lambda x: x[0], reverse=True) + return dict(grouped_entries) -def format_wordpress_line(line: str) -> Optional[str]: - """ - Input example: - https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/: {'str1': 1} - Output example: - [https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/](https://e3sm.org/moab-based-coupler-achieves-bit-for-bit-parity-with-legacy-system/) -- {'str1': 1} - """ +def format_wordpress_line(line: str) -> Optional[str]: dict_start = line.find("{") if dict_start == -1: return None @@ -87,13 +102,6 @@ def format_wordpress_line(line: str) -> Optional[str]: def format_confluence_line(line: str) -> Optional[str]: - """ - Input example: - 3841294373: E3SM Publicity -- {'str1': 85, 'str2': 20, 'str3': 2} - - Output example: - E3SM Publicity: [confluence](https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/3841294373) [e3sm.org](https://e3sm.org/e3sm-publicity) -- {'str1': 85, 'str2': 20, 'str3': 2} - """ dict_start = line.find("{") if dict_start == -1: return None @@ -101,8 +109,6 @@ def format_confluence_line(line: str) -> Optional[str]: counts = line[dict_start:].strip() prefix = line[:dict_start].rstrip() - # Expected prefix format: - # "<page_id>: <title> --" if prefix.endswith("--"): prefix = prefix[:-2].rstrip() @@ -136,21 +142,38 @@ def format_confluence_line(line: str) -> Optional[str]: return md +def write_section( + f, + section_title: str, + grouped_entries: Dict[str, List[Tuple[int, str]]], + formatter, +) -> None: + f.write(f"## {section_title}\n\n") + + def year_sort_key(year_str: str) -> Tuple[int, int]: + if year_str == "Unknown": + return (1, 0) + return (0, -int(year_str)) + + for year in sorted(grouped_entries.keys(), key=year_sort_key): + f.write(f"### {year}\n\n") + for _, line in grouped_entries[year]: + formatted = formatter(line) + if formatted: + f.write(f"- {formatted}\n") + f.write("\n") + + def main() -> None: - entries_e3sm_org: List[Tuple[int, str]] = sort_by_match_sum(INPUT_E3SM_ORG) - entries_confluence: List[Tuple[int, str]] = sort_by_match_sum(INPUT_CONFLUENCE) + entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) + entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) with open(OUTPUT, "w", encoding="utf-8") as f: f.write("# Sensitive Terms Report\n\n") - f.write("## e3sm.org\n\n") - for _, line in entries_e3sm_org: - formatted = format_wordpress_line(line) - if formatted: - f.write(f"- {formatted}\n") + write_section(f, "e3sm.org", entries_e3sm_org, format_wordpress_line) + write_section(f, "Confluence", entries_confluence, format_confluence_line) - f.write("\n## Confluence\n\n") - for _, line in entries_confluence: - formatted = format_confluence_line(line) - if formatted: - f.write(f"- {formatted}\n") + +if __name__ == "__main__": + main() From d264e8240d36377f746b751f3b60c3198101a418 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 8 Apr 2026 15:20:06 -0700 Subject: [PATCH 06/85] Add descriptions to term-reviewer output --- e3sm_comms/term_reviewer/main.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 826bc9f..755b227 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -64,7 +64,7 @@ def sort_and_group_by_year(input_file: str) -> Dict[str, List[Tuple[int, str]]]: continue year, remainder = extract_year_and_remainder(line) - year_key = str(year) if year is not None else "Unknown" + year_key = str(year) if year is not None else "Unknown year" dict_start = remainder.find("{") if dict_start == -1: @@ -145,13 +145,15 @@ def format_confluence_line(line: str) -> Optional[str]: def write_section( f, section_title: str, + section_description: str, grouped_entries: Dict[str, List[Tuple[int, str]]], formatter, ) -> None: f.write(f"## {section_title}\n\n") + f.write(f"Description: {section_description}\n\n") def year_sort_key(year_str: str) -> Tuple[int, int]: - if year_str == "Unknown": + if year_str == "Unknown year": return (1, 0) return (0, -int(year_str)) @@ -165,14 +167,29 @@ def year_sort_key(year_str: str) -> Tuple[int, int]: def main() -> None: + description_e3sm_org: str = ( + "These are the currently publicly-available (whitelisted) e3sm.org pages that include sensitive terms." + ) + description_confluence: str = ( + "These are the Confluence pages (serving as drafts of e3sm.org pages) that include sensitive terms. The 'confluence' links are what the script _actually_ reviewed. The 'e3sm.org' links are _predicted_ based on common URL naming patterns and thus may in fact be broken links. If the Confluence drafts and actual e3sm.org pages have not been kept in sync, remember that the term count is for the Confluence draft, not the actual e3sm.org page." + ) + entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) with open(OUTPUT, "w", encoding="utf-8") as f: f.write("# Sensitive Terms Report\n\n") - write_section(f, "e3sm.org", entries_e3sm_org, format_wordpress_line) - write_section(f, "Confluence", entries_confluence, format_confluence_line) + write_section( + f, "e3sm.org", description_e3sm_org, entries_e3sm_org, format_wordpress_line + ) + write_section( + f, + "Confluence", + description_confluence, + entries_confluence, + format_confluence_line, + ) if __name__ == "__main__": From 5407ece4bf78e3d98b3cf1852543d0ee2c767e34 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 8 Apr 2026 15:32:14 -0700 Subject: [PATCH 07/85] Add summary tables to term-reviewer output --- e3sm_comms/term_reviewer/main.py | 68 +++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 755b227..bc2df30 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -142,6 +142,65 @@ def format_confluence_line(line: str) -> Optional[str]: return md +def year_sort_key(year_str: str) -> Tuple[int, int]: + if year_str == "Unknown year": + return (1, 0) + return (0, -int(year_str)) + + +def build_year_summary( + grouped_entries: Dict[str, List[Tuple[int, str]]], +) -> Dict[str, Dict[str, int]]: + summary: Dict[str, Dict[str, int]] = {} + + for year, entries in grouped_entries.items(): + counts = { + "total": len(entries), + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5+": 0, + } + + for total_terms, _ in entries: + if total_terms == 1: + counts["1"] += 1 + elif total_terms == 2: + counts["2"] += 1 + elif total_terms == 3: + counts["3"] += 1 + elif total_terms == 4: + counts["4"] += 1 + elif total_terms >= 5: + counts["5+"] += 1 + + summary[year] = counts + + return summary + + +def write_summary_table(f, grouped_entries: Dict[str, List[Tuple[int, str]]]) -> None: + summary = build_year_summary(grouped_entries) + + f.write("### Summary Table\n") + f.write( + "How to interpret: each cell's value is the number of pages published in year <row> that contains <col> terms\n" + ) + + f.write("| Year | Total (i.e., any number of terms) | 1 | 2 | 3 | 4 | 5+ |\n") + f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") + + for year in sorted(summary.keys(), key=year_sort_key): + counts = summary[year] + f.write( + f"| {year} | {counts['total']} | {counts['1']} | {counts['2']} | " + f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" + ) + + f.write("\n") + + def write_section( f, section_title: str, @@ -152,17 +211,14 @@ def write_section( f.write(f"## {section_title}\n\n") f.write(f"Description: {section_description}\n\n") - def year_sort_key(year_str: str) -> Tuple[int, int]: - if year_str == "Unknown year": - return (1, 0) - return (0, -int(year_str)) + write_summary_table(f, grouped_entries) for year in sorted(grouped_entries.keys(), key=year_sort_key): f.write(f"### {year}\n\n") - for _, line in grouped_entries[year]: + for idx, (_, line) in enumerate(grouped_entries[year], start=1): formatted = formatter(line) if formatted: - f.write(f"- {formatted}\n") + f.write(f"{idx}. {formatted}\n") f.write("\n") From 64412c2bba091adfa4cf545ee23a87683de0227e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 8 Apr 2026 15:37:45 -0700 Subject: [PATCH 08/85] Get row and col to show up --- e3sm_comms/term_reviewer/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index bc2df30..045fc58 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -185,7 +185,7 @@ def write_summary_table(f, grouped_entries: Dict[str, List[Tuple[int, str]]]) -> f.write("### Summary Table\n") f.write( - "How to interpret: each cell's value is the number of pages published in year <row> that contains <col> terms\n" + "How to interpret: each cell's value is the number of pages published in year `row` that contains `col` terms\n" ) f.write("| Year | Total (i.e., any number of terms) | 1 | 2 | 3 | 4 | 5+ |\n") From 812ac75de7e6108ab570a9d88a7c91a3b55ec768 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 09:40:42 -0700 Subject: [PATCH 09/85] Add website link status --- e3sm_comms/term_reviewer/main.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 045fc58..673ad5f 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -3,6 +3,8 @@ from collections import defaultdict from typing import DefaultDict, Dict, List, Optional, Tuple +import requests # type: ignore + from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR @@ -126,6 +128,7 @@ def format_confluence_line(line: str) -> Optional[str]: confluence_url = build_confluence_url(page_id) + e3sm_url: Optional[str] try: e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) except Exception as exc: @@ -133,10 +136,30 @@ def format_confluence_line(line: str) -> Optional[str]: f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" ) e3sm_url = None + e3sm_url_status: Optional[str] = None + if e3sm_url: + try: + response = requests.get(e3sm_url, timeout=10) + response.raise_for_status() # Raises HTTPError for 4xx/5xx responses + e3sm_url_status = "link works not logged-in" + except requests.exceptions.Timeout: + e3sm_url_status = "link times out" + except requests.exceptions.RequestException as e: + error_message: str = f"{e}" + if error_message.startswith( + "503 Server Error: Service Temporarily Unavailable for url: https://e3sm.org" + ): + e3sm_url_status = "link not whitelisted" + else: + e3sm_url_status = "link raises RequestException" + except Exception: + e3sm_url_status = "link raises Exception" md = f"{title}: [confluence]({confluence_url})" if e3sm_url: md += f" [e3sm.org]({e3sm_url})" + if e3sm_url_status: + md += f" (Note: {e3sm_url_status})" md += f" -- {counts}" return md From fbd3eb70d6ac047572d44c15ff976827857e6e1d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 11:01:48 -0700 Subject: [PATCH 10/85] Add checks for archived pages --- README.md | 4 ++-- e3sm_comms/e3sm_org_reviewer/main.py | 30 ++++++++++++++++++++------ e3sm_comms/page_reviewer/utils_base.py | 23 +++++++++++++++++++- e3sm_comms/term_reviewer/main.py | 28 ++++++------------------ 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4f9a9e4..67cd842 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ This package is for implementing the software needs of the E3SM Communications t ### Simple commands `e3sm-comms-e3sm-org-reviewer` -- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for -- output: txt file listing e3sm.org pages containing those phrases +- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for, txt file listing e3sm.org pages that should be marked as archived +- output: txt file listing e3sm.org pages containing those phrases, txt file listing e3sm.org pages that are accessible even though they should be archived `e3sm-comms-html-reviewer` - input: 1 txt file of html copied from WordPress that includes yellow highlights left over from Confluence. diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index ecb0739..af77300 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,20 +1,27 @@ from typing import Dict, List -from e3sm_comms.page_reviewer.utils_base import LinkedURLs +from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status from e3sm_comms.utils import IO_DIR -INPUT_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/e3sm_org_reviewer/web_pages.txt" +INPUT_ACCESSIBLE_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/e3sm_org_reviewer/web_pages.txt" INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -OUTPUT: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" +OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" + +INPUT_ARCHIVED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/archived_web_pages.txt" +) +OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" +) def main(): - with open(INPUT_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + # Check the accesible pages for search phrases ############################ + with open(INPUT_ACCESSIBLE_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_e3sm_org_paths: List[str] = [line.strip() for line in f] with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: terms: List[str] = [line.rstrip("\n").lower() for line in f] list_search_phrases: List[str] = sorted(terms) - print(f"Checking {len(list_input_e3sm_org_paths)} e3sm.org pages") links = LinkedURLs( list_input_e3sm_org_paths, @@ -22,6 +29,17 @@ def main(): list_sensitive_terms=list_search_phrases, ) relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms - with open(OUTPUT, "w", encoding="utf-8") as f: + with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: for link in relevant_links: f.write(f"{link}: {relevant_links[link]}\n") + + # Check that the inaccessible pages are in fact inaccessible ############### + with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] + with open(OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8") as f: + for e3sm_url in list_input_archived_e3sm_org_paths: + e3sm_url_status = get_e3sm_url_status(e3sm_url) + if e3sm_url_status == "link works not logged-in": + # This URL works, when it should not. + f.write(link) + pass diff --git a/e3sm_comms/page_reviewer/utils_base.py b/e3sm_comms/page_reviewer/utils_base.py index 03a9ffc..d5e9de7 100644 --- a/e3sm_comms/page_reviewer/utils_base.py +++ b/e3sm_comms/page_reviewer/utils_base.py @@ -295,7 +295,7 @@ def remove_output_files(config: Config): print(f"Could not remove {filename}: {e}") -# Functions used by newsletter, resource modes ################################ +# Functions used by newsletter, resource, term modes ########################## def map_confluence_to_e3sm(url: str, page_title: str = "") -> str: if page_title: # Map: @@ -349,6 +349,27 @@ def map_confluence_to_e3sm(url: str, page_title: str = "") -> str: return new_url +# Functions used by e3sm_org, term modes #################################### +def get_e3sm_url_status(e3sm_url: str) -> str: + try: + response = requests.get(e3sm_url, timeout=10) + response.raise_for_status() # Raises HTTPError for 4xx/5xx responses + e3sm_url_status = "link works not logged-in" + except requests.exceptions.Timeout: + e3sm_url_status = "link times out" + except requests.exceptions.RequestException as e: + error_message: str = f"{e}" + if error_message.startswith( + "503 Server Error: Service Temporarily Unavailable for url: https://e3sm.org" + ): + e3sm_url_status = "link not whitelisted" + else: + e3sm_url_status = "link raises RequestException" + except Exception: + e3sm_url_status = "link raises Exception" + return e3sm_url_status + + # Debugging ################################################################### def print_json(data: Dict): print(json.dumps(data, indent=4)) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 673ad5f..2b05113 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -3,9 +3,10 @@ from collections import defaultdict from typing import DefaultDict, Dict, List, Optional, Tuple -import requests # type: ignore - -from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.page_reviewer.utils_base import ( + get_e3sm_url_status, + map_confluence_to_e3sm, +) from e3sm_comms.utils import IO_DIR INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" @@ -138,28 +139,13 @@ def format_confluence_line(line: str) -> Optional[str]: e3sm_url = None e3sm_url_status: Optional[str] = None if e3sm_url: - try: - response = requests.get(e3sm_url, timeout=10) - response.raise_for_status() # Raises HTTPError for 4xx/5xx responses - e3sm_url_status = "link works not logged-in" - except requests.exceptions.Timeout: - e3sm_url_status = "link times out" - except requests.exceptions.RequestException as e: - error_message: str = f"{e}" - if error_message.startswith( - "503 Server Error: Service Temporarily Unavailable for url: https://e3sm.org" - ): - e3sm_url_status = "link not whitelisted" - else: - e3sm_url_status = "link raises RequestException" - except Exception: - e3sm_url_status = "link raises Exception" + e3sm_url_status = get_e3sm_url_status(e3sm_url) md = f"{title}: [confluence]({confluence_url})" if e3sm_url: md += f" [e3sm.org]({e3sm_url})" - if e3sm_url_status: - md += f" (Note: {e3sm_url_status})" + if e3sm_url_status: + md += f" (Note: {e3sm_url_status})" md += f" -- {counts}" return md From d5ef25131545769ec04292a6eefbc80b99fe170a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 11:35:48 -0700 Subject: [PATCH 11/85] Update print lines in e3sm org reviewer --- e3sm_comms/e3sm_org_reviewer/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index af77300..94c0ec4 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -22,7 +22,7 @@ def main(): with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: terms: List[str] = [line.rstrip("\n").lower() for line in f] list_search_phrases: List[str] = sorted(terms) - print(f"Checking {len(list_input_e3sm_org_paths)} e3sm.org pages") + print(f"Checking {len(list_input_e3sm_org_paths)} accessible e3sm.org pages") links = LinkedURLs( list_input_e3sm_org_paths, scan_links_for_sensitive_terms=True, @@ -36,6 +36,7 @@ def main(): # Check that the inaccessible pages are in fact inaccessible ############### with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] + print(f"Checking {len(list_input_archived_e3sm_org_paths)} archived e3sm.org pages") with open(OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8") as f: for e3sm_url in list_input_archived_e3sm_org_paths: e3sm_url_status = get_e3sm_url_status(e3sm_url) From 8bf405169494b012aa87961db5d76f3b1a51bf9a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 13:12:22 -0700 Subject: [PATCH 12/85] Share the archived web pages input --- e3sm_comms/e3sm_org_reviewer/main.py | 4 +--- e3sm_comms/term_reviewer/main.py | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 94c0ec4..6a19705 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -7,9 +7,7 @@ INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" -INPUT_ARCHIVED_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/e3sm_org_reviewer/archived_web_pages.txt" -) +INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 2b05113..268be36 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -11,6 +11,7 @@ INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" +INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" CONFLUENCE_SPACE = "EPWCD" @@ -239,6 +240,13 @@ def main() -> None: "These are the Confluence pages (serving as drafts of e3sm.org pages) that include sensitive terms. The 'confluence' links are what the script _actually_ reviewed. The 'e3sm.org' links are _predicted_ based on common URL naming patterns and thus may in fact be broken links. If the Confluence drafts and actual e3sm.org pages have not been kept in sync, remember that the term count is for the Confluence draft, not the actual e3sm.org page." ) + with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] + if list_input_archived_e3sm_org_paths: + print( + "TODO: sort so the archived files appear at the end, as if the year was 'archived'" + ) + entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) From 04bdb1ae67756d4295ab14216c583da63f3d1eb2 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 13:27:46 -0700 Subject: [PATCH 13/85] AI-generated addition of Archived section --- e3sm_comms/term_reviewer/main.py | 93 +++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 268be36..14ce65a 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -1,7 +1,7 @@ import ast import re from collections import defaultdict -from typing import DefaultDict, Dict, List, Optional, Tuple +from typing import Callable, DefaultDict, Dict, List, Optional, Tuple from e3sm_comms.page_reviewer.utils_base import ( get_e3sm_url_status, @@ -16,6 +16,7 @@ CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" +ARCHIVED_YEAR_LABEL = "Archived" FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") @@ -90,6 +91,76 @@ def sort_and_group_by_year(input_file: str) -> Dict[str, List[Tuple[int, str]]]: return dict(grouped_entries) +def extract_wordpress_url(line: str) -> Optional[str]: + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + if prefix.endswith(":"): + prefix = prefix[:-1].rstrip() + + return prefix + + +def extract_confluence_predicted_e3sm_url(line: str) -> Optional[str]: + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + if prefix.endswith("--"): + prefix = prefix[:-2].rstrip() + + first_colon = prefix.find(":") + if first_colon == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + page_id = prefix[:first_colon].strip() + title = prefix[first_colon + 1 :].strip() + + if not page_id.isdigit(): + print(f"Skipping Confluence line with non-numeric page id: {line}") + return None + + confluence_url = build_confluence_url(page_id) + + try: + return map_confluence_to_e3sm(confluence_url, page_title=title) + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + return None + + +def move_entries_to_archived_year( + grouped_entries: Dict[str, List[Tuple[int, str]]], + archived_paths: List[str], + e3sm_url_extractor: Callable[[str], Optional[str]], +) -> Dict[str, List[Tuple[int, str]]]: + archived_set = {path.strip() for path in archived_paths if path.strip()} + if not archived_set: + return grouped_entries + + updated: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list) + + for year, entries in grouped_entries.items(): + for total, line in entries: + e3sm_url = e3sm_url_extractor(line) + + if e3sm_url and e3sm_url in archived_set: + updated[ARCHIVED_YEAR_LABEL].append((total, line)) + else: + updated[year].append((total, line)) + + for year_key in updated: + updated[year_key].sort(key=lambda x: x[0], reverse=True) + + return dict(updated) + + def format_wordpress_line(line: str) -> Optional[str]: dict_start = line.find("{") if dict_start == -1: @@ -138,6 +209,7 @@ def format_confluence_line(line: str) -> Optional[str]: f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" ) e3sm_url = None + e3sm_url_status: Optional[str] = None if e3sm_url: e3sm_url_status = get_e3sm_url_status(e3sm_url) @@ -153,8 +225,10 @@ def format_confluence_line(line: str) -> Optional[str]: def year_sort_key(year_str: str) -> Tuple[int, int]: - if year_str == "Unknown year": + if year_str == ARCHIVED_YEAR_LABEL: return (1, 0) + if year_str == "Unknown year": + return (2, 0) return (0, -int(year_str)) @@ -242,13 +316,20 @@ def main() -> None: with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] - if list_input_archived_e3sm_org_paths: - print( - "TODO: sort so the archived files appear at the end, as if the year was 'archived'" - ) entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) + entries_e3sm_org = move_entries_to_archived_year( + entries_e3sm_org, + list_input_archived_e3sm_org_paths, + extract_wordpress_url, + ) + entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) + entries_confluence = move_entries_to_archived_year( + entries_confluence, + list_input_archived_e3sm_org_paths, + extract_confluence_predicted_e3sm_url, + ) with open(OUTPUT, "w", encoding="utf-8") as f: f.write("# Sensitive Terms Report\n\n") From 0f8bd748ed8b53b495d29c4e48a2817f101b59c6 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 14:59:41 -0700 Subject: [PATCH 14/85] Add example script and update README --- README.md | 4 ++-- examples/review_terms.bash | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100755 examples/review_terms.bash diff --git a/README.md b/README.md index 67cd842..23e613d 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ This package is for implementing the software needs of the E3SM Communications t - Known issues: more than just `<mark>` tags are changed (presumably no other semantic changes though) `e3sm-comms-term-reviewer` -- input: txt file of sensitive terms (e.g., output from `e3sm-comms-e3sm-org-reviewer` or `e3sm-comms-website-reviewer`) -- output: sorted version of that txt file +- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived +- output: Markdown report of terms found `e3sm-comms-tree-reviewer` - input: 2 txt files showing the website structure in hierarchical form (via indents) -- i.e. in tree form diff --git a/examples/review_terms.bash b/examples/review_terms.bash new file mode 100755 index 0000000..cfbaf6c --- /dev/null +++ b/examples/review_terms.bash @@ -0,0 +1,23 @@ +# Before running: +# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/web_pages.txt +# Also confirm confluence_top_levels_partial.txt is the list of top levels you want to use, otherwise switch it out. + +IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io + +echo "Count of top-level Confluence pages:" +wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt # Excludes MODEL, RESEARCH, DATA +echo "Count of whitelisted e3sm.org pages": +wc -l ${IO_DIR}/input/e3sm_org_reviewer/web_pages.txt + +echo "Step 1. Review Confluence" +echo "Note: this will require a Confluence login" +e3sm-comms-website-reviewer + +echo "Step 2. Review e3sm.org" +e3sm-comms-e3sm-org-reviewer + +echo "Step 3. Synthesize into report" +cp ${IO_DIR}/output/website_reviewer/sensitive_terms.txt ${IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt +cp ${IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt ${IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt +e3sm-comms-term-reviewer +echo "Output: ${IO_DIR}/output/term_reviewer/sensitive_terms.md" From 56cc0abd9c053cc1b706878881884e373ba7245b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 16:11:19 -0700 Subject: [PATCH 15/85] AI-generated check for manually reviewed pages --- e3sm_comms/term_reviewer/main.py | 38 +++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 14ce65a..e1cac52 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -12,11 +12,15 @@ INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" +INPUT_IGNORED_CONFLUENCE_PATHS: str = ( + f"{IO_DIR}/input/term_reviewer/ignored_confluence_paths.txt" +) OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" -ARCHIVED_YEAR_LABEL = "Archived" +ARCHIVED_YEAR_LABEL = "Archived (or should be archived)" +IGNORED_YEAR_LABEL = "IGNORED (manually reviewed)" FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") @@ -135,13 +139,14 @@ def extract_confluence_predicted_e3sm_url(line: str) -> Optional[str]: return None -def move_entries_to_archived_year( +def move_entries_to_label( grouped_entries: Dict[str, List[Tuple[int, str]]], - archived_paths: List[str], + matching_paths: List[str], e3sm_url_extractor: Callable[[str], Optional[str]], + target_label: str, ) -> Dict[str, List[Tuple[int, str]]]: - archived_set = {path.strip() for path in archived_paths if path.strip()} - if not archived_set: + matching_set = {path.strip() for path in matching_paths if path.strip()} + if not matching_set: return grouped_entries updated: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list) @@ -150,8 +155,8 @@ def move_entries_to_archived_year( for total, line in entries: e3sm_url = e3sm_url_extractor(line) - if e3sm_url and e3sm_url in archived_set: - updated[ARCHIVED_YEAR_LABEL].append((total, line)) + if e3sm_url and e3sm_url in matching_set: + updated[target_label].append((total, line)) else: updated[year].append((total, line)) @@ -227,8 +232,10 @@ def format_confluence_line(line: str) -> Optional[str]: def year_sort_key(year_str: str) -> Tuple[int, int]: if year_str == ARCHIVED_YEAR_LABEL: return (1, 0) - if year_str == "Unknown year": + if year_str == IGNORED_YEAR_LABEL: return (2, 0) + if year_str == "Unknown year": + return (3, 0) return (0, -int(year_str)) @@ -317,18 +324,29 @@ def main() -> None: with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] + with open(INPUT_IGNORED_CONFLUENCE_PATHS, "r", encoding="utf-8") as f: + list_input_ignored_confluence_paths: List[str] = [line.strip() for line in f] + entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) - entries_e3sm_org = move_entries_to_archived_year( + entries_e3sm_org = move_entries_to_label( entries_e3sm_org, list_input_archived_e3sm_org_paths, extract_wordpress_url, + ARCHIVED_YEAR_LABEL, ) entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) - entries_confluence = move_entries_to_archived_year( + entries_confluence = move_entries_to_label( entries_confluence, list_input_archived_e3sm_org_paths, extract_confluence_predicted_e3sm_url, + ARCHIVED_YEAR_LABEL, + ) + entries_confluence = move_entries_to_label( + entries_confluence, + list_input_ignored_confluence_paths, + extract_confluence_predicted_e3sm_url, + IGNORED_YEAR_LABEL, ) with open(OUTPUT, "w", encoding="utf-8") as f: From 86127cc10237fac61e28109dee6abd9b3378062e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 16:21:31 -0700 Subject: [PATCH 16/85] Update README for manually reviewed input file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 23e613d..58d1f2d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This package is for implementing the software needs of the E3SM Communications t - Known issues: more than just `<mark>` tags are changed (presumably no other semantic changes though) `e3sm-comms-term-reviewer` -- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived +- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing Confluence pages that have been manually reviewed (and thus can be safely ignored). - output: Markdown report of terms found `e3sm-comms-tree-reviewer` From 6007e5585bd034c8a00d3945da1ca35e1d950e93 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 16:27:57 -0700 Subject: [PATCH 17/85] Convert manual review check from Confluence to e3sm.org --- e3sm_comms/term_reviewer/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index e1cac52..61ddddb 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -12,8 +12,8 @@ INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" -INPUT_IGNORED_CONFLUENCE_PATHS: str = ( - f"{IO_DIR}/input/term_reviewer/ignored_confluence_paths.txt" +INPUT_IGNORED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/term_reviewer/ignored_e3sm_org_paths.txt" ) OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" @@ -324,8 +324,8 @@ def main() -> None: with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] - with open(INPUT_IGNORED_CONFLUENCE_PATHS, "r", encoding="utf-8") as f: - list_input_ignored_confluence_paths: List[str] = [line.strip() for line in f] + with open(INPUT_IGNORED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_ignored_e3sm_org_paths: List[str] = [line.strip() for line in f] entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) entries_e3sm_org = move_entries_to_label( @@ -334,6 +334,12 @@ def main() -> None: extract_wordpress_url, ARCHIVED_YEAR_LABEL, ) + entries_e3sm_org = move_entries_to_label( + entries_e3sm_org, + list_input_ignored_e3sm_org_paths, + extract_wordpress_url, + IGNORED_YEAR_LABEL, + ) entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) entries_confluence = move_entries_to_label( @@ -344,7 +350,7 @@ def main() -> None: ) entries_confluence = move_entries_to_label( entries_confluence, - list_input_ignored_confluence_paths, + list_input_ignored_e3sm_org_paths, extract_confluence_predicted_e3sm_url, IGNORED_YEAR_LABEL, ) From 704317bd81e8c2a5adc1173139a13a52a0dac2e7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 16:50:22 -0700 Subject: [PATCH 18/85] AI replace IGNORED with KNOWN OK and DOES NOT EXIST --- e3sm_comms/term_reviewer/main.py | 45 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 61ddddb..0f331b1 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -12,15 +12,19 @@ INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" -INPUT_IGNORED_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/term_reviewer/ignored_e3sm_org_paths.txt" +INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/term_reviewer/known_ok_e3sm_org_paths.txt" +) +INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/term_reviewer/does_not_exist_e3sm_org_paths.txt" ) OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" ARCHIVED_YEAR_LABEL = "Archived (or should be archived)" -IGNORED_YEAR_LABEL = "IGNORED (manually reviewed)" +KNOWN_OK_LABEL = "KNOWN OK" +DOES_NOT_EXIST_LABEL = "DOES NOT EXIST" FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") @@ -232,10 +236,12 @@ def format_confluence_line(line: str) -> Optional[str]: def year_sort_key(year_str: str) -> Tuple[int, int]: if year_str == ARCHIVED_YEAR_LABEL: return (1, 0) - if year_str == IGNORED_YEAR_LABEL: + if year_str == KNOWN_OK_LABEL: return (2, 0) - if year_str == "Unknown year": + if year_str == DOES_NOT_EXIST_LABEL: return (3, 0) + if year_str == "Unknown year": + return (4, 0) return (0, -int(year_str)) @@ -324,8 +330,13 @@ def main() -> None: with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] - with open(INPUT_IGNORED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_ignored_e3sm_org_paths: List[str] = [line.strip() for line in f] + with open(INPUT_KNOWN_OK_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_known_ok_e3sm_org_paths: List[str] = [line.strip() for line in f] + + with open(INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_does_not_exist_e3sm_org_paths: List[str] = [ + line.strip() for line in f + ] entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) entries_e3sm_org = move_entries_to_label( @@ -336,9 +347,15 @@ def main() -> None: ) entries_e3sm_org = move_entries_to_label( entries_e3sm_org, - list_input_ignored_e3sm_org_paths, + list_input_known_ok_e3sm_org_paths, + extract_wordpress_url, + KNOWN_OK_LABEL, + ) + entries_e3sm_org = move_entries_to_label( + entries_e3sm_org, + list_input_does_not_exist_e3sm_org_paths, extract_wordpress_url, - IGNORED_YEAR_LABEL, + DOES_NOT_EXIST_LABEL, ) entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) @@ -350,9 +367,15 @@ def main() -> None: ) entries_confluence = move_entries_to_label( entries_confluence, - list_input_ignored_e3sm_org_paths, + list_input_known_ok_e3sm_org_paths, + extract_confluence_predicted_e3sm_url, + KNOWN_OK_LABEL, + ) + entries_confluence = move_entries_to_label( + entries_confluence, + list_input_does_not_exist_e3sm_org_paths, extract_confluence_predicted_e3sm_url, - IGNORED_YEAR_LABEL, + DOES_NOT_EXIST_LABEL, ) with open(OUTPUT, "w", encoding="utf-8") as f: From 667c85794c80fb1a440cea0d24ced919236f58fc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 16:52:18 -0700 Subject: [PATCH 19/85] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58d1f2d..fbc3742 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This package is for implementing the software needs of the E3SM Communications t - Known issues: more than just `<mark>` tags are changed (presumably no other semantic changes though) `e3sm-comms-term-reviewer` -- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing Confluence pages that have been manually reviewed (and thus can be safely ignored). +- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing e3sm.org pages that do not contain the search terms (and presumably only show up because their corresponding Confluence pages have the terms somewhere in metadata), txt file listing e3sm.org pages that are known not to exist (either the script couldn't determine the correct e3sm.org path, or it doesn't even exist). - output: Markdown report of terms found `e3sm-comms-tree-reviewer` From 679d9b27039e5fc49d0f6181a586b4a0b17cc3ff Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 17:09:02 -0700 Subject: [PATCH 20/85] Update labels --- e3sm_comms/term_reviewer/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 0f331b1..75c5fb8 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -23,8 +23,10 @@ CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" ARCHIVED_YEAR_LABEL = "Archived (or should be archived)" -KNOWN_OK_LABEL = "KNOWN OK" -DOES_NOT_EXIST_LABEL = "DOES NOT EXIST" +KNOWN_OK_LABEL = "Known OK (Confluence page may be reporting terms that aren't showing up on the e3sm.org page)" +DOES_NOT_EXIST_LABEL = ( + "Does not exist (either script couldn't find e3sm.org URL or none exists)" +) FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") From 9d774f6febf463cf6ba2289a574b95b5f02fff19 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 17:12:40 -0700 Subject: [PATCH 21/85] AI generated add keep-unchanged bucket --- e3sm_comms/term_reviewer/main.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py index 75c5fb8..7a49488 100644 --- a/e3sm_comms/term_reviewer/main.py +++ b/e3sm_comms/term_reviewer/main.py @@ -15,6 +15,9 @@ INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/input/term_reviewer/known_ok_e3sm_org_paths.txt" ) +INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/term_reviewer/keep_unchanged_e3sm_org_paths.txt" +) INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/input/term_reviewer/does_not_exist_e3sm_org_paths.txt" ) @@ -24,6 +27,7 @@ CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" ARCHIVED_YEAR_LABEL = "Archived (or should be archived)" KNOWN_OK_LABEL = "Known OK (Confluence page may be reporting terms that aren't showing up on the e3sm.org page)" +KEEP_UNCHANGED_LABEL = "Keep unchanged" DOES_NOT_EXIST_LABEL = ( "Does not exist (either script couldn't find e3sm.org URL or none exists)" ) @@ -240,10 +244,12 @@ def year_sort_key(year_str: str) -> Tuple[int, int]: return (1, 0) if year_str == KNOWN_OK_LABEL: return (2, 0) - if year_str == DOES_NOT_EXIST_LABEL: + if year_str == KEEP_UNCHANGED_LABEL: return (3, 0) - if year_str == "Unknown year": + if year_str == DOES_NOT_EXIST_LABEL: return (4, 0) + if year_str == "Unknown year": + return (5, 0) return (0, -int(year_str)) @@ -335,6 +341,11 @@ def main() -> None: with open(INPUT_KNOWN_OK_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_known_ok_e3sm_org_paths: List[str] = [line.strip() for line in f] + with open(INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_input_keep_unchanged_e3sm_org_paths: List[str] = [ + line.strip() for line in f + ] + with open(INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: list_input_does_not_exist_e3sm_org_paths: List[str] = [ line.strip() for line in f @@ -353,6 +364,12 @@ def main() -> None: extract_wordpress_url, KNOWN_OK_LABEL, ) + entries_e3sm_org = move_entries_to_label( + entries_e3sm_org, + list_input_keep_unchanged_e3sm_org_paths, + extract_wordpress_url, + KEEP_UNCHANGED_LABEL, + ) entries_e3sm_org = move_entries_to_label( entries_e3sm_org, list_input_does_not_exist_e3sm_org_paths, @@ -373,6 +390,12 @@ def main() -> None: extract_confluence_predicted_e3sm_url, KNOWN_OK_LABEL, ) + entries_confluence = move_entries_to_label( + entries_confluence, + list_input_keep_unchanged_e3sm_org_paths, + extract_confluence_predicted_e3sm_url, + KEEP_UNCHANGED_LABEL, + ) entries_confluence = move_entries_to_label( entries_confluence, list_input_does_not_exist_e3sm_org_paths, From 5c4e2557c81de67a75ad2d88f2614e25e16e762e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 4 May 2026 17:13:45 -0700 Subject: [PATCH 22/85] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbc3742..6f2da6d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This package is for implementing the software needs of the E3SM Communications t - Known issues: more than just `<mark>` tags are changed (presumably no other semantic changes though) `e3sm-comms-term-reviewer` -- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing e3sm.org pages that do not contain the search terms (and presumably only show up because their corresponding Confluence pages have the terms somewhere in metadata), txt file listing e3sm.org pages that are known not to exist (either the script couldn't determine the correct e3sm.org path, or it doesn't even exist). +- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing e3sm.org pages that do not contain the search terms (and presumably only show up because their corresponding Confluence pages have the terms somewhere in metadata), txt file listing e3sm.org pages that are known not to exist (either the script couldn't determine the correct e3sm.org path, or it doesn't even exist), txt file listing e3sm.org pages that are to be kept unchanged. - output: Markdown report of terms found `e3sm-comms-tree-reviewer` From 9b1fd30e43999a856b15e81642d9480caef816fe Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 11:13:19 -0700 Subject: [PATCH 23/85] Produce lists of all WordPress pages and posts --- README.md | 4 +- e3sm_comms/e3sm_org_reviewer/main.py | 70 +++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6f2da6d..e95f721 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ This package is for implementing the software needs of the E3SM Communications t ### Simple commands `e3sm-comms-e3sm-org-reviewer` -- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for, txt file listing e3sm.org pages that should be marked as archived -- output: txt file listing e3sm.org pages containing those phrases, txt file listing e3sm.org pages that are accessible even though they should be archived +- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for, txt file listing e3sm.org pages that should be marked as archived, xml file of Wordpress pages, xml file of Wordpress posts. Note: xml files can be obtained from WordPress under Tools > Export. +- output: txt file listing e3sm.org pages containing those phrases, txt file listing e3sm.org pages that are accessible even though they should be archived, txt file of page URLs found in the xml, txt file of post URLs found in the xml. `e3sm-comms-html-reviewer` - input: 1 txt file of html copied from WordPress that includes yellow highlights left over from Confluence. diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 6a19705..2565c0e 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,3 +1,4 @@ +import xml.etree.ElementTree as ET from typing import Dict, List from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status @@ -12,6 +13,15 @@ f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) +INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" +INPUT_XML_POSTS: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_posts.xml" +OUTPUT_PAGE_URLS_FROM_XML: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/page_urls_from_xml.txt" +) +OUTPUT_POST_URLS_FROM_XML: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/post_urls_from_xml.txt" +) + def main(): # Check the accesible pages for search phrases ############################ @@ -40,5 +50,63 @@ def main(): e3sm_url_status = get_e3sm_url_status(e3sm_url) if e3sm_url_status == "link works not logged-in": # This URL works, when it should not. - f.write(link) + f.write(f"{link}\n") pass + + # Review XML exports from WordPress ####################################### + # urls = get_wordpress_page_urls("wordpress-export.xml", status_filter="publish") + page_urls = get_wordpress_urls(INPUT_XML_PAGES, "page") + post_urls = get_wordpress_urls(INPUT_XML_POSTS, "post") + print(f"Checking {len(page_urls)} page URLs, {len(post_urls)} post URLs") + with open(OUTPUT_PAGE_URLS_FROM_XML, "w", encoding="utf-8") as f: + for url in page_urls: + f.write(f"{url}\n") + with open(OUTPUT_POST_URLS_FROM_XML, "w", encoding="utf-8") as f: + for url in post_urls: + f.write(f"{url}\n") + + +def get_wordpress_urls(xml_file_path: str, wordpress_type: str, status_filter=None): + """ + Read a WordPress export XML file and return a list of hosted page URLs. + + Args: + xml_file_path (str): Path to the WordPress export XML file. + status_filter (str | None): Optional, like 'publish'. If set, only + pages with this wp:status are included. publish, archive + + Returns: + list[str]: List of page URLs. + """ + ns = { + "content": "http://purl.org/rss/1.0/modules/content/", + "excerpt": "http://wordpress.org/export/1.2/excerpt/", + "wfw": "http://wellformedweb.org/CommentAPI/", + "dc": "http://purl.org/dc/elements/1.1/", + "wp": "http://wordpress.org/export/1.2/", + } + + tree = ET.parse(xml_file_path) + root = tree.getroot() + + urls: List[str] = [] + + channel = root.find("channel") + if channel is None: + return urls + + for item in channel.findall("item"): + post_type = item.find("wp:post_type", ns) + status = item.find("wp:status", ns) + link = item.find("link") + + if post_type is None or post_type.text != wordpress_type: + continue + + if status_filter and (status is None or status.text != status_filter): + continue + + if link is not None and link.text: + urls.append(link.text.strip()) + + return urls From 908d3bd819494659522ad8858d5304627a5eba88 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 12:24:26 -0700 Subject: [PATCH 24/85] Add more filtering of e3sm.org paths --- e3sm_comms/e3sm_org_reviewer/main.py | 257 +++++++++++++++++++-------- 1 file changed, 184 insertions(+), 73 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 2565c0e..fb3845b 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,112 +1,223 @@ import xml.etree.ElementTree as ET +from collections import defaultdict from typing import Dict, List from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status from e3sm_comms.utils import IO_DIR -INPUT_ACCESSIBLE_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/e3sm_org_reviewer/web_pages.txt" +INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" +INPUT_XML_POSTS: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_posts.xml" +INPUT_WHITELIST: str = f"{IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt" +INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/shared/archived_web_pages.txt" +) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" -INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" +OUTPUT_WHITELISTED_NOT_PUBLISHED: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/whitelisted_not_published.txt" +) +OUTPUT_PUBLISHED_NOT_WHITELISTED: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/published_not_whitelisted.txt" +) +OUTPUT_SHOULD_BE_ARCHIVED: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/should_be_archived.txt" +) +OUTPUT_EXTRA_ARCHIVED: str = f"{IO_DIR}/output/e3sm_org_reviewer/extra_archived.txt" +OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) -INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" -INPUT_XML_POSTS: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_posts.xml" -OUTPUT_PAGE_URLS_FROM_XML: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/page_urls_from_xml.txt" -) -OUTPUT_POST_URLS_FROM_XML: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/post_urls_from_xml.txt" -) +RUN_CHECKS: bool = False # Set to False for faster debugging def main(): - # Check the accesible pages for search phrases ############################ - with open(INPUT_ACCESSIBLE_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_e3sm_org_paths: List[str] = [line.strip() for line in f] - with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: - terms: List[str] = [line.rstrip("\n").lower() for line in f] - list_search_phrases: List[str] = sorted(terms) - print(f"Checking {len(list_input_e3sm_org_paths)} accessible e3sm.org pages") - links = LinkedURLs( - list_input_e3sm_org_paths, - scan_links_for_sensitive_terms=True, - list_sensitive_terms=list_search_phrases, + # Review XML exports from WordPress ####################################### + pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_PAGES, "page" + ) + posts_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_POSTS, "post" + ) + num_pages: int = get_total_count(pages_by_status) + num_posts: int = get_total_count(posts_by_status) + print(f"Found {num_pages} pages, {num_posts} posts") + # ['archive', 'draft', 'future', 'pending', 'private', 'publish'] + print( + f"Pages have status in {pages_by_status.keys()}; posts have status in {posts_by_status.keys()}" ) - relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms - with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: - for link in relevant_links: - f.write(f"{link}: {relevant_links[link]}\n") - - # Check that the inaccessible pages are in fact inaccessible ############### - with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] - print(f"Checking {len(list_input_archived_e3sm_org_paths)} archived e3sm.org pages") - with open(OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8") as f: - for e3sm_url in list_input_archived_e3sm_org_paths: - e3sm_url_status = get_e3sm_url_status(e3sm_url) - if e3sm_url_status == "link works not logged-in": - # This URL works, when it should not. - f.write(f"{link}\n") - pass - # Review XML exports from WordPress ####################################### - # urls = get_wordpress_page_urls("wordpress-export.xml", status_filter="publish") - page_urls = get_wordpress_urls(INPUT_XML_PAGES, "page") - post_urls = get_wordpress_urls(INPUT_XML_POSTS, "post") - print(f"Checking {len(page_urls)} page URLs, {len(post_urls)} post URLs") - with open(OUTPUT_PAGE_URLS_FROM_XML, "w", encoding="utf-8") as f: - for url in page_urls: + all_urls_by_status: Dict[str, List[str]] = get_combined_urls_by_status( + pages_by_status, posts_by_status + ) + if "publish" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['publish'])} published URLs") + if "archive" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['archive'])} archived URLs") + if "draft" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['draft'])} draft URLs") + if "future" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['future'])} future URLs") + if "pending" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['pending'])} pending URLs") + if "private" in all_urls_by_status: + print(f"Found {len(all_urls_by_status['private'])} private URLs") + non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) + print(f"Total non-published URLs: {len(non_published_urls)}") + + # Compare with expectations ############################################### + with open(INPUT_WHITELIST, "r", encoding="utf-8") as f: + list_whitelisted_paths: List[str] = [line.strip() for line in f] + with open(INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: + list_expected_archived_paths: List[str] = [line.strip() for line in f] + + all_urls: List[str] = get_all_urls(all_urls_by_status) + invalid_whitelisted_paths: List[str] = get_list_difference( + list_whitelisted_paths, all_urls + ) + valid_whitelisted_paths: List[str] = get_list_difference( + list_whitelisted_paths, invalid_whitelisted_paths + ) + invalid_expected_archived_paths: List[str] = get_list_difference( + list_expected_archived_paths, all_urls + ) + valid_expected_archived_paths: List[str] = get_list_difference( + list_expected_archived_paths, invalid_expected_archived_paths + ) + print( + f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs" + ) + print( + f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs" + ) + + whitelisted_but_not_published: List[str] = get_list_difference( + valid_whitelisted_paths, all_urls_by_status["publish"] + ) + published_but_not_whitelisted: List[str] = get_list_difference( + all_urls_by_status["publish"], valid_whitelisted_paths + ) + print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") + print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") + with open(OUTPUT_WHITELISTED_NOT_PUBLISHED, "w", encoding="utf-8") as f: + for url in whitelisted_but_not_published: f.write(f"{url}\n") - with open(OUTPUT_POST_URLS_FROM_XML, "w", encoding="utf-8") as f: - for url in post_urls: + with open(OUTPUT_PUBLISHED_NOT_WHITELISTED, "w", encoding="utf-8") as f: + for url in published_but_not_whitelisted: f.write(f"{url}\n") + should_be_archived: List[str] = get_list_difference( + valid_expected_archived_paths, all_urls_by_status["archive"] + ) + extra_archived: List[str] = get_list_difference( + all_urls_by_status["archive"], valid_expected_archived_paths + ) + print(f"Not archived, but should be archived: {len(should_be_archived)}") + print(f"Archived, but weren't on our expected list: {len(extra_archived)}") + with open(OUTPUT_SHOULD_BE_ARCHIVED, "w", encoding="utf-8") as f: + for url in should_be_archived: + f.write(f"{url}\n") + with open(OUTPUT_EXTRA_ARCHIVED, "w", encoding="utf-8") as f: + for url in extra_archived: + f.write(f"{url}\n") -def get_wordpress_urls(xml_file_path: str, wordpress_type: str, status_filter=None): - """ - Read a WordPress export XML file and return a list of hosted page URLs. - - Args: - xml_file_path (str): Path to the WordPress export XML file. - status_filter (str | None): Optional, like 'publish'. If set, only - pages with this wp:status are included. publish, archive - - Returns: - list[str]: List of page URLs. - """ + # Run checks ############################################################## + if RUN_CHECKS: + print( + f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" + ) + with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: + terms: List[str] = [line.rstrip("\n").lower() for line in f] + list_search_phrases: List[str] = sorted(terms) + links = LinkedURLs( + list_whitelisted_paths, + scan_links_for_sensitive_terms=True, + list_sensitive_terms=list_search_phrases, + ) + relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms + with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: + for link in relevant_links: + f.write(f"{link}: {relevant_links[link]}\n") + + print( + f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" + ) + with open( + OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8" + ) as f: + for e3sm_url in list_expected_archived_paths: + e3sm_url_status = get_e3sm_url_status(e3sm_url) + if e3sm_url_status == "link works not logged-in": + # This URL works, when it should not. + f.write(f"{e3sm_url}\n") + pass + + +def get_wordpress_urls_by_status(xml_file_path: str, post_type: str): ns = { - "content": "http://purl.org/rss/1.0/modules/content/", - "excerpt": "http://wordpress.org/export/1.2/excerpt/", - "wfw": "http://wellformedweb.org/CommentAPI/", - "dc": "http://purl.org/dc/elements/1.1/", "wp": "http://wordpress.org/export/1.2/", } tree = ET.parse(xml_file_path) root = tree.getroot() - urls: List[str] = [] - + grouped = defaultdict(list) channel = root.find("channel") if channel is None: - return urls + return {} for item in channel.findall("item"): - post_type = item.find("wp:post_type", ns) - status = item.find("wp:status", ns) + item_post_type = item.find("wp:post_type", ns) + item_status = item.find("wp:status", ns) link = item.find("link") - if post_type is None or post_type.text != wordpress_type: + if item_post_type is None or item_post_type.text != post_type: continue - if status_filter and (status is None or status.text != status_filter): - continue + status = ( + item_status.text.strip() + if item_status is not None and item_status.text + else "unknown" + ) if link is not None and link.text: - urls.append(link.text.strip()) + grouped[status].append(link.text.strip()) + + return {status: sorted(urls) for status, urls in sorted(grouped.items())} + + +def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: + count: int = 0 + for status in urls_by_status: + count += len(urls_by_status[status]) + return count + + +def get_combined_urls_by_status( + pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] +) -> Dict[str, List[str]]: + merged = defaultdict(list) + for source in (pages_by_status, posts_by_status): + for status, urls in source.items(): + merged[status].extend(urls) + return dict(merged) + + +def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: + diff: List[str] = list(set(list1) - set(list2)) + return sorted(diff) + + +def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + non_published_urls: List[str] = [] + for status in urls_by_status: + non_published_urls += urls_by_status[status] + return non_published_urls + - return urls +def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + non_published_urls: List[str] = [] + for status in urls_by_status: + if status != "publish": + non_published_urls += urls_by_status[status] + return non_published_urls From 5709c637e02a6f55cefc4247af1ce66e66c51e48 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 12:53:50 -0700 Subject: [PATCH 25/85] Change output to Markdown report --- e3sm_comms/e3sm_org_reviewer/main.py | 178 ++++++++++++++++----------- 1 file changed, 107 insertions(+), 71 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index fb3845b..37f73e2 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -13,16 +13,7 @@ ) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -OUTPUT_WHITELISTED_NOT_PUBLISHED: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/whitelisted_not_published.txt" -) -OUTPUT_PUBLISHED_NOT_WHITELISTED: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/published_not_whitelisted.txt" -) -OUTPUT_SHOULD_BE_ARCHIVED: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/should_be_archived.txt" -) -OUTPUT_EXTRA_ARCHIVED: str = f"{IO_DIR}/output/e3sm_org_reviewer/extra_archived.txt" +OUTPUT_MARKDOWN_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/path_report.md" OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" @@ -32,7 +23,7 @@ def main(): - # Review XML exports from WordPress ####################################### + # Review XML exports from WordPress pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( INPUT_XML_PAGES, "page" ) @@ -42,7 +33,6 @@ def main(): num_pages: int = get_total_count(pages_by_status) num_posts: int = get_total_count(posts_by_status) print(f"Found {num_pages} pages, {num_posts} posts") - # ['archive', 'draft', 'future', 'pending', 'private', 'publish'] print( f"Pages have status in {pages_by_status.keys()}; posts have status in {posts_by_status.keys()}" ) @@ -50,40 +40,35 @@ def main(): all_urls_by_status: Dict[str, List[str]] = get_combined_urls_by_status( pages_by_status, posts_by_status ) - if "publish" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['publish'])} published URLs") - if "archive" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['archive'])} archived URLs") - if "draft" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['draft'])} draft URLs") - if "future" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['future'])} future URLs") - if "pending" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['pending'])} pending URLs") - if "private" in all_urls_by_status: - print(f"Found {len(all_urls_by_status['private'])} private URLs") + print_status_counts(all_urls_by_status) + non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) print(f"Total non-published URLs: {len(non_published_urls)}") - # Compare with expectations ############################################### + # Compare with expectations with open(INPUT_WHITELIST, "r", encoding="utf-8") as f: - list_whitelisted_paths: List[str] = [line.strip() for line in f] + list_whitelisted_paths: List[str] = [line.strip() for line in f if line.strip()] with open(INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_expected_archived_paths: List[str] = [line.strip() for line in f] + list_expected_archived_paths: List[str] = [ + line.strip() for line in f if line.strip() + ] all_urls: List[str] = get_all_urls(all_urls_by_status) + invalid_whitelisted_paths: List[str] = get_list_difference( list_whitelisted_paths, all_urls ) valid_whitelisted_paths: List[str] = get_list_difference( list_whitelisted_paths, invalid_whitelisted_paths ) + invalid_expected_archived_paths: List[str] = get_list_difference( list_expected_archived_paths, all_urls ) valid_expected_archived_paths: List[str] = get_list_difference( list_expected_archived_paths, invalid_expected_archived_paths ) + print( f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs" ) @@ -91,37 +76,35 @@ def main(): f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs" ) + published_urls: List[str] = all_urls_by_status.get("publish", []) + archived_urls: List[str] = all_urls_by_status.get("archive", []) + whitelisted_but_not_published: List[str] = get_list_difference( - valid_whitelisted_paths, all_urls_by_status["publish"] + valid_whitelisted_paths, published_urls ) published_but_not_whitelisted: List[str] = get_list_difference( - all_urls_by_status["publish"], valid_whitelisted_paths + published_urls, valid_whitelisted_paths ) - print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") - print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") - with open(OUTPUT_WHITELISTED_NOT_PUBLISHED, "w", encoding="utf-8") as f: - for url in whitelisted_but_not_published: - f.write(f"{url}\n") - with open(OUTPUT_PUBLISHED_NOT_WHITELISTED, "w", encoding="utf-8") as f: - for url in published_but_not_whitelisted: - f.write(f"{url}\n") - should_be_archived: List[str] = get_list_difference( - valid_expected_archived_paths, all_urls_by_status["archive"] - ) - extra_archived: List[str] = get_list_difference( - all_urls_by_status["archive"], valid_expected_archived_paths + valid_expected_archived_paths, archived_urls ) + + print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") + print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") print(f"Not archived, but should be archived: {len(should_be_archived)}") - print(f"Archived, but weren't on our expected list: {len(extra_archived)}") - with open(OUTPUT_SHOULD_BE_ARCHIVED, "w", encoding="utf-8") as f: - for url in should_be_archived: - f.write(f"{url}\n") - with open(OUTPUT_EXTRA_ARCHIVED, "w", encoding="utf-8") as f: - for url in extra_archived: - f.write(f"{url}\n") - - # Run checks ############################################################## + print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") + print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") + + write_markdown_report( + output_path=OUTPUT_MARKDOWN_REPORT, + whitelisted_but_not_published=whitelisted_but_not_published, + published_but_not_whitelisted=published_but_not_whitelisted, + should_be_archived=should_be_archived, + invalid_whitelisted_paths=invalid_whitelisted_paths, + invalid_expected_archived_paths=invalid_expected_archived_paths, + ) + + # Run checks if RUN_CHECKS: print( f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" @@ -129,6 +112,7 @@ def main(): with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: terms: List[str] = [line.rstrip("\n").lower() for line in f] list_search_phrases: List[str] = sorted(terms) + links = LinkedURLs( list_whitelisted_paths, scan_links_for_sensitive_terms=True, @@ -145,15 +129,71 @@ def main(): with open( OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8" ) as f: - for e3sm_url in list_expected_archived_paths: + for e3sm_url in non_published_urls: e3sm_url_status = get_e3sm_url_status(e3sm_url) if e3sm_url_status == "link works not logged-in": - # This URL works, when it should not. f.write(f"{e3sm_url}\n") - pass -def get_wordpress_urls_by_status(xml_file_path: str, post_type: str): +def write_markdown_report( + output_path: str, + whitelisted_but_not_published: List[str], + published_but_not_whitelisted: List[str], + should_be_archived: List[str], + invalid_whitelisted_paths: List[str], + invalid_expected_archived_paths: List[str], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Valid Paths\n\n") + write_markdown_section( + f, + "Whitelisted but not published", + whitelisted_but_not_published, + ) + write_markdown_section( + f, + "Published but not whitelisted", + published_but_not_whitelisted, + ) + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + + f.write("# Invalid Paths\n\n") + write_markdown_section( + f, + "Identified in whitelist input", + invalid_whitelisted_paths, + ) + write_markdown_section( + f, + "Identified in archive input", + invalid_expected_archived_paths, + ) + + +def write_markdown_section(file_obj, title: str, items: List[str]) -> None: + file_obj.write(f"## {title}\n\n") + if not items: + file_obj.write("_None._\n\n") + return + + for i, item in enumerate(items, start=1): + file_obj.write(f"{i}. {item}\n") + file_obj.write("\n") + + +def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: + for status in ["publish", "archive", "draft", "future", "pending", "private"]: + if status in all_urls_by_status: + print(f"Found {len(all_urls_by_status[status])} {status} URLs") + + +def get_wordpress_urls_by_status( + xml_file_path: str, post_type: str +) -> Dict[str, List[str]]: ns = { "wp": "http://wordpress.org/export/1.2/", } @@ -161,7 +201,7 @@ def get_wordpress_urls_by_status(xml_file_path: str, post_type: str): tree = ET.parse(xml_file_path) root = tree.getroot() - grouped = defaultdict(list) + grouped: Dict[str, List[str]] = defaultdict(list) channel = root.find("channel") if channel is None: return {} @@ -187,37 +227,33 @@ def get_wordpress_urls_by_status(xml_file_path: str, post_type: str): def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: - count: int = 0 - for status in urls_by_status: - count += len(urls_by_status[status]) - return count + return sum(len(urls) for urls in urls_by_status.values()) def get_combined_urls_by_status( pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] ) -> Dict[str, List[str]]: - merged = defaultdict(list) + merged: Dict[str, List[str]] = defaultdict(list) for source in (pages_by_status, posts_by_status): for status, urls in source.items(): merged[status].extend(urls) - return dict(merged) + return {status: sorted(urls) for status, urls in merged.items()} def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: - diff: List[str] = list(set(list1) - set(list2)) - return sorted(diff) + return sorted(set(list1) - set(list2)) def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: - non_published_urls: List[str] = [] - for status in urls_by_status: - non_published_urls += urls_by_status[status] - return non_published_urls + all_urls: List[str] = [] + for urls in urls_by_status.values(): + all_urls.extend(urls) + return sorted(all_urls) def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: non_published_urls: List[str] = [] - for status in urls_by_status: + for status, urls in urls_by_status.items(): if status != "publish": - non_published_urls += urls_by_status[status] - return non_published_urls + non_published_urls.extend(urls) + return sorted(non_published_urls) From 68227b742a5a9e87c955aeb50db39baa9bcbbc81 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 12:58:21 -0700 Subject: [PATCH 26/85] Add summary table to Markdown report --- e3sm_comms/e3sm_org_reviewer/main.py | 68 +++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 37f73e2..b83b80e 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,6 +1,6 @@ import xml.etree.ElementTree as ET from collections import defaultdict -from typing import Dict, List +from typing import Dict, List, TextIO from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status from e3sm_comms.utils import IO_DIR @@ -97,6 +97,9 @@ def main(): write_markdown_report( output_path=OUTPUT_MARKDOWN_REPORT, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, whitelisted_but_not_published=whitelisted_but_not_published, published_but_not_whitelisted=published_but_not_whitelisted, should_be_archived=should_be_archived, @@ -137,6 +140,9 @@ def main(): def write_markdown_report( output_path: str, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], whitelisted_but_not_published: List[str], published_but_not_whitelisted: List[str], should_be_archived: List[str], @@ -144,6 +150,13 @@ def write_markdown_report( invalid_expected_archived_paths: List[str], ) -> None: with open(output_path, "w", encoding="utf-8") as f: + write_summary_table( + f, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + ) + f.write("# Valid Paths\n\n") write_markdown_section( f, @@ -174,7 +187,58 @@ def write_markdown_report( ) -def write_markdown_section(file_obj, title: str, items: List[str]) -> None: +def write_summary_table( + file_obj: TextIO, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], +) -> None: + statuses: List[str] = sorted(all_urls_by_status.keys()) + + whitelist_set = set(valid_whitelisted_paths) + expected_archived_set = set(valid_expected_archived_paths) + accounted_for_set = whitelist_set.union(expected_archived_set) + + all_urls_set = set(get_all_urls(all_urls_by_status)) + remaining_set = all_urls_set - accounted_for_set + + rows = [ + ("Whitelisted URLs", whitelist_set), + ("Expected archived", expected_archived_set), + ("All remaining", remaining_set), + ("TOTAL", all_urls_set), + ] + + file_obj.write("# Summary\n\n") + file_obj.write("| Type | " + " | ".join(statuses) + " |\n") + file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " |\n") + + for row_name, row_urls in rows: + counts = get_status_counts_for_urls( + urls=list(row_urls), + all_urls_by_status=all_urls_by_status, + statuses=statuses, + ) + file_obj.write( + f"| {row_name} | " + + " | ".join(str(counts[status]) for status in statuses) + + " |\n" + ) + + file_obj.write("\n") + + +def get_status_counts_for_urls( + urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] +) -> Dict[str, int]: + url_set = set(urls) + counts: Dict[str, int] = {} + for status in statuses: + counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) + return counts + + +def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: file_obj.write(f"## {title}\n\n") if not items: file_obj.write("_None._\n\n") From 4e651311d54a9357f064244f1f8b353b79334651 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 13:00:01 -0700 Subject: [PATCH 27/85] Add total col to summary table --- e3sm_comms/e3sm_org_reviewer/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index b83b80e..8196066 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -210,8 +210,8 @@ def write_summary_table( ] file_obj.write("# Summary\n\n") - file_obj.write("| Type | " + " | ".join(statuses) + " |\n") - file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " |\n") + file_obj.write("| Type | " + " | ".join(statuses) + " | Total |\n") + file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " | --- |\n") for row_name, row_urls in rows: counts = get_status_counts_for_urls( @@ -219,10 +219,11 @@ def write_summary_table( all_urls_by_status=all_urls_by_status, statuses=statuses, ) + total_count = sum(counts.values()) file_obj.write( f"| {row_name} | " + " | ".join(str(counts[status]) for status in statuses) - + " |\n" + + f" | {total_count} |\n" ) file_obj.write("\n") From 2d99f9b9098b1a5451bd36872b845d4a6d1cf5e5 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 13:16:18 -0700 Subject: [PATCH 28/85] Handle expandable patterns --- e3sm_comms/e3sm_org_reviewer/main.py | 96 ++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 8196066..f35cfb3 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,6 +1,6 @@ import xml.etree.ElementTree as ET from collections import defaultdict -from typing import Dict, List, TextIO +from typing import Dict, List, Set, TextIO from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status from e3sm_comms.utils import IO_DIR @@ -55,38 +55,47 @@ def main(): all_urls: List[str] = get_all_urls(all_urls_by_status) - invalid_whitelisted_paths: List[str] = get_list_difference( + invalid_whitelisted_paths: List[str] = get_invalid_patterns( list_whitelisted_paths, all_urls ) - valid_whitelisted_paths: List[str] = get_list_difference( - list_whitelisted_paths, invalid_whitelisted_paths - ) + valid_whitelisted_paths: List[str] = [ + path for path in list_whitelisted_paths if path not in invalid_whitelisted_paths + ] - invalid_expected_archived_paths: List[str] = get_list_difference( + invalid_expected_archived_paths: List[str] = get_invalid_patterns( list_expected_archived_paths, all_urls ) - valid_expected_archived_paths: List[str] = get_list_difference( - list_expected_archived_paths, invalid_expected_archived_paths - ) + valid_expected_archived_paths: List[str] = [ + path + for path in list_expected_archived_paths + if path not in invalid_expected_archived_paths + ] print( - f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs" + f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns" ) print( - f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs" + f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" ) published_urls: List[str] = all_urls_by_status.get("publish", []) archived_urls: List[str] = all_urls_by_status.get("archive", []) + whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( + valid_whitelisted_paths, all_urls + ) + expected_archived_urls_expanded: List[str] = expand_patterns_to_urls( + valid_expected_archived_paths, all_urls + ) + whitelisted_but_not_published: List[str] = get_list_difference( - valid_whitelisted_paths, published_urls + whitelisted_urls_expanded, published_urls ) published_but_not_whitelisted: List[str] = get_list_difference( - published_urls, valid_whitelisted_paths + published_urls, whitelisted_urls_expanded ) should_be_archived: List[str] = get_list_difference( - valid_expected_archived_paths, archived_urls + expected_archived_urls_expanded, archived_urls ) print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") @@ -112,12 +121,15 @@ def main(): print( f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" ) + expanded_whitelist_for_checks: List[str] = expand_patterns_to_urls( + list_whitelisted_paths, all_urls + ) with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: terms: List[str] = [line.rstrip("\n").lower() for line in f] list_search_phrases: List[str] = sorted(terms) links = LinkedURLs( - list_whitelisted_paths, + expanded_whitelist_for_checks, scan_links_for_sensitive_terms=True, list_sensitive_terms=list_search_phrases, ) @@ -194,12 +206,17 @@ def write_summary_table( valid_expected_archived_paths: List[str], ) -> None: statuses: List[str] = sorted(all_urls_by_status.keys()) + all_urls: List[str] = get_all_urls(all_urls_by_status) - whitelist_set = set(valid_whitelisted_paths) - expected_archived_set = set(valid_expected_archived_paths) + whitelist_set: Set[str] = set( + expand_patterns_to_urls(valid_whitelisted_paths, all_urls) + ) + expected_archived_set: Set[str] = set( + expand_patterns_to_urls(valid_expected_archived_paths, all_urls) + ) accounted_for_set = whitelist_set.union(expected_archived_set) - all_urls_set = set(get_all_urls(all_urls_by_status)) + all_urls_set = set(all_urls) remaining_set = all_urls_set - accounted_for_set rows = [ @@ -322,3 +339,46 @@ def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str if status != "publish": non_published_urls.extend(urls) return sorted(non_published_urls) + + +def matches_pattern(pattern: str, url: str) -> bool: + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + prefix = pattern[:-1] + return url.startswith(prefix) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: + matched_urls: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url): + matched_urls.add(url) + return sorted(matched_urls) + + +def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: + invalid_patterns: List[str] = [] + for pattern in patterns: + if not any(matches_pattern(pattern, url) for url in all_urls): + invalid_patterns.append(pattern) + return sorted(invalid_patterns) From df00e121559962d67276b21e82327e8bd24961c3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 13:34:54 -0700 Subject: [PATCH 29/85] Add Confluence URL handling --- e3sm_comms/e3sm_org_reviewer/main.py | 134 ++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index f35cfb3..69a20aa 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,8 +1,12 @@ import xml.etree.ElementTree as ET from collections import defaultdict -from typing import Dict, List, Set, TextIO +from typing import Dict, List, Set, TextIO, Tuple -from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status +from e3sm_comms.page_reviewer.utils_base import ( + LinkedURLs, + get_e3sm_url_status, + map_confluence_to_e3sm, +) from e3sm_comms.utils import IO_DIR INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" @@ -12,6 +16,9 @@ f"{IO_DIR}/input/shared/archived_web_pages.txt" ) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" +INPUT_CONFLUENCE_HIERARCHY: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt" +) OUTPUT_MARKDOWN_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/path_report.md" OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" @@ -21,6 +28,80 @@ RUN_CHECKS: bool = False # Set to False for faster debugging +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + """ + Parses a hierarchy file whose indentation only indicates nesting. + + Expected line format: + <optional spaces><page_id>: <title> + + Returns: + List of (page_id, title) + """ + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +def get_confluence_predicted_e3sm_urls( + input_file: str, +) -> Tuple[List[str], List[str]]: + """ + Reads a Confluence hierarchy file and returns: + - valid_predicted_urls: predicted e3sm.org URLs successfully mapped from Confluence + - unmapped_confluence_pages: human-readable Confluence entries that could not be mapped + """ + valid_predicted_urls: List[str] = [] + unmapped_confluence_pages: List[str] = [] + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + valid_predicted_urls.append(e3sm_url) + else: + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + + return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages) + def main(): # Review XML exports from WordPress @@ -71,12 +152,31 @@ def main(): if path not in invalid_expected_archived_paths ] + confluence_predicted_urls, confluence_unmapped_entries = ( + get_confluence_predicted_e3sm_urls(INPUT_CONFLUENCE_HIERARCHY) + ) + invalid_confluence_paths: List[str] = get_invalid_patterns( + confluence_predicted_urls, all_urls + ) + valid_confluence_paths: List[str] = [ + path + for path in confluence_predicted_urls + if path not in invalid_confluence_paths + ] + print( f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns" ) print( f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" ) + print( + f"Of {len(confluence_predicted_urls)} predicted Confluence e3sm.org paths, " + f"{len(valid_confluence_paths)} are valid URLs" + ) + print( + f"Confluence pages with no predicted e3sm.org URL: {len(confluence_unmapped_entries)}" + ) published_urls: List[str] = all_urls_by_status.get("publish", []) archived_urls: List[str] = all_urls_by_status.get("archive", []) @@ -97,12 +197,21 @@ def main(): should_be_archived: List[str] = get_list_difference( expected_archived_urls_expanded, archived_urls ) + published_but_not_in_confluence: List[str] = get_list_difference( + published_urls, valid_confluence_paths + ) print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") print(f"Not archived, but should be archived: {len(should_be_archived)}") + print( + f"Published, but no matching Confluence path found: {len(published_but_not_in_confluence)}" + ) print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") + print( + f"Invalid Confluence-predicted e3sm.org paths: {len(invalid_confluence_paths)}" + ) write_markdown_report( output_path=OUTPUT_MARKDOWN_REPORT, @@ -112,8 +221,11 @@ def main(): whitelisted_but_not_published=whitelisted_but_not_published, published_but_not_whitelisted=published_but_not_whitelisted, should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, invalid_whitelisted_paths=invalid_whitelisted_paths, invalid_expected_archived_paths=invalid_expected_archived_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, ) # Run checks @@ -158,8 +270,11 @@ def write_markdown_report( whitelisted_but_not_published: List[str], published_but_not_whitelisted: List[str], should_be_archived: List[str], + published_but_not_in_confluence: List[str], invalid_whitelisted_paths: List[str], invalid_expected_archived_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], ) -> None: with open(output_path, "w", encoding="utf-8") as f: write_summary_table( @@ -185,6 +300,11 @@ def write_markdown_report( "Expecting to be archived, but not yet archived", should_be_archived, ) + write_markdown_section( + f, + "Published but no matching Confluence path found", + published_but_not_in_confluence, + ) f.write("# Invalid Paths\n\n") write_markdown_section( @@ -197,6 +317,16 @@ def write_markdown_report( "Identified in archive input", invalid_expected_archived_paths, ) + write_markdown_section( + f, + "Identified in Confluence input", + invalid_confluence_paths, + ) + write_markdown_section( + f, + "Confluence pages with no mappable e3sm.org URL", + confluence_unmapped_entries, + ) def write_summary_table( From 84f4b6e2604240f979fe79e6e166ec57cf296234 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 13:47:51 -0700 Subject: [PATCH 30/85] Add Confluence mapping summary --- e3sm_comms/e3sm_org_reviewer/main.py | 40 +++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 69a20aa..229b61a 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -218,6 +218,7 @@ def main(): all_urls_by_status=all_urls_by_status, valid_whitelisted_paths=valid_whitelisted_paths, valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, whitelisted_but_not_published=whitelisted_but_not_published, published_but_not_whitelisted=published_but_not_whitelisted, should_be_archived=should_be_archived, @@ -267,6 +268,7 @@ def write_markdown_report( all_urls_by_status: Dict[str, List[str]], valid_whitelisted_paths: List[str], valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], whitelisted_but_not_published: List[str], published_but_not_whitelisted: List[str], should_be_archived: List[str], @@ -282,6 +284,9 @@ def write_markdown_report( all_urls_by_status=all_urls_by_status, valid_whitelisted_paths=valid_whitelisted_paths, valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, ) f.write("# Valid Paths\n\n") @@ -334,6 +339,9 @@ def write_summary_table( all_urls_by_status: Dict[str, List[str]], valid_whitelisted_paths: List[str], valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], ) -> None: statuses: List[str] = sorted(all_urls_by_status.keys()) all_urls: List[str] = get_all_urls(all_urls_by_status) @@ -344,16 +352,15 @@ def write_summary_table( expected_archived_set: Set[str] = set( expand_patterns_to_urls(valid_expected_archived_paths, all_urls) ) - accounted_for_set = whitelist_set.union(expected_archived_set) - - all_urls_set = set(all_urls) - remaining_set = all_urls_set - accounted_for_set + both_set: Set[str] = whitelist_set.intersection(expected_archived_set) + neither_set: Set[str] = set(all_urls) - whitelist_set.union(expected_archived_set) rows = [ ("Whitelisted URLs", whitelist_set), ("Expected archived", expected_archived_set), - ("All remaining", remaining_set), - ("TOTAL", all_urls_set), + ("Both whitelisted and expected archived", both_set), + ("Neither whitelisted nor expected archived", neither_set), + ("TOTAL", set(all_urls)), ] file_obj.write("# Summary\n\n") @@ -373,6 +380,27 @@ def write_summary_table( + f" | {total_count} |\n" ) + confluence_valid_set: Set[str] = set(valid_confluence_paths) + all_urls_set: Set[str] = set(all_urls) + + e3sm_with_confluence: Set[str] = all_urls_set.intersection(confluence_valid_set) + e3sm_without_confluence: Set[str] = all_urls_set - confluence_valid_set + confluence_not_valid_count: int = len(invalid_confluence_paths) + len( + confluence_unmapped_entries + ) + + file_obj.write("\n## Confluence Mapping Summary\n\n") + file_obj.write("| Type | Count |\n") + file_obj.write("| --- | --- |\n") + file_obj.write( + f"| Confluence paths that do not map to a valid e3sm.org path | {confluence_not_valid_count} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do not have a Confluence path associated with them | {len(e3sm_without_confluence)} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do have a Confluence counterpart | {len(e3sm_with_confluence)} |\n" + ) file_obj.write("\n") From 4cc8a48d6cf09065963e73ebfd41f4d008c47689 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 13:53:15 -0700 Subject: [PATCH 31/85] Add validation to Confluence summary --- e3sm_comms/e3sm_org_reviewer/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 229b61a..faa98a9 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -385,9 +385,19 @@ def write_summary_table( e3sm_with_confluence: Set[str] = all_urls_set.intersection(confluence_valid_set) e3sm_without_confluence: Set[str] = all_urls_set - confluence_valid_set + confluence_not_valid_count: int = len(invalid_confluence_paths) + len( confluence_unmapped_entries ) + total_confluence_urls: int = len(confluence_valid_set) + confluence_not_valid_count + total_e3sm_urls: int = len(all_urls_set) + + confluence_counts_match: bool = ( + confluence_not_valid_count + len(e3sm_with_confluence) == total_confluence_urls + ) + e3sm_counts_match: bool = ( + len(e3sm_without_confluence) + len(e3sm_with_confluence) == total_e3sm_urls + ) file_obj.write("\n## Confluence Mapping Summary\n\n") file_obj.write("| Type | Count |\n") @@ -401,8 +411,22 @@ def write_summary_table( file_obj.write( f"| e3sm.org paths that do have a Confluence counterpart | {len(e3sm_with_confluence)} |\n" ) + file_obj.write(f"| Total Confluence-derived paths | {total_confluence_urls} |\n") + file_obj.write(f"| Total e3sm.org paths | {total_e3sm_urls} |\n") file_obj.write("\n") + file_obj.write("Validation:\n\n") + file_obj.write( + f"- Confluence counts match: " + f"{confluence_not_valid_count} + {len(e3sm_with_confluence)} = {total_confluence_urls} " + f"({'yes' if confluence_counts_match else 'no'})\n" + ) + file_obj.write( + f"- e3sm.org counts match: " + f"{len(e3sm_without_confluence)} + {len(e3sm_with_confluence)} = {total_e3sm_urls} " + f"({'yes' if e3sm_counts_match else 'no'})\n\n" + ) + def get_status_counts_for_urls( urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] From 514b73543fdc08abf770bc55f228377580e8ed61 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 14:23:54 -0700 Subject: [PATCH 32/85] Fix errors that were causing page skips --- .../page_reviewer/confluence_page_reviewer.py | 121 ++++++++++++++---- 1 file changed, 99 insertions(+), 22 deletions(-) diff --git a/e3sm_comms/page_reviewer/confluence_page_reviewer.py b/e3sm_comms/page_reviewer/confluence_page_reviewer.py index 618ce5a..b80e1da 100644 --- a/e3sm_comms/page_reviewer/confluence_page_reviewer.py +++ b/e3sm_comms/page_reviewer/confluence_page_reviewer.py @@ -1,5 +1,5 @@ import re -from typing import Dict, List +from typing import Dict, List, Optional from e3sm_comms.page_reviewer.utils_base import ( Config, @@ -35,15 +35,21 @@ # Main functionality ########################################################## def run(config: Config): remove_output_files(config) + credentials: Optional[ConfluenceCredentials] = None try: credentials = ConfluenceCredentials() if config.mode in ["resource", "website"]: for tab in config.list_input_confluence_paths: walk_page_and_child_pages(config, credentials, tab) + if config.mode == "newsletter": newsletter_page_list: List[ConfluencePage] = read_page_list(config) for page in newsletter_page_list: - extract_data_from_page(config, credentials, page) + try: + extract_data_from_page(config, credentials, page) + except Exception as e: + print(f"ERROR processing newsletter page {page.url}: {e}") + newsletter_dict: Dict[str, str] if config.newsletter_test_link: newsletter_dict = process_newsletter( @@ -53,7 +59,8 @@ def run(config: Config): newsletter_dict = {} construct_markdown_table(config, newsletter_page_list, newsletter_dict) finally: - del credentials.api_token # Clear the API token from memory, for added security + if credentials is not None and hasattr(credentials, "api_token"): + del credentials.api_token # Clear the API token from memory # Recurse through pages ####################################################### @@ -63,59 +70,82 @@ def walk_page_and_child_pages( page_url: str, current_depth: int = 0, ): - current_page = ConfluencePage(page_url, current_depth) - extract_data_from_page(config, credentials, current_page) - if config.mode == "resource": - process_resource(config, current_page) - for child_page_id in current_page.child_page_ids: - child_page_url = ( - f"https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/{child_page_id}/" - ) - walk_page_and_child_pages( - config, credentials, child_page_url, current_depth=current_depth + 1 - ) + indent = " " * current_depth + current_page: Optional[ConfluencePage] = None + + try: + current_page = ConfluencePage(page_url, current_depth) + print(f"{indent}Visiting page_id={current_page.page_id}, url={page_url}") + extract_data_from_page(config, credentials, current_page) -# Per page analysis ############################################################### + if config.mode == "resource": + process_resource(config, current_page) + + for child_page_id in current_page.child_page_ids: + child_page_url = ( + f"https://e3sm.atlassian.net/wiki/spaces/EPWCD/pages/{child_page_id}/" + ) + walk_page_and_child_pages( + config, credentials, child_page_url, current_depth=current_depth + 1 + ) + + except Exception as e: + page_id = current_page.page_id if current_page else "unknown" + print(f"{indent}ERROR on page_id={page_id}, url={page_url}: {e}") + # Continue traversal for sibling branches by not re-raising + + +# Per page analysis ########################################################## def extract_data_from_page( config: Config, credentials: ConfluenceCredentials, page: ConfluencePage ): extract_data_from_content_url(credentials, page) + if config.mode in ["newsletter", "website"]: extract_data_from_content_url_body(config, credentials, page) + if config.mode == "newsletter": extract_data_from_comments_url(credentials, page) + if config.mode in ["resource", "website"]: extract_data_from_child_pages_url(credentials, page) + if config.mode == "website": write_results(config, page) -# Functions used by all modes ################################################# +# Functions used by all modes ################################################ def extract_data_from_content_url( credentials: ConfluenceCredentials, page: ConfluencePage ): data = get_json(credentials, page.page_id, page.content_url) + if "title" not in data: raise RuntimeError( f"Response for page_id={page.page_id} does not contain 'title'. Full response: {data}" ) + page.title = re.sub(r"[\r\n]+", "", data["title"]) print(f"Extracting data from page_id={page.page_id}, title={page.title}") + if "version" not in data or "number" not in data["version"]: raise RuntimeError( f"Response for page_id={page.page_id} does not contain 'version' or 'version > number'. Full response: {data}" ) - current_version: str = data["version"]["number"] + + current_version = data["version"]["number"] page.current_version = int(current_version) + if "history" not in data or "createdDate" not in data["history"]: raise RuntimeError( f"Response for page_id={page.page_id} does not contain 'history' or 'history > createdDate'. Full response: {data}" ) + page.created_date = data["history"]["createdDate"] -# Functions used by newsletter, website modes ################################### +# Functions used by newsletter, website modes ############################### def extract_data_from_content_url_body( config: Config, credentials: ConfluenceCredentials, page: ConfluencePage ): @@ -125,17 +155,20 @@ def extract_data_from_content_url_body( page.content_url, params={"expand": "body.view.value"}, ) - # print_json(data) # For debugging + raw_html = data.get("body", {}).get("view", {}).get("value", "") if config.mode == "newsletter": raw_html = skip_newsletter_metadata_in_header(raw_html) + page.main_html, page.metadata_html = split_html(raw_html) + if config.check_links_work: page.main_html.linked_urls = LinkedURLs( page.main_html.links, config.scan_links_for_sensitive_terms, config.list_sensitive_terms, ) + if ("sensitive_terms" in config.requested_output) or ( "newsletter_review_table" in config.requested_output ): @@ -155,9 +188,11 @@ def extract_data_from_content_url_body( ) else: print(" Skipping first-person review. Page URL is in the approved list.") + page.main_html.double_spaces_after_periods = find_double_spaces_after_periods( page.main_html.paragraphs ) + lowercase_text: str = page.main_html.text.lower() page.main_html.img_mentions = get_image_mention_frequencies( lowercase_text, page.main_html.num_imgs @@ -165,11 +200,13 @@ def extract_data_from_content_url_body( page.main_html.img_resolutions = get_image_resolutions( page.main_html.img_srcs, "https://e3sm.atlassian.net/wiki", credentials ) + acronyms = get_acronyms( page.main_html.text ) # Use original text, not lowercase_text!! page.main_html.acronyms = filter_acronyms(page.url, acronyms) set_wordpress_keys(page) + if "need_to_sync_wordpress" in config.requested_output: if page.metadata_html: table = extract_confluence_table_to_dict(page.metadata_html) @@ -178,12 +215,52 @@ def extract_data_from_content_url_body( page.need_to_sync_wordpress = True -# Functions used by resource, website modes ################################### +# Functions used by resource, website modes ################################# def extract_data_from_child_pages_url( credentials: ConfluenceCredentials, page: ConfluencePage ): - data = get_json(credentials, page.page_id, page.child_pages_url) - page.child_page_ids = [page["id"] for page in data.get("results", [])] + child_page_ids: List[str] = [] + start = 0 + limit = 100 + + while True: + data = get_json( + credentials, + page.page_id, + page.child_pages_url, + params={"start": str(start), "limit": str(limit)}, + ) + + results = data.get("results", []) + child_page_ids.extend([child["id"] for child in results if "id" in child]) + + batch_count = len(results) + if batch_count == 0: + break + + size = data.get("size") + returned_limit = data.get("limit", limit) + next_link = data.get("_links", {}).get("next") + + print( + f" Retrieved {batch_count} child pages for page_id={page.page_id} " + f"(start={start}, limit={returned_limit})" + ) + + if next_link: + start += batch_count + continue + + if size is not None and batch_count < returned_limit: + break + + if batch_count < limit: + break + + start += batch_count + + page.child_page_ids = child_page_ids + count = len(page.child_page_ids) if count: print(f" Found {count} child pages: {page.child_page_ids}") From 0b3809e30bb2083b64f117628efe51dd991b2d4c Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 14:49:13 -0700 Subject: [PATCH 33/85] Update bash script --- e3sm_comms/e3sm_org_reviewer/main.py | 2 +- examples/review_terms.bash | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index faa98a9..39a419e 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -26,7 +26,7 @@ f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) -RUN_CHECKS: bool = False # Set to False for faster debugging +RUN_CHECKS: bool = True # Set to False for faster debugging CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" diff --git a/examples/review_terms.bash b/examples/review_terms.bash index cfbaf6c..d4b6a48 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -14,6 +14,7 @@ echo "Note: this will require a Confluence login" e3sm-comms-website-reviewer echo "Step 2. Review e3sm.org" +cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt e3sm-comms-e3sm-org-reviewer echo "Step 3. Synthesize into report" From 2629f0625ae0263f7b4bec5b575923fa10668cac Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 15:20:18 -0700 Subject: [PATCH 34/85] Make term reviewer part of e3sm.org reviewer --- e3sm_comms/e3sm_org_reviewer/main.py | 1167 +++++++++++++++++++------- examples/review_terms.bash | 7 +- 2 files changed, 844 insertions(+), 330 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 39a419e..5605713 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,6 +1,9 @@ +import ast +import re import xml.etree.ElementTree as ET from collections import defaultdict -from typing import Dict, List, Set, TextIO, Tuple +from dataclasses import dataclass +from typing import Callable, DefaultDict, Dict, List, Optional, Set, TextIO, Tuple from e3sm_comms.page_reviewer.utils_base import ( LinkedURLs, @@ -19,9 +22,21 @@ INPUT_CONFLUENCE_HIERARCHY: str = ( f"{IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt" ) +INPUT_CONFLUENCE_SENSITIVE_TERMS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt" +) +INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/known_ok_e3sm_org_paths.txt" +) +INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/e3sm_org_reviewer/keep_unchanged_e3sm_org_paths.txt" +) OUTPUT_MARKDOWN_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/path_report.md" OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" +OUTPUT_SENSITIVE_TERMS_REPORT: str = ( + f"{IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" +) OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) @@ -31,21 +46,39 @@ CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" +FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") + +CLASS_PUBLISHED = "Published" +CLASS_ARCHIVED = "Archived" +CLASS_SHOULD_BE_ARCHIVED = "Should be archived" +CLASS_NOT_PUBLISHED = "Not published" +CLASS_KNOWN_OK = "Known OK" +CLASS_KEEP_UNCHANGED = "Keep unchanged" +CLASS_NO_MAPPED_E3SM_URL = "No mapped e3sm.org URL" +CLASS_PREDICTED_URL_NOT_IN_EXPORT = "Predicted e3sm.org URL not in WordPress export" + + +@dataclass +class SensitiveTermRecord: + source: str # "e3sm.org" or "confluence" + raw_line: str + year_label: str + year_int: Optional[int] + total_terms: int + term_counts: Dict[str, int] + source_url: Optional[str] + title: Optional[str] + confluence_url: Optional[str] + e3sm_url: Optional[str] + classification: str + wordpress_status: Optional[str] + def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: - """ - Parses a hierarchy file whose indentation only indicates nesting. - - Expected line format: - <optional spaces><page_id>: <title> - - Returns: - List of (page_id, title) - """ parsed: List[Tuple[str, str]] = [] with open(input_file, "r", encoding="utf-8") as f: @@ -78,11 +111,6 @@ def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: def get_confluence_predicted_e3sm_urls( input_file: str, ) -> Tuple[List[str], List[str]]: - """ - Reads a Confluence hierarchy file and returns: - - valid_predicted_urls: predicted e3sm.org URLs successfully mapped from Confluence - - unmapped_confluence_pages: human-readable Confluence entries that could not be mapped - """ valid_predicted_urls: List[str] = [] unmapped_confluence_pages: List[str] = [] @@ -103,235 +131,142 @@ def get_confluence_predicted_e3sm_urls( return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages) -def main(): - # Review XML exports from WordPress - pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( - INPUT_XML_PAGES, "page" - ) - posts_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( - INPUT_XML_POSTS, "post" - ) - num_pages: int = get_total_count(pages_by_status) - num_posts: int = get_total_count(posts_by_status) - print(f"Found {num_pages} pages, {num_posts} posts") - print( - f"Pages have status in {pages_by_status.keys()}; posts have status in {posts_by_status.keys()}" - ) +def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: + for status in ["publish", "archive", "draft", "future", "pending", "private"]: + if status in all_urls_by_status: + print(f"Found {len(all_urls_by_status[status])} {status} URLs") - all_urls_by_status: Dict[str, List[str]] = get_combined_urls_by_status( - pages_by_status, posts_by_status - ) - print_status_counts(all_urls_by_status) - non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) - print(f"Total non-published URLs: {len(non_published_urls)}") +def get_wordpress_urls_by_status( + xml_file_path: str, post_type: str +) -> Dict[str, List[str]]: + ns = { + "wp": "http://wordpress.org/export/1.2/", + } - # Compare with expectations - with open(INPUT_WHITELIST, "r", encoding="utf-8") as f: - list_whitelisted_paths: List[str] = [line.strip() for line in f if line.strip()] - with open(INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_expected_archived_paths: List[str] = [ - line.strip() for line in f if line.strip() - ] + tree = ET.parse(xml_file_path) + root = tree.getroot() - all_urls: List[str] = get_all_urls(all_urls_by_status) + grouped: Dict[str, List[str]] = defaultdict(list) + channel = root.find("channel") + if channel is None: + return {} - invalid_whitelisted_paths: List[str] = get_invalid_patterns( - list_whitelisted_paths, all_urls - ) - valid_whitelisted_paths: List[str] = [ - path for path in list_whitelisted_paths if path not in invalid_whitelisted_paths - ] + for item in channel.findall("item"): + item_post_type = item.find("wp:post_type", ns) + item_status = item.find("wp:status", ns) + link = item.find("link") - invalid_expected_archived_paths: List[str] = get_invalid_patterns( - list_expected_archived_paths, all_urls - ) - valid_expected_archived_paths: List[str] = [ - path - for path in list_expected_archived_paths - if path not in invalid_expected_archived_paths - ] + if item_post_type is None or item_post_type.text != post_type: + continue - confluence_predicted_urls, confluence_unmapped_entries = ( - get_confluence_predicted_e3sm_urls(INPUT_CONFLUENCE_HIERARCHY) - ) - invalid_confluence_paths: List[str] = get_invalid_patterns( - confluence_predicted_urls, all_urls - ) - valid_confluence_paths: List[str] = [ - path - for path in confluence_predicted_urls - if path not in invalid_confluence_paths - ] + status = ( + item_status.text.strip() + if item_status is not None and item_status.text + else "unknown" + ) - print( - f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns" - ) - print( - f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" - ) - print( - f"Of {len(confluence_predicted_urls)} predicted Confluence e3sm.org paths, " - f"{len(valid_confluence_paths)} are valid URLs" - ) - print( - f"Confluence pages with no predicted e3sm.org URL: {len(confluence_unmapped_entries)}" - ) + if link is not None and link.text: + grouped[status].append(link.text.strip()) - published_urls: List[str] = all_urls_by_status.get("publish", []) - archived_urls: List[str] = all_urls_by_status.get("archive", []) + return {status: sorted(urls) for status, urls in sorted(grouped.items())} - whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( - valid_whitelisted_paths, all_urls - ) - expected_archived_urls_expanded: List[str] = expand_patterns_to_urls( - valid_expected_archived_paths, all_urls - ) - whitelisted_but_not_published: List[str] = get_list_difference( - whitelisted_urls_expanded, published_urls - ) - published_but_not_whitelisted: List[str] = get_list_difference( - published_urls, whitelisted_urls_expanded - ) - should_be_archived: List[str] = get_list_difference( - expected_archived_urls_expanded, archived_urls - ) - published_but_not_in_confluence: List[str] = get_list_difference( - published_urls, valid_confluence_paths - ) +def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: + return sum(len(urls) for urls in urls_by_status.values()) - print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") - print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") - print(f"Not archived, but should be archived: {len(should_be_archived)}") - print( - f"Published, but no matching Confluence path found: {len(published_but_not_in_confluence)}" - ) - print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") - print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") - print( - f"Invalid Confluence-predicted e3sm.org paths: {len(invalid_confluence_paths)}" - ) - write_markdown_report( - output_path=OUTPUT_MARKDOWN_REPORT, - all_urls_by_status=all_urls_by_status, - valid_whitelisted_paths=valid_whitelisted_paths, - valid_expected_archived_paths=valid_expected_archived_paths, - valid_confluence_paths=valid_confluence_paths, - whitelisted_but_not_published=whitelisted_but_not_published, - published_but_not_whitelisted=published_but_not_whitelisted, - should_be_archived=should_be_archived, - published_but_not_in_confluence=published_but_not_in_confluence, - invalid_whitelisted_paths=invalid_whitelisted_paths, - invalid_expected_archived_paths=invalid_expected_archived_paths, - invalid_confluence_paths=invalid_confluence_paths, - confluence_unmapped_entries=confluence_unmapped_entries, - ) +def get_combined_urls_by_status( + pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] +) -> Dict[str, List[str]]: + merged: Dict[str, List[str]] = defaultdict(list) + for source in (pages_by_status, posts_by_status): + for status, urls in source.items(): + merged[status].extend(urls) + return {status: sorted(urls) for status, urls in merged.items()} - # Run checks - if RUN_CHECKS: - print( - f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" - ) - expanded_whitelist_for_checks: List[str] = expand_patterns_to_urls( - list_whitelisted_paths, all_urls - ) - with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: - terms: List[str] = [line.rstrip("\n").lower() for line in f] - list_search_phrases: List[str] = sorted(terms) - links = LinkedURLs( - expanded_whitelist_for_checks, - scan_links_for_sensitive_terms=True, - list_sensitive_terms=list_search_phrases, - ) - relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms - with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: - for link in relevant_links: - f.write(f"{link}: {relevant_links[link]}\n") +def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: + return sorted(set(list1) - set(list2)) - print( - f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" - ) - with open( - OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8" - ) as f: - for e3sm_url in non_published_urls: - e3sm_url_status = get_e3sm_url_status(e3sm_url) - if e3sm_url_status == "link works not logged-in": - f.write(f"{e3sm_url}\n") +def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + all_urls: List[str] = [] + for urls in urls_by_status.values(): + all_urls.extend(urls) + return sorted(all_urls) -def write_markdown_report( - output_path: str, - all_urls_by_status: Dict[str, List[str]], - valid_whitelisted_paths: List[str], - valid_expected_archived_paths: List[str], - valid_confluence_paths: List[str], - whitelisted_but_not_published: List[str], - published_but_not_whitelisted: List[str], - should_be_archived: List[str], - published_but_not_in_confluence: List[str], - invalid_whitelisted_paths: List[str], - invalid_expected_archived_paths: List[str], - invalid_confluence_paths: List[str], - confluence_unmapped_entries: List[str], -) -> None: - with open(output_path, "w", encoding="utf-8") as f: - write_summary_table( - f, - all_urls_by_status=all_urls_by_status, - valid_whitelisted_paths=valid_whitelisted_paths, - valid_expected_archived_paths=valid_expected_archived_paths, - valid_confluence_paths=valid_confluence_paths, - invalid_confluence_paths=invalid_confluence_paths, - confluence_unmapped_entries=confluence_unmapped_entries, - ) - f.write("# Valid Paths\n\n") - write_markdown_section( - f, - "Whitelisted but not published", - whitelisted_but_not_published, - ) - write_markdown_section( - f, - "Published but not whitelisted", - published_but_not_whitelisted, - ) - write_markdown_section( - f, - "Expecting to be archived, but not yet archived", - should_be_archived, - ) - write_markdown_section( - f, - "Published but no matching Confluence path found", - published_but_not_in_confluence, - ) +def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + non_published_urls: List[str] = [] + for status, urls in urls_by_status.items(): + if status != "publish": + non_published_urls.extend(urls) + return sorted(non_published_urls) - f.write("# Invalid Paths\n\n") - write_markdown_section( - f, - "Identified in whitelist input", - invalid_whitelisted_paths, - ) - write_markdown_section( - f, - "Identified in archive input", - invalid_expected_archived_paths, - ) - write_markdown_section( - f, - "Identified in Confluence input", - invalid_confluence_paths, - ) - write_markdown_section( - f, - "Confluence pages with no mappable e3sm.org URL", - confluence_unmapped_entries, - ) + +def matches_pattern(pattern: str, url: str) -> bool: + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + prefix = pattern[:-1] + return url.startswith(prefix) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: + matched_urls: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url): + matched_urls.add(url) + return sorted(matched_urls) + + +def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: + invalid_patterns: List[str] = [] + for pattern in patterns: + if not any(matches_pattern(pattern, url) for url in all_urls): + invalid_patterns.append(pattern) + return sorted(invalid_patterns) + + +def get_status_counts_for_urls( + urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] +) -> Dict[str, int]: + url_set = set(urls) + counts: Dict[str, int] = {} + for status in statuses: + counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) + return counts + + +def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: + file_obj.write(f"## {title}\n\n") + if not items: + file_obj.write("_None._\n\n") + return + + for i, item in enumerate(items, start=1): + file_obj.write(f"{i}. {item}\n") + file_obj.write("\n") def write_summary_table( @@ -428,139 +363,723 @@ def write_summary_table( ) -def get_status_counts_for_urls( - urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] -) -> Dict[str, int]: - url_set = set(urls) - counts: Dict[str, int] = {} - for status in statuses: - counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) - return counts +def write_markdown_report( + output_path: str, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + whitelisted_but_not_published: List[str], + published_but_not_whitelisted: List[str], + should_be_archived: List[str], + published_but_not_in_confluence: List[str], + invalid_whitelisted_paths: List[str], + invalid_expected_archived_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + write_summary_table( + f, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) + + f.write("# Valid Paths\n\n") + write_markdown_section( + f, + "Whitelisted but not published", + whitelisted_but_not_published, + ) + write_markdown_section( + f, + "Published but not whitelisted", + published_but_not_whitelisted, + ) + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + write_markdown_section( + f, + "Published but no matching Confluence path found", + published_but_not_in_confluence, + ) + f.write("# Invalid Paths\n\n") + write_markdown_section( + f, + "Identified in whitelist input", + invalid_whitelisted_paths, + ) + write_markdown_section( + f, + "Identified in archive input", + invalid_expected_archived_paths, + ) + write_markdown_section( + f, + "Identified in Confluence input", + invalid_confluence_paths, + ) + write_markdown_section( + f, + "Confluence pages with no mappable e3sm.org URL", + confluence_unmapped_entries, + ) -def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: - file_obj.write(f"## {title}\n\n") - if not items: - file_obj.write("_None._\n\n") - return - for i, item in enumerate(items, start=1): - file_obj.write(f"{i}. {item}\n") - file_obj.write("\n") +def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: + try: + data = ast.literal_eval(dict_str) + except (SyntaxError, ValueError): + return None + if not isinstance(data, dict): + return None -def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: - for status in ["publish", "archive", "draft", "future", "pending", "private"]: - if status in all_urls_by_status: - print(f"Found {len(all_urls_by_status[status])} {status} URLs") + try: + total = sum(data.values()) + except TypeError: + return None + if not isinstance(total, (int, float)): + return None -def get_wordpress_urls_by_status( - xml_file_path: str, post_type: str -) -> Dict[str, List[str]]: - ns = { - "wp": "http://wordpress.org/export/1.2/", + return data + + +def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: + match = FROM_PREFIX_RE.match(line) + if not match: + return None, line + + year = int(match.group(1)) + remainder = match.group(2).strip() + return year, remainder + + +def build_url_to_status(all_urls_by_status: Dict[str, List[str]]) -> Dict[str, str]: + result: Dict[str, str] = {} + for status, urls in all_urls_by_status.items(): + for url in urls: + result[url] = status + return result + + +def read_nonempty_lines(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + +def classify_e3sm_url( + e3sm_url: Optional[str], + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Tuple[str, Optional[str]]: + if not e3sm_url: + return CLASS_NO_MAPPED_E3SM_URL, None + + if e3sm_url in known_ok_urls: + return CLASS_KNOWN_OK, url_to_status.get(e3sm_url) + + if e3sm_url in keep_unchanged_urls: + return CLASS_KEEP_UNCHANGED, url_to_status.get(e3sm_url) + + wordpress_status = url_to_status.get(e3sm_url) + + if wordpress_status is None: + return CLASS_PREDICTED_URL_NOT_IN_EXPORT, None + + if wordpress_status == "archive": + return CLASS_ARCHIVED, wordpress_status + + if e3sm_url in expected_archived_urls: + return CLASS_SHOULD_BE_ARCHIVED, wordpress_status + + if wordpress_status != "publish": + return CLASS_NOT_PUBLISHED, wordpress_status + + return CLASS_PUBLISHED, wordpress_status + + +def parse_wordpress_sensitive_terms_file(input_file: str) -> List[Tuple[int, str]]: + parsed: List[Tuple[int, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for raw_line in f: + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + dict_start = line.find("{") + if dict_start == -1: + print(f"Skipping malformed WordPress sensitive-terms line: {line}") + continue + + dict_str = line[dict_start:].strip() + dict_data = parse_dict(dict_str) + if dict_data is None: + print(f"Skipping malformed dictionary in WordPress line: {line}") + continue + + total = int(sum(dict_data.values())) + parsed.append((total, line)) + + parsed.sort(key=lambda x: x[0], reverse=True) + return parsed + + +def extract_wordpress_url(line: str) -> Optional[str]: + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + if prefix.endswith(":"): + prefix = prefix[:-1].rstrip() + + return prefix + + +def parse_wordpress_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + dict_start = line.find("{") + if dict_start == -1: + return None + + url = extract_wordpress_url(line) + if not url: + return None + + dict_str = line[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + return None + + total_terms = int(sum(term_counts.values())) + classification, wordpress_status = classify_e3sm_url( + e3sm_url=url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + return SensitiveTermRecord( + source="e3sm.org", + raw_line=line, + year_label="N/A", + year_int=None, + total_terms=total_terms, + term_counts=term_counts, + source_url=url, + title=None, + confluence_url=None, + e3sm_url=url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def extract_confluence_components( + line: str, +) -> Optional[Tuple[Optional[int], str, str, Dict[str, int]]]: + year, remainder = extract_year_and_remainder(line) + + dict_start = remainder.find("{") + if dict_start == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + dict_str = remainder[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + print(f"Skipping malformed dictionary in Confluence line: {line}") + return None + + prefix = remainder[:dict_start].rstrip() + if prefix.endswith("--"): + prefix = prefix[:-2].rstrip() + + first_colon = prefix.find(":") + if first_colon == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + page_id = prefix[:first_colon].strip() + title = prefix[first_colon + 1 :].strip() + + if not page_id.isdigit(): + print(f"Skipping Confluence line with non-numeric page id: {line}") + return None + + return year, page_id, title, term_counts + + +def parse_confluence_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + components = extract_confluence_components(line) + if components is None: + return None + + year, page_id, title, term_counts = components + confluence_url = build_confluence_url(page_id) + + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + e3sm_url = None + + classification, wordpress_status = classify_e3sm_url( + e3sm_url=e3sm_url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + total_terms = int(sum(term_counts.values())) + year_label = str(year) if year is not None else "Unknown year" + + return SensitiveTermRecord( + source="confluence", + raw_line=line, + year_label=year_label, + year_int=year, + total_terms=total_terms, + term_counts=term_counts, + source_url=confluence_url, + title=title, + confluence_url=confluence_url, + e3sm_url=e3sm_url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def classification_sort_key(classification: str) -> Tuple[int, str]: + order = { + CLASS_PUBLISHED: 0, + CLASS_SHOULD_BE_ARCHIVED: 1, + CLASS_ARCHIVED: 2, + CLASS_NOT_PUBLISHED: 3, + CLASS_KNOWN_OK: 4, + CLASS_KEEP_UNCHANGED: 5, + CLASS_NO_MAPPED_E3SM_URL: 6, + CLASS_PREDICTED_URL_NOT_IN_EXPORT: 7, } + return (order.get(classification, 999), classification) + + +def year_sort_key(year_str: str) -> Tuple[int, int]: + if year_str == "Unknown year": + return (1, 0) + if year_str == "N/A": + return (2, 0) + try: + return (0, -int(year_str)) + except ValueError: + return (3, 0) + + +def build_classification_summary( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, int]]: + summary: Dict[str, Dict[str, int]] = {} + + for record in records: + classification = record.classification + if classification not in summary: + summary[classification] = { + "total": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5+": 0, + } + + summary[classification]["total"] += 1 + if record.total_terms == 1: + summary[classification]["1"] += 1 + elif record.total_terms == 2: + summary[classification]["2"] += 1 + elif record.total_terms == 3: + summary[classification]["3"] += 1 + elif record.total_terms == 4: + summary[classification]["4"] += 1 + elif record.total_terms >= 5: + summary[classification]["5+"] += 1 + + return summary + + +def group_records_by_classification_and_year( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, List[SensitiveTermRecord]]]: + grouped: DefaultDict[str, DefaultDict[str, List[SensitiveTermRecord]]] = ( + defaultdict(lambda: defaultdict(list)) + ) - tree = ET.parse(xml_file_path) - root = tree.getroot() + for record in records: + grouped[record.classification][record.year_label].append(record) - grouped: Dict[str, List[str]] = defaultdict(list) - channel = root.find("channel") - if channel is None: - return {} + for classification in grouped: + for year_label in grouped[classification]: + grouped[classification][year_label].sort( + key=lambda r: (r.total_terms, r.e3sm_url or "", r.title or ""), + reverse=True, + ) - for item in channel.findall("item"): - item_post_type = item.find("wp:post_type", ns) - item_status = item.find("wp:status", ns) - link = item.find("link") + return {k: dict(v) for k, v in grouped.items()} - if item_post_type is None or item_post_type.text != post_type: - continue - status = ( - item_status.text.strip() - if item_status is not None and item_status.text - else "unknown" +def write_sensitive_terms_summary_table( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + summary = build_classification_summary(records) + + f.write("### Summary Table\n\n") + f.write("| Classification | Total | 1 | 2 | 3 | 4 | 5+ |\n") + f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") + + for classification in sorted(summary.keys(), key=classification_sort_key): + counts = summary[classification] + f.write( + f"| {classification} | {counts['total']} | {counts['1']} | {counts['2']} | " + f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" ) - if link is not None and link.text: - grouped[status].append(link.text.strip()) + f.write("\n") - return {status: sorted(urls) for status, urls in sorted(grouped.items())} +def format_term_counts(term_counts: Dict[str, int]) -> str: + return str(term_counts) -def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: - return sum(len(urls) for urls in urls_by_status.values()) +def format_e3sm_record(record: SensitiveTermRecord) -> str: + url = record.e3sm_url or record.source_url or "UNKNOWN" + md = f"[{url}]({url})" + if record.wordpress_status: + md += f" (status: {record.wordpress_status})" + md += f" -- {format_term_counts(record.term_counts)}" + return md -def get_combined_urls_by_status( - pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] -) -> Dict[str, List[str]]: - merged: Dict[str, List[str]] = defaultdict(list) - for source in (pages_by_status, posts_by_status): - for status, urls in source.items(): - merged[status].extend(urls) - return {status: sorted(urls) for status, urls in merged.items()} +def format_confluence_record(record: SensitiveTermRecord) -> str: + title = record.title or "Untitled" + confluence_url = record.confluence_url or record.source_url or "" + md = f"{title}: [confluence]({confluence_url})" -def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: - return sorted(set(list1) - set(list2)) + if record.e3sm_url: + md += f" [e3sm.org]({record.e3sm_url})" + try: + e3sm_url_status = get_e3sm_url_status(record.e3sm_url) + except Exception as exc: + print(f"Could not get e3sm.org URL status for {record.e3sm_url}: {exc}") + e3sm_url_status = None -def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: - all_urls: List[str] = [] - for urls in urls_by_status.values(): - all_urls.extend(urls) - return sorted(all_urls) + if e3sm_url_status: + md += f" (Note: {e3sm_url_status})" + if record.wordpress_status: + md += f" (WordPress status: {record.wordpress_status})" -def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: - non_published_urls: List[str] = [] - for status, urls in urls_by_status.items(): - if status != "publish": - non_published_urls.extend(urls) - return sorted(non_published_urls) + md += f" -- {format_term_counts(record.term_counts)}" + return md -def matches_pattern(pattern: str, url: str) -> bool: - if "*" not in pattern: - return pattern == url +def write_sensitive_terms_section( + f: TextIO, + section_title: str, + description: str, + records: List[SensitiveTermRecord], + formatter: Callable[[SensitiveTermRecord], str], +) -> None: + f.write(f"## {section_title}\n\n") + f.write(f"{description}\n\n") - if pattern.count("*") == 1 and pattern.endswith("*"): - prefix = pattern[:-1] - return url.startswith(prefix) + write_sensitive_terms_summary_table(f, records) - parts = pattern.split("*") - position = 0 - for i, part in enumerate(parts): - if not part: - continue - found_at = url.find(part, position) - if found_at == -1: - return False - if i == 0 and not pattern.startswith("*") and found_at != 0: - return False - position = found_at + len(part) + grouped = group_records_by_classification_and_year(records) - if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): - return False + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") - return True + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {formatter(record)}\n") + f.write("\n") -def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: - matched_urls: Set[str] = set() - for pattern in patterns: - for url in all_urls: - if matches_pattern(pattern, url): - matched_urls.add(url) - return sorted(matched_urls) +def write_sensitive_terms_report( + output_path: str, + e3sm_records: List[SensitiveTermRecord], + confluence_records: List[SensitiveTermRecord], +) -> None: + description_e3sm_org = ( + "These are the currently reviewed e3sm.org pages that include sensitive terms. " + "Classification is derived from WordPress status, expected archived inputs, and the manual exception lists for known-ok and keep-unchanged paths." + ) + description_confluence = ( + "These are the Confluence pages that include sensitive terms. The confluence links are what the website reviewer scanned. " + "The e3sm.org links are predicted from Confluence mapping and then classified against the WordPress export." + ) + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Sensitive Terms Report\n\n") -def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: - invalid_patterns: List[str] = [] - for pattern in patterns: - if not any(matches_pattern(pattern, url) for url in all_urls): - invalid_patterns.append(pattern) - return sorted(invalid_patterns) + write_sensitive_terms_section( + f, + "e3sm.org", + description_e3sm_org, + e3sm_records, + format_e3sm_record, + ) + write_sensitive_terms_section( + f, + "Confluence", + description_confluence, + confluence_records, + format_confluence_record, + ) + + +def main(): + pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_PAGES, "page" + ) + posts_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( + INPUT_XML_POSTS, "post" + ) + num_pages: int = get_total_count(pages_by_status) + num_posts: int = get_total_count(posts_by_status) + print(f"Found {num_pages} pages, {num_posts} posts") + print( + f"Pages have status in {pages_by_status.keys()}; posts have status in {posts_by_status.keys()}" + ) + + all_urls_by_status: Dict[str, List[str]] = get_combined_urls_by_status( + pages_by_status, posts_by_status + ) + print_status_counts(all_urls_by_status) + + non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) + print(f"Total non-published URLs: {len(non_published_urls)}") + + list_whitelisted_paths: List[str] = read_nonempty_lines(INPUT_WHITELIST) + list_expected_archived_paths: List[str] = read_nonempty_lines( + INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS + ) + list_known_ok_paths: List[str] = read_nonempty_lines(INPUT_KNOWN_OK_E3SM_ORG_PATHS) + list_keep_unchanged_paths: List[str] = read_nonempty_lines( + INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS + ) + + all_urls: List[str] = get_all_urls(all_urls_by_status) + url_to_status: Dict[str, str] = build_url_to_status(all_urls_by_status) + + invalid_whitelisted_paths: List[str] = get_invalid_patterns( + list_whitelisted_paths, all_urls + ) + valid_whitelisted_paths: List[str] = [ + path for path in list_whitelisted_paths if path not in invalid_whitelisted_paths + ] + + invalid_expected_archived_paths: List[str] = get_invalid_patterns( + list_expected_archived_paths, all_urls + ) + valid_expected_archived_paths: List[str] = [ + path + for path in list_expected_archived_paths + if path not in invalid_expected_archived_paths + ] + + confluence_predicted_urls, confluence_unmapped_entries = ( + get_confluence_predicted_e3sm_urls(INPUT_CONFLUENCE_HIERARCHY) + ) + invalid_confluence_paths: List[str] = get_invalid_patterns( + confluence_predicted_urls, all_urls + ) + valid_confluence_paths: List[str] = [ + path + for path in confluence_predicted_urls + if path not in invalid_confluence_paths + ] + + print( + f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns" + ) + print( + f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" + ) + print( + f"Of {len(confluence_predicted_urls)} predicted Confluence e3sm.org paths, " + f"{len(valid_confluence_paths)} are valid URLs" + ) + print( + f"Confluence pages with no predicted e3sm.org URL: {len(confluence_unmapped_entries)}" + ) + + published_urls: List[str] = all_urls_by_status.get("publish", []) + archived_urls: List[str] = all_urls_by_status.get("archive", []) + + whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( + valid_whitelisted_paths, all_urls + ) + expected_archived_urls_expanded: List[str] = expand_patterns_to_urls( + valid_expected_archived_paths, all_urls + ) + + whitelisted_but_not_published: List[str] = get_list_difference( + whitelisted_urls_expanded, published_urls + ) + published_but_not_whitelisted: List[str] = get_list_difference( + published_urls, whitelisted_urls_expanded + ) + should_be_archived: List[str] = get_list_difference( + expected_archived_urls_expanded, archived_urls + ) + published_but_not_in_confluence: List[str] = get_list_difference( + published_urls, valid_confluence_paths + ) + + print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") + print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") + print(f"Not archived, but should be archived: {len(should_be_archived)}") + print( + f"Published, but no matching Confluence path found: {len(published_but_not_in_confluence)}" + ) + print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") + print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") + print( + f"Invalid Confluence-predicted e3sm.org paths: {len(invalid_confluence_paths)}" + ) + + write_markdown_report( + output_path=OUTPUT_MARKDOWN_REPORT, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + whitelisted_but_not_published=whitelisted_but_not_published, + published_but_not_whitelisted=published_but_not_whitelisted, + should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, + invalid_whitelisted_paths=invalid_whitelisted_paths, + invalid_expected_archived_paths=invalid_expected_archived_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) + + e3sm_records: List[SensitiveTermRecord] = [] + confluence_records: List[SensitiveTermRecord] = [] + + if RUN_CHECKS: + print( + f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" + ) + expanded_whitelist_for_checks: List[str] = expand_patterns_to_urls( + list_whitelisted_paths, all_urls + ) + with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: + terms: List[str] = [line.rstrip("\n").lower() for line in f] + list_search_phrases: List[str] = sorted(terms) + + links = LinkedURLs( + expanded_whitelist_for_checks, + scan_links_for_sensitive_terms=True, + list_sensitive_terms=list_search_phrases, + ) + relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms + with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: + for link in relevant_links: + f.write(f"{link}: {relevant_links[link]}\n") + + expected_archived_urls_set: Set[str] = set(expected_archived_urls_expanded) + known_ok_urls_set: Set[str] = set(list_known_ok_paths) + keep_unchanged_urls_set: Set[str] = set(list_keep_unchanged_paths) + + wordpress_lines = parse_wordpress_sensitive_terms_file(OUTPUT_FOUND_PHRASES) + for _, line in wordpress_lines: + record = parse_wordpress_record( + line=line, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls_set, + known_ok_urls=known_ok_urls_set, + keep_unchanged_urls=keep_unchanged_urls_set, + ) + if record: + e3sm_records.append(record) + + if INPUT_CONFLUENCE_SENSITIVE_TERMS: + try: + with open(INPUT_CONFLUENCE_SENSITIVE_TERMS, "r", encoding="utf-8") as f: + for raw_line in f: + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + record = parse_confluence_record( + line=line, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls_set, + known_ok_urls=known_ok_urls_set, + keep_unchanged_urls=keep_unchanged_urls_set, + ) + if record: + confluence_records.append(record) + except FileNotFoundError: + print( + f"Confluence sensitive terms input not found: {INPUT_CONFLUENCE_SENSITIVE_TERMS}" + ) + + write_sensitive_terms_report( + output_path=OUTPUT_SENSITIVE_TERMS_REPORT, + e3sm_records=e3sm_records, + confluence_records=confluence_records, + ) + + print( + f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" + ) + with open( + OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8" + ) as f: + for e3sm_url in non_published_urls: + e3sm_url_status = get_e3sm_url_status(e3sm_url) + if e3sm_url_status == "link works not logged-in": + f.write(f"{e3sm_url}\n") + else: + write_sensitive_terms_report( + output_path=OUTPUT_SENSITIVE_TERMS_REPORT, + e3sm_records=[], + confluence_records=[], + ) + + +if __name__ == "__main__": + main() diff --git a/examples/review_terms.bash b/examples/review_terms.bash index d4b6a48..1fdcd81 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -15,10 +15,5 @@ e3sm-comms-website-reviewer echo "Step 2. Review e3sm.org" cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt +cp ${IO_DIR}/output/website_reviewer/sensitive_terms.txt ${IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt e3sm-comms-e3sm-org-reviewer - -echo "Step 3. Synthesize into report" -cp ${IO_DIR}/output/website_reviewer/sensitive_terms.txt ${IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt -cp ${IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt ${IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt -e3sm-comms-term-reviewer -echo "Output: ${IO_DIR}/output/term_reviewer/sensitive_terms.md" From b2e637067dc2bae302f6ee4cfd37e4e131083d57 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 15:53:35 -0700 Subject: [PATCH 35/85] Add action items Markdown --- e3sm_comms/e3sm_org_reviewer/main.py | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 5605713..473d186 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -40,6 +40,7 @@ OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" ) +OUTPUT_ACTION_ITEMS_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/action_items.md" RUN_CHECKS: bool = True # Set to False for faster debugging @@ -434,6 +435,54 @@ def write_markdown_report( ) +def write_action_items_report( + output_path: str, + should_be_archived: List[str], + published_but_not_in_confluence: List[str], + confluence_published_sensitive_records: List[SensitiveTermRecord], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Action Items Report\n\n") + + f.write("## Summary\n\n") + f.write("| Action Area | Count |\n") + f.write("| --- | ---: |\n") + f.write( + f"| Expecting to be archived, but not yet archived | {len(should_be_archived)} |\n" + ) + f.write( + f"| Published but no matching Confluence path found | {len(published_but_not_in_confluence)} |\n" + ) + f.write( + f"| Confluence pages with sensitive terms mapped to published e3sm.org pages | {len(confluence_published_sensitive_records)} |\n" + ) + f.write("\n") + + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + + write_markdown_section( + f, + "Published but no matching Confluence path found", + published_but_not_in_confluence, + ) + + f.write( + "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" + ) + if not confluence_published_sensitive_records: + f.write("_None._\n\n") + else: + for idx, record in enumerate( + confluence_published_sensitive_records, start=1 + ): + f.write(f"{idx}. {format_confluence_record(record)}\n") + f.write("\n") + + def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: try: data = ast.literal_eval(dict_str) @@ -1063,6 +1112,19 @@ def main(): confluence_records=confluence_records, ) + confluence_published_sensitive_records: List[SensitiveTermRecord] = [ + record + for record in confluence_records + if record.classification == CLASS_PUBLISHED + ] + + write_action_items_report( + output_path=OUTPUT_ACTION_ITEMS_REPORT, + should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, + confluence_published_sensitive_records=confluence_published_sensitive_records, + ) + print( f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" ) @@ -1079,6 +1141,12 @@ def main(): e3sm_records=[], confluence_records=[], ) + write_action_items_report( + output_path=OUTPUT_ACTION_ITEMS_REPORT, + should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, + confluence_published_sensitive_records=[], + ) if __name__ == "__main__": From 14ce477d0605701b261189e03c8df27a02b902ed Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 16:06:23 -0700 Subject: [PATCH 36/85] Improvements to action items reporting --- e3sm_comms/e3sm_org_reviewer/main.py | 202 +++++++++++++++------------ 1 file changed, 109 insertions(+), 93 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 473d186..68431e4 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -33,13 +33,9 @@ ) OUTPUT_MARKDOWN_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/path_report.md" -OUTPUT_FOUND_PHRASES: str = f"{IO_DIR}/output/e3sm_org_reviewer/found_phrases.txt" OUTPUT_SENSITIVE_TERMS_REPORT: str = ( f"{IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" ) -OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/output/e3sm_org_reviewer/incorrectly_accessible_web_pages.txt" -) OUTPUT_ACTION_ITEMS_REPORT: str = f"{IO_DIR}/output/e3sm_org_reviewer/action_items.md" RUN_CHECKS: bool = True # Set to False for faster debugging @@ -184,7 +180,7 @@ def get_combined_urls_by_status( for source in (pages_by_status, posts_by_status): for status, urls in source.items(): merged[status].extend(urls) - return {status: sorted(urls) for status, urls in merged.items()} + return {status: sorted(urls) for status, urls in sorted(merged.items())} def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: @@ -374,6 +370,8 @@ def write_markdown_report( published_but_not_whitelisted: List[str], should_be_archived: List[str], published_but_not_in_confluence: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], + incorrectly_accessible_non_published_urls: List[str], invalid_whitelisted_paths: List[str], invalid_expected_archived_paths: List[str], invalid_confluence_paths: List[str], @@ -411,6 +409,16 @@ def write_markdown_report( "Published but no matching Confluence path found", published_but_not_in_confluence, ) + write_markdown_section( + f, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, + ) + write_markdown_section( + f, + "Non-published e3sm.org pages that are still accessible without login", + incorrectly_accessible_non_published_urls, + ) f.write("# Invalid Paths\n\n") write_markdown_section( @@ -435,10 +443,33 @@ def write_markdown_report( ) +def write_action_items_confluence_section( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + f.write( + "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" + ) + + if not records: + f.write("_None._\n\n") + return + + grouped = group_records_by_classification_and_year(records) + + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") + + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {format_confluence_record(record)}\n") + f.write("\n") + + def write_action_items_report( output_path: str, should_be_archived: List[str], - published_but_not_in_confluence: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], confluence_published_sensitive_records: List[SensitiveTermRecord], ) -> None: with open(output_path, "w", encoding="utf-8") as f: @@ -451,7 +482,7 @@ def write_action_items_report( f"| Expecting to be archived, but not yet archived | {len(should_be_archived)} |\n" ) f.write( - f"| Published but no matching Confluence path found | {len(published_but_not_in_confluence)} |\n" + f"| Published but not whitelisted and no matching Confluence path found | {len(published_not_whitelisted_and_not_in_confluence)} |\n" ) f.write( f"| Confluence pages with sensitive terms mapped to published e3sm.org pages | {len(confluence_published_sensitive_records)} |\n" @@ -466,21 +497,11 @@ def write_action_items_report( write_markdown_section( f, - "Published but no matching Confluence path found", - published_but_not_in_confluence, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, ) - f.write( - "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" - ) - if not confluence_published_sensitive_records: - f.write("_None._\n\n") - else: - for idx, record in enumerate( - confluence_published_sensitive_records, start=1 - ): - f.write(f"{idx}. {format_confluence_record(record)}\n") - f.write("\n") + write_action_items_confluence_section(f, confluence_published_sensitive_records) def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: @@ -559,28 +580,27 @@ def classify_e3sm_url( return CLASS_PUBLISHED, wordpress_status -def parse_wordpress_sensitive_terms_file(input_file: str) -> List[Tuple[int, str]]: +def parse_wordpress_sensitive_terms_lines(lines: List[str]) -> List[Tuple[int, str]]: parsed: List[Tuple[int, str]] = [] - with open(input_file, "r", encoding="utf-8") as f: - for raw_line in f: - line = raw_line.rstrip("\n") - if not line.strip(): - continue + for raw_line in lines: + line = raw_line.rstrip("\n") + if not line.strip(): + continue - dict_start = line.find("{") - if dict_start == -1: - print(f"Skipping malformed WordPress sensitive-terms line: {line}") - continue + dict_start = line.find("{") + if dict_start == -1: + print(f"Skipping malformed WordPress sensitive-terms line: {line}") + continue - dict_str = line[dict_start:].strip() - dict_data = parse_dict(dict_str) - if dict_data is None: - print(f"Skipping malformed dictionary in WordPress line: {line}") - continue + dict_str = line[dict_start:].strip() + dict_data = parse_dict(dict_str) + if dict_data is None: + print(f"Skipping malformed dictionary in WordPress line: {line}") + continue - total = int(sum(dict_data.values())) - parsed.append((total, line)) + total = int(sum(dict_data.values())) + parsed.append((total, line)) parsed.sort(key=lambda x: x[0], reverse=True) return parsed @@ -1015,6 +1035,9 @@ def main(): published_but_not_in_confluence: List[str] = get_list_difference( published_urls, valid_confluence_paths ) + published_not_whitelisted_and_not_in_confluence: List[str] = get_list_difference( + published_but_not_in_confluence, whitelisted_urls_expanded + ) print(f"Whitelisted, but not published: {len(whitelisted_but_not_published)}") print(f"Published, but not whitelisted: {len(published_but_not_whitelisted)}") @@ -1022,27 +1045,17 @@ def main(): print( f"Published, but no matching Confluence path found: {len(published_but_not_in_confluence)}" ) + print( + "Published, but not whitelisted and no matching Confluence path found: " + f"{len(published_not_whitelisted_and_not_in_confluence)}" + ) print(f"Invalid whitelist paths: {len(invalid_whitelisted_paths)}") print(f"Invalid archive-input paths: {len(invalid_expected_archived_paths)}") print( f"Invalid Confluence-predicted e3sm.org paths: {len(invalid_confluence_paths)}" ) - write_markdown_report( - output_path=OUTPUT_MARKDOWN_REPORT, - all_urls_by_status=all_urls_by_status, - valid_whitelisted_paths=valid_whitelisted_paths, - valid_expected_archived_paths=valid_expected_archived_paths, - valid_confluence_paths=valid_confluence_paths, - whitelisted_but_not_published=whitelisted_but_not_published, - published_but_not_whitelisted=published_but_not_whitelisted, - should_be_archived=should_be_archived, - published_but_not_in_confluence=published_but_not_in_confluence, - invalid_whitelisted_paths=invalid_whitelisted_paths, - invalid_expected_archived_paths=invalid_expected_archived_paths, - invalid_confluence_paths=invalid_confluence_paths, - confluence_unmapped_entries=confluence_unmapped_entries, - ) + incorrectly_accessible_non_published_urls: List[str] = [] e3sm_records: List[SensitiveTermRecord] = [] confluence_records: List[SensitiveTermRecord] = [] @@ -1064,15 +1077,15 @@ def main(): list_sensitive_terms=list_search_phrases, ) relevant_links: Dict[str, Dict[str, int]] = links.links_with_sensitive_terms - with open(OUTPUT_FOUND_PHRASES, "w", encoding="utf-8") as f: - for link in relevant_links: - f.write(f"{link}: {relevant_links[link]}\n") expected_archived_urls_set: Set[str] = set(expected_archived_urls_expanded) known_ok_urls_set: Set[str] = set(list_known_ok_paths) keep_unchanged_urls_set: Set[str] = set(list_keep_unchanged_paths) - wordpress_lines = parse_wordpress_sensitive_terms_file(OUTPUT_FOUND_PHRASES) + wordpress_lines_input: List[str] = [ + f"{link}: {relevant_links[link]}" for link in relevant_links + ] + wordpress_lines = parse_wordpress_sensitive_terms_lines(wordpress_lines_input) for _, line in wordpress_lines: record = parse_wordpress_record( line=line, @@ -1106,47 +1119,50 @@ def main(): f"Confluence sensitive terms input not found: {INPUT_CONFLUENCE_SENSITIVE_TERMS}" ) - write_sensitive_terms_report( - output_path=OUTPUT_SENSITIVE_TERMS_REPORT, - e3sm_records=e3sm_records, - confluence_records=confluence_records, + print( + f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" ) + for e3sm_url in non_published_urls: + e3sm_url_status = get_e3sm_url_status(e3sm_url) + if e3sm_url_status == "link works not logged-in": + incorrectly_accessible_non_published_urls.append(e3sm_url) - confluence_published_sensitive_records: List[SensitiveTermRecord] = [ - record - for record in confluence_records - if record.classification == CLASS_PUBLISHED - ] + write_markdown_report( + output_path=OUTPUT_MARKDOWN_REPORT, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + whitelisted_but_not_published=whitelisted_but_not_published, + published_but_not_whitelisted=published_but_not_whitelisted, + should_be_archived=should_be_archived, + published_but_not_in_confluence=published_but_not_in_confluence, + published_not_whitelisted_and_not_in_confluence=published_not_whitelisted_and_not_in_confluence, + incorrectly_accessible_non_published_urls=incorrectly_accessible_non_published_urls, + invalid_whitelisted_paths=invalid_whitelisted_paths, + invalid_expected_archived_paths=invalid_expected_archived_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) - write_action_items_report( - output_path=OUTPUT_ACTION_ITEMS_REPORT, - should_be_archived=should_be_archived, - published_but_not_in_confluence=published_but_not_in_confluence, - confluence_published_sensitive_records=confluence_published_sensitive_records, - ) + write_sensitive_terms_report( + output_path=OUTPUT_SENSITIVE_TERMS_REPORT, + e3sm_records=e3sm_records, + confluence_records=confluence_records, + ) - print( - f"Checking {len(non_published_urls)} non-published e3sm.org pages are inaccessible" - ) - with open( - OUTPUT_INCORRECTLY_ACCESSIBLE_E3SM_ORG_PATHS, "w", encoding="utf-8" - ) as f: - for e3sm_url in non_published_urls: - e3sm_url_status = get_e3sm_url_status(e3sm_url) - if e3sm_url_status == "link works not logged-in": - f.write(f"{e3sm_url}\n") - else: - write_sensitive_terms_report( - output_path=OUTPUT_SENSITIVE_TERMS_REPORT, - e3sm_records=[], - confluence_records=[], - ) - write_action_items_report( - output_path=OUTPUT_ACTION_ITEMS_REPORT, - should_be_archived=should_be_archived, - published_but_not_in_confluence=published_but_not_in_confluence, - confluence_published_sensitive_records=[], - ) + confluence_published_sensitive_records: List[SensitiveTermRecord] = [ + record + for record in confluence_records + if record.classification == CLASS_PUBLISHED + ] + + write_action_items_report( + output_path=OUTPUT_ACTION_ITEMS_REPORT, + should_be_archived=should_be_archived, + published_not_whitelisted_and_not_in_confluence=published_not_whitelisted_and_not_in_confluence, + confluence_published_sensitive_records=confluence_published_sensitive_records, + ) if __name__ == "__main__": From 771147dbe055ba1da0c80c26a25575b22775ab73 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 16:19:11 -0700 Subject: [PATCH 37/85] Fully remove term-reviewer --- README.md | 11 +- e3sm_comms/e3sm_org_reviewer/main.py | 15 +- e3sm_comms/term_reviewer/__init__.py | 0 e3sm_comms/term_reviewer/main.py | 422 --------------------------- examples/review_terms.bash | 4 + pyproject.toml | 1 - 6 files changed, 19 insertions(+), 434 deletions(-) delete mode 100644 e3sm_comms/term_reviewer/__init__.py delete mode 100644 e3sm_comms/term_reviewer/main.py diff --git a/README.md b/README.md index e95f721..da5df57 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,17 @@ This package is for implementing the software needs of the E3SM Communications t ### Simple commands `e3sm-comms-e3sm-org-reviewer` -- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for, txt file listing e3sm.org pages that should be marked as archived, xml file of Wordpress pages, xml file of Wordpress posts. Note: xml files can be obtained from WordPress under Tools > Export. -- output: txt file listing e3sm.org pages containing those phrases, txt file listing e3sm.org pages that are accessible even though they should be archived, txt file of page URLs found in the xml, txt file of post URLs found in the xml. +- input: + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts + - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages + - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change) +- output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items `e3sm-comms-html-reviewer` - input: 1 txt file of html copied from WordPress that includes yellow highlights left over from Confluence. - output: 1 txt file of html with those highlights removed. - Known issues: more than just `<mark>` tags are changed (presumably no other semantic changes though) -`e3sm-comms-term-reviewer` -- input: 2 txt files of sensitive terms (use the output from `e3sm-comms-e3sm-org-reviewer` & `e3sm-comms-website-reviewer`), txt file listing e3sm.org pages that should be marked as archived, txt file listing e3sm.org pages that do not contain the search terms (and presumably only show up because their corresponding Confluence pages have the terms somewhere in metadata), txt file listing e3sm.org pages that are known not to exist (either the script couldn't determine the correct e3sm.org path, or it doesn't even exist), txt file listing e3sm.org pages that are to be kept unchanged. -- output: Markdown report of terms found - `e3sm-comms-tree-reviewer` - input: 2 txt files showing the website structure in hierarchical form (via indents) -- i.e. in tree form - output: txt file listing the steps of moving subtrees to get from one tree to the other diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 68431e4..9ab7bee 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -12,19 +12,24 @@ ) from e3sm_comms.utils import IO_DIR +# From WordPress under Tools > Export: INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" INPUT_XML_POSTS: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_posts.xml" -INPUT_WHITELIST: str = f"{IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt" -INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/shared/archived_web_pages.txt" -) -INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" + +# From output of `e3sm-comms-website-reviewer`: INPUT_CONFLUENCE_HIERARCHY: str = ( f"{IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt" ) INPUT_CONFLUENCE_SENSITIVE_TERMS: str = ( f"{IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt" ) + +# Other: +INPUT_WHITELIST: str = f"{IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt" +INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS: str = ( + f"{IO_DIR}/input/shared/archived_web_pages.txt" +) +INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( f"{IO_DIR}/input/e3sm_org_reviewer/known_ok_e3sm_org_paths.txt" ) diff --git a/e3sm_comms/term_reviewer/__init__.py b/e3sm_comms/term_reviewer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/e3sm_comms/term_reviewer/main.py b/e3sm_comms/term_reviewer/main.py deleted file mode 100644 index 7a49488..0000000 --- a/e3sm_comms/term_reviewer/main.py +++ /dev/null @@ -1,422 +0,0 @@ -import ast -import re -from collections import defaultdict -from typing import Callable, DefaultDict, Dict, List, Optional, Tuple - -from e3sm_comms.page_reviewer.utils_base import ( - get_e3sm_url_status, - map_confluence_to_e3sm, -) -from e3sm_comms.utils import IO_DIR - -INPUT_E3SM_ORG: str = f"{IO_DIR}/input/term_reviewer/wordpress_sensitive_terms.txt" -INPUT_CONFLUENCE: str = f"{IO_DIR}/input/term_reviewer/confluence_sensitive_terms.txt" -INPUT_ARCHIVED_E3SM_ORG_PATHS: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" -INPUT_KNOWN_OK_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/term_reviewer/known_ok_e3sm_org_paths.txt" -) -INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/term_reviewer/keep_unchanged_e3sm_org_paths.txt" -) -INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS: str = ( - f"{IO_DIR}/input/term_reviewer/does_not_exist_e3sm_org_paths.txt" -) -OUTPUT: str = f"{IO_DIR}/output/term_reviewer/sensitive_terms.md" - -CONFLUENCE_SPACE = "EPWCD" -CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" -ARCHIVED_YEAR_LABEL = "Archived (or should be archived)" -KNOWN_OK_LABEL = "Known OK (Confluence page may be reporting terms that aren't showing up on the e3sm.org page)" -KEEP_UNCHANGED_LABEL = "Keep unchanged" -DOES_NOT_EXIST_LABEL = ( - "Does not exist (either script couldn't find e3sm.org URL or none exists)" -) - -FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") - - -def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: - return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" - - -def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: - try: - data = ast.literal_eval(dict_str) - except (SyntaxError, ValueError): - return None - - if not isinstance(data, dict): - return None - - try: - total = sum(data.values()) - except TypeError: - return None - - if not isinstance(total, (int, float)): - return None - - return data - - -def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: - """ - Supports lines like: - [From 2023-04-12T21:05:24.198Z] 3746136122: Title -- {'str1': 3} - """ - match = FROM_PREFIX_RE.match(line) - if not match: - return None, line - - year = int(match.group(1)) - remainder = match.group(2).strip() - return year, remainder - - -def sort_and_group_by_year(input_file: str) -> Dict[str, List[Tuple[int, str]]]: - grouped_entries: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list) - - with open(input_file, "r", encoding="utf-8") as f: - for raw_line in f: - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - year, remainder = extract_year_and_remainder(line) - year_key = str(year) if year is not None else "Unknown year" - - dict_start = remainder.find("{") - if dict_start == -1: - print(f"Skipping malformed line: {line}") - continue - - dict_str = remainder[dict_start:].strip() - dict_data = parse_dict(dict_str) - if dict_data is None: - print(f"Skipping malformed dictionary: {line}") - continue - - total = int(sum(dict_data.values())) - grouped_entries[year_key].append((total, remainder)) - - for year_key in grouped_entries: - grouped_entries[year_key].sort(key=lambda x: x[0], reverse=True) - - return dict(grouped_entries) - - -def extract_wordpress_url(line: str) -> Optional[str]: - dict_start = line.find("{") - if dict_start == -1: - return None - - prefix = line[:dict_start].rstrip() - if prefix.endswith(":"): - prefix = prefix[:-1].rstrip() - - return prefix - - -def extract_confluence_predicted_e3sm_url(line: str) -> Optional[str]: - dict_start = line.find("{") - if dict_start == -1: - return None - - prefix = line[:dict_start].rstrip() - if prefix.endswith("--"): - prefix = prefix[:-2].rstrip() - - first_colon = prefix.find(":") - if first_colon == -1: - print(f"Skipping malformed Confluence line: {line}") - return None - - page_id = prefix[:first_colon].strip() - title = prefix[first_colon + 1 :].strip() - - if not page_id.isdigit(): - print(f"Skipping Confluence line with non-numeric page id: {line}") - return None - - confluence_url = build_confluence_url(page_id) - - try: - return map_confluence_to_e3sm(confluence_url, page_title=title) - except Exception as exc: - print( - f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" - ) - return None - - -def move_entries_to_label( - grouped_entries: Dict[str, List[Tuple[int, str]]], - matching_paths: List[str], - e3sm_url_extractor: Callable[[str], Optional[str]], - target_label: str, -) -> Dict[str, List[Tuple[int, str]]]: - matching_set = {path.strip() for path in matching_paths if path.strip()} - if not matching_set: - return grouped_entries - - updated: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list) - - for year, entries in grouped_entries.items(): - for total, line in entries: - e3sm_url = e3sm_url_extractor(line) - - if e3sm_url and e3sm_url in matching_set: - updated[target_label].append((total, line)) - else: - updated[year].append((total, line)) - - for year_key in updated: - updated[year_key].sort(key=lambda x: x[0], reverse=True) - - return dict(updated) - - -def format_wordpress_line(line: str) -> Optional[str]: - dict_start = line.find("{") - if dict_start == -1: - return None - - prefix = line[:dict_start].rstrip() - counts = line[dict_start:].strip() - - if prefix.endswith(":"): - prefix = prefix[:-1].rstrip() - - url = prefix - return f"[{url}]({url}) -- {counts}" - - -def format_confluence_line(line: str) -> Optional[str]: - dict_start = line.find("{") - if dict_start == -1: - return None - - counts = line[dict_start:].strip() - prefix = line[:dict_start].rstrip() - - if prefix.endswith("--"): - prefix = prefix[:-2].rstrip() - - first_colon = prefix.find(":") - if first_colon == -1: - print(f"Skipping malformed Confluence line: {line}") - return None - - page_id = prefix[:first_colon].strip() - title = prefix[first_colon + 1 :].strip() - - if not page_id.isdigit(): - print(f"Skipping Confluence line with non-numeric page id: {line}") - return None - - confluence_url = build_confluence_url(page_id) - - e3sm_url: Optional[str] - try: - e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) - except Exception as exc: - print( - f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" - ) - e3sm_url = None - - e3sm_url_status: Optional[str] = None - if e3sm_url: - e3sm_url_status = get_e3sm_url_status(e3sm_url) - - md = f"{title}: [confluence]({confluence_url})" - if e3sm_url: - md += f" [e3sm.org]({e3sm_url})" - if e3sm_url_status: - md += f" (Note: {e3sm_url_status})" - md += f" -- {counts}" - - return md - - -def year_sort_key(year_str: str) -> Tuple[int, int]: - if year_str == ARCHIVED_YEAR_LABEL: - return (1, 0) - if year_str == KNOWN_OK_LABEL: - return (2, 0) - if year_str == KEEP_UNCHANGED_LABEL: - return (3, 0) - if year_str == DOES_NOT_EXIST_LABEL: - return (4, 0) - if year_str == "Unknown year": - return (5, 0) - return (0, -int(year_str)) - - -def build_year_summary( - grouped_entries: Dict[str, List[Tuple[int, str]]], -) -> Dict[str, Dict[str, int]]: - summary: Dict[str, Dict[str, int]] = {} - - for year, entries in grouped_entries.items(): - counts = { - "total": len(entries), - "1": 0, - "2": 0, - "3": 0, - "4": 0, - "5+": 0, - } - - for total_terms, _ in entries: - if total_terms == 1: - counts["1"] += 1 - elif total_terms == 2: - counts["2"] += 1 - elif total_terms == 3: - counts["3"] += 1 - elif total_terms == 4: - counts["4"] += 1 - elif total_terms >= 5: - counts["5+"] += 1 - - summary[year] = counts - - return summary - - -def write_summary_table(f, grouped_entries: Dict[str, List[Tuple[int, str]]]) -> None: - summary = build_year_summary(grouped_entries) - - f.write("### Summary Table\n") - f.write( - "How to interpret: each cell's value is the number of pages published in year `row` that contains `col` terms\n" - ) - - f.write("| Year | Total (i.e., any number of terms) | 1 | 2 | 3 | 4 | 5+ |\n") - f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") - - for year in sorted(summary.keys(), key=year_sort_key): - counts = summary[year] - f.write( - f"| {year} | {counts['total']} | {counts['1']} | {counts['2']} | " - f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" - ) - - f.write("\n") - - -def write_section( - f, - section_title: str, - section_description: str, - grouped_entries: Dict[str, List[Tuple[int, str]]], - formatter, -) -> None: - f.write(f"## {section_title}\n\n") - f.write(f"Description: {section_description}\n\n") - - write_summary_table(f, grouped_entries) - - for year in sorted(grouped_entries.keys(), key=year_sort_key): - f.write(f"### {year}\n\n") - for idx, (_, line) in enumerate(grouped_entries[year], start=1): - formatted = formatter(line) - if formatted: - f.write(f"{idx}. {formatted}\n") - f.write("\n") - - -def main() -> None: - description_e3sm_org: str = ( - "These are the currently publicly-available (whitelisted) e3sm.org pages that include sensitive terms." - ) - description_confluence: str = ( - "These are the Confluence pages (serving as drafts of e3sm.org pages) that include sensitive terms. The 'confluence' links are what the script _actually_ reviewed. The 'e3sm.org' links are _predicted_ based on common URL naming patterns and thus may in fact be broken links. If the Confluence drafts and actual e3sm.org pages have not been kept in sync, remember that the term count is for the Confluence draft, not the actual e3sm.org page." - ) - - with open(INPUT_ARCHIVED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_archived_e3sm_org_paths: List[str] = [line.strip() for line in f] - - with open(INPUT_KNOWN_OK_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_known_ok_e3sm_org_paths: List[str] = [line.strip() for line in f] - - with open(INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_keep_unchanged_e3sm_org_paths: List[str] = [ - line.strip() for line in f - ] - - with open(INPUT_DOES_NOT_EXIST_E3SM_ORG_PATHS, "r", encoding="utf-8") as f: - list_input_does_not_exist_e3sm_org_paths: List[str] = [ - line.strip() for line in f - ] - - entries_e3sm_org = sort_and_group_by_year(INPUT_E3SM_ORG) - entries_e3sm_org = move_entries_to_label( - entries_e3sm_org, - list_input_archived_e3sm_org_paths, - extract_wordpress_url, - ARCHIVED_YEAR_LABEL, - ) - entries_e3sm_org = move_entries_to_label( - entries_e3sm_org, - list_input_known_ok_e3sm_org_paths, - extract_wordpress_url, - KNOWN_OK_LABEL, - ) - entries_e3sm_org = move_entries_to_label( - entries_e3sm_org, - list_input_keep_unchanged_e3sm_org_paths, - extract_wordpress_url, - KEEP_UNCHANGED_LABEL, - ) - entries_e3sm_org = move_entries_to_label( - entries_e3sm_org, - list_input_does_not_exist_e3sm_org_paths, - extract_wordpress_url, - DOES_NOT_EXIST_LABEL, - ) - - entries_confluence = sort_and_group_by_year(INPUT_CONFLUENCE) - entries_confluence = move_entries_to_label( - entries_confluence, - list_input_archived_e3sm_org_paths, - extract_confluence_predicted_e3sm_url, - ARCHIVED_YEAR_LABEL, - ) - entries_confluence = move_entries_to_label( - entries_confluence, - list_input_known_ok_e3sm_org_paths, - extract_confluence_predicted_e3sm_url, - KNOWN_OK_LABEL, - ) - entries_confluence = move_entries_to_label( - entries_confluence, - list_input_keep_unchanged_e3sm_org_paths, - extract_confluence_predicted_e3sm_url, - KEEP_UNCHANGED_LABEL, - ) - entries_confluence = move_entries_to_label( - entries_confluence, - list_input_does_not_exist_e3sm_org_paths, - extract_confluence_predicted_e3sm_url, - DOES_NOT_EXIST_LABEL, - ) - - with open(OUTPUT, "w", encoding="utf-8") as f: - f.write("# Sensitive Terms Report\n\n") - - write_section( - f, "e3sm.org", description_e3sm_org, entries_e3sm_org, format_wordpress_line - ) - write_section( - f, - "Confluence", - description_confluence, - entries_confluence, - format_confluence_line, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/review_terms.bash b/examples/review_terms.bash index 1fdcd81..0a61031 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -17,3 +17,7 @@ echo "Step 2. Review e3sm.org" cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/e3sm_org_reviewer/hierarchical_outline.txt cp ${IO_DIR}/output/website_reviewer/sensitive_terms.txt ${IO_DIR}/input/e3sm_org_reviewer/confluence_sensitive_terms.txt e3sm-comms-e3sm-org-reviewer +echo "Output reports:" +echo "1. ${IO_DIR}/output/e3sm_org_reviewer/path_report.md" +echo "2. ${IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" +echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md diff --git a/pyproject.toml b/pyproject.toml index c108d4e..8795d9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,7 +117,6 @@ e3sm-comms-website-reviewer = "e3sm_comms.website_reviewer.main:main" # These do not: e3sm-comms-e3sm-org-reviewer = "e3sm_comms.e3sm_org_reviewer.main:main" e3sm-comms-html-reviewer = "e3sm_comms.html_reviewer.main:main" -e3sm-comms-term-reviewer = "e3sm_comms.term_reviewer.main:main" e3sm-comms-tree-reviewer = "e3sm_comms.tree_reviewer.main:main" e3sm-comms-video-reviewer = "e3sm_comms.video_reviewer.main:main" From 2849336c8c82e87a7bcc028cd5ae71fc9481fd3a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 16:36:27 -0700 Subject: [PATCH 38/85] Fix whitelist URL count --- e3sm_comms/e3sm_org_reviewer/main.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 9ab7bee..928362a 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1004,8 +1004,11 @@ def main(): if path not in invalid_confluence_paths ] + whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( + valid_whitelisted_paths, all_urls + ) print( - f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns" + f"Of {len(list_whitelisted_paths)} whitelisted paths, {len(valid_whitelisted_paths)} are valid URLs/patterns. Expanding patterns, it's {len(whitelisted_urls_expanded)} valid URLs." ) print( f"Of {len(list_expected_archived_paths)} expected archived paths, {len(valid_expected_archived_paths)} are valid URLs/patterns" @@ -1021,9 +1024,6 @@ def main(): published_urls: List[str] = all_urls_by_status.get("publish", []) archived_urls: List[str] = all_urls_by_status.get("archive", []) - whitelisted_urls_expanded: List[str] = expand_patterns_to_urls( - valid_whitelisted_paths, all_urls - ) expected_archived_urls_expanded: List[str] = expand_patterns_to_urls( valid_expected_archived_paths, all_urls ) @@ -1067,17 +1067,14 @@ def main(): if RUN_CHECKS: print( - f"Checking {len(list_whitelisted_paths)} whitelisted e3sm.org pages for search phrases" - ) - expanded_whitelist_for_checks: List[str] = expand_patterns_to_urls( - list_whitelisted_paths, all_urls + f"Checking {len(whitelisted_urls_expanded)} whitelisted e3sm.org pages for search phrases" ) with open(INPUT_SEARCH_PHRASES, "r", encoding="utf-8") as f: terms: List[str] = [line.rstrip("\n").lower() for line in f] list_search_phrases: List[str] = sorted(terms) links = LinkedURLs( - expanded_whitelist_for_checks, + whitelisted_urls_expanded, scan_links_for_sensitive_terms=True, list_sensitive_terms=list_search_phrases, ) From 5a196ee241993a19e710c4eac9bf5b50591ce880 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 18:22:14 -0700 Subject: [PATCH 39/85] Add exported-xml-reviewer --- README.md | 7 + e3sm_comms/exported_xml_reviewer/__init__.py | 0 e3sm_comms/exported_xml_reviewer/main.py | 359 +++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 367 insertions(+) create mode 100644 e3sm_comms/exported_xml_reviewer/__init__.py create mode 100644 e3sm_comms/exported_xml_reviewer/main.py diff --git a/README.md b/README.md index da5df57..4c3e66b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,13 @@ This package is for implementing the software needs of the E3SM Communications t - input: txt file of time intervals to cut from the video, txt file of initial timestamps - output: txt file of new timestamps after cutting the specified intervals +`e3sm-comms-exported-xml-reviewer` +- input: + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts + - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages + - Other: txt file of sensitive terms +- output: Markdown summary report of sensitive terms found in exported WordPress data + ### Confluence API commands (require Confluence token) `e3sm-comms-newsletter-reviewer` diff --git a/e3sm_comms/exported_xml_reviewer/__init__.py b/e3sm_comms/exported_xml_reviewer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py new file mode 100644 index 0000000..6bb3d49 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import DefaultDict, Dict, List, Optional, Tuple + +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.utils import IO_DIR + +# ----------------------------------------------------------------------------- +# Configuration +# ----------------------------------------------------------------------------- + +INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" +INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" +INPUT_CONFLUENCE_HIERARCHY: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" +) +INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" + +OUTPUT_MARKDOWN_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" +) + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +@dataclass +class WordpressItem: + title: str + url: str + status: str + body: str + + +@dataclass +class ReportRecord: + title: str + e3sm_url: str + status: str + sensitive_terms: Dict[str, int] + confluence_draft_url: Optional[str] + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" + + +def normalize_status(raw_status: Optional[str]) -> str: + if not raw_status: + return "unknown" + + mapping = { + "publish": "published", + "archive": "archived", + "draft": "draft", + "future": "future", + "pending": "pending", + "private": "private", + } + return mapping.get(raw_status.strip().lower(), raw_status.strip().lower()) + + +def strip_html(text: str) -> str: + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r" ", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def read_sensitive_terms(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + terms = [line.strip().lower() for line in f if line.strip()] + return sorted(set(terms)) + + +def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: + counts: Dict[str, int] = {} + lowered = text.lower() + + for term in terms: + escaped = re.escape(term) + pattern = rf"\b{escaped}\b" + matches = re.findall(pattern, lowered) + if matches: + counts[term] = len(matches) + + return counts + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +def get_confluence_mapping(input_file: str) -> Dict[str, str]: + mapping: Dict[str, str] = {} + + if map_confluence_to_e3sm is None: + print( + "Warning: map_confluence_to_e3sm is not available, Confluence mapping will be skipped." + ) + return mapping + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + mapping[e3sm_url] = confluence_url + except Exception as exc: + print(f"Could not map {confluence_url}: {exc}") + + return mapping + + +def extract_item_body(item: ET.Element) -> str: + ns = { + "wp": "http://wordpress.org/export/1.2/", + "content": "http://purl.org/rss/1.0/modules/content/", + "excerpt": "http://wordpress.org/export/1.2/excerpt/", + } + + body_parts: List[str] = [] + + content_elem = item.find("content:encoded", ns) + if content_elem is not None and content_elem.text and content_elem.text.strip(): + body_parts.append(content_elem.text.strip()) + + excerpt_elem = item.find("excerpt:encoded", ns) + if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): + body_parts.append(excerpt_elem.text.strip()) + + for postmeta in item.findall("wp:postmeta", ns): + meta_key_elem = postmeta.find("wp:meta_key", ns) + meta_value_elem = postmeta.find("wp:meta_value", ns) + + meta_key = ( + meta_key_elem.text.strip() + if meta_key_elem is not None and meta_key_elem.text + else "" + ) + meta_value = ( + meta_value_elem.text.strip() + if meta_value_elem is not None and meta_value_elem.text + else "" + ) + + if not meta_value: + continue + + if meta_key.endswith("_free_form_content") and not meta_key.startswith("_"): + body_parts.append(meta_value) + + return "\n".join(body_parts) + + +def parse_wordpress_xml( + xml_file_path: str, expected_post_type: str +) -> List[WordpressItem]: + ns = { + "wp": "http://wordpress.org/export/1.2/", + } + + tree = ET.parse(xml_file_path) + root = tree.getroot() + + channel = root.find("channel") + if channel is None: + return [] + + items: List[WordpressItem] = [] + + for item in channel.findall("item"): + post_type_elem = item.find("wp:post_type", ns) + if post_type_elem is None: + continue + + post_type_text = (post_type_elem.text or "").strip() + if post_type_text != expected_post_type: + continue + + title_elem = item.find("title") + link_elem = item.find("link") + status_elem = item.find("wp:status", ns) + + title = ( + title_elem.text.strip() + if title_elem is not None and title_elem.text is not None + else "Untitled" + ) + link = ( + link_elem.text.strip() + if link_elem is not None and link_elem.text is not None + else "" + ) + status = ( + status_elem.text.strip() + if status_elem is not None and status_elem.text is not None + else "unknown" + ) + body = extract_item_body(item) + + items.append( + WordpressItem( + title=title, + url=link, + status=status, + body=body, + ) + ) + + return items + + +def build_records( + xml_pages: str, + xml_posts: str, + confluence_hierarchy: str, + sensitive_terms_file: str, +) -> List[ReportRecord]: + sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) + confluence_map = get_confluence_mapping(confluence_hierarchy) + + raw_items: List[WordpressItem] = [] + raw_items.extend(parse_wordpress_xml(xml_pages, "page")) + raw_items.extend(parse_wordpress_xml(xml_posts, "post")) + + records: List[ReportRecord] = [] + + for item in raw_items: + plain_text = strip_html(item.body) + term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) + + if not term_counts: + continue + + records.append( + ReportRecord( + title=item.title, + e3sm_url=item.url, + status=normalize_status(item.status), + sensitive_terms=term_counts, + confluence_draft_url=confluence_map.get(item.url), + ) + ) + + return records + + +def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None: + grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) + for record in records: + grouped[record.status].append(record) + + for status in grouped: + grouped[status].sort( + key=lambda r: (-sum(r.sensitive_terms.values()), r.title.lower()) + ) + + ordered_statuses = [ + "published", + "archived", + "draft", + "future", + "pending", + "private", + "unknown", + ] + + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Sensitive Terms Report\n\n") + f.write( + "This report includes only e3sm.org pages/posts where one or more sensitive terms were found.\n\n" + ) + + f.write("| Status | Count |\n") + f.write("| --- | ---: |\n") + + for status in ordered_statuses: + if status in grouped: + f.write(f"| {status} | {len(grouped[status])} |\n") + + extra_statuses = sorted(s for s in grouped if s not in ordered_statuses) + for status in extra_statuses: + f.write(f"| {status} | {len(grouped[status])} |\n") + + f.write("\n") + + all_statuses = ordered_statuses + extra_statuses + seen = set() + + for status in all_statuses: + if status not in grouped or status in seen: + continue + seen.add(status) + + f.write(f"## {status.capitalize()}\n\n") + + for idx, record in enumerate(grouped[status], start=1): + e3sm_md = f"[e3sm.org]({record.e3sm_url})" + confluence_md = ( + f" [confluence draft]({record.confluence_draft_url})" + if record.confluence_draft_url + else "" + ) + + f.write( + f"{idx}. {record.title}: {e3sm_md}{confluence_md} -- {record.sensitive_terms}\n" + ) + + f.write("\n") + + +def main() -> None: + records = build_records( + xml_pages=INPUT_XML_PAGES, + xml_posts=INPUT_XML_POSTS, + confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, + sensitive_terms_file=INPUT_SEARCH_PHRASES, + ) + + write_markdown_report(OUTPUT_MARKDOWN_REPORT, records) + print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 8795d9c..7b954d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ e3sm-comms-resource-reviewer = "e3sm_comms.resource_reviewer.main:main" e3sm-comms-website-reviewer = "e3sm_comms.website_reviewer.main:main" # These do not: e3sm-comms-e3sm-org-reviewer = "e3sm_comms.e3sm_org_reviewer.main:main" +e3sm-comms-exported-xml-reviewer = "e3sm_comms.exported_xml_reviewer.main:main" e3sm-comms-html-reviewer = "e3sm_comms.html_reviewer.main:main" e3sm-comms-tree-reviewer = "e3sm_comms.tree_reviewer.main:main" e3sm-comms-video-reviewer = "e3sm_comms.video_reviewer.main:main" From a360f550fa84246899c2aea9f7bb97b2d07cc45e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 18:31:09 -0700 Subject: [PATCH 40/85] Improve xml-reviewer output --- e3sm_comms/exported_xml_reviewer/main.py | 74 ++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 6bb3d49..b90644b 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -5,7 +5,7 @@ from collections import defaultdict from dataclasses import dataclass from pathlib import Path -from typing import DefaultDict, Dict, List, Optional, Tuple +from typing import DefaultDict, Dict, List, Optional, Set, Tuple from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR @@ -20,6 +20,7 @@ f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" ) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" +INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" OUTPUT_MARKDOWN_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" @@ -78,6 +79,11 @@ def read_sensitive_terms(file_path: str) -> List[str]: return sorted(set(terms)) +def read_whitelist_patterns(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: counts: Dict[str, int] = {} lowered = text.lower() @@ -92,6 +98,41 @@ def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: return counts +def matches_pattern(pattern: str, url: str) -> bool: + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + prefix = pattern[:-1] + return url.startswith(prefix) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> Set[str]: + matched_urls: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url): + matched_urls.add(url) + return matched_urls + + def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: parsed: List[Tuple[str, str]] = [] @@ -246,6 +287,7 @@ def build_records( xml_posts: str, confluence_hierarchy: str, sensitive_terms_file: str, + whitelist_file: str, ) -> List[ReportRecord]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) confluence_map = get_confluence_mapping(confluence_hierarchy) @@ -254,6 +296,10 @@ def build_records( raw_items.extend(parse_wordpress_xml(xml_pages, "page")) raw_items.extend(parse_wordpress_xml(xml_posts, "post")) + whitelist_patterns = read_whitelist_patterns(whitelist_file) + all_urls = [item.url for item in raw_items if item.url] + whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) + records: List[ReportRecord] = [] for item in raw_items: @@ -263,11 +309,18 @@ def build_records( if not term_counts: continue + normalized_status = normalize_status(item.status) + if normalized_status == "published": + if item.url in whitelisted_urls: + normalized_status = "published & whitelisted" + else: + normalized_status = "published & not whitelisted" + records.append( ReportRecord( title=item.title, e3sm_url=item.url, - status=normalize_status(item.status), + status=normalized_status, sensitive_terms=term_counts, confluence_draft_url=confluence_map.get(item.url), ) @@ -287,7 +340,8 @@ def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None ) ordered_statuses = [ - "published", + "published & whitelisted", + "published & not whitelisted", "archived", "draft", "future", @@ -308,14 +362,21 @@ def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None f.write("| Status | Count |\n") f.write("| --- | ---: |\n") + total_count = 0 + for status in ordered_statuses: if status in grouped: - f.write(f"| {status} | {len(grouped[status])} |\n") + count = len(grouped[status]) + total_count += count + f.write(f"| {status} | {count} |\n") extra_statuses = sorted(s for s in grouped if s not in ordered_statuses) for status in extra_statuses: - f.write(f"| {status} | {len(grouped[status])} |\n") + count = len(grouped[status]) + total_count += count + f.write(f"| {status} | {count} |\n") + f.write(f"| TOTAL | {total_count} |\n") f.write("\n") all_statuses = ordered_statuses + extra_statuses @@ -331,7 +392,7 @@ def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None for idx, record in enumerate(grouped[status], start=1): e3sm_md = f"[e3sm.org]({record.e3sm_url})" confluence_md = ( - f" [confluence draft]({record.confluence_draft_url})" + f" [(confluence draft)]({record.confluence_draft_url})" if record.confluence_draft_url else "" ) @@ -349,6 +410,7 @@ def main() -> None: xml_posts=INPUT_XML_POSTS, confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, sensitive_terms_file=INPUT_SEARCH_PHRASES, + whitelist_file=INPUT_WHITELIST, ) write_markdown_report(OUTPUT_MARKDOWN_REPORT, records) From 8fdb757252f2d217fc268012229319e03871dd5b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 7 May 2026 18:37:52 -0700 Subject: [PATCH 41/85] Successful Markdown report from xml-reviewer --- README.md | 2 +- e3sm_comms/exported_xml_reviewer/main.py | 71 +++++++++++++++--------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4c3e66b..5d7c105 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ This package is for implementing the software needs of the E3SM Communications t - input: - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages - - Other: txt file of sensitive terms + - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages - output: Markdown summary report of sensitive terms found in exported WordPress data ### Confluence API commands (require Confluence token) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index b90644b..6e0c672 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -288,7 +288,7 @@ def build_records( confluence_hierarchy: str, sensitive_terms_file: str, whitelist_file: str, -) -> List[ReportRecord]: +) -> Tuple[List[ReportRecord], Dict[str, int]]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) confluence_map = get_confluence_mapping(confluence_hierarchy) @@ -301,14 +301,9 @@ def build_records( whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) records: List[ReportRecord] = [] + status_totals: DefaultDict[str, int] = defaultdict(int) for item in raw_items: - plain_text = strip_html(item.body) - term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) - - if not term_counts: - continue - normalized_status = normalize_status(item.status) if normalized_status == "published": if item.url in whitelisted_urls: @@ -316,6 +311,14 @@ def build_records( else: normalized_status = "published & not whitelisted" + status_totals[normalized_status] += 1 + + plain_text = strip_html(item.body) + term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) + + if not term_counts: + continue + records.append( ReportRecord( title=item.title, @@ -326,10 +329,14 @@ def build_records( ) ) - return records + return records, dict(status_totals) -def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None: +def write_markdown_report( + output_path: str, + records: List[ReportRecord], + status_totals: Dict[str, int], +) -> None: grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) for record in records: grouped[record.status].append(record) @@ -356,27 +363,39 @@ def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None with open(output_file, "w", encoding="utf-8") as f: f.write("# WordPress Sensitive Terms Report\n\n") f.write( - "This report includes only e3sm.org pages/posts where one or more sensitive terms were found.\n\n" + "The detailed sections below include only e3sm.org pages/posts where one or more sensitive terms were found. " + "The summary table includes counts for both flagged and unflagged items.\n\n" ) - f.write("| Status | Count |\n") - f.write("| --- | ---: |\n") + f.write("| Status | With sensitive terms | Without sensitive terms | Total |\n") + f.write("| --- | ---: | ---: | ---: |\n") + + total_with_terms = 0 + total_without_terms = 0 + + all_summary_statuses = set(status_totals) | set(grouped) + extra_statuses = sorted( + s for s in all_summary_statuses if s not in ordered_statuses + ) - total_count = 0 + for status in ordered_statuses + extra_statuses: + total_in_status = status_totals.get(status, 0) + with_terms = len(grouped.get(status, [])) + without_terms = total_in_status - with_terms - for status in ordered_statuses: - if status in grouped: - count = len(grouped[status]) - total_count += count - f.write(f"| {status} | {count} |\n") + if total_in_status == 0 and with_terms == 0: + continue - extra_statuses = sorted(s for s in grouped if s not in ordered_statuses) - for status in extra_statuses: - count = len(grouped[status]) - total_count += count - f.write(f"| {status} | {count} |\n") + total_with_terms += with_terms + total_without_terms += without_terms + f.write( + f"| {status} | {with_terms} | {without_terms} | {total_in_status} |\n" + ) - f.write(f"| TOTAL | {total_count} |\n") + grand_total = total_with_terms + total_without_terms + f.write( + f"| TOTAL | {total_with_terms} | {total_without_terms} | {grand_total} |\n" + ) f.write("\n") all_statuses = ordered_statuses + extra_statuses @@ -405,7 +424,7 @@ def write_markdown_report(output_path: str, records: List[ReportRecord]) -> None def main() -> None: - records = build_records( + records, status_totals = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, @@ -413,7 +432,7 @@ def main() -> None: whitelist_file=INPUT_WHITELIST, ) - write_markdown_report(OUTPUT_MARKDOWN_REPORT, records) + write_markdown_report(OUTPUT_MARKDOWN_REPORT, records, status_totals) print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") From f25483209dca97573185c653fd02d8732f2e2344 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 15 May 2026 10:57:11 -0700 Subject: [PATCH 42/85] Add requested links section --- e3sm_comms/exported_xml_reviewer/main.py | 176 ++++++++++++++++++++--- 1 file changed, 156 insertions(+), 20 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 6e0c672..4f3f954 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -1,11 +1,13 @@ from __future__ import annotations +import csv import re import xml.etree.ElementTree as ET from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import DefaultDict, Dict, List, Optional, Set, Tuple +from urllib.parse import urlsplit, urlunsplit from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR @@ -21,6 +23,7 @@ ) INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" OUTPUT_MARKDOWN_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" @@ -47,6 +50,29 @@ class ReportRecord: confluence_draft_url: Optional[str] +@dataclass +class RequestedLinkRecord: + e3sm_url: str + included_later: bool + current_status: str + currently_whitelisted: bool + requesting_urls: str + + +def normalize_url(url: str) -> str: + url = url.strip() + if not url: + return "" + + parts = urlsplit(url) + scheme = parts.scheme.lower() + netloc = parts.netloc.lower() + path = parts.path.rstrip("/") + + normalized = urlunsplit((scheme, netloc, path, "", "")) + return normalized + + def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" @@ -66,6 +92,19 @@ def normalize_status(raw_status: Optional[str]) -> str: return mapping.get(raw_status.strip().lower(), raw_status.strip().lower()) +def display_status(status: str) -> str: + mapping = { + "published": "Published", + "archived": "Archived", + "draft": "Draft", + "future": "Future", + "pending": "Pending", + "private": "Private", + "unknown": "Unknown", + } + return mapping.get(status, status.title()) + + def strip_html(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r" ", " ", text) @@ -84,6 +123,23 @@ def read_whitelist_patterns(file_path: str) -> List[str]: return [line.strip() for line in f if line.strip()] +def read_requested_links(file_path: str) -> List[Tuple[str, str]]: + rows: List[Tuple[str, str]] = [] + + with open(file_path, "r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + e3sm_url = normalize_url(row.get("e3sm.org link", "")) + requesting_urls = ( + row.get("list of URLs that wants to link to it") or "" + ).strip() + + if e3sm_url: + rows.append((e3sm_url, requesting_urls)) + + return rows + + def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: counts: Dict[str, int] = {} lowered = text.lower() @@ -100,25 +156,35 @@ def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: def matches_pattern(pattern: str, url: str) -> bool: if "*" not in pattern: - return pattern == url + return normalize_url(pattern) == normalize_url(url) + + normalized_url = normalize_url(url) + normalized_pattern = normalize_url(pattern) - if pattern.count("*") == 1 and pattern.endswith("*"): - prefix = pattern[:-1] - return url.startswith(prefix) + if "*" not in normalized_pattern: + return normalized_pattern == normalized_url - parts = pattern.split("*") + if normalized_pattern.count("*") == 1 and normalized_pattern.endswith("*"): + prefix = normalized_pattern[:-1] + return normalized_url.startswith(prefix) + + parts = normalized_pattern.split("*") position = 0 for i, part in enumerate(parts): if not part: continue - found_at = url.find(part, position) + found_at = normalized_url.find(part, position) if found_at == -1: return False - if i == 0 and not pattern.startswith("*") and found_at != 0: + if i == 0 and not normalized_pattern.startswith("*") and found_at != 0: return False position = found_at + len(part) - if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + if ( + not normalized_pattern.endswith("*") + and parts[-1] + and not normalized_url.endswith(parts[-1]) + ): return False return True @@ -176,7 +242,7 @@ def get_confluence_mapping(input_file: str) -> Dict[str, str]: try: e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) if e3sm_url: - mapping[e3sm_url] = confluence_url + mapping[normalize_url(e3sm_url)] = confluence_url except Exception as exc: print(f"Could not map {confluence_url}: {exc}") @@ -259,7 +325,7 @@ def parse_wordpress_xml( else "Untitled" ) link = ( - link_elem.text.strip() + normalize_url(link_elem.text) if link_elem is not None and link_elem.text is not None else "" ) @@ -282,13 +348,47 @@ def parse_wordpress_xml( return items +def build_requested_link_records( + requested_links_file: str, + raw_items: List[WordpressItem], + whitelisted_urls: Set[str], + flagged_urls: Set[str], +) -> List[RequestedLinkRecord]: + requested_rows = read_requested_links(requested_links_file) + item_by_url = {item.url: item for item in raw_items if item.url} + + records: List[RequestedLinkRecord] = [] + for e3sm_url, requesting_urls in requested_rows: + item = item_by_url.get(e3sm_url) + + if item is None: + current_status = "Not found" + currently_whitelisted = False + else: + current_status = display_status(normalize_status(item.status)) + currently_whitelisted = e3sm_url in whitelisted_urls + + records.append( + RequestedLinkRecord( + e3sm_url=e3sm_url, + included_later=e3sm_url in flagged_urls, + current_status=current_status, + currently_whitelisted=currently_whitelisted, + requesting_urls=requesting_urls, + ) + ) + + return records + + def build_records( xml_pages: str, xml_posts: str, confluence_hierarchy: str, sensitive_terms_file: str, whitelist_file: str, -) -> Tuple[List[ReportRecord], Dict[str, int]]: + requested_links_file: str, +) -> Tuple[List[ReportRecord], Dict[str, int], List[RequestedLinkRecord]]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) confluence_map = get_confluence_mapping(confluence_hierarchy) @@ -304,14 +404,16 @@ def build_records( status_totals: DefaultDict[str, int] = defaultdict(int) for item in raw_items: - normalized_status = normalize_status(item.status) - if normalized_status == "published": + base_status = normalize_status(item.status) + report_status = base_status + + if base_status == "published": if item.url in whitelisted_urls: - normalized_status = "published & whitelisted" + report_status = "published & whitelisted" else: - normalized_status = "published & not whitelisted" + report_status = "published & not whitelisted" - status_totals[normalized_status] += 1 + status_totals[report_status] += 1 plain_text = strip_html(item.body) term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) @@ -323,19 +425,28 @@ def build_records( ReportRecord( title=item.title, e3sm_url=item.url, - status=normalized_status, + status=report_status, sensitive_terms=term_counts, confluence_draft_url=confluence_map.get(item.url), ) ) - return records, dict(status_totals) + flagged_urls = {record.e3sm_url for record in records} + requested_link_records = build_requested_link_records( + requested_links_file=requested_links_file, + raw_items=raw_items, + whitelisted_urls=whitelisted_urls, + flagged_urls=flagged_urls, + ) + + return records, dict(status_totals), requested_link_records def write_markdown_report( output_path: str, records: List[ReportRecord], status_totals: Dict[str, int], + requested_link_records: List[RequestedLinkRecord], ) -> None: grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) for record in records: @@ -398,6 +509,25 @@ def write_markdown_report( ) f.write("\n") + if requested_link_records: + f.write("## Requested Links\n\n") + f.write( + "| e3sm.org link | Included later on this page? | Current status | Currently whitelisted? | Requesting URLs |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + for requested_record in requested_link_records: + included_later = "Yes" if requested_record.included_later else "No" + currently_whitelisted = ( + "Yes" if requested_record.currently_whitelisted else "No" + ) + f.write( + f"| {requested_record.e3sm_url} | {included_later} | {requested_record.current_status} | " + f"{currently_whitelisted} | {requested_record.requesting_urls} |\n" + ) + + f.write("\n") + all_statuses = ordered_statuses + extra_statuses seen = set() @@ -424,15 +554,21 @@ def write_markdown_report( def main() -> None: - records, status_totals = build_records( + records, status_totals, requested_link_records = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, sensitive_terms_file=INPUT_SEARCH_PHRASES, whitelist_file=INPUT_WHITELIST, + requested_links_file=INPUT_REQUESTED_LINKS, ) - write_markdown_report(OUTPUT_MARKDOWN_REPORT, records, status_totals) + write_markdown_report( + OUTPUT_MARKDOWN_REPORT, + records, + status_totals, + requested_link_records, + ) print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") From c7de3a21deb14d030971f1253c5bc7394673f892 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 15 May 2026 11:58:34 -0700 Subject: [PATCH 43/85] Minor improvements --- README.md | 2 +- e3sm_comms/website_reviewer/main.py | 12 ++++++------ examples/review_xml.bash | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 examples/review_xml.bash diff --git a/README.md b/README.md index 5d7c105..508d1b2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This package is for implementing the software needs of the E3SM Communications t `e3sm-comms-e3sm-org-reviewer` - input: - - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_posts.xml`. - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change) - output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items diff --git a/e3sm_comms/website_reviewer/main.py b/e3sm_comms/website_reviewer/main.py index 33b314a..0813ebf 100644 --- a/e3sm_comms/website_reviewer/main.py +++ b/e3sm_comms/website_reviewer/main.py @@ -6,15 +6,15 @@ def main(): c = Config("website") # c.file_input_confluence_paths = f"{IO_DIR}/input/website_reviewer/confluence_top_level_tabs_20260109.txt" - # c.file_input_confluence_paths = ( - # f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" - # ) - - # _partial excludes these tabs: RESEARCH, MODEL, DATA c.file_input_confluence_paths = ( - f"{IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt" + f"{IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt" ) + # _partial excludes these tabs: RESEARCH, MODEL, DATA + # c.file_input_confluence_paths = ( + # f"{IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt" + # ) + c.sensitive_terms_file = f"{IO_DIR}/input/shared/sensitive_terms.txt" c.output_dir = f"{IO_DIR}/output/website_reviewer/" # Must end with "/" c.requested_output = [ diff --git a/examples/review_xml.bash b/examples/review_xml.bash new file mode 100644 index 0000000..b50aca9 --- /dev/null +++ b/examples/review_xml.bash @@ -0,0 +1,24 @@ +# Before running: + +# WordPress: Tools > Export > export pages +# WordPress: Tools > Export > export posts +# scp local_path/wordpress_pages.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_pages.xml +# scp local_path/wordpress_posts.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_posts.xml + +# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/web_pages.txt + +IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io + +echo "Count of top-level Confluence pages:" +wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt +echo "Count of whitelisted e3sm.org pages": +wc -l ${IO_DIR}/input/e3sm_org_reviewer/web_pages.txt + +echo "Step 1. Review Confluence" +echo "Note: this will require a Confluence login" +e3sm-comms-website-reviewer + +echo "Step 2. Review e3sm.org" +cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt +e3sm-comms-exported-xml-reviewer +echo "Output report: ${IO_DIR}/output/e3sm_org_reviewer/wordpress_sensitive_terms_report.md" From 4244b0aa236d35268c3684bafef8119489f2cd96 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 15 May 2026 12:01:05 -0700 Subject: [PATCH 44/85] Improve requested_links section --- e3sm_comms/exported_xml_reviewer/main.py | 50 ++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 4f3f954..f2af8f8 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -128,11 +128,49 @@ def read_requested_links(file_path: str) -> List[Tuple[str, str]]: with open(file_path, "r", encoding="utf-8", newline="") as f: reader = csv.DictReader(f) + + if not reader.fieldnames: + print(f"Requested links CSV has no headers: {file_path}") + return rows + + normalized_to_actual = { + header.strip().lower(): header for header in reader.fieldnames if header + } + + e3sm_header = normalized_to_actual.get("e3sm.org link") + requesting_header = normalized_to_actual.get( + "list of urls that wants to link to it" + ) + + if requesting_header is None: + for candidate in [ + "list of urls that want to link to it", + "requesting urls", + "requesting url", + "list of urls", + ]: + requesting_header = normalized_to_actual.get(candidate) + if requesting_header: + break + + if e3sm_header is None: + print( + f"Requested links CSV is missing required column 'e3sm.org link'. " + f"Found headers: {reader.fieldnames}" + ) + return rows + + if requesting_header is None: + print( + "Requested links CSV could not find the requesting URLs column. " + f"Found headers: {reader.fieldnames}" + ) + for row in reader: - e3sm_url = normalize_url(row.get("e3sm.org link", "")) + e3sm_url = normalize_url(row.get(e3sm_header, "")) requesting_urls = ( - row.get("list of URLs that wants to link to it") or "" - ).strip() + row.get(requesting_header, "").strip() if requesting_header else "" + ) if e3sm_url: rows.append((e3sm_url, requesting_urls)) @@ -517,7 +555,11 @@ def write_markdown_report( f.write("| --- | --- | --- | --- | --- |\n") for requested_record in requested_link_records: - included_later = "Yes" if requested_record.included_later else "No" + included_later = ( + "Yes" + if requested_record.included_later + else "No (i.e., contains no sensitive terms)" + ) currently_whitelisted = ( "Yes" if requested_record.currently_whitelisted else "No" ) From d5bc1000118e6717e018fb62d9860283d9e47fdd Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 15 May 2026 15:23:19 -0700 Subject: [PATCH 45/85] Sort requested links --- e3sm_comms/exported_xml_reviewer/main.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index f2af8f8..a06cbfd 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -480,6 +480,26 @@ def build_records( return records, dict(status_totals), requested_link_records +def sort_requested_link_records( + requested_link_records: List[RequestedLinkRecord], +) -> List[RequestedLinkRecord]: + status_order = { + "Published": 0, + "Archived": 1, + "Not found": 3, + } + + return sorted( + requested_link_records, + key=lambda r: ( + 0 if r.included_later else 1, + status_order.get(r.current_status, 2), + 0 if r.currently_whitelisted else 1, + r.e3sm_url.lower(), + ), + ) + + def write_markdown_report( output_path: str, records: List[ReportRecord], @@ -548,6 +568,8 @@ def write_markdown_report( f.write("\n") if requested_link_records: + requested_link_records = sort_requested_link_records(requested_link_records) + f.write("## Requested Links\n\n") f.write( "| e3sm.org link | Included later on this page? | Current status | Currently whitelisted? | Requesting URLs |\n" From eac1a7dba44e6e3e6ce01c3440805bdfcc19cacc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Tue, 19 May 2026 14:05:20 -0700 Subject: [PATCH 46/85] Update run script --- examples/review_xml.bash | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) mode change 100644 => 100755 examples/review_xml.bash diff --git a/examples/review_xml.bash b/examples/review_xml.bash old mode 100644 new mode 100755 index b50aca9..ba09088 --- a/examples/review_xml.bash +++ b/examples/review_xml.bash @@ -2,23 +2,26 @@ # WordPress: Tools > Export > export pages # WordPress: Tools > Export > export posts -# scp local_path/wordpress_pages.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_pages.xml -# scp local_path/wordpress_posts.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_posts.xml +# scp wordpress_pages.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_pages.xml +# scp wordpress_posts.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_posts.xml -# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/web_pages.txt +# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io//input/exported_xml_reviewer/whitelisted_web_pages.txt IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io echo "Count of top-level Confluence pages:" wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt -echo "Count of whitelisted e3sm.org pages": -wc -l ${IO_DIR}/input/e3sm_org_reviewer/web_pages.txt -echo "Step 1. Review Confluence" +echo "Count of requested links:" +wc -l ${IO_DIR}/input/exported_xml_reviewer/requested_links.csv +echo "Count of whitelisted pages:" +wc -l ${IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt + +echo "Step 1. Review Confluence pages" echo "Note: this will require a Confluence login" e3sm-comms-website-reviewer -echo "Step 2. Review e3sm.org" +echo "Step 2. Review xml exported from e3sm.org" cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt e3sm-comms-exported-xml-reviewer -echo "Output report: ${IO_DIR}/output/e3sm_org_reviewer/wordpress_sensitive_terms_report.md" +echo "Output report: ${IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" From 72af824214fc5c6233953fd8a8418db67a0e33b3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 21 May 2026 08:59:49 -0700 Subject: [PATCH 47/85] Add outline of WordPress pages --- e3sm_comms/exported_xml_reviewer/main.py | 102 ++++++++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index a06cbfd..4cc1539 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -28,6 +28,9 @@ OUTPUT_MARKDOWN_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" ) +OUTPUT_HIERARCHICAL_OUTLINE: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.txt" +) CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -35,6 +38,9 @@ @dataclass class WordpressItem: + post_id: str + post_parent: str + post_type: str title: str url: str status: str @@ -356,6 +362,8 @@ def parse_wordpress_xml( title_elem = item.find("title") link_elem = item.find("link") status_elem = item.find("wp:status", ns) + post_id_elem = item.find("wp:post_id", ns) + post_parent_elem = item.find("wp:post_parent", ns) title = ( title_elem.text.strip() @@ -372,10 +380,23 @@ def parse_wordpress_xml( if status_elem is not None and status_elem.text is not None else "unknown" ) + post_id = ( + post_id_elem.text.strip() + if post_id_elem is not None and post_id_elem.text is not None + else "" + ) + post_parent = ( + post_parent_elem.text.strip() + if post_parent_elem is not None and post_parent_elem.text is not None + else "0" + ) body = extract_item_body(item) items.append( WordpressItem( + post_id=post_id, + post_parent=post_parent, + post_type=post_type_text, title=title, url=link, status=status, @@ -426,7 +447,12 @@ def build_records( sensitive_terms_file: str, whitelist_file: str, requested_links_file: str, -) -> Tuple[List[ReportRecord], Dict[str, int], List[RequestedLinkRecord]]: +) -> Tuple[ + List[ReportRecord], + Dict[str, int], + List[RequestedLinkRecord], + List[WordpressItem], +]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) confluence_map = get_confluence_mapping(confluence_hierarchy) @@ -477,7 +503,7 @@ def build_records( flagged_urls=flagged_urls, ) - return records, dict(status_totals), requested_link_records + return records, dict(status_totals), requested_link_records, raw_items def sort_requested_link_records( @@ -617,8 +643,71 @@ def write_markdown_report( f.write("\n") +def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def write_section(f, section_items: List[WordpressItem], heading: str) -> None: + section_items = [item for item in section_items if item.post_id] + + children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + item_by_id: Dict[str, WordpressItem] = {} + + for item in section_items: + item_by_id[item.post_id] = item + + for item in section_items: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in section_items + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + f.write(f"{heading}\n") + + seen: Set[str] = set() + + def walk(node: WordpressItem, depth: int) -> None: + indent = " " * depth + line = f"{indent}{node.title}" + if node.url: + line += f" [{node.url}]" + f.write(line + "\n") + + if node.post_id in seen: + return + + seen.add(node.post_id) + + for child in children_by_parent.get(node.post_id, []): + walk(child, depth + 1) + + for root in roots: + walk(root, 0) + + f.write("\n") + + pages = [item for item in items if item.post_type == "page"] + posts = [item for item in items if item.post_type == "post"] + + with open(output_path, "w", encoding="utf-8") as f: + write_section(f, pages, "Pages") + write_section(f, posts, "Posts") + + def main() -> None: - records, status_totals, requested_link_records = build_records( + records, status_totals, requested_link_records, raw_items = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, @@ -633,7 +722,14 @@ def main() -> None: status_totals, requested_link_records, ) + + write_hierarchical_outline( + OUTPUT_HIERARCHICAL_OUTLINE, + raw_items, + ) + print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") + print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") if __name__ == "__main__": From 9d01f82cb6b3f11dc42006414c5bc18abb46e03c Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 21 May 2026 09:02:37 -0700 Subject: [PATCH 48/85] Add status to WordPress outline --- e3sm_comms/exported_xml_reviewer/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 4cc1539..5599773 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -680,7 +680,8 @@ def write_section(f, section_items: List[WordpressItem], heading: str) -> None: def walk(node: WordpressItem, depth: int) -> None: indent = " " * depth - line = f"{indent}{node.title}" + status_label = display_status(normalize_status(node.status)) + line = f"{indent}{node.title} [{status_label}]" if node.url: line += f" [{node.url}]" f.write(line + "\n") From 15ca65a47ce812b990a383a130db53797e71b68b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 21 May 2026 11:02:57 -0700 Subject: [PATCH 49/85] Add known ok links section --- e3sm_comms/exported_xml_reviewer/main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 5599773..59426b1 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -24,6 +24,7 @@ INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" +INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" OUTPUT_MARKDOWN_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" @@ -129,6 +130,11 @@ def read_whitelist_patterns(file_path: str) -> List[str]: return [line.strip() for line in f if line.strip()] +def read_known_ok_links(file_path: str) -> Set[str]: + with open(file_path, "r", encoding="utf-8") as f: + return {normalize_url(line.strip()) for line in f if line.strip()} + + def read_requested_links(file_path: str) -> List[Tuple[str, str]]: rows: List[Tuple[str, str]] = [] @@ -447,6 +453,7 @@ def build_records( sensitive_terms_file: str, whitelist_file: str, requested_links_file: str, + known_ok_links_file: str, ) -> Tuple[ List[ReportRecord], Dict[str, int], @@ -455,6 +462,7 @@ def build_records( ]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) confluence_map = get_confluence_mapping(confluence_hierarchy) + known_ok_urls = read_known_ok_links(known_ok_links_file) raw_items: List[WordpressItem] = [] raw_items.extend(parse_wordpress_xml(xml_pages, "page")) @@ -473,7 +481,10 @@ def build_records( if base_status == "published": if item.url in whitelisted_urls: - report_status = "published & whitelisted" + if item.url in known_ok_urls: + report_status = "published & whitelisted, known ok" + else: + report_status = "published & whitelisted" else: report_status = "published & not whitelisted" @@ -542,6 +553,7 @@ def write_markdown_report( ) ordered_statuses = [ + "published & whitelisted, known ok", "published & whitelisted", "published & not whitelisted", "archived", @@ -715,6 +727,7 @@ def main() -> None: sensitive_terms_file=INPUT_SEARCH_PHRASES, whitelist_file=INPUT_WHITELIST, requested_links_file=INPUT_REQUESTED_LINKS, + known_ok_links_file=INPUT_KNOWN_OK_LINKS, ) write_markdown_report( From 2e52730631cfd57215f89c29ec4ae991c4bd7fbc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 21 May 2026 11:15:07 -0700 Subject: [PATCH 50/85] Add navigation report --- e3sm_comms/exported_xml_reviewer/main.py | 145 +++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 59426b1..97bc0c1 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -32,6 +32,9 @@ OUTPUT_HIERARCHICAL_OUTLINE: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.txt" ) +OUTPUT_NAVIGATION_ISSUES_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" +) CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -66,6 +69,23 @@ class RequestedLinkRecord: requesting_urls: str +@dataclass +class TopLevelPageIssue: + title: str + url: str + status: str + + +@dataclass +class ArchivedParentPublishedChildIssue: + parent_title: str + parent_url: str + parent_status: str + child_title: str + child_url: str + child_status: str + + def normalize_url(url: str) -> str: url = url.strip() if not url: @@ -517,6 +537,74 @@ def build_records( return records, dict(status_totals), requested_link_records, raw_items +def build_navigation_issue_records( + items: List[WordpressItem], +) -> Tuple[List[TopLevelPageIssue], List[ArchivedParentPublishedChildIssue]]: + allowed_top_level_titles = { + "about", + "news", + "resources", + "tools", + "policies", + } + + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + top_level_issues: List[TopLevelPageIssue] = [] + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue] = [] + + for page in pages: + normalized_status = normalize_status(page.status) + + is_top_level = ( + not page.post_parent + or page.post_parent == "0" + or page.post_parent not in item_by_id + ) + + if is_top_level and page.title.strip().lower() not in allowed_top_level_titles: + top_level_issues.append( + TopLevelPageIssue( + title=page.title, + url=page.url, + status=display_status(normalized_status), + ) + ) + + if ( + page.post_parent + and page.post_parent != "0" + and page.post_parent in item_by_id + and normalized_status == "published" + ): + parent = item_by_id[page.post_parent] + parent_status = normalize_status(parent.status) + + if parent_status == "archived": + archived_parent_published_child_issues.append( + ArchivedParentPublishedChildIssue( + parent_title=parent.title, + parent_url=parent.url, + parent_status=display_status(parent_status), + child_title=page.title, + child_url=page.url, + child_status=display_status(normalized_status), + ) + ) + + top_level_issues.sort(key=lambda x: (x.title.lower(), x.url.lower())) + archived_parent_published_child_issues.sort( + key=lambda x: ( + x.parent_title.lower(), + x.child_title.lower(), + x.child_url.lower(), + ) + ) + + return top_level_issues, archived_parent_published_child_issues + + def sort_requested_link_records( requested_link_records: List[RequestedLinkRecord], ) -> List[RequestedLinkRecord]: @@ -719,6 +807,52 @@ def walk(node: WordpressItem, depth: int) -> None: write_section(f, posts, "Posts") +def write_navigation_issues_report( + output_path: str, + top_level_issues: List[TopLevelPageIssue], + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Navigation Issues Report\n\n") + + f.write("## 1. Top-level pages that are not expected top-level tabs\n\n") + f.write( + "Expected top-level tabs are: About, News, Resources, Tools, Policies.\n\n" + ) + + if top_level_issues: + f.write("| Title | Status | URL |\n") + f.write("| --- | --- | --- |\n") + for top_level_issue in top_level_issues: + f.write( + f"| {top_level_issue.title} | {top_level_issue.status} | {top_level_issue.url} |\n" + ) + else: + f.write("No unexpected top-level pages found.\n") + + f.write("\n") + + f.write("## 2. Published child pages under archived parent pages\n\n") + + if archived_parent_published_child_issues: + f.write( + "| Parent title | Parent status | Parent URL | Child title | Child status | Child URL |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- |\n") + for archived_child_issue in archived_parent_published_child_issues: + f.write( + f"| {archived_child_issue.parent_title} | {archived_child_issue.parent_status} | {archived_child_issue.parent_url} | " + f"{archived_child_issue.child_title} | {archived_child_issue.child_status} | {archived_child_issue.child_url} |\n" + ) + else: + f.write( + "No published child pages were found under archived parent pages.\n" + ) + + def main() -> None: records, status_totals, requested_link_records, raw_items = build_records( xml_pages=INPUT_XML_PAGES, @@ -742,8 +876,19 @@ def main() -> None: raw_items, ) + top_level_issues, archived_parent_published_child_issues = ( + build_navigation_issue_records(raw_items) + ) + + write_navigation_issues_report( + OUTPUT_NAVIGATION_ISSUES_REPORT, + top_level_issues, + archived_parent_published_child_issues, + ) + print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") + print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") if __name__ == "__main__": From 3047d416fce548973c33c82a802261f95a155383 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 21 May 2026 11:22:08 -0700 Subject: [PATCH 51/85] Sort navigation report --- e3sm_comms/exported_xml_reviewer/main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 97bc0c1..661a79c 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -546,6 +546,7 @@ def build_navigation_issue_records( "resources", "tools", "policies", + "home page", } pages = [item for item in items if item.post_type == "page" and item.post_id] @@ -593,7 +594,24 @@ def build_navigation_issue_records( ) ) - top_level_issues.sort(key=lambda x: (x.title.lower(), x.url.lower())) + status_order = { + "Published": 0, + "Draft": 1, + "Pending": 2, + "Future": 3, + "Private": 4, + "Archived": 5, + "Unknown": 6, + } + + top_level_issues.sort( + key=lambda x: ( + status_order.get(x.status, 99), + x.title.lower(), + x.url.lower(), + ) + ) + archived_parent_published_child_issues.sort( key=lambda x: ( x.parent_title.lower(), @@ -820,7 +838,7 @@ def write_navigation_issues_report( f.write("## 1. Top-level pages that are not expected top-level tabs\n\n") f.write( - "Expected top-level tabs are: About, News, Resources, Tools, Policies.\n\n" + "Expected top-level tabs are: About, News, Resources, Tools, Policies, Home Page.\n\n" ) if top_level_issues: From 320b2c593f9f0e9ac075c141ec64a1a9905b6197 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 09:30:09 -0700 Subject: [PATCH 52/85] Add invalid link report --- e3sm_comms/exported_xml_reviewer/main.py | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 661a79c..94f5e16 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -35,6 +35,9 @@ OUTPUT_NAVIGATION_ISSUES_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" ) +OUTPUT_INVALID_INTERNAL_LINKS_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" +) CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -86,6 +89,12 @@ class ArchivedParentPublishedChildIssue: child_status: str +@dataclass +class InvalidInternalLinkGroup: + linked_url: str + source_titles: List[str] + + def normalize_url(url: str) -> str: url = url.strip() if not url: @@ -433,6 +442,58 @@ def parse_wordpress_xml( return items +def extract_internal_e3sm_links(html_text: str) -> Set[str]: + links: Set[str] = set() + + for match in re.finditer( + r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE + ): + href = match.group(1).strip() + if not href: + continue + + try: + parts = urlsplit(href) + except ValueError: + continue + + host = parts.netloc.lower() + + if host.endswith("e3sm.org"): + links.add(normalize_url(href)) + elif not parts.scheme and not parts.netloc and href.startswith("/"): + links.add(normalize_url(f"https://e3sm.org{href}")) + + return links + + +def build_invalid_internal_link_groups( + items: List[WordpressItem], +) -> List[InvalidInternalLinkGroup]: + actual_urls = {normalize_url(item.url) for item in items if item.url} + + linked_to_sources: DefaultDict[str, Set[str]] = defaultdict(set) + + for item in items: + if not item.url or not item.body: + continue + + for linked_url in extract_internal_e3sm_links(item.body): + if linked_url not in actual_urls: + linked_to_sources[linked_url].add(item.title) + + groups = [ + InvalidInternalLinkGroup( + linked_url=linked_url, + source_titles=sorted(source_titles, key=str.lower), + ) + for linked_url, source_titles in linked_to_sources.items() + ] + + groups.sort(key=lambda g: g.linked_url.lower()) + return groups + + def build_requested_link_records( requested_links_file: str, raw_items: List[WordpressItem], @@ -761,6 +822,28 @@ def write_markdown_report( f.write("\n") +def write_invalid_internal_links_report( + output_path: str, + groups: List[InvalidInternalLinkGroup], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Invalid Internal e3sm.org Links\n\n") + + if not groups: + f.write("No invalid internal links found.\n") + return + + f.write("| Invalid linked URL | Referenced on |\n") + f.write("| --- | --- |\n") + + for group in groups: + referenced_on = ", ".join(group.source_titles) + f.write(f"| {group.linked_url} | {referenced_on} |\n") + + def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) @@ -904,9 +987,18 @@ def main() -> None: archived_parent_published_child_issues, ) + invalid_link_groups = build_invalid_internal_link_groups(raw_items) + write_invalid_internal_links_report( + OUTPUT_INVALID_INTERNAL_LINKS_REPORT, + invalid_link_groups, + ) + print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") + print( + f"Wrote invalid internal links report to {OUTPUT_INVALID_INTERNAL_LINKS_REPORT}" + ) if __name__ == "__main__": From 21e11102a9861958f25062c5a637c9525db82ffd Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 09:41:16 -0700 Subject: [PATCH 53/85] Filter the invalid links report --- e3sm_comms/exported_xml_reviewer/main.py | 57 +++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 94f5e16..e67f59c 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -92,7 +92,8 @@ class ArchivedParentPublishedChildIssue: @dataclass class InvalidInternalLinkGroup: linked_url: str - source_titles: List[str] + likely_new_link: str + referenced_on: List[Tuple[str, str]] def normalize_url(url: str) -> str: @@ -458,21 +459,49 @@ def extract_internal_e3sm_links(html_text: str) -> Set[str]: continue host = parts.netloc.lower() + path = parts.path.lower() + + if host == "docs.e3sm.org": + continue if host.endswith("e3sm.org"): + if path.startswith("/wp-content"): + continue links.add(normalize_url(href)) elif not parts.scheme and not parts.netloc and href.startswith("/"): - links.add(normalize_url(f"https://e3sm.org{href}")) + normalized_relative = normalize_url(f"https://e3sm.org{href}") + if "/wp-content" in normalized_relative.lower(): + continue + links.add(normalized_relative) return links +def infer_likely_new_link(linked_url: str) -> str: + parts = urlsplit(linked_url) + path = parts.path.rstrip("/").lower() + + if path in {"/model", "/data"}: + return f"https://e3sm.org/resources{path}" + + if path == "/about/news": + return "https://e3sm.org/news" + + if path == "/resources/policies": + return "https://e3sm.org/policies" + + if path == "/resources/tools": + return "https://e3sm.org/tools" + + return "" + + def build_invalid_internal_link_groups( items: List[WordpressItem], ) -> List[InvalidInternalLinkGroup]: actual_urls = {normalize_url(item.url) for item in items if item.url} - linked_to_sources: DefaultDict[str, Set[str]] = defaultdict(set) + linked_to_sources: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict(set) for item in items: if not item.url or not item.body: @@ -480,14 +509,15 @@ def build_invalid_internal_link_groups( for linked_url in extract_internal_e3sm_links(item.body): if linked_url not in actual_urls: - linked_to_sources[linked_url].add(item.title) + linked_to_sources[linked_url].add((item.title, item.url)) groups = [ InvalidInternalLinkGroup( linked_url=linked_url, - source_titles=sorted(source_titles, key=str.lower), + likely_new_link=infer_likely_new_link(linked_url), + referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), ) - for linked_url, source_titles in linked_to_sources.items() + for linked_url, source_pairs in linked_to_sources.items() ] groups.sort(key=lambda g: g.linked_url.lower()) @@ -836,12 +866,19 @@ def write_invalid_internal_links_report( f.write("No invalid internal links found.\n") return - f.write("| Invalid linked URL | Referenced on |\n") - f.write("| --- | --- |\n") + f.write("| Invalid linked URL | Likely new link | Referenced on |\n") + f.write("| --- | --- | --- |\n") for group in groups: - referenced_on = ", ".join(group.source_titles) - f.write(f"| {group.linked_url} | {referenced_on} |\n") + likely_new_link = ( + f"[{group.likely_new_link}]({group.likely_new_link})" + if group.likely_new_link + else "" + ) + referenced_on = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on + ) + f.write(f"| {group.linked_url} | {likely_new_link} | {referenced_on} |\n") def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: From bbfeb691f503b9fdee9bba590e473c18969d0b7a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 09:46:22 -0700 Subject: [PATCH 54/85] Improve inference rules for link redirection --- e3sm_comms/exported_xml_reviewer/main.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index e67f59c..ebff00c 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -481,17 +481,25 @@ def infer_likely_new_link(linked_url: str) -> str: parts = urlsplit(linked_url) path = parts.path.rstrip("/").lower() - if path in {"/model", "/data"}: - return f"https://e3sm.org/resources{path}" + if path.startswith("/model"): + suffix = parts.path[len("/model") :].lstrip("/") + return f"https://e3sm.org/resources/model/{suffix}".rstrip("/") - if path == "/about/news": - return "https://e3sm.org/news" + if path.startswith("/data"): + suffix = parts.path[len("/data") :].lstrip("/") + return f"https://e3sm.org/resources/data/{suffix}".rstrip("/") - if path == "/resources/policies": - return "https://e3sm.org/policies" + if path.startswith("/about/news"): + suffix = parts.path[len("/about/news") :].lstrip("/") + return f"https://e3sm.org/news/{suffix}".rstrip("/") - if path == "/resources/tools": - return "https://e3sm.org/tools" + if path.startswith("/resources/policies"): + suffix = parts.path[len("/resources/policies") :].lstrip("/") + return f"https://e3sm.org/policies/{suffix}".rstrip("/") + + if path.startswith("/resources/tools"): + suffix = parts.path[len("/resources/tools") :].lstrip("/") + return f"https://e3sm.org/tools/{suffix}".rstrip("/") return "" From 811c407b2d77bbecade86586ca50052b8dcc4d6f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 09:53:09 -0700 Subject: [PATCH 55/85] Add guessed link column --- e3sm_comms/exported_xml_reviewer/main.py | 54 +++++++++++++++++++----- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index ebff00c..141aeaa 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -93,6 +93,7 @@ class ArchivedParentPublishedChildIssue: class InvalidInternalLinkGroup: linked_url: str likely_new_link: str + guessed_link: str referenced_on: List[Tuple[str, str]] @@ -504,6 +505,29 @@ def infer_likely_new_link(linked_url: str) -> str: return "" +def guess_redirect_target(linked_url: str, actual_urls: Set[str]) -> str: + parts = urlsplit(linked_url) + path = parts.path.strip("/").lower() + + if not path: + return "" + + slug = path.split("/")[-1] + candidates = [] + + for actual_url in actual_urls: + actual_parts = urlsplit(actual_url) + actual_path = actual_parts.path.strip("/").lower() + + if actual_path.endswith("/" + slug) or actual_path == slug: + candidates.append(normalize_url(actual_url)) + + if len(candidates) == 1: + return candidates[0] + + return "" + + def build_invalid_internal_link_groups( items: List[WordpressItem], ) -> List[InvalidInternalLinkGroup]: @@ -519,14 +543,21 @@ def build_invalid_internal_link_groups( if linked_url not in actual_urls: linked_to_sources[linked_url].add((item.title, item.url)) - groups = [ - InvalidInternalLinkGroup( - linked_url=linked_url, - likely_new_link=infer_likely_new_link(linked_url), - referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), + groups: List[InvalidInternalLinkGroup] = [] + for linked_url, source_pairs in linked_to_sources.items(): + likely_new_link = infer_likely_new_link(linked_url) + guessed_link = "" + if not likely_new_link: + guessed_link = guess_redirect_target(linked_url, actual_urls) + + groups.append( + InvalidInternalLinkGroup( + linked_url=linked_url, + likely_new_link=likely_new_link, + guessed_link=guessed_link, + referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), + ) ) - for linked_url, source_pairs in linked_to_sources.items() - ] groups.sort(key=lambda g: g.linked_url.lower()) return groups @@ -874,8 +905,8 @@ def write_invalid_internal_links_report( f.write("No invalid internal links found.\n") return - f.write("| Invalid linked URL | Likely new link | Referenced on |\n") - f.write("| --- | --- | --- |\n") + f.write("| Invalid linked URL | Likely new link | Guess | Referenced on |\n") + f.write("| --- | --- | --- | --- |\n") for group in groups: likely_new_link = ( @@ -883,10 +914,13 @@ def write_invalid_internal_links_report( if group.likely_new_link else "" ) + guessed_link = f"{group.guessed_link} (guess)" if group.guessed_link else "" referenced_on = ", ".join( f"[{title}]({url})" for title, url in group.referenced_on ) - f.write(f"| {group.linked_url} | {likely_new_link} | {referenced_on} |\n") + f.write( + f"| {group.linked_url} | {likely_new_link} | {guessed_link} | {referenced_on} |\n" + ) def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: From 43cb4d9d69887803016918212bd0a624661d8375 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 09:58:40 -0700 Subject: [PATCH 56/85] Sort by inference type --- e3sm_comms/exported_xml_reviewer/main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 141aeaa..e170370 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -559,7 +559,17 @@ def build_invalid_internal_link_groups( ) ) - groups.sort(key=lambda g: g.linked_url.lower()) + def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, str]: + if group.likely_new_link: + priority = 0 + elif group.guessed_link: + priority = 1 + else: + priority = 2 + + return (priority, group.linked_url.lower()) + + groups.sort(key=sort_key) return groups From d289d328644322306df8b5bcdcc40ecdfdff8d5b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 10:04:00 -0700 Subject: [PATCH 57/85] Add status column --- e3sm_comms/exported_xml_reviewer/main.py | 34 ++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index e170370..2801048 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -94,6 +94,7 @@ class InvalidInternalLinkGroup: linked_url: str likely_new_link: str guessed_link: str + linked_target_status: str referenced_on: List[Tuple[str, str]] @@ -550,16 +551,33 @@ def build_invalid_internal_link_groups( if not likely_new_link: guessed_link = guess_redirect_target(linked_url, actual_urls) + target_url = likely_new_link or guessed_link + linked_target_status = "" + if target_url: + for item in items: + if normalize_url(item.url) == normalize_url(target_url): + linked_target_status = display_status(normalize_status(item.status)) + break + groups.append( InvalidInternalLinkGroup( linked_url=linked_url, likely_new_link=likely_new_link, guessed_link=guessed_link, + linked_target_status=linked_target_status, referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), ) ) - def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, str]: + def status_rank(status: str) -> int: + status = status.lower() + if status == "published": + return 0 + if status == "archived": + return 1 + return 2 + + def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, str]: if group.likely_new_link: priority = 0 elif group.guessed_link: @@ -567,7 +585,11 @@ def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, str]: else: priority = 2 - return (priority, group.linked_url.lower()) + return ( + priority, + status_rank(group.linked_target_status), + group.linked_url.lower(), + ) groups.sort(key=sort_key) return groups @@ -915,8 +937,10 @@ def write_invalid_internal_links_report( f.write("No invalid internal links found.\n") return - f.write("| Invalid linked URL | Likely new link | Guess | Referenced on |\n") - f.write("| --- | --- | --- | --- |\n") + f.write( + "| Invalid linked URL | Likely new link | Guess | Status | Referenced on |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") for group in groups: likely_new_link = ( @@ -929,7 +953,7 @@ def write_invalid_internal_links_report( f"[{title}]({url})" for title, url in group.referenced_on ) f.write( - f"| {group.linked_url} | {likely_new_link} | {guessed_link} | {referenced_on} |\n" + f"| {group.linked_url} | {likely_new_link} | {guessed_link} | {group.linked_target_status} | {referenced_on} |\n" ) From 00b1ab0c762c368e6abffea75d41bf8781995060 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 10:11:49 -0700 Subject: [PATCH 58/85] Clean up invalid links report --- e3sm_comms/exported_xml_reviewer/main.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 2801048..ceba59b 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -445,6 +445,12 @@ def parse_wordpress_xml( return items +def is_legacy_content_url(url: str) -> bool: + parts = urlsplit(url) + slug = parts.path.strip("/").lower() + return bool(re.fullmatch(r"\d{6,8}[_-].+", slug)) + + def extract_internal_e3sm_links(html_text: str) -> Set[str]: links: Set[str] = set() @@ -469,11 +475,19 @@ def extract_internal_e3sm_links(html_text: str) -> Set[str]: if host.endswith("e3sm.org"): if path.startswith("/wp-content"): continue - links.add(normalize_url(href)) + + normalized = normalize_url(href) + if is_legacy_content_url(normalized): + continue + + links.add(normalized) elif not parts.scheme and not parts.netloc and href.startswith("/"): normalized_relative = normalize_url(f"https://e3sm.org{href}") if "/wp-content" in normalized_relative.lower(): continue + if is_legacy_content_url(normalized_relative): + continue + links.add(normalized_relative) return links @@ -938,7 +952,7 @@ def write_invalid_internal_links_report( return f.write( - "| Invalid linked URL | Likely new link | Guess | Status | Referenced on |\n" + "| Invalid linked URL | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on |\n" ) f.write("| --- | --- | --- | --- | --- |\n") @@ -948,7 +962,7 @@ def write_invalid_internal_links_report( if group.likely_new_link else "" ) - guessed_link = f"{group.guessed_link} (guess)" if group.guessed_link else "" + guessed_link = group.guessed_link if group.guessed_link else "" referenced_on = ", ".join( f"[{title}]({url})" for title, url in group.referenced_on ) From 7ec059ba8470ad9cc6e01432f62424d9d8fc6ed7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 10:21:35 -0700 Subject: [PATCH 59/85] Make Confluence and whitelist inputs optional --- e3sm_comms/exported_xml_reviewer/main.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index ceba59b..1b59f52 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -18,11 +18,13 @@ INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" -INPUT_CONFLUENCE_HIERARCHY: str = ( - f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" -) +# INPUT_CONFLUENCE_HIERARCHY: str = ( +# f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" +# ) +INPUT_CONFLUENCE_HIERARCHY: str = "" INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +# INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +INPUT_WHITELIST: str = "" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" @@ -657,16 +659,23 @@ def build_records( List[WordpressItem], ]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) - confluence_map = get_confluence_mapping(confluence_hierarchy) + + confluence_map = {} + if confluence_hierarchy: + confluence_map = get_confluence_mapping(confluence_hierarchy) + known_ok_urls = read_known_ok_links(known_ok_links_file) raw_items: List[WordpressItem] = [] raw_items.extend(parse_wordpress_xml(xml_pages, "page")) raw_items.extend(parse_wordpress_xml(xml_posts, "post")) - whitelist_patterns = read_whitelist_patterns(whitelist_file) all_urls = [item.url for item in raw_items if item.url] - whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) + if whitelist_file: + whitelist_patterns = read_whitelist_patterns(whitelist_file) + whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) + else: + whitelisted_urls = set(all_urls) records: List[ReportRecord] = [] status_totals: DefaultDict[str, int] = defaultdict(int) From c1bd8e282adb91e757da6c23278a428621577482 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 10:54:51 -0700 Subject: [PATCH 60/85] Add link checking --- e3sm_comms/exported_xml_reviewer/main.py | 92 +++++++++++++++++------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 1b59f52..e117cf3 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -9,6 +9,8 @@ from typing import DefaultDict, Dict, List, Optional, Set, Tuple from urllib.parse import urlsplit, urlunsplit +import requests # type: ignore + from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm from e3sm_comms.utils import IO_DIR @@ -94,8 +96,10 @@ class ArchivedParentPublishedChildIssue: @dataclass class InvalidInternalLinkGroup: linked_url: str - likely_new_link: str - guessed_link: str + redirect_target: str + redirect_status: str + inferred_link: str + found_under_different_prefix: str linked_target_status: str referenced_on: List[Tuple[str, str]] @@ -545,6 +549,24 @@ def guess_redirect_target(linked_url: str, actual_urls: Set[str]) -> str: return "" +def check_redirect_target(link_url: str) -> Tuple[str, str]: + try: + response = requests.get(link_url, timeout=10, allow_redirects=True) + redirect_status = "" + + if response.history: + first_response = response.history[0] + if first_response.status_code in {301, 302, 303, 307, 308} and response.ok: + final_url = normalize_url(response.url) + redirect_status = str(first_response.status_code) + return final_url, redirect_status + + return "", "" + + except (requests.exceptions.Timeout, requests.exceptions.RequestException): + return "", "" + + def build_invalid_internal_link_groups( items: List[WordpressItem], ) -> List[InvalidInternalLinkGroup]: @@ -562,13 +584,17 @@ def build_invalid_internal_link_groups( groups: List[InvalidInternalLinkGroup] = [] for linked_url, source_pairs in linked_to_sources.items(): - likely_new_link = infer_likely_new_link(linked_url) - guessed_link = "" - if not likely_new_link: - guessed_link = guess_redirect_target(linked_url, actual_urls) + redirect_target, redirect_status = check_redirect_target(linked_url) + + inferred_link = infer_likely_new_link(linked_url) + found_under_different_prefix = "" + if not inferred_link: + found_under_different_prefix = guess_redirect_target( + linked_url, actual_urls + ) - target_url = likely_new_link or guessed_link linked_target_status = "" + target_url = redirect_target or inferred_link or found_under_different_prefix if target_url: for item in items: if normalize_url(item.url) == normalize_url(target_url): @@ -578,8 +604,10 @@ def build_invalid_internal_link_groups( groups.append( InvalidInternalLinkGroup( linked_url=linked_url, - likely_new_link=likely_new_link, - guessed_link=guessed_link, + redirect_target=redirect_target, + redirect_status=redirect_status, + inferred_link=inferred_link, + found_under_different_prefix=found_under_different_prefix, linked_target_status=linked_target_status, referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), ) @@ -593,16 +621,17 @@ def status_rank(status: str) -> int: return 1 return 2 - def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, str]: - if group.likely_new_link: - priority = 0 - elif group.guessed_link: - priority = 1 - else: - priority = 2 + def inference_rank(group: InvalidInternalLinkGroup) -> int: + if group.inferred_link: + return 0 + if group.found_under_different_prefix: + return 1 + return 2 + def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, int, str]: return ( - priority, + 0 if group.redirect_target else 1, + inference_rank(group), status_rank(group.linked_target_status), group.linked_url.lower(), ) @@ -961,22 +990,37 @@ def write_invalid_internal_links_report( return f.write( - "| Invalid linked URL | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on |\n" + "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on |\n" ) - f.write("| --- | --- | --- | --- | --- |\n") + f.write("| --- | --- | --- | --- | --- | --- |\n") for group in groups: - likely_new_link = ( - f"[{group.likely_new_link}]({group.likely_new_link})" - if group.likely_new_link + redirect_md = ( + f"[{group.redirect_target}]({group.redirect_target})" + if group.redirect_target + else "" + ) + if redirect_md and group.redirect_status: + redirect_md = ( + f"{redirect_md} (redirect status: {group.redirect_status})" + ) + + inferred_md = ( + f"[{group.inferred_link}]({group.inferred_link})" + if group.inferred_link + else "" + ) + prefix_md = ( + f"[{group.found_under_different_prefix}]({group.found_under_different_prefix})" + if group.found_under_different_prefix else "" ) - guessed_link = group.guessed_link if group.guessed_link else "" referenced_on = ", ".join( f"[{title}]({url})" for title, url in group.referenced_on ) + f.write( - f"| {group.linked_url} | {likely_new_link} | {guessed_link} | {group.linked_target_status} | {referenced_on} |\n" + f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_on} |\n" ) From eeea9c6c4a4675a40b9e051930c20c79e1d3b57d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 11:14:29 -0700 Subject: [PATCH 61/85] Revise inference rules --- e3sm_comms/exported_xml_reviewer/main.py | 31 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index e117cf3..900d2dd 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -571,6 +571,7 @@ def build_invalid_internal_link_groups( items: List[WordpressItem], ) -> List[InvalidInternalLinkGroup]: actual_urls = {normalize_url(item.url) for item in items if item.url} + item_by_url = {normalize_url(item.url): item for item in items if item.url} linked_to_sources: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict(set) @@ -586,20 +587,30 @@ def build_invalid_internal_link_groups( for linked_url, source_pairs in linked_to_sources.items(): redirect_target, redirect_status = check_redirect_target(linked_url) - inferred_link = infer_likely_new_link(linked_url) + inferred_candidate = infer_likely_new_link(linked_url) + inferred_link = ( + inferred_candidate + if inferred_candidate and inferred_candidate in actual_urls + else "" + ) + found_under_different_prefix = "" if not inferred_link: - found_under_different_prefix = guess_redirect_target( - linked_url, actual_urls - ) + guessed_candidate = guess_redirect_target(linked_url, actual_urls) + if guessed_candidate and guessed_candidate in actual_urls: + found_under_different_prefix = guessed_candidate + + resolved_target = ( + redirect_target or inferred_link or found_under_different_prefix + ) linked_target_status = "" - target_url = redirect_target or inferred_link or found_under_different_prefix - if target_url: - for item in items: - if normalize_url(item.url) == normalize_url(target_url): - linked_target_status = display_status(normalize_status(item.status)) - break + if resolved_target: + matched_item = item_by_url.get(normalize_url(resolved_target)) + if matched_item is not None: + linked_target_status = display_status( + normalize_status(matched_item.status) + ) groups.append( InvalidInternalLinkGroup( From 251edaebef1ed2d607ca1f27d6995c4acbca6f50 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 11:25:55 -0700 Subject: [PATCH 62/85] Make distinct tables --- e3sm_comms/exported_xml_reviewer/main.py | 61 ++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 900d2dd..6e6428e 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -993,19 +993,13 @@ def write_invalid_internal_links_report( output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w", encoding="utf-8") as f: - f.write("# Invalid Internal e3sm.org Links\n\n") - - if not groups: - f.write("No invalid internal links found.\n") - return - + def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: f.write( "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on |\n" ) f.write("| --- | --- | --- | --- | --- | --- |\n") - for group in groups: + for group in table_groups: redirect_md = ( f"[{group.redirect_target}]({group.redirect_target})" if group.redirect_target @@ -1034,6 +1028,57 @@ def write_invalid_internal_links_report( f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_on} |\n" ) + working_redirects: List[InvalidInternalLinkGroup] = [] + published_targets: List[InvalidInternalLinkGroup] = [] + archived_targets: List[InvalidInternalLinkGroup] = [] + no_candidate: List[InvalidInternalLinkGroup] = [] + + for group in groups: + if group.redirect_target: + working_redirects.append(group) + elif group.linked_target_status == "Published": + published_targets.append(group) + elif group.linked_target_status == "Archived": + archived_targets.append(group) + else: + no_candidate.append(group) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Invalid Internal e3sm.org Links\n\n") + + if not groups: + f.write("No invalid internal links found.\n") + return + + f.write("## 1. These have working redirections already\n\n") + if working_redirects: + render_table(f, working_redirects) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + "## 2. The target pages are published, we just need to set up the redirections\n\n" + ) + if published_targets: + render_table(f, published_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write("## 3. The target pages are archived\n\n") + if archived_targets: + render_table(f, archived_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write("## 4. Couldn't find a redirection candidate\n\n") + if no_candidate: + render_table(f, no_candidate) + else: + f.write("None found.\n") + def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: output_file = Path(output_path) From d32aa18fcffee955953fe5aaa47b9e2213a7e6ea Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 11:40:30 -0700 Subject: [PATCH 63/85] Fix status check --- e3sm_comms/exported_xml_reviewer/main.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 6e6428e..159eab0 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -590,23 +590,20 @@ def build_invalid_internal_link_groups( inferred_candidate = infer_likely_new_link(linked_url) inferred_link = ( inferred_candidate - if inferred_candidate and inferred_candidate in actual_urls + if inferred_candidate and normalize_url(inferred_candidate) in actual_urls else "" ) found_under_different_prefix = "" if not inferred_link: guessed_candidate = guess_redirect_target(linked_url, actual_urls) - if guessed_candidate and guessed_candidate in actual_urls: + if guessed_candidate and normalize_url(guessed_candidate) in actual_urls: found_under_different_prefix = guessed_candidate - resolved_target = ( - redirect_target or inferred_link or found_under_different_prefix - ) - + status_target = inferred_link or found_under_different_prefix linked_target_status = "" - if resolved_target: - matched_item = item_by_url.get(normalize_url(resolved_target)) + if status_target: + matched_item = item_by_url.get(normalize_url(status_target)) if matched_item is not None: linked_target_status = display_status( normalize_status(matched_item.status) From b819292ca90b3dd2a37f5a40aac8d68793a0528e Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 11:52:13 -0700 Subject: [PATCH 64/85] Add section for archived links --- e3sm_comms/exported_xml_reviewer/main.py | 75 +++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 159eab0..ff8282b 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -104,6 +104,13 @@ class InvalidInternalLinkGroup: referenced_on: List[Tuple[str, str]] +@dataclass +class NonPublishedInternalLinkGroup: + linked_url: str + target_status: str + referenced_on: List[Tuple[str, str]] + + def normalize_url(url: str) -> str: url = url.strip() if not url: @@ -648,6 +655,48 @@ def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, int, str]: return groups +def build_non_published_internal_link_groups( + items: List[WordpressItem], +) -> List[NonPublishedInternalLinkGroup]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + + linked_to_sources: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict(set) + + for item in items: + if not item.url or not item.body: + continue + + for linked_url in extract_internal_e3sm_links(item.body): + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + normalized_target_status = normalize_status(matched_item.status) + if normalized_target_status == "published": + continue + + linked_to_sources[linked_url].add((item.title, item.url)) + + groups: List[NonPublishedInternalLinkGroup] = [] + for linked_url, source_pairs in linked_to_sources.items(): + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + groups.append( + NonPublishedInternalLinkGroup( + linked_url=linked_url, + target_status=display_status(normalize_status(matched_item.status)), + referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), + ) + ) + + groups.sort( + key=lambda group: (group.target_status.lower(), group.linked_url.lower()) + ) + return groups + + def build_requested_link_records( requested_links_file: str, raw_items: List[WordpressItem], @@ -986,6 +1035,7 @@ def write_markdown_report( def write_invalid_internal_links_report( output_path: str, groups: List[InvalidInternalLinkGroup], + non_published_groups: List[NonPublishedInternalLinkGroup], ) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) @@ -1025,6 +1075,20 @@ def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_on} |\n" ) + def render_non_published_table( + f, table_groups: List[NonPublishedInternalLinkGroup] + ) -> None: + f.write("| Valid linked URL | Target status | Referenced on |\n") + f.write("| --- | --- | --- |\n") + + for group in table_groups: + referenced_on = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on + ) + f.write( + f"| {group.linked_url} | {group.target_status} | {referenced_on} |\n" + ) + working_redirects: List[InvalidInternalLinkGroup] = [] published_targets: List[InvalidInternalLinkGroup] = [] archived_targets: List[InvalidInternalLinkGroup] = [] @@ -1043,7 +1107,7 @@ def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: with open(output_file, "w", encoding="utf-8") as f: f.write("# Invalid Internal e3sm.org Links\n\n") - if not groups: + if not groups and not non_published_groups: f.write("No invalid internal links found.\n") return @@ -1075,6 +1139,13 @@ def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: render_table(f, no_candidate) else: f.write("None found.\n") + f.write("\n") + + f.write("## 5. Technically valid links that point to non-published targets\n\n") + if non_published_groups: + render_non_published_table(f, non_published_groups) + else: + f.write("None found.\n") def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: @@ -1221,9 +1292,11 @@ def main() -> None: ) invalid_link_groups = build_invalid_internal_link_groups(raw_items) + non_published_link_groups = build_non_published_internal_link_groups(raw_items) write_invalid_internal_links_report( OUTPUT_INVALID_INTERNAL_LINKS_REPORT, invalid_link_groups, + non_published_link_groups, ) print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") From 319ec7e4ac99dc108cef8fffdb7236d37642959d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Fri, 22 May 2026 12:24:00 -0700 Subject: [PATCH 65/85] Add section counts and split references column --- e3sm_comms/exported_xml_reviewer/main.py | 113 +++++++++++++++++------ 1 file changed, 87 insertions(+), 26 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index ff8282b..17178b1 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -101,14 +101,16 @@ class InvalidInternalLinkGroup: inferred_link: str found_under_different_prefix: str linked_target_status: str - referenced_on: List[Tuple[str, str]] + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] @dataclass class NonPublishedInternalLinkGroup: linked_url: str target_status: str - referenced_on: List[Tuple[str, str]] + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] def normalize_url(url: str) -> str: @@ -580,18 +582,33 @@ def build_invalid_internal_link_groups( actual_urls = {normalize_url(item.url) for item in items if item.url} item_by_url = {normalize_url(item.url): item for item in items if item.url} - linked_to_sources: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict(set) + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) for item in items: if not item.url or not item.body: continue + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + for linked_url in extract_internal_e3sm_links(item.body): if linked_url not in actual_urls: - linked_to_sources[linked_url].add((item.title, item.url)) + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) groups: List[InvalidInternalLinkGroup] = [] - for linked_url, source_pairs in linked_to_sources.items(): + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: redirect_target, redirect_status = check_redirect_target(linked_url) inferred_candidate = infer_likely_new_link(linked_url) @@ -624,7 +641,14 @@ def build_invalid_internal_link_groups( inferred_link=inferred_link, found_under_different_prefix=found_under_different_prefix, linked_target_status=linked_target_status, - referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), ) ) @@ -660,12 +684,20 @@ def build_non_published_internal_link_groups( ) -> List[NonPublishedInternalLinkGroup]: item_by_url = {normalize_url(item.url): item for item in items if item.url} - linked_to_sources: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict(set) + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) for item in items: if not item.url or not item.body: continue + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + for linked_url in extract_internal_e3sm_links(item.body): matched_item = item_by_url.get(normalize_url(linked_url)) if matched_item is None: @@ -675,10 +707,17 @@ def build_non_published_internal_link_groups( if normalized_target_status == "published": continue - linked_to_sources[linked_url].add((item.title, item.url)) + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) groups: List[NonPublishedInternalLinkGroup] = [] - for linked_url, source_pairs in linked_to_sources.items(): + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: matched_item = item_by_url.get(normalize_url(linked_url)) if matched_item is None: continue @@ -687,7 +726,14 @@ def build_non_published_internal_link_groups( NonPublishedInternalLinkGroup( linked_url=linked_url, target_status=display_status(normalize_status(matched_item.status)), - referenced_on=sorted(source_pairs, key=lambda x: x[0].lower()), + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), ) ) @@ -1015,7 +1061,7 @@ def write_markdown_report( continue seen.add(status) - f.write(f"## {status.capitalize()}\n\n") + f.write(f"## {status.capitalize()} ({len(grouped[status])})\n\n") for idx, record in enumerate(grouped[status], start=1): e3sm_md = f"[e3sm.org]({record.e3sm_url})" @@ -1042,9 +1088,9 @@ def write_invalid_internal_links_report( def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: f.write( - "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on |\n" + "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on these published pages | Referenced on these non-published pages |\n" ) - f.write("| --- | --- | --- | --- | --- | --- |\n") + f.write("| --- | --- | --- | --- | --- | --- | --- |\n") for group in table_groups: redirect_md = ( @@ -1067,26 +1113,35 @@ def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: if group.found_under_different_prefix else "" ) - referenced_on = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on + + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published ) f.write( - f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_on} |\n" + f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_published} | {referenced_non_published} |\n" ) def render_non_published_table( f, table_groups: List[NonPublishedInternalLinkGroup] ) -> None: - f.write("| Valid linked URL | Target status | Referenced on |\n") - f.write("| --- | --- | --- |\n") + f.write( + "| Valid linked URL | Target status | Referenced on these published pages | Referenced on these non-published pages |\n" + ) + f.write("| --- | --- | --- | --- |\n") for group in table_groups: - referenced_on = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published ) f.write( - f"| {group.linked_url} | {group.target_status} | {referenced_on} |\n" + f"| {group.linked_url} | {group.target_status} | {referenced_published} | {referenced_non_published} |\n" ) working_redirects: List[InvalidInternalLinkGroup] = [] @@ -1111,7 +1166,9 @@ def render_non_published_table( f.write("No invalid internal links found.\n") return - f.write("## 1. These have working redirections already\n\n") + f.write( + f"## 1. These have working redirections already ({len(working_redirects)})\n\n" + ) if working_redirects: render_table(f, working_redirects) else: @@ -1119,7 +1176,7 @@ def render_non_published_table( f.write("\n") f.write( - "## 2. The target pages are published, we just need to set up the redirections\n\n" + f"## 2. The target pages are published, we just need to set up the redirections ({len(published_targets)})\n\n" ) if published_targets: render_table(f, published_targets) @@ -1127,21 +1184,25 @@ def render_non_published_table( f.write("None found.\n") f.write("\n") - f.write("## 3. The target pages are archived\n\n") + f.write(f"## 3. The target pages are archived ({len(archived_targets)})\n\n") if archived_targets: render_table(f, archived_targets) else: f.write("None found.\n") f.write("\n") - f.write("## 4. Couldn't find a redirection candidate\n\n") + f.write( + f"## 4. Couldn't find a redirection candidate ({len(no_candidate)})\n\n" + ) if no_candidate: render_table(f, no_candidate) else: f.write("None found.\n") f.write("\n") - f.write("## 5. Technically valid links that point to non-published targets\n\n") + f.write( + f"## 5. Technically valid links that point to non-published targets ({len(non_published_groups)})\n\n" + ) if non_published_groups: render_non_published_table(f, non_published_groups) else: From e190b249b4e80126b02aac7222ceda43784cb37d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Tue, 26 May 2026 17:42:40 -0700 Subject: [PATCH 66/85] Add published page link report --- e3sm_comms/exported_xml_reviewer/main.py | 137 +++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 17178b1..f57239e 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -42,6 +42,9 @@ OUTPUT_INVALID_INTERNAL_LINKS_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" ) +OUTPUT_PUBLISHED_PAGES_LINK_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" +) CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -113,6 +116,15 @@ class NonPublishedInternalLinkGroup: referenced_on_non_published: List[Tuple[str, str]] +@dataclass +class PublishedPageLinkSummary: + title: str + url: str + archived_links: List[str] + redirected_links: List[str] + broken_links: List[str] + + def normalize_url(url: str) -> str: url = url.strip() if not url: @@ -576,6 +588,94 @@ def check_redirect_target(link_url: str) -> Tuple[str, str]: return "", "" +def get_page_hierarchy_order(items: List[WordpressItem]) -> List[WordpressItem]: + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + for item in pages: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in pages + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + ordered: List[WordpressItem] = [] + seen: Set[str] = set() + + def walk(node: WordpressItem) -> None: + if node.post_id in seen: + return + seen.add(node.post_id) + ordered.append(node) + for child in children_by_parent.get(node.post_id, []): + walk(child) + + for root in roots: + walk(root) + + return ordered + + +def build_published_page_link_summaries( + items: List[WordpressItem], +) -> List[PublishedPageLinkSummary]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + actual_urls = set(item_by_url.keys()) + + ordered_pages = get_page_hierarchy_order(items) + summaries: List[PublishedPageLinkSummary] = [] + + for page in ordered_pages: + if normalize_status(page.status) != "published": + continue + if not page.url or not page.body: + continue + + archived_links: Set[str] = set() + redirected_links: Set[str] = set() + broken_links: Set[str] = set() + + for linked_url in extract_internal_e3sm_links(page.body): + linked_norm = normalize_url(linked_url) + target_item = item_by_url.get(linked_norm) + + if target_item is not None: + target_status = normalize_status(target_item.status) + if target_status == "archived": + archived_links.add(linked_norm) + continue + + redirect_target, _redirect_status = check_redirect_target(linked_norm) + if redirect_target and normalize_url(redirect_target) in actual_urls: + redirected_links.add(linked_norm) + else: + broken_links.add(linked_norm) + + summaries.append( + PublishedPageLinkSummary( + title=page.title, + url=page.url, + archived_links=sorted(archived_links), + redirected_links=sorted(redirected_links), + broken_links=sorted(broken_links), + ) + ) + + return summaries + + def build_invalid_internal_link_groups( items: List[WordpressItem], ) -> List[InvalidInternalLinkGroup]: @@ -1209,6 +1309,36 @@ def render_non_published_table( f.write("None found.\n") +def write_published_pages_link_report( + output_path: str, + summaries: List[PublishedPageLinkSummary], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Pages Invalid Link Report\n\n") + + if not summaries: + f.write("No published pages with links found.\n") + return + + f.write( + "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work |\n" + ) + f.write("| --- | --- | --- | --- |\n") + + for summary in summaries: + page_md = f"[{summary.title}]({summary.url})" + archived_md = ", ".join(f"[{url}]({url})" for url in summary.archived_links) + redirected_md = ", ".join( + f"[{url}]({url})" for url in summary.redirected_links + ) + broken_md = ", ".join(f"[{url}]({url})" for url in summary.broken_links) + + f.write(f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} |\n") + + def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) @@ -1360,12 +1490,19 @@ def main() -> None: non_published_link_groups, ) + published_page_link_summaries = build_published_page_link_summaries(raw_items) + write_published_pages_link_report( + OUTPUT_PUBLISHED_PAGES_LINK_REPORT, + published_page_link_summaries, + ) + print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") print( f"Wrote invalid internal links report to {OUTPUT_INVALID_INTERNAL_LINKS_REPORT}" ) + print(f"Wrote published pages link report to {OUTPUT_PUBLISHED_PAGES_LINK_REPORT}") if __name__ == "__main__": From 72d4799ed58426de35b3a8e4e6e7f4417a093d92 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Tue, 26 May 2026 17:50:30 -0700 Subject: [PATCH 67/85] Add valid link count --- e3sm_comms/exported_xml_reviewer/main.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index f57239e..f9f9338 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -123,6 +123,7 @@ class PublishedPageLinkSummary: archived_links: List[str] redirected_links: List[str] broken_links: List[str] + valid_link_count: int def normalize_url(url: str) -> str: @@ -646,6 +647,7 @@ def build_published_page_link_summaries( archived_links: Set[str] = set() redirected_links: Set[str] = set() broken_links: Set[str] = set() + valid_link_count = 0 for linked_url in extract_internal_e3sm_links(page.body): linked_norm = normalize_url(linked_url) @@ -655,11 +657,14 @@ def build_published_page_link_summaries( target_status = normalize_status(target_item.status) if target_status == "archived": archived_links.add(linked_norm) + else: + valid_link_count += 1 continue redirect_target, _redirect_status = check_redirect_target(linked_norm) if redirect_target and normalize_url(redirect_target) in actual_urls: redirected_links.add(linked_norm) + valid_link_count += 1 else: broken_links.add(linked_norm) @@ -670,6 +675,7 @@ def build_published_page_link_summaries( archived_links=sorted(archived_links), redirected_links=sorted(redirected_links), broken_links=sorted(broken_links), + valid_link_count=valid_link_count, ) ) @@ -1316,6 +1322,8 @@ def write_published_pages_link_report( output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) + total_valid_links = 0 + with open(output_file, "w", encoding="utf-8") as f: f.write("# Published Pages Invalid Link Report\n\n") @@ -1324,9 +1332,9 @@ def write_published_pages_link_report( return f.write( - "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work |\n" + "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" ) - f.write("| --- | --- | --- | --- |\n") + f.write("| --- | --- | --- | --- | ---: |\n") for summary in summaries: page_md = f"[{summary.title}]({summary.url})" @@ -1336,7 +1344,14 @@ def write_published_pages_link_report( ) broken_md = ", ".join(f"[{url}]({url})" for url in summary.broken_links) - f.write(f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} |\n") + total_valid_links += summary.valid_link_count + + f.write( + f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {summary.valid_link_count} |\n" + ) + + f.write("\n") + f.write(f"Total valid e3sm.org links on published pages: {total_valid_links}\n") def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: From 20c8a663e9d8d06136125d2d398fbd9f283df532 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 27 May 2026 07:39:16 -0700 Subject: [PATCH 68/85] Display pages with invalid links first --- e3sm_comms/exported_xml_reviewer/main.py | 56 +++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index f9f9338..3038f71 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -1322,7 +1322,16 @@ def write_published_pages_link_report( output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) - total_valid_links = 0 + invalid_summaries = [ + s for s in summaries if s.archived_links or s.redirected_links or s.broken_links + ] + valid_only_summaries = [ + s + for s in summaries + if not (s.archived_links or s.redirected_links or s.broken_links) + ] + + total_valid_links = sum(s.valid_link_count for s in summaries) with open(output_file, "w", encoding="utf-8") as f: f.write("# Published Pages Invalid Link Report\n\n") @@ -1331,24 +1340,39 @@ def write_published_pages_link_report( f.write("No published pages with links found.\n") return - f.write( - "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" - ) - f.write("| --- | --- | --- | --- | ---: |\n") - - for summary in summaries: - page_md = f"[{summary.title}]({summary.url})" - archived_md = ", ".join(f"[{url}]({url})" for url in summary.archived_links) - redirected_md = ", ".join( - f"[{url}]({url})" for url in summary.redirected_links + f.write("## Pages with invalid links\n\n") + if invalid_summaries: + f.write( + "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" ) - broken_md = ", ".join(f"[{url}]({url})" for url in summary.broken_links) + f.write("| --- | --- | --- | --- | ---: |\n") - total_valid_links += summary.valid_link_count + for summary in invalid_summaries: + page_md = f"[{summary.title}]({summary.url})" + archived_md = ", ".join( + f"[{url}]({url})" for url in summary.archived_links + ) + redirected_md = ", ".join( + f"[{url}]({url})" for url in summary.redirected_links + ) + broken_md = ", ".join(f"[{url}]({url})" for url in summary.broken_links) - f.write( - f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {summary.valid_link_count} |\n" - ) + f.write( + f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {summary.valid_link_count} |\n" + ) + else: + f.write("No pages with invalid links found.\n") + + f.write("\n## Published pages with no invalid links\n\n") + if valid_only_summaries: + f.write("| Published pages | valid e3sm.org link count |\n") + f.write("| --- | ---: |\n") + + for summary in valid_only_summaries: + page_md = f"[{summary.title}]({summary.url})" + f.write(f"| {page_md} | {summary.valid_link_count} |\n") + else: + f.write("No pages with only valid links found.\n") f.write("\n") f.write(f"Total valid e3sm.org links on published pages: {total_valid_links}\n") From 3133503ab9de9d59137e2ad86a82febeabd403f6 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 27 May 2026 07:48:15 -0700 Subject: [PATCH 69/85] Add column totals --- e3sm_comms/exported_xml_reviewer/main.py | 74 +++++++++++++++++------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 3038f71..172ac2a 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -123,7 +123,7 @@ class PublishedPageLinkSummary: archived_links: List[str] redirected_links: List[str] broken_links: List[str] - valid_link_count: int + valid_links: List[str] def normalize_url(url: str) -> str: @@ -647,7 +647,7 @@ def build_published_page_link_summaries( archived_links: Set[str] = set() redirected_links: Set[str] = set() broken_links: Set[str] = set() - valid_link_count = 0 + valid_links: Set[str] = set() for linked_url in extract_internal_e3sm_links(page.body): linked_norm = normalize_url(linked_url) @@ -658,13 +658,13 @@ def build_published_page_link_summaries( if target_status == "archived": archived_links.add(linked_norm) else: - valid_link_count += 1 + valid_links.add(linked_norm) continue redirect_target, _redirect_status = check_redirect_target(linked_norm) if redirect_target and normalize_url(redirect_target) in actual_urls: redirected_links.add(linked_norm) - valid_link_count += 1 + valid_links.add(linked_norm) else: broken_links.add(linked_norm) @@ -675,7 +675,7 @@ def build_published_page_link_summaries( archived_links=sorted(archived_links), redirected_links=sorted(redirected_links), broken_links=sorted(broken_links), - valid_link_count=valid_link_count, + valid_links=sorted(valid_links), ) ) @@ -1331,7 +1331,8 @@ def write_published_pages_link_report( if not (s.archived_links or s.redirected_links or s.broken_links) ] - total_valid_links = sum(s.valid_link_count for s in summaries) + def render_link_list(urls: List[str]) -> str: + return ", ".join(f"[{url}]({url})" for url in urls) with open(output_file, "w", encoding="utf-8") as f: f.write("# Published Pages Invalid Link Report\n\n") @@ -1340,43 +1341,76 @@ def write_published_pages_link_report( f.write("No published pages with links found.\n") return - f.write("## Pages with invalid links\n\n") + f.write(f"## Pages with invalid links ({len(invalid_summaries)})\n\n") if invalid_summaries: f.write( "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" ) f.write("| --- | --- | --- | --- | ---: |\n") + archived_total = 0 + redirected_total = 0 + broken_total = 0 + valid_total = 0 + + archived_unique: Set[str] = set() + redirected_unique: Set[str] = set() + broken_unique: Set[str] = set() + valid_unique: Set[str] = set() + for summary in invalid_summaries: page_md = f"[{summary.title}]({summary.url})" - archived_md = ", ".join( - f"[{url}]({url})" for url in summary.archived_links - ) - redirected_md = ", ".join( - f"[{url}]({url})" for url in summary.redirected_links - ) - broken_md = ", ".join(f"[{url}]({url})" for url in summary.broken_links) + archived_md = render_link_list(summary.archived_links) + redirected_md = render_link_list(summary.redirected_links) + broken_md = render_link_list(summary.broken_links) + valid_md = render_link_list(summary.valid_links) + + archived_total += len(summary.archived_links) + redirected_total += len(summary.redirected_links) + broken_total += len(summary.broken_links) + valid_total += len(summary.valid_links) + + archived_unique.update(summary.archived_links) + redirected_unique.update(summary.redirected_links) + broken_unique.update(summary.broken_links) + valid_unique.update(summary.valid_links) f.write( - f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {summary.valid_link_count} |\n" + f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" ) + + f.write( + f"| Total link count | {archived_total} | {redirected_total} | {broken_total} | {valid_total} |\n" + ) + f.write( + f"| Unique link count | {len(archived_unique)} | {len(redirected_unique)} | {len(broken_unique)} | {len(valid_unique)} |\n" + ) else: f.write("No pages with invalid links found.\n") - f.write("\n## Published pages with no invalid links\n\n") + f.write("\n") + f.write( + f"## Published pages with no invalid links ({len(valid_only_summaries)})\n\n" + ) + if valid_only_summaries: f.write("| Published pages | valid e3sm.org link count |\n") f.write("| --- | ---: |\n") + total_links = 0 + unique_links: Set[str] = set() + for summary in valid_only_summaries: page_md = f"[{summary.title}]({summary.url})" - f.write(f"| {page_md} | {summary.valid_link_count} |\n") + f.write(f"| {page_md} | {len(summary.valid_links)} |\n") + total_links += len(summary.valid_links) + unique_links.update(summary.valid_links) + + f.write(f"| Total link count | {total_links} |\n") + f.write(f"| Unique link count | {len(unique_links)} |\n") else: f.write("No pages with only valid links found.\n") - f.write("\n") - f.write(f"Total valid e3sm.org links on published pages: {total_valid_links}\n") - def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: output_file = Path(output_path) From 8e402804b90ffc7826c25396bdb4516f7a92c0c7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 27 May 2026 08:10:40 -0700 Subject: [PATCH 70/85] Add posts --- e3sm_comms/exported_xml_reviewer/main.py | 108 ++++++++++++++--------- 1 file changed, 66 insertions(+), 42 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 172ac2a..7907411 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -20,12 +20,8 @@ INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" -# INPUT_CONFLUENCE_HIERARCHY: str = ( -# f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" -# ) INPUT_CONFLUENCE_HIERARCHY: str = "" INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -# INPUT_WHITELIST: str = f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" INPUT_WHITELIST: str = "" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" @@ -117,7 +113,7 @@ class NonPublishedInternalLinkGroup: @dataclass -class PublishedPageLinkSummary: +class PublishedContentLinkSummary: title: str url: str archived_links: List[str] @@ -629,19 +625,27 @@ def walk(node: WordpressItem) -> None: return ordered -def build_published_page_link_summaries( +def build_published_content_link_summaries( items: List[WordpressItem], -) -> List[PublishedPageLinkSummary]: + post_type: str, +) -> List[PublishedContentLinkSummary]: item_by_url = {normalize_url(item.url): item for item in items if item.url} actual_urls = set(item_by_url.keys()) - ordered_pages = get_page_hierarchy_order(items) - summaries: List[PublishedPageLinkSummary] = [] + if post_type == "page": + ordered_items = get_page_hierarchy_order(items) + else: + ordered_items = sorted( + [item for item in items if item.post_type == post_type and item.post_id], + key=lambda x: x.title.lower(), + ) + + summaries: List[PublishedContentLinkSummary] = [] - for page in ordered_pages: - if normalize_status(page.status) != "published": + for item in ordered_items: + if normalize_status(item.status) != "published": continue - if not page.url or not page.body: + if not item.url or not item.body: continue archived_links: Set[str] = set() @@ -649,7 +653,7 @@ def build_published_page_link_summaries( broken_links: Set[str] = set() valid_links: Set[str] = set() - for linked_url in extract_internal_e3sm_links(page.body): + for linked_url in extract_internal_e3sm_links(item.body): linked_norm = normalize_url(linked_url) target_item = item_by_url.get(linked_norm) @@ -669,9 +673,9 @@ def build_published_page_link_summaries( broken_links.add(linked_norm) summaries.append( - PublishedPageLinkSummary( - title=page.title, - url=page.url, + PublishedContentLinkSummary( + title=item.title, + url=item.url, archived_links=sorted(archived_links), redirected_links=sorted(redirected_links), broken_links=sorted(broken_links), @@ -1317,34 +1321,41 @@ def render_non_published_table( def write_published_pages_link_report( output_path: str, - summaries: List[PublishedPageLinkSummary], + page_summaries: List[PublishedContentLinkSummary], + post_summaries: List[PublishedContentLinkSummary], ) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) - invalid_summaries = [ - s for s in summaries if s.archived_links or s.redirected_links or s.broken_links - ] - valid_only_summaries = [ - s - for s in summaries - if not (s.archived_links or s.redirected_links or s.broken_links) - ] - def render_link_list(urls: List[str]) -> str: return ", ".join(f"[{url}]({url})" for url in urls) - with open(output_file, "w", encoding="utf-8") as f: - f.write("# Published Pages Invalid Link Report\n\n") + def write_section( + f, + section_title: str, + summaries: List[PublishedContentLinkSummary], + ) -> None: + invalid_summaries = [ + s + for s in summaries + if s.archived_links or s.redirected_links or s.broken_links + ] + valid_only_summaries = [ + s + for s in summaries + if not (s.archived_links or s.redirected_links or s.broken_links) + ] + + f.write(f"## {section_title}\n\n") if not summaries: - f.write("No published pages with links found.\n") + f.write("No published items with links found.\n\n") return - f.write(f"## Pages with invalid links ({len(invalid_summaries)})\n\n") + f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") if invalid_summaries: f.write( - "| Published page | known archived links | published page, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" + "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" ) f.write("| --- | --- | --- | --- | ---: |\n") @@ -1359,7 +1370,7 @@ def render_link_list(urls: List[str]) -> str: valid_unique: Set[str] = set() for summary in invalid_summaries: - page_md = f"[{summary.title}]({summary.url})" + item_md = f"[{summary.title}]({summary.url})" archived_md = render_link_list(summary.archived_links) redirected_md = render_link_list(summary.redirected_links) broken_md = render_link_list(summary.broken_links) @@ -1376,7 +1387,7 @@ def render_link_list(urls: List[str]) -> str: valid_unique.update(summary.valid_links) f.write( - f"| {page_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" + f"| {item_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" ) f.write( @@ -1386,30 +1397,35 @@ def render_link_list(urls: List[str]) -> str: f"| Unique link count | {len(archived_unique)} | {len(redirected_unique)} | {len(broken_unique)} | {len(valid_unique)} |\n" ) else: - f.write("No pages with invalid links found.\n") + f.write("No items with invalid links found.\n") f.write("\n") - f.write( - f"## Published pages with no invalid links ({len(valid_only_summaries)})\n\n" - ) + f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") if valid_only_summaries: - f.write("| Published pages | valid e3sm.org link count |\n") + f.write("| Published item | valid e3sm.org link count |\n") f.write("| --- | ---: |\n") total_links = 0 unique_links: Set[str] = set() for summary in valid_only_summaries: - page_md = f"[{summary.title}]({summary.url})" - f.write(f"| {page_md} | {len(summary.valid_links)} |\n") + item_md = f"[{summary.title}]({summary.url})" + f.write(f"| {item_md} | {len(summary.valid_links)} |\n") total_links += len(summary.valid_links) unique_links.update(summary.valid_links) f.write(f"| Total link count | {total_links} |\n") f.write(f"| Unique link count | {len(unique_links)} |\n") else: - f.write("No pages with only valid links found.\n") + f.write("No items with only valid links found.\n") + + f.write("\n") + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Content Invalid Link Report\n\n") + write_section(f, "Published Pages", page_summaries) + write_section(f, "Published Posts", post_summaries) def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: @@ -1563,10 +1579,18 @@ def main() -> None: non_published_link_groups, ) - published_page_link_summaries = build_published_page_link_summaries(raw_items) + published_page_link_summaries = build_published_content_link_summaries( + raw_items, + "page", + ) + published_post_link_summaries = build_published_content_link_summaries( + raw_items, + "post", + ) write_published_pages_link_report( OUTPUT_PUBLISHED_PAGES_LINK_REPORT, published_page_link_summaries, + published_post_link_summaries, ) print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") From 42d68886f9139d63d50d135ef0399d5e58cd90b3 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 13:13:07 -0700 Subject: [PATCH 71/85] Address comments --- README.md | 4 ++-- e3sm_comms/exported_xml_reviewer/main.py | 16 ++++++++++--- e3sm_comms/page_reviewer/utils_base.py | 24 +++++++++---------- .../page_reviewer/utils_website_reviewer.py | 2 +- examples/review_terms.bash | 10 ++++---- pyproject.toml | 1 + 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 508d1b2..56fb045 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This package is for implementing the software needs of the E3SM Communications t `e3sm-comms-e3sm-org-reviewer` - input: - - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml forsyth@perlmutter.nersc.gov:perlmutter_path/wordpress_posts.xml`. + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml user@host:/remote/path/wordpress_posts.xml`. - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change) - output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items @@ -31,7 +31,7 @@ This package is for implementing the software needs of the E3SM Communications t - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages -- output: Markdown summary report of sensitive terms found in exported WordPress data + - output: 5 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md` ### Confluence API commands (require Confluence token) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 7907411..d0ec83b 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -18,14 +18,24 @@ # Configuration # ----------------------------------------------------------------------------- +# Required inputs: INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" -INPUT_CONFLUENCE_HIERARCHY: str = "" INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" -INPUT_WHITELIST: str = "" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" +# Optional inputs: +INPUT_CONFLUENCE_HIERARCHY: str = "" +# INPUT_CONFLUENCE_HIERARCHY: str = ( +# f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" +# ) +INPUT_WHITELIST: str = "" +# INPUT_WHITELIST: str = ( +# f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +# ) + +# Outputs: OUTPUT_MARKDOWN_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" ) @@ -1357,7 +1367,7 @@ def write_section( f.write( "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" ) - f.write("| --- | --- | --- | --- | ---: |\n") + f.write("| --- | --- | --- | --- | --- |\n") archived_total = 0 redirected_total = 0 diff --git a/e3sm_comms/page_reviewer/utils_base.py b/e3sm_comms/page_reviewer/utils_base.py index d5e9de7..ee1625e 100644 --- a/e3sm_comms/page_reviewer/utils_base.py +++ b/e3sm_comms/page_reviewer/utils_base.py @@ -354,20 +354,20 @@ def get_e3sm_url_status(e3sm_url: str) -> str: try: response = requests.get(e3sm_url, timeout=10) response.raise_for_status() # Raises HTTPError for 4xx/5xx responses - e3sm_url_status = "link works not logged-in" + return "link works not logged-in" except requests.exceptions.Timeout: - e3sm_url_status = "link times out" - except requests.exceptions.RequestException as e: - error_message: str = f"{e}" - if error_message.startswith( - "503 Server Error: Service Temporarily Unavailable for url: https://e3sm.org" - ): - e3sm_url_status = "link not whitelisted" - else: - e3sm_url_status = "link raises RequestException" + return "link times out" + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code if e.response is not None else None + if status_code == 503 and ( + e.response.url if e.response is not None else "" + ).startswith("https://e3sm.org"): + return "link not whitelisted" + return "link raises RequestException" + except requests.exceptions.RequestException: + return "link raises RequestException" except Exception: - e3sm_url_status = "link raises Exception" - return e3sm_url_status + return "link raises Exception" # Debugging ################################################################### diff --git a/e3sm_comms/page_reviewer/utils_website_reviewer.py b/e3sm_comms/page_reviewer/utils_website_reviewer.py index 785ea96..62ba61a 100644 --- a/e3sm_comms/page_reviewer/utils_website_reviewer.py +++ b/e3sm_comms/page_reviewer/utils_website_reviewer.py @@ -26,7 +26,7 @@ def write_results(config: Config, page: ConfluencePage): with open( f"{config.output_dir}hierarchical_outline.txt", "a", encoding="utf-8" ) as f: - f.write(f"{page.depth * " "}{line_id}\n") + f.write(f"{page.depth * ' '}{line_id}\n") if "sensitive_terms" in config.requested_output: # Append if sensitive terms were found. diff --git a/examples/review_terms.bash b/examples/review_terms.bash index 0a61031..01dceec 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -1,13 +1,15 @@ # Before running: -# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/web_pages.txt +# WordPress: Tools > Export > export pages (wordpress_pages.xml) and posts (wordpress_posts.xml) +# Copy those XMLs into /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/ +# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/whitelisted_web_pages.txt # Also confirm confluence_top_levels_partial.txt is the list of top levels you want to use, otherwise switch it out. IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io echo "Count of top-level Confluence pages:" wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt # Excludes MODEL, RESEARCH, DATA -echo "Count of whitelisted e3sm.org pages": -wc -l ${IO_DIR}/input/e3sm_org_reviewer/web_pages.txt +echo "Count of whitelisted e3sm.org pages:" +wc -l ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt echo "Step 1. Review Confluence" echo "Note: this will require a Confluence login" @@ -20,4 +22,4 @@ e3sm-comms-e3sm-org-reviewer echo "Output reports:" echo "1. ${IO_DIR}/output/e3sm_org_reviewer/path_report.md" echo "2. ${IO_DIR}/output/e3sm_org_reviewer/sensitive_terms.md" -echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md +echo "3. ${IO_DIR}/output/e3sm_org_reviewer/action_items.md" diff --git a/pyproject.toml b/pyproject.toml index 7b954d1..1d24d7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ dependencies = [ "beautifulsoup4", + "requests", ] [project.optional-dependencies] From d3ce206a251e3f37d9b27a95f133b237a3b5f46b Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 15:30:20 -0700 Subject: [PATCH 72/85] Break exported_xml_reviewer into multiple files --- e3sm_comms/exported_xml_reviewer/README.md | 10 + e3sm_comms/exported_xml_reviewer/builders.py | 390 +++++ .../exported_xml_reviewer/confluence.py | 63 + .../exported_xml_reviewer/link_analysis.py | 314 ++++ e3sm_comms/exported_xml_reviewer/main.py | 1523 +---------------- e3sm_comms/exported_xml_reviewer/readers.py | 204 +++ e3sm_comms/exported_xml_reviewer/reporters.py | 488 ++++++ e3sm_comms/exported_xml_reviewer/utils.py | 119 ++ 8 files changed, 1604 insertions(+), 1507 deletions(-) create mode 100644 e3sm_comms/exported_xml_reviewer/README.md create mode 100644 e3sm_comms/exported_xml_reviewer/builders.py create mode 100644 e3sm_comms/exported_xml_reviewer/confluence.py create mode 100644 e3sm_comms/exported_xml_reviewer/link_analysis.py create mode 100644 e3sm_comms/exported_xml_reviewer/readers.py create mode 100644 e3sm_comms/exported_xml_reviewer/reporters.py create mode 100644 e3sm_comms/exported_xml_reviewer/utils.py diff --git a/e3sm_comms/exported_xml_reviewer/README.md b/e3sm_comms/exported_xml_reviewer/README.md new file mode 100644 index 0000000..069f167 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/README.md @@ -0,0 +1,10 @@ +# Dependency hierarchy + +It is important to not introduce circular dependencies. +To avoid this, the dependency hierarchy is listed below: + +- Level 1: `main.py` +- Level 2: `builders.py`, +- Level 3: `confluence.py`, `link_analysis.py`, +- Level 4: `readers.py`, +- Level 5: `utils.py` diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py new file mode 100644 index 0000000..ec3d915 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, Dict, List, Optional, Set, Tuple + +from e3sm_comms.exported_xml_reviewer.confluence import get_confluence_mapping +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + check_redirect_target, + extract_internal_e3sm_links, +) +from e3sm_comms.exported_xml_reviewer.readers import ( + WordpressItem, + parse_wordpress_xml, + read_known_ok_links, + read_requested_links, + read_sensitive_terms, + read_whitelist_patterns, +) +from e3sm_comms.exported_xml_reviewer.utils import ( + count_sensitive_terms, + display_status, + expand_patterns_to_urls, + normalize_status, + normalize_url, + strip_html, +) + + +@dataclass +class ReportRecord: + title: str + e3sm_url: str + status: str + sensitive_terms: Dict[str, int] + confluence_draft_url: Optional[str] + + +@dataclass +class RequestedLinkRecord: + e3sm_url: str + included_later: bool + current_status: str + currently_whitelisted: bool + requesting_urls: str + + +@dataclass +class TopLevelPageIssue: + title: str + url: str + status: str + + +@dataclass +class ArchivedParentPublishedChildIssue: + parent_title: str + parent_url: str + parent_status: str + child_title: str + child_url: str + child_status: str + + +@dataclass +class PublishedContentLinkSummary: + title: str + url: str + archived_links: List[str] + redirected_links: List[str] + broken_links: List[str] + valid_links: List[str] + + +def build_records( + xml_pages: str, + xml_posts: str, + confluence_hierarchy: str, + sensitive_terms_file: str, + whitelist_file: str, + requested_links_file: str, + known_ok_links_file: str, +) -> Tuple[ + List[ReportRecord], + Dict[str, int], + List[RequestedLinkRecord], + List[WordpressItem], +]: + sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) + + confluence_map = {} + if confluence_hierarchy: + confluence_map = get_confluence_mapping(confluence_hierarchy) + + known_ok_urls = read_known_ok_links(known_ok_links_file) + + raw_items: List[WordpressItem] = [] + raw_items.extend(parse_wordpress_xml(xml_pages, "page")) + raw_items.extend(parse_wordpress_xml(xml_posts, "post")) + + all_urls = [item.url for item in raw_items if item.url] + if whitelist_file: + whitelist_patterns = read_whitelist_patterns(whitelist_file) + whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) + else: + whitelisted_urls = set(all_urls) + + records: List[ReportRecord] = [] + status_totals: DefaultDict[str, int] = defaultdict(int) + + for item in raw_items: + base_status = normalize_status(item.status) + report_status = base_status + + if base_status == "published": + if item.url in whitelisted_urls: + if item.url in known_ok_urls: + report_status = "published & whitelisted, known ok" + else: + report_status = "published & whitelisted" + else: + report_status = "published & not whitelisted" + + status_totals[report_status] += 1 + + plain_text = strip_html(item.body) + term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) + + if not term_counts: + continue + + records.append( + ReportRecord( + title=item.title, + e3sm_url=item.url, + status=report_status, + sensitive_terms=term_counts, + confluence_draft_url=confluence_map.get(item.url), + ) + ) + + flagged_urls = {record.e3sm_url for record in records} + requested_link_records = build_requested_link_records( + requested_links_file=requested_links_file, + raw_items=raw_items, + whitelisted_urls=whitelisted_urls, + flagged_urls=flagged_urls, + ) + + return records, dict(status_totals), requested_link_records, raw_items + + +def build_navigation_issue_records( + items: List[WordpressItem], +) -> Tuple[List[TopLevelPageIssue], List[ArchivedParentPublishedChildIssue]]: + allowed_top_level_titles = { + "about", + "news", + "resources", + "tools", + "policies", + "home page", + } + + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + top_level_issues: List[TopLevelPageIssue] = [] + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue] = [] + + for page in pages: + normalized_status = normalize_status(page.status) + + is_top_level = ( + not page.post_parent + or page.post_parent == "0" + or page.post_parent not in item_by_id + ) + + if is_top_level and page.title.strip().lower() not in allowed_top_level_titles: + top_level_issues.append( + TopLevelPageIssue( + title=page.title, + url=page.url, + status=display_status(normalized_status), + ) + ) + + if ( + page.post_parent + and page.post_parent != "0" + and page.post_parent in item_by_id + and normalized_status == "published" + ): + parent = item_by_id[page.post_parent] + parent_status = normalize_status(parent.status) + + if parent_status == "archived": + archived_parent_published_child_issues.append( + ArchivedParentPublishedChildIssue( + parent_title=parent.title, + parent_url=parent.url, + parent_status=display_status(parent_status), + child_title=page.title, + child_url=page.url, + child_status=display_status(normalized_status), + ) + ) + + status_order = { + "Published": 0, + "Draft": 1, + "Pending": 2, + "Future": 3, + "Private": 4, + "Archived": 5, + "Unknown": 6, + } + + top_level_issues.sort( + key=lambda x: ( + status_order.get(x.status, 99), + x.title.lower(), + x.url.lower(), + ) + ) + + archived_parent_published_child_issues.sort( + key=lambda x: ( + x.parent_title.lower(), + x.child_title.lower(), + x.child_url.lower(), + ) + ) + + return top_level_issues, archived_parent_published_child_issues + + +def build_published_content_link_summaries( + items: List[WordpressItem], + post_type: str, +) -> List[PublishedContentLinkSummary]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + actual_urls = set(item_by_url.keys()) + + if post_type == "page": + ordered_items = get_page_hierarchy_order(items) + else: + ordered_items = sorted( + [item for item in items if item.post_type == post_type and item.post_id], + key=lambda x: x.title.lower(), + ) + + summaries: List[PublishedContentLinkSummary] = [] + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + + archived_links: Set[str] = set() + redirected_links: Set[str] = set() + broken_links: Set[str] = set() + valid_links: Set[str] = set() + + for linked_url in extract_internal_e3sm_links(item.body): + linked_norm = normalize_url(linked_url) + target_item = item_by_url.get(linked_norm) + + if target_item is not None: + target_status = normalize_status(target_item.status) + if target_status == "archived": + archived_links.add(linked_norm) + else: + valid_links.add(linked_norm) + continue + + redirect_target, _redirect_status = check_redirect_target(linked_norm) + if redirect_target and normalize_url(redirect_target) in actual_urls: + redirected_links.add(linked_norm) + valid_links.add(linked_norm) + else: + broken_links.add(linked_norm) + + summaries.append( + PublishedContentLinkSummary( + title=item.title, + url=item.url, + archived_links=sorted(archived_links), + redirected_links=sorted(redirected_links), + broken_links=sorted(broken_links), + valid_links=sorted(valid_links), + ) + ) + + return summaries + + +def build_requested_link_records( + requested_links_file: str, + raw_items: List[WordpressItem], + whitelisted_urls: Set[str], + flagged_urls: Set[str], +) -> List[RequestedLinkRecord]: + requested_rows = read_requested_links(requested_links_file) + item_by_url = {item.url: item for item in raw_items if item.url} + + records: List[RequestedLinkRecord] = [] + for e3sm_url, requesting_urls in requested_rows: + item = item_by_url.get(e3sm_url) + + if item is None: + current_status = "Not found" + currently_whitelisted = False + else: + current_status = display_status(normalize_status(item.status)) + currently_whitelisted = e3sm_url in whitelisted_urls + + records.append( + RequestedLinkRecord( + e3sm_url=e3sm_url, + included_later=e3sm_url in flagged_urls, + current_status=current_status, + currently_whitelisted=currently_whitelisted, + requesting_urls=requesting_urls, + ) + ) + + return records + + +def get_page_hierarchy_order(items: List[WordpressItem]) -> List[WordpressItem]: + pages = [item for item in items if item.post_type == "page" and item.post_id] + item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} + + children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + for item in pages: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in pages + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + ordered: List[WordpressItem] = [] + seen: Set[str] = set() + + def walk(node: WordpressItem) -> None: + if node.post_id in seen: + return + seen.add(node.post_id) + ordered.append(node) + for child in children_by_parent.get(node.post_id, []): + walk(child) + + for root in roots: + walk(root) + + return ordered + + +def sort_requested_link_records( + requested_link_records: List[RequestedLinkRecord], +) -> List[RequestedLinkRecord]: + status_order = { + "Published": 0, + "Archived": 1, + "Not found": 3, + } + + return sorted( + requested_link_records, + key=lambda r: ( + 0 if r.included_later else 1, + status_order.get(r.current_status, 2), + 0 if r.currently_whitelisted else 1, + r.e3sm_url.lower(), + ), + ) diff --git a/e3sm_comms/exported_xml_reviewer/confluence.py b/e3sm_comms/exported_xml_reviewer/confluence.py new file mode 100644 index 0000000..ee68c70 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/confluence.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import Dict, List, Tuple + +from e3sm_comms.exported_xml_reviewer.utils import normalize_url +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def get_confluence_mapping(input_file: str) -> Dict[str, str]: + mapping: Dict[str, str] = {} + + if map_confluence_to_e3sm is None: + print( + "Warning: map_confluence_to_e3sm is not available, Confluence mapping will be skipped." + ) + return mapping + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + mapping[normalize_url(e3sm_url)] = confluence_url + except Exception as exc: + print(f"Could not map {confluence_url}: {exc}") + + return mapping + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" diff --git a/e3sm_comms/exported_xml_reviewer/link_analysis.py b/e3sm_comms/exported_xml_reviewer/link_analysis.py new file mode 100644 index 0000000..fe44b0a --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/link_analysis.py @@ -0,0 +1,314 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import DefaultDict, List, Set, Tuple +from urllib.parse import urlsplit + +import requests # type: ignore + +from e3sm_comms.exported_xml_reviewer.readers import WordpressItem +from e3sm_comms.exported_xml_reviewer.utils import ( + display_status, + is_legacy_content_url, + normalize_status, + normalize_url, +) + + +@dataclass +class InvalidInternalLinkGroup: + linked_url: str + redirect_target: str + redirect_status: str + inferred_link: str + found_under_different_prefix: str + linked_target_status: str + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] + + +@dataclass +class NonPublishedInternalLinkGroup: + linked_url: str + target_status: str + referenced_on_published: List[Tuple[str, str]] + referenced_on_non_published: List[Tuple[str, str]] + + +def extract_internal_e3sm_links(html_text: str) -> Set[str]: + links: Set[str] = set() + + for match in re.finditer( + r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE + ): + href = match.group(1).strip() + if not href: + continue + + try: + parts = urlsplit(href) + except ValueError: + continue + + host = parts.netloc.lower() + path = parts.path.lower() + + if host == "docs.e3sm.org": + continue + + if host.endswith("e3sm.org"): + if path.startswith("/wp-content"): + continue + + normalized = normalize_url(href) + if is_legacy_content_url(normalized): + continue + + links.add(normalized) + elif not parts.scheme and not parts.netloc and href.startswith("/"): + normalized_relative = normalize_url(f"https://e3sm.org{href}") + if "/wp-content" in normalized_relative.lower(): + continue + if is_legacy_content_url(normalized_relative): + continue + + links.add(normalized_relative) + + return links + + +def check_redirect_target(link_url: str) -> Tuple[str, str]: + try: + response = requests.get(link_url, timeout=10, allow_redirects=True) + redirect_status = "" + + if response.history: + first_response = response.history[0] + if first_response.status_code in {301, 302, 303, 307, 308} and response.ok: + final_url = normalize_url(response.url) + redirect_status = str(first_response.status_code) + return final_url, redirect_status + + return "", "" + + except (requests.exceptions.Timeout, requests.exceptions.RequestException): + return "", "" + + +def infer_likely_new_link(linked_url: str) -> str: + parts = urlsplit(linked_url) + path = parts.path.rstrip("/").lower() + + if path.startswith("/model"): + suffix = parts.path[len("/model") :].lstrip("/") + return f"https://e3sm.org/resources/model/{suffix}".rstrip("/") + + if path.startswith("/data"): + suffix = parts.path[len("/data") :].lstrip("/") + return f"https://e3sm.org/resources/data/{suffix}".rstrip("/") + + if path.startswith("/about/news"): + suffix = parts.path[len("/about/news") :].lstrip("/") + return f"https://e3sm.org/news/{suffix}".rstrip("/") + + if path.startswith("/resources/policies"): + suffix = parts.path[len("/resources/policies") :].lstrip("/") + return f"https://e3sm.org/policies/{suffix}".rstrip("/") + + if path.startswith("/resources/tools"): + suffix = parts.path[len("/resources/tools") :].lstrip("/") + return f"https://e3sm.org/tools/{suffix}".rstrip("/") + + return "" + + +def guess_redirect_target(linked_url: str, actual_urls: Set[str]) -> str: + parts = urlsplit(linked_url) + path = parts.path.strip("/").lower() + + if not path: + return "" + + slug = path.split("/")[-1] + candidates = [] + + for actual_url in actual_urls: + actual_parts = urlsplit(actual_url) + actual_path = actual_parts.path.strip("/").lower() + + if actual_path.endswith("/" + slug) or actual_path == slug: + candidates.append(normalize_url(actual_url)) + + if len(candidates) == 1: + return candidates[0] + + return "" + + +def build_invalid_internal_link_groups( + items: List[WordpressItem], +) -> List[InvalidInternalLinkGroup]: + actual_urls = {normalize_url(item.url) for item in items if item.url} + item_by_url = {normalize_url(item.url): item for item in items if item.url} + + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) + + for item in items: + if not item.url or not item.body: + continue + + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + + for linked_url in extract_internal_e3sm_links(item.body): + if linked_url not in actual_urls: + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) + + groups: List[InvalidInternalLinkGroup] = [] + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: + redirect_target, redirect_status = check_redirect_target(linked_url) + + inferred_candidate = infer_likely_new_link(linked_url) + inferred_link = ( + inferred_candidate + if inferred_candidate and normalize_url(inferred_candidate) in actual_urls + else "" + ) + + found_under_different_prefix = "" + if not inferred_link: + guessed_candidate = guess_redirect_target(linked_url, actual_urls) + if guessed_candidate and normalize_url(guessed_candidate) in actual_urls: + found_under_different_prefix = guessed_candidate + + status_target = inferred_link or found_under_different_prefix + linked_target_status = "" + if status_target: + matched_item = item_by_url.get(normalize_url(status_target)) + if matched_item is not None: + linked_target_status = display_status( + normalize_status(matched_item.status) + ) + + groups.append( + InvalidInternalLinkGroup( + linked_url=linked_url, + redirect_target=redirect_target, + redirect_status=redirect_status, + inferred_link=inferred_link, + found_under_different_prefix=found_under_different_prefix, + linked_target_status=linked_target_status, + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + ) + ) + + def status_rank(status: str) -> int: + status = status.lower() + if status == "published": + return 0 + if status == "archived": + return 1 + return 2 + + def inference_rank(group: InvalidInternalLinkGroup) -> int: + if group.inferred_link: + return 0 + if group.found_under_different_prefix: + return 1 + return 2 + + def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, int, str]: + return ( + 0 if group.redirect_target else 1, + inference_rank(group), + status_rank(group.linked_target_status), + group.linked_url.lower(), + ) + + groups.sort(key=sort_key) + return groups + + +def build_non_published_internal_link_groups( + items: List[WordpressItem], +) -> List[NonPublishedInternalLinkGroup]: + item_by_url = {normalize_url(item.url): item for item in items if item.url} + + linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( + set + ) + linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( + defaultdict(set) + ) + + for item in items: + if not item.url or not item.body: + continue + + source_pair = (item.title, item.url) + is_published_source = normalize_status(item.status) == "published" + + for linked_url in extract_internal_e3sm_links(item.body): + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + normalized_target_status = normalize_status(matched_item.status) + if normalized_target_status == "published": + continue + + if is_published_source: + linked_to_sources_published[linked_url].add(source_pair) + else: + linked_to_sources_non_published[linked_url].add(source_pair) + + groups: List[NonPublishedInternalLinkGroup] = [] + all_linked_urls = set(linked_to_sources_published) | set( + linked_to_sources_non_published + ) + + for linked_url in all_linked_urls: + matched_item = item_by_url.get(normalize_url(linked_url)) + if matched_item is None: + continue + + groups.append( + NonPublishedInternalLinkGroup( + linked_url=linked_url, + target_status=display_status(normalize_status(matched_item.status)), + referenced_on_published=sorted( + linked_to_sources_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + referenced_on_non_published=sorted( + linked_to_sources_non_published.get(linked_url, set()), + key=lambda x: x[0].lower(), + ), + ) + ) + + groups.sort( + key=lambda group: (group.target_status.lower(), group.linked_url.lower()) + ) + return groups diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index d0ec83b..a87b3db 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -1,17 +1,21 @@ from __future__ import annotations -import csv -import re -import xml.etree.ElementTree as ET -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path -from typing import DefaultDict, Dict, List, Optional, Set, Tuple -from urllib.parse import urlsplit, urlunsplit - -import requests # type: ignore - -from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.exported_xml_reviewer.builders import ( + build_navigation_issue_records, + build_published_content_link_summaries, + build_records, +) +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + build_invalid_internal_link_groups, + build_non_published_internal_link_groups, +) +from e3sm_comms.exported_xml_reviewer.reporters import ( + write_hierarchical_outline, + write_invalid_internal_links_report, + write_markdown_report, + write_navigation_issues_report, + write_published_pages_link_report, +) from e3sm_comms.utils import IO_DIR # ----------------------------------------------------------------------------- @@ -52,1501 +56,6 @@ f"{IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" ) -CONFLUENCE_SPACE = "EPWCD" -CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" - - -@dataclass -class WordpressItem: - post_id: str - post_parent: str - post_type: str - title: str - url: str - status: str - body: str - - -@dataclass -class ReportRecord: - title: str - e3sm_url: str - status: str - sensitive_terms: Dict[str, int] - confluence_draft_url: Optional[str] - - -@dataclass -class RequestedLinkRecord: - e3sm_url: str - included_later: bool - current_status: str - currently_whitelisted: bool - requesting_urls: str - - -@dataclass -class TopLevelPageIssue: - title: str - url: str - status: str - - -@dataclass -class ArchivedParentPublishedChildIssue: - parent_title: str - parent_url: str - parent_status: str - child_title: str - child_url: str - child_status: str - - -@dataclass -class InvalidInternalLinkGroup: - linked_url: str - redirect_target: str - redirect_status: str - inferred_link: str - found_under_different_prefix: str - linked_target_status: str - referenced_on_published: List[Tuple[str, str]] - referenced_on_non_published: List[Tuple[str, str]] - - -@dataclass -class NonPublishedInternalLinkGroup: - linked_url: str - target_status: str - referenced_on_published: List[Tuple[str, str]] - referenced_on_non_published: List[Tuple[str, str]] - - -@dataclass -class PublishedContentLinkSummary: - title: str - url: str - archived_links: List[str] - redirected_links: List[str] - broken_links: List[str] - valid_links: List[str] - - -def normalize_url(url: str) -> str: - url = url.strip() - if not url: - return "" - - parts = urlsplit(url) - scheme = parts.scheme.lower() - netloc = parts.netloc.lower() - path = parts.path.rstrip("/") - - normalized = urlunsplit((scheme, netloc, path, "", "")) - return normalized - - -def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: - return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" - - -def normalize_status(raw_status: Optional[str]) -> str: - if not raw_status: - return "unknown" - - mapping = { - "publish": "published", - "archive": "archived", - "draft": "draft", - "future": "future", - "pending": "pending", - "private": "private", - } - return mapping.get(raw_status.strip().lower(), raw_status.strip().lower()) - - -def display_status(status: str) -> str: - mapping = { - "published": "Published", - "archived": "Archived", - "draft": "Draft", - "future": "Future", - "pending": "Pending", - "private": "Private", - "unknown": "Unknown", - } - return mapping.get(status, status.title()) - - -def strip_html(text: str) -> str: - text = re.sub(r"<[^>]+>", " ", text) - text = re.sub(r" ", " ", text) - text = re.sub(r"\s+", " ", text) - return text.strip() - - -def read_sensitive_terms(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - terms = [line.strip().lower() for line in f if line.strip()] - return sorted(set(terms)) - - -def read_whitelist_patterns(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - return [line.strip() for line in f if line.strip()] - - -def read_known_ok_links(file_path: str) -> Set[str]: - with open(file_path, "r", encoding="utf-8") as f: - return {normalize_url(line.strip()) for line in f if line.strip()} - - -def read_requested_links(file_path: str) -> List[Tuple[str, str]]: - rows: List[Tuple[str, str]] = [] - - with open(file_path, "r", encoding="utf-8", newline="") as f: - reader = csv.DictReader(f) - - if not reader.fieldnames: - print(f"Requested links CSV has no headers: {file_path}") - return rows - - normalized_to_actual = { - header.strip().lower(): header for header in reader.fieldnames if header - } - - e3sm_header = normalized_to_actual.get("e3sm.org link") - requesting_header = normalized_to_actual.get( - "list of urls that wants to link to it" - ) - - if requesting_header is None: - for candidate in [ - "list of urls that want to link to it", - "requesting urls", - "requesting url", - "list of urls", - ]: - requesting_header = normalized_to_actual.get(candidate) - if requesting_header: - break - - if e3sm_header is None: - print( - f"Requested links CSV is missing required column 'e3sm.org link'. " - f"Found headers: {reader.fieldnames}" - ) - return rows - - if requesting_header is None: - print( - "Requested links CSV could not find the requesting URLs column. " - f"Found headers: {reader.fieldnames}" - ) - - for row in reader: - e3sm_url = normalize_url(row.get(e3sm_header, "")) - requesting_urls = ( - row.get(requesting_header, "").strip() if requesting_header else "" - ) - - if e3sm_url: - rows.append((e3sm_url, requesting_urls)) - - return rows - - -def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: - counts: Dict[str, int] = {} - lowered = text.lower() - - for term in terms: - escaped = re.escape(term) - pattern = rf"\b{escaped}\b" - matches = re.findall(pattern, lowered) - if matches: - counts[term] = len(matches) - - return counts - - -def matches_pattern(pattern: str, url: str) -> bool: - if "*" not in pattern: - return normalize_url(pattern) == normalize_url(url) - - normalized_url = normalize_url(url) - normalized_pattern = normalize_url(pattern) - - if "*" not in normalized_pattern: - return normalized_pattern == normalized_url - - if normalized_pattern.count("*") == 1 and normalized_pattern.endswith("*"): - prefix = normalized_pattern[:-1] - return normalized_url.startswith(prefix) - - parts = normalized_pattern.split("*") - position = 0 - for i, part in enumerate(parts): - if not part: - continue - found_at = normalized_url.find(part, position) - if found_at == -1: - return False - if i == 0 and not normalized_pattern.startswith("*") and found_at != 0: - return False - position = found_at + len(part) - - if ( - not normalized_pattern.endswith("*") - and parts[-1] - and not normalized_url.endswith(parts[-1]) - ): - return False - - return True - - -def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> Set[str]: - matched_urls: Set[str] = set() - for pattern in patterns: - for url in all_urls: - if matches_pattern(pattern, url): - matched_urls.add(url) - return matched_urls - - -def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: - parsed: List[Tuple[str, str]] = [] - - with open(input_file, "r", encoding="utf-8") as f: - for line_number, raw_line in enumerate(f, start=1): - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - stripped = line.lstrip() - if ":" not in stripped: - print(f"Skipping malformed Confluence line {line_number}: {line}") - continue - - page_id, title = stripped.split(":", 1) - page_id = page_id.strip() - title = title.strip() - - if not page_id.isdigit(): - print( - f"Skipping Confluence line {line_number} with non-numeric page id: {line}" - ) - continue - - parsed.append((page_id, title)) - - return parsed - - -def get_confluence_mapping(input_file: str) -> Dict[str, str]: - mapping: Dict[str, str] = {} - - if map_confluence_to_e3sm is None: - print( - "Warning: map_confluence_to_e3sm is not available, Confluence mapping will be skipped." - ) - return mapping - - for page_id, title in parse_confluence_hierarchy_file(input_file): - confluence_url = build_confluence_url(page_id) - try: - e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) - if e3sm_url: - mapping[normalize_url(e3sm_url)] = confluence_url - except Exception as exc: - print(f"Could not map {confluence_url}: {exc}") - - return mapping - - -def extract_item_body(item: ET.Element) -> str: - ns = { - "wp": "http://wordpress.org/export/1.2/", - "content": "http://purl.org/rss/1.0/modules/content/", - "excerpt": "http://wordpress.org/export/1.2/excerpt/", - } - - body_parts: List[str] = [] - - content_elem = item.find("content:encoded", ns) - if content_elem is not None and content_elem.text and content_elem.text.strip(): - body_parts.append(content_elem.text.strip()) - - excerpt_elem = item.find("excerpt:encoded", ns) - if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): - body_parts.append(excerpt_elem.text.strip()) - - for postmeta in item.findall("wp:postmeta", ns): - meta_key_elem = postmeta.find("wp:meta_key", ns) - meta_value_elem = postmeta.find("wp:meta_value", ns) - - meta_key = ( - meta_key_elem.text.strip() - if meta_key_elem is not None and meta_key_elem.text - else "" - ) - meta_value = ( - meta_value_elem.text.strip() - if meta_value_elem is not None and meta_value_elem.text - else "" - ) - - if not meta_value: - continue - - if meta_key.endswith("_free_form_content") and not meta_key.startswith("_"): - body_parts.append(meta_value) - - return "\n".join(body_parts) - - -def parse_wordpress_xml( - xml_file_path: str, expected_post_type: str -) -> List[WordpressItem]: - ns = { - "wp": "http://wordpress.org/export/1.2/", - } - - tree = ET.parse(xml_file_path) - root = tree.getroot() - - channel = root.find("channel") - if channel is None: - return [] - - items: List[WordpressItem] = [] - - for item in channel.findall("item"): - post_type_elem = item.find("wp:post_type", ns) - if post_type_elem is None: - continue - - post_type_text = (post_type_elem.text or "").strip() - if post_type_text != expected_post_type: - continue - - title_elem = item.find("title") - link_elem = item.find("link") - status_elem = item.find("wp:status", ns) - post_id_elem = item.find("wp:post_id", ns) - post_parent_elem = item.find("wp:post_parent", ns) - - title = ( - title_elem.text.strip() - if title_elem is not None and title_elem.text is not None - else "Untitled" - ) - link = ( - normalize_url(link_elem.text) - if link_elem is not None and link_elem.text is not None - else "" - ) - status = ( - status_elem.text.strip() - if status_elem is not None and status_elem.text is not None - else "unknown" - ) - post_id = ( - post_id_elem.text.strip() - if post_id_elem is not None and post_id_elem.text is not None - else "" - ) - post_parent = ( - post_parent_elem.text.strip() - if post_parent_elem is not None and post_parent_elem.text is not None - else "0" - ) - body = extract_item_body(item) - - items.append( - WordpressItem( - post_id=post_id, - post_parent=post_parent, - post_type=post_type_text, - title=title, - url=link, - status=status, - body=body, - ) - ) - - return items - - -def is_legacy_content_url(url: str) -> bool: - parts = urlsplit(url) - slug = parts.path.strip("/").lower() - return bool(re.fullmatch(r"\d{6,8}[_-].+", slug)) - - -def extract_internal_e3sm_links(html_text: str) -> Set[str]: - links: Set[str] = set() - - for match in re.finditer( - r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE - ): - href = match.group(1).strip() - if not href: - continue - - try: - parts = urlsplit(href) - except ValueError: - continue - - host = parts.netloc.lower() - path = parts.path.lower() - - if host == "docs.e3sm.org": - continue - - if host.endswith("e3sm.org"): - if path.startswith("/wp-content"): - continue - - normalized = normalize_url(href) - if is_legacy_content_url(normalized): - continue - - links.add(normalized) - elif not parts.scheme and not parts.netloc and href.startswith("/"): - normalized_relative = normalize_url(f"https://e3sm.org{href}") - if "/wp-content" in normalized_relative.lower(): - continue - if is_legacy_content_url(normalized_relative): - continue - - links.add(normalized_relative) - - return links - - -def infer_likely_new_link(linked_url: str) -> str: - parts = urlsplit(linked_url) - path = parts.path.rstrip("/").lower() - - if path.startswith("/model"): - suffix = parts.path[len("/model") :].lstrip("/") - return f"https://e3sm.org/resources/model/{suffix}".rstrip("/") - - if path.startswith("/data"): - suffix = parts.path[len("/data") :].lstrip("/") - return f"https://e3sm.org/resources/data/{suffix}".rstrip("/") - - if path.startswith("/about/news"): - suffix = parts.path[len("/about/news") :].lstrip("/") - return f"https://e3sm.org/news/{suffix}".rstrip("/") - - if path.startswith("/resources/policies"): - suffix = parts.path[len("/resources/policies") :].lstrip("/") - return f"https://e3sm.org/policies/{suffix}".rstrip("/") - - if path.startswith("/resources/tools"): - suffix = parts.path[len("/resources/tools") :].lstrip("/") - return f"https://e3sm.org/tools/{suffix}".rstrip("/") - - return "" - - -def guess_redirect_target(linked_url: str, actual_urls: Set[str]) -> str: - parts = urlsplit(linked_url) - path = parts.path.strip("/").lower() - - if not path: - return "" - - slug = path.split("/")[-1] - candidates = [] - - for actual_url in actual_urls: - actual_parts = urlsplit(actual_url) - actual_path = actual_parts.path.strip("/").lower() - - if actual_path.endswith("/" + slug) or actual_path == slug: - candidates.append(normalize_url(actual_url)) - - if len(candidates) == 1: - return candidates[0] - - return "" - - -def check_redirect_target(link_url: str) -> Tuple[str, str]: - try: - response = requests.get(link_url, timeout=10, allow_redirects=True) - redirect_status = "" - - if response.history: - first_response = response.history[0] - if first_response.status_code in {301, 302, 303, 307, 308} and response.ok: - final_url = normalize_url(response.url) - redirect_status = str(first_response.status_code) - return final_url, redirect_status - - return "", "" - - except (requests.exceptions.Timeout, requests.exceptions.RequestException): - return "", "" - - -def get_page_hierarchy_order(items: List[WordpressItem]) -> List[WordpressItem]: - pages = [item for item in items if item.post_type == "page" and item.post_id] - item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} - - children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) - for item in pages: - parent_id = ( - item.post_parent if item.post_parent and item.post_parent != "0" else "" - ) - children_by_parent[parent_id].append(item) - - for child_list in children_by_parent.values(): - child_list.sort(key=lambda x: x.title.lower()) - - roots = [ - item - for item in pages - if not item.post_parent - or item.post_parent == "0" - or item.post_parent not in item_by_id - ] - roots.sort(key=lambda x: x.title.lower()) - - ordered: List[WordpressItem] = [] - seen: Set[str] = set() - - def walk(node: WordpressItem) -> None: - if node.post_id in seen: - return - seen.add(node.post_id) - ordered.append(node) - for child in children_by_parent.get(node.post_id, []): - walk(child) - - for root in roots: - walk(root) - - return ordered - - -def build_published_content_link_summaries( - items: List[WordpressItem], - post_type: str, -) -> List[PublishedContentLinkSummary]: - item_by_url = {normalize_url(item.url): item for item in items if item.url} - actual_urls = set(item_by_url.keys()) - - if post_type == "page": - ordered_items = get_page_hierarchy_order(items) - else: - ordered_items = sorted( - [item for item in items if item.post_type == post_type and item.post_id], - key=lambda x: x.title.lower(), - ) - - summaries: List[PublishedContentLinkSummary] = [] - - for item in ordered_items: - if normalize_status(item.status) != "published": - continue - if not item.url or not item.body: - continue - - archived_links: Set[str] = set() - redirected_links: Set[str] = set() - broken_links: Set[str] = set() - valid_links: Set[str] = set() - - for linked_url in extract_internal_e3sm_links(item.body): - linked_norm = normalize_url(linked_url) - target_item = item_by_url.get(linked_norm) - - if target_item is not None: - target_status = normalize_status(target_item.status) - if target_status == "archived": - archived_links.add(linked_norm) - else: - valid_links.add(linked_norm) - continue - - redirect_target, _redirect_status = check_redirect_target(linked_norm) - if redirect_target and normalize_url(redirect_target) in actual_urls: - redirected_links.add(linked_norm) - valid_links.add(linked_norm) - else: - broken_links.add(linked_norm) - - summaries.append( - PublishedContentLinkSummary( - title=item.title, - url=item.url, - archived_links=sorted(archived_links), - redirected_links=sorted(redirected_links), - broken_links=sorted(broken_links), - valid_links=sorted(valid_links), - ) - ) - - return summaries - - -def build_invalid_internal_link_groups( - items: List[WordpressItem], -) -> List[InvalidInternalLinkGroup]: - actual_urls = {normalize_url(item.url) for item in items if item.url} - item_by_url = {normalize_url(item.url): item for item in items if item.url} - - linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( - set - ) - linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( - defaultdict(set) - ) - - for item in items: - if not item.url or not item.body: - continue - - source_pair = (item.title, item.url) - is_published_source = normalize_status(item.status) == "published" - - for linked_url in extract_internal_e3sm_links(item.body): - if linked_url not in actual_urls: - if is_published_source: - linked_to_sources_published[linked_url].add(source_pair) - else: - linked_to_sources_non_published[linked_url].add(source_pair) - - groups: List[InvalidInternalLinkGroup] = [] - all_linked_urls = set(linked_to_sources_published) | set( - linked_to_sources_non_published - ) - - for linked_url in all_linked_urls: - redirect_target, redirect_status = check_redirect_target(linked_url) - - inferred_candidate = infer_likely_new_link(linked_url) - inferred_link = ( - inferred_candidate - if inferred_candidate and normalize_url(inferred_candidate) in actual_urls - else "" - ) - - found_under_different_prefix = "" - if not inferred_link: - guessed_candidate = guess_redirect_target(linked_url, actual_urls) - if guessed_candidate and normalize_url(guessed_candidate) in actual_urls: - found_under_different_prefix = guessed_candidate - - status_target = inferred_link or found_under_different_prefix - linked_target_status = "" - if status_target: - matched_item = item_by_url.get(normalize_url(status_target)) - if matched_item is not None: - linked_target_status = display_status( - normalize_status(matched_item.status) - ) - - groups.append( - InvalidInternalLinkGroup( - linked_url=linked_url, - redirect_target=redirect_target, - redirect_status=redirect_status, - inferred_link=inferred_link, - found_under_different_prefix=found_under_different_prefix, - linked_target_status=linked_target_status, - referenced_on_published=sorted( - linked_to_sources_published.get(linked_url, set()), - key=lambda x: x[0].lower(), - ), - referenced_on_non_published=sorted( - linked_to_sources_non_published.get(linked_url, set()), - key=lambda x: x[0].lower(), - ), - ) - ) - - def status_rank(status: str) -> int: - status = status.lower() - if status == "published": - return 0 - if status == "archived": - return 1 - return 2 - - def inference_rank(group: InvalidInternalLinkGroup) -> int: - if group.inferred_link: - return 0 - if group.found_under_different_prefix: - return 1 - return 2 - - def sort_key(group: InvalidInternalLinkGroup) -> Tuple[int, int, int, str]: - return ( - 0 if group.redirect_target else 1, - inference_rank(group), - status_rank(group.linked_target_status), - group.linked_url.lower(), - ) - - groups.sort(key=sort_key) - return groups - - -def build_non_published_internal_link_groups( - items: List[WordpressItem], -) -> List[NonPublishedInternalLinkGroup]: - item_by_url = {normalize_url(item.url): item for item in items if item.url} - - linked_to_sources_published: DefaultDict[str, Set[Tuple[str, str]]] = defaultdict( - set - ) - linked_to_sources_non_published: DefaultDict[str, Set[Tuple[str, str]]] = ( - defaultdict(set) - ) - - for item in items: - if not item.url or not item.body: - continue - - source_pair = (item.title, item.url) - is_published_source = normalize_status(item.status) == "published" - - for linked_url in extract_internal_e3sm_links(item.body): - matched_item = item_by_url.get(normalize_url(linked_url)) - if matched_item is None: - continue - - normalized_target_status = normalize_status(matched_item.status) - if normalized_target_status == "published": - continue - - if is_published_source: - linked_to_sources_published[linked_url].add(source_pair) - else: - linked_to_sources_non_published[linked_url].add(source_pair) - - groups: List[NonPublishedInternalLinkGroup] = [] - all_linked_urls = set(linked_to_sources_published) | set( - linked_to_sources_non_published - ) - - for linked_url in all_linked_urls: - matched_item = item_by_url.get(normalize_url(linked_url)) - if matched_item is None: - continue - - groups.append( - NonPublishedInternalLinkGroup( - linked_url=linked_url, - target_status=display_status(normalize_status(matched_item.status)), - referenced_on_published=sorted( - linked_to_sources_published.get(linked_url, set()), - key=lambda x: x[0].lower(), - ), - referenced_on_non_published=sorted( - linked_to_sources_non_published.get(linked_url, set()), - key=lambda x: x[0].lower(), - ), - ) - ) - - groups.sort( - key=lambda group: (group.target_status.lower(), group.linked_url.lower()) - ) - return groups - - -def build_requested_link_records( - requested_links_file: str, - raw_items: List[WordpressItem], - whitelisted_urls: Set[str], - flagged_urls: Set[str], -) -> List[RequestedLinkRecord]: - requested_rows = read_requested_links(requested_links_file) - item_by_url = {item.url: item for item in raw_items if item.url} - - records: List[RequestedLinkRecord] = [] - for e3sm_url, requesting_urls in requested_rows: - item = item_by_url.get(e3sm_url) - - if item is None: - current_status = "Not found" - currently_whitelisted = False - else: - current_status = display_status(normalize_status(item.status)) - currently_whitelisted = e3sm_url in whitelisted_urls - - records.append( - RequestedLinkRecord( - e3sm_url=e3sm_url, - included_later=e3sm_url in flagged_urls, - current_status=current_status, - currently_whitelisted=currently_whitelisted, - requesting_urls=requesting_urls, - ) - ) - - return records - - -def build_records( - xml_pages: str, - xml_posts: str, - confluence_hierarchy: str, - sensitive_terms_file: str, - whitelist_file: str, - requested_links_file: str, - known_ok_links_file: str, -) -> Tuple[ - List[ReportRecord], - Dict[str, int], - List[RequestedLinkRecord], - List[WordpressItem], -]: - sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) - - confluence_map = {} - if confluence_hierarchy: - confluence_map = get_confluence_mapping(confluence_hierarchy) - - known_ok_urls = read_known_ok_links(known_ok_links_file) - - raw_items: List[WordpressItem] = [] - raw_items.extend(parse_wordpress_xml(xml_pages, "page")) - raw_items.extend(parse_wordpress_xml(xml_posts, "post")) - - all_urls = [item.url for item in raw_items if item.url] - if whitelist_file: - whitelist_patterns = read_whitelist_patterns(whitelist_file) - whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) - else: - whitelisted_urls = set(all_urls) - - records: List[ReportRecord] = [] - status_totals: DefaultDict[str, int] = defaultdict(int) - - for item in raw_items: - base_status = normalize_status(item.status) - report_status = base_status - - if base_status == "published": - if item.url in whitelisted_urls: - if item.url in known_ok_urls: - report_status = "published & whitelisted, known ok" - else: - report_status = "published & whitelisted" - else: - report_status = "published & not whitelisted" - - status_totals[report_status] += 1 - - plain_text = strip_html(item.body) - term_counts = count_sensitive_terms(plain_text, sensitive_terms_list) - - if not term_counts: - continue - - records.append( - ReportRecord( - title=item.title, - e3sm_url=item.url, - status=report_status, - sensitive_terms=term_counts, - confluence_draft_url=confluence_map.get(item.url), - ) - ) - - flagged_urls = {record.e3sm_url for record in records} - requested_link_records = build_requested_link_records( - requested_links_file=requested_links_file, - raw_items=raw_items, - whitelisted_urls=whitelisted_urls, - flagged_urls=flagged_urls, - ) - - return records, dict(status_totals), requested_link_records, raw_items - - -def build_navigation_issue_records( - items: List[WordpressItem], -) -> Tuple[List[TopLevelPageIssue], List[ArchivedParentPublishedChildIssue]]: - allowed_top_level_titles = { - "about", - "news", - "resources", - "tools", - "policies", - "home page", - } - - pages = [item for item in items if item.post_type == "page" and item.post_id] - item_by_id: Dict[str, WordpressItem] = {item.post_id: item for item in pages} - - top_level_issues: List[TopLevelPageIssue] = [] - archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue] = [] - - for page in pages: - normalized_status = normalize_status(page.status) - - is_top_level = ( - not page.post_parent - or page.post_parent == "0" - or page.post_parent not in item_by_id - ) - - if is_top_level and page.title.strip().lower() not in allowed_top_level_titles: - top_level_issues.append( - TopLevelPageIssue( - title=page.title, - url=page.url, - status=display_status(normalized_status), - ) - ) - - if ( - page.post_parent - and page.post_parent != "0" - and page.post_parent in item_by_id - and normalized_status == "published" - ): - parent = item_by_id[page.post_parent] - parent_status = normalize_status(parent.status) - - if parent_status == "archived": - archived_parent_published_child_issues.append( - ArchivedParentPublishedChildIssue( - parent_title=parent.title, - parent_url=parent.url, - parent_status=display_status(parent_status), - child_title=page.title, - child_url=page.url, - child_status=display_status(normalized_status), - ) - ) - - status_order = { - "Published": 0, - "Draft": 1, - "Pending": 2, - "Future": 3, - "Private": 4, - "Archived": 5, - "Unknown": 6, - } - - top_level_issues.sort( - key=lambda x: ( - status_order.get(x.status, 99), - x.title.lower(), - x.url.lower(), - ) - ) - - archived_parent_published_child_issues.sort( - key=lambda x: ( - x.parent_title.lower(), - x.child_title.lower(), - x.child_url.lower(), - ) - ) - - return top_level_issues, archived_parent_published_child_issues - - -def sort_requested_link_records( - requested_link_records: List[RequestedLinkRecord], -) -> List[RequestedLinkRecord]: - status_order = { - "Published": 0, - "Archived": 1, - "Not found": 3, - } - - return sorted( - requested_link_records, - key=lambda r: ( - 0 if r.included_later else 1, - status_order.get(r.current_status, 2), - 0 if r.currently_whitelisted else 1, - r.e3sm_url.lower(), - ), - ) - - -def write_markdown_report( - output_path: str, - records: List[ReportRecord], - status_totals: Dict[str, int], - requested_link_records: List[RequestedLinkRecord], -) -> None: - grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) - for record in records: - grouped[record.status].append(record) - - for status in grouped: - grouped[status].sort( - key=lambda r: (-sum(r.sensitive_terms.values()), r.title.lower()) - ) - - ordered_statuses = [ - "published & whitelisted, known ok", - "published & whitelisted", - "published & not whitelisted", - "archived", - "draft", - "future", - "pending", - "private", - "unknown", - ] - - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, "w", encoding="utf-8") as f: - f.write("# WordPress Sensitive Terms Report\n\n") - f.write( - "The detailed sections below include only e3sm.org pages/posts where one or more sensitive terms were found. " - "The summary table includes counts for both flagged and unflagged items.\n\n" - ) - - f.write("| Status | With sensitive terms | Without sensitive terms | Total |\n") - f.write("| --- | ---: | ---: | ---: |\n") - - total_with_terms = 0 - total_without_terms = 0 - - all_summary_statuses = set(status_totals) | set(grouped) - extra_statuses = sorted( - s for s in all_summary_statuses if s not in ordered_statuses - ) - - for status in ordered_statuses + extra_statuses: - total_in_status = status_totals.get(status, 0) - with_terms = len(grouped.get(status, [])) - without_terms = total_in_status - with_terms - - if total_in_status == 0 and with_terms == 0: - continue - - total_with_terms += with_terms - total_without_terms += without_terms - f.write( - f"| {status} | {with_terms} | {without_terms} | {total_in_status} |\n" - ) - - grand_total = total_with_terms + total_without_terms - f.write( - f"| TOTAL | {total_with_terms} | {total_without_terms} | {grand_total} |\n" - ) - f.write("\n") - - if requested_link_records: - requested_link_records = sort_requested_link_records(requested_link_records) - - f.write("## Requested Links\n\n") - f.write( - "| e3sm.org link | Included later on this page? | Current status | Currently whitelisted? | Requesting URLs |\n" - ) - f.write("| --- | --- | --- | --- | --- |\n") - - for requested_record in requested_link_records: - included_later = ( - "Yes" - if requested_record.included_later - else "No (i.e., contains no sensitive terms)" - ) - currently_whitelisted = ( - "Yes" if requested_record.currently_whitelisted else "No" - ) - f.write( - f"| {requested_record.e3sm_url} | {included_later} | {requested_record.current_status} | " - f"{currently_whitelisted} | {requested_record.requesting_urls} |\n" - ) - - f.write("\n") - - all_statuses = ordered_statuses + extra_statuses - seen = set() - - for status in all_statuses: - if status not in grouped or status in seen: - continue - seen.add(status) - - f.write(f"## {status.capitalize()} ({len(grouped[status])})\n\n") - - for idx, record in enumerate(grouped[status], start=1): - e3sm_md = f"[e3sm.org]({record.e3sm_url})" - confluence_md = ( - f" [(confluence draft)]({record.confluence_draft_url})" - if record.confluence_draft_url - else "" - ) - - f.write( - f"{idx}. {record.title}: {e3sm_md}{confluence_md} -- {record.sensitive_terms}\n" - ) - - f.write("\n") - - -def write_invalid_internal_links_report( - output_path: str, - groups: List[InvalidInternalLinkGroup], - non_published_groups: List[NonPublishedInternalLinkGroup], -) -> None: - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: - f.write( - "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on these published pages | Referenced on these non-published pages |\n" - ) - f.write("| --- | --- | --- | --- | --- | --- | --- |\n") - - for group in table_groups: - redirect_md = ( - f"[{group.redirect_target}]({group.redirect_target})" - if group.redirect_target - else "" - ) - if redirect_md and group.redirect_status: - redirect_md = ( - f"{redirect_md} (redirect status: {group.redirect_status})" - ) - - inferred_md = ( - f"[{group.inferred_link}]({group.inferred_link})" - if group.inferred_link - else "" - ) - prefix_md = ( - f"[{group.found_under_different_prefix}]({group.found_under_different_prefix})" - if group.found_under_different_prefix - else "" - ) - - referenced_published = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on_published - ) - referenced_non_published = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on_non_published - ) - - f.write( - f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_published} | {referenced_non_published} |\n" - ) - - def render_non_published_table( - f, table_groups: List[NonPublishedInternalLinkGroup] - ) -> None: - f.write( - "| Valid linked URL | Target status | Referenced on these published pages | Referenced on these non-published pages |\n" - ) - f.write("| --- | --- | --- | --- |\n") - - for group in table_groups: - referenced_published = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on_published - ) - referenced_non_published = ", ".join( - f"[{title}]({url})" for title, url in group.referenced_on_non_published - ) - f.write( - f"| {group.linked_url} | {group.target_status} | {referenced_published} | {referenced_non_published} |\n" - ) - - working_redirects: List[InvalidInternalLinkGroup] = [] - published_targets: List[InvalidInternalLinkGroup] = [] - archived_targets: List[InvalidInternalLinkGroup] = [] - no_candidate: List[InvalidInternalLinkGroup] = [] - - for group in groups: - if group.redirect_target: - working_redirects.append(group) - elif group.linked_target_status == "Published": - published_targets.append(group) - elif group.linked_target_status == "Archived": - archived_targets.append(group) - else: - no_candidate.append(group) - - with open(output_file, "w", encoding="utf-8") as f: - f.write("# Invalid Internal e3sm.org Links\n\n") - - if not groups and not non_published_groups: - f.write("No invalid internal links found.\n") - return - - f.write( - f"## 1. These have working redirections already ({len(working_redirects)})\n\n" - ) - if working_redirects: - render_table(f, working_redirects) - else: - f.write("None found.\n") - f.write("\n") - - f.write( - f"## 2. The target pages are published, we just need to set up the redirections ({len(published_targets)})\n\n" - ) - if published_targets: - render_table(f, published_targets) - else: - f.write("None found.\n") - f.write("\n") - - f.write(f"## 3. The target pages are archived ({len(archived_targets)})\n\n") - if archived_targets: - render_table(f, archived_targets) - else: - f.write("None found.\n") - f.write("\n") - - f.write( - f"## 4. Couldn't find a redirection candidate ({len(no_candidate)})\n\n" - ) - if no_candidate: - render_table(f, no_candidate) - else: - f.write("None found.\n") - f.write("\n") - - f.write( - f"## 5. Technically valid links that point to non-published targets ({len(non_published_groups)})\n\n" - ) - if non_published_groups: - render_non_published_table(f, non_published_groups) - else: - f.write("None found.\n") - - -def write_published_pages_link_report( - output_path: str, - page_summaries: List[PublishedContentLinkSummary], - post_summaries: List[PublishedContentLinkSummary], -) -> None: - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - def render_link_list(urls: List[str]) -> str: - return ", ".join(f"[{url}]({url})" for url in urls) - - def write_section( - f, - section_title: str, - summaries: List[PublishedContentLinkSummary], - ) -> None: - invalid_summaries = [ - s - for s in summaries - if s.archived_links or s.redirected_links or s.broken_links - ] - valid_only_summaries = [ - s - for s in summaries - if not (s.archived_links or s.redirected_links or s.broken_links) - ] - - f.write(f"## {section_title}\n\n") - - if not summaries: - f.write("No published items with links found.\n\n") - return - - f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") - if invalid_summaries: - f.write( - "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" - ) - f.write("| --- | --- | --- | --- | --- |\n") - - archived_total = 0 - redirected_total = 0 - broken_total = 0 - valid_total = 0 - - archived_unique: Set[str] = set() - redirected_unique: Set[str] = set() - broken_unique: Set[str] = set() - valid_unique: Set[str] = set() - - for summary in invalid_summaries: - item_md = f"[{summary.title}]({summary.url})" - archived_md = render_link_list(summary.archived_links) - redirected_md = render_link_list(summary.redirected_links) - broken_md = render_link_list(summary.broken_links) - valid_md = render_link_list(summary.valid_links) - - archived_total += len(summary.archived_links) - redirected_total += len(summary.redirected_links) - broken_total += len(summary.broken_links) - valid_total += len(summary.valid_links) - - archived_unique.update(summary.archived_links) - redirected_unique.update(summary.redirected_links) - broken_unique.update(summary.broken_links) - valid_unique.update(summary.valid_links) - - f.write( - f"| {item_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" - ) - - f.write( - f"| Total link count | {archived_total} | {redirected_total} | {broken_total} | {valid_total} |\n" - ) - f.write( - f"| Unique link count | {len(archived_unique)} | {len(redirected_unique)} | {len(broken_unique)} | {len(valid_unique)} |\n" - ) - else: - f.write("No items with invalid links found.\n") - - f.write("\n") - f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") - - if valid_only_summaries: - f.write("| Published item | valid e3sm.org link count |\n") - f.write("| --- | ---: |\n") - - total_links = 0 - unique_links: Set[str] = set() - - for summary in valid_only_summaries: - item_md = f"[{summary.title}]({summary.url})" - f.write(f"| {item_md} | {len(summary.valid_links)} |\n") - total_links += len(summary.valid_links) - unique_links.update(summary.valid_links) - - f.write(f"| Total link count | {total_links} |\n") - f.write(f"| Unique link count | {len(unique_links)} |\n") - else: - f.write("No items with only valid links found.\n") - - f.write("\n") - - with open(output_file, "w", encoding="utf-8") as f: - f.write("# Published Content Invalid Link Report\n\n") - write_section(f, "Published Pages", page_summaries) - write_section(f, "Published Posts", post_summaries) - - -def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - def write_section(f, section_items: List[WordpressItem], heading: str) -> None: - section_items = [item for item in section_items if item.post_id] - - children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) - item_by_id: Dict[str, WordpressItem] = {} - - for item in section_items: - item_by_id[item.post_id] = item - - for item in section_items: - parent_id = ( - item.post_parent if item.post_parent and item.post_parent != "0" else "" - ) - children_by_parent[parent_id].append(item) - - for child_list in children_by_parent.values(): - child_list.sort(key=lambda x: x.title.lower()) - - roots = [ - item - for item in section_items - if not item.post_parent - or item.post_parent == "0" - or item.post_parent not in item_by_id - ] - roots.sort(key=lambda x: x.title.lower()) - - f.write(f"{heading}\n") - - seen: Set[str] = set() - - def walk(node: WordpressItem, depth: int) -> None: - indent = " " * depth - status_label = display_status(normalize_status(node.status)) - line = f"{indent}{node.title} [{status_label}]" - if node.url: - line += f" [{node.url}]" - f.write(line + "\n") - - if node.post_id in seen: - return - - seen.add(node.post_id) - - for child in children_by_parent.get(node.post_id, []): - walk(child, depth + 1) - - for root in roots: - walk(root, 0) - - f.write("\n") - - pages = [item for item in items if item.post_type == "page"] - posts = [item for item in items if item.post_type == "post"] - - with open(output_path, "w", encoding="utf-8") as f: - write_section(f, pages, "Pages") - write_section(f, posts, "Posts") - - -def write_navigation_issues_report( - output_path: str, - top_level_issues: List[TopLevelPageIssue], - archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue], -) -> None: - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, "w", encoding="utf-8") as f: - f.write("# WordPress Navigation Issues Report\n\n") - - f.write("## 1. Top-level pages that are not expected top-level tabs\n\n") - f.write( - "Expected top-level tabs are: About, News, Resources, Tools, Policies, Home Page.\n\n" - ) - - if top_level_issues: - f.write("| Title | Status | URL |\n") - f.write("| --- | --- | --- |\n") - for top_level_issue in top_level_issues: - f.write( - f"| {top_level_issue.title} | {top_level_issue.status} | {top_level_issue.url} |\n" - ) - else: - f.write("No unexpected top-level pages found.\n") - - f.write("\n") - - f.write("## 2. Published child pages under archived parent pages\n\n") - - if archived_parent_published_child_issues: - f.write( - "| Parent title | Parent status | Parent URL | Child title | Child status | Child URL |\n" - ) - f.write("| --- | --- | --- | --- | --- | --- |\n") - for archived_child_issue in archived_parent_published_child_issues: - f.write( - f"| {archived_child_issue.parent_title} | {archived_child_issue.parent_status} | {archived_child_issue.parent_url} | " - f"{archived_child_issue.child_title} | {archived_child_issue.child_status} | {archived_child_issue.child_url} |\n" - ) - else: - f.write( - "No published child pages were found under archived parent pages.\n" - ) - def main() -> None: records, status_totals, requested_link_records, raw_items = build_records( diff --git a/e3sm_comms/exported_xml_reviewer/readers.py b/e3sm_comms/exported_xml_reviewer/readers.py new file mode 100644 index 0000000..9e21f6b --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/readers.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import csv +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from typing import List, Set, Tuple + +from e3sm_comms.exported_xml_reviewer.utils import normalize_url + + +@dataclass +class WordpressItem: + post_id: str + post_parent: str + post_type: str + title: str + url: str + status: str + body: str + + +def read_sensitive_terms(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + terms = [line.strip().lower() for line in f if line.strip()] + return sorted(set(terms)) + + +def read_known_ok_links(file_path: str) -> Set[str]: + with open(file_path, "r", encoding="utf-8") as f: + return {normalize_url(line.strip()) for line in f if line.strip()} + + +def read_requested_links(file_path: str) -> List[Tuple[str, str]]: + rows: List[Tuple[str, str]] = [] + + with open(file_path, "r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + + if not reader.fieldnames: + print(f"Requested links CSV has no headers: {file_path}") + return rows + + normalized_to_actual = { + header.strip().lower(): header for header in reader.fieldnames if header + } + + e3sm_header = normalized_to_actual.get("e3sm.org link") + requesting_header = normalized_to_actual.get( + "list of urls that wants to link to it" + ) + + if requesting_header is None: + for candidate in [ + "list of urls that want to link to it", + "requesting urls", + "requesting url", + "list of urls", + ]: + requesting_header = normalized_to_actual.get(candidate) + if requesting_header: + break + + if e3sm_header is None: + print( + f"Requested links CSV is missing required column 'e3sm.org link'. " + f"Found headers: {reader.fieldnames}" + ) + return rows + + if requesting_header is None: + print( + "Requested links CSV could not find the requesting URLs column. " + f"Found headers: {reader.fieldnames}" + ) + + for row in reader: + e3sm_url = normalize_url(row.get(e3sm_header, "")) + requesting_urls = ( + row.get(requesting_header, "").strip() if requesting_header else "" + ) + + if e3sm_url: + rows.append((e3sm_url, requesting_urls)) + + return rows + + +def read_whitelist_patterns(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + +def parse_wordpress_xml( + xml_file_path: str, expected_post_type: str +) -> List[WordpressItem]: + ns = { + "wp": "http://wordpress.org/export/1.2/", + } + + tree = ET.parse(xml_file_path) + root = tree.getroot() + + channel = root.find("channel") + if channel is None: + return [] + + items: List[WordpressItem] = [] + + for item in channel.findall("item"): + post_type_elem = item.find("wp:post_type", ns) + if post_type_elem is None: + continue + + post_type_text = (post_type_elem.text or "").strip() + if post_type_text != expected_post_type: + continue + + title_elem = item.find("title") + link_elem = item.find("link") + status_elem = item.find("wp:status", ns) + post_id_elem = item.find("wp:post_id", ns) + post_parent_elem = item.find("wp:post_parent", ns) + + title = ( + title_elem.text.strip() + if title_elem is not None and title_elem.text is not None + else "Untitled" + ) + link = ( + normalize_url(link_elem.text) + if link_elem is not None and link_elem.text is not None + else "" + ) + status = ( + status_elem.text.strip() + if status_elem is not None and status_elem.text is not None + else "unknown" + ) + post_id = ( + post_id_elem.text.strip() + if post_id_elem is not None and post_id_elem.text is not None + else "" + ) + post_parent = ( + post_parent_elem.text.strip() + if post_parent_elem is not None and post_parent_elem.text is not None + else "0" + ) + body = extract_item_body(item) + + items.append( + WordpressItem( + post_id=post_id, + post_parent=post_parent, + post_type=post_type_text, + title=title, + url=link, + status=status, + body=body, + ) + ) + + return items + + +def extract_item_body(item: ET.Element) -> str: + ns = { + "wp": "http://wordpress.org/export/1.2/", + "content": "http://purl.org/rss/1.0/modules/content/", + "excerpt": "http://wordpress.org/export/1.2/excerpt/", + } + + body_parts: List[str] = [] + + content_elem = item.find("content:encoded", ns) + if content_elem is not None and content_elem.text and content_elem.text.strip(): + body_parts.append(content_elem.text.strip()) + + excerpt_elem = item.find("excerpt:encoded", ns) + if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): + body_parts.append(excerpt_elem.text.strip()) + + for postmeta in item.findall("wp:postmeta", ns): + meta_key_elem = postmeta.find("wp:meta_key", ns) + meta_value_elem = postmeta.find("wp:meta_value", ns) + + meta_key = ( + meta_key_elem.text.strip() + if meta_key_elem is not None and meta_key_elem.text + else "" + ) + meta_value = ( + meta_value_elem.text.strip() + if meta_value_elem is not None and meta_value_elem.text + else "" + ) + + if not meta_value: + continue + + if meta_key.endswith("_free_form_content") and not meta_key.startswith("_"): + body_parts.append(meta_value) + + return "\n".join(body_parts) diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py new file mode 100644 index 0000000..37e5c82 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -0,0 +1,488 @@ +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import DefaultDict, Dict, List, Set + +from e3sm_comms.exported_xml_reviewer.builders import ( + ArchivedParentPublishedChildIssue, + PublishedContentLinkSummary, + ReportRecord, + RequestedLinkRecord, + TopLevelPageIssue, + sort_requested_link_records, +) +from e3sm_comms.exported_xml_reviewer.link_analysis import ( + InvalidInternalLinkGroup, + NonPublishedInternalLinkGroup, +) +from e3sm_comms.exported_xml_reviewer.readers import WordpressItem +from e3sm_comms.exported_xml_reviewer.utils import display_status, normalize_status + + +def write_markdown_report( + output_path: str, + records: List[ReportRecord], + status_totals: Dict[str, int], + requested_link_records: List[RequestedLinkRecord], +) -> None: + grouped: DefaultDict[str, List[ReportRecord]] = defaultdict(list) + for record in records: + grouped[record.status].append(record) + + for status in grouped: + grouped[status].sort( + key=lambda r: (-sum(r.sensitive_terms.values()), r.title.lower()) + ) + + ordered_statuses = [ + "published & whitelisted, known ok", + "published & whitelisted", + "published & not whitelisted", + "archived", + "draft", + "future", + "pending", + "private", + "unknown", + ] + + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Sensitive Terms Report\n\n") + f.write( + "The detailed sections below include only e3sm.org pages/posts where one or more sensitive terms were found. " + "The summary table includes counts for both flagged and unflagged items.\n\n" + ) + + f.write("| Status | With sensitive terms | Without sensitive terms | Total |\n") + f.write("| --- | ---: | ---: | ---: |\n") + + total_with_terms = 0 + total_without_terms = 0 + + all_summary_statuses = set(status_totals) | set(grouped) + extra_statuses = sorted( + s for s in all_summary_statuses if s not in ordered_statuses + ) + + for status in ordered_statuses + extra_statuses: + total_in_status = status_totals.get(status, 0) + with_terms = len(grouped.get(status, [])) + without_terms = total_in_status - with_terms + + if total_in_status == 0 and with_terms == 0: + continue + + total_with_terms += with_terms + total_without_terms += without_terms + f.write( + f"| {status} | {with_terms} | {without_terms} | {total_in_status} |\n" + ) + + grand_total = total_with_terms + total_without_terms + f.write( + f"| TOTAL | {total_with_terms} | {total_without_terms} | {grand_total} |\n" + ) + f.write("\n") + + if requested_link_records: + requested_link_records = sort_requested_link_records(requested_link_records) + + f.write("## Requested Links\n\n") + f.write( + "| e3sm.org link | Included later on this page? | Current status | Currently whitelisted? | Requesting URLs |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + for requested_record in requested_link_records: + included_later = ( + "Yes" + if requested_record.included_later + else "No (i.e., contains no sensitive terms)" + ) + currently_whitelisted = ( + "Yes" if requested_record.currently_whitelisted else "No" + ) + f.write( + f"| {requested_record.e3sm_url} | {included_later} | {requested_record.current_status} | " + f"{currently_whitelisted} | {requested_record.requesting_urls} |\n" + ) + + f.write("\n") + + all_statuses = ordered_statuses + extra_statuses + seen = set() + + for status in all_statuses: + if status not in grouped or status in seen: + continue + seen.add(status) + + f.write(f"## {status.capitalize()} ({len(grouped[status])})\n\n") + + for idx, record in enumerate(grouped[status], start=1): + e3sm_md = f"[e3sm.org]({record.e3sm_url})" + confluence_md = ( + f" [(confluence draft)]({record.confluence_draft_url})" + if record.confluence_draft_url + else "" + ) + + f.write( + f"{idx}. {record.title}: {e3sm_md}{confluence_md} -- {record.sensitive_terms}\n" + ) + + f.write("\n") + + +def write_invalid_internal_links_report( + output_path: str, + groups: List[InvalidInternalLinkGroup], + non_published_groups: List[NonPublishedInternalLinkGroup], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_table(f, table_groups: List[InvalidInternalLinkGroup]) -> None: + f.write( + "| Invalid linked URL | Does it redirect to a working link? | Inferred by inference rules | Found under different prefix | Status of inferred/found page/post | Referenced on these published pages | Referenced on these non-published pages |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- | --- |\n") + + for group in table_groups: + redirect_md = ( + f"[{group.redirect_target}]({group.redirect_target})" + if group.redirect_target + else "" + ) + if redirect_md and group.redirect_status: + redirect_md = ( + f"{redirect_md} (redirect status: {group.redirect_status})" + ) + + inferred_md = ( + f"[{group.inferred_link}]({group.inferred_link})" + if group.inferred_link + else "" + ) + prefix_md = ( + f"[{group.found_under_different_prefix}]({group.found_under_different_prefix})" + if group.found_under_different_prefix + else "" + ) + + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published + ) + + f.write( + f"| {group.linked_url} | {redirect_md} | {inferred_md} | {prefix_md} | {group.linked_target_status} | {referenced_published} | {referenced_non_published} |\n" + ) + + def render_non_published_table( + f, table_groups: List[NonPublishedInternalLinkGroup] + ) -> None: + f.write( + "| Valid linked URL | Target status | Referenced on these published pages | Referenced on these non-published pages |\n" + ) + f.write("| --- | --- | --- | --- |\n") + + for group in table_groups: + referenced_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_published + ) + referenced_non_published = ", ".join( + f"[{title}]({url})" for title, url in group.referenced_on_non_published + ) + f.write( + f"| {group.linked_url} | {group.target_status} | {referenced_published} | {referenced_non_published} |\n" + ) + + working_redirects: List[InvalidInternalLinkGroup] = [] + published_targets: List[InvalidInternalLinkGroup] = [] + archived_targets: List[InvalidInternalLinkGroup] = [] + no_candidate: List[InvalidInternalLinkGroup] = [] + + for group in groups: + if group.redirect_target: + working_redirects.append(group) + elif group.linked_target_status == "Published": + published_targets.append(group) + elif group.linked_target_status == "Archived": + archived_targets.append(group) + else: + no_candidate.append(group) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Invalid Internal e3sm.org Links\n\n") + + if not groups and not non_published_groups: + f.write("No invalid internal links found.\n") + return + + f.write( + f"## 1. These have working redirections already ({len(working_redirects)})\n\n" + ) + if working_redirects: + render_table(f, working_redirects) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 2. The target pages are published, we just need to set up the redirections ({len(published_targets)})\n\n" + ) + if published_targets: + render_table(f, published_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write(f"## 3. The target pages are archived ({len(archived_targets)})\n\n") + if archived_targets: + render_table(f, archived_targets) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 4. Couldn't find a redirection candidate ({len(no_candidate)})\n\n" + ) + if no_candidate: + render_table(f, no_candidate) + else: + f.write("None found.\n") + f.write("\n") + + f.write( + f"## 5. Technically valid links that point to non-published targets ({len(non_published_groups)})\n\n" + ) + if non_published_groups: + render_non_published_table(f, non_published_groups) + else: + f.write("None found.\n") + + +def write_published_pages_link_report( + output_path: str, + page_summaries: List[PublishedContentLinkSummary], + post_summaries: List[PublishedContentLinkSummary], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_link_list(urls: List[str]) -> str: + return ", ".join(f"[{url}]({url})" for url in urls) + + def write_section( + f, + section_title: str, + summaries: List[PublishedContentLinkSummary], + ) -> None: + invalid_summaries = [ + s + for s in summaries + if s.archived_links or s.redirected_links or s.broken_links + ] + valid_only_summaries = [ + s + for s in summaries + if not (s.archived_links or s.redirected_links or s.broken_links) + ] + + f.write(f"## {section_title}\n\n") + + if not summaries: + f.write("No published items with links found.\n\n") + return + + f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") + if invalid_summaries: + f.write( + "| Published item | known archived links | published item, wrong URL, but redirection working | link does not work | valid e3sm.org links |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + archived_total = 0 + redirected_total = 0 + broken_total = 0 + valid_total = 0 + + archived_unique: Set[str] = set() + redirected_unique: Set[str] = set() + broken_unique: Set[str] = set() + valid_unique: Set[str] = set() + + for summary in invalid_summaries: + item_md = f"[{summary.title}]({summary.url})" + archived_md = render_link_list(summary.archived_links) + redirected_md = render_link_list(summary.redirected_links) + broken_md = render_link_list(summary.broken_links) + valid_md = render_link_list(summary.valid_links) + + archived_total += len(summary.archived_links) + redirected_total += len(summary.redirected_links) + broken_total += len(summary.broken_links) + valid_total += len(summary.valid_links) + + archived_unique.update(summary.archived_links) + redirected_unique.update(summary.redirected_links) + broken_unique.update(summary.broken_links) + valid_unique.update(summary.valid_links) + + f.write( + f"| {item_md} | {archived_md} | {redirected_md} | {broken_md} | {valid_md} |\n" + ) + + f.write( + f"| Total link count | {archived_total} | {redirected_total} | {broken_total} | {valid_total} |\n" + ) + f.write( + f"| Unique link count | {len(archived_unique)} | {len(redirected_unique)} | {len(broken_unique)} | {len(valid_unique)} |\n" + ) + else: + f.write("No items with invalid links found.\n") + + f.write("\n") + f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") + + if valid_only_summaries: + f.write("| Published item | valid e3sm.org link count |\n") + f.write("| --- | ---: |\n") + + total_links = 0 + unique_links: Set[str] = set() + + for summary in valid_only_summaries: + item_md = f"[{summary.title}]({summary.url})" + f.write(f"| {item_md} | {len(summary.valid_links)} |\n") + total_links += len(summary.valid_links) + unique_links.update(summary.valid_links) + + f.write(f"| Total link count | {total_links} |\n") + f.write(f"| Unique link count | {len(unique_links)} |\n") + else: + f.write("No items with only valid links found.\n") + + f.write("\n") + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Content Invalid Link Report\n\n") + write_section(f, "Published Pages", page_summaries) + write_section(f, "Published Posts", post_summaries) + + +def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def write_section(f, section_items: List[WordpressItem], heading: str) -> None: + section_items = [item for item in section_items if item.post_id] + + children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + item_by_id: Dict[str, WordpressItem] = {} + + for item in section_items: + item_by_id[item.post_id] = item + + for item in section_items: + parent_id = ( + item.post_parent if item.post_parent and item.post_parent != "0" else "" + ) + children_by_parent[parent_id].append(item) + + for child_list in children_by_parent.values(): + child_list.sort(key=lambda x: x.title.lower()) + + roots = [ + item + for item in section_items + if not item.post_parent + or item.post_parent == "0" + or item.post_parent not in item_by_id + ] + roots.sort(key=lambda x: x.title.lower()) + + f.write(f"{heading}\n") + + seen: Set[str] = set() + + def walk(node: WordpressItem, depth: int) -> None: + indent = " " * depth + status_label = display_status(normalize_status(node.status)) + line = f"{indent}{node.title} [{status_label}]" + if node.url: + line += f" [{node.url}]" + f.write(line + "\n") + + if node.post_id in seen: + return + + seen.add(node.post_id) + + for child in children_by_parent.get(node.post_id, []): + walk(child, depth + 1) + + for root in roots: + walk(root, 0) + + f.write("\n") + + pages = [item for item in items if item.post_type == "page"] + posts = [item for item in items if item.post_type == "post"] + + with open(output_path, "w", encoding="utf-8") as f: + write_section(f, pages, "Pages") + write_section(f, posts, "Posts") + + +def write_navigation_issues_report( + output_path: str, + top_level_issues: List[TopLevelPageIssue], + archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# WordPress Navigation Issues Report\n\n") + + f.write("## 1. Top-level pages that are not expected top-level tabs\n\n") + f.write( + "Expected top-level tabs are: About, News, Resources, Tools, Policies, Home Page.\n\n" + ) + + if top_level_issues: + f.write("| Title | Status | URL |\n") + f.write("| --- | --- | --- |\n") + for top_level_issue in top_level_issues: + f.write( + f"| {top_level_issue.title} | {top_level_issue.status} | {top_level_issue.url} |\n" + ) + else: + f.write("No unexpected top-level pages found.\n") + + f.write("\n") + + f.write("## 2. Published child pages under archived parent pages\n\n") + + if archived_parent_published_child_issues: + f.write( + "| Parent title | Parent status | Parent URL | Child title | Child status | Child URL |\n" + ) + f.write("| --- | --- | --- | --- | --- | --- |\n") + for archived_child_issue in archived_parent_published_child_issues: + f.write( + f"| {archived_child_issue.parent_title} | {archived_child_issue.parent_status} | {archived_child_issue.parent_url} | " + f"{archived_child_issue.child_title} | {archived_child_issue.child_status} | {archived_child_issue.child_url} |\n" + ) + else: + f.write( + "No published child pages were found under archived parent pages.\n" + ) diff --git a/e3sm_comms/exported_xml_reviewer/utils.py b/e3sm_comms/exported_xml_reviewer/utils.py new file mode 100644 index 0000000..7749fe7 --- /dev/null +++ b/e3sm_comms/exported_xml_reviewer/utils.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Set +from urllib.parse import urlsplit, urlunsplit + + +def normalize_url(url: str) -> str: + url = url.strip() + if not url: + return "" + + parts = urlsplit(url) + scheme = parts.scheme.lower() + netloc = parts.netloc.lower() + path = parts.path.rstrip("/") + + normalized = urlunsplit((scheme, netloc, path, "", "")) + return normalized + + +def normalize_status(raw_status: Optional[str]) -> str: + if not raw_status: + return "unknown" + + mapping = { + "publish": "published", + "archive": "archived", + "draft": "draft", + "future": "future", + "pending": "pending", + "private": "private", + } + return mapping.get(raw_status.strip().lower(), raw_status.strip().lower()) + + +def display_status(status: str) -> str: + mapping = { + "published": "Published", + "archived": "Archived", + "draft": "Draft", + "future": "Future", + "pending": "Pending", + "private": "Private", + "unknown": "Unknown", + } + return mapping.get(status, status.title()) + + +def strip_html(text: str) -> str: + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r" ", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def matches_pattern(pattern: str, url: str) -> bool: + if "*" not in pattern: + return normalize_url(pattern) == normalize_url(url) + + normalized_url = normalize_url(url) + normalized_pattern = normalize_url(pattern) + + if "*" not in normalized_pattern: + return normalized_pattern == normalized_url + + if normalized_pattern.count("*") == 1 and normalized_pattern.endswith("*"): + prefix = normalized_pattern[:-1] + return normalized_url.startswith(prefix) + + parts = normalized_pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = normalized_url.find(part, position) + if found_at == -1: + return False + if i == 0 and not normalized_pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if ( + not normalized_pattern.endswith("*") + and parts[-1] + and not normalized_url.endswith(parts[-1]) + ): + return False + + return True + + +def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> Set[str]: + matched_urls: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url): + matched_urls.add(url) + return matched_urls + + +def is_legacy_content_url(url: str) -> bool: + parts = urlsplit(url) + slug = parts.path.strip("/").lower() + return bool(re.fullmatch(r"\d{6,8}[_-].+", slug)) + + +def count_sensitive_terms(text: str, terms: List[str]) -> Dict[str, int]: + counts: Dict[str, int] = {} + lowered = text.lower() + + for term in terms: + escaped = re.escape(term) + pattern = rf"\b{escaped}\b" + matches = re.findall(pattern, lowered) + if matches: + counts[term] = len(matches) + + return counts From 09682298443ca3f95a69bdc2078f1966d50ed0ea Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 15:35:30 -0700 Subject: [PATCH 73/85] Rename report --- e3sm_comms/exported_xml_reviewer/main.py | 10 +++++----- e3sm_comms/exported_xml_reviewer/reporters.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index a87b3db..3005d63 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -12,9 +12,9 @@ from e3sm_comms.exported_xml_reviewer.reporters import ( write_hierarchical_outline, write_invalid_internal_links_report, - write_markdown_report, write_navigation_issues_report, write_published_pages_link_report, + write_terms_report, ) from e3sm_comms.utils import IO_DIR @@ -40,7 +40,7 @@ # ) # Outputs: -OUTPUT_MARKDOWN_REPORT: str = ( +OUTPUT_TERMS_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" ) OUTPUT_HIERARCHICAL_OUTLINE: str = ( @@ -68,8 +68,8 @@ def main() -> None: known_ok_links_file=INPUT_KNOWN_OK_LINKS, ) - write_markdown_report( - OUTPUT_MARKDOWN_REPORT, + write_terms_report( + OUTPUT_TERMS_REPORT, records, status_totals, requested_link_records, @@ -112,7 +112,7 @@ def main() -> None: published_post_link_summaries, ) - print(f"Wrote report to {OUTPUT_MARKDOWN_REPORT}") + print(f"Wrote report to {OUTPUT_TERMS_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") print( diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index 37e5c82..9fc654b 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -20,7 +20,7 @@ from e3sm_comms.exported_xml_reviewer.utils import display_status, normalize_status -def write_markdown_report( +def write_terms_report( output_path: str, records: List[ReportRecord], status_totals: Dict[str, int], From 2743ea5510f084a6dfbe114be479298261c211d2 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 15:41:40 -0700 Subject: [PATCH 74/85] Add flags for optional inputs --- e3sm_comms/exported_xml_reviewer/main.py | 42 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 3005d63..952662d 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -1,5 +1,7 @@ from __future__ import annotations +import argparse + from e3sm_comms.exported_xml_reviewer.builders import ( build_navigation_issue_records, build_published_content_link_summaries, @@ -30,14 +32,12 @@ INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" # Optional inputs: -INPUT_CONFLUENCE_HIERARCHY: str = "" -# INPUT_CONFLUENCE_HIERARCHY: str = ( -# f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" -# ) -INPUT_WHITELIST: str = "" -# INPUT_WHITELIST: str = ( -# f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" -# ) +DEFAULT_CONFLUENCE_HIERARCHY: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt" +) +DEFAULT_WHITELIST: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" +) # Outputs: OUTPUT_TERMS_REPORT: str = ( @@ -57,13 +57,35 @@ ) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Review exported WordPress XML files.") + parser.add_argument( + "--use-confluence", + action="store_true", + help="Use the Confluence hierarchy file for hierarchical outline generation.", + ) + parser.add_argument( + "--use-whitelist", + action="store_true", + help="Use the whitelisted web pages file to filter results.", + ) + return parser.parse_args() + + def main() -> None: + args = parse_args() + + input_confluence_hierarchy = ( + DEFAULT_CONFLUENCE_HIERARCHY if args.use_confluence else "" + ) + input_whitelist = DEFAULT_WHITELIST if args.use_whitelist else "" + records, status_totals, requested_link_records, raw_items = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, - confluence_hierarchy=INPUT_CONFLUENCE_HIERARCHY, + confluence_hierarchy=input_confluence_hierarchy, sensitive_terms_file=INPUT_SEARCH_PHRASES, - whitelist_file=INPUT_WHITELIST, + whitelist_file=input_whitelist, requested_links_file=INPUT_REQUESTED_LINKS, known_ok_links_file=INPUT_KNOWN_OK_LINKS, ) From 8aec40fa067df74dc32c4434b2fa9a4cb4760da8 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 16:29:48 -0700 Subject: [PATCH 75/85] Break e3sm_org_reviewer into multiple files --- e3sm_comms/e3sm_org_reviewer/README.md | 8 + e3sm_comms/e3sm_org_reviewer/classifiers.py | 81 ++ e3sm_comms/e3sm_org_reviewer/confluence.py | 63 ++ e3sm_comms/e3sm_org_reviewer/main.py | 937 +------------------- e3sm_comms/e3sm_org_reviewer/parsers.py | 197 ++++ e3sm_comms/e3sm_org_reviewer/readers.py | 44 + e3sm_comms/e3sm_org_reviewer/record.py | 18 + e3sm_comms/e3sm_org_reviewer/reporters.py | 428 +++++++++ e3sm_comms/e3sm_org_reviewer/utils.py | 101 +++ 9 files changed, 970 insertions(+), 907 deletions(-) create mode 100644 e3sm_comms/e3sm_org_reviewer/README.md create mode 100644 e3sm_comms/e3sm_org_reviewer/classifiers.py create mode 100644 e3sm_comms/e3sm_org_reviewer/confluence.py create mode 100644 e3sm_comms/e3sm_org_reviewer/parsers.py create mode 100644 e3sm_comms/e3sm_org_reviewer/readers.py create mode 100644 e3sm_comms/e3sm_org_reviewer/record.py create mode 100644 e3sm_comms/e3sm_org_reviewer/reporters.py create mode 100644 e3sm_comms/e3sm_org_reviewer/utils.py diff --git a/e3sm_comms/e3sm_org_reviewer/README.md b/e3sm_comms/e3sm_org_reviewer/README.md new file mode 100644 index 0000000..1a0fecb --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/README.md @@ -0,0 +1,8 @@ +# Dependency hierarchy + +It is important to not introduce circular dependencies. +To avoid this, the dependency hierarchy is listed below: + +- Level 1: `main.py` +- Level 2: `parsers.py`, `reporters.py` +- Level 3: `classifiers.py`, `confluence.py`, `readers.py`, `record.py`, `utils.py` diff --git a/e3sm_comms/e3sm_org_reviewer/classifiers.py b/e3sm_comms/e3sm_org_reviewer/classifiers.py new file mode 100644 index 0000000..dc63e69 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/classifiers.py @@ -0,0 +1,81 @@ +import re +from typing import Dict, Optional, Set, Tuple + +CLASS_PUBLISHED = "Published" +CLASS_ARCHIVED = "Archived" +CLASS_SHOULD_BE_ARCHIVED = "Should be archived" +CLASS_NOT_PUBLISHED = "Not published" +CLASS_KNOWN_OK = "Known OK" +CLASS_KEEP_UNCHANGED = "Keep unchanged" +CLASS_NO_MAPPED_E3SM_URL = "No mapped e3sm.org URL" +CLASS_PREDICTED_URL_NOT_IN_EXPORT = "Predicted e3sm.org URL not in WordPress export" + +FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") + + +def classify_e3sm_url( + e3sm_url: Optional[str], + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Tuple[str, Optional[str]]: + if not e3sm_url: + return CLASS_NO_MAPPED_E3SM_URL, None + + if e3sm_url in known_ok_urls: + return CLASS_KNOWN_OK, url_to_status.get(e3sm_url) + + if e3sm_url in keep_unchanged_urls: + return CLASS_KEEP_UNCHANGED, url_to_status.get(e3sm_url) + + wordpress_status = url_to_status.get(e3sm_url) + + if wordpress_status is None: + return CLASS_PREDICTED_URL_NOT_IN_EXPORT, None + + if wordpress_status == "archive": + return CLASS_ARCHIVED, wordpress_status + + if e3sm_url in expected_archived_urls: + return CLASS_SHOULD_BE_ARCHIVED, wordpress_status + + if wordpress_status != "publish": + return CLASS_NOT_PUBLISHED, wordpress_status + + return CLASS_PUBLISHED, wordpress_status + + +def classification_sort_key(classification: str) -> Tuple[int, str]: + order = { + CLASS_PUBLISHED: 0, + CLASS_SHOULD_BE_ARCHIVED: 1, + CLASS_ARCHIVED: 2, + CLASS_NOT_PUBLISHED: 3, + CLASS_KNOWN_OK: 4, + CLASS_KEEP_UNCHANGED: 5, + CLASS_NO_MAPPED_E3SM_URL: 6, + CLASS_PREDICTED_URL_NOT_IN_EXPORT: 7, + } + return (order.get(classification, 999), classification) + + +def year_sort_key(year_str: str) -> Tuple[int, int]: + if year_str == "Unknown year": + return (1, 0) + if year_str == "N/A": + return (2, 0) + try: + return (0, -int(year_str)) + except ValueError: + return (3, 0) + + +def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: + match = FROM_PREFIX_RE.match(line) + if not match: + return None, line + + year = int(match.group(1)) + remainder = match.group(2).strip() + return year, remainder diff --git a/e3sm_comms/e3sm_org_reviewer/confluence.py b/e3sm_comms/e3sm_org_reviewer/confluence.py new file mode 100644 index 0000000..884b22e --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/confluence.py @@ -0,0 +1,63 @@ +from typing import List, Tuple + +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm + +CONFLUENCE_SPACE = "EPWCD" +CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" + + +def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: + return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +def get_confluence_predicted_e3sm_urls( + input_file: str, +) -> Tuple[List[str], List[str]]: + valid_predicted_urls: List[str] = [] + unmapped_confluence_pages: List[str] = [] + + for page_id, title in parse_confluence_hierarchy_file(input_file): + confluence_url = build_confluence_url(page_id) + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + if e3sm_url: + valid_predicted_urls.append(e3sm_url) + else: + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + unmapped_confluence_pages.append(f"{title}: {confluence_url}") + + return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages) diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 928362a..47f1ed3 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -1,15 +1,34 @@ -import ast -import re -import xml.etree.ElementTree as ET -from collections import defaultdict -from dataclasses import dataclass -from typing import Callable, DefaultDict, Dict, List, Optional, Set, TextIO, Tuple - -from e3sm_comms.page_reviewer.utils_base import ( - LinkedURLs, - get_e3sm_url_status, - map_confluence_to_e3sm, +from typing import Dict, List, Set + +from e3sm_comms.e3sm_org_reviewer.classifiers import CLASS_PUBLISHED +from e3sm_comms.e3sm_org_reviewer.confluence import get_confluence_predicted_e3sm_urls +from e3sm_comms.e3sm_org_reviewer.parsers import ( + parse_confluence_record, + parse_wordpress_record, + parse_wordpress_sensitive_terms_lines, +) +from e3sm_comms.e3sm_org_reviewer.readers import ( + get_wordpress_urls_by_status, + print_status_counts, +) +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.e3sm_org_reviewer.reporters import ( + write_action_items_report, + write_markdown_report, + write_sensitive_terms_report, +) +from e3sm_comms.e3sm_org_reviewer.utils import ( + build_url_to_status, + expand_patterns_to_urls, + get_all_non_published_urls, + get_all_urls, + get_combined_urls_by_status, + get_invalid_patterns, + get_list_difference, + get_total_count, + read_nonempty_lines, ) +from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status from e3sm_comms.utils import IO_DIR # From WordPress under Tools > Export: @@ -45,902 +64,6 @@ RUN_CHECKS: bool = True # Set to False for faster debugging -CONFLUENCE_SPACE = "EPWCD" -CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" - -FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$") - -CLASS_PUBLISHED = "Published" -CLASS_ARCHIVED = "Archived" -CLASS_SHOULD_BE_ARCHIVED = "Should be archived" -CLASS_NOT_PUBLISHED = "Not published" -CLASS_KNOWN_OK = "Known OK" -CLASS_KEEP_UNCHANGED = "Keep unchanged" -CLASS_NO_MAPPED_E3SM_URL = "No mapped e3sm.org URL" -CLASS_PREDICTED_URL_NOT_IN_EXPORT = "Predicted e3sm.org URL not in WordPress export" - - -@dataclass -class SensitiveTermRecord: - source: str # "e3sm.org" or "confluence" - raw_line: str - year_label: str - year_int: Optional[int] - total_terms: int - term_counts: Dict[str, int] - source_url: Optional[str] - title: Optional[str] - confluence_url: Optional[str] - e3sm_url: Optional[str] - classification: str - wordpress_status: Optional[str] - - -def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: - return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" - - -def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: - parsed: List[Tuple[str, str]] = [] - - with open(input_file, "r", encoding="utf-8") as f: - for line_number, raw_line in enumerate(f, start=1): - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - stripped = line.lstrip() - - if ":" not in stripped: - print(f"Skipping malformed Confluence line {line_number}: {line}") - continue - - page_id, title = stripped.split(":", 1) - page_id = page_id.strip() - title = title.strip() - - if not page_id.isdigit(): - print( - f"Skipping Confluence line {line_number} with non-numeric page id: {line}" - ) - continue - - parsed.append((page_id, title)) - - return parsed - - -def get_confluence_predicted_e3sm_urls( - input_file: str, -) -> Tuple[List[str], List[str]]: - valid_predicted_urls: List[str] = [] - unmapped_confluence_pages: List[str] = [] - - for page_id, title in parse_confluence_hierarchy_file(input_file): - confluence_url = build_confluence_url(page_id) - try: - e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) - if e3sm_url: - valid_predicted_urls.append(e3sm_url) - else: - unmapped_confluence_pages.append(f"{title}: {confluence_url}") - except Exception as exc: - print( - f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" - ) - unmapped_confluence_pages.append(f"{title}: {confluence_url}") - - return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages) - - -def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: - for status in ["publish", "archive", "draft", "future", "pending", "private"]: - if status in all_urls_by_status: - print(f"Found {len(all_urls_by_status[status])} {status} URLs") - - -def get_wordpress_urls_by_status( - xml_file_path: str, post_type: str -) -> Dict[str, List[str]]: - ns = { - "wp": "http://wordpress.org/export/1.2/", - } - - tree = ET.parse(xml_file_path) - root = tree.getroot() - - grouped: Dict[str, List[str]] = defaultdict(list) - channel = root.find("channel") - if channel is None: - return {} - - for item in channel.findall("item"): - item_post_type = item.find("wp:post_type", ns) - item_status = item.find("wp:status", ns) - link = item.find("link") - - if item_post_type is None or item_post_type.text != post_type: - continue - - status = ( - item_status.text.strip() - if item_status is not None and item_status.text - else "unknown" - ) - - if link is not None and link.text: - grouped[status].append(link.text.strip()) - - return {status: sorted(urls) for status, urls in sorted(grouped.items())} - - -def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: - return sum(len(urls) for urls in urls_by_status.values()) - - -def get_combined_urls_by_status( - pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] -) -> Dict[str, List[str]]: - merged: Dict[str, List[str]] = defaultdict(list) - for source in (pages_by_status, posts_by_status): - for status, urls in source.items(): - merged[status].extend(urls) - return {status: sorted(urls) for status, urls in sorted(merged.items())} - - -def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: - return sorted(set(list1) - set(list2)) - - -def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: - all_urls: List[str] = [] - for urls in urls_by_status.values(): - all_urls.extend(urls) - return sorted(all_urls) - - -def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: - non_published_urls: List[str] = [] - for status, urls in urls_by_status.items(): - if status != "publish": - non_published_urls.extend(urls) - return sorted(non_published_urls) - - -def matches_pattern(pattern: str, url: str) -> bool: - if "*" not in pattern: - return pattern == url - - if pattern.count("*") == 1 and pattern.endswith("*"): - prefix = pattern[:-1] - return url.startswith(prefix) - - parts = pattern.split("*") - position = 0 - for i, part in enumerate(parts): - if not part: - continue - found_at = url.find(part, position) - if found_at == -1: - return False - if i == 0 and not pattern.startswith("*") and found_at != 0: - return False - position = found_at + len(part) - - if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): - return False - - return True - - -def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: - matched_urls: Set[str] = set() - for pattern in patterns: - for url in all_urls: - if matches_pattern(pattern, url): - matched_urls.add(url) - return sorted(matched_urls) - - -def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: - invalid_patterns: List[str] = [] - for pattern in patterns: - if not any(matches_pattern(pattern, url) for url in all_urls): - invalid_patterns.append(pattern) - return sorted(invalid_patterns) - - -def get_status_counts_for_urls( - urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] -) -> Dict[str, int]: - url_set = set(urls) - counts: Dict[str, int] = {} - for status in statuses: - counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) - return counts - - -def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: - file_obj.write(f"## {title}\n\n") - if not items: - file_obj.write("_None._\n\n") - return - - for i, item in enumerate(items, start=1): - file_obj.write(f"{i}. {item}\n") - file_obj.write("\n") - - -def write_summary_table( - file_obj: TextIO, - all_urls_by_status: Dict[str, List[str]], - valid_whitelisted_paths: List[str], - valid_expected_archived_paths: List[str], - valid_confluence_paths: List[str], - invalid_confluence_paths: List[str], - confluence_unmapped_entries: List[str], -) -> None: - statuses: List[str] = sorted(all_urls_by_status.keys()) - all_urls: List[str] = get_all_urls(all_urls_by_status) - - whitelist_set: Set[str] = set( - expand_patterns_to_urls(valid_whitelisted_paths, all_urls) - ) - expected_archived_set: Set[str] = set( - expand_patterns_to_urls(valid_expected_archived_paths, all_urls) - ) - both_set: Set[str] = whitelist_set.intersection(expected_archived_set) - neither_set: Set[str] = set(all_urls) - whitelist_set.union(expected_archived_set) - - rows = [ - ("Whitelisted URLs", whitelist_set), - ("Expected archived", expected_archived_set), - ("Both whitelisted and expected archived", both_set), - ("Neither whitelisted nor expected archived", neither_set), - ("TOTAL", set(all_urls)), - ] - - file_obj.write("# Summary\n\n") - file_obj.write("| Type | " + " | ".join(statuses) + " | Total |\n") - file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " | --- |\n") - - for row_name, row_urls in rows: - counts = get_status_counts_for_urls( - urls=list(row_urls), - all_urls_by_status=all_urls_by_status, - statuses=statuses, - ) - total_count = sum(counts.values()) - file_obj.write( - f"| {row_name} | " - + " | ".join(str(counts[status]) for status in statuses) - + f" | {total_count} |\n" - ) - - confluence_valid_set: Set[str] = set(valid_confluence_paths) - all_urls_set: Set[str] = set(all_urls) - - e3sm_with_confluence: Set[str] = all_urls_set.intersection(confluence_valid_set) - e3sm_without_confluence: Set[str] = all_urls_set - confluence_valid_set - - confluence_not_valid_count: int = len(invalid_confluence_paths) + len( - confluence_unmapped_entries - ) - total_confluence_urls: int = len(confluence_valid_set) + confluence_not_valid_count - total_e3sm_urls: int = len(all_urls_set) - - confluence_counts_match: bool = ( - confluence_not_valid_count + len(e3sm_with_confluence) == total_confluence_urls - ) - e3sm_counts_match: bool = ( - len(e3sm_without_confluence) + len(e3sm_with_confluence) == total_e3sm_urls - ) - - file_obj.write("\n## Confluence Mapping Summary\n\n") - file_obj.write("| Type | Count |\n") - file_obj.write("| --- | --- |\n") - file_obj.write( - f"| Confluence paths that do not map to a valid e3sm.org path | {confluence_not_valid_count} |\n" - ) - file_obj.write( - f"| e3sm.org paths that do not have a Confluence path associated with them | {len(e3sm_without_confluence)} |\n" - ) - file_obj.write( - f"| e3sm.org paths that do have a Confluence counterpart | {len(e3sm_with_confluence)} |\n" - ) - file_obj.write(f"| Total Confluence-derived paths | {total_confluence_urls} |\n") - file_obj.write(f"| Total e3sm.org paths | {total_e3sm_urls} |\n") - file_obj.write("\n") - - file_obj.write("Validation:\n\n") - file_obj.write( - f"- Confluence counts match: " - f"{confluence_not_valid_count} + {len(e3sm_with_confluence)} = {total_confluence_urls} " - f"({'yes' if confluence_counts_match else 'no'})\n" - ) - file_obj.write( - f"- e3sm.org counts match: " - f"{len(e3sm_without_confluence)} + {len(e3sm_with_confluence)} = {total_e3sm_urls} " - f"({'yes' if e3sm_counts_match else 'no'})\n\n" - ) - - -def write_markdown_report( - output_path: str, - all_urls_by_status: Dict[str, List[str]], - valid_whitelisted_paths: List[str], - valid_expected_archived_paths: List[str], - valid_confluence_paths: List[str], - whitelisted_but_not_published: List[str], - published_but_not_whitelisted: List[str], - should_be_archived: List[str], - published_but_not_in_confluence: List[str], - published_not_whitelisted_and_not_in_confluence: List[str], - incorrectly_accessible_non_published_urls: List[str], - invalid_whitelisted_paths: List[str], - invalid_expected_archived_paths: List[str], - invalid_confluence_paths: List[str], - confluence_unmapped_entries: List[str], -) -> None: - with open(output_path, "w", encoding="utf-8") as f: - write_summary_table( - f, - all_urls_by_status=all_urls_by_status, - valid_whitelisted_paths=valid_whitelisted_paths, - valid_expected_archived_paths=valid_expected_archived_paths, - valid_confluence_paths=valid_confluence_paths, - invalid_confluence_paths=invalid_confluence_paths, - confluence_unmapped_entries=confluence_unmapped_entries, - ) - - f.write("# Valid Paths\n\n") - write_markdown_section( - f, - "Whitelisted but not published", - whitelisted_but_not_published, - ) - write_markdown_section( - f, - "Published but not whitelisted", - published_but_not_whitelisted, - ) - write_markdown_section( - f, - "Expecting to be archived, but not yet archived", - should_be_archived, - ) - write_markdown_section( - f, - "Published but no matching Confluence path found", - published_but_not_in_confluence, - ) - write_markdown_section( - f, - "Published but not whitelisted and no matching Confluence path found", - published_not_whitelisted_and_not_in_confluence, - ) - write_markdown_section( - f, - "Non-published e3sm.org pages that are still accessible without login", - incorrectly_accessible_non_published_urls, - ) - - f.write("# Invalid Paths\n\n") - write_markdown_section( - f, - "Identified in whitelist input", - invalid_whitelisted_paths, - ) - write_markdown_section( - f, - "Identified in archive input", - invalid_expected_archived_paths, - ) - write_markdown_section( - f, - "Identified in Confluence input", - invalid_confluence_paths, - ) - write_markdown_section( - f, - "Confluence pages with no mappable e3sm.org URL", - confluence_unmapped_entries, - ) - - -def write_action_items_confluence_section( - f: TextIO, records: List[SensitiveTermRecord] -) -> None: - f.write( - "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" - ) - - if not records: - f.write("_None._\n\n") - return - - grouped = group_records_by_classification_and_year(records) - - for classification in sorted(grouped.keys(), key=classification_sort_key): - f.write(f"### {classification}\n\n") - - for year_label in sorted(grouped[classification].keys(), key=year_sort_key): - f.write(f"#### {year_label}\n\n") - for idx, record in enumerate(grouped[classification][year_label], start=1): - f.write(f"{idx}. {format_confluence_record(record)}\n") - f.write("\n") - - -def write_action_items_report( - output_path: str, - should_be_archived: List[str], - published_not_whitelisted_and_not_in_confluence: List[str], - confluence_published_sensitive_records: List[SensitiveTermRecord], -) -> None: - with open(output_path, "w", encoding="utf-8") as f: - f.write("# Action Items Report\n\n") - - f.write("## Summary\n\n") - f.write("| Action Area | Count |\n") - f.write("| --- | ---: |\n") - f.write( - f"| Expecting to be archived, but not yet archived | {len(should_be_archived)} |\n" - ) - f.write( - f"| Published but not whitelisted and no matching Confluence path found | {len(published_not_whitelisted_and_not_in_confluence)} |\n" - ) - f.write( - f"| Confluence pages with sensitive terms mapped to published e3sm.org pages | {len(confluence_published_sensitive_records)} |\n" - ) - f.write("\n") - - write_markdown_section( - f, - "Expecting to be archived, but not yet archived", - should_be_archived, - ) - - write_markdown_section( - f, - "Published but not whitelisted and no matching Confluence path found", - published_not_whitelisted_and_not_in_confluence, - ) - - write_action_items_confluence_section(f, confluence_published_sensitive_records) - - -def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: - try: - data = ast.literal_eval(dict_str) - except (SyntaxError, ValueError): - return None - - if not isinstance(data, dict): - return None - - try: - total = sum(data.values()) - except TypeError: - return None - - if not isinstance(total, (int, float)): - return None - - return data - - -def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]: - match = FROM_PREFIX_RE.match(line) - if not match: - return None, line - - year = int(match.group(1)) - remainder = match.group(2).strip() - return year, remainder - - -def build_url_to_status(all_urls_by_status: Dict[str, List[str]]) -> Dict[str, str]: - result: Dict[str, str] = {} - for status, urls in all_urls_by_status.items(): - for url in urls: - result[url] = status - return result - - -def read_nonempty_lines(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - return [line.strip() for line in f if line.strip()] - - -def classify_e3sm_url( - e3sm_url: Optional[str], - url_to_status: Dict[str, str], - expected_archived_urls: Set[str], - known_ok_urls: Set[str], - keep_unchanged_urls: Set[str], -) -> Tuple[str, Optional[str]]: - if not e3sm_url: - return CLASS_NO_MAPPED_E3SM_URL, None - - if e3sm_url in known_ok_urls: - return CLASS_KNOWN_OK, url_to_status.get(e3sm_url) - - if e3sm_url in keep_unchanged_urls: - return CLASS_KEEP_UNCHANGED, url_to_status.get(e3sm_url) - - wordpress_status = url_to_status.get(e3sm_url) - - if wordpress_status is None: - return CLASS_PREDICTED_URL_NOT_IN_EXPORT, None - - if wordpress_status == "archive": - return CLASS_ARCHIVED, wordpress_status - - if e3sm_url in expected_archived_urls: - return CLASS_SHOULD_BE_ARCHIVED, wordpress_status - - if wordpress_status != "publish": - return CLASS_NOT_PUBLISHED, wordpress_status - - return CLASS_PUBLISHED, wordpress_status - - -def parse_wordpress_sensitive_terms_lines(lines: List[str]) -> List[Tuple[int, str]]: - parsed: List[Tuple[int, str]] = [] - - for raw_line in lines: - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - dict_start = line.find("{") - if dict_start == -1: - print(f"Skipping malformed WordPress sensitive-terms line: {line}") - continue - - dict_str = line[dict_start:].strip() - dict_data = parse_dict(dict_str) - if dict_data is None: - print(f"Skipping malformed dictionary in WordPress line: {line}") - continue - - total = int(sum(dict_data.values())) - parsed.append((total, line)) - - parsed.sort(key=lambda x: x[0], reverse=True) - return parsed - - -def extract_wordpress_url(line: str) -> Optional[str]: - dict_start = line.find("{") - if dict_start == -1: - return None - - prefix = line[:dict_start].rstrip() - if prefix.endswith(":"): - prefix = prefix[:-1].rstrip() - - return prefix - - -def parse_wordpress_record( - line: str, - url_to_status: Dict[str, str], - expected_archived_urls: Set[str], - known_ok_urls: Set[str], - keep_unchanged_urls: Set[str], -) -> Optional[SensitiveTermRecord]: - dict_start = line.find("{") - if dict_start == -1: - return None - - url = extract_wordpress_url(line) - if not url: - return None - - dict_str = line[dict_start:].strip() - term_counts = parse_dict(dict_str) - if term_counts is None: - return None - - total_terms = int(sum(term_counts.values())) - classification, wordpress_status = classify_e3sm_url( - e3sm_url=url, - url_to_status=url_to_status, - expected_archived_urls=expected_archived_urls, - known_ok_urls=known_ok_urls, - keep_unchanged_urls=keep_unchanged_urls, - ) - - return SensitiveTermRecord( - source="e3sm.org", - raw_line=line, - year_label="N/A", - year_int=None, - total_terms=total_terms, - term_counts=term_counts, - source_url=url, - title=None, - confluence_url=None, - e3sm_url=url, - classification=classification, - wordpress_status=wordpress_status, - ) - - -def extract_confluence_components( - line: str, -) -> Optional[Tuple[Optional[int], str, str, Dict[str, int]]]: - year, remainder = extract_year_and_remainder(line) - - dict_start = remainder.find("{") - if dict_start == -1: - print(f"Skipping malformed Confluence line: {line}") - return None - - dict_str = remainder[dict_start:].strip() - term_counts = parse_dict(dict_str) - if term_counts is None: - print(f"Skipping malformed dictionary in Confluence line: {line}") - return None - - prefix = remainder[:dict_start].rstrip() - if prefix.endswith("--"): - prefix = prefix[:-2].rstrip() - - first_colon = prefix.find(":") - if first_colon == -1: - print(f"Skipping malformed Confluence line: {line}") - return None - - page_id = prefix[:first_colon].strip() - title = prefix[first_colon + 1 :].strip() - - if not page_id.isdigit(): - print(f"Skipping Confluence line with non-numeric page id: {line}") - return None - - return year, page_id, title, term_counts - - -def parse_confluence_record( - line: str, - url_to_status: Dict[str, str], - expected_archived_urls: Set[str], - known_ok_urls: Set[str], - keep_unchanged_urls: Set[str], -) -> Optional[SensitiveTermRecord]: - components = extract_confluence_components(line) - if components is None: - return None - - year, page_id, title, term_counts = components - confluence_url = build_confluence_url(page_id) - - try: - e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) - except Exception as exc: - print( - f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" - ) - e3sm_url = None - - classification, wordpress_status = classify_e3sm_url( - e3sm_url=e3sm_url, - url_to_status=url_to_status, - expected_archived_urls=expected_archived_urls, - known_ok_urls=known_ok_urls, - keep_unchanged_urls=keep_unchanged_urls, - ) - - total_terms = int(sum(term_counts.values())) - year_label = str(year) if year is not None else "Unknown year" - - return SensitiveTermRecord( - source="confluence", - raw_line=line, - year_label=year_label, - year_int=year, - total_terms=total_terms, - term_counts=term_counts, - source_url=confluence_url, - title=title, - confluence_url=confluence_url, - e3sm_url=e3sm_url, - classification=classification, - wordpress_status=wordpress_status, - ) - - -def classification_sort_key(classification: str) -> Tuple[int, str]: - order = { - CLASS_PUBLISHED: 0, - CLASS_SHOULD_BE_ARCHIVED: 1, - CLASS_ARCHIVED: 2, - CLASS_NOT_PUBLISHED: 3, - CLASS_KNOWN_OK: 4, - CLASS_KEEP_UNCHANGED: 5, - CLASS_NO_MAPPED_E3SM_URL: 6, - CLASS_PREDICTED_URL_NOT_IN_EXPORT: 7, - } - return (order.get(classification, 999), classification) - - -def year_sort_key(year_str: str) -> Tuple[int, int]: - if year_str == "Unknown year": - return (1, 0) - if year_str == "N/A": - return (2, 0) - try: - return (0, -int(year_str)) - except ValueError: - return (3, 0) - - -def build_classification_summary( - records: List[SensitiveTermRecord], -) -> Dict[str, Dict[str, int]]: - summary: Dict[str, Dict[str, int]] = {} - - for record in records: - classification = record.classification - if classification not in summary: - summary[classification] = { - "total": 0, - "1": 0, - "2": 0, - "3": 0, - "4": 0, - "5+": 0, - } - - summary[classification]["total"] += 1 - if record.total_terms == 1: - summary[classification]["1"] += 1 - elif record.total_terms == 2: - summary[classification]["2"] += 1 - elif record.total_terms == 3: - summary[classification]["3"] += 1 - elif record.total_terms == 4: - summary[classification]["4"] += 1 - elif record.total_terms >= 5: - summary[classification]["5+"] += 1 - - return summary - - -def group_records_by_classification_and_year( - records: List[SensitiveTermRecord], -) -> Dict[str, Dict[str, List[SensitiveTermRecord]]]: - grouped: DefaultDict[str, DefaultDict[str, List[SensitiveTermRecord]]] = ( - defaultdict(lambda: defaultdict(list)) - ) - - for record in records: - grouped[record.classification][record.year_label].append(record) - - for classification in grouped: - for year_label in grouped[classification]: - grouped[classification][year_label].sort( - key=lambda r: (r.total_terms, r.e3sm_url or "", r.title or ""), - reverse=True, - ) - - return {k: dict(v) for k, v in grouped.items()} - - -def write_sensitive_terms_summary_table( - f: TextIO, records: List[SensitiveTermRecord] -) -> None: - summary = build_classification_summary(records) - - f.write("### Summary Table\n\n") - f.write("| Classification | Total | 1 | 2 | 3 | 4 | 5+ |\n") - f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") - - for classification in sorted(summary.keys(), key=classification_sort_key): - counts = summary[classification] - f.write( - f"| {classification} | {counts['total']} | {counts['1']} | {counts['2']} | " - f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" - ) - - f.write("\n") - - -def format_term_counts(term_counts: Dict[str, int]) -> str: - return str(term_counts) - - -def format_e3sm_record(record: SensitiveTermRecord) -> str: - url = record.e3sm_url or record.source_url or "UNKNOWN" - md = f"[{url}]({url})" - if record.wordpress_status: - md += f" (status: {record.wordpress_status})" - md += f" -- {format_term_counts(record.term_counts)}" - return md - - -def format_confluence_record(record: SensitiveTermRecord) -> str: - title = record.title or "Untitled" - confluence_url = record.confluence_url or record.source_url or "" - md = f"{title}: [confluence]({confluence_url})" - - if record.e3sm_url: - md += f" [e3sm.org]({record.e3sm_url})" - - try: - e3sm_url_status = get_e3sm_url_status(record.e3sm_url) - except Exception as exc: - print(f"Could not get e3sm.org URL status for {record.e3sm_url}: {exc}") - e3sm_url_status = None - - if e3sm_url_status: - md += f" (Note: {e3sm_url_status})" - - if record.wordpress_status: - md += f" (WordPress status: {record.wordpress_status})" - - md += f" -- {format_term_counts(record.term_counts)}" - return md - - -def write_sensitive_terms_section( - f: TextIO, - section_title: str, - description: str, - records: List[SensitiveTermRecord], - formatter: Callable[[SensitiveTermRecord], str], -) -> None: - f.write(f"## {section_title}\n\n") - f.write(f"{description}\n\n") - - write_sensitive_terms_summary_table(f, records) - - grouped = group_records_by_classification_and_year(records) - - for classification in sorted(grouped.keys(), key=classification_sort_key): - f.write(f"### {classification}\n\n") - - for year_label in sorted(grouped[classification].keys(), key=year_sort_key): - f.write(f"#### {year_label}\n\n") - for idx, record in enumerate(grouped[classification][year_label], start=1): - f.write(f"{idx}. {formatter(record)}\n") - f.write("\n") - - -def write_sensitive_terms_report( - output_path: str, - e3sm_records: List[SensitiveTermRecord], - confluence_records: List[SensitiveTermRecord], -) -> None: - description_e3sm_org = ( - "These are the currently reviewed e3sm.org pages that include sensitive terms. " - "Classification is derived from WordPress status, expected archived inputs, and the manual exception lists for known-ok and keep-unchanged paths." - ) - description_confluence = ( - "These are the Confluence pages that include sensitive terms. The confluence links are what the website reviewer scanned. " - "The e3sm.org links are predicted from Confluence mapping and then classified against the WordPress export." - ) - - with open(output_path, "w", encoding="utf-8") as f: - f.write("# Sensitive Terms Report\n\n") - - write_sensitive_terms_section( - f, - "e3sm.org", - description_e3sm_org, - e3sm_records, - format_e3sm_record, - ) - write_sensitive_terms_section( - f, - "Confluence", - description_confluence, - confluence_records, - format_confluence_record, - ) - def main(): pages_by_status: Dict[str, List[str]] = get_wordpress_urls_by_status( diff --git a/e3sm_comms/e3sm_org_reviewer/parsers.py b/e3sm_comms/e3sm_org_reviewer/parsers.py new file mode 100644 index 0000000..f1abdf7 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/parsers.py @@ -0,0 +1,197 @@ +import ast +from typing import Dict, List, Optional, Set, Tuple + +from e3sm_comms.e3sm_org_reviewer.classifiers import ( + classify_e3sm_url, + extract_year_and_remainder, +) +from e3sm_comms.e3sm_org_reviewer.confluence import build_confluence_url +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm + + +def parse_dict(dict_str: str) -> Optional[Dict[str, int]]: + try: + data = ast.literal_eval(dict_str) + except (SyntaxError, ValueError): + return None + + if not isinstance(data, dict): + return None + + try: + total = sum(data.values()) + except TypeError: + return None + + if not isinstance(total, (int, float)): + return None + + return data + + +def parse_wordpress_sensitive_terms_lines(lines: List[str]) -> List[Tuple[int, str]]: + parsed: List[Tuple[int, str]] = [] + + for raw_line in lines: + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + dict_start = line.find("{") + if dict_start == -1: + print(f"Skipping malformed WordPress sensitive-terms line: {line}") + continue + + dict_str = line[dict_start:].strip() + dict_data = parse_dict(dict_str) + if dict_data is None: + print(f"Skipping malformed dictionary in WordPress line: {line}") + continue + + total = int(sum(dict_data.values())) + parsed.append((total, line)) + + parsed.sort(key=lambda x: x[0], reverse=True) + return parsed + + +def parse_wordpress_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + dict_start = line.find("{") + if dict_start == -1: + return None + + url = extract_wordpress_url(line) + if not url: + return None + + dict_str = line[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + return None + + total_terms = int(sum(term_counts.values())) + classification, wordpress_status = classify_e3sm_url( + e3sm_url=url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + return SensitiveTermRecord( + source="e3sm.org", + raw_line=line, + year_label="N/A", + year_int=None, + total_terms=total_terms, + term_counts=term_counts, + source_url=url, + title=None, + confluence_url=None, + e3sm_url=url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def parse_confluence_record( + line: str, + url_to_status: Dict[str, str], + expected_archived_urls: Set[str], + known_ok_urls: Set[str], + keep_unchanged_urls: Set[str], +) -> Optional[SensitiveTermRecord]: + components = extract_confluence_components(line) + if components is None: + return None + + year, page_id, title, term_counts = components + confluence_url = build_confluence_url(page_id) + + try: + e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title) + except Exception as exc: + print( + f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}" + ) + e3sm_url = None + + classification, wordpress_status = classify_e3sm_url( + e3sm_url=e3sm_url, + url_to_status=url_to_status, + expected_archived_urls=expected_archived_urls, + known_ok_urls=known_ok_urls, + keep_unchanged_urls=keep_unchanged_urls, + ) + + total_terms = int(sum(term_counts.values())) + year_label = str(year) if year is not None else "Unknown year" + + return SensitiveTermRecord( + source="confluence", + raw_line=line, + year_label=year_label, + year_int=year, + total_terms=total_terms, + term_counts=term_counts, + source_url=confluence_url, + title=title, + confluence_url=confluence_url, + e3sm_url=e3sm_url, + classification=classification, + wordpress_status=wordpress_status, + ) + + +def extract_wordpress_url(line: str) -> Optional[str]: + dict_start = line.find("{") + if dict_start == -1: + return None + + prefix = line[:dict_start].rstrip() + if prefix.endswith(":"): + prefix = prefix[:-1].rstrip() + + return prefix + + +def extract_confluence_components( + line: str, +) -> Optional[Tuple[Optional[int], str, str, Dict[str, int]]]: + year, remainder = extract_year_and_remainder(line) + + dict_start = remainder.find("{") + if dict_start == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + dict_str = remainder[dict_start:].strip() + term_counts = parse_dict(dict_str) + if term_counts is None: + print(f"Skipping malformed dictionary in Confluence line: {line}") + return None + + prefix = remainder[:dict_start].rstrip() + if prefix.endswith("--"): + prefix = prefix[:-2].rstrip() + + first_colon = prefix.find(":") + if first_colon == -1: + print(f"Skipping malformed Confluence line: {line}") + return None + + page_id = prefix[:first_colon].strip() + title = prefix[first_colon + 1 :].strip() + + if not page_id.isdigit(): + print(f"Skipping Confluence line with non-numeric page id: {line}") + return None + + return year, page_id, title, term_counts diff --git a/e3sm_comms/e3sm_org_reviewer/readers.py b/e3sm_comms/e3sm_org_reviewer/readers.py new file mode 100644 index 0000000..6398f74 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/readers.py @@ -0,0 +1,44 @@ +import xml.etree.ElementTree as ET +from collections import defaultdict +from typing import Dict, List + + +def get_wordpress_urls_by_status( + xml_file_path: str, post_type: str +) -> Dict[str, List[str]]: + ns = { + "wp": "http://wordpress.org/export/1.2/", + } + + tree = ET.parse(xml_file_path) + root = tree.getroot() + + grouped: Dict[str, List[str]] = defaultdict(list) + channel = root.find("channel") + if channel is None: + return {} + + for item in channel.findall("item"): + item_post_type = item.find("wp:post_type", ns) + item_status = item.find("wp:status", ns) + link = item.find("link") + + if item_post_type is None or item_post_type.text != post_type: + continue + + status = ( + item_status.text.strip() + if item_status is not None and item_status.text + else "unknown" + ) + + if link is not None and link.text: + grouped[status].append(link.text.strip()) + + return {status: sorted(urls) for status, urls in sorted(grouped.items())} + + +def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: + for status in ["publish", "archive", "draft", "future", "pending", "private"]: + if status in all_urls_by_status: + print(f"Found {len(all_urls_by_status[status])} {status} URLs") diff --git a/e3sm_comms/e3sm_org_reviewer/record.py b/e3sm_comms/e3sm_org_reviewer/record.py new file mode 100644 index 0000000..d358a04 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/record.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass +from typing import Dict, Optional + + +@dataclass +class SensitiveTermRecord: + source: str # "e3sm.org" or "confluence" + raw_line: str + year_label: str + year_int: Optional[int] + total_terms: int + term_counts: Dict[str, int] + source_url: Optional[str] + title: Optional[str] + confluence_url: Optional[str] + e3sm_url: Optional[str] + classification: str + wordpress_status: Optional[str] diff --git a/e3sm_comms/e3sm_org_reviewer/reporters.py b/e3sm_comms/e3sm_org_reviewer/reporters.py new file mode 100644 index 0000000..d182e2b --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/reporters.py @@ -0,0 +1,428 @@ +from collections import defaultdict +from typing import Callable, DefaultDict, Dict, List, Set, TextIO + +from e3sm_comms.e3sm_org_reviewer.classifiers import ( + classification_sort_key, + year_sort_key, +) +from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord +from e3sm_comms.e3sm_org_reviewer.utils import ( + expand_patterns_to_urls, + get_all_urls, + get_status_counts_for_urls, +) +from e3sm_comms.page_reviewer.utils_base import get_e3sm_url_status + + +def write_markdown_report( + output_path: str, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + whitelisted_but_not_published: List[str], + published_but_not_whitelisted: List[str], + should_be_archived: List[str], + published_but_not_in_confluence: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], + incorrectly_accessible_non_published_urls: List[str], + invalid_whitelisted_paths: List[str], + invalid_expected_archived_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + write_summary_table( + f, + all_urls_by_status=all_urls_by_status, + valid_whitelisted_paths=valid_whitelisted_paths, + valid_expected_archived_paths=valid_expected_archived_paths, + valid_confluence_paths=valid_confluence_paths, + invalid_confluence_paths=invalid_confluence_paths, + confluence_unmapped_entries=confluence_unmapped_entries, + ) + + f.write("# Valid Paths\n\n") + write_markdown_section( + f, + "Whitelisted but not published", + whitelisted_but_not_published, + ) + write_markdown_section( + f, + "Published but not whitelisted", + published_but_not_whitelisted, + ) + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + write_markdown_section( + f, + "Published but no matching Confluence path found", + published_but_not_in_confluence, + ) + write_markdown_section( + f, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, + ) + write_markdown_section( + f, + "Non-published e3sm.org pages that are still accessible without login", + incorrectly_accessible_non_published_urls, + ) + + f.write("# Invalid Paths\n\n") + write_markdown_section( + f, + "Identified in whitelist input", + invalid_whitelisted_paths, + ) + write_markdown_section( + f, + "Identified in archive input", + invalid_expected_archived_paths, + ) + write_markdown_section( + f, + "Identified in Confluence input", + invalid_confluence_paths, + ) + write_markdown_section( + f, + "Confluence pages with no mappable e3sm.org URL", + confluence_unmapped_entries, + ) + + +def write_summary_table( + file_obj: TextIO, + all_urls_by_status: Dict[str, List[str]], + valid_whitelisted_paths: List[str], + valid_expected_archived_paths: List[str], + valid_confluence_paths: List[str], + invalid_confluence_paths: List[str], + confluence_unmapped_entries: List[str], +) -> None: + statuses: List[str] = sorted(all_urls_by_status.keys()) + all_urls: List[str] = get_all_urls(all_urls_by_status) + + whitelist_set: Set[str] = set( + expand_patterns_to_urls(valid_whitelisted_paths, all_urls) + ) + expected_archived_set: Set[str] = set( + expand_patterns_to_urls(valid_expected_archived_paths, all_urls) + ) + both_set: Set[str] = whitelist_set.intersection(expected_archived_set) + neither_set: Set[str] = set(all_urls) - whitelist_set.union(expected_archived_set) + + rows = [ + ("Whitelisted URLs", whitelist_set), + ("Expected archived", expected_archived_set), + ("Both whitelisted and expected archived", both_set), + ("Neither whitelisted nor expected archived", neither_set), + ("TOTAL", set(all_urls)), + ] + + file_obj.write("# Summary\n\n") + file_obj.write("| Type | " + " | ".join(statuses) + " | Total |\n") + file_obj.write("| --- | " + " | ".join("---" for _ in statuses) + " | --- |\n") + + for row_name, row_urls in rows: + counts = get_status_counts_for_urls( + urls=list(row_urls), + all_urls_by_status=all_urls_by_status, + statuses=statuses, + ) + total_count = sum(counts.values()) + file_obj.write( + f"| {row_name} | " + + " | ".join(str(counts[status]) for status in statuses) + + f" | {total_count} |\n" + ) + + confluence_valid_set: Set[str] = set(valid_confluence_paths) + all_urls_set: Set[str] = set(all_urls) + + e3sm_with_confluence: Set[str] = all_urls_set.intersection(confluence_valid_set) + e3sm_without_confluence: Set[str] = all_urls_set - confluence_valid_set + + confluence_not_valid_count: int = len(invalid_confluence_paths) + len( + confluence_unmapped_entries + ) + total_confluence_urls: int = len(confluence_valid_set) + confluence_not_valid_count + total_e3sm_urls: int = len(all_urls_set) + + confluence_counts_match: bool = ( + confluence_not_valid_count + len(e3sm_with_confluence) == total_confluence_urls + ) + e3sm_counts_match: bool = ( + len(e3sm_without_confluence) + len(e3sm_with_confluence) == total_e3sm_urls + ) + + file_obj.write("\n## Confluence Mapping Summary\n\n") + file_obj.write("| Type | Count |\n") + file_obj.write("| --- | --- |\n") + file_obj.write( + f"| Confluence paths that do not map to a valid e3sm.org path | {confluence_not_valid_count} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do not have a Confluence path associated with them | {len(e3sm_without_confluence)} |\n" + ) + file_obj.write( + f"| e3sm.org paths that do have a Confluence counterpart | {len(e3sm_with_confluence)} |\n" + ) + file_obj.write(f"| Total Confluence-derived paths | {total_confluence_urls} |\n") + file_obj.write(f"| Total e3sm.org paths | {total_e3sm_urls} |\n") + file_obj.write("\n") + + file_obj.write("Validation:\n\n") + file_obj.write( + f"- Confluence counts match: " + f"{confluence_not_valid_count} + {len(e3sm_with_confluence)} = {total_confluence_urls} " + f"({'yes' if confluence_counts_match else 'no'})\n" + ) + file_obj.write( + f"- e3sm.org counts match: " + f"{len(e3sm_without_confluence)} + {len(e3sm_with_confluence)} = {total_e3sm_urls} " + f"({'yes' if e3sm_counts_match else 'no'})\n\n" + ) + + +def write_markdown_section(file_obj: TextIO, title: str, items: List[str]) -> None: + file_obj.write(f"## {title}\n\n") + if not items: + file_obj.write("_None._\n\n") + return + + for i, item in enumerate(items, start=1): + file_obj.write(f"{i}. {item}\n") + file_obj.write("\n") + + +def write_action_items_report( + output_path: str, + should_be_archived: List[str], + published_not_whitelisted_and_not_in_confluence: List[str], + confluence_published_sensitive_records: List[SensitiveTermRecord], +) -> None: + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Action Items Report\n\n") + + f.write("## Summary\n\n") + f.write("| Action Area | Count |\n") + f.write("| --- | ---: |\n") + f.write( + f"| Expecting to be archived, but not yet archived | {len(should_be_archived)} |\n" + ) + f.write( + f"| Published but not whitelisted and no matching Confluence path found | {len(published_not_whitelisted_and_not_in_confluence)} |\n" + ) + f.write( + f"| Confluence pages with sensitive terms mapped to published e3sm.org pages | {len(confluence_published_sensitive_records)} |\n" + ) + f.write("\n") + + write_markdown_section( + f, + "Expecting to be archived, but not yet archived", + should_be_archived, + ) + + write_markdown_section( + f, + "Published but not whitelisted and no matching Confluence path found", + published_not_whitelisted_and_not_in_confluence, + ) + + write_action_items_confluence_section(f, confluence_published_sensitive_records) + + +def write_action_items_confluence_section( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + f.write( + "## Confluence pages with sensitive terms mapped to published e3sm.org pages\n\n" + ) + + if not records: + f.write("_None._\n\n") + return + + grouped = group_records_by_classification_and_year(records) + + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") + + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {format_confluence_record(record)}\n") + f.write("\n") + + +def write_sensitive_terms_report( + output_path: str, + e3sm_records: List[SensitiveTermRecord], + confluence_records: List[SensitiveTermRecord], +) -> None: + description_e3sm_org = ( + "These are the currently reviewed e3sm.org pages that include sensitive terms. " + "Classification is derived from WordPress status, expected archived inputs, and the manual exception lists for known-ok and keep-unchanged paths." + ) + description_confluence = ( + "These are the Confluence pages that include sensitive terms. The confluence links are what the website reviewer scanned. " + "The e3sm.org links are predicted from Confluence mapping and then classified against the WordPress export." + ) + + with open(output_path, "w", encoding="utf-8") as f: + f.write("# Sensitive Terms Report\n\n") + + write_sensitive_terms_section( + f, + "e3sm.org", + description_e3sm_org, + e3sm_records, + format_e3sm_record, + ) + write_sensitive_terms_section( + f, + "Confluence", + description_confluence, + confluence_records, + format_confluence_record, + ) + + +def write_sensitive_terms_section( + f: TextIO, + section_title: str, + description: str, + records: List[SensitiveTermRecord], + formatter: Callable[[SensitiveTermRecord], str], +) -> None: + f.write(f"## {section_title}\n\n") + f.write(f"{description}\n\n") + + write_sensitive_terms_summary_table(f, records) + + grouped = group_records_by_classification_and_year(records) + + for classification in sorted(grouped.keys(), key=classification_sort_key): + f.write(f"### {classification}\n\n") + + for year_label in sorted(grouped[classification].keys(), key=year_sort_key): + f.write(f"#### {year_label}\n\n") + for idx, record in enumerate(grouped[classification][year_label], start=1): + f.write(f"{idx}. {formatter(record)}\n") + f.write("\n") + + +def write_sensitive_terms_summary_table( + f: TextIO, records: List[SensitiveTermRecord] +) -> None: + summary = build_classification_summary(records) + + f.write("### Summary Table\n\n") + f.write("| Classification | Total | 1 | 2 | 3 | 4 | 5+ |\n") + f.write("| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n") + + for classification in sorted(summary.keys(), key=classification_sort_key): + counts = summary[classification] + f.write( + f"| {classification} | {counts['total']} | {counts['1']} | {counts['2']} | " + f"{counts['3']} | {counts['4']} | {counts['5+']} |\n" + ) + + f.write("\n") + + +def format_e3sm_record(record: SensitiveTermRecord) -> str: + url = record.e3sm_url or record.source_url or "UNKNOWN" + md = f"[{url}]({url})" + if record.wordpress_status: + md += f" (status: {record.wordpress_status})" + md += f" -- {format_term_counts(record.term_counts)}" + return md + + +def format_confluence_record(record: SensitiveTermRecord) -> str: + title = record.title or "Untitled" + confluence_url = record.confluence_url or record.source_url or "" + md = f"{title}: [confluence]({confluence_url})" + + if record.e3sm_url: + md += f" [e3sm.org]({record.e3sm_url})" + + try: + e3sm_url_status = get_e3sm_url_status(record.e3sm_url) + except Exception as exc: + print(f"Could not get e3sm.org URL status for {record.e3sm_url}: {exc}") + e3sm_url_status = None + + if e3sm_url_status: + md += f" (Note: {e3sm_url_status})" + + if record.wordpress_status: + md += f" (WordPress status: {record.wordpress_status})" + + md += f" -- {format_term_counts(record.term_counts)}" + return md + + +def format_term_counts(term_counts: Dict[str, int]) -> str: + return str(term_counts) + + +def build_classification_summary( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, int]]: + summary: Dict[str, Dict[str, int]] = {} + + for record in records: + classification = record.classification + if classification not in summary: + summary[classification] = { + "total": 0, + "1": 0, + "2": 0, + "3": 0, + "4": 0, + "5+": 0, + } + + summary[classification]["total"] += 1 + if record.total_terms == 1: + summary[classification]["1"] += 1 + elif record.total_terms == 2: + summary[classification]["2"] += 1 + elif record.total_terms == 3: + summary[classification]["3"] += 1 + elif record.total_terms == 4: + summary[classification]["4"] += 1 + elif record.total_terms >= 5: + summary[classification]["5+"] += 1 + + return summary + + +def group_records_by_classification_and_year( + records: List[SensitiveTermRecord], +) -> Dict[str, Dict[str, List[SensitiveTermRecord]]]: + grouped: DefaultDict[str, DefaultDict[str, List[SensitiveTermRecord]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + + for record in records: + grouped[record.classification][record.year_label].append(record) + + for classification in grouped: + for year_label in grouped[classification]: + grouped[classification][year_label].sort( + key=lambda r: (r.total_terms, r.e3sm_url or "", r.title or ""), + reverse=True, + ) + + return {k: dict(v) for k, v in grouped.items()} diff --git a/e3sm_comms/e3sm_org_reviewer/utils.py b/e3sm_comms/e3sm_org_reviewer/utils.py new file mode 100644 index 0000000..14ad571 --- /dev/null +++ b/e3sm_comms/e3sm_org_reviewer/utils.py @@ -0,0 +1,101 @@ +from collections import defaultdict +from typing import Dict, List, Set + + +def matches_pattern(pattern: str, url: str) -> bool: + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + prefix = pattern[:-1] + return url.startswith(prefix) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: + matched_urls: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url): + matched_urls.add(url) + return sorted(matched_urls) + + +def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: + invalid_patterns: List[str] = [] + for pattern in patterns: + if not any(matches_pattern(pattern, url) for url in all_urls): + invalid_patterns.append(pattern) + return sorted(invalid_patterns) + + +def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: + return sorted(set(list1) - set(list2)) + + +def get_all_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + all_urls: List[str] = [] + for urls in urls_by_status.values(): + all_urls.extend(urls) + return sorted(all_urls) + + +def get_all_non_published_urls(urls_by_status: Dict[str, List[str]]) -> List[str]: + non_published_urls: List[str] = [] + for status, urls in urls_by_status.items(): + if status != "publish": + non_published_urls.extend(urls) + return sorted(non_published_urls) + + +def get_total_count(urls_by_status: Dict[str, List[str]]) -> int: + return sum(len(urls) for urls in urls_by_status.values()) + + +def get_combined_urls_by_status( + pages_by_status: Dict[str, List[str]], posts_by_status: Dict[str, List[str]] +) -> Dict[str, List[str]]: + merged: Dict[str, List[str]] = defaultdict(list) + for source in (pages_by_status, posts_by_status): + for status, urls in source.items(): + merged[status].extend(urls) + return {status: sorted(urls) for status, urls in sorted(merged.items())} + + +def get_status_counts_for_urls( + urls: List[str], all_urls_by_status: Dict[str, List[str]], statuses: List[str] +) -> Dict[str, int]: + url_set = set(urls) + counts: Dict[str, int] = {} + for status in statuses: + counts[status] = len(url_set.intersection(all_urls_by_status.get(status, []))) + return counts + + +def read_nonempty_lines(file_path: str) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + +def build_url_to_status(all_urls_by_status: Dict[str, List[str]]) -> Dict[str, str]: + result: Dict[str, str] = {} + for status, urls in all_urls_by_status.items(): + for url in urls: + result[url] = status + return result From 223f226ec90688518626de936adcfb90db7a0bb8 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 16:46:12 -0700 Subject: [PATCH 76/85] Update example scripts --- examples/review_terms.bash | 3 +-- examples/review_xml.bash | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/review_terms.bash b/examples/review_terms.bash index 01dceec..d99abbe 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -2,12 +2,11 @@ # WordPress: Tools > Export > export pages (wordpress_pages.xml) and posts (wordpress_posts.xml) # Copy those XMLs into /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/ # e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/whitelisted_web_pages.txt -# Also confirm confluence_top_levels_partial.txt is the list of top levels you want to use, otherwise switch it out. IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io echo "Count of top-level Confluence pages:" -wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_partial.txt # Excludes MODEL, RESEARCH, DATA +wc -l ${IO_DIR}/input/website_reviewer/confluence_top_levels_ALL.txt echo "Count of whitelisted e3sm.org pages:" wc -l ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt diff --git a/examples/review_xml.bash b/examples/review_xml.bash index ba09088..f220c4f 100755 --- a/examples/review_xml.bash +++ b/examples/review_xml.bash @@ -5,7 +5,7 @@ # scp wordpress_pages.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_pages.xml # scp wordpress_posts.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_posts.xml -# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io//input/exported_xml_reviewer/whitelisted_web_pages.txt +# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/whitelisted_web_pages.txt IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io @@ -23,5 +23,10 @@ e3sm-comms-website-reviewer echo "Step 2. Review xml exported from e3sm.org" cp ${IO_DIR}/output/website_reviewer/hierarchical_outline.txt ${IO_DIR}/input/exported_xml_reviewer/hierarchical_outline.txt -e3sm-comms-exported-xml-reviewer -echo "Output report: ${IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" +e3sm-comms-exported-xml-reviewer --use-confluence --use-whitelist +echo "Output reports:" +echo "1. ${IO_DIR}/output/exported_xml_reviewer/wordpress_sensitive_terms_report.md" +echo "2. ${IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.txt" +echo "3. ${IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" +echo "4. ${IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" +echo "5. ${IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" From 782b1f312aac84793b951cfb7c8e566a5ecb69fe Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 1 Jun 2026 17:48:16 -0700 Subject: [PATCH 77/85] Factor out common code --- e3sm_comms/e3sm_org_reviewer/confluence.py | 31 +-- e3sm_comms/e3sm_org_reviewer/main.py | 24 +- e3sm_comms/e3sm_org_reviewer/readers.py | 37 --- e3sm_comms/e3sm_org_reviewer/reporters.py | 7 +- e3sm_comms/e3sm_org_reviewer/utils.py | 50 +--- e3sm_comms/exported_xml_reviewer/README.md | 7 +- e3sm_comms/exported_xml_reviewer/builders.py | 16 +- .../exported_xml_reviewer/confluence.py | 33 +-- .../exported_xml_reviewer/link_analysis.py | 3 +- e3sm_comms/exported_xml_reviewer/readers.py | 136 +--------- e3sm_comms/exported_xml_reviewer/reporters.py | 2 +- e3sm_comms/exported_xml_reviewer/utils.py | 63 +---- e3sm_comms/page_reviewer/utils_base.py | 13 +- e3sm_comms/utils.py | 241 ++++++++++++++++++ 14 files changed, 283 insertions(+), 380 deletions(-) diff --git a/e3sm_comms/e3sm_org_reviewer/confluence.py b/e3sm_comms/e3sm_org_reviewer/confluence.py index 884b22e..70e4d9c 100644 --- a/e3sm_comms/e3sm_org_reviewer/confluence.py +++ b/e3sm_comms/e3sm_org_reviewer/confluence.py @@ -1,6 +1,7 @@ from typing import List, Tuple from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.utils import parse_confluence_hierarchy_file CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -10,36 +11,6 @@ def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" -def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: - parsed: List[Tuple[str, str]] = [] - - with open(input_file, "r", encoding="utf-8") as f: - for line_number, raw_line in enumerate(f, start=1): - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - stripped = line.lstrip() - - if ":" not in stripped: - print(f"Skipping malformed Confluence line {line_number}: {line}") - continue - - page_id, title = stripped.split(":", 1) - page_id = page_id.strip() - title = title.strip() - - if not page_id.isdigit(): - print( - f"Skipping Confluence line {line_number} with non-numeric page id: {line}" - ) - continue - - parsed.append((page_id, title)) - - return parsed - - def get_confluence_predicted_e3sm_urls( input_file: str, ) -> Tuple[List[str], List[str]]: diff --git a/e3sm_comms/e3sm_org_reviewer/main.py b/e3sm_comms/e3sm_org_reviewer/main.py index 47f1ed3..0efb7bf 100644 --- a/e3sm_comms/e3sm_org_reviewer/main.py +++ b/e3sm_comms/e3sm_org_reviewer/main.py @@ -7,10 +7,7 @@ parse_wordpress_record, parse_wordpress_sensitive_terms_lines, ) -from e3sm_comms.e3sm_org_reviewer.readers import ( - get_wordpress_urls_by_status, - print_status_counts, -) +from e3sm_comms.e3sm_org_reviewer.readers import print_status_counts from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord from e3sm_comms.e3sm_org_reviewer.reporters import ( write_action_items_report, @@ -19,17 +16,20 @@ ) from e3sm_comms.e3sm_org_reviewer.utils import ( build_url_to_status, - expand_patterns_to_urls, get_all_non_published_urls, get_all_urls, get_combined_urls_by_status, - get_invalid_patterns, get_list_difference, get_total_count, - read_nonempty_lines, ) from e3sm_comms.page_reviewer.utils_base import LinkedURLs, get_e3sm_url_status -from e3sm_comms.utils import IO_DIR +from e3sm_comms.utils import ( + IO_DIR, + expand_patterns_to_urls, + get_invalid_patterns, + get_wordpress_urls_by_status, + read_lines, +) # From WordPress under Tools > Export: INPUT_XML_PAGES: str = f"{IO_DIR}/input/e3sm_org_reviewer/wordpress_pages.xml" @@ -87,12 +87,12 @@ def main(): non_published_urls: List[str] = get_all_non_published_urls(all_urls_by_status) print(f"Total non-published URLs: {len(non_published_urls)}") - list_whitelisted_paths: List[str] = read_nonempty_lines(INPUT_WHITELIST) - list_expected_archived_paths: List[str] = read_nonempty_lines( + list_whitelisted_paths: List[str] = read_lines(INPUT_WHITELIST) + list_expected_archived_paths: List[str] = read_lines( INPUT_EXPECTED_ARCHIVED_E3SM_ORG_PATHS ) - list_known_ok_paths: List[str] = read_nonempty_lines(INPUT_KNOWN_OK_E3SM_ORG_PATHS) - list_keep_unchanged_paths: List[str] = read_nonempty_lines( + list_known_ok_paths: List[str] = read_lines(INPUT_KNOWN_OK_E3SM_ORG_PATHS) + list_keep_unchanged_paths: List[str] = read_lines( INPUT_KEEP_UNCHANGED_E3SM_ORG_PATHS ) diff --git a/e3sm_comms/e3sm_org_reviewer/readers.py b/e3sm_comms/e3sm_org_reviewer/readers.py index 6398f74..3214274 100644 --- a/e3sm_comms/e3sm_org_reviewer/readers.py +++ b/e3sm_comms/e3sm_org_reviewer/readers.py @@ -1,43 +1,6 @@ -import xml.etree.ElementTree as ET -from collections import defaultdict from typing import Dict, List -def get_wordpress_urls_by_status( - xml_file_path: str, post_type: str -) -> Dict[str, List[str]]: - ns = { - "wp": "http://wordpress.org/export/1.2/", - } - - tree = ET.parse(xml_file_path) - root = tree.getroot() - - grouped: Dict[str, List[str]] = defaultdict(list) - channel = root.find("channel") - if channel is None: - return {} - - for item in channel.findall("item"): - item_post_type = item.find("wp:post_type", ns) - item_status = item.find("wp:status", ns) - link = item.find("link") - - if item_post_type is None or item_post_type.text != post_type: - continue - - status = ( - item_status.text.strip() - if item_status is not None and item_status.text - else "unknown" - ) - - if link is not None and link.text: - grouped[status].append(link.text.strip()) - - return {status: sorted(urls) for status, urls in sorted(grouped.items())} - - def print_status_counts(all_urls_by_status: Dict[str, List[str]]) -> None: for status in ["publish", "archive", "draft", "future", "pending", "private"]: if status in all_urls_by_status: diff --git a/e3sm_comms/e3sm_org_reviewer/reporters.py b/e3sm_comms/e3sm_org_reviewer/reporters.py index d182e2b..acadbe2 100644 --- a/e3sm_comms/e3sm_org_reviewer/reporters.py +++ b/e3sm_comms/e3sm_org_reviewer/reporters.py @@ -6,12 +6,9 @@ year_sort_key, ) from e3sm_comms.e3sm_org_reviewer.record import SensitiveTermRecord -from e3sm_comms.e3sm_org_reviewer.utils import ( - expand_patterns_to_urls, - get_all_urls, - get_status_counts_for_urls, -) +from e3sm_comms.e3sm_org_reviewer.utils import get_all_urls, get_status_counts_for_urls from e3sm_comms.page_reviewer.utils_base import get_e3sm_url_status +from e3sm_comms.utils import expand_patterns_to_urls def write_markdown_report( diff --git a/e3sm_comms/e3sm_org_reviewer/utils.py b/e3sm_comms/e3sm_org_reviewer/utils.py index 14ad571..3c72a75 100644 --- a/e3sm_comms/e3sm_org_reviewer/utils.py +++ b/e3sm_comms/e3sm_org_reviewer/utils.py @@ -1,48 +1,5 @@ from collections import defaultdict -from typing import Dict, List, Set - - -def matches_pattern(pattern: str, url: str) -> bool: - if "*" not in pattern: - return pattern == url - - if pattern.count("*") == 1 and pattern.endswith("*"): - prefix = pattern[:-1] - return url.startswith(prefix) - - parts = pattern.split("*") - position = 0 - for i, part in enumerate(parts): - if not part: - continue - found_at = url.find(part, position) - if found_at == -1: - return False - if i == 0 and not pattern.startswith("*") and found_at != 0: - return False - position = found_at + len(part) - - if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): - return False - - return True - - -def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> List[str]: - matched_urls: Set[str] = set() - for pattern in patterns: - for url in all_urls: - if matches_pattern(pattern, url): - matched_urls.add(url) - return sorted(matched_urls) - - -def get_invalid_patterns(patterns: List[str], all_urls: List[str]) -> List[str]: - invalid_patterns: List[str] = [] - for pattern in patterns: - if not any(matches_pattern(pattern, url) for url in all_urls): - invalid_patterns.append(pattern) - return sorted(invalid_patterns) +from typing import Dict, List def get_list_difference(list1: List[str], list2: List[str]) -> List[str]: @@ -88,11 +45,6 @@ def get_status_counts_for_urls( return counts -def read_nonempty_lines(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - return [line.strip() for line in f if line.strip()] - - def build_url_to_status(all_urls_by_status: Dict[str, List[str]]) -> Dict[str, str]: result: Dict[str, str] = {} for status, urls in all_urls_by_status.items(): diff --git a/e3sm_comms/exported_xml_reviewer/README.md b/e3sm_comms/exported_xml_reviewer/README.md index 069f167..f14df1f 100644 --- a/e3sm_comms/exported_xml_reviewer/README.md +++ b/e3sm_comms/exported_xml_reviewer/README.md @@ -4,7 +4,6 @@ It is important to not introduce circular dependencies. To avoid this, the dependency hierarchy is listed below: - Level 1: `main.py` -- Level 2: `builders.py`, -- Level 3: `confluence.py`, `link_analysis.py`, -- Level 4: `readers.py`, -- Level 5: `utils.py` +- Level 2: `builders.py` +- Level 3: `confluence.py`, `link_analysis.py`, `readers.py` +- Level 4: `utils.py` diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py index ec3d915..e016075 100644 --- a/e3sm_comms/exported_xml_reviewer/builders.py +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -10,8 +10,6 @@ extract_internal_e3sm_links, ) from e3sm_comms.exported_xml_reviewer.readers import ( - WordpressItem, - parse_wordpress_xml, read_known_ok_links, read_requested_links, read_sensitive_terms, @@ -20,11 +18,15 @@ from e3sm_comms.exported_xml_reviewer.utils import ( count_sensitive_terms, display_status, - expand_patterns_to_urls, normalize_status, - normalize_url, strip_html, ) +from e3sm_comms.utils import ( + WordpressItem, + expand_patterns_to_urls, + normalize_url, + parse_wordpress_xml_items, +) @dataclass @@ -95,13 +97,13 @@ def build_records( known_ok_urls = read_known_ok_links(known_ok_links_file) raw_items: List[WordpressItem] = [] - raw_items.extend(parse_wordpress_xml(xml_pages, "page")) - raw_items.extend(parse_wordpress_xml(xml_posts, "post")) + raw_items.extend(parse_wordpress_xml_items(xml_pages, "page")) + raw_items.extend(parse_wordpress_xml_items(xml_posts, "post")) all_urls = [item.url for item in raw_items if item.url] if whitelist_file: whitelist_patterns = read_whitelist_patterns(whitelist_file) - whitelisted_urls = expand_patterns_to_urls(whitelist_patterns, all_urls) + whitelisted_urls = set(expand_patterns_to_urls(whitelist_patterns, all_urls)) else: whitelisted_urls = set(all_urls) diff --git a/e3sm_comms/exported_xml_reviewer/confluence.py b/e3sm_comms/exported_xml_reviewer/confluence.py index ee68c70..fd7db0f 100644 --- a/e3sm_comms/exported_xml_reviewer/confluence.py +++ b/e3sm_comms/exported_xml_reviewer/confluence.py @@ -1,9 +1,9 @@ from __future__ import annotations -from typing import Dict, List, Tuple +from typing import Dict -from e3sm_comms.exported_xml_reviewer.utils import normalize_url from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm +from e3sm_comms.utils import normalize_url, parse_confluence_hierarchy_file CONFLUENCE_SPACE = "EPWCD" CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki" @@ -30,34 +30,5 @@ def get_confluence_mapping(input_file: str) -> Dict[str, str]: return mapping -def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: - parsed: List[Tuple[str, str]] = [] - - with open(input_file, "r", encoding="utf-8") as f: - for line_number, raw_line in enumerate(f, start=1): - line = raw_line.rstrip("\n") - if not line.strip(): - continue - - stripped = line.lstrip() - if ":" not in stripped: - print(f"Skipping malformed Confluence line {line_number}: {line}") - continue - - page_id, title = stripped.split(":", 1) - page_id = page_id.strip() - title = title.strip() - - if not page_id.isdigit(): - print( - f"Skipping Confluence line {line_number} with non-numeric page id: {line}" - ) - continue - - parsed.append((page_id, title)) - - return parsed - - def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str: return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}" diff --git a/e3sm_comms/exported_xml_reviewer/link_analysis.py b/e3sm_comms/exported_xml_reviewer/link_analysis.py index fe44b0a..dfcafe0 100644 --- a/e3sm_comms/exported_xml_reviewer/link_analysis.py +++ b/e3sm_comms/exported_xml_reviewer/link_analysis.py @@ -8,13 +8,12 @@ import requests # type: ignore -from e3sm_comms.exported_xml_reviewer.readers import WordpressItem from e3sm_comms.exported_xml_reviewer.utils import ( display_status, is_legacy_content_url, normalize_status, - normalize_url, ) +from e3sm_comms.utils import WordpressItem, normalize_url @dataclass diff --git a/e3sm_comms/exported_xml_reviewer/readers.py b/e3sm_comms/exported_xml_reviewer/readers.py index 9e21f6b..adb822e 100644 --- a/e3sm_comms/exported_xml_reviewer/readers.py +++ b/e3sm_comms/exported_xml_reviewer/readers.py @@ -1,28 +1,13 @@ from __future__ import annotations import csv -import xml.etree.ElementTree as ET -from dataclasses import dataclass from typing import List, Set, Tuple -from e3sm_comms.exported_xml_reviewer.utils import normalize_url - - -@dataclass -class WordpressItem: - post_id: str - post_parent: str - post_type: str - title: str - url: str - status: str - body: str +from e3sm_comms.utils import normalize_url, read_lines def read_sensitive_terms(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - terms = [line.strip().lower() for line in f if line.strip()] - return sorted(set(terms)) + return sorted(set(read_lines(file_path, lowercase=True))) def read_known_ok_links(file_path: str) -> Set[str]: @@ -86,119 +71,4 @@ def read_requested_links(file_path: str) -> List[Tuple[str, str]]: def read_whitelist_patterns(file_path: str) -> List[str]: - with open(file_path, "r", encoding="utf-8") as f: - return [line.strip() for line in f if line.strip()] - - -def parse_wordpress_xml( - xml_file_path: str, expected_post_type: str -) -> List[WordpressItem]: - ns = { - "wp": "http://wordpress.org/export/1.2/", - } - - tree = ET.parse(xml_file_path) - root = tree.getroot() - - channel = root.find("channel") - if channel is None: - return [] - - items: List[WordpressItem] = [] - - for item in channel.findall("item"): - post_type_elem = item.find("wp:post_type", ns) - if post_type_elem is None: - continue - - post_type_text = (post_type_elem.text or "").strip() - if post_type_text != expected_post_type: - continue - - title_elem = item.find("title") - link_elem = item.find("link") - status_elem = item.find("wp:status", ns) - post_id_elem = item.find("wp:post_id", ns) - post_parent_elem = item.find("wp:post_parent", ns) - - title = ( - title_elem.text.strip() - if title_elem is not None and title_elem.text is not None - else "Untitled" - ) - link = ( - normalize_url(link_elem.text) - if link_elem is not None and link_elem.text is not None - else "" - ) - status = ( - status_elem.text.strip() - if status_elem is not None and status_elem.text is not None - else "unknown" - ) - post_id = ( - post_id_elem.text.strip() - if post_id_elem is not None and post_id_elem.text is not None - else "" - ) - post_parent = ( - post_parent_elem.text.strip() - if post_parent_elem is not None and post_parent_elem.text is not None - else "0" - ) - body = extract_item_body(item) - - items.append( - WordpressItem( - post_id=post_id, - post_parent=post_parent, - post_type=post_type_text, - title=title, - url=link, - status=status, - body=body, - ) - ) - - return items - - -def extract_item_body(item: ET.Element) -> str: - ns = { - "wp": "http://wordpress.org/export/1.2/", - "content": "http://purl.org/rss/1.0/modules/content/", - "excerpt": "http://wordpress.org/export/1.2/excerpt/", - } - - body_parts: List[str] = [] - - content_elem = item.find("content:encoded", ns) - if content_elem is not None and content_elem.text and content_elem.text.strip(): - body_parts.append(content_elem.text.strip()) - - excerpt_elem = item.find("excerpt:encoded", ns) - if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): - body_parts.append(excerpt_elem.text.strip()) - - for postmeta in item.findall("wp:postmeta", ns): - meta_key_elem = postmeta.find("wp:meta_key", ns) - meta_value_elem = postmeta.find("wp:meta_value", ns) - - meta_key = ( - meta_key_elem.text.strip() - if meta_key_elem is not None and meta_key_elem.text - else "" - ) - meta_value = ( - meta_value_elem.text.strip() - if meta_value_elem is not None and meta_value_elem.text - else "" - ) - - if not meta_value: - continue - - if meta_key.endswith("_free_form_content") and not meta_key.startswith("_"): - body_parts.append(meta_value) - - return "\n".join(body_parts) + return read_lines(file_path) diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index 9fc654b..845cfb4 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -16,8 +16,8 @@ InvalidInternalLinkGroup, NonPublishedInternalLinkGroup, ) -from e3sm_comms.exported_xml_reviewer.readers import WordpressItem from e3sm_comms.exported_xml_reviewer.utils import display_status, normalize_status +from e3sm_comms.utils import WordpressItem def write_terms_report( diff --git a/e3sm_comms/exported_xml_reviewer/utils.py b/e3sm_comms/exported_xml_reviewer/utils.py index 7749fe7..9f34350 100644 --- a/e3sm_comms/exported_xml_reviewer/utils.py +++ b/e3sm_comms/exported_xml_reviewer/utils.py @@ -1,22 +1,8 @@ from __future__ import annotations import re -from typing import Dict, List, Optional, Set -from urllib.parse import urlsplit, urlunsplit - - -def normalize_url(url: str) -> str: - url = url.strip() - if not url: - return "" - - parts = urlsplit(url) - scheme = parts.scheme.lower() - netloc = parts.netloc.lower() - path = parts.path.rstrip("/") - - normalized = urlunsplit((scheme, netloc, path, "", "")) - return normalized +from typing import Dict, List, Optional +from urllib.parse import urlsplit def normalize_status(raw_status: Optional[str]) -> str: @@ -54,51 +40,6 @@ def strip_html(text: str) -> str: return text.strip() -def matches_pattern(pattern: str, url: str) -> bool: - if "*" not in pattern: - return normalize_url(pattern) == normalize_url(url) - - normalized_url = normalize_url(url) - normalized_pattern = normalize_url(pattern) - - if "*" not in normalized_pattern: - return normalized_pattern == normalized_url - - if normalized_pattern.count("*") == 1 and normalized_pattern.endswith("*"): - prefix = normalized_pattern[:-1] - return normalized_url.startswith(prefix) - - parts = normalized_pattern.split("*") - position = 0 - for i, part in enumerate(parts): - if not part: - continue - found_at = normalized_url.find(part, position) - if found_at == -1: - return False - if i == 0 and not normalized_pattern.startswith("*") and found_at != 0: - return False - position = found_at + len(part) - - if ( - not normalized_pattern.endswith("*") - and parts[-1] - and not normalized_url.endswith(parts[-1]) - ): - return False - - return True - - -def expand_patterns_to_urls(patterns: List[str], all_urls: List[str]) -> Set[str]: - matched_urls: Set[str] = set() - for pattern in patterns: - for url in all_urls: - if matches_pattern(pattern, url): - matched_urls.add(url) - return matched_urls - - def is_legacy_content_url(url: str) -> bool: parts = urlsplit(url) slug = parts.path.strip("/").lower() diff --git a/e3sm_comms/page_reviewer/utils_base.py b/e3sm_comms/page_reviewer/utils_base.py index ee1625e..3c9d8a8 100644 --- a/e3sm_comms/page_reviewer/utils_base.py +++ b/e3sm_comms/page_reviewer/utils_base.py @@ -10,6 +10,8 @@ from bs4 import BeautifulSoup from requests.auth import HTTPBasicAuth # type: ignore +from e3sm_comms.utils import count_sensitive_terms + # Classes ##################################################################### # Set these values in newsletter_review/main.py, resource_reviewer/main.py, website_reviewer/main.py @@ -262,14 +264,9 @@ def split_html(raw_html: str) -> Tuple[ParsedHTML, Optional[ParsedHTML]]: def find_sensitive_terms( list_sensitive_terms: List[str], lowercase_text: str ) -> Dict[str, int]: - result = {} - for term in list_sensitive_terms: - pattern = re.escape(term) - matches = re.findall(pattern, lowercase_text) - count = len(matches) - if count > 0: - result[term] = count - return result + return count_sensitive_terms( + lowercase_text, list_sensitive_terms, whole_words_only=False + ) def remove_output_files(config: Config): diff --git a/e3sm_comms/utils.py b/e3sm_comms/utils.py index 817a879..aa680da 100644 --- a/e3sm_comms/utils.py +++ b/e3sm_comms/utils.py @@ -1,5 +1,246 @@ +import re +import xml.etree.ElementTree as ET +from collections import defaultdict +from dataclasses import dataclass +from typing import Dict, List, Set, Tuple +from urllib.parse import urlsplit, urlunsplit + +# IO DIR ###################################################################### # Chrysalis # IO_DIR="/home/ac.forsyth2/ez/e3sm-comms-io" # Perlmutter IO_DIR = "/global/homes/f/forsyth/ez/e3sm-comms-io" + +# Confluence ################################################################## + + +def parse_confluence_hierarchy_file(input_file: str) -> List[Tuple[str, str]]: + parsed: List[Tuple[str, str]] = [] + + with open(input_file, "r", encoding="utf-8") as f: + for line_number, raw_line in enumerate(f, start=1): + line = raw_line.rstrip("\n") + if not line.strip(): + continue + + stripped = line.lstrip() + if ":" not in stripped: + print(f"Skipping malformed Confluence line {line_number}: {line}") + continue + + page_id, title = stripped.split(":", 1) + page_id = page_id.strip() + title = title.strip() + + if not page_id.isdigit(): + print( + f"Skipping Confluence line {line_number} with non-numeric page id: {line}" + ) + continue + + parsed.append((page_id, title)) + + return parsed + + +# Pattern matching ############################################################ + + +def normalize_url(url: str) -> str: + url = url.strip() + if not url: + return "" + parts = urlsplit(url) + scheme = parts.scheme.lower() + netloc = parts.netloc.lower() + path = parts.path.rstrip("/") + return urlunsplit((scheme, netloc, path, "", "")) + + +def matches_pattern(pattern: str, url: str, normalize: bool = True) -> bool: + if normalize: + pattern = normalize_url(pattern) + url = normalize_url(url) + + if "*" not in pattern: + return pattern == url + + if pattern.count("*") == 1 and pattern.endswith("*"): + return url.startswith(pattern[:-1]) + + parts = pattern.split("*") + position = 0 + for i, part in enumerate(parts): + if not part: + continue + found_at = url.find(part, position) + if found_at == -1: + return False + if i == 0 and not pattern.startswith("*") and found_at != 0: + return False + position = found_at + len(part) + + if not pattern.endswith("*") and parts[-1] and not url.endswith(parts[-1]): + return False + + return True + + +def expand_patterns_to_urls( + patterns: List[str], all_urls: List[str], normalize: bool = True +) -> List[str]: + matched: Set[str] = set() + for pattern in patterns: + for url in all_urls: + if matches_pattern(pattern, url, normalize=normalize): + matched.add(url) + return sorted(matched) + + +def get_invalid_patterns( + patterns: List[str], all_urls: List[str], normalize: bool = True +) -> List[str]: + return sorted( + pattern + for pattern in patterns + if not any( + matches_pattern(pattern, url, normalize=normalize) for url in all_urls + ) + ) + + +# Sensitive term counting ##################################################### + + +def count_sensitive_terms( + text: str, terms: List[str], whole_words_only: bool = True +) -> Dict[str, int]: + counts: Dict[str, int] = {} + for term in terms: + escaped = re.escape(term) + pattern = rf"\b{escaped}\b" if whole_words_only else escaped + matches = re.findall(pattern, text) + if matches: + counts[term] = len(matches) + return counts + + +# WordPress XML parsing ####################################################### + +WORDPRESS_NS = {"wp": "http://wordpress.org/export/1.2/"} + + +@dataclass +class WordpressItem: + post_id: str + post_parent: str + post_type: str + title: str + url: str + status: str + body: str + + +def _get_item_text(item: ET.Element, tag: str, ns: bool = True) -> str: + elem = item.find(tag, WORDPRESS_NS) if ns else item.find(tag) + return elem.text.strip() if elem is not None and elem.text else "" + + +def parse_wordpress_xml_items( + xml_file_path: str, post_type: str +) -> List[WordpressItem]: + tree = ET.parse(xml_file_path) + root = tree.getroot() + channel = root.find("channel") + if channel is None: + return [] + + items: List[WordpressItem] = [] + for item in channel.findall("item"): + post_type_elem = item.find("wp:post_type", WORDPRESS_NS) + if post_type_elem is None or (post_type_elem.text or "").strip() != post_type: + continue + + items.append( + WordpressItem( + post_id=_get_item_text(item, "wp:post_id"), + post_parent=_get_item_text(item, "wp:post_parent") or "0", + post_type=post_type, + title=_get_item_text(item, "title", ns=False) or "Untitled", + url=normalize_url(_get_item_text(item, "link", ns=False)), + status=_get_item_text(item, "wp:status") or "unknown", + body=_extract_item_body(item), + ) + ) + + return items + + +def get_wordpress_urls_by_status( + xml_file_path: str, post_type: str +) -> Dict[str, List[str]]: + grouped: Dict[str, List[str]] = defaultdict(list) + for item in parse_wordpress_xml_items(xml_file_path, post_type): + if item.url: + grouped[item.status].append(item.url) + return {status: sorted(urls) for status, urls in sorted(grouped.items())} + + +def _extract_item_body(item: ET.Element) -> str: + ns = { + "wp": "http://wordpress.org/export/1.2/", + "content": "http://purl.org/rss/1.0/modules/content/", + "excerpt": "http://wordpress.org/export/1.2/excerpt/", + } + body_parts: List[str] = [] + + content_elem = item.find("content:encoded", ns) + if content_elem is not None and content_elem.text and content_elem.text.strip(): + body_parts.append(content_elem.text.strip()) + + excerpt_elem = item.find("excerpt:encoded", ns) + if excerpt_elem is not None and excerpt_elem.text and excerpt_elem.text.strip(): + body_parts.append(excerpt_elem.text.strip()) + + for postmeta in item.findall("wp:postmeta", ns): + meta_key_elem = postmeta.find("wp:meta_key", ns) + meta_value_elem = postmeta.find("wp:meta_value", ns) + meta_key = ( + meta_key_elem.text.strip() + if meta_key_elem is not None and meta_key_elem.text + else "" + ) + meta_value = ( + meta_value_elem.text.strip() + if meta_value_elem is not None and meta_value_elem.text + else "" + ) + if ( + meta_value + and meta_key.endswith("_free_form_content") + and not meta_key.startswith("_") + ): + body_parts.append(meta_value) + + return "\n".join(body_parts) + + +# File reading ################################################################ + + +def read_lines( + file_path: str, + strip: bool = True, + skip_empty: bool = True, + lowercase: bool = False, +) -> List[str]: + with open(file_path, "r", encoding="utf-8") as f: + lines = [line.rstrip("\n") for line in f] + if strip: + lines = [line.strip() for line in lines] + if skip_empty: + lines = [line for line in lines if line] + if lowercase: + lines = [line.lower() for line in lines] + return lines From 26a2c958eb693aa35afa2022f13a992856cf2546 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 24 Jun 2026 10:42:27 -0700 Subject: [PATCH 78/85] Claude-generated code to check external links --- e3sm_comms/exported_xml_reviewer/builders.py | 90 ++++++++++++++ .../exported_xml_reviewer/link_analysis.py | 75 ++++++++++++ e3sm_comms/exported_xml_reviewer/main.py | 14 +++ e3sm_comms/exported_xml_reviewer/reporters.py | 111 ++++++++++++++++++ 4 files changed, 290 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py index e016075..d9bac8f 100644 --- a/e3sm_comms/exported_xml_reviewer/builders.py +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -6,7 +6,9 @@ from e3sm_comms.exported_xml_reviewer.confluence import get_confluence_mapping from e3sm_comms.exported_xml_reviewer.link_analysis import ( + check_external_link, check_redirect_target, + extract_external_links, extract_internal_e3sm_links, ) from e3sm_comms.exported_xml_reviewer.readers import ( @@ -74,6 +76,16 @@ class PublishedContentLinkSummary: valid_links: List[str] +@dataclass +class ExternalContentLinkSummary: + title: str + url: str + not_found_links: List[str] + timed_out_links: List[str] + security_error_links: List[str] + valid_links: List[str] + + def build_records( xml_pages: str, xml_posts: str, @@ -299,6 +311,84 @@ def build_published_content_link_summaries( return summaries +def build_external_content_link_summaries( + items: List[WordpressItem], + post_type: str, +) -> List[ExternalContentLinkSummary]: + """ + Mirror of build_published_content_link_summaries, but for external links. + + Deduplicates URLs across all items before fetching so each external URL + is checked exactly once. + """ + if post_type == "page": + ordered_items = get_page_hierarchy_order(items) + else: + ordered_items = sorted( + [item for item in items if item.post_type == post_type and item.post_id], + key=lambda x: x.title.lower(), + ) + + # Collect all unique external URLs first to avoid redundant fetches. + all_external_urls: Set[str] = set() + item_to_external_urls: Dict[str, Set[str]] = {} + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + found = extract_external_links(item.body) + item_to_external_urls[item.url] = found + all_external_urls.update(found) + + # Check each unique URL once. + url_results: Dict[str, str] = {} + for ext_url in sorted(all_external_urls): + url_results[ext_url] = check_external_link(ext_url).status + + summaries: List[ExternalContentLinkSummary] = [] + + for item in ordered_items: + if normalize_status(item.status) != "published": + continue + if not item.url or not item.body: + continue + + ext_urls = item_to_external_urls.get(item.url, set()) + if not ext_urls: + continue + + not_found: Set[str] = set() + timed_out: Set[str] = set() + security_error: Set[str] = set() + valid: Set[str] = set() + + for ext_url in ext_urls: + status = url_results.get(ext_url, "not_found") + if status == "valid": + valid.add(ext_url) + elif status == "timed_out": + timed_out.add(ext_url) + elif status == "security_error": + security_error.add(ext_url) + else: + not_found.add(ext_url) + + summaries.append( + ExternalContentLinkSummary( + title=item.title, + url=item.url, + not_found_links=sorted(not_found), + timed_out_links=sorted(timed_out), + security_error_links=sorted(security_error), + valid_links=sorted(valid), + ) + ) + + return summaries + + def build_requested_link_records( requested_links_file: str, raw_items: List[WordpressItem], diff --git a/e3sm_comms/exported_xml_reviewer/link_analysis.py b/e3sm_comms/exported_xml_reviewer/link_analysis.py index dfcafe0..b3433b5 100644 --- a/e3sm_comms/exported_xml_reviewer/link_analysis.py +++ b/e3sm_comms/exported_xml_reviewer/link_analysis.py @@ -15,6 +15,8 @@ ) from e3sm_comms.utils import WordpressItem, normalize_url +_EXTERNAL_TIMEOUT = 15 # seconds + @dataclass class InvalidInternalLinkGroup: @@ -36,6 +38,14 @@ class NonPublishedInternalLinkGroup: referenced_on_non_published: List[Tuple[str, str]] +@dataclass +class ExternalLinkResult: + """Outcome of checking a single external URL.""" + + url: str + status: str # "valid" | "not_found" | "timed_out" | "security_error" + + def extract_internal_e3sm_links(html_text: str) -> Set[str]: links: Set[str] = set() @@ -96,6 +106,71 @@ def check_redirect_target(link_url: str) -> Tuple[str, str]: return "", "" +def extract_external_links(html_text: str) -> Set[str]: + """Return all non-e3sm.org absolute hrefs found in *html_text*.""" + links: Set[str] = set() + for match in re.finditer( + r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE + ): + href = match.group(1).strip() + if not href or href.startswith("#") or href.startswith("mailto:"): + continue + try: + parts = urlsplit(href) + except ValueError: + continue + if not parts.scheme or not parts.netloc: + continue + if parts.netloc.lower().endswith("e3sm.org"): + continue + links.add(href) + return links + + +def check_external_link(url: str) -> ExternalLinkResult: + """ + HEAD-then-GET a URL and return an ExternalLinkResult. + + Outcomes + -------- + "valid" – 2xx response (or safe redirect to one) + "not_found" – 4xx / 5xx response, or connection error + "timed_out" – requests.Timeout + "security_error" – SSL error (mirrors Firefox "potential security risk ahead") + """ + headers = {"User-Agent": "Mozilla/5.0 (compatible; e3sm-link-checker/1.0)"} + try: + try: + resp = requests.head( + url, + timeout=_EXTERNAL_TIMEOUT, + allow_redirects=True, + headers=headers, + ) + if resp.status_code == 405: + raise requests.exceptions.HTTPError("HEAD not allowed") + except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError): + resp = requests.get( + url, + timeout=_EXTERNAL_TIMEOUT, + allow_redirects=True, + headers=headers, + stream=True, + ) + resp.close() + + if resp.ok: + return ExternalLinkResult(url=url, status="valid") + return ExternalLinkResult(url=url, status="not_found") + + except requests.exceptions.Timeout: + return ExternalLinkResult(url=url, status="timed_out") + except requests.exceptions.SSLError: + return ExternalLinkResult(url=url, status="security_error") + except requests.exceptions.RequestException: + return ExternalLinkResult(url=url, status="not_found") + + def infer_likely_new_link(linked_url: str) -> str: parts = urlsplit(linked_url) path = parts.path.rstrip("/").lower() diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 952662d..57ffbd3 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -3,6 +3,7 @@ import argparse from e3sm_comms.exported_xml_reviewer.builders import ( + build_external_content_link_summaries, build_navigation_issue_records, build_published_content_link_summaries, build_records, @@ -12,6 +13,7 @@ build_non_published_internal_link_groups, ) from e3sm_comms.exported_xml_reviewer.reporters import ( + write_external_links_report, write_hierarchical_outline, write_invalid_internal_links_report, write_navigation_issues_report, @@ -55,6 +57,9 @@ OUTPUT_PUBLISHED_PAGES_LINK_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" ) +OUTPUT_EXTERNAL_LINKS_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_external_links_report.md" +) def parse_args() -> argparse.Namespace: @@ -134,6 +139,14 @@ def main() -> None: published_post_link_summaries, ) + external_page_summaries = build_external_content_link_summaries(raw_items, "page") + external_post_summaries = build_external_content_link_summaries(raw_items, "post") + write_external_links_report( + OUTPUT_EXTERNAL_LINKS_REPORT, + external_page_summaries, + external_post_summaries, + ) + print(f"Wrote report to {OUTPUT_TERMS_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") @@ -141,6 +154,7 @@ def main() -> None: f"Wrote invalid internal links report to {OUTPUT_INVALID_INTERNAL_LINKS_REPORT}" ) print(f"Wrote published pages link report to {OUTPUT_PUBLISHED_PAGES_LINK_REPORT}") + print(f"Wrote external links report to {OUTPUT_EXTERNAL_LINKS_REPORT}") if __name__ == "__main__": diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index 845cfb4..a827ac3 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -6,6 +6,7 @@ from e3sm_comms.exported_xml_reviewer.builders import ( ArchivedParentPublishedChildIssue, + ExternalContentLinkSummary, PublishedContentLinkSummary, ReportRecord, RequestedLinkRecord, @@ -378,6 +379,116 @@ def write_section( write_section(f, "Published Posts", post_summaries) +def write_external_links_report( + output_path: str, + page_summaries: List[ExternalContentLinkSummary], + post_summaries: List[ExternalContentLinkSummary], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + def render_link_list(urls: List[str]) -> str: + return ", ".join(f"[{url}]({url})" for url in urls) + + def write_section( + f, + section_title: str, + summaries: List[ExternalContentLinkSummary], + ) -> None: + invalid_summaries = [ + s + for s in summaries + if s.not_found_links or s.timed_out_links or s.security_error_links + ] + valid_only_summaries = [ + s + for s in summaries + if not (s.not_found_links or s.timed_out_links or s.security_error_links) + ] + + f.write(f"## {section_title}\n\n") + + if not summaries: + f.write("No published items with external links found.\n\n") + return + + f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") + + if invalid_summaries: + f.write( + "| Published item | Link not found | Link timed out | Security error | Valid link |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + + not_found_total = 0 + timed_out_total = 0 + security_total = 0 + valid_total = 0 + + not_found_unique: Set[str] = set() + timed_out_unique: Set[str] = set() + security_unique: Set[str] = set() + valid_unique: Set[str] = set() + + for s in invalid_summaries: + item_md = f"[{s.title}]({s.url})" + not_found_md = render_link_list(s.not_found_links) + timed_out_md = render_link_list(s.timed_out_links) + security_md = render_link_list(s.security_error_links) + valid_md = render_link_list(s.valid_links) + + not_found_total += len(s.not_found_links) + timed_out_total += len(s.timed_out_links) + security_total += len(s.security_error_links) + valid_total += len(s.valid_links) + + not_found_unique.update(s.not_found_links) + timed_out_unique.update(s.timed_out_links) + security_unique.update(s.security_error_links) + valid_unique.update(s.valid_links) + + f.write( + f"| {item_md} | {not_found_md} | {timed_out_md} | {security_md} | {valid_md} |\n" + ) + + f.write( + f"| Total link count | {not_found_total} | {timed_out_total} | {security_total} | {valid_total} |\n" + ) + f.write( + f"| Unique link count | {len(not_found_unique)} | {len(timed_out_unique)} | {len(security_unique)} | {len(valid_unique)} |\n" + ) + else: + f.write("No items with invalid external links found.\n") + + f.write("\n") + f.write(f"### Items with no invalid links ({len(valid_only_summaries)})\n\n") + + if valid_only_summaries: + f.write("| Published item | Valid external link count |\n") + f.write("| --- | ---: |\n") + + total_links = 0 + unique_links: Set[str] = set() + + for s in valid_only_summaries: + item_md = f"[{s.title}]({s.url})" + f.write(f"| {item_md} | {len(s.valid_links)} |\n") + total_links += len(s.valid_links) + unique_links.update(s.valid_links) + + f.write(f"| Total link count | {total_links} |\n") + f.write(f"| Unique link count | {len(unique_links)} |\n") + else: + f.write("No items with only valid external links found.\n") + + f.write("\n") + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Published Content External Links Report\n\n") + write_section(f, "Published Pages", page_summaries) + write_section(f, "Published Posts", post_summaries) + + def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) From 23708b3a206846069ffe2a211230b97d62b45335 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 24 Jun 2026 10:52:07 -0700 Subject: [PATCH 79/85] Add comment explaining inputs --- e3sm_comms/exported_xml_reviewer/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 57ffbd3..75267a8 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -29,6 +29,7 @@ # Required inputs: INPUT_XML_PAGES: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_pages.xml" INPUT_XML_POSTS: str = f"{IO_DIR}/input/exported_xml_reviewer/wordpress_posts.xml" +# These 3 inputs are only used for wordpress_sensitive_terms_report.md: INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" From 5d147f4e28aa965536e8b10a21350bd911fbcf61 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 24 Jun 2026 12:18:36 -0700 Subject: [PATCH 80/85] Fix link extraction --- e3sm_comms/exported_xml_reviewer/link_analysis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/link_analysis.py b/e3sm_comms/exported_xml_reviewer/link_analysis.py index b3433b5..2b11832 100644 --- a/e3sm_comms/exported_xml_reviewer/link_analysis.py +++ b/e3sm_comms/exported_xml_reviewer/link_analysis.py @@ -107,7 +107,6 @@ def check_redirect_target(link_url: str) -> Tuple[str, str]: def extract_external_links(html_text: str) -> Set[str]: - """Return all non-e3sm.org absolute hrefs found in *html_text*.""" links: Set[str] = set() for match in re.finditer( r'href=["\']([^"\']+)["\']', html_text, flags=re.IGNORECASE @@ -119,7 +118,11 @@ def extract_external_links(html_text: str) -> Set[str]: parts = urlsplit(href) except ValueError: continue - if not parts.scheme or not parts.netloc: + if parts.scheme not in ("http", "https"): # changed from `if not parts.scheme` + continue + if not parts.netloc: + continue + if " " in parts.netloc: continue if parts.netloc.lower().endswith("e3sm.org"): continue From 40dab1f6b61bb8aa66853a321f3b805e0c7f3983 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 24 Jun 2026 17:03:50 -0700 Subject: [PATCH 81/85] Claude-generated code to skip known inaccessible links --- e3sm_comms/exported_xml_reviewer/builders.py | 18 +++++++++-- e3sm_comms/exported_xml_reviewer/main.py | 14 +++++++-- e3sm_comms/exported_xml_reviewer/readers.py | 15 +++++++++ e3sm_comms/exported_xml_reviewer/reporters.py | 31 ++++++++++++++----- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py index d9bac8f..8ac47d1 100644 --- a/e3sm_comms/exported_xml_reviewer/builders.py +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -83,6 +83,7 @@ class ExternalContentLinkSummary: not_found_links: List[str] timed_out_links: List[str] security_error_links: List[str] + inaccessible_links: List[str] valid_links: List[str] @@ -314,12 +315,16 @@ def build_published_content_link_summaries( def build_external_content_link_summaries( items: List[WordpressItem], post_type: str, + inaccessible_prefixes: Tuple[str, ...] = (), ) -> List[ExternalContentLinkSummary]: """ Mirror of build_published_content_link_summaries, but for external links. Deduplicates URLs across all items before fetching so each external URL is checked exactly once. + + URLs whose prefix matches any entry in `inaccessible_prefixes` are skipped + entirely (no network request) and reported in the "inaccessible" column. """ if post_type == "page": ordered_items = get_page_hierarchy_order(items) @@ -342,10 +347,15 @@ def build_external_content_link_summaries( item_to_external_urls[item.url] = found all_external_urls.update(found) - # Check each unique URL once. + # Check each unique URL once, skipping known-inaccessible prefixes. url_results: Dict[str, str] = {} for ext_url in sorted(all_external_urls): - url_results[ext_url] = check_external_link(ext_url).status + if inaccessible_prefixes and any( + ext_url.startswith(prefix) for prefix in inaccessible_prefixes + ): + url_results[ext_url] = "inaccessible" + else: + url_results[ext_url] = check_external_link(ext_url).status summaries: List[ExternalContentLinkSummary] = [] @@ -362,6 +372,7 @@ def build_external_content_link_summaries( not_found: Set[str] = set() timed_out: Set[str] = set() security_error: Set[str] = set() + inaccessible: Set[str] = set() valid: Set[str] = set() for ext_url in ext_urls: @@ -372,6 +383,8 @@ def build_external_content_link_summaries( timed_out.add(ext_url) elif status == "security_error": security_error.add(ext_url) + elif status == "inaccessible": + inaccessible.add(ext_url) else: not_found.add(ext_url) @@ -382,6 +395,7 @@ def build_external_content_link_summaries( not_found_links=sorted(not_found), timed_out_links=sorted(timed_out), security_error_links=sorted(security_error), + inaccessible_links=sorted(inaccessible), valid_links=sorted(valid), ) ) diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index 75267a8..b3218cc 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -12,6 +12,7 @@ build_invalid_internal_link_groups, build_non_published_internal_link_groups, ) +from e3sm_comms.exported_xml_reviewer.readers import read_inaccessible_prefixes from e3sm_comms.exported_xml_reviewer.reporters import ( write_external_links_report, write_hierarchical_outline, @@ -33,6 +34,9 @@ INPUT_SEARCH_PHRASES: str = f"{IO_DIR}/input/shared/sensitive_terms.txt" INPUT_REQUESTED_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/requested_links.csv" INPUT_KNOWN_OK_LINKS: str = f"{IO_DIR}/input/exported_xml_reviewer/known_ok_links.txt" +INPUT_INACCESSIBLE_PREFIXES: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/inaccessible_prefixes.txt" +) # Optional inputs: DEFAULT_CONFLUENCE_HIERARCHY: str = ( @@ -86,6 +90,8 @@ def main() -> None: ) input_whitelist = DEFAULT_WHITELIST if args.use_whitelist else "" + inaccessible_prefixes = read_inaccessible_prefixes(INPUT_INACCESSIBLE_PREFIXES) + records, status_totals, requested_link_records, raw_items = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, @@ -140,8 +146,12 @@ def main() -> None: published_post_link_summaries, ) - external_page_summaries = build_external_content_link_summaries(raw_items, "page") - external_post_summaries = build_external_content_link_summaries(raw_items, "post") + external_page_summaries = build_external_content_link_summaries( + raw_items, "page", inaccessible_prefixes + ) + external_post_summaries = build_external_content_link_summaries( + raw_items, "post", inaccessible_prefixes + ) write_external_links_report( OUTPUT_EXTERNAL_LINKS_REPORT, external_page_summaries, diff --git a/e3sm_comms/exported_xml_reviewer/readers.py b/e3sm_comms/exported_xml_reviewer/readers.py index adb822e..12816a1 100644 --- a/e3sm_comms/exported_xml_reviewer/readers.py +++ b/e3sm_comms/exported_xml_reviewer/readers.py @@ -15,6 +15,21 @@ def read_known_ok_links(file_path: str) -> Set[str]: return {normalize_url(line.strip()) for line in f if line.strip()} +def read_inaccessible_prefixes(file_path: str) -> Tuple[str, ...]: + """ + Read a plain-text file of URL prefixes (one per line) that are known to + block automated access. Blank lines and lines starting with '#' are + ignored. Returns a tuple suitable for use with str.startswith(). + """ + with open(file_path, "r", encoding="utf-8") as f: + prefixes = [ + line.strip() + for line in f + if line.strip() and not line.strip().startswith("#") + ] + return tuple(prefixes) + + def read_requested_links(file_path: str) -> List[Tuple[str, str]]: rows: List[Tuple[str, str]] = [] diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index a827ac3..652d70f 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -398,12 +398,20 @@ def write_section( invalid_summaries = [ s for s in summaries - if s.not_found_links or s.timed_out_links or s.security_error_links + if s.not_found_links + or s.timed_out_links + or s.security_error_links + or s.inaccessible_links ] valid_only_summaries = [ s for s in summaries - if not (s.not_found_links or s.timed_out_links or s.security_error_links) + if not ( + s.not_found_links + or s.timed_out_links + or s.security_error_links + or s.inaccessible_links + ) ] f.write(f"## {section_title}\n\n") @@ -416,18 +424,20 @@ def write_section( if invalid_summaries: f.write( - "| Published item | Link not found | Link timed out | Security error | Valid link |\n" + "| Published item | Link not found | Link timed out | Security error | Known inaccessible to script | Valid link |\n" ) - f.write("| --- | --- | --- | --- | --- |\n") + f.write("| --- | --- | --- | --- | --- | --- |\n") not_found_total = 0 timed_out_total = 0 security_total = 0 + inaccessible_total = 0 valid_total = 0 not_found_unique: Set[str] = set() timed_out_unique: Set[str] = set() security_unique: Set[str] = set() + inaccessible_unique: Set[str] = set() valid_unique: Set[str] = set() for s in invalid_summaries: @@ -435,27 +445,30 @@ def write_section( not_found_md = render_link_list(s.not_found_links) timed_out_md = render_link_list(s.timed_out_links) security_md = render_link_list(s.security_error_links) + inaccessible_md = render_link_list(s.inaccessible_links) valid_md = render_link_list(s.valid_links) not_found_total += len(s.not_found_links) timed_out_total += len(s.timed_out_links) security_total += len(s.security_error_links) + inaccessible_total += len(s.inaccessible_links) valid_total += len(s.valid_links) not_found_unique.update(s.not_found_links) timed_out_unique.update(s.timed_out_links) security_unique.update(s.security_error_links) + inaccessible_unique.update(s.inaccessible_links) valid_unique.update(s.valid_links) f.write( - f"| {item_md} | {not_found_md} | {timed_out_md} | {security_md} | {valid_md} |\n" + f"| {item_md} | {not_found_md} | {timed_out_md} | {security_md} | {inaccessible_md} | {valid_md} |\n" ) f.write( - f"| Total link count | {not_found_total} | {timed_out_total} | {security_total} | {valid_total} |\n" + f"| Total link count | {not_found_total} | {timed_out_total} | {security_total} | {inaccessible_total} | {valid_total} |\n" ) f.write( - f"| Unique link count | {len(not_found_unique)} | {len(timed_out_unique)} | {len(security_unique)} | {len(valid_unique)} |\n" + f"| Unique link count | {len(not_found_unique)} | {len(timed_out_unique)} | {len(security_unique)} | {len(inaccessible_unique)} | {len(valid_unique)} |\n" ) else: f.write("No items with invalid external links found.\n") @@ -494,9 +507,11 @@ def write_hierarchical_outline(output_path: str, items: List[WordpressItem]) -> output_file.parent.mkdir(parents=True, exist_ok=True) def write_section(f, section_items: List[WordpressItem], heading: str) -> None: + from collections import defaultdict as _defaultdict + section_items = [item for item in section_items if item.post_id] - children_by_parent: DefaultDict[str, List[WordpressItem]] = defaultdict(list) + children_by_parent: DefaultDict[str, List[WordpressItem]] = _defaultdict(list) item_by_id: Dict[str, WordpressItem] = {} for item in section_items: From faa7ca81f85e2066ff56ca97015bc5e31402793f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Wed, 24 Jun 2026 17:04:37 -0700 Subject: [PATCH 82/85] Update docs for external links report --- README.md | 4 ++-- examples/review_xml.bash | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 56fb045..fb86eba 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ This package is for implementing the software needs of the E3SM Communications t - input: - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages - - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages - - output: 5 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md` + - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages, txt file of links known to be inaccessible for automated review + - output: 5 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`, (6) `wordpress_invalid_external_links_report.md` ### Confluence API commands (require Confluence token) diff --git a/examples/review_xml.bash b/examples/review_xml.bash index f220c4f..ea93169 100755 --- a/examples/review_xml.bash +++ b/examples/review_xml.bash @@ -30,3 +30,4 @@ echo "2. ${IO_DIR}/output/exported_xml_reviewer/wordpress_hierarchical_outline.t echo "3. ${IO_DIR}/output/exported_xml_reviewer/wordpress_navigation_issues_report.md" echo "4. ${IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_internal_links_report.md" echo "5. ${IO_DIR}/output/exported_xml_reviewer/wordpress_published_pages_link_report.md" +echo "6. ${IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_external_links_report.md" From 0d3938c1d9cdac39428a5ac0434652b65a5dcfd8 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 25 Jun 2026 10:33:41 -0700 Subject: [PATCH 83/85] Update inaccessible link description --- e3sm_comms/exported_xml_reviewer/reporters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index 652d70f..ccdd4f5 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -423,6 +423,9 @@ def write_section( f.write(f"### Items with invalid links ({len(invalid_summaries)})\n\n") if invalid_summaries: + f.write( + "Known inaccessible to script: these are likely accessible manually, or are fake Lorem Ipsum links\n" + ) f.write( "| Published item | Link not found | Link timed out | Security error | Known inaccessible to script | Valid link |\n" ) From 519896865194930cbd728c8b2e8ace6d9004ba22 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Thu, 2 Jul 2026 10:10:44 -0700 Subject: [PATCH 84/85] Address Copilot review comments --- README.md | 4 ++-- e3sm_comms/exported_xml_reviewer/confluence.py | 6 ------ e3sm_comms/page_reviewer/confluence_page_reviewer.py | 7 ++++++- e3sm_comms/utils.py | 5 +++++ examples/review_terms.bash | 5 +++-- examples/review_xml.bash | 7 ++++--- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fb86eba..4c564d5 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ This package is for implementing the software needs of the E3SM Communications t `e3sm-comms-exported-xml-reviewer` - input: - - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts + - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. NOTE: Only ever use the XML files downloaded directly from WordPress; the source must be trusted. - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages, txt file of links known to be inaccessible for automated review - - output: 5 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`, (6) `wordpress_invalid_external_links_report.md` +- output: 6 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`, (6) `wordpress_invalid_external_links_report.md` ### Confluence API commands (require Confluence token) diff --git a/e3sm_comms/exported_xml_reviewer/confluence.py b/e3sm_comms/exported_xml_reviewer/confluence.py index fd7db0f..215e616 100644 --- a/e3sm_comms/exported_xml_reviewer/confluence.py +++ b/e3sm_comms/exported_xml_reviewer/confluence.py @@ -12,12 +12,6 @@ def get_confluence_mapping(input_file: str) -> Dict[str, str]: mapping: Dict[str, str] = {} - if map_confluence_to_e3sm is None: - print( - "Warning: map_confluence_to_e3sm is not available, Confluence mapping will be skipped." - ) - return mapping - for page_id, title in parse_confluence_hierarchy_file(input_file): confluence_url = build_confluence_url(page_id) try: diff --git a/e3sm_comms/page_reviewer/confluence_page_reviewer.py b/e3sm_comms/page_reviewer/confluence_page_reviewer.py index b80e1da..7e7ccff 100644 --- a/e3sm_comms/page_reviewer/confluence_page_reviewer.py +++ b/e3sm_comms/page_reviewer/confluence_page_reviewer.py @@ -119,7 +119,12 @@ def extract_data_from_page( def extract_data_from_content_url( credentials: ConfluenceCredentials, page: ConfluencePage ): - data = get_json(credentials, page.page_id, page.content_url) + data = get_json( + credentials, + page.page_id, + page.content_url, + params={"expand": "version,history"}, + ) if "title" not in data: raise RuntimeError( diff --git a/e3sm_comms/utils.py b/e3sm_comms/utils.py index aa680da..385d984 100644 --- a/e3sm_comms/utils.py +++ b/e3sm_comms/utils.py @@ -5,6 +5,11 @@ from typing import Dict, List, Set, Tuple from urllib.parse import urlsplit, urlunsplit +# Note: to use a more hardened XML library, instead use: +# import defusedxml.ElementTree as ET +# And add defusedxml to conda/dev.yml. +# However, we are only using XML files downloaded directly from WordPress. + # IO DIR ###################################################################### # Chrysalis # IO_DIR="/home/ac.forsyth2/ez/e3sm-comms-io" diff --git a/examples/review_terms.bash b/examples/review_terms.bash index d99abbe..acf6b35 100755 --- a/examples/review_terms.bash +++ b/examples/review_terms.bash @@ -1,8 +1,9 @@ # Before running: # WordPress: Tools > Export > export pages (wordpress_pages.xml) and posts (wordpress_posts.xml) -# Copy those XMLs into /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/ -# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/e3sm_org_reviewer/whitelisted_web_pages.txt +# Copy those XMLs into ${IO_DIR}/input/e3sm_org_reviewer/ +# e3sm.org > CMP Settings > CMP Advanced Setup: copy the list of pages to ${IO_DIR}/input/e3sm_org_reviewer/whitelisted_web_pages.txt +# Replace this with your IO dir! IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io echo "Count of top-level Confluence pages:" diff --git a/examples/review_xml.bash b/examples/review_xml.bash index ea93169..0b1507b 100755 --- a/examples/review_xml.bash +++ b/examples/review_xml.bash @@ -2,11 +2,12 @@ # WordPress: Tools > Export > export pages # WordPress: Tools > Export > export posts -# scp wordpress_pages.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_pages.xml -# scp wordpress_posts.xml forsyth@perlmutter.nersc.gov:/global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/wordpress_posts.xml +# scp wordpress_pages.xml user@host:/path/to/e3sm-comms-io/input/exported_xml_reviewer/wordpress_pages.xml +# scp wordpress_posts.xml user@host:/path/to/e3sm-comms-io/input/exported_xml_reviewer/wordpress_posts.xml -# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to /global/homes/f/forsyth/ez/e3sm-comms-io/input/exported_xml_reviewer/whitelisted_web_pages.txt +# WordPress: CMP Settings > CMP Advanced Setup: copy the list of pages to ${IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt +# Replace this with your IO dir! IO_DIR=/global/homes/f/forsyth/ez/e3sm-comms-io echo "Count of top-level Confluence pages:" From 8512e8c58eb79a7ee4be64a57e743e7c590dffd6 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth <forsyth2@llnl.gov> Date: Mon, 10 Aug 2026 17:06:33 -0700 Subject: [PATCH 85/85] Port remaining e3sm_org_reviewer checks into exported_xml_reviewer exported_xml_reviewer replaces most of e3sm_org_reviewer, but four checks added during e3sm_org_reviewer's rewrite were never carried over. Ports them, each opt-in so default behavior is unchanged: - --use-expected-archived: pages expected to be archived that aren't - --use-keep-unchanged: exception list for pages we won't touch - --use-confluence (existing flag): flags published/whitelisted pages with no matching Confluence page - --check-non-published-access: live check that non-published pages are actually inaccessible First three add sections to wordpress_navigation_issues_report.md; the last gets its own wordpress_non_published_accessibility_report.md. Not ported: e3sm_org_reviewer's live-page term scan, since scanning the WordPress XML export directly is the more reliable source. e3sm_org_reviewer now has no unique capability left and can be dropped. --- e3sm_comms/exported_xml_reviewer/builders.py | 139 +++++++++++++++++- e3sm_comms/exported_xml_reviewer/main.py | 61 +++++++- e3sm_comms/exported_xml_reviewer/readers.py | 9 ++ e3sm_comms/exported_xml_reviewer/reporters.py | 63 ++++++++ 4 files changed, 270 insertions(+), 2 deletions(-) diff --git a/e3sm_comms/exported_xml_reviewer/builders.py b/e3sm_comms/exported_xml_reviewer/builders.py index 8ac47d1..9f69963 100644 --- a/e3sm_comms/exported_xml_reviewer/builders.py +++ b/e3sm_comms/exported_xml_reviewer/builders.py @@ -12,6 +12,8 @@ extract_internal_e3sm_links, ) from e3sm_comms.exported_xml_reviewer.readers import ( + read_expected_archived_patterns, + read_keep_unchanged_links, read_known_ok_links, read_requested_links, read_sensitive_terms, @@ -23,6 +25,7 @@ normalize_status, strip_html, ) +from e3sm_comms.page_reviewer.utils_base import get_e3sm_url_status from e3sm_comms.utils import ( WordpressItem, expand_patterns_to_urls, @@ -87,6 +90,14 @@ class ExternalContentLinkSummary: valid_links: List[str] +@dataclass +class AccessibleNonPublishedIssue: + title: str + url: str + status: str + e3sm_url_status: str + + def build_records( xml_pages: str, xml_posts: str, @@ -95,11 +106,15 @@ def build_records( whitelist_file: str, requested_links_file: str, known_ok_links_file: str, + expected_archived_file: str = "", + keep_unchanged_links_file: str = "", ) -> Tuple[ List[ReportRecord], Dict[str, int], List[RequestedLinkRecord], List[WordpressItem], + List[Tuple[str, str]], + List[Tuple[str, str]], ]: sensitive_terms_list = read_sensitive_terms(sensitive_terms_file) @@ -108,6 +123,11 @@ def build_records( confluence_map = get_confluence_mapping(confluence_hierarchy) known_ok_urls = read_known_ok_links(known_ok_links_file) + keep_unchanged_urls = ( + read_keep_unchanged_links(keep_unchanged_links_file) + if keep_unchanged_links_file + else set() + ) raw_items: List[WordpressItem] = [] raw_items.extend(parse_wordpress_xml_items(xml_pages, "page")) @@ -136,6 +156,9 @@ def build_records( else: report_status = "published & not whitelisted" + if item.url in keep_unchanged_urls: + report_status = f"{report_status}, keep unchanged" + status_totals[report_status] += 1 plain_text = strip_html(item.body) @@ -162,7 +185,121 @@ def build_records( flagged_urls=flagged_urls, ) - return records, dict(status_totals), requested_link_records, raw_items + should_be_archived = build_should_be_archived( + raw_items=raw_items, + all_urls=all_urls, + expected_archived_file=expected_archived_file, + ) + + published_not_in_confluence = build_published_not_in_confluence( + raw_items=raw_items, + confluence_map=confluence_map, + whitelisted_urls=whitelisted_urls, + ) + + return ( + records, + dict(status_totals), + requested_link_records, + raw_items, + should_be_archived, + published_not_in_confluence, + ) + + +def build_should_be_archived( + raw_items: List[WordpressItem], + all_urls: List[str], + expected_archived_file: str, +) -> List[Tuple[str, str]]: + """ + Ported from e3sm_org_reviewer.classifiers: cross-reference a manually + curated list of e3sm.org paths that are expected to be archived against + each page/post's actual WordPress status, and flag any that have not + actually been archived yet. Returns (title, url) pairs, sorted by title. + """ + if not expected_archived_file: + return [] + + expected_archived_patterns = read_expected_archived_patterns(expected_archived_file) + expected_archived_urls = set( + expand_patterns_to_urls(expected_archived_patterns, all_urls) + ) + + if not expected_archived_urls: + return [] + + flagged: List[Tuple[str, str]] = [] + for item in raw_items: + if not item.url or item.url not in expected_archived_urls: + continue + if normalize_status(item.status) != "archived": + flagged.append((item.title, item.url)) + + return sorted(flagged, key=lambda x: x[0].lower()) + + +def build_published_not_in_confluence( + raw_items: List[WordpressItem], + confluence_map: Dict[str, str], + whitelisted_urls: Set[str], +) -> List[Tuple[str, str]]: + """ + Ported from e3sm_org_reviewer.main: published pages/posts with no + corresponding entry in the Confluence-predicted URL map, i.e. content + that's live on e3sm.org but has no source-of-truth Confluence page. + Restricted to whitelisted URLs, mirroring the original tool's + `published_not_whitelisted_and_not_in_confluence` check. Returns + (title, url) pairs, sorted by title. Empty if no Confluence hierarchy + file was supplied. + """ + if not confluence_map: + return [] + + flagged: List[Tuple[str, str]] = [] + for item in raw_items: + if not item.url or normalize_status(item.status) != "published": + continue + if item.url not in whitelisted_urls: + continue + if normalize_url(item.url) not in confluence_map: + flagged.append((item.title, item.url)) + + return sorted(flagged, key=lambda x: x[0].lower()) + + +def build_accessible_non_published_issues( + raw_items: List[WordpressItem], +) -> List[AccessibleNonPublishedIssue]: + """ + Ported from e3sm_org_reviewer.main: for every non-published page/post, + make a live, logged-out HTTP request to its e3sm.org URL and flag it if + the page is actually reachable. Draft/private/pending/future content + should 404 or redirect to a login when fetched without credentials; if + it returns 200, it's effectively public despite its WordPress status. + + This makes one live network request per non-published URL, so it's + opt-in (see `--check-non-published-access` in main.py) and can be slow + on a site with many drafts. + """ + flagged: List[AccessibleNonPublishedIssue] = [] + + for item in raw_items: + if not item.url or normalize_status(item.status) == "published": + continue + + e3sm_url_status = get_e3sm_url_status(item.url) + if e3sm_url_status == "link works not logged-in": + flagged.append( + AccessibleNonPublishedIssue( + title=item.title, + url=item.url, + status=display_status(normalize_status(item.status)), + e3sm_url_status=e3sm_url_status, + ) + ) + + return sorted(flagged, key=lambda x: x.title.lower()) def build_navigation_issue_records( diff --git a/e3sm_comms/exported_xml_reviewer/main.py b/e3sm_comms/exported_xml_reviewer/main.py index b3218cc..5585502 100644 --- a/e3sm_comms/exported_xml_reviewer/main.py +++ b/e3sm_comms/exported_xml_reviewer/main.py @@ -3,6 +3,7 @@ import argparse from e3sm_comms.exported_xml_reviewer.builders import ( + build_accessible_non_published_issues, build_external_content_link_summaries, build_navigation_issue_records, build_published_content_link_summaries, @@ -18,6 +19,7 @@ write_hierarchical_outline, write_invalid_internal_links_report, write_navigation_issues_report, + write_non_published_accessibility_report, write_published_pages_link_report, write_terms_report, ) @@ -45,6 +47,11 @@ DEFAULT_WHITELIST: str = ( f"{IO_DIR}/input/exported_xml_reviewer/whitelisted_web_pages.txt" ) +# Ported from e3sm_org_reviewer: same shared file it used for this input. +DEFAULT_EXPECTED_ARCHIVED: str = f"{IO_DIR}/input/shared/archived_web_pages.txt" +DEFAULT_KEEP_UNCHANGED: str = ( + f"{IO_DIR}/input/exported_xml_reviewer/keep_unchanged_web_pages.txt" +) # Outputs: OUTPUT_TERMS_REPORT: str = ( @@ -65,6 +72,9 @@ OUTPUT_EXTERNAL_LINKS_REPORT: str = ( f"{IO_DIR}/output/exported_xml_reviewer/wordpress_invalid_external_links_report.md" ) +OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT: str = ( + f"{IO_DIR}/output/exported_xml_reviewer/wordpress_non_published_accessibility_report.md" +) def parse_args() -> argparse.Namespace: @@ -79,6 +89,29 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Use the whitelisted web pages file to filter results.", ) + parser.add_argument( + "--use-expected-archived", + action="store_true", + help=( + "Flag pages/posts that are expected to be archived (per the shared " + "archived_web_pages.txt list) but are not yet archived in WordPress." + ), + ) + parser.add_argument( + "--use-keep-unchanged", + action="store_true", + help="Annotate pages/posts that are on the keep-unchanged exception list.", + ) + parser.add_argument( + "--check-non-published-access", + action="store_true", + help=( + "Make a live, logged-out HTTP request to every non-published " + "page/post and flag any that are actually reachable. Makes one " + "network request per non-published URL, so this is slow and " + "off by default." + ), + ) return parser.parse_args() @@ -89,10 +122,21 @@ def main() -> None: DEFAULT_CONFLUENCE_HIERARCHY if args.use_confluence else "" ) input_whitelist = DEFAULT_WHITELIST if args.use_whitelist else "" + input_expected_archived = ( + DEFAULT_EXPECTED_ARCHIVED if args.use_expected_archived else "" + ) + input_keep_unchanged = DEFAULT_KEEP_UNCHANGED if args.use_keep_unchanged else "" inaccessible_prefixes = read_inaccessible_prefixes(INPUT_INACCESSIBLE_PREFIXES) - records, status_totals, requested_link_records, raw_items = build_records( + ( + records, + status_totals, + requested_link_records, + raw_items, + should_be_archived, + published_not_in_confluence, + ) = build_records( xml_pages=INPUT_XML_PAGES, xml_posts=INPUT_XML_POSTS, confluence_hierarchy=input_confluence_hierarchy, @@ -100,6 +144,8 @@ def main() -> None: whitelist_file=input_whitelist, requested_links_file=INPUT_REQUESTED_LINKS, known_ok_links_file=INPUT_KNOWN_OK_LINKS, + expected_archived_file=input_expected_archived, + keep_unchanged_links_file=input_keep_unchanged, ) write_terms_report( @@ -122,6 +168,8 @@ def main() -> None: OUTPUT_NAVIGATION_ISSUES_REPORT, top_level_issues, archived_parent_published_child_issues, + should_be_archived=should_be_archived, + published_not_in_confluence=published_not_in_confluence, ) invalid_link_groups = build_invalid_internal_link_groups(raw_items) @@ -158,6 +206,17 @@ def main() -> None: external_post_summaries, ) + if args.check_non_published_access: + accessibility_issues = build_accessible_non_published_issues(raw_items) + write_non_published_accessibility_report( + OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT, + accessibility_issues, + ) + print( + "Wrote non-published accessibility report to " + f"{OUTPUT_NON_PUBLISHED_ACCESSIBILITY_REPORT}" + ) + print(f"Wrote report to {OUTPUT_TERMS_REPORT}") print(f"Wrote hierarchical outline to {OUTPUT_HIERARCHICAL_OUTLINE}") print(f"Wrote navigation issues report to {OUTPUT_NAVIGATION_ISSUES_REPORT}") diff --git a/e3sm_comms/exported_xml_reviewer/readers.py b/e3sm_comms/exported_xml_reviewer/readers.py index 12816a1..d9e1fea 100644 --- a/e3sm_comms/exported_xml_reviewer/readers.py +++ b/e3sm_comms/exported_xml_reviewer/readers.py @@ -87,3 +87,12 @@ def read_requested_links(file_path: str) -> List[Tuple[str, str]]: def read_whitelist_patterns(file_path: str) -> List[str]: return read_lines(file_path) + + +def read_expected_archived_patterns(file_path: str) -> List[str]: + return read_lines(file_path) + + +def read_keep_unchanged_links(file_path: str) -> Set[str]: + with open(file_path, "r", encoding="utf-8") as f: + return {normalize_url(line.strip()) for line in f if line.strip()} diff --git a/e3sm_comms/exported_xml_reviewer/reporters.py b/e3sm_comms/exported_xml_reviewer/reporters.py index ccdd4f5..a2cd031 100644 --- a/e3sm_comms/exported_xml_reviewer/reporters.py +++ b/e3sm_comms/exported_xml_reviewer/reporters.py @@ -5,6 +5,7 @@ from typing import DefaultDict, Dict, List, Set from e3sm_comms.exported_xml_reviewer.builders import ( + AccessibleNonPublishedIssue, ArchivedParentPublishedChildIssue, ExternalContentLinkSummary, PublishedContentLinkSummary, @@ -571,10 +572,40 @@ def walk(node: WordpressItem, depth: int) -> None: write_section(f, posts, "Posts") +def write_non_published_accessibility_report( + output_path: str, + issues: List[AccessibleNonPublishedIssue], +) -> None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + f.write("# Non-Published Pages That Are Still Accessible\n\n") + f.write( + "For every page/post whose WordPress status is not \"publish\", this " + "checks the live e3sm.org URL (logged out) and flags it here if it's " + "actually reachable without logging in.\n\n" + ) + + if not issues: + f.write("No incorrectly accessible non-published pages found.\n") + return + + f.write(f"## Found {len(issues)} issue(s)\n\n") + f.write("| Title | WordPress status | URL | Live check result |\n") + f.write("| --- | --- | --- | --- |\n") + for issue in issues: + f.write( + f"| {issue.title} | {issue.status} | {issue.url} | {issue.e3sm_url_status} |\n" + ) + + def write_navigation_issues_report( output_path: str, top_level_issues: List[TopLevelPageIssue], archived_parent_published_child_issues: List[ArchivedParentPublishedChildIssue], + should_be_archived: List[tuple] = (), + published_not_in_confluence: List[tuple] = (), ) -> None: output_file = Path(output_path) output_file.parent.mkdir(parents=True, exist_ok=True) @@ -615,3 +646,35 @@ def write_navigation_issues_report( f.write( "No published child pages were found under archived parent pages.\n" ) + + f.write("\n") + + f.write( + "## 3. Expecting to be archived, but not yet archived " + f"({len(should_be_archived)})\n\n" + ) + if should_be_archived: + f.write("| Title | URL |\n") + f.write("| --- | --- |\n") + for title, url in should_be_archived: + f.write(f"| {title} | {url} |\n") + else: + f.write( + "No pages/posts found (or `--use-expected-archived` was not passed).\n" + ) + + f.write("\n") + + f.write( + "## 4. Published & whitelisted, but no matching Confluence page found " + f"({len(published_not_in_confluence)})\n\n" + ) + if published_not_in_confluence: + f.write("| Title | URL |\n") + f.write("| --- | --- |\n") + for title, url in published_not_in_confluence: + f.write(f"| {title} | {url} |\n") + else: + f.write( + "No pages/posts found (or `--use-confluence` was not passed).\n" + )