Add functionality to review web pages

README.md

@@ -7,8 +7,11 @@ This package is for implementing the software needs of the E3SM Communications t
 ### Simple commands
 
 `e3sm-comms-e3sm-org-reviewer`
-- input: txt file listing e3sm.org pages to review, txt file containing phrases to search for
-- output: txt file listing e3sm.org pages containing those phrases
+- input:
+  - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. Note: your export will go to your local machine. Example transfer command: `scp local_path/wordpress_posts.xml user@host:/remote/path/wordpress_posts.xml`.
+  - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages, txt file of sensitive terms found on Confluence pages
+  - Other: txt file of whitelisted e3sm.org pages, txt file of e3sm.org pages expected to be archived, txt file of sensitive terms, txt file of known-ok e3sm.org pages (that is, script is picking up errors we don't care about), txt file of keep-unchanged e3sm.org pages (that is, pages we don't want to change)
+- output: 3 Markdown summary reports: (1) An analysis of the e3sm.org paths, (2) An analysis of the sensitive terms found, (3) the key action items
 
 `e3sm-comms-html-reviewer`
 - input: 1 txt file of html copied from WordPress that includes yellow highlights left over from Confluence.
@@ -23,6 +26,13 @@ This package is for implementing the software needs of the E3SM Communications t
 - input: txt file of time intervals to cut from the video, txt file of initial timestamps
 - output: txt file of new timestamps after cutting the specified intervals
 
+`e3sm-comms-exported-xml-reviewer`
+- input:
+  - From WordPress under Tools > Export: xml file of WordPress pages, xml file of WordPress posts. NOTE: Only ever use the XML files downloaded directly from WordPress; the source must be trusted.
+  - From output of `e3sm-comms-website-reviewer`: txt file of hierarchical outline of Confluence pages
+  - Other: txt file of sensitive terms, txt file of whitelisted e3sm.org pages, txt file of links known to be inaccessible for automated review
+- output: 6 files under `output/exported_xml_reviewer/`: (1) `wordpress_sensitive_terms_report.md`, (2) `wordpress_hierarchical_outline.txt`, (3) `wordpress_navigation_issues_report.md`, (4) `wordpress_invalid_internal_links_report.md`, (5) `wordpress_published_pages_link_report.md`, (6) `wordpress_invalid_external_links_report.md`
+
 ### Confluence API commands (require Confluence token)
 
 `e3sm-comms-newsletter-reviewer`

e3sm_comms/e3sm_org_reviewer/README.md

@@ -0,0 +1,8 @@
+# Dependency hierarchy
+
+It is important to not introduce circular dependencies.
+To avoid this, the dependency hierarchy is listed below:
+
+- Level 1: `main.py`
+- Level 2: `parsers.py`, `reporters.py`
+- Level 3: `classifiers.py`, `confluence.py`, `readers.py`, `record.py`, `utils.py`

e3sm_comms/e3sm_org_reviewer/classifiers.py

@@ -0,0 +1,81 @@
+import re
+from typing import Dict, Optional, Set, Tuple
+
+CLASS_PUBLISHED = "Published"
+CLASS_ARCHIVED = "Archived"
+CLASS_SHOULD_BE_ARCHIVED = "Should be archived"
+CLASS_NOT_PUBLISHED = "Not published"
+CLASS_KNOWN_OK = "Known OK"
+CLASS_KEEP_UNCHANGED = "Keep unchanged"
+CLASS_NO_MAPPED_E3SM_URL = "No mapped e3sm.org URL"
+CLASS_PREDICTED_URL_NOT_IN_EXPORT = "Predicted e3sm.org URL not in WordPress export"
+
+FROM_PREFIX_RE = re.compile(r"^\[From\s+(\d{4})-\d{2}-\d{2}T[^\]]+\]\s*(.*)$")
+
+
+def classify_e3sm_url(
+    e3sm_url: Optional[str],
+    url_to_status: Dict[str, str],
+    expected_archived_urls: Set[str],
+    known_ok_urls: Set[str],
+    keep_unchanged_urls: Set[str],
+) -> Tuple[str, Optional[str]]:
+    if not e3sm_url:
+        return CLASS_NO_MAPPED_E3SM_URL, None
+
+    if e3sm_url in known_ok_urls:
+        return CLASS_KNOWN_OK, url_to_status.get(e3sm_url)
+
+    if e3sm_url in keep_unchanged_urls:
+        return CLASS_KEEP_UNCHANGED, url_to_status.get(e3sm_url)
+
+    wordpress_status = url_to_status.get(e3sm_url)
+
+    if wordpress_status is None:
+        return CLASS_PREDICTED_URL_NOT_IN_EXPORT, None
+
+    if wordpress_status == "archive":
+        return CLASS_ARCHIVED, wordpress_status
+
+    if e3sm_url in expected_archived_urls:
+        return CLASS_SHOULD_BE_ARCHIVED, wordpress_status
+
+    if wordpress_status != "publish":
+        return CLASS_NOT_PUBLISHED, wordpress_status
+
+    return CLASS_PUBLISHED, wordpress_status
+
+
+def classification_sort_key(classification: str) -> Tuple[int, str]:
+    order = {
+        CLASS_PUBLISHED: 0,
+        CLASS_SHOULD_BE_ARCHIVED: 1,
+        CLASS_ARCHIVED: 2,
+        CLASS_NOT_PUBLISHED: 3,
+        CLASS_KNOWN_OK: 4,
+        CLASS_KEEP_UNCHANGED: 5,
+        CLASS_NO_MAPPED_E3SM_URL: 6,
+        CLASS_PREDICTED_URL_NOT_IN_EXPORT: 7,
+    }
+    return (order.get(classification, 999), classification)
+
+
+def year_sort_key(year_str: str) -> Tuple[int, int]:
+    if year_str == "Unknown year":
+        return (1, 0)
+    if year_str == "N/A":
+        return (2, 0)
+    try:
+        return (0, -int(year_str))
+    except ValueError:
+        return (3, 0)
+
+
+def extract_year_and_remainder(line: str) -> Tuple[Optional[int], str]:
+    match = FROM_PREFIX_RE.match(line)
+    if not match:
+        return None, line
+
+    year = int(match.group(1))
+    remainder = match.group(2).strip()
+    return year, remainder

e3sm_comms/e3sm_org_reviewer/confluence.py

@@ -0,0 +1,34 @@
+from typing import List, Tuple
+
+from e3sm_comms.page_reviewer.utils_base import map_confluence_to_e3sm
+from e3sm_comms.utils import parse_confluence_hierarchy_file
+
+CONFLUENCE_SPACE = "EPWCD"
+CONFLUENCE_BASE = "https://e3sm.atlassian.net/wiki"
+
+
+def build_confluence_url(page_id: str, space_key: str = CONFLUENCE_SPACE) -> str:
+    return f"{CONFLUENCE_BASE}/spaces/{space_key}/pages/{page_id}"
+
+
+def get_confluence_predicted_e3sm_urls(
+    input_file: str,
+) -> Tuple[List[str], List[str]]:
+    valid_predicted_urls: List[str] = []
+    unmapped_confluence_pages: List[str] = []
+
+    for page_id, title in parse_confluence_hierarchy_file(input_file):
+        confluence_url = build_confluence_url(page_id)
+        try:
+            e3sm_url = map_confluence_to_e3sm(confluence_url, page_title=title)
+            if e3sm_url:
+                valid_predicted_urls.append(e3sm_url)
+            else:
+                unmapped_confluence_pages.append(f"{title}: {confluence_url}")
+        except Exception as exc:
+            print(
+                f"Could not map Confluence URL to e3sm.org URL for {confluence_url}: {exc}"
+            )
+            unmapped_confluence_pages.append(f"{title}: {confluence_url}")
+
+    return sorted(set(valid_predicted_urls)), sorted(unmapped_confluence_pages)