fix: harden requirement extraction and rescore visibility

src/main/java/com/jobpilot/extraction/DeterministicRequirementExtractor.java

@@ -3,6 +3,7 @@
 import com.jobpilot.jobs.domain.ExtractedRequirements;
 import com.jobpilot.jobs.domain.Job;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -15,6 +16,75 @@
 public class DeterministicRequirementExtractor {
     private static final Pattern YEARS = Pattern.compile(
             "(?i)(?:minimum|min\\.?|at least|requires?|with)?\\s*(\\d+(?:\\.\\d+)?)\\s*\\+?\\s*(?:years?|yrs?)(?:\\s+of)?(?:\\s+(?:commercial|professional|industry))?\\s+experience");
+    /** Technical tokens REST is credibly listed beside; longer alternatives come first. */
+    private static final String REST_TECH_TOKEN = "(?:OOP|OO|MVVM|MVC|MVP|SOAP|JSON|XML|YAML"
+            + "|HTTPS|HTTP|gRPC|GraphQL|SOA|microservices|NoSQL|PostgreSQL|MySQL|MongoDB|SQL"
+            + "|Redis|Kafka|RabbitMQ|Spring Boot|Spring|Hibernate|JPA|JDBC|Maven|Gradle|JUnit"
+            + "|Mockito|Selenium|Cucumber|BDD|TDD|Docker|Kubernetes|CI/CD|Jenkins|Linux|AWS"
+            + "|Azure|GCP|HTML|CSS|React|Angular|Node|Django|Flask|\\.NET|APIs|API|JavaScript"
+            + "|TypeScript|Java|Kotlin|Scala|Python|C#|C\\+\\+|Git)";
+    /** Never a bare space: that is what keeps "Rest Assured" out of every list form. */
+    private static final String REST_LIST_DELIMITER = "(?:\\s*[,;/]\\s*|\\s+(?:and|or)\\s+)";
+    private static final String REST_LIST_END = "(?=\\s*[.)\\];:]|\\s*$)";
+    private static final String REST_INTRODUCER = "(?:experience\\s+(?:with|in)|knowledge\\s+of"
+            + "|understanding\\s+of|familiarity\\s+with|proficiency\\s+(?:in|with)"
+            + "|proficient\\s+(?:in|with)|skills?\\s+(?:in|with))";
+
+    /**
+     * Qualified REST. Bare "rest" is an ordinary English noun and "Rest Assured" is a test
+     * library, so neither is API evidence on its own, but production showed the token also
+     * carries genuine evidence as technical shorthand ("HTTP/REST", "REST/JSON", "APIs (REST)"),
+     * directly after a technical introducer, and as an item in a delimited technology list
+     * ("OO, MVC, REST", "Java, Spring Boot, REST, SQL").
+     *
+     * <p>The list forms require REST to be a <em>complete</em> item — a delimiter on both sides,
+     * or a delimiter and the end of the list — and the delimiter never matches a bare space. That
+     * is what keeps "Selenium, Cucumber, BDD, Rest Assured and CI/CD" negative: "Rest" there is
+     * followed by a word, not by a delimiter. A posting carrying both dirty and genuine text
+     * still matches, because the scan reaches the genuine occurrence.
+     */
+    private static final String QUALIFIED_REST = "(?:"
+            + "RESTful"
+            + "|REST[\\s-]?(?:API|service|endpoint|controller)s?"
+            + "|JAX-?RS"
+            + "|Spring\\s+REST"
+            + "|HTTP\\s*/\\s*REST"
+            + "|REST\\s*/\\s*JSON"
+            + "|APIs?\\s*\\(\\s*REST\\s*\\)"
+            + "|" + REST_INTRODUCER + "\\s+REST"
+            + "|" + REST_TECH_TOKEN + REST_LIST_DELIMITER + "REST"
+                  + REST_LIST_DELIMITER + REST_TECH_TOKEN
+            + "|" + REST_TECH_TOKEN + REST_LIST_DELIMITER + REST_TECH_TOKEN
+                  + REST_LIST_DELIMITER + "REST" + REST_LIST_END
+            + "|REST" + REST_LIST_DELIMITER + REST_TECH_TOKEN
+                  + REST_LIST_DELIMITER + REST_TECH_TOKEN
+            + ")";
+
+    private static final String LANGUAGE_TOKEN = "(?:Java|Python|Rust|Kotlin|Scala|Ruby|PHP"
+            + "|C\\+\\+|C#|TypeScript|JavaScript|Elixir|Erlang|Swift|Haskell)";
+    private static final String LANGUAGE_DELIMITER = "(?:\\s*[,/]\\s*|\\s+(?:and|or)\\s+)";
+    private static final String TECHNICAL_INTRODUCER = "(?:programming\\s+languages?|languages?"
+            + "|technologies|tech\\s+stack|stack|experience\\s+(?:with|in)"
+            + "|proficient\\s+(?:with|in)|written\\s+in|skills\\s+(?:with|in))";
+    /** Go has to end as a list item; without this "experience with Java and go to market" matches. */
+    private static final String GO_LIST_TAIL =
+            "(?=\\s*[,;./)\\]]|\\s+(?:and|or)\\s+" + LANGUAGE_TOKEN + "|\\s*$)";
+
+    /**
+     * "Go" is an ordinary English verb, so a bare token is never evidence. Either it is spelled
+     * unambiguously, or it sits <em>between</em> two recognised languages in a delimited list, or
+     * it follows an explicit technical introducer <em>and</em> terminates as a list item. Simple
+     * two-item adjacency is deliberately rejected: "ready to go, Java developers should apply"
+     * would otherwise qualify. Conservative false negatives are preferred here — Go carries no
+     * scoring weight, while a false Go pollutes the evidence diagnostics depend on.
+     */
+    private static final String QUALIFIED_GO = "(?:Golang|goroutines?"
+            + "|Go[\\s-]?(?:developer|engineer|programming|language|module|routine)s?"
+            + "|" + LANGUAGE_TOKEN + LANGUAGE_DELIMITER + "Go" + LANGUAGE_DELIMITER + LANGUAGE_TOKEN
+            + "|" + TECHNICAL_INTRODUCER + "\\s*:?\\s*(?:" + LANGUAGE_TOKEN + LANGUAGE_DELIMITER
+            + "){0,3}Go" + GO_LIST_TAIL
+            + ")";
+
     private static final Map<String, Pattern> TECHNOLOGIES = technologies();
     private static final List<String> PROGRAMMING = List.of("Java", "TypeScript", "JavaScript", "Python", "Kotlin", "C#", "C++", "Go");
     // Whole words only: "internal" or "international" must not count as an internship.
@@ -96,17 +166,40 @@ public ExtractedRequirements extract(Job job) {
                 salary(text), job.getDeadline(), "DETERMINISTIC");
     }
 
+    /**
+     * Insertion-ordered on purpose. {@link Map#copyOf} salts its iteration order per JVM run, so
+     * the extracted technology list — and therefore {@link ExtractedRequirements} record equality —
+     * used to differ between runs for identical text. Nothing depended on that while the rescore
+     * plan compared scores only; requirement-aware plan membership does.
+     */
     private static Map<String, Pattern> technologies() {
         Map<String, Pattern> map = new LinkedHashMap<>();
-        for (String tech : List.of("Java", "Spring Boot", "Spring MVC", "Spring Security", "REST", "SQL",
-                "PostgreSQL", "JPA", "Hibernate", "Maven", "JUnit", "Mockito", "React", "React Native",
-                "TypeScript", "JavaScript", "HTML", "CSS", "Git", "GitHub Actions", "CI/CD", "Docker",
-                "Kubernetes", "Python", "Kotlin", "C#", "C++", "Go", "AWS", "Azure", "GCP")) {
-            String token = "CI/CD".equals(tech) ? "CI(?:/|\\s*)CD" : Pattern.quote(tech);
-            String expression = "(?i)(?<![\\p{L}\\p{N}])" + token + "(?![\\p{L}\\p{N}])";
+        for (String tech : List.of("Java", "JVM", "Spring Boot", "Spring MVC", "Spring Security",
+                "REST", "SQL", "PostgreSQL", "JPA", "Hibernate", "Maven", "JUnit", "Mockito",
+                "React", "React Native", "TypeScript", "JavaScript", "HTML", "CSS", "Git",
+                "GitHub Actions", "CI/CD", "Docker", "Kubernetes", "Python", "Kotlin", "C#", "C++",
+                "Go", "AWS", "Azure", "GCP")) {
+            String expression = "(?i)(?<![\\p{L}\\p{N}])" + token(tech) + "(?![\\p{L}\\p{N}])";
             map.put(tech, Pattern.compile(expression));
         }
-        return Map.copyOf(map);
+        return Collections.unmodifiableMap(map);
+    }
+
+    /**
+     * A literal token by default. Only terms carrying a spelling variant, or an English homograph
+     * that would otherwise produce false evidence, override it. "PostgreSQL" stays literal: the
+     * "Postgres" alias is a measured gap deliberately deferred to the scoring redesign, because
+     * PostgreSQL is an equally weighted backend skill today and recovering it would promote
+     * infrastructure postings this change is not meant to touch.
+     */
+    private static String token(String tech) {
+        return switch (tech) {
+            case "CI/CD" -> "CI(?:/|\\s*)CD";
+            case "REST" -> QUALIFIED_REST;
+            case "Go" -> QUALIFIED_GO;
+            case "Spring Boot" -> "Spring[\\s-]?Boot";
+            default -> Pattern.quote(tech);
+        };
     }
 
     private List<String> extractTechnologies(String text) {

src/main/java/com/jobpilot/matching/preview/ScoreRescorePreviewReport.java

@@ -5,12 +5,13 @@
 import com.jobpilot.matching.ScoreBand;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /** Immutable, deterministic result; it contains no description, URL, or candidate data. */
 public record ScoreRescorePreviewReport(
         int inspectedJobs,
         int exactMatches,
-        int changedScoreCount,
+        int scoreDeltaChangedCount,
         int changedBandCount,
         int blockerRemovedCount,
         int blockerAddedCount,
@@ -24,7 +25,84 @@ public record ScoreRescorePreviewReport(
         QueueProjection matchesQueue,
         QueueProjection reviewQueue,
         List<JobPreview> changedJobs,
-        JobPreview juniorJavaDeveloper) {
+        JobPreview juniorJavaDeveloper,
+        ChangeCounts changeCounts,
+        List<RequirementsPreview> requirementsChangedJobs) {
+
+    /**
+     * Plan membership, decomposed. {@code scoreDeltaChangedCount} above counts a numeric score
+     * move; {@code scoreChangedCount} here counts a {@code ScoreCard} identity change, which is
+     * what actually decides whether a row is written.
+     */
+    public record ChangeCounts(
+            int scoreChangedCount,
+            int requirementsChangedCount,
+            int requirementsOnlyChangedCount,
+            int changedPlanCount,
+            int unchangedCount) {
+    }
+
+    /** Every persisted component of {@code ExtractedRequirements}, in record order. */
+    public enum RequirementField {
+        SENIORITY,
+        INTERNSHIP_OR_TRAINEE,
+        REQUIRED_EXPERIENCE_YEARS,
+        REQUIRED_EDUCATION,
+        FINAL_YEAR_MANDATORY,
+        TECHNOLOGIES,
+        PROGRAMMING_LANGUAGES,
+        SPOKEN_LANGUAGES,
+        LOCATION,
+        REMOTE_ELIGIBILITY,
+        MENTORSHIP_SIGNALS,
+        WORK_AUTHORIZATION,
+        SALARY,
+        APPLICATION_DEADLINE,
+        EXTRACTION_METHOD
+    }
+
+    /** One differing persisted field. Both values arrive already sanitized and bounded. */
+    public record RequirementFieldChange(
+            RequirementField field,
+            String storedValue,
+            String computedValue) {
+
+        public static final int MAX_VALUE_LENGTH = 80;
+
+        public RequirementFieldChange {
+            Objects.requireNonNull(field, "Requirement field is required");
+            requireBounded(storedValue, "Stored");
+            requireBounded(computedValue, "Computed");
+        }
+
+        private static void requireBounded(String value, String side) {
+            if (value == null || value.length() > MAX_VALUE_LENGTH) {
+                throw new IllegalArgumentException(
+                        side + " requirement value must be sanitized and bounded");
+            }
+        }
+    }
+
+    /**
+     * Why one row entered the plan. Carries no description, URL, or candidate data, and cannot be
+     * constructed without naming at least one field, so a planned row can never be invisible.
+     */
+    public record RequirementsPreview(
+            long jobId,
+            String title,
+            boolean scoreChanged,
+            int storedScore,
+            int computedScore,
+            List<RequirementFieldChange> changedFields) {
+
+        public RequirementsPreview {
+            changedFields = List.copyOf(changedFields);
+            if (changedFields.isEmpty()) {
+                throw new IllegalArgumentException(
+                        "A requirements preview must name at least one changed field");
+            }
+        }
+    }
 
     public record BoundaryCrossings(
             List<Long> unsuitable,

src/main/java/com/jobpilot/matching/preview/ScoreRescorePreviewReportRenderer.java

@@ -4,6 +4,8 @@
 import com.jobpilot.matching.preview.ScoreRescorePreviewReport.JobPreview;
 import com.jobpilot.matching.preview.ScoreRescorePreviewReport.QueueEntry;
 import com.jobpilot.matching.preview.ScoreRescorePreviewReport.QueueProjection;
+import com.jobpilot.matching.preview.ScoreRescorePreviewReport.RequirementFieldChange;
+import com.jobpilot.matching.preview.ScoreRescorePreviewReport.RequirementsPreview;
 import java.util.ArrayList;
 import java.util.List;
 import org.springframework.stereotype.Component;
@@ -24,14 +26,21 @@ public List<String> render(ScoreRescorePreviewResult result) {
         List<String> lines = new ArrayList<>();
         lines.add("SCORE_RESCORE_PREVIEW status=SUCCESS inspected=" + report.inspectedJobs()
                 + " exact=" + report.exactMatches()
-                + " changedScores=" + report.changedScoreCount()
+                + " scoreDeltaChanged=" + report.scoreDeltaChangedCount()
                 + " changedBands=" + report.changedBandCount()
                 + " blockerRemoved=" + report.blockerRemovedCount()
                 + " blockerAdded=" + report.blockerAddedCount()
                 + " increases=" + report.scoreIncreaseCount()
                 + " decreases=" + report.scoreDecreaseCount()
                 + " zeroToPositive=" + report.storedZeroToPositiveCount()
                 + " positiveToZero=" + report.storedPositiveToZeroCount());
+        lines.add("SCORE_RESCORE_PREVIEW_PLAN scoreChanged="
+                + report.changeCounts().scoreChangedCount()
+                + " requirementsChanged=" + report.changeCounts().requirementsChangedCount()
+                + " requirementsOnlyChanged="
+                + report.changeCounts().requirementsOnlyChangedCount()
+                + " changedPlan=" + report.changeCounts().changedPlanCount()
+                + " unchanged=" + report.changeCounts().unchangedCount());
         lines.add(bound("SCORE_RESCORE_PREVIEW_DELTA distribution="
                 + report.scoreDeltaDistribution()));
         lines.add(bound("SCORE_RESCORE_PREVIEW_BOUNDARIES unsuitable="
@@ -43,6 +52,9 @@ public List<String> render(ScoreRescorePreviewResult result) {
         for (JobPreview job : report.changedJobs()) {
             lines.add(renderJob("SCORE_RESCORE_PREVIEW_CHANGED", job));
         }
+        for (RequirementsPreview requirements : report.requirementsChangedJobs()) {
+            lines.add(renderRequirements(requirements));
+        }
         renderQueue(lines, "matches", report.matchesQueue());
         renderQueue(lines, "review", report.reviewQueue());
         if (report.juniorJavaDeveloper() == null) {
@@ -81,6 +93,23 @@ private String renderJob(String prefix, JobPreview job) {
                 + "->" + position(job.newQueuePosition()));
     }
 
+    /** Names every persisted field the write would change, so no planned row is opaque. */
+    private String renderRequirements(RequirementsPreview requirements) {
+        StringBuilder line = new StringBuilder("SCORE_RESCORE_PREVIEW_REQUIREMENTS jobId=")
+                .append(requirements.jobId())
+                .append(" title=").append(quote(requirements.title()))
+                .append(" scoreChanged=").append(requirements.scoreChanged())
+                .append(" score=").append(requirements.storedScore())
+                .append("->").append(requirements.computedScore())
+                .append(" changedFields=").append(requirements.changedFields().size());
+        for (RequirementFieldChange change : requirements.changedFields()) {
+            line.append(' ').append(change.field()).append('=')
+                    .append(quote(change.storedValue()))
+                    .append("->").append(quote(change.computedValue()));
+        }
+        return bound(line.toString());
+    }
+
     private void renderQueue(List<String> lines, String name, QueueProjection queue) {
         lines.add("SCORE_RESCORE_PREVIEW_QUEUE queue=/" + name
                 + " phase=BEFORE total=" + queue.before().size());