diff --git a/src/main/java/com/jobpilot/sources/lever/LeverJobSource.java b/src/main/java/com/jobpilot/sources/lever/LeverJobSource.java index 5dad802..5fd7144 100644 --- a/src/main/java/com/jobpilot/sources/lever/LeverJobSource.java +++ b/src/main/java/com/jobpilot/sources/lever/LeverJobSource.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.jobpilot.common.ExternalHttpClient; +import com.jobpilot.common.ExternalHttpException; import com.jobpilot.config.JobPilotProperties; import com.jobpilot.jobs.domain.RawJob; import com.jobpilot.jobs.domain.RawCareerData; @@ -11,12 +12,39 @@ import org.springframework.beans.factory.annotation.Autowired; import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import org.springframework.stereotype.Component; import org.springframework.web.util.HtmlUtils; +/** + * Public, bounded Lever Postings API adapter. + * + *

Lever returns one flat JSON array per request, so an unpaginated tenant fetch grows + * with the board: the largest configured tenant broke the shared response-size bound and + * failed as {@code RESPONSE_TOO_LARGE}. Every tenant is therefore fetched in fixed-size + * {@code skip}/{@code limit} pages, which keeps each response small without touching the + * transport limit. + */ @Component public class LeverJobSource implements JobSource { + /** + * Conservative fixed page size. Lever accepts a larger {@code limit}, but response + * bytes scale with it, and a Lever posting carries its complete description; 50 keeps + * every page far below the shared transport bound even for verbose boards. + */ + static final int PAGE_SIZE = 50; + + /** + * Hard stop on requests per tenant, and therefore on postings: at most + * {@code 40 x 50 = 2000} postings per board, comfortably above every configured tenant + * while keeping a pathological or hostile board from paging forever. + */ + static final int MAX_PAGES_PER_TENANT = 40; + + private static final String API_ROOT = "https://api.lever.co/v0/postings/"; + private final ExternalHttpClient http; private final List companies; private final TenantFetchMonitor monitor; @@ -41,15 +69,72 @@ public String getSourceName() { @Override public List fetchJobs() { List jobs = new ArrayList<>(); - // One monitored attempt per tenant: a failure is classified, recorded, and - // isolated to that tenant so the remaining companies are still fetched. + // One monitored attempt per tenant, covering the tenant's complete pagination: a + // failure on any page is classified, recorded, and isolated to that tenant, so no + // partial board escapes and the remaining companies are still fetched. for (String company : companies) { - jobs.addAll(monitor.fetch(getSourceName(), company, () -> parse(company, http.getJson( - "https://api.lever.co/v0/postings/" + company + "?mode=json")))); + jobs.addAll(monitor.fetch(getSourceName(), company, () -> fetchCompany(company))); } return jobs; } + /** + * Pages one tenant with {@code skip} and {@code limit}, in provider order. + * + *

{@code skip} advances by the number of postings the page actually returned rather + * than by the requested page size. Both agree on a full page, but only the returned + * count is safe if Lever ever trims a page: stepping by the constant could then skip + * over postings that were never read. Paging stops on the first empty or short page, + * postings repeated across pages are returned once, and a full page that contributes + * nothing new means the provider ignored {@code skip} — rejected rather than retried. + */ + List fetchCompany(String company) { + List jobs = new ArrayList<>(); + Set seenIds = new LinkedHashSet<>(); + int skip = 0; + for (int page = 1; ; page++) { + if (page > MAX_PAGES_PER_TENANT) { + // Structural cap, reported with the existing oversize semantics: it carries + // no URL, response, or posting data. + throw new ExternalHttpException( + ExternalHttpException.Category.RESPONSE_TOO_LARGE, null); + } + String stage = "page" + page; + JsonNode items = http.getJson(pageUrl(company, skip)); + if (!items.isArray()) { + schemaFailure(stage + ": expected ARRAY but was " + items.getNodeType().name()); + } + if (items.size() > PAGE_SIZE) { + schemaFailure(stage + " returned more postings than the requested limit"); + } + if (items.isEmpty()) return List.copyOf(jobs); + + int before = jobs.size(); + for (JsonNode item : items) { + if (seenIds.add(item.path("id").asText())) jobs.add(parseOne(company, item)); + } + if (jobs.size() == before) { + schemaFailure(stage + " repeated only postings earlier pages already returned"); + } + if (items.size() < PAGE_SIZE) return List.copyOf(jobs); + skip += items.size(); + } + } + + private String pageUrl(String company, int skip) { + return API_ROOT + company + "?mode=json&skip=" + skip + "&limit=" + PAGE_SIZE; + } + + /** + * Rejects a response and names the rule that rejected it. The detail is built only from + * compile-time literals and a JSON type name, so it can never carry a field + * value, response fragment, URL, or header. + */ + private static void schemaFailure(String detail) { + throw new ExternalHttpException(ExternalHttpException.Category.MALFORMED_JSON, null) + .parseDetail(detail); + } + public List parse(String company, JsonNode root) { List result = new ArrayList<>(); for (JsonNode item : root) { diff --git a/src/test/java/com/jobpilot/sources/lever/LeverJobSourceTest.java b/src/test/java/com/jobpilot/sources/lever/LeverJobSourceTest.java index 8ac5703..e102cba 100644 --- a/src/test/java/com/jobpilot/sources/lever/LeverJobSourceTest.java +++ b/src/test/java/com/jobpilot/sources/lever/LeverJobSourceTest.java @@ -1,14 +1,43 @@ package com.jobpilot.sources.lever; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.jobpilot.common.ExternalHttpClient; +import com.jobpilot.common.ExternalHttpException; +import com.jobpilot.config.JobPilotProperties; +import com.jobpilot.jobs.domain.RawJob; +import com.jobpilot.sources.health.SourceTenantHealth; +import com.jobpilot.sources.health.SourceTenantHealthRecorder; +import com.jobpilot.sources.health.TenantAttemptStatus; +import com.jobpilot.sources.health.TenantFailure; +import com.jobpilot.sources.health.TenantFailureCategory; +import com.jobpilot.sources.health.TenantFailureClassifier; +import com.jobpilot.sources.health.TenantFetchMonitor; import com.jobpilot.support.TestProperties; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.IntFunction; import org.junit.jupiter.api.Test; class LeverJobSourceTest { + private static final String COMPANY = "acme"; + private static final String OTHER = "othercorp"; + + private final ObjectMapper mapper = new ObjectMapper(); + private final List requests = new ArrayList<>(); + @Test void parsesPublicPostingPayload() throws Exception { var source = new LeverJobSource(mock(ExternalHttpClient.class), TestProperties.create()); @@ -23,10 +52,286 @@ void parsesPublicPostingPayload() throws Exception { assertThat(jobs).singleElement().satisfies(job -> { assertThat(job.externalId()).isEqualTo("abc"); + assertThat(job.url()).isEqualTo("https://jobs.example/abc"); + assertThat(job.title()).isEqualTo("Software Engineer Intern"); assertThat(job.location()).isEqualTo("Romania"); + assertThat(job.locationData().structuredLocations()).containsExactly("Romania"); assertThat(job.description()).isEqualTo("Java & REST Requirements No experience required"); assertThat(job.employmentType()).isEqualTo("Internship"); assertThat(job.careerData().providerSeniority()).isEqualTo("Entry Level"); }); } + + @Test + void aShortFirstPageIsFetchedInASingleRequest() { + ExternalHttpClient http = pages(skip -> page(skip, 3)); + + var jobs = source(http, List.of(COMPANY)).fetchCompany(COMPANY); + + assertThat(jobs).extracting(RawJob::externalId) + .containsExactly("job-0", "job-1", "job-2"); + assertThat(jobs).allSatisfy(job -> { + assertThat(job.source()).isEqualTo("lever"); + assertThat(job.providerTenant()).isEqualTo(COMPANY); + }); + assertThat(requests).containsExactly(url(COMPANY, 0)); + } + + @Test + void aFullPageFollowedByAnEmptyPageReturnsEveryPosting() { + ExternalHttpClient http = pages(skip -> skip == 0 ? page(0, 50) : page(0, 0)); + + var jobs = source(http, List.of(COMPANY)).fetchCompany(COMPANY); + + assertThat(jobs).hasSize(50); + assertThat(jobs).extracting(RawJob::externalId).startsWith("job-0").endsWith("job-49"); + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50)); + } + + @Test + void everyPageIsReturnedInProviderOrderWithAdvancingSkip() { + ExternalHttpClient http = pages(skip -> switch (skip) { + case 0 -> page(0, 50); + case 50 -> page(50, 50); + default -> page(100, 20); + }); + + var jobs = source(http, List.of(COMPANY)).fetchCompany(COMPANY); + + assertThat(jobs).extracting(RawJob::externalId) + .containsExactlyElementsOf(ids(0, 120)); + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50), url(COMPANY, 100)); + } + + @Test + void aShortPageAfterAFullPageStopsWithoutAnotherRequest() { + ExternalHttpClient http = pages(skip -> skip == 0 ? page(0, 50) : page(50, 10)); + + var jobs = source(http, List.of(COMPANY)).fetchCompany(COMPANY); + + assertThat(jobs).extracting(RawJob::externalId).containsExactlyElementsOf(ids(0, 60)); + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50)); + } + + @Test + void postingsRepeatedAcrossAPageBoundaryAreReturnedOnce() { + // The second page re-sends the last posting of the first page, as an overlapping + // board would; the third page ends the tenant. + ExternalHttpClient http = pages(skip -> switch (skip) { + case 0 -> page(0, 50); + case 50 -> page(49, 50); + default -> page(0, 0); + }); + + var jobs = source(http, List.of(COMPANY)).fetchCompany(COMPANY); + + assertThat(jobs).extracting(RawJob::externalId) + .doesNotHaveDuplicates() + .containsExactlyElementsOf(ids(0, 99)); + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50), url(COMPANY, 100)); + } + + @Test + void aBoardThatIgnoresSkipFailsInsteadOfPagingForever() { + ExternalHttpClient http = pages(skip -> page(0, 50)); + + assertParseFailure(() -> source(http, List.of(COMPANY)).fetchCompany(COMPANY)); + + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50)); + assertThat(new TenantFailureClassifier().classify("lever", COMPANY, thrown(http)).category()) + .isEqualTo(TenantFailureCategory.RESPONSE_PARSE_ERROR); + } + + @Test + void aBoardThatNeverEndsStopsAtTheConfiguredPageCap() { + ExternalHttpClient http = pages(skip -> page(skip, 50)); + + ExternalHttpException failure = thrown(http); + + assertThat(failure.category()).isEqualTo(ExternalHttpException.Category.RESPONSE_TOO_LARGE); + assertThat(requests).hasSize(LeverJobSource.MAX_PAGES_PER_TENANT); + assertThat(new TenantFailureClassifier().classify("lever", COMPANY, failure).category()) + .isEqualTo(TenantFailureCategory.RESPONSE_TOO_LARGE); + } + + @Test + void aLaterPageFailureDiscardsTheTenantAndLeavesOtherTenantsFetchable() { + ExternalHttpClient http = mock(ExternalHttpClient.class); + when(http.getJson(anyString())).thenAnswer(invocation -> { + String url = invocation.getArgument(0, String.class); + requests.add(url); + if (!url.contains(COMPANY)) return page(0, 2); + if (skip(url) == 0) return page(0, 50); + throw new ExternalHttpException(ExternalHttpException.Category.HTTP_STATUS, 500); + }); + + var jobs = source(http, List.of(COMPANY, OTHER)).fetchJobs(); + + // Not one posting of the failed tenant's first page escapes, and the later tenant ran. + assertThat(jobs).extracting(RawJob::providerTenant).containsOnly(OTHER); + assertThat(jobs).hasSize(2); + assertThat(requests).containsExactly(url(COMPANY, 0), url(COMPANY, 50), url(OTHER, 0)); + } + + @Test + void oneMonitoredAttemptPerTenantRecordsThePostingsOfEveryPage() { + ExternalHttpClient http = mock(ExternalHttpClient.class); + when(http.getJson(anyString())).thenAnswer(invocation -> { + String url = invocation.getArgument(0, String.class); + requests.add(url); + if (!url.contains(COMPANY)) { + throw new ExternalHttpException(ExternalHttpException.Category.HTTP_STATUS, 500); + } + return skip(url) == 0 ? page(0, 50) : page(50, 10); + }); + RecordingRecorder recorder = new RecordingRecorder(); + TenantFetchMonitor monitor = new TenantFetchMonitor(new TenantFailureClassifier(), recorder, + Clock.fixed(Instant.parse("2026-08-06T12:00:00Z"), ZoneOffset.UTC)); + + var jobs = source(http, List.of(COMPANY, OTHER), monitor).fetchJobs(); + + assertThat(jobs).hasSize(60); + // Two paginated requests for the healthy tenant, but exactly one attempt per tenant. + assertThat(recorder.calls).hasSize(2); + assertThat(recorder.byTenant(COMPANY).status).isEqualTo(TenantAttemptStatus.SUCCESS); + assertThat(recorder.byTenant(COMPANY).fetchedCount).isEqualTo(60); + assertThat(recorder.byTenant(OTHER).status).isEqualTo(TenantAttemptStatus.FAILURE); + assertThat(recorder.byTenant(OTHER).failure.category()) + .isEqualTo(TenantFailureCategory.SERVER_ERROR); + assertThat(recorder.byTenant(OTHER).fetchedCount).isZero(); + } + + @Test + void aPageThatIsNotAJsonArrayIsRejected() { + ExternalHttpClient http = mock(ExternalHttpClient.class); + ObjectNode notAnArray = mapper.createObjectNode(); + notAnArray.putArray("postings").add(mapper.createObjectNode().put("id", "job-0")); + when(http.getJson(anyString())).thenReturn(notAnArray); + + assertParseFailure(() -> source(http, List.of(COMPANY)).fetchCompany(COMPANY)); + assertThatThrownBy(() -> source(http, List.of(COMPANY)).fetchCompany(COMPANY)) + .isInstanceOfSatisfying(ExternalHttpException.class, failure -> { + assertThat(failure.parseDetail()) + .isEqualTo("page1: expected ARRAY but was OBJECT"); + // Rule and JSON type only: no value, URL, tenant, or fragment. + assertThat(failure.parseDetail()).doesNotContain("http", COMPANY, "job-0"); + }); + } + + @Test + void noConfiguredTenantMakesNoRequest() { + ExternalHttpClient http = mock(ExternalHttpClient.class); + + assertThat(source(http, List.of()).fetchJobs()).isEmpty(); + + verifyNoInteractions(http); + } + + private ExternalHttpException thrown(ExternalHttpClient http) { + requests.clear(); + try { + source(http, List.of(COMPANY)).fetchCompany(COMPANY); + } catch (ExternalHttpException failure) { + return failure; + } + throw new AssertionError("The tenant fetch was expected to fail"); + } + + private void assertParseFailure(org.assertj.core.api.ThrowableAssert.ThrowingCallable call) { + assertThatThrownBy(call).isInstanceOfSatisfying(ExternalHttpException.class, + failure -> assertThat(failure.category()) + .isEqualTo(ExternalHttpException.Category.MALFORMED_JSON)); + } + + /** Answers every request from the requested {@code skip} and records the URL. */ + private ExternalHttpClient pages(IntFunction bySkip) { + ExternalHttpClient http = mock(ExternalHttpClient.class); + when(http.getJson(anyString())).thenAnswer(invocation -> { + String url = invocation.getArgument(0, String.class); + requests.add(url); + return bySkip.apply(skip(url)); + }); + return http; + } + + private LeverJobSource source(ExternalHttpClient http, List companies) { + return source(http, companies, TenantFetchMonitor.disabled()); + } + + private LeverJobSource source(ExternalHttpClient http, List companies, + TenantFetchMonitor monitor) { + JobPilotProperties base = TestProperties.create(); + JobPilotProperties.Sources sources = new JobPilotProperties.Sources( + List.of(), companies, List.of(), List.of(), List.of()); + JobPilotProperties properties = new JobPilotProperties(base.telegram(), sources, + base.eligibility(), base.candidate(), base.http(), base.manualUrl(), base.llm(), + base.scheduling(), base.searchTerms(), base.locations()); + return new LeverJobSource(http, properties, monitor); + } + + private ArrayNode page(int firstId, int count) { + ArrayNode page = mapper.createArrayNode(); + for (int index = 0; index < count; index++) { + String id = "job-" + (firstId + index); + ObjectNode item = page.addObject(); + item.put("id", id).put("text", "Java Engineer " + id) + .put("hostedUrl", "https://jobs.example.test/" + id) + .put("descriptionPlain", "Java role " + id) + .put("createdAt", 1784200000000L); + item.putObject("categories").put("location", "Bucharest") + .put("commitment", "Internship").put("level", "Entry Level"); + } + return page; + } + + private List ids(int firstId, int count) { + List ids = new ArrayList<>(count); + for (int index = 0; index < count; index++) ids.add("job-" + (firstId + index)); + return ids; + } + + private String url(String company, int skip) { + return "https://api.lever.co/v0/postings/" + company + "?mode=json&skip=" + skip + + "&limit=50"; + } + + private int skip(String url) { + int start = url.indexOf("skip=") + "skip=".length(); + int end = url.indexOf('&', start); + return Integer.parseInt(end < 0 ? url.substring(start) : url.substring(start, end)); + } + + private static final class Call { + private String tenant; + private TenantAttemptStatus status; + private TenantFailure failure; + private int fetchedCount; + } + + private static final class RecordingRecorder extends SourceTenantHealthRecorder { + private final List calls = new ArrayList<>(); + + private RecordingRecorder() { + super(null, null); + } + + private Call byTenant(String tenant) { + return calls.stream().filter(call -> call.tenant.equals(tenant)).findFirst() + .orElseThrow(() -> new AssertionError("No attempt recorded for " + tenant)); + } + + @Override + public SourceTenantHealth record(UUID runId, String provider, String tenant, + TenantAttemptStatus status, TenantFailure failure, + int fetchedCount, long durationMs, + Instant startedAt, Instant finishedAt) { + Call call = new Call(); + call.tenant = tenant; + call.status = status; + call.failure = failure; + call.fetchedCount = fetchedCount; + calls.add(call); + return null; + } + } }