diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 511dd51..d1328ca 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.6.0" + ".": "2.7.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 33b7080..3b484e3 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 37 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-960cb623c7ec84bf4dc0f5945cbc19eec9cca48271071f400d96066eaa55dbd6.yml -openapi_spec_hash: 84fd39e3f4dc964bf0c32d4e95da1b34 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-758c3a2fbd5b7be61c8e6e0ad2e6a5ec30695747bb675960d888c497647d13d7.yml +openapi_spec_hash: 00002d90bde02f67e174368ae470c597 config_hash: 2bea1743c84d63bd61f8501a6ea63065 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a6d1da..f635b52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2.7.0 (2026-08-01) + +Full Changelog: [v2.6.0...v2.7.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.6.0...v2.7.0) + +### Features + +* **api:** api update ([4c828e0](https://github.com/context-dot-dev/context-go-sdk/commit/4c828e0356eb08bae31f2e3d027c36f371f8f601)) +* **api:** api update ([3caf6a2](https://github.com/context-dot-dev/context-go-sdk/commit/3caf6a2bd7c4f4f4d10d468dc6d8481f5532c4df)) +* **api:** api update ([c68d11b](https://github.com/context-dot-dev/context-go-sdk/commit/c68d11b9b68dc1fb625ae620c9f66beafef8bb0e)) + ## 2.6.0 (2026-07-31) Full Changelog: [v2.5.0...v2.6.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.5.0...v2.6.0) diff --git a/README.md b/README.md index 291ae6f..46b184b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Or to pin the version: ```sh -go get -u 'github.com/context-dot-dev/context-go-sdk@v2.6.0' +go get -u 'github.com/context-dot-dev/context-go-sdk@v2.7.0' ``` diff --git a/api.md b/api.md index bac920a..8cc5939 100644 --- a/api.md +++ b/api.md @@ -123,8 +123,10 @@ Methods: Response Types: -- contextdev.ErrorCount -- contextdev.Error +- contextdev.PageErrorCount +- contextdev.Failure +- contextdev.CrawlControls +- contextdev.Intake - contextdev.BatchGetResponse - contextdev.BatchListResponse - contextdev.BatchCancelResponse diff --git a/batch.go b/batch.go index e7ccac6..9415b9b 100644 --- a/batch.go +++ b/batch.go @@ -39,9 +39,7 @@ func NewBatchService(opts ...option.RequestOption) (r BatchService) { return } -// Check progress and get download links when the batch finishes. Also returns the -// rejected-URL list and webhook signing secret from submission, so nothing is lost -// if the submit response was dropped. +// Check progress, and get download links once the batch finishes. func (r *BatchService) Get(ctx context.Context, batchID string, opts ...option.RequestOption) (res *BatchGetResponse, err error) { opts = slices.Concat(r.options, opts) if batchID == "" { @@ -75,9 +73,8 @@ func (r *BatchService) Cancel(ctx context.Context, batchID string, opts ...optio return res, err } -// Page through the result records of a finished batch as JSON, in the same order -// as the downloadable result files. Use this instead of downloading and parsing -// the NDJSON files yourself. +// Page through a finished batch's results as JSON instead of downloading the +// NDJSON files. func (r *BatchService) GetResults(ctx context.Context, batchID string, query BatchGetResultsParams, opts ...option.RequestOption) (res *BatchGetResultsResponse, err error) { opts = slices.Concat(r.options, opts) if batchID == "" { @@ -98,7 +95,7 @@ func (r *BatchService) Submit(ctx context.Context, body BatchSubmitParams, opts } // Page failures sharing one error code. -type ErrorCount struct { +type PageErrorCount struct { // Error code for these failures. Code string `json:"code" api:"required"` // Pages that failed with this code. @@ -113,16 +110,17 @@ type ErrorCount struct { } // Returns the unmodified JSON received from the API -func (r ErrorCount) RawJSON() string { return r.JSON.raw } -func (r *ErrorCount) UnmarshalJSON(data []byte) error { +func (r PageErrorCount) RawJSON() string { return r.JSON.raw } +func (r *PageErrorCount) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Why the batch failed. -type Error struct { - // Batch error code. +// A failure of the batch as a whole, distinct from the per-page failures in +// `page_errors`. +type Failure struct { + // Why the batch itself stopped. Code string `json:"code" api:"required"` - // Batch error message. + // Human-readable explanation. Message string `json:"message" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { @@ -134,32 +132,189 @@ type Error struct { } // Returns the unmodified JSON received from the API -func (r Error) RawJSON() string { return r.JSON.raw } -func (r *Error) UnmarshalJSON(data []byte) error { +func (r Failure) RawJSON() string { return r.JSON.raw } +func (r *Failure) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// The crawl controls as submitted, so the limits requested can be compared against +// what the crawl reached. +type CrawlControls struct { + // Whether links to subdomains were followed. Always false for a sitemap crawl. + FollowSubdomains bool `json:"follow_subdomains" api:"required"` + // Link depth limit. Always 0 for a sitemap crawl, which never follows links off + // its URLs; null when a `start_url` crawl set no limit. + MaxDepth int64 `json:"max_depth" api:"required"` + // The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs + // its sitemap actually lists, up to this many, so `input.reserved` is often lower. + MaxPages int64 `json:"max_pages" api:"required"` + // Where the crawl started. + Source CrawlControlsSourceUnion `json:"source" api:"required"` + // RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + URLPattern string `json:"url_pattern" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + FollowSubdomains respjson.Field + MaxDepth respjson.Field + MaxPages respjson.Field + Source respjson.Field + URLPattern respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r CrawlControls) RawJSON() string { return r.JSON.raw } +func (r *CrawlControls) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// CrawlControlsSourceUnion contains all possible properties and values from +// [CrawlControlsSourceObject], [CrawlControlsSourceObject2]. +// +// Use the methods beginning with 'As' to cast the union to one of its variants. +type CrawlControlsSourceUnion struct { + Type string `json:"type"` + // This field is from variant [CrawlControlsSourceObject]. + URL string `json:"url"` + // This field is from variant [CrawlControlsSourceObject2]. + Domain string `json:"domain"` + JSON struct { + Type respjson.Field + URL respjson.Field + Domain respjson.Field + raw string + } `json:"-"` +} + +func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject() (v CrawlControlsSourceObject) { + apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) + return +} + +func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject2() (v CrawlControlsSourceObject2) { + apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) + return +} + +// Returns the unmodified JSON received from the API +func (u CrawlControlsSourceUnion) RawJSON() string { return u.JSON.raw } + +func (r *CrawlControlsSourceUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type CrawlControlsSourceObject struct { + // Any of "start_url". + Type string `json:"type" api:"required"` + // Page the crawl started from. + URL string `json:"url" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Type respjson.Field + URL respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r CrawlControlsSourceObject) RawJSON() string { return r.JSON.raw } +func (r *CrawlControlsSourceObject) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type CrawlControlsSourceObject2 struct { + // Domain whose sitemap supplied the pages. + Domain string `json:"domain" api:"required"` + // Any of "sitemap". + Type string `json:"type" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Domain respjson.Field + Type respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r CrawlControlsSourceObject2) RawJSON() string { return r.JSON.raw } +func (r *CrawlControlsSourceObject2) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// What submission took in, and what it charged for. +type Intake struct { + // URLs dropped before reserving because another entry resolved to the same page. + // Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than + // once. + Duplicates int64 `json:"duplicates" api:"required"` + // URLs from your list rejected as unusable; the same ones are itemised in + // `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable + // page is rejected outright with a 400 rather than accepted with an empty list. + Invalid int64 `json:"invalid" api:"required"` + // Pages credits were reserved for. Everything else — progress, the refund, the + // completion percentage — is measured against this. + Reserved int64 `json:"reserved" api:"required"` + // Whether `reserved` is an upper bound the batch may finish under. True only for a + // crawl that follows links, whose reachable page count is unknowable until it + // runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact + // page count. + ReservedIsCeiling bool `json:"reserved_is_ceiling" api:"required"` + // URLs in the list you sent, before validation and de-duplication. Null for a + // crawl, which is given a source rather than a list. + Submitted int64 `json:"submitted" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Duplicates respjson.Field + Invalid respjson.Field + Reserved respjson.Field + ReservedIsCeiling respjson.Field + Submitted respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r Intake) RawJSON() string { return r.JSON.raw } +func (r *Intake) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } type BatchGetResponse struct { // Batch ID used to retrieve or cancel the job. ID string `json:"id" api:"required"` - // Reserved and used credits. + // The crawl controls as submitted, so the limits requested can be compared against + // what the crawl reached. + Crawl CrawlControls `json:"crawl" api:"required"` + // What this batch has done to your credit balance. Credits BatchGetResponseCredits `json:"credits" api:"required"` - // Why the batch failed. - Error Error `json:"error" api:"required"` - // Page failures grouped by error code. - Errors []ErrorCount `json:"errors" api:"required"` - // Submission counts. - Input BatchGetResponseInput `json:"input" api:"required"` + // A failure of the batch as a whole, distinct from the per-page failures in + // `page_errors`. + Failure Failure `json:"failure" api:"required"` + // What each page is returned as. Matches `input.data.format` on the submit + // request. + // + // Any of "markdown", "html". + Format BatchGetResponseFormat `json:"format" api:"required"` + // What submission took in, and what it charged for. + Input Intake `json:"input" api:"required"` // Rejected URLs, up to 100. These are not charged. InvalidURLs []BatchGetResponseInvalidURL `json:"invalid_urls" api:"required"` - // How pages are selected. + // How pages were selected. Matches `input.mode` on the submit request. // // Any of "scrape", "crawl". Mode BatchGetResponseMode `json:"mode" api:"required"` - // Current processing counts. Use `status` to check completion. + // Individual page failures grouped by error code, sorted by count. Unrelated to + // `failure`, which is the batch itself failing. + PageErrors []PageErrorCount `json:"page_errors" api:"required"` + // Pages attempted so far. Use `status` to check completion. Progress BatchGetResponseProgress `json:"progress" api:"required"` - // Download links available when the batch finishes. GET /batch/{batch_id}/results - // serves the same records as paginated JSON. + // Download links, available once the batch reaches a final status and null before + // then. GET /batch/{batch_id}/results serves the same records as paginated JSON. Results BatchGetResponseResults `json:"results" api:"required"` // Current state. `completed`, `cancelled`, and `failed` are final. // @@ -168,33 +323,27 @@ type BatchGetResponse struct { // Tags stored on the batch at submission. Tags []string `json:"tags" api:"required"` Timing BatchGetResponseTiming `json:"timing" api:"required"` - // Output format. - // - // Any of "markdown", "html". - Type BatchGetResponseType `json:"type" api:"required"` // API key usage for this request. KeyMetadata BatchGetResponseKeyMetadata `json:"key_metadata"` - // Webhook signing secret. Also returned by GET /batch/{batch_id}. - WebhookSecret string `json:"webhook_secret"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - ID respjson.Field - Credits respjson.Field - Error respjson.Field - Errors respjson.Field - Input respjson.Field - InvalidURLs respjson.Field - Mode respjson.Field - Progress respjson.Field - Results respjson.Field - Status respjson.Field - Tags respjson.Field - Timing respjson.Field - Type respjson.Field - KeyMetadata respjson.Field - WebhookSecret respjson.Field - ExtraFields map[string]respjson.Field - raw string + ID respjson.Field + Crawl respjson.Field + Credits respjson.Field + Failure respjson.Field + Format respjson.Field + Input respjson.Field + InvalidURLs respjson.Field + Mode respjson.Field + PageErrors respjson.Field + Progress respjson.Field + Results respjson.Field + Status respjson.Field + Tags respjson.Field + Timing respjson.Field + KeyMetadata respjson.Field + ExtraFields map[string]respjson.Field + raw string } `json:"-"` } @@ -204,16 +353,22 @@ func (r *BatchGetResponse) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Reserved and used credits. +// What this batch has done to your credit balance. type BatchGetResponseCredits struct { - // Credits used by successful pages. - Charged int64 `json:"charged" api:"required"` - // Credits reserved when the batch was accepted. - Estimated int64 `json:"estimated" api:"required"` + // `reserved` minus `refunded` — what the batch has cost so far. Equal to + // `reserved` until the batch settles. + Net int64 `json:"net" api:"required"` + // Credits returned for pages that did not succeed. Stays 0 until the batch reaches + // a final status, then settles in one movement. + Refunded int64 `json:"refunded" api:"required"` + // Credits debited from your balance the moment the batch was accepted. This is a + // charge, not a forecast — the whole amount leaves the balance up front. + Reserved int64 `json:"reserved" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Charged respjson.Field - Estimated respjson.Field + Net respjson.Field + Refunded respjson.Field + Reserved respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -225,32 +380,14 @@ func (r *BatchGetResponseCredits) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Submission counts. -type BatchGetResponseInput struct { - // Pages accepted, or the crawl page limit. Credits are reserved for this count. - Accepted int64 `json:"accepted" api:"required"` - // Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - Duplicates int64 `json:"duplicates" api:"required"` - // Pages rejected during validation. - Invalid int64 `json:"invalid" api:"required"` - // Pages submitted before validation. For a crawl, the page limit. - Submitted int64 `json:"submitted" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Accepted respjson.Field - Duplicates respjson.Field - Invalid respjson.Field - Submitted respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} +// What each page is returned as. Matches `input.data.format` on the submit +// request. +type BatchGetResponseFormat string -// Returns the unmodified JSON received from the API -func (r BatchGetResponseInput) RawJSON() string { return r.JSON.raw } -func (r *BatchGetResponseInput) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} +const ( + BatchGetResponseFormatMarkdown BatchGetResponseFormat = "markdown" + BatchGetResponseFormatHTML BatchGetResponseFormat = "html" +) type BatchGetResponseInvalidURL struct { // Why it was rejected. @@ -272,7 +409,7 @@ func (r *BatchGetResponseInvalidURL) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// How pages are selected. +// How pages were selected. Matches `input.mode` on the submit request. type BatchGetResponseMode string const ( @@ -280,12 +417,13 @@ const ( BatchGetResponseModeCrawl BatchGetResponseMode = "crawl" ) -// Current processing counts. Use `status` to check completion. +// Pages attempted so far. Use `status` to check completion. type BatchGetResponseProgress struct { // Pages that could not be scraped. Failed int64 `json:"failed" api:"required"` - // Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - // finish under its page limit when the site has no more reachable pages. + // Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + // never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + // final, because its unspent budget was never real pages. Pending int64 `json:"pending" api:"required"` // Pages scraped successfully. Succeeded int64 `json:"succeeded" api:"required"` @@ -305,8 +443,8 @@ func (r *BatchGetResponseProgress) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Download links available when the batch finishes. GET /batch/{batch_id}/results -// serves the same records as paginated JSON. +// Download links, available once the batch reaches a final status and null before +// then. GET /batch/{batch_id}/results serves the same records as paginated JSON. type BatchGetResponseResults struct { // When the download URLs expire. ExpiresAt string `json:"expires_at" api:"required"` @@ -385,14 +523,6 @@ func (r *BatchGetResponseTiming) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Output format. -type BatchGetResponseType string - -const ( - BatchGetResponseTypeMarkdown BatchGetResponseType = "markdown" - BatchGetResponseTypeHTML BatchGetResponseType = "html" -) - // API key usage for this request. type BatchGetResponseKeyMetadata struct { // The number of credits consumed by this request. @@ -445,22 +575,32 @@ func (r *BatchListResponse) UnmarshalJSON(data []byte) error { type BatchListResponseData struct { // Batch ID used to retrieve or cancel the job. ID string `json:"id" api:"required"` - // Reserved and used credits. + // The crawl controls as submitted, so the limits requested can be compared against + // what the crawl reached. + Crawl CrawlControls `json:"crawl" api:"required"` + // What this batch has done to your credit balance. Credits BatchListResponseDataCredits `json:"credits" api:"required"` - // Why the batch failed. - Error Error `json:"error" api:"required"` - // Page failures grouped by error code. - Errors []ErrorCount `json:"errors" api:"required"` - // Submission counts. - Input BatchListResponseDataInput `json:"input" api:"required"` - // How pages are selected. + // A failure of the batch as a whole, distinct from the per-page failures in + // `page_errors`. + Failure Failure `json:"failure" api:"required"` + // What each page is returned as. Matches `input.data.format` on the submit + // request. + // + // Any of "markdown", "html". + Format string `json:"format" api:"required"` + // What submission took in, and what it charged for. + Input Intake `json:"input" api:"required"` + // How pages were selected. Matches `input.mode` on the submit request. // // Any of "scrape", "crawl". Mode string `json:"mode" api:"required"` - // Current processing counts. Use `status` to check completion. + // Individual page failures grouped by error code, sorted by count. Unrelated to + // `failure`, which is the batch itself failing. + PageErrors []PageErrorCount `json:"page_errors" api:"required"` + // Pages attempted so far. Use `status` to check completion. Progress BatchListResponseDataProgress `json:"progress" api:"required"` - // Download links available when the batch finishes. GET /batch/{batch_id}/results - // serves the same records as paginated JSON. + // Download links, available once the batch reaches a final status and null before + // then. GET /batch/{batch_id}/results serves the same records as paginated JSON. Results BatchListResponseDataResults `json:"results" api:"required"` // Current state. `completed`, `cancelled`, and `failed` are final. // @@ -469,24 +609,21 @@ type BatchListResponseData struct { // Tags stored on the batch at submission. Tags []string `json:"tags" api:"required"` Timing BatchListResponseDataTiming `json:"timing" api:"required"` - // Output format. - // - // Any of "markdown", "html". - Type string `json:"type" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field + Crawl respjson.Field Credits respjson.Field - Error respjson.Field - Errors respjson.Field + Failure respjson.Field + Format respjson.Field Input respjson.Field Mode respjson.Field + PageErrors respjson.Field Progress respjson.Field Results respjson.Field Status respjson.Field Tags respjson.Field Timing respjson.Field - Type respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -498,16 +635,22 @@ func (r *BatchListResponseData) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Reserved and used credits. +// What this batch has done to your credit balance. type BatchListResponseDataCredits struct { - // Credits used by successful pages. - Charged int64 `json:"charged" api:"required"` - // Credits reserved when the batch was accepted. - Estimated int64 `json:"estimated" api:"required"` + // `reserved` minus `refunded` — what the batch has cost so far. Equal to + // `reserved` until the batch settles. + Net int64 `json:"net" api:"required"` + // Credits returned for pages that did not succeed. Stays 0 until the batch reaches + // a final status, then settles in one movement. + Refunded int64 `json:"refunded" api:"required"` + // Credits debited from your balance the moment the batch was accepted. This is a + // charge, not a forecast — the whole amount leaves the balance up front. + Reserved int64 `json:"reserved" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Charged respjson.Field - Estimated respjson.Field + Net respjson.Field + Refunded respjson.Field + Reserved respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -519,39 +662,13 @@ func (r *BatchListResponseDataCredits) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Submission counts. -type BatchListResponseDataInput struct { - // Pages accepted, or the crawl page limit. Credits are reserved for this count. - Accepted int64 `json:"accepted" api:"required"` - // Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - Duplicates int64 `json:"duplicates" api:"required"` - // Pages rejected during validation. - Invalid int64 `json:"invalid" api:"required"` - // Pages submitted before validation. For a crawl, the page limit. - Submitted int64 `json:"submitted" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Accepted respjson.Field - Duplicates respjson.Field - Invalid respjson.Field - Submitted respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchListResponseDataInput) RawJSON() string { return r.JSON.raw } -func (r *BatchListResponseDataInput) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Current processing counts. Use `status` to check completion. +// Pages attempted so far. Use `status` to check completion. type BatchListResponseDataProgress struct { // Pages that could not be scraped. Failed int64 `json:"failed" api:"required"` - // Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - // finish under its page limit when the site has no more reachable pages. + // Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + // never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + // final, because its unspent budget was never real pages. Pending int64 `json:"pending" api:"required"` // Pages scraped successfully. Succeeded int64 `json:"succeeded" api:"required"` @@ -571,8 +688,8 @@ func (r *BatchListResponseDataProgress) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Download links available when the batch finishes. GET /batch/{batch_id}/results -// serves the same records as paginated JSON. +// Download links, available once the batch reaches a final status and null before +// then. GET /batch/{batch_id}/results serves the same records as paginated JSON. type BatchListResponseDataResults struct { // When the download URLs expire. ExpiresAt string `json:"expires_at" api:"required"` @@ -662,52 +779,51 @@ func (r *BatchListResponseKeyMetadata) UnmarshalJSON(data []byte) error { } type BatchCancelResponse struct { - // Batch ID used to retrieve or cancel the job. + // Batch ID. ID string `json:"id" api:"required"` - // Reserved and used credits. + // The crawl controls as submitted, so the limits requested can be compared against + // what the crawl reached. + Crawl CrawlControls `json:"crawl" api:"required"` + // What this batch cost so far. Credits BatchCancelResponseCredits `json:"credits" api:"required"` - // Why the batch failed. - Error Error `json:"error" api:"required"` - // Page failures grouped by error code. - Errors []ErrorCount `json:"errors" api:"required"` - // Submission counts. - Input BatchCancelResponseInput `json:"input" api:"required"` - // How pages are selected. + // What each page is returned as. + // + // Any of "markdown", "html". + Format BatchCancelResponseFormat `json:"format" api:"required"` + // What submission took in, and what it charged for. + Input Intake `json:"input" api:"required"` + // How pages were selected. // // Any of "scrape", "crawl". Mode BatchCancelResponseMode `json:"mode" api:"required"` - // Current processing counts. Use `status` to check completion. + // Page failures so far, grouped by error code and sorted by count. + PageErrors []PageErrorCount `json:"page_errors" api:"required"` + // How far the batch got before cancellation. Progress BatchCancelResponseProgress `json:"progress" api:"required"` - // Download links available when the batch finishes. GET /batch/{batch_id}/results - // serves the same records as paginated JSON. - Results BatchCancelResponseResults `json:"results" api:"required"` - // Current state. `completed`, `cancelled`, and `failed` are final. + // Always `cancelling`. Work already in flight finishes; the batch reaches + // `cancelled` shortly after. // - // Any of "queued", "running", "cancelling", "completed", "cancelled", "failed". + // Any of "cancelling". Status BatchCancelResponseStatus `json:"status" api:"required"` // Tags stored on the batch at submission. - Tags []string `json:"tags" api:"required"` + Tags []string `json:"tags" api:"required"` + // There is no finish time yet — the batch is still winding down. Timing BatchCancelResponseTiming `json:"timing" api:"required"` - // Output format. - // - // Any of "markdown", "html". - Type BatchCancelResponseType `json:"type" api:"required"` // API key usage for this request. KeyMetadata BatchCancelResponseKeyMetadata `json:"key_metadata"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { ID respjson.Field + Crawl respjson.Field Credits respjson.Field - Error respjson.Field - Errors respjson.Field + Format respjson.Field Input respjson.Field Mode respjson.Field + PageErrors respjson.Field Progress respjson.Field - Results respjson.Field Status respjson.Field Tags respjson.Field Timing respjson.Field - Type respjson.Field KeyMetadata respjson.Field ExtraFields map[string]respjson.Field raw string @@ -720,16 +836,14 @@ func (r *BatchCancelResponse) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Reserved and used credits. +// What this batch cost so far. type BatchCancelResponseCredits struct { - // Credits used by successful pages. - Charged int64 `json:"charged" api:"required"` - // Credits reserved when the batch was accepted. - Estimated int64 `json:"estimated" api:"required"` + // Credits debited at submission. The unspent remainder is refunded once the batch + // settles — read `credits.refunded` from GET /batch/{batch_id} then. + Reserved int64 `json:"reserved" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Charged respjson.Field - Estimated respjson.Field + Reserved respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` @@ -741,34 +855,15 @@ func (r *BatchCancelResponseCredits) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Submission counts. -type BatchCancelResponseInput struct { - // Pages accepted, or the crawl page limit. Credits are reserved for this count. - Accepted int64 `json:"accepted" api:"required"` - // Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - Duplicates int64 `json:"duplicates" api:"required"` - // Pages rejected during validation. - Invalid int64 `json:"invalid" api:"required"` - // Pages submitted before validation. For a crawl, the page limit. - Submitted int64 `json:"submitted" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Accepted respjson.Field - Duplicates respjson.Field - Invalid respjson.Field - Submitted respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} +// What each page is returned as. +type BatchCancelResponseFormat string -// Returns the unmodified JSON received from the API -func (r BatchCancelResponseInput) RawJSON() string { return r.JSON.raw } -func (r *BatchCancelResponseInput) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} +const ( + BatchCancelResponseFormatMarkdown BatchCancelResponseFormat = "markdown" + BatchCancelResponseFormatHTML BatchCancelResponseFormat = "html" +) -// How pages are selected. +// How pages were selected. type BatchCancelResponseMode string const ( @@ -776,14 +871,13 @@ const ( BatchCancelResponseModeCrawl BatchCancelResponseMode = "crawl" ) -// Current processing counts. Use `status` to check completion. +// How far the batch got before cancellation. type BatchCancelResponseProgress struct { - // Pages that could not be scraped. + // Pages that could not be scraped before the request landed. Failed int64 `json:"failed" api:"required"` - // Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - // finish under its page limit when the site has no more reachable pages. + // Reserved pages that will now be skipped, and refunded when the batch settles. Pending int64 `json:"pending" api:"required"` - // Pages scraped successfully. + // Pages scraped successfully before the request landed. Succeeded int64 `json:"succeeded" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { @@ -801,73 +895,22 @@ func (r *BatchCancelResponseProgress) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Download links available when the batch finishes. GET /batch/{batch_id}/results -// serves the same records as paginated JSON. -type BatchCancelResponseResults struct { - // When the download URLs expire. - ExpiresAt string `json:"expires_at" api:"required"` - // Result files. Order is not guaranteed. - Files []BatchCancelResponseResultsFile `json:"files" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - ExpiresAt respjson.Field - Files respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchCancelResponseResults) RawJSON() string { return r.JSON.raw } -func (r *BatchCancelResponseResults) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -type BatchCancelResponseResultsFile struct { - // Compressed file size in bytes. - Bytes int64 `json:"bytes" api:"required"` - // Results in this file. - Items int64 `json:"items" api:"required"` - // Temporary URL for a gzipped NDJSON file. - URL string `json:"url" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Bytes respjson.Field - Items respjson.Field - URL respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchCancelResponseResultsFile) RawJSON() string { return r.JSON.raw } -func (r *BatchCancelResponseResultsFile) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Current state. `completed`, `cancelled`, and `failed` are final. +// Always `cancelling`. Work already in flight finishes; the batch reaches +// `cancelled` shortly after. type BatchCancelResponseStatus string const ( - BatchCancelResponseStatusQueued BatchCancelResponseStatus = "queued" - BatchCancelResponseStatusRunning BatchCancelResponseStatus = "running" BatchCancelResponseStatusCancelling BatchCancelResponseStatus = "cancelling" - BatchCancelResponseStatusCompleted BatchCancelResponseStatus = "completed" - BatchCancelResponseStatusCancelled BatchCancelResponseStatus = "cancelled" - BatchCancelResponseStatusFailed BatchCancelResponseStatus = "failed" ) +// There is no finish time yet — the batch is still winding down. type BatchCancelResponseTiming struct { - // When processing finished. Null while active. - CompletedAt string `json:"completed_at" api:"required"` // When the batch was created. CreatedAt string `json:"created_at" api:"required"` - // When processing started. Null while queued. + // When processing started. Null if it was cancelled while still queued. StartedAt string `json:"started_at" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - CompletedAt respjson.Field CreatedAt respjson.Field StartedAt respjson.Field ExtraFields map[string]respjson.Field @@ -881,14 +924,6 @@ func (r *BatchCancelResponseTiming) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Output format. -type BatchCancelResponseType string - -const ( - BatchCancelResponseTypeMarkdown BatchCancelResponseType = "markdown" - BatchCancelResponseTypeHTML BatchCancelResponseType = "html" -) - // API key usage for this request. type BatchCancelResponseKeyMetadata struct { // The number of credits consumed by this request. diff --git a/internal/version.go b/internal/version.go index e0668d4..166fc82 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,4 +2,4 @@ package internal -const PackageVersion = "2.6.0" // x-release-please-version +const PackageVersion = "2.7.0" // x-release-please-version diff --git a/web.go b/web.go index 6a6a9b6..b4c3e51 100644 --- a/web.go +++ b/web.go @@ -2659,9 +2659,8 @@ type WebScreenshotParams struct { // // Any of "light", "dark". ColorScheme WebScreenshotParamsColorScheme `query:"colorScheme,omitzero" json:"-"` - // Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - // residential proxy exit location. Must be one of Context.dev's supported - // countries. When provided, Context.dev fetches the target page from that country. + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). // // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", @@ -2732,9 +2731,8 @@ const ( WebScreenshotParamsColorSchemeDark WebScreenshotParamsColorScheme = "dark" ) -// Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev -// residential proxy exit location. Must be one of Context.dev's supported -// countries. When provided, Context.dev fetches the target page from that country. +// Fetch the target page through a residential proxy in this country (ISO 3166-1 +// alpha-2). type WebScreenshotParamsCountry string const ( @@ -3437,9 +3435,8 @@ type WebWebCrawlMdParams struct { // Optional browser wait time in milliseconds after initial page load for each // crawled page. Min: 0. Max: 30000 (30 seconds). WaitForMs param.Opt[int64] `json:"waitForMs,omitzero"` - // Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - // residential proxy exit location. Must be one of Context.dev's supported - // countries. When provided, Context.dev fetches the target page from that country. + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). // // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", @@ -3490,9 +3487,8 @@ func (r *WebWebCrawlMdParams) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev -// residential proxy exit location. Must be one of Context.dev's supported -// countries. When provided, Context.dev fetches the target page from that country. +// Fetch the target page through a residential proxy in this country (ISO 3166-1 +// alpha-2). type WebWebCrawlMdParamsCountry string const ( @@ -3765,9 +3761,8 @@ type WebWebScrapeHTMLParams struct { // kept and everything else is dropped. When omitted, the entire document is kept. // Examples: "article.main", "#content", "[role=main]". IncludeSelectors []string `query:"includeSelectors,omitzero" json:"-"` - // Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - // residential proxy exit location. Must be one of Context.dev's supported - // countries. When provided, Context.dev fetches the target page from that country. + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). // // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", @@ -3879,9 +3874,8 @@ func (r WebWebScrapeHTMLParamsActionPerform) URLQuery() (v url.Values, err error }) } -// Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev -// residential proxy exit location. Must be one of Context.dev's supported -// countries. When provided, Context.dev fetches the target page from that country. +// Fetch the target page through a residential proxy in this country (ISO 3166-1 +// alpha-2). type WebWebScrapeHTMLParamsCountry string const ( @@ -4442,9 +4436,8 @@ type WebWebScrapeMdParams struct { // descendants) are kept before conversion to Markdown. When omitted, the entire // document is kept. Examples: "article.main", "#content", "[role=main]". IncludeSelectors []string `query:"includeSelectors,omitzero" json:"-"` - // Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - // residential proxy exit location. Must be one of Context.dev's supported - // countries. When provided, Context.dev fetches the target page from that country. + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). // // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", @@ -4562,9 +4555,8 @@ func (r WebWebScrapeMdParamsActionPerform) URLQuery() (v url.Values, err error) }) } -// Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev -// residential proxy exit location. Must be one of Context.dev's supported -// countries. When provided, Context.dev fetches the target page from that country. +// Fetch the target page through a residential proxy in this country (ISO 3166-1 +// alpha-2). type WebWebScrapeMdParamsCountry string const (