diff --git a/.github/release/dashboard.mjs b/.github/release/dashboard.mjs index 8804c7b95..91c301bc2 100644 --- a/.github/release/dashboard.mjs +++ b/.github/release/dashboard.mjs @@ -10,7 +10,7 @@ import { createGitHub } from "./lib/github.mjs"; const MARKER = ""; const TITLE = "🚀 Releases en cours"; const LABEL = "release"; -const PAIRS = new Set(["dev>beta", "beta>main"]); +const PAIRS = new Set(["dev>main", "dev>beta", "beta>main"]); const TITLE_VERSION = /^chore\(release\): (v\S+)$/; // GitHub turns #N into a link to the pull request. diff --git a/.github/release/lib/flow.mjs b/.github/release/lib/flow.mjs new file mode 100644 index 000000000..a411da054 --- /dev/null +++ b/.github/release/lib/flow.mjs @@ -0,0 +1,21 @@ +// The promotion chain. Until a v2 (maybe never), `beta` is skipped: dev is promoted straight to main +// as a stable, and a release is brought back to dev straight from main. +// To bring beta back: set BETA to true here, then put `beta` back in promote.yml's push branches (the +// workflows' other conditions accept both chains). + +export const BETA = false; + +/** The promotions the bot opens, in release order. */ +export function promotions(beta = BETA) { + return beta + ? [ + { source: "dev", target: "beta" }, + { source: "beta", target: "main" }, + ] + : [{ source: "dev", target: "main" }]; +} + +/** The branch `source` is promoted to, or null when it is not promoted. */ +export function promotionTarget(source, beta = BETA) { + return promotions(beta).find((p) => p.source === source)?.target ?? null; +} diff --git a/.github/release/lib/version.mjs b/.github/release/lib/version.mjs index c65fe6d1c..7b4e39c9c 100644 --- a/.github/release/lib/version.mjs +++ b/.github/release/lib/version.mjs @@ -1,4 +1,5 @@ -// Version computation for the dev -> beta -> main release flow. +// Version computation for the release flow: dev -> main while beta is skipped (lib/flow.mjs), else +// dev -> beta -> main. // Tags are `vX.Y.Z` (stable) and `vX.Y.Z-BETA.N` (pre-release). const TAG = /^v(\d+)\.(\d+)\.(\d+)(?:-BETA\.(\d+))?$/; @@ -74,6 +75,20 @@ export function nextBeta(lastStable, betaTags, subjects) { return format(target, n); } +/** + * Next stable promoted straight from dev (beta skipped): lastStable bumped by the commits, or the highest + * target among the betas newer than lastStable when that is higher; null when no commit is releasable. + */ +export function nextDirectStable(lastStable, betaTags, subjects) { + const kind = bumpKind(subjects, lastStable); + if (kind === null) return null; + let target = bump(parseVersion(lastStable), kind); + for (const beta of betaTags.map(parseVersion)) { + if (beta.beta !== null && compare(beta, target) > 0) target = beta; + } + return format(target); +} + export function nextStable(betaTag) { const version = parseVersion(betaTag); if (version.beta === null) throw new Error(`Not a beta tag: ${betaTag}`); diff --git a/.github/release/promote.mjs b/.github/release/promote.mjs index cb4e6cd84..4f0761411 100644 --- a/.github/release/promote.mjs +++ b/.github/release/promote.mjs @@ -1,10 +1,12 @@ // Opens (in draft) or refreshes the single promotion PR from `source` to `target`. -// CLI: node .github/release/promote.mjs --source dev --target beta -// (env: GH_TOKEN, GITHUB_REPOSITORY=owner/repo, APP_SLUG = the release App's slug) +// CLI: node .github/release/promote.mjs --source dev [--target main] +// (--target defaults to the source's next branch in lib/flow.mjs; env: GH_TOKEN, +// GITHUB_REPOSITORY=owner/repo, APP_SLUG = the release App's slug) import { parseArgs } from "node:util"; import { pathToFileURL } from "node:url"; -import { parseVersion, bumpKind, nextBeta, nextStable } from "./lib/version.mjs"; +import { parseVersion, bumpKind, nextBeta, nextDirectStable, nextStable } from "./lib/version.mjs"; +import { promotionTarget } from "./lib/flow.mjs"; import { buildBody, parseBody, scheduledText, seenId, setPromotedHead, setScheduledText, updateBody } from "./lib/pr-body.mjs"; import { activeSchedule, NO_SCHEDULE } from "./lib/schedule.mjs"; import { createGitHub } from "./lib/github.mjs"; @@ -37,16 +39,20 @@ export function compareVersions(a, b) { } /** - * Stable for a promotion to `main`, beta otherwise; null when there is nothing to release. `commits` - * are the promoted ones (release commits left out). ready.mjs checks the PR title against it. + * Stable for a promotion to `main` (dev -> main bumps the last stable, beta -> main finishes the last + * beta), beta otherwise; null when there is nothing to release. `commits` are the promoted ones + * (release commits left out). ready.mjs checks the PR title against it. */ -export function releaseVersion(target, tags, commits) { +export function releaseVersion(source, target, tags, commits) { const versions = versionTags(tags).sort(compareVersions); const stables = versions.filter((v) => v.beta === null); const lastStable = stables.length ? stables[stables.length - 1] : { tag: "v0.0.0", ...parseVersion("v0.0.0") }; const betas = versions.filter((v) => v.beta !== null && compareVersions(v, lastStable) > 0); - if (target === "main") return betas.length ? nextStable(betas[betas.length - 1].tag) : null; const messages = commits.map((c) => c.message); + if (target === "main" && source === "dev") { + return nextDirectStable(lastStable.tag, betas.map((v) => v.tag), messages); + } + if (target === "main") return betas.length ? nextStable(betas[betas.length - 1].tag) : null; if (bumpKind(messages, lastStable.tag) === null && betas.length === 0) return null; return nextBeta(lastStable.tag, betas.map((v) => v.tag), messages); } @@ -97,7 +103,7 @@ export async function promote({ gh, source, target, owner, repo, appSlug }) { return { action: "closed", number: pull.number }; } - const version = releaseVersion(target, await gh.listTags(), commits); + const version = releaseVersion(source, target, await gh.listTags(), commits); if (version === null) return withNumber("none"); const title = `chore(release): ${version}`; @@ -137,7 +143,8 @@ export async function promote({ gh, source, target, owner, repo, appSlug }) { if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { const { values } = parseArgs({ options: { source: { type: "string" }, target: { type: "string" } } }); - if (!values.source || !values.target) throw new Error("Usage: promote.mjs --source --target "); + values.target ??= values.source ? (promotionTarget(values.source) ?? undefined) : undefined; + if (!values.source || !values.target) throw new Error("Usage: promote.mjs --source [--target ]"); const [owner, repo] = (process.env.GITHUB_REPOSITORY ?? "").split("/"); const gh = createGitHub({ token: process.env.GH_TOKEN, owner, repo }); const result = await promote({ ...values, gh, owner, repo, appSlug: process.env.APP_SLUG }); diff --git a/.github/release/publish.mjs b/.github/release/publish.mjs index fd26eb781..9800ee02d 100644 --- a/.github/release/publish.mjs +++ b/.github/release/publish.mjs @@ -1,5 +1,6 @@ // A promotion (or hotfix) PR was merged: tag its merge commit, and after a stable or a hotfix open -// the backport PRs that bring it back to dev (main -> beta, then beta -> dev once that one is merged). The tag is pushed with the App token, so its push event runs +// the backport PRs that bring it back to dev, with auto-merge (main -> dev while beta is skipped, else +// main -> beta, then beta -> dev once that one is merged). The tag is pushed with the App token, so its push event runs // release.yml, the single publication path (it reads the PR's notes itself). After a beta, the beta -> main // promotion is retitled by publish.yml's own job, in the promote-beta concurrency group, not here. // CLI: node .github/release/publish.mjs --number 12 @@ -10,6 +11,7 @@ import { pathToFileURL } from "node:url"; import { createGitHub } from "./lib/github.mjs"; import { refreshDashboardSafely } from "./dashboard.mjs"; import { kindOf, titleVersion } from "./ready.mjs"; +import { BETA } from "./lib/flow.mjs"; const short = (sha) => sha.slice(0, 7); const pattern = (kind) => (kind === "beta" ? "vX.Y.Z-BETA.N" : "vX.Y.Z"); @@ -32,40 +34,63 @@ function isBackportToBeta(pr, repository) { return pr.headRepo === repository && pr.head === "main" && pr.base === "beta" && pr.labels?.includes("backport"); } -function backportBody(tag, pr, from, to) { +function backportBody(tag, pr, from, to, autoMerge) { return ( `Report de \`${tag}\` (#${pr.number}) de \`${from}\` vers \`${to}\`.\n\n` + - `Fusionne cette PR par commit de merge pour ramener la release sur \`${to}\` ; tant qu'elle est ` + - `ouverte, la promotion depuis et vers \`${to}\` est refusĂ©e.\n` + (autoMerge + ? `L'auto-merge est activĂ© : la PR se fusionne d'elle-mĂȘme (commit de merge) dĂšs que les checks passent` + : `Fusionne cette PR par commit de merge pour ramener la release sur \`${to}\``) + + ` ; tant qu'elle est ouverte, la promotion depuis et vers \`${to}\` est refusĂ©e.\n` ); } +/** Auto-merge (merge commit) pinned to the backport's head; a failure leaves the merge to the owner. */ +async function autoMergeBackport(gh, created, title) { + try { + await gh.enableAutoMerge(created, { expectedHeadOid: created.headSha, commitHeadline: title }); + return true; + } catch (error) { + console.warn(`::warning::#${created.number}: could not enable auto-merge: ${error.message}`); + return false; + } +} + /** - * Opens the backport PR `from` -> `to` unless one is open (it is reused) or there is nothing to bring. + * Opens the backport PR `from` -> `to`, with auto-merge, unless one is open (it is reused) or there is + * nothing to bring. * -> {number} when opened, {pending: true} when one is already open, null when there is nothing to bring. */ async function openBackport(gh, tag, pr, from, to) { const [open, commits] = await Promise.all([gh.listPulls({ base: to, head: from }), gh.compareCommits(to, from)]); if (open.length) return { pending: true }; if (commits.length === 0) return null; + const title = `chore(release): backport ${tag} to ${to}`; const created = await gh.createPull({ - title: `chore(release): backport ${tag} to ${to}`, + title, base: to, head: from, draft: false, labels: ["backport"], - body: backportBody(tag, pr, from, to), + body: backportBody(tag, pr, from, to, true), }); console.log(`opened the backport #${created.number} ${from} -> ${to}`); + if (!(await autoMergeBackport(gh, created, title))) { + await gh.updatePull(created.number, { body: backportBody(tag, pr, from, to, false) }); + } return { number: created.number }; } /** * Brings the release back to dev, whose version line would otherwise conflict with the next - * promotion: a stable through beta -> dev; a hotfix through main -> beta first, beta -> dev being - * opened once that one is merged (or right away when beta needs nothing). Returns the PRs opened. + * promotion. Beta skipped (lib/flow.mjs): main -> dev. Otherwise a stable through beta -> dev; a hotfix + * through main -> beta first, beta -> dev being opened once that one is merged (or right away when beta + * needs nothing). Returns the PRs opened. */ -async function openBackports(gh, tag, pr, kind) { +async function openBackports(gh, tag, pr, kind, beta) { + if (!beta && kind !== "backport") { + const toDev = await openBackport(gh, tag, pr, "main", "dev"); + return toDev?.number ? [toDev.number] : []; + } if (kind === "hotfix") { const toBeta = await openBackport(gh, tag, pr, "main", "beta"); if (toBeta !== null) return toBeta.number ? [toBeta.number] : []; @@ -74,31 +99,33 @@ async function openBackports(gh, tag, pr, kind) { return toDev?.number ? [toDev.number] : []; } -async function release(gh, pr, kind) { +async function release(gh, pr, kind, beta) { if (!pr.merged || !pr.mergeCommitSha) throw new Error(`#${pr.number} is not merged`); const tag = titleVersion(pr.title, kind); if (tag === null) throw new Error(`#${pr.number}: the title does not carry the version to publish: ${pr.title}`); await ensureTag(gh, tag, pr.mergeCommitSha, pr.title); - const backports = kind === "beta" ? [] : await openBackports(gh, tag, pr, kind); + // A stable promoted from beta goes back through beta -> dev, whatever the current chain. + const through = kind === "stable" && pr.head === "beta" ? true : beta; + const backports = kind === "beta" ? [] : await openBackports(gh, tag, pr, kind, through); return { tag, backports }; } /** * -> {tag, backports} (numbers of the backport PRs opened). Throws on an unmerged or unrelated PR; a - * failure on a release PR is also reported on it. + * failure on a release PR is also reported on it. `beta`: whether the chain goes through beta. */ -export async function publish({ gh, pr, owner, repo }) { +export async function publish({ gh, pr, owner, repo, beta = BETA }) { if (isBackportToBeta(pr, `${owner}/${repo}`)) { if (!pr.merged) throw new Error(`#${pr.number} is not merged`); const tag = BACKPORT_TITLE.exec(pr.title)?.[1]; if (!tag) throw new Error(`#${pr.number}: the title does not carry the backported version: ${pr.title}`); - return { tag, backports: await openBackports(gh, tag, pr, "stable") }; + return { tag, backports: await openBackports(gh, tag, pr, "backport", true) }; } const kind = kindOf(pr, `${owner}/${repo}`); if (kind === null) throw new Error(`#${pr.number} is not a promotion or hotfix pull request`); try { - return await release(gh, pr, kind); + return await release(gh, pr, kind, beta); } catch (error) { const version = titleVersion(pr.title, kind); if (pr.merged) { diff --git a/.github/release/ready.mjs b/.github/release/ready.mjs index 41697e07e..a3cfb2eb2 100644 --- a/.github/release/ready.mjs +++ b/.github/release/ready.mjs @@ -30,11 +30,14 @@ export function bumpGradleVersion(text, version) { return text.replace(VERSION_LINE, (_, prefix) => `${prefix}"${version}"`); } -/** Promotions dev -> beta and beta -> main, and hotfixes into main, all from this repository. */ +/** + * Promotions dev -> main (beta skipped, lib/flow.mjs), dev -> beta and beta -> main, and hotfixes into + * main, all from this repository. + */ export function kindOf(pull, repository) { if (pull.headRepo !== repository) return null; if (pull.head === "dev" && pull.base === "beta") return "beta"; - if (pull.head === "beta" && pull.base === "main") return "stable"; + if ((pull.head === "dev" || pull.head === "beta") && pull.base === "main") return "stable"; if (pull.head.startsWith("hotfix/") && pull.base === "main") return "hotfix"; return null; } @@ -164,7 +167,7 @@ async function prepare({ gh, pr, pull, kind, owner, repo, appSlug, promote, wait if (promotion) { const [tags, compared] = await Promise.all([gh.listTags(), gh.compareCommits(pull.base, pull.head)]); const promoted = compared.filter((c) => !isReleaseCommit(c, appSlug)); - const expected = releaseVersion(pull.base, tags, promoted); + const expected = releaseVersion(pull.head, pull.base, tags, promoted); if (expected !== version) return refuseAndRefresh(gh, pull, MESSAGES.staleVersion(version, expected), refresh); } diff --git a/.github/release/schedule.mjs b/.github/release/schedule.mjs index 63a9fb7bf..3c2aa86b1 100644 --- a/.github/release/schedule.mjs +++ b/.github/release/schedule.mjs @@ -1,9 +1,10 @@ // Scheduled releases. --confirm answers the owner's edits of a promotion PR: only the App's comment // makes a date active. --due lists the source branches whose date has come, and --run takes their PR // out of draft, then follows the same path as the owner doing it. -// CLI: node .github/release/schedule.mjs --confirm | --due | --run --source dev --target beta +// CLI: node .github/release/schedule.mjs --confirm | --due | --run --source dev [--target main] // (env: GH_TOKEN, GITHUB_REPOSITORY=owner/repo, APP_SLUG = the release App's slug; GITHUB_EVENT_PATH -// for --confirm, GITHUB_OUTPUT for --due) +// for --confirm, GITHUB_OUTPUT for --due). --run takes the target from lib/flow.mjs: --target is only +// a fallback, so an older release-schedule.yml (still on main until the next release) stays right. import { appendFileSync, readFileSync } from "node:fs"; import { parseArgs } from "node:util"; @@ -13,11 +14,7 @@ import { activeSchedule, formatParis, NO_SCHEDULE, parseScheduleDate, scheduleMa import { createGitHub } from "./lib/github.mjs"; import { isReleaseCommit } from "./promote.mjs"; import { kindOf, onReady } from "./ready.mjs"; - -export const PROMOTIONS = [ - { source: "dev", target: "beta" }, - { source: "beta", target: "main" }, -]; +import { BETA, promotions, promotionTarget } from "./lib/flow.mjs"; // Past this delay, the start of the release says it is late (GitHub's schedules may be delayed). export const LATE_NOTICE_MS = 60 * 60 * 1000; @@ -72,9 +69,9 @@ export async function confirm({ gh, event, owner, repo, appSlug, now = new Date( } /** The source branches of the open promotions whose scheduled date has come. */ -export async function dueSources({ gh, appSlug, now = new Date() }) { +export async function dueSources({ gh, appSlug, now = new Date(), beta = BETA }) { const due = []; - for (const { source, target } of PROMOTIONS) { + for (const { source, target } of promotions(beta)) { const [pull] = await gh.listPulls({ base: target, head: source }); if (!pull) continue; const active = activeSchedule(await gh.listComments(pull.number), appSlug); @@ -156,9 +153,10 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { const due = await dueSources({ gh, appSlug }); console.log(due); appendFileSync(process.env.GITHUB_OUTPUT, `sources=${JSON.stringify(due)}\n`); - } else if (values.run && values.source && values.target) { - console.log(await run({ gh, source: values.source, target: values.target, owner, repo, appSlug })); + } else if (values.run && values.source && (promotionTarget(values.source) ?? values.target)) { + const target = promotionTarget(values.source) ?? values.target; + console.log(await run({ gh, source: values.source, target, owner, repo, appSlug })); } else { - throw new Error("Usage: schedule.mjs --confirm | --due | --run --source --target "); + throw new Error("Usage: schedule.mjs --confirm | --due | --run --source [--target ]"); } } diff --git a/.github/release/test/dashboard.test.mjs b/.github/release/test/dashboard.test.mjs index 1b78fdc8c..fb5baf078 100644 --- a/.github/release/test/dashboard.test.mjs +++ b/.github/release/test/dashboard.test.mjs @@ -71,15 +71,17 @@ test("refresh_derivesStateFromThePullRequests", async () => { pull(1), pull(2, { draft: false }), pull(3, { draft: false, autoMerge: true, head: "beta", base: "main", title: "chore(release): v1.0.0" }), - pull(4, { draft: false, base: "main", head: "dev" }), // not a promotion pair + pull(4, { draft: false, base: "main", head: "feat/x" }), // not a promotion pair pull(5, { labels: ["backport"], head: "main", base: "beta" }), pull(6, { labels: [] }), + pull(7, { base: "main", head: "dev", title: "chore(release): v1.2.0" }), // beta skipped ], mergeStates: { 2: "DIRTY" }, }); await refreshDashboard({ gh }); const body = gh.calls.find(([name]) => name === "create")[1].body; assert.match(body, /v1\.0\.0-BETA\.1 \| draft \| #1/); + assert.match(body, /`dev` → `main` \| v1\.2\.0 \| draft \| #7/); assert.match(body, /v1\.0\.0-BETA\.2 \| en conflit \| #2/); assert.match(body, /\| v1\.0\.0 \| prĂȘte \(auto-merge\) \| #3/); assert.doesNotMatch(body, /#4|#6/); diff --git a/.github/release/test/promote.test.mjs b/.github/release/test/promote.test.mjs index 7d7306dbc..3080b82b0 100644 --- a/.github/release/test/promote.test.mjs +++ b/.github/release/test/promote.test.mjs @@ -171,6 +171,24 @@ test("betaToMain_usesTheStableOfTheLatestBeta", async () => { assert.equal(gh.calls[0][1].title, "chore(release): v1.0.0"); }); +test("devToMain_bumpsTheLastStableStraightAway", async () => { + const gh = fakeGitHub({ + tags: ["v1.0.0-BETA.102", "v1.1.0-BETA.1", "v1.1.0"], + ahead: { "main...dev": [commit("a", "feat: x"), commit("b", "fix: y")] }, + }); + assert.deepEqual(await run(gh, "dev", "main"), { action: "created", number: 100 }); + const [[, created]] = gh.calls; + assert.equal(created.title, "chore(release): v1.2.0"); + assert.equal(created.base, "main"); + assert.equal(created.head, "dev"); +}); + +test("devToMain_withoutReleasableCommit_doesNothing", async () => { + const gh = fakeGitHub({ tags: ["v1.1.0"], ahead: { "main...dev": [commit("a", "not conventional")] } }); + assert.deepEqual(await run(gh, "dev", "main"), { action: "none" }); + assert.deepEqual(gh.calls, []); +}); + test("betaToMain_withoutBeta_doesNothing", async () => { const gh = fakeGitHub({ tags: ["v1.0.0"], ahead: { "main...beta": [commit("a", "fix: x")] } }); assert.deepEqual(await run(gh, "beta", "main"), { action: "none" }); diff --git a/.github/release/test/publish.test.mjs b/.github/release/test/publish.test.mjs index 4b90b1c85..4606a4787 100644 --- a/.github/release/test/publish.test.mjs +++ b/.github/release/test/publish.test.mjs @@ -9,7 +9,7 @@ const sha = (c) => c.repeat(40); const CHANGELOG = "### FonctionnalitĂ©s\n\n- abc1234 edited by hand"; // In-memory GitHub: tags map name -> commit SHA, pulls the open ones. -function fakeGitHub({ tags = {}, pulls = [], compare = {} } = {}) { +function fakeGitHub({ tags = {}, pulls = [], compare = {}, autoMergeFails = false } = {}) { const calls = []; const record = (name, ...args) => calls.push([name, ...args]); return { @@ -32,7 +32,14 @@ function fakeGitHub({ tags = {}, pulls = [], compare = {} } = {}) { }, async createPull(fields) { record("createPull", fields); - return { number: 100 + calls.length, ...fields }; + return { number: 100 + calls.length, nodeId: "PR_N", headSha: sha("h"), ...fields }; + }, + async enableAutoMerge(pull, options) { + record("enableAutoMerge", pull.number, options); + if (autoMergeFails) throw new Error("Pull request is in clean status"); + }, + async updatePull(number, fields) { + record("updatePull", number, fields); }, async comment(number, text) { record("comment", number, text); @@ -51,7 +58,9 @@ function mergedPull({ return { number: 7, title, body, state: "closed", merged, base, head, headRepo, mergeCommitSha: sha("m") }; } -const run = (gh, pr) => publish({ gh, pr, owner: OWNER, repo: REPO }); +// `beta`: whether the chain goes through beta (lib/flow.mjs); left out, the current chain. +const run = (gh, pr, beta) => publish({ gh, pr, owner: OWNER, repo: REPO, beta }); +const withBeta = (gh, pr) => run(gh, pr, true); const named = (gh, name) => gh.calls.filter((c) => c[0] === name); const commit = (c, subject) => ({ sha: sha(c), subject, message: subject, author: OWNER, parents: [sha("0")] }); @@ -128,7 +137,7 @@ test("hotfixMerge_publishesPatch_andOpensOnlyTheBackportToBeta", async () => { "dev...beta": [commit("r", "chore(release): v1.0.1-BETA.1")], }, }); - const result = await run(gh, hotfix()); + const result = await withBeta(gh, hotfix()); assert.equal(result.tag, "v1.0.1"); assert.equal(result.backports.length, 1); assert.deepEqual(named(gh, "createTag"), [ @@ -143,14 +152,14 @@ test("hotfixBackports_skipOpenOrEmptyPairs", async () => { pulls: [{ number: 40, base: "beta", head: "main", labels: ["backport"] }], compare: { "beta...main": [commit("h", "fix: login")], "dev...beta": [commit("r", "fix: r")] }, }); - const result = await run(gh, hotfix()); + const result = await withBeta(gh, hotfix()); assert.deepEqual(result.backports, []); assert.deepEqual(named(gh, "createPull"), [], "the open main -> beta is reused; beta -> dev waits for it"); }); test("hotfixWithNothingToBringToBeta_opensTheBackportToDevRightAway", async () => { const gh = fakeGitHub({ compare: { "dev...beta": [commit("r", "fix: r")] } }); - const result = await run(gh, hotfix()); + const result = await withBeta(gh, hotfix()); assert.equal(result.backports.length, 1); assert.deepEqual(opened(gh), [backport("dev", "beta", "v1.0.1")]); }); @@ -173,6 +182,44 @@ test("stableMerge_reusesAnOpenBackportToDev", async () => { assert.deepEqual(named(gh, "createPull"), []); }); +test("backport_isOpenedWithAutoMergePinnedToItsHead", async () => { + const gh = fakeGitHub({ compare: { "dev...beta": [commit("r", "chore(release): v1.0.0")] } }); + const [number] = (await run(gh, mergedPull({ base: "main", head: "beta", title: "chore(release): v1.0.0" }))).backports; + assert.deepEqual(named(gh, "enableAutoMerge"), [ + ["enableAutoMerge", number, { expectedHeadOid: sha("h"), commitHeadline: "chore(release): backport v1.0.0 to dev" }], + ]); + assert.match(named(gh, "createPull")[0][1].body, /auto-merge est activĂ©/); + assert.deepEqual(named(gh, "updatePull"), []); +}); + +test("backportWhoseAutoMergeFails_isLeftToTheOwner", async () => { + const gh = fakeGitHub({ compare: { "dev...main": [commit("r", "fix: r")] }, autoMergeFails: true }); + const result = await run(gh, mergedPull({ base: "main", head: "dev", title: "chore(release): v1.2.0" }), false); + assert.equal(result.backports.length, 1, "the backport stays open"); + const [[, number, { body }]] = named(gh, "updatePull"); + assert.equal(number, result.backports[0]); + assert.match(body, /Fusionne cette PR par commit de merge/); +}); + +// Beta skipped (lib/flow.mjs): dev -> main is a stable, brought back to dev straight from main. +test("devToMainMerge_tagsTheStable_andOpensTheBackportOfMainToDev", async () => { + const gh = fakeGitHub({ compare: { "dev...main": [commit("r", "chore(release): v1.2.0")] } }); + const result = await run(gh, mergedPull({ base: "main", head: "dev", title: "chore(release): v1.2.0" }), false); + assert.equal(result.tag, "v1.2.0"); + assert.deepEqual(named(gh, "createTag"), [ + ["createTag", { tag: "v1.2.0", sha: sha("m"), message: "chore(release): v1.2.0" }], + ]); + assert.deepEqual(opened(gh), [backport("dev", "main", "v1.2.0")]); + assert.equal(named(gh, "enableAutoMerge").length, 1); +}); + +test("hotfixWithoutBeta_opensTheBackportOfMainToDev", async () => { + const gh = fakeGitHub({ compare: { "beta...main": [commit("h", "fix: login")], "dev...main": [commit("h", "fix: login")] } }); + const result = await run(gh, hotfix(), false); + assert.equal(result.backports.length, 1); + assert.deepEqual(opened(gh), [backport("dev", "main", "v1.0.1")]); +}); + test("betaMerge_opensNoBackport", async () => { const gh = fakeGitHub({ compare: { "dev...beta": [commit("r", "fix: r")] } }); await run(gh, mergedPull()); diff --git a/.github/release/test/ready.test.mjs b/.github/release/test/ready.test.mjs index 9b300b6ab..c7d3fe3c6 100644 --- a/.github/release/test/ready.test.mjs +++ b/.github/release/test/ready.test.mjs @@ -251,6 +251,20 @@ test("stablePromotion_bumpsToTheStableVersion", async () => { assert.match(file.text, /version = "1\.0\.0"\n/); }); +test("devToMainPromotion_bumpsDevToTheStableVersion", async () => { + const gh = fakeGitHub({ + base: "main", + head: "dev", + title: "chore(release): v1.2.0", + tags: ["v1.1.0-BETA.1", "v1.1.0"], + ahead: [commit("a", "feat(web): one")], + }); + assert.equal(await run(gh), "auto-merge"); + const [[, file]] = named(gh, "commitFile"); + assert.equal(file.branch, "dev"); + assert.match(file.text, /version = "1\.2\.0"\n/); +}); + test("hotfixWithoutVersionTitle_getsTheNextPatch", async () => { const tags = ["v0.0.0", "v1.0.0"]; const gh = fakeGitHub({ base: "main", head: "hotfix/login", title: "fix: login", marker: null, tags }); @@ -279,7 +293,7 @@ test("otherSender_isIgnored", async () => { }); test("unrelatedPr_isIgnored", async () => { - for (const [base, head] of [["dev", "feat/x"], ["main", "dev"], ["beta", "hotfix/x"], ["main", "feat/x"]]) { + for (const [base, head] of [["dev", "feat/x"], ["dev", "main"], ["beta", "hotfix/x"], ["main", "feat/x"]]) { const gh = fakeGitHub({ base, head }); assert.equal(await run(gh), "ignored"); assert.deepEqual(gh.calls, [["getPull", 7]]); diff --git a/.github/release/test/schedule.test.mjs b/.github/release/test/schedule.test.mjs index 63fb24b2a..bff2fe713 100644 --- a/.github/release/test/schedule.test.mjs +++ b/.github/release/test/schedule.test.mjs @@ -171,8 +171,16 @@ test("the App's own draft changes never cancel", async () => { test("a release is due only once its date has come", async () => { const gh = fakeGitHub({ comments: [said(scheduleMarker(AT, sha("a")))] }); - assert.deepEqual(await dueSources({ gh, appSlug: APP, now: new Date(AT - 1) }), []); - assert.deepEqual(await dueSources({ gh, appSlug: APP, now: AT }), ["dev"]); + assert.deepEqual(await dueSources({ gh, appSlug: APP, now: new Date(AT - 1), beta: true }), []); + assert.deepEqual(await dueSources({ gh, appSlug: APP, now: AT, beta: true }), ["dev"]); +}); + +test("beta skipped, the due release is the dev -> main promotion", async () => { + const marker = [said(scheduleMarker(AT, sha("a")))]; + const direct = fakeGitHub({ pull: { base: "main", title: "chore(release): v1.2.0" }, comments: marker }); + assert.deepEqual(await dueSources({ gh: direct, appSlug: APP, now: AT }), ["dev"]); + const viaBeta = fakeGitHub({ comments: marker }); + assert.deepEqual(await dueSources({ gh: viaBeta, appSlug: APP, now: AT }), [], "dev -> beta is not scheduled"); }); test("before the date, the run does nothing", async () => { diff --git a/.github/release/test/version.test.mjs b/.github/release/test/version.test.mjs index 1c63a2b9e..be7618548 100644 --- a/.github/release/test/version.test.mjs +++ b/.github/release/test/version.test.mjs @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { parseVersion, bumpKind, nextBeta, nextStable } from "../lib/version.mjs"; +import { parseVersion, bumpKind, nextBeta, nextDirectStable, nextStable } from "../lib/version.mjs"; test("parseVersion reads stable and beta tags", () => { assert.deepEqual(parseVersion("v1.0.0-BETA.3"), { major: 1, minor: 0, patch: 0, beta: 3 }); @@ -56,6 +56,18 @@ test("nextBeta with no new commits continues the existing beta line, else throws assert.throws(() => nextBeta("v1.0.0", [], [])); }); +test("nextDirectStable bumps the last stable from the commits, or returns null", () => { + assert.equal(nextDirectStable("v1.1.0", [], ["fix: a"]), "v1.1.1"); + assert.equal(nextDirectStable("v1.1.0", [], ["fix: a", "feat: b"]), "v1.2.0"); + assert.equal(nextDirectStable("v1.1.0", [], ["feat!: c"]), "v2.0.0"); + assert.equal(nextDirectStable("v1.1.0", [], ["docs no prefix"]), null); +}); + +test("nextDirectStable never goes below a pending beta's target", () => { + assert.equal(nextDirectStable("v1.1.0", ["v1.2.0-BETA.1"], ["fix: a"]), "v1.2.0"); + assert.equal(nextDirectStable("v1.1.0", ["v1.0.0-BETA.9"], ["fix: a"]), "v1.1.1"); +}); + test("nextStable strips the beta suffix", () => { assert.equal(nextStable("v1.1.0-BETA.3"), "v1.1.0"); assert.throws(() => nextStable("v1.1.0")); diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3f81a99c..b4b29799c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,17 +1,17 @@ name: Build +# Pull requests (into any branch, stacked ones included) rather than `push`: a push to a branch with +# an open PR ran the same job twice. dev's head is checked by its dev -> main promotion PR, main's by +# the backport main -> dev, and a version tag push runs release.yml. on: - # Branches only: a version tag push runs release.yml, not the checks again - push: - branches: ['**'] # Lets renovate-merge.yml run it on dev after a checkbox merge, which GITHUB_TOKEN pushes cannot trigger workflow_dispatch: pull_request: - branches: - - master # until it is renamed dev - - dev - - beta - - main + +# A new push to a pull request cancels its previous run; a dispatched run is never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: build: diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 70e6fcdcb..36e8ce4f4 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -1,20 +1,23 @@ name: Commit lint +# Pull requests only: the commits pushed to dev were linted in their pull request. on: pull_request: - push: - branches: - - master # until it is renamed dev - - dev + +# A new push to a pull request cancels its previous run; a dispatched run is never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: commitlint: # Promotion and backport PRs carry commits already on a long-lived branch (old subjects, merge # commits): their job is skipped, and a skipped job still satisfies the required check. if: >- - github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository || !( + (github.event.pull_request.head.ref == 'dev' && github.event.pull_request.base.ref == 'main') || + (github.event.pull_request.head.ref == 'main' && github.event.pull_request.base.ref == 'dev') || (github.event.pull_request.head.ref == 'dev' && github.event.pull_request.base.ref == 'beta') || (github.event.pull_request.head.ref == 'beta' && github.event.pull_request.base.ref == 'main') || (github.event.pull_request.head.ref == 'main' && github.event.pull_request.base.ref == 'beta') || diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 455d01db6..d35ea0f5b 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -1,8 +1,10 @@ name: Promote on: + # Beta is skipped (.github/release/lib/flow.mjs): dev is promoted straight to main. Put `beta` back + # here when it is brought back. push: - branches: [dev, beta] + branches: [dev] # Lets renovate-merge.yml run it on dev after a checkbox merge, which GITHUB_TOKEN pushes cannot trigger workflow_dispatch: @@ -33,5 +35,5 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} APP_SLUG: ${{ steps.app-token.outputs.app-slug }} SOURCE: ${{ github.ref_name }} - TARGET: ${{ github.ref_name == 'dev' && 'beta' || 'main' }} - run: node .github/release/promote.mjs --source "$SOURCE" --target "$TARGET" + # The target is the source's next branch in lib/flow.mjs + run: node .github/release/promote.mjs --source "$SOURCE" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 50e743892..091bd05a8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,9 @@ name: Publish # Tags the merge commit only on an actual merge: a closed but unmerged promotion PR releases nothing. # The tag is pushed with the App token, whose pushes do trigger workflows: that tag push runs -# release.yml, the single publication path. A merged hotfix backport main -> beta opens beta -> dev. +# release.yml, the single publication path. A stable or a hotfix opens its backport to dev: +# main -> dev while beta is skipped (.github/release/lib/flow.mjs); with beta, a merged hotfix +# backport main -> beta opens beta -> dev. on: pull_request: types: [closed] @@ -20,6 +22,7 @@ jobs: github.event.pull_request.merged == true && github.event.pull_request.head.repo.full_name == github.repository && ( + (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'beta' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'beta') || (github.event.pull_request.base.ref == 'main' && startsWith(github.event.pull_request.head.ref, 'hotfix/')) || diff --git a/.github/workflows/release-ready.yml b/.github/workflows/release-ready.yml index ea591e484..5723e1d43 100644 --- a/.github/workflows/release-ready.yml +++ b/.github/workflows/release-ready.yml @@ -2,7 +2,8 @@ name: Release ready # `pull_request` rather than `pull_request_target`: `main` holds no workflow until the first stable # release, so a target-based trigger would never fire for beta -> main. The job only runs for the -# owner on this repository's release branches, and it runs dev's scripts. +# owner on this repository's release branches, and it runs dev's scripts. Beta is skipped for now +# (.github/release/lib/flow.mjs): the promotion is dev -> main; the beta pairs stay accepted. on: pull_request: types: [ready_for_review] @@ -21,6 +22,7 @@ jobs: github.event.sender.login == github.repository_owner && github.event.pull_request.head.repo.full_name == github.repository && ( + (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'beta' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'beta') || (github.event.pull_request.base.ref == 'main' && startsWith(github.event.pull_request.head.ref, 'hotfix/')) diff --git a/.github/workflows/release-schedule.yml b/.github/workflows/release-schedule.yml index c9fdc89a9..b610d7ee4 100644 --- a/.github/workflows/release-schedule.yml +++ b/.github/workflows/release-schedule.yml @@ -18,6 +18,7 @@ jobs: github.event.sender.login == github.repository_owner && github.event.pull_request.head.repo.full_name == github.repository && ( + (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'beta' && github.event.pull_request.head.ref == 'dev') || (github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.ref == 'beta') ) @@ -98,5 +99,5 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} APP_SLUG: ${{ steps.app-token.outputs.app-slug }} SOURCE: ${{ matrix.source }} - TARGET: ${{ matrix.source == 'dev' && 'beta' || 'main' }} - run: node .github/release/schedule.mjs --run --source "$SOURCE" --target "$TARGET" + # The target is the source's next branch in lib/flow.mjs (dev -> main while beta is skipped) + run: node .github/release/schedule.mjs --run --source "$SOURCE" diff --git a/.github/workflows/release-scripts.yml b/.github/workflows/release-scripts.yml index d65c915ff..f8bb0242f 100644 --- a/.github/workflows/release-scripts.yml +++ b/.github/workflows/release-scripts.yml @@ -1,10 +1,13 @@ name: Release scripts +# Pull requests only, no `push`: a branch push with an open PR would run the same job twice. on: pull_request: - # Branches only: a version tag push runs release.yml, not the checks again - push: - branches: ['**'] + +# A new push to a pull request cancels its previous run; a dispatched run is never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: # Not `test`: ready.mjs requires the check named `test` (Tests workflow) before merging a release. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ad5af7ef..92ffc3aba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,17 +1,17 @@ name: Tests +# Pull requests (into any branch, stacked ones included) rather than `push`: a push to a branch with +# an open PR ran the same job twice. dev's head is checked by its dev -> main promotion PR, main's by +# the backport main -> dev, and a version tag push runs release.yml. on: - # Branches only: a version tag push runs release.yml, not the checks again - push: - branches: ['**'] # Lets renovate-merge.yml run it on dev after a checkbox merge, which GITHUB_TOKEN pushes cannot trigger workflow_dispatch: pull_request: - branches: - - master # until it is renamed dev - - dev - - beta - - main + +# A new push to a pull request cancels its previous run; a dispatched run is never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: test: diff --git a/CHANGELOG.md b/CHANGELOG.md index ec2075bfc..dc84cfcea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,114 @@ Toutes les versions publiĂ©es de Catapult, la plus rĂ©cente en premier. Ce fichier est rĂ©gĂ©nĂ©rĂ© par le bot Ă  chaque release Ă  partir des notes des [GitHub Releases](https://github.com/enimaloc/catapult/releases) : pour corriger une entrĂ©e, modifie les notes de la release (ou le changelog de la PR de release avant sa publication), pas ce fichier. +## [v1.1.0](https://github.com/enimaloc/catapult/releases/tag/v1.1.0) - 2026-11-10 + +### Features +- **api:** send the stream title along with the category (7fee7ca) +- **api:** let channels edit their stream title settings (0cdb5b4) +- **web:** let streamers set their stream title per game or for every game (df86277) +- **api:** write stream title variables as {{GAME}} and add {{TITLE}} (8bbfcd0) +- **api:** add {{ORIGINAL_TITLE}} and accept only exactly written title variables (653a1aa) +- **api:** keep title variables not written exactly as typed instead of refusing them (1a5fb6c) +- give custom bindings a stream title of their own (c9f1fe7) +- **web:** color the variables Catapult replaces in the stream title fields (62eb121) +- add {{CATEGORY}} and titles for the "no game" and incomplete game categories (2cfb76a) +- merge stream title into dev (7ed07b3) +- **api:** gate drops, title conditions and title preview behind their experiments (7d16eaf) +- **api:** render conditions in stream titles (5acd43c) +- **api:** keep the running Twitch drops campaigns from ttvdrops (8621d5a) +- **api:** evaluate drops and binding conditions when sending the stream title (1f3dd9f) +- show running Twitch drops on the channel bindings (22ea797) +- **api:** expose running drops at /api/me/drops (b49d8a4) +- **web:** relay /api/me/drops like the other public routes (1958811) +- preview the stream title as it would be sent (48e49d4) +- **api:** let an experiment spec target the members another experiment gave a variant (1b5bbfa) +- **api:** offer the title conditions and preview only to the channels stream.title enabled (e1aacd8) +- merge stream title conditions and Twitch drops into dev (5464a77) + +### Fixes +- **web:** give the custom binding title field the form's full width and its own height (88c4a0b) +- **ci:** refresh the beta -> main promotion in the promote-beta concurrency group (44d3ae8) +- **ci:** merge the promote-beta concurrency fix into dev (60d1654) +- **api:** keep only http links, ttvdrops' own images and plain ids from the drops campaigns (8074cf5) +- **api:** refuse a title preview longer than a title before rendering it (58e56ef) +- **api:** render brace-adjacent title variables as before and keep only a failed ternary raw (ef47535) +- **api:** preview an empty own title as the channel-wide one that would be sent (8d1a5f7) +- **api:** keep the drops when one campaign's channels can't be read and cap the listing (4f74b60) +- **api:** show how to send the personal token in the API docs (1511746) +- **api:** merge the API docs token guide into dev (c9989db) +- **api:** enrol the channels in stream.title.conditions from their settings (e08d937) +- **web:** highlight the title ternaries and the drop variables (76fdaed) + +### Performance +- **web:** ask a binding row's title preview only once its field is used (22d1e45) + +### Documentation +- **api:** explain the stream title conditions and drops variables in the card's help (8bc67fe) + +### Other +- **api:** roll the stream title out to one channel in ten (a3bc4b6) +- **api:** mock the stream title service in the controller and Twitchat revert slices (6cbe9c7) +- **api:** budget the settings route's stream title gate lookup (6acae3c) +- **common:** cover the stream title DTO constructors and older settings payloads (748c327) +- **release:** merge the v1.0.1 backport into dev (5bbb792) +- **api:** declare /api/me/drops among the public routes (fb8c440) +- merge dev into feat/stream-title-conditions (5ce9b3f) + +## [v1.1.0-BETA.1](https://github.com/enimaloc/catapult/releases/tag/v1.1.0-BETA.1) - 2026-11-10 + +### Features +- **api:** send the stream title along with the category (7fee7ca) +- **api:** let channels edit their stream title settings (0cdb5b4) +- **web:** let streamers set their stream title per game or for every game (df86277) +- **api:** write stream title variables as {{GAME}} and add {{TITLE}} (8bbfcd0) +- **api:** add {{ORIGINAL_TITLE}} and accept only exactly written title variables (653a1aa) +- **api:** keep title variables not written exactly as typed instead of refusing them (1a5fb6c) +- give custom bindings a stream title of their own (c9f1fe7) +- **web:** color the variables Catapult replaces in the stream title fields (62eb121) +- add {{CATEGORY}} and titles for the "no game" and incomplete game categories (2cfb76a) +- merge stream title into dev (7ed07b3) +- **api:** gate drops, title conditions and title preview behind their experiments (7d16eaf) +- **api:** render conditions in stream titles (5acd43c) +- **api:** keep the running Twitch drops campaigns from ttvdrops (8621d5a) +- **api:** evaluate drops and binding conditions when sending the stream title (1f3dd9f) +- show running Twitch drops on the channel bindings (22ea797) +- **api:** expose running drops at /api/me/drops (b49d8a4) +- **web:** relay /api/me/drops like the other public routes (1958811) +- preview the stream title as it would be sent (48e49d4) +- **api:** let an experiment spec target the members another experiment gave a variant (1b5bbfa) +- **api:** offer the title conditions and preview only to the channels stream.title enabled (e1aacd8) +- merge stream title conditions and Twitch drops into dev (5464a77) + +### Fixes +- **web:** give the custom binding title field the form's full width and its own height (88c4a0b) +- **ci:** refresh the beta -> main promotion in the promote-beta concurrency group (44d3ae8) +- **ci:** merge the promote-beta concurrency fix into dev (60d1654) +- **api:** show how to send the personal token in the API docs (1511746) +- **api:** merge the API docs token guide into dev (c9989db) +- **api:** keep only http links, ttvdrops' own images and plain ids from the drops campaigns (8074cf5) +- **api:** refuse a title preview longer than a title before rendering it (58e56ef) +- **api:** render brace-adjacent title variables as before and keep only a failed ternary raw (ef47535) +- **api:** preview an empty own title as the channel-wide one that would be sent (8d1a5f7) +- **api:** keep the drops when one campaign's channels can't be read and cap the listing (4f74b60) +- **api:** enrol the channels in stream.title.conditions from their settings (e08d937) +- **web:** highlight the title ternaries and the drop variables (76fdaed) + +### Performance +- **web:** ask a binding row's title preview only once its field is used (22d1e45) + +### Documentation +- **api:** explain the stream title conditions and drops variables in the card's help (8bc67fe) + +### Other +- **api:** roll the stream title out to one channel in ten (a3bc4b6) +- **api:** mock the stream title service in the controller and Twitchat revert slices (6cbe9c7) +- **api:** budget the settings route's stream title gate lookup (6acae3c) +- **common:** cover the stream title DTO constructors and older settings payloads (748c327) +- **release:** merge the v1.0.1 backport into dev (5bbb792) +- **api:** declare /api/me/drops among the public routes (fb8c440) +- merge dev into feat/stream-title-conditions (5ce9b3f) + ## [v1.0.1](https://github.com/enimaloc/catapult/releases/tag/v1.0.1) - 2026-11-10 ### Fixes diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/ApiHelpController.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/ApiHelpController.java index b254d5986..f2aa0856e 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/ApiHelpController.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/ApiHelpController.java @@ -2,6 +2,7 @@ import fr.enimaloc.catapult.common.dto.HelpContent; import fr.enimaloc.catapult.service.minecraft.MinecraftGateService; +import fr.enimaloc.catapult.service.title.TitleHelpSections; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.MessageSource; @@ -22,6 +23,7 @@ public class ApiHelpController { private final MessageSource messageSource; private final MinecraftGateService gateService; + private final TitleHelpSections titleHelpSections; @Value("${app.account.deletion-delay-days:7}") private int accountDeletionDelayDays; @@ -49,9 +51,27 @@ public HelpContent help(@PathVariable String cardId, Locale locale, body += messageSource.getMessage("help.connections.body.minecraft", args, "", locale); } } + if ("title-settings".equals(cardId)) { + TitleHelpSections.Offered offered = titleSections(jwt); + if (offered.conditions()) { + body += messageSource.getMessage("help.title-settings.body.conditions", args, "", locale); + } + if (offered.drops()) { + body += messageSource.getMessage("help.title-settings.body.drops", args, "", locale); + } + } return new HelpContent(title, body); } + private TitleHelpSections.Offered titleSections(Jwt jwt) { + if (jwt == null) return TitleHelpSections.Offered.NONE; + try { + return titleHelpSections.forAccount(UUID.fromString(jwt.getSubject())); + } catch (IllegalArgumentException e) { + return TitleHelpSections.Offered.NONE; + } + } + private boolean isMinecraftVisible(Jwt jwt) { if (jwt == null) return false; try { diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesController.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesController.java new file mode 100644 index 000000000..d8076d0f5 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesController.java @@ -0,0 +1,72 @@ +package fr.enimaloc.catapult.api.admin; + +import fr.enimaloc.catapult.api.ApiPaths; +import fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.service.dtdd.DtddCacheAdminService; +import fr.enimaloc.catapult.service.dtdd.DtddCacheKind; +import fr.enimaloc.catapult.service.dtdd.DtddCacheRefetchJob; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +/** + * The DTDD caches (a title's topics, the titles, the searches): their summary and the keys' quotas, each one paged, + * everything cached about an entry, an entry fetched again, and the bulk refetch of them all ({@code /refetch-all}: started, followed, cancelled). + */ +@RestController +@RequestMapping(ApiPaths.ADMIN + "/dtdd-caches") +@RequiredArgsConstructor +public class ApiAdminDtddCachesController { + + private final DtddCacheAdminService caches; + private final DtddCacheRefetchJob job; + + @GetMapping + public DtddCachesSummaryDto summary() { + return caches.summary(); + } + + @GetMapping("/refetch-all") + public DtddRefetchJobDto job() { + return job.status(); + } + + @PostMapping("/refetch-all") + @ResponseStatus(HttpStatus.ACCEPTED) + public DtddRefetchJobDto startJob() { + return job.start(); + } + + @DeleteMapping("/refetch-all") + public DtddRefetchJobDto cancelJob() { + return job.cancel(); + } + + @GetMapping("/{kind}") + public Page list(@PathVariable String kind, @RequestParam(required = false) String q, + @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "20") int size) { + return caches.list(DtddCacheKind.of(kind), q, page, size); + } + + /** Everything cached about an entry; {@code key}: the title's ID, or the search's query. */ + @GetMapping("/{kind}/entry") + public DtddCacheEntryDetailDto entry(@PathVariable String kind, @RequestParam String key) { + return caches.detail(DtddCacheKind.of(kind), key); + } + + /** {@code key}: the title's ID, or the search's query. */ + @PostMapping("/{kind}/refetch") + public Object refetch(@PathVariable String kind, @RequestParam String key) { + return caches.refetch(DtddCacheKind.of(kind), key); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsController.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsController.java new file mode 100644 index 000000000..0b126f886 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsController.java @@ -0,0 +1,38 @@ +package fr.enimaloc.catapult.api.admin; + +import fr.enimaloc.catapult.api.ApiPaths; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimiter; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * The rate limits Catapult learned from the platforms' 429s (hosts and their routes), an admin's correction of one + * and its reset. {@code id}: the scope's ID ({@code HOST store.steampowered.com}, + * {@code GET store.steampowered.com/app/{id}}). + */ +@RestController +@RequestMapping(ApiPaths.ADMIN + "/rate-limits") +@RequiredArgsConstructor +public class ApiAdminRateLimitsController { + + private final LearnedRateLimiter limiter; + + @GetMapping + public List list() { + return limiter.list(); + } + + @PutMapping("/entry") + public LearnedRateLimitDto correct(@RequestParam String id, @RequestBody LearnedRateLimitCorrectionDto body) { + return limiter.correct(id, body.limitCalls(), body.limitWindowSeconds(), body.resumeSeconds(), body.scope()); + } + + @DeleteMapping("/entry") + public LearnedRateLimitDto reset(@RequestParam String id) { + return limiter.reset(id); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminTwController.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminTwController.java index 54e8674d1..7b96fb45f 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminTwController.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/admin/ApiAdminTwController.java @@ -3,11 +3,15 @@ import fr.enimaloc.catapult.api.ApiPaths; import fr.enimaloc.catapult.common.dto.admin.AdminTwCreateBody; import fr.enimaloc.catapult.common.dto.admin.DtddTopicsBody; +import fr.enimaloc.catapult.common.dto.admin.TwDtddTopicDto; +import fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping; +import fr.enimaloc.catapult.service.tw.TwDtddTopicLevels; import fr.enimaloc.catapult.common.dto.admin.IgdbDescriptorsBody; import fr.enimaloc.catapult.common.dto.admin.SteamIdsBody; import fr.enimaloc.catapult.common.dto.admin.SteamKeywordBody; import fr.enimaloc.catapult.common.dto.admin.SteamKeywordsBody; import fr.enimaloc.catapult.common.dto.admin.SteamSignalTestBody; +import fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto; import fr.enimaloc.catapult.common.dto.admin.UpdateBody; import fr.enimaloc.catapult.domain.tw.TwDefinition; import fr.enimaloc.catapult.service.notification.AdminEventPublisher; @@ -31,6 +35,7 @@ import org.springframework.web.server.ResponseStatusException; import java.util.List; +import java.util.Locale; import java.util.Set; @RestController @@ -47,6 +52,8 @@ public class ApiAdminTwController { private final AdminEventPublisher events; + private final TwDtddTopicLevels topicLevels; + @GetMapping public List list() { return service.list(); @@ -82,9 +89,20 @@ public ResponseEntity delete(@PathVariable String id) { return ResponseEntity.noContent().build(); } + /** The TW's DTDD topics with their least severe level, and the levels the cached titles give each. */ + @GetMapping("/{id}/dtdd-topics") + public List listDtdd(@PathVariable String id) { + if (service.get(id).isEmpty()) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Unknown TW " + id); + List topics = service.listDtddTopics(id); + Map> levels = topicLevels.of(topics.stream().map(TwDtddTopicMapping::getDtddTopicName).toList()); + return topics.stream().map(m -> new TwDtddTopicDto(m.getDtddTopicName(), m.getMinSeverity(), + levels.getOrDefault(m.getDtddTopicName().toLowerCase(Locale.ROOT), List.of()))).toList(); + } + @PutMapping("/{id}/dtdd-topics") public ResponseEntity setDtdd(@PathVariable String id, @RequestBody DtddTopicsBody b) { - service.replaceDtddTopics(id, b.topics() == null ? Set.of() : b.topics()); + service.replaceDtddTopics(id, b.topics() == null ? Set.of() : b.topics(), + b.minSeverities() == null ? Map.of() : b.minSeverities()); return ResponseEntity.noContent().build(); } @@ -129,32 +147,39 @@ public AdminTwService.SteamSignalTestResult testSteamKw(@PathVariable String id, } @PostMapping("/rebuild") - public ResponseEntity rebuildNonOverriddenMappings() { - return backfillService - .map(backfill -> { - backfill.rebuildAllNonOverridden(); - return ResponseEntity.accepted().build(); - }) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "The TW backfill is not available")); + public ResponseEntity rebuildNonOverriddenMappings() { + return start(TwBackfillService.Mode.NON_OVERRIDDEN); } @PostMapping("/rebuild/force") - public ResponseEntity rebuildMappings() { - return backfillService - .map(backfill -> { - backfill.rebuildAll(); - return ResponseEntity.accepted().build(); - }) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "The TW backfill is not available")); + public ResponseEntity rebuildMappings() { + return start(TwBackfillService.Mode.UNTOUCHED); } @PostMapping("/rebuild/force-pinned") - public ResponseEntity rebuildMappingsIncludingPinned() { - return backfillService - .map(backfill -> { - backfill.forceRebuildAllIncludingPinned(); - return ResponseEntity.accepted().build(); - }) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "The TW backfill is not available")); + public ResponseEntity rebuildMappingsIncludingPinned() { + return start(TwBackfillService.Mode.FORCE_PINNED); + } + + /** The TWs' recalculation running or last run (IDLE without the backfill: none ever). */ + @GetMapping("/rebuild/status") + public TwRebuildStatusDto rebuildStatus() { + return backfillService.map(TwBackfillService::status).orElse(TwRebuildStatusDto.IDLE); + } + + /** Asks the running recalculation to stop after its current binding. */ + @PostMapping("/rebuild/cancel") + public TwRebuildStatusDto cancelRebuild() { + return backfill().cancel(); + } + + /** 202 with the recalculation started; 409 while one runs already, 503 without the backfill. */ + private ResponseEntity start(TwBackfillService.Mode mode) { + return ResponseEntity.accepted().body(backfill().start(mode)); + } + + private TwBackfillService backfill() { + return backfillService.orElseThrow( + () -> new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "The TW backfill is not available")); } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ApiChannelTwController.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ApiChannelTwController.java index 4d86e9dd3..b395517a2 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ApiChannelTwController.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ApiChannelTwController.java @@ -1,5 +1,10 @@ package fr.enimaloc.catapult.api.channel; +import fr.enimaloc.catapult.common.dto.channel.TwDetailsDto; +import fr.enimaloc.catapult.service.binding.TwDetailsService; +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; + import fr.enimaloc.catapult.api.ApiPaths; import fr.enimaloc.catapult.api.ApiUserResolver; import fr.enimaloc.catapult.common.dto.channel.SaveBody; @@ -43,6 +48,7 @@ public class ApiChannelTwController { private final ApiUserResolver userResolver; private final ChannelDashboardActions dashboardActions; private final BindingService bindingService; + private final TwDetailsService twDetailsService; @PreAuthorize("@authz.isSelf(#userId)") @PutMapping("/tws") @@ -87,6 +93,17 @@ public ResponseEntity toggleTwEnabled(@PathVariable UUID userId, return ResponseEntity.noContent().build(); } + /** Why the binding has its TWs, as the detection tells it now; 404 when it isn't the caller's. */ + @PreAuthorize("@authz.isSelf(#userId)") + @GetMapping("/tws/details") + public TwDetailsDto twDetails(@PathVariable UUID userId, + @PathVariable UUID bindingId, + @AuthenticationPrincipal Jwt jwt) { + UserAccount user = userResolver.viewerByTwitchId(jwt); + return twDetailsService.details(user, bindingId) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Unknown binding")); + } + @PreAuthorize("@authz.isSelf(#userId)") @GetMapping("/tws/suggestions") public Map> suggestTws(@PathVariable UUID userId, diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssembler.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssembler.java index 4d5db3ff7..dc026dd32 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssembler.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssembler.java @@ -265,10 +265,10 @@ public UserSettingsDto settings(UserAccount channel) { return defaults; }); boolean titleAvailable = streamTitle.isAvailableFor(channel); - if (titleAvailable) { - // The title's fields are where the conditions are written: offered them, the channel may be enrolled - streamTitle.conditionsAvailableFor(channel); - } + // The title's fields are where the conditions are written: offered them, the channel may be enrolled + boolean titleConditions = titleAvailable && streamTitle.conditionsAvailableFor(channel); + // What the title reads of the drops (StreamTitleService#context): the conditions and the drops both on + boolean titleDrops = titleConditions && dropsGate.isAvailableFor(channel); return new UserSettingsDto( settings.isCclFeatureEnabled(), settings.getBlockedCcls(), @@ -295,7 +295,9 @@ public UserSettingsDto settings(UserAccount channel) { titleAvailable, settings.getNoGameTitle(), settings.getIncompleteFallbackTitle(), - streamTitle.previewAvailableFor(channel)); + streamTitle.previewAvailableFor(channel), + titleConditions, + titleDrops); } // --- DTDD mapping ---------------------------------------------------------------------------- diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponse.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponse.java index 0b0911fa8..206cae84a 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponse.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponse.java @@ -2,10 +2,43 @@ import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import java.util.Arrays; import java.util.List; public record DtddDetailResponse(String url, List yesTopics, List noTopics, List mostlyTopics, String overview, String[] genres, long releaseYear, String itemTypeName, long tmdbId, String imdbId, String posterImage, String backgroundImage, - DtddApiClient.DtddTopicItemStat[] topicItemStats) {} + DtddTopicItemStat[] topicItemStats) { + + /** + * A topic's votes as the public API answers them; {@code severity}: how severe it is on the title, null when the + * topic has no levels. Named as DTDD's, so the OpenAPI schema keeps its name. + */ + public record DtddTopicItemStat(long topicItemId, long yesSum, long noSum, long numComments, long topicId, + String topicName, long itemId, DtddSeverity severity) { + + static DtddTopicItemStat[] of(DtddApiClient.DtddTopicItemStat[] stats) { + if (stats == null) return null; + return Arrays.stream(stats) + .map(s -> new DtddTopicItemStat(s.topicItemId(), s.yesSum(), s.noSum(), s.numComments(), + s.topicId(), s.topicName(), s.itemId(), DtddSeverity.of(s.severity()))) + .toArray(DtddTopicItemStat[]::new); + } + } + + /** + * How severe a topic is on the title, as DTDD's members voted it: {@code levels} its labels, mildest first; + * {@code counts} and {@code percents} the votes for each, by position; {@code total} the level votes; + * {@code verified} the level DTDD's reviewers gave (1-based), null when they gave none. + */ + public record DtddSeverity(List levels, List counts, List percents, int total, + Integer verified) { + + static DtddSeverity of(DtddApiClient.DtddSeverity severity) { + if (severity == null) return null; + return new DtddSeverity(severity.levels(), severity.counts(), severity.percents(), severity.total(), + severity.verified()); + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/GameInfoAssembler.java b/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/GameInfoAssembler.java index d176e2526..90c20e845 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/GameInfoAssembler.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/api/userapi/GameInfoAssembler.java @@ -170,7 +170,8 @@ public Optional dtdd(long dtddId) { item.map(DtddApiClient.DtddItem::imdbId).orElse(null), item.map(DtddApiClient.DtddItem::posterImage).orElse(null), item.map(DtddApiClient.DtddItem::backgroundImage).orElse(null), - item.map(DtddApiClient.DtddItem::topicItemStats).orElse(null))); + item.map(DtddApiClient.DtddItem::topicItemStats).map(DtddDetailResponse.DtddTopicItemStat::of) + .orElse(null))); } /** The binding's tw/ccl configuration, empty when there is no binding with that id. */ diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/config/WebClientConfig.java b/catapult-api/src/main/java/fr/enimaloc/catapult/config/WebClientConfig.java index 59b03eb09..464ec5f50 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/config/WebClientConfig.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/config/WebClientConfig.java @@ -1,6 +1,7 @@ package fr.enimaloc.catapult.config; import com.fasterxml.jackson.databind.ObjectMapper; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimitInterceptor; import org.springframework.boot.restclient.RestClientCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,16 +25,18 @@ public class WebClientConfig { * Spring Boot hands out. Wraps the request factory in a * {@link BufferingClientHttpRequestFactory} so the interceptor can read * the response body without exhausting the underlying stream for the - * actual caller. + * actual caller. The learned rate limits apply to every call too. */ @Bean - public RestClientCustomizer traceLoggingCustomizer(HttpTraceLoggingInterceptor interceptor) { + public RestClientCustomizer traceLoggingCustomizer(HttpTraceLoggingInterceptor interceptor, + LearnedRateLimitInterceptor rateLimits) { JdkClientHttpRequestFactory requestFactory = new JdkClientHttpRequestFactory( HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build()); requestFactory.setReadTimeout(READ_TIMEOUT); return builder -> builder .requestFactory(new BufferingClientHttpRequestFactory(requestFactory)) - .requestInterceptor(interceptor); + .requestInterceptor(interceptor) + .requestInterceptor(rateLimits); } /** diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameBinding.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameBinding.java index 40fb87394..243a7400d 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameBinding.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameBinding.java @@ -61,6 +61,10 @@ public enum Status { @Column(nullable = false) private boolean ignored = false; + /** true: the streamer chose {@link #ignored} by hand, an admin mapping's ignore no longer changes it. */ + @Column(name = "ignored_override", nullable = false) + private boolean ignoredOverride = false; + @Column(name = "ccl_enabled", nullable = false) private boolean cclEnabled = true; diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameMapping.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameMapping.java index 60270195c..f5cbe4578 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameMapping.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/binding/GameMapping.java @@ -32,12 +32,17 @@ public class GameMapping { @Column(name = "source_id", nullable = false) private String sourceId; - @Column(name = "twitch_game_id", nullable = false) + /** null (with {@link #twitchGameName}) only when {@link #ignoredByDefault}: the bindings keep IGDB's category. */ + @Column(name = "twitch_game_id") private String twitchGameId; - @Column(name = "twitch_game_name", nullable = false) + @Column(name = "twitch_game_name") private String twitchGameName; + /** true: the game's bindings are ignored, but those whose streamer chose ({@link GameBinding#isIgnoredOverride}). */ + @Column(name = "ignored_by_default", nullable = false) + private boolean ignoredByDefault = false; + /** false: the bindings keep IGDB's CCLs; true: they get {@link #ccls}, even when empty. */ @Column(name = "ccls_overridden", nullable = false) private boolean cclsOverridden = false; @@ -73,4 +78,9 @@ public class GameMapping { /** The admin who last saved it. */ @Column(name = "updated_by") private UUID updatedBy; + + /** Whether it sets its bindings' category, else they keep IGDB's (an ignore-only mapping). */ + public boolean hasCategory() { + return twitchGameId != null; + } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicSeverity.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicSeverity.java new file mode 100644 index 000000000..47625abbf --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicSeverity.java @@ -0,0 +1,11 @@ +package fr.enimaloc.catapult.domain.dtdd; + +import java.util.List; + +/** + * A topic's severity on a title, as cached: {@code levels} its labels, mildest first; {@code counts} and + * {@code percents} the votes for each, by position; {@code total} the level votes; {@code verified} the reviewers' + * level (1-based), null when they gave none. + */ +public record DtddTopicSeverity(List levels, List counts, List percents, int total, + Integer verified) {} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicsCache.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicsCache.java index 5cc2f6557..c97aa2432 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicsCache.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/dtdd/DtddTopicsCache.java @@ -7,6 +7,7 @@ import java.time.Instant; import java.util.List; +import java.util.Map; @Entity @Table(name = "dtdd_topics_cache") @@ -34,4 +35,15 @@ public class DtddTopicsCache { @Column(name = "fetched_at", nullable = false) private Instant fetchedAt; + + /** Each topic's severity by its name; null when fetched before they were kept, a topic without levels absent. */ + @JdbcTypeCode(SqlTypes.JSON) + @Column(name = "severities", columnDefinition = "jsonb") + private Map severities; + + /** Topics without their severities. */ + public DtddTopicsCache(Long dtddId, List yesTopics, List noTopics, List mostlyTopics, + Instant fetchedAt) { + this(dtddId, yesTopics, noTopics, mostlyTopics, fetchedAt, null); + } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/ratelimit/LearnedRateLimit.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/ratelimit/LearnedRateLimit.java new file mode 100644 index 000000000..3b97c4542 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/ratelimit/LearnedRateLimit.java @@ -0,0 +1,79 @@ +package fr.enimaloc.catapult.domain.ratelimit; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.Instant; + +/** + * What was learned of one scope's rate limit: a host (its shared budget; {@code method} and {@code path} null) or a + * route ({@code scope} HOST: counted in its host's budget, ROUTE: its own). {@code override*}: an admin's correction. + */ +@Entity +@Table(name = "learned_rate_limits") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder(toBuilder = true) +public class LearnedRateLimit { + + @Id + @Column(name = "id", nullable = false, length = 800) + private String id; + + @Column(name = "host", nullable = false) + private String host; + + @Column(name = "method") + private String method; + + @Column(name = "path") + private String path; + + @Column(name = "scope") + private String scope; + + @Column(name = "scope_pinned", nullable = false) + private boolean scopePinned; + + @Column(name = "limit_calls") + private Integer limitCalls; + + @Column(name = "limit_window_seconds") + private Integer limitWindowSeconds; + + @Column(name = "resume_ms", nullable = false) + private long resumeMs; + + @Column(name = "resume_fail_ms") + private Long resumeFailMs; + + @Column(name = "resume_ok_ms") + private Long resumeOkMs; + + @Column(name = "last_fail_at") + private Instant lastFailAt; + + @Column(name = "rate_limited_count", nullable = false) + private int rateLimitedCount; + + @Column(name = "last_rate_limited_at") + private Instant lastRateLimitedAt; + + @Column(name = "override_calls") + private Integer overrideCalls; + + @Column(name = "override_window_seconds") + private Integer overrideWindowSeconds; + + @Column(name = "override_resume_ms") + private Long overrideResumeMs; + + /** The last admin correction or reset: an instance that hasn't taken it yet doesn't write over it. */ + @Column(name = "admin_changed_at") + private Instant adminChangedAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/tw/TwDtddTopicMapping.java b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/tw/TwDtddTopicMapping.java index 70b13586a..2331a99dd 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/domain/tw/TwDtddTopicMapping.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/domain/tw/TwDtddTopicMapping.java @@ -19,4 +19,8 @@ public class TwDtddTopicMapping { @Column(name = "dtdd_topic_name", nullable = false, length = 200) private String dtddTopicName; + + /** The least severe level of the topic that turns the TW on (one of its DTDD labels); null: any level. */ + @Column(name = "min_severity", length = 40) + private String minSeverity; } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/event/TwsMissedSteamEvent.java b/catapult-api/src/main/java/fr/enimaloc/catapult/event/TwsMissedSteamEvent.java new file mode 100644 index 000000000..2c70d6618 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/event/TwsMissedSteamEvent.java @@ -0,0 +1,10 @@ +package fr.enimaloc.catapult.event; + +import java.time.Instant; +import java.util.UUID; + +/** + * A Steam game's binding had its TWs resolved while the Steam store was rate limited: they miss what Steam would + * have added, and are to be resolved again once it may be called ({@code until}). + */ +public record TwsMissedSteamEvent(UUID bindingId, Instant until) {} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiClient.java b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiClient.java index 9d28c2c12..d955ada43 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiClient.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiClient.java @@ -3,6 +3,7 @@ import org.jetbrains.annotations.Nullable; import java.util.List; +import java.util.Map; import java.util.Optional; public interface DtddApiClient { @@ -19,13 +20,49 @@ default Optional> search(String query) { Optional item(long dtddId); + /** + * The same calls, made only with keys that keep more than {@code monthlyReserve} requests of their month (none + * left: they answer empty) — for bulk work, which must leave the reserve to the detection. + */ + default DtddApiClient reserving(int monthlyReserve) { + return this; + } + record DtddSearchResult(long dtddId, String name, String slug, String mediaType, String posterUrl) {} - record DtddTopics(List yesTopics, List noTopics, List mostlyTopics) {} + /** A title's topics by its votes, and each one's severity by its name (a topic without levels absent). */ + record DtddTopics(List yesTopics, List noTopics, List mostlyTopics, + Map severities) { + + /** Topics without their severities. */ + public DtddTopics(List yesTopics, List noTopics, List mostlyTopics) { + this(yesTopics, noTopics, mostlyTopics, Map.of()); + } + } record DtddItem(long id, String name, String[] genres, long releaseYear, long itemTypeId, String itemTypeName, long tmdbId, String imdbId, String backgroundImage, String posterImage, String overview, DtddTopicItemStat[] topicItemStats) {} - record DtddTopicItemStat(long topicItemId, long yesSum, long noSum, long numComments, long topicId, String topicName, long itemId) {} + /** A topic's votes on a title; {@code severity}: its level votes, null when the topic has no levels. */ + record DtddTopicItemStat(long topicItemId, long yesSum, long noSum, long numComments, long topicId, String topicName, + long itemId, DtddSeverity severity) {} + + /** + * How severe a topic is on a title, as DTDD's members voted it: {@code levels} the topic's labels, mildest first + * ("Implied", "Depicted", "Excessive"
); {@code counts} and {@code percents} the votes for each, by position; + * {@code total} the level votes; {@code verified} the level DTDD's reviewers gave (1-based, as a rating's + * severity), null when they gave none. + */ + record DtddSeverity(List levels, List counts, List percents, int total, + Integer verified) { + + /** The reviewers' level as its label, empty when they gave none (or one out of {@code levels}). */ + public Optional verifiedLevel() { + if (verified == null || levels == null || verified < 1 || verified > levels.size()) { + return Optional.empty(); + } + return Optional.of(levels.get(verified - 1)); + } + } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotator.java b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotator.java index 9eafccd38..33da0c9e0 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotator.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotator.java @@ -23,6 +23,7 @@ public class DtddApiKeyRotator { private final DtddApiKeyRepository repository; private final MeterRegistry meterRegistry; private final Map keyBlockedUntil = new ConcurrentHashMap<>(); + private final Map quotas = new ConcurrentHashMap<>(); private final AtomicInteger counter = new AtomicInteger(0); private volatile List keys = List.of(); @@ -88,6 +89,66 @@ public boolean isAllKeysBlocked() { return snapshot.stream().allMatch(k -> keyBlockedUntil.getOrDefault(k, 0L) > now); } + /** + * What DTDD said was left of a key's quota with its last answer ({@code X-RateLimit-Remaining-*}), null when it + * didn't say; {@code at} when it said it (epoch ms). + */ + public record Quota(Integer remainingMinute, Integer remainingMonth, long at) {} + + /** + * Remembers what DTDD said was left of {@code key}'s quota. Nothing left this minute: the key rests until the + * next one instead of asking for a 429. + */ + public void onQuota(String key, Integer remainingMinute, Integer remainingMonth) { + if (remainingMinute == null && remainingMonth == null) return; + long now = System.currentTimeMillis(); + quotas.put(key, new Quota(remainingMinute, remainingMonth, now)); + if (remainingMinute != null && remainingMinute <= 0) { + keyBlockedUntil.merge(key, now + 60_000L, Math::max); + } + } + + /** Each key's last known quota (keys DTDD never answered for are absent). */ + public Map getQuotas() { + return Map.copyOf(quotas); + } + + /** + * A key usable now that keeps more than {@code monthlyReserve} requests of its month (one whose month is unknown + * counts as keeping them), round robin; empty when none is — for bulk work, which must leave the reserve to the + * detection. Unlike {@link #nextKey}, never a blocked one. + */ + public Optional nextKeyAbove(int monthlyReserve) { + long now = System.currentTimeMillis(); + List available = keys.stream() + .filter(k -> aboveReserve(k, monthlyReserve)) + .filter(k -> keyBlockedUntil.getOrDefault(k, 0L) <= now) + .toList(); + if (available.isEmpty()) return Optional.empty(); + return Optional.of(available.get(Math.floorMod(counter.getAndIncrement(), available.size()))); + } + + /** + * When a key that keeps more than {@code monthlyReserve} requests of its month is usable (epoch ms, now or + * earlier when one already is); empty when no key keeps them: the month is spent down to the reserve. + */ + public Optional nextUsableAt(int monthlyReserve) { + return keys.stream() + .filter(k -> aboveReserve(k, monthlyReserve)) + .map(k -> keyBlockedUntil.getOrDefault(k, 0L)) + .min(Long::compare); + } + + /** How many keys keep more than {@code monthlyReserve} requests of their month, blocked or not. */ + public int keysAbove(int monthlyReserve) { + return (int) keys.stream().filter(k -> aboveReserve(k, monthlyReserve)).count(); + } + + private boolean aboveReserve(String key, int monthlyReserve) { + Quota quota = quotas.get(key); + return quota == null || quota.remainingMonth() == null || quota.remainingMonth() > monthlyReserve; + } + public void onKeyRateLimited(String key, int retryAfterSeconds) { keyBlockedUntil.put(key, System.currentTimeMillis() + retryAfterSeconds * 1000L); meterRegistry.counter("catapult.external.rate_limited", "api", "dtdd", "scope", "key").increment(); diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClient.java b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClient.java index 2ff2d13fe..5d2851fc7 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClient.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClient.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import fr.enimaloc.catapult.service.metrics.ExternalApiObservations; import lombok.extern.slf4j.Slf4j; @@ -9,26 +10,35 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatusCode; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; import org.springframework.web.client.RestClientResponseException; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; +import java.util.function.Supplier; @Slf4j @Component @ConditionalOnBooleanProperty("dtdd.enabled") public class RealDtddApiClient implements DtddApiClient { - private static final ObjectMapper MAPPER = new ObjectMapper(); + // DTDD adds fields to its answers (severity, 2026-11): one we don't know must not fail the whole call + private static final ObjectMapper MAPPER = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); private final RestClient client; private final DtddApiKeyRotator rotator; private final ExternalApiObservations apiObservations; + /** The key each call takes: any usable one, or (a {@link #reserving} view) only those above a reserve. */ + private final Supplier> keys; @Autowired public RealDtddApiClient(@Value("${dtdd.api-base-url}") String baseUrl, @@ -42,9 +52,20 @@ public RealDtddApiClient(@Value("${dtdd.api-base-url}") String baseUrl, // visible for tests RealDtddApiClient(RestClient client, DtddApiKeyRotator rotator, ExternalApiObservations apiObservations) { + this(client, rotator, apiObservations, rotator::nextKey); + } + + private RealDtddApiClient(RestClient client, DtddApiKeyRotator rotator, ExternalApiObservations apiObservations, + Supplier> keys) { this.client = client; this.rotator = rotator; this.apiObservations = apiObservations; + this.keys = keys; + } + + @Override + public DtddApiClient reserving(int monthlyReserve) { + return new RealDtddApiClient(client, rotator, apiObservations, () -> rotator.nextKeyAbove(monthlyReserve)); } @Override @@ -54,7 +75,7 @@ public Optional> search(String query, @Nullable String me .uri("/v3/items?q={q}", query) .header("X-API-KEY", key) .retrieve() - .body(String.class), this::parseSearch) + .toEntity(String.class), this::parseSearch) .map(list -> list.stream() .filter(result -> mediaType == null || mediaType.equals(result.mediaType())) .toList()) @@ -66,12 +87,16 @@ public Optional fetchTopics(long dtddId) { return apiObservations.observe("dtdd", "fetch_topics", () -> item(dtddId).map(item -> { List yes = new ArrayList<>(), no = new ArrayList<>(), mostly = new ArrayList<>(); + Map severities = new LinkedHashMap<>(); for (DtddTopicItemStat topicItemStat : item.topicItemStats()) { if (topicItemStat.yesSum() > topicItemStat.noSum()) yes.add(topicItemStat.topicName()); else if (topicItemStat.yesSum() < topicItemStat.noSum()) no.add(topicItemStat.topicName()); else mostly.add(topicItemStat.topicName()); + if (topicItemStat.severity() != null && topicItemStat.topicName() != null) { + severities.put(topicItemStat.topicName(), topicItemStat.severity()); + } } - return new DtddTopics(yes, no, mostly); + return new DtddTopics(yes, no, mostly, severities); }) ); } @@ -83,18 +108,20 @@ public Optional item(long dtddId) { .uri("/v3/items/{id}", dtddId) .header("X-API-KEY", key) .retrieve() - .body(String.class), this::parseItem) + .toEntity(String.class), this::parseItem) ); } - private Optional callWithRetry(Function call, Function parser) { + private Optional callWithRetry(Function> call, Function parser) { for (int attempt = 0; attempt < 2; attempt++) { - Optional keyOpt = rotator.nextKey(); + Optional keyOpt = keys.get(); if (keyOpt.isEmpty()) return Optional.empty(); String key = keyOpt.get(); try { - String body = call.apply(key); - return Optional.ofNullable(parser.apply(body)); + ResponseEntity response = call.apply(key); + rotator.onQuota(key, header(response.getHeaders(), "X-RateLimit-Remaining-Minute"), + header(response.getHeaders(), "X-RateLimit-Remaining-Month")); + return Optional.ofNullable(parser.apply(response.getBody())); } catch (RestClientResponseException e) { HttpStatusCode status = e.getStatusCode(); if (status.value() == 404) return Optional.empty(); @@ -115,6 +142,17 @@ private Optional callWithRetry(Function call, Function excludedHosts = new ArrayList<>(List.of("api.steampowered.com", "api.twitch.tv", + "id.twitch.tv", "api.igdb.com", "www.doesthedogdie.com", "git.enimaloc.fr", "oauth.battle.net", + "*.battle.net", "*.api.blizzard.com")); + private double bulkMargin = 0.8; + private Duration liveMaxWait = Duration.ofSeconds(2); + private Duration raiseAfter = Duration.ofHours(1); + private Duration forgetFailAfter = Duration.ofHours(24); + /** Past this many routes, a host's new ones share a single route ({@code /{other}}): what is stored stays bounded. */ + private int maxRoutesPerHost = 50; +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiter.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiter.java new file mode 100644 index 000000000..28db6b966 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiter.java @@ -0,0 +1,337 @@ +package fr.enimaloc.catapult.ratelimit; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; +import fr.enimaloc.catapult.domain.ratelimit.LearnedRateLimit; +import fr.enimaloc.catapult.ratelimit.LimitEstimator.Estimate; +import fr.enimaloc.catapult.ratelimit.RateLimitScope.Kind; +import fr.enimaloc.catapult.repository.ratelimit.LearnedRateLimitRepository; +import lombok.AccessLevel; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.server.ResponseStatusException; + +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Rate limits the platforms don't document, guessed from their 429s and kept to before calling them: a route counts + * in its host's budget until it shows a limit of its own. A live call passes while under the limit and gives up + * after a short wait; bulk work keeps a margin under it and waits its turn. What is learned is stored, shared by the + * instances, and can be corrected or reset by an admin. + */ +@Slf4j +@Component +public class LearnedRateLimiter { + + /** The header a refusal made here (no call sent) carries: never learned from. */ + public static final String LOCAL_REFUSAL_HEADER = "X-Catapult-Local-Limit"; + private static final long SLICE_MS = 1_000L; + static final String OTHER_PATH = "/{other}"; + /** The scope an admin gives a route back to the learning with. */ + static final String LEARNED_SCOPE = "LEARNED"; + + interface Sleeper { + void sleep(long millis) throws InterruptedException; + } + + private final LearnedRateLimitProperties props; + private final LearnedRateLimitRepository repository; + private final Clock clock; + /** Its own transaction for each write: never part of (nor failing) the caller's. */ + private final TransactionTemplate writes; + private final Map scopes = new ConcurrentHashMap<>(); + + @Setter(AccessLevel.PACKAGE) + private Sleeper sleeper = Thread::sleep; + + /** + * {@code clock}: the application's when there is one (not every profile defines it), else the system's; + * {@code transactions}: each write's own transaction (without one, the repository's). + */ + public LearnedRateLimiter(LearnedRateLimitProperties props, LearnedRateLimitRepository repository, + Optional clock, Optional transactions) { + this.props = props; + this.repository = repository; + this.clock = clock.orElseGet(Clock::systemUTC); + this.writes = transactions.map(TransactionTemplate::new).orElse(null); + if (writes != null) writes.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + } + + /** Whether calls to {@code host} are learned from (not one of the hosts left to their documented limits). */ + public boolean learns(String host) { + if (!props.isEnabled() || host == null || host.isEmpty()) return false; + String h = host.toLowerCase(Locale.ROOT); + for (String excluded : props.getExcludedHosts()) { + String e = excluded.trim().toLowerCase(Locale.ROOT); + if (e.startsWith("*.") ? h.endsWith(e.substring(1)) : h.equals(e)) return false; + } + return true; + } + + /** + * Waits until the route may be called, counting the call in. Refuses with a local 429 (carrying + * {@link #LOCAL_REFUSAL_HEADER}) when a live call would wait longer than allowed, or bulk work was cancelled. + */ + public void acquire(RouteKey key) { + if (!learns(key.host())) return; + RateLimitScope route = route(key); + RateLimitScope host = host(key.host()); + boolean bulk = RateLimitPriority.current() == RateLimitPriority.BULK; + Instant deadline = bulk ? null : clock.instant().plus(props.getLiveMaxWait()); + while (true) { + Instant now = clock.instant(); + RateLimitScope budget = route.kind() == Kind.ROUTE ? route : host; + boolean[] raised = {false}; + Optional retryAt = budget.admit(now, bulk, props.getBulkMargin(), raised); + if (raised[0]) save(budget); + if (retryAt.isEmpty()) { + if (budget != route) route.record(now); + return; + } + Instant at = retryAt.get(); + if (deadline != null && at.isAfter(deadline)) throw refusal(now, at); + if (bulk && RateLimitPriority.cancelled()) throw refusal(now, at); + try { + sleeper.sleep(Math.max(1, Math.min(SLICE_MS, Duration.between(now, at).toMillis()))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw refusal(now, at); + } + } + } + + /** What the platform answered a call {@link #acquire}d for: learns from a 429, ends a probe on an answer. */ + public void onResponse(RouteKey key, HttpStatusCode status, HttpHeaders headers) { + if (!learns(key.host())) return; + if (headers != null && headers.containsHeader(LOCAL_REFUSAL_HEADER)) return; + RateLimitScope route = route(key); + RateLimitScope host = host(key.host()); + Instant now = clock.instant(); + if (status.value() != HttpStatus.TOO_MANY_REQUESTS.value()) { + host.answered(route.id(), now); + RateLimitScope budget = route.kind() == Kind.ROUTE ? route : host; + if (budget.onAnswer(now)) save(budget); + return; + } + Optional retryAfter = retryAfter(headers == null ? null : headers.getFirst(HttpHeaders.RETRY_AFTER), now); + if (route.kind() == Kind.HOST && ownLimitShown(route, host, now) && route.splitOff()) { + log.info("Learned rate limits: {} has a limit of its own, apart from {}", route.id(), host.id()); + } + if (route.kind() == Kind.ROUTE) { + route.on429(now, retryAfter); + } else { + route.countRateLimited(now); + host.on429(now, retryAfter); + save(host); + } + save(route); + log.warn("Rate limited (429) on {}: left alone until {}", route.id(), + (route.kind() == Kind.ROUTE ? route : host).blockedUntil(now).orElse(now)); + } + + /** + * A 429 while the host's budget is well under its learned limit and another of its routes is still answered: + * the route's limit is its own. + */ + private boolean ownLimitShown(RateLimitScope route, RateLimitScope host, Instant now) { + Optional limit = host.limit(); + if (limit.isEmpty()) return false; + Duration window = limit.get().window(); + return host.used(now) * 2 < limit.get().calls() + && host.otherRouteAnsweredSince(route.id(), now.minus(window)); + } + + /** Until when calls to this GET route wait after a 429 (its own block, or its host's). */ + public Optional blockedUntil(String host, String path) { + if (!learns(host)) return Optional.empty(); + RateLimitScope route = route(new RouteKey(host.toLowerCase(Locale.ROOT), "GET", path)); + RateLimitScope budget = route.kind() == Kind.ROUTE ? route : host(host.toLowerCase(Locale.ROOT)); + return budget.blockedUntil(clock.instant()); + } + + /** Every scope learned or seen: hosts and their routes, stored ones included, by host then route. */ + public List list() { + Instant now = clock.instant(); + Map out = new TreeMap<>(); + try { + for (LearnedRateLimit e : repository.findAll()) { + RateLimitScope scope = scopes.computeIfAbsent(e.getId(), id -> fromEntity(e)); + out.put(e.getId(), scope.toDto(now)); + } + } catch (RuntimeException ex) { + log.warn("Learned rate limits: the stored ones could not be read: {}", ex.getMessage()); + } + scopes.values().forEach(s -> out.putIfAbsent(s.id(), s.toDto(now))); + return out.values().stream() + .sorted(Comparator.comparing(LearnedRateLimitDto::host) + .thenComparing(d -> d.hostScope() ? "" : d.path() + " " + d.method())) + .toList(); + } + + /** + * An admin's correction of a scope (404 if unknown): the limit ({@code calls} per {@code windowSeconds}, both or + * neither), the resume delay, a route's scope (HOST, ROUTE, or LEARNED: given back to the learning); null leaves + * that part to the learning. + */ + public LearnedRateLimitDto correct(String id, Integer calls, Integer windowSeconds, Integer resumeSeconds, + String scope) { + RateLimitScope s = known(id); + if ((calls == null) != (windowSeconds == null) || (calls != null && (calls < 1 || windowSeconds < 1)) + || (resumeSeconds != null && resumeSeconds < 1)) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid correction"); + } + String named = scope == null ? "" : scope.trim().toUpperCase(Locale.ROOT); + boolean unpin = named.equals(LEARNED_SCOPE); + Kind kind; + try { + kind = named.isEmpty() || unpin ? null : Kind.valueOf(named); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Unknown scope " + scope); + } + s.correct(calls == null ? null : new Estimate(calls, Duration.ofSeconds(windowSeconds)), + resumeSeconds == null ? null : Duration.ofSeconds(resumeSeconds), kind, unpin, clock.instant()); + store(s); + return s.toDto(clock.instant()); + } + + /** Forgets what was learned and corrected of a scope (404 if unknown). */ + public LearnedRateLimitDto reset(String id) { + RateLimitScope s = known(id); + s.reset(clock.instant()); + store(s); + return s.toDto(clock.instant()); + } + + /** Takes what the other instances learned or an admin corrected meanwhile. */ + @Scheduled(fixedDelayString = "${rate-limit.learning.sync-every:60s}") + public void sync() { + try { + for (LearnedRateLimit e : repository.findAll()) { + RateLimitScope scope = scopes.get(e.getId()); + if (scope != null) scope.apply(e); + } + } catch (RuntimeException ex) { + log.warn("Learned rate limits: could not sync with the stored ones: {}", ex.getMessage()); + } + } + + private RateLimitScope known(String id) { + RateLimitScope s = scopes.get(id); + if (s != null) return s; + return repository.findById(id).map(e -> scopes.computeIfAbsent(id, k -> fromEntity(e))) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Unknown rate limit " + id)); + } + + private RateLimitScope host(String host) { + return scope(RateLimitScope.hostId(host), () -> new RateLimitScope(host, null, props, clock.instant())); + } + + /** The route's scope; past the host's {@code maxRoutesPerHost}, a new route shares {@code /{other}}. */ + private RateLimitScope route(RouteKey key) { + RouteKey k = key; + if (!scopes.containsKey(k.id())) { + String prefix = k.method() + " " + k.host() + "/"; + long known = scopes.keySet().stream().filter(id -> id.startsWith(prefix)).count(); + if (known >= props.getMaxRoutesPerHost()) k = new RouteKey(k.host(), k.method(), OTHER_PATH); + } + RouteKey routeKey = k; + return scope(routeKey.id(), () -> new RateLimitScope(routeKey.host(), routeKey, props, clock.instant())); + } + + /** The scope, loaded from what was stored the first time it is needed, stored when new. */ + private RateLimitScope scope(String id, java.util.function.Supplier fresh) { + RateLimitScope existing = scopes.get(id); + if (existing != null) return existing; + Optional stored; + try { + stored = repository.findById(id); + } catch (RuntimeException ex) { + log.warn("Learned rate limits: {} could not be read: {}", id, ex.getMessage()); + stored = Optional.empty(); + } + RateLimitScope created = fresh.get(); + stored.ifPresent(created::apply); + RateLimitScope winner = scopes.putIfAbsent(id, created); + if (winner != null) return winner; + if (stored.isEmpty()) save(created); + return created; + } + + private RateLimitScope fromEntity(LearnedRateLimit e) { + RateLimitScope s = e.getPath() == null + ? new RateLimitScope(e.getHost(), null, props, clock.instant()) + : new RateLimitScope(e.getHost(), new RouteKey(e.getHost(), e.getMethod(), e.getPath()), props, + clock.instant()); + s.apply(e); + return s; + } + + /** + * Stores what the learning changed, unless an admin corrected or reset the scope meanwhile (on another instance): + * then the scope takes that instead, rather than writing over it. + */ + private void save(RateLimitScope scope) { + write(scope, () -> { + Optional stored = repository.findById(scope.id()); + if (stored.isPresent() && scope.behind(stored.get())) { + scope.apply(stored.get()); + return; + } + repository.save(scope.toEntity(clock.instant())); + }); + } + + /** Stores an admin's correction or reset as it is. */ + private void store(RateLimitScope scope) { + write(scope, () -> repository.save(scope.toEntity(clock.instant()))); + } + + private void write(RateLimitScope scope, Runnable body) { + try { + if (writes == null) body.run(); + else writes.executeWithoutResult(status -> body.run()); + } catch (RuntimeException ex) { + log.warn("Learned rate limits: {} could not be stored: {}", scope.id(), ex.getMessage()); + } + } + + private static HttpClientErrorException refusal(Instant now, Instant until) { + HttpHeaders headers = new HttpHeaders(); + headers.set(LOCAL_REFUSAL_HEADER, "true"); + headers.set(HttpHeaders.RETRY_AFTER, String.valueOf(Math.max(1, Duration.between(now, until).toSeconds()))); + return HttpClientErrorException.create("Learned rate limit reached (no call sent)", + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, new byte[0], StandardCharsets.UTF_8); + } + + /** A Retry-After header: seconds or an HTTP date. */ + public static Optional retryAfter(String header, Instant now) { + if (header == null || header.isBlank()) return Optional.empty(); + String value = header.trim(); + try { + return Optional.of(Duration.ofSeconds(Math.max(1, Long.parseLong(value)))); + } catch (NumberFormatException notSeconds) { + try { + Instant at = ZonedDateTime.parse(value, DateTimeFormatter.RFC_1123_DATE_TIME).toInstant(); + return Optional.of(at.isAfter(now) ? Duration.between(now, at) : Duration.ofSeconds(1)); + } catch (DateTimeParseException notADate) { + return Optional.empty(); + } + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LimitEstimator.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LimitEstimator.java new file mode 100644 index 000000000..017b17dd4 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/LimitEstimator.java @@ -0,0 +1,119 @@ +package fr.enimaloc.catapult.ratelimit; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; + +/** + * A scope's limit, guessed from its 429s: the calls sent in the window the platform seems to count, minus the one it + * refused. Only tightened by a 429; raised by a tenth after a long time near it without any. Not thread-safe: its + * {@link RateLimitScope} guards it. + */ +final class LimitEstimator { + + /** The windows a platform may count calls in, shortest first. */ + static final List WINDOWS = List.of(Duration.ofSeconds(10), Duration.ofMinutes(1), + Duration.ofMinutes(5), Duration.ofHours(1)); + private static final Duration KEPT = WINDOWS.getLast(); + private static final int MAX_KEPT = 10_000; + /** The fewest calls a first limit is learned from: a 429 after fewer tells too little (other instances' calls). */ + static final int MIN_FIRST_LIMIT = 5; + + /** A limit: at most {@code calls} in any {@code window}. */ + record Estimate(int calls, Duration window) {} + + private final Duration raiseAfter; + private final Deque calls = new ArrayDeque<>(); + private Estimate estimate; + private Instant last429; + private Instant lastRaise; + /** Since when every call is counted: a 429 tells a limit only for a window counted whole. */ + private Instant countingSince; + + LimitEstimator(Duration raiseAfter, Instant countingSince) { + this.raiseAfter = raiseAfter; + this.countingSince = countingSince; + } + + void record(Instant at) { + calls.addLast(at); + prune(at); + } + + int countWithin(Duration window, Instant now) { + Instant from = now.minus(window); + int n = 0; + for (Iterator it = calls.descendingIterator(); it.hasNext(); ) { + if (!it.next().isAfter(from)) break; + n++; + } + return n; + } + + /** The oldest call still counted in {@code window}: the limit frees a call when it leaves the window. */ + Optional oldestWithin(Duration window, Instant now) { + Instant from = now.minus(window); + return calls.stream().filter(at -> at.isAfter(from)).findFirst(); + } + + Optional estimate() { + return Optional.ofNullable(estimate); + } + + /** + * A 429: the window is the shortest one at least as long as the wait the platform asks or the one learned + * ({@code resumeHint}), the limit the calls sent in it before the refused one. Never loosened by a 429, and at + * most halved by one in the same window: a 429 after few calls tells more of a longer window than of a tiny limit. + * Nothing is learned when the window wasn't counted whole (just started, or a wait ended within it): the calls + * the platform counted aren't all known; nor a first limit from fewer than {@link #MIN_FIRST_LIMIT} calls. + */ + void on429(Instant now, Duration resumeHint) { + last429 = now; + Duration window = WINDOWS.stream().filter(w -> w.compareTo(resumeHint) >= 0).findFirst().orElse(KEPT); + if (countingSince.isAfter(now.minus(window))) return; + int limit = Math.max(1, countWithin(window, now) - 1); + if (estimate != null && estimate.window().equals(window)) { + int halved = (estimate.calls() + 1) / 2; + estimate = new Estimate(Math.min(estimate.calls(), Math.max(limit, halved)), window); + } else if (limit >= MIN_FIRST_LIMIT) { + estimate = new Estimate(limit, window); + } + } + + /** Raises the limit by a tenth (at least one call) when it was nearly reached for long without a 429. */ + void maybeRaise(Instant now) { + if (estimate == null) return; + Instant since = last429 == null ? lastRaise + : lastRaise == null || last429.isAfter(lastRaise) ? last429 : lastRaise; + if (since != null && since.plus(raiseAfter).isAfter(now)) return; + if (countWithin(estimate.window(), now) * 10 < estimate.calls() * 8) return; + estimate = new Estimate(estimate.calls() + Math.max(1, estimate.calls() / 10), estimate.window()); + lastRaise = now; + } + + /** Forgets the calls counted: after a 429's wait (ending {@code from}), the platform counts afresh. */ + void forgetCalls(Instant from) { + calls.clear(); + countingSince = from; + } + + void set(Estimate estimate, Instant last429) { + this.estimate = estimate; + this.last429 = last429; + } + + void reset() { + estimate = null; + last429 = null; + lastRaise = null; + } + + private void prune(Instant now) { + Instant from = now.minus(KEPT); + while (!calls.isEmpty() && (calls.size() > MAX_KEPT || !calls.peekFirst().isAfter(from))) calls.removeFirst(); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitPriority.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitPriority.java new file mode 100644 index 000000000..2158c5004 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitPriority.java @@ -0,0 +1,47 @@ +package fr.enimaloc.catapult.ratelimit; + +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +/** + * Who is calling: {@link #LIVE} (a streamer waiting, the default) passes while under a route's learned limit; + * {@link #BULK} (a recalculation, a refetch) keeps a margin under it and waits its turn. Set around the calls with + * {@link #run}/{@link #call}, read by {@link LearnedRateLimiter} on the calling thread. + */ +public enum RateLimitPriority { + LIVE, BULK; + + private record Context(RateLimitPriority priority, BooleanSupplier cancelled) {} + + private static final Context DEFAULT = new Context(LIVE, () -> false); + private static final ThreadLocal CURRENT = ThreadLocal.withInitial(() -> DEFAULT); + + /** The calling thread's priority. */ + public static RateLimitPriority current() { + return CURRENT.get().priority(); + } + + /** Whether the calling thread's work was cancelled: a wait for the limit then gives up. */ + public static boolean cancelled() { + return CURRENT.get().cancelled().getAsBoolean(); + } + + /** Runs {@code body} at this priority; {@code cancelled} tells a wait for the limit to give up. */ + public static T call(RateLimitPriority priority, BooleanSupplier cancelled, Supplier body) { + Context previous = CURRENT.get(); + CURRENT.set(new Context(priority, cancelled)); + try { + return body.get(); + } finally { + CURRENT.set(previous); + } + } + + /** As {@link #call}, without a result. */ + public static void run(RateLimitPriority priority, BooleanSupplier cancelled, Runnable body) { + call(priority, cancelled, () -> { + body.run(); + return null; + }); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitScope.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitScope.java new file mode 100644 index 000000000..27c1bb8cc --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RateLimitScope.java @@ -0,0 +1,258 @@ +package fr.enimaloc.catapult.ratelimit; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; +import fr.enimaloc.catapult.domain.ratelimit.LearnedRateLimit; +import fr.enimaloc.catapult.ratelimit.LimitEstimator.Estimate; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * One scope's learning and state, guarded by itself: a host (the budget its {@link Kind#HOST} routes share) or a + * route. Only the scope a call is counted in (its route when {@link Kind#ROUTE}, else its host) blocks and learns; + * a route in its host's budget keeps its own 429 count and calls for the admin tab. + */ +final class RateLimitScope { + + /** A route's budget: its host's, or its own. */ + enum Kind { HOST, ROUTE } + + private final String id; + private final String host; + private final RouteKey route; + private final LimitEstimator estimator; + private final ResumeDelay resume; + private Kind kind = Kind.HOST; + private boolean kindPinned; + private Estimate overrideLimit; + private Duration overrideResume; + private Instant blockedUntil; + /** Blocked after a 429: the next answer tells whether the wait was long enough. */ + private boolean probing; + /** The block's wait was the learned one (no Retry-After): its outcome teaches the resume delay. */ + private boolean learningResume; + private int rateLimitedCount; + private Instant lastRateLimitedAt; + /** The last admin correction or reset taken: a newer one stored is taken before writing over it. */ + private Instant adminSeenAt; + /** A host's: when each of its routes was last answered (no 429), by route ID. */ + private final Map lastAnswerByRoute = new ConcurrentHashMap<>(); + + RateLimitScope(String host, RouteKey route, LearnedRateLimitProperties props, Instant now) { + this.host = host; + this.route = route; + this.id = route == null ? hostId(host) : route.id(); + this.estimator = new LimitEstimator(props.getRaiseAfter(), now); + this.resume = new ResumeDelay(props.getForgetFailAfter()); + } + + static String hostId(String host) { + return "HOST " + host; + } + + String id() { + return id; + } + + synchronized Kind kind() { + return kind; + } + + /** The limit that applies: the admin's correction, else the one learned. */ + synchronized Optional limit() { + return overrideLimit != null ? Optional.of(overrideLimit) : estimator.estimate(); + } + + synchronized Duration resumeDelay() { + return overrideResume != null ? overrideResume : resume.current(); + } + + synchronized Optional blockedUntil(Instant now) { + return blockedUntil != null && blockedUntil.isAfter(now) ? Optional.of(blockedUntil) : Optional.empty(); + } + + /** + * Counts a call in if the scope lets it go now ({@code bulk}: under {@code margin} of the limit); else when to + * try again. Returns whether the learned limit was raised, through {@code raised}. + */ + synchronized Optional admit(Instant now, boolean bulk, double margin, boolean[] raised) { + if (blockedUntil != null && blockedUntil.isAfter(now)) return Optional.of(blockedUntil); + if (overrideLimit == null) { + Optional before = estimator.estimate(); + estimator.maybeRaise(now); + raised[0] = !before.equals(estimator.estimate()); + } + Optional limit = limit(); + if (limit.isPresent()) { + Estimate l = limit.get(); + int cap = bulk ? Math.max(1, (int) Math.floor(l.calls() * margin)) : l.calls(); + if (estimator.countWithin(l.window(), now) >= cap) { + Instant freed = estimator.oldestWithin(l.window(), now).map(at -> at.plus(l.window())).orElse(now); + return Optional.of(freed.isAfter(now) ? freed : now.plusMillis(10)); + } + } + estimator.record(now); + return Optional.empty(); + } + + /** Counts a call made through another scope's budget (a route in its host's). */ + synchronized void record(Instant now) { + estimator.record(now); + } + + /** The calls counted here in the limit's window (a minute without a limit). */ + synchronized int used(Instant now) { + return estimator.countWithin(limit().map(Estimate::window).orElse(Duration.ofMinutes(1)), now); + } + + /** + * A 429 for this budget: blocked for {@code retryAfter} when the platform says, else for the learned delay (made + * longer first when the last one wasn't enough); the limit tightened from the calls before it, unless the 429 + * answers the first call after a wait (the wait was too short, not the limit too high). + */ + synchronized void on429(Instant now, Optional retryAfter) { + countRateLimited(now); + boolean afterWait = probing; + Duration wait; + if (retryAfter.isPresent()) { + wait = retryAfter.get(); + learningResume = false; + } else { + if (probing && learningResume && overrideResume == null) resume.onFailure(now); + wait = resumeDelay(); + learningResume = true; + } + if (overrideLimit == null && !afterWait) estimator.on429(now, wait); + Instant until = now.plus(wait); + if (blockedUntil == null || until.isAfter(blockedUntil)) blockedUntil = until; + // When the wait ends the platform counts afresh: the calls before it no longer fill the window + estimator.forgetCalls(blockedUntil); + probing = true; + } + + /** A route in its host's budget got a 429: counted for the admin tab only. */ + synchronized void countRateLimited(Instant now) { + rateLimitedCount++; + lastRateLimitedAt = now; + } + + /** An answer that is no 429: ends a probe, teaching the resume delay. Returns whether anything was learned. */ + synchronized boolean onAnswer(Instant now) { + if (!probing) return false; + if (learningResume && overrideResume == null) resume.onSuccess(now); + probing = false; + learningResume = false; + return true; + } + + void answered(String routeId, Instant now) { + lastAnswerByRoute.put(routeId, now); + } + + /** Whether a route of this host other than {@code routeId} was answered since {@code since}. */ + boolean otherRouteAnsweredSince(String routeId, Instant since) { + return lastAnswerByRoute.entrySet().stream() + .anyMatch(e -> !e.getKey().equals(routeId) && !e.getValue().isBefore(since)); + } + + /** The learning splits the route off its host's budget; never while an admin chose its scope. */ + synchronized boolean splitOff() { + if (kindPinned || kind == Kind.ROUTE) return false; + kind = Kind.ROUTE; + return true; + } + + /** + * An admin's correction: null values leave that part to the learning; a route's scope chosen ({@code pinnedKind}), + * or given back to the learning ({@code unpin}), else left as it is. + */ + synchronized void correct(Estimate limit, Duration resumeDelay, Kind pinnedKind, boolean unpin, Instant now) { + overrideLimit = limit; + overrideResume = resumeDelay; + if (route != null && pinnedKind != null) { + kind = pinnedKind; + kindPinned = true; + } else if (route != null && unpin) { + kindPinned = false; + } + adminSeenAt = now; + } + + /** Whether {@code stored} carries an admin's correction or reset this scope hasn't taken yet. */ + synchronized boolean behind(LearnedRateLimit stored) { + Instant changed = stored.getAdminChangedAt(); + return changed != null && (adminSeenAt == null || changed.isAfter(adminSeenAt)); + } + + /** Forgets everything learned and corrected: the scope starts again without a known limit. */ + synchronized void reset(Instant now) { + adminSeenAt = now; + estimator.reset(); + resume.reset(); + overrideLimit = null; + overrideResume = null; + blockedUntil = null; + probing = false; + learningResume = false; + rateLimitedCount = 0; + lastRateLimitedAt = null; + if (route != null) { + kind = Kind.HOST; + kindPinned = false; + } + } + + synchronized LearnedRateLimit toEntity(Instant now) { + Optional learned = estimator.estimate(); + ResumeDelay.State r = resume.state(); + return LearnedRateLimit.builder() + .id(id).host(host) + .method(route == null ? null : route.method()) + .path(route == null ? null : route.path()) + .scope(route == null ? null : kind.name()) + .scopePinned(kindPinned) + .limitCalls(learned.map(Estimate::calls).orElse(null)) + .limitWindowSeconds(learned.map(e -> (int) e.window().toSeconds()).orElse(null)) + .resumeMs(r.current().toMillis()) + .resumeFailMs(r.fail() == null ? null : r.fail().toMillis()) + .resumeOkMs(r.ok() == null ? null : r.ok().toMillis()) + .lastFailAt(r.lastFail()) + .rateLimitedCount(rateLimitedCount) + .lastRateLimitedAt(lastRateLimitedAt) + .overrideCalls(overrideLimit == null ? null : overrideLimit.calls()) + .overrideWindowSeconds(overrideLimit == null ? null : (int) overrideLimit.window().toSeconds()) + .overrideResumeMs(overrideResume == null ? null : overrideResume.toMillis()) + .adminChangedAt(adminSeenAt) + .updatedAt(now) + .build(); + } + + /** Takes what was stored (learned, corrected, the scope), keeping the calls counted here. */ + synchronized void apply(LearnedRateLimit e) { + estimator.set(e.getLimitCalls() == null || e.getLimitWindowSeconds() == null ? null + : new Estimate(e.getLimitCalls(), Duration.ofSeconds(e.getLimitWindowSeconds())), e.getLastRateLimitedAt()); + resume.restore(new ResumeDelay.State(Duration.ofMillis(e.getResumeMs()), + e.getResumeFailMs() == null ? null : Duration.ofMillis(e.getResumeFailMs()), + e.getResumeOkMs() == null ? null : Duration.ofMillis(e.getResumeOkMs()), e.getLastFailAt())); + overrideLimit = e.getOverrideCalls() == null || e.getOverrideWindowSeconds() == null ? null + : new Estimate(e.getOverrideCalls(), Duration.ofSeconds(e.getOverrideWindowSeconds())); + overrideResume = e.getOverrideResumeMs() == null ? null : Duration.ofMillis(e.getOverrideResumeMs()); + rateLimitedCount = e.getRateLimitedCount(); + lastRateLimitedAt = e.getLastRateLimitedAt(); + if (route != null && e.getScope() != null) kind = Kind.valueOf(e.getScope()); + kindPinned = e.isScopePinned(); + adminSeenAt = e.getAdminChangedAt(); + } + + synchronized LearnedRateLimitDto toDto(Instant now) { + Optional limit = limit(); + return new LearnedRateLimitDto(id, host, route == null ? null : route.method(), + route == null ? null : route.path(), route == null ? null : kind.name(), kindPinned, + limit.map(Estimate::calls).orElse(null), limit.map(l -> (int) l.window().toSeconds()).orElse(null), + overrideLimit != null, resumeDelay().toMillis(), overrideResume != null, rateLimitedCount, + lastRateLimitedAt, used(now), blockedUntil(now).orElse(null)); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/ResumeDelay.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/ResumeDelay.java new file mode 100644 index 000000000..fcd47fe8b --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/ResumeDelay.java @@ -0,0 +1,73 @@ +package fr.enimaloc.catapult.ratelimit; + +import java.time.Duration; +import java.time.Instant; + +/** + * How long a scope waits after a 429 without a Retry-After, learned from the first call after each wait: an answer + * tries a fifth shorter next time, another 429 doubles it. Never shorter than the longest wait that failed lately + * ({@code fail}, forgotten after a long time without failing). Not thread-safe: its {@link RateLimitScope} guards it. + */ +final class ResumeDelay { + + static final Duration INITIAL = Duration.ofSeconds(60); + static final Duration MAX = Duration.ofMinutes(10); + private static final Duration MIN = Duration.ofSeconds(1); + + /** What is kept of it. */ + record State(Duration current, Duration fail, Duration ok, Instant lastFail) {} + + private final Duration forgetFailAfter; + private Duration current = INITIAL; + private Duration fail; + private Duration ok; + private Instant lastFail; + + ResumeDelay(Duration forgetFailAfter) { + this.forgetFailAfter = forgetFailAfter; + } + + Duration current() { + return current; + } + + /** The first call after waiting {@link #current} got a 429 again: too short. */ + void onFailure(Instant now) { + forgetOldFailure(now); + fail = fail == null || current.compareTo(fail) > 0 ? current : fail; + Duration doubled = current.multipliedBy(2); + current = doubled.compareTo(MAX) > 0 ? MAX : doubled; + lastFail = now; + } + + /** The first call after waiting {@link #current} was answered: long enough, a shorter wait is tried next. */ + void onSuccess(Instant now) { + forgetOldFailure(now); + ok = ok == null || current.compareTo(ok) < 0 ? current : ok; + Duration shorter = current.multipliedBy(4).dividedBy(5); + if (fail != null && shorter.compareTo(fail) < 0) shorter = fail; + current = shorter.compareTo(MIN) < 0 ? MIN : shorter; + } + + State state() { + return new State(current, fail, ok, lastFail); + } + + void restore(State state) { + current = state.current() == null ? INITIAL : state.current(); + fail = state.fail(); + ok = state.ok(); + lastFail = state.lastFail(); + } + + void reset() { + restore(new State(INITIAL, null, null, null)); + } + + private void forgetOldFailure(Instant now) { + if (lastFail != null && lastFail.plus(forgetFailAfter).isBefore(now)) { + fail = null; + lastFail = null; + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RouteKey.java b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RouteKey.java new file mode 100644 index 000000000..a408269f8 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/ratelimit/RouteKey.java @@ -0,0 +1,61 @@ +package fr.enimaloc.catapult.ratelimit; + +import org.springframework.http.HttpRequest; + +import java.net.URI; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * One route of a platform: the method, the host and the path as written by the caller ({@code /app/{id}}), never the + * resolved URL, so every app's page counts as the same route. Identifier-looking segments become {@code {id}}, also + * in a template a caller built by concatenating them. + */ +public record RouteKey(String host, String method, String path) { + + /** The request attribute {@code RestClient} keeps the URI template in. */ + static final String URI_TEMPLATE_ATTRIBUTE = "org.springframework.web.client.DefaultRestClient.uriTemplate"; + + /** The longest path kept (the rest cut): what is stored stays bounded. */ + static final int MAX_PATH = 300; + private static final Pattern ID_SEGMENT = Pattern.compile("\\d+|[A-Za-z0-9_-]{16,}|.*\\(.*\\).*"); + + public static RouteKey of(HttpRequest request) { + URI uri = request.getURI(); + String host = uri.getHost() == null ? "" : uri.getHost().toLowerCase(Locale.ROOT); + Object template = request.getAttributes().get(URI_TEMPLATE_ATTRIBUTE); + String path = normalizedPath(template instanceof String t ? templatePath(t) : uri.getRawPath()); + return new RouteKey(host, request.getMethod().name(), path); + } + + /** The route's ID: {@code GET store.steampowered.com/app/{id}}. */ + public String id() { + return method + " " + host + path; + } + + /** A template's path: no scheme, host or query. */ + static String templatePath(String template) { + String path = template; + int scheme = path.indexOf("://"); + if (scheme >= 0) { + int slash = path.indexOf('/', scheme + 3); + path = slash < 0 ? "/" : path.substring(slash); + } + int query = path.indexOf('?'); + if (query >= 0) path = path.substring(0, query); + return path.isEmpty() ? "/" : path; + } + + /** A resolved path whose identifier-looking segments become {@code {id}}. */ + static String normalizedPath(String rawPath) { + if (rawPath == null || rawPath.isEmpty()) return "/"; + StringBuilder out = new StringBuilder(); + for (String segment : rawPath.split("/", -1)) { + if (segment.isEmpty()) continue; + out.append('/').append(ID_SEGMENT.matcher(segment).matches() ? "{id}" : segment); + } + if (rawPath.endsWith("/") && !out.isEmpty()) out.append('/'); + if (out.length() > MAX_PATH) return out.substring(0, MAX_PATH) + "
"; + return out.isEmpty() ? "/" : out.toString(); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddGameCacheRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddGameCacheRepository.java index 16bc05aa6..00ffed273 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddGameCacheRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddGameCacheRepository.java @@ -1,15 +1,28 @@ package fr.enimaloc.catapult.repository.dtdd; import fr.enimaloc.catapult.domain.dtdd.DtddGameCache; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import java.time.Instant; import java.util.List; public interface DtddGameCacheRepository extends JpaRepository { @Query("SELECT g FROM DtddGameCache g WHERE LOWER(g.name) LIKE LOWER(CONCAT('%', :q, '%'))") List searchByNameLike(@Param("q") String q, Pageable pageable); + + /** The cached titles whose ID starts with {@code q} or whose name contains it (lower-cased by the caller). */ + @Query("SELECT g FROM DtddGameCache g WHERE CAST(g.dtddId AS string) LIKE CONCAT(:q, '%') " + + "OR LOWER(g.name) LIKE CONCAT('%', :q, '%')") + Page search(@Param("q") String q, Pageable pageable); + + long countByFetchedAtBefore(Instant before); + + /** Each title's ID and when it was fetched, the oldest first: what a bulk refetch goes through. */ + @Query("SELECT g.dtddId, g.fetchedAt FROM DtddGameCache g ORDER BY g.fetchedAt") + List findAllAges(); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddSearchCacheRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddSearchCacheRepository.java index fa4e216ab..c72532793 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddSearchCacheRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddSearchCacheRepository.java @@ -1,7 +1,34 @@ package fr.enimaloc.catapult.repository.dtdd; import fr.enimaloc.catapult.domain.dtdd.DtddSearchCache; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.time.Instant; +import java.util.List; public interface DtddSearchCacheRepository extends JpaRepository { + + /** The cached searches whose query contains {@code q} (lower-cased by the caller, as the queries are). */ + @Query("SELECT s FROM DtddSearchCache s WHERE s.queryNormalized LIKE CONCAT('%', :q, '%')") + Page search(@Param("q") String q, Pageable pageable); + + long countBySearchedAtBefore(Instant before); + + /** The cached searches that found the title {@code dtddId}, the latest first. */ + @Query(value = "SELECT * FROM dtdd_search_cache WHERE dtdd_ids @> CAST(:ids AS jsonb) ORDER BY searched_at DESC", + nativeQuery = true) + List findAllFinding(@Param("ids") String jsonIds); + + /** The cached searches that found the title {@code dtddId}, the latest first. */ + default List findAllFinding(long dtddId) { + return findAllFinding("[" + dtddId + "]"); + } + + /** Each query and when it was searched, the oldest first: what a bulk refetch goes through. */ + @Query("SELECT s.queryNormalized, s.searchedAt FROM DtddSearchCache s ORDER BY s.searchedAt") + List findAllAges(); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddTopicsCacheRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddTopicsCacheRepository.java index ca38cb3df..c3f8c927e 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddTopicsCacheRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/dtdd/DtddTopicsCacheRepository.java @@ -1,7 +1,35 @@ package fr.enimaloc.catapult.repository.dtdd; import fr.enimaloc.catapult.domain.dtdd.DtddTopicsCache; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.time.Instant; +import java.util.List; public interface DtddTopicsCacheRepository extends JpaRepository { + + /** The cached topics whose title's ID starts with {@code q}, or whose cached game's name contains it. */ + @Query("SELECT t FROM DtddTopicsCache t WHERE CAST(t.dtddId AS string) LIKE CONCAT(:q, '%') " + + "OR t.dtddId IN (SELECT g.dtddId FROM DtddGameCache g WHERE LOWER(g.name) LIKE CONCAT('%', :q, '%'))") + Page search(@Param("q") String q, Pageable pageable); + + long countByFetchedAtBefore(Instant before); + + /** + * The severity levels of these topics (lower-cased names) as the last cached title having them gives them: each + * row its lower-cased name and its levels as a JSON array. + */ + @Query(value = "SELECT DISTINCT ON (lower(s.key)) lower(s.key), CAST(s.value -> 'levels' AS text) " + + "FROM dtdd_topics_cache t, jsonb_each(t.severities) s " + + "WHERE t.severities IS NOT NULL AND jsonb_typeof(s.value) = 'object' AND lower(s.key) IN (:lowered) " + + "ORDER BY lower(s.key), t.fetched_at DESC", nativeQuery = true) + List findLevelsOf(@Param("lowered") java.util.Collection lowered); + + /** Each title's ID and when its topics were fetched, the oldest first: what a bulk refetch goes through. */ + @Query("SELECT t.dtddId, t.fetchedAt FROM DtddTopicsCache t ORDER BY t.fetchedAt") + List findAllAges(); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/ratelimit/LearnedRateLimitRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/ratelimit/LearnedRateLimitRepository.java new file mode 100644 index 000000000..6acfc525f --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/ratelimit/LearnedRateLimitRepository.java @@ -0,0 +1,7 @@ +package fr.enimaloc.catapult.repository.ratelimit; + +import fr.enimaloc.catapult.domain.ratelimit.LearnedRateLimit; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LearnedRateLimitRepository extends JpaRepository { +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwDtddTopicMappingRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwDtddTopicMappingRepository.java index 14ffa9261..c8d4bffd7 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwDtddTopicMappingRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwDtddTopicMappingRepository.java @@ -16,5 +16,9 @@ public interface TwDtddTopicMappingRepository extends JpaRepository findTwIdsByDtddTopicNameInIgnoreCase(@Param("lowered") Set lowered); + /** The mappings of these DTDD topics (lower-cased names), each one with its TW and its least severe level. */ + @Query("SELECT m FROM TwDtddTopicMapping m WHERE LOWER(m.dtddTopicName) IN :lowered") + List findAllByDtddTopicNameInIgnoreCase(@Param("lowered") Set lowered); + void deleteAllByTwId(String twId); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwIgdbDescriptorMappingRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwIgdbDescriptorMappingRepository.java index 26b5c491d..791570f28 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwIgdbDescriptorMappingRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwIgdbDescriptorMappingRepository.java @@ -15,5 +15,8 @@ public interface TwIgdbDescriptorMappingRepository extends JpaRepository findTwIdsByDescriptorIds(@Param("ids") Set descriptorIds); + /** The mappings of these IGDB descriptors, each one with its TW. */ + List findAllByDescriptorIdIn(Set descriptorIds); + void deleteAllByTwId(String twId); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwSteamContentIdMappingRepository.java b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwSteamContentIdMappingRepository.java index 7cb696c05..818d97c78 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwSteamContentIdMappingRepository.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/repository/tw/TwSteamContentIdMappingRepository.java @@ -15,5 +15,8 @@ public interface TwSteamContentIdMappingRepository extends JpaRepository findTwIdsBySteamContentIdIn(@Param("ids") Set contentIds); + /** The mappings of these Steam content descriptors, each one with its TW. */ + List findAllBySteamContentIdIn(Set contentIds); + void deleteAllByTwId(String twId); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/account/AdminMigrationService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/account/AdminMigrationService.java index 6a0146312..b1b926e62 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/account/AdminMigrationService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/account/AdminMigrationService.java @@ -99,6 +99,7 @@ private void copyBindings(UserAccount source, UserAccount target) { tgt.setSourceName(src.getSourceName()); tgt.setStatus(src.getStatus()); tgt.setIgnored(src.isIgnored()); + tgt.setIgnoredOverride(src.isIgnoredOverride()); tgt.setCclEnabled(src.isCclEnabled()); tgt.setCcls(new HashSet<>(src.getCcls())); tgt.setExtensionSlots(new HashSet<>(src.getExtensionSlots())); @@ -115,6 +116,7 @@ private void copyBindings(UserAccount source, UserAccount target) { tgt.setTwitchGameName(src.getTwitchGameName()); tgt.setStatus(src.getStatus()); tgt.setIgnored(src.isIgnored()); + tgt.setIgnoredOverride(src.isIgnoredOverride()); tgt.setCclEnabled(src.isCclEnabled()); tgt.setCcls(new HashSet<>(src.getCcls())); tgt.setExtensionSlots(new HashSet<>(src.getExtensionSlots())); diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/AdminGameMappingService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/AdminGameMappingService.java index d35d26f79..45be8b8c0 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/AdminGameMappingService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/AdminGameMappingService.java @@ -33,8 +33,10 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -48,6 +50,9 @@ * The admin game mappings: an admin sets the Twitch category of a detected game (platform + its ID) for every * streamer. Saving one applies it to the game's AUTO, INCOMPLETE and MAPPED bindings (never to a streamer's MANUAL * choice), deleting one sends its MAPPED bindings back to IGDB, in the same transaction as the mapping itself. + * A mapping can also ignore its game by default (a background app such as Wallpaper Engine), its category then + * optional: turning that on ignores the bindings, turning it off (or deleting the mapping) stops ignoring them — but + * those whose streamer chose ({@code ignoredOverride}). * After the commit, each changed binding is pushed to its streamer's dashboard and, when asked, the channels live * on that game get it on Twitch through the detection's own path * ({@link CategoryChain}, {@link MappingChanged}); a Twitch failure there is logged and counted, the mapping stays. @@ -96,10 +101,34 @@ static GameKey of(Object sourceType, Object sourceId) { } } - /** What a write did in its transaction: the mapping (null once deleted), the bindings changed, the live ones. */ - private record Written(GameMappingDto mapping, List changed, List live) {} + /** + * What a write did in its transaction: the mapping (null once deleted), the bindings changed, those whose ignore + * it switched, the live ones. + */ + private record Written(GameMappingDto mapping, List changed, Set ignoreSwitched, + List live) {} + + /** The bindings a write changed, each once, and those whose ignore it switched. */ + private static final class Changes { + private final Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + private final List changed = new ArrayList<>(); + private final Set ignoreSwitched = Collections.newSetFromMap(new IdentityHashMap<>()); + + void add(GameBinding binding) { + if (seen.add(binding)) changed.add(binding); + } + + void addAll(Collection bindings) { + bindings.forEach(this::add); + } - /** A validated request: the platform, the trimmed ID and category. */ + void ignoreSwitched(GameBinding binding) { + add(binding); + ignoreSwitched.add(binding); + } + } + + /** A validated request: the platform, the trimmed ID and category (both null: none, an ignore-only mapping). */ private record Target(SourceType sourceType, String sourceId, String twitchGameId, String twitchGameName) {} // ---- Listings @@ -144,11 +173,16 @@ public GameMappingResult create(GameMappingRequest request, UserAccount admin) { GameMapping mapping = new GameMapping(); fill(mapping, target, request, admin); GameMapping saved = save(mapping); - List changed = applyToBindings(saved, suggestions); - log.info("Game mapping {} {} created by {} — '{}' (twitchId={}), {} binding(s) updated", + Changes changes = new Changes(); + changes.addAll(applyToBindings(saved, suggestions)); + if (saved.isIgnoredByDefault()) { + applyIgnore(saved.getSourceType(), saved.getSourceId(), true, changes); + } + log.info("Game mapping {} {} created by {} — '{}' (twitchId={}, ignored={}), {} binding(s) updated", saved.getSourceType(), saved.getSourceId(), admin == null ? null : admin.getId(), - saved.getTwitchGameName(), saved.getTwitchGameId(), changed.size()); - return new Written(toDto(saved, admin), changed, List.of()); + saved.getTwitchGameName(), saved.getTwitchGameId(), saved.isIgnoredByDefault(), + changes.changed.size()); + return written(toDto(saved, admin), changes); }); } @@ -159,21 +193,32 @@ public GameMappingResult update(UUID id, GameMappingRequest request, UserAccount GameMapping mapping = existing(id); boolean moved = mapping.getSourceType() != target.sourceType() || !mapping.getSourceId().equals(target.sourceId()); - List changed = new ArrayList<>(); + // What the game's bindings were told before: on another game, nothing + boolean wasIgnored = !moved && mapping.isIgnoredByDefault(); + Changes changes = new Changes(); if (moved) { if (mappingRepository.findBySourceTypeAndSourceId(target.sourceType(), target.sourceId()).isPresent()) { throw duplicate(); } - // Now about another game: the old one's bindings go back to IGDB. - changed.addAll(backToIgdb(mapping.getSourceType(), mapping.getSourceId())); + // Now about another game: the old one's bindings go back to IGDB, and are no longer ignored by it. + changes.addAll(backToIgdb(mapping.getSourceType(), mapping.getSourceId())); + if (mapping.isIgnoredByDefault()) { + applyIgnore(mapping.getSourceType(), mapping.getSourceId(), false, changes); + } } fill(mapping, target, request, admin); GameMapping saved = save(mapping); - changed.addAll(applyToBindings(saved, suggestions)); - log.info("Game mapping {} {} updated by {} — '{}' (twitchId={}), {} binding(s) updated", + // No category any more: the bindings that followed it go back to IGDB + if (!saved.hasCategory()) changes.addAll(backToIgdb(saved.getSourceType(), saved.getSourceId())); + changes.addAll(applyToBindings(saved, suggestions)); + if (saved.isIgnoredByDefault() != wasIgnored) { + applyIgnore(saved.getSourceType(), saved.getSourceId(), saved.isIgnoredByDefault(), changes); + } + log.info("Game mapping {} {} updated by {} — '{}' (twitchId={}, ignored={}), {} binding(s) updated", saved.getSourceType(), saved.getSourceId(), admin == null ? null : admin.getId(), - saved.getTwitchGameName(), saved.getTwitchGameId(), changed.size()); - return new Written(toDto(saved, admin), changed, List.of()); + saved.getTwitchGameName(), saved.getTwitchGameId(), saved.isIgnoredByDefault(), + changes.changed.size()); + return written(toDto(saved, admin), changes); }); } @@ -181,10 +226,14 @@ public GameMappingResult delete(UUID id, boolean applyLive) { return write(applyLive, () -> { GameMapping mapping = existing(id); mappingRepository.delete(mapping); - List changed = backToIgdb(mapping.getSourceType(), mapping.getSourceId()); - log.info("Game mapping {} {} deleted — {} binding(s) back to IGDB", - mapping.getSourceType(), mapping.getSourceId(), changed.size()); - return new Written(null, changed, List.of()); + Changes changes = new Changes(); + changes.addAll(backToIgdb(mapping.getSourceType(), mapping.getSourceId())); + if (mapping.isIgnoredByDefault()) { + applyIgnore(mapping.getSourceType(), mapping.getSourceId(), false, changes); + } + log.info("Game mapping {} {} deleted — {} binding(s) back to IGDB or no longer ignored", + mapping.getSourceType(), mapping.getSourceId(), changes.changed.size()); + return written(null, changes); }); } @@ -195,7 +244,8 @@ public GameMappingResult delete(UUID id, boolean applyLive) { private GameMappingResult write(boolean applyLive, Supplier work) { Written written = transactions.execute(status -> { Written w = work.get(); - return new Written(w.mapping(), w.changed(), applyLive ? liveOnTheirGame(w.changed()) : List.of()); + return new Written(w.mapping(), w.changed(), w.ignoreSwitched(), + applyLive ? liveOnTheirGame(w.changed(), w.ignoreSwitched()) : List.of()); }); Objects.requireNonNull(written); @@ -244,13 +294,20 @@ private GameMappingResult write(boolean applyLive, Supplier work) { return new GameMappingResult(written.mapping(), written.changed().size(), updated, failed); } - /** The changed bindings whose streamer is live and plays their game now (not ignored ones: no-game fallback). */ - private List liveOnTheirGame(List changed) { + private static Written written(GameMappingDto mapping, Changes changes) { + return new Written(mapping, changes.changed, changes.ignoreSwitched, List.of()); + } + + /** + * The changed bindings whose streamer is live and plays their game now: not the ignored ones (no-game fallback + * whatever their category), but those the write just ignored, which go to it. + */ + private List liveOnTheirGame(List changed, Set ignoreSwitched) { if (changed.isEmpty()) return List.of(); Map playing = gameStateService.lastKnownGames(); List live = new ArrayList<>(); for (GameBinding binding : changed) { - if (binding.isIgnored()) continue; + if (binding.isIgnored() && !ignoreSwitched.contains(binding)) continue; DetectedGame game = playing.get(binding.getUser().getId()); if (game == null || game.getSourceType() != binding.getSourceType() || !Objects.equals(game.getSourceId(), binding.getSourceId())) continue; @@ -267,7 +324,10 @@ private List liveOnTheirGame(List changed) { * name) for the CCLs / TWs the mapping keeps from IGDB, so no IGDB call happens while its bindings are locked. */ private BindingService.MappingSuggestions igdbSuggestions(Target target, GameMappingRequest request) { - if (request.ccls() != null && request.tws() != null) return BindingService.MappingSuggestions.NONE; + // An ignore-only mapping leaves the bindings to IGDB; one that replaces both needs none of its suggestions + if (target.twitchGameId() == null || request.ccls() != null && request.tws() != null) { + return BindingService.MappingSuggestions.NONE; + } GameKey game = GameKey.of(target.sourceType(), target.sourceId()); String name = mostFrequentNames(List.of(game)).get(game); return bindingService.igdbSuggestions(new DetectedGame(target.sourceId(), target.sourceType(), name), @@ -305,6 +365,20 @@ private List backToIgdb(SourceType sourceType, String sourceId) { return changed; } + /** + * Ignores the game's bindings (or stops ignoring them) as its mapping says, but MANUAL ones and those whose + * streamer chose; they are saved. The rows are locked first, as in {@link #applyToBindings}. + */ + private void applyIgnore(SourceType sourceType, String sourceId, boolean ignored, Changes changes) { + for (GameBinding binding : bindingRepository.lockAllBySourceTypeAndSourceIdAndStatusIn( + sourceType, sourceId, MAPPABLE)) { + if (bindingService.applyMappingIgnore(binding, ignored)) { + bindingRepository.save(binding); + changes.ignoreSwitched(binding); + } + } + } + private GameMapping save(GameMapping mapping) { try { return mappingRepository.saveAndFlush(mapping); @@ -325,6 +399,7 @@ private static void fill(GameMapping mapping, Target target, GameMappingRequest mapping.setTwsOverridden(request.tws() != null); mapping.getTws().clear(); if (request.tws() != null) mapping.getTws().addAll(request.tws()); + mapping.setIgnoredByDefault(request.ignoredByDefault()); mapping.setUpdatedBy(admin == null ? null : admin.getId()); } @@ -344,6 +419,10 @@ private static Target validate(GameMappingRequest request) { if (sourceId.isEmpty()) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Missing game ID"); String twitchGameId = request.twitchGameId() == null ? "" : request.twitchGameId().trim(); String twitchGameName = request.twitchGameName() == null ? "" : request.twitchGameName().trim(); + if (twitchGameId.isEmpty() && twitchGameName.isEmpty() && request.ignoredByDefault()) { + // Ignored by default, the game needs no category: its bindings keep IGDB's + return new Target(platform, sourceId, null, null); + } if (twitchGameId.isEmpty() || twitchGameName.isEmpty()) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Missing Twitch category"); } @@ -395,7 +474,7 @@ private List toDtos(List mappings, UserAccount admi m.isCclsOverridden() ? Set.copyOf(m.getCcls()) : null, m.isTwsOverridden() ? Set.copyOf(m.getTws()) : null, followers.getOrDefault(key, 0L), m.getUpdatedAt(), - m.getUpdatedBy() == null ? null : logins.get(m.getUpdatedBy())); + m.getUpdatedBy() == null ? null : logins.get(m.getUpdatedBy()), m.isIgnoredByDefault()); }).toList(); } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/BindingService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/BindingService.java index 671d2bf6a..bf32f9700 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/BindingService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/BindingService.java @@ -6,6 +6,7 @@ import fr.enimaloc.catapult.domain.binding.GameBinding; import fr.enimaloc.catapult.domain.binding.GameMapping; import fr.enimaloc.catapult.domain.igdb.IgdbGameDetails; +import fr.enimaloc.catapult.event.TwsMissedSteamEvent; import fr.enimaloc.catapult.getter.DetectedGame; import fr.enimaloc.catapult.repository.binding.GameBindingRepository; import fr.enimaloc.catapult.repository.binding.GameMappingRepository; @@ -18,6 +19,7 @@ import fr.enimaloc.catapult.service.twitch.TwitchService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -44,6 +46,8 @@ public class BindingService { private final GameBindingLookup gameBindingLookup; private final CategoryChain categoryChain; private final ExtensionLayoutApplier extensionLayoutApplier; + /** Tells a binding whose Steam signals were missed (the store rate limited). */ + private final ApplicationEventPublisher events; /** * Applies {@code change} to the user's binding (if it exists), saves it, and applies it when it is the game @@ -62,10 +66,13 @@ private void editBinding(UserAccount user, UUID bindingId, Consumer /** The TWs suggested for a game from its IGDB descriptors, Steam signals (Steam games only) and name. */ private Set suggestTws(String igdbId, DetectedGame game) { + return twResolverService.suggest(twInput(igdbId, game)); + } + + private TwResolverService.SuggestInput twInput(String igdbId, DetectedGame game) { Set descriptorIds = igdbService.fetchDescriptorIds(igdbId); String steamAppId = game.getSourceType() == GameBinding.SourceType.STEAM ? game.getSourceId() : null; - return twResolverService.suggest(new TwResolverService.SuggestInput( - igdbId, descriptorIds, steamAppId, game.getSourceName())); + return new TwResolverService.SuggestInput(igdbId, descriptorIds, steamAppId, game.getSourceName()); } private static DetectedGame detectedOf(GameBinding binding) { @@ -125,26 +132,79 @@ private GameBinding createWithIgdbResolution(UserAccount user, DetectedGame dete binding.setSourceType(detectedGame.getSourceType()); binding.setSourceName(detectedGame.getSourceName()); + // A game an admin mapping ignores by default comes in ignored + binding.setIgnored(mappingOf(detectedGame).map(GameMapping::isIgnoredByDefault).orElse(false)); + log.info("Creating binding for user {} — game '{}'", user.getId(), detectedGame.getSourceName()); return updateWithIgdbResolution(user, detectedGame, binding); } + /** The admin game mapping of a detected game, if any (none without a source ID). */ + private Optional mappingOf(DetectedGame detectedGame) { + return detectedGame.getSourceId() != null + ? gameMappingRepository.findBySourceTypeAndSourceId( + detectedGame.getSourceType(), detectedGame.getSourceId()) + : Optional.empty(); + } + /** - * Resolves the binding's category and saves it: from the admin game mapping of its game when there is one + * Resolves the binding's category and saves it: from the admin game mapping of its game when it sets one * (MAPPED), else from IGDB (AUTO, or INCOMPLETE when IGDB finds no Twitch category). */ private GameBinding updateWithIgdbResolution(UserAccount user, DetectedGame detectedGame, GameBinding binding) { - Optional mapping = detectedGame.getSourceId() != null - ? gameMappingRepository.findBySourceTypeAndSourceId( - detectedGame.getSourceType(), detectedGame.getSourceId()) - : Optional.empty(); + Optional mapping = mappingOf(detectedGame).filter(GameMapping::hasCategory); if (mapping.isPresent()) { applyMapping(binding, mapping.get()); log.info("Binding updated as MAPPED for user {} — game '{}' mapped by an admin to '{}' (twitchId={})", user.getId(), detectedGame.getSourceName(), binding.getTwitchGameName(), binding.getTwitchGameId()); - return gameBindingRepository.save(binding); + return missedSteam(gameBindingRepository.save(binding), mapping.get().isTwsOverridden()); + } + return missedSteam(updateWithoutMapping(user, detectedGame, binding), false); + } + + /** + * A Steam game's binding just resolved while the Steam store is rate limited: its TWs (when the detection chose + * them: not the streamer's, nor an admin's) miss Steam's signals, and are resolved again once it may be called. + */ + private GameBinding missedSteam(GameBinding saved, boolean adminTws) { + if (adminTws || saved.isTwOverride() + || saved.getSourceType() != GameBinding.SourceType.STEAM || saved.getId() == null) { + return saved; + } + twResolverService.steamRateLimitedUntil() + .ifPresent(until -> events.publishEvent(new TwsMissedSteamEvent(saved.getId(), until))); + return saved; + } + + /** + * Resolves again the TWs of a binding whose Steam signals were missed (the store rate limited), when the detection + * still chooses them; saved when they changed. Still limited, it is told again for later. + * + * @return the binding when its TWs changed, to push to its dashboard + */ + @Transactional + public Optional retryTwsAfterSteamLimit(UUID bindingId) { + Optional found = gameBindingRepository.findById(bindingId); + if (found.isEmpty()) return Optional.empty(); + GameBinding binding = found.get(); + boolean adminTws = binding.getStatus() == GameBinding.Status.MAPPED && binding.getSourceId() != null + && gameMappingRepository.findBySourceTypeAndSourceId(binding.getSourceType(), binding.getSourceId()) + .map(GameMapping::isTwsOverridden).orElse(false); + if (binding.isTwOverride() || adminTws || binding.getStatus() == GameBinding.Status.MANUAL) { + return Optional.empty(); + } + Optional igdbId = igdbService.resolveIgdbIdForBinding(binding); + if (igdbId.isEmpty()) return Optional.empty(); + TwResolverService.Suggestion suggestion = twResolverService.suggestReporting(twInput(igdbId.get(), detectedOf(binding))); + if (suggestion.steamLimitedUntil().isPresent()) { + events.publishEvent(new TwsMissedSteamEvent(bindingId, suggestion.steamLimitedUntil().get())); + return Optional.empty(); } - return updateWithoutMapping(user, detectedGame, binding); + if (binding.getTws().equals(suggestion.tws())) return Optional.empty(); + binding.getTws().clear(); + binding.getTws().addAll(suggestion.tws()); + log.info("[TW] binding {} resolved again after the Steam store's rate limit: {}", bindingId, suggestion.tws()); + return Optional.of(gameBindingRepository.save(binding)); } /** @@ -191,7 +251,7 @@ public MappingSuggestions igdbSuggestions(DetectedGame game, boolean ccls, boole * @return whether the binding changed; always false for a MANUAL binding, which is never touched. */ public boolean applyMapping(GameBinding binding, GameMapping mapping, MappingSuggestions suggestions) { - if (binding.getStatus() == GameBinding.Status.MANUAL) return false; + if (binding.getStatus() == GameBinding.Status.MANUAL || !mapping.hasCategory()) return false; Set ccls = mapping.isCclsOverridden() ? Set.copyOf(mapping.getCcls()) : suggestions.ccls(); Set tws = binding.isTwOverride() ? Set.copyOf(binding.getTws()) @@ -214,6 +274,19 @@ public boolean applyMapping(GameBinding binding, GameMapping mapping, MappingSug return changed; } + /** + * Sets whether the binding is ignored as an admin mapping says, without saving it — unless it is MANUAL or its + * streamer chose ({@code ignoredOverride}). + * + * @return whether the binding changed + */ + public boolean applyMappingIgnore(GameBinding binding, boolean ignored) { + if (binding.getStatus() == GameBinding.Status.MANUAL || binding.isIgnoredOverride() + || binding.isIgnored() == ignored) return false; + binding.setIgnored(ignored); + return true; + } + /** Resolves and saves the binding from IGDB alone, its game's mapping ignored — for a deleted mapping. */ @Transactional public GameBinding resolveWithoutMapping(GameBinding binding) { @@ -229,7 +302,8 @@ private GameBinding updateWithoutMapping(UserAccount user, DetectedGame detected String twitchId = igdbGameDetailsService.getDetails(igdbId) .map(IgdbGameDetails::getWebsites) .map(websites -> websites.get("twitch")) - .filter(id -> !id.isBlank()) + // A category id (external_games' uid), not a Twitch website's URL kept under the same key + .filter(BindingService::isTwitchCategoryId) .or(() -> igdbService.findTwitchGameId(igdbId)) .or(() -> twitchService.findCategoryIdByName(user, gameName)) .orElse(null); @@ -264,6 +338,11 @@ private GameBinding updateWithoutMapping(UserAccount user, DetectedGame detected return gameBindingRepository.save(binding); } + /** Twitch's category ids are numeric; anything else (a website URL) is refused as the PATCH's game_id. */ + static boolean isTwitchCategoryId(String id) { + return !id.isEmpty() && id.chars().allMatch(c -> c >= '0' && c <= '9'); + } + private Optional resolveViaIgdb(DetectedGame detectedGame) { return detectedGame.getSourceId() != null ? igdbService.findByExternalAppId(detectedGame.getSourceType(), detectedGame.getSourceId()) @@ -290,6 +369,7 @@ public void updateBinding(UserAccount user, UUID bindingId, String twitchGameId, binding.getTws().clear(); binding.getTws().addAll(tws); binding.setTwOverride(true); + if (binding.isIgnored() != ignored) binding.setIgnoredOverride(true); binding.setIgnored(ignored); if (twitchGameId != null && !twitchGameId.isBlank()) { binding.setStatus(GameBinding.Status.MANUAL); @@ -336,15 +416,17 @@ public void resetTws(UserAccount user, UUID bindingId) { } /** - * Puts the binding back as first detected: resolved anew (category, CCLs, TWs), its toggles on, not ignored, its - * extension layout emptied (pushed to Twitch when it is the configuration in place). + * Puts the binding back as first detected: resolved anew (category, CCLs, TWs), its toggles on, ignored only when + * its game's admin mapping ignores it by default, its extension layout emptied (pushed to Twitch when it is the + * configuration in place). * @throws ResponseStatusException 404 when the binding isn't the user's */ @Transactional public void resetToDefaults(UserAccount user, UUID bindingId) { GameBinding binding = gameBindingRepository.findByIdAndUser(bindingId, user) .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Unknown binding")); - binding.setIgnored(false); + binding.setIgnoredOverride(false); + binding.setIgnored(mappingOf(detectedOf(binding)).map(GameMapping::isIgnoredByDefault).orElse(false)); binding.setCclEnabled(true); binding.setTwEnabled(true); binding.setTitle(null); @@ -387,6 +469,17 @@ public void setTwsForBinding(UserAccount user, UUID bindingId, Set tws) }); } + /** A binding and why each TW is suggested for its game now (nothing when IGDB doesn't know the game). */ + public record TwExplanation(GameBinding binding, TwResolverService.Explanation explanation) {} + + /** As {@link #previewTws}, with the binding and each suggestion's sources; empty when it isn't the user's. */ + public Optional explainTws(UserAccount user, UUID bindingId) { + return gameBindingRepository.findByIdAndUser(bindingId, user).map(binding -> new TwExplanation(binding, + igdbService.resolveIgdbIdForBinding(binding) + .map(igdbId -> twResolverService.explain(twInput(igdbId, detectedOf(binding)))) + .orElseGet(() -> new TwResolverService.Explanation(List.of())))); + } + public Set previewTws(UserAccount user, UUID bindingId) { return gameBindingRepository.findByIdAndUser(bindingId, user) .flatMap(binding -> igdbService.resolveIgdbIdForBinding(binding) @@ -402,7 +495,11 @@ public void setTitle(UserAccount user, UUID bindingId, String title) { @Transactional public void toggleIgnored(UserAccount user, UUID bindingId, boolean ignored) { - editBinding(user, bindingId, binding -> binding.setIgnored(ignored)); + editBinding(user, bindingId, binding -> { + binding.setIgnored(ignored); + // The streamer's choice: an admin mapping no longer changes it + binding.setIgnoredOverride(true); + }); } @Transactional diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwDetailsService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwDetailsService.java new file mode 100644 index 000000000..694a6cd9e --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwDetailsService.java @@ -0,0 +1,85 @@ +package fr.enimaloc.catapult.service.binding; + +import fr.enimaloc.catapult.common.dto.channel.TwDetailsDto; +import fr.enimaloc.catapult.domain.binding.GameBinding; +import fr.enimaloc.catapult.domain.tw.TwDefinition; +import fr.enimaloc.catapult.repository.binding.GameMappingRepository; +import fr.enimaloc.catapult.repository.tw.TwDefinitionRepository; +import fr.enimaloc.catapult.service.tw.TwResolverService; +import fr.enimaloc.catapult.service.tw.TwSeverityRule; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Why a streamer's binding has its TWs: each one's sources as the detection finds them now (recomputed, never + * stored), the streamer's or an admin's choice otherwise, and the DTDD topics their severity kept off. + */ +@Service +@RequiredArgsConstructor +public class TwDetailsService { + + private final BindingService bindingService; + private final GameMappingRepository mappingRepository; + private final TwDefinitionRepository definitions; + + /** The binding's TWs explained; empty when it isn't the user's. */ + public Optional details(fr.enimaloc.catapult.domain.account.UserAccount user, UUID bindingId) { + return bindingService.explainTws(user, bindingId).map(this::details); + } + + private TwDetailsDto details(BindingService.TwExplanation explained) { + GameBinding binding = explained.binding(); + TwResolverService.Explanation explanation = explained.explanation(); + Map> bySource = explanation.signals().stream() + .filter(TwResolverService.Signal::triggers) + .collect(Collectors.groupingBy(TwResolverService.Signal::twId)); + boolean adminChose = binding.getStatus() == GameBinding.Status.MAPPED && binding.getSourceId() != null + && mappingRepository.findBySourceTypeAndSourceId(binding.getSourceType(), binding.getSourceId()) + .map(m -> m.isTwsOverridden()).orElse(false); + Map defs = definitions.findAll().stream() + .collect(Collectors.toMap(TwDefinition::getId, Function.identity(), (a, b) -> a)); + + // The binding's TWs, then the ones the detection suggests that it hasn't + Set ids = new LinkedHashSet<>(binding.getTws()); + ids.addAll(bySource.keySet()); + List tws = new ArrayList<>(); + for (String id : ids) { + boolean active = binding.getTws().contains(id); + List sources = bySource.getOrDefault(id, List.of()); + String origin = !sources.isEmpty() ? "DETECTED" + : binding.isTwOverride() ? "STREAMER" + : adminChose ? "ADMIN" + : "NO_SOURCE"; + // An admin's TWs replace the detection's: those it suggests, the binding doesn't have, aren't turned off + if (!active && adminChose && !binding.isTwOverride()) continue; + TwDefinition def = defs.get(id); + tws.add(new TwDetailsDto.Tw(id, def == null ? id : def.getLabel(), active, origin, + sources.stream().map(s -> source(s, defs)).toList())); + } + tws.sort(Comparator.comparing((TwDetailsDto.Tw t) -> !t.active()) + .thenComparing(t -> defs.containsKey(t.id()) ? defs.get(t.id()).getSortOrder() : Integer.MAX_VALUE) + .thenComparing(TwDetailsDto.Tw::id)); + return new TwDetailsDto(binding.isTwOverride(), tws, + explanation.dropped().stream().map(s -> source(s, defs)).toList()); + } + + private static TwDetailsDto.Source source(TwResolverService.Signal signal, Map defs) { + TwSeverityRule.Verdict v = signal.severity(); + TwDefinition def = defs.get(signal.twId()); + return new TwDetailsDto.Source(signal.twId(), def == null ? signal.twId() : def.getLabel(), + signal.source().name(), signal.value(), v == null ? null + : new TwDetailsDto.Severity(v.reason().name(), v.minSeverity(), v.level(), v.votes(), + v.percentAtOrAbove())); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwSteamRetry.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwSteamRetry.java new file mode 100644 index 000000000..155ca2eb3 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/binding/TwSteamRetry.java @@ -0,0 +1,51 @@ +package fr.enimaloc.catapult.service.binding; + +import fr.enimaloc.catapult.event.TwsMissedSteamEvent; +import fr.enimaloc.catapult.service.notification.ChannelEventPublisher; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionalEventListener; + +import java.time.Duration; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A binding whose TWs missed the Steam store's signals (rate limited) has them resolved again once the store may be + * called — a moment after its Retry-After, never holding up the detection — then pushed to its dashboard. Once per + * binding at a time; still limited by then, it is planned again. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class TwSteamRetry { + + /** After the limit, so the store is surely callable again. */ + static final Duration MARGIN = Duration.ofSeconds(2); + + private final BindingService bindingService; + private final ChannelEventPublisher channelEvents; + private final TaskScheduler scheduler; + private final Set planned = ConcurrentHashMap.newKeySet(); + + /** Once the binding is saved (its transaction committed, or none). */ + @TransactionalEventListener(fallbackExecution = true) + public void onMissedSteam(TwsMissedSteamEvent event) { + if (!planned.add(event.bindingId())) return; + scheduler.schedule(() -> retry(event.bindingId()), event.until().plus(MARGIN)); + log.debug("[TW] binding {} resolved again after the Steam store's limit ({})", event.bindingId(), event.until()); + } + + void retry(UUID bindingId) { + planned.remove(bindingId); + try { + bindingService.retryTwsAfterSteamLimit(bindingId) + .ifPresent(binding -> channelEvents.bindingUpserted(binding.getUser().getId(), binding)); + } catch (RuntimeException e) { + log.warn("[TW] binding {} not resolved again after the Steam store's limit: {}", bindingId, e.getMessage()); + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminService.java new file mode 100644 index 000000000..e191d083f --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminService.java @@ -0,0 +1,257 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddGameCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddKeyQuotaDto; +import fr.enimaloc.catapult.common.dto.admin.DtddSearchCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicSeverityDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; +import fr.enimaloc.catapult.domain.dtdd.DtddApiKeyEntry; +import fr.enimaloc.catapult.domain.dtdd.DtddGameCache; +import fr.enimaloc.catapult.domain.dtdd.DtddSearchCache; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicSeverity; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicsCache; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiKeyRotator; +import fr.enimaloc.catapult.repository.dtdd.DtddApiKeyRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import fr.enimaloc.catapult.service.ApiKeyHasher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * The DTDD caches as the admin sees them: each one paged and searched, the oldest first, with what is past its own + * lifetime ({@code dtdd.cache.topics-ttl-hours}, {@code game-info-ttl-days}, {@code search-ttl-days}); an entry + * fetched again on demand; the keys' quotas and the bulk refetch ({@link DtddCacheRefetchJob}). + */ +@Service +public class DtddCacheAdminService { + + static final int MAX_PAGE_SIZE = 100; + + private final DtddTopicsCacheRepository topicsRepo; + private final DtddGameCacheRepository gameRepo; + private final DtddSearchCacheRepository searchRepo; + private final DtddApiKeyRepository keyRepo; + private final Optional client; + private final Optional rotator; + private final DtddCacheRefetcher refetcher; + private final DtddCacheRefetchJob job; + private final Map lifetimes; + private final Clock clock; + + @Autowired + public DtddCacheAdminService(DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo, DtddApiKeyRepository keyRepo, + Optional client, Optional rotator, + DtddCacheRefetcher refetcher, DtddCacheRefetchJob job, + @Value("${dtdd.cache.topics-ttl-hours:168}") int topicsTtlHours, + @Value("${dtdd.cache.game-info-ttl-days:180}") int gamesTtlDays, + @Value("${dtdd.cache.search-ttl-days:30}") int searchesTtlDays) { + this(topicsRepo, gameRepo, searchRepo, keyRepo, client, rotator, refetcher, job, + Map.of(DtddCacheKind.TOPICS, Duration.ofHours(topicsTtlHours), + DtddCacheKind.GAMES, Duration.ofDays(gamesTtlDays), + DtddCacheKind.SEARCHES, Duration.ofDays(searchesTtlDays)), + Clock.systemUTC()); + } + + DtddCacheAdminService(DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo, DtddApiKeyRepository keyRepo, + Optional client, Optional rotator, + DtddCacheRefetcher refetcher, DtddCacheRefetchJob job, + Map lifetimes, Clock clock) { + this.topicsRepo = topicsRepo; + this.gameRepo = gameRepo; + this.searchRepo = searchRepo; + this.keyRepo = keyRepo; + this.client = client; + this.rotator = rotator; + this.refetcher = refetcher; + this.job = job; + this.lifetimes = Map.copyOf(lifetimes); + this.clock = clock; + } + + public DtddCachesSummaryDto summary() { + return new DtddCachesSummaryDto( + topicsRepo.count(), topicsRepo.countByFetchedAtBefore(staleBefore(DtddCacheKind.TOPICS)), + gameRepo.count(), gameRepo.countByFetchedAtBefore(staleBefore(DtddCacheKind.GAMES)), + searchRepo.count(), searchRepo.countBySearchedAtBefore(staleBefore(DtddCacheKind.SEARCHES)), + hours(DtddCacheKind.TOPICS), hours(DtddCacheKind.GAMES), hours(DtddCacheKind.SEARCHES), + quotas(), job.monthlyReserve(), job.status()); + } + + /** The {@code kind} cache's entries matching {@code q} (none: all), the oldest first. */ + public Page list(DtddCacheKind kind, String q, int page, int size) { + String needle = q == null ? "" : q.trim().toLowerCase(Locale.ROOT); + return switch (kind) { + case TOPICS -> topics(needle, pageable(page, size, "fetchedAt")); + case GAMES -> gameRepo.search(needle, pageable(page, size, "fetchedAt")).map(this::game); + case SEARCHES -> searchRepo.search(needle, pageable(page, size, "searchedAt")).map(this::search); + }; + } + + /** + * Fetches the {@code kind} entry {@code key} again (a title's ID, a search's query) and gives it as now cached. + * + * @throws ResponseStatusException 503 while DTDD is off, 404 for an entry not cached, 502 when DTDD gave + * nothing (an unknown title, no usable key, DTDD unavailable) + */ + public Object refetch(DtddCacheKind kind, String key) { + DtddApiClient dtdd = client.orElseThrow( + () -> new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "DTDD is disabled")); + String normalized = key == null ? "" : key.trim(); + if (kind == DtddCacheKind.SEARCHES) normalized = normalized.toLowerCase(Locale.ROOT); + if (!cached(kind, normalized)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Not in the DTDD cache"); + } + if (!refetcher.refetch(dtdd, kind, normalized)) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, "DTDD gave nothing for it"); + } + return switch (kind) { + case TOPICS -> topic(topicsRepo.findById(Long.parseLong(normalized)).orElseThrow(), names( + List.of(Long.parseLong(normalized)))); + case GAMES -> game(gameRepo.findById(Long.parseLong(normalized)).orElseThrow()); + case SEARCHES -> search(searchRepo.findById(normalized).orElseThrow()); + }; + } + + /** + * Everything cached about the {@code kind} entry {@code key}: a title (from its topics or its record) with both + * and the searches that found it; a search with the titles it found. + * + * @throws ResponseStatusException 404 for an entry not cached, 400 for a title key that isn't an ID + */ + public DtddCacheEntryDetailDto detail(DtddCacheKind kind, String key) { + String normalized = key == null ? "" : key.trim(); + if (kind == DtddCacheKind.SEARCHES) normalized = normalized.toLowerCase(Locale.ROOT); + if (!cached(kind, normalized)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Not in the DTDD cache"); + } + if (kind == DtddCacheKind.SEARCHES) { + DtddSearchCache found = searchRepo.findById(normalized).orElseThrow(); + List ids = found.getDtddIds() == null ? List.of() : found.getDtddIds(); + Map names = names(ids); + Set withTopics = new HashSet<>(); + topicsRepo.findAllById(ids).forEach(t -> withTopics.add(t.getDtddId())); + return new DtddCacheEntryDetailDto(null, normalized, null, null, List.of(search(found)), + ids.stream().map(id -> new DtddCacheEntryDetailDto.FoundTitle(id, names.get(id), + withTopics.contains(id))).toList()); + } + long id = Long.parseLong(normalized); + Map names = names(List.of(id)); + return new DtddCacheEntryDetailDto(id, null, + gameRepo.findById(id).map(this::game).orElse(null), + topicsRepo.findById(id).map(t -> topic(t, names)).orElse(null), + searchRepo.findAllFinding(id).stream().map(this::search).toList(), + List.of()); + } + + private boolean cached(DtddCacheKind kind, String key) { + if (kind == DtddCacheKind.SEARCHES) return searchRepo.existsById(key); + long id; + try { + id = Long.parseLong(key); + } catch (NumberFormatException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Not a DTDD title ID"); + } + return kind == DtddCacheKind.TOPICS ? topicsRepo.existsById(id) : gameRepo.existsById(id); + } + + private Page topics(String needle, Pageable pageable) { + Page found = topicsRepo.search(needle, pageable); + Map names = names(found.getContent().stream().map(DtddTopicsCache::getDtddId).toList()); + return found.map(t -> topic(t, names)); + } + + /** The cached names of these titles. */ + private Map names(List ids) { + Map names = new HashMap<>(); + gameRepo.findAllById(ids).forEach(g -> names.put(g.getDtddId(), g.getName())); + return names; + } + + private DtddTopicsCacheEntryDto topic(DtddTopicsCache t, Map names) { + Map severities = null; + if (t.getSeverities() != null) { + severities = new LinkedHashMap<>(); + for (Map.Entry e : t.getSeverities().entrySet()) { + DtddTopicSeverity s = e.getValue(); + severities.put(e.getKey(), new DtddTopicSeverityDto(s.levels(), s.counts(), s.percents(), s.total(), + s.verified())); + } + } + return new DtddTopicsCacheEntryDto(t.getDtddId(), names.get(t.getDtddId()), t.getYesTopics(), + t.getNoTopics(), t.getMostlyTopics(), t.getFetchedAt(), stale(DtddCacheKind.TOPICS, t.getFetchedAt()), + severities); + } + + private DtddGameCacheEntryDto game(DtddGameCache g) { + return new DtddGameCacheEntryDto(g.getDtddId(), g.getName(), g.getMediaType(), g.getPosterUrl(), + g.getFetchedAt(), stale(DtddCacheKind.GAMES, g.getFetchedAt())); + } + + private DtddSearchCacheEntryDto search(DtddSearchCache s) { + return new DtddSearchCacheEntryDto(s.getQueryNormalized(), s.getDtddIds(), s.getSearchedAt(), + stale(DtddCacheKind.SEARCHES, s.getSearchedAt())); + } + + /** Each pooled key's last known quota and rest, masked as the keys tab shows them. */ + private List quotas() { + if (rotator.isEmpty()) return List.of(); + Map quotas = rotator.get().getQuotas(); + Map blockedUntil = rotator.get().getKeyBlockedUntil(); + long now = clock.millis(); + List out = new ArrayList<>(); + for (DtddApiKeyEntry entry : keyRepo.findByExclusiveFalse()) { + String key = entry.getApiKey(); + DtddApiKeyRotator.Quota quota = quotas.get(key); + long until = blockedUntil.getOrDefault(key, 0L); + out.add(new DtddKeyQuotaDto(ApiKeyHasher.id(key), DtddKeyPoolService.mask(key), + quota == null ? null : quota.remainingMinute(), quota == null ? null : quota.remainingMonth(), + quota == null ? null : Instant.ofEpochMilli(quota.at()), + until > now ? Instant.ofEpochMilli(until) : null)); + } + return out; + } + + private Instant staleBefore(DtddCacheKind kind) { + return clock.instant().minus(lifetimes.get(kind)); + } + + private boolean stale(DtddCacheKind kind, Instant at) { + return at == null || at.isBefore(staleBefore(kind)); + } + + private int hours(DtddCacheKind kind) { + return (int) lifetimes.get(kind).toHours(); + } + + private static Pageable pageable(int page, int size, String oldestFirst) { + int bounded = size <= 0 ? 20 : Math.min(size, MAX_PAGE_SIZE); + return PageRequest.of(Math.max(page, 0), bounded, Sort.by(Sort.Direction.ASC, oldestFirst)); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheKind.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheKind.java new file mode 100644 index 000000000..38734ce12 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheKind.java @@ -0,0 +1,20 @@ +package fr.enimaloc.catapult.service.dtdd; + +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; + +import java.util.Locale; + +/** The DTDD caches the admin sees: a title's topics, the titles found, the searches made. */ +public enum DtddCacheKind { + TOPICS, GAMES, SEARCHES; + + /** The kind a route names ({@code topics}, {@code games}, {@code searches}); 404 for another. */ + public static DtddCacheKind of(String path) { + try { + return valueOf(path.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException | NullPointerException e) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Unknown DTDD cache"); + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJob.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJob.java new file mode 100644 index 000000000..ba1b1374c --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJob.java @@ -0,0 +1,220 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiKeyRotator; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * The admin's bulk refetch of every cached DTDD entry, the oldest first, one at a time on this instance (its state + * lives here: a restart ends it). It keeps to DTDD's rate limit: under each key's requests per minute + * ({@code dtdd.refetch.per-minute}, shared over the keys it may use), waiting while every such key rests, and only + * with keys that keep more than {@code dtdd.refetch.monthly-reserve} requests of their month — the detection's. + * Once none does, it stops (QUOTA). It can be cancelled. + */ +@Slf4j +@Component +public class DtddCacheRefetchJob { + + /** Sleeps that long at most at once, so a cancel is seen soon. */ + private static final long SLEEP_SLICE_MS = 1_000L; + + /** Waits {@code millis}; interrupted, the job ends as cancelled. */ + @FunctionalInterface + interface Sleeper { + void sleep(long millis) throws InterruptedException; + } + + private record Target(DtddCacheKind kind, String key, Instant at) {} + + private final Optional client; + private final Optional rotator; + private final DtddTopicsCacheRepository topicsRepo; + private final DtddGameCacheRepository gameRepo; + private final DtddSearchCacheRepository searchRepo; + private final DtddCacheRefetcher refetcher; + private final int monthlyReserve; + private final int perMinute; + private final Executor executor; + private final Sleeper sleeper; + private final Clock clock; + + private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicReference status = new AtomicReference<>(DtddRefetchJobDto.IDLE); + private volatile boolean cancelled; + + @Autowired + public DtddCacheRefetchJob(Optional client, Optional rotator, + DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo, DtddCacheRefetcher refetcher, + @Value("${dtdd.refetch.monthly-reserve:500}") int monthlyReserve, + @Value("${dtdd.refetch.per-minute:28}") int perMinute) { + this(client, rotator, topicsRepo, gameRepo, searchRepo, refetcher, monthlyReserve, perMinute, + task -> Thread.ofVirtual().name("dtdd-refetch").start(task), Thread::sleep, Clock.systemUTC()); + } + + DtddCacheRefetchJob(Optional client, Optional rotator, + DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo, DtddCacheRefetcher refetcher, int monthlyReserve, + int perMinute, Executor executor, Sleeper sleeper, Clock clock) { + this.client = client; + this.rotator = rotator; + this.topicsRepo = topicsRepo; + this.gameRepo = gameRepo; + this.searchRepo = searchRepo; + this.refetcher = refetcher; + this.monthlyReserve = monthlyReserve; + this.perMinute = Math.max(1, perMinute); + this.executor = executor; + this.sleeper = sleeper; + this.clock = clock; + } + + public int monthlyReserve() { + return monthlyReserve; + } + + public DtddRefetchJobDto status() { + return status.get(); + } + + /** + * Starts the refetch of every cached entry, the oldest first. + * + * @throws ResponseStatusException 503 while DTDD is off, 409 while one runs already + */ + public DtddRefetchJobDto start() { + if (client.isEmpty() || rotator.isEmpty()) { + throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "DTDD is disabled"); + } + if (!running.compareAndSet(false, true)) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "A DTDD refetch is running already"); + } + try { + cancelled = false; + List targets = targets(); + status.set(new DtddRefetchJobDto("RUNNING", targets.size(), 0, 0, null, clock.instant(), null, null)); + log.info("DTDD caches refetch started: {} entries, monthly reserve {} per key", targets.size(), + monthlyReserve); + executor.execute(() -> run(targets)); + } catch (RuntimeException e) { + running.set(false); + throw e; + } + return status.get(); + } + + /** Asks the running refetch to stop after its current entry; nothing when none runs. */ + public DtddRefetchJobDto cancel() { + if (running.get()) cancelled = true; + return status.get(); + } + + private List targets() { + List targets = new ArrayList<>(); + for (Object[] row : topicsRepo.findAllAges()) { + targets.add(new Target(DtddCacheKind.TOPICS, String.valueOf(row[0]), (Instant) row[1])); + } + for (Object[] row : gameRepo.findAllAges()) { + targets.add(new Target(DtddCacheKind.GAMES, String.valueOf(row[0]), (Instant) row[1])); + } + for (Object[] row : searchRepo.findAllAges()) { + targets.add(new Target(DtddCacheKind.SEARCHES, (String) row[0], (Instant) row[1])); + } + targets.sort(Comparator.comparing(Target::at, Comparator.nullsFirst(Comparator.naturalOrder()))); + return targets; + } + + private void run(List targets) { + DtddApiKeyRotator keys = rotator.orElseThrow(); + DtddApiClient bulk = client.orElseThrow().reserving(monthlyReserve); + int done = 0; + int failed = 0; + try { + for (Target target : targets) { + // A key that keeps the reserve, once one is usable + while (true) { + if (cancelled) { + finish("CANCELLED", null, done, failed); + return; + } + Optional usableAt = keys.nextUsableAt(monthlyReserve); + if (usableAt.isEmpty()) { + boolean noKey = keys.availableKeyCount() + keys.blockedKeyCount() == 0; + finish("STOPPED", noKey ? "NO_KEY" : "QUOTA", done, failed); + return; + } + long waitMs = usableAt.get() - clock.millis(); + if (waitMs <= 0) break; + progress(done, failed, Instant.ofEpochMilli(usableAt.get())); + pause(waitMs); + } + boolean refetched; + try { + refetched = refetcher.refetch(bulk, target.kind(), target.key()); + } catch (RuntimeException e) { + log.warn("DTDD refetch of {} {} failed: {}", target.kind(), target.key(), e.getMessage()); + refetched = false; + } + done++; + if (!refetched) failed++; + progress(done, failed, null); + // Under each key's requests per minute, shared over the keys it may use + pause(60_000L / ((long) perMinute * Math.max(1, keys.keysAbove(monthlyReserve)))); + } + finish("DONE", null, done, failed); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + finish("CANCELLED", null, done, failed); + } catch (RuntimeException e) { + log.warn("DTDD caches refetch ended on an error: {}", e.getMessage(), e); + finish("STOPPED", "ERROR", done, failed); + } finally { + // Whatever ended it, another one may start + running.set(false); + } + } + + /** Sleeps {@code millis} in slices, back early once cancelled. */ + private void pause(long millis) throws InterruptedException { + long left = millis; + while (left > 0 && !cancelled) { + long slice = Math.min(left, SLEEP_SLICE_MS); + sleeper.sleep(slice); + left -= slice; + } + } + + private void progress(int done, int failed, Instant waitingUntil) { + DtddRefetchJobDto current = status.get(); + status.set(new DtddRefetchJobDto("RUNNING", current.total(), done, failed, null, current.startedAt(), null, + waitingUntil)); + } + + private void finish(String state, String stopReason, int done, int failed) { + DtddRefetchJobDto current = status.get(); + status.set(new DtddRefetchJobDto(state, current.total(), done, failed, stopReason, current.startedAt(), + clock.instant(), null)); + log.info("DTDD caches refetch {}{}: {}/{} done, {} without an answer", state, + stopReason == null ? "" : " (" + stopReason + ")", done, current.total(), failed); + running.set(false); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcher.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcher.java new file mode 100644 index 000000000..3b006a662 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcher.java @@ -0,0 +1,107 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.domain.dtdd.DtddGameCache; +import fr.enimaloc.catapult.domain.dtdd.DtddSearchCache; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicSeverity; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicsCache; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSearchResult; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Fetches one cached DTDD entry again and saves what DTDD answers, as the cache's own readers would have: a title's + * topics, a title (its item), a search (and the titles it finds). Nothing is changed when DTDD answers nothing. + */ +@Component +public class DtddCacheRefetcher { + + private final DtddTopicsCacheRepository topicsRepo; + private final DtddGameCacheRepository gameRepo; + private final DtddSearchCacheRepository searchRepo; + private final Clock clock; + + @Autowired + public DtddCacheRefetcher(DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo) { + this(topicsRepo, gameRepo, searchRepo, Clock.systemUTC()); + } + + DtddCacheRefetcher(DtddTopicsCacheRepository topicsRepo, DtddGameCacheRepository gameRepo, + DtddSearchCacheRepository searchRepo, Clock clock) { + this.topicsRepo = topicsRepo; + this.gameRepo = gameRepo; + this.searchRepo = searchRepo; + this.clock = clock; + } + + /** + * Refetches the {@code kind} entry {@code key} (a title's ID, or a search's query) through {@code client}. + * + * @return whether DTDD answered and the entry was saved; false when it answered nothing (unknown title, no key + * usable, DTDD unavailable) + */ + public boolean refetch(DtddApiClient client, DtddCacheKind kind, String key) { + Instant now = clock.instant(); + return switch (kind) { + case TOPICS -> client.fetchTopics(id(key)).map(topics -> { + topicsRepo.save(new DtddTopicsCache(id(key), topics.yesTopics(), topics.noTopics(), + topics.mostlyTopics(), now, severities(topics))); + return true; + }).orElse(false); + case GAMES -> client.item(id(key)).map(item -> { + DtddGameCache game = gameRepo.findById(id(key)).orElseGet(() -> { + DtddGameCache created = new DtddGameCache(); + created.setDtddId(id(key)); + created.setSlug("https://www.doesthedogdie.com/media/" + id(key)); + return created; + }); + if (item.name() != null) game.setName(item.name()); + if (item.itemTypeName() != null) game.setMediaType(item.itemTypeName()); + // The search gives a poster URL; the item only a full one is taken from + if (item.posterImage() != null && item.posterImage().startsWith("http")) { + game.setPosterUrl(item.posterImage()); + } + if (game.getName() == null) return false; + game.setFetchedAt(now); + gameRepo.save(game); + return true; + }).orElse(false); + // As the searches were made: the video games only + case SEARCHES -> client.search(key).map(results -> { + for (DtddSearchResult r : results) { + gameRepo.save(new DtddGameCache(r.dtddId(), r.name(), r.slug(), r.mediaType(), r.posterUrl(), now)); + } + searchRepo.save(new DtddSearchCache(key, results.stream().map(DtddSearchResult::dtddId).toList(), now)); + return true; + }).orElse(false); + }; + } + + /** The topics' severities as the cache keeps them, by topic name. */ + static Map severities(DtddApiClient.DtddTopics topics) { + Map kept = new LinkedHashMap<>(); + if (topics.severities() == null) return kept; + topics.severities().forEach((topic, s) -> kept.put(topic, + new DtddTopicSeverity(s.levels(), s.counts(), s.percents(), s.total(), s.verified()))); + return kept; + } + + private static long id(String key) { + try { + return Long.parseLong(key.trim()); + } catch (NumberFormatException | NullPointerException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Not a DTDD title ID"); + } + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddKeyPoolService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddKeyPoolService.java index 8acd2602f..2cabf1d0c 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddKeyPoolService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddKeyPoolService.java @@ -34,7 +34,7 @@ public DtddKeyPoolService(DtddApiKeyRepository repository, this.events = events; } - private static String mask(String key) { + static String mask(String key) { return key.length() > 8 ? key.substring(0, 4) + "
" + key.substring(key.length() - 4) : "
"; diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddService.java index f2e5abfce..397c253de 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddService.java @@ -17,6 +17,8 @@ import java.time.Duration; import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Optional; @Slf4j @@ -46,7 +48,7 @@ public Optional getTopics(long dtddId) { if (cached.isPresent()) { DtddTopicsCache c = cached.get(); if (!isFresh(c.getFetchedAt())) refreshAsync(dtddId); - return Optional.of(new DtddTopics(c.getYesTopics(), c.getNoTopics(), c.getMostlyTopics())); + return Optional.of(new DtddTopics(c.getYesTopics(), c.getNoTopics(), c.getMostlyTopics(), severities(c))); } Optional remote = apiClient.fetchTopics(dtddId); @@ -65,7 +67,16 @@ void refreshAsync(long dtddId) { private void persist(long dtddId, DtddTopics t) { topicsRepo.save(new DtddTopicsCache( - dtddId, t.yesTopics(), t.noTopics(), t.mostlyTopics(), Instant.now())); + dtddId, t.yesTopics(), t.noTopics(), t.mostlyTopics(), Instant.now(), DtddCacheRefetcher.severities(t))); + } + + /** The cached severities as DTDD gives them, by topic name; null when fetched before they were kept. */ + static Map severities(DtddTopicsCache cached) { + if (cached.getSeverities() == null) return null; + Map out = new LinkedHashMap<>(); + cached.getSeverities().forEach((topic, s) -> out.put(topic, + new DtddApiClient.DtddSeverity(s.levels(), s.counts(), s.percents(), s.total(), s.verified()))); + return out; } private boolean isFresh(Instant ts) { diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddSignalService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddSignalService.java index 17cd69e70..caa20bd47 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddSignalService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/dtdd/DtddSignalService.java @@ -1,10 +1,12 @@ package fr.enimaloc.catapult.service.dtdd; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSeverity; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; import org.springframework.stereotype.Service; import java.util.HashSet; +import java.util.Map; import java.util.Set; @Service @@ -14,13 +16,26 @@ public class DtddSignalService { private final DtddService dtddService; + /** + * A game's DTDD topics marked yes or mostly, with each one's severity by its name: {@code severities} null when + * they weren't cached (fetched before they were kept), a topic without levels absent from it. + */ + public record TopicSignals(Set topics, Map severities) { + public static final TopicSignals NONE = new TopicSignals(Set.of(), Map.of()); + } + /** Returns the union of DTDD topics marked yes and mostly for the given game. */ public Set getYesMostlyTopics(String igdbId, String name) { + return getYesMostlySignals(igdbId, name).topics(); + } + + /** As {@link #getYesMostlyTopics}, with the topics' severities. */ + public TopicSignals getYesMostlySignals(String igdbId, String name) { return dtddService.getTopics(igdbId, name).map(t -> { Set out = new HashSet<>(); if (t.yesTopics() != null) out.addAll(t.yesTopics()); if (t.mostlyTopics() != null) out.addAll(t.mostlyTopics()); - return out; - }).orElse(Set.of()); + return new TopicSignals(out, t.severities()); + }).orElse(TopicSignals.NONE); } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreService.java index d1832dd86..2aa107faf 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreService.java @@ -27,6 +27,14 @@ public static SteamTwSignals empty() { Map fetchTwSignals(Collection appIds); + /** + * Until when the Steam store asked to be left alone (a 429): it isn't called meanwhile, an app it would have + * answered for answering nothing. Empty while it may be called. + */ + default Optional rateLimitedUntil() { + return Optional.empty(); + } + /** * Store-page short description for the given app, in the given locale (mapped to Steam's * own language names internally — Steam does not accept ISO codes). Empty when Steam has diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceImpl.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceImpl.java index 83171f1b0..1d5e582ed 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceImpl.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceImpl.java @@ -2,14 +2,21 @@ import fr.enimaloc.catapult.domain.steam.SteamAppParentEntry; import fr.enimaloc.catapult.getter.steam.SteamPlaytestRedirectResolver; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimiter; +import fr.enimaloc.catapult.ratelimit.RateLimitPriority; import fr.enimaloc.catapult.repository.steam.SteamAppParentRepository; import fr.enimaloc.catapult.service.metrics.ExternalApiObservations; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClient; +import java.time.Clock; import java.time.Instant; import java.util.*; @@ -18,7 +25,9 @@ @RequiredArgsConstructor public class SteamStoreServiceImpl implements SteamStoreService { - private static final String APP_DETAILS_URL = "https://store.steampowered.com/api/appdetails"; + private static final String STORE_HOST = "store.steampowered.com"; + private static final String APP_DETAILS_PATH = "/api/appdetails"; + private static final String APP_DETAILS_URL = "https://" + STORE_HOST + APP_DETAILS_PATH; private static final Map> KEYWORDS = Map.of( "ViolentGraphic", Set.of("blood", "gore", "violence", "violent", "killing", "combat", "death", "injury"), @@ -32,9 +41,51 @@ public class SteamStoreServiceImpl implements SteamStoreService { private final ExternalApiObservations apiObservations; private final SteamAppParentRepository steamAppParentRepository; private final SteamPlaytestRedirectResolver redirectResolver; + private final LearnedRateLimiter rateLimiter; private static final java.time.Duration PARENT_CACHE_TTL = java.time.Duration.ofDays(30); + @Setter(AccessLevel.PACKAGE) + private Clock clock = Clock.systemUTC(); + /** Until when the learned rate limit refused the store's calls without sending them (live calls don't wait). */ + private volatile Instant refusedUntil; + /** How long a 429 nothing learned from (the learning off) leaves the store alone without a Retry-After. */ + private static final java.time.Duration UNBLOCKED_429_WAIT = java.time.Duration.ofMinutes(1); + + /** + * Until when the store's {@code appdetails} isn't called: its learned rate limit blocks it after a 429, or + * refused a live call without sending it. + */ + @Override + public Optional rateLimitedUntil() { + Instant now = clock.instant(); + Instant refused = refusedUntil; + Optional blocked = rateLimiter.blockedUntil(STORE_HOST, APP_DETAILS_PATH); + if (refused == null || !refused.isAfter(now)) return blocked; + return blocked.filter(b -> b.isAfter(refused)).or(() -> Optional.of(refused)); + } + + /** + * A 429 on {@code appdetails} ({@code appDetails}; another route's never stops it): a live call the learned rate + * limit refused (no call sent) tells until when, bulk work it gave up on (cancelled) doesn't. A real one is + * already learned; while nothing blocks the route (the learning off), it is left alone for its Retry-After, else + * a minute, so the TWs resolved meanwhile know Steam's signals were missed. + */ + private void onRateLimited(HttpClientErrorException.TooManyRequests e, boolean appDetails) { + HttpHeaders headers = e.getResponseHeaders(); + boolean local = headers != null && headers.containsHeader(LearnedRateLimiter.LOCAL_REFUSAL_HEADER); + if (!local) log.warn("Steam store rate limited (429): left alone until {}", rateLimitedUntil().orElse(null)); + if (!appDetails) return; + if (local && RateLimitPriority.current() != RateLimitPriority.LIVE) return; + if (!local && rateLimiter.blockedUntil(STORE_HOST, APP_DETAILS_PATH).isPresent()) return; + Instant now = clock.instant(); + java.time.Duration wait = LearnedRateLimiter.retryAfter(headers == null ? null + : headers.getFirst(HttpHeaders.RETRY_AFTER), now).orElse(UNBLOCKED_429_WAIT); + Instant until = now.plus(wait); + Instant refused = refusedUntil; + if (refused == null || until.isAfter(refused)) refusedUntil = until; + } + /** One app's entry in an {@code appdetails} response, keyed by appId. */ record AppDetailsEntry(boolean success, SteamStorePage data) {} @@ -207,14 +258,21 @@ public Optional fetchData(String appId, Locale locale, boolean r .map(ResolvedParentApp::appId) .orElse(appId) : appId; SteamLanguage lang = SteamLanguage.fromLocale(locale); + if (rateLimitedUntil().isPresent()) { + log.debug("Steam store left alone (rate limited): no appdetails for appId={}", appId); + return Optional.empty(); + } try { Map response = restClient.get() - .uri(APP_DETAILS_URL + "?appids=" + effectiveAppId + "&l=" + lang) + .uri(APP_DETAILS_URL + "?appids={appId}&l={lang}", effectiveAppId, lang) .retrieve() .body(new ParameterizedTypeReference<>() {}); AppDetailsEntry entry = response == null ? null : response.get(effectiveAppId); if (entry == null || !entry.success()) return Optional.empty(); return Optional.ofNullable(entry.data()); + } catch (HttpClientErrorException.TooManyRequests e) { + onRateLimited(e, true); + return Optional.empty(); } catch (Exception e) { log.warn("Steam fetchData failed for appId={}: {}", appId, e.getMessage(), e); return Optional.empty(); @@ -229,13 +287,17 @@ public Optional fetchIADisclosure(String appId, Locale locale) { .map(ResolvedParentApp::appId) .orElse(appId); SteamLanguage lang = SteamLanguage.fromLocale(locale); + if (rateLimitedUntil().isPresent()) return Optional.empty(); try { String response = restClient.get() - .uri("https://store.steampowered.com/app/" + effectiveAppId + "?l=" + lang) + .uri("https://" + STORE_HOST + "/app/{appId}?l={lang}", effectiveAppId, lang) .retrieve() .body(String.class); if (response == null || response.isBlank()) return Optional.empty(); return Optional.of(response); + } catch (HttpClientErrorException.TooManyRequests e) { + onRateLimited(e, false); + return Optional.empty(); } catch (Exception e) { log.warn("Steam fetchIADisclosure failed for appId={}: {}", appId, e.getMessage(), e); return Optional.empty(); diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/title/TitleHelpSections.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/title/TitleHelpSections.java new file mode 100644 index 000000000..d944db360 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/title/TitleHelpSections.java @@ -0,0 +1,34 @@ +package fr.enimaloc.catapult.service.title; + +import fr.enimaloc.catapult.repository.account.UserAccountRepository; +import fr.enimaloc.catapult.service.drops.DropsGate; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.UUID; + +/** + * Which parts of the title's help an account is offered, as its title fields are: the conditions (experiment + * stream.title.conditions), and the drops with them (experiment drops). Read-only: the settings page enrols. + */ +@Service +@RequiredArgsConstructor +public class TitleHelpSections { + + /** The help's parts offered; the drops never without the conditions, as the title reads them. */ + public record Offered(boolean conditions, boolean drops) { + public static final Offered NONE = new Offered(false, false); + } + + private final UserAccountRepository accounts; + private final StreamTitleService streamTitleService; + private final DropsGate dropsGate; + + /** The parts offered to the account {@code accountId}; none for an unknown one. */ + public Offered forAccount(UUID accountId) { + return accounts.findById(accountId).map(account -> { + boolean conditions = streamTitleService.conditionsAvailableReadOnly(account); + return new Offered(conditions, conditions && dropsGate.isAvailableReadOnly(account)); + }).orElse(Offered.NONE); + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/AdminTwService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/AdminTwService.java index 5880d6950..6307fa0f9 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/AdminTwService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/AdminTwService.java @@ -31,6 +31,7 @@ public class AdminTwService { private static final Pattern SLUG = Pattern.compile("^[a-z0-9_]{1,40}$"); private static final int LABEL_MAX = 80; private static final int TOPIC_MAX = 200; + private static final int MIN_SEVERITY_MAX = 40; private static final int KEYWORD_MAX = 50; private static final Set STEAM_IDS = Set.of(1, 2, 3, 4, 5); @@ -89,19 +90,44 @@ public void delete(String id) { @Transactional public void replaceDtddTopics(String twId, Set topics) { + replaceDtddTopics(twId, topics, Map.of()); + } + + /** + * Replaces the TW's DTDD topics, each with the least severe level of it that turns the TW on + * ({@code minSeverities}, by topic as given; absent or blank: any level). + */ + @Transactional + public void replaceDtddTopics(String twId, Set topics, Map minSeverities) { requireExists(twId); dtddRepo.deleteAllByTwId(twId); + Set seen = new HashSet<>(); for (String t : topics) { String trimmed = Objects.requireNonNull(t).trim(); if (trimmed.isEmpty() || trimmed.length() > TOPIC_MAX) continue; + if (!seen.add(trimmed.toLowerCase(Locale.ROOT))) continue; + String min = minSeverities == null ? null : minSeverities.get(t); + min = min == null || min.isBlank() ? null : min.trim(); + if (min != null && min.length() > MIN_SEVERITY_MAX) { + throw new IllegalArgumentException("Severity level too long: " + min); + } TwDtddTopicMapping m = new TwDtddTopicMapping(); m.setTwId(twId); m.setDtddTopicName(trimmed); + m.setMinSeverity(min); dtddRepo.save(m); } definitionsChanged(); } + /** The TW's DTDD topics with their thresholds, by name. */ + public List listDtddTopics(String twId) { + requireExists(twId); + return dtddRepo.findAllByTwId(twId).stream() + .sorted(Comparator.comparing(m -> m.getDtddTopicName().toLowerCase(Locale.ROOT))) + .toList(); + } + @Transactional public void replaceIgdbDescriptors(String twId, Set descriptorIds) { requireExists(twId); diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwBackfillService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwBackfillService.java index bc6c4f99a..1e9f8c79d 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwBackfillService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwBackfillService.java @@ -1,14 +1,17 @@ package fr.enimaloc.catapult.service.tw; -import fr.enimaloc.catapult.service.debug.DebugContext; +import fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto; import fr.enimaloc.catapult.domain.binding.GameBinding; import fr.enimaloc.catapult.domain.binding.GameMapping; import fr.enimaloc.catapult.domain.config.AppState; +import fr.enimaloc.catapult.ratelimit.RateLimitPriority; import fr.enimaloc.catapult.repository.binding.GameBindingRepository; import fr.enimaloc.catapult.repository.binding.GameMappingRepository; import fr.enimaloc.catapult.repository.config.AppStateRepository; +import fr.enimaloc.catapult.service.debug.DebugContext; import fr.enimaloc.catapult.service.igdb.IgdbService; import io.micrometer.core.instrument.MeterRegistry; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -18,12 +21,20 @@ import org.springframework.context.event.EventListener; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import java.time.Clock; +import java.time.Duration; import java.time.Instant; +import java.util.Locale; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; @Slf4j @Service @@ -31,6 +42,20 @@ @RequiredArgsConstructor public class TwBackfillService { + /** The recalculations: the start-up backfill, and the admin's three rebuilds. */ + public enum Mode { BACKFILL, NON_OVERRIDDEN, UNTOUCHED, FORCE_PINNED } + + /** How many times a binding waits for the Steam store's rate limit before it counts as failed. */ + static final int MAX_STEAM_WAITS = 10; + /** Sleeps that long at most at once, so a cancel is seen soon. */ + private static final long SLEEP_SLICE_MS = 1_000L; + + /** Waits {@code millis}; interrupted, the recalculation ends as cancelled. */ + @FunctionalInterface + interface Sleeper { + void sleep(long millis) throws InterruptedException; + } + private final AppStateRepository appStateRepo; private final GameBindingRepository bindingRepo; private final GameMappingRepository mappingRepo; @@ -46,6 +71,36 @@ public class TwBackfillService { @Value("${tw.backfill.batch-size:50}") int batchSize; + @Setter(AccessLevel.PACKAGE) + private Sleeper sleeper = Thread::sleep; + @Setter(AccessLevel.PACKAGE) + private Clock clock = Clock.systemUTC(); + + private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicReference status = new AtomicReference<>(TwRebuildStatusDto.IDLE); + private volatile boolean cancelled; + + public TwRebuildStatusDto status() { + return status.get(); + } + + /** Asks the running recalculation to stop after its current binding; nothing when none runs. */ + public TwRebuildStatusDto cancel() { + if (running.get()) cancelled = true; + return status.get(); + } + + /** + * Starts an admin rebuild in the background, its progress in {@link #status()}. + * + * @throws ResponseStatusException 409 while a recalculation runs already + */ + public TwRebuildStatusDto start(Mode mode) { + claim(mode); + Thread.ofVirtual().name("tw-rebuild").start(() -> runClaimed(mode)); + return status.get(); + } + @Async @EventListener(ApplicationReadyEvent.class) public void run() { @@ -53,13 +108,16 @@ public void run() { log.debug("TW backfill already completed; skipping."); return; } - log.info("TW backfill starting (throttle={}/s, batchSize={})", throttle, batchSize); - Counts c = process(true); + if (!running.compareAndSet(false, true)) { + log.info("TW backfill not started: a recalculation runs already."); + return; + } + begin(Mode.BACKFILL); + Counts c = runClaimed(Mode.BACKFILL); appStateRepo.save(new AppState( "tw_backfill_completed", "processed=%d skipped=%d failed=%d".formatted(c.processed, c.skipped, c.failed), Instant.now())); - log.info("TW backfill done: processed={} skipped={} failed={}", c.processed, c.skipped, c.failed); } /** @@ -70,22 +128,18 @@ public void run() { */ @Async public void rebuildAllNonOverridden() { - log.info("TW rebuild starting (throttle={}/s, batchSize={})", throttle, batchSize); - Counts c = process(false); - log.info("TW rebuild done: processed={} skipped={} failed={}", c.processed, c.skipped, c.failed); + claim(Mode.NON_OVERRIDDEN); + runClaimed(Mode.NON_OVERRIDDEN); } /** - * Admin-triggered rebuild: re-resolve TW suggestions for every binding - * whose streamer has not pinned them ({@code twOverride=false}), including - * those that already carry a non-empty TW set. Used after the mapping - * tables change so existing bindings pick up the new signals. + * Admin-triggered rebuild: re-resolve TW suggestions for the bindings that + * carry none yet and whose streamer has not pinned them, as the start-up backfill. */ @Async public void rebuildAll() { - log.info("TW rebuild starting (throttle={}/s, batchSize={})", throttle, batchSize); - Counts c = process(true); - log.info("TW rebuild done: processed={} skipped={} failed={}", c.processed, c.skipped, c.failed); + claim(Mode.UNTOUCHED); + runClaimed(Mode.UNTOUCHED); } /** @@ -97,9 +151,8 @@ public void rebuildAll() { */ @Async public void forceRebuildAllIncludingPinned() { - log.info("TW force-rebuild (including pinned) starting (throttle={}/s, batchSize={})", throttle, batchSize); - Counts c = processForceAll(); - log.info("TW force-rebuild done: processed={} skipped={} failed={}", c.processed, c.skipped, c.failed); + claim(Mode.FORCE_PINNED); + runClaimed(Mode.FORCE_PINNED); } private static class Counts { @@ -108,56 +161,84 @@ private static class Counts { int failed; } - /** - * Shared loop body. {@code backfillOnly=true} restricts to the original - * empty-TW candidates and applies the historical inner skip guard; - * {@code false} processes every non-overridden binding (rebuild path). - */ - private Counts process(boolean backfillOnly) { - long sleepMs = 1000L / Math.max(throttle, 1); + /** Takes the one recalculation slot for {@code mode}, its status reset. */ + private void claim(Mode mode) { + if (!running.compareAndSet(false, true)) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "A TW recalculation runs already"); + } + try { + begin(mode); + } catch (RuntimeException e) { + running.set(false); + throw e; + } + } + + private void begin(Mode mode) { + cancelled = false; + long total = candidates(mode, PageRequest.of(0, Math.max(batchSize, 1))).getTotalElements(); + status.set(new TwRebuildStatusDto("RUNNING", mode.name(), total, 0, 0, 0, clock.instant(), null, null)); + log.info("TW {} starting: {} binding(s) (throttle={}/s, batchSize={})", mode, total, throttle, batchSize); + } + + /** Runs the recalculation the slot was claimed for, then frees it. */ + private Counts runClaimed(Mode mode) { Counts c = new Counts(); - int page = 0; - Page batch; - String mode = backfillOnly ? "backfill" : "rebuild"; - do { - batch = backfillOnly - ? bindingRepo.findCandidatesForTwBackfill(PageRequest.of(page++, batchSize)) - : bindingRepo.findCandidatesForTwRebuild(PageRequest.of(page++, batchSize)); - for (GameBinding b : batch) { - if (b.isTwOverride() || (backfillOnly && !b.getTws().isEmpty()) || hasAdminTws(b)) { - c.skipped++; - meterRegistry.counter("catapult.tw." + mode + ".skipped").increment(); - continue; - } - if (!remap(b, mode, c, sleepMs)) { - return c; - } - } - } while (batch.hasNext()); + String state = "DONE"; + try { + if (!process(mode, c)) state = "CANCELLED"; + } catch (RuntimeException e) { + log.warn("TW {} ended on an error: {}", mode, e.getMessage(), e); + } finally { + TwRebuildStatusDto current = status.get(); + status.set(new TwRebuildStatusDto(state, mode.name(), current.total(), c.processed, c.skipped, c.failed, + current.startedAt(), clock.instant(), null)); + running.set(false); + log.info("TW {} {}: processed={} skipped={} failed={}", mode, state.toLowerCase(Locale.ROOT), + c.processed, c.skipped, c.failed); + } return c; } - /** Same loop as {@link #process}, but never skips on {@code twOverride} (the admin's mapped TWs still win). */ - private Counts processForceAll() { + private Page candidates(Mode mode, Pageable page) { + return switch (mode) { + case BACKFILL, UNTOUCHED -> bindingRepo.findCandidatesForTwBackfill(page); + case NON_OVERRIDDEN -> bindingRepo.findCandidatesForTwRebuild(page); + case FORCE_PINNED -> bindingRepo.findAllCandidatesForTwForceRebuild(page); + }; + } + + /** + * The loop: every candidate of {@code mode}, the streamer's pinned TWs left alone but by {@code FORCE_PINNED}, + * an admin's TWs always. Returns {@code false} when cancelled (or interrupted). + */ + private boolean process(Mode mode, Counts c) { long sleepMs = 1000L / Math.max(throttle, 1); - Counts c = new Counts(); + boolean backfillOnly = mode == Mode.BACKFILL || mode == Mode.UNTOUCHED; + String metric = switch (mode) { + case BACKFILL, UNTOUCHED -> "backfill"; + case NON_OVERRIDDEN -> "rebuild"; + case FORCE_PINNED -> "force_all"; + }; int page = 0; Page batch; - String mode = "force_all"; do { - batch = bindingRepo.findAllCandidatesForTwForceRebuild(PageRequest.of(page++, batchSize)); + batch = candidates(mode, PageRequest.of(page++, batchSize)); for (GameBinding b : batch) { - if (hasAdminTws(b)) { + if (cancelled) return false; + boolean pinned = mode != Mode.FORCE_PINNED && b.isTwOverride(); + if (pinned || (backfillOnly && !b.getTws().isEmpty()) || hasAdminTws(b)) { c.skipped++; - meterRegistry.counter("catapult.tw." + mode + ".skipped").increment(); + meterRegistry.counter("catapult.tw." + metric + ".skipped").increment(); + progress(c, null); continue; } - if (!remap(b, mode, c, sleepMs)) { - return c; + if (!remap(b, metric, c, sleepMs)) { + return false; } } } while (batch.hasNext()); - return c; + return true; } /** @@ -173,37 +254,76 @@ private boolean hasAdminTws(GameBinding b) { /** * Re-resolves one binding's TW suggestions, records the outcome in {@code c}, then - * throttles. Returns {@code false} when interrupted during the throttle sleep. + * throttles. Missing the Steam store's signals (rate limited), it waits for the limit's end and tries again, + * never saving TWs without them. Returns {@code false} when cancelled or interrupted. */ - private boolean remap(GameBinding b, String mode, Counts c, long sleepMs) { - DebugContext.runAs(b.getUser().getId(), () -> resolveTws(b, mode, c)); + private boolean remap(GameBinding b, String metric, Counts c, long sleepMs) { try { - Thread.sleep(sleepMs); + for (int waits = 0; ; waits++) { + AtomicReference limited = new AtomicReference<>(); + // Bulk traffic: kept under the learned rate limits with a margin for live calls, waiting its turn + RateLimitPriority.run(RateLimitPriority.BULK, () -> cancelled, + () -> DebugContext.runAs(b.getUser().getId(), () -> limited.set(resolveTws(b, metric, c)))); + if (limited.get() == null) break; + if (waits >= MAX_STEAM_WAITS) { + log.warn("TW {} gave up on binding {}: the Steam store stayed rate limited", metric, b.getId()); + c.failed++; + meterRegistry.counter("catapult.tw." + metric + ".failed").increment(); + break; + } + progress(c, limited.get()); + if (!pause(Duration.between(clock.instant(), limited.get()).toMillis() + 1_000L)) return false; + } + progress(c, null); + return pause(sleepMs); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return false; } - return true; } - /** Re-resolves one binding's TW suggestions as its owner and records the outcome in {@code c}. */ - private void resolveTws(GameBinding b, String mode, Counts c) { + /** Sleeps {@code millis} in slices; {@code false} once cancelled. */ + private boolean pause(long millis) throws InterruptedException { + long left = millis; + while (left > 0) { + if (cancelled) return false; + long slice = Math.min(left, SLEEP_SLICE_MS); + sleeper.sleep(slice); + left -= slice; + } + return !cancelled; + } + + private void progress(Counts c, Instant waitingUntil) { + TwRebuildStatusDto current = status.get(); + status.set(new TwRebuildStatusDto("RUNNING", current.mode(), current.total(), c.processed, c.skipped, c.failed, + current.startedAt(), null, waitingUntil)); + } + + /** + * Re-resolves one binding's TW suggestions as its owner and records the outcome in {@code c}; returns until when + * the Steam store is rate limited when its signals were missed (nothing saved then), else null. + */ + private Instant resolveTws(GameBinding b, String metric, Counts c) { try { Optional igdbId = igdbService.resolveIgdbIdForBinding(b); Set descIds = igdbId.map(igdbService::fetchDescriptorIds).orElse(Set.of()); String steamApp = b.getSourceType() == GameBinding.SourceType.STEAM ? b.getSourceId() : null; - Set tws = resolver.suggest(new TwResolverService.SuggestInput( + TwResolverService.Suggestion suggestion = resolver.suggestReporting(new TwResolverService.SuggestInput( igdbId.orElse(null), descIds, steamApp, b.getSourceName())); + if (suggestion.steamLimitedUntil().isPresent()) return suggestion.steamLimitedUntil().get(); + Set tws = suggestion.tws(); log.debug("[TW] {} mapping for binding {} game '{}' (igdb={}): {}", - mode, b.getId(), b.getSourceName(), igdbId.orElse(null), tws); + metric, b.getId(), b.getSourceName(), igdbId.orElse(null), tws); b.setTws(tws); bindingRepo.save(b); c.processed++; - meterRegistry.counter("catapult.tw." + mode + ".processed").increment(); + meterRegistry.counter("catapult.tw." + metric + ".processed").increment(); } catch (Exception e) { - log.warn("TW {} failed for binding {}: {}", mode, b.getId(), e.getMessage()); + log.warn("TW {} failed for binding {}: {}", metric, b.getId(), e.getMessage()); c.failed++; - meterRegistry.counter("catapult.tw." + mode + ".failed").increment(); + meterRegistry.counter("catapult.tw." + metric + ".failed").increment(); } + return null; } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwDtddTopicLevels.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwDtddTopicLevels.java new file mode 100644 index 000000000..1c85b1116 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwDtddTopicLevels.java @@ -0,0 +1,39 @@ +package fr.enimaloc.catapult.service.tw; + +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.json.JsonMapper; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * The severity levels of DTDD topics as the cached titles give them (DTDD lists no topic's levels on its own): what + * the admin picks a TW mapping's least severe level from. + */ +@Component +@RequiredArgsConstructor +public class TwDtddTopicLevels { + + private static final TypeReference> LEVELS = new TypeReference<>() {}; + + private final DtddTopicsCacheRepository topicsRepo; + private final JsonMapper json; + + /** Each topic's levels by its lower-cased name, mildest first; a topic no cached title has levels for absent. */ + public Map> of(Collection topics) { + if (topics.isEmpty()) return Map.of(); + Map> out = new HashMap<>(); + for (Object[] row : topicsRepo.findLevelsOf(topics.stream() + .map(t -> t.toLowerCase(Locale.ROOT)).collect(Collectors.toSet()))) { + out.put((String) row[0], json.readValue((String) row[1], LEVELS)); + } + return out; + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwResolverService.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwResolverService.java index 78e2480f9..ebd34584c 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwResolverService.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwResolverService.java @@ -1,18 +1,25 @@ package fr.enimaloc.catapult.service.tw; +import fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping; +import fr.enimaloc.catapult.domain.tw.TwIgdbDescriptorMapping; +import fr.enimaloc.catapult.domain.tw.TwSteamContentIdMapping; import fr.enimaloc.catapult.domain.tw.TwSteamKeyword; import fr.enimaloc.catapult.repository.tw.TwDtddTopicMappingRepository; import fr.enimaloc.catapult.repository.tw.TwIgdbDescriptorMappingRepository; import fr.enimaloc.catapult.repository.tw.TwSteamContentIdMappingRepository; import fr.enimaloc.catapult.repository.tw.TwSteamKeywordRepository; import fr.enimaloc.catapult.service.dtdd.DtddSignalService; +import fr.enimaloc.catapult.service.dtdd.DtddSignalService.TopicSignals; import fr.enimaloc.catapult.service.steam.SteamStoreService; import fr.enimaloc.catapult.service.steam.SteamStoreService.SteamTwSignals; import lombok.RequiredArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; import org.springframework.stereotype.Service; +import java.time.Instant; import java.util.*; import java.util.stream.Collectors; @@ -29,6 +36,9 @@ public class TwResolverService { private final Optional dtddSignalService; private final SteamStoreService steamStoreService; + /** The level votes a DTDD topic needs for its severity to keep a TW off (see {@link TwSeverityRule}). */ + @Setter @Value("${dtdd.severity.min-votes:3}") private int dtddSeverityMinVotes = 3; + public record SuggestInput( String igdbId, Set igdbDescriptorIds, @@ -36,38 +46,104 @@ public record SuggestInput( String sourceName ) {} + /** Where a signal comes from. */ + public enum SourceType { IGDB_DESCRIPTOR, STEAM_CONTENT, STEAM_KEYWORD, DTDD_TOPIC } + + /** + * One signal for a TW: its source and what in it ({@code value}: the IGDB descriptor's or the Steam content + * descriptor's ID, the Steam keyword, the DTDD topic); {@code severity} a DTDD topic's verdict, null for the rest. + */ + public record Signal(String twId, SourceType source, String value, TwSeverityRule.Verdict severity) { + public boolean triggers() { + return severity == null || severity.triggers(); + } + } + + /** + * Every signal found for a game: those that turn their TW on, and the DTDD topics kept off by their severity + * ({@link #dropped}). {@link #twIds} is what {@link #suggest} gives. {@code steamLimitedUntil}: until when the + * Steam store was rate limited when a Steam game's signals were asked for (empty: they came, or weren't needed); + * without them, the TWs miss what Steam would have added. + */ + public record Explanation(List signals, Optional steamLimitedUntil) { + + /** Signals Steam didn't fail to give. */ + public Explanation(List signals) { + this(signals, Optional.empty()); + } + + public Set twIds() { + return signals.stream().filter(Signal::triggers).map(Signal::twId).collect(Collectors.toSet()); + } + + public List dropped() { + return signals.stream().filter(s -> !s.triggers()).toList(); + } + } + + /** The TWs suggested, and until when the Steam store was rate limited when its signals were missed. */ + public record Suggestion(Set tws, Optional steamLimitedUntil) {} + + /** Until when the Steam store is rate limited (a Steam game's TWs resolved meanwhile miss its signals). */ + public Optional steamRateLimitedUntil() { + return steamStoreService.rateLimitedUntil(); + } + + /** As {@link #suggest}, telling when the Steam signals were missed because the store was rate limited. */ + public Suggestion suggestReporting(SuggestInput in) { + Explanation explanation = explain(in); + return new Suggestion(explanation.twIds(), explanation.steamLimitedUntil()); + } + /** Computes the set of TW slugs that should be pre-activated for this binding input. */ public Set suggest(SuggestInput in) { - Set out = new HashSet<>(); + return explain(in).twIds(); + } + + /** As {@link #suggest}, telling which source turned each TW on, and which DTDD topics their severity kept off. */ + public Explanation explain(SuggestInput in) { + List out = new ArrayList<>(); + Optional steamLimitedUntil = Optional.empty(); if (in.igdbDescriptorIds() != null && !in.igdbDescriptorIds().isEmpty()) { - out.addAll(igdbRepo.findTwIdsByDescriptorIds(in.igdbDescriptorIds())); + for (TwIgdbDescriptorMapping m : igdbRepo.findAllByDescriptorIdIn(in.igdbDescriptorIds())) { + out.add(new Signal(m.getTwId(), SourceType.IGDB_DESCRIPTOR, String.valueOf(m.getDescriptorId()), null)); + } } if (in.steamAppId() != null) { - SteamTwSignals s = steamStoreService.fetchTwSignals(List.of(in.steamAppId())) - .getOrDefault(in.steamAppId(), SteamTwSignals.empty()); + Map fetched = steamStoreService.fetchTwSignals(List.of(in.steamAppId())); + if (!fetched.containsKey(in.steamAppId())) steamLimitedUntil = steamStoreService.rateLimitedUntil(); + SteamTwSignals s = fetched.getOrDefault(in.steamAppId(), SteamTwSignals.empty()); if (!s.contentDescriptorIds().isEmpty()) { - out.addAll(steamIdRepo.findTwIdsBySteamContentIdIn(s.contentDescriptorIds())); + for (TwSteamContentIdMapping m : steamIdRepo.findAllBySteamContentIdIn(s.contentDescriptorIds())) { + out.add(new Signal(m.getTwId(), SourceType.STEAM_CONTENT, String.valueOf(m.getSteamContentId()), null)); + } } if (!s.notesLowercase().isBlank()) { String notes = s.notesLowercase(); - out.addAll(steamKwRepo.findAll().stream() - .filter(k -> notes.contains(k.getKeyword())) - .map(TwSteamKeyword::getTwId) - .collect(Collectors.toSet())); + for (TwSteamKeyword k : steamKwRepo.findAll()) { + if (notes.contains(k.getKeyword())) { + out.add(new Signal(k.getTwId(), SourceType.STEAM_KEYWORD, k.getKeyword(), null)); + } + } } } dtddSignalService.ifPresent(svc -> { - Set topics = svc.getYesMostlyTopics(in.igdbId(), in.sourceName()); - if (!topics.isEmpty()) { - Set lowered = topics.stream() - .map(t -> t.toLowerCase(Locale.ROOT)).collect(Collectors.toSet()); - out.addAll(dtddRepo.findTwIdsByDtddTopicNameInIgnoreCase(lowered)); + TopicSignals topics = svc.getYesMostlySignals(in.igdbId(), in.sourceName()); + if (topics.topics().isEmpty()) return; + Map byLowered = new HashMap<>(); + topics.topics().forEach(t -> byLowered.put(t.toLowerCase(Locale.ROOT), t)); + Map severities = topics.severities(); + for (TwDtddTopicMapping m : dtddRepo.findAllByDtddTopicNameInIgnoreCase(byLowered.keySet())) { + String topic = byLowered.getOrDefault(m.getDtddTopicName().toLowerCase(Locale.ROOT), m.getDtddTopicName()); + TwSeverityRule.Verdict verdict = TwSeverityRule.evaluate(m.getMinSeverity(), severities != null, + severities == null ? null : topics.severities().get(topic), dtddSeverityMinVotes); + out.add(new Signal(m.getTwId(), SourceType.DTDD_TOPIC, topic, verdict)); } }); - return out; + return new Explanation(out, steamLimitedUntil); } } diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwSeverityRule.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwSeverityRule.java new file mode 100644 index 000000000..bd348b1c6 --- /dev/null +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/tw/TwSeverityRule.java @@ -0,0 +1,104 @@ +package fr.enimaloc.catapult.service.tw; + +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSeverity; + +import java.util.List; +import java.util.Locale; + +/** + * Whether a DTDD topic marked yes or mostly is severe enough for its TW mapping's least severe level + * ({@code minSeverity}). It turns the TW on but when DTDD can tell the topic is below it: its reviewers' level, else + * at least half of its members' level votes at or above it — only from {@code minVotes} level votes on. Anything DTDD + * can't tell (no threshold, a topic without levels, severities not cached, a threshold not among the topic's levels, + * too few votes) turns it on, as before the severities. + */ +public final class TwSeverityRule { + + /** Why a topic turned its TW on, or didn't ({@link #BELOW_VERIFIED}, {@link #BELOW_VOTES}). */ + public enum Reason { + NO_THRESHOLD, NOT_CACHED, NO_LEVELS, UNKNOWN_LEVEL, TOO_FEW_VOTES, VERIFIED, VOTES, BELOW_VERIFIED, BELOW_VOTES; + + public boolean triggers() { + return this != BELOW_VERIFIED && this != BELOW_VOTES; + } + } + + /** + * The verdict: {@code level} the level that decided (the reviewers', else the members' most voted; null when none), + * {@code votes} the level votes, {@code percentAtOrAbove} their share at or above the threshold (0 without one). + */ + public record Verdict(Reason reason, String minSeverity, String level, int votes, int percentAtOrAbove) { + public boolean triggers() { + return reason.triggers(); + } + } + + private TwSeverityRule() { + } + + /** + * @param severitiesKnown whether the topics' severities were cached at all (false: fetched before they were kept) + * @param severity the topic's, null when it has no levels + */ + public static Verdict evaluate(String minSeverity, boolean severitiesKnown, DtddSeverity severity, int minVotes) { + String min = minSeverity == null || minSeverity.isBlank() ? null : minSeverity.trim(); + if (min == null) return new Verdict(Reason.NO_THRESHOLD, null, null, votes(severity), 0); + if (!severitiesKnown) return new Verdict(Reason.NOT_CACHED, min, null, 0, 0); + if (severity == null || severity.levels() == null || severity.levels().isEmpty()) { + return new Verdict(Reason.NO_LEVELS, min, null, 0, 0); + } + List levels = severity.levels(); + int minIndex = indexOf(levels, min); + if (minIndex < 0) return new Verdict(Reason.UNKNOWN_LEVEL, min, null, votes(severity), 0); + + Integer verified = severity.verified(); + if (verified != null && verified >= 1 && verified <= levels.size()) { + String level = levels.get(verified - 1); + return new Verdict(verified - 1 >= minIndex ? Reason.VERIFIED : Reason.BELOW_VERIFIED, min, level, + votes(severity), 0); + } + int total = votes(severity); + String mostVoted = mostVoted(severity); + if (total < Math.max(1, minVotes)) return new Verdict(Reason.TOO_FEW_VOTES, min, mostVoted, total, 0); + int atOrAbove = 0; + List counts = severity.counts(); + for (int i = minIndex; i < counts.size(); i++) { + atOrAbove += counts.get(i) == null ? 0 : counts.get(i); + } + int percent = Math.round(100f * atOrAbove / total); + // At least half: a tie leans to the warning + return new Verdict(2 * atOrAbove >= total ? Reason.VOTES : Reason.BELOW_VOTES, min, mostVoted, total, percent); + } + + /** The level votes: DTDD's total, else the counts' sum. */ + private static int votes(DtddSeverity severity) { + if (severity == null) return 0; + if (severity.total() > 0) return severity.total(); + return severity.counts() == null ? 0 + : severity.counts().stream().mapToInt(c -> c == null ? 0 : c).sum(); + } + + /** The members' most voted level (the more severe on a tie), null without a vote. */ + private static String mostVoted(DtddSeverity severity) { + List counts = severity.counts(); + if (counts == null) return null; + int best = -1; + int bestCount = 0; + for (int i = 0; i < Math.min(counts.size(), severity.levels().size()); i++) { + int c = counts.get(i) == null ? 0 : counts.get(i); + if (c > 0 && c >= bestCount) { + best = i; + bestCount = c; + } + } + return best < 0 ? null : severity.levels().get(best); + } + + private static int indexOf(List levels, String level) { + String wanted = level.toLowerCase(Locale.ROOT); + for (int i = 0; i < levels.size(); i++) { + if (levels.get(i) != null && levels.get(i).toLowerCase(Locale.ROOT).equals(wanted)) return i; + } + return -1; + } +} diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/twitch/TwitchServiceImpl.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/twitch/TwitchServiceImpl.java index 7bbb6a3e1..a1337f257 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/twitch/TwitchServiceImpl.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/twitch/TwitchServiceImpl.java @@ -214,7 +214,8 @@ private ChannelUpdateOutcome doUpdateChannel(UserAccount user, GameBinding bindi if (e.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS) { log.warn("Twitch rate limit hit for user {} — will retry next cycle", user.getId()); } else { - log.error("Twitch API error for user {}: {} {}", user.getId(), e.getStatusCode(), e.getMessage()); + log.error("Twitch API error for user {} (game_id={}): {} {}", + user.getId(), binding.getTwitchGameId(), e.getStatusCode(), e.getMessage()); } throw refused[0] = e; } catch (RuntimeException e) { diff --git a/catapult-api/src/main/java/fr/enimaloc/catapult/service/xbox/XboxStoreServiceImpl.java b/catapult-api/src/main/java/fr/enimaloc/catapult/service/xbox/XboxStoreServiceImpl.java index bc66c7016..e4f81f342 100644 --- a/catapult-api/src/main/java/fr/enimaloc/catapult/service/xbox/XboxStoreServiceImpl.java +++ b/catapult-api/src/main/java/fr/enimaloc/catapult/service/xbox/XboxStoreServiceImpl.java @@ -35,8 +35,8 @@ public Optional fetchProduct(String productId, Locale locale) { ? "en-US" : locale.toLanguageTag(); String market = locale != null && !locale.getCountry().isBlank() ? locale.getCountry() : "US"; DisplayCatalogResponse response = restClient.get() - .uri(DISPLAY_CATALOG_URL + productId + "?market=" + market - + "&languages=" + languageTag + "&fieldsTemplate=Details") + .uri(DISPLAY_CATALOG_URL + "{productId}?market={market}&languages={languages}" + + "&fieldsTemplate=Details", productId, market, languageTag) .retrieve() .body(DisplayCatalogResponse.class); if (response == null || response.product() == null) { diff --git a/catapult-api/src/main/resources/application.properties b/catapult-api/src/main/resources/application.properties index 1294849e7..ce4b50464 100644 --- a/catapult-api/src/main/resources/application.properties +++ b/catapult-api/src/main/resources/application.properties @@ -147,6 +147,13 @@ dtdd.api-base-url=https://www.doesthedogdie.com/api dtdd.cache.search-ttl-days=30 dtdd.cache.game-info-ttl-days=180 dtdd.cache.topics-ttl-hours=168 +# The admin's bulk refetch of the caches: each key's requests per minute it keeps under (DTDD's free tier: 30), and +# the requests of its month it leaves each key, for the detection +dtdd.refetch.per-minute=28 +dtdd.refetch.monthly-reserve=500 +# The level votes a DTDD topic needs for its severity to keep a TW off (below its mapping's least severe level); +# with fewer (and no reviewers' level), the topic turns its TW on as before +dtdd.severity.min-votes=3 # Matching a game to a DTDD title: the score to accept a match, the one to list a candidate, and the weights dtdd.match.min-confidence=0.85 dtdd.match.min-candidate-score=0.30 @@ -160,6 +167,23 @@ tw.enabled=${TW_ENABLED:true} tw.backfill.batch-size=50 tw.backfill.throttle-per-second=2 +# ============================================================ +# Learned rate limits (platforms without a documented one, guessed from their 429s) +# ============================================================ +rate-limit.learning.enabled=${RATE_LIMIT_LEARNING_ENABLED:true} +# Left to their documented limits (*.host: every subdomain) +rate-limit.learning.excluded-hosts=api.steampowered.com,api.twitch.tv,id.twitch.tv,api.igdb.com,\ + www.doesthedogdie.com,git.enimaloc.fr,oauth.battle.net,*.battle.net,*.api.blizzard.com +# The share of a limit bulk work (TW recalculation) uses, leaving the rest to live calls +rate-limit.learning.bulk-margin=0.8 +# How long a live call waits for the limit before giving up (a local 429) +rate-limit.learning.live-max-wait=2s +# Near a limit this long without a 429, it is raised by a tenth; a resume delay that failed is forgotten after +rate-limit.learning.raise-after=1h +rate-limit.learning.forget-fail-after=24h +# How often an instance takes what the others learned and the admins corrected +rate-limit.learning.sync-every=60s + # ============================================================ # Feedback form (GitLab issues) # ============================================================ diff --git a/catapult-api/src/main/resources/db/migration/V104__game_mapping_ignored.sql b/catapult-api/src/main/resources/db/migration/V104__game_mapping_ignored.sql new file mode 100644 index 000000000..fac328293 --- /dev/null +++ b/catapult-api/src/main/resources/db/migration/V104__game_mapping_ignored.sql @@ -0,0 +1,11 @@ +-- An admin game mapping can ignore its game by default (a background app such as Wallpaper Engine): its bindings are +-- ignored, but those the streamer ignored or not by hand. Such a mapping needs no Twitch category: without one, its +-- bindings keep IGDB's. +ALTER TABLE game_mapping ADD COLUMN ignored_by_default BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE game_mapping ALTER COLUMN twitch_game_id DROP NOT NULL; +ALTER TABLE game_mapping ALTER COLUMN twitch_game_name DROP NOT NULL; + +-- true: the streamer chose whether the binding is ignored, an admin mapping no longer changes it. +ALTER TABLE game_binding ADD COLUMN ignored_override BOOLEAN NOT NULL DEFAULT FALSE; +-- Before mappings could ignore a game, only the streamer did. +UPDATE game_binding SET ignored_override = TRUE WHERE ignored; diff --git a/catapult-api/src/main/resources/db/migration/V105__dtdd_topics_severities.sql b/catapult-api/src/main/resources/db/migration/V105__dtdd_topics_severities.sql new file mode 100644 index 000000000..ae2fbc623 --- /dev/null +++ b/catapult-api/src/main/resources/db/migration/V105__dtdd_topics_severities.sql @@ -0,0 +1,3 @@ +-- Each cached topic's severity, as DTDD's members voted it (levels, counts, percents, total, the reviewers' level), +-- by topic name. NULL: fetched before the severities were kept; a topic without levels is absent. +ALTER TABLE dtdd_topics_cache ADD COLUMN severities JSONB; diff --git a/catapult-api/src/main/resources/db/migration/V106__tw_dtdd_min_severity.sql b/catapult-api/src/main/resources/db/migration/V106__tw_dtdd_min_severity.sql new file mode 100644 index 000000000..d9c19b68e --- /dev/null +++ b/catapult-api/src/main/resources/db/migration/V106__tw_dtdd_min_severity.sql @@ -0,0 +1,20 @@ +-- The least severe level of its DTDD topic that turns the TW on (one of the topic's severity labels, as DTDD names +-- them); NULL: any level, as before. Below it, the topic doesn't turn the TW on — unless DTDD's members cast too few +-- level votes to tell (dtdd.severity.min-votes) and its reviewers gave none. +ALTER TABLE tw_dtdd_topic_mapping ADD COLUMN min_severity VARCHAR(40); + +-- Only where the mildest level is harmless to show (mentioned, implied, fake: a phobia's subject only when it isn't +-- there for real); never for self-harm, sexual violence, child abuse nor flashing lights. +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Depicted' WHERE lower(dtdd_topic_name) = 'there''s blood/gore'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Clear' WHERE lower(dtdd_topic_name) = 'there''s audio gore'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Heard' WHERE lower(dtdd_topic_name) = 'someone vomits'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Startling' WHERE lower(dtdd_topic_name) = 'there are jump scares'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Startling' WHERE lower(dtdd_topic_name) = 'there are sudden loud noises'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Mild' WHERE lower(dtdd_topic_name) = 'there''s screaming'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Crude' WHERE lower(dtdd_topic_name) = 'there is obscene language/gestures'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Clear' WHERE lower(dtdd_topic_name) = 'shaky cam is used'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Depicted' WHERE lower(dtdd_topic_name) = 'needles/syringes are used'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Drunk' WHERE lower(dtdd_topic_name) = 'alcohol abuse'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Used' WHERE lower(dtdd_topic_name) = 'someone uses drugs'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Harmless' WHERE lower(dtdd_topic_name) = 'there are snakes'; +UPDATE tw_dtdd_topic_mapping SET min_severity = 'Harmless' WHERE lower(dtdd_topic_name) = 'there are spiders'; diff --git a/catapult-api/src/main/resources/db/migration/V107__learned_rate_limits.sql b/catapult-api/src/main/resources/db/migration/V107__learned_rate_limits.sql new file mode 100644 index 000000000..46bf5b688 --- /dev/null +++ b/catapult-api/src/main/resources/db/migration/V107__learned_rate_limits.sql @@ -0,0 +1,25 @@ +-- What Catapult learned of the rate limits platforms don't document, from their 429s: one row per host (its shared +-- budget, method and path NULL) and one per route seen (scope HOST: counted in its host's budget; ROUTE: its own). +-- override_*: an admin's correction, kept until reset; the learning leaves it alone. admin_changed_at: the last +-- correction or reset, which an instance that hasn't taken it yet must not write over. +CREATE TABLE learned_rate_limits ( + id VARCHAR(800) PRIMARY KEY, + host VARCHAR(255) NOT NULL, + method VARCHAR(16), + path VARCHAR(512), + scope VARCHAR(8), + scope_pinned BOOLEAN NOT NULL DEFAULT FALSE, + limit_calls INTEGER, + limit_window_seconds INTEGER, + resume_ms BIGINT NOT NULL, + resume_fail_ms BIGINT, + resume_ok_ms BIGINT, + last_fail_at TIMESTAMPTZ, + rate_limited_count INTEGER NOT NULL DEFAULT 0, + last_rate_limited_at TIMESTAMPTZ, + override_calls INTEGER, + override_window_seconds INTEGER, + override_resume_ms BIGINT, + admin_changed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL +); diff --git a/catapult-api/src/main/resources/lang/messages.properties b/catapult-api/src/main/resources/lang/messages.properties index e98d37d96..ec35909a5 100644 --- a/catapult-api/src/main/resources/lang/messages.properties +++ b/catapult-api/src/main/resources/lang/messages.properties @@ -33,7 +33,9 @@ help.ccl-settings.body=

This section controls the content labels (CCLs) that { help.tw-settings.title=Trigger Warnings help.tw-settings.body=Use {tw#active} for the aggregated list, or {tw#<slug>} for a specific category. Auto-suggestion uses IGDB age ratings, Steam content descriptors, Steam mature-content notes, and DoesTheDogDie.com topics. help.title-settings.title=Stream title -help.title-settings.body=When it is on, Catapult changes your stream title each time it changes your category. What has a title of its own uses it: a game (in its row, under Overview), a custom binding, and the "no game" and incomplete game categories (in their cards); the rest uses the title for every game. In a title, {{GAME}} becomes the game detected (a custom binding''s name, nothing without a game), {{CATEGORY}} its Twitch category, {{TITLE}} the title the stream had just before, and {{ORIGINAL_TITLE}} your own title, the one you set before Catapult changed it (change it by hand and that one becomes your own). Write them exactly so, in capitals: anything else between double braces stays as typed. With no title to use, yours is left as it is; when the title before can''t be read from Twitch, it is left as it is too. Twitch cuts a title at 140 characters.

When conditions are offered to your channel, {{DROP_ENABLED ? [[[Drops] ]] : [[]]}} keeps the first part when the condition holds and the second otherwise. The conditions are DROP_ENABLED (a Twitch drops campaign runs for your channel on the category), HAS_GAME, IS_CUSTOM, IS_INCOMPLETE, IS_NO_GAME, HAS_TW and IS_MATURE; combine them with !, && and ||, put variables or other conditions in the parts. {{DROP_NAME}} and {{DROP_END}} name the drops campaign and its end. A condition that doesn''t read through stays as typed. +help.title-settings.body=When it is on, Catapult changes your stream title each time it changes your category. What has a title of its own uses it: a game (in its row, under Overview), a custom binding, and the "no game" and incomplete game categories (in their cards); the rest uses the title for every game. In a title, {{GAME}} becomes the game detected (a custom binding''s name, nothing without a game), {{CATEGORY}} its Twitch category, {{TITLE}} the title the stream had just before (even when Catapult set it), and {{ORIGINAL_TITLE}} your own title, the one you set before Catapult changed it (change it by hand and that one becomes your own). Write them exactly so, in capitals: anything else between double braces stays as typed. With no title to use, yours is left as it is; when the title before can''t be read from Twitch, it is left as it is too. Twitch cuts a title at 140 characters. +help.title-settings.body.conditions=

{{HAS_TW ? [[[TW] ]] : [[]]}} keeps the first part when the condition holds and the second otherwise. The conditions are IS_NO_GAME (no game detected), HAS_GAME (a game is detected), IS_CUSTOM (it is a custom binding), IS_INCOMPLETE (the game is detected but no Twitch category was found), HAS_TW (Catapult found warnings, the TWs, on the game) and IS_MATURE (Catapult found the Mature CCL on the game); combine them with !, && and ||, put variables or other conditions in the parts. A condition that doesn''t read through stays as typed. +help.title-settings.body.drops=

For Twitch drops, the DROP_ENABLED condition holds when a drops campaign runs for your channel on the category; {{DROP_NAME}} then names the campaign and {{DROP_END}} gives its end date (dd/MM, Paris time). Without a campaign, both are empty. help.dtdd.title=DTDD association help.dtdd.body=

{2} links the current game to its page on DoesTheDogDie.com: the sensitive topics it lists are used to suggest the game''s trigger warnings.

  • The percentage is the confidence of the association found automatically.
  • Validate confirms it is right; this is possible as long as it isn''t verified and no correction is pending.
  • Correct lets you search for the right page and propose it: the correction applies once an administrator validates it.
diff --git a/catapult-api/src/main/resources/lang/messages_fr.properties b/catapult-api/src/main/resources/lang/messages_fr.properties index fbdec01c8..bd3056757 100644 --- a/catapult-api/src/main/resources/lang/messages_fr.properties +++ b/catapult-api/src/main/resources/lang/messages_fr.properties @@ -32,7 +32,9 @@ help.ccl-settings.body=

Cette section contrĂŽle les labels de contenu (CCL) qu help.tw-settings.title=Trigger Warnings help.tw-settings.body=Utilise {tw#active} pour la liste agrĂ©gĂ©e, ou {tw#<slug>} pour une catĂ©gorie spĂ©cifique. L''auto-suggestion s''appuie sur les classifications IGDB, les content_descriptors Steam, les notes mature Steam, et les topics DoesTheDogDie.com. help.title-settings.title=Titre du stream -help.title-settings.body=ActivĂ©, Catapult change le titre de ton stream Ă  chaque fois qu''il change ta catĂ©gorie. Ce qui a son propre titre l''utilise : un jeu (dans sa ligne, sous Vue d''ensemble), un binding personnalisĂ©, et les catĂ©gories « aucun jeu » et « jeu incomplet » (dans leurs cartes) ; le reste prend le titre pour tous les jeux. Dans un titre, {{GAME}} devient le jeu dĂ©tectĂ© (le nom d''un binding personnalisĂ©, rien sans jeu), {{CATEGORY}} sa catĂ©gorie Twitch, {{TITLE}} le titre qu''avait le stream juste avant et {{ORIGINAL_TITLE}} ton propre titre, celui que tu avais mis avant que Catapult le change (change-le Ă  la main et c''est lui qui devient le tien). Écris-les exactement ainsi, en majuscules : tout le reste entre doubles accolades reste tel quel. Sans titre Ă  utiliser, le tien n''est pas touchĂ© ; si le titre d''avant ne peut pas ĂȘtre lu sur Twitch, il n''est pas touchĂ© non plus. Twitch coupe un titre Ă  140 caractĂšres.

Quand les conditions sont proposĂ©es Ă  ta chaĂźne, {{DROP_ENABLED ? [[[Drops] ]] : [[]]}} garde la premiĂšre partie si la condition est vraie, la seconde sinon. Les conditions sont DROP_ENABLED (une campagne de drops Twitch est en cours pour ta chaĂźne sur la catĂ©gorie), HAS_GAME, IS_CUSTOM, IS_INCOMPLETE, IS_NO_GAME, HAS_TW et IS_MATURE ; combine-les avec !, && et ||, et mets des variables ou d''autres conditions dans les parties. {{DROP_NAME}} et {{DROP_END}} donnent le nom de la campagne de drops et sa date de fin. Une condition mal Ă©crite reste telle quelle. +help.title-settings.body=ActivĂ©, Catapult change le titre de ton stream Ă  chaque fois qu''il change ta catĂ©gorie. Ce qui a son propre titre l''utilise : un jeu (dans sa ligne, sous Vue d''ensemble), un binding personnalisĂ©, et les catĂ©gories « aucun jeu » et « jeu incomplet » (dans leurs cartes) ; le reste prend le titre pour tous les jeux. Dans un titre, {{GAME}} devient le jeu dĂ©tectĂ© (le nom d''un binding personnalisĂ©, rien sans jeu), {{CATEGORY}} sa catĂ©gorie Twitch, {{TITLE}} le titre qu''avait le stream juste avant (mĂȘme s''il avait Ă©tĂ© mis par Catapult) et {{ORIGINAL_TITLE}} ton propre titre, celui que tu avais mis avant que Catapult le change (change-le Ă  la main et c''est lui qui devient le tien). Écris-les exactement ainsi, en majuscules : tout le reste entre doubles accolades reste tel quel. Sans titre Ă  utiliser, le tien n''est pas touchĂ© ; si le titre d''avant ne peut pas ĂȘtre lu sur Twitch, il n''est pas touchĂ© non plus. Twitch coupe un titre Ă  140 caractĂšres. +help.title-settings.body.conditions=

{{HAS_TW ? [[[TW] ]] : [[]]}} garde la premiÚre partie si la condition est vraie, la seconde sinon. Les conditions sont IS_NO_GAME (aucun jeu détecté), HAS_GAME (un jeu est détecté), IS_CUSTOM (c''est un binding personnalisé), IS_INCOMPLETE (le jeu est détecté mais aucune catégorie Twitch n''a été trouvée), HAS_TW (Catapult a trouvé des avertissements, les TW, sur le jeu) et IS_MATURE (Catapult a trouvé le CCL Mature sur le jeu) ; combine-les avec !, && et ||, mets des variables ou d''autres conditions dans les parties. Une condition mal écrite reste telle quelle. +help.title-settings.body.drops=

Pour les drops Twitch, la condition DROP_ENABLED est vraie quand une campagne de drops est en cours pour ta chaßne sur la catégorie ; {{DROP_NAME}} donne alors le nom de la campagne et {{DROP_END}} sa date de fin (jj/MM, heure de Paris). Sans campagne, ces deux variables sont vides. help.dtdd.title=Association DTDD help.dtdd.body=

{2} relie le jeu en cours à sa fiche sur DoesTheDogDie.com : les sujets sensibles qu''elle recense servent à suggérer les trigger warnings du jeu.

  • Le pourcentage indique la confiance de l''association trouvĂ©e automatiquement.
  • Valider confirme qu''elle est juste ; c''est possible tant qu''elle n''est pas vĂ©rifiĂ©e et qu''aucune correction n''est en attente.
  • Corriger permet de chercher la bonne fiche et de la proposer : la correction est appliquĂ©e une fois validĂ©e par un administrateur.
diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/ApiHelpControllerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/ApiHelpControllerTest.java index 25e91525d..c2d7fb03b 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/api/ApiHelpControllerTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/ApiHelpControllerTest.java @@ -1,6 +1,7 @@ package fr.enimaloc.catapult.api; import fr.enimaloc.catapult.service.minecraft.MinecraftGateService; +import fr.enimaloc.catapult.service.title.TitleHelpSections; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,6 +27,7 @@ class ApiHelpControllerTest { @Mock private MessageSource messageSource; @Mock private MinecraftGateService gateService; + @Mock private TitleHelpSections titleHelpSections; @Mock private Jwt jwt; @InjectMocks private ApiHelpController controller; @@ -45,6 +47,12 @@ void setup() { .thenReturn("

minecraft

"); when(messageSource.getMessage(eq("help.connections.body.xbox"), any(), any(), eq(locale))) .thenReturn("

xbox

"); + when(messageSource.getMessage(eq("help.title-settings.body"), any(), any(), eq(locale))) + .thenReturn("

titre

"); + when(messageSource.getMessage(eq("help.title-settings.body.conditions"), any(), any(), eq(locale))) + .thenReturn("

conditions

"); + when(messageSource.getMessage(eq("help.title-settings.body.drops"), any(), any(), eq(locale))) + .thenReturn("

drops

"); } @Test @@ -120,4 +128,39 @@ void connections_xboxDisabled_omitsXboxSection() { assertThat(content.body()).isEqualTo("

corps

"); } + + @Test + void titleSettings_conditionsAndDrops_appendsBothSections() { + when(titleHelpSections.forAccount(userId)).thenReturn(new TitleHelpSections.Offered(true, true)); + + var content = controller.help("title-settings", locale, jwt); + + assertThat(content.body()).isEqualTo("

titre

conditions

drops

"); + } + + @Test + void titleSettings_conditionsWithoutDrops_appendsConditionsOnly() { + when(titleHelpSections.forAccount(userId)).thenReturn(new TitleHelpSections.Offered(true, false)); + + var content = controller.help("title-settings", locale, jwt); + + assertThat(content.body()).isEqualTo("

titre

conditions

"); + } + + @Test + void titleSettings_noneOffered_omitsBoth() { + when(titleHelpSections.forAccount(userId)).thenReturn(TitleHelpSections.Offered.NONE); + + var content = controller.help("title-settings", locale, jwt); + + assertThat(content.body()).isEqualTo("

titre

"); + } + + @Test + void titleSettings_anonymous_omitsBoth() { + var content = controller.help("title-settings", locale, null); + + assertThat(content.body()).isEqualTo("

titre

"); + org.mockito.Mockito.verifyNoInteractions(titleHelpSections); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesControllerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesControllerTest.java new file mode 100644 index 000000000..f45ca2a79 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminDtddCachesControllerTest.java @@ -0,0 +1,50 @@ +package fr.enimaloc.catapult.api.admin; + +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.service.dtdd.DtddCacheAdminService; +import fr.enimaloc.catapult.service.dtdd.DtddCacheKind; +import fr.enimaloc.catapult.service.dtdd.DtddCacheRefetchJob; +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Page; +import org.springframework.web.server.ResponseStatusException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.*; + +class ApiAdminDtddCachesControllerTest { + + private final DtddCacheAdminService caches = mock(DtddCacheAdminService.class); + private final DtddCacheRefetchJob job = mock(DtddCacheRefetchJob.class); + private final ApiAdminDtddCachesController controller = new ApiAdminDtddCachesController(caches, job); + + @Test + void aCache_isNamedByItsKind() { + doReturn(Page.empty()).when(caches).list(DtddCacheKind.SEARCHES, "halo", 2, 50); + + assertThat(controller.list("searches", "halo", 2, 50)).isEmpty(); + controller.refetch("games", "7"); + verify(caches).refetch(DtddCacheKind.GAMES, "7"); + controller.entry("topics", "7"); + verify(caches).detail(DtddCacheKind.TOPICS, "7"); + } + + @Test + void anotherKind_isNotFound() { + assertThatThrownBy(() -> controller.list("keys", null, 0, 20)) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("404"); + verifyNoInteractions(caches); + } + + @Test + void theBulkRefetch_isStartedFollowedAndCancelled() { + DtddRefetchJobDto running = new DtddRefetchJobDto("RUNNING", 3, 0, 0, null, null, null, null); + when(job.start()).thenReturn(running); + when(job.status()).thenReturn(running); + when(job.cancel()).thenReturn(running); + + assertThat(controller.startJob()).isEqualTo(running); + assertThat(controller.job()).isEqualTo(running); + assertThat(controller.cancelJob()).isEqualTo(running); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsControllerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsControllerTest.java new file mode 100644 index 000000000..98a08776b --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminRateLimitsControllerTest.java @@ -0,0 +1,33 @@ +package fr.enimaloc.catapult.api.admin; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimiter; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +class ApiAdminRateLimitsControllerTest { + + private static final String HOST = "HOST store.steampowered.com"; + private static final LearnedRateLimitDto ROW = new LearnedRateLimitDto(HOST, "store.steampowered.com", null, null, + null, false, 20, 60, true, 30_000, false, 1, null, 0, null); + + private final LearnedRateLimiter limiter = mock(LearnedRateLimiter.class); + private final ApiAdminRateLimitsController controller = new ApiAdminRateLimitsController(limiter); + + @Test + void theLearnedLimits_areListedCorrectedAndReset() { + when(limiter.list()).thenReturn(List.of(ROW)); + when(limiter.correct(HOST, 20, 60, null, "ROUTE")).thenReturn(ROW); + when(limiter.reset(HOST)).thenReturn(ROW); + + assertThat(controller.list()).containsExactly(ROW); + assertThat(controller.correct(HOST, new LearnedRateLimitCorrectionDto(20, 60, null, "ROUTE"))).isEqualTo(ROW); + assertThat(controller.reset(HOST)).isEqualTo(ROW); + assertThat(ROW.hostScope()).isTrue(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerTest.java index 335a1f9aa..5ab04a28c 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerTest.java @@ -28,6 +28,7 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -45,13 +46,14 @@ class ApiAdminTwControllerTest { @MockitoBean AdminCclService adminCclService; @MockitoBean TwitchLoginSuccessHandler twitchLoginSuccessHandler; @MockitoBean TwBackfillService twBackfillService; + @MockitoBean fr.enimaloc.catapult.service.tw.TwDtddTopicLevels topicLevels; @Test void rebuildForcePinned_delegatesToBackfillService() throws Exception { MockMvc mvc = MockMvcBuilders.webAppContextSetup(wac).build(); mvc.perform(post("/api/internal/admin/tws/rebuild/force-pinned").with(csrf())) .andExpect(status().isAccepted()); - verify(twBackfillService).forceRebuildAllIncludingPinned(); + verify(twBackfillService).start(TwBackfillService.Mode.FORCE_PINNED); } @Test @@ -129,4 +131,62 @@ void testSteamKeywords_returnsMatchResult() throws Exception { .andExpect(jsonPath("$.draftKeywordMatch").value(true)) .andExpect(jsonPath("$.matchedContentDescriptorIds[0]").value(2)); } + + @Test + void theRebuild_isFollowedAndCancelled() throws Exception { + MockMvc mvc = MockMvcBuilders.webAppContextSetup(wac).build(); + var running = new fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto("RUNNING", "NON_OVERRIDDEN", 10, 3, 1, + 0, null, null, null); + when(twBackfillService.status()).thenReturn(running); + when(twBackfillService.cancel()).thenReturn(running); + + mvc.perform(get("/api/internal/admin/tws/rebuild/status")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.state").value("RUNNING")) + .andExpect(jsonPath("$.total").value(10)); + mvc.perform(post("/api/internal/admin/tws/rebuild/cancel").with(csrf())).andExpect(status().isOk()); + verify(twBackfillService).cancel(); + } + + @Test + void dtddTopics_comeWithTheirThresholdAndTheLevelsCached() throws Exception { + MockMvc mvc = MockMvcBuilders.webAppContextSetup(wac).build(); + TwDefinition gore = new TwDefinition(); + gore.setId("gore"); + when(adminTwService.get("gore")).thenReturn(Optional.of(gore)); + var blood = new fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping(); + blood.setTwId("gore"); + blood.setDtddTopicName("there's blood/gore"); + blood.setMinSeverity("Depicted"); + var excessive = new fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping(); + excessive.setTwId("gore"); + excessive.setDtddTopicName("there's excessive gore"); + when(adminTwService.listDtddTopics("gore")).thenReturn(List.of(blood, excessive)); + when(topicLevels.of(any())).thenReturn(java.util.Map.of("there's blood/gore", + List.of("Minor", "Depicted", "Excessive"))); + + mvc.perform(get("/api/internal/admin/tws/gore/dtdd-topics")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[0].topic").value("there's blood/gore")) + .andExpect(jsonPath("$[0].minSeverity").value("Depicted")) + .andExpect(jsonPath("$[0].knownLevels[2]").value("Excessive")) + .andExpect(jsonPath("$[1].minSeverity").doesNotExist()) + .andExpect(jsonPath("$[1].knownLevels").isEmpty()); + mvc.perform(get("/api/internal/admin/tws/nope/dtdd-topics")).andExpect(status().isNotFound()); + } + + @Test + void dtddTopics_areReplacedWithTheirThresholds_theOldBodyStillRead() throws Exception { + MockMvc mvc = MockMvcBuilders.webAppContextSetup(wac).build(); + mvc.perform(put("/api/internal/admin/tws/gore/dtdd-topics").with(csrf()).contentType("application/json") + .content("{\"topics\":[\"there's blood/gore\"],\"minSeverities\":{\"there's blood/gore\":\"Depicted\"}}")) + .andExpect(status().isNoContent()); + verify(adminTwService).replaceDtddTopics("gore", Set.of("there's blood/gore"), + java.util.Map.of("there's blood/gore", "Depicted")); + + mvc.perform(put("/api/internal/admin/tws/violence/dtdd-topics").with(csrf()).contentType("application/json") + .content("{\"topics\":[\"someone dies\"]}")) + .andExpect(status().isNoContent()); + verify(adminTwService).replaceDtddTopics("violence", Set.of("someone dies"), java.util.Map.of()); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerWithoutBackfillTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerWithoutBackfillTest.java index 94bc28942..0ab99172b 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerWithoutBackfillTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/admin/ApiAdminTwControllerWithoutBackfillTest.java @@ -17,7 +17,8 @@ class ApiAdminTwControllerWithoutBackfillTest { private final ApiAdminTwController controller = new ApiAdminTwController( - mock(AdminTwService.class), Optional.empty(), mock(AdminEventPublisher.class)); + mock(AdminTwService.class), Optional.empty(), mock(AdminEventPublisher.class), + mock(fr.enimaloc.catapult.service.tw.TwDtddTopicLevels.class)); @Test void rebuildNonOverridden_withoutBackfill_isServiceUnavailable() { diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ApiChannelTwControllerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ApiChannelTwControllerTest.java index 97a65329f..98b0bc4c1 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ApiChannelTwControllerTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ApiChannelTwControllerTest.java @@ -45,6 +45,7 @@ class ApiChannelTwControllerTest { @Autowired ObjectMapper om; @Autowired UserAccountRepository userAccountRepository; @MockitoBean BindingService bindingService; + @MockitoBean fr.enimaloc.catapult.service.binding.TwDetailsService twDetailsService; @MockitoBean TwitchLoginSuccessHandler twitchLoginSuccessHandler; private UserAccount seededUser() { @@ -152,4 +153,27 @@ void anotherMembersBindings_areForbidden() throws Exception { verifyNoInteractions(bindingService); } + + @Test + void twDetails_areTheOwners_andAnotherOnesBindingIs404() throws Exception { + MockMvc mvc = MockMvcBuilders.webAppContextSetup(wac).apply(springSecurity()).build(); + UserAccount user = seededUser(); + UUID bindingId = UUID.randomUUID(); + when(twDetailsService.details(any(UserAccount.class), eq(bindingId))).thenReturn(java.util.Optional.of( + new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto(false, java.util.List.of( + new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Tw("gore", "Gore", true, "DETECTED", + java.util.List.of())), java.util.List.of()))); + + mvc.perform(get("/api/internal/users/{userId}/bindings/{bindingId}/tws/details", user.getId(), bindingId) + .with(jwt().jwt(j -> j.subject(user.getId().toString()).claim("twitchId", user.getTwitchId())))) + .andExpect(status().isOk()) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath("$.tws[0].origin") + .value("DETECTED")); + + UUID other = UUID.randomUUID(); + when(twDetailsService.details(any(UserAccount.class), eq(other))).thenReturn(java.util.Optional.empty()); + mvc.perform(get("/api/internal/users/{userId}/bindings/{bindingId}/tws/details", user.getId(), other) + .with(jwt().jwt(j -> j.subject(user.getId().toString()).claim("twitchId", user.getTwitchId())))) + .andExpect(status().isNotFound()); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssemblerTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssemblerTest.java index 03b0e56df..4a3cfc844 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssemblerTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/channel/ChannelDashboardAssemblerTest.java @@ -643,6 +643,23 @@ void streamTitle_enrolsTheChannelInTheConditions_onlyWhileTheTitleIsOffered() { verify(streamTitle, never()).conditionsAvailableFor(owner); } + /** The title's drops are read only with its conditions (StreamTitleService#context): the help says the same. */ + @Test + void streamTitle_offersTheDropsOnlyWithTheConditions() { + when(streamTitle.isAvailableFor(owner)).thenReturn(true); + when(dropsGate.isAvailableFor(owner)).thenReturn(true); + + when(streamTitle.conditionsAvailableFor(owner)).thenReturn(true); + UserSettingsDto both = assembler().settings(owner); + when(streamTitle.conditionsAvailableFor(owner)).thenReturn(false); + UserSettingsDto noConditions = assembler().settings(owner); + + assertThat(both.titleConditionsAvailable()).isTrue(); + assertThat(both.titleDropsAvailable()).isTrue(); + assertThat(noConditions.titleConditionsAvailable()).isFalse(); + assertThat(noConditions.titleDropsAvailable()).isFalse(); + } + @Test void raidRestore_isOfferedOnlyWhileTheExperimentIsOnForTheChannel() { UserSettings saved = new UserSettings(); diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponseTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponseTest.java new file mode 100644 index 000000000..c9e26481c --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/api/userapi/DtddDetailResponseTest.java @@ -0,0 +1,45 @@ +package fr.enimaloc.catapult.api.userapi; + +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.json.JsonMapper; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** The public DTDD detail's topic stats: DTDD's own, with the topic's severity as DTDD gives it. */ +class DtddDetailResponseTest { + + private static DtddApiClient.DtddTopicItemStat stat(DtddApiClient.DtddSeverity severity) { + return new DtddApiClient.DtddTopicItemStat(395866, 57, 3, 7, 153, "a dog dies", 10752, severity); + } + + @Test + void topicItemStats_carryTheSeverity() { + DtddApiClient.DtddSeverity severity = new DtddApiClient.DtddSeverity( + List.of("Implied", "Depicted", "Excessive"), List.of(2, 9, 1), List.of(17, 75, 8), 12, 2); + + DtddDetailResponse.DtddTopicItemStat[] stats = + DtddDetailResponse.DtddTopicItemStat.of(new DtddApiClient.DtddTopicItemStat[]{stat(severity)}); + + assertThat(JsonMapper.builder().build().writeValueAsString(stats[0])).isEqualTo( + "{\"topicItemId\":395866,\"yesSum\":57,\"noSum\":3,\"numComments\":7,\"topicId\":153," + + "\"topicName\":\"a dog dies\",\"itemId\":10752,\"severity\":{\"levels\":[\"Implied\"," + + "\"Depicted\",\"Excessive\"],\"counts\":[2,9,1],\"percents\":[17,75,8],\"total\":12," + + "\"verified\":2}}"); + } + + @Test + void aTopicWithoutLevels_hasNoSeverity() { + DtddDetailResponse.DtddTopicItemStat[] stats = + DtddDetailResponse.DtddTopicItemStat.of(new DtddApiClient.DtddTopicItemStat[]{stat(null)}); + + assertThat(stats[0].severity()).isNull(); + } + + @Test + void noStats_staysNull() { + assertThat(DtddDetailResponse.DtddTopicItemStat.of(null)).isNull(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/event/listener/CustomBindingDetectionTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/event/listener/CustomBindingDetectionTest.java index 42efed344..3f701ec9a 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/event/listener/CustomBindingDetectionTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/event/listener/CustomBindingDetectionTest.java @@ -302,7 +302,7 @@ void editingAGameBinding_duringActivation_isSavedNotApplied() { BindingService realBindings = new BindingService(gameBindings, mock(IgdbService.class), mock(IgdbGameDetailsService.class), twitchService, mock(TwResolverService.class), mock(GameMappingRepository.class), new GameBindingLookup(gameBindings), realChain, - mock(ExtensionLayoutApplier.class)); + mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); realBindings.setTwitchGame(user, gameBinding.getId(), "1234", "Other"); diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotatorTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotatorTest.java index 9ac0d11f1..7de636f59 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotatorTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/DtddApiKeyRotatorTest.java @@ -94,4 +94,69 @@ void isAllKeysBlocked_trueWhenAllBlocked() { rotator.onKeyRateLimited("KEY_A", 60); assertThat(rotator.isAllKeysBlocked()).isTrue(); } + + private void twoKeys() { + when(repository.findByExclusiveFalse()).thenReturn(List.of( + new DtddApiKeyEntry("KEY_A"), + new DtddApiKeyEntry("KEY_B") + )); + rotator.refreshKeys(); + } + + @Test + void nextKeyAbove_skipsAKeyWhoseMonthIsDownToTheReserve() { + twoKeys(); + rotator.onQuota("KEY_A", 20, 500); + rotator.onQuota("KEY_B", 20, 501); + + for (int i = 0; i < 5; i++) { + assertThat(rotator.nextKeyAbove(500)).contains("KEY_B"); + } + // The detection still gets it + assertThat(List.of(rotator.nextKey().orElseThrow(), rotator.nextKey().orElseThrow())) + .containsExactlyInAnyOrder("KEY_A", "KEY_B"); + } + + @Test + void nextKeyAbove_countsAnUnknownMonthAsEnough_butNeverGivesABlockedKey() { + twoKeys(); + rotator.onKeyRateLimited("KEY_A", 60); + + assertThat(rotator.nextKeyAbove(500)).contains("KEY_B"); + rotator.onKeyRateLimited("KEY_B", 60); + assertThat(rotator.nextKeyAbove(500)).isEmpty(); + } + + @Test + void aMinuteSpent_restsTheKeyUntilTheNextOne() { + twoKeys(); + long before = System.currentTimeMillis(); + + rotator.onQuota("KEY_A", 0, 4000); + + assertThat(rotator.getKeyBlockedUntil().get("KEY_A")).isBetween(before + 59_000L, before + 61_000L); + assertThat(rotator.getQuotas().get("KEY_A").remainingMonth()).isEqualTo(4000); + } + + @Test + void nextUsableAt_isEmptyOnceEveryMonthIsDownToTheReserve() { + twoKeys(); + rotator.onKeyRateLimited("KEY_A", 30); + rotator.onQuota("KEY_B", 10, 100); + + assertThat(rotator.keysAbove(500)).isEqualTo(1); + assertThat(rotator.nextUsableAt(500)).hasValueSatisfying(at -> + assertThat(at).isGreaterThan(System.currentTimeMillis() + 25_000L)); + + rotator.onQuota("KEY_A", 10, 499); + assertThat(rotator.nextUsableAt(500)).isEmpty(); + assertThat(rotator.keysAbove(500)).isZero(); + } + + @Test + void aQuotaWithoutNumbers_isNotRemembered() { + twoKeys(); + rotator.onQuota("KEY_A", null, null); + assertThat(rotator.getQuotas()).isEmpty(); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClientTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClientTest.java index d8a8cc8aa..863ae9e14 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClientTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/getter/dtdd/RealDtddApiClientTest.java @@ -131,4 +131,88 @@ void fetchTopics_parsesPayload_on200() { assertThat(topics.get().mostlyTopics()).containsExactly("Flashing lights"); assertThat(topics.get().noTopics()).containsExactly("Bad ending"); } + + /** DTDD's answer as its v3 docs give it since 2026-11: a severity per topic, and fields we don't model. */ + @Test + void item_readsTheSeverity_andIgnoresUnknownFields() { + when(rotator.nextKey()).thenReturn(Optional.of("KEY_A")); + server.enqueue(new MockResponse() + .setHeader("Content-Type", "application/json") + .setBody(""" + { "id": 10752, "name": "A game", "somethingNew": { "x": 1 }, + "topicItemStats": [ + { "topicItemId": 395866, "yesSum": 57, "noSum": 3, "numComments": 7, "topicId": 153, + "topicName": "a dog dies", "itemId": 10752, + "severity": { "levels": ["Implied", "Depicted", "Excessive"], "counts": [2, 9, 1], + "percents": [17, 75, 8], "total": 12, "verified": 2 } }, + { "topicItemId": 395867, "yesSum": 1, "noSum": 4, "numComments": 0, "topicId": 154, + "topicName": "a cat dies", "itemId": 10752, "severity": null } + ] } + """)); + + Optional item = client.item(10752L); + + assertThat(item).isPresent(); + DtddApiClient.DtddTopicItemStat[] stats = item.get().topicItemStats(); + assertThat(stats).hasSize(2); + DtddApiClient.DtddSeverity severity = stats[0].severity(); + assertThat(severity.levels()).containsExactly("Implied", "Depicted", "Excessive"); + assertThat(severity.counts()).containsExactly(2, 9, 1); + assertThat(severity.percents()).containsExactly(17, 75, 8); + assertThat(severity.total()).isEqualTo(12); + assertThat(severity.verifiedLevel()).contains("Depicted"); + assertThat(stats[1].severity()).isNull(); + } + + @Test + void verifiedLevel_isEmpty_withoutAReviewersLevelInRange() { + List levels = List.of("Mild", "Shown", "Severe"); + assertThat(new DtddApiClient.DtddSeverity(levels, List.of(), List.of(), 0, null).verifiedLevel()).isEmpty(); + assertThat(new DtddApiClient.DtddSeverity(levels, List.of(), List.of(), 0, 0).verifiedLevel()).isEmpty(); + assertThat(new DtddApiClient.DtddSeverity(levels, List.of(), List.of(), 0, 4).verifiedLevel()).isEmpty(); + assertThat(new DtddApiClient.DtddSeverity(levels, List.of(), List.of(), 0, 1).verifiedLevel()).contains("Mild"); + } + + @Test + void everyAnswer_tellsTheRotatorWhatIsLeftOfTheKey() { + when(rotator.nextKey()).thenReturn(Optional.of("KEY_A")); + server.enqueue(new MockResponse() + .setHeader("Content-Type", "application/json") + .setHeader("X-RateLimit-Remaining-Minute", "28") + .setHeader("X-RateLimit-Remaining-Month", "4321") + .setBody("[]")); + + client.search("x", null); + + verify(rotator).onQuota("KEY_A", 28, 4321); + } + + @Test + void aReservingView_takesOnlyKeysAboveTheReserve_andAnswersEmptyWithoutOne() { + when(rotator.nextKeyAbove(500)).thenReturn(Optional.empty()); + + assertThat(client.reserving(500).item(1L)).isEmpty(); + assertThat(server.getRequestCount()).isZero(); + verify(rotator, never()).nextKey(); + } + + @Test + void fetchTopics_keepsEachTopicsSeverity_byItsName() { + when(rotator.nextKey()).thenReturn(Optional.of("KEY_A")); + server.enqueue(new MockResponse() + .setHeader("Content-Type", "application/json") + .setBody(""" + { "id": 7, "topicItemStats": [ + { "topicName": "a dog dies", "yesSum": 5, "noSum": 1, + "severity": { "levels": ["Implied", "Depicted"], "counts": [1, 4], "percents": [20, 80], + "total": 5, "verified": 2 } }, + { "topicName": "a cat dies", "yesSum": 0, "noSum": 3, "severity": null } + ] } + """)); + + DtddApiClient.DtddTopics topics = client.fetchTopics(7L).orElseThrow(); + + assertThat(topics.severities()).containsOnlyKeys("a dog dies"); + assertThat(topics.severities().get("a dog dies").verifiedLevel()).contains("Depicted"); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/monitoring/HttpClientUriTagConventionTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/monitoring/HttpClientUriTagConventionTest.java new file mode 100644 index 000000000..150706888 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/monitoring/HttpClientUriTagConventionTest.java @@ -0,0 +1,50 @@ +package fr.enimaloc.catapult.monitoring; + +import io.micrometer.common.KeyValue; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.observation.ClientRequestObservationContext; +import org.springframework.mock.http.client.MockClientHttpRequest; + +import static org.assertj.core.api.Assertions.assertThat; + +class HttpClientUriTagConventionTest { + + private final HttpClientUriTagConvention convention = new HttpClientUriTagConvention(); + + /* + * The Helix calls put the streamer's id in the query ("/helix/channels?broadcaster_id=" + id): one uri tag + * per streamer, until Micrometer's cap (100) drops every new uri, seen in production. + */ + @Test + void theQueryString_isLeftOutOfTheUriTag() { + assertThat(uriTag("https://api.twitch.tv/helix/channels?broadcaster_id=12345")) + .isEqualTo("/helix/channels"); + assertThat(uriTag("https://api.twitch.tv/helix/games?name={name}")).isEqualTo("/helix/games"); + } + + @Test + void aTemplateWithoutQuery_isTaggedAsBefore() { + assertThat(uriTag("https://api.twitch.tv/helix/users/extensions")).isEqualTo("/helix/users/extensions"); + assertThat(uriTag("campaigns/{id}/")).isEqualTo("/campaigns/{id}/"); + } + + @Test + void aRequestWithoutTemplate_isTaggedNone() { + assertThat(uriTag(null)).isEqualTo(KeyValue.NONE_VALUE); + } + + @Test + void keepsSpringsObservationName() { + assertThat(convention.getName()).isEqualTo("http.client.requests"); + } + + private String uriTag(String template) { + ClientRequestObservationContext context = new ClientRequestObservationContext( + new MockClientHttpRequest(HttpMethod.GET, "https://api.twitch.tv/helix/channels")); + context.setUriTemplate(template); + return convention.getLowCardinalityKeyValues(context).stream() + .filter(keyValue -> keyValue.getKey().equals("uri")) + .findFirst().orElseThrow().getValue(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimitInterceptorTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimitInterceptorTest.java new file mode 100644 index 000000000..93d0f81c0 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimitInterceptorTest.java @@ -0,0 +1,79 @@ +package fr.enimaloc.catapult.ratelimit; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClient; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +class LearnedRateLimitInterceptorTest { + + private static final RouteKey PAGE = new RouteKey("store.steampowered.com", "GET", "/app/{id}"); + + private LearnedRateLimiter limiter; + private MockRestServiceServer server; + private RestClient client; + + @BeforeEach + void setUp() { + limiter = mock(LearnedRateLimiter.class); + when(limiter.learns("store.steampowered.com")).thenReturn(true); + RestClient.Builder builder = RestClient.builder().requestInterceptor(new LearnedRateLimitInterceptor(limiter)); + server = MockRestServiceServer.bindTo(builder).build(); + client = builder.build(); + } + + @Test + void aCall_waitsForTheLimit_byItsTemplate_thenTellsTheAnswer() { + server.expect(requestTo("https://store.steampowered.com/app/620?l=french")).andRespond(withSuccess()); + + client.get().uri("https://store.steampowered.com/app/{id}?l={l}", 620, "french").retrieve().toBodilessEntity(); + + verify(limiter).acquire(PAGE); + verify(limiter).onResponse(eq(PAGE), eq(HttpStatus.OK), any(HttpHeaders.class)); + } + + @Test + void a429_isTold_beforeTheCallerSeesIt() { + server.expect(requestTo("https://store.steampowered.com/app/620")).andRespond(withStatus(HttpStatus.TOO_MANY_REQUESTS)); + + assertThatThrownBy(() -> client.get().uri("https://store.steampowered.com/app/{id}", 620).retrieve() + .toBodilessEntity()).isInstanceOf(HttpClientErrorException.TooManyRequests.class); + + verify(limiter).onResponse(eq(PAGE), eq(HttpStatus.TOO_MANY_REQUESTS), any(HttpHeaders.class)); + } + + @Test + void aLocalRefusal_sendsNothing_andIsNotLearned() { + HttpHeaders local = new HttpHeaders(); + local.set(LearnedRateLimiter.LOCAL_REFUSAL_HEADER, "true"); + doThrow(HttpClientErrorException.create(HttpStatus.TOO_MANY_REQUESTS, "local", local, new byte[0], null)) + .when(limiter).acquire(PAGE); + + assertThatThrownBy(() -> client.get().uri("https://store.steampowered.com/app/{id}", 620).retrieve() + .toBodilessEntity()).isInstanceOf(HttpClientErrorException.TooManyRequests.class); + + verify(limiter, never()).onResponse(any(), any(), any()); + server.verify(); + } + + @Test + void aDocumentedHost_passesUntouched() { + server.expect(requestTo("https://api.steampowered.com/ISteamUser/x")).andRespond(withSuccess()); + + client.get().uri("https://api.steampowered.com/ISteamUser/x").retrieve().toBodilessEntity(); + + verify(limiter, never()).acquire(any()); + verify(limiter, never()).onResponse(any(), any(), any()); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiterTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiterTest.java new file mode 100644 index 000000000..2c9ac5e25 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearnedRateLimiterTest.java @@ -0,0 +1,370 @@ +package fr.enimaloc.catapult.ratelimit; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; +import fr.enimaloc.catapult.domain.ratelimit.LearnedRateLimit; +import fr.enimaloc.catapult.repository.ratelimit.LearnedRateLimitRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.server.ResponseStatusException; + +import java.time.*; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +class LearnedRateLimiterTest { + + private static final String STORE = "store.steampowered.com"; + private static final RouteKey APPDETAILS = new RouteKey(STORE, "GET", "/api/appdetails"); + private static final RouteKey PAGE = new RouteKey(STORE, "GET", "/app/{id}"); + + /** A clock the test (and the limiter's waits) move. */ + static final class MovingClock extends Clock { + Instant now = Instant.parse("2026-11-12T10:00:00Z"); + + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now; } + void advance(Duration d) { now = now.plus(d); } + } + + private final MovingClock clock = new MovingClock(); + private final LearnedRateLimitProperties props = new LearnedRateLimitProperties(); + private final Map stored = new HashMap<>(); + private LearnedRateLimitRepository repository; + private LearnedRateLimiter limiter; + private final AtomicInteger slept = new AtomicInteger(); + + @BeforeEach + void setUp() { + repository = mock(LearnedRateLimitRepository.class); + when(repository.findById(anyString())).thenAnswer(i -> Optional.ofNullable(stored.get(i.getArgument(0)))); + when(repository.findAll()).thenAnswer(i -> new ArrayList<>(stored.values())); + when(repository.save(any())).thenAnswer(i -> { + LearnedRateLimit e = i.getArgument(0); + stored.put(e.getId(), e); + return e; + }); + limiter = new LearnedRateLimiter(props, repository, Optional.of(clock), Optional.empty()); + limiter.setSleeper(ms -> { + slept.addAndGet((int) ms); + clock.advance(Duration.ofMillis(ms)); + }); + } + + private void calls(RouteKey key, int n) { + for (int i = 0; i < n; i++) { + limiter.acquire(key); + limiter.onResponse(key, HttpStatus.OK, new HttpHeaders()); + clock.advance(Duration.ofSeconds(1)); + } + } + + private void tooMany(RouteKey key, String retryAfter) { + limiter.acquire(key); + HttpHeaders h = new HttpHeaders(); + if (retryAfter != null) h.set(HttpHeaders.RETRY_AFTER, retryAfter); + limiter.onResponse(key, HttpStatus.TOO_MANY_REQUESTS, h); + } + + /** The store's scopes known for an hour: a 429 then tells a limit. */ + private void known() { + limiter.blockedUntil(STORE, "/api/appdetails"); + limiter.blockedUntil(STORE, "/app/{id}"); + clock.advance(Duration.ofHours(1)); + } + + private LearnedRateLimitDto row(String id) { + return limiter.list().stream().filter(d -> d.id().equals(id)).findFirst().orElseThrow(); + } + + @Test + void documentedHosts_areLeftAlone() { + assertThat(limiter.learns("api.steampowered.com")).isFalse(); + assertThat(limiter.learns("eu.api.blizzard.com")).isFalse(); + assertThat(limiter.learns("")).isFalse(); + assertThat(limiter.learns(null)).isFalse(); + assertThat(limiter.learns(STORE)).isTrue(); + + RouteKey twitch = new RouteKey("api.twitch.tv", "GET", "/helix/users"); + limiter.acquire(twitch); + limiter.onResponse(twitch, HttpStatus.TOO_MANY_REQUESTS, new HttpHeaders()); + assertThat(stored).isEmpty(); + assertThat(limiter.blockedUntil("api.twitch.tv", "/helix/users")).isEmpty(); + + props.setEnabled(false); + assertThat(limiter.learns(STORE)).isFalse(); + } + + @Test + void a429_blocksTheWholeHost_andALiveCallElsewhereIsRefusedWithoutWaiting() { + known(); + calls(APPDETAILS, 20); + tooMany(APPDETAILS, null); + + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).contains(clock.now.plusSeconds(60)); + assertThat(limiter.blockedUntil(STORE, "/app/{id}")).contains(clock.now.plusSeconds(60)); + assertThatThrownBy(() -> limiter.acquire(PAGE)) + .isInstanceOfSatisfying(HttpClientErrorException.TooManyRequests.class, e -> { + assertThat(e.getResponseHeaders().getFirst(LearnedRateLimiter.LOCAL_REFUSAL_HEADER)).isEqualTo("true"); + assertThat(e.getResponseHeaders().getFirst(HttpHeaders.RETRY_AFTER)).isEqualTo("60"); + }); + assertThat(slept.get()).isZero(); + + LearnedRateLimitDto host = row("HOST " + STORE); + assertThat(host.limitCalls()).isEqualTo(20); + assertThat(host.limitWindowSeconds()).isEqualTo(60); + assertThat(host.rateLimitedCount()).isEqualTo(1); + assertThat(row(APPDETAILS.id()).rateLimitedCount()).isEqualTo(1); + assertThat(row(APPDETAILS.id()).scope()).isEqualTo("HOST"); + } + + @Test + void aLocalRefusal_isNeverLearned() { + HttpHeaders local = new HttpHeaders(); + local.set(LearnedRateLimiter.LOCAL_REFUSAL_HEADER, "true"); + limiter.onResponse(APPDETAILS, HttpStatus.TOO_MANY_REQUESTS, local); + assertThat(stored).isEmpty(); + } + + @Test + void theRetryAfter_isFollowed_andTeachesNoResumeDelay() { + known(); + calls(APPDETAILS, 5); + tooMany(APPDETAILS, "7"); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).contains(clock.now.plusSeconds(7)); + assertThat(row("HOST " + STORE).limitWindowSeconds()).isEqualTo(10); + + clock.advance(Duration.ofSeconds(8)); + calls(APPDETAILS, 1); + assertThat(row("HOST " + STORE).resumeMs()).isEqualTo(60_000); + } + + @Test + void theResumeDelay_isLearnedFromTheFirstCallAfterEachWait() { + calls(APPDETAILS, 3); + tooMany(APPDETAILS, null); + clock.advance(Duration.ofSeconds(60)); + calls(APPDETAILS, 1); + assertThat(row("HOST " + STORE).resumeMs()).isEqualTo(48_000); + + clock.advance(Duration.ofMinutes(5)); + tooMany(APPDETAILS, null); + clock.advance(Duration.ofSeconds(48)); + tooMany(APPDETAILS, null); + assertThat(row("HOST " + STORE).resumeMs()).isEqualTo(96_000); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).contains(clock.now.plusSeconds(96)); + } + + @Test + void bulkWork_keepsAMarginForLiveCalls_andWaitsItsTurn() { + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).isEmpty(); + limiter.correct("HOST " + STORE, 10, 60, null, null); + RateLimitPriority.run(RateLimitPriority.BULK, () -> false, () -> { + for (int i = 0; i < 8; i++) limiter.acquire(APPDETAILS); + }); + assertThat(slept.get()).isZero(); + + limiter.acquire(APPDETAILS); + limiter.acquire(APPDETAILS); + assertThatThrownBy(() -> limiter.acquire(PAGE)).isInstanceOf(HttpClientErrorException.TooManyRequests.class); + + RateLimitPriority.run(RateLimitPriority.BULK, () -> false, () -> limiter.acquire(APPDETAILS)); + assertThat(Duration.ofMillis(slept.get())).isEqualTo(Duration.ofSeconds(60)); + } + + @Test + void cancelledBulkWork_stopsWaiting() { + calls(APPDETAILS, 2); + tooMany(APPDETAILS, "600"); + AtomicInteger checks = new AtomicInteger(); + + assertThatThrownBy(() -> RateLimitPriority.run(RateLimitPriority.BULK, () -> checks.incrementAndGet() > 2, + () -> limiter.acquire(APPDETAILS))).isInstanceOf(HttpClientErrorException.TooManyRequests.class); + assertThat(slept.get()).isEqualTo(2_000); + } + + @Test + void anInterruptedWait_isRefused() { + calls(APPDETAILS, 2); + tooMany(APPDETAILS, "600"); + limiter.setSleeper(ms -> { throw new InterruptedException(); }); + + assertThatThrownBy(() -> RateLimitPriority.run(RateLimitPriority.BULK, () -> false, + () -> limiter.acquire(APPDETAILS))).isInstanceOf(HttpClientErrorException.TooManyRequests.class); + assertThat(Thread.interrupted()).isTrue(); + } + + @Test + void aCorrection_isKeptByTheLearning_untilReset() { + limiter.acquire(APPDETAILS); + limiter.correct("HOST " + STORE, 50, 60, 30, null); + calls(APPDETAILS, 10); + tooMany(APPDETAILS, null); + + LearnedRateLimitDto host = row("HOST " + STORE); + assertThat(host.limitCalls()).isEqualTo(50); + assertThat(host.limitCorrected()).isTrue(); + assertThat(host.resumeMs()).isEqualTo(30_000); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).contains(clock.now.plusSeconds(30)); + + LearnedRateLimitDto reset = limiter.reset("HOST " + STORE); + assertThat(reset.limitCalls()).isNull(); + assertThat(reset.limitCorrected()).isFalse(); + assertThat(reset.rateLimitedCount()).isZero(); + assertThat(reset.resumeMs()).isEqualTo(60_000); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).isEmpty(); + } + + @Test + void aRouteWithALimitOfItsOwn_isSplitOffItsHost() { + known(); + calls(APPDETAILS, 30); + tooMany(APPDETAILS, null); + clock.advance(Duration.ofMinutes(10)); + calls(PAGE, 2); + calls(APPDETAILS, 2); + + tooMany(APPDETAILS, null); + + assertThat(row(APPDETAILS.id()).scope()).isEqualTo("ROUTE"); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).isPresent(); + assertThat(limiter.blockedUntil(STORE, "/app/{id}")).isEmpty(); + limiter.acquire(PAGE); + } + + @Test + void anAdminsScope_isNeverChangedByTheLearning() { + limiter.acquire(APPDETAILS); + limiter.correct(APPDETAILS.id(), null, null, null, "route"); + assertThat(row(APPDETAILS.id()).scopePinned()).isTrue(); + tooMany(APPDETAILS, null); + assertThat(limiter.blockedUntil(STORE, "/app/{id}")).isEmpty(); + + limiter.correct(APPDETAILS.id(), null, null, null, "HOST"); + calls(PAGE, 30); + clock.advance(Duration.ofMinutes(5)); + tooMany(APPDETAILS, null); + assertThat(row(APPDETAILS.id()).scope()).isEqualTo("HOST"); + + limiter.correct(APPDETAILS.id(), null, null, null, "learned"); + assertThat(row(APPDETAILS.id()).scopePinned()).isFalse(); + assertThat(row(APPDETAILS.id()).scope()).isEqualTo("HOST"); + + limiter.correct(APPDETAILS.id(), null, null, null, "ROUTE"); + limiter.reset(APPDETAILS.id()); + assertThat(row(APPDETAILS.id()).scopePinned()).isFalse(); + } + + @Test + void anAdminsCorrectionOrReset_onAnotherInstance_isNeverWrittenOver() { + known(); + calls(APPDETAILS, 10); + LearnedRateLimiter other = new LearnedRateLimiter(props, repository, Optional.of(clock), Optional.empty()); + other.correct("HOST " + STORE, 50, 60, null, null); + + // This instance learns before syncing: it takes the correction rather than writing over it + tooMany(APPDETAILS, null); + assertThat(stored.get("HOST " + STORE).getOverrideCalls()).isEqualTo(50); + assertThat(row("HOST " + STORE).limitCalls()).isEqualTo(50); + + clock.advance(Duration.ofMinutes(5)); + other.reset("HOST " + STORE); + calls(APPDETAILS, 1); + clock.advance(Duration.ofHours(2)); + tooMany(APPDETAILS, null); + assertThat(stored.get("HOST " + STORE).getOverrideCalls()).isNull(); + } + + @Test + void aBadCorrection_isRefused() { + limiter.acquire(APPDETAILS); + String id = "HOST " + STORE; + assertThatThrownBy(() -> limiter.correct(id, 5, null, null, null)).isInstanceOf(ResponseStatusException.class); + assertThatThrownBy(() -> limiter.correct(id, 0, 60, null, null)).isInstanceOf(ResponseStatusException.class); + assertThatThrownBy(() -> limiter.correct(id, 5, 0, null, null)).isInstanceOf(ResponseStatusException.class); + assertThatThrownBy(() -> limiter.correct(id, null, null, 0, null)).isInstanceOf(ResponseStatusException.class); + assertThatThrownBy(() -> limiter.correct(id, null, null, null, "nope")).isInstanceOf(ResponseStatusException.class); + assertThatThrownBy(() -> limiter.reset("HOST unknown")).isInstanceOf(ResponseStatusException.class); + assertThat(limiter.correct(id, null, null, null, " ").limitCorrected()).isFalse(); + } + + @Test + void whatWasStored_isTakenBack_andSyncedFromTheOtherInstances() { + known(); + calls(APPDETAILS, 10); + tooMany(APPDETAILS, null); + LearnedRateLimiter other = new LearnedRateLimiter(props, repository, Optional.of(clock), + Optional.of(mock(org.springframework.transaction.PlatformTransactionManager.class))); + + assertThat(other.blockedUntil(STORE, "/api/appdetails")).isEmpty(); + assertThat(other.list()).extracting(LearnedRateLimitDto::id) + .containsExactly("HOST " + STORE, APPDETAILS.id(), PAGE.id()); + assertThat(other.list().getFirst().limitCalls()).isEqualTo(10); + assertThat(other.list().getFirst().hostScope()).isTrue(); + + other.correct(APPDETAILS.id(), 3, 10, 5, "ROUTE"); + limiter.sync(); + assertThat(row(APPDETAILS.id()).limitCalls()).isEqualTo(3); + assertThat(row(APPDETAILS.id()).scope()).isEqualTo("ROUTE"); + } + + @Test + void aFailingStore_neverFailsTheCalls() { + doThrow(new IllegalStateException("db down")).when(repository).findById(anyString()); + doThrow(new IllegalStateException("db down")).when(repository).save(any()); + doThrow(new IllegalStateException("db down")).when(repository).findAll(); + + calls(APPDETAILS, 2); + tooMany(APPDETAILS, null); + limiter.sync(); + + assertThat(limiter.list()).hasSize(2); + assertThat(limiter.blockedUntil(STORE, "/api/appdetails")).isPresent(); + } + + @Test + void longNearTheLimit_raisesIt() { + known(); + calls(APPDETAILS, 6); + tooMany(APPDETAILS, null); + assertThat(row("HOST " + STORE).limitCalls()).isEqualTo(6); + clock.advance(Duration.ofHours(2)); + calls(APPDETAILS, 6); + assertThat(row("HOST " + STORE).limitCalls()).isEqualTo(7); + verify(repository, atLeast(3)).save(any()); + } + + @Test + void pastTheRoutesAHostMayHave_newOnesShareOne() { + props.setMaxRoutesPerHost(2); + limiter.acquire(APPDETAILS); + limiter.acquire(PAGE); + limiter.acquire(new RouteKey(STORE, "GET", "/a")); + limiter.acquire(new RouteKey(STORE, "GET", "/b")); + limiter.acquire(APPDETAILS); + + assertThat(limiter.list()).extracting(LearnedRateLimitDto::path) + .containsExactly(null, "/api/appdetails", "/app/{id}", LearnedRateLimiter.OTHER_PATH); + } + + @Test + void retryAfter_readsSecondsOrADate() { + Instant now = Instant.parse("2026-11-12T10:00:00Z"); + assertThat(LearnedRateLimiter.retryAfter("30", now)).contains(Duration.ofSeconds(30)); + assertThat(LearnedRateLimiter.retryAfter("0", now)).contains(Duration.ofSeconds(1)); + assertThat(LearnedRateLimiter.retryAfter("Thu, 12 Nov 2026 10:01:00 GMT", now)).contains(Duration.ofMinutes(1)); + assertThat(LearnedRateLimiter.retryAfter("Thu, 12 Nov 2026 09:00:00 GMT", now)).contains(Duration.ofSeconds(1)); + assertThat(LearnedRateLimiter.retryAfter("soon", now)).isEmpty(); + assertThat(LearnedRateLimiter.retryAfter(" ", now)).isEmpty(); + assertThat(LearnedRateLimiter.retryAfter(null, now)).isEmpty(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearningTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearningTest.java new file mode 100644 index 000000000..f61ea6514 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/LearningTest.java @@ -0,0 +1,142 @@ +package fr.enimaloc.catapult.ratelimit; + +import fr.enimaloc.catapult.ratelimit.LimitEstimator.Estimate; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; + +/** The limit and the resume delay learned from 429s. */ +class LearningTest { + + private static final Instant T0 = Instant.parse("2026-11-12T10:00:00Z"); + + private static LimitEstimator calls(int n, Duration spread) { + LimitEstimator e = new LimitEstimator(Duration.ofHours(1), T0.minus(Duration.ofHours(1))); + for (int i = 0; i < n; i++) e.record(T0.plusMillis(spread.toMillis() * i / n)); + return e; + } + + @Test + void a429_limitsToTheCallsBeforeTheRefusedOne_inTheWindowOfTheWait() { + LimitEstimator e = calls(30, Duration.ofSeconds(50)); + + e.on429(T0.plusSeconds(50), Duration.ofSeconds(60)); + + assertThat(e.estimate()).contains(new Estimate(29, Duration.ofMinutes(1))); + } + + @Test + void a429_neverLoosensTheLimit_butAnotherWindowReplacesIt() { + LimitEstimator e = calls(30, Duration.ofSeconds(50)); + e.on429(T0.plusSeconds(50), Duration.ofSeconds(60)); + for (int i = 0; i < 20; i++) e.record(T0.plusSeconds(51)); + + e.on429(T0.plusSeconds(52), Duration.ofSeconds(30)); + assertThat(e.estimate()).contains(new Estimate(29, Duration.ofMinutes(1))); + + e.on429(T0.plusSeconds(52), Duration.ofSeconds(5)); + assertThat(e.estimate()).contains(new Estimate(23, Duration.ofSeconds(10))); + + e.on429(T0.plusSeconds(52), Duration.ofHours(3)); + assertThat(e.estimate()).contains(new Estimate(49, Duration.ofHours(1))); + } + + @Test + void oneFewCalls429_atMostHalvesTheLimit() { + LimitEstimator e = calls(30, Duration.ofSeconds(50)); + e.on429(T0.plusSeconds(50), Duration.ofSeconds(60)); + Instant later = T0.plus(Duration.ofMinutes(10)); + e.record(later); + + e.on429(later, Duration.ofSeconds(60)); + assertThat(e.estimate()).contains(new Estimate(15, Duration.ofMinutes(1))); + } + + @Test + void aWindowNotCountedWhole_teachesNoLimit() { + LimitEstimator e = new LimitEstimator(Duration.ofHours(1), T0); + e.record(T0.plusSeconds(1)); + e.on429(T0.plusSeconds(2), Duration.ofSeconds(60)); + assertThat(e.estimate()).isEmpty(); + } + + @Test + void aFirstLimit_needsEnoughCalls() { + LimitEstimator e = new LimitEstimator(Duration.ofHours(1), T0.minus(Duration.ofHours(1))); + e.on429(T0, Duration.ofSeconds(1)); + assertThat(e.estimate()).isEmpty(); + assertThat(e.oldestWithin(Duration.ofSeconds(10), T0)).isEmpty(); + + for (int i = 0; i < 6; i++) e.record(T0.plusSeconds(1)); + e.on429(T0.plusSeconds(2), Duration.ofSeconds(1)); + assertThat(e.estimate()).contains(new Estimate(5, Duration.ofSeconds(10))); + } + + @Test + void longNearTheLimitWithoutA429_raisesItByATenth() { + LimitEstimator e = calls(30, Duration.ofSeconds(50)); + e.on429(T0.plusSeconds(50), Duration.ofSeconds(60)); + Instant later = T0.plus(Duration.ofHours(2)); + + e.maybeRaise(later); + assertThat(e.estimate()).contains(new Estimate(29, Duration.ofMinutes(1))); + + for (int i = 24; i >= 0; i--) e.record(later.minusSeconds(i)); + e.maybeRaise(later); + assertThat(e.estimate()).contains(new Estimate(31, Duration.ofMinutes(1))); + + e.maybeRaise(later.plusSeconds(1)); + assertThat(e.estimate()).contains(new Estimate(31, Duration.ofMinutes(1))); + assertThat(e.oldestWithin(Duration.ofMinutes(1), later)).contains(later.minusSeconds(24)); + + e.forgetCalls(later); + assertThat(e.countWithin(Duration.ofHours(1), later)).isZero(); + e.on429(later.plusSeconds(30), Duration.ofSeconds(60)); + assertThat(e.estimate()).contains(new Estimate(31, Duration.ofMinutes(1))); + e.reset(); + e.maybeRaise(later); + assertThat(e.estimate()).isEmpty(); + e.set(new Estimate(5, Duration.ofSeconds(10)), null); + assertThat(e.estimate()).contains(new Estimate(5, Duration.ofSeconds(10))); + } + + @Test + void theResumeDelay_convergesWithoutGoingUnderWhatFailed() { + ResumeDelay d = new ResumeDelay(Duration.ofHours(24)); + assertThat(d.current()).isEqualTo(Duration.ofSeconds(60)); + + d.onSuccess(T0); + assertThat(d.current()).isEqualTo(Duration.ofSeconds(48)); + d.onFailure(T0); + assertThat(d.current()).isEqualTo(Duration.ofSeconds(96)); + d.onSuccess(T0); + assertThat(d.current()).isEqualTo(Duration.ofMillis(76_800)); + for (int i = 0; i < 10; i++) d.onSuccess(T0); + assertThat(d.current()).isEqualTo(Duration.ofSeconds(48)); + assertThat(d.state().ok()).isEqualTo(Duration.ofSeconds(48)); + assertThat(d.state().fail()).isEqualTo(Duration.ofSeconds(48)); + } + + @Test + void theResumeDelay_forgetsAnOldFailure_isCapped_andResets() { + ResumeDelay d = new ResumeDelay(Duration.ofHours(24)); + for (int i = 0; i < 6; i++) d.onFailure(T0); + assertThat(d.current()).isEqualTo(ResumeDelay.MAX); + + d.onSuccess(T0.plus(Duration.ofDays(2))); + assertThat(d.state().fail()).isNull(); + assertThat(d.current()).isEqualTo(Duration.ofMinutes(8)); + + d.restore(new ResumeDelay.State(Duration.ofMillis(1100), null, null, null)); + d.onSuccess(T0); + assertThat(d.current()).isEqualTo(Duration.ofSeconds(1)); + d.restore(new ResumeDelay.State(null, null, null, null)); + assertThat(d.current()).isEqualTo(ResumeDelay.INITIAL); + d.onFailure(T0); + d.reset(); + assertThat(d.state()).isEqualTo(new ResumeDelay.State(ResumeDelay.INITIAL, null, null, null)); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/RouteKeyTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/RouteKeyTest.java new file mode 100644 index 000000000..650b88092 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/ratelimit/RouteKeyTest.java @@ -0,0 +1,75 @@ +package fr.enimaloc.catapult.ratelimit; + +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpMethod; +import org.springframework.mock.http.client.MockClientHttpRequest; + +import java.net.URI; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +class RouteKeyTest { + + @Test + void theCallersTemplate_isTheRoute() { + MockClientHttpRequest request = new MockClientHttpRequest(HttpMethod.GET, + URI.create("https://Store.SteamPowered.com/app/620?l=french")); + request.getAttributes().put(RouteKey.URI_TEMPLATE_ATTRIBUTE, "https://store.steampowered.com/app/{id}?l={l}"); + + RouteKey key = RouteKey.of(request); + + assertThat(key).isEqualTo(new RouteKey("store.steampowered.com", "GET", "/app/{id}")); + assertThat(key.id()).isEqualTo("GET store.steampowered.com/app/{id}"); + } + + @Test + void aConcatenatedTemplate_isNormalizedToo() { + MockClientHttpRequest request = new MockClientHttpRequest(HttpMethod.GET, + URI.create("https://displaycatalog.mp.microsoft.com/v7.0/products/9NBLGGH4R315ABCD")); + request.getAttributes().put(RouteKey.URI_TEMPLATE_ATTRIBUTE, + "https://displaycatalog.mp.microsoft.com/v7.0/products/9NBLGGH4R315ABCD?market=FR"); + + assertThat(RouteKey.of(request).path()).isEqualTo("/v7.0/products/{id}"); + } + + @Test + void withoutATemplate_identifiersBecomeId() { + MockClientHttpRequest request = new MockClientHttpRequest(HttpMethod.GET, + URI.create("https://displaycatalog.mp.microsoft.com/v7.0/products/9NBLGGH4R315ABCD?market=FR")); + + assertThat(RouteKey.of(request).path()).isEqualTo("/v7.0/products/{id}"); + assertThat(RouteKey.normalizedPath("/users/xuid(123)/titles/42/")).isEqualTo("/users/{id}/titles/{id}/"); + assertThat(RouteKey.normalizedPath("")).isEqualTo("/"); + assertThat(RouteKey.normalizedPath(null)).isEqualTo("/"); + } + + @Test + void aTemplatesPath_dropsSchemeHostAndQuery() { + assertThat(RouteKey.templatePath("/api/appdetails?appids={id}")).isEqualTo("/api/appdetails"); + assertThat(RouteKey.templatePath("https://host.example")).isEqualTo("/"); + assertThat(RouteKey.templatePath("?x={x}")).isEqualTo("/"); + } + + @Test + void thePriority_isLiveUnlessSetAndComesBackAfter() { + AtomicReference inner = new AtomicReference<>(); + AtomicReference innerCancelled = new AtomicReference<>(); + + assertThat(RateLimitPriority.current()).isEqualTo(RateLimitPriority.LIVE); + assertThat(RateLimitPriority.cancelled()).isFalse(); + RateLimitPriority.run(RateLimitPriority.BULK, () -> true, () -> { + String nested = RateLimitPriority.call(RateLimitPriority.LIVE, () -> false, () -> { + inner.set(RateLimitPriority.current()); + return "ok"; + }); + assertThat(nested).isEqualTo("ok"); + innerCancelled.set(RateLimitPriority.cancelled()); + assertThat(RateLimitPriority.current()).isEqualTo(RateLimitPriority.BULK); + }); + + assertThat(inner.get()).isEqualTo(RateLimitPriority.LIVE); + assertThat(innerCancelled.get()).isTrue(); + assertThat(RateLimitPriority.current()).isEqualTo(RateLimitPriority.LIVE); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/AdminGameMappingServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/AdminGameMappingServiceTest.java index 1ba517c65..0a4de32c7 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/AdminGameMappingServiceTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/AdminGameMappingServiceTest.java @@ -88,7 +88,7 @@ void setUp() { chain = spy(new CategoryChain(activations, streamStateService, userSettingsRepository, raidRestoreHold, gameStateService, mock(UserAccountRepository.class), new CategoryApplier(twitchService, streamStateService), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class))); bindingService = spy(new BindingService(bindingRepo, igdbService, igdbGameDetailsService, twitchService, - twResolverService, mappingRepo, new GameBindingLookup(bindingRepo), chain, mock(ExtensionLayoutApplier.class))); + twResolverService, mappingRepo, new GameBindingLookup(bindingRepo), chain, mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class))); service = new AdminGameMappingService(mappingRepo, bindingRepo, userRepo, bindingService, gameStateService, streamStateService, channelEventPublisher, chain, transactionManager); @@ -548,4 +548,167 @@ void streamOfflineAfterTheCommit_isSkipped_neitherCountedNorPending() { verify(twitchService, never()).updateChannelReporting(any(), any()); verify(streamStateService, never()).storePending(any(), any()); } + + // ---- Ignored by default + + private static GameMappingRequest ignoring(String sourceId, String twitchId, boolean applyLive) { + return new GameMappingRequest("STEAM", sourceId, twitchId, twitchId == null ? null : "Category " + twitchId, + null, null, applyLive, true); + } + + @Test + void ignoredByDefault_withoutCategory_ignoresTheBindings_andLeavesTheirCategoryToIgdb() { + GameBinding auto = binding(user("a"), "730", Status.AUTO); + GameBinding incomplete = binding(user("b"), "730", Status.INCOMPLETE); + bindingsOfTheGame(auto, incomplete); + + GameMappingResult result = service.create(ignoring("730", null, false), admin); + + assertThat(result.bindingsUpdated()).isEqualTo(2); + assertThat(List.of(auto, incomplete)).allSatisfy(b -> assertThat(b.isIgnored()).isTrue()); + assertThat(auto.getStatus()).isEqualTo(Status.AUTO); + assertThat(auto.getTwitchGameId()).isEqualTo("old"); + assertThat(incomplete.getStatus()).isEqualTo(Status.INCOMPLETE); + assertThat(result.mapping().ignoredByDefault()).isTrue(); + assertThat(result.mapping().twitchGameId()).isNull(); + // Nothing to ask IGDB for a mapping that sets no category + verifyNoInteractions(igdbService); + } + + @Test + void ignoredByDefault_withACategory_mapsAndIgnores() { + GameBinding auto = binding(user("a"), "730", Status.AUTO); + bindingsOfTheGame(auto); + + GameMappingResult result = service.create(ignoring("730", "32399", false), admin); + + assertThat(result.bindingsUpdated()).isEqualTo(1); + assertThat(auto.getStatus()).isEqualTo(Status.MAPPED); + assertThat(auto.getTwitchGameId()).isEqualTo("32399"); + assertThat(auto.isIgnored()).isTrue(); + } + + @Test + void ignoredByDefault_leavesManualBindingsAndTheStreamersChoiceAlone() { + GameBinding manual = binding(user("a"), "730", Status.MANUAL); + GameBinding chosen = binding(user("b"), "730", Status.AUTO); + chosen.setIgnoredOverride(true); + bindingsOfTheGame(manual, chosen); + + GameMappingResult result = service.create(ignoring("730", null, false), admin); + + assertThat(result.bindingsUpdated()).isZero(); + assertThat(manual.isIgnored()).isFalse(); + assertThat(chosen.isIgnored()).isFalse(); + verify(bindingRepo, never()).save(any(GameBinding.class)); + } + + @Test + void withoutCategory_isRefused_unlessIgnoredByDefault() { + assertThatThrownBy(() -> service.create( + new GameMappingRequest("STEAM", "730", " ", null, null, null, false, false), admin)) + .isInstanceOf(ResponseStatusException.class) + .hasMessageContaining("Missing Twitch category"); + // Half a category is refused all the same + assertThatThrownBy(() -> service.create( + new GameMappingRequest("STEAM", "730", "32399", " ", null, null, false, true), admin)) + .isInstanceOf(ResponseStatusException.class) + .hasMessageContaining("Missing Twitch category"); + } + + @Test + void turningTheIgnoreOff_stopsIgnoringOnlyWhatItIgnored() { + GameMappingResult created = service.create(ignoring("730", null, false), admin); + GameBinding byTheMapping = binding(user("a"), "730", Status.AUTO); + byTheMapping.setIgnored(true); + GameBinding byTheStreamer = binding(user("b"), "730", Status.AUTO); + byTheStreamer.setIgnored(true); + byTheStreamer.setIgnoredOverride(true); + bindingsOfTheGame(byTheMapping, byTheStreamer); + + GameMappingResult result = service.update(created.mapping().id(), request("730", "32399", false), admin); + + assertThat(byTheMapping.isIgnored()).isFalse(); + assertThat(byTheMapping.getStatus()).isEqualTo(Status.MAPPED); + assertThat(byTheStreamer.isIgnored()).isTrue(); + // Mapped, but still ignored by its streamer + assertThat(byTheStreamer.getStatus()).isEqualTo(Status.MAPPED); + assertThat(result.bindingsUpdated()).isEqualTo(2); + assertThat(result.mapping().ignoredByDefault()).isFalse(); + } + + @Test + void anUpdateKeepingTheIgnore_doesNotIgnoreAgainWhatTheStreamerStoppedIgnoring() { + GameMappingResult created = service.create(ignoring("730", null, false), admin); + // Ignored by the mapping, then not by its streamer + GameBinding unignored = binding(user("a"), "730", Status.AUTO); + unignored.setIgnoredOverride(true); + bindingsOfTheGame(unignored); + + service.update(created.mapping().id(), ignoring("730", "32399", false), admin); + + assertThat(unignored.isIgnored()).isFalse(); + } + + @Test + void anUpdateDroppingTheCategory_sendsTheMappedBindingsBackToIgdb() { + GameMappingResult created = service.create(request("730", "32399", false), admin); + GameBinding mapped = binding(user("a"), "730", Status.MAPPED); + bindingsOfTheGame(mapped); + + service.update(created.mapping().id(), ignoring("730", null, false), admin); + + verify(bindingService).resolveWithoutMapping(mapped); + // IGDB finds nothing here: incomplete again, and ignored + assertThat(mapped.getStatus()).isEqualTo(Status.INCOMPLETE); + assertThat(mapped.isIgnored()).isTrue(); + } + + @Test + void deletingAnIgnoringMapping_stopsIgnoringItsBindings() { + GameMappingResult created = service.create(ignoring("730", null, false), admin); + GameBinding ignored = binding(user("a"), "730", Status.AUTO); + ignored.setIgnored(true); + bindingsOfTheGame(ignored); + + GameMappingResult result = service.delete(created.mapping().id(), false); + + assertThat(ignored.isIgnored()).isFalse(); + assertThat(result.bindingsUpdated()).isEqualTo(1); + } + + @Test + void movingAnIgnoringMapping_stopsIgnoringTheOldGame() { + GameMappingResult created = service.create(ignoring("730", null, false), admin); + GameBinding oldGame = binding(user("a"), "730", Status.AUTO); + oldGame.setIgnored(true); + GameBinding newGame = binding(user("b"), "570", Status.AUTO); + bindingsOfTheGame(oldGame); + when(bindingRepo.lockAllBySourceTypeAndSourceIdAndStatusIn(eq(SourceType.STEAM), eq("570"), anyCollection())) + .thenReturn(new ArrayList<>(List.of(newGame))); + + service.update(created.mapping().id(), ignoring("570", null, false), admin); + + assertThat(oldGame.isIgnored()).isFalse(); + assertThat(newGame.isIgnored()).isTrue(); + } + + /** A binding it just ignored goes to the no-game fallback live, unlike one ignored already. */ + @Test + void applyLive_pushesTheBindingsItJustIgnored() { + UserAccount onIt = user("on-it"); + UserAccount alreadyIgnored = user("already"); + GameBinding justIgnored = binding(onIt, "730", Status.AUTO); + GameBinding stillIgnored = binding(alreadyIgnored, "730", Status.AUTO); + stillIgnored.setIgnored(true); + stillIgnored.setIgnoredOverride(true); + bindingsOfTheGame(justIgnored, stillIgnored); + liveOn(onIt, "730"); + liveOn(alreadyIgnored, "730"); + + service.create(ignoring("730", "32399", true), admin); + + verify(chain).apply(eq(onIt), any(MappingChanged.class)); + verify(chain, never()).apply(eq(alreadyIgnored), any()); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceEditsTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceEditsTest.java index f180b46e8..b40a02a20 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceEditsTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceEditsTest.java @@ -55,7 +55,7 @@ class BindingServiceEditsTest { private final BindingService service = new BindingService(bindings, igdb, igdbDetails, twitch, twResolver, mappings, new GameBindingLookup(bindings), new CategoryChain(mock(CustomBindingActivations.class), streamState, mock(UserSettingsRepository.class), new RaidRestoreHold(), - gameState, mock(UserAccountRepository.class), new CategoryApplier(twitch, streamState), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), layouts); + gameState, mock(UserAccountRepository.class), new CategoryApplier(twitch, streamState), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), layouts, org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); private UserAccount user; private GameBinding binding; diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceMappingTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceMappingTest.java index 45ed8b0a8..8868b8e15 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceMappingTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceMappingTest.java @@ -51,7 +51,7 @@ class BindingServiceMappingTest { new BindingService(bindings, igdb, igdbDetails, twitch, twResolver, mappings, new GameBindingLookup(bindings), new CategoryChain(mock(CustomBindingActivations.class), mock(StreamStateService.class), mock(UserSettingsRepository.class), new RaidRestoreHold(), - gameState, mock(UserAccountRepository.class), new CategoryApplier(twitch, mock(StreamStateService.class)), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), mock(ExtensionLayoutApplier.class)); + gameState, mock(UserAccountRepository.class), new CategoryApplier(twitch, mock(StreamStateService.class)), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); private final DetectedGame cs2 = new DetectedGame("730", GameBinding.SourceType.STEAM, "Counter-Strike 2"); private UserAccount user; @@ -299,4 +299,76 @@ void resolveWithoutMapping_fallsBackToIgdb() { assertThat(service.resolveWithoutMapping(lost).getStatus()).isEqualTo(GameBinding.Status.INCOMPLETE); } + + @Test + void newDetection_ofAGameIgnoredByDefault_comesInIgnored_withIgdbsCategoryWithoutOne() { + mapping.setIgnoredByDefault(true); + mapping.setTwitchGameId(null); + mapping.setTwitchGameName(null); + when(bindings.findByUserAndSourceIdAndSourceType(user, "730", GameBinding.SourceType.STEAM)) + .thenReturn(Optional.empty()); + + GameBinding created = service.resolveOrCreate(user, cs2); + + assertThat(created.isIgnored()).isTrue(); + assertThat(created.isIgnoredOverride()).isFalse(); + assertThat(created.getStatus()).isEqualTo(GameBinding.Status.AUTO); + assertThat(created.getTwitchGameId()).isEqualTo("igdb-twitch"); + } + + @Test + void applyMapping_ofAnIgnoreOnlyMapping_changesNothing() { + mapping.setTwitchGameId(null); + mapping.setTwitchGameName(null); + GameBinding binding = binding(GameBinding.Status.AUTO); + + assertThat(service.applyMapping(binding, mapping, BindingService.MappingSuggestions.NONE)).isFalse(); + assertThat(binding.getStatus()).isEqualTo(GameBinding.Status.AUTO); + assertThat(binding.getTwitchGameId()).isEqualTo("old"); + } + + @Test + void toggleIgnored_isTheStreamersChoice() { + GameBinding binding = binding(GameBinding.Status.MAPPED); + + service.toggleIgnored(user, binding.getId(), true); + + assertThat(binding.isIgnored()).isTrue(); + assertThat(binding.isIgnoredOverride()).isTrue(); + // An admin mapping no longer changes it + assertThat(service.applyMappingIgnore(binding, false)).isFalse(); + assertThat(binding.isIgnored()).isTrue(); + } + + @Test + void updateBinding_marksTheStreamersChoice_onlyWhenTheIgnoreChanges() { + GameBinding kept = binding(GameBinding.Status.AUTO); + GameBinding switched = binding(GameBinding.Status.AUTO); + + service.updateBinding(user, kept.getId(), "1", "One", Set.of(), false); + service.updateBinding(user, switched.getId(), "1", "One", Set.of(), true); + + assertThat(kept.isIgnoredOverride()).isFalse(); + assertThat(switched.isIgnoredOverride()).isTrue(); + } + + @Test + void applyMappingIgnore_leavesManualBindingsAlone() { + GameBinding manual = binding(GameBinding.Status.MANUAL); + + assertThat(service.applyMappingIgnore(manual, true)).isFalse(); + assertThat(manual.isIgnored()).isFalse(); + } + + @Test + void resetToDefaults_followsTheMappingsIgnore_andForgetsTheStreamersChoice() { + mapping.setIgnoredByDefault(true); + GameBinding binding = binding(GameBinding.Status.MAPPED); + binding.setIgnoredOverride(true); + + service.resetToDefaults(user, binding.getId()); + + assertThat(binding.isIgnored()).isTrue(); + assertThat(binding.isIgnoredOverride()).isFalse(); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceSteamLimitTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceSteamLimitTest.java new file mode 100644 index 000000000..291851d7f --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceSteamLimitTest.java @@ -0,0 +1,177 @@ +package fr.enimaloc.catapult.service.binding; + +import fr.enimaloc.catapult.domain.account.UserAccount; +import fr.enimaloc.catapult.domain.binding.GameBinding; +import fr.enimaloc.catapult.domain.binding.GameMapping; +import fr.enimaloc.catapult.event.TwsMissedSteamEvent; +import fr.enimaloc.catapult.getter.DetectedGame; +import fr.enimaloc.catapult.repository.account.UserAccountRepository; +import fr.enimaloc.catapult.repository.account.UserSettingsRepository; +import fr.enimaloc.catapult.repository.binding.GameBindingRepository; +import fr.enimaloc.catapult.repository.binding.GameMappingRepository; +import fr.enimaloc.catapult.service.category.CategoryApplier; +import fr.enimaloc.catapult.service.category.CategoryChain; +import fr.enimaloc.catapult.service.category.ExtensionLayoutApplier; +import fr.enimaloc.catapult.service.igdb.IgdbGameDetailsService; +import fr.enimaloc.catapult.service.igdb.IgdbService; +import fr.enimaloc.catapult.service.tw.TwResolverService; +import fr.enimaloc.catapult.service.twitch.TwitchService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; + +import java.time.Instant; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** A Steam game's TWs resolved while the Steam store is rate limited: told, then resolved again once it may be called. */ +class BindingServiceSteamLimitTest { + + private static final Instant UNTIL = Instant.parse("2026-11-12T10:05:00Z"); + + private final GameBindingRepository bindings = mock(GameBindingRepository.class); + private final GameMappingRepository mappings = mock(GameMappingRepository.class); + private final IgdbService igdb = mock(IgdbService.class); + private final IgdbGameDetailsService igdbDetails = mock(IgdbGameDetailsService.class); + private final TwitchService twitch = mock(TwitchService.class); + private final TwResolverService twResolver = mock(TwResolverService.class); + private final ApplicationEventPublisher events = mock(ApplicationEventPublisher.class); + private final BindingService service = new BindingService(bindings, igdb, igdbDetails, twitch, twResolver, mappings, + new GameBindingLookup(bindings), new CategoryChain(mock(CustomBindingActivations.class), + mock(StreamStateService.class), mock(UserSettingsRepository.class), new RaidRestoreHold(), + mock(GameStateService.class), mock(UserAccountRepository.class), + new CategoryApplier(twitch, mock(StreamStateService.class)), mock(RaidRestoreGateService.class), + mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), mock(ExtensionLayoutApplier.class), events); + + private final UserAccount user = new UserAccount(); + private final DetectedGame steamGame = new DetectedGame("730", GameBinding.SourceType.STEAM, "Counter-Strike 2"); + + @BeforeEach + void setUp() { + user.setId(UUID.randomUUID()); + when(bindings.save(any())).thenAnswer(call -> { + GameBinding b = call.getArgument(0); + if (b.getId() == null) b.setId(UUID.randomUUID()); + return b; + }); + when(mappings.findBySourceTypeAndSourceId(any(), any())).thenReturn(Optional.empty()); + when(igdb.findByExternalAppId(GameBinding.SourceType.STEAM, "730")) + .thenReturn(Optional.of(new IgdbService.IgdbGame("igdb-1", "Counter-Strike 2"))); + when(igdbDetails.getDetails(anyString())).thenReturn(Optional.empty()); + when(igdb.findTwitchGameId("igdb-1")).thenReturn(Optional.of("32399")); + when(igdb.resolveIgdbIdForBinding(any())).thenReturn(Optional.of("igdb-1")); + when(twResolver.suggest(any())).thenReturn(Set.of("violence")); + when(twResolver.steamRateLimitedUntil()).thenReturn(Optional.of(UNTIL)); + } + + private GameBinding binding(GameBinding.Status status) { + GameBinding b = new GameBinding(); + b.setId(UUID.randomUUID()); + b.setUser(user); + b.setSourceType(GameBinding.SourceType.STEAM); + b.setSourceId("730"); + b.setSourceName("Counter-Strike 2"); + b.setStatus(status); + b.getTws().add("old"); + when(bindings.findById(b.getId())).thenReturn(Optional.of(b)); + return b; + } + + @Test + void aSteamGameResolvedWhileLimited_isTold() { + when(bindings.findByUserAndSourceIdAndSourceType(user, "730", GameBinding.SourceType.STEAM)) + .thenReturn(Optional.empty()); + + GameBinding created = service.resolveOrCreate(user, steamGame); + + verify(events).publishEvent(new TwsMissedSteamEvent(created.getId(), UNTIL)); + } + + @Test + void notWhenTheStoreMayBeCalled_norForAnotherPlatform() { + when(twResolver.steamRateLimitedUntil()).thenReturn(Optional.empty()); + when(bindings.findByUserAndSourceIdAndSourceType(any(), any(), any())).thenReturn(Optional.empty()); + service.resolveOrCreate(user, steamGame); + + when(twResolver.steamRateLimitedUntil()).thenReturn(Optional.of(UNTIL)); + when(igdb.findByExternalAppId(GameBinding.SourceType.XBOX, "x1")) + .thenReturn(Optional.of(new IgdbService.IgdbGame("igdb-2", "Halo"))); + service.resolveOrCreate(user, new DetectedGame("x1", GameBinding.SourceType.XBOX, "Halo")); + + verify(events, never()).publishEvent(any(Object.class)); + } + + @Test + void notWhenAnAdminMappingSetsTheTws() { + GameMapping mapping = new GameMapping(); + mapping.setTwitchGameId("32399"); + mapping.setTwitchGameName("CS"); + mapping.setTwsOverridden(true); + when(mappings.findBySourceTypeAndSourceId(GameBinding.SourceType.STEAM, "730")).thenReturn(Optional.of(mapping)); + when(bindings.findByUserAndSourceIdAndSourceType(any(), any(), any())).thenReturn(Optional.empty()); + + service.resolveOrCreate(user, steamGame); + + verify(events, never()).publishEvent(any(Object.class)); + } + + @Test + void theRetry_savesTheTwsOnceSteamAnswers() { + GameBinding binding = binding(GameBinding.Status.AUTO); + when(twResolver.suggestReporting(any())).thenReturn( + new TwResolverService.Suggestion(Set.of("violence", "gore"), Optional.empty())); + + Optional retried = service.retryTwsAfterSteamLimit(binding.getId()); + + assertThat(retried).contains(binding); + assertThat(binding.getTws()).containsExactlyInAnyOrder("violence", "gore"); + verify(bindings).save(binding); + } + + @Test + void theRetry_stillLimited_isToldAgain_andChangesNothing() { + GameBinding binding = binding(GameBinding.Status.AUTO); + Instant later = UNTIL.plusSeconds(120); + when(twResolver.suggestReporting(any())).thenReturn( + new TwResolverService.Suggestion(Set.of("violence"), Optional.of(later))); + + assertThat(service.retryTwsAfterSteamLimit(binding.getId())).isEmpty(); + assertThat(binding.getTws()).containsExactly("old"); + verify(events).publishEvent(new TwsMissedSteamEvent(binding.getId(), later)); + verify(bindings, never()).save(any()); + } + + @Test + void theRetry_leavesTheStreamersAndAnAdminsChoiceAlone() { + GameBinding pinned = binding(GameBinding.Status.AUTO); + pinned.setTwOverride(true); + GameBinding manual = binding(GameBinding.Status.MANUAL); + GameBinding mapped = binding(GameBinding.Status.MAPPED); + GameMapping mapping = new GameMapping(); + mapping.setTwsOverridden(true); + when(mappings.findBySourceTypeAndSourceId(GameBinding.SourceType.STEAM, "730")).thenReturn(Optional.of(mapping)); + + assertThat(service.retryTwsAfterSteamLimit(pinned.getId())).isEmpty(); + assertThat(service.retryTwsAfterSteamLimit(manual.getId())).isEmpty(); + assertThat(service.retryTwsAfterSteamLimit(mapped.getId())).isEmpty(); + verify(twResolver, never()).suggestReporting(any()); + } + + @Test + void theRetry_ofUnchangedTws_pushesNothing() { + GameBinding binding = binding(GameBinding.Status.AUTO); + when(twResolver.suggestReporting(any())).thenReturn(new TwResolverService.Suggestion(Set.of("old"), Optional.empty())); + + assertThat(service.retryTwsAfterSteamLimit(binding.getId())).isEmpty(); + verify(bindings, never()).save(any()); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceTest.java index e74559ede..917b30b51 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/BindingServiceTest.java @@ -60,7 +60,7 @@ void setup() { bindingService = new BindingService(gameBindingRepository, igdbService, igdbGameDetailsService, twitchService, twResolverService, gameMappingRepository, new GameBindingLookup(gameBindingRepository), new CategoryChain(mock(CustomBindingActivations.class), streamState, mock(UserSettingsRepository.class), new RaidRestoreHold(), gameStateService, - mock(UserAccountRepository.class), new CategoryApplier(twitchService, streamState), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), mock(ExtensionLayoutApplier.class)); + mock(UserAccountRepository.class), new CategoryApplier(twitchService, streamState), mock(RaidRestoreGateService.class), mock(GameBindingLookup.class), mock(ExtensionLayoutApplier.class)), mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); user = new UserAccount(); bindingId = UUID.randomUUID(); @@ -261,6 +261,34 @@ void resolveOrCreate_usesIgdbExternalGamesTwitchId_beforeNameMatching() { verifyNoInteractions(twitchService); } + /* + * A game without a Twitch external_game but with a Twitch website keeps that website's URL under "twitch" + * (IgdbWebsiteKeys#links): sent as the game_id, Twitch refused every update ("The ID in game_id is not valid"). + */ + @Test + void resolveOrCreate_ignoresATwitchWebsiteUrl_forTheCategoryId() { + DetectedGame game = new DetectedGame("steam-123", GameBinding.SourceType.STEAM, "Portal"); + when(gameBindingRepository.findByUserAndSourceIdAndSourceType(user, "steam-123", GameBinding.SourceType.STEAM)) + .thenReturn(Optional.empty()); + IgdbService.IgdbGame igdbGame = new IgdbService.IgdbGame("1234", "Portal"); + when(igdbService.findByExternalAppId(GameBinding.SourceType.STEAM, "steam-123")).thenReturn(Optional.of(igdbGame)); + + IgdbGameDetails details = new IgdbGameDetails(); + details.setIgdbId("1234"); + details.setWebsites(Map.of("twitch", "https://www.twitch.tv/valve")); + when(igdbGameDetailsService.getDetails("1234")).thenReturn(Optional.of(details)); + when(igdbService.findTwitchGameId("1234")).thenReturn(Optional.empty()); + when(twitchService.findCategoryIdByName(user, "Portal")).thenReturn(Optional.of("6187")); + + when(igdbService.suggestCcls("1234")).thenReturn(Set.of()); + when(gameBindingRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + + GameBinding result = bindingService.resolveOrCreate(user, game); + + assertThat(result.getStatus()).isEqualTo(GameBinding.Status.AUTO); + assertThat(result.getTwitchGameId()).isEqualTo("6187"); + } + @Test void resolveOrCreate_noExisting_igdbNotFound_createsIncompleteBinding() { DetectedGame game = new DetectedGame(null, GameBinding.SourceType.MANUAL, "Unknown Game"); diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwDetailsServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwDetailsServiceTest.java new file mode 100644 index 000000000..3ab9f38a0 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwDetailsServiceTest.java @@ -0,0 +1,136 @@ +package fr.enimaloc.catapult.service.binding; + +import fr.enimaloc.catapult.common.dto.channel.TwDetailsDto; +import fr.enimaloc.catapult.domain.account.UserAccount; +import fr.enimaloc.catapult.domain.binding.GameBinding; +import fr.enimaloc.catapult.domain.binding.GameMapping; +import fr.enimaloc.catapult.domain.tw.TwDefinition; +import fr.enimaloc.catapult.repository.binding.GameMappingRepository; +import fr.enimaloc.catapult.repository.tw.TwDefinitionRepository; +import fr.enimaloc.catapult.service.tw.TwResolverService; +import fr.enimaloc.catapult.service.tw.TwResolverService.Signal; +import fr.enimaloc.catapult.service.tw.TwResolverService.SourceType; +import fr.enimaloc.catapult.service.tw.TwSeverityRule; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Why a binding has its TWs: the detection's sources, the streamer's or an admin's choice, the topics kept off. */ +class TwDetailsServiceTest { + + private final BindingService bindings = mock(BindingService.class); + private final GameMappingRepository mappings = mock(GameMappingRepository.class); + private final TwDefinitionRepository definitions = mock(TwDefinitionRepository.class); + private final TwDetailsService service = new TwDetailsService(bindings, mappings, definitions); + private final UserAccount user = new UserAccount(); + private final UUID bindingId = UUID.randomUUID(); + private GameBinding binding; + + private static TwDefinition def(String id, String label, int order) { + TwDefinition d = new TwDefinition(); + d.setId(id); + d.setLabel(label); + d.setSortOrder(order); + return d; + } + + private static TwSeverityRule.Verdict verdict(TwSeverityRule.Reason reason) { + return new TwSeverityRule.Verdict(reason, "Depicted", "Minor", 10, 20); + } + + private void explained(Signal... signals) { + when(bindings.explainTws(user, bindingId)).thenReturn(Optional.of( + new BindingService.TwExplanation(binding, new TwResolverService.Explanation(List.of(signals))))); + } + + @BeforeEach + void setUp() { + binding = new GameBinding(); + binding.setStatus(GameBinding.Status.AUTO); + binding.setSourceType(GameBinding.SourceType.STEAM); + binding.setSourceId("730"); + when(definitions.findAll()).thenReturn(List.of(def("gore", "Gore", 10), def("violence", "Violence", 20), + def("spiders", "Spiders", 30))); + } + + @Test + void eachDetectedTw_comesWithItsSources_andTheTopicsKeptOffAreTold() { + binding.getTws().addAll(List.of("violence", "gore")); + explained( + new Signal("violence", SourceType.STEAM_CONTENT, "2", null), + new Signal("violence", SourceType.DTDD_TOPIC, "there's blood/gore", verdict(TwSeverityRule.Reason.NO_THRESHOLD)), + new Signal("gore", SourceType.STEAM_KEYWORD, "gore", null), + new Signal("gore", SourceType.DTDD_TOPIC, "there's blood/gore", verdict(TwSeverityRule.Reason.BELOW_VOTES))); + + TwDetailsDto details = service.details(user, bindingId).orElseThrow(); + + assertThat(details.streamerChose()).isFalse(); + // By the TWs' order + assertThat(details.tws()).extracting(TwDetailsDto.Tw::id).containsExactly("gore", "violence"); + assertThat(details.tws().get(0)).satisfies(gore -> { + assertThat(gore.label()).isEqualTo("Gore"); + assertThat(gore.origin()).isEqualTo("DETECTED"); + assertThat(gore.sources()).singleElement().satisfies(s -> assertThat(s.value()).isEqualTo("gore")); + }); + assertThat(details.tws().get(1).sources()).extracting(TwDetailsDto.Source::type) + .containsExactly("STEAM_CONTENT", "DTDD_TOPIC"); + assertThat(details.keptOff()).singleElement().satisfies(s -> { + assertThat(s.twId()).isEqualTo("gore"); + assertThat(s.twLabel()).isEqualTo("Gore"); + assertThat(s.severity().reason()).isEqualTo("BELOW_VOTES"); + assertThat(s.severity().minSeverity()).isEqualTo("Depicted"); + assertThat(s.severity().percentAtOrAbove()).isEqualTo(20); + }); + } + + @Test + void theStreamersChoice_isToldAsSuch_andWhatTheyTurnedOff_isShownOff() { + binding.setTwOverride(true); + binding.getTws().add("spiders"); + explained(new Signal("gore", SourceType.STEAM_KEYWORD, "gore", null)); + + TwDetailsDto details = service.details(user, bindingId).orElseThrow(); + + assertThat(details.streamerChose()).isTrue(); + assertThat(details.tws()).extracting(TwDetailsDto.Tw::id, TwDetailsDto.Tw::active, TwDetailsDto.Tw::origin) + .containsExactly(org.assertj.core.api.Assertions.tuple("spiders", true, "STREAMER"), + org.assertj.core.api.Assertions.tuple("gore", false, "DETECTED")); + } + + @Test + void anAdminsTws_areToldAsSuch_andTheDetectionsOthersLeftOut() { + binding.setStatus(GameBinding.Status.MAPPED); + binding.getTws().add("spiders"); + GameMapping mapping = new GameMapping(); + mapping.setTwsOverridden(true); + when(mappings.findBySourceTypeAndSourceId(GameBinding.SourceType.STEAM, "730")).thenReturn(Optional.of(mapping)); + explained(new Signal("gore", SourceType.STEAM_KEYWORD, "gore", null)); + + assertThat(service.details(user, bindingId).orElseThrow().tws()) + .extracting(TwDetailsDto.Tw::id, TwDetailsDto.Tw::origin) + .containsExactly(org.assertj.core.api.Assertions.tuple("spiders", "ADMIN")); + } + + @Test + void aTwNothingSuggestsAnyMore_isToldSo() { + binding.getTws().add("spiders"); + explained(); + + assertThat(service.details(user, bindingId).orElseThrow().tws()).singleElement() + .satisfies(t -> assertThat(t.origin()).isEqualTo("NO_SOURCE")); + } + + @Test + void anotherOnesBinding_hasNoDetails() { + when(bindings.explainTws(user, bindingId)).thenReturn(Optional.empty()); + + assertThat(service.details(user, bindingId)).isEmpty(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwSteamRetryTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwSteamRetryTest.java new file mode 100644 index 000000000..05fa1d928 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/binding/TwSteamRetryTest.java @@ -0,0 +1,63 @@ +package fr.enimaloc.catapult.service.binding; + +import fr.enimaloc.catapult.domain.account.UserAccount; +import fr.enimaloc.catapult.domain.binding.GameBinding; +import fr.enimaloc.catapult.event.TwsMissedSteamEvent; +import fr.enimaloc.catapult.service.notification.ChannelEventPublisher; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.scheduling.TaskScheduler; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** The deferred TW resolution: planned after the limit, once per binding, its result pushed. */ +class TwSteamRetryTest { + + private final BindingService bindings = mock(BindingService.class); + private final ChannelEventPublisher channelEvents = mock(ChannelEventPublisher.class); + private final TaskScheduler scheduler = mock(TaskScheduler.class); + private final TwSteamRetry retry = new TwSteamRetry(bindings, channelEvents, scheduler); + private final UUID bindingId = UUID.randomUUID(); + private final Instant until = Instant.parse("2026-11-12T10:05:00Z"); + + @Test + void itIsPlannedJustAfterTheLimit_oncePerBindingAtATime() { + retry.onMissedSteam(new TwsMissedSteamEvent(bindingId, until)); + retry.onMissedSteam(new TwsMissedSteamEvent(bindingId, until.plusSeconds(30))); + + verify(scheduler, times(1)).schedule(any(Runnable.class), eq(until.plus(TwSteamRetry.MARGIN))); + } + + @Test + void itsResult_isPushedToTheDashboard_andAnotherMayBePlannedThen() { + UserAccount owner = new UserAccount(); + owner.setId(UUID.randomUUID()); + GameBinding binding = new GameBinding(); + binding.setUser(owner); + when(bindings.retryTwsAfterSteamLimit(bindingId)).thenReturn(Optional.of(binding)); + retry.onMissedSteam(new TwsMissedSteamEvent(bindingId, until)); + ArgumentCaptor task = ArgumentCaptor.forClass(Runnable.class); + verify(scheduler).schedule(task.capture(), any(Instant.class)); + + task.getValue().run(); + + verify(channelEvents).bindingUpserted(owner.getId(), binding); + retry.onMissedSteam(new TwsMissedSteamEvent(bindingId, until.plusSeconds(60))); + verify(scheduler, times(2)).schedule(any(Runnable.class), any(Instant.class)); + } + + @Test + void aFailure_isLogged_notThrown() { + when(bindings.retryTwsAfterSteamLimit(bindingId)).thenThrow(new IllegalStateException("boom")); + + retry.retry(bindingId); + + verifyNoInteractions(channelEvents); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/category/ChannelFixture.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/category/ChannelFixture.java index ddf61b4c6..28833a807 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/category/ChannelFixture.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/category/ChannelFixture.java @@ -232,7 +232,7 @@ public ChannelFixture() { new CategoryApplier(twitch, streamStateService), new RaidRestoreGateService(experimentService), lookup, mock(ExtensionLayoutApplier.class)); bindingService = new BindingService(bindingRepository, igdbService, mock(IgdbGameDetailsService.class), twitch, mock(TwResolverService.class), mappingRepository, lookup, categoryChain, - mock(ExtensionLayoutApplier.class)); + mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); raidRestoreGameService = new RaidRestoreGameService(categoryChain, new CategoryApplier(twitch, streamStateService), raidRestoreHold); listener = new GameEventListener(bindingService, streamStateService, publisher, diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/debug/DebugAttributionTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/debug/DebugAttributionTest.java index 92ff12aa4..2c3204859 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/debug/DebugAttributionTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/debug/DebugAttributionTest.java @@ -315,7 +315,7 @@ void incompleteBindingRefresh_attributesEachBindingToItsOwner() { }); BindingService service = new BindingService(bindings, mock(IgdbService.class), mock(IgdbGameDetailsService.class), mock(TwitchService.class), mock(TwResolverService.class), - mappings, new GameBindingLookup(bindings), mock(CategoryChain.class), mock(ExtensionLayoutApplier.class)); + mappings, new GameBindingLookup(bindings), mock(CategoryChain.class), mock(ExtensionLayoutApplier.class), org.mockito.Mockito.mock(org.springframework.context.ApplicationEventPublisher.class)); service.refreshIncompleteBindings(); diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminServiceTest.java new file mode 100644 index 000000000..dfabf7337 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheAdminServiceTest.java @@ -0,0 +1,223 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; +import fr.enimaloc.catapult.domain.dtdd.DtddApiKeyEntry; +import fr.enimaloc.catapult.domain.dtdd.DtddGameCache; +import fr.enimaloc.catapult.domain.dtdd.DtddSearchCache; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicsCache; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiKeyRotator; +import fr.enimaloc.catapult.repository.dtdd.DtddApiKeyRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** The DTDD caches as the admin sees them: paged the oldest first, their stale entries, the keys' quotas. */ +class DtddCacheAdminServiceTest { + + private static final Instant NOW = Instant.parse("2026-11-12T10:00:00Z"); + private static final int TTL_HOURS = 168; + private static final Map LIFETIMES = Map.of(DtddCacheKind.TOPICS, + Duration.ofHours(TTL_HOURS), DtddCacheKind.GAMES, Duration.ofDays(180), DtddCacheKind.SEARCHES, + Duration.ofDays(30)); + + private final DtddTopicsCacheRepository topics = mock(DtddTopicsCacheRepository.class); + private final DtddGameCacheRepository games = mock(DtddGameCacheRepository.class); + private final DtddSearchCacheRepository searches = mock(DtddSearchCacheRepository.class); + private final DtddApiKeyRepository keys = mock(DtddApiKeyRepository.class); + private final DtddApiClient client = mock(DtddApiClient.class); + private final DtddApiKeyRotator rotator = mock(DtddApiKeyRotator.class); + private final DtddCacheRefetcher refetcher = mock(DtddCacheRefetcher.class); + private final DtddCacheRefetchJob job = mock(DtddCacheRefetchJob.class); + private final DtddCacheAdminService service = new DtddCacheAdminService(topics, games, searches, keys, + Optional.of(client), Optional.of(rotator), refetcher, job, LIFETIMES, Clock.fixed(NOW, ZoneOffset.UTC)); + + @Test + void summary_countsWhatIsPastTheLifetime_andTellsTheKeysQuotas() { + when(topics.count()).thenReturn(10L); + when(topics.countByFetchedAtBefore(NOW.minus(Duration.ofHours(TTL_HOURS)))).thenReturn(3L); + // Each cache by its own lifetime + when(games.countByFetchedAtBefore(NOW.minus(Duration.ofDays(180)))).thenReturn(1L); + when(searches.countBySearchedAtBefore(NOW.minus(Duration.ofDays(30)))).thenReturn(2L); + when(keys.findByExclusiveFalse()).thenReturn(List.of(new DtddApiKeyEntry("ddd_abcdefghijklmnop"), + new DtddApiKeyEntry("ddd_qrstuvwxyz012345"))); + when(rotator.getQuotas()).thenReturn(Map.of("ddd_abcdefghijklmnop", + new DtddApiKeyRotator.Quota(12, 4321, NOW.minusSeconds(5).toEpochMilli()))); + when(rotator.getKeyBlockedUntil()).thenReturn(Map.of("ddd_qrstuvwxyz012345", NOW.plusSeconds(30).toEpochMilli())); + when(job.monthlyReserve()).thenReturn(500); + when(job.status()).thenReturn(DtddRefetchJobDto.IDLE); + + DtddCachesSummaryDto summary = service.summary(); + + assertThat(summary.topics()).isEqualTo(10L); + assertThat(summary.topicsStale()).isEqualTo(3L); + assertThat(summary.gamesStale()).isEqualTo(1L); + assertThat(summary.searchesStale()).isEqualTo(2L); + assertThat(summary.topicsTtlHours()).isEqualTo(TTL_HOURS); + assertThat(summary.gamesTtlHours()).isEqualTo(180 * 24); + assertThat(summary.monthlyReserve()).isEqualTo(500); + assertThat(summary.quotas()).hasSize(2); + assertThat(summary.quotas().get(0)).satisfies(q -> { + assertThat(q.maskedKey()).isEqualTo("ddd_
mnop"); + assertThat(q.remainingMonth()).isEqualTo(4321); + assertThat(q.blockedUntil()).isNull(); + }); + assertThat(summary.quotas().get(1)).satisfies(q -> { + assertThat(q.remainingMonth()).isNull(); + assertThat(q.blockedUntil()).isEqualTo(NOW.plusSeconds(30)); + }); + // The keys themselves never leave + assertThat(summary.toString()).doesNotContain("abcdefghijklmnop"); + } + + @Test + void topics_arePagedTheOldestFirst_withTheirTitlesName_andMarkedStale() { + DtddTopicsCache old = new DtddTopicsCache(7L, List.of("a dog dies"), List.of(), List.of(), + NOW.minus(Duration.ofHours(TTL_HOURS + 1))); + when(topics.search(eq("halo"), any())).thenReturn(new PageImpl<>(List.of(old))); + when(games.findAllById(List.of(7L))).thenReturn(List.of( + new DtddGameCache(7L, "Halo", null, null, null, NOW))); + + Page page = service.list(DtddCacheKind.TOPICS, " Halo ", 0, 500); + + assertThat(page.getContent()).singleElement().isEqualTo(new DtddTopicsCacheEntryDto(7L, "Halo", + List.of("a dog dies"), List.of(), List.of(), old.getFetchedAt(), true)); + ArgumentCaptor pageable = ArgumentCaptor.forClass(Pageable.class); + verify(topics).search(eq("halo"), pageable.capture()); + assertThat(pageable.getValue().getPageSize()).isEqualTo(DtddCacheAdminService.MAX_PAGE_SIZE); + assertThat(pageable.getValue().getSort()).isEqualTo(Sort.by(Sort.Direction.ASC, "fetchedAt")); + } + + @Test + void refetch_givesTheEntryAsNowCached() { + when(games.existsById(7L)).thenReturn(true); + when(refetcher.refetch(client, DtddCacheKind.GAMES, "7")).thenReturn(true); + when(games.findById(7L)).thenReturn(Optional.of(new DtddGameCache(7L, "Halo", null, "Video Game", null, NOW))); + + Object entry = service.refetch(DtddCacheKind.GAMES, " 7 "); + + assertThat(entry).hasFieldOrPropertyWithValue("name", "Halo").hasFieldOrPropertyWithValue("stale", false); + } + + @Test + void refetch_ofASearch_readsItsQueryAsTheCacheKeepsIt() { + when(searches.existsById("halo infinite")).thenReturn(true); + when(refetcher.refetch(client, DtddCacheKind.SEARCHES, "halo infinite")).thenReturn(false); + + assertThatThrownBy(() -> service.refetch(DtddCacheKind.SEARCHES, " Halo Infinite ")) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("502"); + } + + @Test + void refetch_ofWhatIsNotCached_isNotFound_andAsksNothing() { + assertThatThrownBy(() -> service.refetch(DtddCacheKind.TOPICS, "7")) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("404"); + verifyNoInteractions(refetcher); + } + + @Test + void refetch_withoutDtdd_isUnavailable() { + DtddCacheAdminService off = new DtddCacheAdminService(topics, games, searches, keys, Optional.empty(), + Optional.empty(), refetcher, job, LIFETIMES, Clock.fixed(NOW, ZoneOffset.UTC)); + + assertThatThrownBy(() -> off.refetch(DtddCacheKind.TOPICS, "7")) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("503"); + assertThat(off.summary().quotas()).isEmpty(); + } + + @Test + void detail_ofATitle_givesItsRecordItsTopicsInFull_andTheSearchesThatFoundIt() { + when(topics.existsById(7L)).thenReturn(true); + when(topics.findById(7L)).thenReturn(Optional.of(new DtddTopicsCache(7L, List.of("a dog dies", "gore"), + List.of("a cat dies"), List.of("jump scares"), NOW))); + when(games.findById(7L)).thenReturn(Optional.of( + new DtddGameCache(7L, "Halo", "https://www.doesthedogdie.com/media/7", "Video Game", null, NOW))); + when(games.findAllById(List.of(7L))).thenReturn(List.of(new DtddGameCache(7L, "Halo", null, null, null, NOW))); + when(searches.findAllFinding(7L)).thenReturn(List.of(new DtddSearchCache("halo", List.of(7L, 8L), NOW))); + + var detail = service.detail(DtddCacheKind.TOPICS, "7"); + + assertThat(detail.dtddId()).isEqualTo(7L); + assertThat(detail.topics().name()).isEqualTo("Halo"); + assertThat(detail.topics().yesTopics()).containsExactly("a dog dies", "gore"); + assertThat(detail.topics().noTopics()).containsExactly("a cat dies"); + assertThat(detail.topics().mostlyTopics()).containsExactly("jump scares"); + assertThat(detail.game().mediaType()).isEqualTo("Video Game"); + assertThat(detail.searches()).singleElement().satisfies(s -> assertThat(s.query()).isEqualTo("halo")); + assertThat(detail.titles()).isEmpty(); + } + + @Test + void detail_ofATitleWithoutTopics_hasNoneButItsRecord() { + when(games.existsById(9L)).thenReturn(true); + when(games.findById(9L)).thenReturn(Optional.of(new DtddGameCache(9L, "Portal", null, null, null, NOW))); + + var detail = service.detail(DtddCacheKind.GAMES, "9"); + + assertThat(detail.topics()).isNull(); + assertThat(detail.game().name()).isEqualTo("Portal"); + assertThat(detail.searches()).isEmpty(); + } + + @Test + void detail_ofASearch_givesTheTitlesItFound_asCached() { + when(searches.existsById("halo")).thenReturn(true); + when(searches.findById("halo")).thenReturn(Optional.of(new DtddSearchCache("halo", List.of(7L, 8L), NOW))); + when(games.findAllById(List.of(7L, 8L))).thenReturn(List.of(new DtddGameCache(7L, "Halo", null, null, null, NOW))); + when(topics.findAllById(List.of(7L, 8L))).thenReturn(List.of( + new DtddTopicsCache(8L, List.of(), List.of(), List.of(), NOW))); + + var detail = service.detail(DtddCacheKind.SEARCHES, " HALO "); + + assertThat(detail.query()).isEqualTo("halo"); + assertThat(detail.searches()).singleElement().satisfies(s -> assertThat(s.dtddIds()).containsExactly(7L, 8L)); + assertThat(detail.titles()).containsExactly( + new fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto.FoundTitle(7L, "Halo", false), + new fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto.FoundTitle(8L, null, true)); + } + + @Test + void detail_ofWhatIsNotCached_isNotFound() { + assertThatThrownBy(() -> service.detail(DtddCacheKind.SEARCHES, "nope")) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("404"); + } + + @Test + void detail_givesEachTopicsSeverity_andNoneWhenTheyWereNotKept() { + when(topics.existsById(7L)).thenReturn(true); + when(topics.findById(7L)).thenReturn(Optional.of(new DtddTopicsCache(7L, List.of("a dog dies"), List.of(), + List.of(), NOW, Map.of("a dog dies", new fr.enimaloc.catapult.domain.dtdd.DtddTopicSeverity( + List.of("Implied", "Depicted"), List.of(1, 4), List.of(20, 80), 5, 2))))); + + var severity = service.detail(DtddCacheKind.TOPICS, "7").topics().severities().get("a dog dies"); + assertThat(severity.verifiedLevel()).isEqualTo("Depicted"); + assertThat(severity.level(1)).isEqualTo("Implied"); + + when(topics.findById(7L)).thenReturn(Optional.of(new DtddTopicsCache(7L, List.of("a dog dies"), List.of(), + List.of(), NOW))); + assertThat(service.detail(DtddCacheKind.TOPICS, "7").topics().severities()).isNull(); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJobTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJobTest.java new file mode 100644 index 000000000..c3ed4779b --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetchJobTest.java @@ -0,0 +1,195 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiKeyRotator; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** The bulk refetch: the oldest entries first, under the rate limit, keeping the monthly reserve, cancellable. */ +class DtddCacheRefetchJobTest { + + private static final int RESERVE = 500; + private static final Instant T0 = Instant.parse("2026-11-12T10:00:00Z"); + + /** A clock the sleeps move forward. */ + private static final class FakeClock extends Clock { + final AtomicLong millis = new AtomicLong(T0.toEpochMilli()); + + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return Instant.ofEpochMilli(millis.get()); } + } + + private final DtddApiClient client = mock(DtddApiClient.class); + private final DtddApiClient bulk = mock(DtddApiClient.class); + private final DtddApiKeyRotator rotator = mock(DtddApiKeyRotator.class); + private final DtddTopicsCacheRepository topics = mock(DtddTopicsCacheRepository.class); + private final DtddGameCacheRepository games = mock(DtddGameCacheRepository.class); + private final DtddSearchCacheRepository searches = mock(DtddSearchCacheRepository.class); + private final DtddCacheRefetcher refetcher = mock(DtddCacheRefetcher.class); + private final FakeClock clock = new FakeClock(); + private final List sleeps = new ArrayList<>(); + private Runnable onSleep = () -> {}; + private Executor executor = Runnable::run; + private DtddCacheRefetchJob job; + + @BeforeEach + void setUp() { + when(client.reserving(RESERVE)).thenReturn(bulk); + when(rotator.nextUsableAt(RESERVE)).thenAnswer(inv -> Optional.of(clock.millis())); + when(rotator.keysAbove(RESERVE)).thenReturn(1); + when(rotator.availableKeyCount()).thenReturn(1); + when(refetcher.refetch(eq(bulk), any(), any())).thenReturn(true); + when(topics.findAllAges()).thenReturn(List.of(new Object[]{10L, T0.minusSeconds(100)})); + when(games.findAllAges()).thenReturn(List.of(new Object[]{20L, T0.minusSeconds(300)})); + when(searches.findAllAges()).thenReturn(List.of(new Object[]{"halo", T0.minusSeconds(200)})); + job = job(); + } + + private DtddCacheRefetchJob job() { + return new DtddCacheRefetchJob(Optional.of(client), Optional.of(rotator), topics, games, searches, refetcher, + RESERVE, 30, task -> executor.execute(task), millis -> { + sleeps.add(millis); + clock.millis.addAndGet(millis); + onSleep.run(); + }, clock); + } + + @Test + void refetchesEverything_theOldestFirst_withTheReservingClient() { + DtddRefetchJobDto started = job.start(); + + assertThat(started.total()).isEqualTo(3); + var order = inOrder(refetcher); + order.verify(refetcher).refetch(bulk, DtddCacheKind.GAMES, "20"); + order.verify(refetcher).refetch(bulk, DtddCacheKind.SEARCHES, "halo"); + order.verify(refetcher).refetch(bulk, DtddCacheKind.TOPICS, "10"); + verify(client, never()).fetchTopics(anyLong()); + assertThat(job.status()).satisfies(s -> { + assertThat(s.state()).isEqualTo("DONE"); + assertThat(s.done()).isEqualTo(3); + assertThat(s.failed()).isZero(); + assertThat(s.finishedAt()).isNotNull(); + }); + } + + @Test + void keepsUnderEachKeysRequestsPerMinute_sharedOverTheKeys() { + job.start(); + // 30 a minute on one key: 2 s apart, in slices of a second at most + assertThat(sleeps.stream().mapToLong(Long::longValue).sum()).isEqualTo(3 * 2_000L); + assertThat(sleeps).allMatch(s -> s <= 1_000L); + + sleeps.clear(); + when(rotator.keysAbove(RESERVE)).thenReturn(2); + job.start(); + assertThat(sleeps.stream().mapToLong(Long::longValue).sum()).isEqualTo(3 * 1_000L); + } + + @Test + void anEntryDtddGivesNothingFor_isCountedAndTheRestGoesOn() { + when(refetcher.refetch(bulk, DtddCacheKind.SEARCHES, "halo")).thenReturn(false); + when(refetcher.refetch(bulk, DtddCacheKind.TOPICS, "10")).thenThrow(new IllegalStateException("boom")); + + job.start(); + + assertThat(job.status().done()).isEqualTo(3); + assertThat(job.status().failed()).isEqualTo(2); + assertThat(job.status().state()).isEqualTo("DONE"); + } + + @Test + void waitsWhileEveryKeyRests() { + long restUntil = clock.millis() + 5_500L; + when(rotator.nextUsableAt(RESERVE)).thenAnswer(inv -> + Optional.of(Math.max(restUntil, clock.millis()))); + List waitingShown = new ArrayList<>(); + onSleep = () -> waitingShown.add(job.status().waitingUntil()); + + job.start(); + + assertThat(waitingShown.getFirst()).isEqualTo(Instant.ofEpochMilli(restUntil)); + verify(refetcher, times(3)).refetch(eq(bulk), any(), any()); + assertThat(job.status().state()).isEqualTo("DONE"); + assertThat(job.status().waitingUntil()).isNull(); + } + + @Test + void stopsOnceEveryKeyIsDownToTheMonthlyReserve() { + when(rotator.nextUsableAt(RESERVE)).thenReturn(Optional.of(clock.millis()), Optional.empty()); + + job.start(); + + verify(refetcher, times(1)).refetch(eq(bulk), any(), any()); + assertThat(job.status().state()).isEqualTo("STOPPED"); + assertThat(job.status().stopReason()).isEqualTo("QUOTA"); + assertThat(job.status().done()).isEqualTo(1); + } + + @Test + void stopsWithoutAnyKey() { + when(rotator.nextUsableAt(RESERVE)).thenReturn(Optional.empty()); + when(rotator.availableKeyCount()).thenReturn(0); + + job.start(); + + assertThat(job.status().stopReason()).isEqualTo("NO_KEY"); + verifyNoInteractions(refetcher); + } + + @Test + void aCancel_endsItAfterTheCurrentEntry() { + onSleep = () -> job.cancel(); + + job.start(); + + verify(refetcher, times(1)).refetch(eq(bulk), any(), any()); + assertThat(job.status().state()).isEqualTo("CANCELLED"); + // Back from its pause at once + assertThat(sleeps).hasSize(1); + } + + @Test + void oneAtATime_andAnotherOnceItEnded() { + List queued = new ArrayList<>(); + executor = queued::add; + + job.start(); + assertThatThrownBy(job::start).isInstanceOf(ResponseStatusException.class).hasMessageContaining("409"); + + queued.getFirst().run(); + assertThat(job.status().state()).isEqualTo("DONE"); + job.start(); + assertThat(queued).hasSize(2); + } + + @Test + void withoutDtdd_isUnavailable() { + DtddCacheRefetchJob off = new DtddCacheRefetchJob(Optional.empty(), Optional.empty(), topics, games, searches, + refetcher, RESERVE, 30, Runnable::run, millis -> {}, clock); + + assertThatThrownBy(off::start).isInstanceOf(ResponseStatusException.class).hasMessageContaining("503"); + assertThat(off.status()).isEqualTo(DtddRefetchJobDto.IDLE); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcherTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcherTest.java new file mode 100644 index 000000000..ff18bdbd1 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/dtdd/DtddCacheRefetcherTest.java @@ -0,0 +1,121 @@ +package fr.enimaloc.catapult.service.dtdd; + +import fr.enimaloc.catapult.domain.dtdd.DtddGameCache; +import fr.enimaloc.catapult.domain.dtdd.DtddSearchCache; +import fr.enimaloc.catapult.domain.dtdd.DtddTopicsCache; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddItem; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSearchResult; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddTopics; +import fr.enimaloc.catapult.repository.dtdd.DtddGameCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddSearchCacheRepository; +import fr.enimaloc.catapult.repository.dtdd.DtddTopicsCacheRepository; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** One cached DTDD entry fetched again: saved as its readers would, untouched when DTDD answers nothing. */ +class DtddCacheRefetcherTest { + + private static final Instant NOW = Instant.parse("2026-11-12T10:00:00Z"); + + private final DtddTopicsCacheRepository topics = mock(DtddTopicsCacheRepository.class); + private final DtddGameCacheRepository games = mock(DtddGameCacheRepository.class); + private final DtddSearchCacheRepository searches = mock(DtddSearchCacheRepository.class); + private final DtddApiClient client = mock(DtddApiClient.class); + private final DtddCacheRefetcher refetcher = + new DtddCacheRefetcher(topics, games, searches, Clock.fixed(NOW, ZoneOffset.UTC)); + + private static DtddItem item(String name, String poster) { + return new DtddItem(7, name, null, 2024, 1, "Video Game", 0, null, null, poster, null, null); + } + + @Test + void topics_areSavedAnew() { + when(client.fetchTopics(7L)).thenReturn(Optional.of( + new DtddTopics(List.of("a dog dies"), List.of("a cat dies"), List.of()))); + + assertThat(refetcher.refetch(client, DtddCacheKind.TOPICS, "7")).isTrue(); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DtddTopicsCache.class); + verify(topics).save(saved.capture()); + assertThat(saved.getValue().getYesTopics()).containsExactly("a dog dies"); + assertThat(saved.getValue().getFetchedAt()).isEqualTo(NOW); + } + + @Test + void aGame_takesTheItemsNameAndType_andKeepsItsPosterWithoutAFullOne() { + DtddGameCache cached = new DtddGameCache(7L, "Old", "slug", "Movie", "https://poster", NOW.minusSeconds(9)); + when(games.findById(7L)).thenReturn(Optional.of(cached)); + when(client.item(7L)).thenReturn(Optional.of(item("New name", "/relative.jpg"))); + + assertThat(refetcher.refetch(client, DtddCacheKind.GAMES, "7")).isTrue(); + + assertThat(cached.getName()).isEqualTo("New name"); + assertThat(cached.getMediaType()).isEqualTo("Video Game"); + assertThat(cached.getPosterUrl()).isEqualTo("https://poster"); + assertThat(cached.getSlug()).isEqualTo("slug"); + assertThat(cached.getFetchedAt()).isEqualTo(NOW); + verify(games).save(cached); + } + + @Test + void aSearch_savesTheTitlesItFinds_withTheVideoGamesFilter() { + when(client.search("halo")).thenReturn(Optional.of(List.of( + new DtddSearchResult(1, "Halo", "s1", "Video Game", null), + new DtddSearchResult(2, "Halo 2", "s2", "Video Game", null)))); + + assertThat(refetcher.refetch(client, DtddCacheKind.SEARCHES, "halo")).isTrue(); + + verify(games, times(2)).save(any(DtddGameCache.class)); + ArgumentCaptor saved = ArgumentCaptor.forClass(DtddSearchCache.class); + verify(searches).save(saved.capture()); + assertThat(saved.getValue().getDtddIds()).containsExactly(1L, 2L); + assertThat(saved.getValue().getSearchedAt()).isEqualTo(NOW); + } + + @Test + void dtddAnsweringNothing_changesNothing() { + when(client.fetchTopics(7L)).thenReturn(Optional.empty()); + when(client.item(7L)).thenReturn(Optional.empty()); + when(client.search("halo")).thenReturn(Optional.empty()); + + assertThat(refetcher.refetch(client, DtddCacheKind.TOPICS, "7")).isFalse(); + assertThat(refetcher.refetch(client, DtddCacheKind.GAMES, "7")).isFalse(); + assertThat(refetcher.refetch(client, DtddCacheKind.SEARCHES, "halo")).isFalse(); + verifyNoInteractions(topics, searches); + verify(games, never()).save(any()); + } + + @Test + void aTitleKey_mustBeAnId() { + assertThatThrownBy(() -> refetcher.refetch(client, DtddCacheKind.TOPICS, "halo")) + .isInstanceOf(ResponseStatusException.class).hasMessageContaining("400"); + } + + @Test + void topics_keepTheirSeverities() { + when(client.fetchTopics(7L)).thenReturn(Optional.of(new DtddTopics(List.of("a dog dies"), List.of(), List.of(), + java.util.Map.of("a dog dies", new DtddApiClient.DtddSeverity(List.of("Implied", "Depicted"), + List.of(1, 4), List.of(20, 80), 5, 2))))); + + refetcher.refetch(client, DtddCacheKind.TOPICS, "7"); + + ArgumentCaptor saved = ArgumentCaptor.forClass(DtddTopicsCache.class); + verify(topics).save(saved.capture()); + assertThat(saved.getValue().getSeverities()).containsOnlyKeys("a dog dies"); + assertThat(saved.getValue().getSeverities().get("a dog dies").verified()).isEqualTo(2); + assertThat(saved.getValue().getSeverities().get("a dog dies").percents()).containsExactly(20, 80); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceTest.java index 739e57cec..cc4e97cfb 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/steam/SteamStoreServiceTest.java @@ -8,6 +8,10 @@ import fr.enimaloc.catapult.service.steam.SteamStoreService.SteamTwSignals; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.micrometer.observation.ObservationRegistry; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimitInterceptor; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimitProperties; +import fr.enimaloc.catapult.ratelimit.LearnedRateLimiter; +import fr.enimaloc.catapult.repository.ratelimit.LearnedRateLimitRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -56,13 +60,21 @@ class SteamStoreServiceTest { private MockRestServiceServer steam; private SteamStoreServiceImpl service; + private final MovingClock clock = new MovingClock(); + private LearnedRateLimiter rateLimiter; + @BeforeEach void setUp() { - RestClient.Builder builder = RestClient.builder(); + LearnedRateLimitRepository limits = org.mockito.Mockito.mock(LearnedRateLimitRepository.class); + doReturn(Optional.empty()).when(limits).findById(any()); + rateLimiter = new LearnedRateLimiter(new LearnedRateLimitProperties(), limits, Optional.of(clock), Optional.empty()); + RestClient.Builder builder = RestClient.builder().requestInterceptor(new LearnedRateLimitInterceptor(rateLimiter)); steam = MockRestServiceServer.bindTo(builder).ignoreExpectOrder(true).build(); ExternalApiObservations observations = new ExternalApiObservations(ObservationRegistry.create(), new SimpleMeterRegistry()); - service = new SteamStoreServiceImpl(builder.build(), observations, steamAppParentRepository, redirectResolver); + service = new SteamStoreServiceImpl(builder.build(), observations, steamAppParentRepository, redirectResolver, + rateLimiter); + service.setClock(clock); doReturn(Optional.empty()).when(steamAppParentRepository).findById(any()); } @@ -445,4 +457,87 @@ void fetchIADisclosure_storeError_returnsEmpty() { assertThat(service.fetchIADisclosure("100", Locale.ENGLISH)).isEmpty(); } + + // --- rate limit (429) --------------------------------------------------------------------- + + /** A clock the test moves on. */ + private static final class MovingClock extends java.time.Clock { + Instant now = Instant.parse("2026-11-12T10:00:00Z"); + + @Override public java.time.ZoneId getZone() { return java.time.ZoneOffset.UTC; } + @Override public java.time.Clock withZone(java.time.ZoneId zone) { return this; } + @Override public Instant instant() { return now; } + } + + private void given429(String appId, String retryAfter) { + var response = withStatus(HttpStatus.TOO_MANY_REQUESTS); + if (retryAfter != null) response = response.header("Retry-After", retryAfter); + steam.expect(org.springframework.test.web.client.ExpectedCount.once(), + requestTo(APP_DETAILS + "?appids=" + appId + "&l=english")).andRespond(response); + } + + @Test + void a429_leavesTheStoreAloneForItsRetryAfter_thenAsksAgain() { + given429("100", "120"); + + assertThat(service.fetchTwSignals(List.of("100"))).isEmpty(); + assertThat(service.rateLimitedUntil()).contains(clock.now.plusSeconds(120)); + // Not asked meanwhile (a second request would fail the mock) + assertThat(service.fetchTwSignals(List.of("100"))).isEmpty(); + assertThat(service.fetchIADisclosure("100", Locale.ENGLISH)).isEmpty(); + steam.verify(); + + clock.now = clock.now.plusSeconds(121); + assertThat(service.rateLimitedUntil()).isEmpty(); + steam.reset(); + givenAppDetails("100", success("100", "{\"type\": \"game\", \"name\": \"Some Game\"}")); + assertThat(service.fetchTwSignals(List.of("100"))).containsKey("100"); + } + + @Test + void aCallTheLearnedLimitRefuses_tellsUntilWhen() { + assertThat(service.rateLimitedUntil()).isEmpty(); + rateLimiter.correct("HOST store.steampowered.com", 1, 10, null, null); + givenAppDetails("100", success("100", "{\"type\": \"game\", \"name\": \"Some Game\"}")); + assertThat(service.fetchData("100", Locale.ENGLISH, false)).isPresent(); + steam.reset(); + + // A call per 10 s: the next is refused without being sent + assertThat(service.fetchData("100", Locale.ENGLISH, false)).isEmpty(); + assertThat(service.rateLimitedUntil()).contains(clock.now.plusSeconds(10)); + clock.now = clock.now.plusSeconds(11); + assertThat(service.rateLimitedUntil()).isEmpty(); + } + + @Test + void cancelledBulkWork_neverStopsTheLiveCalls() { + assertThat(service.rateLimitedUntil()).isEmpty(); + rateLimiter.correct("HOST store.steampowered.com", 1, 10, null, null); + givenAppDetails("100", success("100", "{\"type\": \"game\", \"name\": \"Some Game\"}")); + assertThat(service.fetchData("100", Locale.ENGLISH, false)).isPresent(); + steam.reset(); + + assertThat(fr.enimaloc.catapult.ratelimit.RateLimitPriority.call(fr.enimaloc.catapult.ratelimit.RateLimitPriority.BULK, + () -> true, () -> service.fetchData("100", Locale.ENGLISH, false))).isEmpty(); + assertThat(service.rateLimitedUntil()).isEmpty(); + } + + @Test + void withTheLearningOff_a429_stillLeavesTheStoreAlone() { + rateLimiter = null; + LearnedRateLimitRepository limits = org.mockito.Mockito.mock(LearnedRateLimitRepository.class); + LearnedRateLimitProperties off = new LearnedRateLimitProperties(); + off.setEnabled(false); + LearnedRateLimiter disabled = new LearnedRateLimiter(off, limits, Optional.of(clock), Optional.empty()); + RestClient.Builder builder = RestClient.builder().requestInterceptor(new LearnedRateLimitInterceptor(disabled)); + steam = MockRestServiceServer.bindTo(builder).ignoreExpectOrder(true).build(); + service = new SteamStoreServiceImpl(builder.build(), + new ExternalApiObservations(ObservationRegistry.create(), new SimpleMeterRegistry()), + steamAppParentRepository, redirectResolver, disabled); + service.setClock(clock); + given429("100", null); + + assertThat(service.fetchTwSignals(List.of("100"))).isEmpty(); + assertThat(service.rateLimitedUntil()).contains(clock.now.plusSeconds(60)); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/title/TitleHelpSectionsTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/title/TitleHelpSectionsTest.java new file mode 100644 index 000000000..eb88abd95 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/title/TitleHelpSectionsTest.java @@ -0,0 +1,42 @@ +package fr.enimaloc.catapult.service.title; + +import fr.enimaloc.catapult.domain.account.UserAccount; +import fr.enimaloc.catapult.repository.account.UserAccountRepository; +import fr.enimaloc.catapult.service.drops.DropsGate; +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** The title help's parts an account is offered: the drops only with the conditions, nothing for an unknown one. */ +class TitleHelpSectionsTest { + + private final UserAccountRepository accounts = mock(UserAccountRepository.class); + private final StreamTitleService streamTitle = mock(StreamTitleService.class); + private final DropsGate drops = mock(DropsGate.class); + private final TitleHelpSections sections = new TitleHelpSections(accounts, streamTitle, drops); + private final UUID id = UUID.randomUUID(); + private final UserAccount account = new UserAccount(); + + @Test + void theDrops_needTheConditions() { + when(accounts.findById(id)).thenReturn(Optional.of(account)); + when(drops.isAvailableReadOnly(account)).thenReturn(true); + + when(streamTitle.conditionsAvailableReadOnly(account)).thenReturn(true); + assertThat(sections.forAccount(id)).isEqualTo(new TitleHelpSections.Offered(true, true)); + when(streamTitle.conditionsAvailableReadOnly(account)).thenReturn(false); + assertThat(sections.forAccount(id)).isEqualTo(TitleHelpSections.Offered.NONE); + } + + @Test + void anUnknownAccount_isOfferedNothing() { + when(accounts.findById(id)).thenReturn(Optional.empty()); + + assertThat(sections.forAccount(id)).isEqualTo(TitleHelpSections.Offered.NONE); + } +} diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/AdminTwServiceMappingsTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/AdminTwServiceMappingsTest.java index 6905c0f1f..062ac1a3d 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/AdminTwServiceMappingsTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/AdminTwServiceMappingsTest.java @@ -219,4 +219,23 @@ void signalTest_withoutDraft_orStoreData() { assertThat(result.draftKeywordMatch()).isNull(); assertThat(result.matchedContentDescriptorIds()).isEmpty(); } + + @Test + void dtddTopics_keepTheirLeastSevereLevel_trimmedAndOncePerTopic() { + service.replaceDtddTopics("gore", new java.util.LinkedHashSet<>(List.of(" there's blood/gore ", "THERE'S BLOOD/GORE", + "someone dies")), java.util.Map.of(" there's blood/gore ", " Depicted ", "someone dies", " ")); + + var saved = org.mockito.ArgumentCaptor.forClass(fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping.class); + verify(dtdd, times(2)).save(saved.capture()); + assertThat(saved.getAllValues()).extracting(fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping::getDtddTopicName, + fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping::getMinSeverity) + .containsExactly(org.assertj.core.api.Assertions.tuple("there's blood/gore", "Depicted"), + org.assertj.core.api.Assertions.tuple("someone dies", null)); + } + + @Test + void aTooLongLevel_isRefused() { + assertThatThrownBy(() -> service.replaceDtddTopics("gore", Set.of("x"), java.util.Map.of("x", "y".repeat(41)))) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwBackfillServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwBackfillServiceTest.java index 32bab70e0..ff5ea24b0 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwBackfillServiceTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwBackfillServiceTest.java @@ -17,6 +17,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.time.Instant; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -25,6 +26,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; class TwBackfillServiceTest { @@ -75,11 +77,20 @@ void processesEligibleBindings_skipsOverride_persistsFlag() { when(igdb.resolveIgdbIdForBinding(b1)).thenReturn(Optional.of("igdb42")); when(igdb.fetchDescriptorIds("igdb42")).thenReturn(Set.of(10L)); TwResolverService res = mock(TwResolverService.class); - when(res.suggest(any())).thenReturn(Set.of("violence_graphic")); + java.util.List priorities = new java.util.ArrayList<>(); + when(res.suggestReporting(any())).thenAnswer(i -> { + priorities.add(fr.enimaloc.catapult.ratelimit.RateLimitPriority.current()); + return new TwResolverService.Suggestion(Set.of("violence_graphic"), Optional.empty()); + }); TwBackfillService svc = build(appRepo, bRepo, igdb, res); svc.run(); + // Resolved as bulk traffic: under the learned rate limits' margin, leaving room for live calls + assertThat(priorities).containsOnly(fr.enimaloc.catapult.ratelimit.RateLimitPriority.BULK); + assertThat(fr.enimaloc.catapult.ratelimit.RateLimitPriority.current()) + .isEqualTo(fr.enimaloc.catapult.ratelimit.RateLimitPriority.LIVE); + verify(bRepo, atLeastOnce()).save(b1); assertThat(b1.getTws()).containsExactly("violence_graphic"); verify(appRepo).save(any(AppState.class)); @@ -107,7 +118,7 @@ void forceRebuildAllIncludingPinned_doesNotSkipOverriddenBindings() { when(igdb.resolveIgdbIdForBinding(pinned)).thenReturn(Optional.of("igdb99")); when(igdb.fetchDescriptorIds("igdb99")).thenReturn(Set.of(20L)); TwResolverService res = mock(TwResolverService.class); - when(res.suggest(any())).thenReturn(Set.of("new_tw")); + when(res.suggestReporting(any())).thenReturn(new TwResolverService.Suggestion(Set.of("new_tw"), Optional.empty())); TwBackfillService svc = build(appRepo, bRepo, igdb, res); svc.forceRebuildAllIncludingPinned(); @@ -131,7 +142,7 @@ private MappedPair mappedPair() { IgdbService igdb = mock(IgdbService.class); when(igdb.resolveIgdbIdForBinding(any())).thenReturn(Optional.of("igdb")); TwResolverService res = mock(TwResolverService.class); - when(res.suggest(any())).thenReturn(Set.of("igdb_tw")); + when(res.suggestReporting(any())).thenReturn(new TwResolverService.Suggestion(Set.of("igdb_tw"), Optional.empty())); return new MappedPair(adminTws, igdbTws, igdb, res); } @@ -200,4 +211,133 @@ private TwBackfillService build(AppStateRepository app, GameBindingRepository b, s.setThrottle(1000); return s; } + + // --- progress, one at a time, the Steam store's rate limit --------------------------------- + + private static GameBinding steamBinding(String appId) { + GameBinding b = new GameBinding(); + b.setUser(owner()); + b.setId(UUID.randomUUID()); + b.setStatus(GameBinding.Status.AUTO); + b.setSourceType(GameBinding.SourceType.STEAM); + b.setSourceId(appId); + b.setSourceName("Game " + appId); + return b; + } + + @Test + void aRebuild_tellsItsProgress_andItsEnd() { + GameBindingRepository bRepo = mock(GameBindingRepository.class); + GameBinding pinned = steamBinding("1"); + pinned.setTwOverride(true); + GameBinding plain = steamBinding("2"); + when(bRepo.findCandidatesForTwRebuild(any(Pageable.class))).thenReturn(new PageImpl<>(List.of(pinned, plain))); + IgdbService igdb = mock(IgdbService.class); + TwResolverService res = mock(TwResolverService.class); + when(res.suggestReporting(any())).thenReturn(new TwResolverService.Suggestion(Set.of("gore"), Optional.empty())); + TwBackfillService svc = build(mock(AppStateRepository.class), bRepo, igdb, res); + svc.setSleeper(ms -> {}); + + svc.rebuildAllNonOverridden(); + + var status = svc.status(); + assertThat(status.state()).isEqualTo("DONE"); + assertThat(status.mode()).isEqualTo("NON_OVERRIDDEN"); + assertThat(status.total()).isEqualTo(2); + assertThat(status.processed()).isEqualTo(1); + assertThat(status.skipped()).isEqualTo(1); + assertThat(status.done()).isEqualTo(2); + assertThat(status.finishedAt()).isNotNull(); + } + + @Test + void theSteamStoresLimit_isWaitedFor_andTheBindingTriedAgain_neverSavedWithoutSteam() { + GameBindingRepository bRepo = mock(GameBindingRepository.class); + GameBinding plain = steamBinding("2"); + when(bRepo.findCandidatesForTwRebuild(any(Pageable.class))).thenReturn(new PageImpl<>(List.of(plain))); + TwResolverService res = mock(TwResolverService.class); + Instant start = Instant.parse("2026-11-12T10:00:00Z"); + Instant until = start.plusSeconds(5); + when(res.suggestReporting(any())).thenReturn( + new TwResolverService.Suggestion(Set.of("partial"), Optional.of(until)), + new TwResolverService.Suggestion(Set.of("gore"), Optional.empty())); + TwBackfillService svc = build(mock(AppStateRepository.class), bRepo, mock(IgdbService.class), res); + svc.setClock(java.time.Clock.fixed(start, java.time.ZoneOffset.UTC)); + List waitingShown = new java.util.ArrayList<>(); + java.util.concurrent.atomic.AtomicLong slept = new java.util.concurrent.atomic.AtomicLong(); + svc.setSleeper(ms -> { + slept.addAndGet(ms); + waitingShown.add(svc.status().waitingUntil()); + }); + + svc.rebuildAllNonOverridden(); + + assertThat(waitingShown.getFirst()).isEqualTo(until); + // Until the limit's end, and a second more + assertThat(slept.get()).isGreaterThanOrEqualTo(6_000L); + verify(bRepo, times(1)).save(plain); + assertThat(plain.getTws()).containsExactly("gore"); + assertThat(svc.status().processed()).isEqualTo(1); + } + + @Test + void aBindingTheStoreStaysLimitedFor_isFailed_afterSoManyWaits() { + GameBindingRepository bRepo = mock(GameBindingRepository.class); + GameBinding plain = steamBinding("2"); + when(bRepo.findCandidatesForTwRebuild(any(Pageable.class))).thenReturn(new PageImpl<>(List.of(plain))); + TwResolverService res = mock(TwResolverService.class); + Instant start = Instant.parse("2026-11-12T10:00:00Z"); + when(res.suggestReporting(any())).thenReturn( + new TwResolverService.Suggestion(Set.of(), Optional.of(start.plusSeconds(1)))); + TwBackfillService svc = build(mock(AppStateRepository.class), bRepo, mock(IgdbService.class), res); + svc.setClock(java.time.Clock.fixed(start, java.time.ZoneOffset.UTC)); + svc.setSleeper(ms -> {}); + + svc.rebuildAllNonOverridden(); + + verify(res, times(TwBackfillService.MAX_STEAM_WAITS + 1)).suggestReporting(any()); + verify(bRepo, never()).save(any()); + assertThat(svc.status().failed()).isEqualTo(1); + } + + @Test + void aCancel_endsItAfterTheCurrentBinding() { + GameBindingRepository bRepo = mock(GameBindingRepository.class); + when(bRepo.findCandidatesForTwRebuild(any(Pageable.class))) + .thenReturn(new PageImpl<>(List.of(steamBinding("1"), steamBinding("2")))); + TwResolverService res = mock(TwResolverService.class); + when(res.suggestReporting(any())).thenReturn(new TwResolverService.Suggestion(Set.of("gore"), Optional.empty())); + TwBackfillService svc = build(mock(AppStateRepository.class), bRepo, mock(IgdbService.class), res); + svc.setSleeper(ms -> svc.cancel()); + + svc.rebuildAllNonOverridden(); + + assertThat(svc.status().state()).isEqualTo("CANCELLED"); + assertThat(svc.status().processed()).isEqualTo(1); + } + + @Test + void oneAtATime() throws Exception { + GameBindingRepository bRepo = mock(GameBindingRepository.class); + when(bRepo.findCandidatesForTwRebuild(any(Pageable.class))) + .thenReturn(new PageImpl<>(List.of(steamBinding("1")))); + TwResolverService res = mock(TwResolverService.class); + java.util.concurrent.CountDownLatch inside = new java.util.concurrent.CountDownLatch(1); + java.util.concurrent.CountDownLatch release = new java.util.concurrent.CountDownLatch(1); + when(res.suggestReporting(any())).thenAnswer(inv -> { + inside.countDown(); + release.await(); + return new TwResolverService.Suggestion(Set.of(), Optional.empty()); + }); + TwBackfillService svc = build(mock(AppStateRepository.class), bRepo, mock(IgdbService.class), res); + svc.setSleeper(ms -> {}); + + assertThat(svc.start(TwBackfillService.Mode.NON_OVERRIDDEN).state()).isEqualTo("RUNNING"); + assertThat(inside.await(5, java.util.concurrent.TimeUnit.SECONDS)).isTrue(); + org.assertj.core.api.Assertions.assertThatThrownBy(() -> svc.start(TwBackfillService.Mode.FORCE_PINNED)) + .isInstanceOf(org.springframework.web.server.ResponseStatusException.class).hasMessageContaining("409"); + release.countDown(); + for (int i = 0; i < 100 && "RUNNING".equals(svc.status().state()); i++) Thread.sleep(20); + assertThat(svc.status().state()).isEqualTo("DONE"); + } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwResolverServiceTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwResolverServiceTest.java index 346c944e1..ce17aaa02 100644 --- a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwResolverServiceTest.java +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwResolverServiceTest.java @@ -1,5 +1,12 @@ package fr.enimaloc.catapult.service.tw; +import fr.enimaloc.catapult.domain.tw.TwDtddTopicMapping; +import fr.enimaloc.catapult.domain.tw.TwIgdbDescriptorMapping; +import fr.enimaloc.catapult.domain.tw.TwSteamContentIdMapping; +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSeverity; +import fr.enimaloc.catapult.service.dtdd.DtddSignalService.TopicSignals; +import static org.assertj.core.api.Assertions.tuple; + import fr.enimaloc.catapult.domain.tw.TwSteamKeyword; import fr.enimaloc.catapult.repository.tw.TwDtddTopicMappingRepository; import fr.enimaloc.catapult.repository.tw.TwIgdbDescriptorMappingRepository; @@ -17,48 +24,130 @@ class TwResolverServiceTest { + private final TwIgdbDescriptorMappingRepository igdbRepo = mock(TwIgdbDescriptorMappingRepository.class); + private final TwSteamContentIdMappingRepository steamIdRepo = mock(TwSteamContentIdMappingRepository.class); + private final TwSteamKeywordRepository steamKwRepo = mock(TwSteamKeywordRepository.class); + private final TwDtddTopicMappingRepository dtddRepo = mock(TwDtddTopicMappingRepository.class); + private final DtddSignalService dtdd = mock(DtddSignalService.class); + private final SteamStoreService steam = mock(SteamStoreService.class); + private final TwResolverService svc = + new TwResolverService(igdbRepo, steamIdRepo, steamKwRepo, dtddRepo, Optional.of(dtdd), steam); + + private static TwDtddTopicMapping dtddMapping(String twId, String topic, String minSeverity) { + TwDtddTopicMapping m = new TwDtddTopicMapping(); + m.setTwId(twId); + m.setDtddTopicName(topic); + m.setMinSeverity(minSeverity); + return m; + } + + private static DtddSeverity gore(int minor, int depicted, int excessive) { + return new DtddSeverity(List.of("Minor", "Depicted", "Excessive"), List.of(minor, depicted, excessive), + List.of(), minor + depicted + excessive, null); + } + @Test void aggregatesAllSources_dtddIgdbSteamIdsAndKeywords() { - TwIgdbDescriptorMappingRepository igdbRepo = mock(TwIgdbDescriptorMappingRepository.class); - TwSteamContentIdMappingRepository steamIdRepo = mock(TwSteamContentIdMappingRepository.class); - TwSteamKeywordRepository steamKwRepo = mock(TwSteamKeywordRepository.class); - TwDtddTopicMappingRepository dtddRepo = mock(TwDtddTopicMappingRepository.class); - DtddSignalService dtdd = mock(DtddSignalService.class); - SteamStoreService steam = mock(SteamStoreService.class); - - when(igdbRepo.findTwIdsByDescriptorIds(Set.of(10L,11L))) - .thenReturn(Set.of("violence_graphic")); + TwIgdbDescriptorMapping igdb = new TwIgdbDescriptorMapping(); + igdb.setTwId("violence_graphic"); + igdb.setDescriptorId(10L); + when(igdbRepo.findAllByDescriptorIdIn(Set.of(10L, 11L))).thenReturn(List.of(igdb)); when(steam.fetchTwSignals(List.of("app42"))).thenReturn(Map.of( "app42", new SteamTwSignals(Set.of(2, 5), "blood and gore present"))); - when(steamIdRepo.findTwIdsBySteamContentIdIn(Set.of(2,5))) - .thenReturn(Set.of("violence_graphic")); + TwSteamContentIdMapping steamId = new TwSteamContentIdMapping(); + steamId.setTwId("violence_graphic"); + steamId.setSteamContentId(2); + when(steamIdRepo.findAllBySteamContentIdIn(Set.of(2, 5))).thenReturn(List.of(steamId)); TwSteamKeyword kw1 = new TwSteamKeyword(); kw1.setKeyword("gore"); kw1.setTwId("violence_graphic"); TwSteamKeyword kw2 = new TwSteamKeyword(); kw2.setKeyword("flashing"); kw2.setTwId("flashing_lights"); when(steamKwRepo.findAll()).thenReturn(List.of(kw1, kw2)); - when(dtdd.getYesMostlyTopics("igdb42", "Stardew")) - .thenReturn(Set.of("A dog dies")); - when(dtddRepo.findTwIdsByDtddTopicNameInIgnoreCase(Set.of("a dog dies"))) - .thenReturn(Set.of("death_of_animal")); + when(dtdd.getYesMostlySignals("igdb42", "Stardew")) + .thenReturn(new TopicSignals(Set.of("A dog dies"), Map.of())); + when(dtddRepo.findAllByDtddTopicNameInIgnoreCase(Set.of("a dog dies"))) + .thenReturn(List.of(dtddMapping("death_of_animal", "a dog dies", null))); - TwResolverService svc = new TwResolverService( - igdbRepo, steamIdRepo, steamKwRepo, dtddRepo, Optional.of(dtdd), steam); - - var out = svc.suggest(new TwResolverService.SuggestInput( - "igdb42", Set.of(10L,11L), "app42", "Stardew")); - assertThat(out).containsExactlyInAnyOrder("violence_graphic", "death_of_animal"); + var in = new TwResolverService.SuggestInput("igdb42", Set.of(10L, 11L), "app42", "Stardew"); + assertThat(svc.suggest(in)).containsExactlyInAnyOrder("violence_graphic", "death_of_animal"); // Note: "flashing" not in notes → flashing_lights not selected. + + // Each source told, the DTDD topic under its own name + assertThat(svc.explain(in).signals()).extracting(TwResolverService.Signal::source, TwResolverService.Signal::value) + .containsExactlyInAnyOrder( + tuple(TwResolverService.SourceType.IGDB_DESCRIPTOR, "10"), + tuple(TwResolverService.SourceType.STEAM_CONTENT, "2"), + tuple(TwResolverService.SourceType.STEAM_KEYWORD, "gore"), + tuple(TwResolverService.SourceType.DTDD_TOPIC, "A dog dies")); + } + + @Test + void aDtddTopicBelowItsThreshold_keepsItsTwOff_andIsTold() { + when(dtdd.getYesMostlySignals("igdb42", "Game")).thenReturn(new TopicSignals( + Set.of("there's blood/gore", "a dog dies"), Map.of("there's blood/gore", gore(8, 1, 1)))); + when(dtddRepo.findAllByDtddTopicNameInIgnoreCase(Set.of("there's blood/gore", "a dog dies"))).thenReturn(List.of( + dtddMapping("gore", "there's blood/gore", "Depicted"), + dtddMapping("violence", "there's blood/gore", null), + dtddMapping("death_of_animal", "a dog dies", "Depicted"))); + + var explanation = svc.explain(new TwResolverService.SuggestInput("igdb42", Set.of(), null, "Game")); + + // gore kept off (80 % Minor); the same topic without a threshold still turns violence on; a topic without + // levels still turns its TW on + assertThat(explanation.twIds()).containsExactlyInAnyOrder("violence", "death_of_animal"); + assertThat(explanation.dropped()).singleElement().satisfies(s -> { + assertThat(s.twId()).isEqualTo("gore"); + assertThat(s.severity().reason()).isEqualTo(TwSeverityRule.Reason.BELOW_VOTES); + assertThat(s.severity().percentAtOrAbove()).isEqualTo(20); + }); + } + + @Test + void severitiesNotCachedYet_turnTheTwOnAsBefore() { + when(dtdd.getYesMostlySignals("igdb42", "Game")).thenReturn(new TopicSignals(Set.of("there's blood/gore"), null)); + when(dtddRepo.findAllByDtddTopicNameInIgnoreCase(Set.of("there's blood/gore"))) + .thenReturn(List.of(dtddMapping("gore", "there's blood/gore", "Depicted"))); + + assertThat(svc.suggest(new TwResolverService.SuggestInput("igdb42", Set.of(), null, "Game"))) + .containsExactly("gore"); + } + + @Test + void theVotesNeeded_areConfigurable() { + when(dtdd.getYesMostlySignals("igdb42", "Game")).thenReturn(new TopicSignals( + Set.of("there's blood/gore"), Map.of("there's blood/gore", gore(4, 0, 0)))); + when(dtddRepo.findAllByDtddTopicNameInIgnoreCase(Set.of("there's blood/gore"))) + .thenReturn(List.of(dtddMapping("gore", "there's blood/gore", "Depicted"))); + var in = new TwResolverService.SuggestInput("igdb42", Set.of(), null, "Game"); + + assertThat(svc.suggest(in)).isEmpty(); + svc.setDtddSeverityMinVotes(5); + assertThat(svc.suggest(in)).containsExactly("gore"); } @Test void dtddAbsent_skipsSilently() { + TwResolverService none = new TwResolverService(igdbRepo, steamIdRepo, steamKwRepo, dtddRepo, Optional.empty(), + steam); + assertThat(none.suggest(new TwResolverService.SuggestInput("x", Set.of(), null, "x"))).isEmpty(); + } + + @Test + void steamSignalsMissedWhileRateLimited_areReported_andOtherwiseNot() { + SteamStoreService steam = mock(SteamStoreService.class); TwResolverService svc = new TwResolverService( - mock(TwIgdbDescriptorMappingRepository.class), - mock(TwSteamContentIdMappingRepository.class), - mock(TwSteamKeywordRepository.class), - mock(TwDtddTopicMappingRepository.class), - Optional.empty(), - mock(SteamStoreService.class)); - assertThat(svc.suggest(new TwResolverService.SuggestInput( - "x", Set.of(), null, "x"))).isEmpty(); + mock(TwIgdbDescriptorMappingRepository.class), mock(TwSteamContentIdMappingRepository.class), + mock(TwSteamKeywordRepository.class), mock(TwDtddTopicMappingRepository.class), Optional.empty(), steam); + java.time.Instant until = java.time.Instant.parse("2026-11-12T10:05:00Z"); + when(steam.fetchTwSignals(List.of("app42"))).thenReturn(Map.of()); + when(steam.rateLimitedUntil()).thenReturn(Optional.of(until)); + + assertThat(svc.suggestReporting(new TwResolverService.SuggestInput("x", Set.of(), "app42", "x")) + .steamLimitedUntil()).contains(until); + // Not a Steam game: nothing asked of Steam, nothing missed + assertThat(svc.suggestReporting(new TwResolverService.SuggestInput("x", Set.of(), null, "x")) + .steamLimitedUntil()).isEmpty(); + // Steam answered + when(steam.fetchTwSignals(List.of("app42"))).thenReturn(Map.of("app42", SteamTwSignals.empty())); + assertThat(svc.suggestReporting(new TwResolverService.SuggestInput("x", Set.of(), "app42", "x")) + .steamLimitedUntil()).isEmpty(); } } diff --git a/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwSeverityRuleTest.java b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwSeverityRuleTest.java new file mode 100644 index 000000000..f8feb7f72 --- /dev/null +++ b/catapult-api/src/test/java/fr/enimaloc/catapult/service/tw/TwSeverityRuleTest.java @@ -0,0 +1,70 @@ +package fr.enimaloc.catapult.service.tw; + +import fr.enimaloc.catapult.getter.dtdd.DtddApiClient.DtddSeverity; +import fr.enimaloc.catapult.service.tw.TwSeverityRule.Reason; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** A DTDD topic's severity against its TW mapping's least severe level. */ +class TwSeverityRuleTest { + + private static final List LEVELS = List.of("Minor", "Depicted", "Excessive"); + + private static DtddSeverity votes(int minor, int depicted, int excessive, Integer verified) { + return new DtddSeverity(LEVELS, List.of(minor, depicted, excessive), List.of(), minor + depicted + excessive, + verified); + } + + @Test + void withoutAThreshold_itTurnsOn() { + assertThat(TwSeverityRule.evaluate(null, true, votes(9, 0, 0, null), 3).reason()).isEqualTo(Reason.NO_THRESHOLD); + assertThat(TwSeverityRule.evaluate(" ", true, null, 3).triggers()).isTrue(); + } + + @Test + void whatDtddCantTell_turnsItOn() { + assertThat(TwSeverityRule.evaluate("Depicted", false, null, 3).reason()).isEqualTo(Reason.NOT_CACHED); + assertThat(TwSeverityRule.evaluate("Depicted", true, null, 3).reason()).isEqualTo(Reason.NO_LEVELS); + assertThat(TwSeverityRule.evaluate("Graphic", true, votes(9, 0, 0, null), 3).reason()) + .isEqualTo(Reason.UNKNOWN_LEVEL); + var tooFew = TwSeverityRule.evaluate("Depicted", true, votes(2, 0, 0, null), 3); + assertThat(tooFew.reason()).isEqualTo(Reason.TOO_FEW_VOTES); + assertThat(tooFew.triggers()).isTrue(); + assertThat(tooFew.level()).isEqualTo("Minor"); + assertThat(tooFew.votes()).isEqualTo(2); + } + + @Test + void theReviewersLevel_decides_whateverTheVotes() { + var below = TwSeverityRule.evaluate("Depicted", true, votes(0, 0, 9, 1), 3); + assertThat(below.reason()).isEqualTo(Reason.BELOW_VERIFIED); + assertThat(below.triggers()).isFalse(); + assertThat(below.level()).isEqualTo("Minor"); + // Even from a single vote + assertThat(TwSeverityRule.evaluate("Depicted", true, votes(1, 0, 0, 2), 3).reason()).isEqualTo(Reason.VERIFIED); + } + + @Test + void halfTheVotesAtOrAboveTheThreshold_turnItOn() { + // Minor is the most voted, yet 60 % are Depicted or worse + var spread = TwSeverityRule.evaluate("Depicted", true, votes(4, 3, 3, null), 3); + assertThat(spread.reason()).isEqualTo(Reason.VOTES); + assertThat(spread.percentAtOrAbove()).isEqualTo(60); + assertThat(spread.level()).isEqualTo("Minor"); + // A tie leans to the warning + assertThat(TwSeverityRule.evaluate("Depicted", true, votes(2, 2, 0, null), 3).triggers()).isTrue(); + + var mild = TwSeverityRule.evaluate("depicted", true, votes(7, 2, 1, null), 3); + assertThat(mild.reason()).isEqualTo(Reason.BELOW_VOTES); + assertThat(mild.triggers()).isFalse(); + assertThat(mild.percentAtOrAbove()).isEqualTo(30); + } + + @Test + void theMostVotedLevel_isTheMoreSevereOnATie() { + assertThat(TwSeverityRule.evaluate("Excessive", true, votes(3, 3, 0, null), 3).level()).isEqualTo("Depicted"); + } +} diff --git a/catapult-api/src/test/resources/access-matrix.csv b/catapult-api/src/test/resources/access-matrix.csv index d8f579361..bb1dad6a5 100644 --- a/catapult-api/src/test/resources/access-matrix.csv +++ b/catapult-api/src/test/resources/access-matrix.csv @@ -46,6 +46,7 @@ PUT,/api/internal/users/{userId}/bindings/{bindingId}/tws,userId=self,,401,401,P DELETE,/api/internal/users/{userId}/bindings/{bindingId}/tws,userId=self,,401,401,PASS,PASS,PASS,PASS,401 POST,/api/internal/users/{userId}/bindings/{bindingId}/reset,userId=self,,401,401,PASS,PASS,PASS,PASS,401 GET,/api/internal/users/{userId}/bindings/{bindingId}/tws/suggestions,userId=self,,401,401,PASS,PASS,PASS,PASS,401 +GET,/api/internal/users/{userId}/bindings/{bindingId}/tws/details,userId=self,,401,401,PASS,PASS,PASS,PASS,401 GET,/api/internal/dtdd-mappings/{igdbId},igdbId=1,,401,PASS,PASS,PASS,PASS,PASS,401 POST,/api/internal/dtdd-mappings/{igdbId}/proposals,igdbId=1,,401,401,PASS,PASS,PASS,PASS,401 GET,/api/internal/dtdd-games,?q=matrix,,401,PASS,PASS,PASS,PASS,PASS,401 @@ -120,6 +121,7 @@ GET,/api/internal/users/{userId}/obs,unmapped,,401,401,403,403,PASS,403,401 GET,/api/internal/users/{userId}/chat-command-settings,unmapped,,401,401,403,403,GATED,403,401 PUT,/api/internal/users/{userId}/chat-command-settings/{key},unmapped,,401,401,403,403,GATED,403,401 GET,/api/internal/users/{userId}/bindings/{bindingId}/tws/suggestions,unmapped,,401,401,403,403,PASS,403,401 +GET,/api/internal/users/{userId}/bindings/{bindingId}/tws/details,unmapped,,401,401,403,403,PASS,403,401 PUT,/api/internal/users/{userId}/bindings/{bindingId}/tws,unmapped,,401,401,403,403,PASS,403,401 GET,/api/internal/users/{userId}/api-token,unmapped,,401,401,403,403,PASS,403,401 DELETE,/api/internal/users/{userId}/api-token,unmapped,,401,401,403,403,PASS,403,401 @@ -217,6 +219,16 @@ GET,/api/internal/admin/dtdd-keys,,,401,PASS,403,403,403,PASS,401 POST,/api/internal/admin/dtdd-keys,,"{""apiKey"":""matrix""}",401,PASS,403,403,403,PASS,401 DELETE,/api/internal/admin/dtdd-keys/{keyId},keyId=matrix,,401,PASS,403,403,403,PASS,401 POST,/api/internal/admin/dtdd-keys/refresh,,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/dtdd-caches,,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/dtdd-caches/refetch-all,,,401,PASS,403,403,403,PASS,401 +POST,/api/internal/admin/dtdd-caches/refetch-all,,,401,PASS,403,403,403,PASS,401 +DELETE,/api/internal/admin/dtdd-caches/refetch-all,,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/dtdd-caches/{kind},kind=topics,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/dtdd-caches/{kind}/entry,kind=topics;?key=matrix,,401,PASS,403,403,403,PASS,401 +POST,/api/internal/admin/dtdd-caches/{kind}/refetch,kind=topics;?key=matrix,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/rate-limits,,,401,PASS,403,403,403,PASS,401 +PUT,/api/internal/admin/rate-limits/entry,?id=matrix,,401,PASS,403,403,403,PASS,401 +DELETE,/api/internal/admin/rate-limits/entry,?id=matrix,,401,PASS,403,403,403,PASS,401 GET,/api/internal/admin/dtdd-mapping-proposals,,,401,PASS,403,403,403,PASS,401 # handler reads the JWT itself: 401 without one POST,/api/internal/admin/dtdd-mapping-proposals/{id}/approve,,,401,401,403,403,403,PASS,401 @@ -326,10 +338,13 @@ POST,/api/internal/admin/tws,,"{""id"":""matrix"",""label"":""matrix""}",401,PAS POST,/api/internal/admin/tws/rebuild,,,401,PASS,403,403,403,PASS,401 POST,/api/internal/admin/tws/rebuild/force,,,401,PASS,403,403,403,PASS,401 POST,/api/internal/admin/tws/rebuild/force-pinned,,,401,PASS,403,403,403,PASS,401 +GET,/api/internal/admin/tws/rebuild/status,,,401,PASS,403,403,403,PASS,401 +POST,/api/internal/admin/tws/rebuild/cancel,,,401,PASS,403,403,403,PASS,401 DELETE,/api/internal/admin/tws/{id},,,401,PASS,403,403,403,PASS,401 GET,/api/internal/admin/tws/{id},,,401,PASS,403,403,403,PASS,401 # ERROR: the dead id is no TW, and the handler throws IllegalArgumentException for it (test artefact) PATCH,/api/internal/admin/tws/{id},,,401,ERROR,403,403,403,ERROR,401 +GET,/api/internal/admin/tws/{id}/dtdd-topics,,,401,PASS,403,403,403,PASS,401 # ERROR: the dead id is no TW, and the handler throws IllegalArgumentException for it (test artefact) PUT,/api/internal/admin/tws/{id}/dtdd-topics,,,401,ERROR,403,403,403,ERROR,401 # ERROR: the dead id is no TW, and the handler throws IllegalArgumentException for it (test artefact) diff --git a/catapult-api/src/test/resources/frozen-routes/api-docs.json b/catapult-api/src/test/resources/frozen-routes/api-docs.json index 53cd27072..94257e309 100644 --- a/catapult-api/src/test/resources/frozen-routes/api-docs.json +++ b/catapult-api/src/test/resources/frozen-routes/api-docs.json @@ -4,5 +4,5 @@ "Cache-Control" : [ "no-cache, no-store, max-age=0, must-revalidate" ], "Content-Type" : [ "application/json" ] }, - "body" : "{\"openapi\":\"3.1.0\",\"info\":{\"title\":\"Catapult API\",\"description\":\"Your channel's game, bindings and custom bindings, and public per-platform game detail.\\n\\n## Authentication\\n\\nThe `/api/me` routes take a personal API token, generated from the account settings (or from the keys card on this page). The `/api/games` routes are public and take none.\\n\\n| Token | Prefix | Opens |\\n|---|---|---|\\n| Full | `cat_
` | every `/api/me` route, reads and changes |\\n| Read-only | `catr_
` | the `/api/me` `GET` routes only |\\n\\nSend it in the `Authorization` header:\\n\\n```\\ncurl -H \\\"Authorization: Bearer cat_
\\\" -X DELETE http://localhost/api/me/custom-bindings/active\\ncurl -H \\\"Authorization: Bearer catr_
\\\" http://localhost/api/me/game\\n```\\n\\nWhere no header can be set (a URL pasted into an OBS browser source, a chat bot), a `GET` takes the read-only token in the query string instead, `?token=catr_
`; the full token is refused there:\\n\\n```\\ncurl \\\"http://localhost/api/me/game/summary?token=catr_
\\\"\\n```\\n\\nHere, **Authorize** keeps a token for **Try it out**: `personalToken` for the header, `readToken` for the query string.\\n\",\"version\":\"2\"},\"servers\":[{\"url\":\"http://localhost\"}],\"tags\":[{\"name\":\"Me — drops\",\"description\":\"The Twitch drops campaigns running for your channel, by your personal API token (Authorization: Bearer cat_
 or catr_
).\"},{\"name\":\"Me — custom bindings\",\"description\":\"Trigger your channel's custom bindings, by your personal API token (Authorization: Bearer cat_
) generated from the account settings; the read-only token (catr_
) lists them only.\"},{\"name\":\"Games\",\"description\":\"Public per-platform game detail, keyed by the platform's own id; no token needed.\"},{\"name\":\"Me — game\",\"description\":\"The game your channel plays now, by your personal API token (Authorization: Bearer cat_
 or catr_
, or ?token=catr_
 for a pasted URL).\"},{\"name\":\"Me — bindings\",\"description\":\"Your game bindings (Catapult's trigger-warning and content-label configuration of a game), by your personal API token.\"}],\"paths\":{\"/api/me/custom-bindings/active\":{\"put\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"Apply a custom binding now (204), live or not; 400 no idOrName, 404 unknown, 409 Twitch link broken or Catapult paused, 502 Twitch failed; errors are application/problem+json with a code\",\"description\":\"**Token:** `Authorization: Bearer cat_
` (the full token only; the read-only one and the query string are refused).\",\"operationId\":\"activate\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/ActivateRequest\"}}},\"required\":true},\"responses\":{\"200\":{\"description\":\"OK\"}},\"security\":[{\"personalToken\":[]}]},\"delete\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"End the active custom binding, if any: the game open now takes over\",\"description\":\"**Token:** `Authorization: Bearer cat_
` (the full token only; the read-only one and the query string are refused).\",\"operationId\":\"deactivate\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"204\":{\"description\":\"No Content\"}},\"security\":[{\"personalToken\":[]}]}},\"/api/me/game\":{\"get\":{\"tags\":[\"Me — game\"],\"summary\":\"Get your channel's currently-detected game, fully aggregated\",\"description\":\"Every platform object that could be resolved for the game your channel plays now (igdb/steam/xbox/dtdd/catapult, each null if unavailable), each carrying a \\\"more\\\" link to its own detail route.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"gameInfo\",\"parameters\":[{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"f\",\"in\":\"query\",\"description\":\"One field of the answer, as plain text instead of JSON: a dotted path (name, igdb.url, releaseDate.releaseDate). For chat bots that can't read JSON; a list is joined with \\\", \\\", a missing field is empty.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"igdb.url\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"inGame=false with every platform object null while no game is detected; otherwise the aggregated game info.\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"object\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/game/summary\":{\"get\":{\"tags\":[\"Me — game\"],\"summary\":\"Get a compact summary of your channel's currently-detected game\",\"description\":\"Same aggregation as GET /api/me/game, but only inGame plus the cross-platform derived fields (name, description, tws, releaseDate — the first non-null value found across igdb/steam/xbox/dtdd/catapult, in that priority order), not the per-platform objects.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"summary\",\"parameters\":[{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"f\",\"in\":\"query\",\"description\":\"One field of the answer, as plain text instead of JSON: a dotted path (name, igdb.url, releaseDate.releaseDate). For chat bots that can't read JSON; a list is joined with \\\", \\\", a missing field is empty.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"name\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"inGame=false while no game is detected; otherwise the summarized game info.\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"object\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/drops\":{\"get\":{\"tags\":[\"Me — drops\"],\"summary\":\"The drops campaigns running for your channel on its current category, soonest end first\",\"description\":\"Empty while no game is detected, or when the game's category has no campaign running for your channel; 403 feature_disabled while drops aren't offered to your channel.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"drops\",\"parameters\":[{\"name\":\"categoryId\",\"in\":\"query\",\"description\":\"A Twitch category id, instead of the current game's\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":1282866981},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"OK\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/DropCampaignResponse\"}}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/custom-bindings\":{\"get\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"The custom bindings and the active one\",\"description\":\"**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"list\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"OK\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/CustomBindingsStateDto\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/bindings/{bindingId}\":{\"get\":{\"tags\":[\"Me — bindings\"],\"summary\":\"Get one of your bindings (tw/ccl config)\",\"description\":\"Lookup by GameBinding id, as the \\\"more\\\" link on the catapult object of GET /api/me/game names it.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"binding\",\"parameters\":[{\"name\":\"bindingId\",\"in\":\"path\",\"description\":\"GameBinding id.\",\"required\":true,\"schema\":{\"type\":\"string\",\"format\":\"uuid\"}},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize the returned twLabels. Falls back to the Accept-Language request header, then English.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The binding's tw/ccl configuration.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/CatapultDetailResponse\"}}}},\"404\":{\"description\":\"No binding of yours with that id.\"}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/games/xbox/{productId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full Microsoft Store detail for a product\",\"description\":\"Public lookup by Microsoft Store product id, via the unofficial, unauthenticated Display Catalog API (no XSTS/linked-account session needed — see XboxStoreServiceImpl).\",\"operationId\":\"xboxDetail\",\"parameters\":[{\"name\":\"productId\",\"in\":\"path\",\"description\":\"Microsoft Store product id.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":\"9NBLGGH2JHXJ\"},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The Xbox/Microsoft Store product's detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/XboxProduct\"}}}},\"404\":{\"description\":\"No product with that id, or the catalog fetch failed.\"}}}},\"/api/games/steam/{appId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full Steam store detail for an app\",\"description\":\"Public lookup by Steam appId — fetches the store appdetails page live (no local cache), plus content-label (ccl) data and, if this app is a demo/beta/playtest, the resolved parent app. releaseDate is a parsed Instant, not Steam's raw locale-formatted string (see SteamReleaseDate).\",\"operationId\":\"steamDetail\",\"parameters\":[{\"name\":\"appId\",\"in\":\"path\",\"description\":\"Steam appId.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":400},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag Steam's appdetails API is queried with (see SteamLanguages) — controls the store description/date-format language, not just a display hint. Unlisted tags silently fall back to English.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\",\"enum\":[\"en\",\"fr\",\"de\",\"it\",\"es\",\"pt\",\"pt-BR\",\"zh\",\"zh-TW\",\"zh-HK\",\"ru\",\"ja\",\"ko\",\"th\",\"tr\",\"uk\",\"nl\",\"da\",\"fi\",\"no\",\"sv\",\"pl\",\"hu\",\"cs\",\"ro\",\"bg\",\"el\",\"vi\",\"ar\",\"id\"]},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The Steam app's store-page detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/SteamDetailResponse\"}}}},\"404\":{\"description\":\"No Steam app with that id, or the store fetch failed.\"}}}},\"/api/games/igdb/{igdbId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full IGDB detail for a game\",\"description\":\"Public lookup by IGDB game id — served from IgdbGameDetailsService's stale-while-revalidate cache, so a cache miss triggers a live IGDB fetch (rating, genres, media, dlcs/similarGames with their own \\\"more\\\" links back into this same endpoint, etc.).\",\"operationId\":\"igdbDetail\",\"parameters\":[{\"name\":\"igdbId\",\"in\":\"path\",\"description\":\"IGDB game id.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":1234},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The IGDB game's detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/IgdbDetailResponse\"}}}},\"404\":{\"description\":\"No IGDB game with that id.\"}}}},\"/api/games/dtdd/{dtddId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full \\\"Does the Dog Die\\\" content-warning detail for a title\",\"description\":\"Public lookup by DTDD numeric id — yes/no/mostly trigger-warning topic votes plus item metadata (overview, genres, poster/background images, per-topic vote counts). 404 if the dtdd.enabled feature flag is off, not just if the id doesn't resolve.\",\"operationId\":\"dtddDetail\",\"parameters\":[{\"name\":\"dtddId\",\"in\":\"path\",\"description\":\"DTDD (doesthedogdie.com) numeric item id.\",\"required\":true,\"schema\":{\"type\":\"integer\",\"format\":\"int64\"},\"example\":16758},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The title's DTDD content-warning detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/DtddDetailResponse\"}}}},\"404\":{\"description\":\"DTDD support disabled, or no title with that id.\"}}}}},\"components\":{\"schemas\":{\"ActivateRequest\":{\"type\":\"object\",\"properties\":{\"idOrName\":{\"type\":\"string\",\"description\":\"The custom binding's id or name, case ignored.\",\"example\":\"Art\"}},\"required\":[\"idOrName\"]},\"DropCampaignResponse\":{\"type\":\"object\",\"description\":\"A Twitch drops campaign running for your channel\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"The campaign's Twitch id\"},\"name\":{\"type\":\"string\",\"description\":\"Its name\",\"example\":\"SEASON AURORA - WARFARE\"},\"gameId\":{\"type\":\"string\",\"description\":\"The Twitch category it runs on\"},\"gameName\":{\"type\":\"string\",\"description\":\"That category's name\"},\"startAt\":{\"type\":\"string\",\"format\":\"date-time\",\"description\":\"When it started\"},\"endAt\":{\"type\":\"string\",\"format\":\"date-time\",\"description\":\"When it ends\"},\"detailsUrl\":{\"type\":\"string\",\"description\":\"Its page, null when it has none\"},\"imageUrl\":{\"type\":\"string\",\"description\":\"Its image, null when it has none\"}}},\"CustomBindingDto\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"string\",\"format\":\"uuid\"},\"name\":{\"type\":\"string\"},\"twitchGameId\":{\"type\":\"string\"},\"twitchGameName\":{\"type\":\"string\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"tws\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"uninterruptible\":{\"type\":\"boolean\"},\"title\":{\"type\":\"string\"}}},\"CustomBindingsStateDto\":{\"type\":\"object\",\"properties\":{\"enabled\":{\"type\":\"boolean\"},\"bindings\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/CustomBindingDto\"}},\"activeId\":{\"type\":\"string\",\"format\":\"uuid\"},\"activatedAt\":{\"type\":\"string\",\"format\":\"date-time\"}}},\"CatapultDetailResponse\":{\"type\":\"object\",\"properties\":{\"createdAt\":{\"type\":\"string\",\"format\":\"date-time\"},\"updatedAt\":{\"type\":\"string\",\"format\":\"date-time\"},\"sourceType\":{\"type\":\"string\",\"enum\":[\"STEAM\",\"XBOX\",\"BATTLENET\",\"MANUAL\",\"MINECRAFT\"]},\"sourceId\":{\"type\":\"string\"},\"sourceName\":{\"type\":\"string\"},\"twitchGameId\":{\"type\":\"string\"},\"twitchGameName\":{\"type\":\"string\"},\"ignored\":{\"type\":\"boolean\"},\"cclEnabled\":{\"type\":\"boolean\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"twEnabled\":{\"type\":\"boolean\"},\"twOverride\":{\"type\":\"boolean\"},\"tws\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"twLabels\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true}}},\"XboxImage\":{\"type\":\"object\",\"properties\":{\"uri\":{\"type\":\"string\"},\"imagePurpose\":{\"type\":\"string\"},\"width\":{\"type\":\"integer\",\"format\":\"int32\"},\"height\":{\"type\":\"integer\",\"format\":\"int32\"},\"caption\":{\"type\":\"string\"}}},\"XboxProduct\":{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"},\"description\":{\"type\":\"string\"},\"publisherName\":{\"type\":\"string\"},\"developerName\":{\"type\":\"string\"},\"publisherAddress\":{\"type\":\"string\"},\"publisherWebsiteUri\":{\"type\":\"string\"},\"supportUri\":{\"type\":\"string\"},\"supportPhone\":{\"type\":\"string\"},\"lastModifiedDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"images\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/XboxImage\"}},\"videos\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/XboxVideo\"}},\"franchises\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"gamePassAffirmations\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"storeUrl\":{\"type\":\"string\"}}},\"XboxVideo\":{\"type\":\"object\",\"properties\":{\"caption\":{\"type\":\"string\"},\"dashUrl\":{\"type\":\"string\"},\"hlsUrl\":{\"type\":\"string\"},\"previewImageUri\":{\"type\":\"string\"},\"width\":{\"type\":\"integer\",\"format\":\"int32\"},\"height\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"Achievement\":{\"type\":\"object\",\"properties\":{\"total\":{\"type\":\"integer\",\"format\":\"int32\"},\"highlighted\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Highlighted\"}}}},\"Category\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"}}},\"ContentDescriptors\":{\"type\":\"object\",\"properties\":{\"ids\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"note\":{\"type\":\"string\"}}},\"Demo\":{\"type\":\"object\",\"properties\":{\"appid\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Dlc\":{\"type\":\"object\",\"properties\":{\"appId\":{\"type\":\"integer\",\"format\":\"int32\"},\"more\":{\"type\":\"string\"}}},\"FullGame\":{\"type\":\"object\",\"properties\":{\"appid\":{\"type\":\"integer\",\"format\":\"int32\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Genre\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"}}},\"Highlighted\":{\"type\":\"object\",\"properties\":{\"icon\":{\"type\":\"string\"},\"localizedName\":{\"type\":\"string\"},\"archived\":{\"type\":\"boolean\"},\"hidden\":{\"type\":\"boolean\"},\"name\":{\"type\":\"string\"},\"path\":{\"type\":\"string\"}}},\"IaDisclosure\":{\"type\":\"object\",\"properties\":{\"hasDisclosure\":{\"type\":\"boolean\"},\"note\":{\"type\":\"string\"}}},\"Metacritic\":{\"type\":\"object\",\"properties\":{\"score\":{\"type\":\"integer\",\"format\":\"int32\"},\"url\":{\"type\":\"string\"}}},\"Movie\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"name\":{\"type\":\"string\"},\"thumbnail\":{\"type\":\"string\"},\"dashAv1\":{\"type\":\"string\"},\"dashH264\":{\"type\":\"string\"},\"hlsH264\":{\"type\":\"string\"},\"highlight\":{\"type\":\"boolean\"}}},\"Packages\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"},\"title\":{\"type\":\"string\"},\"description\":{\"type\":\"string\"},\"selectionText\":{\"type\":\"string\"},\"saveText\":{\"type\":\"string\"},\"displayType\":{\"type\":\"integer\",\"format\":\"int32\"},\"isRecurringSubscription\":{\"type\":\"string\"},\"subs\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Sub\"}}}},\"Page\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"steamAppId\":{\"type\":\"integer\",\"format\":\"int32\"},\"requiredAge\":{\"type\":\"integer\",\"format\":\"int32\"},\"isFree\":{\"type\":\"boolean\"},\"controllerSupport\":{\"type\":\"string\"},\"dlc\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Dlc\"}},\"detailedDescription\":{\"type\":\"string\"},\"aboutTheGame\":{\"type\":\"string\"},\"shortDescription\":{\"type\":\"string\"},\"fullgame\":{\"$ref\":\"#/components/schemas/FullGame\"},\"supportedLanguages\":{\"type\":\"string\"},\"reviews\":{\"type\":\"string\"},\"headerImage\":{\"type\":\"string\"},\"capsuleImage\":{\"type\":\"string\"},\"capsuleImageV5\":{\"type\":\"string\"},\"website\":{\"type\":\"string\"},\"pcRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"macRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"linuxRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"legalNotice\":{\"type\":\"string\"},\"developers\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"publishers\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"demos\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Demo\"}},\"priceOverview\":{\"$ref\":\"#/components/schemas/PriceOverview\"},\"packages\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"packageGroups\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Packages\"}},\"platforms\":{\"$ref\":\"#/components/schemas/Platforms\"},\"metacritic\":{\"$ref\":\"#/components/schemas/Metacritic\"},\"categories\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Category\"}},\"genres\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Genre\"}},\"screenshots\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Screenshot\"}},\"movies\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Movie\"}},\"recommendations\":{\"$ref\":\"#/components/schemas/Recommendations\"},\"achievements\":{\"$ref\":\"#/components/schemas/Achievement\"},\"releaseDate\":{\"$ref\":\"#/components/schemas/ReleaseDate\"},\"supportInfo\":{\"$ref\":\"#/components/schemas/SupportInfo\"},\"background\":{\"type\":\"string\"},\"backgroundRaw\":{\"type\":\"string\"},\"contentDescriptors\":{\"$ref\":\"#/components/schemas/ContentDescriptors\"},\"ratings\":{\"$ref\":\"#/components/schemas/Ratings\"},\"demo\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Demo\"}}}},\"ParentApp\":{\"type\":\"object\",\"properties\":{\"appId\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Platforms\":{\"type\":\"object\",\"properties\":{\"windows\":{\"type\":\"boolean\"},\"mac\":{\"type\":\"boolean\"},\"linux\":{\"type\":\"boolean\"}}},\"PriceOverview\":{\"type\":\"object\",\"properties\":{\"currency\":{\"type\":\"string\"},\"initial\":{\"type\":\"integer\",\"format\":\"int32\"},\"final0\":{\"type\":\"integer\",\"format\":\"int32\"},\"discountPercent\":{\"type\":\"integer\",\"format\":\"int32\"},\"initialFormatted\":{\"type\":\"string\"},\"finalFormatted\":{\"type\":\"string\"}}},\"Rating\":{\"type\":\"object\",\"properties\":{\"ratingGenerated\":{\"type\":\"string\"},\"rating\":{\"type\":\"string\"},\"requiredAge\":{\"type\":\"string\"},\"banned\":{\"type\":\"string\"},\"useAgeGate\":{\"type\":\"string\"},\"descriptors\":{\"type\":\"string\"},\"interactiveElements\":{\"type\":\"string\"},\"displayOnlineNotice\":{\"type\":\"string\"},\"displayOnlineMusicNotice\":{\"type\":\"string\"}}},\"Ratings\":{\"type\":\"object\",\"properties\":{\"esrb\":{\"$ref\":\"#/components/schemas/Rating\"},\"pegi\":{\"$ref\":\"#/components/schemas/Rating\"},\"bbfc\":{\"$ref\":\"#/components/schemas/Rating\"},\"usk\":{\"$ref\":\"#/components/schemas/Rating\"},\"cero\":{\"$ref\":\"#/components/schemas/Rating\"},\"kggrb\":{\"$ref\":\"#/components/schemas/Rating\"},\"fpb\":{\"$ref\":\"#/components/schemas/Rating\"},\"csrr\":{\"$ref\":\"#/components/schemas/Rating\"},\"crl\":{\"$ref\":\"#/components/schemas/Rating\"},\"agcom\":{\"$ref\":\"#/components/schemas/Rating\"},\"oflc\":{\"$ref\":\"#/components/schemas/Rating\"},\"nzoflc\":{\"$ref\":\"#/components/schemas/Rating\"},\"cadpa\":{\"$ref\":\"#/components/schemas/Rating\"},\"dejus\":{\"$ref\":\"#/components/schemas/Rating\"},\"germanyRating\":{\"$ref\":\"#/components/schemas/Rating\"},\"igrs\":{\"$ref\":\"#/components/schemas/Rating\"},\"steamAustralia\":{\"$ref\":\"#/components/schemas/Rating\"},\"mda\":{\"$ref\":\"#/components/schemas/Rating\"},\"gmedia\":{\"$ref\":\"#/components/schemas/Rating\"}}},\"Recommendations\":{\"type\":\"object\",\"properties\":{\"total\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"ReleaseDate\":{\"type\":\"object\",\"properties\":{\"releaseDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"comingSoon\":{\"type\":\"boolean\"}}},\"Requirement\":{\"type\":\"object\",\"properties\":{\"minimum\":{\"type\":\"string\"},\"recommended\":{\"type\":\"string\"}}},\"Screenshot\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"pathThumbnail\":{\"type\":\"string\"},\"pathFull\":{\"type\":\"string\"}}},\"SteamDetailResponse\":{\"type\":\"object\",\"properties\":{\"storeUrl\":{\"type\":\"string\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"ia\":{\"$ref\":\"#/components/schemas/IaDisclosure\"},\"parentApp\":{\"$ref\":\"#/components/schemas/ParentApp\"},\"page\":{\"$ref\":\"#/components/schemas/Page\"}}},\"Sub\":{\"type\":\"object\",\"properties\":{\"packageid\":{\"type\":\"integer\",\"format\":\"int32\"},\"percentSavingText\":{\"type\":\"string\"},\"percentSaving\":{\"type\":\"integer\",\"format\":\"int32\"},\"optionText\":{\"type\":\"string\"},\"optionDescription\":{\"type\":\"string\"},\"canGetFreeLicense\":{\"type\":\"string\"},\"isFreeLicense\":{\"type\":\"boolean\"},\"priceInCentsWithDiscount\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"SupportInfo\":{\"type\":\"object\",\"properties\":{\"url\":{\"type\":\"string\"},\"email\":{\"type\":\"string\"}}},\"IgdbDetailResponse\":{\"type\":\"object\",\"properties\":{\"slug\":{\"type\":\"string\"},\"url\":{\"type\":\"string\"},\"summary\":{\"type\":\"string\"},\"storyline\":{\"type\":\"string\"},\"platforms\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"rating\":{\"type\":\"number\",\"format\":\"double\"},\"aggregatedRating\":{\"type\":\"number\",\"format\":\"double\"},\"totalRating\":{\"type\":\"number\",\"format\":\"double\"},\"totalRatingCount\":{\"type\":\"integer\",\"format\":\"int32\"},\"releaseDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"websites\":{\"type\":\"object\",\"additionalProperties\":{}},\"dlcs\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/IgdbRef\"}},\"similarGames\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/IgdbRef\"}},\"genres\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"gameModes\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"themes\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"playerPerspectives\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"keywords\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"franchiseNames\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"coverUrl\":{\"type\":\"string\"},\"screenshotUrls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"videoUrls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"IgdbRef\":{\"type\":\"object\",\"properties\":{\"igdbId\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"DtddDetailResponse\":{\"type\":\"object\",\"properties\":{\"url\":{\"type\":\"string\"},\"yesTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"noTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"mostlyTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"overview\":{\"type\":\"string\"},\"genres\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"releaseYear\":{\"type\":\"integer\",\"format\":\"int64\"},\"itemTypeName\":{\"type\":\"string\"},\"tmdbId\":{\"type\":\"integer\",\"format\":\"int64\"},\"imdbId\":{\"type\":\"string\"},\"posterImage\":{\"type\":\"string\"},\"backgroundImage\":{\"type\":\"string\"},\"topicItemStats\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/DtddTopicItemStat\"}}}},\"DtddTopicItemStat\":{\"type\":\"object\",\"properties\":{\"topicItemId\":{\"type\":\"integer\",\"format\":\"int64\"},\"yesSum\":{\"type\":\"integer\",\"format\":\"int64\"},\"noSum\":{\"type\":\"integer\",\"format\":\"int64\"},\"numComments\":{\"type\":\"integer\",\"format\":\"int64\"},\"topicId\":{\"type\":\"integer\",\"format\":\"int64\"},\"topicName\":{\"type\":\"string\"},\"itemId\":{\"type\":\"integer\",\"format\":\"int64\"}}}},\"securitySchemes\":{\"personalToken\":{\"type\":\"http\",\"description\":\"A personal API token from the account settings, in the Authorization header: the full token (cat_
) reads and changes everything, the read-only token (catr_
) only the GET routes.\",\"scheme\":\"bearer\"},\"readToken\":{\"type\":\"apiKey\",\"description\":\"The read-only personal token (catr_
) in the query string, for a URL that cannot carry a header. GET routes only, and only when the request has no Authorization header; the full token is refused here.\",\"name\":\"token\",\"in\":\"query\"}}}}" + "body" : "{\"openapi\":\"3.1.0\",\"info\":{\"title\":\"Catapult API\",\"description\":\"Your channel's game, bindings and custom bindings, and public per-platform game detail.\\n\\n## Authentication\\n\\nThe `/api/me` routes take a personal API token, generated from the account settings (or from the keys card on this page). The `/api/games` routes are public and take none.\\n\\n| Token | Prefix | Opens |\\n|---|---|---|\\n| Full | `cat_
` | every `/api/me` route, reads and changes |\\n| Read-only | `catr_
` | the `/api/me` `GET` routes only |\\n\\nSend it in the `Authorization` header:\\n\\n```\\ncurl -H \\\"Authorization: Bearer cat_
\\\" -X DELETE http://localhost/api/me/custom-bindings/active\\ncurl -H \\\"Authorization: Bearer catr_
\\\" http://localhost/api/me/game\\n```\\n\\nWhere no header can be set (a URL pasted into an OBS browser source, a chat bot), a `GET` takes the read-only token in the query string instead, `?token=catr_
`; the full token is refused there:\\n\\n```\\ncurl \\\"http://localhost/api/me/game/summary?token=catr_
\\\"\\n```\\n\\nHere, **Authorize** keeps a token for **Try it out**: `personalToken` for the header, `readToken` for the query string.\\n\",\"version\":\"2\"},\"servers\":[{\"url\":\"http://localhost\"}],\"tags\":[{\"name\":\"Me — drops\",\"description\":\"The Twitch drops campaigns running for your channel, by your personal API token (Authorization: Bearer cat_
 or catr_
).\"},{\"name\":\"Me — custom bindings\",\"description\":\"Trigger your channel's custom bindings, by your personal API token (Authorization: Bearer cat_
) generated from the account settings; the read-only token (catr_
) lists them only.\"},{\"name\":\"Games\",\"description\":\"Public per-platform game detail, keyed by the platform's own id; no token needed.\"},{\"name\":\"Me — game\",\"description\":\"The game your channel plays now, by your personal API token (Authorization: Bearer cat_
 or catr_
, or ?token=catr_
 for a pasted URL).\"},{\"name\":\"Me — bindings\",\"description\":\"Your game bindings (Catapult's trigger-warning and content-label configuration of a game), by your personal API token.\"}],\"paths\":{\"/api/me/custom-bindings/active\":{\"put\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"Apply a custom binding now (204), live or not; 400 no idOrName, 404 unknown, 409 Twitch link broken or Catapult paused, 502 Twitch failed; errors are application/problem+json with a code\",\"description\":\"**Token:** `Authorization: Bearer cat_
` (the full token only; the read-only one and the query string are refused).\",\"operationId\":\"activate\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"$ref\":\"#/components/schemas/ActivateRequest\"}}},\"required\":true},\"responses\":{\"200\":{\"description\":\"OK\"}},\"security\":[{\"personalToken\":[]}]},\"delete\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"End the active custom binding, if any: the game open now takes over\",\"description\":\"**Token:** `Authorization: Bearer cat_
` (the full token only; the read-only one and the query string are refused).\",\"operationId\":\"deactivate\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"204\":{\"description\":\"No Content\"}},\"security\":[{\"personalToken\":[]}]}},\"/api/me/game\":{\"get\":{\"tags\":[\"Me — game\"],\"summary\":\"Get your channel's currently-detected game, fully aggregated\",\"description\":\"Every platform object that could be resolved for the game your channel plays now (igdb/steam/xbox/dtdd/catapult, each null if unavailable), each carrying a \\\"more\\\" link to its own detail route.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"gameInfo\",\"parameters\":[{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"f\",\"in\":\"query\",\"description\":\"One field of the answer, as plain text instead of JSON: a dotted path (name, igdb.url, releaseDate.releaseDate). For chat bots that can't read JSON; a list is joined with \\\", \\\", a missing field is empty.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"igdb.url\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"inGame=false with every platform object null while no game is detected; otherwise the aggregated game info.\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"object\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/game/summary\":{\"get\":{\"tags\":[\"Me — game\"],\"summary\":\"Get a compact summary of your channel's currently-detected game\",\"description\":\"Same aggregation as GET /api/me/game, but only inGame plus the cross-platform derived fields (name, description, tws, releaseDate — the first non-null value found across igdb/steam/xbox/dtdd/catapult, in that priority order), not the per-platform objects.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"summary\",\"parameters\":[{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"f\",\"in\":\"query\",\"description\":\"One field of the answer, as plain text instead of JSON: a dotted path (name, igdb.url, releaseDate.releaseDate). For chat bots that can't read JSON; a list is joined with \\\", \\\", a missing field is empty.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"name\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"inGame=false while no game is detected; otherwise the summarized game info.\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"object\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/drops\":{\"get\":{\"tags\":[\"Me — drops\"],\"summary\":\"The drops campaigns running for your channel on its current category, soonest end first\",\"description\":\"Empty while no game is detected, or when the game's category has no campaign running for your channel; 403 feature_disabled while drops aren't offered to your channel.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"drops\",\"parameters\":[{\"name\":\"categoryId\",\"in\":\"query\",\"description\":\"A Twitch category id, instead of the current game's\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":1282866981},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"OK\",\"content\":{\"*/*\":{\"schema\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/DropCampaignResponse\"}}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/custom-bindings\":{\"get\":{\"tags\":[\"Me — custom bindings\"],\"summary\":\"The custom bindings and the active one\",\"description\":\"**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"list\",\"parameters\":[{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"OK\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/CustomBindingsStateDto\"}}}}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/me/bindings/{bindingId}\":{\"get\":{\"tags\":[\"Me — bindings\"],\"summary\":\"Get one of your bindings (tw/ccl config)\",\"description\":\"Lookup by GameBinding id, as the \\\"more\\\" link on the catapult object of GET /api/me/game names it.\\n\\n**Token:** `Authorization: Bearer cat_
` or `Authorization: Bearer catr_
`, or, without that header, `?token=catr_
` (the read-only token only).\",\"operationId\":\"binding\",\"parameters\":[{\"name\":\"bindingId\",\"in\":\"path\",\"description\":\"GameBinding id.\",\"required\":true,\"schema\":{\"type\":\"string\",\"format\":\"uuid\"}},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize the returned twLabels. Falls back to the Accept-Language request header, then English.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The binding's tw/ccl configuration.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/CatapultDetailResponse\"}}}},\"404\":{\"description\":\"No binding of yours with that id.\"}},\"security\":[{\"personalToken\":[]},{\"readToken\":[]}]}},\"/api/games/xbox/{productId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full Microsoft Store detail for a product\",\"description\":\"Public lookup by Microsoft Store product id, via the unofficial, unauthenticated Display Catalog API (no XSTS/linked-account session needed — see XboxStoreServiceImpl).\",\"operationId\":\"xboxDetail\",\"parameters\":[{\"name\":\"productId\",\"in\":\"path\",\"description\":\"Microsoft Store product id.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":\"9NBLGGH2JHXJ\"},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag used to localize/re-fetch platform data (Steam store text and date format, trigger warning labels, Xbox store text). Falls back to the Accept-Language request header, then English. For the list of locales Steam actually supports, see the \\\"lang\\\" parameter on GET /steam/{appId}.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\"},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The Xbox/Microsoft Store product's detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/XboxProduct\"}}}},\"404\":{\"description\":\"No product with that id, or the catalog fetch failed.\"}}}},\"/api/games/steam/{appId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full Steam store detail for an app\",\"description\":\"Public lookup by Steam appId — fetches the store appdetails page live (no local cache), plus content-label (ccl) data and, if this app is a demo/beta/playtest, the resolved parent app. releaseDate is a parsed Instant, not Steam's raw locale-formatted string (see SteamReleaseDate).\",\"operationId\":\"steamDetail\",\"parameters\":[{\"name\":\"appId\",\"in\":\"path\",\"description\":\"Steam appId.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":400},{\"name\":\"lang\",\"in\":\"query\",\"description\":\"BCP 47 language tag Steam's appdetails API is queried with (see SteamLanguages) — controls the store description/date-format language, not just a display hint. Unlisted tags silently fall back to English.\",\"required\":false,\"schema\":{\"type\":\"string\",\"default\":\"en\",\"enum\":[\"en\",\"fr\",\"de\",\"it\",\"es\",\"pt\",\"pt-BR\",\"zh\",\"zh-TW\",\"zh-HK\",\"ru\",\"ja\",\"ko\",\"th\",\"tr\",\"uk\",\"nl\",\"da\",\"fi\",\"no\",\"sv\",\"pl\",\"hu\",\"cs\",\"ro\",\"bg\",\"el\",\"vi\",\"ar\",\"id\"]},\"example\":\"fr\"},{\"name\":\"Accept-Language\",\"in\":\"header\",\"description\":\"Fallback for \\\"lang\\\" when that query param is omitted — used by clients that can set headers (this route is also meant to be pasted as a plain URL into an OBS browser source, which sends no custom headers, hence \\\"lang\\\" taking priority). The highest-quality (\\\"q=\\\") tag is used; unparseable values are ignored.\",\"required\":false,\"schema\":{\"type\":\"string\"},\"example\":\"fr-FR,fr;q=0.9\"},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The Steam app's store-page detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/SteamDetailResponse\"}}}},\"404\":{\"description\":\"No Steam app with that id, or the store fetch failed.\"}}}},\"/api/games/igdb/{igdbId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full IGDB detail for a game\",\"description\":\"Public lookup by IGDB game id — served from IgdbGameDetailsService's stale-while-revalidate cache, so a cache miss triggers a live IGDB fetch (rating, genres, media, dlcs/similarGames with their own \\\"more\\\" links back into this same endpoint, etc.).\",\"operationId\":\"igdbDetail\",\"parameters\":[{\"name\":\"igdbId\",\"in\":\"path\",\"description\":\"IGDB game id.\",\"required\":true,\"schema\":{\"type\":\"string\"},\"example\":1234},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The IGDB game's detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/IgdbDetailResponse\"}}}},\"404\":{\"description\":\"No IGDB game with that id.\"}}}},\"/api/games/dtdd/{dtddId}\":{\"get\":{\"tags\":[\"Games\"],\"summary\":\"Get full \\\"Does the Dog Die\\\" content-warning detail for a title\",\"description\":\"Public lookup by DTDD numeric id — yes/no/mostly trigger-warning topic votes plus item metadata (overview, genres, poster/background images, per-topic vote counts). 404 if the dtdd.enabled feature flag is off, not just if the id doesn't resolve.\",\"operationId\":\"dtddDetail\",\"parameters\":[{\"name\":\"dtddId\",\"in\":\"path\",\"description\":\"DTDD (doesthedogdie.com) numeric item id.\",\"required\":true,\"schema\":{\"type\":\"integer\",\"format\":\"int64\"},\"example\":16758},{\"name\":\"X-API-Version\",\"in\":\"header\",\"schema\":{\"type\":\"string\",\"default\":\"2.0.0\",\"enum\":[\"2\"]}}],\"responses\":{\"200\":{\"description\":\"The title's DTDD content-warning detail.\",\"content\":{\"*/*\":{\"schema\":{\"$ref\":\"#/components/schemas/DtddDetailResponse\"}}}},\"404\":{\"description\":\"DTDD support disabled, or no title with that id.\"}}}}},\"components\":{\"schemas\":{\"ActivateRequest\":{\"type\":\"object\",\"properties\":{\"idOrName\":{\"type\":\"string\",\"description\":\"The custom binding's id or name, case ignored.\",\"example\":\"Art\"}},\"required\":[\"idOrName\"]},\"DropCampaignResponse\":{\"type\":\"object\",\"description\":\"A Twitch drops campaign running for your channel\",\"properties\":{\"id\":{\"type\":\"string\",\"description\":\"The campaign's Twitch id\"},\"name\":{\"type\":\"string\",\"description\":\"Its name\",\"example\":\"SEASON AURORA - WARFARE\"},\"gameId\":{\"type\":\"string\",\"description\":\"The Twitch category it runs on\"},\"gameName\":{\"type\":\"string\",\"description\":\"That category's name\"},\"startAt\":{\"type\":\"string\",\"format\":\"date-time\",\"description\":\"When it started\"},\"endAt\":{\"type\":\"string\",\"format\":\"date-time\",\"description\":\"When it ends\"},\"detailsUrl\":{\"type\":\"string\",\"description\":\"Its page, null when it has none\"},\"imageUrl\":{\"type\":\"string\",\"description\":\"Its image, null when it has none\"}}},\"CustomBindingDto\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"string\",\"format\":\"uuid\"},\"name\":{\"type\":\"string\"},\"twitchGameId\":{\"type\":\"string\"},\"twitchGameName\":{\"type\":\"string\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"tws\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"uninterruptible\":{\"type\":\"boolean\"},\"title\":{\"type\":\"string\"}}},\"CustomBindingsStateDto\":{\"type\":\"object\",\"properties\":{\"enabled\":{\"type\":\"boolean\"},\"bindings\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/CustomBindingDto\"}},\"activeId\":{\"type\":\"string\",\"format\":\"uuid\"},\"activatedAt\":{\"type\":\"string\",\"format\":\"date-time\"}}},\"CatapultDetailResponse\":{\"type\":\"object\",\"properties\":{\"createdAt\":{\"type\":\"string\",\"format\":\"date-time\"},\"updatedAt\":{\"type\":\"string\",\"format\":\"date-time\"},\"sourceType\":{\"type\":\"string\",\"enum\":[\"STEAM\",\"XBOX\",\"BATTLENET\",\"MANUAL\",\"MINECRAFT\"]},\"sourceId\":{\"type\":\"string\"},\"sourceName\":{\"type\":\"string\"},\"twitchGameId\":{\"type\":\"string\"},\"twitchGameName\":{\"type\":\"string\"},\"ignored\":{\"type\":\"boolean\"},\"cclEnabled\":{\"type\":\"boolean\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"twEnabled\":{\"type\":\"boolean\"},\"twOverride\":{\"type\":\"boolean\"},\"tws\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"twLabels\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true}}},\"XboxImage\":{\"type\":\"object\",\"properties\":{\"uri\":{\"type\":\"string\"},\"imagePurpose\":{\"type\":\"string\"},\"width\":{\"type\":\"integer\",\"format\":\"int32\"},\"height\":{\"type\":\"integer\",\"format\":\"int32\"},\"caption\":{\"type\":\"string\"}}},\"XboxProduct\":{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"},\"description\":{\"type\":\"string\"},\"publisherName\":{\"type\":\"string\"},\"developerName\":{\"type\":\"string\"},\"publisherAddress\":{\"type\":\"string\"},\"publisherWebsiteUri\":{\"type\":\"string\"},\"supportUri\":{\"type\":\"string\"},\"supportPhone\":{\"type\":\"string\"},\"lastModifiedDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"images\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/XboxImage\"}},\"videos\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/XboxVideo\"}},\"franchises\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"gamePassAffirmations\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"storeUrl\":{\"type\":\"string\"}}},\"XboxVideo\":{\"type\":\"object\",\"properties\":{\"caption\":{\"type\":\"string\"},\"dashUrl\":{\"type\":\"string\"},\"hlsUrl\":{\"type\":\"string\"},\"previewImageUri\":{\"type\":\"string\"},\"width\":{\"type\":\"integer\",\"format\":\"int32\"},\"height\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"Achievement\":{\"type\":\"object\",\"properties\":{\"total\":{\"type\":\"integer\",\"format\":\"int32\"},\"highlighted\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Highlighted\"}}}},\"Category\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"}}},\"ContentDescriptors\":{\"type\":\"object\",\"properties\":{\"ids\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"note\":{\"type\":\"string\"}}},\"Demo\":{\"type\":\"object\",\"properties\":{\"appid\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Dlc\":{\"type\":\"object\",\"properties\":{\"appId\":{\"type\":\"integer\",\"format\":\"int32\"},\"more\":{\"type\":\"string\"}}},\"FullGame\":{\"type\":\"object\",\"properties\":{\"appid\":{\"type\":\"integer\",\"format\":\"int32\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Genre\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"description\":{\"type\":\"string\"}}},\"Highlighted\":{\"type\":\"object\",\"properties\":{\"icon\":{\"type\":\"string\"},\"localizedName\":{\"type\":\"string\"},\"archived\":{\"type\":\"boolean\"},\"hidden\":{\"type\":\"boolean\"},\"name\":{\"type\":\"string\"},\"path\":{\"type\":\"string\"}}},\"IaDisclosure\":{\"type\":\"object\",\"properties\":{\"hasDisclosure\":{\"type\":\"boolean\"},\"note\":{\"type\":\"string\"}}},\"Metacritic\":{\"type\":\"object\",\"properties\":{\"score\":{\"type\":\"integer\",\"format\":\"int32\"},\"url\":{\"type\":\"string\"}}},\"Movie\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"name\":{\"type\":\"string\"},\"thumbnail\":{\"type\":\"string\"},\"dashAv1\":{\"type\":\"string\"},\"dashH264\":{\"type\":\"string\"},\"hlsH264\":{\"type\":\"string\"},\"highlight\":{\"type\":\"boolean\"}}},\"Packages\":{\"type\":\"object\",\"properties\":{\"name\":{\"type\":\"string\"},\"title\":{\"type\":\"string\"},\"description\":{\"type\":\"string\"},\"selectionText\":{\"type\":\"string\"},\"saveText\":{\"type\":\"string\"},\"displayType\":{\"type\":\"integer\",\"format\":\"int32\"},\"isRecurringSubscription\":{\"type\":\"string\"},\"subs\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Sub\"}}}},\"Page\":{\"type\":\"object\",\"properties\":{\"type\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"steamAppId\":{\"type\":\"integer\",\"format\":\"int32\"},\"requiredAge\":{\"type\":\"integer\",\"format\":\"int32\"},\"isFree\":{\"type\":\"boolean\"},\"controllerSupport\":{\"type\":\"string\"},\"dlc\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Dlc\"}},\"detailedDescription\":{\"type\":\"string\"},\"aboutTheGame\":{\"type\":\"string\"},\"shortDescription\":{\"type\":\"string\"},\"fullgame\":{\"$ref\":\"#/components/schemas/FullGame\"},\"supportedLanguages\":{\"type\":\"string\"},\"reviews\":{\"type\":\"string\"},\"headerImage\":{\"type\":\"string\"},\"capsuleImage\":{\"type\":\"string\"},\"capsuleImageV5\":{\"type\":\"string\"},\"website\":{\"type\":\"string\"},\"pcRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"macRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"linuxRequirements\":{\"$ref\":\"#/components/schemas/Requirement\"},\"legalNotice\":{\"type\":\"string\"},\"developers\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"publishers\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"demos\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Demo\"}},\"priceOverview\":{\"$ref\":\"#/components/schemas/PriceOverview\"},\"packages\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"packageGroups\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Packages\"}},\"platforms\":{\"$ref\":\"#/components/schemas/Platforms\"},\"metacritic\":{\"$ref\":\"#/components/schemas/Metacritic\"},\"categories\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Category\"}},\"genres\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Genre\"}},\"screenshots\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Screenshot\"}},\"movies\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Movie\"}},\"recommendations\":{\"$ref\":\"#/components/schemas/Recommendations\"},\"achievements\":{\"$ref\":\"#/components/schemas/Achievement\"},\"releaseDate\":{\"$ref\":\"#/components/schemas/ReleaseDate\"},\"supportInfo\":{\"$ref\":\"#/components/schemas/SupportInfo\"},\"background\":{\"type\":\"string\"},\"backgroundRaw\":{\"type\":\"string\"},\"contentDescriptors\":{\"$ref\":\"#/components/schemas/ContentDescriptors\"},\"ratings\":{\"$ref\":\"#/components/schemas/Ratings\"},\"demo\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/Demo\"}}}},\"ParentApp\":{\"type\":\"object\",\"properties\":{\"appId\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"Platforms\":{\"type\":\"object\",\"properties\":{\"windows\":{\"type\":\"boolean\"},\"mac\":{\"type\":\"boolean\"},\"linux\":{\"type\":\"boolean\"}}},\"PriceOverview\":{\"type\":\"object\",\"properties\":{\"currency\":{\"type\":\"string\"},\"initial\":{\"type\":\"integer\",\"format\":\"int32\"},\"final0\":{\"type\":\"integer\",\"format\":\"int32\"},\"discountPercent\":{\"type\":\"integer\",\"format\":\"int32\"},\"initialFormatted\":{\"type\":\"string\"},\"finalFormatted\":{\"type\":\"string\"}}},\"Rating\":{\"type\":\"object\",\"properties\":{\"ratingGenerated\":{\"type\":\"string\"},\"rating\":{\"type\":\"string\"},\"requiredAge\":{\"type\":\"string\"},\"banned\":{\"type\":\"string\"},\"useAgeGate\":{\"type\":\"string\"},\"descriptors\":{\"type\":\"string\"},\"interactiveElements\":{\"type\":\"string\"},\"displayOnlineNotice\":{\"type\":\"string\"},\"displayOnlineMusicNotice\":{\"type\":\"string\"}}},\"Ratings\":{\"type\":\"object\",\"properties\":{\"esrb\":{\"$ref\":\"#/components/schemas/Rating\"},\"pegi\":{\"$ref\":\"#/components/schemas/Rating\"},\"bbfc\":{\"$ref\":\"#/components/schemas/Rating\"},\"usk\":{\"$ref\":\"#/components/schemas/Rating\"},\"cero\":{\"$ref\":\"#/components/schemas/Rating\"},\"kggrb\":{\"$ref\":\"#/components/schemas/Rating\"},\"fpb\":{\"$ref\":\"#/components/schemas/Rating\"},\"csrr\":{\"$ref\":\"#/components/schemas/Rating\"},\"crl\":{\"$ref\":\"#/components/schemas/Rating\"},\"agcom\":{\"$ref\":\"#/components/schemas/Rating\"},\"oflc\":{\"$ref\":\"#/components/schemas/Rating\"},\"nzoflc\":{\"$ref\":\"#/components/schemas/Rating\"},\"cadpa\":{\"$ref\":\"#/components/schemas/Rating\"},\"dejus\":{\"$ref\":\"#/components/schemas/Rating\"},\"germanyRating\":{\"$ref\":\"#/components/schemas/Rating\"},\"igrs\":{\"$ref\":\"#/components/schemas/Rating\"},\"steamAustralia\":{\"$ref\":\"#/components/schemas/Rating\"},\"mda\":{\"$ref\":\"#/components/schemas/Rating\"},\"gmedia\":{\"$ref\":\"#/components/schemas/Rating\"}}},\"Recommendations\":{\"type\":\"object\",\"properties\":{\"total\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"ReleaseDate\":{\"type\":\"object\",\"properties\":{\"releaseDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"comingSoon\":{\"type\":\"boolean\"}}},\"Requirement\":{\"type\":\"object\",\"properties\":{\"minimum\":{\"type\":\"string\"},\"recommended\":{\"type\":\"string\"}}},\"Screenshot\":{\"type\":\"object\",\"properties\":{\"id\":{\"type\":\"integer\",\"format\":\"int32\"},\"pathThumbnail\":{\"type\":\"string\"},\"pathFull\":{\"type\":\"string\"}}},\"SteamDetailResponse\":{\"type\":\"object\",\"properties\":{\"storeUrl\":{\"type\":\"string\"},\"ccls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"uniqueItems\":true},\"ia\":{\"$ref\":\"#/components/schemas/IaDisclosure\"},\"parentApp\":{\"$ref\":\"#/components/schemas/ParentApp\"},\"page\":{\"$ref\":\"#/components/schemas/Page\"}}},\"Sub\":{\"type\":\"object\",\"properties\":{\"packageid\":{\"type\":\"integer\",\"format\":\"int32\"},\"percentSavingText\":{\"type\":\"string\"},\"percentSaving\":{\"type\":\"integer\",\"format\":\"int32\"},\"optionText\":{\"type\":\"string\"},\"optionDescription\":{\"type\":\"string\"},\"canGetFreeLicense\":{\"type\":\"string\"},\"isFreeLicense\":{\"type\":\"boolean\"},\"priceInCentsWithDiscount\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"SupportInfo\":{\"type\":\"object\",\"properties\":{\"url\":{\"type\":\"string\"},\"email\":{\"type\":\"string\"}}},\"IgdbDetailResponse\":{\"type\":\"object\",\"properties\":{\"slug\":{\"type\":\"string\"},\"url\":{\"type\":\"string\"},\"summary\":{\"type\":\"string\"},\"storyline\":{\"type\":\"string\"},\"platforms\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"rating\":{\"type\":\"number\",\"format\":\"double\"},\"aggregatedRating\":{\"type\":\"number\",\"format\":\"double\"},\"totalRating\":{\"type\":\"number\",\"format\":\"double\"},\"totalRatingCount\":{\"type\":\"integer\",\"format\":\"int32\"},\"releaseDate\":{\"type\":\"string\",\"format\":\"date-time\"},\"websites\":{\"type\":\"object\",\"additionalProperties\":{}},\"dlcs\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/IgdbRef\"}},\"similarGames\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/IgdbRef\"}},\"genres\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"gameModes\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"themes\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"playerPerspectives\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"keywords\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"franchiseNames\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"coverUrl\":{\"type\":\"string\"},\"screenshotUrls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"videoUrls\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}}}},\"IgdbRef\":{\"type\":\"object\",\"properties\":{\"igdbId\":{\"type\":\"string\"},\"name\":{\"type\":\"string\"},\"more\":{\"type\":\"string\"}}},\"DtddDetailResponse\":{\"type\":\"object\",\"properties\":{\"url\":{\"type\":\"string\"},\"yesTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"noTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"mostlyTopics\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"overview\":{\"type\":\"string\"},\"genres\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"releaseYear\":{\"type\":\"integer\",\"format\":\"int64\"},\"itemTypeName\":{\"type\":\"string\"},\"tmdbId\":{\"type\":\"integer\",\"format\":\"int64\"},\"imdbId\":{\"type\":\"string\"},\"posterImage\":{\"type\":\"string\"},\"backgroundImage\":{\"type\":\"string\"},\"topicItemStats\":{\"type\":\"array\",\"items\":{\"$ref\":\"#/components/schemas/DtddTopicItemStat\"}}}},\"DtddSeverity\":{\"type\":\"object\",\"properties\":{\"levels\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"counts\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"percents\":{\"type\":\"array\",\"items\":{\"type\":\"integer\",\"format\":\"int32\"}},\"total\":{\"type\":\"integer\",\"format\":\"int32\"},\"verified\":{\"type\":\"integer\",\"format\":\"int32\"}}},\"DtddTopicItemStat\":{\"type\":\"object\",\"properties\":{\"topicItemId\":{\"type\":\"integer\",\"format\":\"int64\"},\"yesSum\":{\"type\":\"integer\",\"format\":\"int64\"},\"noSum\":{\"type\":\"integer\",\"format\":\"int64\"},\"numComments\":{\"type\":\"integer\",\"format\":\"int64\"},\"topicId\":{\"type\":\"integer\",\"format\":\"int64\"},\"topicName\":{\"type\":\"string\"},\"itemId\":{\"type\":\"integer\",\"format\":\"int64\"},\"severity\":{\"$ref\":\"#/components/schemas/DtddSeverity\"}}}},\"securitySchemes\":{\"personalToken\":{\"type\":\"http\",\"description\":\"A personal API token from the account settings, in the Authorization header: the full token (cat_
) reads and changes everything, the read-only token (catr_
) only the GET routes.\",\"scheme\":\"bearer\"},\"readToken\":{\"type\":\"apiKey\",\"description\":\"The read-only personal token (catr_
) in the query string, for a URL that cannot carry a header. GET routes only, and only when the request has no Authorization header; the full token is refused here.\",\"name\":\"token\",\"in\":\"query\"}}}}" } diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCacheEntryDetailDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCacheEntryDetailDto.java new file mode 100644 index 000000000..23110ed7e --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCacheEntryDetailDto.java @@ -0,0 +1,17 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.util.List; + +/** + * Everything the DTDD caches keep about one entry. A title ({@code dtddId}): its cached record ({@code game}, null + * when not cached), its topics in full ({@code topics}, null when not cached) and the cached searches that found it. + * A search ({@code query}): itself ({@code searches}, its only element) and the titles it found ({@code titles}), + * each as cached ({@code name} null when not) with whether its topics are cached too. + */ +public record DtddCacheEntryDetailDto(Long dtddId, String query, DtddGameCacheEntryDto game, + DtddTopicsCacheEntryDto topics, List searches, + List titles) { + + /** A title a search found: its ID, its cached name (null when not cached), whether its topics are cached. */ + public record FoundTitle(long dtddId, String name, boolean topicsCached) {} +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCachesSummaryDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCachesSummaryDto.java new file mode 100644 index 000000000..bfea683c1 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddCachesSummaryDto.java @@ -0,0 +1,11 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.util.List; + +/** + * The DTDD caches at a glance: each one's size and how many are past its own lifetime (the {@code *TtlHours}), the + * pooled keys' quotas, the reserve of its month the bulk refetch leaves each key, and that refetch. + */ +public record DtddCachesSummaryDto(long topics, long topicsStale, long games, long gamesStale, long searches, + long searchesStale, int topicsTtlHours, int gamesTtlHours, int searchesTtlHours, + List quotas, int monthlyReserve, DtddRefetchJobDto job) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddGameCacheEntryDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddGameCacheEntryDto.java new file mode 100644 index 000000000..12ca9e652 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddGameCacheEntryDto.java @@ -0,0 +1,7 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; + +/** A cached DTDD title, as DTDD's search found it; {@code stale} past the cache's lifetime. */ +public record DtddGameCacheEntryDto(long dtddId, String name, String mediaType, String posterUrl, Instant fetchedAt, + boolean stale) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddKeyQuotaDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddKeyQuotaDto.java new file mode 100644 index 000000000..38e0a1db2 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddKeyQuotaDto.java @@ -0,0 +1,10 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; + +/** + * What DTDD said was left of a pooled key's quota with its last answer ({@code maskedKey} as the keys tab shows it): + * null counts when it never said; {@code at} when it said it; {@code blockedUntil} while the key rests (rate limit). + */ +public record DtddKeyQuotaDto(String keyId, String maskedKey, Integer remainingMinute, Integer remainingMonth, + Instant at, Instant blockedUntil) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddRefetchJobDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddRefetchJobDto.java new file mode 100644 index 000000000..eae8edcfb --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddRefetchJobDto.java @@ -0,0 +1,15 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; + +/** + * The DTDD caches' bulk refetch, one at a time: {@code state} IDLE (never run), RUNNING, DONE, STOPPED (it stopped + * itself, {@code stopReason} QUOTA: every key down to the monthly reserve, NO_KEY: no key at all, ERROR) or + * CANCELLED; {@code done} entries refetched out of {@code total}, {@code failed} of them DTDD gave nothing for; + * {@code waitingUntil} while it waits for a key to be usable again. + */ +public record DtddRefetchJobDto(String state, int total, int done, int failed, String stopReason, Instant startedAt, + Instant finishedAt, Instant waitingUntil) { + + public static final DtddRefetchJobDto IDLE = new DtddRefetchJobDto("IDLE", 0, 0, 0, null, null, null, null); +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddSearchCacheEntryDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddSearchCacheEntryDto.java new file mode 100644 index 000000000..323c21617 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddSearchCacheEntryDto.java @@ -0,0 +1,7 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; +import java.util.List; + +/** A cached DTDD search: its query (lower-cased, trimmed) and the titles DTDD found; {@code stale} past its lifetime. */ +public record DtddSearchCacheEntryDto(String query, List dtddIds, Instant searchedAt, boolean stale) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicSeverityDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicSeverityDto.java new file mode 100644 index 000000000..5a08bfec6 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicSeverityDto.java @@ -0,0 +1,23 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.util.List; + +/** + * A cached topic's severity on a title: {@code levels} its labels, mildest first; {@code counts} and {@code percents} + * the votes for each, by position; {@code total} the level votes; {@code verified} the reviewers' level (1-based), + * null when they gave none. + */ +public record DtddTopicSeverityDto(List levels, List counts, List percents, int total, + Integer verified) { + + /** The label at {@code position} (1-based) among the levels, null out of them. */ + public String level(Integer position) { + if (position == null || levels == null || position < 1 || position > levels.size()) return null; + return levels.get(position - 1); + } + + /** The reviewers' level as its label, null when they gave none. */ + public String verifiedLevel() { + return level(verified); + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsBody.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsBody.java index a8a3e5f65..9d2f2ce79 100644 --- a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsBody.java +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsBody.java @@ -1,5 +1,16 @@ package fr.enimaloc.catapult.common.dto.admin; +import java.util.Map; import java.util.Set; -public record DtddTopicsBody(Set topics) {} +/** + * A TW's DTDD topics, replacing them all; {@code minSeverities} the least severe level of each that turns the TW on, + * by topic (a topic absent, or null: any level). + */ +public record DtddTopicsBody(Set topics, Map minSeverities) { + + /** Topics without thresholds. */ + public DtddTopicsBody(Set topics) { + this(topics, null); + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsCacheEntryDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsCacheEntryDto.java new file mode 100644 index 000000000..024dc57d3 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/DtddTopicsCacheEntryDto.java @@ -0,0 +1,21 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +/** + * A DTDD title's cached topics: {@code name} the title's, when its game is cached too (else null); {@code stale} + * past the cache's lifetime (the next read refreshes it); {@code severities} each topic's by its name, null when + * fetched before they were kept (a topic without levels absent). + */ +public record DtddTopicsCacheEntryDto(long dtddId, String name, List yesTopics, List noTopics, + List mostlyTopics, Instant fetchedAt, boolean stale, + Map severities) { + + /** Topics without their severities. */ + public DtddTopicsCacheEntryDto(long dtddId, String name, List yesTopics, List noTopics, + List mostlyTopics, Instant fetchedAt, boolean stale) { + this(dtddId, name, yesTopics, noTopics, mostlyTopics, fetchedAt, stale, null); + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingDto.java index e3fbf5bc6..fae0ee5ef 100644 --- a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingDto.java +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingDto.java @@ -9,7 +9,23 @@ * every streamer. {@code ccls} / {@code tws} are null when the bindings keep IGDB's, else the admin's (even empty). * {@code sourceName} is the game's most frequent detected name, null when it was never detected; * {@code bindingCount} the number of bindings that follow it (MAPPED); {@code updatedBy} the admin's Twitch login. + * {@code ignoredByDefault}: the game's bindings are ignored, but those whose streamer chose; its category + * ({@code twitchGameId}, {@code twitchGameName}) is then null when it sets none. */ public record GameMappingDto(UUID id, String sourceType, String sourceId, String sourceName, String twitchGameId, String twitchGameName, Set ccls, Set tws, long bindingCount, - Instant updatedAt, String updatedBy) {} + Instant updatedAt, String updatedBy, Boolean ignoredByDefault) { + + /** Boxed so a catapult-api that predates it still reads (Jackson refuses a missing primitive): absent, false. */ + public GameMappingDto { + ignoredByDefault = Boolean.TRUE.equals(ignoredByDefault); + } + + /** A mapping that doesn't ignore its game. */ + public GameMappingDto(UUID id, String sourceType, String sourceId, String sourceName, String twitchGameId, + String twitchGameName, Set ccls, Set tws, long bindingCount, + Instant updatedAt, String updatedBy) { + this(id, sourceType, sourceId, sourceName, twitchGameId, twitchGameName, ccls, tws, bindingCount, updatedAt, + updatedBy, false); + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingRequest.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingRequest.java index 2ee5092a1..e09faec32 100644 --- a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingRequest.java +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/GameMappingRequest.java @@ -5,7 +5,20 @@ /** * Creates or changes an admin game mapping. {@code ccls} / {@code tws}: null keeps IGDB's, a set (even empty) * replaces them. {@code applyLive}: also update now the channels live on this game, else their bindings only serve - * the next detection. + * the next detection. {@code ignoredByDefault}: the game's bindings are ignored, but those whose streamer chose; the + * category is then optional (none: the bindings keep IGDB's). */ public record GameMappingRequest(String sourceType, String sourceId, String twitchGameId, String twitchGameName, - Set ccls, Set tws, boolean applyLive) {} + Set ccls, Set tws, boolean applyLive, Boolean ignoredByDefault) { + + /** Boxed so a body without it still reads (Jackson refuses a missing primitive): absent, the game isn't ignored. */ + public GameMappingRequest { + ignoredByDefault = Boolean.TRUE.equals(ignoredByDefault); + } + + /** A request that doesn't ignore the game. */ + public GameMappingRequest(String sourceType, String sourceId, String twitchGameId, String twitchGameName, + Set ccls, Set tws, boolean applyLive) { + this(sourceType, sourceId, twitchGameId, twitchGameName, ccls, tws, applyLive, false); + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitCorrectionDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitCorrectionDto.java new file mode 100644 index 000000000..4e15caf38 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitCorrectionDto.java @@ -0,0 +1,9 @@ +package fr.enimaloc.catapult.common.dto.admin; + +/** + * An admin's correction of a learned rate limit: {@code limitCalls} per {@code limitWindowSeconds} (both or neither), + * {@code resumeSeconds}, and for a route its {@code scope} (HOST or ROUTE). A null value leaves that part to the + * learning; what is set stays until the scope is reset. + */ +public record LearnedRateLimitCorrectionDto(Integer limitCalls, Integer limitWindowSeconds, Integer resumeSeconds, + String scope) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitDto.java new file mode 100644 index 000000000..26f3273b7 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/LearnedRateLimitDto.java @@ -0,0 +1,23 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; + +/** + * What Catapult learned of one scope's rate limit from its 429s: a host ({@code method} and {@code path} null: the + * budget its {@code HOST} routes share) or one of its routes ({@code scope} HOST: in its host's budget, ROUTE: its + * own limit; {@code scopePinned} when an admin chose it). {@code limitCalls} per {@code limitWindowSeconds} and + * {@code resumeMs} are what applies (an admin's correction when {@code limitCorrected}/{@code resumeCorrected}, else + * what was learned; no limit while none was learned). {@code usedCalls}: the calls in that window (a minute without a + * limit) on the instance that answered; {@code blockedUntil} while it waits after a 429. + */ +public record LearnedRateLimitDto(String id, String host, String method, String path, String scope, + boolean scopePinned, Integer limitCalls, Integer limitWindowSeconds, + boolean limitCorrected, long resumeMs, boolean resumeCorrected, + int rateLimitedCount, Instant lastRateLimitedAt, int usedCalls, + Instant blockedUntil) { + + /** A host's row rather than a route's. */ + public boolean hostScope() { + return path == null; + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwDtddTopicDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwDtddTopicDto.java new file mode 100644 index 000000000..e43699e61 --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwDtddTopicDto.java @@ -0,0 +1,9 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.util.List; + +/** + * A DTDD topic that turns a TW on: {@code minSeverity} its least severe level that does (null: any), and + * {@code knownLevels} the topic's levels as the cached titles give them, mildest first (empty when none has them). + */ +public record TwDtddTopicDto(String topic, String minSeverity, List knownLevels) {} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwRebuildStatusDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwRebuildStatusDto.java new file mode 100644 index 000000000..465d8f20a --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/admin/TwRebuildStatusDto.java @@ -0,0 +1,20 @@ +package fr.enimaloc.catapult.common.dto.admin; + +import java.time.Instant; + +/** + * The TWs' recalculation, one at a time: {@code state} IDLE (none since the start), RUNNING, DONE or CANCELLED; + * {@code mode} BACKFILL (at start, the bindings without TWs), NON_OVERRIDDEN, UNTOUCHED or FORCE_PINNED; + * {@code processed}, {@code skipped} and {@code failed} out of {@code total} bindings; {@code waitingUntil} while it + * waits for the Steam store's rate limit to end. + */ +public record TwRebuildStatusDto(String state, String mode, long total, int processed, int skipped, int failed, + Instant startedAt, Instant finishedAt, Instant waitingUntil) { + + public static final TwRebuildStatusDto IDLE = new TwRebuildStatusDto("IDLE", null, 0, 0, 0, 0, null, null, null); + + /** The bindings gone through so far. */ + public long done() { + return (long) processed + skipped + failed; + } +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/TwDetailsDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/TwDetailsDto.java new file mode 100644 index 000000000..9c7a78bae --- /dev/null +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/TwDetailsDto.java @@ -0,0 +1,30 @@ +package fr.enimaloc.catapult.common.dto.channel; + +import java.util.List; + +/** + * Why a binding has its TWs, as the detection tells it now. {@code tws}: each TW the binding has, or the detection + * suggests ({@code active} false: the streamer turned it off), with where it comes from ({@code origin} DETECTED: + * its {@code sources}; STREAMER: the streamer's own choice; ADMIN: an admin's game mapping; NO_SOURCE: nothing + * suggests it any more, the next detection drops it). {@code keptOff}: the DTDD topics a TW's least severe level kept + * it off for. {@code streamerChose}: the streamer edited the binding's TWs. + */ +public record TwDetailsDto(boolean streamerChose, List tws, List keptOff) { + + public record Tw(String id, String label, boolean active, String origin, List sources) {} + + /** + * One signal for the TW {@code twId} ({@code twLabel} its label): {@code type} IGDB_DESCRIPTOR, STEAM_CONTENT, + * STEAM_KEYWORD or DTDD_TOPIC, {@code value} the descriptor's ID, the keyword or the topic; {@code severity} a + * DTDD topic's verdict, null for the rest. + */ + public record Source(String twId, String twLabel, String type, String value, Severity severity) {} + + /** + * A DTDD topic's severity against its TW's least severe level: {@code reason} NO_THRESHOLD, NOT_CACHED, NO_LEVELS, + * UNKNOWN_LEVEL, TOO_FEW_VOTES, VERIFIED, VOTES (the TW on), BELOW_VERIFIED, BELOW_VOTES (kept off); + * {@code level} the reviewers' level, else the members' most voted; {@code percentAtOrAbove} the votes' share at + * or above {@code minSeverity}. + */ + public record Severity(String reason, String minSeverity, String level, int votes, int percentAtOrAbove) {} +} diff --git a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/UserSettingsDto.java b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/UserSettingsDto.java index c7266cf73..939751e8d 100644 --- a/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/UserSettingsDto.java +++ b/catapult-common/src/main/java/fr/enimaloc/catapult/common/dto/channel/UserSettingsDto.java @@ -37,8 +37,32 @@ public record UserSettingsDto( String noGameTitle, String incompleteFallbackTitle, // Whether the title fields get their preview (experiment stream.title.preview); null: not offered - Boolean titlePreviewAvailable + Boolean titlePreviewAvailable, + // Whether the titles read their conditions (experiment stream.title.conditions); null: not offered + Boolean titleConditionsAvailable, + // Whether the titles get the drops ({{DROP_NAME}}, {{DROP_END}}, DROP_ENABLED): the conditions and the + // experiment drops; null: not offered + Boolean titleDropsAvailable ) { + /** Settings without the title's conditions nor drops. */ + public UserSettingsDto(boolean cclFeatureEnabled, Set blockedCcls, String noGameTwitchGameId, + String noGameTwitchGameName, Set noGameCcls, boolean applyDefaultOnStreamStart, + boolean applyDefaultOnNoGame, boolean applyDefaultOnStreamEnd, + String incompleteFallbackTwitchGameId, String incompleteFallbackTwitchGameName, + Set incompleteFallbackCcls, List availableCcls, boolean twFeatureEnabled, + Set blockedTws, List availableTws, Integer raidRestoreGameMinutes, + boolean raidRestoreGameAvailable, List noGameExtensions, + List incompleteFallbackExtensions, Boolean titleFeatureEnabled, + String titleTemplate, Boolean titleAvailable, String noGameTitle, + String incompleteFallbackTitle, Boolean titlePreviewAvailable) { + this(cclFeatureEnabled, blockedCcls, noGameTwitchGameId, noGameTwitchGameName, noGameCcls, + applyDefaultOnStreamStart, applyDefaultOnNoGame, applyDefaultOnStreamEnd, + incompleteFallbackTwitchGameId, incompleteFallbackTwitchGameName, incompleteFallbackCcls, + availableCcls, twFeatureEnabled, blockedTws, availableTws, raidRestoreGameMinutes, + raidRestoreGameAvailable, noGameExtensions, incompleteFallbackExtensions, titleFeatureEnabled, + titleTemplate, titleAvailable, noGameTitle, incompleteFallbackTitle, titlePreviewAvailable, null, null); + } + /** Settings without the title's preview. */ public UserSettingsDto(boolean cclFeatureEnabled, Set blockedCcls, String noGameTwitchGameId, String noGameTwitchGameName, Set noGameCcls, boolean applyDefaultOnStreamStart, @@ -55,7 +79,7 @@ public UserSettingsDto(boolean cclFeatureEnabled, Set blockedCcls, Strin incompleteFallbackTwitchGameId, incompleteFallbackTwitchGameName, incompleteFallbackCcls, availableCcls, twFeatureEnabled, blockedTws, availableTws, raidRestoreGameMinutes, raidRestoreGameAvailable, noGameExtensions, incompleteFallbackExtensions, titleFeatureEnabled, - titleTemplate, titleAvailable, noGameTitle, incompleteFallbackTitle, null); + titleTemplate, titleAvailable, noGameTitle, incompleteFallbackTitle, null, null, null); } /** Settings without the fallbacks' own titles. */ @@ -73,7 +97,7 @@ public UserSettingsDto(boolean cclFeatureEnabled, Set blockedCcls, Strin incompleteFallbackTwitchGameId, incompleteFallbackTwitchGameName, incompleteFallbackCcls, availableCcls, twFeatureEnabled, blockedTws, availableTws, raidRestoreGameMinutes, raidRestoreGameAvailable, noGameExtensions, incompleteFallbackExtensions, titleFeatureEnabled, - titleTemplate, titleAvailable, null, null, null); + titleTemplate, titleAvailable, null, null, null, null, null); } /** Settings without the stream title (off, and not offered). */ diff --git a/catapult-common/src/test/java/fr/enimaloc/catapult/common/dto/AdminDtosTest.java b/catapult-common/src/test/java/fr/enimaloc/catapult/common/dto/AdminDtosTest.java new file mode 100644 index 000000000..fc40e044c --- /dev/null +++ b/catapult-common/src/test/java/fr/enimaloc/catapult/common/dto/AdminDtosTest.java @@ -0,0 +1,75 @@ +package fr.enimaloc.catapult.common.dto; + +import fr.enimaloc.catapult.common.dto.admin.DtddTopicSeverityDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsBody; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.GameMappingDto; +import fr.enimaloc.catapult.common.dto.admin.GameMappingRequest; +import fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto; +import org.junit.jupiter.api.Test; +import tools.jackson.databind.json.JsonMapper; + +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +/** The admin DTOs' shorter constructors and their read of payloads from before a field. */ +class AdminDtosTest { + + private final JsonMapper json = JsonMapper.builder().build(); + + @Test + void aSeverity_namesItsLevels_andTheReviewersOne() { + DtddTopicSeverityDto severity = new DtddTopicSeverityDto(List.of("Implied", "Depicted", "Excessive"), + List.of(2, 9, 1), List.of(17, 75, 8), 12, 2); + + assertThat(severity.verifiedLevel()).isEqualTo("Depicted"); + assertThat(severity.level(1)).isEqualTo("Implied"); + assertThat(severity.level(0)).isNull(); + assertThat(severity.level(4)).isNull(); + assertThat(severity.level(null)).isNull(); + assertThat(new DtddTopicSeverityDto(null, List.of(), List.of(), 0, null).verifiedLevel()).isNull(); + } + + @Test + void cachedTopics_withoutTheirSeverities_haveNone() { + DtddTopicsCacheEntryDto entry = new DtddTopicsCacheEntryDto(7, "Halo", List.of("gore"), List.of(), List.of(), + Instant.EPOCH, false); + + assertThat(entry.severities()).isNull(); + } + + @Test + void dtddTopics_withoutThresholds_haveNone() { + assertThat(new DtddTopicsBody(Set.of("gore")).minSeverities()).isNull(); + assertThat(json.readValue("{\"topics\":[\"gore\"]}", DtddTopicsBody.class).minSeverities()).isNull(); + } + + @Test + void aGameMapping_withoutItsIgnore_doesNotIgnoreTheGame() { + GameMappingRequest request = new GameMappingRequest("STEAM", "730", "1", "CS", null, null, false); + assertThat(request.ignoredByDefault()).isFalse(); + assertThat(new GameMappingRequest("STEAM", "730", null, null, null, null, false, null).ignoredByDefault()) + .isFalse(); + assertThat(json.readValue("{\"sourceType\":\"STEAM\",\"sourceId\":\"730\",\"applyLive\":true}", + GameMappingRequest.class).ignoredByDefault()).isFalse(); + + GameMappingDto mapping = new GameMappingDto(null, "STEAM", "730", "CS2", "1", "CS", null, null, 3, + Instant.EPOCH, "admin"); + assertThat(mapping.ignoredByDefault()).isFalse(); + assertThat(json.readValue("{\"sourceType\":\"STEAM\",\"sourceId\":\"730\",\"bindingCount\":0}", + GameMappingDto.class).ignoredByDefault()).isFalse(); + } + + @Test + void aRecalculation_countsWhatItWentThrough() { + TwRebuildStatusDto status = new TwRebuildStatusDto("RUNNING", "NON_OVERRIDDEN", 40, 10, 3, 1, Instant.EPOCH, + null, null); + + assertThat(status.done()).isEqualTo(14L); + assertThat(TwRebuildStatusDto.IDLE.done()).isZero(); + assertThat(TwRebuildStatusDto.IDLE.state()).isEqualTo("IDLE"); + } +} diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminDtddCachesController.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminDtddCachesController.java new file mode 100644 index 000000000..ca3682792 --- /dev/null +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminDtddCachesController.java @@ -0,0 +1,47 @@ +package fr.enimaloc.catapult.controller; + +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.service.ApiService; +import fr.enimaloc.catapult.service.http.ApiClient; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** + * The admin DTDD caches tab's actions, forwarded to catapult-api with the session's JWT: an entry fetched again + * ({@code key}: the title's ID or the search's query), and the bulk refetch started or cancelled. They answer + * catapult-api's status (and the refetch, as JSON); 502 when it is unreachable. + */ +@RestController +@RequestMapping("/admin/dtdd-caches") +@RequiredArgsConstructor +public class AdminDtddCachesController { + private final ApiService apiService; + private final ModelFiller filler; + + @PostMapping("/{kind}/refetch") + public ResponseEntity refetch(@PathVariable String kind, @RequestParam String key) { + AdminAccess.require(filler); + return ResponseEntity.status(apiService.adminRefetchDtddCache(kind, key)).build(); + } + + @PostMapping("/refetch-all/start") + public ResponseEntity start() { + AdminAccess.require(filler); + return answer(apiService.adminStartDtddRefetch()); + } + + @PostMapping("/refetch-all/cancel") + public ResponseEntity cancel() { + AdminAccess.require(filler); + return answer(apiService.adminCancelDtddRefetch()); + } + + private static ResponseEntity answer(ApiClient.Result result) { + return ResponseEntity.status(result.status()).body(result.body()); + } +} diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminRateLimitsController.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminRateLimitsController.java new file mode 100644 index 000000000..ce7a4fa63 --- /dev/null +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminRateLimitsController.java @@ -0,0 +1,35 @@ +package fr.enimaloc.catapult.controller; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.service.ApiService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** + * The admin rate limits tab's actions, forwarded to catapult-api with the session's JWT: a learned limit corrected + * ({@code id}: the scope's), or reset. They answer catapult-api's status; 502 when it is unreachable. + */ +@RestController +@RequestMapping("/admin/rate-limits") +@RequiredArgsConstructor +public class AdminRateLimitsController { + private final ApiService apiService; + private final ModelFiller filler; + + @PostMapping("/correct") + public ResponseEntity correct(@RequestParam String id, @RequestBody LearnedRateLimitCorrectionDto body) { + AdminAccess.require(filler); + return ResponseEntity.status(apiService.adminCorrectRateLimit(id, body)).build(); + } + + @PostMapping("/reset") + public ResponseEntity reset(@RequestParam String id) { + AdminAccess.require(filler); + return ResponseEntity.status(apiService.adminResetRateLimit(id)).build(); + } +} diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTabsFragmentController.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTabsFragmentController.java index 260249d92..6dd38f9fc 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTabsFragmentController.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTabsFragmentController.java @@ -35,6 +35,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.server.ResponseStatusException; import java.time.Instant; import java.time.format.DateTimeParseException; @@ -371,6 +372,55 @@ String gameMappings(Model model, HttpServletResponse response, @RequestParam(def }); } + /** The DTDD caches' summary: their sizes and stale entries, the keys' quotas and the bulk refetch. */ + @GetMapping("/dtdd-caches") + String dtddCaches(Model model, HttpServletResponse response) { + return guarded(model, response, () -> { + model.addAttribute("summary", apiService.adminDtddCaches()); + return "pages/admin-tab/dtdd-caches :: list"; + }); + } + + /** One DTDD cache's entries ({@code kind}: topics, games, searches), searched and paged, the oldest first. */ + @GetMapping("/dtdd-caches/{kind}") + String dtddCacheEntries(Model model, HttpServletResponse response, @PathVariable String kind, + @RequestParam(required = false) String q, @RequestParam(defaultValue = "0") int page) { + return guarded(model, response, () -> { + int shown = Math.max(0, page); + Object entries = switch (kind) { + case "topics" -> apiService.adminDtddTopicsCache(q, shown); + case "games" -> apiService.adminDtddGamesCache(q, shown); + case "searches" -> apiService.adminDtddSearchesCache(q, shown); + default -> throw new ResponseStatusException(HttpStatus.NOT_FOUND); + }; + model.addAttribute("kind", kind); + model.addAttribute("q", q == null ? "" : q); + model.addAttribute("entries", entries); + return "pages/admin-tab/dtdd-caches :: entries"; + }); + } + + /** Everything the DTDD caches keep about one entry, the body of the entries' dialog. */ + @GetMapping("/dtdd-caches/{kind}/entry") + String dtddCacheEntry(Model model, HttpServletResponse response, @PathVariable String kind, + @RequestParam String key) { + return guarded(model, response, () -> { + model.addAttribute("kind", kind); + model.addAttribute("key", key); + model.addAttribute("entry", apiService.adminDtddCacheEntry(kind, key)); + return "pages/admin-tab/dtdd-caches :: entry"; + }); + } + + /** The rate limits learned from the platforms' 429s: each host, then its routes. */ + @GetMapping("/rate-limits") + String rateLimits(Model model, HttpServletResponse response) { + return guarded(model, response, () -> { + model.addAttribute("limits", apiService.adminRateLimits()); + return "pages/admin-tab/rate-limits :: list"; + }); + } + @GetMapping("/groups") String groups(Model model, HttpServletResponse response) { return guarded(model, response, () -> { @@ -392,6 +442,7 @@ String whitelist(Model model, HttpServletResponse response) { String tw(Model model, HttpServletResponse response) { return guarded(model, response, () -> { model.addAttribute("definitions", apiService.adminTws()); + model.addAttribute("rebuild", apiService.adminTwRebuildStatus()); return "pages/admin-tab/tw :: content"; }); } @@ -411,6 +462,21 @@ String twKeywords(Model model, HttpServletResponse response, @PathVariable Strin }); } + /** One definition's DTDD topics with their least severe level; the missing fragment (404) when it is unknown. */ + @GetMapping("/tw/{id}/dtdd") + String twDtddTopics(Model model, HttpServletResponse response, @PathVariable String id) { + return guarded(model, response, () -> { + TwDefinitionDto definition = apiService.adminTw(id); + if (definition == null) { + response.setStatus(HttpStatus.NOT_FOUND.value()); + return "pages/admin-tab/tw :: missing"; + } + model.addAttribute("definition", definition); + model.addAttribute("topics", apiService.adminTwDtddTopics(id)); + return "pages/admin-tab/tw :: dtdd"; + }); + } + /** Steam and DTDD keys cards, then the Minecraft service accounts card. */ @GetMapping("/keys") String keys(Model model, HttpServletResponse response) { diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTwController.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTwController.java index 7e8fa4bbd..6a583d45d 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTwController.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/AdminTwController.java @@ -13,7 +13,12 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; import static org.springframework.http.HttpStatus.NOT_FOUND; @@ -40,6 +45,8 @@ public class AdminTwController { private static final int KEYWORD_MIN = 2; private static final int KEYWORD_MAX = 50; private static final Pattern APP_ID = Pattern.compile("[0-9]{1,12}"); + private static final int DTDD_TOPIC_MAX = 200; + private static final int DTDD_LEVEL_MAX = 40; private final ApiService apiService; private final ModelFiller filler; @@ -93,6 +100,38 @@ public ResponseEntity testKeyword(@PathVariable String id, @Re return result == null ? ResponseEntity.status(502).build() : ResponseEntity.ok(result); } + /** Asks the running recalculation to stop after its current binding. */ + @PostMapping("/rebuild/cancel") + public ResponseEntity cancelRebuild() { + AdminAccess.require(filler); + return ResponseEntity.status(apiService.adminCancelTwRebuild()).build(); + } + + /** A DTDD topic and the least severe level of it that turns the TW on (null or blank: any). */ + public record DtddTopicBody(String topic, String minSeverity) {} + + public record DtddTopicsBody(List topics) {} + + /** + * Replaces the definition's DTDD topics with {@code topics}, in order, each one once (by name, ignoring case); + * a topic of 1 to 200 characters and a level of 40 at most, else 400. + */ + @PostMapping("/{id}/dtdd") + public ResponseEntity setDtddTopics(@PathVariable String id, @RequestBody DtddTopicsBody body) { + AdminAccess.require(filler); + Map minByTopic = new LinkedHashMap<>(); + Set seen = new HashSet<>(); + for (DtddTopicBody t : body.topics() == null ? List.of() : body.topics()) { + String topic = t == null || t.topic() == null ? "" : t.topic().trim(); + String min = t == null || t.minSeverity() == null ? "" : t.minSeverity().trim(); + if (topic.isEmpty() || topic.length() > DTDD_TOPIC_MAX || min.length() > DTDD_LEVEL_MAX) { + return ResponseEntity.badRequest().build(); + } + if (seen.add(topic.toLowerCase(Locale.ROOT))) minByTopic.put(topic, min.isEmpty() ? null : min); + } + return ResponseEntity.status(apiService.adminSetTwDtddTopics(id, minByTopic)).build(); + } + @PostMapping("/rebuild/{mode}") public ResponseEntity rebuild(@PathVariable String mode) { AdminAccess.require(filler); diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SpaFragmentController.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SpaFragmentController.java index 3c5f5fda3..03b7263f9 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SpaFragmentController.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SpaFragmentController.java @@ -105,6 +105,13 @@ ResponseEntity> channelState(@PathVariable String usern apiService.dtddMappingStatus(username))); } + /** Why the caller's binding has its TWs, the body of its TW panel's details dialog. */ + @GetMapping("/channel/bindings/{bindingId}/tw-details") + String twDetails(Model model, @PathVariable String bindingId) { + model.addAttribute("details", apiService.twDetails(bindingId)); + return "pages/channel-tab/tw-details :: details"; + } + @GetMapping("/channel/{username}/dock") String dock(Model model, @PathVariable String username, HttpServletResponse response) { int refused = filler.channelDock(model, username); diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SseStreams.java b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SseStreams.java index 65dae5893..deba4f8ec 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SseStreams.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/controller/SseStreams.java @@ -194,7 +194,7 @@ void offer(SseEmitter.SseEventBuilder event) { if (size.incrementAndGet() > MAX_PENDING) { log.debug("Closing an SSE stream that does not keep up"); closed(); - emitter.complete(); + complete(); return; } pending.add(event); @@ -226,15 +226,20 @@ private void write() { size.decrementAndGet(); if (event == CLOSE) { closed(); - emitter.complete(); + complete(); return; } try { emitter.send(event); - } catch (IOException | IllegalStateException e) { - // Gone, or completed meanwhile. + } catch (IOException e) { + // Gone: the container reports the error itself (onError) and ends the request; completing + // it from here as well is refused (Spring: not to be used after an error while sending). + closed(); + return; + } catch (IllegalStateException e) { + // Completed meanwhile. closed(); - emitter.complete(); + complete(); return; } } @@ -245,6 +250,18 @@ private void write() { if (!closed.get() && !pending.isEmpty()) schedule(); } + /** + * Ends the request, unless the container ended it first: a client gone between our last write and this + * call has the container fail it (onError), after which Tomcat refuses an application thread's dispatch. + */ + private void complete() { + try { + emitter.complete(); + } catch (IllegalStateException e) { + log.debug("SSE stream already ended by the container: {}", e.getMessage()); + } + } + /** Forgotten, once; {@code onClosed} told. */ void closed() { if (closed.compareAndSet(false, true)) { diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/ApiService.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/ApiService.java index e130a2bb2..867aaa2e5 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/ApiService.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/ApiService.java @@ -8,6 +8,8 @@ import fr.enimaloc.catapult.service.http.ApiClient; import fr.enimaloc.catapult.service.http.ApiError; import fr.enimaloc.catapult.common.dto.*; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; import fr.enimaloc.catapult.common.dto.admin.AdminMemberDto; import fr.enimaloc.catapult.common.dto.admin.AdminMembersPage; import fr.enimaloc.catapult.common.dto.admin.AdminCclPage; @@ -35,6 +37,12 @@ import fr.enimaloc.catapult.common.dto.admin.MemberTargetingDto; import fr.enimaloc.catapult.common.dto.admin.AdminPage; import fr.enimaloc.catapult.common.dto.admin.DetectedGameDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddGameCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.common.dto.admin.DtddSearchCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; import fr.enimaloc.catapult.common.dto.admin.GameMappingDto; import fr.enimaloc.catapult.common.dto.admin.GameMappingRequest; import fr.enimaloc.catapult.common.dto.admin.GameMappingResult; @@ -83,6 +91,8 @@ public interface ApiService { int updateBinding(String username, String bindingId, String twitchGameId, String twitchGameName, java.util.Set ccls); int saveTws(String bindingId, java.util.Set tws); int resetTws(String bindingId); + /** Why the caller's binding has its TWs, as the detection tells it now; null when it isn't theirs or catapult-api fails. */ + fr.enimaloc.catapult.common.dto.channel.TwDetailsDto twDetails(String bindingId); int resetBinding(String bindingId); int toggleTwEnabled(String bindingId, boolean enabled); int saveSteamToken(String username, String token, boolean shared); @@ -345,6 +355,40 @@ default int adminCreateNotification(AdminNotificationCreateBody body) { /** catapult-api's status (200; 404 unknown) and result (no mapping), 502 when unreachable. */ ApiClient.Result adminDeleteGameMapping(UUID id, boolean applyLive); + /** The DTDD caches' sizes and stale entries, the keys' quotas and the bulk refetch; null when catapult-api fails. */ + DtddCachesSummaryDto adminDtddCaches(); + /** One page of the cached DTDD topics matching {@code q}, the oldest first; null when catapult-api fails. */ + AdminPage adminDtddTopicsCache(String q, int page); + /** One page of the cached DTDD titles matching {@code q}, the oldest first; null when catapult-api fails. */ + AdminPage adminDtddGamesCache(String q, int page); + /** One page of the cached DTDD searches matching {@code q}, the oldest first; null when catapult-api fails. */ + AdminPage adminDtddSearchesCache(String q, int page); + /** + * Fetches a cached DTDD entry again ({@code kind}: topics, games, searches; {@code key}: the title's ID or the + * search's query): catapult-api's status (200; 400 not an ID; 404 not cached; 502 DTDD gave nothing; 503 DTDD + * off), 502 when unreachable. + */ + int adminRefetchDtddCache(String kind, String key); + /** + * Everything the DTDD caches keep about an entry ({@code kind}: topics, games, searches; {@code key}: the title's + * ID or the search's query); null when it isn't cached or catapult-api fails. + */ + DtddCacheEntryDetailDto adminDtddCacheEntry(String kind, String key); + /** catapult-api's status (202; 409 one runs already; 503 DTDD off) and the refetch, 502 when unreachable. */ + ApiClient.Result adminStartDtddRefetch(); + /** catapult-api's status (200) and the refetch asked to stop, 502 when unreachable. */ + ApiClient.Result adminCancelDtddRefetch(); + + /** The rate limits catapult-api learned from the platforms' 429s, hosts then their routes; null when it fails. */ + List adminRateLimits(); + /** + * Corrects a learned rate limit ({@code id}: the scope's): catapult-api's status (200; 400 an invalid correction; + * 404 unknown), 502 when unreachable. + */ + int adminCorrectRateLimit(String id, LearnedRateLimitCorrectionDto correction); + /** Forgets what was learned and corrected of a rate limit: catapult-api's status (200; 404 unknown), 502 when unreachable. */ + int adminResetRateLimit(String id); + /** The member groups, with their member counts; null when catapult-api fails. */ List adminGroups(); /** catapult-api's status (201; 400 blank key; 409 key already used), 502 when unreachable. */ @@ -480,8 +524,19 @@ record MinecraftDeviceCodeStart(int status, MinecraftDeviceCodeDto code, ApiErro int adminCreateTw(String id, String label, int sortOrder); /** Starts a background rebuild of the mappings; catapult-api's status (202; 503 when trigger warnings are off), 502 when unreachable. */ int adminRebuildTws(TwRebuild mode); + /** The TWs' recalculation running or last run; null when catapult-api fails. */ + fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto adminTwRebuildStatus(); + /** Asks the running recalculation to stop: catapult-api's status (200; 503 TWs disabled), 502 when unreachable. */ + int adminCancelTwRebuild(); /** A definition's Steam keywords, sorted; null when unknown or when catapult-api fails. */ List adminTwKeywords(String id); + /** A TW's DTDD topics with their least severe level and the levels known for each; null when catapult-api fails. */ + List adminTwDtddTopics(String id); + /** + * Replaces a TW's DTDD topics, each with its least severe level ({@code minByTopic}: null or blank, any): + * catapult-api's status (204), 502 when unreachable. + */ + int adminSetTwDtddTopics(String id, Map minByTopic); /** Adds a keyword (already there changes nothing); catapult-api's status (204), 502 when unreachable. */ int adminAddTwKeyword(String id, String keyword); /** Removes a keyword (also an absent one); catapult-api's status (204), 502 when unreachable. */ diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockApiService.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockApiService.java index cf8b98a9f..db51486fe 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockApiService.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockApiService.java @@ -176,6 +176,8 @@ public class MockApiService implements ApiService { private final MockDtddProposalsStore dtddProposals = new MockDtddProposalsStore(); /** Application-wide: the game mappings an admin edits are the same for every mock session. */ private final MockGameMappingsStore gameMappings = new MockGameMappingsStore(); + private final MockDtddCachesStore dtddCaches = new MockDtddCachesStore(); + private final MockRateLimitsStore rateLimits = new MockRateLimitsStore(Clock.systemUTC()); /** Application-wide: the groups an admin creates are the same for every mock session. */ private final MockGroupsStore groups = new MockGroupsStore(); /** Application-wide: the experiments an admin edits are the same for every mock session. */ @@ -389,6 +391,36 @@ public int saveTws(String bindingId, Set tws) { return NO_CONTENT; } + /** + * The mock's TWs explained: each one the binding has detected by a sample source (the first by Steam and a DTDD + * topic over its threshold), and jump scares kept off by a DTDD topic under its own. + */ + @Override + public fr.enimaloc.catapult.common.dto.channel.TwDetailsDto twDetails(String bindingId) { + log.trace("twDetails({})", bindingId); + BindingDto binding = currentData().bindingById(bindingId); + if (binding == null) return null; + Map labels = new HashMap<>(); + MockPresets.AVAILABLE_TWS.forEach(t -> labels.put(t.id(), t.label())); + List tws = new ArrayList<>(); + for (String id : binding.tws()) { + String label = labels.getOrDefault(id, id); + List sources = tws.isEmpty() + ? List.of(new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Source(id, label, "STEAM_CONTENT", "2", null), + new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Source(id, label, "DTDD_TOPIC", + "there's blood/gore", new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Severity( + "VOTES", "Depicted", "Excessive", 12, 83))) + : List.of(new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Source(id, label, "STEAM_KEYWORD", id, null)); + tws.add(new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Tw(id, label, true, + binding.twOverride() ? "STREAMER" : "DETECTED", binding.twOverride() ? List.of() : sources)); + } + return new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto(binding.twOverride(), tws, List.of( + new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Source("jumpscares", + labels.getOrDefault("jumpscares", "Jump scares"), "DTDD_TOPIC", "there are jump scares", + new fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.Severity("BELOW_VOTES", "Startling", "Mild", + 9, 22)))); + } + @Override public int resetTws(String bindingId) { log.trace("resetTws({})", bindingId); @@ -787,12 +819,79 @@ public ApiClient.Result adminDeleteGameMapping(UUID id, boole return gameMappings.delete(id, applyLive); } + @Override + public fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto adminDtddCaches() { + log.trace("adminDtddCaches()"); + return dtddCaches.summary(); + } + + @Override + public AdminPage adminDtddTopicsCache(String q, int page) { + log.trace("adminDtddTopicsCache({}, {})", q, page); + return dtddCaches.topics(q, page); + } + + @Override + public AdminPage adminDtddGamesCache(String q, int page) { + log.trace("adminDtddGamesCache({}, {})", q, page); + return dtddCaches.games(q, page); + } + + @Override + public AdminPage adminDtddSearchesCache(String q, int page) { + log.trace("adminDtddSearchesCache({}, {})", q, page); + return dtddCaches.searches(q, page); + } + + @Override + public int adminRefetchDtddCache(String kind, String key) { + log.trace("adminRefetchDtddCache({}, {})", kind, key); + return dtddCaches.refetch(kind, key); + } + + @Override + public fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto adminDtddCacheEntry(String kind, String key) { + log.trace("adminDtddCacheEntry({}, {})", kind, key); + return dtddCaches.detail(kind, key); + } + + @Override + public ApiClient.Result adminStartDtddRefetch() { + log.trace("adminStartDtddRefetch()"); + return dtddCaches.start(); + } + + @Override + public ApiClient.Result adminCancelDtddRefetch() { + log.trace("adminCancelDtddRefetch()"); + return dtddCaches.cancel(); + } + @Override public List adminDtddProposals(String status) { log.trace("adminDtddProposals({})", status); return dtddProposals.list(status); } + @Override + public List adminRateLimits() { + log.trace("adminRateLimits()"); + return rateLimits.list(); + } + + @Override + public int adminCorrectRateLimit(String id, + fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto correction) { + log.trace("adminCorrectRateLimit({}, {})", id, correction); + return rateLimits.correct(id, correction); + } + + @Override + public int adminResetRateLimit(String id) { + log.trace("adminResetRateLimit({})", id); + return rateLimits.reset(id); + } + @Override public List adminGroups() { log.trace("adminGroups()"); @@ -1688,7 +1787,19 @@ public int adminCreateTw(String id, String label, int sortOrder) { @Override public int adminRebuildTws(TwRebuild mode) { log.trace("adminRebuildTws({})", mode); - return tws.rebuild(); + return tws.rebuild(mode.name()); + } + + @Override + public fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto adminTwRebuildStatus() { + log.trace("adminTwRebuildStatus()"); + return tws.rebuildStatus(); + } + + @Override + public int adminCancelTwRebuild() { + log.trace("adminCancelTwRebuild()"); + return tws.cancelRebuild(); } @Override @@ -1697,6 +1808,18 @@ public List adminTwKeywords(String id) { return tws.keywords(id); } + @Override + public List adminTwDtddTopics(String id) { + log.trace("adminTwDtddTopics({})", id); + return tws.dtddTopics(id); + } + + @Override + public int adminSetTwDtddTopics(String id, Map minByTopic) { + log.trace("adminSetTwDtddTopics({}, {})", id, minByTopic); + return tws.setDtddTopics(id, minByTopic); + } + @Override public int adminAddTwKeyword(String id, String keyword) { log.trace("adminAddTwKeyword({}, {})", id, keyword); diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockData.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockData.java index d748aed54..5a5ea2f8a 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockData.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockData.java @@ -217,7 +217,7 @@ private static MockData fromConfig(MockConfig config) { null, null, Set.of(), AVAILABLE_CCLS, config.settings.twFeatureEnabled, Set.copyOf(config.settings.blockedTws), AVAILABLE_TWS, // raid.restore-game and stream.title: the mock has no experiments, the settings are offered - null, true, List.of(), List.of(), false, null, true, null, null, true); + null, true, List.of(), List.of(), false, null, true, null, null, true, true, true); MockConfig.Steam steam = config.steam; MockData data = new MockData(channel, List.of(binding), settings, List.of(), @@ -892,7 +892,7 @@ UserSettingsDto build() { from.availableCcls(), twEnabled, blockedTws, from.availableTws(), raidMinutes, from.raidRestoreGameAvailable(), noGameExtensions, fallbackExtensions, titleEnabled, titleTemplate, from.titleAvailable(), noGameTitle, incompleteFallbackTitle, - from.titlePreviewAvailable()); + from.titlePreviewAvailable(), from.titleConditionsAvailable(), from.titleDropsAvailable()); } } } diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockDtddCachesStore.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockDtddCachesStore.java new file mode 100644 index 000000000..3bef5a7bb --- /dev/null +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockDtddCachesStore.java @@ -0,0 +1,229 @@ +package fr.enimaloc.catapult.service.mock; + +import fr.enimaloc.catapult.common.dto.admin.AdminPage; +import fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddGameCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddKeyQuotaDto; +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.common.dto.admin.DtddSearchCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicSeverityDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; +import fr.enimaloc.catapult.service.http.ApiClient; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; + +/** + * The mock's DTDD caches: a few titles' topics, titles and searches, some past their lifetime, two keys' quotas, and + * a bulk refetch that moves on by one entry each time it is read, so the panel's progress shows. Answers like + * catapult-api: 404 for an entry not cached, 409 for a second refetch. + */ +class MockDtddCachesStore { + static final int DEFAULT_PAGE_SIZE = 3; + static final int TOPICS_TTL_HOURS = 168; + static final int GAMES_TTL_HOURS = 180 * 24; + static final int SEARCHES_TTL_HOURS = 30 * 24; + static final int MONTHLY_RESERVE = 500; + + private final Clock clock; + private final int pageSize; + private final List topics = new ArrayList<>(); + private final List games = new ArrayList<>(); + private final List searches = new ArrayList<>(); + private DtddRefetchJobDto job = DtddRefetchJobDto.IDLE; + + MockDtddCachesStore() { + this(Clock.systemUTC(), DEFAULT_PAGE_SIZE); + } + + MockDtddCachesStore(Clock clock, int pageSize) { + this.clock = clock; + this.pageSize = pageSize; + Instant now = clock.instant(); + topics.add(new DtddTopicsCacheEntryDto(16758, "Halo Infinite", List.of("a dog dies"), List.of("a cat dies"), + List.of("jump scares"), now.minus(Duration.ofDays(9)), false, Map.of( + "a dog dies", new DtddTopicSeverityDto(List.of("Implied", "Depicted", "Excessive"), + List.of(2, 9, 1), List.of(17, 75, 8), 12, 2), + "jump scares", new DtddTopicSeverityDto(List.of("Mild", "Shown", "Severe"), + List.of(5, 3, 0), List.of(63, 37, 0), 8, null)))); + topics.add(new DtddTopicsCacheEntryDto(20311, "Elden Ring", List.of("gore", "body horror"), List.of(), + List.of("spiders"), now.minus(Duration.ofDays(2)), false)); + topics.add(new DtddTopicsCacheEntryDto(18842, null, List.of(), List.of("a dog dies"), List.of(), + now.minus(Duration.ofHours(5)), false)); + topics.add(new DtddTopicsCacheEntryDto(21007, "Stardew Valley", List.of(), List.of("gore"), List.of(), + now.minus(Duration.ofDays(30)), false)); + games.add(new DtddGameCacheEntryDto(16758, "Halo Infinite", "Video Game", null, now.minus(Duration.ofDays(200)), false)); + games.add(new DtddGameCacheEntryDto(20311, "Elden Ring", "Video Game", null, now.minus(Duration.ofDays(40)), false)); + games.add(new DtddGameCacheEntryDto(21007, "Stardew Valley", "Video Game", null, now.minus(Duration.ofDays(3)), false)); + searches.add(new DtddSearchCacheEntryDto("halo", List.of(16758L), now.minus(Duration.ofDays(45)), false)); + searches.add(new DtddSearchCacheEntryDto("elden ring", List.of(20311L), now.minus(Duration.ofDays(4)), false)); + } + + synchronized DtddCachesSummaryDto summary() { + advance(); + return new DtddCachesSummaryDto(topics.size(), stale(topics, DtddTopicsCacheEntryDto::fetchedAt, TOPICS_TTL_HOURS), + games.size(), stale(games, DtddGameCacheEntryDto::fetchedAt, GAMES_TTL_HOURS), + searches.size(), stale(searches, DtddSearchCacheEntryDto::searchedAt, SEARCHES_TTL_HOURS), + TOPICS_TTL_HOURS, GAMES_TTL_HOURS, SEARCHES_TTL_HOURS, + List.of(new DtddKeyQuotaDto("a1b2c3d4e5f60718", "ddd_
mnop", 27, 4120, + clock.instant().minusSeconds(40), null), + new DtddKeyQuotaDto("0f1e2d3c4b5a6978", "ddd_
2345", 0, 612, clock.instant().minusSeconds(10), + clock.instant().plusSeconds(50))), + MONTHLY_RESERVE, job); + } + + synchronized AdminPage topics(String q, int page) { + return page(topics, t -> matches(q, t.dtddId(), t.name()), DtddTopicsCacheEntryDto::fetchedAt, page, + t -> new DtddTopicsCacheEntryDto(t.dtddId(), t.name(), t.yesTopics(), t.noTopics(), t.mostlyTopics(), + t.fetchedAt(), isStale(t.fetchedAt(), TOPICS_TTL_HOURS), t.severities())); + } + + synchronized AdminPage games(String q, int page) { + return page(games, g -> matches(q, g.dtddId(), g.name()), DtddGameCacheEntryDto::fetchedAt, page, + g -> new DtddGameCacheEntryDto(g.dtddId(), g.name(), g.mediaType(), g.posterUrl(), g.fetchedAt(), + isStale(g.fetchedAt(), GAMES_TTL_HOURS))); + } + + synchronized AdminPage searches(String q, int page) { + String needle = q == null ? "" : q.trim().toLowerCase(Locale.ROOT); + return page(searches, s -> s.query().contains(needle), DtddSearchCacheEntryDto::searchedAt, page, + s -> new DtddSearchCacheEntryDto(s.query(), s.dtddIds(), s.searchedAt(), + isStale(s.searchedAt(), SEARCHES_TTL_HOURS))); + } + + /** Everything cached about an entry, as catapult-api gives it; null when it isn't cached. */ + synchronized DtddCacheEntryDetailDto detail(String kind, String key) { + String trimmed = key == null ? "" : key.trim(); + if (kind.equals("searches")) { + String query = trimmed.toLowerCase(Locale.ROOT); + return searches(query, 0).content().stream().filter(s -> s.query().equals(query)).findFirst() + .map(s -> new DtddCacheEntryDetailDto(null, query, null, null, List.of(s), s.dtddIds().stream() + .map(id -> new DtddCacheEntryDetailDto.FoundTitle(id, + games.stream().filter(g -> g.dtddId() == id).map(DtddGameCacheEntryDto::name) + .findFirst().orElse(null), + topics.stream().anyMatch(t -> t.dtddId() == id))) + .toList())) + .orElse(null); + } + long id; + try { + id = Long.parseLong(trimmed); + } catch (NumberFormatException e) { + return null; + } + DtddTopicsCacheEntryDto topic = topics(String.valueOf(id), 0).content().stream() + .filter(t -> t.dtddId() == id).findFirst().orElse(null); + DtddGameCacheEntryDto game = games(String.valueOf(id), 0).content().stream() + .filter(g -> g.dtddId() == id).findFirst().orElse(null); + if (kind.equals("topics") ? topic == null : game == null) return null; + return new DtddCacheEntryDetailDto(id, null, game, topic, + searches.stream().filter(s -> s.dtddIds().contains(id)) + .map(s -> new DtddSearchCacheEntryDto(s.query(), s.dtddIds(), s.searchedAt(), + isStale(s.searchedAt(), SEARCHES_TTL_HOURS))) + .toList(), + List.of()); + } + + /** The entry as fetched just now: 200, 404 when it isn't cached, 400 for an unknown cache or a title key not an ID. */ + synchronized int refetch(String kind, String key) { + Instant now = clock.instant(); + String trimmed = key == null ? "" : key.trim(); + switch (kind) { + case "topics", "games" -> { + long id; + try { + id = Long.parseLong(trimmed); + } catch (NumberFormatException e) { + return 400; + } + if (kind.equals("topics")) { + int i = indexOf(topics, t -> t.dtddId() == id); + if (i < 0) return 404; + DtddTopicsCacheEntryDto t = topics.get(i); + topics.set(i, new DtddTopicsCacheEntryDto(id, t.name(), t.yesTopics(), t.noTopics(), + t.mostlyTopics(), now, false, t.severities())); + } else { + int i = indexOf(games, g -> g.dtddId() == id); + if (i < 0) return 404; + DtddGameCacheEntryDto g = games.get(i); + games.set(i, new DtddGameCacheEntryDto(id, g.name(), g.mediaType(), g.posterUrl(), now, false)); + } + return 200; + } + case "searches" -> { + String query = trimmed.toLowerCase(Locale.ROOT); + int i = indexOf(searches, s -> s.query().equals(query)); + if (i < 0) return 404; + searches.set(i, new DtddSearchCacheEntryDto(query, searches.get(i).dtddIds(), now, false)); + return 200; + } + default -> { + return 404; + } + } + } + + synchronized ApiClient.Result start() { + if ("RUNNING".equals(job.state())) return new ApiClient.Result<>(409, null); + int total = topics.size() + games.size() + searches.size(); + job = new DtddRefetchJobDto("RUNNING", total, 0, 0, null, clock.instant(), null, null); + return new ApiClient.Result<>(202, job); + } + + synchronized ApiClient.Result cancel() { + if ("RUNNING".equals(job.state())) { + job = new DtddRefetchJobDto("CANCELLED", job.total(), job.done(), job.failed(), null, job.startedAt(), + clock.instant(), null); + } + return new ApiClient.Result<>(200, job); + } + + /** A running refetch moves on by one entry each time it is read; the mock's last one gives nothing. */ + private void advance() { + if (!"RUNNING".equals(job.state())) return; + int done = job.done() + 1; + int failed = done == job.total() ? 1 : job.failed(); + job = done >= job.total() + ? new DtddRefetchJobDto("DONE", job.total(), done, failed, null, job.startedAt(), clock.instant(), null) + : new DtddRefetchJobDto("RUNNING", job.total(), done, failed, null, job.startedAt(), null, null); + } + + private boolean isStale(Instant at, int ttlHours) { + return at.isBefore(clock.instant().minus(Duration.ofHours(ttlHours))); + } + + private long stale(List all, Function at, int ttlHours) { + return all.stream().filter(e -> isStale(at.apply(e), ttlHours)).count(); + } + + private static boolean matches(String q, long id, String name) { + String needle = q == null ? "" : q.trim().toLowerCase(Locale.ROOT); + return String.valueOf(id).startsWith(needle) || name != null && name.toLowerCase(Locale.ROOT).contains(needle); + } + + private static int indexOf(List all, Predicate match) { + for (int i = 0; i < all.size(); i++) { + if (match.test(all.get(i))) return i; + } + return -1; + } + + /** The {@code page} of {@code all} matching, the oldest first, each one as now ({@code fresh}: its staleness). */ + private AdminPage page(List all, Predicate match, Function at, int page, + Function fresh) { + List found = all.stream().filter(match).sorted(Comparator.comparing(at)).map(fresh).toList(); + int from = (int) Math.min((long) Math.max(page, 0) * pageSize, found.size()); + int to = Math.min(from + pageSize, found.size()); + int totalPages = (found.size() + pageSize - 1) / pageSize; + return new AdminPage<>(found.subList(from, to), Math.max(page, 0), totalPages, found.size()); + } +} diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockGameMappingsStore.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockGameMappingsStore.java index aeb492650..686322d81 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockGameMappingsStore.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockGameMappingsStore.java @@ -49,6 +49,8 @@ class MockGameMappingsStore { mappings.add(new GameMappingDto(UUID.randomUUID(), "XBOX", "1942", "Halo Infinite", "506416", "Halo Infinite", java.util.Set.of("violent-graphic"), java.util.Set.of("violence"), 6, now.minus(1, ChronoUnit.DAYS), "enimaloc")); + mappings.add(new GameMappingDto(UUID.randomUUID(), "STEAM", "431960", "Wallpaper Engine", null, null, + null, null, 0, now.minus(3, ChronoUnit.DAYS), "enimaloc", true)); } synchronized AdminPage list(int page) { @@ -73,7 +75,8 @@ synchronized List detected(String sourceType, String q) { } synchronized ApiClient.Result create(GameMappingRequest request, String updatedBy) { - if (request == null || invalid(request.sourceType(), request.sourceId(), request.twitchGameId())) { + if (request == null || invalid(request.sourceType(), request.sourceId(), request.twitchGameId(), + request.ignoredByDefault())) { return new ApiClient.Result<>(400, null); } String sourceId = request.sourceId().trim(); @@ -83,7 +86,8 @@ synchronized ApiClient.Result create(GameMappingRequest reque .findFirst().orElse(null); GameMappingDto mapping = new GameMappingDto(UUID.randomUUID(), request.sourceType(), sourceId, game == null ? null : game.sourceName(), request.twitchGameId(), request.twitchGameName(), - request.ccls(), request.tws(), game == null ? 0 : game.streamerCount(), clock.instant(), updatedBy); + request.ccls(), request.tws(), game == null ? 0 : game.streamerCount(), clock.instant(), updatedBy, + request.ignoredByDefault()); mappings.add(mapping); return new ApiClient.Result<>(201, result(mapping, request.applyLive())); } @@ -93,12 +97,12 @@ synchronized ApiClient.Result update(UUID id, GameMappingRequ int index = indexOf(id); if (index < 0) return new ApiClient.Result<>(404, null); GameMappingDto current = mappings.get(index); - if (request == null || request.twitchGameId() == null || request.twitchGameId().isBlank()) { + if (request == null || blank(request.twitchGameId()) && !request.ignoredByDefault()) { return new ApiClient.Result<>(400, null); } GameMappingDto changed = new GameMappingDto(id, current.sourceType(), current.sourceId(), current.sourceName(), request.twitchGameId(), request.twitchGameName(), request.ccls(), request.tws(), - current.bindingCount(), clock.instant(), updatedBy); + current.bindingCount(), clock.instant(), updatedBy, request.ignoredByDefault()); mappings.set(index, changed); return new ApiClient.Result<>(200, result(changed, request.applyLive())); } @@ -116,9 +120,14 @@ private AdminPage page(List all, int page) { return new AdminPage<>(all.subList(from, to), Math.max(page, 0), (all.size() + pageSize - 1) / pageSize, all.size()); } - private static boolean invalid(String sourceType, String sourceId, String twitchGameId) { - return sourceType == null || sourceType.isBlank() || "MANUAL".equalsIgnoreCase(sourceType) - || sourceId == null || sourceId.isBlank() || twitchGameId == null || twitchGameId.isBlank(); + /** No category is only valid for a game ignored by default, as catapult-api checks it. */ + private static boolean invalid(String sourceType, String sourceId, String twitchGameId, boolean ignoredByDefault) { + return blank(sourceType) || "MANUAL".equalsIgnoreCase(sourceType) + || blank(sourceId) || blank(twitchGameId) && !ignoredByDefault; + } + + private static boolean blank(String s) { + return s == null || s.isBlank(); } private static GameMappingResult result(GameMappingDto mapping, boolean applyLive) { diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockPresets.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockPresets.java index 8f701a6cb..fefef7b61 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockPresets.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockPresets.java @@ -116,7 +116,7 @@ public static String getDefaultAvatar(Random random, int size) { null, null, Set.of(), AVAILABLE_CCLS, false, Set.of(), AVAILABLE_TWS, // raid.restore-game and stream.title: the mock has no experiments, the settings are offered - null, true, List.of(), List.of(), false, null, true, null, null, true); + null, true, List.of(), List.of(), false, null, true, null, null, true, true, true); private MockPresets() { } diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockRateLimitsStore.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockRateLimitsStore.java new file mode 100644 index 000000000..a61043e0b --- /dev/null +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockRateLimitsStore.java @@ -0,0 +1,78 @@ +package fr.enimaloc.catapult.service.mock; + +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * The mock's learned rate limits: the Steam store's host with its routes (the page split off with a limit of its + * own) and an Xbox host never limited. Corrected and reset like catapult-api: 400 for an invalid correction, 404 for + * an unknown scope. + */ +class MockRateLimitsStore { + + private static final String STORE = "store.steampowered.com"; + private final List limits = new ArrayList<>(); + + MockRateLimitsStore(Clock clock) { + Instant now = clock.instant(); + limits.add(new LearnedRateLimitDto("HOST " + STORE, STORE, null, null, null, false, 190, 300, false, 76_800, + false, 4, now.minus(Duration.ofMinutes(42)), 23, null)); + limits.add(new LearnedRateLimitDto("GET " + STORE + "/api/appdetails", STORE, "GET", "/api/appdetails", "HOST", + false, null, null, false, 60_000, false, 3, now.minus(Duration.ofMinutes(42)), 21, null)); + limits.add(new LearnedRateLimitDto("GET " + STORE + "/app/{id}", STORE, "GET", "/app/{id}", "ROUTE", false, 12, + 60, false, 30_000, false, 1, now.minus(Duration.ofMinutes(3)), 12, now.plusSeconds(25))); + limits.add(new LearnedRateLimitDto("HOST titlehub.xboxlive.com", "titlehub.xboxlive.com", null, null, null, + false, null, null, false, 60_000, false, 0, null, 2, null)); + } + + List list() { + return List.copyOf(limits); + } + + int correct(String id, LearnedRateLimitCorrectionDto c) { + int i = indexOf(id); + if (i < 0) return 404; + if (c == null || (c.limitCalls() == null) != (c.limitWindowSeconds() == null) + || (c.limitCalls() != null && (c.limitCalls() < 1 || c.limitWindowSeconds() < 1)) + || (c.resumeSeconds() != null && c.resumeSeconds() < 1)) { + return 400; + } + LearnedRateLimitDto d = limits.get(i); + String named = c.scope() == null ? "" : c.scope().trim().toUpperCase(Locale.ROOT); + boolean unpin = named.equals("LEARNED"); + String scope = named.isEmpty() || unpin || d.hostScope() ? d.scope() : named; + if (scope != null && !scope.equals("HOST") && !scope.equals("ROUTE")) return 400; + boolean pinned = !d.hostScope() && !unpin && (d.scopePinned() || !named.isEmpty()); + boolean limitSet = c.limitCalls() != null; + boolean resumeSet = c.resumeSeconds() != null; + limits.set(i, new LearnedRateLimitDto(d.id(), d.host(), d.method(), d.path(), scope, + pinned, + limitSet ? c.limitCalls() : d.limitCalls(), limitSet ? c.limitWindowSeconds() : d.limitWindowSeconds(), + limitSet, resumeSet ? c.resumeSeconds() * 1000L : d.resumeMs(), resumeSet, d.rateLimitedCount(), + d.lastRateLimitedAt(), d.usedCalls(), d.blockedUntil())); + return 200; + } + + int reset(String id) { + int i = indexOf(id); + if (i < 0) return 404; + LearnedRateLimitDto d = limits.get(i); + limits.set(i, new LearnedRateLimitDto(d.id(), d.host(), d.method(), d.path(), d.hostScope() ? null : "HOST", false, + null, null, false, 60_000, false, 0, null, d.usedCalls(), null)); + return 200; + } + + private int indexOf(String id) { + for (int i = 0; i < limits.size(); i++) { + if (limits.get(i).id().equals(id)) return i; + } + return -1; + } +} diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockTwStore.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockTwStore.java index ea3889f78..dc0fe3910 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockTwStore.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/mock/MockTwStore.java @@ -1,8 +1,11 @@ package fr.enimaloc.catapult.service.mock; import fr.enimaloc.catapult.common.dto.admin.TwDefinitionDto; +import fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto; +import fr.enimaloc.catapult.common.dto.admin.TwDtddTopicDto; import fr.enimaloc.catapult.common.dto.admin.TwKeywordTestDto; +import java.time.Instant; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -28,12 +31,43 @@ class MockTwStore { private final List definitions = new ArrayList<>(); private final Map> keywords = new java.util.HashMap<>(); + /** Each definition's DTDD topics with their least severe level, as catapult-api keeps them. */ + private final Map> dtddTopics = new java.util.HashMap<>(); + /** The levels the mock's cached titles give: blood/gore and jump scares only. */ + private static final Map> KNOWN_LEVELS = Map.of( + "there's blood/gore", List.of("Minor", "Depicted", "Excessive"), + "there are jump scares", List.of("Mild", "Startling", "Jarring")); MockTwStore() { definitions.add(new TwDefinitionDto("violence", "Violence", null, true, 10)); definitions.add(new TwDefinitionDto("jumpscares", "Jump scares", null, true, 20)); definitions.add(new TwDefinitionDto("flashing_lights", "Flashing lights", null, false, 30)); keywords.put("violence", new TreeSet<>(List.of("blood", "gore"))); + Map violence = new java.util.TreeMap<>(); + violence.put("there's blood/gore", "Depicted"); + violence.put("someone is stabbed", null); + dtddTopics.put("violence", violence); + Map jumps = new java.util.TreeMap<>(); + jumps.put("there are jump scares", null); + dtddTopics.put("jumpscares", jumps); + } + + /** A definition's DTDD topics with their threshold and known levels, by name; null when it is unknown. */ + synchronized List dtddTopics(String id) { + if (find(id).isEmpty()) return null; + return dtddTopics.getOrDefault(id, Map.of()).entrySet().stream() + .map(e -> new TwDtddTopicDto(e.getKey(), e.getValue(), + KNOWN_LEVELS.getOrDefault(e.getKey().toLowerCase(Locale.ROOT), List.of()))) + .toList(); + } + + /** Replaces them all: 204, 404 for an unknown definition. */ + synchronized int setDtddTopics(String id, Map minByTopic) { + if (find(id).isEmpty()) return 404; + Map kept = new java.util.TreeMap<>(); + minByTopic.forEach((topic, min) -> kept.put(topic.trim(), min == null || min.isBlank() ? null : min.trim())); + dtddTopics.put(id, kept); + return 204; } synchronized List list() { @@ -54,11 +88,48 @@ synchronized int create(String id, String label, int sortOrder) { return 201; } - /** The rebuild runs in the background in catapult-api: accepted, nothing to change here. */ - int rebuild() { + /** How many bindings the mock's recalculation goes through, three more each time it is read. */ + static final int REBUILD_TOTAL = 12; + private static final int REBUILD_STEP = 3; + private TwRebuildStatusDto rebuild = TwRebuildStatusDto.IDLE; + + /** Runs "in the background" as catapult-api does: 202, 409 while one runs already. */ + synchronized int rebuild(String mode) { + if ("RUNNING".equals(rebuild.state())) return 409; + rebuild = new TwRebuildStatusDto("RUNNING", mode, REBUILD_TOTAL, 0, 0, 0, Instant.now(), null, null); return 202; } + /** + * The recalculation, three bindings further each time it is read; halfway, it waits a minute for the Steam + * store's rate limit once, as catapult-api would show it. + */ + synchronized TwRebuildStatusDto rebuildStatus() { + if (!"RUNNING".equals(rebuild.state())) return rebuild; + TwRebuildStatusDto r = rebuild; + if (r.done() == REBUILD_TOTAL / 2 && r.waitingUntil() == null) { + rebuild = new TwRebuildStatusDto("RUNNING", r.mode(), r.total(), r.processed(), r.skipped(), r.failed(), + r.startedAt(), null, Instant.now().plusSeconds(60)); + return rebuild; + } + int processed = r.processed() + REBUILD_STEP - 1; + int skipped = r.skipped() + 1; + boolean done = processed + skipped >= REBUILD_TOTAL; + rebuild = new TwRebuildStatusDto(done ? "DONE" : "RUNNING", r.mode(), r.total(), + Math.min(processed, REBUILD_TOTAL - skipped), skipped, r.failed(), r.startedAt(), + done ? Instant.now() : null, null); + return rebuild; + } + + synchronized int cancelRebuild() { + if ("RUNNING".equals(rebuild.state())) { + TwRebuildStatusDto r = rebuild; + rebuild = new TwRebuildStatusDto("CANCELLED", r.mode(), r.total(), r.processed(), r.skipped(), r.failed(), + r.startedAt(), Instant.now(), null); + } + return 200; + } + /** A definition's keywords, sorted; null when the definition is unknown. */ synchronized List keywords(String id) { return find(id).isEmpty() ? null : List.copyOf(keywords.getOrDefault(id, new TreeSet<>())); diff --git a/catapult-web/src/main/java/fr/enimaloc/catapult/service/real/RealApiService.java b/catapult-web/src/main/java/fr/enimaloc/catapult/service/real/RealApiService.java index 0fe7c015f..162e4ac90 100644 --- a/catapult-web/src/main/java/fr/enimaloc/catapult/service/real/RealApiService.java +++ b/catapult-web/src/main/java/fr/enimaloc/catapult/service/real/RealApiService.java @@ -8,6 +8,8 @@ import fr.enimaloc.catapult.common.dto.premium.PremiumStatusDto; import fr.enimaloc.catapult.common.dto.channel.DeleteAccountRequest; import fr.enimaloc.catapult.common.dto.TokenResponse; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitCorrectionDto; +import fr.enimaloc.catapult.common.dto.admin.LearnedRateLimitDto; import fr.enimaloc.catapult.common.dto.admin.AdminMemberDto; import fr.enimaloc.catapult.common.dto.admin.AdminMembersPage; import fr.enimaloc.catapult.common.dto.admin.AdminCclPage; @@ -49,6 +51,12 @@ import fr.enimaloc.catapult.common.dto.admin.MemberTargetingDto; import fr.enimaloc.catapult.common.dto.admin.AdminPage; import fr.enimaloc.catapult.common.dto.admin.DetectedGameDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCacheEntryDetailDto; +import fr.enimaloc.catapult.common.dto.admin.DtddCachesSummaryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddGameCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddRefetchJobDto; +import fr.enimaloc.catapult.common.dto.admin.DtddSearchCacheEntryDto; +import fr.enimaloc.catapult.common.dto.admin.DtddTopicsCacheEntryDto; import fr.enimaloc.catapult.common.dto.admin.GameMappingDto; import fr.enimaloc.catapult.common.dto.admin.GameMappingRequest; import fr.enimaloc.catapult.common.dto.admin.GameMappingResult; @@ -157,6 +165,9 @@ public class RealApiService implements ApiService { private static final ParameterizedTypeReference> DEBUG_EXCHANGES = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> DEBUG_SERVICES = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> GAME_MAPPINGS = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> DTDD_TOPICS = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> DTDD_GAMES = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> DTDD_SEARCHES = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> MISSING_GAMES = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> DETECTED_GAMES = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> CATEGORIES = new ParameterizedTypeReference<>() {}; @@ -164,6 +175,7 @@ public class RealApiService implements ApiService { private static final ParameterizedTypeReference> TWS = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> ENABLED_COMMANDS = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> GROUPS = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> RATE_LIMITS = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> EXPERIMENTS = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> FLAG_KEYS = new ParameterizedTypeReference<>() {}; private static final ParameterizedTypeReference> ADMIN_NOTIFICATIONS = new ParameterizedTypeReference<>() {}; @@ -266,6 +278,14 @@ public int resetTws(String bindingId) { return client.delete("/api/internal/users/{userId}/bindings/{bindingId}/tws", me, bindingId); } + @Override + public fr.enimaloc.catapult.common.dto.channel.TwDetailsDto twDetails(String bindingId) { + UUID me = me(); + if (me == null) return null; + return client.get("/api/internal/users/{userId}/bindings/{bindingId}/tws/details", + fr.enimaloc.catapult.common.dto.channel.TwDetailsDto.class, me, bindingId); + } + @Override public int resetBinding(String bindingId) { UUID me = me(); @@ -989,6 +1009,69 @@ public ApiClient.Result adminDeleteGameMapping(UUID id, boole GameMappingResult.class, id, applyLive); } + @Override + public DtddCachesSummaryDto adminDtddCaches() { + return client.get("/api/internal/admin/dtdd-caches", DtddCachesSummaryDto.class); + } + + @Override + public AdminPage adminDtddTopicsCache(String q, int page) { + return client.get("/api/internal/admin/dtdd-caches/{kind}?q={q}&page={page}&size=20", DTDD_TOPICS, "topics", + q == null ? "" : q, page); + } + + @Override + public AdminPage adminDtddGamesCache(String q, int page) { + return client.get("/api/internal/admin/dtdd-caches/{kind}?q={q}&page={page}&size=20", DTDD_GAMES, "games", + q == null ? "" : q, page); + } + + @Override + public AdminPage adminDtddSearchesCache(String q, int page) { + return client.get("/api/internal/admin/dtdd-caches/{kind}?q={q}&page={page}&size=20", DTDD_SEARCHES, "searches", + q == null ? "" : q, page); + } + + @Override + public int adminRefetchDtddCache(String kind, String key) { + return client.call(HttpMethod.POST, "/api/internal/admin/dtdd-caches/{kind}/refetch?key={key}", null, + Void.class, kind, key).status(); + } + + @Override + public DtddCacheEntryDetailDto adminDtddCacheEntry(String kind, String key) { + return client.get("/api/internal/admin/dtdd-caches/{kind}/entry?key={key}", DtddCacheEntryDetailDto.class, + kind, key); + } + + @Override + public ApiClient.Result adminStartDtddRefetch() { + return client.call(HttpMethod.POST, "/api/internal/admin/dtdd-caches/refetch-all", null, DtddRefetchJobDto.class); + } + + @Override + public ApiClient.Result adminCancelDtddRefetch() { + return client.call(HttpMethod.DELETE, "/api/internal/admin/dtdd-caches/refetch-all", null, + DtddRefetchJobDto.class); + } + + @Override + public List adminRateLimits() { + return client.get("/api/internal/admin/rate-limits", RATE_LIMITS); + } + + @Override + public int adminCorrectRateLimit(String id, LearnedRateLimitCorrectionDto correction) { + return client.call(HttpMethod.PUT, "/api/internal/admin/rate-limits/entry?id={id}", correction, + LearnedRateLimitDto.class, id).status(); + } + + @Override + public int adminResetRateLimit(String id) { + return client.call(HttpMethod.DELETE, "/api/internal/admin/rate-limits/entry?id={id}", null, + LearnedRateLimitDto.class, id).status(); + } + @Override public List adminGroups() { return client.get("/api/internal/admin/groups", GROUPS); @@ -1330,6 +1413,16 @@ public int adminCreateTw(String id, String label, int sortOrder) { return client.call(HttpMethod.POST, "/api/internal/admin/tws", new AdminTwCreateBody(id, label, null, sortOrder), Void.class).status(); } + @Override + public fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto adminTwRebuildStatus() { + return client.get("/api/internal/admin/tws/rebuild/status", fr.enimaloc.catapult.common.dto.admin.TwRebuildStatusDto.class); + } + + @Override + public int adminCancelTwRebuild() { + return client.call(HttpMethod.POST, "/api/internal/admin/tws/rebuild/cancel", null, Void.class).status(); + } + @Override public int adminRebuildTws(TwRebuild mode) { ApiClient.Result result = switch (mode) { @@ -1345,6 +1438,23 @@ public List adminTwKeywords(String id) { return client.get("/api/internal/admin/tws/{id}/steam-keywords", new ParameterizedTypeReference>() {}, id); } + @Override + public List adminTwDtddTopics(String id) { + return client.get("/api/internal/admin/tws/{id}/dtdd-topics", + new ParameterizedTypeReference>() {}, id); + } + + @Override + public int adminSetTwDtddTopics(String id, Map minByTopic) { + Map thresholds = new java.util.LinkedHashMap<>(); + minByTopic.forEach((topic, min) -> { + if (min != null && !min.isBlank()) thresholds.put(topic, min); + }); + return client.call(HttpMethod.PUT, "/api/internal/admin/tws/{id}/dtdd-topics", + new fr.enimaloc.catapult.common.dto.admin.DtddTopicsBody(minByTopic.keySet(), thresholds), Void.class, + id).status(); + } + @Override public int adminAddTwKeyword(String id, String keyword) { return client.call(HttpMethod.POST, "/api/internal/admin/tws/{id}/steam-keywords", new SteamKeywordBody(keyword), Void.class, id).status(); diff --git a/catapult-web/src/main/resources/lang/messages.properties b/catapult-web/src/main/resources/lang/messages.properties index d036293cc..bf4e4d307 100644 --- a/catapult-web/src/main/resources/lang/messages.properties +++ b/catapult-web/src/main/resources/lang/messages.properties @@ -208,6 +208,33 @@ channel.bindings.title.label=Stream title for this game channel.bindings.title.helper={{GAME}}: the game detected, {{CATEGORY}}: its Twitch category, {{TITLE}}: the title just before, {{ORIGINAL_TITLE}}: your own title, before Catapult changed it. Empty: the title set in Settings channel.bindings.tw.enabled=Enable trigger warnings channel.bindings.tw.reset=Reset +channel.bindings.tw.details=Details +channel.bindings.tw.details.title=Why these TWs? +channel.bindings.tw.details.hint=What the detection finds for this game right now, source by source. +channel.bindings.tw.details.streamer_chose=You changed this game's TWs: your choices apply, the detection no longer changes them until you reset them. +channel.bindings.tw.details.none=No TW for this game. +channel.bindings.tw.details.unavailable=The details aren't available right now. +channel.bindings.tw.details.turned_off=turned off by you +channel.bindings.tw.details.origin.streamer=Added by you. +channel.bindings.tw.details.origin.admin=Set by a Catapult admin for this game. +channel.bindings.tw.details.origin.no_source=Nothing suggests it any more: it goes at the next recomputation. +channel.bindings.tw.details.kept_off=Kept off by their severity +channel.bindings.tw.details.source.igdb=IGDB: content descriptor #{0} +channel.bindings.tw.details.source.steam_content=Steam: {0} +channel.bindings.tw.details.source.steam_keyword=Steam: the content notes mention “{0}” +channel.bindings.tw.details.source.dtdd=DoesTheDogDie: “{0}” +channel.bindings.tw.details.steam.1=some nudity or sexual content +channel.bindings.tw.details.steam.2=frequent violence or gore +channel.bindings.tw.details.steam.3=adult only sexual content +channel.bindings.tw.details.steam.4=frequent nudity or sexual content +channel.bindings.tw.details.steam.5=general mature content +channel.bindings.tw.details.severity.not_cached=Severity not known yet: on by default. +channel.bindings.tw.details.severity.unknown_level=The least severe level “{0}” doesn''t exist for this topic: on by default. +channel.bindings.tw.details.severity.too_few_votes=Too few votes to judge its severity ({0,choice,0#no vote|1#1 vote|1<{0} votes}): on by default. +channel.bindings.tw.details.severity.verified=Severity verified by DoesTheDogDie: {0} (least {1}). +channel.bindings.tw.details.severity.votes={0}% of the votes at {1} or above ({2} votes). +channel.bindings.tw.details.severity.below_verified=Severity verified by DoesTheDogDie: {0}, below the least {1}. +channel.bindings.tw.details.severity.below_votes=Only {0}% of the votes at {1} or above ({2} votes, mostly {3}). channel.bindings.reset=Reset to default channel.bindings.reset.title=Reset this binding to default? channel.bindings.reset.prompt=Its category, CCLs and TWs go back to automatic detection, its extensions are emptied and its switches turned back on. @@ -278,6 +305,8 @@ channel.settings.stream_title.preview=Preview: channel.settings.stream_title.preview_none=(title left as it is) channel.settings.fallback_title=Stream title with this category channel.settings.fallback_title_helper={{GAME}}: the game detected, {{CATEGORY}}: its Twitch category, {{TITLE}}: the title just before, {{ORIGINAL_TITLE}}: your own title, before Catapult changed it. Empty: the title for every game +channel.title.helper_conditions=Conditions: {{HAS_TW ? [[if it holds]] : [[otherwise]]}} with IS_NO_GAME, HAS_GAME, IS_CUSTOM, IS_INCOMPLETE, HAS_TW or IS_MATURE, detailed in the help. +channel.title.helper_drops=Drops: {{DROP_NAME}}: the running campaign's name, {{DROP_END}}: its end date (dd/MM), DROP_ENABLED: holds during a campaign. channel.settings.danger.title=Danger zone channel.settings.danger.description=Deleting your account is irreversible after {0} days. channel.settings.danger.delete_button=Delete my account @@ -442,6 +471,74 @@ admin.caches.entry.title=Entry {0} admin.caches.entry.expires_in=Expires in {0} admin.caches.entry.expired_since=Expired {0} ago admin.caches.entry.not_found=No stored record for this entry. +admin.dtdd-caches.hint=What Catapult keeps of DoesTheDogDie: each title's topics, the titles found and the searches made. An entry older than its cache's lifetime is refreshed the next time it is read, or here. +admin.dtdd-caches.kind.topics=Topics +admin.dtdd-caches.kind.games=Titles +admin.dtdd-caches.kind.searches=Searches +admin.dtdd-caches.kind.summary={0,choice,0#No entry|1#1 entry|1<{0} entries} · {1,choice,0#none stale|1#1 stale|1<{1} stale} · lifetime {2} +admin.dtdd-caches.lifetime.days={0} d +admin.dtdd-caches.lifetime.hours={0} h +admin.dtdd-caches.job.title=Refetch everything +admin.dtdd-caches.job.hint=Fetches every entry again, the oldest first, under DTDD''s limit per minute and leaving each key {0} requests of its month for the detection. It stops by itself once no key keeps them. +admin.dtdd-caches.job.state.IDLE=Never run +admin.dtdd-caches.job.state.RUNNING=Running +admin.dtdd-caches.job.state.DONE=Done +admin.dtdd-caches.job.state.STOPPED=Stopped +admin.dtdd-caches.job.state.CANCELLED=Cancelled +admin.dtdd-caches.job.progress={0} / {1} entries · {2,choice,0#none unanswered|1#1 unanswered|1<{2} unanswered} +admin.dtdd-caches.job.waiting=Waiting for a key until {0} +admin.dtdd-caches.job.stop.QUOTA=No key keeps its monthly reserve any more. +admin.dtdd-caches.job.stop.NO_KEY=No DTDD key in the pool. +admin.dtdd-caches.job.stop.ERROR=An error interrupted it (see catapult-api's logs). +admin.dtdd-caches.job.finished=Finished on {0} +admin.dtdd-caches.job.start=Refetch everything +admin.dtdd-caches.job.cancel=Stop +admin.dtdd-caches.job.start.confirm=Refetch every DTDD cache? +admin.dtdd-caches.job.start.description=Each entry costs a DTDD request. The refetch leaves each key its monthly reserve and stops by itself before touching it. +admin.dtdd-caches.quotas.title=Keys' quotas +admin.dtdd-caches.quotas.none=No DTDD key in the pool. +admin.dtdd-caches.quotas.unknown=not read yet +admin.dtdd-caches.quotas.rest=resting until {0} +admin.dtdd-caches.quotas.low=reserve reached +admin.dtdd-caches.column.key=Key +admin.dtdd-caches.column.minute=Left this minute +admin.dtdd-caches.column.month=Left this month +admin.dtdd-caches.column.at=Read on +admin.dtdd-caches.column.id=ID +admin.dtdd-caches.column.title=Title +admin.dtdd-caches.column.yes=Yes +admin.dtdd-caches.column.no=No +admin.dtdd-caches.column.mostly=Mixed +admin.dtdd-caches.column.type=Type +admin.dtdd-caches.column.query=Query +admin.dtdd-caches.column.found=Titles found +admin.dtdd-caches.column.fetched=Fetched on +admin.dtdd-caches.stale=Stale +admin.dtdd-caches.refetch=Refetch +admin.dtdd-caches.refetched=Entry fetched again. +admin.dtdd-caches.detail=See the whole entry +admin.dtdd-caches.detail.not_found=This entry is no longer in the cache. +admin.dtdd-caches.detail.title=Title {0} +admin.dtdd-caches.detail.poster=Poster +admin.dtdd-caches.detail.game_fetched=Record fetched on +admin.dtdd-caches.detail.not_cached=not cached +admin.dtdd-caches.detail.found_by=Found by the searches +admin.dtdd-caches.detail.no_search=No cached search found it. +admin.dtdd-caches.detail.searched=Searched on +admin.dtdd-caches.detail.nothing_found=The search found nothing. +admin.dtdd-caches.detail.topics_cached=topics cached +admin.dtdd-caches.detail.topics_not_cached=topics not cached +admin.dtdd-caches.severity.hint=Severity as DTDD's members voted it: each level's share of the votes (the votes), the reviewers' level in bold. +admin.dtdd-caches.severity.not_kept=Severity not cached yet: refetch the entry to get it. +admin.dtdd-caches.severity.none=no levels +admin.dtdd-caches.severity.verified=verified: {0} +admin.dtdd-caches.severity.level={0} {1}% ({2}) +admin.dtdd-caches.empty=This cache is empty. +admin.dtdd-caches.error.400=That is not a DTDD title ID. +admin.dtdd-caches.error.404=This entry is no longer in the cache. +admin.dtdd-caches.error.409=A full refetch is running already. +admin.dtdd-caches.error.502=DTDD gave nothing: an unknown title, no usable key left, or DTDD unavailable. +admin.dtdd-caches.error.503=DTDD is disabled. admin.data.repos={0,choice,0#No repository|1#1 repository|1<{0} repositories} admin.data.rows={0,choice,0#No row|1#1 row|1<{0} rows} admin.data.none=No repository. @@ -633,7 +730,43 @@ admin.dtdd-mapping.conflict=Another admin already resolved this proposal. admin.dtdd-mapping.unknown=This proposal no longer exists. admin.dtdd-mapping.unavailable=The proposals are unavailable right now. admin.tab.game-mappings=Game mappings -admin.game-mappings.hint=Sets the Twitch category of a detected game for every streamer. A streamer's own choice still comes first; the games without a mapping keep IGDB's. +admin.tab.dtdd-caches=DTDD caches +admin.tab.rate-limits=Rate limits +admin.rate-limits.hint=What Catapult guessed from the 429s of the platforms that don't document their limit: each host's budget, shared by its routes unless one shows a limit of its own, and how long to wait before calling again. Live calls pass first; bulk work (TW recalculation) keeps a margin under the limit. +admin.rate-limits.none=No platform limited Catapult yet. +admin.rate-limits.column.scope=Host / route +admin.rate-limits.column.limit=Limit +admin.rate-limits.column.resume=Resume after +admin.rate-limits.column.rate_limited=429s +admin.rate-limits.column.used=Calls in the window +admin.rate-limits.scope.HOST=Host's budget +admin.rate-limits.scope.ROUTE=Its own limit +admin.rate-limits.scope.pinned=chosen by an admin +admin.rate-limits.scope.learned=Learned +admin.rate-limits.limit.none=none known +admin.rate-limits.limit={0} per {1} +admin.rate-limits.window.10=10 s +admin.rate-limits.window.60=minute +admin.rate-limits.window.300=5 min +admin.rate-limits.window.3600=hour +admin.rate-limits.corrected=corrected +admin.rate-limits.seconds={0} s +admin.rate-limits.last=last on {0} +admin.rate-limits.blocked=waiting until {0} +admin.rate-limits.edit=Correct +admin.rate-limits.edit.hint=An empty field is left to the learning; what you set stays until the limit is reset. +admin.rate-limits.field.calls=Calls +admin.rate-limits.field.window=Per +admin.rate-limits.field.resume=Resume after (seconds) +admin.rate-limits.field.scope=Scope +admin.rate-limits.save=Save +admin.rate-limits.saved=Correction saved. +admin.rate-limits.reset=Reset +admin.rate-limits.reset.confirm=Reset this limit? +admin.rate-limits.reset.description=What was learned and corrected is forgotten: Catapult starts again without a known limit. +admin.rate-limits.error.400=Invalid correction: the calls and their window go together, and every value is at least 1. +admin.rate-limits.error.404=This limit is no longer known. +admin.game-mappings.hint=Sets the Twitch category of a detected game for every streamer, or ignores it by default (a wallpaper app such as Wallpaper Engine). A streamer's own choice still comes first; the games without a mapping keep IGDB's. admin.game-mappings.new=New mapping admin.game-mappings.empty=No game mapping yet. admin.game-mappings.column.platform=Platform @@ -673,6 +806,9 @@ admin.game-mappings.form.ccls=Content classification labels admin.game-mappings.form.tws=Trigger warnings admin.game-mappings.form.override=Replace IGDB's admin.game-mappings.form.no-choice=Nothing to choose from right now. +admin.game-mappings.ignored=Ignored by default +admin.game-mappings.form.ignored=Ignore this game by default +admin.game-mappings.form.ignored-helper=Its bindings are ignored (the streamer's "no game" category), but for the streamers who chose themselves whether to ignore it. The Twitch category becomes optional: without one, they keep IGDB's. admin.game-mappings.form.apply-live=Apply to the streams live now admin.game-mappings.form.save=Save admin.game-mappings.form.close=Close @@ -682,7 +818,7 @@ admin.game-mappings.error.400=Invalid mapping: check the platform, the ID and th admin.game-mappings.error.404=This mapping no longer exists. admin.game-mappings.error.409=This game already has a mapping. admin.game-mappings.error.id=Enter the game's ID. -admin.game-mappings.error.category=Pick a Twitch category from the search. +admin.game-mappings.error.category=Pick a Twitch category from the search, or ignore the game by default. admin.game-mappings.unavailable=The game mappings are unavailable right now. admin.game-mappings.delete.title=Delete the mapping of {0}? admin.game-mappings.delete.description=The bindings that follow it go back to IGDB's category. @@ -908,12 +1044,35 @@ admin.tw.rebuild.force-pinned.button=Force-rebuild ALL (overwrites pinned) admin.tw.rebuild.force-pinned.tooltip=Re-runs the TW resolution for every eligible binding, including those a streamer pinned by hand — their choice is overwritten. Asynchronous, see the logs. admin.tw.rebuild.force-pinned.confirm=This will OVERWRITE the TW choices streamers pinned by hand, for every binding. It cannot be undone automatically. Continue? admin.tw.action.manage-keywords=Manage keywords +admin.tw.action.manage-dtdd=DTDD topics +admin.tw.dtdd.title=DTDD topics of {0} +admin.tw.dtdd.hint=A topic marked yes or mostly on DTDD turns this TW on. With a least severe level, only when DTDD rates it at least that severe: its reviewers' level, else at least half of its members' votes. With too few votes, it still does. The levels offered are those of the cached titles. +admin.tw.dtdd.empty=No DTDD topic. +admin.tw.dtdd.min=Least severe level +admin.tw.dtdd.any=Any level +admin.tw.dtdd.remove=Remove the topic +admin.tw.dtdd.add.label=DTDD topic +admin.tw.dtdd.add.placeholder=there's blood/gore +admin.tw.dtdd.saved=DTDD topics saved. +admin.tw.dtdd.invalid=A topic must be 1 to 200 characters long. admin.tw.invalid=Invalid identifier (a-z, 0-9, _; 40 max.) or label (80 max.). admin.tw.conflict=This identifier already exists. admin.tw.unavailable=The trigger warnings are unavailable right now. admin.tw.disabled=Trigger warnings are disabled (tw.enabled=false). admin.tw.created=Trigger warning added. admin.tw.rebuild-started=Rebuild started, it continues in the background. +admin.tw.rebuild.running=A TW recalculation is running already. +admin.tw.rebuild.state.RUNNING=Recalculating the TWs +admin.tw.rebuild.state.DONE=TW recalculation done +admin.tw.rebuild.state.CANCELLED=TW recalculation stopped +admin.tw.rebuild.mode.BACKFILL=at start, games without TWs +admin.tw.rebuild.mode.UNTOUCHED=games without TWs +admin.tw.rebuild.mode.NON_OVERRIDDEN=games their streamer didn't change +admin.tw.rebuild.mode.FORCE_PINNED=every game, the streamers' choices included +admin.tw.rebuild.progress={0} / {1} games · {2} recalculated, {3} skipped, {4} failed +admin.tw.rebuild.waiting=Steam is rate limiting: going on at {0}. +admin.tw.rebuild.finished=Finished on {0} +admin.tw.rebuild.cancel=Stop admin.tw.kw.title=Trigger warning — {0} admin.tw.kw.empty=No Steam keyword defined for this trigger warning. admin.tw.kw.add.label=New keyword diff --git a/catapult-web/src/main/resources/lang/messages_en.properties b/catapult-web/src/main/resources/lang/messages_en.properties index 497f01e99..21c611b9a 100644 --- a/catapult-web/src/main/resources/lang/messages_en.properties +++ b/catapult-web/src/main/resources/lang/messages_en.properties @@ -208,6 +208,33 @@ channel.bindings.title.label=Stream title for this game channel.bindings.title.helper={{GAME}}: the game detected, {{CATEGORY}}: its Twitch category, {{TITLE}}: the title just before, {{ORIGINAL_TITLE}}: your own title, before Catapult changed it. Empty: the title set in Settings channel.bindings.tw.enabled=Enable trigger warnings channel.bindings.tw.reset=Reset +channel.bindings.tw.details=Details +channel.bindings.tw.details.title=Why these TWs? +channel.bindings.tw.details.hint=What the detection finds for this game right now, source by source. +channel.bindings.tw.details.streamer_chose=You changed this game's TWs: your choices apply, the detection no longer changes them until you reset them. +channel.bindings.tw.details.none=No TW for this game. +channel.bindings.tw.details.unavailable=The details aren't available right now. +channel.bindings.tw.details.turned_off=turned off by you +channel.bindings.tw.details.origin.streamer=Added by you. +channel.bindings.tw.details.origin.admin=Set by a Catapult admin for this game. +channel.bindings.tw.details.origin.no_source=Nothing suggests it any more: it goes at the next recomputation. +channel.bindings.tw.details.kept_off=Kept off by their severity +channel.bindings.tw.details.source.igdb=IGDB: content descriptor #{0} +channel.bindings.tw.details.source.steam_content=Steam: {0} +channel.bindings.tw.details.source.steam_keyword=Steam: the content notes mention “{0}” +channel.bindings.tw.details.source.dtdd=DoesTheDogDie: “{0}” +channel.bindings.tw.details.steam.1=some nudity or sexual content +channel.bindings.tw.details.steam.2=frequent violence or gore +channel.bindings.tw.details.steam.3=adult only sexual content +channel.bindings.tw.details.steam.4=frequent nudity or sexual content +channel.bindings.tw.details.steam.5=general mature content +channel.bindings.tw.details.severity.not_cached=Severity not known yet: on by default. +channel.bindings.tw.details.severity.unknown_level=The least severe level “{0}” doesn''t exist for this topic: on by default. +channel.bindings.tw.details.severity.too_few_votes=Too few votes to judge its severity ({0,choice,0#no vote|1#1 vote|1<{0} votes}): on by default. +channel.bindings.tw.details.severity.verified=Severity verified by DoesTheDogDie: {0} (least {1}). +channel.bindings.tw.details.severity.votes={0}% of the votes at {1} or above ({2} votes). +channel.bindings.tw.details.severity.below_verified=Severity verified by DoesTheDogDie: {0}, below the least {1}. +channel.bindings.tw.details.severity.below_votes=Only {0}% of the votes at {1} or above ({2} votes, mostly {3}). channel.bindings.reset=Reset to default channel.bindings.reset.title=Reset this binding to default? channel.bindings.reset.prompt=Its category, CCLs and TWs go back to automatic detection, its extensions are emptied and its switches turned back on. @@ -278,6 +305,8 @@ channel.settings.stream_title.preview=Preview: channel.settings.stream_title.preview_none=(title left as it is) channel.settings.fallback_title=Stream title with this category channel.settings.fallback_title_helper={{GAME}}: the game detected, {{CATEGORY}}: its Twitch category, {{TITLE}}: the title just before, {{ORIGINAL_TITLE}}: your own title, before Catapult changed it. Empty: the title for every game +channel.title.helper_conditions=Conditions: {{HAS_TW ? [[if it holds]] : [[otherwise]]}} with IS_NO_GAME, HAS_GAME, IS_CUSTOM, IS_INCOMPLETE, HAS_TW or IS_MATURE, detailed in the help. +channel.title.helper_drops=Drops: {{DROP_NAME}}: the running campaign's name, {{DROP_END}}: its end date (dd/MM), DROP_ENABLED: holds during a campaign. channel.settings.danger.title=Danger zone channel.settings.danger.description=Deleting your account is irreversible after {0} days. channel.settings.danger.delete_button=Delete my account @@ -442,6 +471,74 @@ admin.caches.entry.title=Entry {0} admin.caches.entry.expires_in=Expires in {0} admin.caches.entry.expired_since=Expired {0} ago admin.caches.entry.not_found=No stored record for this entry. +admin.dtdd-caches.hint=What Catapult keeps of DoesTheDogDie: each title's topics, the titles found and the searches made. An entry older than its cache's lifetime is refreshed the next time it is read, or here. +admin.dtdd-caches.kind.topics=Topics +admin.dtdd-caches.kind.games=Titles +admin.dtdd-caches.kind.searches=Searches +admin.dtdd-caches.kind.summary={0,choice,0#No entry|1#1 entry|1<{0} entries} · {1,choice,0#none stale|1#1 stale|1<{1} stale} · lifetime {2} +admin.dtdd-caches.lifetime.days={0} d +admin.dtdd-caches.lifetime.hours={0} h +admin.dtdd-caches.job.title=Refetch everything +admin.dtdd-caches.job.hint=Fetches every entry again, the oldest first, under DTDD''s limit per minute and leaving each key {0} requests of its month for the detection. It stops by itself once no key keeps them. +admin.dtdd-caches.job.state.IDLE=Never run +admin.dtdd-caches.job.state.RUNNING=Running +admin.dtdd-caches.job.state.DONE=Done +admin.dtdd-caches.job.state.STOPPED=Stopped +admin.dtdd-caches.job.state.CANCELLED=Cancelled +admin.dtdd-caches.job.progress={0} / {1} entries · {2,choice,0#none unanswered|1#1 unanswered|1<{2} unanswered} +admin.dtdd-caches.job.waiting=Waiting for a key until {0} +admin.dtdd-caches.job.stop.QUOTA=No key keeps its monthly reserve any more. +admin.dtdd-caches.job.stop.NO_KEY=No DTDD key in the pool. +admin.dtdd-caches.job.stop.ERROR=An error interrupted it (see catapult-api's logs). +admin.dtdd-caches.job.finished=Finished on {0} +admin.dtdd-caches.job.start=Refetch everything +admin.dtdd-caches.job.cancel=Stop +admin.dtdd-caches.job.start.confirm=Refetch every DTDD cache? +admin.dtdd-caches.job.start.description=Each entry costs a DTDD request. The refetch leaves each key its monthly reserve and stops by itself before touching it. +admin.dtdd-caches.quotas.title=Keys' quotas +admin.dtdd-caches.quotas.none=No DTDD key in the pool. +admin.dtdd-caches.quotas.unknown=not read yet +admin.dtdd-caches.quotas.rest=resting until {0} +admin.dtdd-caches.quotas.low=reserve reached +admin.dtdd-caches.column.key=Key +admin.dtdd-caches.column.minute=Left this minute +admin.dtdd-caches.column.month=Left this month +admin.dtdd-caches.column.at=Read on +admin.dtdd-caches.column.id=ID +admin.dtdd-caches.column.title=Title +admin.dtdd-caches.column.yes=Yes +admin.dtdd-caches.column.no=No +admin.dtdd-caches.column.mostly=Mixed +admin.dtdd-caches.column.type=Type +admin.dtdd-caches.column.query=Query +admin.dtdd-caches.column.found=Titles found +admin.dtdd-caches.column.fetched=Fetched on +admin.dtdd-caches.stale=Stale +admin.dtdd-caches.refetch=Refetch +admin.dtdd-caches.refetched=Entry fetched again. +admin.dtdd-caches.detail=See the whole entry +admin.dtdd-caches.detail.not_found=This entry is no longer in the cache. +admin.dtdd-caches.detail.title=Title {0} +admin.dtdd-caches.detail.poster=Poster +admin.dtdd-caches.detail.game_fetched=Record fetched on +admin.dtdd-caches.detail.not_cached=not cached +admin.dtdd-caches.detail.found_by=Found by the searches +admin.dtdd-caches.detail.no_search=No cached search found it. +admin.dtdd-caches.detail.searched=Searched on +admin.dtdd-caches.detail.nothing_found=The search found nothing. +admin.dtdd-caches.detail.topics_cached=topics cached +admin.dtdd-caches.detail.topics_not_cached=topics not cached +admin.dtdd-caches.severity.hint=Severity as DTDD's members voted it: each level's share of the votes (the votes), the reviewers' level in bold. +admin.dtdd-caches.severity.not_kept=Severity not cached yet: refetch the entry to get it. +admin.dtdd-caches.severity.none=no levels +admin.dtdd-caches.severity.verified=verified: {0} +admin.dtdd-caches.severity.level={0} {1}% ({2}) +admin.dtdd-caches.empty=This cache is empty. +admin.dtdd-caches.error.400=That is not a DTDD title ID. +admin.dtdd-caches.error.404=This entry is no longer in the cache. +admin.dtdd-caches.error.409=A full refetch is running already. +admin.dtdd-caches.error.502=DTDD gave nothing: an unknown title, no usable key left, or DTDD unavailable. +admin.dtdd-caches.error.503=DTDD is disabled. admin.data.repos={0,choice,0#No repository|1#1 repository|1<{0} repositories} admin.data.rows={0,choice,0#No row|1#1 row|1<{0} rows} admin.data.none=No repository. @@ -633,7 +730,43 @@ admin.dtdd-mapping.conflict=Another admin already resolved this proposal. admin.dtdd-mapping.unknown=This proposal no longer exists. admin.dtdd-mapping.unavailable=The proposals are unavailable right now. admin.tab.game-mappings=Game mappings -admin.game-mappings.hint=Sets the Twitch category of a detected game for every streamer. A streamer's own choice still comes first; the games without a mapping keep IGDB's. +admin.tab.dtdd-caches=DTDD caches +admin.tab.rate-limits=Rate limits +admin.rate-limits.hint=What Catapult guessed from the 429s of the platforms that don't document their limit: each host's budget, shared by its routes unless one shows a limit of its own, and how long to wait before calling again. Live calls pass first; bulk work (TW recalculation) keeps a margin under the limit. +admin.rate-limits.none=No platform limited Catapult yet. +admin.rate-limits.column.scope=Host / route +admin.rate-limits.column.limit=Limit +admin.rate-limits.column.resume=Resume after +admin.rate-limits.column.rate_limited=429s +admin.rate-limits.column.used=Calls in the window +admin.rate-limits.scope.HOST=Host's budget +admin.rate-limits.scope.ROUTE=Its own limit +admin.rate-limits.scope.pinned=chosen by an admin +admin.rate-limits.scope.learned=Learned +admin.rate-limits.limit.none=none known +admin.rate-limits.limit={0} per {1} +admin.rate-limits.window.10=10 s +admin.rate-limits.window.60=minute +admin.rate-limits.window.300=5 min +admin.rate-limits.window.3600=hour +admin.rate-limits.corrected=corrected +admin.rate-limits.seconds={0} s +admin.rate-limits.last=last on {0} +admin.rate-limits.blocked=waiting until {0} +admin.rate-limits.edit=Correct +admin.rate-limits.edit.hint=An empty field is left to the learning; what you set stays until the limit is reset. +admin.rate-limits.field.calls=Calls +admin.rate-limits.field.window=Per +admin.rate-limits.field.resume=Resume after (seconds) +admin.rate-limits.field.scope=Scope +admin.rate-limits.save=Save +admin.rate-limits.saved=Correction saved. +admin.rate-limits.reset=Reset +admin.rate-limits.reset.confirm=Reset this limit? +admin.rate-limits.reset.description=What was learned and corrected is forgotten: Catapult starts again without a known limit. +admin.rate-limits.error.400=Invalid correction: the calls and their window go together, and every value is at least 1. +admin.rate-limits.error.404=This limit is no longer known. +admin.game-mappings.hint=Sets the Twitch category of a detected game for every streamer, or ignores it by default (a wallpaper app such as Wallpaper Engine). A streamer's own choice still comes first; the games without a mapping keep IGDB's. admin.game-mappings.new=New mapping admin.game-mappings.empty=No game mapping yet. admin.game-mappings.column.platform=Platform @@ -673,6 +806,9 @@ admin.game-mappings.form.ccls=Content classification labels admin.game-mappings.form.tws=Trigger warnings admin.game-mappings.form.override=Replace IGDB's admin.game-mappings.form.no-choice=Nothing to choose from right now. +admin.game-mappings.ignored=Ignored by default +admin.game-mappings.form.ignored=Ignore this game by default +admin.game-mappings.form.ignored-helper=Its bindings are ignored (the streamer's "no game" category), but for the streamers who chose themselves whether to ignore it. The Twitch category becomes optional: without one, they keep IGDB's. admin.game-mappings.form.apply-live=Apply to the streams live now admin.game-mappings.form.save=Save admin.game-mappings.form.close=Close @@ -682,7 +818,7 @@ admin.game-mappings.error.400=Invalid mapping: check the platform, the ID and th admin.game-mappings.error.404=This mapping no longer exists. admin.game-mappings.error.409=This game already has a mapping. admin.game-mappings.error.id=Enter the game's ID. -admin.game-mappings.error.category=Pick a Twitch category from the search. +admin.game-mappings.error.category=Pick a Twitch category from the search, or ignore the game by default. admin.game-mappings.unavailable=The game mappings are unavailable right now. admin.game-mappings.delete.title=Delete the mapping of {0}? admin.game-mappings.delete.description=The bindings that follow it go back to IGDB's category. @@ -908,12 +1044,35 @@ admin.tw.rebuild.force-pinned.button=Force-rebuild ALL (overwrites pinned) admin.tw.rebuild.force-pinned.tooltip=Re-runs the TW resolution for every eligible binding, including those a streamer pinned by hand — their choice is overwritten. Asynchronous, see the logs. admin.tw.rebuild.force-pinned.confirm=This will OVERWRITE the TW choices streamers pinned by hand, for every binding. It cannot be undone automatically. Continue? admin.tw.action.manage-keywords=Manage keywords +admin.tw.action.manage-dtdd=DTDD topics +admin.tw.dtdd.title=DTDD topics of {0} +admin.tw.dtdd.hint=A topic marked yes or mostly on DTDD turns this TW on. With a least severe level, only when DTDD rates it at least that severe: its reviewers' level, else at least half of its members' votes. With too few votes, it still does. The levels offered are those of the cached titles. +admin.tw.dtdd.empty=No DTDD topic. +admin.tw.dtdd.min=Least severe level +admin.tw.dtdd.any=Any level +admin.tw.dtdd.remove=Remove the topic +admin.tw.dtdd.add.label=DTDD topic +admin.tw.dtdd.add.placeholder=there's blood/gore +admin.tw.dtdd.saved=DTDD topics saved. +admin.tw.dtdd.invalid=A topic must be 1 to 200 characters long. admin.tw.invalid=Invalid identifier (a-z, 0-9, _; 40 max.) or label (80 max.). admin.tw.conflict=This identifier already exists. admin.tw.unavailable=The trigger warnings are unavailable right now. admin.tw.disabled=Trigger warnings are disabled (tw.enabled=false). admin.tw.created=Trigger warning added. admin.tw.rebuild-started=Rebuild started, it continues in the background. +admin.tw.rebuild.running=A TW recalculation is running already. +admin.tw.rebuild.state.RUNNING=Recalculating the TWs +admin.tw.rebuild.state.DONE=TW recalculation done +admin.tw.rebuild.state.CANCELLED=TW recalculation stopped +admin.tw.rebuild.mode.BACKFILL=at start, games without TWs +admin.tw.rebuild.mode.UNTOUCHED=games without TWs +admin.tw.rebuild.mode.NON_OVERRIDDEN=games their streamer didn't change +admin.tw.rebuild.mode.FORCE_PINNED=every game, the streamers' choices included +admin.tw.rebuild.progress={0} / {1} games · {2} recalculated, {3} skipped, {4} failed +admin.tw.rebuild.waiting=Steam is rate limiting: going on at {0}. +admin.tw.rebuild.finished=Finished on {0} +admin.tw.rebuild.cancel=Stop admin.tw.kw.title=Trigger warning — {0} admin.tw.kw.empty=No Steam keyword defined for this trigger warning. admin.tw.kw.add.label=New keyword diff --git a/catapult-web/src/main/resources/lang/messages_fr.properties b/catapult-web/src/main/resources/lang/messages_fr.properties index 3318e9bb9..0e48e67af 100644 --- a/catapult-web/src/main/resources/lang/messages_fr.properties +++ b/catapult-web/src/main/resources/lang/messages_fr.properties @@ -208,6 +208,33 @@ channel.bindings.title.label=Titre du stream pour ce jeu channel.bindings.title.helper={{GAME}} : le jeu dĂ©tectĂ©, {{CATEGORY}} : sa catĂ©gorie Twitch, {{TITLE}} : le titre juste avant, {{ORIGINAL_TITLE}} : ton propre titre, avant que Catapult le change. Vide : le titre dĂ©fini dans les ParamĂštres channel.bindings.tw.enabled=Activer les avertissements channel.bindings.tw.reset=RĂ©initialiser +channel.bindings.tw.details=DĂ©tails +channel.bindings.tw.details.title=Pourquoi ces TW ? +channel.bindings.tw.details.hint=Ce que la dĂ©tection trouve pour ce jeu en ce moment, source par source. +channel.bindings.tw.details.streamer_chose=Tu as modifiĂ© les TW de ce jeu : ce sont tes choix qui s'appliquent, la dĂ©tection ne les change plus tant que tu ne les rĂ©initialises pas. +channel.bindings.tw.details.none=Aucun TW pour ce jeu. +channel.bindings.tw.details.unavailable=Les dĂ©tails ne sont pas disponibles pour le moment. +channel.bindings.tw.details.turned_off=dĂ©sactivĂ© par toi +channel.bindings.tw.details.origin.streamer=AjoutĂ© par toi. +channel.bindings.tw.details.origin.admin=FixĂ© par un admin de Catapult pour ce jeu. +channel.bindings.tw.details.origin.no_source=Plus aucune source ne le suggĂšre : il sera retirĂ© au prochain recalcul. +channel.bindings.tw.details.kept_off=ÉcartĂ©s par leur gravitĂ© +channel.bindings.tw.details.source.igdb=IGDB : descripteur de contenu n°{0} +channel.bindings.tw.details.source.steam_content=Steam : {0} +channel.bindings.tw.details.source.steam_keyword=Steam : la note de contenu mentionne « {0} » +channel.bindings.tw.details.source.dtdd=DoesTheDogDie : « {0} » +channel.bindings.tw.details.steam.1=un peu de nuditĂ© ou de contenu sexuel +channel.bindings.tw.details.steam.2=violence ou gore frĂ©quents +channel.bindings.tw.details.steam.3=contenu sexuel pour adultes +channel.bindings.tw.details.steam.4=nuditĂ© ou contenu sexuel frĂ©quents +channel.bindings.tw.details.steam.5=contenu pour adultes en gĂ©nĂ©ral +channel.bindings.tw.details.severity.not_cached=GravitĂ© pas encore connue : activĂ© par dĂ©faut. +channel.bindings.tw.details.severity.unknown_level=Le niveau minimum « {0} » n''existe pas pour ce sujet : activĂ© par dĂ©faut. +channel.bindings.tw.details.severity.too_few_votes=Pas assez de votes pour juger la gravitĂ© ({0,choice,0#aucun vote|1#1 vote|1<{0} votes}) : activĂ© par dĂ©faut. +channel.bindings.tw.details.severity.verified=GravitĂ© vĂ©rifiĂ©e par DoesTheDogDie : {0} (minimum {1}). +channel.bindings.tw.details.severity.votes={0} % des votes au niveau {1} ou plus ({2} votes). +channel.bindings.tw.details.severity.below_verified=GravitĂ© vĂ©rifiĂ©e par DoesTheDogDie : {0}, sous le minimum {1}. +channel.bindings.tw.details.severity.below_votes=Seulement {0} % des votes au niveau {1} ou plus ({2} votes, surtout {3}). channel.bindings.reset=Remettre par dĂ©faut channel.bindings.reset.title=Remettre ce binding par dĂ©faut ? channel.bindings.reset.prompt=Sa catĂ©gorie, ses CCL et ses TW reviennent Ă  la dĂ©tection automatique, ses extensions sont vidĂ©es et ses interrupteurs rĂ©activĂ©s. @@ -278,6 +305,8 @@ channel.settings.stream_title.preview=Aperçu : channel.settings.stream_title.preview_none=(titre laissĂ© tel quel) channel.settings.fallback_title=Titre du stream avec cette catĂ©gorie channel.settings.fallback_title_helper={{GAME}} : le jeu dĂ©tectĂ©, {{CATEGORY}} : sa catĂ©gorie Twitch, {{TITLE}} : le titre juste avant, {{ORIGINAL_TITLE}} : ton propre titre, avant que Catapult le change. Vide : le titre pour tous les jeux +channel.title.helper_conditions=Conditions : {{HAS_TW ? [[si vraie]] : [[sinon]]}} avec IS_NO_GAME, HAS_GAME, IS_CUSTOM, IS_INCOMPLETE, HAS_TW ou IS_MATURE, dĂ©taillĂ©es dans l'aide. +channel.title.helper_drops=Drops : {{DROP_NAME}} : le nom de la campagne en cours, {{DROP_END}} : sa date de fin (jj/MM), DROP_ENABLED : vraie pendant une campagne. channel.settings.danger.title=Zone danger channel.settings.danger.description=La suppression de compte est irrĂ©versible aprĂšs {0} jours. channel.settings.danger.delete_button=Supprimer mon compte @@ -442,6 +471,74 @@ admin.caches.entry.title=EntrĂ©e {0} admin.caches.entry.expires_in=Expire dans {0} admin.caches.entry.expired_since=ExpirĂ©e depuis {0} admin.caches.entry.not_found=Aucune donnĂ©e en base pour cette entrĂ©e. +admin.dtdd-caches.hint=Ce que Catapult garde de DoesTheDogDie : les sujets de chaque titre, les titres trouvĂ©s et les recherches faites. Une entrĂ©e plus vieille que la durĂ©e de vie de son cache est rafraĂźchie Ă  sa prochaine lecture, ou ici. +admin.dtdd-caches.kind.topics=Sujets +admin.dtdd-caches.kind.games=Titres +admin.dtdd-caches.kind.searches=Recherches +admin.dtdd-caches.kind.summary={0,choice,0#Aucune entrĂ©e|1#1 entrĂ©e|1<{0} entrĂ©es} · {1,choice,0#aucune pĂ©rimĂ©e|1#1 pĂ©rimĂ©e|1<{1} pĂ©rimĂ©es} · durĂ©e de vie {2} +admin.dtdd-caches.lifetime.days={0} j +admin.dtdd-caches.lifetime.hours={0} h +admin.dtdd-caches.job.title=Tout refetch +admin.dtdd-caches.job.hint=Refait chaque entrĂ©e, de la plus ancienne Ă  la plus rĂ©cente, sans dĂ©passer la limite par minute de DTDD et en laissant {0} requĂȘtes du mois Ă  chaque clĂ© pour la dĂ©tection. Il s''arrĂȘte de lui-mĂȘme quand plus aucune clĂ© ne les garde. +admin.dtdd-caches.job.state.IDLE=Jamais lancĂ© +admin.dtdd-caches.job.state.RUNNING=En cours +admin.dtdd-caches.job.state.DONE=TerminĂ© +admin.dtdd-caches.job.state.STOPPED=ArrĂȘtĂ© +admin.dtdd-caches.job.state.CANCELLED=AnnulĂ© +admin.dtdd-caches.job.progress={0} / {1} entrĂ©es · {2,choice,0#aucune sans rĂ©ponse|1#1 sans rĂ©ponse|1<{2} sans rĂ©ponse} +admin.dtdd-caches.job.waiting=En attente d''une clĂ© jusqu''Ă  {0} +admin.dtdd-caches.job.stop.QUOTA=Plus aucune clĂ© ne garde la rĂ©serve du mois. +admin.dtdd-caches.job.stop.NO_KEY=Aucune clĂ© DTDD dans le pool. +admin.dtdd-caches.job.stop.ERROR=Une erreur l'a interrompu (voir les logs de catapult-api). +admin.dtdd-caches.job.finished=Fini le {0} +admin.dtdd-caches.job.start=Tout refetch +admin.dtdd-caches.job.cancel=ArrĂȘter +admin.dtdd-caches.job.start.confirm=Refetch tous les caches DTDD ? +admin.dtdd-caches.job.start.description=Chaque entrĂ©e coĂ»te une requĂȘte Ă  DTDD. Le refetch laisse sa rĂ©serve du mois Ă  chaque clĂ© et s'arrĂȘte de lui-mĂȘme avant d'y toucher. +admin.dtdd-caches.quotas.title=Quotas des clĂ©s +admin.dtdd-caches.quotas.none=Aucune clĂ© DTDD dans le pool. +admin.dtdd-caches.quotas.unknown=pas encore relevĂ© +admin.dtdd-caches.quotas.rest=au repos jusqu''Ă  {0} +admin.dtdd-caches.quotas.low=rĂ©serve atteinte +admin.dtdd-caches.column.key=ClĂ© +admin.dtdd-caches.column.minute=Reste cette minute +admin.dtdd-caches.column.month=Reste ce mois +admin.dtdd-caches.column.at=RelevĂ© le +admin.dtdd-caches.column.id=ID +admin.dtdd-caches.column.title=Titre +admin.dtdd-caches.column.yes=Oui +admin.dtdd-caches.column.no=Non +admin.dtdd-caches.column.mostly=MitigĂ© +admin.dtdd-caches.column.type=Type +admin.dtdd-caches.column.query=RequĂȘte +admin.dtdd-caches.column.found=Titres trouvĂ©s +admin.dtdd-caches.column.fetched=RĂ©cupĂ©rĂ© le +admin.dtdd-caches.stale=PĂ©rimĂ© +admin.dtdd-caches.refetch=Refetch +admin.dtdd-caches.refetched=EntrĂ©e rĂ©cupĂ©rĂ©e Ă  nouveau. +admin.dtdd-caches.detail=Voir l'entrĂ©e complĂšte +admin.dtdd-caches.detail.not_found=Cette entrĂ©e n'est plus dans le cache. +admin.dtdd-caches.detail.title=Titre {0} +admin.dtdd-caches.detail.poster=Affiche +admin.dtdd-caches.detail.game_fetched=Fiche rĂ©cupĂ©rĂ©e le +admin.dtdd-caches.detail.not_cached=pas en cache +admin.dtdd-caches.detail.found_by=TrouvĂ© par les recherches +admin.dtdd-caches.detail.no_search=Aucune recherche en cache ne l'a trouvĂ©. +admin.dtdd-caches.detail.searched=RecherchĂ© le +admin.dtdd-caches.detail.nothing_found=La recherche n'a rien trouvĂ©. +admin.dtdd-caches.detail.topics_cached=sujets en cache +admin.dtdd-caches.detail.topics_not_cached=sujets pas en cache +admin.dtdd-caches.severity.hint=SĂ©vĂ©ritĂ© votĂ©e par les membres de DTDD : part des votes par niveau (nombre de votes), le niveau retenu par les relecteurs en gras. +admin.dtdd-caches.severity.not_kept=SĂ©vĂ©ritĂ© pas encore en cache : refetch l'entrĂ©e pour l'avoir. +admin.dtdd-caches.severity.none=sans niveaux +admin.dtdd-caches.severity.verified=vĂ©rifiĂ© : {0} +admin.dtdd-caches.severity.level={0} {1} % ({2}) +admin.dtdd-caches.empty=Ce cache est vide. +admin.dtdd-caches.error.400=Ce n'est pas un ID de titre DTDD. +admin.dtdd-caches.error.404=Cette entrĂ©e n'est plus dans le cache. +admin.dtdd-caches.error.409=Un refetch global tourne dĂ©jĂ . +admin.dtdd-caches.error.502=DTDD n'a rien renvoyĂ© : titre inconnu, plus de clĂ© utilisable ou DTDD indisponible. +admin.dtdd-caches.error.503=DTDD est dĂ©sactivĂ©. admin.data.repos={0,choice,0#Aucun dĂ©pĂŽt|1#1 dĂ©pĂŽt|1<{0} dĂ©pĂŽts} admin.data.rows={0,choice,0#Aucune ligne|1#1 ligne|1<{0} lignes} admin.data.none=Aucun dĂ©pĂŽt. @@ -633,7 +730,43 @@ admin.dtdd-mapping.conflict=Un autre administrateur a dĂ©jĂ  traitĂ© cette propo admin.dtdd-mapping.unknown=Cette proposition n'existe plus. admin.dtdd-mapping.unavailable=Les propositions sont indisponibles pour le moment. admin.tab.game-mappings=Mappings de jeu -admin.game-mappings.hint=DĂ©finit la catĂ©gorie Twitch d'un jeu dĂ©tectĂ© pour tous les streamers. Le choix d'un streamer reste prioritaire ; les jeux sans mapping gardent celle d'IGDB. +admin.tab.dtdd-caches=Caches DTDD +admin.tab.rate-limits=Limites de dĂ©bit +admin.rate-limits.hint=Ce que Catapult a devinĂ© des 429 des plateformes qui ne documentent pas leur limite : le budget de chaque hĂŽte, partagĂ© par ses routes sauf celles qui montrent une limite Ă  elles, et l'attente avant de rappeler. Les appels en live passent d'abord ; le travail de masse (recalcul des TW) garde une marge sous la limite. +admin.rate-limits.none=Aucune plateforme n'a encore limitĂ© Catapult. +admin.rate-limits.column.scope=HĂŽte / route +admin.rate-limits.column.limit=Limite +admin.rate-limits.column.resume=Reprise aprĂšs +admin.rate-limits.column.rate_limited=429 +admin.rate-limits.column.used=Appels dans la fenĂȘtre +admin.rate-limits.scope.HOST=Budget de l'hĂŽte +admin.rate-limits.scope.ROUTE=Limite propre +admin.rate-limits.scope.pinned=choisi par un admin +admin.rate-limits.scope.learned=Appris +admin.rate-limits.limit.none=aucune connue +admin.rate-limits.limit={0} par {1} +admin.rate-limits.window.10=10 s +admin.rate-limits.window.60=minute +admin.rate-limits.window.300=5 min +admin.rate-limits.window.3600=heure +admin.rate-limits.corrected=corrigĂ© +admin.rate-limits.seconds={0} s +admin.rate-limits.last=dernier le {0} +admin.rate-limits.blocked=en attente jusqu''Ă  {0} +admin.rate-limits.edit=Corriger +admin.rate-limits.edit.hint=Un champ vide est laissĂ© Ă  l'apprentissage ; ce que tu fixes reste jusqu'Ă  la remise Ă  zĂ©ro. +admin.rate-limits.field.calls=Appels +admin.rate-limits.field.window=Par +admin.rate-limits.field.resume=Reprise aprĂšs (secondes) +admin.rate-limits.field.scope=PortĂ©e +admin.rate-limits.save=Enregistrer +admin.rate-limits.saved=Correction enregistrĂ©e. +admin.rate-limits.reset=Remettre Ă  zĂ©ro +admin.rate-limits.reset.confirm=Remettre cette limite Ă  zĂ©ro ? +admin.rate-limits.reset.description=Ce qui a Ă©tĂ© appris et corrigĂ© est oubliĂ© : Catapult repart sans limite connue. +admin.rate-limits.error.400=Correction invalide : les appels vont avec leur fenĂȘtre, et chaque valeur vaut au moins 1. +admin.rate-limits.error.404=Cette limite n'est plus connue. +admin.game-mappings.hint=DĂ©finit la catĂ©gorie Twitch d'un jeu dĂ©tectĂ© pour tous les streamers, ou l'ignore par dĂ©faut (un fond d'Ă©cran comme Wallpaper Engine). Le choix d'un streamer reste prioritaire ; les jeux sans mapping gardent celle d'IGDB. admin.game-mappings.new=Nouveau mapping admin.game-mappings.empty=Aucun mapping de jeu. admin.game-mappings.column.platform=Plateforme @@ -673,6 +806,9 @@ admin.game-mappings.form.ccls=Classifications de contenu (CCL) admin.game-mappings.form.tws=Trigger warnings admin.game-mappings.form.override=Remplacer ceux d'IGDB admin.game-mappings.form.no-choice=Aucun choix disponible pour l'instant. +admin.game-mappings.ignored=IgnorĂ© par dĂ©faut +admin.game-mappings.form.ignored=Ignorer ce jeu par dĂ©faut +admin.game-mappings.form.ignored-helper=Ses bindings passent en ignorĂ© (la catĂ©gorie « aucun jeu » du streamer), sauf chez les streamers qui l'ont ignorĂ© ou non eux-mĂȘmes. La catĂ©gorie Twitch devient facultative : sans elle, ils gardent celle d'IGDB. admin.game-mappings.form.apply-live=Appliquer aux streams en cours admin.game-mappings.form.save=Enregistrer admin.game-mappings.form.close=Fermer @@ -682,7 +818,7 @@ admin.game-mappings.error.400=Mapping invalide : vĂ©rifiez la plateforme, l'ID e admin.game-mappings.error.404=Ce mapping n'existe plus. admin.game-mappings.error.409=Ce jeu a dĂ©jĂ  un mapping. admin.game-mappings.error.id=Saisissez l'ID du jeu. -admin.game-mappings.error.category=Choisissez une catĂ©gorie Twitch dans la recherche. +admin.game-mappings.error.category=Choisissez une catĂ©gorie Twitch dans la recherche, ou ignorez le jeu par dĂ©faut. admin.game-mappings.unavailable=Les mappings de jeu sont indisponibles pour l'instant. admin.game-mappings.delete.title=Supprimer le mapping de {0} ? admin.game-mappings.delete.description=Les associations qui le suivent reviennent Ă  la catĂ©gorie d'IGDB. @@ -908,12 +1044,35 @@ admin.tw.rebuild.force-pinned.button=Forcer TOUT (Ă©crase les figĂ©s) admin.tw.rebuild.force-pinned.tooltip=Relance la rĂ©solution TW pour tous les bindings Ă©ligibles, y compris ceux qu'un streamer a figĂ©s manuellement — Ă©crase leur choix. Asynchrone, voir les logs. admin.tw.rebuild.force-pinned.confirm=Cette action va ÉCRASER les choix TW figĂ©s manuellement par les streamers, pour tous les bindings. Impossible Ă  annuler automatiquement. Continuer ? admin.tw.action.manage-keywords=GĂ©rer les mots-clĂ©s +admin.tw.action.manage-dtdd=Sujets DTDD +admin.tw.dtdd.title=Sujets DTDD de {0} +admin.tw.dtdd.hint=Un sujet marquĂ© oui ou mitigĂ© sur DTDD active ce TW. Avec un niveau minimum, il ne l'active que si DTDD le juge au moins aussi grave : le niveau des relecteurs, sinon la moitiĂ© au moins des votes des membres. Avec trop peu de votes, il l'active quand mĂȘme. Les niveaux proposĂ©s sont ceux des titres en cache. +admin.tw.dtdd.empty=Aucun sujet DTDD. +admin.tw.dtdd.min=Niveau minimum +admin.tw.dtdd.any=Tout niveau +admin.tw.dtdd.remove=Retirer le sujet +admin.tw.dtdd.add.label=Sujet DTDD +admin.tw.dtdd.add.placeholder=there's blood/gore +admin.tw.dtdd.saved=Sujets DTDD enregistrĂ©s. +admin.tw.dtdd.invalid=Un sujet doit faire de 1 Ă  200 caractĂšres. admin.tw.invalid=Identifiant (a-z, 0-9, _ ; 40 max.) ou libellĂ© (80 max.) invalide. admin.tw.conflict=Cet identifiant existe dĂ©jĂ . admin.tw.unavailable=Les trigger warnings sont indisponibles pour le moment. admin.tw.disabled=Les trigger warnings sont dĂ©sactivĂ©s (tw.enabled=false). admin.tw.created=Trigger warning ajoutĂ©. admin.tw.rebuild-started=Recalcul lancĂ©, il se poursuit en arriĂšre-plan. +admin.tw.rebuild.running=Un recalcul des TW est dĂ©jĂ  en cours. +admin.tw.rebuild.state.RUNNING=Recalcul des TW en cours +admin.tw.rebuild.state.DONE=Recalcul des TW terminĂ© +admin.tw.rebuild.state.CANCELLED=Recalcul des TW arrĂȘtĂ© +admin.tw.rebuild.mode.BACKFILL=au dĂ©marrage, jeux sans TW +admin.tw.rebuild.mode.UNTOUCHED=jeux sans TW +admin.tw.rebuild.mode.NON_OVERRIDDEN=jeux non modifiĂ©s par leur streamer +admin.tw.rebuild.mode.FORCE_PINNED=tous les jeux, choix des streamers compris +admin.tw.rebuild.progress={0} / {1} jeux · {2} recalculĂ©s, {3} ignorĂ©s, {4} en Ă©chec +admin.tw.rebuild.waiting=Steam limite les requĂȘtes : reprise Ă  {0}. +admin.tw.rebuild.finished=Fini le {0} +admin.tw.rebuild.cancel=ArrĂȘter admin.tw.kw.title=Trigger warning — {0} admin.tw.kw.empty=Aucun mot-clĂ© Steam dĂ©fini pour ce trigger warning. admin.tw.kw.add.label=Nouveau mot-clĂ© diff --git a/catapult-web/src/main/resources/static/css/catapult.css b/catapult-web/src/main/resources/static/css/catapult.css index e11859f49..9e20e4cbd 100644 --- a/catapult-web/src/main/resources/static/css/catapult.css +++ b/catapult-web/src/main/resources/static/css/catapult.css @@ -2097,10 +2097,10 @@ span.box-art { * The tables below are one grid each, every row (header included) a subgrid of it: as separate grids, * the rows sized their action column to their own buttons, so no cell sat under its header. */ -.admin-table:has(> :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row)) { display: grid; column-gap: 12px; } -.admin-table:has(> :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row)) > * { grid-column: 1 / -1; } -.admin-table > :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row) { grid-template-columns: subgrid; } -.admin-table > :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row) > .admin-row-actions { justify-content: flex-end; } +.admin-table:has(> :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row, .admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row)) { display: grid; column-gap: 12px; } +.admin-table:has(> :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row, .admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row)) > * { grid-column: 1 / -1; } +.admin-table > :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row, .admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row) { grid-template-columns: subgrid; } +.admin-table > :is(.admin-dtdd-row, .admin-groups-row, .admin-whitelist-invite-row, .admin-whitelist-row, .admin-keys-row, .admin-minecraft-row, .admin-tw-row, .admin-meta-tag-row, .admin-social-link-row, .invite-row, .admin-gm-row, .admin-gm-missing-row, .admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row) > .admin-row-actions { justify-content: flex-end; } .admin-pagination { display: flex; @@ -2357,6 +2357,8 @@ mdui-snackbar.broadcast-warning { --mdui-color-inverse-surface: var(--mdui-color .admin-gm-row, .admin-table:has(> .admin-gm-row) { grid-template-columns: 7.5rem minmax(0, 1fr) minmax(0, 1.5fr) minmax(0, 1.5fr) minmax(0, 1fr) minmax(0, 1fr) 5.5rem 9rem auto; } .admin-gm-missing-row, .admin-table:has(> .admin-gm-missing-row) { grid-template-columns: 7.5rem minmax(0, 1fr) minmax(0, 2fr) 6rem auto; } .admin-gm-labels { min-width: 0; overflow-wrap: anywhere; } +.admin-gm-category { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 8px; } +.admin-gm-ignored-helper { margin: -8px 0 0; } .admin-gm-row > .admin-member-id { display: flex; flex-direction: column; } .admin-gm-form { display: flex; flex-direction: column; gap: 12px; min-width: min(560px, 100%); } .admin-gm-form [hidden] { display: none; } @@ -2368,6 +2370,33 @@ mdui-snackbar.broadcast-warning { --mdui-color-inverse-surface: var(--mdui-color .admin-gm-choices { display: flex; flex-wrap: wrap; gap: 0 16px; } .admin-gm-error { margin: 0; color: rgb(var(--mdui-color-error)); } .admin-gm-result { margin: 0; color: var(--catapult-text-secondary); } +.admin-dc-hint { margin: 0 0 16px; } +.admin-dc-job { display: flex; flex-direction: column; gap: 8px; margin-bottom: 16px; } +.admin-dc-job h3, .admin-dc-job p { margin: 0; } +.admin-dc-job-state { display: flex; flex-direction: column; gap: 6px; } +.admin-dc-stop { color: rgb(var(--mdui-color-error)); } +.admin-dc-quotas-header { margin-top: 24px; } +.admin-dc-topics-row, .admin-table:has(> .admin-dc-topics-row) { grid-template-columns: 6rem minmax(0, 1.2fr) minmax(0, 1.5fr) minmax(0, 1.5fr) minmax(0, 1fr) 10rem auto; } +.admin-dc-games-row, .admin-table:has(> .admin-dc-games-row) { grid-template-columns: 6rem minmax(0, 2fr) minmax(0, 1fr) 10rem auto; } +.admin-dc-searches-row, .admin-table:has(> .admin-dc-searches-row) { grid-template-columns: minmax(0, 2fr) minmax(0, 1.5fr) 10rem auto; } +.admin-dc-quota-row, .admin-table:has(> .admin-dc-quota-row) { grid-template-columns: minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.2fr) 10rem; } +.admin-dc-topics { min-width: 0; overflow-wrap: anywhere; } +.admin-dc-fetched { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 8px; } +.admin-dc-detail-section { margin: 16px 0 4px; } +.admin-dc-topic-lists { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; } +.admin-dc-topic-list h4 { margin: 0 0 4px; } +/* Wider than the other admin dialogs: three lists of topics, each with its severity, side by side */ +.admin-dc-dialog { --mdui-comp-dialog-max-width: min(1100px, 100vw - 32px); } +.admin-dc-detail-body { width: min(1040px, 100vw - 96px); max-width: 100%; } +.admin-dc-topic-items { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 10px; } +.admin-dc-topic-items > li { display: flex; flex-direction: column; align-items: flex-start; gap: 4px; } +.admin-dc-topic-name { font-weight: 500; overflow-wrap: anywhere; } +.admin-dc-levels { display: flex; flex-wrap: wrap; gap: 2px 10px; font-size: 0.875rem; color: var(--catapult-text-secondary); } +.admin-dc-level-verified { color: var(--catapult-text); font-weight: 600; } +.admin-dc-poster { max-width: 120px; border-radius: 8px; } +@media (max-width: 650px) { + .admin-dc-topic-lists { grid-template-columns: minmax(0, 1fr); } +} @media (max-width: 650px) { .admin-gm-row, .admin-table:has(> .admin-gm-row), .admin-gm-missing-row, .admin-table:has(> .admin-gm-missing-row) { grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); } @@ -2375,6 +2404,14 @@ mdui-snackbar.broadcast-warning { --mdui-color-inverse-surface: var(--mdui-color .admin-gm-row > .admin-row-actions, .admin-gm-missing-row > .admin-row-actions { grid-column: 1 / -1; } .admin-gm-row > .admin-cell, .admin-gm-missing-row > .admin-cell { white-space: normal; } } +@media (max-width: 650px) { + .admin-dc-topics-row, .admin-table:has(> .admin-dc-topics-row), + .admin-dc-games-row, .admin-table:has(> .admin-dc-games-row), + .admin-dc-searches-row, .admin-table:has(> .admin-dc-searches-row), + .admin-dc-quota-row, .admin-table:has(> .admin-dc-quota-row) { grid-template-columns: minmax(0, 1fr); } + :is(.admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row) > [data-label]::before { content: attr(data-label) " : "; color: var(--catapult-text-muted); } + :is(.admin-dc-topics-row, .admin-dc-games-row, .admin-dc-searches-row, .admin-dc-quota-row) > .admin-cell { white-space: normal; } +} /* The admin Groups tab. */ .admin-groups-create { padding: 16px; margin-bottom: 16px; } @@ -2514,6 +2551,11 @@ mdui-snackbar.broadcast-warning { --mdui-color-inverse-surface: var(--mdui-color .admin-tw-kw-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 12px; align-items: center; padding: 4px 0; } .admin-tw-kw-row + .admin-tw-kw-row { border-top: 1px solid var(--catapult-border, rgba(127, 127, 127, 0.2)); } .admin-tw-kw-row > .admin-member-id { min-width: 0; overflow-wrap: anywhere; } +.admin-tw-dtdd-row { grid-template-columns: minmax(0, 1fr) minmax(10rem, 14rem) auto; } +@media (max-width: 650px) { + .admin-tw-dtdd-row { grid-template-columns: minmax(0, 1fr) auto; } + .admin-tw-dtdd-row > .admin-tw-dtdd-min { grid-column: 1 / -1; grid-row: 2; } +} .admin-tw-kw-form { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 12px; align-items: start; } .admin-tw-test-result { margin-top: 12px; overflow-wrap: anywhere; } .admin-tw-test-result p { margin: 4px 0; } @@ -2898,3 +2940,15 @@ mdui-text-field[data-title-highlight].title-highlighted::part(input) { .binding-edit-panel .title-preview { grid-column: 1 / -1; } + +/* Why a binding has its TWs (channel page, its TW panel's details dialog). */ +.tw-details-dialog { --mdui-comp-dialog-max-width: min(760px, 100vw - 32px); } +.tw-details-body { width: min(700px, 100vw - 96px); max-width: 100%; } +.tw-details-hint { color: var(--catapult-text-secondary); margin: 0 0 12px; } +.tw-details-tw, .tw-details-kept-off { padding: 8px 0; } +.tw-details-tw + .tw-details-tw, .tw-details-kept-off { border-top: 1px solid var(--catapult-border, rgba(127, 127, 127, 0.2)); } +.tw-details-tw h4, .tw-details-kept-off h4 { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin: 0 0 4px; } +.tw-details-tw p { margin: 0 0 4px; color: var(--catapult-text-secondary); } +.tw-details-tw ul, .tw-details-kept-off ul { margin: 4px 0 0; padding-left: 20px; } +.tw-details-source { margin: 2px 0; overflow-wrap: anywhere; } +.tw-details-severity { display: block; color: var(--catapult-text-secondary); font-size: 0.875rem; } diff --git a/catapult-web/src/main/resources/static/js/admin-dtdd-caches.js b/catapult-web/src/main/resources/static/js/admin-dtdd-caches.js new file mode 100644 index 000000000..5bc664471 --- /dev/null +++ b/catapult-web/src/main/resources/static/js/admin-dtdd-caches.js @@ -0,0 +1,112 @@ +/** + * The admin DTDD caches tab. Its summary starts the bulk refetch once confirmed, or asks it to stop, and follows it + * while it runs: the summary is patched every few seconds (admin-tabs.js) as long as its job reads RUNNING and the + * tab is the one shown. A cache's entries each open in full in a dialog (what the caches keep about it), or fetch + * again on demand, then the results shown are patched (same search, same page). A refusal is told with the view's data-error-{status}, else its data-error ({0}: the status). + * Listeners are delegated to the panel: its content comes and goes. + */ +window.CatapultAdminDtddCaches = (function () { + const TAB = "dtdd-caches"; + const BASE = "/admin/dtdd-caches"; + const POLL_MS = 3000; + + let timer = null; + + const panelOf = () => document.querySelector(`mdui-tab-panel[value="${TAB}"]`); + + /** Whether the summary shown reads a running refetch, in the tab shown. */ + function following(panel) { + const job = panel.querySelector('.admin-dtdd-caches [data-admin-value="job"]'); + return job?.dataset.state === "RUNNING" && document.getElementById("admin-tabs")?.value === TAB; + } + + /** Patches the summary every POLL_MS while it reads a running refetch; stops by itself otherwise. */ + function follow(panel) { + clearTimeout(timer); + timer = null; + if (!following(panel)) return; + timer = setTimeout(async () => { + if (!panel.isConnected || !following(panel)) return; + await CatapultAdminTabs.patch(panel); + follow(panel); + }, POLL_MS); + } + + async function refetch(panel, root, button) { + const query = new URLSearchParams({ key: button.dataset.key }); + const sent = await CatapultAdminTabs.post(root, button, + `${BASE}/${encodeURIComponent(root.dataset.kind)}/refetch?${query}`, {}); + if (sent !== "ok") { + // gone meanwhile: the list as it is now + if (sent === 404) await CatapultAdminTabs.patch(panel); + return; + } + button.disabled = false; + window.mdui?.snackbar?.({ message: root.dataset.refetched }); + await CatapultAdminTabs.patch(panel); + } + + /** The entry in full, into the view's dialog; a failure is told. */ + async function detail(root, button) { + const dialog = root.querySelector("#admin-dc-dialog"); + const query = new URLSearchParams({ key: button.dataset.key }); + let response = null; + try { + response = await fetch(`/spa/admin/${TAB}/${encodeURIComponent(root.dataset.kind)}/entry?${query}`); + } catch { + // not fetched: told below + } + if (!response?.ok || !dialog) { + CatapultChannel.showError(CatapultAdminTabs.refusal(root, response?.status)); + return; + } + dialog.querySelector(".admin-dialog-body").innerHTML = await response.text(); + dialog.open = true; + } + + async function start(panel, root, button) { + if (!await CatapultAdminTabs.confirmed(root, root.dataset.startConfirm, root.dataset.startDescription)) return; + const sent = await CatapultAdminTabs.post(root, button, `${BASE}/refetch-all/start`, {}); + // one started elsewhere meanwhile: follow it all the same + if (sent !== "ok" && sent !== 409) return; + button.disabled = false; + await CatapultAdminTabs.patch(panel); + follow(panel); + } + + async function cancel(panel, root, button) { + if (await CatapultAdminTabs.post(root, button, `${BASE}/refetch-all/cancel`, {}) !== "ok") return; + button.disabled = false; + await CatapultAdminTabs.patch(panel); + } + + function attach() { + const panel = panelOf(); + if (!panel || panel.dataset.dtddCachesWired) return; + panel.dataset.dtddCachesWired = "true"; + + panel.addEventListener("click", event => { + const button = event.target.closest(".admin-dc-detail, .admin-dc-refetch, #admin-dc-start, #admin-dc-cancel, .admin-dialog-close"); + if (!button || button.disabled) return; + const root = button.closest(".admin-view"); + if (!root) return; + if (button.matches(".admin-dialog-close")) button.closest("mdui-dialog").open = false; + else if (button.matches(".admin-dc-detail")) detail(root, button); + else if (button.matches(".admin-dc-refetch")) refetch(panel, root, button); + else if (button.id === "admin-dc-start") start(panel, root, button); + else cancel(panel, root, button); + }); + // Back on the tab: a refetch running is followed again (left, it stopped being) + document.getElementById("admin-tabs")?.addEventListener("change", event => { + if (event.target.id === "admin-tabs") follow(panel); + }); + } + + document.addEventListener("catapult:render", attach); + // A view shown (the tab, back from a cache): a refetch running is followed again + document.addEventListener("catapult:admin-panel", event => { + if (event.detail?.tab === TAB) follow(event.detail.panel); + }); + + return { attach, follow }; +})(); diff --git a/catapult-web/src/main/resources/static/js/admin-game-mappings.js b/catapult-web/src/main/resources/static/js/admin-game-mappings.js index 386271aee..bac3db70f 100644 --- a/catapult-web/src/main/resources/static/js/admin-game-mappings.js +++ b/catapult-web/src/main/resources/static/js/admin-game-mappings.js @@ -8,7 +8,7 @@ * changes one (from its row). Its game field suggests the platform's detected games as the ID or name is typed * ("name — ID · Streamers: n"; picking one fills the ID, any ID typed is kept as is); its category field searches * Twitch through GameSearch. The CCLs and TWs stay IGDB's (null) unless their switch replaces them with the boxes - * checked (even none). After a save the form shows what changed and turns into the saved mapping's edit form. + * checked (even none). Its "ignored by default" switch makes the category optional. After a save the form shows what changed and turns into the saved mapping's edit form. * A deletion asks first, in its own dialog. A refusal is told by its status, never by catapult-api's message * (catapult-web relays none): 400 invalid, 404 gone, 409 already mapped, anything else unavailable. * Text from data only ever reaches the page through textContent. Listeners are delegated to the panel. @@ -125,6 +125,7 @@ window.CatapultAdminGameMappings = (function () { gameName: el.querySelector(".admin-gm-game-name"), suggestions: el.querySelector(".admin-gm-suggestions"), category: el.querySelector('[name="category"]'), + ignored: el.querySelector('[name="ignoredByDefault"]'), applyLive: el.querySelector('[name="applyLive"]'), error: el.querySelector(".admin-gm-error"), result: el.querySelector(".admin-gm-result"), @@ -240,7 +241,10 @@ window.CatapultAdminGameMappings = (function () { return [...checked, ...form.kept[key]]; } - /** Fills the form with `mapping` (`{ id?, sourceType, sourceId, sourceName, twitchGameId, twitchGameName, ccls, tws }`) and opens it. */ + /** + * Fills the form with `mapping` (`{ id?, sourceType, sourceId, sourceName, twitchGameId, twitchGameName, ccls, tws, + * ignoredByDefault? }`) and opens it. + */ function open(form, mapping) { form.id = mapping.id ?? null; form.title.textContent = form.root.dataset[form.id ? "labelEditTitle" : "labelCreateTitle"]; @@ -252,6 +256,7 @@ window.CatapultAdminGameMappings = (function () { form.search.select(mapping.twitchGameId ? { id: mapping.twitchGameId, name: mapping.twitchGameName } : null); setOverride(form, "ccls", mapping.ccls ?? null); setOverride(form, "tws", mapping.tws ?? null); + form.ignored.checked = mapping.ignoredByDefault === true; form.applyLive.checked = false; hideMessages(form); form.dialog.open = true; @@ -269,7 +274,8 @@ window.CatapultAdminGameMappings = (function () { twitchGameId: row.dataset.twitchGameId, twitchGameName: row.dataset.twitchGameName, ccls: list("ccls"), - tws: list("tws") + tws: list("tws"), + ignoredByDefault: row.hasAttribute("data-ignored-by-default") }; } @@ -277,16 +283,19 @@ window.CatapultAdminGameMappings = (function () { hideMessages(form); const sourceId = form.sourceId.value.trim(); const category = form.search.selected(); + const ignoredByDefault = form.ignored.checked; if (sourceId === "") return tell(form.error, form.root.dataset.labelErrorId); - if (!category) return tell(form.error, form.root.dataset.labelErrorCategory); + // A game ignored by default needs no category: its bindings keep IGDB's + if (!category && !ignoredByDefault) return tell(form.error, form.root.dataset.labelErrorCategory); const body = { sourceType: form.platform.value, sourceId, - twitchGameId: category.id, - twitchGameName: category.name, + twitchGameId: category?.id ?? null, + twitchGameName: category?.name ?? null, ccls: overrideOf(form, "ccls"), tws: overrideOf(form, "tws"), - applyLive: form.applyLive.checked + applyLive: form.applyLive.checked, + ignoredByDefault }; form.save.disabled = true; const { status, body: result } = form.id diff --git a/catapult-web/src/main/resources/static/js/admin-rate-limits.js b/catapult-web/src/main/resources/static/js/admin-rate-limits.js new file mode 100644 index 000000000..61ab2e354 --- /dev/null +++ b/catapult-web/src/main/resources/static/js/admin-rate-limits.js @@ -0,0 +1,86 @@ +/** + * The admin rate limits tab. A learned limit is corrected in the view's dialog, filled from its row (an empty field + * leaves that part to the learning; a host has no scope to choose), or reset once confirmed; the list is then patched + * (admin-tabs.js). A refusal is told with the view's data-error-{status}, else its data-error ({0}: the status). + * Listeners are delegated to the panel: its content comes and goes. + */ +window.CatapultAdminRateLimits = (function () { + const TAB = "rate-limits"; + const BASE = "/admin/rate-limits"; + + const panelOf = () => document.querySelector(`mdui-tab-panel[value="${TAB}"]`); + + /** A number typed in a field, null when left empty. */ + function numberOf(field) { + const value = String(field?.value ?? "").trim(); + return value === "" ? null : Number(value); + } + + /** The dialog filled from the row's correction (what an admin set; empty: learned). */ + function edit(root, row) { + const dialog = root.querySelector("#admin-rl-dialog"); + if (!dialog) return; + dialog.dataset.id = row.dataset.id; + dialog.querySelector(".admin-rl-dialog-id code").textContent = row.dataset.id; + dialog.querySelector('[name="calls"]').value = row.dataset.calls ?? ""; + dialog.querySelector('[name="window"]').value = row.dataset.window || "60"; + dialog.querySelector('[name="resume"]').value = row.dataset.resume ?? ""; + const scope = dialog.querySelector('[name="scope"]'); + scope.hidden = row.dataset.host === "true"; + scope.value = row.dataset.scopePinned === "true" ? row.dataset.scope : "LEARNED"; + dialog.open = true; + } + + /** The correction the dialog holds: a limit only with its calls, a scope only for a route (LEARNED: unpinned). */ + function correctionOf(dialog) { + const calls = numberOf(dialog.querySelector('[name="calls"]')); + const scope = dialog.querySelector('[name="scope"]'); + return { + limitCalls: calls, + limitWindowSeconds: calls === null ? null : numberOf(dialog.querySelector('[name="window"]')), + resumeSeconds: numberOf(dialog.querySelector('[name="resume"]')), + scope: scope.hidden || !scope.value ? null : scope.value + }; + } + + async function save(panel, root, button) { + const dialog = button.closest("mdui-dialog"); + const query = new URLSearchParams({ id: dialog.dataset.id }); + if (await CatapultAdminTabs.post(root, button, `${BASE}/correct?${query}`, correctionOf(dialog)) !== "ok") return; + button.disabled = false; + dialog.open = false; + window.mdui?.snackbar?.({ message: root.dataset.saved }); + await CatapultAdminTabs.patch(panel); + } + + async function reset(panel, root, button) { + if (!await CatapultAdminTabs.confirmed(root, root.dataset.resetConfirm, root.dataset.resetDescription)) return; + const query = new URLSearchParams({ id: button.closest("[data-id]").dataset.id }); + const sent = await CatapultAdminTabs.post(root, button, `${BASE}/reset?${query}`, {}); + // gone meanwhile: the list as it is now + if (sent !== "ok" && sent !== 404) return; + button.disabled = false; + await CatapultAdminTabs.patch(panel); + } + + function attach() { + const panel = panelOf(); + if (!panel || panel.dataset.rateLimitsWired) return; + panel.dataset.rateLimitsWired = "true"; + + panel.addEventListener("click", event => { + const button = event.target.closest(".admin-rl-edit, .admin-rl-reset, .admin-rl-save, .admin-dialog-close"); + if (!button || button.disabled) return; + const root = button.closest(".admin-view"); + if (!root) return; + if (button.matches(".admin-dialog-close")) button.closest("mdui-dialog").open = false; + else if (button.matches(".admin-rl-edit")) edit(root, button.closest("[data-id]")); + else if (button.matches(".admin-rl-save")) save(panel, root, button); + else reset(panel, root, button); + }); + } + + document.addEventListener("catapult:render", attach); + + return { attach, correctionOf }; +})(); diff --git a/catapult-web/src/main/resources/static/js/admin-tw-dtdd.js b/catapult-web/src/main/resources/static/js/admin-tw-dtdd.js new file mode 100644 index 000000000..a0f76ebe6 --- /dev/null +++ b/catapult-web/src/main/resources/static/js/admin-tw-dtdd.js @@ -0,0 +1,86 @@ +/** + * The admin Trigger warnings tab's DTDD topics sub-view (#tw/{id}/dtdd): any change (a least severe level picked, a + * topic removed or added) saves them all, in the order shown, to AdminTwController, then patches the list + * (admin-tabs.js). A topic is free text: kept in data-topic and sent in the body, never in a path. A refusal is told + * with the view's data-error-{status}, else its data-error ({0}: the status). + * Listeners are delegated to the panel: its content comes and goes. + */ +window.CatapultAdminTwDtdd = (function () { + const TOPIC_MAX = 200; + + const viewOf = element => element.closest(".admin-tw-dtdd-view"); + + /** The topics shown, each with its least severe level ("" any), plus `extra` when given. */ + function topicsOf(root, extra) { + const topics = Array.from(root.querySelectorAll(".admin-tw-dtdd-row")).map(row => ({ + topic: row.dataset.topic, + minSeverity: row.querySelector(".admin-tw-dtdd-min")?.value ?? "" + })); + if (extra) topics.push(extra); + return topics; + } + + /** Saves `topics` with `button` disabled meanwhile; patches the list once saved. */ + async function save(panel, root, button, topics) { + const sent = await CatapultAdminTabs.post(root, button, + `/admin/tw/${encodeURIComponent(root.dataset.twId)}/dtdd`, { topics }); + if (sent !== "ok") return false; + button.disabled = false; + window.mdui?.snackbar?.({ message: root.dataset.saved }); + await CatapultAdminTabs.patch(panel); + return true; + } + + async function add(panel, root, button) { + const field = root.querySelector('[name="topic"]'); + const topic = (field.value ?? "").trim(); + if (topic === "" || topic.length > TOPIC_MAX) { + field.setAttribute("invalid", ""); + field.reportValidity?.(); + return; + } + const known = topicsOf(root).some(t => t.topic.toLowerCase() === topic.toLowerCase()); + if (known) { + field.value = ""; + return; + } + if (await save(panel, root, button, topicsOf(root, { topic, minSeverity: "" }))) field.value = ""; + } + + function attach() { + const panel = document.querySelector('mdui-tab-panel[value="tw"]'); + if (!panel || panel.dataset.twDtddWired) return; + panel.dataset.twDtddWired = "true"; + + panel.addEventListener("click", event => { + const button = event.target.closest(".admin-tw-dtdd-add, .admin-tw-dtdd-remove"); + if (!button || button.disabled) return; + const root = viewOf(button); + if (!root) return; + if (button.matches(".admin-tw-dtdd-add")) { + add(panel, root, button); + return; + } + const row = button.closest(".admin-tw-dtdd-row"); + save(panel, root, button, topicsOf(root).filter(t => t.topic !== row.dataset.topic)); + }); + // mdui's select tells its change on itself + panel.addEventListener("change", event => { + const select = event.target.closest?.(".admin-tw-dtdd-min"); + if (!select) return; + const root = viewOf(select); + // disabled while it saves, as a button would be + if (root) save(panel, root, select, topicsOf(root)); + }); + panel.addEventListener("keydown", event => { + if (event.key !== "Enter" || !event.target.matches?.('.admin-tw-dtdd-view [name="topic"]')) return; + event.preventDefault(); + const root = viewOf(event.target); + add(panel, root, root.querySelector(".admin-tw-dtdd-add")); + }); + } + + document.addEventListener("catapult:render", attach); + + return { attach }; +})(); diff --git a/catapult-web/src/main/resources/static/js/admin-tw.js b/catapult-web/src/main/resources/static/js/admin-tw.js index 9d5cc9e13..2b6c1a036 100644 --- a/catapult-web/src/main/resources/static/js/admin-tw.js +++ b/catapult-web/src/main/resources/static/js/admin-tw.js @@ -6,6 +6,9 @@ * by its status, never by catapult-api's message: 400 (invalid), 409 (id taken), 503 (trigger warnings * disabled), else unavailable. "Manage keywords" is a plain data-admin-open link handled by admin-tabs.js. * + * A recalculation started (or one already running: 409) shows its progress: the tab's card is patched every few + * seconds (admin-tabs.js) while it reads RUNNING and the tab is the one shown, and it can be asked to stop. + * * The keywords sub-view (#tw/{id}) adds and removes (confirmed) Steam keywords of 2 to 50 characters, the * keyword travelling in the JSON body, and tests the saved keywords plus the one being typed on a Steam app: * the answer is built with textContent only (the notes are data). Adding or removing one patches the list @@ -76,18 +79,55 @@ window.CatapultAdminTw = (function () { } } - async function rebuild(root, button) { + async function rebuild(panel, root, button) { button.disabled = true; // before the confirmation: a second click must not open another try { if (!await CatapultAdminTabs.confirmed(root, button.dataset.confirm)) return; const status = await post(button, `/admin/tw/rebuild/${button.dataset.rebuild}`, {}); if (ok(status)) window.mdui?.snackbar?.({ message: root.dataset.labelRebuildStarted }); - else CatapultChannel.showError(refusal(root, status)); + else CatapultChannel.showError(status === 409 ? root.dataset.labelRebuildRunning : refusal(root, status)); + // started, or one runs already: its progress is shown + if (ok(status) || status === 409) { + await CatapultAdminTabs.patch(panel); + follow(panel); + } } finally { button.disabled = false; } } + async function cancelRebuild(panel, root, button) { + const status = await post(button, "/admin/tw/rebuild/cancel", {}); + if (!ok(status)) { + CatapultChannel.showError(refusal(root, status)); + return; + } + await CatapultAdminTabs.patch(panel); + } + + // --- following a recalculation ------------------------------------------------------------- + + const FOLLOW_MS = 3000; + let followTimer = null; + + /** Whether the list shown reads a running recalculation, in the tab shown. */ + function following(panel) { + const progress = panel.querySelector('.admin-tw [data-admin-value="tw-rebuild"]'); + return progress?.dataset.state === "RUNNING" && document.getElementById("admin-tabs")?.value === "tw"; + } + + /** Patches the list every FOLLOW_MS while it reads a running recalculation; stops by itself otherwise. */ + function follow(panel) { + clearTimeout(followTimer); + followTimer = null; + if (!following(panel)) return; + followTimer = setTimeout(async () => { + if (!panel.isConnected || !following(panel)) return; + await CatapultAdminTabs.patch(panel); + follow(panel); + }, FOLLOW_MS); + } + // --- the keywords sub-view --------------------------------------------------------------- const KEYWORD_MIN = 2; @@ -180,7 +220,7 @@ window.CatapultAdminTw = (function () { if (!panel) return; panel.addEventListener("click", event => { - const target = event.target.closest?.(".admin-tw-add-save, .admin-tw-rebuild," + const target = event.target.closest?.(".admin-tw-add-save, .admin-tw-rebuild, .admin-tw-rebuild-cancel," + " .admin-tw-keyword-add, .admin-tw-keyword-remove, .admin-tw-test-run"); if (!target || !panel.contains(target) || target.disabled) return; const keywordsRoot = target.closest(".admin-tw-kw"); @@ -193,12 +233,21 @@ window.CatapultAdminTw = (function () { const root = panel.querySelector(".admin-tw"); if (!root) return; if (target.matches(".admin-tw-add-save")) addDefinition(panel, root, target); - else rebuild(root, target); + else if (target.matches(".admin-tw-rebuild-cancel")) cancelRebuild(panel, root, target); + else rebuild(panel, root, target); }); panel.addEventListener("input", event => event.target.removeAttribute?.("invalid")); + // Back on the tab: a recalculation running is followed again (left, it stopped being) + document.getElementById("admin-tabs")?.addEventListener("change", event => { + if (event.target.id === "admin-tabs") follow(panel); + }); } document.addEventListener("catapult:render", attach); + // The list shown (the tab, back from a sub-view): a recalculation running is followed + document.addEventListener("catapult:admin-panel", event => { + if (event.detail?.tab === "tw") follow(event.detail.panel); + }); return { attach }; })(); diff --git a/catapult-web/src/main/resources/static/js/channel-bindings.js b/catapult-web/src/main/resources/static/js/channel-bindings.js index d630709a0..730d1b948 100644 --- a/catapult-web/src/main/resources/static/js/channel-bindings.js +++ b/catapult-web/src/main/resources/static/js/channel-bindings.js @@ -8,6 +8,26 @@ document.addEventListener("catapult:render", function () { let resetting = null; resetDialog?.querySelector(".binding-reset-dismiss").addEventListener("click", () => { resetDialog.open = false; }); + + // Why a binding has its TWs: catapult-web renders it (as catapult-api tells it now) into the dialog + const twDetailsDialog = document.getElementById("binding-tw-details-dialog"); + twDetailsDialog?.querySelector(".binding-tw-details-close").addEventListener("click", () => { twDetailsDialog.open = false; }); + + async function showTwDetails(row) { + if (!twDetailsDialog) return; + let response = null; + try { + response = await fetch(`/spa/channel/bindings/${encodeURIComponent(row.dataset.bindingId)}/tw-details`); + } catch { + // not fetched: told below + } + if (!response?.ok) { + CatapultChannel.showError(twDetailsDialog.dataset.unavailable); + return; + } + twDetailsDialog.querySelector(".tw-details-body").innerHTML = await response.text(); + twDetailsDialog.open = true; + } resetDialog?.querySelector(".binding-reset-confirm").addEventListener("click", async () => { const row = resetting; resetDialog.open = false; @@ -138,6 +158,8 @@ document.addEventListener("catapult:render", function () { panel.querySelector(".binding-tw-reset-btn").addEventListener("click", () => postJson(twUrl(row, "tws/reset"), {})); + + panel.querySelector(".binding-tw-details-btn")?.addEventListener("click", () => showTwDetails(row)); }); } }); diff --git a/catapult-web/src/main/resources/templates/index.html b/catapult-web/src/main/resources/templates/index.html index f0c2b7cb2..18e0e84b6 100644 --- a/catapult-web/src/main/resources/templates/index.html +++ b/catapult-web/src/main/resources/templates/index.html @@ -228,11 +228,14 @@ + + + diff --git a/catapult-web/src/main/resources/templates/pages/admin-tab/dtdd-caches.html b/catapult-web/src/main/resources/templates/pages/admin-tab/dtdd-caches.html new file mode 100644 index 000000000..49b0f78f7 --- /dev/null +++ b/catapult-web/src/main/resources/templates/pages/admin-tab/dtdd-caches.html @@ -0,0 +1,297 @@ + + + + +
+
+

DTDD caches

+
+

hint

+ + + +

Refetch everything

+

hint

+ +
+ Never run + + + 0 / 0 + + waiting + stopped + finished +
+ Refetch everything + Stop +
+
+ + + + + + topics + + 0 + + + games + + 0 + + + searches + + 0 + + + + +
+

Keys' quotas

+
+
+ + + + + +
+ ddd_
 + + 30 + resting + + + 5000 + reserve + + when +
+ +
+ +
+ +
+ + + +
+
+ +
+ + + +
+ 1 + title + yes + no + mixed + + +
+ + + + +
+ 1 + title + type + + +
+ + + + +
+ query + 1, 2 + + +
+ + +
+
+ + +
+ Close + +
+ + +
+

Yes

+

—

+
    +
  • + topic + + verified +
    + Implied 17 % (2) +
    +     + no levels +
    • +
+
+ + +
+

not cached

+ +

Title

+
+
ID
+
1
+ +
Type
+
type
+
Poster
+
—
+
+
Record fetched
+
+ + +
Record fetched
+
not cached
+ +
+ +

Topics

+

not cached

+ +

+

not kept

+

hint

+
+ + + +
+ + +

Found by

+

none

+
    +
  • query · +
    • +
+ + + +

query

+
+
Searched
+
+
+

Titles found

+

nothing

+
    +
  • + 1 · + name + topics +
    • +
+ +
+ + + + when + Stale + + + + + when + Stale + + + + + + + diff --git a/catapult-web/src/main/resources/templates/pages/admin-tab/game-mappings.html b/catapult-web/src/main/resources/templates/pages/admin-tab/game-mappings.html index a17f7d97c..c08a29530 100644 --- a/catapult-web/src/main/resources/templates/pages/admin-tab/game-mappings.html +++ b/catapult-web/src/main/resources/templates/pages/admin-tab/game-mappings.html @@ -4,7 +4,7 @@ + + +
+
+

Rate limits

+
+

hint

+ + + + + + +
+ + store.steampowered.com + + GET /app/{id} + host's budget + chosen + + + + none known + + 20 / min + corrected + + + + 60 s + corrected + + + 0 + last + + + 0 + waiting + + + + + +
+ + + + +

hint

+

+
+ + + min + + + + learned + host + route + +
+ Cancel + Save + +
diff --git a/catapult-web/src/main/resources/templates/pages/admin-tab/tw.html b/catapult-web/src/main/resources/templates/pages/admin-tab/tw.html index 3bec1ede2..0528e45f9 100644 --- a/catapult-web/src/main/resources/templates/pages/admin-tab/tw.html +++ b/catapult-web/src/main/resources/templates/pages/admin-tab/tw.html @@ -1,4 +1,5 @@ + +
+ +

+ Running + mode +

+ +

0 / 0

+

waiting for Steam

+

finished

+
+ Stop +
+ +
+

Add a trigger warning

@@ -60,6 +88,8 @@

Add a trigger warning

Manage keywords + DTDD topics
@@ -115,6 +145,46 @@

Test against a Steam game

+ +
+
+ +

DTDD topics

+
+

hint

+ + + +

No topic.

+
+
+ topic + + + Any level + level + saved + + +
+
+
+ + Add +
+ +
+
diff --git a/catapult-web/src/main/resources/templates/pages/admin.html b/catapult-web/src/main/resources/templates/pages/admin.html index 0e27abb52..55eb136ce 100644 --- a/catapult-web/src/main/resources/templates/pages/admin.html +++ b/catapult-web/src/main/resources/templates/pages/admin.html @@ -9,14 +9,14 @@

admin.title

- + admin.tab -
diff --git a/catapult-web/src/main/resources/templates/pages/channel-tab/overview.html b/catapult-web/src/main/resources/templates/pages/channel-tab/overview.html index 645428836..a4c298851 100644 --- a/catapult-web/src/main/resources/templates/pages/channel-tab/overview.html +++ b/catapult-web/src/main/resources/templates/pages/channel-tab/overview.html @@ -1,6 +1,6 @@ - +

Statut

@@ -198,7 +198,7 @@

Associations

+ th:attr="label=#{channel.bindings.title.label},helper=|#{channel.bindings.title.helper}${titleHelperExtra}|">

Aperçu :

@@ -246,6 +246,9 @@

Associations

spa:on="hasOverride:TwUpdatedEvent.tws,hasOverride:!TwResetEvent" th:text="#{channel.bindings.tw.reset}">RĂ©initialiser + + DĂ©tails
@@ -261,6 +264,12 @@

Associations

+ +
+ Fermer + +

Sa catégorie, ses CCL et ses TW reviennent à la détection automatique.

@@ -396,7 +405,7 @@

Bindings personnalisés

+ th:attr="label=#{channel.custom_bindings.settings.title},helper=|#{channel.bindings.title.helper}${titleHelperExtra}|">

Aperçu :

diff --git a/catapult-web/src/main/resources/templates/pages/channel-tab/settings.html b/catapult-web/src/main/resources/templates/pages/channel-tab/settings.html index 276ddbec3..b269b740c 100644 --- a/catapult-web/src/main/resources/templates/pages/channel-tab/settings.html +++ b/catapult-web/src/main/resources/templates/pages/channel-tab/settings.html @@ -1,7 +1,7 @@ - +