From 2e1125029d4bf87fa936bb8a370e32b86bae6046 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 18:12:06 +0200 Subject: [PATCH 1/7] feat(store-oracle): add Oracle Database 19c+ JobStore Add threadmill-store-oracle with OracleJobStore, held to the shared AbstractJobStoreContractTest like the in-memory, PostgreSQL, and Redis stores. - Standard JDBC only; applications supply the driver (ojdbc11 is test-only). - Startup refuses servers older than 19c, non-AL32UTF8 databases, and sessions with NLS_COMP other than BINARY. - Partial indexes are emulated with indexed virtual columns (CASE WHEN THEN END); per-key top-n probes stay sort-free under the 19c optimizer. - Claiming uses index-ordered FOR UPDATE SKIP LOCKED cursors fetched only as far as the page budget, ROWNUM top-n branches per key, MIN loose scans, and EXISTS admission probes; no lateral joins. - Long CLOB values are written through temporary Clobs: direct binds split surrogate pairs at the driver's chunk boundary. - Text keyset pages order in binary, independent of the client locale. - OracleMigrationRunner records per-statement progress and resumes a crashed migration, since Oracle commits DDL implicitly; a lock row on a second connection serializes migrators. - No remote wake channel: Oracle has no lightweight LISTEN/NOTIFY. Refs #137 --- build.gradle.kts | 2 + gradle/libs.versions.toml | 2 + gradle/verification-metadata.xml | 8 + settings.gradle.kts | 1 + threadmill-store-oracle/build.gradle.kts | 39 + threadmill-store-oracle/gradle.lockfile | 61 + .../threadmill/store/oracle/OracleClobs.java | 60 + .../store/oracle/OracleConnectionWork.java | 22 + .../store/oracle/OracleDeadlockRetry.java | 104 + .../threadmill/store/oracle/OracleJdbc.java | 178 + .../store/oracle/OracleJobStore.java | 2850 +++++++++++++++++ .../store/oracle/OracleMigrationRunner.java | 601 ++++ .../threadmill/store/oracle/OracleServer.java | 118 + .../oracle/OracleTransactionBoundary.java | 54 + .../store/oracle/OracleTransactions.java | 52 + .../OwningOracleTransactionBoundary.java | 33 + .../store/oracle/PendingKeyCursors.java | 67 + .../store/oracle/migrations/V1__baseline.sql | 451 +++ .../store/oracle/NonAutoCommitDataSource.java | 93 + .../store/oracle/OracleDeadlockRetryTest.java | 79 + .../oracle/OracleJobStoreContractTest.java | 38 + .../oracle/OracleJobStoreRegressionTest.java | 606 ++++ .../oracle/OracleMigrationRunnerTest.java | 247 ++ .../store/oracle/OracleQueryPlanTest.java | 296 ++ .../store/oracle/OracleServerTest.java | 60 + .../oracle/OracleStatementSplitTest.java | 52 + .../store/oracle/OracleTestDatabase.java | 135 + 27 files changed, 6309 insertions(+) create mode 100644 threadmill-store-oracle/build.gradle.kts create mode 100644 threadmill-store-oracle/gradle.lockfile create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleClobs.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleConnectionWork.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetry.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleMigrationRunner.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactionBoundary.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactions.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OwningOracleTransactionBoundary.java create mode 100644 threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/PendingKeyCursors.java create mode 100644 threadmill-store-oracle/src/main/resources/com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/NonAutoCommitDataSource.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetryTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreContractTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreRegressionTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleMigrationRunnerTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleServerTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleStatementSplitTest.java create mode 100644 threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleTestDatabase.java diff --git a/build.gradle.kts b/build.gradle.kts index 21eb143b..7845ffbb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,6 +36,7 @@ val publishedProjects = project(":threadmill-core"), project(":threadmill-store-memory"), project(":threadmill-store-postgres"), + project(":threadmill-store-oracle"), project(":threadmill-store-redis"), project(":threadmill-spring-boot"), project(":threadmill-test-support"), @@ -337,6 +338,7 @@ val productionCheck by dependsOn(subprojects.map { it.tasks.matching { task -> task.name == "javadoc" } }) dependsOn( ":threadmill-store-postgres:test", + ":threadmill-store-oracle:test", ":threadmill-store-redis:test", ":threadmill-dashboard-spring:browserTest", ":threadmill-soak:soakRegression", diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 21a6e753..56ec9cd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,7 @@ assertj = "3.27.7" awaitility = "4.2.2" testcontainers = "2.0.5" postgresql = "42.7.12" +oracleJdbc = "23.26.3.0.0" lettuce = "6.8.2.RELEASE" netty = "4.1.137.Final" networkntJsonschema = "1.5.4" @@ -58,6 +59,7 @@ testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version testcontainers-junit-jupiter = { module = "org.testcontainers:testcontainers-junit-jupiter" } testcontainers-postgresql = { module = "org.testcontainers:testcontainers-postgresql" } postgresql-jdbc = { module = "org.postgresql:postgresql", version.ref = "postgresql" } +oracle-jdbc = { module = "com.oracle.database.jdbc:ojdbc11", version.ref = "oracleJdbc" } lettuce-core = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" } netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } networknt-jsonschema = { module = "com.networknt:json-schema-validator", version.ref = "networkntJsonschema" } diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8e7f54cf..1c52956f 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -751,6 +751,14 @@ + + + + + + + + diff --git a/settings.gradle.kts b/settings.gradle.kts index b1239e6a..c0b6b187 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,6 +13,7 @@ include( "threadmill-core", "threadmill-store-memory", "threadmill-store-postgres", + "threadmill-store-oracle", "threadmill-store-redis", "threadmill-spring-boot", "threadmill-test-support", diff --git a/threadmill-store-oracle/build.gradle.kts b/threadmill-store-oracle/build.gradle.kts new file mode 100644 index 00000000..efee8c53 --- /dev/null +++ b/threadmill-store-oracle/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + id("threadmill.java-module") + id("threadmill.publish") +} + +dependencies { + api(project(":threadmill-core")) + // The store uses standard JDBC only. Applications supply the Oracle driver + // (ojdbc11) themselves, so the published module carries no dependency on + // Oracle-licensed artifacts; the tests pin the driver below. + + testImplementation(project(":threadmill-test-support")) + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.junit.jupiter) + testRuntimeOnly(libs.junit.platform.launcher) + testImplementation(libs.assertj.core) + testImplementation(libs.slf4j.simple) + testImplementation(platform(libs.testcontainers.bom)) + testImplementation(libs.testcontainers.junit.jupiter) + testImplementation("org.testcontainers:testcontainers") + testImplementation(libs.oracle.jdbc) + testImplementation(libs.hikaricp) +} + +// Database selection for the integration tests (see OracleTestDatabase): +// -PoracleImage=gvenzl/oracle-xe:21-slim-faststart another gvenzl image (CI uses 21c XE) +// -PoracleJdbcUrl=... -PoracleUser=... -PoraclePassword=... an existing database, for +// example a real 19c instance; its schema is reset by the tests +tasks.named("test") { + mapOf( + "oracleImage" to "threadmill.oracle.image", + "oracleJdbcUrl" to "threadmill.oracle.jdbcUrl", + "oracleUser" to "threadmill.oracle.user", + "oraclePassword" to "threadmill.oracle.password", + ) + .forEach { (property, systemProperty) -> + providers.gradleProperty(property).orNull?.let { systemProperty(systemProperty, it) } + } +} diff --git a/threadmill-store-oracle/gradle.lockfile b/threadmill-store-oracle/gradle.lockfile new file mode 100644 index 00000000..0b19d6be --- /dev/null +++ b/threadmill-store-oracle/gradle.lockfile @@ -0,0 +1,61 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.21=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-api:3.7.1=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport-zerodep:3.7.1=testCompileClasspath,testRuntimeClasspath +com.github.docker-java:docker-java-transport:3.7.1=testCompileClasspath,testRuntimeClasspath +com.gradleup.gratatouille:gratatouille-tasks-runtime:0.2.3=nmcpTasks +com.gradleup.nmcp:nmcp-tasks:1.6.1=nmcpTasks +com.oracle.database.jdbc:ojdbc11:23.26.3.0.0=testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp-jvm:5.4.0=nmcpTasks +com.squareup.okhttp3:okhttp:5.4.0=nmcpTasks +com.squareup.okio:okio-jvm:3.17.0=nmcpTasks +com.squareup.okio:okio:3.17.0=nmcpTasks +com.zaxxer:HikariCP:7.0.2=testCompileClasspath,testRuntimeClasspath +commons-codec:commons-codec:1.19.0=testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.20.0=testCompileClasspath,testRuntimeClasspath +io.github.pdvrieze.xmlutil:core-jvm:1.0.0-rc3=nmcpTasks +io.github.pdvrieze.xmlutil:core:1.0.0-rc3=nmcpTasks +io.github.pdvrieze.xmlutil:serialization-jvm:1.0.0-rc3=nmcpTasks +io.github.pdvrieze.xmlutil:serialization:1.0.0-rc3=nmcpTasks +net.bytebuddy:byte-buddy:1.18.3=testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.18.1=testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-compress:1.28.0=testCompileClasspath,testRuntimeClasspath +org.apache.commons:commons-lang3:3.18.0=testCompileClasspath,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath +org.assertj:assertj-core:3.27.7=testCompileClasspath,testRuntimeClasspath +org.awaitility:awaitility:4.2.2=testCompileClasspath,testRuntimeClasspath +org.hamcrest:hamcrest:2.1=testCompileClasspath,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.4.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-coroutines-bom:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-coroutines-core:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-serialization-bom:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-serialization-core:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.11.0=nmcpTasks +org.jetbrains.kotlinx:kotlinx-serialization-json:1.11.0=nmcpTasks +org.jetbrains:annotations:17.0.0=testCompileClasspath,testRuntimeClasspath +org.jetbrains:annotations:23.0.0=nmcpTasks +org.junit.jupiter:junit-jupiter-api:5.11.4=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.11.4=testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.11.4=testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.11.4=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.11.4=testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.11.4=testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.11.4=testRuntimeClasspath +org.junit:junit-bom:5.11.4=testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath +org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.16=compileClasspath,runtimeClasspath +org.slf4j:slf4j-api:2.0.17=testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-simple:2.0.16=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-bom:2.0.5=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers-junit-jupiter:2.0.5=testCompileClasspath,testRuntimeClasspath +org.testcontainers:testcontainers:2.0.5=testCompileClasspath,testRuntimeClasspath +empty=annotationProcessor,testAnnotationProcessor diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleClobs.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleClobs.java new file mode 100644 index 00000000..7273c802 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleClobs.java @@ -0,0 +1,60 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Clob; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; + +/** + * Binds text to {@code CLOB} parameters for the statements of one scope and + * frees the temporary LOBs it created when the scope closes. + * + *

Short values are bound directly with {@code setString}. Longer values are + * written through a temporary {@link Clob}: the Oracle driver streams a long + * directly bound string in fixed-size chunks and corrupts a supplementary + * character (a surrogate pair, such as an emoji) that straddles a chunk + * boundary, while {@link Clob#setString} keeps characters whole. Temporary LOBs + * live in the session until freed, so a pooled connection would otherwise + * accumulate them; close this object after the statements executed. + */ +final class OracleClobs implements AutoCloseable { + + /** + * Longest value bound directly. At most 3 bytes per UTF-16 unit in AL32UTF8, + * this stays below the driver's 32 KiB direct-bind limit, so it is never + * chunked. + */ + static final int DIRECT_BIND_MAX_CHARS = 8_000; + + private final List created = new ArrayList<>(); + + void bind(PreparedStatement ps, int index, String value) throws SQLException { + if (value == null) { + ps.setNull(index, Types.CLOB); + } else if (value.length() <= DIRECT_BIND_MAX_CHARS) { + ps.setString(index, value); + } else { + Clob clob = ps.getConnection().createClob(); + created.add(clob); + clob.setString(1, value); + ps.setClob(index, clob); + } + } + + @Override + public void close() throws SQLException { + SQLException failure = null; + for (Clob clob : created) { + try { + clob.free(); + } catch (SQLException e) { + if (failure == null) failure = e; + else failure.addSuppressed(e); + } + } + created.clear(); + if (failure != null) throw failure; + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleConnectionWork.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleConnectionWork.java new file mode 100644 index 00000000..360761ac --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleConnectionWork.java @@ -0,0 +1,22 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Unit of Oracle store work that runs with one JDBC connection. + * + * @param result type + */ +@FunctionalInterface +public interface OracleConnectionWork { + + /** + * Run the work. + * + * @param connection the connection scoped to the active transaction boundary + * @return the work result + * @throws SQLException when a statement fails + */ + T execute(Connection connection) throws SQLException; +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetry.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetry.java new file mode 100644 index 00000000..a9d53186 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetry.java @@ -0,0 +1,104 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.SQLException; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Bounded retry for transient lock conflicts on hot, multi-index rows. + * + *

Deadlocks on a busy queue table are normal, not + * exceptional. Oracle reports them as {@code ORA-00060}; serializable + * transactions additionally see {@code ORA-08177}. Oracle rolls back only the + * statement that detected the deadlock, but a self-owned store transaction is + * rolled back as a whole and retried after a short, jittered sleep. + * + *

Any other exception is rethrown unchanged. + */ +public final class OracleDeadlockRetry { + + /** ORA-00060: deadlock detected while waiting for resource. */ + public static final int ORA_DEADLOCK = 60; + + /** ORA-08177: can't serialize access for this transaction. */ + public static final int ORA_SERIALIZATION_FAILURE = 8177; + + private OracleDeadlockRetry() {} + + /** + * An action that may fail with a transient conflict. + * + * @param result type + */ + @FunctionalInterface + public interface SqlAction { + /** + * Run the action once. + * + * @return the result + * @throws SQLException when the action fails + */ + T run() throws SQLException; + } + + /** + * Run {@code action} with the default retry budget (5 attempts, 5–50 ms backoff). + * + * @param action the action to run + * @param result type + * @return the action result + * @throws SQLException the last failure when it is not retryable or retries are exhausted + */ + public static T run(SqlAction action) throws SQLException { + return run(action, 5, 5, 50); + } + + /** + * Run {@code action}, retrying on {@code ORA-00060} / {@code ORA-08177} up to + * {@code maxAttempts} times with an exponentially backing-off, jittered + * sleep between {@code minBackoffMs} and {@code maxBackoffMs} milliseconds. + * + * @param action the action to run + * @param maxAttempts maximum number of attempts, at least one + * @param minBackoffMs initial backoff in milliseconds + * @param maxBackoffMs backoff cap in milliseconds + * @param result type + * @return the action result + * @throws SQLException the last failure when it is not retryable or retries are exhausted + */ + public static T run( + SqlAction action, int maxAttempts, long minBackoffMs, long maxBackoffMs) + throws SQLException { + for (int attempt = 1; ; attempt++) { + try { + return action.run(); + } catch (SQLException e) { + if (!isRetryable(e) || attempt >= maxAttempts) { + throw e; + } + sleepBackoff(attempt, minBackoffMs, maxBackoffMs); + } + } + } + + /** + * Whether {@code e} reports a transient lock conflict anywhere in its chains. + * + * @param e the failure + * @return {@code true} for {@code ORA-00060} and {@code ORA-08177} + */ + public static boolean isRetryable(SQLException e) { + return OracleJdbc.hasErrorCode(e, ORA_DEADLOCK) + || OracleJdbc.hasErrorCode(e, ORA_SERIALIZATION_FAILURE); + } + + private static void sleepBackoff(int attempt, long minMs, long maxMs) { + long base = Math.min(maxMs, minMs << Math.min(attempt - 1, 6)); + long jitter = ThreadLocalRandom.current().nextLong(base + 1); + try { + Thread.sleep(base + jitter); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while retrying a transient SQL conflict", ie); + } + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java new file mode 100644 index 00000000..85c7f154 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java @@ -0,0 +1,178 @@ +package com.hemju.threadmill.store.oracle; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +/** + * Standard-JDBC value mapping for the Oracle schema. The store never touches + * {@code oracle.jdbc} types, so applications choose and supply their own driver. + * + *

    + *
  • UUIDs are {@code RAW(16)} in big-endian byte order, whose unsigned + * byte-wise comparison equals {@code JobId}'s canonical natural order.
    • + *
  • Instants are UTC {@code TIMESTAMP(9)} values bound as + * {@link LocalDateTime}. Binding {@code java.sql.Timestamp} would apply + * the JVM's default time zone, so the stored wall time would depend on + * the host configuration.
    • + *
  • CLOB text is bound through {@link OracleClobs}.
    • + *
+ */ +final class OracleJdbc { + + private OracleJdbc() {} + + /** Largest number of expressions Oracle accepts in one {@code IN} list. */ + static final int MAX_IN_LIST = 1000; + + static byte[] bytes(UUID value) { + return ByteBuffer.allocate(16) + .putLong(value.getMostSignificantBits()) + .putLong(value.getLeastSignificantBits()) + .array(); + } + + static UUID uuid(byte[] value) { + if (value == null) { + return null; + } + if (value.length != 16) { + throw new IllegalStateException( + "Expected a 16-byte RAW UUID, got " + value.length + " bytes"); + } + var buffer = ByteBuffer.wrap(value); + return new UUID(buffer.getLong(), buffer.getLong()); + } + + static void setUuid(PreparedStatement ps, int index, UUID value) throws SQLException { + if (value == null) { + ps.setNull(index, Types.BINARY); + } else { + ps.setBytes(index, bytes(value)); + } + } + + static UUID getUuid(ResultSet rs, int index) throws SQLException { + return uuid(rs.getBytes(index)); + } + + static UUID getUuid(ResultSet rs, String column) throws SQLException { + return uuid(rs.getBytes(column)); + } + + static void setInstant(PreparedStatement ps, int index, Instant value) throws SQLException { + if (value == null) { + ps.setNull(index, Types.TIMESTAMP); + } else { + ps.setObject(index, LocalDateTime.ofInstant(value, ZoneOffset.UTC)); + } + } + + static Instant getInstant(ResultSet rs, int index) throws SQLException { + var value = rs.getObject(index, LocalDateTime.class); + return value == null ? null : value.toInstant(ZoneOffset.UTC); + } + + static Instant getInstant(ResultSet rs, String column) throws SQLException { + var value = rs.getObject(column, LocalDateTime.class); + return value == null ? null : value.toInstant(ZoneOffset.UTC); + } + + /** Seconds as an exact decimal, for {@code NUMTODSINTERVAL(?, 'SECOND')}. */ + static void setSeconds(PreparedStatement ps, int index, Duration value) throws SQLException { + ps.setBigDecimal(index, BigDecimal.valueOf(value.toMillis(), 3)); + } + + /** + * Bind-list size for {@code count} values. Sizes are rounded up to a small + * set of buckets so statements built from variable-length lists share a + * bounded number of cursors in the shared pool; unused slots are bound to + * NULL, which matches nothing. + */ + static int bucket(int count) { + if (count > MAX_IN_LIST) { + throw new IllegalArgumentException("At most " + MAX_IN_LIST + " values per list: " + count); + } + int size = 8; + while (size < count) { + size <<= 1; + } + return Math.min(size, MAX_IN_LIST); + } + + /** {@code ?, ?, ...} with {@code size} placeholders. */ + static String placeholders(int size) { + return String.join(", ", Collections.nCopies(size, "?")); + } + + /** Bind strings into {@code size} consecutive slots, padding with NULL. */ + static int bindStrings(PreparedStatement ps, int start, Collection values, int size) + throws SQLException { + int index = start; + for (String value : values) { + ps.setString(index++, value); + } + while (index < start + size) { + ps.setNull(index++, Types.VARCHAR); + } + return index; + } + + /** Bind UUIDs into {@code size} consecutive slots, padding with NULL. */ + static int bindUuids(PreparedStatement ps, int start, Collection values, int size) + throws SQLException { + int index = start; + for (UUID value : values) { + ps.setBytes(index++, bytes(value)); + } + while (index < start + size) { + ps.setNull(index++, Types.BINARY); + } + return index; + } + + /** Split {@code values} into chunks no larger than {@code max}. */ + static List> chunks(List values, int max) { + if (values.size() <= max) { + return List.of(values); + } + var chunks = new ArrayList>(); + for (int from = 0; from < values.size(); from += max) { + chunks.add(values.subList(from, Math.min(values.size(), from + max))); + } + return chunks; + } + + /** Whether {@code e} or a chained/caused exception carries Oracle error {@code code} (ORA-n). */ + static boolean hasErrorCode(SQLException e, int code) { + for (SQLException cur = e; cur != null; cur = cur.getNextException()) { + if (cur.getErrorCode() == code) { + return true; + } + } + for (Throwable cause = e.getCause(); cause != null; cause = cause.getCause()) { + if (cause instanceof SQLException sql && sql.getErrorCode() == code) { + return true; + } + } + return false; + } + + /** ORA-00001: unique constraint violated. */ + static final int UNIQUE_VIOLATION = 1; + + /** ORA-02291: integrity constraint violated, parent key not found. */ + static final int PARENT_KEY_NOT_FOUND = 2291; +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java new file mode 100644 index 00000000..5955ea30 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java @@ -0,0 +1,2850 @@ +package com.hemju.threadmill.store.oracle; + +import static com.hemju.threadmill.store.oracle.OracleJdbc.MAX_IN_LIST; +import static com.hemju.threadmill.store.oracle.OracleJdbc.PARENT_KEY_NOT_FOUND; +import static com.hemju.threadmill.store.oracle.OracleJdbc.UNIQUE_VIOLATION; +import static com.hemju.threadmill.store.oracle.OracleJdbc.bindStrings; +import static com.hemju.threadmill.store.oracle.OracleJdbc.bindUuids; +import static com.hemju.threadmill.store.oracle.OracleJdbc.bucket; +import static com.hemju.threadmill.store.oracle.OracleJdbc.chunks; +import static com.hemju.threadmill.store.oracle.OracleJdbc.getInstant; +import static com.hemju.threadmill.store.oracle.OracleJdbc.getUuid; +import static com.hemju.threadmill.store.oracle.OracleJdbc.hasErrorCode; +import static com.hemju.threadmill.store.oracle.OracleJdbc.placeholders; +import static com.hemju.threadmill.store.oracle.OracleJdbc.setInstant; +import static com.hemju.threadmill.store.oracle.OracleJdbc.setSeconds; +import static com.hemju.threadmill.store.oracle.OracleJdbc.setUuid; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Savepoint; +import java.sql.Types; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobReplacement; +import com.hemju.threadmill.core.JobReplacements; +import com.hemju.threadmill.core.JobSnapshot; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.JobStateEntry; +import com.hemju.threadmill.core.Names; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.OversizedJobException; +import com.hemju.threadmill.core.StaleJobException; +import com.hemju.threadmill.core.engine.RemoteWakeChannel; +import com.hemju.threadmill.core.internal.ExecutionHeartbeats; +import com.hemju.threadmill.core.internal.RetentionPosition; +import com.hemju.threadmill.core.schedule.CronExpression; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JobSerializer; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.serialization.SerializationException; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.BulkInsertBudget; +import com.hemju.threadmill.core.store.JobSearch; +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.JobStoreCapabilities; +import com.hemju.threadmill.core.store.Mutexes; +import com.hemju.threadmill.core.store.NodeHeartbeat; +import com.hemju.threadmill.core.store.RetentionCursor; +import com.hemju.threadmill.core.store.RetentionPage; + +/** + * Oracle Database (19c or later) implementation of {@link JobStore}. + * + *

Design highlights: + *

    + *
  • The body column (JSON in a {@code CLOB}) is the source of + * truth; the indexed scalar columns are denormalized so hot + * queries hit indexes without parsing the body.
    • + *
  • Oracle has no partial indexes. Virtual columns of the form + * {@code CASE WHEN THEN END} (for example + * {@code keyed_queue}, {@code pending_key}) are indexed instead: rows + * that do not match the predicate have all-NULL keys, which a B-tree + * index does not store. Queries name these columns directly.
    • + *
  • {@link #claimReady} locks candidates with + * {@code FOR UPDATE SKIP LOCKED}, so contending workers never collide + * and never wait. Oracle forbids row limits on locking queries, so + * locking cursors are index-ordered and fetched only as far as the + * budget; Oracle locks {@code SKIP LOCKED} rows as they are fetched.
    • + *
  • Per-key candidate and admission lookups use plain index probes + * ({@code MIN}/{@code MAX}, {@code ROWNUM} top-n branches, and + * {@code EXISTS} range probes), never lateral joins: Oracle keeps a + * sort over each key's whole range inside a correlated top-n lateral + * view, so its cost would grow with each key's backlog.
    • + *
  • Every self-owned write uses an explicit transaction, independent of + * the {@link DataSource}'s default auto-commit mode, and is retried by + * {@link OracleDeadlockRetry} after {@code ORA-00060}.
    • + *
  • Per-state and per-queue counts come from sharded counter tables + * maintained by a trigger; a {@code COUNT(*)} would contend with claims.
    • + *
  • Only standard JDBC is used. The application supplies the driver.
    • + *
+ * + *

There is no cross-node wake channel: Oracle has no lightweight + * equivalent of PostgreSQL's {@code LISTEN}/{@code NOTIFY}, and wakes are + * only latency hints. Same-JVM wakes still work; other nodes pick up new work + * within their poll interval. + */ +public final class OracleJobStore implements JobStore { + + private static final Logger LOG = LoggerFactory.getLogger(OracleJobStore.class); + private static final String MAINTENANCE_LEASE = "maintenance"; + + /** Current server time as a UTC {@code TIMESTAMP}, matching the stored UTC wall time. */ + private static final String NOW_UTC = "SYS_EXTRACT_UTC(SYSTIMESTAMP)"; + + /** States whose rows are finished and no longer hold claim-time concurrency. */ + private static final String TERMINAL_STATES = "('SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED')"; + + /** Keys enumerated per gathering pass; a per-queue cursor rotates later polls through the rest. */ + static final int MAX_PENDING_KEYS_PER_PASS = 128; + + /** Top-n branches per statement when gathering per-key or per-hold heads. */ + static final int MAX_BRANCHES_PER_STATEMENT = 64; + + /** Process-local fairness hints retained across dynamic queues. */ + private static final int MAX_TRACKED_PENDING_KEY_CURSORS = 1024; + + private static final String CLAIM_COLUMNS = + "id, version, priority, concurrency_key, concurrency_mode, workflow_root_id, current_state_at"; + + private static final String JOB_PROJECTION = "body, owner_heartbeat_at"; + + private static final String JOB_INSERT = "INSERT INTO threadmill_jobs (id, state, queue, " + + "priority, handler_signature, scheduled_at, owner_node_id, owner_heartbeat_at, " + + "last_checkin_at, current_state_at, version, body, created_at, concurrency_key, " + + "concurrency_mode, workflow_root_id, parent_job_id) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + private static final String CRON_TASK_COLUMNS = "name, trigger_kind, trigger_value, " + + "handler_signature, payload_type_tag, payload_serialized, queue, priority, timeout_seconds, " + + "max_attempts, is_exclusive, missed_run_policy, time_zone, enabled"; + + private final DataSource dataSource; + private final OracleTransactionBoundary transactionBoundary; + + /** Boundary for self-owned writes that must never join an external transaction. */ + private final OracleTransactionBoundary owningBoundary; + + private final JobSerializer serializer; + private final JobStoreCapabilities capabilities; + private final OracleServer.Facts server; + private final PendingKeyCursors pendingKeyCursors = + new PendingKeyCursors(MAX_TRACKED_PENDING_KEY_CURSORS); + + private String idleGroupAfter; + private String idleQueueAfter; + + /** + * Create a store with the JSON serializer and default capabilities. + * + * @param dataSource host-owned connection source for a 19c+ AL32UTF8 database + */ + public OracleJobStore(DataSource dataSource) { + this(dataSource, new JsonJobSerializer(), JobStoreCapabilities.defaults()); + } + + /** + * Create a store that owns its transactions. + * + * @param dataSource host-owned connection source for a 19c+ AL32UTF8 database + * @param serializer the job wire-format serializer + * @param capabilities size limits and feature flags + */ + public OracleJobStore( + DataSource dataSource, JobSerializer serializer, JobStoreCapabilities capabilities) { + this(dataSource, serializer, capabilities, OracleTransactionBoundary.owning(dataSource)); + } + + /** + * Create a store whose job writes run inside {@code transactionBoundary}. + * + * @param dataSource host-owned connection source for a 19c+ AL32UTF8 database + * @param serializer the job wire-format serializer + * @param capabilities size limits and feature flags + * @param transactionBoundary boundary for enqueue writes, for example one that + * joins a framework-managed transaction + * @throws com.hemju.threadmill.core.JobEngineFatalException if the server is + * older than 19c or its character set is not AL32UTF8 + */ + public OracleJobStore( + DataSource dataSource, + JobSerializer serializer, + JobStoreCapabilities capabilities, + OracleTransactionBoundary transactionBoundary) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + this.transactionBoundary = Objects.requireNonNull(transactionBoundary, "transactionBoundary"); + this.owningBoundary = OracleTransactionBoundary.owning(this.dataSource); + this.serializer = Objects.requireNonNull(serializer, "serializer"); + this.capabilities = Objects.requireNonNull(capabilities, "capabilities"); + this.server = OracleServer.require(OracleServer.read(this.dataSource)); + } + + /** + * Validate that {@code dataSource} points at a supported Oracle server. + * + * @param dataSource the connection source to check + * @throws com.hemju.threadmill.core.JobEngineFatalException if the server is + * older than 19c or its character set is not AL32UTF8 + */ + public static void requireSupportedServer(DataSource dataSource) { + OracleServer.require(OracleServer.read(dataSource)); + } + + // ---------------------------------------------------------------- capabilities + + @Override + public JobStoreCapabilities capabilities() { + return capabilities; + } + + /** + * Liveness probe for the dispatcher's store-outage circuit breaker: a real + * round trip that fails while the database is unreachable. + */ + @Override + public void verifyWritable() { + try (Connection conn = dataSource.getConnection(); + var st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT 1 FROM dual")) { + rs.next(); + } catch (SQLException e) { + throw new JdbcException("verifyWritable probe failed", e); + } + } + + @Override + public String describe() { + return server.describe(); + } + + @Override + public JobStore delegate() { + return this; + } + + @Override + public boolean supportsExternalTransactions() { + return transactionBoundary.supportsExternalTransactions(); + } + + /** Oracle offers no lightweight cross-session notification; wakes fall back to polling. */ + @Override + public Optional createRemoteWakeChannel(String channelName) { + return Optional.empty(); + } + + private T writeTransaction(OracleConnectionWork work) throws SQLException { + if (transactionBoundary.externallyManagedTransactionActive()) { + return transactionBoundary.inTransaction(work); + } + return OracleDeadlockRetry.run(() -> transactionBoundary.inTransaction(work)); + } + + /** + * Run one self-owned write in a transaction Threadmill commits, never the + * caller's. The explicit boundary is required even for a single statement: + * a pool may hand out {@code autoCommit=false} connections, and closing such + * a connection does not commit it. Spring defers nudges to + * {@code afterCommit}, where a joining boundary would hand back the caller's + * already-committed connection and nobody would commit the nudge. + */ + private T ownedTransaction(OracleConnectionWork work) throws SQLException { + return OracleDeadlockRetry.run(() -> owningBoundary.inTransaction(work)); + } + + // ---------------------------------------------------------------- single-job + + @Override + public void insert(Job job) { + Objects.requireNonNull(job, "job"); + Names.requireName("queue", job.queue()); + long version = 1L; + try { + writeTransaction(conn -> { + JobSnapshot snapshot = snapshotForInsert(conn, job, version); + String body = serializer.serializeJob(snapshot, capabilities); + insertSnapshot( + conn, snapshot, body, lastTransitionTime(snapshot, snapshot.currentState()), version); + noteInsertedWorkflowDescendant(conn, snapshot); + return null; + }); + } catch (SQLException e) { + if (hasErrorCode(e, UNIQUE_VIOLATION)) { + throw new IllegalStateException("Job already exists: " + job.id(), e); + } + throw new JdbcException("Insert failed", e); + } + job.adoptVersion(version); + } + + @Override + public List insertAll(List jobsToInsert) { + Objects.requireNonNull(jobsToInsert, "jobs"); + if (jobsToInsert.isEmpty()) return List.of(); + var budget = new BulkInsertBudget(jobsToInsert.size(), capabilities); + // Pre-flight: serialize every snapshot before any write, so an oversized + // job rejects the whole batch without mutating any input version. + for (var job : jobsToInsert) { + Objects.requireNonNull(job, "job"); + Names.requireName("queue", job.queue()); + budget.include(serializer.serializeJob(job.snapshot(), capabilities)); + } + long version = 1L; + try { + writeTransaction(conn -> { + // Re-snapshot inside the transaction so workflow_root_id resolves + // against live store state. + var finalBudget = new BulkInsertBudget(jobsToInsert.size(), capabilities); + var snapshots = new ArrayList(jobsToInsert.size()); + var bodies = new ArrayList(jobsToInsert.size()); + for (var job : jobsToInsert) { + JobSnapshot snapshot = snapshotForInsert(conn, job, version); + String body = serializer.serializeJob(snapshot, capabilities); + finalBudget.include(body); + snapshots.add(snapshot); + bodies.add(body); + } + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement(JOB_INSERT)) { + for (int i = 0; i < snapshots.size(); i++) { + JobSnapshot snapshot = snapshots.get(i); + bindJobInsert( + clobs, + ps, + snapshot, + bodies.get(i), + lastTransitionTime(snapshot, snapshot.currentState()), + version); + ps.addBatch(); + } + ps.executeBatch(); + } + // Lock the distinct concurrency groups once, in sorted order: locking + // per job in batch order lets concurrent batches with reversed key + // orders deadlock (fatal when joined to a caller's transaction). + var keys = new TreeSet(); + for (JobSnapshot snapshot : snapshots) { + if (snapshot.concurrencyKey() != null) keys.add(snapshot.concurrencyKey()); + } + if (!keys.isEmpty()) { + lockConcurrencyGroups(conn, keys); + for (JobSnapshot snapshot : snapshots) { + incrementWorkflowHoldOutstanding(conn, snapshot); + } + } + return null; + }); + } catch (SQLException e) { + if (hasErrorCode(e, UNIQUE_VIOLATION)) { + throw new IllegalStateException("Duplicate job id in batch", e); + } + throw new JdbcException("insertAll failed", e); + } + var ids = new ArrayList(jobsToInsert.size()); + for (var job : jobsToInsert) { + job.adoptVersion(version); + ids.add(job.id()); + } + return List.copyOf(ids); + } + + @Override + public EnqueueResult enqueueIfAbsent(Job job, String dedupKey, Duration ttl, Instant now) { + Objects.requireNonNull(job, "job"); + Objects.requireNonNull(ttl, "ttl"); + Objects.requireNonNull(now, "now"); + Names.requireName("queue", job.queue()); + if (dedupKey == null || dedupKey.isBlank()) { + throw new IllegalArgumentException("dedupKey must not be blank"); + } + long version = 1L; + try { + EnqueueResult result = writeTransaction(conn -> { + JobSnapshot snapshot = snapshotForInsert(conn, job, version); + String body = serializer.serializeJob(snapshot, capabilities); + Instant currentStateAt = lastTransitionTime(snapshot, snapshot.currentState()); + Optional existing = findActiveDedup(conn, job.queue(), dedupKey, now); + if (existing.isPresent()) { + return new EnqueueResult.Coalesced(existing.get()); + } + // A concurrent producer may commit the same dedup key while this + // insert waits on its unique index. Oracle rolls back only the failed + // statement, so undo this job's writes to a savepoint and coalesce + // onto the winner — inside a joined caller transaction as well. + Savepoint beforeInsert = conn.setSavepoint(); + insertSnapshot(conn, snapshot, body, currentStateAt, version); + noteInsertedWorkflowDescendant(conn, snapshot); + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO threadmill_dedup_keys (queue, dedup_key, job_id, expires_at) " + + "VALUES (?, ?, ?, ?)")) { + ps.setString(1, job.queue()); + ps.setString(2, dedupKey); + setUuid(ps, 3, job.id().asUuid()); + setInstant(ps, 4, now.plus(ttl)); + ps.executeUpdate(); + } catch (SQLException e) { + if (!hasErrorCode(e, UNIQUE_VIOLATION)) throw e; + conn.rollback(beforeInsert); + Optional winner = findActiveDedup(conn, job.queue(), dedupKey, now); + if (winner.isPresent()) return new EnqueueResult.Coalesced(winner.get()); + throw e; + } + return new EnqueueResult.Created(job.id()); + }); + if (result instanceof EnqueueResult.Created) { + job.adoptVersion(version); + } + return result; + } catch (SQLException e) { + throw new JdbcException("enqueueIfAbsent failed", e); + } + } + + @Override + public Optional findById(JobId id) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE id = ?")) { + setUuid(ps, 1, id.asUuid()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return Optional.empty(); + return Optional.of(readJobWithHeartbeat(rs)); + } + } catch (SQLException e) { + throw new JdbcException("findById failed", e); + } + } + + @Override + public void saveAtomic(Job job, long expectedVersion) { + Objects.requireNonNull(job, "job"); + long nextVersion = expectedVersion + 1; + JobSnapshot snapshot = withVersion(job, nextVersion); + // Serialize before any database work so OversizedJobException cannot corrupt state. + String body = serializer.serializeJob(snapshot, capabilities); + Instant currentStateAt = lastTransitionTime(snapshot, snapshot.currentState()); + boolean saved; + try { + saved = ownedTransaction(conn -> { + JobSnapshot oldSnapshot; + try (PreparedStatement ps = conn.prepareStatement( + "SELECT body, version FROM threadmill_jobs WHERE id = ? FOR UPDATE")) { + setUuid(ps, 1, snapshot.id().asUuid()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next() || rs.getLong(2) != expectedVersion) { + return false; + } + oldSnapshot = serializer.deserializeJob(rs.getString(1)).snapshot(); + } + } + if (oldSnapshot.concurrencyKey() != null) { + lockConcurrencyGroup(conn, oldSnapshot.concurrencyKey()); + } + adjustWorkflowHoldOnTransition(conn, oldSnapshot, snapshot.currentState()); + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "state = ?, queue = ?, priority = ?, handler_signature = ?, " + + "scheduled_at = ?, owner_node_id = ?, owner_heartbeat_at = ?, last_checkin_at = ?, " + + "current_state_at = ?, version = ?, body = ?, " + + "concurrency_key = ?, concurrency_mode = ?, workflow_root_id = ?, parent_job_id = ? " + + "WHERE id = ? AND version = ?")) { + ps.setString(1, snapshot.currentState().name()); + ps.setString(2, snapshot.queue()); + ps.setInt(3, snapshot.priority()); + ps.setString(4, snapshot.spec().handlerType()); + setInstant(ps, 5, snapshot.scheduledFor()); + setUuid( + ps, + 6, + snapshot.ownerNodeId() == null ? null : snapshot.ownerNodeId().asUuid()); + setInstant(ps, 7, snapshot.ownerHeartbeatAt()); + setInstant(ps, 8, snapshot.lastCheckinAt()); + setInstant(ps, 9, currentStateAt); + ps.setLong(10, nextVersion); + clobs.bind(ps, 11, body); + setNullableConcurrency(ps, 12, snapshot.concurrencyKey(), snapshot.concurrencyMode()); + setUuid(ps, 14, snapshot.workflowRootId().asUuid()); + setUuid(ps, 15, parentJobId(snapshot)); + setUuid(ps, 16, snapshot.id().asUuid()); + ps.setLong(17, expectedVersion); + return ps.executeUpdate() > 0; + } + }); + } catch (SQLException e) { + throw new JdbcException("saveAtomic failed", e); + } + if (!saved) { + throw new StaleJobException(job.id(), expectedVersion); + } + job.adoptVersion(nextVersion); + } + + @Override + public boolean softDelete(JobId id) { + try { + return ownedTransaction(conn -> { + String body; + long version; + try (PreparedStatement ps = conn.prepareStatement( + "SELECT body, version FROM threadmill_jobs WHERE id = ? FOR UPDATE")) { + setUuid(ps, 1, id.asUuid()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return false; + body = rs.getString(1); + version = rs.getLong(2); + } + } + Job job = serializer.deserializeJob(body); + if (job.currentState() == JobState.DELETED) { + return false; + } + JobSnapshot oldSnapshot = job.snapshot(); + if (oldSnapshot.concurrencyKey() != null) { + lockConcurrencyGroup(conn, oldSnapshot.concurrencyKey()); + } + job.transitionTo(JobState.DELETED, Instant.now(), "user.delete", null); + long nextVersion = version + 1; + JobSnapshot snapshot = withVersion(job, nextVersion); + String newBody = serializer.serializeJob(snapshot, capabilities); + adjustWorkflowHoldOnTransition(conn, oldSnapshot, JobState.DELETED); + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET state = ?, " + + "version = ?, body = ?, current_state_at = ? WHERE id = ?")) { + ps.setString(1, JobState.DELETED.name()); + ps.setLong(2, nextVersion); + clobs.bind(ps, 3, newBody); + setInstant(ps, 4, lastTransitionTime(snapshot, JobState.DELETED)); + setUuid(ps, 5, id.asUuid()); + ps.executeUpdate(); + } + return true; + }); + } catch (SQLException e) { + throw new JdbcException("softDelete failed", e); + } + } + + // ---------------------------------------------------------------- claim & heartbeat + + @Override + public List claimReady(NodeId nodeId, String queue, int max, Instant heartbeatAt) { + Objects.requireNonNull(nodeId, "nodeId"); + Names.requireName("queue", queue); + Objects.requireNonNull(heartbeatAt, "heartbeatAt"); + if (max <= 0) return List.of(); + if (isQueuePaused(queue)) return List.of(); + int cap = Math.min(max, capabilities.maxClaimBatch()); + try { + return ownedTransaction(conn -> { + List result = new ArrayList<>(); + // Version-matched as defense in depth: correctness rests on the + // SKIP LOCKED row locks taken while gathering, but if a refactor ever + // fetched candidates without them, this turns a silent double claim + // into a loud failure. + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "state = 'PROCESSING', owner_node_id = ?, owner_heartbeat_at = ?, " + + "last_checkin_at = NULL, execution_revision = 0, current_state_at = ?, version = ?, " + + "body = ? WHERE id = ? AND version = ?")) { + var alreadyBatched = new HashSet(); + while (result.size() < cap) { + // Rows batched in an earlier pass are still ENQUEUED (the batch + // executes after the loop) and locked by this transaction, so + // SKIP LOCKED does not hide them from the re-gather. + List pending = + lockClaimCandidates(conn, queue, cap - result.size()).stream() + .filter(p -> !alreadyBatched.contains(p.id())) + .toList(); + if (pending.isEmpty()) { + break; + } + List claimable = claimableCandidates(conn, pending, cap - result.size()); + Map bodies = fetchBodies(conn, claimable); + int quarantined = 0; + int before = result.size(); + for (var candidate : claimable) { + if (result.size() >= cap) break; + String original = bodies.get(candidate.id()); + Job job; + try { + job = serializer.deserializeJob(original); + } catch (RuntimeException corrupt) { + // An unreadable body must not fail the whole claim and wedge + // the queue: quarantine it and continue with the rest. + quarantineUnreadable(conn, candidate, heartbeatAt, original); + quarantined++; + continue; + } + job.transitionTo(JobState.PROCESSING, heartbeatAt, "engine.claim", null); + job.assignOwner(nodeId, heartbeatAt); + job.incrementAttempts(); + long nextVersion = candidate.version() + 1; + String newBody; + try { + newBody = serializer.serializeJob(withVersion(job, nextVersion), capabilities); + } catch (OversizedJobException | SerializationException poison) { + quarantineUnreadable(conn, candidate, heartbeatAt, original); + quarantined++; + continue; + } + acquireWorkflowHold(conn, job.snapshot()); + setUuid(ps, 1, nodeId.asUuid()); + setInstant(ps, 2, heartbeatAt); + setInstant(ps, 3, heartbeatAt); + ps.setLong(4, nextVersion); + clobs.bind(ps, 5, newBody); + setUuid(ps, 6, candidate.id()); + ps.setLong(7, candidate.version()); + ps.addBatch(); + alreadyBatched.add(candidate.id()); + result.add(serializer.deserializeJob(newBody)); + } + // Every gathered head is concurrency-inadmissible right now. The + // keyed cursor has already advanced, so the next poll sees the + // next page; chasing pages here would grow the lock set with key + // count. Quarantines are progress: the poison left ENQUEUED. + if (result.size() == before && quarantined == 0) { + break; + } + } + if (!alreadyBatched.isEmpty()) { + for (int count : ps.executeBatch()) { + if (count != 1) { + throw new IllegalStateException("Claim UPDATE matched " + count + " rows — the " + + "SKIP LOCKED row lock no longer guarantees claim exclusivity"); + } + } + } + } + return result; + }); + } catch (SQLException e) { + throw new JdbcException("claimReady failed", e); + } + } + + /** + * Gather and row-lock claim candidates with cost bounded by claimable work, + * never by backlog depth: + * + *

    + *
  • Unkeyed lane — an index-ordered {@code FOR UPDATE SKIP + * LOCKED} cursor over {@code threadmill_jobs_unkeyed_idx}, fetched + * only as far as the page budget.
    • + *
  • Keyed lane — distinct pending keys come from a recursive + * {@code MIN} loose scan (one index probe per key); each key + * contributes its earliest ENQUEUED heads through an index-ordered + * top-n probe in the engine's {@code (current_state_at, id)} in-key + * order.
    • + *
  • Hold lane — members of an active workflow hold are + * admissible regardless of their position in the key's pending + * order, so each active hold's earliest members are gathered by + * {@code (concurrency_key, workflow_root_id)} as well.
    • + *
+ * + *

The merged result is ordered {@code (priority DESC, id)} and capped at + * the narrow page budget; keyed and hold candidates are then locked with + * {@code SKIP LOCKED}, and rows another claimer holds are dropped. + */ + private List lockClaimCandidates(Connection conn, String queue, int want) + throws SQLException { + int page = narrowClaimPageSize(want); + var candidates = new LinkedHashMap(); + collectUnkeyedCandidates(conn, queue, page, candidates); + var keyed = new LinkedHashMap(); + collectKeyedCandidates(conn, queue, want, keyed); + keyed.keySet().removeAll(candidates.keySet()); + candidates.putAll(keyed); + if (candidates.isEmpty()) { + return List.of(); + } + List ordered = candidates.values().stream() + .sorted(Comparator.comparingInt(PendingClaim::priority) + .reversed() + .thenComparing(PendingClaim::id, JobId::compareCanonical)) + .limit(page) + .toList(); + var unlocked = + ordered.stream().map(PendingClaim::id).filter(keyed::containsKey).toList(); + if (unlocked.isEmpty()) { + return ordered; + } + Map locked = lockByIds(conn, unlocked); + var result = new ArrayList(ordered.size()); + for (var candidate : ordered) { + if (!keyed.containsKey(candidate.id())) { + result.add(candidate); + } else if (locked.containsKey(candidate.id())) { + result.add(locked.get(candidate.id())); + } + } + return result; + } + + private void collectUnkeyedCandidates( + Connection conn, String queue, int page, Map into) throws SQLException { + try (PreparedStatement ps = conn.prepareStatement(UNKEYED_CLAIM_SQL)) { + ps.setString(1, queue); + // Oracle locks SKIP LOCKED rows as they are fetched: fetch exactly one + // page so the cursor locks no more than the budget. + ps.setFetchSize(page); + ps.setMaxRows(page); + readPendingClaims(ps, page, into); + } + } + + private void collectKeyedCandidates( + Connection conn, String queue, int want, Map into) throws SQLException { + List keys = distinctEnqueuedKeys(conn, queue); + if (keys.isEmpty()) { + return; + } + int perKey = Math.max(1, want); + for (var chunk : chunks(keys, MAX_BRANCHES_PER_STATEMENT)) { + int size = branchBucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement(keyedHeadsSql(size))) { + int parameter = 1; + for (int i = 0; i < size; i++) { + ps.setString(parameter++, queue); + if (i < chunk.size()) { + ps.setString(parameter++, chunk.get(i)); + } else { + ps.setNull(parameter++, Types.VARCHAR); + } + ps.setInt(parameter++, perKey); + } + ps.setFetchSize(Math.min(1000, chunk.size() * perKey)); + readPendingClaims(ps, Integer.MAX_VALUE, into); + } + } + collectActiveHoldMemberCandidates(conn, queue, perKey, keys, into); + } + + private void collectActiveHoldMemberCandidates( + Connection conn, String queue, int perKey, List keys, Map into) + throws SQLException { + var holds = new ArrayList(); + int size = bucket(keys.size()); + try (PreparedStatement ps = conn.prepareStatement("SELECT concurrency_key, workflow_root_id " + + "FROM threadmill_concurrency_workflow_holds WHERE concurrency_key IN (" + + placeholders(size) + ")")) { + bindStrings(ps, 1, keys, size); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + holds.add(new WorkflowKey(rs.getString(1), getUuid(rs, 2))); + } + } + } + for (var chunk : chunks(holds, MAX_BRANCHES_PER_STATEMENT)) { + int branches = branchBucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement(holdMembersSql(branches))) { + int parameter = 1; + for (int i = 0; i < branches; i++) { + if (i < chunk.size()) { + ps.setString(parameter++, chunk.get(i).concurrencyKey()); + setUuid(ps, parameter++, chunk.get(i).workflowRootId()); + } else { + ps.setNull(parameter++, Types.VARCHAR); + setUuid(ps, parameter++, null); + } + ps.setString(parameter++, queue); + ps.setInt(parameter++, perKey); + } + readPendingClaims(ps, Integer.MAX_VALUE, into); + } + } + } + + /** Branch counts are bucketed so variable key counts share a few cursors. */ + private static int branchBucket(int count) { + int size = 4; + while (size < count) { + size <<= 1; + } + return Math.min(size, MAX_BRANCHES_PER_STATEMENT); + } + + private Map lockByIds(Connection conn, List ids) throws SQLException { + var locked = new HashMap(); + for (var chunk : chunks(ids, MAX_IN_LIST)) { + int size = bucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement(lockByIdsSql(size))) { + bindUuids(ps, 1, chunk, size); + ps.setFetchSize(size); + readPendingClaims(ps, Integer.MAX_VALUE, locked); + } + } + return locked; + } + + private static void readPendingClaims( + PreparedStatement ps, int limit, Map into) throws SQLException { + try (ResultSet rs = ps.executeQuery()) { + int read = 0; + while (read < limit && rs.next()) { + read++; + String mode = rs.getString(5); + var claim = new PendingClaim( + getUuid(rs, 1), + rs.getLong(2), + rs.getInt(3), + rs.getString(4), + mode == null ? null : ConcurrencyMode.valueOf(mode), + getUuid(rs, 6), + getInstant(rs, 7)); + into.putIfAbsent(claim.id(), claim); + } + } + } + + /** + * Distinct concurrency keys with ENQUEUED members in this queue, via a + * recursive {@code MIN} loose scan over {@code threadmill_jobs_keyed_idx}: + * one index probe per key, independent of how many rows each key holds. + * Each store instance keeps a bounded, process-local keyset cursor per + * active queue so a page of blocked early keys cannot permanently hide + * later claimable keys; reaching the end wraps the next scan to the first + * key. Cursor mutation is a disposable fairness hint, deliberately not + * transactional: a retried transaction may skip or repeat a page without + * affecting claim correctness. + */ + private List distinctEnqueuedKeys(Connection conn, String queue) throws SQLException { + var observed = pendingKeyCursors.current(queue); + var keys = distinctEnqueuedKeysAfter(conn, queue, observed == null ? null : observed.after()); + if (keys.size() > MAX_PENDING_KEYS_PER_PASS) { + keys.removeLast(); + pendingKeyCursors.advance(queue, observed, keys.getLast()); + return keys; + } + pendingKeyCursors.clear(queue, observed); + if (keys.isEmpty() && observed != null) { + // The tail disappeared after the previous look-ahead (for example, + // another node claimed it). Restart now instead of reporting a false + // empty keyed lane for this poll. + keys = distinctEnqueuedKeysAfter(conn, queue, null); + if (keys.size() > MAX_PENDING_KEYS_PER_PASS) { + keys.removeLast(); + pendingKeyCursors.advance(queue, null, keys.getLast()); + } + } + return keys; + } + + private List distinctEnqueuedKeysAfter(Connection conn, String queue, String after) + throws SQLException { + try (PreparedStatement ps = conn.prepareStatement(pendingKeysSql(after != null))) { + int parameter = 1; + ps.setString(parameter++, queue); + if (after != null) ps.setString(parameter++, after); + ps.setString(parameter++, queue); + ps.setInt(parameter, MAX_PENDING_KEYS_PER_PASS); + ps.setFetchSize(MAX_PENDING_KEYS_PER_PASS + 1); + try (ResultSet rs = ps.executeQuery()) { + var keys = new ArrayList(); + while (rs.next()) { + keys.add(rs.getString(1)); + } + return keys; + } + } + } + + private List claimableCandidates( + Connection conn, List pending, int remaining) throws SQLException { + var keys = new TreeSet(); + var keyed = new ArrayList(); + for (var candidate : pending) { + if (candidate.concurrencyKey() != null) { + keys.add(candidate.concurrencyKey()); + keyed.add(candidate); + } + } + if (keys.isEmpty()) { + return pending.subList(0, Math.min(remaining, pending.size())); + } + lockConcurrencyGroups(conn, keys); + Map groups = loadGroupStates(conn, keys); + Set activeHolds = loadActiveWorkflowHolds(conn, keyed, keys); + Map admissions = loadAdmissions(conn, keyed); + var claimable = new ArrayList(Math.min(remaining, pending.size())); + for (var candidate : pending) { + if (claimable.size() >= remaining) { + break; + } + if (canClaim(candidate, groups, activeHolds, admissions)) { + claimable.add(candidate); + } + } + return claimable; + } + + /** + * Whether an earlier pending job, and an earlier pending EXCLUSIVE job, + * exists in each candidate's key, as {@code EXISTS} range probes bounded at + * the candidate's own {@code current_state_at}: the probe reads the key's + * first index entry and stops, independent of the key's backlog. + */ + private Map loadAdmissions(Connection conn, List keyed) + throws SQLException { + var admissions = new HashMap(); + var ids = keyed.stream().map(PendingClaim::id).toList(); + for (var chunk : chunks(ids, MAX_IN_LIST)) { + int size = bucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement(admissionSql(size))) { + bindUuids(ps, 1, chunk, size); + ps.setFetchSize(size); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + admissions.put(getUuid(rs, 1), new Admission(rs.getInt(2) == 1, rs.getInt(3) == 1)); + } + } + } + } + return admissions; + } + + private Map fetchBodies(Connection conn, List claimable) + throws SQLException { + if (claimable.isEmpty()) { + return Map.of(); + } + var bodies = new HashMap(); + var ids = claimable.stream().map(PendingClaim::id).toList(); + for (var chunk : chunks(ids, MAX_IN_LIST)) { + int size = bucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement( + "SELECT id, body FROM threadmill_jobs WHERE id IN (" + placeholders(size) + ")")) { + bindUuids(ps, 1, chunk, size); + ps.setFetchSize(size); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + bodies.put(getUuid(rs, 1), rs.getString(2)); + } + } + } + } + return bodies; + } + + private String quarantineBody(String original, long version, Instant now) { + try { + var rejected = serializer.deserializeJob(original); + rejected.transitionTo( + JobState.QUARANTINED, now, "engine.claim-poison", "Cannot prepare processing state"); + return serializer.serializeJob(withVersion(rejected, version), capabilities); + } catch (RuntimeException unreadable) { + // Preserve raw evidence when no valid bounded envelope can be written. + return null; + } + } + + /** + * Move an ENQUEUED job whose body cannot be prepared out of the claim path + * with a scalar update; the counter trigger reconciles ENQUEUED to + * QUARANTINED in the same transaction. + */ + private void quarantineUnreadable( + Connection conn, PendingClaim candidate, Instant now, String originalBody) + throws SQLException { + String rejectedBody = quarantineBody(originalBody, candidate.version() + 1, now); + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "state = 'QUARANTINED', current_state_at = ?, version = ?, body = NVL(?, body) " + + "WHERE id = ? AND version = ? AND state = 'ENQUEUED'")) { + setInstant(ps, 1, now); + ps.setLong(2, candidate.version() + 1); + clobs.bind(ps, 3, rejectedBody); + setUuid(ps, 4, candidate.id()); + ps.setLong(5, candidate.version()); + if (ps.executeUpdate() == 0) { + return; // raced away — nothing was quarantined + } + } + // QUARANTINED is terminal: if the job's workflow root holds the key, this + // member's share must be released, or the key stays held forever. + if (candidate.concurrencyKey() != null) { + releaseWorkflowHoldShare( + conn, + candidate.concurrencyKey(), + candidate.concurrencyMode(), + candidate.workflowRootId()); + } + LOG.warn( + "Quarantined job {} during claim: its persisted body could not be deserialized", + candidate.id()); + } + + private static void lockConcurrencyGroups(Connection conn, Set keys) throws SQLException { + for (String key : new TreeSet<>(keys)) { + lockConcurrencyGroup(conn, key); + } + } + + /** + * Lock (creating if absent) the concurrency-group row for {@code key}. A + * failed INSERT is a statement-level rollback in Oracle, so the loop is safe + * inside a caller-owned transaction as well. + */ + private static void lockConcurrencyGroup(Connection conn, String key) throws SQLException { + try (var lock = conn.prepareStatement("SELECT concurrency_key FROM " + + "threadmill_concurrency_groups WHERE concurrency_key = ? FOR UPDATE"); + var insert = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " + + "(concurrency_key, exclusive_in_flight, shared_in_flight, last_modified) " + + "VALUES (?, 0, 0, " + NOW_UTC + ")")) { + lock.setString(1, key); + insert.setString(1, key); + for (int attempt = 0; attempt < 10; attempt++) { + try (var row = lock.executeQuery()) { + if (row.next()) return; + } + try { + insert.executeUpdate(); + return; // the inserting transaction holds the new row's lock + } catch (SQLException e) { + if (!hasErrorCode(e, UNIQUE_VIOLATION)) throw e; + // A concurrent transaction created it; lock it on the next attempt. + } + } + throw new SQLException("Concurrency group kept disappearing while acquiring its lock"); + } + } + + private static Map loadGroupStates(Connection conn, Set keys) + throws SQLException { + var groups = new HashMap(); + for (var chunk : chunks(List.copyOf(keys), MAX_IN_LIST)) { + int size = bucket(chunk.size()); + try (PreparedStatement ps = conn.prepareStatement("SELECT concurrency_key, " + + "exclusive_in_flight, shared_in_flight FROM threadmill_concurrency_groups " + + "WHERE concurrency_key IN (" + placeholders(size) + ")")) { + bindStrings(ps, 1, chunk, size); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + groups.put(rs.getString(1), new GroupState(rs.getInt(2), rs.getInt(3))); + } + } + } + } + return groups; + } + + private static Set loadActiveWorkflowHolds( + Connection conn, List keyed, Set keys) throws SQLException { + var roots = keyed.stream().map(PendingClaim::workflowRootId).distinct().toList(); + var holds = new HashSet(); + for (var keyChunk : chunks(List.copyOf(keys), 500)) { + for (var rootChunk : chunks(roots, 500)) { + int keySize = bucket(keyChunk.size()); + int rootSize = bucket(rootChunk.size()); + try (PreparedStatement ps = conn.prepareStatement("SELECT concurrency_key, " + + "workflow_root_id FROM threadmill_concurrency_workflow_holds WHERE concurrency_key IN (" + + placeholders(keySize) + ") AND workflow_root_id IN (" + placeholders(rootSize) + + ")")) { + int next = bindStrings(ps, 1, keyChunk, keySize); + bindUuids(ps, next, rootChunk, rootSize); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + holds.add(new WorkflowKey(rs.getString(1), getUuid(rs, 2))); + } + } + } + } + } + return holds; + } + + private static boolean canClaim( + PendingClaim candidate, + Map groups, + Set activeHolds, + Map admissions) { + if (candidate.concurrencyKey() == null) { + return true; + } + if (activeHolds.contains( + new WorkflowKey(candidate.concurrencyKey(), candidate.workflowRootId()))) { + return true; + } + Admission admission = admissions.get(candidate.id()); + if (admission == null) { + return false; // left the table since it was gathered + } + GroupState group = groups.getOrDefault(candidate.concurrencyKey(), GroupState.IDLE); + if (candidate.concurrencyMode() == ConcurrencyMode.EXCLUSIVE) { + return group.isIdle() && !admission.earlierPending(); + } + return group.exclusiveInFlight() == 0 && !admission.earlierExclusive(); + } + + private record PendingClaim( + UUID id, + long version, + int priority, + String concurrencyKey, + ConcurrencyMode concurrencyMode, + UUID workflowRootId, + Instant currentStateAt) {} + + private record GroupState(int exclusiveInFlight, int sharedInFlight) { + static final GroupState IDLE = new GroupState(0, 0); + + boolean isIdle() { + return exclusiveInFlight == 0 && sharedInFlight == 0; + } + } + + private record Admission(boolean earlierPending, boolean earlierExclusive) {} + + private record WorkflowKey(String concurrencyKey, UUID workflowRootId) {} + + // ---------------------------------------------------------------- claim-path SQL + // Package-private so the plan regression tests run exactly these statements. + + /** + * Unkeyed lane: an index-ordered locking cursor. Oracle rejects a row limit + * on a locking query, so the caller fetches only its page; Oracle locks + * {@code SKIP LOCKED} rows as they are fetched. + */ + static final String UNKEYED_CLAIM_SQL = + "SELECT /*+ INDEX_RS_ASC(j threadmill_jobs_unkeyed_idx) */ " + + CLAIM_COLUMNS + " FROM threadmill_jobs j WHERE unkeyed_queue = ? " + + "ORDER BY unkeyed_rank, unkeyed_id FOR UPDATE SKIP LOCKED"; + + /** + * Distinct keys with ENQUEUED members in one queue: a recursive loose scan + * whose every step is one {@code MIN} index probe, bounded to one page plus + * one look-ahead key. + */ + static String pendingKeysSql(boolean afterCursor) { + return "WITH keys (k, n) AS (" + + "SELECT /*+ INDEX(j threadmill_jobs_keyed_idx) */ MIN(keyed_key), 1 " + + "FROM threadmill_jobs j WHERE keyed_queue = ?" + (afterCursor ? " AND keyed_key > ?" : "") + + " UNION ALL " + + "SELECT (SELECT /*+ INDEX(j threadmill_jobs_keyed_idx) */ MIN(keyed_key) " + + "FROM threadmill_jobs j WHERE keyed_queue = ? AND keyed_key > keys.k), keys.n + 1 " + + "FROM keys WHERE keys.k IS NOT NULL AND keys.n <= ?) " + + "SELECT k FROM keys WHERE k IS NOT NULL ORDER BY n"; + } + + /** + * Earliest ENQUEUED heads of {@code branches} keys: one {@code ROWNUM} top-n + * branch per key that reads the keyed index in order and stops after n + * entries. Deliberately not a lateral join: Oracle keeps a sort over the + * key's whole range inside a correlated top-n lateral view, and + * collection-driven lateral top-n views have returned wrong results. + */ + static String keyedHeadsSql(int branches) { + var branch = "SELECT * FROM (SELECT /*+ INDEX_RS_ASC(j threadmill_jobs_keyed_idx) */ " + + CLAIM_COLUMNS + " FROM threadmill_jobs j WHERE keyed_queue = ? AND keyed_key = ? " + + "ORDER BY keyed_at, keyed_id) WHERE ROWNUM <= ?"; + return String.join(" UNION ALL ", Collections.nCopies(branches, branch)); + } + + /** + * Earliest ENQUEUED members of {@code branches} active workflow holds. One + * root's members are bounded by its workflow's size, not by the backlog. + */ + static String holdMembersSql(int branches) { + var branch = "SELECT * FROM (SELECT /*+ INDEX(j threadmill_jobs_workflow_idx) */ " + + CLAIM_COLUMNS + " FROM threadmill_jobs j WHERE outstanding_key = ? " + + "AND outstanding_root = ? AND state = 'ENQUEUED' AND queue = ? " + + "ORDER BY current_state_at, id) WHERE ROWNUM <= ?"; + return String.join(" UNION ALL ", Collections.nCopies(branches, branch)); + } + + /** Lock gathered keyed candidates by primary key, skipping rows another claimer holds. */ + static String lockByIdsSql(int size) { + return "SELECT /*+ INDEX(j threadmill_jobs_pk) */ " + CLAIM_COLUMNS + + " FROM threadmill_jobs j WHERE id IN (" + placeholders(size) + ") " + + "AND state = 'ENQUEUED' FOR UPDATE SKIP LOCKED"; + } + + /** + * Claim-time admission flags per candidate: whether an earlier pending job, + * and an earlier pending EXCLUSIVE job, exists in its key. Each is an + * {@code EXISTS} range probe bounded at the candidate's own + * {@code current_state_at}, so it reads the key's first index entries and + * stops, independent of the key's backlog. + */ + static String admissionSql(int size) { + return "SELECT /*+ INDEX(c threadmill_jobs_pk) */ c.id, " + + "CASE WHEN EXISTS (SELECT 1 FROM threadmill_jobs p " + + "WHERE p.pending_key = c.concurrency_key AND p.pending_at <= c.current_state_at " + + "AND (p.pending_at < c.current_state_at OR p.pending_id < c.id)) THEN 1 ELSE 0 END, " + + "CASE WHEN EXISTS (SELECT 1 FROM threadmill_jobs x " + + "WHERE x.exclusive_key = c.concurrency_key AND x.exclusive_at <= c.current_state_at " + + "AND (x.exclusive_at < c.current_state_at OR x.exclusive_id < c.id)) THEN 1 ELSE 0 END " + + "FROM threadmill_jobs c WHERE c.id IN (" + placeholders(size) + ")"; + } + + // ---------------------------------------------------------------- queue pauses + + @Override + public void pauseQueue(String queue, String reason) { + Names.requireName("queue", queue); + try { + ownedTransaction(conn -> { + try (var clobs = new OracleClobs()) { + upsert( + conn, + "UPDATE threadmill_queue_pauses SET paused_at = ?, paused_by = ? WHERE queue = ?", + ps -> { + setInstant(ps, 1, Instant.now()); + clobs.bind(ps, 2, reason); + ps.setString(3, queue); + }, + "INSERT INTO threadmill_queue_pauses (queue, paused_at, paused_by) VALUES (?, ?, ?)", + ps -> { + ps.setString(1, queue); + setInstant(ps, 2, Instant.now()); + clobs.bind(ps, 3, reason); + }); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("pauseQueue failed", e); + } + } + + @Override + public void resumeQueue(String queue) { + Names.requireName("queue", queue); + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = + conn.prepareStatement("DELETE FROM threadmill_queue_pauses WHERE queue = ?")) { + ps.setString(1, queue); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("resumeQueue failed", e); + } + } + + @Override + public Set listPausedQueues() { + var out = new HashSet(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("SELECT queue FROM threadmill_queue_pauses"); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) out.add(rs.getString(1)); + } catch (SQLException e) { + throw new JdbcException("listPausedQueues failed", e); + } + return Set.copyOf(out); + } + + private boolean isQueuePaused(String queue) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = + conn.prepareStatement("SELECT 1 FROM threadmill_queue_pauses WHERE queue = ?")) { + ps.setString(1, queue); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } catch (SQLException e) { + throw new JdbcException("isQueuePaused failed", e); + } + } + + // ---------------------------------------------------------------- heartbeats + + @Override + public void touchExecutionHeartbeats(NodeId nodeId, Map activeClaims, Instant now) { + Objects.requireNonNull(nodeId, "nodeId"); + Objects.requireNonNull(now, "now"); + var claims = ExecutionHeartbeats.snapshot(activeClaims); + if (claims.isEmpty()) return; + try { + ownedTransaction(conn -> { + try (var update = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "owner_heartbeat_at = GREATEST(NVL(owner_heartbeat_at, ?), ?) " + + "WHERE id = ? AND version = ? AND owner_node_id = ? AND state = 'PROCESSING'")) { + for (var claim : claims.entrySet()) { + setInstant(update, 1, now); + setInstant(update, 2, now); + setUuid(update, 3, claim.getKey().asUuid()); + update.setLong(4, claim.getValue()); + setUuid(update, 5, nodeId.asUuid()); + update.addBatch(); + } + update.executeBatch(); + } + return null; + }); + } catch (SQLException failure) { + throw new JdbcException("touchExecutionHeartbeats failed", failure); + } + } + + @Override + public void touchOwnerHeartbeat(NodeId nodeId, Instant now) { + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "owner_heartbeat_at = GREATEST(NVL(owner_heartbeat_at, ?), ?) " + + "WHERE state = 'PROCESSING' AND owner_node_id = ?")) { + setInstant(ps, 1, now); + setInstant(ps, 2, now); + setUuid(ps, 3, nodeId.asUuid()); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("touchOwnerHeartbeat failed", e); + } + } + + @Override + public boolean saveExecutionUpdate(Job job, NodeId nodeId) { + Objects.requireNonNull(job, "job"); + Objects.requireNonNull(nodeId, "nodeId"); + var incoming = job.snapshot(); + try { + boolean saved = ownedTransaction(conn -> { + Instant heartbeat = incoming.ownerHeartbeatAt(); + try (var select = conn.prepareStatement("SELECT owner_heartbeat_at, last_checkin_at " + + "FROM threadmill_jobs WHERE id = ? AND state = 'PROCESSING' AND owner_node_id = ? " + + "AND version = ? AND execution_revision = ? FOR UPDATE")) { + setUuid(select, 1, incoming.id().asUuid()); + setUuid(select, 2, nodeId.asUuid()); + select.setLong(3, incoming.version()); + select.setLong(4, incoming.executionRevision()); + try (var rs = select.executeQuery()) { + if (!rs.next()) return false; + Instant persistedHeartbeat = getInstant(rs, 1); + Instant persistedCheckIn = getInstant(rs, 2); + if (persistedCheckIn != null + && (incoming.lastCheckinAt() == null + || incoming.lastCheckinAt().isBefore(persistedCheckIn))) { + return false; + } + if (persistedHeartbeat != null + && (heartbeat == null || heartbeat.isBefore(persistedHeartbeat))) { + heartbeat = persistedHeartbeat; + } + } + } + var updated = incoming.withExecutionUpdate(incoming.executionRevision() + 1, heartbeat); + var body = serializer.serializeJob(updated, capabilities); + try (var clobs = new OracleClobs(); + var update = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "owner_heartbeat_at = ?, last_checkin_at = ?, body = ?, execution_revision = ? " + + "WHERE id = ?")) { + setInstant(update, 1, heartbeat); + setInstant(update, 2, updated.lastCheckinAt()); + clobs.bind(update, 3, body); + update.setLong(4, updated.executionRevision()); + setUuid(update, 5, updated.id().asUuid()); + return update.executeUpdate() == 1; + } + }); + if (saved) job.adoptExecutionRevision(incoming.executionRevision() + 1); + return saved; + } catch (SQLException e) { + throw new JdbcException("saveExecutionUpdate failed", e); + } + } + + @Override + public void recordNodeHeartbeat(NodeId nodeId, Instant now) { + try { + ownedTransaction(conn -> { + upsert( + conn, + "UPDATE threadmill_nodes SET last_heartbeat_at = ? WHERE id = ?", + ps -> { + setInstant(ps, 1, now); + setUuid(ps, 2, nodeId.asUuid()); + }, + "INSERT INTO threadmill_nodes (id, last_heartbeat_at) VALUES (?, ?)", + ps -> { + setUuid(ps, 1, nodeId.asUuid()); + setInstant(ps, 2, now); + }); + return null; + }); + } catch (SQLException e) { + throw new JdbcException("recordNodeHeartbeat failed", e); + } + } + + @Override + public Optional readNodeHeartbeat(NodeId nodeId) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = + conn.prepareStatement("SELECT last_heartbeat_at FROM threadmill_nodes WHERE id = ?")) { + setUuid(ps, 1, nodeId.asUuid()); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() ? Optional.of(getInstant(rs, 1)) : Optional.empty(); + } + } catch (SQLException e) { + throw new JdbcException("readNodeHeartbeat failed", e); + } + } + + @Override + public boolean acquireOrRenewMaintenanceLease(NodeId nodeId, Duration leaseDuration) { + Objects.requireNonNull(nodeId, "nodeId"); + Mutexes.requirePositive(leaseDuration); + try { + // Expiry uses server time for both write and compare, so a node whose + // clock runs ahead cannot steal an unexpired lease. + return ownedTransaction(conn -> conditionalUpsert( + conn, + "UPDATE threadmill_leases SET holder = ?, expires_at = " + NOW_UTC + + " + NUMTODSINTERVAL(?, 'SECOND') WHERE name = ? AND (holder = ? OR expires_at <= " + + NOW_UTC + ")", + ps -> { + setUuid(ps, 1, nodeId.asUuid()); + setSeconds(ps, 2, leaseDuration); + ps.setString(3, MAINTENANCE_LEASE); + setUuid(ps, 4, nodeId.asUuid()); + }, + "INSERT INTO threadmill_leases (name, holder, expires_at) VALUES (?, ?, " + NOW_UTC + + " + NUMTODSINTERVAL(?, 'SECOND'))", + ps -> { + ps.setString(1, MAINTENANCE_LEASE); + setUuid(ps, 2, nodeId.asUuid()); + setSeconds(ps, 3, leaseDuration); + })); + } catch (SQLException e) { + throw new JdbcException("acquireOrRenewMaintenanceLease failed", e); + } + } + + @Override + public void releaseMaintenanceLease(NodeId nodeId) { + Objects.requireNonNull(nodeId, "nodeId"); + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = + conn.prepareStatement("DELETE FROM threadmill_leases WHERE name = ? AND holder = ?")) { + ps.setString(1, MAINTENANCE_LEASE); + setUuid(ps, 2, nodeId.asUuid()); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("releaseMaintenanceLease failed", e); + } + } + + @Override + public Optional readMaintenanceLeaseOwner() { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("SELECT holder FROM threadmill_leases " + + "WHERE name = ? AND expires_at > " + NOW_UTC)) { + ps.setString(1, MAINTENANCE_LEASE); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() ? Optional.of(NodeId.of(getUuid(rs, 1))) : Optional.empty(); + } + } catch (SQLException e) { + throw new JdbcException("readMaintenanceLeaseOwner failed", e); + } + } + + // ---------------------------------------------------------------- housekeeping queries + + @Override + public List findDueForPromotion(Instant now, int max) { + int limit = Math.max(0, max); + return queryJobs(PROMOTION_SQL, limit, ps -> { + setInstant(ps, 1, now); + ps.setInt(2, limit); + }); + } + + @Override + public List findOrphaned(Instant heartbeatExpiry, int max) { + int limit = Math.max(0, max); + return queryJobs(ORPHAN_SQL, limit, ps -> { + setInstant(ps, 1, heartbeatExpiry); + ps.setInt(2, limit); + }); + } + + // Maintenance SQL is package-private so the plan regression tests explain exactly these. + + static final String PROMOTION_SQL = + firstRows("SELECT /*+ INDEX(j threadmill_jobs_scheduled_idx) */ " + JOB_PROJECTION + + " FROM threadmill_jobs j WHERE scheduled_due <= ? ORDER BY scheduled_due"); + + static final String ORPHAN_SQL = + firstRows("SELECT /*+ INDEX(j threadmill_jobs_liveness_idx) */ " + JOB_PROJECTION + + " FROM threadmill_jobs j WHERE processing_liveness <= ? ORDER BY processing_liveness"); + + static final String AWAITING_SQL = + firstRows("SELECT /*+ INDEX(j threadmill_jobs_awaiting_idx) */ body FROM threadmill_jobs j " + + "WHERE awaiting_parent = ? ORDER BY awaiting_at, awaiting_id"); + + /** Oldest ENQUEUED transition in one queue: a single MIN/MAX index probe. */ + static final String QUEUE_AGE_SQL = + "SELECT /*+ INDEX(j threadmill_jobs_queue_age_idx) */ MIN(enqueued_at) " + + "FROM threadmill_jobs j WHERE enqueued_queue = ?"; + + /** Earliest SCHEDULED due time: a single MIN/MAX index probe. */ + static final String OLDEST_SCHEDULED_SQL = + "SELECT /*+ INDEX(j threadmill_jobs_scheduled_idx) */ MIN(scheduled_due) " + + "FROM threadmill_jobs j"; + + /** Earliest transition into one state: a single MIN/MAX probe of the retention index. */ + static final String OLDEST_IN_STATE_SQL = + "SELECT /*+ INDEX(j threadmill_jobs_retention_idx) */ MIN(current_state_at) " + + "FROM threadmill_jobs j WHERE state = ?"; + + /** Oldest owner heartbeat among PROCESSING jobs: a single MIN/MAX index probe. */ + static final String OLDEST_HEARTBEAT_SQL = + "SELECT /*+ INDEX(j threadmill_jobs_heartbeat_idx) */ MIN(processing_heartbeat) " + + "FROM threadmill_jobs j"; + + /** + * Top-n over an ordered query, bound to a trailing {@code ?} row count. The + * classic {@code ROWNUM} form runs as {@code COUNT STOPKEY} over the index + * order on every supported release; with the 19c optimizer, {@code FETCH + * FIRST} can become a {@code WINDOW SORT} that reads the whole index range. + */ + private static String firstRows(String orderedQuery) { + return "SELECT * FROM (" + orderedQuery + ") WHERE ROWNUM <= ?"; + } + + static final String COUNTS_SQL = + "SELECT state, SUM(job_count) FROM threadmill_job_counts GROUP BY state"; + + static final String QUEUE_DEPTHS_SQL = "SELECT queue, SUM(job_count) " + + "FROM threadmill_queue_counts GROUP BY queue HAVING SUM(job_count) > 0"; + + /** Cutoff-eligible retention candidates in {@code (current_state_at, id)} keyset order. */ + static String retentionCandidatesSql(boolean withBody, boolean afterCursor) { + return "SELECT /*+ INDEX_RS_ASC(j threadmill_jobs_retention_idx) */ id, current_state_at" + + (withBody ? ", body" : "") + + " FROM threadmill_jobs j WHERE state = ? AND current_state_at <= ? " + + (afterCursor ? "AND current_state_at >= ? AND (current_state_at > ? OR id > ?) " : "") + + "ORDER BY current_state_at, id FOR UPDATE SKIP LOCKED"; + } + + /** + * Reclaimable concurrency groups in binary key order, after an optional + * keyset cursor, through the RAW {@code idle_sort} key so neither the index + * use nor the order depends on the session's {@code NLS_SORT}. + */ + static String idleGroupsSql(boolean afterCursor) { + return "SELECT /*+ INDEX_RS_ASC(g threadmill_concurrency_idle_idx) */ concurrency_key " + + "FROM threadmill_concurrency_groups g WHERE idle_sort " + + (afterCursor ? "> NLSSORT(?, 'NLS_SORT = BINARY')" : ">= HEXTORAW('00')") + + " AND last_modified <= " + NOW_UTC + " - INTERVAL '1' MINUTE " + + "ORDER BY idle_sort FOR UPDATE SKIP LOCKED"; + } + + /** + * Binary order for text keyset pages. Comparisons ({@code name > ?}) are + * binary under the default {@code NLS_COMP}, but {@code ORDER BY} follows + * {@code NLS_SORT}, which the driver derives from the client JVM locale; a + * linguistic page order would skip rows between pages. + */ + private static String binary(String column) { + return "NLSSORT(" + column + ", 'NLS_SORT = BINARY')"; + } + + // ---------------------------------------------------------------- counts & search + + @Override + public Map countsByState() { + var counts = new EnumMap(JobState.class); + for (JobState s : JobState.values()) counts.put(s, 0L); + // Counts are sharded 16 ways per state; only the SUM is meaningful. + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(COUNTS_SQL); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + try { + counts.put(JobState.valueOf(rs.getString(1)), rs.getLong(2)); + } catch (IllegalArgumentException ignored) { + // unknown state from another schema version — ignore + } + } + } catch (SQLException e) { + throw new JdbcException("countsByState failed", e); + } + return counts; + } + + @Override + public Map queueDepths() { + var depths = new HashMap(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(QUEUE_DEPTHS_SQL); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + depths.put(rs.getString(1), rs.getLong(2)); + } + } catch (SQLException e) { + throw new JdbcException("queueDepths failed", e); + } + return depths; + } + + @Override + public List listEnqueuedQueues() { + return queueDepths().keySet().stream().sorted().toList(); + } + + @Override + public List scanJobs(JobState state, JobId after, int max) { + Objects.requireNonNull(state, "state"); + int limit = Math.clamp(max, 0, 500); + return queryJobs( + firstRows("SELECT /*+ INDEX(j threadmill_jobs_state_id_idx) */ " + JOB_PROJECTION + + " FROM threadmill_jobs j WHERE state = ? " + (after == null ? "" : "AND id > ? ") + + "ORDER BY id"), + limit, + ps -> { + ps.setString(1, state.name()); + if (after != null) setUuid(ps, 2, after.asUuid()); + ps.setInt(after == null ? 2 : 3, limit); + }); + } + + @Override + public List scanCronTasks(String after, int max) { + int limit = Math.clamp(max, 0, 500); + var result = new ArrayList(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(firstRows("SELECT " + CRON_TASK_COLUMNS + + " FROM threadmill_cron_tasks " + (after == null ? "" : "WHERE name > ? ") + + "ORDER BY " + binary("name")))) { + if (after != null) ps.setString(1, after); + ps.setInt(after == null ? 1 : 2, limit); + ps.setFetchSize(Math.max(1, limit)); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) result.add(readCronTask(rs)); + } + return result; + } catch (SQLException e) { + throw new JdbcException("scanCronTasks failed", e); + } + } + + @Override + public List searchJobs(JobSearch search) { + Objects.requireNonNull(search, "search"); + var sql = new StringBuilder("SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE 1 = 1"); + var args = new ArrayList(); + if (search.state() != null) { + sql.append(" AND state = ?"); + args.add(search.state().name()); + } + if (search.queue() != null) { + sql.append(" AND queue = ?"); + args.add(search.queue()); + } + if (search.handlerType() != null) { + sql.append(" AND handler_signature = ?"); + args.add(search.handlerType()); + } + sql.append(" ORDER BY current_state_at DESC, id ASC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"); + args.add(search.offset()); + args.add(search.limit()); + return queryJobs(sql.toString(), search.limit(), ps -> { + for (int i = 0; i < args.size(); i++) { + ps.setObject(i + 1, args.get(i)); + } + }); + } + + @Override + public Optional oldestEnqueuedAt(String queue) { + Names.requireName("queue", queue); + return firstInstant(QUEUE_AGE_SQL, ps -> ps.setString(1, queue), "oldestEnqueuedAt"); + } + + @Override + public Optional oldestMaintenanceAt(JobState state) { + Objects.requireNonNull(state, "state"); + if (state == JobState.SCHEDULED) { + return firstInstant(OLDEST_SCHEDULED_SQL, ps -> {}, "oldestMaintenanceAt"); + } + return firstInstant( + OLDEST_IN_STATE_SQL, ps -> ps.setString(1, state.name()), "oldestMaintenanceAt"); + } + + @Override + public Optional oldestProcessingHeartbeat() { + return firstInstant(OLDEST_HEARTBEAT_SQL, ps -> {}, "oldestProcessingHeartbeat"); + } + + @Override + public List listNodeHeartbeats() { + var out = new ArrayList(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT id, last_heartbeat_at FROM threadmill_nodes ORDER BY id"); + ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + out.add(new NodeHeartbeat(NodeId.of(getUuid(rs, 1)), getInstant(rs, 2))); + } + return out; + } catch (SQLException e) { + throw new JdbcException("listNodeHeartbeats failed", e); + } + } + + @Override + public long deleteNodeHeartbeatsOlderThan(Instant cutoff) { + Objects.requireNonNull(cutoff, "cutoff"); + try { + return ownedTransaction(conn -> { + try (PreparedStatement ps = + conn.prepareStatement("DELETE FROM threadmill_nodes WHERE last_heartbeat_at <= ?")) { + setInstant(ps, 1, cutoff); + return (long) ps.executeUpdate(); + } + }); + } catch (SQLException e) { + throw new JdbcException("deleteNodeHeartbeatsOlderThan failed", e); + } + } + + private record KeyPage(List keys, long removed) {} + + @Override + public synchronized long deleteIdleConcurrencyGroups(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + String after = idleGroupAfter; + try { + var page = writeTransaction(conn -> { + var keys = new ArrayList(); + try (var query = conn.prepareStatement(idleGroupsSql(after != null))) { + if (after != null) query.setString(1, after); + query.setFetchSize(limit); + query.setMaxRows(limit); + try (var rows = query.executeQuery()) { + while (keys.size() < limit && rows.next()) keys.add(rows.getString(1)); + } + } + long removed = 0; + try (var delete = conn.prepareStatement("DELETE FROM threadmill_concurrency_groups g " + + "WHERE concurrency_key = ? AND exclusive_in_flight = 0 AND shared_in_flight = 0 " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_concurrency_workflow_holds h " + + "WHERE h.concurrency_key = g.concurrency_key) " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE " + + "j.outstanding_key = g.concurrency_key)")) { + for (var key : keys) { + delete.setString(1, key); + removed += delete.executeUpdate(); + } + } + return new KeyPage(keys, removed); + }); + idleGroupAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleConcurrencyGroups failed", failure); + } + } + + @Override + public synchronized long deleteIdleQueueMetadata(int max) { + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return 0; + String after = idleQueueAfter; + try { + var page = writeTransaction(conn -> { + var queues = new ArrayList(); + var idleQueues = new ArrayList(); + var candidates = + firstRows("SELECT queue, SUM(job_count) AS total FROM threadmill_queue_counts " + + (after == null ? "" : "WHERE queue > ? ") + "GROUP BY queue ORDER BY " + + binary("queue")); + try (var query = conn.prepareStatement("SELECT c.queue, CASE WHEN c.total = 0 " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j WHERE j.enqueued_queue = c.queue) " + + "THEN 1 ELSE 0 END FROM (" + candidates + ") c ORDER BY " + binary("c.queue"))) { + if (after != null) query.setString(1, after); + query.setInt(after == null ? 1 : 2, limit); + try (var rows = query.executeQuery()) { + while (rows.next()) { + queues.add(rows.getString(1)); + if (rows.getInt(2) == 1) idleQueues.add(rows.getString(1)); + } + } + } + long removed = 0; + for (var queue : idleQueues) { + removed += deleteBalancedQueueShards(conn, queue); + } + return new KeyPage(queues, removed); + }); + idleQueueAfter = page.keys().size() < limit ? null : page.keys().getLast(); + return page.removed(); + } catch (SQLException failure) { + throw new JdbcException("deleteIdleQueueMetadata failed", failure); + } + } + + /** + * Delete only the locked, zero-sum subset of a queue's counter shards. This + * keeps totals exact even with negative shards or concurrent trigger writes + * to shards this transaction could not lock. The idle check is repeated + * under the locks because a producer may have arrived since the prefilter. + */ + private static long deleteBalancedQueueShards(Connection conn, String queue) throws SQLException { + var shards = new ArrayList(); + long total = 0; + try (var lock = conn.prepareStatement("SELECT shard_no, job_count FROM threadmill_queue_counts " + + "WHERE queue = ? FOR UPDATE SKIP LOCKED")) { + lock.setString(1, queue); + lock.setFetchSize(16); + try (var rows = lock.executeQuery()) { + while (rows.next()) { + shards.add(rows.getInt(1)); + total += rows.getLong(2); + } + } + } + if (shards.isEmpty() || total != 0) { + return 0; + } + int size = bucket(shards.size()); + try (var delete = conn.prepareStatement("DELETE FROM threadmill_queue_counts WHERE queue = ? " + + "AND shard_no IN (" + placeholders(size) + ") AND NOT EXISTS (SELECT 1 FROM " + + "threadmill_jobs j WHERE j.enqueued_queue = ?)")) { + delete.setString(1, queue); + int parameter = 2; + for (int shard : shards) delete.setInt(parameter++, shard); + while (parameter < 2 + size) delete.setNull(parameter++, Types.INTEGER); + delete.setString(parameter, queue); + return delete.executeUpdate(); + } + } + + @Override + public long deleteExpiredDedupKeys(Instant now, int max) { + Objects.requireNonNull(now, "now"); + if (max <= 0) return 0L; + try { + return ownedTransaction(conn -> { + try (PreparedStatement ps = conn.prepareStatement("DELETE FROM threadmill_dedup_keys " + + "WHERE ROWID IN (SELECT d.ROWID FROM threadmill_dedup_keys d " + + "WHERE d.expires_at <= ? AND NOT EXISTS (SELECT 1 FROM threadmill_jobs j " + + "WHERE j.id = d.job_id AND j.state NOT IN " + TERMINAL_STATES + ") " + + "AND ROWNUM <= ?)")) { + setInstant(ps, 1, now); + ps.setInt(2, max); + return (long) ps.executeUpdate(); + } + }); + } catch (SQLException e) { + throw new JdbcException("deleteExpiredDedupKeys failed", e); + } + } + + @Override + public List findByHandlerSignature(String handlerType, int max) { + Objects.requireNonNull(handlerType, "handlerType"); + int limit = Math.max(0, max); + return queryJobs( + "SELECT " + JOB_PROJECTION + " FROM threadmill_jobs WHERE handler_signature = ? " + + "AND ROWNUM <= ?", + limit, + ps -> { + ps.setString(1, handlerType); + ps.setInt(2, limit); + }); + } + + // ---------------------------------------------------------------- retention + + @Override + public RetentionPage deleteFinishedPage( + Instant cutoff, JobState state, int max, RetentionCursor after) { + Objects.requireNonNull(cutoff, "cutoff"); + if (state != JobState.SUCCEEDED + && state != JobState.FAILED + && state != JobState.DELETED + && state != JobState.QUARANTINED) { + throw new IllegalArgumentException("Retention requires a finished state"); + } + int limit = Math.clamp(max, 0, 100); + if (limit == 0) return new RetentionPage(0, null); + var position = after == null ? null : RetentionPosition.from(after); + try { + return ownedTransaction(conn -> { + long deleted = 0; + int inspected = 0; + RetentionPosition last = null; + try (var candidates = conn.prepareStatement( + retentionCandidatesSql(state == JobState.FAILED, position != null)); + var remove = conn.prepareStatement("DELETE FROM threadmill_jobs j WHERE id = ? " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_dedup_keys d WHERE d.job_id = j.id " + + "AND d.expires_at > " + NOW_UTC + ") " + + "AND NOT EXISTS (SELECT 1 FROM threadmill_jobs child WHERE " + + "child.awaiting_parent = j.id)")) { + candidates.setString(1, state.name()); + setInstant(candidates, 2, cutoff); + if (position != null) { + setInstant(candidates, 3, position.at()); + setInstant(candidates, 4, position.at()); + setUuid(candidates, 5, position.id().asUuid()); + } + candidates.setFetchSize(limit); + candidates.setMaxRows(limit); + try (var rows = candidates.executeQuery()) { + while (inspected < limit && rows.next()) { + inspected++; + last = new RetentionPosition( + getInstant(rows, "current_state_at"), JobId.of(getUuid(rows, "id"))); + if (state == JobState.FAILED) { + try { + if (serializer + .deserializeJob(rows.getString("body")) + .failureDecision() + .map(decision -> decision.willRetry()) + .orElse(true)) continue; + } catch (SerializationException unreadable) { + continue; // Preserve unknown failure outcomes, but advance the scan. + } + } + setUuid(remove, 1, last.id().asUuid()); + deleted += remove.executeUpdate(); + } + } + } + return new RetentionPage(deleted, inspected == limit ? last.cursor() : null); + }); + } catch (SQLException e) { + throw new JdbcException("deleteFinishedPage failed", e); + } + } + + // ---------------------------------------------------------------- relationships & mutexes + + @Override + public List findAwaitingByParent(JobId parentId, int max) { + Objects.requireNonNull(parentId, "parentId"); + if (max <= 0) return List.of(); + var out = new ArrayList(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(AWAITING_SQL)) { + setUuid(ps, 1, parentId.asUuid()); + ps.setInt(2, max); + ps.setFetchSize(Math.min(max, 500)); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + out.add(serializer.deserializeJob(rs.getString(1))); + } + } + return out; + } catch (SQLException e) { + throw new JdbcException("findAwaitingByParent failed", e); + } + } + + @Override + public boolean tryAcquireMutex(String name, String holder, Duration leaseDuration) { + Names.requireName("mutex", name); + Objects.requireNonNull(holder, "holder"); + Mutexes.requirePositive(leaseDuration); + try { + // Server-side time for write and compare, like the maintenance lease. + // DECODE treats two NULLs as equal, so an empty holder (stored as NULL + // by Oracle) still matches itself. + return ownedTransaction(conn -> conditionalUpsert( + conn, + "UPDATE threadmill_mutexes SET holder = ?, expires_at = " + NOW_UTC + + " + NUMTODSINTERVAL(?, 'SECOND') WHERE name = ? AND (expires_at <= " + NOW_UTC + + " OR DECODE(holder, ?, 1, 0) = 1)", + ps -> { + ps.setString(1, holder); + setSeconds(ps, 2, leaseDuration); + ps.setString(3, name); + ps.setString(4, holder); + }, + "INSERT INTO threadmill_mutexes (name, holder, expires_at) VALUES (?, ?, " + NOW_UTC + + " + NUMTODSINTERVAL(?, 'SECOND'))", + ps -> { + ps.setString(1, name); + ps.setString(2, holder); + setSeconds(ps, 3, leaseDuration); + })); + } catch (SQLException e) { + throw new JdbcException("tryAcquireMutex failed", e); + } + } + + @Override + public void releaseMutex(String name, String holder) { + Names.requireName("mutex", name); + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "DELETE FROM threadmill_mutexes WHERE name = ? AND DECODE(holder, ?, 1, 0) = 1")) { + ps.setString(1, name); + ps.setString(2, holder); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("releaseMutex failed", e); + } + } + + @Override + public boolean replaceJob(JobId id, long expectedVersion, JobReplacement replacement) { + Objects.requireNonNull(id, "id"); + Objects.requireNonNull(replacement, "replacement"); + try { + return ownedTransaction(conn -> { + String body; + long version; + String state; + try (PreparedStatement ps = conn.prepareStatement( + "SELECT body, version, state FROM threadmill_jobs WHERE id = ? FOR UPDATE")) { + setUuid(ps, 1, id.asUuid()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return false; + body = rs.getString(1); + version = rs.getLong(2); + state = rs.getString(3); + } + } + if (version != expectedVersion) { + throw new StaleJobException(id, expectedVersion); + } + if (!isReplaceableState(state)) { + return false; + } + Job replaced = JobReplacements.apply(serializer.deserializeJob(body), replacement); + long nextVersion = version + 1; + JobSnapshot snapshot = withVersion(replaced, nextVersion); + String newBody = serializer.serializeJob(snapshot, capabilities); + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_jobs SET " + + "queue = ?, priority = ?, handler_signature = ?, scheduled_at = ?, " + + "current_state_at = ?, version = ?, body = ?, concurrency_key = ?, " + + "concurrency_mode = ?, workflow_root_id = ?, parent_job_id = ? " + + "WHERE id = ? AND version = ?")) { + ps.setString(1, snapshot.queue()); + ps.setInt(2, snapshot.priority()); + ps.setString(3, snapshot.spec().handlerType()); + setInstant(ps, 4, snapshot.scheduledFor()); + setInstant(ps, 5, lastTransitionTime(snapshot, snapshot.currentState())); + ps.setLong(6, nextVersion); + clobs.bind(ps, 7, newBody); + setNullableConcurrency(ps, 8, snapshot.concurrencyKey(), snapshot.concurrencyMode()); + setUuid(ps, 10, snapshot.workflowRootId().asUuid()); + setUuid(ps, 11, parentJobId(snapshot)); + setUuid(ps, 12, id.asUuid()); + ps.setLong(13, expectedVersion); + return ps.executeUpdate() > 0; + } + }); + } catch (SQLException e) { + throw new JdbcException("replaceJob failed", e); + } + } + + private static boolean isReplaceableState(String state) { + return "ENQUEUED".equals(state) || "SCHEDULED".equals(state) || "AWAITING".equals(state); + } + + // ---------------------------------------------------------------- cron tasks + + @Override + public void upsertCronTask(CronTask task) { + Objects.requireNonNull(task, "task"); + Names.requireName("cronTask", task.name()); + Names.requireName("queue", task.queue()); + String kind; + String value; + if (task.trigger() instanceof CronTask.Trigger.CronExpr cron) { + kind = "CRON"; + value = cron.expression().expression(); + } else if (task.trigger() instanceof CronTask.Trigger.Interval interval) { + kind = "INTERVAL"; + value = interval.interval().toString(); + } else { + throw new IllegalStateException("Unknown trigger kind: " + task.trigger()); + } + try { + ownedTransaction(conn -> { + try (var clobs = new OracleClobs()) { + upsert( + conn, + "UPDATE threadmill_cron_tasks SET trigger_kind = ?, trigger_value = ?, " + + "handler_signature = ?, payload_type_tag = ?, payload_serialized = ?, queue = ?, " + + "priority = ?, timeout_seconds = ?, max_attempts = ?, is_exclusive = ?, " + + "missed_run_policy = ?, time_zone = ?, enabled = ? WHERE name = ?", + ps -> { + bindCronTaskValues(clobs, ps, 1, task, kind, value); + ps.setString(14, task.name()); + }, + "INSERT INTO threadmill_cron_tasks (trigger_kind, trigger_value, handler_signature, " + + "payload_type_tag, payload_serialized, queue, priority, timeout_seconds, " + + "max_attempts, is_exclusive, missed_run_policy, time_zone, enabled, name) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ps -> { + bindCronTaskValues(clobs, ps, 1, task, kind, value); + ps.setString(14, task.name()); + }); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("upsertCronTask failed", e); + } + } + + private static void bindCronTaskValues( + OracleClobs clobs, PreparedStatement ps, int start, CronTask task, String kind, String value) + throws SQLException { + ps.setString(start, kind); + ps.setString(start + 1, value); + ps.setString(start + 2, task.handlerType()); + ps.setString(start + 3, task.payloadArgument().typeTag()); + clobs.bind(ps, start + 4, task.payloadArgument().serialized()); + ps.setString(start + 5, task.queue()); + ps.setInt(start + 6, task.priority()); + if (task.timeout() == null) { + ps.setNull(start + 7, Types.BIGINT); + } else { + ps.setLong(start + 7, task.timeout().toSeconds()); + } + if (task.maxAttempts() == null) { + ps.setNull(start + 8, Types.INTEGER); + } else { + ps.setInt(start + 8, task.maxAttempts()); + } + ps.setInt(start + 9, task.exclusive() ? 1 : 0); + ps.setString(start + 10, task.missedRunPolicy().name()); + ps.setString(start + 11, task.zone().getId()); + ps.setInt(start + 12, task.enabled() ? 1 : 0); + } + + @Override + public Optional findCronTask(String name) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT " + CRON_TASK_COLUMNS + " FROM threadmill_cron_tasks WHERE name = ?")) { + ps.setString(1, name); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() ? Optional.of(readCronTask(rs)) : Optional.empty(); + } + } catch (SQLException e) { + throw new JdbcException("findCronTask failed", e); + } + } + + @Override + public List listCronTasks() { + var out = new ArrayList(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("SELECT " + CRON_TASK_COLUMNS + + " FROM threadmill_cron_tasks ORDER BY " + binary("name"))) { + ps.setFetchSize(500); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) out.add(readCronTask(rs)); + } + } catch (SQLException e) { + throw new JdbcException("listCronTasks failed", e); + } + return out; + } + + @Override + public void deleteCronTask(String name) { + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = + conn.prepareStatement("DELETE FROM threadmill_cron_tasks WHERE name = ?")) { + ps.setString(1, name); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("deleteCronTask failed", e); + } + } + + @Override + public void recordCronTaskOwnership(String namespace, String taskName) { + Names.requireName("cronTaskNamespace", namespace); + Names.requireName("cronTask", taskName); + try { + ownedTransaction(conn -> { + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO threadmill_cron_task_ownership (namespace, task_name) VALUES (?, ?)")) { + ps.setString(1, namespace); + ps.setString(2, taskName); + ps.executeUpdate(); + } catch (SQLException e) { + if (!hasErrorCode(e, UNIQUE_VIOLATION)) throw e; // already recorded + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("recordCronTaskOwnership failed", e); + } + } + + @Override + public Set listCronTaskNamesOwnedBy(String namespace) { + Names.requireName("cronTaskNamespace", namespace); + var out = new HashSet(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "SELECT task_name FROM threadmill_cron_task_ownership WHERE namespace = ?")) { + ps.setString(1, namespace); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) out.add(rs.getString(1)); + } + return Set.copyOf(out); + } catch (SQLException e) { + throw new JdbcException("listCronTaskNamesOwnedBy failed", e); + } + } + + @Override + public void upsertCronTaskState(CronTaskScheduleState state) { + Objects.requireNonNull(state, "state"); + try { + ownedTransaction(conn -> { + // The nudge columns are deliberately absent: only requestCronNudge and + // clearCronNudge write them, so this blanket upsert cannot clobber a + // concurrently accepted nudge. + upsert( + conn, + "UPDATE threadmill_cron_task_state SET last_run_at = ?, last_run_job_id = ?, " + + "next_run_at = ?, in_flight_job_id = ?, timing_fingerprint = ? WHERE task_name = ?", + ps -> { + bindCronStateValues(ps, state); + ps.setString(6, state.taskName()); + }, + "INSERT INTO threadmill_cron_task_state (last_run_at, last_run_job_id, next_run_at, " + + "in_flight_job_id, timing_fingerprint, task_name) VALUES (?, ?, ?, ?, ?, ?)", + ps -> { + bindCronStateValues(ps, state); + ps.setString(6, state.taskName()); + }); + return null; + }); + } catch (SQLException e) { + throw new JdbcException("upsertCronTaskState failed", e); + } + } + + private static void bindCronStateValues(PreparedStatement ps, CronTaskScheduleState state) + throws SQLException { + setInstant(ps, 1, state.lastRunAt()); + setUuid(ps, 2, state.lastRunJobId()); + setInstant(ps, 3, state.nextRunAt()); + setUuid(ps, 4, state.inFlightJobId()); + ps.setString(5, state.timingFingerprint()); + } + + @Override + public Optional findCronTaskState(String name) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("SELECT task_name, last_run_at, " + + "last_run_job_id, next_run_at, in_flight_job_id, timing_fingerprint, " + + "nudge_requested_at, nudge_revision FROM threadmill_cron_task_state " + + "WHERE task_name = ?")) { + ps.setString(1, name); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return Optional.empty(); + long revision = rs.getLong(8); + boolean revisionNull = rs.wasNull(); + return Optional.of(new CronTaskScheduleState( + rs.getString(1), + getInstant(rs, 2), + getUuid(rs, 3), + getInstant(rs, 4), + getUuid(rs, 5), + rs.getString(6), + getInstant(rs, 7), + revisionNull ? null : revision)); + } + } catch (SQLException e) { + throw new JdbcException("findCronTaskState failed", e); + } + } + + @Override + public NudgeOutcome requestCronNudge(String taskName, Instant requestedAt) { + Names.requireName("cronTask", taskName); + Objects.requireNonNull(requestedAt, "requestedAt"); + try { + return ownedTransaction(conn -> { + // Lock the task row so acceptance is atomic with the existence and + // enabled checks: a concurrent delete or disable waits for this + // transaction, and a nudge racing removal cannot resurrect state. + try (PreparedStatement ps = conn.prepareStatement( + "SELECT enabled FROM threadmill_cron_tasks WHERE name = ? FOR UPDATE")) { + ps.setString(1, taskName); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return NudgeOutcome.UNKNOWN_TASK; + if (rs.getInt(1) != 1) return NudgeOutcome.DISABLED; + } + } + // The revision advances on every acceptance and never resets while + // the schedule-state row exists, giving compare-and-clear a + // collision-free identity within one task lifecycle. + try { + upsert( + conn, + "UPDATE threadmill_cron_task_state SET nudge_requested_at = ?, " + + "nudge_revision = NVL(nudge_revision, 0) + 1 WHERE task_name = ?", + ps -> { + setInstant(ps, 1, requestedAt); + ps.setString(2, taskName); + }, + "INSERT INTO threadmill_cron_task_state (task_name, nudge_requested_at, " + + "nudge_revision) VALUES (?, ?, 1)", + ps -> { + ps.setString(1, taskName); + setInstant(ps, 2, requestedAt); + }); + } catch (SQLException e) { + if (hasErrorCode(e, PARENT_KEY_NOT_FOUND)) return NudgeOutcome.UNKNOWN_TASK; + throw e; + } + return NudgeOutcome.ACCEPTED; + }); + } catch (SQLException e) { + throw new JdbcException("requestCronNudge failed", e); + } + } + + @Override + public void clearCronNudge(String taskName, long observedRevision) { + Names.requireName("cronTask", taskName); + try { + ownedTransaction(conn -> { + // Clears the pending flag only; the revision is never reset, so + // cleared identities cannot be reused by a later acceptance. + try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_cron_task_state " + + "SET nudge_requested_at = NULL WHERE task_name = ? AND nudge_revision = ? " + + "AND nudge_requested_at IS NOT NULL")) { + ps.setString(1, taskName); + ps.setLong(2, observedRevision); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new JdbcException("clearCronNudge failed", e); + } + } + + private CronTask readCronTask(ResultSet rs) throws SQLException { + String kind = rs.getString(2); + String value = rs.getString(3); + CronTask.Trigger trigger = + switch (kind) { + case "CRON" -> new CronTask.Trigger.CronExpr(CronExpression.parse(value)); + case "INTERVAL" -> new CronTask.Trigger.Interval(Duration.parse(value)); + default -> throw new SQLException("Unknown trigger_kind: " + kind); + }; + long timeoutSeconds = rs.getLong(9); + Duration timeout = rs.wasNull() ? null : Duration.ofSeconds(timeoutSeconds); + int maxAttemptsValue = rs.getInt(10); + Integer maxAttempts = rs.wasNull() ? null : maxAttemptsValue; + String payload = rs.getString(6); + return new CronTask( + rs.getString(1), + trigger, + rs.getString(4), + // Oracle stores an empty string as NULL. + new JobArgument(rs.getString(5), payload == null ? "" : payload), + rs.getString(7), + rs.getInt(8), + timeout, + maxAttempts, + rs.getInt(11) == 1, + CronTask.MissedRunPolicy.valueOf(rs.getString(12)), + ZoneId.of(rs.getString(13)), + rs.getInt(14) == 1); + } + + // ---------------------------------------------------------------- helpers + + @FunctionalInterface + private interface StatementSetup { + void apply(PreparedStatement ps) throws SQLException; + } + + /** + * Update-then-insert upsert. Two writers that both miss the UPDATE race on + * the INSERT; the loser sees ORA-00001 after the winner commits and retries + * its UPDATE. A failed statement does not abort an Oracle transaction. + */ + private static void upsert( + Connection conn, + String updateSql, + StatementSetup updateSetup, + String insertSql, + StatementSetup insertSetup) + throws SQLException { + for (int attempt = 0; ; attempt++) { + try (PreparedStatement update = conn.prepareStatement(updateSql)) { + updateSetup.apply(update); + if (update.executeUpdate() > 0) return; + } + try (PreparedStatement insert = conn.prepareStatement(insertSql)) { + insertSetup.apply(insert); + insert.executeUpdate(); + return; + } catch (SQLException e) { + if (!hasErrorCode(e, UNIQUE_VIOLATION) || attempt >= 2) throw e; + } + } + } + + /** + * Conditional upsert for leases: the UPDATE applies only while its + * condition holds. When it matches nothing, the INSERT either creates the + * row (acquired) or hits ORA-00001 because another holder owns it. + */ + private static boolean conditionalUpsert( + Connection conn, + String updateSql, + StatementSetup updateSetup, + String insertSql, + StatementSetup insertSetup) + throws SQLException { + try (PreparedStatement update = conn.prepareStatement(updateSql)) { + updateSetup.apply(update); + if (update.executeUpdate() > 0) return true; + } + try (PreparedStatement insert = conn.prepareStatement(insertSql)) { + insertSetup.apply(insert); + insert.executeUpdate(); + return true; + } catch (SQLException e) { + if (hasErrorCode(e, UNIQUE_VIOLATION)) return false; + throw e; + } + } + + private Job readJobWithHeartbeat(ResultSet rs) throws SQLException { + var job = serializer.deserializeJob(rs.getString(1)); + Instant heartbeat = getInstant(rs, 2); + if (heartbeat != null && job.ownerNodeId().isPresent()) job.updateHeartbeat(heartbeat); + return job; + } + + private List queryJobs(String sql, int expectedRows, StatementSetup setup) { + var out = new ArrayList(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + setup.apply(ps); + ps.setFetchSize(Math.clamp(expectedRows, 1, 500)); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + out.add(readJobWithHeartbeat(rs)); + } + } + } catch (SQLException e) { + throw new JdbcException("query failed: " + sql, e); + } + return out; + } + + private Optional firstInstant(String sql, StatementSetup setup, String operation) { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + setup.apply(ps); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() ? Optional.ofNullable(getInstant(rs, 1)) : Optional.empty(); + } + } catch (SQLException e) { + throw new JdbcException(operation + " failed", e); + } + } + + private Optional findActiveDedup( + Connection conn, String queue, String dedupKey, Instant now) throws SQLException { + UUID jobId; + Instant expiresAt; + try (PreparedStatement ps = conn.prepareStatement("SELECT job_id, expires_at " + + "FROM threadmill_dedup_keys WHERE queue = ? AND dedup_key = ? FOR UPDATE")) { + ps.setString(1, queue); + ps.setString(2, dedupKey); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) return Optional.empty(); + jobId = getUuid(rs, 1); + expiresAt = getInstant(rs, 2); + } + } + String state = null; + try (PreparedStatement ps = + conn.prepareStatement("SELECT state FROM threadmill_jobs WHERE id = ?")) { + setUuid(ps, 1, jobId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) state = rs.getString(1); + } + } + // Expired keys still coalesce while the referenced job is active. + if (state != null && (expiresAt.isAfter(now) || !isTerminal(JobState.valueOf(state)))) { + return Optional.of(JobId.of(jobId)); + } + try (PreparedStatement ps = conn.prepareStatement( + "DELETE FROM threadmill_dedup_keys WHERE queue = ? AND dedup_key = ?")) { + ps.setString(1, queue); + ps.setString(2, dedupKey); + ps.executeUpdate(); + } + return Optional.empty(); + } + + private JobSnapshot snapshotForInsert(Connection conn, Job job, long version) + throws SQLException { + if (job.version() > version) { + throw new IllegalStateException( + "Insert requires a new job; persisted version cannot be reset to " + version); + } + JobSnapshot s = withVersion(job, version); + if (s.relationship() == null) { + return s; + } + try (PreparedStatement ps = conn.prepareStatement("SELECT workflow_root_id, " + + "concurrency_key, concurrency_mode FROM threadmill_jobs WHERE id = ?")) { + setUuid(ps, 1, s.relationship().parentId().asUuid()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + return s; + } + String modeValue = rs.getString(3); + return new JobSnapshot( + s.id(), + s.spec(), + s.queue(), + s.priority(), + s.createdAt(), + s.cronTaskName(), + s.relationship(), + JobId.of(getUuid(rs, 1)), + rs.getString(2), + modeValue == null ? null : ConcurrencyMode.valueOf(modeValue), + s.stateHistory(), + new HashMap<>(s.metadata()), + s.log(), + s.progress(), + version, + s.ownerNodeId(), + s.ownerHeartbeatAt(), + s.lastCheckinAt(), + s.scheduledFor(), + s.result(), + s.attempts(), + s.failureDecision(), + s.executionRevision()); + } + } + } + + /** First claim page: locks at most twice the budget in rows. */ + static int narrowClaimPageSize(int cap) { + return Math.max(cap, cap * 2); + } + + private static boolean hasActiveWorkflowHoldForRoot(Connection conn, JobSnapshot candidate) + throws SQLException { + try (PreparedStatement ps = conn.prepareStatement( + "SELECT 1 FROM " + + "threadmill_concurrency_workflow_holds WHERE concurrency_key = ? AND workflow_root_id = ?")) { + ps.setString(1, candidate.concurrencyKey()); + setUuid(ps, 2, candidate.workflowRootId().asUuid()); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } + } + + private static void noteInsertedWorkflowDescendant(Connection conn, JobSnapshot snapshot) + throws SQLException { + if (snapshot.concurrencyKey() == null) { + return; + } + lockConcurrencyGroup(conn, snapshot.concurrencyKey()); + incrementWorkflowHoldOutstanding(conn, snapshot); + } + + /** Caller must already hold the concurrency-group row lock for the snapshot's key. */ + private static void incrementWorkflowHoldOutstanding(Connection conn, JobSnapshot snapshot) + throws SQLException { + if (snapshot.concurrencyKey() == null) { + return; + } + try (PreparedStatement ps = conn.prepareStatement( + "UPDATE threadmill_concurrency_workflow_holds " + + "SET outstanding = outstanding + 1 WHERE concurrency_key = ? AND workflow_root_id = ?")) { + ps.setString(1, snapshot.concurrencyKey()); + setUuid(ps, 2, snapshot.workflowRootId().asUuid()); + ps.executeUpdate(); + } + } + + /** Caller must already hold the concurrency-group row lock for the snapshot's key. */ + private static void acquireWorkflowHold(Connection conn, JobSnapshot snapshot) + throws SQLException { + if (snapshot.concurrencyKey() == null || hasActiveWorkflowHoldForRoot(conn, snapshot)) { + return; + } + int outstanding = Math.max(1, countOutstandingWorkflowJobs(conn, snapshot)); + try (PreparedStatement ps = conn.prepareStatement("INSERT INTO " + + "threadmill_concurrency_workflow_holds (concurrency_key, workflow_root_id, outstanding) " + + "VALUES (?, ?, ?)")) { + ps.setString(1, snapshot.concurrencyKey()); + setUuid(ps, 2, snapshot.workflowRootId().asUuid()); + ps.setInt(3, outstanding); + ps.executeUpdate(); + } + String column = snapshot.concurrencyMode() == ConcurrencyMode.EXCLUSIVE + ? "exclusive_in_flight" + : "shared_in_flight"; + try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_concurrency_groups SET " + + column + " = " + column + " + 1, last_modified = " + NOW_UTC + + " WHERE concurrency_key = ?")) { + ps.setString(1, snapshot.concurrencyKey()); + ps.executeUpdate(); + } + } + + private static int countOutstandingWorkflowJobs(Connection conn, JobSnapshot snapshot) + throws SQLException { + try (PreparedStatement ps = conn.prepareStatement("SELECT /*+ INDEX(j " + + "threadmill_jobs_workflow_idx) */ COUNT(*) FROM threadmill_jobs j WHERE " + + "outstanding_key = ? AND outstanding_root = ?")) { + ps.setString(1, snapshot.concurrencyKey()); + setUuid(ps, 2, snapshot.workflowRootId().asUuid()); + try (ResultSet rs = ps.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + } + + private static void adjustWorkflowHoldOnTransition( + Connection conn, JobSnapshot oldSnapshot, JobState newState) throws SQLException { + if (oldSnapshot.concurrencyKey() == null) { + return; + } + if (isTerminal(oldSnapshot.currentState()) && !isTerminal(newState)) { + // Mirror of the release for the terminal -> non-terminal resurrect (the + // FAILED -> SCHEDULED retry path). Without it the job is decremented + // twice and an EXCLUSIVE key can be released while a descendant still + // runs. Zero rows matched is fine: a standalone job whose hold was fully + // released re-registers at its next claim. + try (PreparedStatement ps = conn.prepareStatement("UPDATE " + + "threadmill_concurrency_workflow_holds SET outstanding = outstanding + 1 " + + "WHERE concurrency_key = ? AND workflow_root_id = ?")) { + ps.setString(1, oldSnapshot.concurrencyKey()); + setUuid(ps, 2, oldSnapshot.workflowRootId().asUuid()); + ps.executeUpdate(); + } + return; + } + if (isTerminal(oldSnapshot.currentState()) || !isTerminal(newState)) { + return; + } + releaseWorkflowHoldShare( + conn, + oldSnapshot.concurrencyKey(), + oldSnapshot.concurrencyMode(), + oldSnapshot.workflowRootId().asUuid()); + } + + /** + * One member of the root's workflow left the non-terminal population: + * decrement the hold's outstanding count and, when it reaches zero, delete + * the hold and free its in-flight slot. A missing hold row is a no-op — + * never-claimed standalone jobs hold nothing. Caller holds the group lock. + */ + private static void releaseWorkflowHoldShare( + Connection conn, String concurrencyKey, ConcurrencyMode mode, UUID root) throws SQLException { + int outstanding; + try (PreparedStatement ps = conn.prepareStatement("SELECT outstanding FROM " + + "threadmill_concurrency_workflow_holds WHERE concurrency_key = ? AND workflow_root_id = ? " + + "FOR UPDATE")) { + ps.setString(1, concurrencyKey); + setUuid(ps, 2, root); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + return; + } + outstanding = rs.getInt(1) - 1; + } + } + if (outstanding > 0) { + try (PreparedStatement ps = conn.prepareStatement("UPDATE " + + "threadmill_concurrency_workflow_holds SET outstanding = ? " + + "WHERE concurrency_key = ? AND workflow_root_id = ?")) { + ps.setInt(1, outstanding); + ps.setString(2, concurrencyKey); + setUuid(ps, 3, root); + ps.executeUpdate(); + } + return; + } + try (PreparedStatement ps = conn.prepareStatement( + "DELETE FROM " + + "threadmill_concurrency_workflow_holds WHERE concurrency_key = ? AND workflow_root_id = ?")) { + ps.setString(1, concurrencyKey); + setUuid(ps, 2, root); + ps.executeUpdate(); + } + String column = mode == ConcurrencyMode.EXCLUSIVE ? "exclusive_in_flight" : "shared_in_flight"; + try (PreparedStatement ps = conn.prepareStatement("UPDATE threadmill_concurrency_groups SET " + + column + " = GREATEST(" + column + " - 1, 0), last_modified = " + NOW_UTC + + " WHERE concurrency_key = ?")) { + ps.setString(1, concurrencyKey); + ps.executeUpdate(); + } + } + + private static void insertSnapshot( + Connection conn, JobSnapshot snapshot, String body, Instant currentStateAt, long version) + throws SQLException { + try (var clobs = new OracleClobs(); + PreparedStatement ps = conn.prepareStatement(JOB_INSERT)) { + bindJobInsert(clobs, ps, snapshot, body, currentStateAt, version); + ps.executeUpdate(); + } + } + + private static void bindJobInsert( + OracleClobs clobs, + PreparedStatement ps, + JobSnapshot snapshot, + String body, + Instant currentStateAt, + long version) + throws SQLException { + setUuid(ps, 1, snapshot.id().asUuid()); + ps.setString(2, snapshot.currentState().name()); + ps.setString(3, snapshot.queue()); + ps.setInt(4, snapshot.priority()); + ps.setString(5, snapshot.spec().handlerType()); + setInstant(ps, 6, snapshot.scheduledFor()); + setUuid( + ps, 7, snapshot.ownerNodeId() == null ? null : snapshot.ownerNodeId().asUuid()); + setInstant(ps, 8, snapshot.ownerHeartbeatAt()); + setInstant(ps, 9, snapshot.lastCheckinAt()); + setInstant(ps, 10, currentStateAt); + ps.setLong(11, version); + clobs.bind(ps, 12, body); + setInstant(ps, 13, snapshot.createdAt()); + setNullableConcurrency(ps, 14, snapshot.concurrencyKey(), snapshot.concurrencyMode()); + setUuid(ps, 16, snapshot.workflowRootId().asUuid()); + setUuid(ps, 17, parentJobId(snapshot)); + } + + private static UUID parentJobId(JobSnapshot snapshot) { + return snapshot.relationship() == null + ? null + : snapshot.relationship().parentId().asUuid(); + } + + private static void setNullableConcurrency( + PreparedStatement ps, int startIndex, String key, ConcurrencyMode mode) throws SQLException { + if (key == null) { + ps.setNull(startIndex, Types.VARCHAR); + ps.setNull(startIndex + 1, Types.VARCHAR); + } else { + ps.setString(startIndex, key); + ps.setString(startIndex + 1, mode.name()); + } + } + + private static Instant lastTransitionTime(JobSnapshot snapshot, JobState state) { + List history = snapshot.stateHistory(); + for (int i = history.size() - 1; i >= 0; i--) { + if (history.get(i).state() == state) return history.get(i).at(); + } + return snapshot.createdAt(); + } + + private static JobSnapshot withVersion(Job job, long version) { + JobSnapshot s = job.snapshot(); + return new JobSnapshot( + s.id(), + s.spec(), + s.queue(), + s.priority(), + s.createdAt(), + s.cronTaskName(), + s.relationship(), + s.workflowRootId(), + s.concurrencyKey(), + s.concurrencyMode(), + s.stateHistory(), + new HashMap<>(s.metadata()), + s.log(), + s.progress(), + version, + s.ownerNodeId(), + s.ownerHeartbeatAt(), + s.lastCheckinAt(), + s.scheduledFor(), + s.result(), + s.attempts(), + s.failureDecision(), + s.executionRevision()); + } + + private static boolean isTerminal(JobState state) { + return switch (state) { + case SUCCEEDED, FAILED, DELETED, QUARANTINED -> true; + case AWAITING, SCHEDULED, ENQUEUED, PROCESSING, PROCESSED -> false; + }; + } + + /** Translates {@link SQLException}s from a {@link JobStore} method into an unchecked form. */ + public static class JdbcException extends RuntimeException { + /** + * Wrap a JDBC failure. + * + * @param message the failed operation + * @param cause the driver's exception + */ + public JdbcException(String message, SQLException cause) { + super(message, cause); + } + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleMigrationRunner.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleMigrationRunner.java new file mode 100644 index 00000000..c0484d93 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleMigrationRunner.java @@ -0,0 +1,601 @@ +package com.hemju.threadmill.store.oracle; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Minimal in-process schema migrator for Oracle Database. + * + *

Migrations are SQL files on the classpath named {@code V__.sql} + * and listed explicitly (no classpath scanning, for native-image friendliness). + * Statements inside a file end with a line holding only {@code /}, the SQL*Plus + * convention, so {@link #emitPendingSql()} output runs unchanged in SQL*Plus or + * SQLcl for hosts whose application user has no DDL rights. + * + *

Oracle commits every DDL statement implicitly, so a migration cannot be + * applied atomically. The runner therefore: + *

    + *
  • serializes concurrent migrators with a row lock on + * {@code threadmill_schema_lock}, held on a second connection because + * each DDL statement would commit (and so release) a lock taken on the + * migrating connection;
    • + *
  • records progress in {@code threadmill_schema_history} after every + * statement, and resumes a crashed migration at the first unrecorded + * statement;
    • + *
  • tolerates an "object already exists" error only for that first resumed + * statement, the one statement whose DDL may have committed before the + * crash prevented its progress record;
    • + *
  • validates the description and SHA-256 checksum of every applied + * migration before applying anything.
    • + *
+ */ +public final class OracleMigrationRunner { + + private static final Pattern FILE_PATTERN = Pattern.compile("V(\\d+)__([A-Za-z0-9_]+)\\.sql"); + private static final String RESOURCE_ROOT = "com/hemju/threadmill/store/oracle/migrations/"; + private static final List SHIPPED_MIGRATIONS = List.of("V1__baseline.sql"); + private static final Logger LOG = LoggerFactory.getLogger(OracleMigrationRunner.class); + private static final Duration LOCK_ACQUIRE_TIMEOUT = Duration.ofMinutes(5); + private static final Duration LOCK_POLL_INTERVAL = Duration.ofMillis(250); + + /** Tables dropped by {@link #dropThreadmillObjects()}, children before parents. */ + private static final List THREADMILL_TABLES = List.of( + "threadmill_mutexes", + "threadmill_concurrency_workflow_holds", + "threadmill_concurrency_groups", + "threadmill_dedup_keys", + "threadmill_cron_task_ownership", + "threadmill_cron_task_state", + "threadmill_cron_tasks", + "threadmill_jobs", + "threadmill_nodes", + "threadmill_leases", + "threadmill_job_counts", + "threadmill_queue_counts", + "threadmill_queue_pauses", + "threadmill_schema_history"); + + /** ORA-00054: resource busy and acquire with NOWAIT specified. */ + private static final int RESOURCE_BUSY = 54; + + /** ORA-00942: table or view does not exist. */ + private static final int TABLE_MISSING = 942; + + /** ORA-00955: name is already used by an existing object. */ + private static final int NAME_IN_USE = 955; + + /** + * Errors that prove a re-executed DDL statement already took effect before + * a crash: the object, column, constraint, or index exists, or the dropped + * object is already gone. + */ + private static final Set ALREADY_APPLIED = Set.of( + NAME_IN_USE, + 1408, // such column list already indexed + 1418, // specified index does not exist (DROP INDEX) + 1430, // column being added already exists + 1442, // column to be modified to NOT NULL is already NOT NULL + 1451, // column to be modified to NULL cannot be modified to NULL + 2260, // table can have only one primary key + 2261, // such unique or primary key already exists + 2264, // name already used by an existing constraint + 2275, // such a referential constraint already exists + TABLE_MISSING); + + private final DataSource dataSource; + + /** + * Create a runner for the schema reachable through {@code dataSource}. + * + * @param dataSource connections whose current schema receives the Threadmill objects + */ + public OracleMigrationRunner(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + } + + /** Apply every pending migration, resuming one that a crash interrupted. Idempotent. */ + public void migrate() { + List all = loadAll(); + try (Connection lock = dataSource.getConnection(); + Connection conn = dataSource.getConnection()) { + bootstrapLockTable(conn); + boolean previousLockAutoCommit = lock.getAutoCommit(); + acquireMigrationLock(lock); + try { + ensureHistoryTable(conn); + Map applied = validateAppliedMigrations(conn, all, false); + for (Migration m : all) { + AppliedMigration progress = applied.get(m.version()); + if (progress != null && progress.complete()) continue; + applyOne(conn, m, progress); + } + } finally { + releaseMigrationLock(lock, previousLockAutoCommit); + } + } catch (SQLException e) { + throw new MigrationException("Migration failed", e); + } + } + + /** + * Return the SQL for migrations that have not been applied yet, in SQL*Plus + * format. Strictly read-only, so it works for roles without DDL rights. When + * the history table does not exist, its DDL is prepended (never executed). + * + * @return the pending SQL script, empty when the schema is current + */ + public String emitPendingSql() { + try (Connection conn = dataSource.getConnection()) { + var sb = new StringBuilder(); + Map applied; + if (tableExists(conn, "THREADMILL_SCHEMA_HISTORY")) { + applied = readAppliedMigrations(conn); + } else { + appendStatement(sb, historyTableSql()); + applied = Map.of(); + } + for (Migration m : loadAll()) { + AppliedMigration progress = applied.get(m.version()); + if (progress != null && progress.complete()) continue; + appendMigrationSql(sb, m, progress == null ? 0 : progress.statementsApplied()); + } + return sb.toString(); + } catch (SQLException e) { + throw new MigrationException("Emit-pending-SQL failed", e); + } + } + + /** + * Return the SQL that installs Threadmill's full schema into an empty schema. + * + * @return the clean-install SQL script in SQL*Plus format + */ + public String emitCleanInstallSql() { + var sb = new StringBuilder(); + appendStatement(sb, historyTableSql()); + for (Migration m : loadAll()) { + appendMigrationSql(sb, m, 0); + } + return sb.toString(); + } + + /** Validate that the schema has exactly the shipped migrations, each completely applied. */ + public void validate() { + try (Connection conn = dataSource.getConnection()) { + if (!tableExists(conn, "THREADMILL_SCHEMA_HISTORY")) { + throw new MigrationException( + "Threadmill schema is missing: threadmill_schema_history does not exist"); + } + validateAppliedMigrations(conn, loadAll(), true); + } catch (SQLException e) { + throw new MigrationException("Schema validation failed", e); + } + } + + /** + * Drop every Threadmill table (with its indexes and trigger). Destroys all + * Threadmill data; intended for disposable environments. The small + * {@code threadmill_schema_lock} table is kept because this call holds its lock. + */ + public void dropThreadmillObjects() { + try (Connection lock = dataSource.getConnection(); + Connection conn = dataSource.getConnection()) { + bootstrapLockTable(conn); + boolean previousLockAutoCommit = lock.getAutoCommit(); + acquireMigrationLock(lock); + try (Statement st = conn.createStatement()) { + for (String table : THREADMILL_TABLES) { + try { + st.execute("DROP TABLE " + table + " CASCADE CONSTRAINTS PURGE"); + } catch (SQLException e) { + if (e.getErrorCode() != TABLE_MISSING) throw e; + } + } + } finally { + releaseMigrationLock(lock, previousLockAutoCommit); + } + } catch (SQLException e) { + throw new MigrationException("Failed to drop Threadmill schema objects", e); + } + } + + // ------------------------------------------------------------------ applying + + private void applyOne(Connection conn, Migration m, AppliedMigration progress) + throws SQLException { + int next = progress == null ? 0 : progress.statementsApplied(); + if (progress == null) { + OracleTransactions.execute(conn, tx -> { + try (PreparedStatement ps = tx.prepareStatement( + "INSERT INTO threadmill_schema_history (version, description, checksum, " + + "statements_applied, statements_total, success) VALUES (?, ?, ?, 0, ?, 0)")) { + ps.setInt(1, m.version()); + ps.setString(2, m.description()); + ps.setString(3, m.checksum()); + ps.setInt(4, m.statements().size()); + ps.executeUpdate(); + } + return null; + }); + } else { + LOG.warn( + "Resuming interrupted Threadmill migration {} at statement {} of {}", + m.fileName(), + next + 1, + m.statements().size()); + } + boolean resumed = progress != null; + for (int i = next; i < m.statements().size(); i++) { + String sql = m.statements().get(i); + int applied = i + 1; + boolean firstResumed = resumed && i == next; + try { + OracleTransactions.execute(conn, tx -> { + // DML commits atomically with its progress record; DDL commits on + // its own, which is why a resumed first statement may already exist. + executeStatement(tx, sql, firstResumed); + try (PreparedStatement ps = tx.prepareStatement( + "UPDATE threadmill_schema_history SET statements_applied = ? WHERE version = ?")) { + ps.setInt(1, applied); + ps.setInt(2, m.version()); + ps.executeUpdate(); + } + return null; + }); + } catch (SQLException e) { + throw new MigrationException( + "Migration " + m.fileName() + " failed at statement " + applied + " of " + + m.statements().size() + "; rerun to resume after fixing the cause", + e); + } + } + OracleTransactions.execute(conn, tx -> { + try (PreparedStatement ps = tx.prepareStatement("UPDATE threadmill_schema_history " + + "SET success = 1, applied_at = SYS_EXTRACT_UTC(SYSTIMESTAMP) WHERE version = ?")) { + ps.setInt(1, m.version()); + ps.executeUpdate(); + } + return null; + }); + } + + private static void executeStatement(Connection conn, String sql, boolean tolerateApplied) + throws SQLException { + try (Statement st = conn.createStatement()) { + st.setEscapeProcessing(false); + st.execute(sql); + } catch (SQLException e) { + if (tolerateApplied && ALREADY_APPLIED.contains(e.getErrorCode())) { + LOG.info("Statement already applied before the interruption ({})", e.getMessage()); + return; + } + throw e; + } + } + + // ------------------------------------------------------------------ history + + private static Map validateAppliedMigrations( + Connection conn, List shipped, boolean requireComplete) throws SQLException { + Map applied = readAppliedMigrations(conn); + if (requireComplete && applied.size() != shipped.size()) { + throw new MigrationException("Threadmill schema history has " + applied.size() + + " migration(s), but this version ships " + shipped.size() + + "; run migrations or repair the schema history"); + } + for (AppliedMigration actual : applied.values()) { + Migration expected = shipped.stream() + .filter(m -> m.version() == actual.version()) + .findFirst() + .orElseThrow(() -> new MigrationException("Threadmill schema history contains unknown " + + "version " + actual.version() + + "; the schema was migrated by a newer Threadmill binary")); + if (!expected.description().equals(actual.description())) { + throw new MigrationException("Threadmill schema history version " + actual.version() + + " with description '" + actual.description() + "', expected '" + + expected.description() + "'"); + } + if (!expected.checksum().equals(actual.checksum())) { + throw new MigrationException("Threadmill schema history version " + actual.version() + + " has checksum " + actual.checksum() + " but the shipped migration hashes to " + + expected.checksum() + " — the migration file was edited after it was applied"); + } + if (requireComplete && !actual.complete()) { + throw new MigrationException("Threadmill migration " + expected.fileName() + + " is incomplete (" + actual.statementsApplied() + " of " + + expected.statements().size() + " statements); run migrations to resume it"); + } + } + return applied; + } + + private static Map readAppliedMigrations(Connection conn) + throws SQLException { + var applied = new HashMap(); + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT version, description, checksum, " + + "statements_applied, success FROM threadmill_schema_history ORDER BY version")) { + while (rs.next()) { + applied.put( + rs.getInt(1), + new AppliedMigration( + rs.getInt(1), rs.getString(2), rs.getString(3), rs.getInt(4), rs.getInt(5) == 1)); + } + } + return applied; + } + + private static void ensureHistoryTable(Connection conn) throws SQLException { + createIgnoringExisting(conn, historyTableSql()); + } + + private static String historyTableSql() { + return "CREATE TABLE threadmill_schema_history (" + + "version NUMBER(10) NOT NULL, " + + "description VARCHAR2(200 CHAR) NOT NULL, " + + "checksum VARCHAR2(64) NOT NULL, " + + "statements_applied NUMBER(10) NOT NULL, " + + "statements_total NUMBER(10) NOT NULL, " + + "success NUMBER(1) NOT NULL, " + + "applied_at TIMESTAMP(6) DEFAULT SYS_EXTRACT_UTC(SYSTIMESTAMP) NOT NULL, " + + "CONSTRAINT threadmill_schema_history_pk PRIMARY KEY (version), " + + "CONSTRAINT threadmill_schema_history_ck CHECK (success IN (0, 1)))"; + } + + private static boolean tableExists(Connection conn, String upperCaseName) throws SQLException { + try (PreparedStatement ps = conn.prepareStatement("SELECT COUNT(*) FROM all_tables " + + "WHERE owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND table_name = ?")) { + ps.setString(1, upperCaseName); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() && rs.getInt(1) > 0; + } + } + } + + // ------------------------------------------------------------------ locking + + private static void bootstrapLockTable(Connection conn) throws SQLException { + createIgnoringExisting( + conn, + "CREATE TABLE threadmill_schema_lock (id NUMBER(1) NOT NULL, " + + "CONSTRAINT threadmill_schema_lock_pk PRIMARY KEY (id), " + + "CONSTRAINT threadmill_schema_lock_ck CHECK (id = 1))"); + OracleTransactions.execute(conn, tx -> { + try (Statement st = tx.createStatement()) { + st.executeUpdate("INSERT INTO threadmill_schema_lock (id) " + + "SELECT 1 FROM dual WHERE NOT EXISTS (SELECT 1 FROM threadmill_schema_lock)"); + } catch (SQLException e) { + // A concurrent migrator inserted the row first. + if (e.getErrorCode() != OracleJdbc.UNIQUE_VIOLATION) throw e; + } + return null; + }); + } + + private static void createIgnoringExisting(Connection conn, String ddl) throws SQLException { + try (Statement st = conn.createStatement()) { + st.execute(ddl); + } catch (SQLException e) { + if (e.getErrorCode() != NAME_IN_USE) throw e; + } + } + + /** + * Take the migration row lock on a dedicated connection with auto-commit off. + * Polls with {@code NOWAIT} and a bounded timeout, so a rolling deploy where + * another node is migrating logs a diagnostic and eventually fails with an + * actionable message instead of hanging startup silently. + */ + private static void acquireMigrationLock(Connection lock) throws SQLException { + lock.setAutoCommit(false); + long deadlineNanos = System.nanoTime() + LOCK_ACQUIRE_TIMEOUT.toNanos(); + boolean warned = false; + while (true) { + try (Statement st = lock.createStatement(); + ResultSet rs = st.executeQuery( + "SELECT id FROM threadmill_schema_lock WHERE id = 1 FOR UPDATE NOWAIT")) { + if (rs.next()) return; + throw new MigrationException("threadmill_schema_lock has no lock row"); + } catch (SQLException e) { + if (e.getErrorCode() != RESOURCE_BUSY) throw e; + } + if (System.nanoTime() >= deadlineNanos) { + throw new MigrationException("Timed out after " + LOCK_ACQUIRE_TIMEOUT + + " waiting for the Threadmill migration lock (row lock on threadmill_schema_lock)." + + " Another node is likely migrating — inspect v$locked_object."); + } + if (!warned) { + LOG.warn("Waiting for the Threadmill migration lock; another node is migrating."); + warned = true; + } + try { + Thread.sleep(LOCK_POLL_INTERVAL.toMillis()); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new MigrationException("Interrupted while waiting for the migration lock", ie); + } + } + } + + private static void releaseMigrationLock(Connection lock, boolean previousAutoCommit) { + // Swallow and log so an unlock failure cannot supersede the migration + // failure that names the failing statement. Oracle releases the row lock + // when the session ends anyway. + try { + lock.rollback(); + lock.setAutoCommit(previousAutoCommit); + } catch (SQLException unlockError) { + LOG.warn("Failed to release the Threadmill migration lock", unlockError); + } + } + + // ------------------------------------------------------------------ emitting + + private static void appendMigrationSql(StringBuilder sb, Migration m, int fromStatement) { + String nl = System.lineSeparator(); + sb.append("-- ").append(m.fileName()); + if (fromStatement > 0) { + sb.append(" (resuming at statement ").append(fromStatement + 1).append(')'); + } + sb.append(nl); + sb.append("-- Oracle commits each DDL statement; stop at the first error and rerun from it.") + .append(nl); + for (int i = fromStatement; i < m.statements().size(); i++) { + appendStatement(sb, m.statements().get(i)); + } + String description = m.description().replace("'", "''"); + if (fromStatement == 0) { + appendStatement( + sb, + "INSERT INTO threadmill_schema_history (version, description, checksum, " + + "statements_applied, statements_total, success) VALUES (" + m.version() + ", '" + + description + "', '" + m.checksum() + "', " + m.statements().size() + ", " + + m.statements().size() + ", 1)"); + } else { + appendStatement( + sb, + "UPDATE threadmill_schema_history SET statements_applied = " + + m.statements().size() + + ", success = 1, applied_at = SYS_EXTRACT_UTC(SYSTIMESTAMP) WHERE version = " + + m.version()); + } + appendStatement(sb, "COMMIT"); + } + + private static void appendStatement(StringBuilder sb, String sql) { + String nl = System.lineSeparator(); + sb.append(sql).append(nl).append('/').append(nl); + } + + // ------------------------------------------------------------------ loading + + private List loadAll() { + var out = new ArrayList(); + for (String name : SHIPPED_MIGRATIONS) { + var matcher = FILE_PATTERN.matcher(name); + if (!matcher.matches()) { + throw new MigrationException("Migration resource has invalid name: " + name); + } + String sql = readResource(RESOURCE_ROOT + name); + out.add(new Migration( + Integer.parseInt(matcher.group(1)), + name, + matcher.group(2).replace('_', ' '), + checksum(sql), + splitStatements(name, sql))); + } + out.sort(Comparator.comparingInt(Migration::version)); + return out; + } + + /** + * Split a migration into statements at lines holding only {@code /}. Leading + * comment lines are removed from each statement so a driver never has to + * classify a statement that starts with a comment. + */ + static List splitStatements(String fileName, String sql) { + var statements = new ArrayList(); + var current = new StringBuilder(); + for (String line : sql.split("\n", -1)) { + if (line.strip().equals("/")) { + addStatement(statements, current); + current.setLength(0); + } else { + current.append(line).append('\n'); + } + } + if (!stripLeadingComments(current.toString()).isBlank()) { + throw new MigrationException( + fileName + " has text after its last '/' terminator; end every statement with '/'"); + } + return List.copyOf(statements); + } + + private static void addStatement(List statements, StringBuilder text) { + String statement = stripLeadingComments(text.toString()).strip(); + if (!statement.isEmpty()) { + statements.add(statement); + } + } + + private static String stripLeadingComments(String text) { + var lines = text.split("\n", -1); + int first = 0; + while (first < lines.length + && (lines[first].isBlank() || lines[first].stripLeading().startsWith("--"))) { + first++; + } + return String.join("\n", List.of(lines).subList(first, lines.length)); + } + + private static String checksum(String sql) { + try { + byte[] digest = + MessageDigest.getInstance("SHA-256").digest(sql.getBytes(StandardCharsets.UTF_8)); + var sb = new StringBuilder(digest.length * 2); + for (byte b : digest) { + sb.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new MigrationException("SHA-256 unavailable", e); + } + } + + private String readResource(String path) { + try (InputStream in = OracleMigrationRunner.class.getClassLoader().getResourceAsStream(path)) { + if (in == null) throw new MigrationException("Migration resource not found: " + path); + try (var reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + var sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line).append('\n'); + } + return sb.toString(); + } + } catch (IOException e) { + throw new MigrationException("Failed to read migration: " + path, e); + } + } + + private record AppliedMigration( + int version, String description, String checksum, int statementsApplied, boolean complete) {} + + private record Migration( + int version, String fileName, String description, String checksum, List statements) {} + + /** Thrown when a migration operation fails. */ + public static class MigrationException extends RuntimeException { + public MigrationException(String message) { + super(message); + } + + public MigrationException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java new file mode 100644 index 00000000..98a1e40a --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java @@ -0,0 +1,118 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + +import com.hemju.threadmill.core.JobEngineFatalException; + +/** + * Startup requirements for the Oracle server Threadmill connects to. + * + *

Threadmill requires Oracle Database 19c or later, the oldest release still + * in long-term support, an {@code AL32UTF8} database character set so job + * bodies and names round-trip every Unicode character, including 4-byte ones, + * and binary text comparison ({@code NLS_COMP=BINARY}, the default). + * The check runs once at store construction so a misconfigured host fails fast + * with an actionable message instead of corrupting text at the first write. + */ +final class OracleServer { + + /** Oldest supported major release. */ + static final int MINIMUM_MAJOR_VERSION = 19; + + /** The only supported database character set. */ + static final String REQUIRED_CHARACTER_SET = "AL32UTF8"; + + /** + * The only supported session text comparison. Keyset predicates and the + * claim path's key ranges rely on binary comparisons matching index order. + */ + static final String REQUIRED_COMPARISON = "BINARY"; + + private static final Pattern VERSION = Pattern.compile("\\d+(?:\\.\\d+)+"); + + private OracleServer() {} + + /** + * What Threadmill needs to know about the server. + * + * @param majorVersion the database major release, for example 19 + * @param version the full release string, for example {@code 19.24.0.0.0} + * @param characterSet the database character set + * @param database the database (service) name + * @param schema the schema that holds the Threadmill objects + * @param comparison the session's {@code NLS_COMP} + */ + record Facts( + int majorVersion, + String version, + String characterSet, + String database, + String schema, + String comparison) { + + String describe() { + return "Oracle Database " + version + " @ " + database + "/" + schema; + } + } + + static Facts read(DataSource dataSource) { + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement()) { + var meta = conn.getMetaData(); + int major = meta.getDatabaseMajorVersion(); + String product = meta.getDatabaseProductVersion(); + var matcher = VERSION.matcher(product == null ? "" : product); + String version = matcher.find() ? matcher.group() : Integer.toString(major); + String characterSet; + try (ResultSet rs = st.executeQuery( + "SELECT value FROM nls_database_parameters " + "WHERE parameter = 'NLS_CHARACTERSET'")) { + characterSet = rs.next() ? rs.getString(1) : "unknown"; + } + String database; + String schema; + String comparison; + try (ResultSet rs = st.executeQuery("SELECT SYS_CONTEXT('USERENV', 'DB_NAME'), " + + "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA'), (SELECT value FROM nls_session_parameters " + + "WHERE parameter = 'NLS_COMP') FROM dual")) { + rs.next(); + database = rs.getString(1); + schema = rs.getString(2); + comparison = rs.getString(3); + } + return new Facts(major, version, characterSet, database, schema, comparison); + } catch (SQLException e) { + throw new JobEngineFatalException("Failed to verify the Oracle server version", e); + } + } + + /** + * Refuse unsupported servers. + * + * @throws JobEngineFatalException when the release is older than 19c, the + * character set is not {@code AL32UTF8}, or the session compares text + * linguistically + */ + static Facts require(Facts facts) { + if (facts.majorVersion() < MINIMUM_MAJOR_VERSION) { + throw new JobEngineFatalException("Threadmill requires Oracle Database 19c or later — found " + + facts.version() + ". Upgrade the server or use a different backend."); + } + if (!REQUIRED_CHARACTER_SET.equalsIgnoreCase(facts.characterSet())) { + throw new JobEngineFatalException("Threadmill requires the AL32UTF8 database character set " + + "so job bodies and names round-trip all Unicode text — found " + + facts.characterSet() + "."); + } + if (!REQUIRED_COMPARISON.equalsIgnoreCase(facts.comparison())) { + throw new JobEngineFatalException("Threadmill requires NLS_COMP=BINARY (the Oracle default) " + + "so key ranges match index order — found " + facts.comparison() + + ". Remove the NLS_COMP override for Threadmill's connections."); + } + return facts; + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactionBoundary.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactionBoundary.java new file mode 100644 index 00000000..7d991043 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactionBoundary.java @@ -0,0 +1,54 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.SQLException; + +import javax.sql.DataSource; + +/** + * Strategy that supplies the JDBC transaction boundary for Oracle store writes. + * + *

The default boundary opens a connection, starts a JDBC transaction, commits + * successful work, and rolls back failed work. Framework integrations may + * provide a boundary that joins an already active host transaction without + * putting framework types in this module. + */ +public interface OracleTransactionBoundary { + + /** + * Run one store write unit with a JDBC connection scoped to the desired + * transaction boundary. + * + * @param work the callback to run + * @param result type + * @return the callback result + * @throws SQLException when the connection or callback fails + */ + T inTransaction(OracleConnectionWork work) throws SQLException; + + /** + * Whether this boundary can join an external transaction, such as a Spring + * transaction bound to the same {@link DataSource}. + * + * @return {@code true} when writes may join a caller's transaction + */ + boolean supportsExternalTransactions(); + + /** + * Whether the current call is already inside an externally managed + * transaction. Store-level deadlock retry is disabled in this case: the + * caller owns the transaction, so only the caller can decide to retry it. + * + * @return {@code true} while a caller-owned transaction is active + */ + boolean externallyManagedTransactionActive(); + + /** + * Return the default boundary that owns its own JDBC transactions. + * + * @param dataSource the connection source + * @return a boundary that commits or rolls back every unit itself + */ + static OracleTransactionBoundary owning(DataSource dataSource) { + return new OwningOracleTransactionBoundary(dataSource); + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactions.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactions.java new file mode 100644 index 00000000..e0463be8 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleTransactions.java @@ -0,0 +1,52 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Shared cleanup policy for self-owned store and migration transactions. + * + *

The connection's previous auto-commit mode is restored afterwards: a pool + * may hand out {@code autoCommit=false} connections, and closing such a + * connection does not commit it, so every self-owned write commits explicitly. + */ +final class OracleTransactions { + private OracleTransactions() {} + + static T execute(Connection connection, OracleConnectionWork work) throws SQLException { + boolean previousAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + Throwable failure = null; + boolean restoreMode = true; + try { + var result = work.execute(connection); + connection.commit(); + return result; + } catch (SQLException | RuntimeException | Error original) { + failure = original; + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error rollbackFailure) { + original.addSuppressed(rollbackFailure); + // Restoring auto-commit after an unsuccessful rollback would commit + // the failed work. Discard this connection instead of returning it. + restoreMode = false; + try { + connection.abort(Runnable::run); + } catch (SQLException | RuntimeException | Error abortFailure) { + original.addSuppressed(abortFailure); + } + } + throw original; + } finally { + if (restoreMode) { + try { + connection.setAutoCommit(previousAutoCommit); + } catch (SQLException | RuntimeException | Error resetFailure) { + if (failure == null) throw resetFailure; + failure.addSuppressed(resetFailure); + } + } + } + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OwningOracleTransactionBoundary.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OwningOracleTransactionBoundary.java new file mode 100644 index 00000000..25ade7a0 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OwningOracleTransactionBoundary.java @@ -0,0 +1,33 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import javax.sql.DataSource; + +final class OwningOracleTransactionBoundary implements OracleTransactionBoundary { + + private final DataSource dataSource; + + OwningOracleTransactionBoundary(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + } + + @Override + public T inTransaction(OracleConnectionWork work) throws SQLException { + try (Connection conn = dataSource.getConnection()) { + return OracleTransactions.execute(conn, work); + } + } + + @Override + public boolean supportsExternalTransactions() { + return false; + } + + @Override + public boolean externallyManagedTransactionActive() { + return false; + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/PendingKeyCursors.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/PendingKeyCursors.java new file mode 100644 index 00000000..93177a99 --- /dev/null +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/PendingKeyCursors.java @@ -0,0 +1,67 @@ +package com.hemju.threadmill.store.oracle; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Bounded per-queue keyset cursors for the keyed claim lane, guarded by short + * in-memory critical sections; JDBC work always happens outside this object. + * + *

A cursor is a disposable fairness hint: it lets later polls reach keys + * beyond a page of blocked early keys. Mutations are accepted only from the + * exact generation a poll observed (reference identity), so a slow concurrent + * poll cannot overwrite a newer cursor. Access-order eviction keeps active + * queues' progress and drops the least recently polled hint when more queues + * are active than the bound retains. + */ +final class PendingKeyCursors { + + /** One cursor generation; equal {@code after} values remain distinct generations. */ + static final class Cursor { + private final String after; + + Cursor(String after) { + this.after = after; + } + + String after() { + return after; + } + } + + private final int maxTracked; + private final Map cursors = new LinkedHashMap<>(16, 0.75f, true); + + PendingKeyCursors(int maxTracked) { + if (maxTracked < 1) { + throw new IllegalArgumentException("maxTracked must be positive"); + } + this.maxTracked = maxTracked; + } + + synchronized Cursor current(String queue) { + return cursors.get(queue); + } + + synchronized void advance(String queue, Cursor expected, String nextKey) { + if (expected != null) { + if (cursors.get(queue) == expected) { + cursors.put(queue, new Cursor(nextKey)); + } + return; + } + if (cursors.containsKey(queue)) { + return; + } + if (cursors.size() >= maxTracked) { + cursors.remove(cursors.keySet().iterator().next()); + } + cursors.put(queue, new Cursor(nextKey)); + } + + synchronized void clear(String queue, Cursor expected) { + if (expected != null && cursors.get(queue) == expected) { + cursors.remove(queue); + } + } +} diff --git a/threadmill-store-oracle/src/main/resources/com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql b/threadmill-store-oracle/src/main/resources/com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql new file mode 100644 index 00000000..366ba8a4 --- /dev/null +++ b/threadmill-store-oracle/src/main/resources/com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql @@ -0,0 +1,451 @@ +-- Threadmill v1 baseline schema (Oracle Database 19c+). +-- +-- The whole schema installs in one migration. Post-release changes ship as +-- additive V2__*.sql, V3__*.sql, ... files; this baseline is never edited +-- again once released. OracleMigrationRunner bootstraps +-- threadmill_schema_history itself. +-- +-- Statement format: every statement ends with a line containing only "/" +-- (the SQL*Plus / SQLcl convention), so the file runs unchanged in those +-- tools. SQL statements carry no trailing semicolon; PL/SQL blocks do. +-- +-- Oracle commits DDL implicitly, so a migration is not atomic. The runner +-- records progress after every statement and resumes a crashed migration at +-- the first unrecorded statement. +-- +-- Type mapping and conventions: +-- * Job and node ids are RAW(16) in big-endian UUID byte order. RAW compares +-- unsigned byte by byte, which equals JobId's canonical natural order. +-- * Timestamps are TIMESTAMP(9) holding UTC wall time: nanosecond-exact for +-- java.time.Instant, time-zone independent, and directly indexable. +-- * Booleans are NUMBER(1) with a 0/1 check (BOOLEAN is 23ai-only). +-- * Text uses CHAR length semantics where values are user names, and the +-- database character set must be AL32UTF8 (checked at startup). +-- * Oracle has no partial indexes. Each one is emulated with VIRTUAL columns of +-- the form CASE WHEN THEN END and an index over them: +-- Oracle B-tree indexes skip rows whose key columns are all NULL, so the +-- index holds only rows matching . Virtual columns (rather than +-- expression indexes) give the queries plain column names and let the 19c +-- optimizer read index order directly, so per-key top-n probes stop after +-- n index entries instead of sorting the key's whole range. +-- +-- The body column is the source-of-truth wire form. The other job columns are +-- denormalised, indexed scalars kept in sync with the body on every write. + +CREATE TABLE threadmill_jobs ( + id RAW(16) NOT NULL, + state VARCHAR2(16) NOT NULL, + queue VARCHAR2(128 CHAR) NOT NULL, + priority NUMBER(10) DEFAULT 0 NOT NULL, + handler_signature VARCHAR2(1000 CHAR) NOT NULL, + scheduled_at TIMESTAMP(9), + owner_node_id RAW(16), + owner_heartbeat_at TIMESTAMP(9), + last_checkin_at TIMESTAMP(9), + current_state_at TIMESTAMP(9) NOT NULL, + version NUMBER(19) NOT NULL, + execution_revision NUMBER(19) DEFAULT 0 NOT NULL, + body CLOB NOT NULL, + created_at TIMESTAMP(9) NOT NULL, + concurrency_key VARCHAR2(256 BYTE), + concurrency_mode VARCHAR2(16), + workflow_root_id RAW(16) NOT NULL, + parent_job_id RAW(16), + -- Claim path, unkeyed lane: ENQUEUED jobs without a concurrency key, per + -- queue, in (priority DESC, id) order. The rank is -priority so the index + -- is ascending throughout. + unkeyed_queue VARCHAR2(128 CHAR) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NULL THEN queue END) VIRTUAL, + unkeyed_rank NUMBER(11) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NULL THEN -priority END) VIRTUAL, + unkeyed_id RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NULL THEN id END) VIRTUAL, + -- Claim path, keyed lane: per-queue distinct keys and each key's ENQUEUED + -- heads in the engine's (current_state_at, id) in-key order. + keyed_queue VARCHAR2(128 CHAR) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NOT NULL THEN queue END) VIRTUAL, + keyed_key VARCHAR2(256 BYTE) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NOT NULL THEN concurrency_key END) + VIRTUAL, + keyed_at TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NOT NULL THEN current_state_at END) + VIRTUAL, + keyed_id RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' AND concurrency_key IS NOT NULL THEN id END) VIRTUAL, + -- Claim-time admission: pending (ENQUEUED, SCHEDULED, AWAITING) keyed jobs. + pending_key VARCHAR2(256 BYTE) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + THEN concurrency_key END) VIRTUAL, + pending_at TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + AND concurrency_key IS NOT NULL THEN current_state_at END) VIRTUAL, + pending_id RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + AND concurrency_key IS NOT NULL THEN id END) VIRTUAL, + -- Claim-time admission: pending EXCLUSIVE jobs (the leapfrog rule). + exclusive_key VARCHAR2(256 BYTE) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + AND concurrency_mode = 'EXCLUSIVE' THEN concurrency_key END) VIRTUAL, + exclusive_at TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + AND concurrency_mode = 'EXCLUSIVE' THEN current_state_at END) VIRTUAL, + exclusive_id RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state IN ('ENQUEUED', 'SCHEDULED', 'AWAITING') + AND concurrency_mode = 'EXCLUSIVE' THEN id END) VIRTUAL, + -- Workflow holds: non-terminal keyed jobs by (key, workflow root). + outstanding_key VARCHAR2(256 BYTE) GENERATED ALWAYS AS ( + CASE WHEN state NOT IN ('SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED') + THEN concurrency_key END) VIRTUAL, + outstanding_root RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state NOT IN ('SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED') + AND concurrency_key IS NOT NULL THEN workflow_root_id END) VIRTUAL, + -- Due-for-promotion scan. + scheduled_due TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'SCHEDULED' THEN scheduled_at END) VIRTUAL, + -- Orphan recovery by the newer of owner heartbeat and check-in. COALESCE + -- keeps GREATEST from returning NULL when only one marker exists. + processing_liveness TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'PROCESSING' THEN GREATEST( + COALESCE(owner_heartbeat_at, last_checkin_at), + COALESCE(last_checkin_at, owner_heartbeat_at)) END) VIRTUAL, + -- Oldest processing heartbeat (monitoring). + processing_heartbeat TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'PROCESSING' THEN owner_heartbeat_at END) VIRTUAL, + -- Workflow successor promotion: AWAITING jobs of one parent. + awaiting_parent RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state = 'AWAITING' THEN parent_job_id END) VIRTUAL, + awaiting_at TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'AWAITING' AND parent_job_id IS NOT NULL + THEN current_state_at END) VIRTUAL, + awaiting_id RAW(16) GENERATED ALWAYS AS ( + CASE WHEN state = 'AWAITING' AND parent_job_id IS NOT NULL THEN id END) VIRTUAL, + -- Oldest ENQUEUED job per queue and idle-queue checks. + enqueued_queue VARCHAR2(128 CHAR) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' THEN queue END) VIRTUAL, + enqueued_at TIMESTAMP(9) GENERATED ALWAYS AS ( + CASE WHEN state = 'ENQUEUED' THEN current_state_at END) VIRTUAL, + CONSTRAINT threadmill_jobs_pk PRIMARY KEY (id), + CONSTRAINT threadmill_jobs_state_ck CHECK (state IN ('AWAITING', 'SCHEDULED', 'ENQUEUED', + 'PROCESSING', 'SUCCEEDED', 'FAILED', 'DELETED', 'QUARANTINED', 'PROCESSED')), + CONSTRAINT threadmill_jobs_mode_ck + CHECK (concurrency_mode IS NULL OR concurrency_mode IN ('SHARED', 'EXCLUSIVE')), + CONSTRAINT threadmill_jobs_shape_ck CHECK ( + (concurrency_key IS NULL AND concurrency_mode IS NULL) + OR (concurrency_key IS NOT NULL AND concurrency_mode IS NOT NULL)), + CONSTRAINT threadmill_jobs_version_ck CHECK (version >= 0), + CONSTRAINT threadmill_jobs_exec_rev_ck CHECK (execution_revision >= 0) +) +/ + +CREATE INDEX threadmill_jobs_unkeyed_idx + ON threadmill_jobs (unkeyed_queue, unkeyed_rank, unkeyed_id) +/ + +CREATE INDEX threadmill_jobs_keyed_idx + ON threadmill_jobs (keyed_queue, keyed_key, keyed_at, keyed_id) +/ + +CREATE INDEX threadmill_jobs_pending_idx + ON threadmill_jobs (pending_key, pending_at, pending_id) +/ + +CREATE INDEX threadmill_jobs_exclusive_idx + ON threadmill_jobs (exclusive_key, exclusive_at, exclusive_id) +/ + +CREATE INDEX threadmill_jobs_workflow_idx + ON threadmill_jobs (outstanding_key, outstanding_root) +/ + +CREATE INDEX threadmill_jobs_scheduled_idx ON threadmill_jobs (scheduled_due) +/ + +CREATE INDEX threadmill_jobs_liveness_idx ON threadmill_jobs (processing_liveness) +/ + +CREATE INDEX threadmill_jobs_heartbeat_idx ON threadmill_jobs (processing_heartbeat) +/ + +CREATE INDEX threadmill_jobs_awaiting_idx + ON threadmill_jobs (awaiting_parent, awaiting_at, awaiting_id) +/ + +CREATE INDEX threadmill_jobs_queue_age_idx ON threadmill_jobs (enqueued_queue, enqueued_at) +/ + +-- Retention keyset pages by (current_state_at, id) within one finished state. +CREATE INDEX threadmill_jobs_retention_idx + ON threadmill_jobs (state, current_state_at, id) +/ + +-- Stable maintenance keyset pages by id within one state. +CREATE INDEX threadmill_jobs_state_id_idx ON threadmill_jobs (state, id) +/ + +-- Dashboard history pages, newest transition first. +CREATE INDEX threadmill_jobs_state_page_idx + ON threadmill_jobs (state, current_state_at DESC, id) +/ + +-- Dashboard search filtered by state, queue, and handler. +CREATE INDEX threadmill_jobs_search_idx + ON threadmill_jobs (state, queue, handler_signature, current_state_at DESC, id) +/ + +CREATE INDEX threadmill_jobs_handler_idx ON threadmill_jobs (handler_signature) +/ + +CREATE TABLE threadmill_nodes ( + id RAW(16) NOT NULL, + last_heartbeat_at TIMESTAMP(9) NOT NULL, + CONSTRAINT threadmill_nodes_pk PRIMARY KEY (id) +) +/ + +-- Recurring task identity. payload_serialized is nullable only because Oracle +-- stores an empty string as NULL; the store reads NULL back as "". +CREATE TABLE threadmill_cron_tasks ( + name VARCHAR2(128 CHAR) NOT NULL, + trigger_kind VARCHAR2(16) NOT NULL, + trigger_value VARCHAR2(1000 CHAR) NOT NULL, + handler_signature VARCHAR2(1000 CHAR) NOT NULL, + payload_type_tag VARCHAR2(1000 CHAR) NOT NULL, + payload_serialized CLOB, + queue VARCHAR2(128 CHAR) DEFAULT 'default' NOT NULL, + priority NUMBER(10) DEFAULT 0 NOT NULL, + timeout_seconds NUMBER(19), + max_attempts NUMBER(10), + is_exclusive NUMBER(1) DEFAULT 0 NOT NULL, + missed_run_policy VARCHAR2(16) DEFAULT 'DROP' NOT NULL, + time_zone VARCHAR2(64 CHAR) DEFAULT 'UTC' NOT NULL, + enabled NUMBER(1) DEFAULT 1 NOT NULL, + CONSTRAINT threadmill_cron_tasks_pk PRIMARY KEY (name), + CONSTRAINT threadmill_cron_kind_ck CHECK (trigger_kind IN ('CRON', 'INTERVAL')), + CONSTRAINT threadmill_cron_policy_ck CHECK (missed_run_policy IN ('DROP', 'CATCH_UP')), + CONSTRAINT threadmill_cron_timeout_ck CHECK (timeout_seconds IS NULL OR timeout_seconds > 0), + CONSTRAINT threadmill_cron_attempts_ck CHECK (max_attempts IS NULL OR max_attempts > 0), + CONSTRAINT threadmill_cron_exclusive_ck CHECK (is_exclusive IN (0, 1)), + CONSTRAINT threadmill_cron_enabled_ck CHECK (enabled IN (0, 1)) +) +/ + +-- Recurring schedule state. The nudge columns are written only by +-- requestCronNudge / clearCronNudge; the blanket state upsert leaves them +-- untouched so re-registration cannot clobber a concurrently accepted nudge. +CREATE TABLE threadmill_cron_task_state ( + task_name VARCHAR2(128 CHAR) NOT NULL, + last_run_at TIMESTAMP(9), + last_run_job_id RAW(16), + next_run_at TIMESTAMP(9), + in_flight_job_id RAW(16), + timing_fingerprint VARCHAR2(4000 BYTE), + nudge_requested_at TIMESTAMP(9), + nudge_revision NUMBER(19), + CONSTRAINT threadmill_cron_state_pk PRIMARY KEY (task_name), + CONSTRAINT threadmill_cron_state_task_fk FOREIGN KEY (task_name) + REFERENCES threadmill_cron_tasks (name) ON DELETE CASCADE +) +/ + +CREATE INDEX threadmill_cron_state_due_idx ON threadmill_cron_task_state (next_run_at) +/ + +CREATE TABLE threadmill_cron_task_ownership ( + namespace VARCHAR2(128 CHAR) NOT NULL, + task_name VARCHAR2(128 CHAR) NOT NULL, + CONSTRAINT threadmill_cron_owner_pk PRIMARY KEY (namespace, task_name), + CONSTRAINT threadmill_cron_owner_task_fk FOREIGN KEY (task_name) + REFERENCES threadmill_cron_tasks (name) ON DELETE CASCADE +) +/ + +CREATE INDEX threadmill_cron_owner_task_idx ON threadmill_cron_task_ownership (task_name) +/ + +-- Cross-cluster named mutex with a lease. holder is nullable only because +-- Oracle stores an empty string as NULL; comparisons use DECODE, which treats +-- two NULLs as equal. +CREATE TABLE threadmill_mutexes ( + name VARCHAR2(128 CHAR) NOT NULL, + holder VARCHAR2(4000 BYTE), + expires_at TIMESTAMP(9) NOT NULL, + CONSTRAINT threadmill_mutexes_pk PRIMARY KEY (name) +) +/ + +-- Store-backed leadership lease for the maintenance cycle. +CREATE TABLE threadmill_leases ( + name VARCHAR2(64) NOT NULL, + holder RAW(16) NOT NULL, + expires_at TIMESTAMP(9) NOT NULL, + CONSTRAINT threadmill_leases_pk PRIMARY KEY (name) +) +/ + +-- Producer-side deduplication. Cleanup is gated on the referenced job being +-- terminal so a long-running active job keeps its dedup protection. +CREATE TABLE threadmill_dedup_keys ( + queue VARCHAR2(128 CHAR) NOT NULL, + dedup_key VARCHAR2(256 BYTE) NOT NULL, + job_id RAW(16) NOT NULL, + expires_at TIMESTAMP(9) NOT NULL, + CONSTRAINT threadmill_dedup_keys_pk PRIMARY KEY (queue, dedup_key), + CONSTRAINT threadmill_dedup_keys_job_fk FOREIGN KEY (job_id) + REFERENCES threadmill_jobs (id) ON DELETE CASCADE +) +/ + +CREATE INDEX threadmill_dedup_keys_exp_idx ON threadmill_dedup_keys (expires_at) +/ + +-- Without this index every retention delete scans the dedup table for the +-- cascade check. +CREATE INDEX threadmill_dedup_keys_job_idx ON threadmill_dedup_keys (job_id) +/ + +-- Claim-time per-key concurrency bookkeeping, updated in the same transaction +-- as the job state transition. +CREATE TABLE threadmill_concurrency_groups ( + concurrency_key VARCHAR2(256 BYTE) NOT NULL, + exclusive_in_flight NUMBER(10) DEFAULT 0 NOT NULL, + shared_in_flight NUMBER(10) DEFAULT 0 NOT NULL, + last_modified TIMESTAMP(9) NOT NULL, + -- Reclaimable groups (no job in flight) in binary key order. A RAW sort key + -- orders identically in every session, whatever its NLS_SORT; ordering the + -- text column would follow a linguistic NLS_SORT (derived from the client + -- JVM locale) and could neither use the index nor match keyset predicates. + idle_sort RAW(2000) GENERATED ALWAYS AS ( + CASE WHEN exclusive_in_flight = 0 AND shared_in_flight = 0 + THEN NLSSORT(concurrency_key, 'NLS_SORT = BINARY') END) VIRTUAL, + CONSTRAINT threadmill_concurrency_pk PRIMARY KEY (concurrency_key), + CONSTRAINT threadmill_concurrency_ex_ck + CHECK (exclusive_in_flight >= 0 AND exclusive_in_flight <= 1), + CONSTRAINT threadmill_concurrency_sh_ck CHECK (shared_in_flight >= 0), + CONSTRAINT threadmill_concurrency_mode_ck + CHECK (exclusive_in_flight = 0 OR shared_in_flight = 0) +) +/ + +-- Bounded keyset pages over reclaimable groups. +CREATE INDEX threadmill_concurrency_idle_idx ON threadmill_concurrency_groups (idle_sort) +/ + +-- Workflow-root outstanding counts: the key stays held from the root's claim +-- to the last descendant's terminal save. +CREATE TABLE threadmill_concurrency_workflow_holds ( + concurrency_key VARCHAR2(256 BYTE) NOT NULL, + workflow_root_id RAW(16) NOT NULL, + outstanding NUMBER(10) NOT NULL, + CONSTRAINT threadmill_workflow_holds_pk PRIMARY KEY (concurrency_key, workflow_root_id), + CONSTRAINT threadmill_workflow_holds_ck CHECK (outstanding >= 0) +) +/ + +-- Per-queue pause primitive. +CREATE TABLE threadmill_queue_pauses ( + queue VARCHAR2(128 CHAR) NOT NULL, + paused_at TIMESTAMP(9) NOT NULL, + paused_by CLOB, + CONSTRAINT threadmill_queue_pauses_pk PRIMARY KEY (queue) +) +/ + +-- Incrementally maintained per-state counts, sharded 16 ways by session so +-- concurrent writers update disjoint rows. Only the SUM per state is +-- meaningful; individual shards may go negative. Never COUNT(*) the jobs table. +CREATE TABLE threadmill_job_counts ( + state VARCHAR2(16) NOT NULL, + shard_no NUMBER(2) NOT NULL, + job_count NUMBER(19) DEFAULT 0 NOT NULL, + CONSTRAINT threadmill_job_counts_pk PRIMARY KEY (state, shard_no) +) +/ + +INSERT INTO threadmill_job_counts (state, shard_no, job_count) +SELECT s.state, sh.shard_no, 0 +FROM (SELECT 'AWAITING' AS state FROM dual + UNION ALL SELECT 'SCHEDULED' FROM dual + UNION ALL SELECT 'ENQUEUED' FROM dual + UNION ALL SELECT 'PROCESSING' FROM dual + UNION ALL SELECT 'PROCESSED' FROM dual + UNION ALL SELECT 'SUCCEEDED' FROM dual + UNION ALL SELECT 'FAILED' FROM dual + UNION ALL SELECT 'DELETED' FROM dual + UNION ALL SELECT 'QUARANTINED' FROM dual) s +CROSS JOIN (SELECT LEVEL - 1 AS shard_no FROM dual CONNECT BY LEVEL <= 16) sh +/ + +-- Incrementally maintained ENQUEUED depth per queue, sharded the same way. +-- Monitoring work scales with queue cardinality, not queued job count. +CREATE TABLE threadmill_queue_counts ( + queue VARCHAR2(128 CHAR) NOT NULL, + shard_no NUMBER(2) NOT NULL, + job_count NUMBER(19) NOT NULL, + CONSTRAINT threadmill_queue_counts_pk PRIMARY KEY (queue, shard_no) +) +/ + +-- Maintains both counter tables row by row. The shard is the session id +-- modulo 16, the Oracle analogue of PostgreSQL's backend pid. +CREATE OR REPLACE TRIGGER threadmill_jobs_counts_trg +AFTER INSERT OR DELETE OR UPDATE OF state, queue ON threadmill_jobs +FOR EACH ROW +DECLARE + sh PLS_INTEGER := MOD(TO_NUMBER(SYS_CONTEXT('USERENV', 'SID')), 16); + old_queue threadmill_jobs.queue%TYPE; + new_queue threadmill_jobs.queue%TYPE; + + PROCEDURE adjust_state(st VARCHAR2, delta PLS_INTEGER) IS + BEGIN + UPDATE threadmill_job_counts SET job_count = job_count + delta + WHERE state = st AND shard_no = sh; + END; + + PROCEDURE adjust_queue(q VARCHAR2, delta PLS_INTEGER) IS + BEGIN + UPDATE threadmill_queue_counts SET job_count = job_count + delta + WHERE queue = q AND shard_no = sh; + IF SQL%ROWCOUNT = 0 THEN + BEGIN + INSERT INTO threadmill_queue_counts (queue, shard_no, job_count) + VALUES (q, sh, delta); + EXCEPTION + WHEN DUP_VAL_ON_INDEX THEN + UPDATE threadmill_queue_counts SET job_count = job_count + delta + WHERE queue = q AND shard_no = sh; + END; + END IF; + END; +BEGIN + IF INSERTING THEN + adjust_state(:NEW.state, 1); + ELSIF DELETING THEN + adjust_state(:OLD.state, -1); + ELSIF :OLD.state <> :NEW.state THEN + adjust_state(:OLD.state, -1); + adjust_state(:NEW.state, 1); + END IF; + + IF NOT INSERTING AND :OLD.state = 'ENQUEUED' THEN + old_queue := :OLD.queue; + END IF; + IF NOT DELETING AND :NEW.state = 'ENQUEUED' THEN + new_queue := :NEW.queue; + END IF; + IF old_queue IS NOT NULL AND new_queue IS NOT NULL AND old_queue = new_queue THEN + RETURN; + END IF; + -- Stable order for queue replacements reduces cross-queue lock inversions. + IF old_queue IS NOT NULL AND new_queue IS NOT NULL AND new_queue < old_queue THEN + adjust_queue(new_queue, 1); + adjust_queue(old_queue, -1); + ELSE + IF old_queue IS NOT NULL THEN + adjust_queue(old_queue, -1); + END IF; + IF new_queue IS NOT NULL THEN + adjust_queue(new_queue, 1); + END IF; + END IF; +END; +/ diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/NonAutoCommitDataSource.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/NonAutoCommitDataSource.java new file mode 100644 index 00000000..5c6382fa --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/NonAutoCommitDataSource.java @@ -0,0 +1,93 @@ +package com.hemju.threadmill.store.oracle; + +import java.io.PrintWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.logging.Logger; + +import javax.sql.DataSource; + +/** Pool-alike test fixture whose borrowed connections start and must end with auto-commit disabled. */ +final class NonAutoCommitDataSource implements DataSource { + + private final DataSource delegate; + + NonAutoCommitDataSource(DataSource delegate) { + this.delegate = delegate; + } + + @Override + public Connection getConnection() throws SQLException { + return guarded(delegate.getConnection()); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + return guarded(delegate.getConnection(username, password)); + } + + private static Connection guarded(Connection connection) throws SQLException { + connection.setAutoCommit(false); + return (Connection) Proxy.newProxyInstance( + NonAutoCommitDataSource.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if (method.getName().equals("close") && method.getParameterCount() == 0) { + SQLException modeFailure = connection.getAutoCommit() + ? new SQLException("borrowed connection was not restored to autoCommit=false") + : null; + try { + connection.close(); + } catch (SQLException closeFailure) { + if (modeFailure == null) throw closeFailure; + modeFailure.addSuppressed(closeFailure); + } + if (modeFailure != null) throw modeFailure; + return null; + } + try { + return method.invoke(connection, arguments); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + + @Override + public PrintWriter getLogWriter() throws SQLException { + return delegate.getLogWriter(); + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException { + delegate.setLogWriter(out); + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException { + delegate.setLoginTimeout(seconds); + } + + @Override + public int getLoginTimeout() throws SQLException { + return delegate.getLoginTimeout(); + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return delegate.getParentLogger(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + return delegate.unwrap(iface); + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return delegate.isWrapperFor(iface); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetryTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetryTest.java new file mode 100644 index 00000000..cf722803 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleDeadlockRetryTest.java @@ -0,0 +1,79 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.sql.BatchUpdateException; +import java.sql.SQLException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +class OracleDeadlockRetryTest { + + private static SQLException ora(int code) { + return new SQLException("ORA-" + code, "61000", code); + } + + @Test + void retriesDeadlocksAndSerializationFailuresUntilSuccess() throws SQLException { + var attempts = new AtomicInteger(); + String result = OracleDeadlockRetry.run( + () -> { + int attempt = attempts.incrementAndGet(); + if (attempt == 1) throw ora(OracleDeadlockRetry.ORA_DEADLOCK); + if (attempt == 2) throw ora(OracleDeadlockRetry.ORA_SERIALIZATION_FAILURE); + return "done"; + }, + 5, + 1, + 2); + + assertThat(result).isEqualTo("done"); + assertThat(attempts).hasValue(3); + } + + @Test + void rethrowsOtherFailuresImmediately() { + var attempts = new AtomicInteger(); + var uniqueViolation = ora(OracleJdbc.UNIQUE_VIOLATION); + + assertThatThrownBy(() -> OracleDeadlockRetry.run( + () -> { + attempts.incrementAndGet(); + throw uniqueViolation; + }, + 5, + 1, + 2)) + .isSameAs(uniqueViolation); + assertThat(attempts).hasValue(1); + } + + @Test + void givesUpAfterTheAttemptBudget() { + var attempts = new AtomicInteger(); + + assertThatThrownBy(() -> OracleDeadlockRetry.run( + () -> { + attempts.incrementAndGet(); + throw ora(OracleDeadlockRetry.ORA_DEADLOCK); + }, + 3, + 1, + 2)) + .isInstanceOf(SQLException.class); + assertThat(attempts).hasValue(3); + } + + @Test + void recognisesErrorCodesThroughBatchAndCauseChains() { + var batch = new BatchUpdateException("batch failed", new int[0]); + batch.setNextException(ora(OracleDeadlockRetry.ORA_DEADLOCK)); + var wrapped = new SQLException("wrapper", ora(OracleJdbc.UNIQUE_VIOLATION)); + + assertThat(OracleDeadlockRetry.isRetryable(batch)).isTrue(); + assertThat(OracleJdbc.hasErrorCode(wrapped, OracleJdbc.UNIQUE_VIOLATION)).isTrue(); + assertThat(OracleDeadlockRetry.isRetryable(wrapped)).isFalse(); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreContractTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreContractTest.java new file mode 100644 index 00000000..f91c17d9 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreContractTest.java @@ -0,0 +1,38 @@ +package com.hemju.threadmill.store.oracle; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.test.AbstractJobStoreContractTest; +import com.hemju.threadmill.test.ClaimPoisonRegression; + +/** + * Runs the {@link AbstractJobStoreContractTest} against a real Oracle database. + * The exact suite the in-memory, PostgreSQL, and Redis stores pass must pass + * here — with every borrowed connection starting at {@code autoCommit=false} + * and asserting that the store restores that mode. + */ +class OracleJobStoreContractTest extends AbstractJobStoreContractTest { + + private DataSource dataSource; + + @BeforeEach + void resetDatabase() throws Exception { + dataSource = OracleTestDatabase.dataSource(); + OracleTestDatabase.reset(); + } + + @Test + void poisonSerializationDoesNotDiscardEarlierClaims() { + ClaimPoisonRegression.verify( + new OracleJobStore(dataSource, ClaimPoisonRegression.serializer(), store.capabilities())); + } + + @Override + protected JobStore createStore() { + return new OracleJobStore(new NonAutoCommitDataSource(OracleTestDatabase.dataSource())); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreRegressionTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreRegressionTest.java new file mode 100644 index 00000000..688ec0a4 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleJobStoreRegressionTest.java @@ -0,0 +1,606 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.JobState; +import com.hemju.threadmill.core.NodeId; +import com.hemju.threadmill.core.schedule.CronExpression; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; +import com.hemju.threadmill.core.store.JobStore; + +/** + * Oracle-specific regression tests. The contract suite + * ({@link OracleJobStoreContractTest}) covers the SPI semantics; these pin the + * Oracle-only mechanics behind them: lock-as-you-fetch claiming, plans bounded + * by indexes rather than backlog, Oracle's empty-string-is-NULL rule, time-zone + * independent timestamps, CLOB bodies, and counter-table exactness. + */ +class OracleJobStoreRegressionTest { + + private DataSource dataSource; + + @BeforeEach + void reset() throws SQLException { + dataSource = OracleTestDatabase.dataSource(); + OracleTestDatabase.reset(); + } + + private JobStore store() { + return new OracleJobStore(dataSource); + } + + private static Job job() { + return Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .build(); + } + + private static Job keyedJob(String key, ConcurrencyMode mode, int priority) { + return Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .concurrencyKey(key) + .concurrencyMode(mode) + .priority(priority) + .build(); + } + + private static Job keyedJob(String key) { + return keyedJob(key, ConcurrencyMode.EXCLUSIVE, 0); + } + + // ---------------------------------------------------------------- claim mechanics + + @Test + void unkeyedClaimCursorLocksOnlyTheRowsItFetches() throws Exception { + // The claim path relies on Oracle locking SKIP LOCKED rows as they are + // fetched (a locking query cannot carry a row limit). If the driver or + // server locked the whole result at open time, every claimer would pin + // the entire queue. + var store = store(); + for (int i = 0; i < 50; i++) store.insert(job()); + try (Connection claimer = dataSource.getConnection(); + Connection observer = dataSource.getConnection()) { + claimer.setAutoCommit(false); + observer.setAutoCommit(false); + try (PreparedStatement ps = claimer.prepareStatement(OracleJobStore.UNKEYED_CLAIM_SQL)) { + ps.setString(1, "default"); + ps.setFetchSize(4); + ps.setMaxRows(4); + try (ResultSet rs = ps.executeQuery()) { + int read = 0; + while (rs.next()) read++; + assertThat(read).isEqualTo(4); + } + assertThat(lockableEnqueuedRows(observer)).isEqualTo(46); + } finally { + claimer.rollback(); + observer.rollback(); + } + } + } + + private static int lockableEnqueuedRows(Connection conn) throws SQLException { + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery( + "SELECT id FROM threadmill_jobs WHERE state = 'ENQUEUED' FOR UPDATE SKIP LOCKED")) { + int rows = 0; + while (rs.next()) rows++; + return rows; + } + } + + @Test + void unkeyedClaimLocksOnlyANarrowPageSoConcurrentClaimersAreNotStarved() throws Exception { + var store = store(); + var jobs = new ArrayList(); + for (int i = 0; i < 300; i++) jobs.add(job()); + store.insertAll(jobs); + + int claimers = 8; + int perClaim = 10; + var start = new CountDownLatch(1); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>>(); + for (int i = 0; i < claimers; i++) { + futures.add(executor.submit(() -> { + start.await(); + return store.claimReady(NodeId.newId(), "default", perClaim, Instant.now()); + })); + } + start.countDown(); + Set seen = new HashSet<>(); + for (var future : futures) { + var claimed = future.get(60, TimeUnit.SECONDS); + assertThat(claimed).hasSize(perClaim); + for (Job j : claimed) assertThat(seen.add(j.id().asUuid())).isTrue(); + } + } + } + + @Test + void claimReadyIsAtomicAcrossManyConcurrentVirtualThreads() throws Exception { + var store = store(); + int total = 200; + var jobs = new ArrayList(); + for (int i = 0; i < total; i++) { + jobs.add(i % 2 == 0 ? job() : keyedJob("atomic:" + (i % 20), ConcurrencyMode.SHARED, 0)); + } + store.insertAll(jobs); + var claimed = new HashSet(); + var start = new CountDownLatch(1); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>>(); + for (int worker = 0; worker < 16; worker++) { + futures.add(executor.submit(() -> { + start.await(); + var mine = new ArrayList(); + List batch; + do { + batch = store.claimReady(NodeId.newId(), "default", 7, Instant.now()); + mine.addAll(batch); + } while (!batch.isEmpty()); + return mine; + })); + } + start.countDown(); + for (var future : futures) { + for (Job j : future.get(120, TimeUnit.SECONDS)) { + assertThat(claimed.add(j.id().asUuid())) + .as("double claim of %s", j.id()) + .isTrue(); + } + } + } + assertThat(claimed).hasSize(total); + assertThat(store.countsByState().get(JobState.PROCESSING)).isEqualTo(total); + } + + @Test + void keyedClaimGathersHeadsFromEveryKeyInOneCall() { + // Guards the per-key head gathering. A collection-driven LATERAL top-n + // returned no rows on Oracle while an equivalent query returned all of + // them; this pins that every key contributes its heads in one call. + var store = store(); + var jobs = new ArrayList(); + for (int key = 0; key < 12; key++) { + for (int i = 0; i < 3; i++) jobs.add(keyedJob("shared:" + key, ConcurrencyMode.SHARED, 0)); + } + store.insertAll(jobs); + + assertThat(store.claimReady(NodeId.newId(), "default", jobs.size(), Instant.now())) + .hasSize(jobs.size()); + } + + @Test + void keyedClaimRotationReachesClaimableWorkBeyondTheFirstBlockedPage() { + var setupStore = store(); + int blockedKeyCount = OracleJobStore.MAX_PENDING_KEYS_PER_PASS + 1; + var blockers = new ArrayList(); + for (int i = 0; i < blockedKeyCount; i++) blockers.add(keyedJob("blocked:%04d".formatted(i))); + setupStore.insertAll(blockers); + assertThat(setupStore.claimReady(NodeId.newId(), "default", blockedKeyCount, Instant.now())) + .hasSize(blockedKeyCount); + + var waiters = new ArrayList(); + for (int i = 0; i < blockedKeyCount; i++) waiters.add(keyedJob("blocked:%04d".formatted(i))); + setupStore.insertAll(waiters); + var laterClaimable = keyedJob("claimable:after-blocked-page"); + setupStore.insert(laterClaimable); + + // A fresh store starts at the first key: the first page holds only + // blocked keys, the second the extra blocker plus the independent key. + var claimingStore = store(); + assertThat(claimingStore.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .isEmpty(); + assertThat(claimingStore.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(laterClaimable.id()); + + // The short tail clears the cursor; work at an early key is reached when + // the next poll wraps to the beginning. + var earlyClaimable = keyedJob("available:after-wrap", ConcurrencyMode.EXCLUSIVE, 10); + setupStore.insert(earlyClaimable); + assertThat(claimingStore.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(earlyClaimable.id()); + } + + @Test + void drainedTailRestartsKeyedScanWithoutAnEmptyPoll() { + var setupStore = store(); + int blockedKeyCount = OracleJobStore.MAX_PENDING_KEYS_PER_PASS + 1; + var blockers = new ArrayList(); + for (int i = 0; i < blockedKeyCount; i++) blockers.add(keyedJob("blocked:%04d".formatted(i))); + setupStore.insertAll(blockers); + assertThat(setupStore.claimReady(NodeId.newId(), "default", blockedKeyCount, Instant.now())) + .hasSize(blockedKeyCount); + var waiters = new ArrayList(); + for (int i = 0; i < blockedKeyCount; i++) waiters.add(keyedJob("blocked:%04d".formatted(i))); + setupStore.insertAll(waiters); + + var claimingStore = store(); + assertThat(claimingStore.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .isEmpty(); + + // The only key after the first page was the look-ahead key. Drain it, + // then add work before the saved cursor: the next tail scan is empty and + // must restart from the beginning in the same poll. + assertThat(setupStore.softDelete(waiters.getLast().id())).isTrue(); + var earlyClaimable = keyedJob("available:after-drained-tail", ConcurrencyMode.EXCLUSIVE, 10); + setupStore.insert(earlyClaimable); + + assertThat(claimingStore.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(earlyClaimable.id()); + } + + @Test + void insertAllWithReversedKeyOrdersDoesNotManufactureDeadlocks() throws Exception { + var store = store(); + var keys = new ArrayList(); + for (int i = 0; i < 8; i++) keys.add("lock-order:" + i); + int rounds = 15; + for (int round = 0; round < rounds; round++) { + var ascending = new ArrayList(); + var descending = new ArrayList(); + for (int i = 0; i < keys.size(); i++) { + ascending.add(keyedJob(keys.get(i))); + descending.add(keyedJob(keys.get(keys.size() - 1 - i))); + } + var start = new CountDownLatch(1); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + Future a = executor.submit(() -> { + start.await(); + return store.insertAll(ascending); + }); + Future b = executor.submit(() -> { + start.await(); + return store.insertAll(descending); + }); + start.countDown(); + a.get(30, TimeUnit.SECONDS); + b.get(30, TimeUnit.SECONDS); + } + } + assertThat(store.countsByState().get(JobState.ENQUEUED)).isEqualTo(rounds * keys.size() * 2L); + } + + @Test + void unreadableBodyAtTheQueueHeadIsQuarantinedWithoutStallingTheQueue() throws SQLException { + var store = store(); + var head = job(); + var good1 = job(); + var good2 = job(); + store.insertAll(List.of(head, good1, good2)); + corruptBody(head); + + assertThat(store.claimReady(NodeId.newId(), "default", 3, Instant.now())) + .extracting(Job::id) + .containsExactlyInAnyOrder(good1.id(), good2.id()); + assertThat(stateOf(head)).isEqualTo("QUARANTINED"); + assertThat(store.countsByState().get(JobState.QUARANTINED)).isEqualTo(1L); + } + + @Test + void quarantinedKeyedWorkflowMemberReleasesTheWorkflowHold() throws SQLException { + var store = store(); + var root = keyedJob("project:corrupt"); + store.insert(root); + var claimedRoot = + store.claimReady(NodeId.newId(), "default", 1, Instant.now()).getFirst(); + var member = Job.builder() + .spec(JobSpec.of("com.example.Member", new JobArgument("java.lang.String", "\"x\""))) + .concurrencyKey("project:corrupt") + .concurrencyMode(ConcurrencyMode.EXCLUSIVE) + .workflowRootId(claimedRoot.id()) + .build(); + store.insert(member); + corruptBody(member); + long rootVersion = claimedRoot.version(); + claimedRoot.transitionTo(JobState.SUCCEEDED, Instant.now(), "test", null); + claimedRoot.clearOwner(); + store.saveAtomic(claimedRoot, rootVersion); + + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())).isEmpty(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("SELECT COUNT(*) FROM " + + "threadmill_concurrency_workflow_holds WHERE concurrency_key = ?")) { + ps.setString(1, "project:corrupt"); + try (ResultSet rs = ps.executeQuery()) { + rs.next(); + assertThat(rs.getInt(1)).as("hold rows left for the key").isZero(); + } + } + var next = keyedJob("project:corrupt"); + store.insert(next); + assertThat(store.claimReady(NodeId.newId(), "default", 1, Instant.now())) + .extracting(Job::id) + .containsExactly(next.id()); + } + + private void corruptBody(Job job) throws SQLException { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement( + "UPDATE threadmill_jobs SET body = '{not valid json' WHERE id = ?")) { + ps.setBytes(1, OracleJdbc.bytes(job.id().asUuid())); + ps.executeUpdate(); + } + } + + private String stateOf(Job job) throws SQLException { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = + conn.prepareStatement("SELECT state FROM threadmill_jobs WHERE id = ?")) { + ps.setBytes(1, OracleJdbc.bytes(job.id().asUuid())); + try (ResultSet rs = ps.executeQuery()) { + assertThat(rs.next()).isTrue(); + return rs.getString(1); + } + } + } + + // ---------------------------------------------------------------- value mapping + + @Test + void supplementaryCharactersSurviveLongClobBodiesAndPayloads() { + // Binding a long string straight into a CLOB made the Oracle driver split + // a surrogate pair that straddled its ~32 KiB chunk boundary, silently + // corrupting emoji in large job bodies. Every character here is a + // surrogate pair, so any chunk boundary falls inside one. Both bind paths + // are covered: direct (short) and temporary-LOB (long). + var store = store(); + for (int rockets : new int[] { + OracleClobs.DIRECT_BIND_MAX_CHARS / 2 - 400, OracleClobs.DIRECT_BIND_MAX_CHARS, 15_000 + }) { + String text = "\uD83D\uDE80".repeat(rockets); + var job = Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .metadata("note", text) + .build(); + job.log().info("𐀀 shipping ✈"); + store.insert(job); + + var loaded = store.findById(job.id()).orElseThrow(); + assertThat(loaded.metadata().get("note")).as("%s rockets", rockets).contains(text); + assertThat(loaded.log().snapshot().getFirst().message()).isEqualTo("𐀀 shipping ✈"); + + var task = new CronTask( + "rockets-" + rockets, + new CronTask.Trigger.CronExpr(CronExpression.parse("*/5 * * * *")), + "com.example.H", + new JobArgument("java.lang.String", text), + "default", + 0, + null, + null, + false, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true); + store.upsertCronTask(task); + assertThat(store.findCronTask(task.name())).contains(task); + } + } + + @Test + void instantsRoundTripIndependentlyOfJvmAndSessionTimeZones() throws SQLException { + var original = TimeZone.getDefault(); + var scheduledFor = Instant.parse("2026-03-29T01:30:00.123456789Z"); // inside a DST change + try { + TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Chatham")); + var store = new OracleJobStore(session("ALTER SESSION SET TIME_ZONE = 'Asia/Tokyo'")); + var job = Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .initialState(JobState.SCHEDULED) + .scheduledFor(scheduledFor) + .build(); + store.insert(job); + + TimeZone.setDefault(TimeZone.getTimeZone("America/Sao_Paulo")); + assertThat(store.findDueForPromotion(scheduledFor.minusNanos(1), 10)).isEmpty(); + assertThat(store.findDueForPromotion(scheduledFor, 10)) + .extracting(Job::id) + .containsExactly(job.id()); + assertThat(store.oldestMaintenanceAt(JobState.SCHEDULED)).contains(scheduledFor); + } finally { + TimeZone.setDefault(original); + } + } + + /** Connections that run {@code alterSession} first, like a host's logon setting. */ + private DataSource session(String alterSession) { + return (DataSource) Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[] {DataSource.class}, (proxy, m, args) -> { + try { + var result = m.invoke(dataSource, args); + if (m.getName().equals("getConnection") && result instanceof Connection conn) { + try (Statement st = conn.createStatement()) { + st.execute(alterSession); + } + } + return result; + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + + @Test + void keysetPagesFollowBinaryOrderUnderALinguisticClientLocale() throws SQLException { + // The driver derives NLS_LANGUAGE (and so NLS_SORT) from the client JVM + // locale. A German locale sorts a < Ä < B while keyset predicates compare + // binary (B < a < Ä): ordering pages by the text column skipped rows. + var store = new OracleJobStore(session("ALTER SESSION SET NLS_LANGUAGE = 'GERMAN'")); + var names = List.of("a", "B", "Ä", "ab", "b", "Z", "z", "é", "A", "Ab", "aB", "ä", "zz", "0"); + for (String name : names) { + store.upsertCronTask(new CronTask( + name, + new CronTask.Trigger.CronExpr(CronExpression.parse("*/5 * * * *")), + "com.example.H", + new JobArgument("java.lang.String", "\"x\""), + "default", + 0, + null, + null, + false, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true)); + } + var scanned = new ArrayList(); + String after = null; + for (int page = 0; page < names.size(); page++) { + var batch = store.scanCronTasks(after, 3); + batch.forEach(task -> scanned.add(task.name())); + if (batch.size() < 3) break; + after = batch.getLast().name(); + } + assertThat(scanned).containsExactlyInAnyOrderElementsOf(names).doesNotHaveDuplicates(); + + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement("INSERT INTO threadmill_concurrency_groups " + + "(concurrency_key, last_modified) VALUES (?, SYS_EXTRACT_UTC(SYSTIMESTAMP) " + + "- INTERVAL '1' HOUR)")) { + conn.setAutoCommit(true); + for (String name : names) { + ps.setString(1, name); + ps.executeUpdate(); + } + } + long removed = 0; + for (int page = 0; page < names.size() / 3 + 1; page++) { + long expected = Math.min(3, names.size() - removed); + assertThat(store.deleteIdleConcurrencyGroups(3)).as("page %s", page).isEqualTo(expected); + removed += expected; + } + assertThat(removed).isEqualTo(names.size()); + } + + @Test + void emptyStringsSurviveOraclesEmptyStringIsNullRule() { + var store = store(); + // Recurring payloads may serialize to the empty string. + var task = new CronTask( + "empty-payload", + new CronTask.Trigger.CronExpr(CronExpression.parse("*/5 * * * *")), + "com.example.H", + new JobArgument("java.lang.String", ""), + "default", + 0, + null, + null, + false, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true); + store.upsertCronTask(task); + assertThat(store.findCronTask("empty-payload")).contains(task); + + // A mutex holder may be empty; it must still match itself. + assertThat(store.tryAcquireMutex("empty-holder", "", Duration.ofSeconds(30))) + .isTrue(); + assertThat(store.tryAcquireMutex("empty-holder", "", Duration.ofSeconds(30))) + .isTrue(); + assertThat(store.tryAcquireMutex("empty-holder", "other", Duration.ofSeconds(30))) + .isFalse(); + store.releaseMutex("empty-holder", ""); + assertThat(store.tryAcquireMutex("empty-holder", "other", Duration.ofSeconds(30))) + .isTrue(); + + store.pauseQueue("empty-reason", ""); + assertThat(store.listPausedQueues()).contains("empty-reason"); + } + + // ---------------------------------------------------------------- counters & health + + @Test + void queueCountersStayExactAcrossClaimsMovesDeletesAndCleanup() { + var store = store(); + var jobs = new ArrayList(); + for (int i = 0; i < 6; i++) { + jobs.add(Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .queue(i % 2 == 0 ? "alpha" : "beta") + .build()); + } + store.insertAll(jobs); + assertThat(store.queueDepths()).containsEntry("alpha", 3L).containsEntry("beta", 3L); + + store.claimReady(NodeId.newId(), "alpha", 2, Instant.now()); + store.softDelete(jobs.get(1).id()); + assertThat(store.queueDepths()).containsEntry("alpha", 1L).containsEntry("beta", 2L); + assertThat(store.countsByState()) + .containsEntry(JobState.ENQUEUED, 3L) + .containsEntry(JobState.PROCESSING, 2L) + .containsEntry(JobState.DELETED, 1L); + + store.claimReady(NodeId.newId(), "beta", 5, Instant.now()); + store.claimReady(NodeId.newId(), "alpha", 5, Instant.now()); + assertThat(store.queueDepths()).isEmpty(); + assertThat(store.deleteIdleQueueMetadata(100)).isPositive(); + assertThat(store.listEnqueuedQueues()).isEmpty(); + } + + @Test + void verifyWritableProbesTheDatabaseAndFailsWhileUnreachable() { + var down = new AtomicBoolean(false); + var flaky = (DataSource) Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[] {DataSource.class}, (proxy, m, args) -> { + if (m.getName().equals("getConnection") && down.get()) { + throw new SQLException("simulated outage"); + } + try { + return m.invoke(dataSource, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + var store = new OracleJobStore(flaky); + + store.verifyWritable(); + down.set(true); + assertThatThrownBy(store::verifyWritable).isInstanceOf(OracleJobStore.JdbcException.class); + down.set(false); + store.verifyWritable(); + } + + @Test + void describeNamesTheServerReleaseAndSchemaAndOffersNoRemoteWake() { + var store = store(); + + assertThat(store.describe()).startsWith("Oracle Database ").contains(" @ "); + assertThat(store.createRemoteWakeChannel("threadmill_wake")).isEmpty(); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleMigrationRunnerTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleMigrationRunnerTest.java new file mode 100644 index 00000000..ec351d95 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleMigrationRunnerTest.java @@ -0,0 +1,247 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.Job; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; + +/** + * Oracle migration-runner regressions. Oracle commits each DDL statement, so + * the runner records progress per statement and must resume, serialize + * migrators, and stay honest about incomplete or edited migrations. + */ +class OracleMigrationRunnerTest { + + private DataSource dataSource; + private OracleMigrationRunner runner; + + @BeforeEach + void migrated() { + dataSource = OracleTestDatabase.dataSource(); + runner = new OracleMigrationRunner(dataSource); + runner.migrate(); + } + + @AfterEach + void restoreSchema() throws SQLException { + // Leave a complete, empty schema for the other test classes. + runner.dropThreadmillObjects(); + runner.migrate(); + OracleTestDatabase.reset(); + } + + @Test + void migrationsAreIdempotentAndValidate() { + runner.migrate(); + runner.validate(); + assertThat(runner.emitPendingSql()).isEmpty(); + } + + @Test + void emitPendingSqlOnAFreshSchemaIsReadOnlyAndPrependsHistoryDdl() throws SQLException { + runner.dropThreadmillObjects(); + + String sql = runner.emitPendingSql(); + + assertThat(sql) + .contains("CREATE TABLE threadmill_schema_history") + .contains("-- V1__baseline.sql") + .contains("CREATE TABLE threadmill_jobs") + .contains("INSERT INTO threadmill_schema_history"); + assertThat(tableExists("THREADMILL_SCHEMA_HISTORY")).isFalse(); + assertThat(tableExists("THREADMILL_JOBS")).isFalse(); + } + + @Test + void emittedCleanInstallSqlAppliesToACleanSchemaAndValidates() throws SQLException { + runner.dropThreadmillObjects(); + + // Apply the emitted script the way SQL*Plus does: statement by statement. + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + for (String statement : OracleMigrationRunner.splitStatements( + "clean-install.sql", runner.emitCleanInstallSql())) { + st.execute(statement); + } + } + + runner.validate(); + var store = new OracleJobStore(dataSource); + var job = job(); + store.insert(job); + assertThat(store.findById(job.id())).isPresent(); + } + + @Test + void interruptedMigrationResumesAndToleratesTheUnrecordedCommittedStatement() throws Exception { + runner.dropThreadmillObjects(); + var statements = baselineStatements(); + // Simulate a crash after statement 3 committed its DDL but before its + // progress record: statements 1-3 exist, history says 2 were applied. + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + for (int i = 0; i < 3; i++) st.execute(statements.get(i)); + st.execute(historyDdl()); + try (PreparedStatement ps = conn.prepareStatement( + "INSERT INTO threadmill_schema_history (version, description, checksum, " + + "statements_applied, statements_total, success) VALUES (1, 'baseline', ?, 2, ?, 0)")) { + ps.setString(1, baselineChecksum()); + ps.setInt(2, statements.size()); + ps.executeUpdate(); + } + } + assertThatThrownBy(runner::validate).hasMessageContaining("incomplete"); + + runner.migrate(); + + runner.validate(); + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT COUNT(*) FROM threadmill_job_counts")) { + rs.next(); + assertThat(rs.getInt(1)).isEqualTo(9 * 16); + } + } + + @Test + void migrateAndValidateRefuseAnUnknownVersion() throws SQLException { + execute("INSERT INTO threadmill_schema_history (version, description, checksum, " + + "statements_applied, statements_total, success) VALUES (99, 'future', 'x', 1, 1, 1)"); + try { + assertThatThrownBy(runner::migrate).hasMessageContaining("newer Threadmill binary"); + assertThatThrownBy(runner::validate) + .isInstanceOf(OracleMigrationRunner.MigrationException.class); + } finally { + execute("DELETE FROM threadmill_schema_history WHERE version = 99"); + } + } + + @Test + void migrateAndValidateRefuseAnEditedMigration() throws SQLException { + execute("UPDATE threadmill_schema_history SET checksum = 'edited' WHERE version = 1"); + + assertThatThrownBy(runner::validate).hasMessageContaining("edited after it was applied"); + assertThatThrownBy(runner::migrate) + .isInstanceOf(OracleMigrationRunner.MigrationException.class); + } + + @Test + void concurrentCleanMigrationsAreSerialized() throws Exception { + runner.dropThreadmillObjects(); + var start = new CountDownLatch(1); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + var futures = new ArrayList>(); + for (int i = 0; i < 3; i++) { + futures.add(executor.submit(() -> { + start.await(); + new OracleMigrationRunner(dataSource).migrate(); + return null; + })); + } + start.countDown(); + for (var future : futures) future.get(5, TimeUnit.MINUTES); + } + runner.validate(); + } + + @Test + void migrationCommitsWhenConnectionsDefaultToNonAutoCommit() { + runner.dropThreadmillObjects(); + + new OracleMigrationRunner(new NonAutoCommitDataSource(dataSource)).migrate(); + + runner.validate(); + } + + @Test + void integrityConstraintsRejectInvalidScalarState() { + assertThatThrownBy(() -> execute("INSERT INTO threadmill_jobs (id, state, queue, priority, " + + "handler_signature, current_state_at, version, body, created_at, workflow_root_id) " + + "VALUES (SYS_GUID(), 'BOGUS', 'q', 0, 'h', SYSTIMESTAMP, 1, '{}', SYSTIMESTAMP, " + + "SYS_GUID())")) + .isInstanceOf(SQLException.class) + .hasMessageContaining("THREADMILL_JOBS_STATE_CK"); + assertThatThrownBy(() -> execute("INSERT INTO threadmill_jobs (id, state, queue, priority, " + + "handler_signature, current_state_at, version, body, created_at, workflow_root_id, " + + "concurrency_key) VALUES (SYS_GUID(), 'ENQUEUED', 'q', 0, 'h', SYSTIMESTAMP, 1, '{}', " + + "SYSTIMESTAMP, SYS_GUID(), 'key-without-mode')")) + .isInstanceOf(SQLException.class) + .hasMessageContaining("THREADMILL_JOBS_SHAPE_CK"); + } + + private static Job job() { + return Job.builder() + .spec(JobSpec.of("com.example.H", new JobArgument("java.lang.String", "\"x\""))) + .build(); + } + + private void execute(String sql) throws SQLException { + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + st.execute(sql); + } + } + + private boolean tableExists(String name) throws SQLException { + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = + conn.prepareStatement("SELECT COUNT(*) FROM user_tables WHERE table_name = ?")) { + ps.setString(1, name); + try (ResultSet rs = ps.executeQuery()) { + rs.next(); + return rs.getInt(1) > 0; + } + } + } + + private static String baselineSql() throws IOException { + try (var in = OracleMigrationRunnerTest.class + .getClassLoader() + .getResourceAsStream("com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql")) { + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static List baselineStatements() throws IOException { + return OracleMigrationRunner.splitStatements("V1__baseline.sql", baselineSql()); + } + + private String historyDdl() { + // The runner bootstraps this table; take its exact DDL from the emitted script. + return OracleMigrationRunner.splitStatements("emitted", runner.emitCleanInstallSql()) + .getFirst(); + } + + private static String baselineChecksum() throws Exception { + var digest = + MessageDigest.getInstance("SHA-256").digest(baselineSql().getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java new file mode 100644 index 00000000..2a5d77ea --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java @@ -0,0 +1,296 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Execution-plan regressions for the store's exact SQL (package-private + * constants and builders of {@link OracleJobStore}). + * + *

Every hot or periodic query must reach its rows through the index built + * for it, with cost bounded by the page size, the number of keys, or a key's + * first entries — never by backlog depth. Each statement runs for real with + * typed binds (bind types change plans: a {@code RAW} column compared with a + * character bind is converted and can no longer use its index), and the test + * reads the plan of the executed cursor. The schema is seeded with tens of + * thousands of mixed-state rows, including one hot key, and real statistics. + * Sessions run with the 19c optimizer feature set (see {@link OracleTestDatabase}). + */ +class OracleQueryPlanTest { + + private static final int ROWS = 40_000; + + @BeforeAll + static void seed() throws SQLException { + assumeTrue( + OracleTestDatabase.canInspectPlans(), + "plan tests need SELECT ANY DICTIONARY (or V$SESSION/V$SQL/V$SQL_PLAN access)"); + OracleTestDatabase.reset(); + try (Connection conn = OracleTestDatabase.dataSource().getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + st.executeUpdate("INSERT INTO threadmill_jobs (id, state, queue, priority, " + + "handler_signature, current_state_at, version, body, created_at, concurrency_key, " + + "concurrency_mode, workflow_root_id, scheduled_at, owner_node_id, owner_heartbeat_at, " + + "parent_job_id) " + + "SELECT SYS_GUID(), CASE MOD(LEVEL, 10) WHEN 0 THEN 'SUCCEEDED' WHEN 1 THEN " + + "'SCHEDULED' WHEN 2 THEN 'PROCESSING' WHEN 3 THEN 'AWAITING' WHEN 4 THEN 'FAILED' " + + "ELSE 'ENQUEUED' END, 'q' || MOD(LEVEL, 2), MOD(LEVEL, 5), 'com.example.H', " + + "SYS_EXTRACT_UTC(SYSTIMESTAMP) - NUMTODSINTERVAL(LEVEL, 'SECOND'), 1, '{}', " + + "SYS_EXTRACT_UTC(SYSTIMESTAMP), " + + "CASE WHEN MOD(LEVEL, 3) = 0 THEN CASE WHEN MOD(LEVEL, 2) = 0 THEN 'hot' " + + "ELSE 'k' || MOD(LEVEL, 300) END END, " + + "CASE WHEN MOD(LEVEL, 3) = 0 THEN CASE WHEN MOD(LEVEL, 4) = 0 THEN 'EXCLUSIVE' " + + "ELSE 'SHARED' END END, SYS_GUID(), " + + "CASE WHEN MOD(LEVEL, 10) = 1 THEN SYS_EXTRACT_UTC(SYSTIMESTAMP) " + + "+ NUMTODSINTERVAL(LEVEL, 'SECOND') END, " + + "CASE WHEN MOD(LEVEL, 10) = 2 THEN SYS_GUID() END, " + + "CASE WHEN MOD(LEVEL, 10) = 2 THEN SYS_EXTRACT_UTC(SYSTIMESTAMP) END, " + + "CASE WHEN MOD(LEVEL, 10) = 3 THEN SYS_GUID() END " + + "FROM dual CONNECT BY LEVEL <= " + ROWS); + st.executeUpdate("INSERT INTO threadmill_concurrency_groups (concurrency_key, " + + "last_modified) SELECT 'g' || LEVEL, SYS_EXTRACT_UTC(SYSTIMESTAMP) - INTERVAL '1' HOUR " + + "FROM dual CONNECT BY LEVEL <= 2000"); + st.execute("BEGIN DBMS_STATS.GATHER_TABLE_STATS(USER, 'THREADMILL_JOBS', cascade => TRUE); " + + "DBMS_STATS.GATHER_TABLE_STATS(USER, 'THREADMILL_CONCURRENCY_GROUPS', " + + "cascade => TRUE); END;"); + } + } + + @AfterAll + static void clear() throws SQLException { + OracleTestDatabase.reset(); + } + + @Test + void unkeyedClaimWalksItsIndexInOrderWithoutSorting() throws SQLException { + assertThat(executedPlan(OracleJobStore.UNKEYED_CLAIM_SQL, ps -> ps.setString(1, "q1"))) + .contains("THREADMILL_JOBS_UNKEYED_IDX") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void keyEnumerationIsOneMinProbePerKey() throws SQLException { + assertThat(executedPlan(OracleJobStore.pendingKeysSql(false), ps -> { + ps.setString(1, "q1"); + ps.setString(2, "q1"); + ps.setInt(3, OracleJobStore.MAX_PENDING_KEYS_PER_PASS); + })) + .contains("INDEX RANGE SCAN (MIN/MAX)") + .contains("THREADMILL_JOBS_KEYED_IDX") + .doesNotContain("FAST FULL SCAN") + .doesNotContain("TABLE ACCESS FULL"); + assertThat(executedPlan(OracleJobStore.pendingKeysSql(true), ps -> { + ps.setString(1, "q1"); + ps.setString(2, "k1"); + ps.setString(3, "q1"); + ps.setInt(4, OracleJobStore.MAX_PENDING_KEYS_PER_PASS); + })) + .contains("INDEX RANGE SCAN (MIN/MAX)") + .doesNotContain("FAST FULL SCAN") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void keyedHeadsStopAfterEachKeysFirstRowsEvenOnAHotKey() throws SQLException { + assertThat(executedPlan(OracleJobStore.keyedHeadsSql(4), ps -> { + int parameter = 1; + for (String key : List.of("hot", "k3", "k9", "k15")) { + ps.setString(parameter++, "q1"); + ps.setString(parameter++, key); + ps.setInt(parameter++, 10); + } + })) + .contains("THREADMILL_JOBS_KEYED_IDX") + .contains("COUNT STOPKEY") + .doesNotContain("SORT ORDER BY") + .doesNotContain("WINDOW SORT") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void admissionProbesUseThePendingIndexes() throws SQLException { + var ids = ids("SELECT id FROM threadmill_jobs WHERE state = 'ENQUEUED' " + + "AND concurrency_key IS NOT NULL AND ROWNUM <= 8"); + assertThat(executedPlan(OracleJobStore.admissionSql(8), ps -> bindRaw(ps, 1, ids))) + .contains("THREADMILL_JOBS_PENDING_IDX") + .contains("THREADMILL_JOBS_EXCLUSIVE_IDX") + .contains("THREADMILL_JOBS_PK") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void candidateLockingProbesThePrimaryKey() throws SQLException { + var ids = ids("SELECT id FROM threadmill_jobs WHERE state = 'ENQUEUED' AND ROWNUM <= 8"); + assertThat(executedPlan(OracleJobStore.lockByIdsSql(8), ps -> bindRaw(ps, 1, ids))) + .contains("THREADMILL_JOBS_PK") + .doesNotContain("THREADMILL_JOBS_STATE_ID_IDX") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void holdMembersUseTheWorkflowIndex() throws SQLException { + var roots = ids("SELECT workflow_root_id FROM threadmill_jobs WHERE state = 'ENQUEUED' " + + "AND concurrency_key = 'hot' AND ROWNUM <= 4"); + assertThat(executedPlan(OracleJobStore.holdMembersSql(4), ps -> { + int parameter = 1; + for (byte[] root : roots) { + ps.setString(parameter++, "hot"); + ps.setBytes(parameter++, root); + ps.setString(parameter++, "q1"); + ps.setInt(parameter++, 10); + } + })) + .contains("THREADMILL_JOBS_WORKFLOW_IDX") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void promotionAndOrphanScansReadTheirIndexesInOrder() throws SQLException { + var now = Instant.now(); + assertThat(executedPlan(OracleJobStore.PROMOTION_SQL, ps -> { + ps.setObject(1, utc(now.plusSeconds(ROWS))); + ps.setInt(2, 500); + })) + .contains("THREADMILL_JOBS_SCHEDULED_IDX") + .contains("COUNT STOPKEY") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + assertThat(executedPlan(OracleJobStore.ORPHAN_SQL, ps -> { + ps.setObject(1, utc(now)); + ps.setInt(2, 500); + })) + .contains("THREADMILL_JOBS_LIVENESS_IDX") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void workflowAndQueueMonitoringReadTheirIndexesInOrder() throws SQLException { + var parents = ids( + "SELECT parent_job_id FROM threadmill_jobs WHERE state = 'AWAITING' " + "AND ROWNUM <= 1"); + assertThat(executedPlan(OracleJobStore.AWAITING_SQL, ps -> { + ps.setBytes(1, parents.getFirst()); + ps.setInt(2, 500); + })) + .contains("THREADMILL_JOBS_AWAITING_IDX") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + assertThat(executedPlan(OracleJobStore.QUEUE_AGE_SQL, ps -> ps.setString(1, "q1"))) + .contains("INDEX RANGE SCAN (MIN/MAX)") + .contains("THREADMILL_JOBS_QUEUE_AGE_IDX"); + assertThat(executedPlan(OracleJobStore.OLDEST_SCHEDULED_SQL, ps -> {})) + .contains("(MIN/MAX)") + .contains("THREADMILL_JOBS_SCHEDULED_IDX"); + assertThat(executedPlan(OracleJobStore.OLDEST_HEARTBEAT_SQL, ps -> {})) + .contains("(MIN/MAX)") + .contains("THREADMILL_JOBS_HEARTBEAT_IDX"); + assertThat(executedPlan(OracleJobStore.OLDEST_IN_STATE_SQL, ps -> ps.setString(1, "FAILED"))) + .contains("(MIN/MAX)") + .doesNotContain("TABLE ACCESS FULL"); + } + + @Test + void retentionAndIdleGroupPagesReadTheirIndexesInOrder() throws SQLException { + var cutoff = utc(Instant.now()); + assertThat(executedPlan(OracleJobStore.retentionCandidatesSql(true, false), ps -> { + ps.setString(1, "FAILED"); + ps.setObject(2, cutoff); + })) + .contains("THREADMILL_JOBS_RETENTION_IDX") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + var cursorId = ids("SELECT id FROM threadmill_jobs WHERE state = 'FAILED' AND ROWNUM <= 1"); + assertThat(executedPlan(OracleJobStore.retentionCandidatesSql(true, true), ps -> { + ps.setString(1, "FAILED"); + ps.setObject(2, cutoff); + ps.setObject(3, utc(Instant.now().minusSeconds(ROWS / 2))); + ps.setObject(4, utc(Instant.now().minusSeconds(ROWS / 2))); + ps.setBytes(5, cursorId.getFirst()); + })) + .contains("THREADMILL_JOBS_RETENTION_IDX") + .doesNotContain("SORT ORDER BY") + .doesNotContain("TABLE ACCESS FULL"); + assertThat(executedPlan(OracleJobStore.idleGroupsSql(false), ps -> {})) + .contains("THREADMILL_CONCURRENCY_IDLE_IDX") + .doesNotContain("SORT ORDER BY"); + assertThat(executedPlan(OracleJobStore.idleGroupsSql(true), ps -> ps.setString(1, "g5"))) + .contains("THREADMILL_CONCURRENCY_IDLE_IDX") + .doesNotContain("SORT ORDER BY"); + } + + @Test + void countsReadTheCounterTablesNeverTheJobsTable() throws SQLException { + assertThat(executedPlan(OracleJobStore.COUNTS_SQL, ps -> {})).doesNotContain("THREADMILL_JOBS"); + assertThat(executedPlan(OracleJobStore.QUEUE_DEPTHS_SQL, ps -> {})) + .doesNotContain("THREADMILL_JOBS"); + } + + @FunctionalInterface + private interface Binder { + void bind(PreparedStatement ps) throws SQLException; + } + + /** Run {@code sql} with typed binds and return the plan of the executed cursor. */ + private static String executedPlan(String sql, Binder binder) throws SQLException { + try (Connection conn = OracleTestDatabase.dataSource().getConnection()) { + conn.setAutoCommit(false); + try { + try (PreparedStatement ps = conn.prepareStatement(sql)) { + binder.bind(ps); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + // drain, so the cursor completes like the store's reads + } + } + } + var plan = new StringBuilder(); + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT plan_table_output FROM TABLE(" + + "DBMS_XPLAN.DISPLAY_CURSOR(NULL, NULL, 'BASIC'))")) { + while (rs.next()) plan.append(rs.getString(1)).append('\n'); + } + assertThat(plan.toString()).as("executed-cursor plan").contains("Plan hash value"); + return plan.toString(); + } finally { + conn.rollback(); + } + } + } + + private static List ids(String sql) throws SQLException { + var ids = new ArrayList(); + try (Connection conn = OracleTestDatabase.dataSource().getConnection(); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery(sql)) { + while (rs.next()) ids.add(rs.getBytes(1)); + } + assertThat(ids).as("seed rows for %s", sql).isNotEmpty(); + return ids; + } + + private static void bindRaw(PreparedStatement ps, int start, List ids) + throws SQLException { + for (int i = 0; i < ids.size(); i++) ps.setBytes(start + i, ids.get(i)); + } + + private static LocalDateTime utc(Instant instant) { + return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleServerTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleServerTest.java new file mode 100644 index 00000000..ecf7d508 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleServerTest.java @@ -0,0 +1,60 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +import com.hemju.threadmill.core.JobEngineFatalException; + +/** The startup gate: Oracle 19c or later with an AL32UTF8 database character set. */ +class OracleServerTest { + + @Test + void refusesToStartAgainstPreNineteenServers() { + var eighteen = new OracleServer.Facts(18, "18.4.0.0.0", "AL32UTF8", "XE", "APP", "BINARY"); + + assertThatThrownBy(() -> OracleServer.require(eighteen)) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("19c or later") + .hasMessageContaining("18.4.0.0.0"); + } + + @Test + void acceptsNineteenAndLaterReleases() { + for (int major : new int[] {19, 21, 23}) { + var facts = + new OracleServer.Facts(major, major + ".0.0.0.0", "AL32UTF8", "DB", "APP", "BINARY"); + assertThat(OracleServer.require(facts)).isSameAs(facts); + } + } + + @Test + void refusesNonUnicodeDatabaseCharacterSets() { + var latin1 = new OracleServer.Facts(19, "19.24.0.0.0", "WE8ISO8859P1", "ORCL", "APP", "BINARY"); + + assertThatThrownBy(() -> OracleServer.require(latin1)) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("AL32UTF8") + .hasMessageContaining("WE8ISO8859P1"); + } + + @Test + void refusesLinguisticTextComparison() { + var linguistic = + new OracleServer.Facts(19, "19.24.0.0.0", "AL32UTF8", "ORCL", "APP", "LINGUISTIC"); + + assertThatThrownBy(() -> OracleServer.require(linguistic)) + .isInstanceOf(JobEngineFatalException.class) + .hasMessageContaining("NLS_COMP=BINARY") + .hasMessageContaining("LINGUISTIC"); + } + + @Test + void describesTheServerWithoutCredentials() { + var facts = + new OracleServer.Facts(19, "19.24.0.0.0", "AL32UTF8", "ORCL", "THREADMILL", "BINARY"); + + assertThat(facts.describe()).isEqualTo("Oracle Database 19.24.0.0.0 @ ORCL/THREADMILL"); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleStatementSplitTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleStatementSplitTest.java new file mode 100644 index 00000000..408152ed --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleStatementSplitTest.java @@ -0,0 +1,52 @@ +package com.hemju.threadmill.store.oracle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +/** Migration files use SQL*Plus statement terminators: a line holding only {@code /}. */ +class OracleStatementSplitTest { + + @Test + void splitsAtSlashLinesAndDropsLeadingComments() { + var sql = """ + -- header comment + CREATE TABLE a (id NUMBER) + / + + -- explains the trigger + CREATE TRIGGER t BEFORE INSERT ON a FOR EACH ROW + BEGIN + -- inline comment stays + NULL; + END; + / + """; + + var statements = OracleMigrationRunner.splitStatements("V1__x.sql", sql); + + assertThat(statements).hasSize(2); + assertThat(statements.get(0)).isEqualTo("CREATE TABLE a (id NUMBER)"); + assertThat(statements.get(1)) + .startsWith("CREATE TRIGGER t") + .contains("-- inline comment stays") + .endsWith("END;"); + } + + @Test + void rejectsTextAfterTheLastTerminator() { + assertThatThrownBy(() -> OracleMigrationRunner.splitStatements( + "V2__x.sql", "CREATE TABLE a (id NUMBER)\n/\nCREATE TABLE b (id NUMBER)\n")) + .isInstanceOf(OracleMigrationRunner.MigrationException.class) + .hasMessageContaining("V2__x.sql"); + } + + @Test + void allowsTrailingCommentsAndBlankLines() { + var statements = + OracleMigrationRunner.splitStatements("V3__x.sql", "SELECT 1 FROM dual\n/\n\n-- end\n"); + + assertThat(statements).containsExactly("SELECT 1 FROM dual"); + } +} diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleTestDatabase.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleTestDatabase.java new file mode 100644 index 00000000..8bb0f225 --- /dev/null +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleTestDatabase.java @@ -0,0 +1,135 @@ +package com.hemju.threadmill.store.oracle; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; + +import javax.sql.DataSource; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * One Oracle database per test JVM, shared by every test class because Oracle + * starts slowly. + * + *

By default a Testcontainers {@code gvenzl/oracle-free} container (multi-arch, + * Oracle 23ai Free) is started. CI selects the closest free release to the + * supported 19c baseline with {@code -Dthreadmill.oracle.image=gvenzl/oracle-xe:21-slim-faststart}. + * A real 19c (or any other) database is used instead of a container when + * {@code threadmill.oracle.jdbcUrl}, {@code threadmill.oracle.user} and + * {@code threadmill.oracle.password} are set; the account's schema is reset by + * the tests, so point it at a disposable schema only. + * + *

Every session runs with {@code OPTIMIZER_FEATURES_ENABLE='19.1.0'}, so + * execution-plan assertions reflect the 19c optimizer on newer servers too. + */ +final class OracleTestDatabase { + + static final String DEFAULT_IMAGE = "gvenzl/oracle-free:23-slim-faststart"; + + private static final String USER = "threadmill"; + private static final String PASSWORD = "threadmill"; + private static final String SESSION_INIT = + "ALTER SESSION SET OPTIMIZER_FEATURES_ENABLE = '19.1.0'"; + + private static DataSource pooled; + + private OracleTestDatabase() {} + + /** A pooled data source for the shared database, starting it on first use. */ + static synchronized DataSource dataSource() { + if (pooled == null) { + pooled = start(); + new OracleMigrationRunner(pooled).migrate(); + } + return pooled; + } + + /** Delete every Threadmill row and zero the counters, keeping the schema. */ + static void reset() throws SQLException { + try (Connection conn = dataSource().getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + for (String table : new String[] { + "threadmill_dedup_keys", + "threadmill_cron_task_ownership", + "threadmill_cron_task_state", + "threadmill_cron_tasks", + "threadmill_jobs", + "threadmill_nodes", + "threadmill_mutexes", + "threadmill_leases", + "threadmill_queue_pauses", + "threadmill_concurrency_groups", + "threadmill_concurrency_workflow_holds", + "threadmill_queue_counts" + }) { + st.executeUpdate("DELETE FROM " + table); + } + st.executeUpdate("UPDATE threadmill_job_counts SET job_count = 0"); + } + } + + @SuppressWarnings("resource") // lives for the whole test JVM; Testcontainers' Ryuk reaps it + private static DataSource start() { + String url = System.getProperty("threadmill.oracle.jdbcUrl"); + String user = System.getProperty("threadmill.oracle.user", USER); + String password = System.getProperty("threadmill.oracle.password", PASSWORD); + if (url == null || url.isBlank()) { + String image = System.getProperty("threadmill.oracle.image", DEFAULT_IMAGE); + var container = new GenericContainer<>(DockerImageName.parse(image)) + .withEnv("ORACLE_PASSWORD", PASSWORD) + .withEnv("APP_USER", USER) + .withEnv("APP_USER_PASSWORD", PASSWORD) + .withExposedPorts(1521) + .waitingFor(Wait.forLogMessage(".*DATABASE IS READY TO USE!.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(10))); + container.start(); + String service = image.contains("oracle-xe") ? "XEPDB1" : "FREEPDB1"; + url = "jdbc:oracle:thin:@//" + container.getHost() + ":" + container.getMappedPort(1521) + "/" + + service; + user = USER; + password = PASSWORD; + grantPlanInspection(url); + } + var config = new HikariConfig(); + config.setJdbcUrl(url); + config.setUsername(user); + config.setPassword(password); + config.setMaximumPoolSize(24); + config.setConnectionInitSql(SESSION_INIT); + config.setPoolName("threadmill-oracle-test"); + return new HikariDataSource(config); + } + + /** + * Let the test user read executed-cursor plans ({@code DBMS_XPLAN.DISPLAY_CURSOR} + * needs the {@code V$} views). Only the throwaway container's admin account + * is used for this. + */ + private static void grantPlanInspection(String url) { + try (Connection admin = DriverManager.getConnection(url, "system", PASSWORD); + Statement st = admin.createStatement()) { + st.execute("GRANT SELECT ANY DICTIONARY TO " + USER); + } catch (SQLException e) { + throw new IllegalStateException("Could not grant plan inspection to the test user", e); + } + } + + /** Whether the test user can read executed-cursor plans. */ + static boolean canInspectPlans() { + try (Connection conn = dataSource().getConnection(); + Statement st = conn.createStatement()) { + st.executeQuery("SELECT COUNT(*) FROM v$session WHERE ROWNUM = 1").close(); + return true; + } catch (SQLException e) { + return false; + } + } +} From 51bacbc1cdeac423eedd0c1aa1b4e262970243ff Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 18:16:59 +0200 Subject: [PATCH 2/7] feat(spring-boot): auto-configure the Oracle store ThreadmillOracleAutoConfiguration wires OracleJobStore from the application's DataSource, gated by @ConditionalOnClass so applications without threadmill-store-oracle never load Oracle types. - threadmill.store.oracle.schema-mode (migrate, validate, none, drop-and-migrate) mirrors the PostgreSQL schema handling, including the explicit destructive-reset flag. - join_transaction enqueue mode now also works with Oracle through SpringOracleTransactionBoundary. - With both JDBC store modules on the classpath, JdbcStoreSelection picks the store: threadmill.store.jdbc-type, then the spring.datasource.url scheme, then the module present (PostgreSQL on a tie, unchanged behaviour). - OracleJobStore names a non-Oracle DataSource in its startup error. Refs #137 --- threadmill-spring-boot/build.gradle.kts | 4 + threadmill-spring-boot/gradle.lockfile | 1 + .../threadmill/spring/JdbcStoreSelection.java | 79 ++++++ .../threadmill/spring/JdbcStoreType.java | 20 ++ .../threadmill/spring/OracleSchemaMode.java | 19 ++ .../threadmill/spring/SpringEnqueueMode.java | 2 +- .../SpringOracleTransactionBoundary.java | 57 +++++ .../spring/ThreadmillAutoConfiguration.java | 6 +- .../ThreadmillOracleAutoConfiguration.java | 80 ++++++ .../ThreadmillPostgresAutoConfiguration.java | 6 +- .../spring/ThreadmillProperties.java | 44 ++++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../spring/JdbcStoreSelectionTest.java | 64 +++++ .../threadmill/spring/OracleTestDatabase.java | 92 +++++++ .../SpringOracleAutoConfigurationTest.java | 136 ++++++++++ .../SpringOracleTransactionBoundaryTest.java | 238 ++++++++++++++++++ .../threadmill/store/oracle/OracleServer.java | 12 +- 17 files changed, 853 insertions(+), 8 deletions(-) create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreSelection.java create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreType.java create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/OracleSchemaMode.java create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundary.java create mode 100644 threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillOracleAutoConfiguration.java create mode 100644 threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/JdbcStoreSelectionTest.java create mode 100644 threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/OracleTestDatabase.java create mode 100644 threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleAutoConfigurationTest.java create mode 100644 threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java diff --git a/threadmill-spring-boot/build.gradle.kts b/threadmill-spring-boot/build.gradle.kts index 71167bf0..da92c93a 100644 --- a/threadmill-spring-boot/build.gradle.kts +++ b/threadmill-spring-boot/build.gradle.kts @@ -23,6 +23,8 @@ dependencies { // use Redis exclusively) don't pull in the JDBC driver. The auto-config // bean for Postgres is gated by @ConditionalOnClass so this is safe. compileOnly(project(":threadmill-store-postgres")) + // Same for the Oracle store, gated by @ConditionalOnClass(OracleJobStore.class). + compileOnly(project(":threadmill-store-oracle")) compileOnly(libs.spring.boot.autoconfigure) compileOnly(libs.spring.context) compileOnly(libs.spring.jdbc) @@ -32,6 +34,8 @@ dependencies { ) testImplementation(project(":threadmill-store-postgres")) + testImplementation(project(":threadmill-store-oracle")) + testImplementation(libs.oracle.jdbc) testImplementation(project(":threadmill-store-redis")) testImplementation(project(":threadmill-test-support")) testImplementation(libs.spring.boot.autoconfigure) diff --git a/threadmill-spring-boot/gradle.lockfile b/threadmill-spring-boot/gradle.lockfile index 9c80c82c..19d015b0 100644 --- a/threadmill-spring-boot/gradle.lockfile +++ b/threadmill-spring-boot/gradle.lockfile @@ -16,6 +16,7 @@ com.google.errorprone:error_prone_annotations:2.38.0=testCompileClasspath com.gradleup.gratatouille:gratatouille-tasks-runtime:0.2.3=nmcpTasks com.gradleup.nmcp:nmcp-tasks:1.6.1=nmcpTasks com.jayway.jsonpath:json-path:2.10.0=testCompileClasspath,testRuntimeClasspath +com.oracle.database.jdbc:ojdbc11:23.26.3.0.0=testCompileClasspath,testRuntimeClasspath com.squareup.okhttp3:okhttp-jvm:5.4.0=nmcpTasks com.squareup.okhttp3:okhttp:5.4.0=nmcpTasks com.squareup.okio:okio-jvm:3.17.0=nmcpTasks diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreSelection.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreSelection.java new file mode 100644 index 00000000..870a49b7 --- /dev/null +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreSelection.java @@ -0,0 +1,79 @@ +package com.hemju.threadmill.spring; + +import java.util.Locale; + +import org.springframework.boot.autoconfigure.condition.ConditionOutcome; +import org.springframework.boot.autoconfigure.condition.SpringBootCondition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.util.ClassUtils; + +/** + * Decides which JDBC store auto-configuration owns the application's + * {@code DataSource} when more than one store module is on the classpath. + * + *

Precedence: {@code threadmill.store.jdbc-type}, then the scheme of + * {@code spring.datasource.url}, then the store modules present (PostgreSQL + * when both are, preserving the behaviour from before the Oracle store). + * Decided from configuration rather than a database round trip, because + * conditions run before any bean exists; the chosen store still verifies the + * server at startup and fails fast on a mismatch. + */ +final class JdbcStoreSelection { + + static final String PROPERTY = "threadmill.store.jdbc-type"; + + private static final String POSTGRES_STORE = + "com.hemju.threadmill.store.postgres.PostgresJobStore"; + private static final String ORACLE_STORE = "com.hemju.threadmill.store.oracle.OracleJobStore"; + + private JdbcStoreSelection() {} + + static JdbcStoreType select(Environment environment, ClassLoader classLoader) { + String configured = environment.getProperty(PROPERTY); + if (configured != null && !configured.isBlank()) { + return JdbcStoreType.valueOf(configured.trim().replace('-', '_').toUpperCase(Locale.ROOT)); + } + String url = environment.getProperty("spring.datasource.url", ""); + if (url.startsWith("jdbc:oracle:")) { + return JdbcStoreType.ORACLE; + } + if (url.startsWith("jdbc:postgresql:")) { + return JdbcStoreType.POSTGRES; + } + boolean postgres = ClassUtils.isPresent(POSTGRES_STORE, classLoader); + boolean oracle = ClassUtils.isPresent(ORACLE_STORE, classLoader); + return oracle && !postgres ? JdbcStoreType.ORACLE : JdbcStoreType.POSTGRES; + } + + private abstract static class Selected extends SpringBootCondition { + private final JdbcStoreType type; + + Selected(JdbcStoreType type) { + this.type = type; + } + + @Override + public ConditionOutcome getMatchOutcome( + ConditionContext context, AnnotatedTypeMetadata metadata) { + JdbcStoreType selected = select(context.getEnvironment(), context.getClassLoader()); + String message = "Threadmill JDBC store selection chose " + selected; + return selected == type ? ConditionOutcome.match(message) : ConditionOutcome.noMatch(message); + } + } + + /** Matches when the PostgreSQL store owns the {@code DataSource}. */ + static final class PostgresSelected extends Selected { + PostgresSelected() { + super(JdbcStoreType.POSTGRES); + } + } + + /** Matches when the Oracle store owns the {@code DataSource}. */ + static final class OracleSelected extends Selected { + OracleSelected() { + super(JdbcStoreType.ORACLE); + } + } +} diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreType.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreType.java new file mode 100644 index 00000000..1b821c4a --- /dev/null +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/JdbcStoreType.java @@ -0,0 +1,20 @@ +package com.hemju.threadmill.spring; + +/** + * The JDBC store Spring Boot auto-configures from the application's + * {@code DataSource}, set with {@code threadmill.store.jdbc-type}. + * + *

Only needed when both {@code threadmill-store-postgres} and + * {@code threadmill-store-oracle} are on the classpath and the database cannot + * be told from {@code spring.datasource.url}. Without it, a + * {@code jdbc:oracle:} URL selects Oracle, a {@code jdbc:postgresql:} URL + * selects PostgreSQL, and otherwise the one store module present is used + * (PostgreSQL when both are). + */ +public enum JdbcStoreType { + /** {@code PostgresJobStore} (PostgreSQL 18+). */ + POSTGRES, + + /** {@code OracleJobStore} (Oracle Database 19c+). */ + ORACLE +} diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/OracleSchemaMode.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/OracleSchemaMode.java new file mode 100644 index 00000000..4aa9673b --- /dev/null +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/OracleSchemaMode.java @@ -0,0 +1,19 @@ +package com.hemju.threadmill.spring; + +/** Oracle schema action for Spring Boot auto-configured Threadmill stores. */ +public enum OracleSchemaMode { + /** + * Apply every pending Threadmill migration before creating the store, + * resuming a migration that a crash interrupted. + */ + MIGRATE, + + /** Validate that the current schema exactly matches the shipped migration set. */ + VALIDATE, + + /** Do not inspect or modify the schema. */ + NONE, + + /** Drop Threadmill-owned schema objects, then apply all migrations. */ + DROP_AND_MIGRATE +} diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringEnqueueMode.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringEnqueueMode.java index bc3bdd12..2eca8648 100644 --- a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringEnqueueMode.java +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringEnqueueMode.java @@ -5,7 +5,7 @@ public enum SpringEnqueueMode { /** Reserve ids immediately, then write jobs in an {@code afterCommit} callback. */ AFTER_COMMIT, - /** Write Postgres jobs inside the caller's Spring JDBC transaction. */ + /** Write PostgreSQL or Oracle jobs inside the caller's Spring JDBC transaction. */ JOIN_TRANSACTION, /** Write jobs immediately, regardless of Spring transaction state. */ diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundary.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundary.java new file mode 100644 index 00000000..4ebc17e2 --- /dev/null +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundary.java @@ -0,0 +1,57 @@ +package com.hemju.threadmill.spring; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.springframework.jdbc.datasource.DataSourceUtils; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import com.hemju.threadmill.store.oracle.OracleConnectionWork; +import com.hemju.threadmill.store.oracle.OracleTransactionBoundary; + +/** + * Joins Oracle store writes to the caller's Spring JDBC transaction on the same + * {@link DataSource}; outside a transaction, Threadmill owns its own. + */ +final class SpringOracleTransactionBoundary implements OracleTransactionBoundary { + + private final DataSource dataSource; + private final OracleTransactionBoundary owningBoundary; + + SpringOracleTransactionBoundary(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + this.owningBoundary = OracleTransactionBoundary.owning(dataSource); + } + + @Override + public T inTransaction(OracleConnectionWork work) throws SQLException { + if (!TransactionSynchronizationManager.isActualTransactionActive()) { + return owningBoundary.inTransaction(work); + } + if (!TransactionSynchronizationManager.hasResource(dataSource)) { + throw new IllegalStateException( + "threadmill.spring.enqueue-mode=join_transaction requires the caller's Spring transaction" + + " to be bound to the same DataSource as Threadmill's OracleJobStore"); + } + Connection conn = DataSourceUtils.getConnection(dataSource); + try { + return work.execute(conn); + } finally { + DataSourceUtils.releaseConnection(conn, dataSource); + } + } + + @Override + public boolean supportsExternalTransactions() { + return true; + } + + @Override + public boolean externallyManagedTransactionActive() { + return TransactionSynchronizationManager.isActualTransactionActive() + && TransactionSynchronizationManager.hasResource(dataSource); + } +} diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillAutoConfiguration.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillAutoConfiguration.java index aadda10d..0d6ab20f 100644 --- a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillAutoConfiguration.java +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillAutoConfiguration.java @@ -238,13 +238,13 @@ public JobScheduler threadmillJobScheduler( case IMMEDIATE -> new JobScheduler(store, serializer, registry, config, wakeBus); case JOIN_TRANSACTION -> { // Routed through the generic JobStore SPI flag so this method does not - // need to reference any postgres-store class — keeping the auto-config - // loadable even when threadmill-store-postgres is not on the classpath. + // need to reference any JDBC store class — keeping the auto-config + // loadable when threadmill-store-postgres or -oracle is absent. if (!unwrapStore(store).supportsExternalTransactions()) { throw new IllegalStateException( "threadmill.spring.enqueue-mode=join_transaction requires a JobStore that supports" + " external transactions (today: the Spring auto-configured PostgresJobStore" - + " using the same DataSource as the caller's transaction)"); + + " or OracleJobStore using the same DataSource as the caller's transaction)"); } yield new TransactionJoinedJobScheduler(store, serializer, registry, config, wakeBus); } diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillOracleAutoConfiguration.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillOracleAutoConfiguration.java new file mode 100644 index 00000000..21612f64 --- /dev/null +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillOracleAutoConfiguration.java @@ -0,0 +1,80 @@ +package com.hemju.threadmill.spring; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; + +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.core.store.JobStoreCapabilities; +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.oracle.OracleMigrationRunner; + +/** + * Oracle-specific Threadmill auto-configuration. + * + *

Like {@link ThreadmillPostgresAutoConfiguration}, it is separate from + * {@link ThreadmillAutoConfiguration} and gated by + * {@code @ConditionalOnClass(OracleJobStore.class)}, so applications without + * {@code threadmill-store-oracle} never load Oracle types. It applies when a + * {@link DataSource} bean exists, Redis is not configured, and + * {@link JdbcStoreSelection} chooses Oracle; it sorts after the real + * {@code DataSourceAutoConfiguration} so the {@code DataSource} condition sees + * a property-configured pool. + */ +@AutoConfiguration +@AutoConfigureBefore(ThreadmillAutoConfiguration.class) +@AutoConfigureAfter( + name = "org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration") +@ConditionalOnClass(OracleJobStore.class) +public class ThreadmillOracleAutoConfiguration { + + private static final Logger LOG = + LoggerFactory.getLogger(ThreadmillOracleAutoConfiguration.class); + + @Bean + @ConditionalOnMissingBean(JobStore.class) + @ConditionalOnBean(DataSource.class) + @Conditional({OnRedisStoreNotConfigured.class, JdbcStoreSelection.OracleSelected.class}) + public JobStore threadmillJobStore(ThreadmillProperties properties, DataSource dataSource) { + LOG.info("Threadmill: using Oracle store wired from the application's DataSource"); + OracleJobStore.requireSupportedServer(dataSource); + applyOracleSchemaMode(dataSource, properties.getStore().getOracle()); + if (properties.getSpring().getEnqueueMode() == SpringEnqueueMode.JOIN_TRANSACTION) { + return new OracleJobStore( + dataSource, + new JsonJobSerializer(), + JobStoreCapabilities.defaults(), + new SpringOracleTransactionBoundary(dataSource)); + } + return new OracleJobStore(dataSource); + } + + private static void applyOracleSchemaMode( + DataSource dataSource, ThreadmillProperties.OracleProperties properties) { + var migrations = new OracleMigrationRunner(dataSource); + switch (properties.getSchemaMode()) { + case MIGRATE -> migrations.migrate(); + case VALIDATE -> migrations.validate(); + case NONE -> {} + case DROP_AND_MIGRATE -> { + if (!properties.isAllowDestructiveSchemaReset()) { + throw new IllegalStateException( + "threadmill.store.oracle.schema-mode=drop-and-migrate requires" + + " threadmill.store.oracle.allow-destructive-schema-reset=true"); + } + migrations.dropThreadmillObjects(); + migrations.migrate(); + } + } + } +} diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillPostgresAutoConfiguration.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillPostgresAutoConfiguration.java index cd370606..2a20c338 100644 --- a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillPostgresAutoConfiguration.java +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillPostgresAutoConfiguration.java @@ -35,7 +35,9 @@ * {@link DataSource} bean is present, and Redis is not configured) the * Postgres store wins the {@link ConditionalOnMissingBean} race in the core * config and Redis precedence is preserved by the - * {@link OnRedisStoreNotConfigured} gate. + * {@link OnRedisStoreNotConfigured} gate. When {@code threadmill-store-oracle} + * is on the classpath too, {@link JdbcStoreSelection} decides which JDBC store + * owns the {@code DataSource}. */ @AutoConfiguration @AutoConfigureBefore(ThreadmillAutoConfiguration.class) @@ -55,7 +57,7 @@ public class ThreadmillPostgresAutoConfiguration { @Bean @ConditionalOnMissingBean(JobStore.class) @ConditionalOnBean(DataSource.class) - @Conditional(OnRedisStoreNotConfigured.class) + @Conditional({OnRedisStoreNotConfigured.class, JdbcStoreSelection.PostgresSelected.class}) public JobStore threadmillJobStore(ThreadmillProperties properties, DataSource dataSource) { LOG.info("Threadmill: using Postgres store wired from the application's DataSource"); PostgresJobStore.requireSupportedServer(dataSource); diff --git a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillProperties.java b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillProperties.java index 7c845448..44bcb55a 100644 --- a/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillProperties.java +++ b/threadmill-spring-boot/src/main/java/com/hemju/threadmill/spring/ThreadmillProperties.java @@ -438,10 +438,25 @@ public void setChannel(String channel) { } public static final class StoreProperties { + private JdbcStoreType jdbcType; private MemoryProperties memory = new MemoryProperties(); private PostgresProperties postgres = new PostgresProperties(); + private OracleProperties oracle = new OracleProperties(); private RedisProperties redis = new RedisProperties(); + /** + * The JDBC store to wire from the application's {@code DataSource} when both + * JDBC store modules are on the classpath; unset selects by + * {@code spring.datasource.url}. + */ + public JdbcStoreType getJdbcType() { + return jdbcType; + } + + public void setJdbcType(JdbcStoreType jdbcType) { + this.jdbcType = jdbcType; + } + public MemoryProperties getMemory() { return memory; } @@ -458,6 +473,14 @@ public void setPostgres(PostgresProperties postgres) { this.postgres = postgres; } + public OracleProperties getOracle() { + return oracle; + } + + public void setOracle(OracleProperties oracle) { + this.oracle = oracle; + } + public RedisProperties getRedis() { return redis; } @@ -500,6 +523,27 @@ public void setAllowDestructiveSchemaReset(boolean allowDestructiveSchemaReset) } } + public static final class OracleProperties { + private OracleSchemaMode schemaMode = OracleSchemaMode.MIGRATE; + private boolean allowDestructiveSchemaReset; + + public OracleSchemaMode getSchemaMode() { + return schemaMode; + } + + public void setSchemaMode(OracleSchemaMode schemaMode) { + this.schemaMode = schemaMode; + } + + public boolean isAllowDestructiveSchemaReset() { + return allowDestructiveSchemaReset; + } + + public void setAllowDestructiveSchemaReset(boolean allowDestructiveSchemaReset) { + this.allowDestructiveSchemaReset = allowDestructiveSchemaReset; + } + } + public static final class RedisProperties { private String mode = "standalone"; private String uri; diff --git a/threadmill-spring-boot/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/threadmill-spring-boot/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 61d017c8..07a5b9cb 100644 --- a/threadmill-spring-boot/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/threadmill-spring-boot/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,3 +1,4 @@ com.hemju.threadmill.spring.ThreadmillAutoConfiguration com.hemju.threadmill.spring.ThreadmillPostgresAutoConfiguration +com.hemju.threadmill.spring.ThreadmillOracleAutoConfiguration com.hemju.threadmill.spring.ThreadmillRedisAutoConfiguration diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/JdbcStoreSelectionTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/JdbcStoreSelectionTest.java new file mode 100644 index 00000000..72555d59 --- /dev/null +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/JdbcStoreSelectionTest.java @@ -0,0 +1,64 @@ +package com.hemju.threadmill.spring; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.mock.env.MockEnvironment; + +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.postgres.PostgresJobStore; + +/** Which JDBC store owns the application's DataSource when both modules are present. */ +class JdbcStoreSelectionTest { + + private static final ClassLoader BOTH = JdbcStoreSelectionTest.class.getClassLoader(); + + @Test + void explicitTypeWinsOverTheDataSourceUrl() { + var environment = new MockEnvironment() + .withProperty("threadmill.store.jdbc-type", "postgres") + .withProperty("spring.datasource.url", "jdbc:oracle:thin:@//db:1521/ORCL"); + + assertThat(JdbcStoreSelection.select(environment, BOTH)).isEqualTo(JdbcStoreType.POSTGRES); + environment.setProperty("threadmill.store.jdbc-type", "ORACLE"); + assertThat(JdbcStoreSelection.select(environment, BOTH)).isEqualTo(JdbcStoreType.ORACLE); + } + + @Test + void dataSourceUrlSchemeSelectsTheStore() { + assertThat(JdbcStoreSelection.select( + new MockEnvironment() + .withProperty("spring.datasource.url", "jdbc:oracle:thin:@//db:1521/ORCL"), + BOTH)) + .isEqualTo(JdbcStoreType.ORACLE); + assertThat(JdbcStoreSelection.select( + new MockEnvironment() + .withProperty("spring.datasource.url", "jdbc:postgresql://db/threadmill"), + BOTH)) + .isEqualTo(JdbcStoreType.POSTGRES); + } + + @Test + void withoutConfigurationThePresentModuleWinsAndPostgresBreaksTies() { + var environment = new MockEnvironment(); + + assertThat(JdbcStoreSelection.select(environment, BOTH)).isEqualTo(JdbcStoreType.POSTGRES); + assertThat( + JdbcStoreSelection.select(environment, new FilteredClassLoader(PostgresJobStore.class))) + .isEqualTo(JdbcStoreType.ORACLE); + assertThat( + JdbcStoreSelection.select(environment, new FilteredClassLoader(OracleJobStore.class))) + .isEqualTo(JdbcStoreType.POSTGRES); + } + + @Test + void jdbcTypeBindsAsAStoreProperty() { + var properties = new ThreadmillProperties(); + properties.getStore().setJdbcType(JdbcStoreType.ORACLE); + + assertThat(properties.getStore().getJdbcType()).isEqualTo(JdbcStoreType.ORACLE); + assertThat(properties.getStore().getOracle().getSchemaMode()) + .isEqualTo(OracleSchemaMode.MIGRATE); + } +} diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/OracleTestDatabase.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/OracleTestDatabase.java new file mode 100644 index 00000000..17bcbf70 --- /dev/null +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/OracleTestDatabase.java @@ -0,0 +1,92 @@ +package com.hemju.threadmill.spring; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; + +import javax.sql.DataSource; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * One Oracle Free container per test JVM for the Spring integration tests + * (Oracle starts slowly). The Oracle store module's own suite covers 21c XE + * and real 19c; these tests exercise the Spring wiring. + */ +final class OracleTestDatabase { + + private static final String CREDENTIAL = "threadmill"; + private static String url; + private static HikariDataSource pooled; + + private OracleTestDatabase() {} + + @SuppressWarnings("resource") // lives for the whole test JVM; Ryuk reaps it + static synchronized String url() { + if (url == null) { + var container = new GenericContainer<>( + DockerImageName.parse("gvenzl/oracle-free:23-slim-faststart")) + .withEnv("ORACLE_PASSWORD", CREDENTIAL) + .withEnv("APP_USER", CREDENTIAL) + .withEnv("APP_USER_PASSWORD", CREDENTIAL) + .withExposedPorts(1521) + .waitingFor(Wait.forLogMessage(".*DATABASE IS READY TO USE!.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(10))); + container.start(); + url = "jdbc:oracle:thin:@//" + container.getHost() + ":" + container.getMappedPort(1521) + + "/FREEPDB1"; + } + return url; + } + + static String user() { + return CREDENTIAL; + } + + static String password() { + return CREDENTIAL; + } + + static synchronized DataSource dataSource() { + if (pooled == null) { + pooled = newPool(); + } + return pooled; + } + + /** A separate pool on the same database: a different {@code DataSource} identity. */ + static HikariDataSource newPool() { + var config = new HikariConfig(); + config.setJdbcUrl(url()); + config.setUsername(user()); + config.setPassword(password()); + config.setMaximumPoolSize(8); + return new HikariDataSource(config); + } + + /** Delete every Threadmill row and zero the counters, keeping the schema. */ + static void reset() throws SQLException { + try (Connection conn = dataSource().getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + for (String table : new String[] { + "threadmill_dedup_keys", + "threadmill_cron_task_ownership", + "threadmill_cron_task_state", + "threadmill_cron_tasks", + "threadmill_jobs", + "threadmill_concurrency_groups", + "threadmill_concurrency_workflow_holds", + "threadmill_queue_counts" + }) { + st.executeUpdate("DELETE FROM " + table); + } + st.executeUpdate("UPDATE threadmill_job_counts SET job_count = 0"); + } + } +} diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleAutoConfigurationTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleAutoConfigurationTest.java new file mode 100644 index 00000000..4b790b91 --- /dev/null +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleAutoConfigurationTest.java @@ -0,0 +1,136 @@ +package com.hemju.threadmill.spring; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.sql.ResultSet; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.jdbc.datasource.DelegatingDataSource; + +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.oracle.OracleMigrationRunner; + +/** + * Spring Boot wiring of the Oracle store: JDBC store selection with both store + * modules on the classpath, schema modes, and join-transaction enqueue mode. + */ +@EnabledIf("com.hemju.threadmill.spring.DockerAvailable#check") +class SpringOracleAutoConfigurationTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of( + ThreadmillRedisAutoConfiguration.class, + ThreadmillPostgresAutoConfiguration.class, + ThreadmillOracleAutoConfiguration.class, + ThreadmillAutoConfiguration.class)) + .withBean(DataSource.class, () -> new DelegatingDataSource(OracleTestDatabase.dataSource())) + .withPropertyValues("threadmill.enabled=false"); + + @BeforeEach + void dropSchema() { + new OracleMigrationRunner(OracleTestDatabase.dataSource()).dropThreadmillObjects(); + } + + @Test + void anOracleDataSourceUrlSelectsTheOracleStoreAndMigrates() { + contextRunner + .withPropertyValues("spring.datasource.url=" + OracleTestDatabase.url()) + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(JobStore.class)).isInstanceOf(OracleJobStore.class); + assertThat(context.getBean(JobStore.class).describe()).startsWith("Oracle Database"); + }); + + new OracleMigrationRunner(OracleTestDatabase.dataSource()).validate(); + } + + @Test + void explicitJdbcTypeSelectsTheOracleStore() { + contextRunner.withPropertyValues("threadmill.store.jdbc-type=oracle").run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(JobStore.class)).isInstanceOf(OracleJobStore.class); + }); + } + + @Test + void aMisselectedPostgresStoreFailsFastAgainstOracle() { + // Both modules present and nothing names the database: PostgreSQL wins + // the tie, and its own server check refuses the Oracle DataSource. + contextRunner.run(context -> assertThat(context).hasFailed()); + } + + @Test + void validateSchemaModeFailsOnEmptySchemaAndPassesAfterMigration() { + var oracle = contextRunner.withPropertyValues( + "threadmill.store.jdbc-type=oracle", "threadmill.store.oracle.schema-mode=validate"); + oracle.run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()).hasMessageContaining("threadmill_schema_history"); + }); + + new OracleMigrationRunner(OracleTestDatabase.dataSource()).migrate(); + oracle.run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(JobStore.class)).isInstanceOf(OracleJobStore.class); + }); + } + + @Test + void noneSchemaModeDoesNotCreateSchema() throws Exception { + contextRunner + .withPropertyValues( + "threadmill.store.jdbc-type=oracle", "threadmill.store.oracle.schema-mode=none") + .run(context -> assertThat(context).hasNotFailed()); + + assertThat(tableExists("THREADMILL_SCHEMA_HISTORY")).isFalse(); + } + + @Test + void dropAndMigrateRequiresTheExplicitDestructiveResetFlag() { + new OracleMigrationRunner(OracleTestDatabase.dataSource()).migrate(); + var dropAndMigrate = contextRunner.withPropertyValues( + "threadmill.store.jdbc-type=oracle", + "threadmill.store.oracle.schema-mode=drop-and-migrate"); + + dropAndMigrate.run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()) + .hasMessageContaining("allow-destructive-schema-reset=true"); + }); + dropAndMigrate + .withPropertyValues("threadmill.store.oracle.allow-destructive-schema-reset=true") + .run(context -> assertThat(context).hasNotFailed()); + new OracleMigrationRunner(OracleTestDatabase.dataSource()).validate(); + } + + @Test + void joinTransactionModeUsesTheTransactionJoinedScheduler() { + contextRunner + .withPropertyValues( + "threadmill.store.jdbc-type=oracle", "threadmill.spring.enqueue-mode=join_transaction") + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(JobStore.class).supportsExternalTransactions()) + .isTrue(); + assertThat(context.getBean(JobScheduler.class)) + .isInstanceOf(TransactionJoinedJobScheduler.class); + }); + } + + private static boolean tableExists(String table) throws Exception { + try (var conn = OracleTestDatabase.dataSource().getConnection(); + var ps = conn.prepareStatement("SELECT COUNT(*) FROM user_tables WHERE table_name = ?")) { + ps.setString(1, table); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() && rs.getInt(1) > 0; + } + } + } +} diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java new file mode 100644 index 00000000..221c5a8b --- /dev/null +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java @@ -0,0 +1,238 @@ +package com.hemju.threadmill.spring; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicReference; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DelegatingDataSource; +import org.springframework.transaction.support.TransactionTemplate; + +import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.engine.LocalWakeBus; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.handler.JobExecutionContext; +import com.hemju.threadmill.core.handler.JobHandler; +import com.hemju.threadmill.core.handler.JobPayload; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.schedule.CronTaskScheduleState; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.store.JobStoreCapabilities; +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.oracle.OracleMigrationRunner; + +/** {@code join_transaction} enqueue mode on Oracle: jobs commit and roll back with the caller. */ +@EnabledIf("com.hemju.threadmill.spring.DockerAvailable#check") +class SpringOracleTransactionBoundaryTest { + + private OracleJobStore store; + private TransactionTemplate transactions; + private TransactionJoinedJobScheduler scheduler; + private CopyOnWriteArrayList wakes; + + public static final class GreetPayload implements JobPayload { + public String tag; + + public GreetPayload() {} + + public GreetPayload(String tag) { + this.tag = tag; + } + } + + public static final class GreetHandler implements JobHandler { + @Override + public void run(GreetPayload p, JobExecutionContext c) {} + } + + @BeforeEach + void setUp() throws Exception { + DataSource dataSource = OracleTestDatabase.dataSource(); + new OracleMigrationRunner(dataSource).migrate(); + OracleTestDatabase.reset(); + store = new OracleJobStore( + dataSource, + new JsonJobSerializer(), + JobStoreCapabilities.defaults(), + new SpringOracleTransactionBoundary(dataSource)); + transactions = new TransactionTemplate(new DataSourceTransactionManager(dataSource)); + var wakeBus = new LocalWakeBus(); + wakes = new CopyOnWriteArrayList<>(); + wakeBus.register(wakes::add); + scheduler = new TransactionJoinedJobScheduler( + store, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + wakeBus); + } + + @Test + void enqueueCommitsWithCallerTransactionAndWakesAfterCommit() { + AtomicReference id = new AtomicReference<>(); + + transactions.executeWithoutResult(status -> { + id.set(scheduler.enqueue(GreetHandler.class, new GreetPayload("commit"))); + assertThat(wakes).isEmpty(); + }); + + assertThat(store.findById(id.get())).isPresent(); + assertThat(wakes).containsExactly("default"); + } + + @Test + void enqueueRollsBackWithCallerTransaction() { + AtomicReference id = new AtomicReference<>(); + + transactions.executeWithoutResult(status -> { + id.set(scheduler.enqueue(GreetHandler.class, new GreetPayload("rollback"))); + status.setRollbackOnly(); + }); + + assertThat(store.findById(id.get())).isEmpty(); + assertThat(store.queueDepths()).isEmpty(); + assertThat(wakes).isEmpty(); + } + + @Test + void dedupRollsBackWithCallerTransaction() { + AtomicReference first = new AtomicReference<>(); + + transactions.executeWithoutResult(status -> { + first.set(scheduler.enqueueIfAbsent( + GreetHandler.class, new GreetPayload("rollback"), "tenant:greet", Duration.ofMinutes(5))); + status.setRollbackOnly(); + }); + + assertThat(first.get()).isInstanceOf(EnqueueResult.Created.class); + assertThat(scheduler.enqueueIfAbsent( + GreetHandler.class, new GreetPayload("retry"), "tenant:greet", Duration.ofMinutes(5))) + .isInstanceOf(EnqueueResult.Created.class); + } + + @Test + void dedupInsideAJoinedTransactionCoalescesOntoTheCommittedJob() { + var committed = scheduler.enqueueIfAbsent( + GreetHandler.class, new GreetPayload("first"), "tenant:once", Duration.ofMinutes(5)); + AtomicReference second = new AtomicReference<>(); + + transactions.executeWithoutResult(status -> second.set(scheduler.enqueueIfAbsent( + GreetHandler.class, new GreetPayload("second"), "tenant:once", Duration.ofMinutes(5)))); + + assertThat(second.get()).isInstanceOf(EnqueueResult.Coalesced.class); + assertThat(((EnqueueResult.Coalesced) second.get()).existingId()) + .isEqualTo(((EnqueueResult.Created) committed).id()); + } + + @Test + void nudgeTakesEffectOnCommitAndIsDiscardedOnRollback() { + registerPumpTask("outbox-pump"); + + transactions.executeWithoutResult(status -> { + scheduler.nudgeRecurring("outbox-pump"); + status.setRollbackOnly(); + }); + assertThat(store.findCronTaskState("outbox-pump").orElseThrow().nudgeRequestedAt()) + .isNull(); + + transactions.executeWithoutResult(status -> { + scheduler.nudgeRecurring("outbox-pump"); + assertThat(store.findCronTaskState("outbox-pump").orElseThrow().nudgeRequestedAt()) + .as("the nudge write is deferred to after commit") + .isNull(); + }); + assertThat(store.findCronTaskState("outbox-pump").orElseThrow().nudgeRequestedAt()) + .isNotNull(); + } + + @Test + void nudgeIsCommittedEvenWhenThePoolHandsOutNonAutoCommitConnections() { + // The nudge runs in afterCommit, when Spring has committed but not yet + // unbound the caller's connection: it must run in Threadmill's own + // committed transaction, not on that connection. + DataSource nonAutoCommit = new NonAutoCommitDataSource(OracleTestDatabase.dataSource()); + var joiningStore = new OracleJobStore( + nonAutoCommit, + new JsonJobSerializer(), + JobStoreCapabilities.defaults(), + new SpringOracleTransactionBoundary(nonAutoCommit)); + var joiningScheduler = new TransactionJoinedJobScheduler( + joiningStore, + new JsonJobSerializer(), + new TestRegistry(), + ProcessingNodeConfig.builder().build(), + new LocalWakeBus()); + registerPumpTask("pooled-pump"); + + new TransactionTemplate(new DataSourceTransactionManager(nonAutoCommit)) + .executeWithoutResult(status -> joiningScheduler.nudgeRecurring("pooled-pump")); + + assertThat(store.findCronTaskState("pooled-pump").orElseThrow().nudgeRequestedAt()) + .isNotNull(); + } + + @Test + void joinTransactionFailsFastWhenCallerTransactionUsesDifferentDataSource() { + try (var other = OracleTestDatabase.newPool()) { + new TransactionTemplate(new DataSourceTransactionManager(other)) + .executeWithoutResult(status -> assertThatThrownBy( + () -> scheduler.enqueue(GreetHandler.class, new GreetPayload("wrong-ds"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("same DataSource")); + } + assertThat(wakes).isEmpty(); + } + + private void registerPumpTask(String name) { + var task = new CronTask( + name, + new CronTask.Trigger.Interval(Duration.ofHours(6)), + GreetHandler.class.getName(), + new JobArgument(GreetPayload.class.getName(), "{}"), + "default", + 0, + CronTask.MissedRunPolicy.DROP, + ZoneId.of("UTC"), + true); + store.upsertCronTask(task); + store.upsertCronTaskState(CronTaskScheduleState.initial( + name, + Instant.now().plus(Duration.ofHours(6)), + CronTaskScheduleState.timingFingerprintOf(task))); + } + + /** A pool-alike whose connections arrive with {@code autoCommit=false}. */ + private static final class NonAutoCommitDataSource extends DelegatingDataSource { + NonAutoCommitDataSource(DataSource target) { + super(target); + } + + @Override + public Connection getConnection() throws SQLException { + Connection connection = super.getConnection(); + connection.setAutoCommit(false); + return connection; + } + } + + private static final class TestRegistry extends ThreadmillJobRegistry { + TestRegistry() { + super(new ThreadmillJobRegistry.Registration( + GreetPayload.class, GreetHandler.class, "default", 0, 5, Duration.ofMinutes(5), null)); + } + } +} diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java index 98a1e40a..24cac914 100644 --- a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleServer.java @@ -4,6 +4,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Locale; import java.util.regex.Pattern; import javax.sql.DataSource; @@ -65,9 +66,16 @@ static Facts read(DataSource dataSource) { try (Connection conn = dataSource.getConnection(); Statement st = conn.createStatement()) { var meta = conn.getMetaData(); + String product = String.valueOf(meta.getDatabaseProductName()); + if (!product.toLowerCase(Locale.ROOT).contains("oracle")) { + throw new JobEngineFatalException( + "Threadmill's OracleJobStore needs an Oracle database, but" + + " the DataSource points at " + product + ". With Spring Boot, set" + + " threadmill.store.jdbc-type for the intended store."); + } int major = meta.getDatabaseMajorVersion(); - String product = meta.getDatabaseProductVersion(); - var matcher = VERSION.matcher(product == null ? "" : product); + String release = meta.getDatabaseProductVersion(); + var matcher = VERSION.matcher(release == null ? "" : release); String version = matcher.find() ? matcher.group() : Integer.toString(major); String characterSet; try (ResultSet rs = st.executeQuery( From e548e6a2ce5b13ca6e958e51e64c7b320f269c81 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 18:22:56 +0200 Subject: [PATCH 3/7] docs: document the Oracle store and run it on 21c XE in CI - threadmill-store-oracle README and docs/oracle-schema.md: requirements, privileges, type mapping, virtual-column indexes, claim path, driver and locale notes, resumable migrations, SQL*Plus-ready manual DDL, and how to test against 21c XE or a real 19c database. - Configuration, quickstart, Spring README, transactions, operations, compatibility, execution model, and index cover the Oracle properties, JDBC store selection, join_transaction on Oracle, and the absent remote wake channel. - Release checklist adds the real-19c run; CI runs the Oracle suite on 21c XE next to the 23ai Free run inside check. - AGENTS.md records the Oracle design decisions and regression coverage. Refs #137 --- .github/workflows/ci.yml | 34 +++++++ AGENTS.md | 45 +++++++- README.md | 32 ++++-- docs/backend-execution-model.md | 13 +++ docs/compatibility.md | 4 +- docs/configuration.md | 20 +++- docs/index.md | 5 + docs/operations.md | 7 +- docs/oracle-schema.md | 132 ++++++++++++++++++++++++ docs/quickstart.md | 12 ++- docs/release-checklist.md | 13 +++ docs/transactions.md | 34 ++++--- threadmill-spring-boot/README.md | 18 +++- threadmill-store-oracle/README.md | 164 ++++++++++++++++++++++++++++++ 14 files changed, 495 insertions(+), 38 deletions(-) create mode 100644 docs/oracle-schema.md create mode 100644 threadmill-store-oracle/README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50c91854..37b7a36e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,40 @@ jobs: if-no-files-found: ignore retention-days: 7 + oracle-xe: + name: Oracle store on 21c XE + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 25 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: temurin + java-version: "25" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3 + + # `check` runs the Oracle suite on 23ai Free. Oracle publishes no free 19c + # image, so 21c XE — the closest free release, without 23ai-only syntax — + # guards the supported 19c floor here; real 19c is a release-checklist run. + - name: Run the Oracle store suite on 21c XE + run: >- + ./gradlew :threadmill-store-oracle:test + -PoracleImage=gvenzl/oracle-xe:21-slim-faststart --stacktrace + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: oracle-xe-test-reports + path: threadmill-store-oracle/build/reports/tests/** + if-no-files-found: ignore + retention-days: 7 + dashboard-browser: name: Dashboard browser smoke runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 04f1a9b9..91af3903 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ This is the standing brief for any AI agent (or human) working on Threadmill. Read it fully before touching the repository. -**Project status:** v1.0.0 is the release version and the v1 feature set is complete. Follow-on improvements continue through the public issue tracker. Three storage backends (in-memory, PostgreSQL 18+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. +**Project status:** v1.0.0 is the release version and the v1 feature set is complete. Follow-on improvements continue through the public issue tracker. Four storage backends (in-memory, PostgreSQL 18+, Oracle Database 19c+, Redis with standalone/Sentinel/Cluster), the processing engine, scheduling and recurring APIs, the per-queue pause primitive, the bulk-enqueue path, claim-time per-key concurrency with workflow inheritance, queue-family lanes, Spring Boot integration with explicit enqueue transaction modes (`after_commit`, `join_transaction`, `immediate`), Micrometer metrics, optional OpenTelemetry tracing, the data-first dashboard model, Spring dashboard API, static React dashboard UI, docs, examples, and a soak/load module are all in place. Store-backed maintenance leadership, crash-safe Redis claim, producer-side deduplication, long-running job check-ins, idle-worker wake signal, cross-node remote wake hints, and failure-detail truncation are part of the production-readiness baseline. The soak harness verifies its invariants live (bounded streaming checks, fail-fast, `progress.json`), supports node churn and external datastores, and ships a dual-backend `soakEndurance` orchestrator for hours-scale Postgres+Redis sign-off runs. --- @@ -27,8 +27,9 @@ Commercial support is available through **hemju.com** for Threadmill **1.0**; co - No preview features in the public API. - **Build tool:** Gradle (≥ 9.5), multi-module. The wrapper is committed and its distribution SHA-256 is pinned. Gradle dependency locking and checksum verification cover every project, including `buildSrc`; dependency changes update both the relevant `gradle.lockfile` files and `gradle/verification-metadata.xml` in the same contribution. Pull requests, a nightly `main` run, and tag releases use pinned Node/npm plus a pinned, checksum-verified OSV Scanner; the gate scans every Git-tracked `gradle.lockfile` and npm `package-lock.json`, then runs `npm audit` at the `low` threshold. Never point it at the unsupported version catalog or scan historical entries in verification metadata as though they were resolved dependencies. Scanner exceptions in `osv-scanner.toml` are advisory-specific, justified, and time-bounded; never add a package-wide or permanent vulnerability ignore. - **Frameworks:** usable from Spring Boot. The core is framework-agnostic so additional adapters can return later as real integrations. Native image remains a future target — minimise reflection. -- **Storage backends:** PostgreSQL and Redis are both first-class, plus an in-memory store for tests and local development. All three satisfy one storage SPI and pass one shared contract test suite. +- **Storage backends:** PostgreSQL, Oracle Database, and Redis are first-class, plus an in-memory store for tests and local development. All four satisfy one storage SPI and pass one shared contract test suite. - **PostgreSQL 18+ only.** Build, tests, and example all run against `postgres:18-alpine`. No back-compat shims for earlier majors. + - **Oracle Database 19c+ only, `AL32UTF8`, `NLS_COMP=BINARY`.** SQL uses only the 19c feature set. No free 19c image exists: CI runs 23ai Free (`check`) and 21c XE (separate job) with `OPTIMIZER_FEATURES_ENABLE='19.1.0'`; a real-19c run is a release-checklist item. Standard JDBC only; applications supply the driver (`ojdbc11` is test-only). --- @@ -40,6 +41,7 @@ Modules are physically separate. No storage implementation and no UI may ever be |---|---|---| | `threadmill-core` | `com.hemju.threadmill.core` | Job model, state machine, `JobStore` SPI, serialization, processing engine, interceptors, scheduling API. **No storage implementation, no UI, no framework code.** | | `threadmill-store-postgres` | `com.hemju.threadmill.store.postgres` | PostgreSQL job store, schema, in-process migration runner. | +| `threadmill-store-oracle` | `com.hemju.threadmill.store.oracle` | Oracle Database 19c+ job store, schema, in-process resumable migration runner. | | `threadmill-store-redis` | `com.hemju.threadmill.store.redis` | Redis job store, atomic Lua scripts. | | `threadmill-store-memory` | `com.hemju.threadmill.store.memory` | In-memory job store for tests and dev — held to the same contract as the real backends, not a simplified fake. | | `threadmill-spring-boot` | `com.hemju.threadmill.spring` | Spring Boot 4.x auto-configuration. | @@ -82,7 +84,9 @@ Use these terms for these concepts. Refining a name during implementation is all | A queue with reserved capacity inside one node | `QueueLane` | | The persistence SPI | `JobStore` | | Cross-node wake hint SPI | `RemoteWakeChannel` | -| In-memory / Postgres / Redis stores | `InMemoryJobStore`, `PostgresJobStore`, `RedisJobStore` | +| In-memory / Postgres / Oracle / Redis stores | `InMemoryJobStore`, `PostgresJobStore`, `OracleJobStore`, `RedisJobStore` | +| Oracle schema migrator | `OracleMigrationRunner` | +| Spring choice between the JDBC stores | `JdbcStoreType` (`threadmill.store.jdbc-type`) | | Serialized description of the work to run | `JobSpec` | | A typed argument inside the spec | `JobArgument` | | A recurring (cron / interval) definition | `CronTask` | @@ -161,7 +165,9 @@ The build uses Gradle (≥ 9.5) and the project's Java 25 toolchain. - `./gradlew :threadmill-simulation:simulateNudgePostgres` / `simulateNudgeRedis` — run the fixed process-separated nudge simulation with a hard-killed maintenance leader and a producer killed between its durable work write and nudge. `simulateNudge` runs both real backends. - `./gradlew :threadmill-example:run` — compile and run the public getting-started example. - `./gradlew productionCheck` — release-candidate validation from clean outputs for every subproject: complete checks, Javadoc, real store tests, soak, short correctness and process-separated nudge simulations, dependency scan hook, example, and artifact inspection. Central Portal publication depends on this task and an exact release-tag/project-version match, so the verified artifacts are uploaded from the same task graph. +- `./gradlew :threadmill-store-oracle:test` — Oracle contract, regression, plan, and migration tests on `gvenzl/oracle-free` 23ai; `-PoracleImage=gvenzl/oracle-xe:21-slim-faststart` for 21c XE (amd64), or `-PoracleJdbcUrl=… -PoracleUser=… -PoraclePassword=…` for an existing (for example real 19c) database whose Threadmill rows the tests delete. - Integration tests for the real backends use **Testcontainers** and need a working container runtime (Docker / Podman / Colima / OrbStack). +- On a JDK newer than 25 (for example 27) the pinned Palantir formatter fails with `NoSuchFieldError … endPositions`; run Gradle with a JDK 25 `JAVA_HOME`, and pass `--no-build-cache` once if a Spotless step cached on the newer JDK keeps failing. The Gradle wrapper is committed; new clones run `./gradlew` without a system Gradle install. @@ -171,7 +177,7 @@ The Gradle wrapper is committed; new clones run `./gradlew` without a system Gra Testing is a first-class deliverable; treat the test suite as equal in weight to the code. -- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** shared contract suite — that is the only thing guaranteeing all three backends behave the same. +- The `JobStore` contract is an **abstract test suite written first**. Every store (in-memory, Postgres, Oracle, Redis) extends `AbstractJobStoreContractTest` and is held to the **identical** shared contract suite — that is the only thing guaranteeing all four backends behave the same. - Integration tests run against **real PostgreSQL and real Redis via Testcontainers**. Never mock the datastore — the hard bugs live in the datastore's real locking, atomicity, and encoding behaviour. - **Every bug becomes a named, permanent regression test** added with (or before) the fix. See the regression-coverage matrix in §11. - Concurrency tests (N simulated nodes claiming from one store), serialization round-trip tests (including 4-byte Unicode and oversized payloads), and the soak/load module are first-class. @@ -269,6 +275,24 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Store and migration transactions share failure cleanup.** `PostgresTransactions` rolls back SQL exceptions, runtime exceptions, and errors before restoring the prior auto-commit mode. Cleanup failures are suppressed on the original failure; a failed rollback aborts the connection rather than risking a commit during mode restoration. Real PostgreSQL regressions `transactionErrorRollsBackWritesBeforeRestoringAutoCommit` and `transactionCleanupFailuresDoNotMaskTheOriginalError` pin this behavior (issue #135). - **Spring Boot Postgres schema handling is explicit.** Auto-configured Postgres stores run `threadmill.store.postgres.schema-mode=migrate` by default before constructing `PostgresJobStore`. `validate` is for externally-applied DDL, `none` skips schema handling, and `drop-and-migrate` requires `threadmill.store.postgres.allow-destructive-schema-reset=true` because it destroys Threadmill job data. +### Oracle-specific + +- **Oracle Database 19c or later, `AL32UTF8`, `NLS_COMP=BINARY`, enforced at startup.** `OracleServer.require` runs in the `OracleJobStore` constructor and refuses older releases, other database character sets (4-byte Unicode must round-trip), linguistic comparison (keyset predicates and index ranges assume binary), and non-Oracle `DataSource`s with a message naming the product. `OracleServerTest` pins all four. +- **Standard JDBC only; the application supplies the driver.** No `oracle.jdbc` types, no `CREATE TYPE`, no `DBMS_*` grants; the schema user needs `CREATE TABLE`, `CREATE TRIGGER`, and quota. `ojdbc11` is a test dependency of the store, never an API/implementation dependency, so the published module carries no Oracle-licensed artifact. +- **Partial indexes are indexed `VIRTUAL` columns, never expression indexes.** Every PostgreSQL partial index becomes columns of the form `CASE WHEN THEN END` (`keyed_queue`, `pending_key`, `scheduled_due`, …) plus an index; rows outside the predicate have all-`NULL` keys, which a B-tree does not store. The first cut used function-based indexes over the same expressions: under the 19c optimizer feature set Oracle then re-sorted every per-key top-n (it re-evaluates the expression from table columns instead of reading index order), while virtual columns read index order and stop after n entries. Queries name the columns directly, which also removes expression-matching drift. A new expression column clashes with an existing function-based index over the same expression (`ORA-54015`). +- **No lateral joins on the claim path.** A correlated top-n `LATERAL`/`APPLY` keeps `SORT ORDER BY STOPKEY` over each key's whole range, and a top-n lateral driven by a `TABLE(SYS.ODCIVARCHAR2LIST(...))` collection returned **wrong results** (zero rows) on Oracle 23.26. Per-key heads are one `ROWNUM` top-n branch per key joined with `UNION ALL` (bucketed branch counts, at most 64 per statement), key enumeration is a recursive `MIN` loose scan, and admission is an `EXISTS` range probe bounded at the candidate's own `current_state_at` per candidate (`earlier pending` / `earlier EXCLUSIVE`). +- **`ROWNUM`, not `FETCH FIRST`, for hot top-n queries.** With `OPTIMIZER_FEATURES_ENABLE='19.1.0'`, `FETCH FIRST n ROWS` became `WINDOW SORT PUSHED RANK` over the full range; the classic `SELECT * FROM (… ORDER BY …) WHERE ROWNUM <= ?` is `COUNT STOPKEY` on every release. Monitoring "oldest" reads are `MIN(...)` (`INDEX RANGE SCAN (MIN/MAX)`). +- **Locking queries cannot carry a row limit (`ORA-02014`).** Locking cursors are index-ordered `FOR UPDATE SKIP LOCKED` statements read with `setFetchSize`/`setMaxRows` = the page budget; Oracle locks `SKIP LOCKED` rows as they are fetched (`unkeyedClaimCursorLocksOnlyTheRowsItFetches` pins this). Keyed candidates are read unlocked and then locked by primary key with `SKIP LOCKED`, so a candidate another claimer holds is dropped rather than waited for. +- **Hot statements carry index hints and their executed plans are asserted.** The optimizer chose full scans and fast-full scans on realistic statistics; hints make the plans deterministic. `OracleQueryPlanTest` runs the store's exact package-private SQL with **typed** binds and reads `DBMS_XPLAN.DISPLAY_CURSOR`: `EXPLAIN PLAN` treats binds as `VARCHAR2`, wraps `RAW` columns in `RAWTOHEX`, and reports full scans that never happen. The test container grants the user `SELECT ANY DICTIONARY`; elsewhere the plan tests skip. +- **Long `CLOB` values go through a temporary `Clob`.** Binding a string longer than ~16K characters directly (`setString` or `setCharacterStream`) made `ojdbc11` 23.26 split a surrogate pair straddling its ~32 KiB chunk boundary, silently corrupting emoji in job bodies. `OracleClobs` binds values up to 8,000 characters directly (below the direct-bind limit even at 3 bytes per UTF-16 unit) and longer ones through `createClob()` + `Clob.setString`, freeing them after the statement so pooled sessions do not accumulate temporary LOBs. `supplementaryCharactersSurviveLongClobBodiesAndPayloads` pins both paths. +- **Text ordering is explicitly binary.** The driver derives `NLS_LANGUAGE`, and so `NLS_SORT`, from the client JVM locale (a German JVM sorts `a < Ä < B`). Comparisons follow `NLS_COMP` (binary), but `ORDER BY` follows `NLS_SORT`, so linguistic page order made keyset pages (`name > ?`) skip rows and prevented index-ordered reads. Small tables order by `NLSSORT(col, 'NLS_SORT = BINARY')`; the idle concurrency-group page reads a `RAW(2000)` virtual sort key (`idle_sort`). Never `ORDER BY` a text column in a paging query. `keysetPagesFollowBinaryOrderUnderALinguisticClientLocale` runs under a German session. +- **UTC `TIMESTAMP(9)` bound as `LocalDateTime`.** Nanosecond-exact and independent of JVM and session time zones (`java.sql.Timestamp` binding would apply the JVM default zone). Server-side time is `SYS_EXTRACT_UTC(SYSTIMESTAMP)`. +- **Oracle stores `''` as `NULL`.** Recurring payloads and pause reasons map `NULL` back to `""`; mutex holders compare with `DECODE`, which treats two `NULL`s as equal. +- **Statement-level rollback.** A failed statement does not abort an Oracle transaction. Upserts are update-then-insert with an `ORA-00001` retry; concurrency-group locking inserts and retries on a duplicate; the deduplicated enqueue rolls back to a savepoint and coalesces onto the concurrent winner even inside a joined Spring transaction (PostgreSQL cannot). `requestCronNudge` locks the task row (`FOR UPDATE`) so acceptance is atomic with the existence and enabled checks. +- **Migrations are resumable because Oracle commits DDL.** `OracleMigrationRunner` records progress per statement in `threadmill_schema_history` (`statements_applied`, `success`), resumes at the first unrecorded statement, and tolerates an "already exists" error only for that first resumed statement (its DDL may have committed before the crash). Migrators serialize on a `threadmill_schema_lock` row held on a **second** connection, because DDL on the migrating connection would commit and release it. Files use SQL*Plus `/` terminators so emitted SQL runs unchanged; leading comments are stripped from each executed statement. The single `V1__baseline.sql` already contains the full PostgreSQL V1–V11 end state; future Oracle changes ship as additive files. +- **No remote wake channel.** `createRemoteWakeChannel` returns empty: `DBMS_ALERT.SIGNAL` locks the alert name until commit and would serialize enqueuing transactions, and AQ needs grants and infrastructure. Cross-node pickup falls back to `pollInterval`. +- **Spring chooses between the JDBC stores from configuration, not a database round trip.** `JdbcStoreSelection` runs in conditions, before any bean exists: `threadmill.store.jdbc-type`, then the `spring.datasource.url` scheme, then the module present (PostgreSQL on a tie, preserving prior behaviour). The chosen store still verifies the server at startup. + ### Redis-specific - **Redis claim-lock cleanup covers uncertain acquisition.** A `SET NX PX` reply can time out or be interrupted after Redis acquired the token. The store attempts token-checked cleanup even without an acknowledged acquisition, releases earlier locks if a later bulk acquisition fails, and releases locks when single/deduplicated workflow re-snapshotting fails. Cleanup continues across ordinary release failures, preserves the acquisition failure, and temporarily clears/restores caller interruption for Lettuce's synchronous release. It must never delete a replacement owner's token. The real-Redis `RedisClaimLockRecoveryTest` injects faults after actual Redis commands and covers lost replies, partial acquisition, workflow re-reads, interrupted cleanup, replacement ownership and cleanup-error suppression. Expiry remains the crash/outage fallback; no submission or exactly-once guarantee is added. @@ -552,6 +576,19 @@ Every hard-won failure mode that has come up during development, and the test th | Metrics refresh re-entered on the same thread by a registry callback that reads a meter during `reconcileQueueMeters`, recursing into the `ConcurrentHashMap` mapping function for the queue being registered; the swallowed `Recursive update` surfaces as a `threadmill.metrics.queue.meter.errors` increment on a healthy store and registry (issue #132) | `ThreadmillMetricsTest.queueGaugeCallbackDuringRegistrationDoesNotReEnterTheRefresh` | | Handler cannot see the watchdog deadline (`@Job(timeout)` documented only as a cap; the interrupt aborts virtual-thread socket I/O mid-step and cascades through pooled connections); no handler-facing accessor for the running context; a shutdown interrupt surfacing as `SocketException` billed as a handler fault and burning an attempt (issue #119) | `ProcessingNodeTest.deadlineMatchesTheWatchdogRuleBeforeAndAfterCheckIn` + `cooperativeHandlerThatStopsOnRemainingIsNeverInterrupted` + `timeoutCancellationIsRecordedBeforeTheInterruptLands` + `shutdownInterruptSurfacingAsASocketFailureIsStillClassifiedAsShutdown` + `closeCollapsesTheDeadlineToTheShutdownGracePeriod` + `currentContextResolvesInsideRunAndIsEmptyOutside` + `JobExecutionContextTest` (defaults: unbounded deadline, zero-clamped `remaining()`, `current()` empty outside a job) + `timedOutHandlerThatChecksInDuringCleanupIsInterruptedAgain` + `forcedShutdownLatchedBeforeTheSweepStillCancelsALateRegisteringAttempt` + `absurdTimeoutMetadataFallsBackToTheGlobalTimeout` + `ProcessingNodeConfigTest.timeoutsBeyondTheMaximumAreRejected` | | Store decorator implements `JobStore` directly and silently answers an interface default instead of forwarding it (tracing missed `supportsExternalTransactions()` and `createRemoteWakeChannel(String)`; issue #131) | `ForwardingJobStoreTest.forwardsEveryStoreOperationIncludingInterfaceDefaults` + `decoratorContractNamesADefaultMethodThatFellThroughToTheInterface` (red path) + `TracingJobStoreForwardingTest` + `MeteredJobStoreForwardingTest` (reflective forwarding plus named external-transaction and remote-wake regressions) + `ForwardingJobStoreContractTest` + `TracingJobStoreContractTest` + `MeteredJobStoreContractTest` | +| Oracle store diverges from the shared contract (issue #137) | `OracleJobStoreContractTest` (full `AbstractJobStoreContractTest` through connections that start with `autoCommit=false`) | +| Oracle accepted on an unsupported server: pre-19c, non-`AL32UTF8`, linguistic `NLS_COMP`, or a non-Oracle `DataSource` | `OracleServerTest` (four refusal / acceptance cases) | +| Oracle locking cursor locks the whole result at open instead of the fetched page (claimers pin the queue) | `OracleJobStoreRegressionTest.unkeyedClaimCursorLocksOnlyTheRowsItFetches` + `unkeyedClaimLocksOnlyANarrowPageSoConcurrentClaimersAreNotStarved` + `claimReadyIsAtomicAcrossManyConcurrentVirtualThreads` | +| Oracle per-key heads regress to a lateral top-n (collection-driven `LATERAL … FETCH FIRST` returned zero rows) or to backlog-sorting plans | `OracleJobStoreRegressionTest.keyedClaimGathersHeadsFromEveryKeyInOneCall` + `OracleQueryPlanTest.keyedHeadsStopAfterEachKeysFirstRowsEvenOnAHotKey` | +| Oracle claim, admission, maintenance, retention, or monitoring statement regresses to a full scan, a full-range sort, or a `WINDOW SORT` under the 19c optimizer | `OracleQueryPlanTest` (executed-cursor plans with typed binds) | +| Oracle keyed claim rotation never reaches claimable keys beyond the first blocked page, or reports an empty poll after a drained tail | `OracleJobStoreRegressionTest.keyedClaimRotationReachesClaimableWorkBeyondTheFirstBlockedPage` + `drainedTailRestartsKeyedScanWithoutAnEmptyPoll` | +| Long Oracle `CLOB` bind splits a surrogate pair at the driver's chunk boundary (corrupted emoji in job bodies / recurring payloads) | `OracleJobStoreRegressionTest.supplementaryCharactersSurviveLongClobBodiesAndPayloads` | +| Oracle keyset pages ordered by a linguistic `NLS_SORT` (client JVM locale) skip recurring tasks or idle concurrency groups | `OracleJobStoreRegressionTest.keysetPagesFollowBinaryOrderUnderALinguisticClientLocale` | +| Oracle timestamps shift with the JVM or session time zone | `OracleJobStoreRegressionTest.instantsRoundTripIndependentlyOfJvmAndSessionTimeZones` | +| Oracle's empty-string-is-NULL rule breaks empty recurring payloads, empty mutex holders, or empty pause reasons | `OracleJobStoreRegressionTest.emptyStringsSurviveOraclesEmptyStringIsNullRule` | +| Oracle quarantine of an unreadable keyed member leaves the workflow hold (and the key) held | `OracleJobStoreRegressionTest.quarantinedKeyedWorkflowMemberReleasesTheWorkflowHold` + `unreadableBodyAtTheQueueHeadIsQuarantinedWithoutStallingTheQueue` | +| Oracle migration crash between a committed DDL statement and its progress record wedges the schema; concurrent migrators race; edited or unknown migrations accepted | `OracleMigrationRunnerTest.interruptedMigrationResumesAndToleratesTheUnrecordedCommittedStatement` + `concurrentCleanMigrationsAreSerialized` + `migrateAndValidateRefuseAnEditedMigration` + `migrateAndValidateRefuseAnUnknownVersion` + `migrationCommitsWhenConnectionsDefaultToNonAutoCommit` + `emittedCleanInstallSqlAppliesToACleanSchemaAndValidates` | +| Spring wires the wrong JDBC store when both modules are present, or Oracle `join_transaction` does not commit/roll back with the caller | `JdbcStoreSelectionTest` + `SpringOracleAutoConfigurationTest` + `SpringOracleTransactionBoundaryTest` | ### Postgres-layer improvements (engagement notes) diff --git a/README.md b/README.md index 0590b4bb..1fc7fb0b 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ A modern, lightweight background-job-processing library for Java 25. Threadmill runs your idempotent background work durably, on a cluster of -application nodes, against a real datastore (PostgreSQL or Redis) — or an -in-memory store for local development. The design uses well-understood +application nodes, against a real datastore (PostgreSQL, Oracle Database, or +Redis) — or an in-memory store for local development. The design uses well-understood techniques: a durable job store, optimistic concurrency control, polling worker loops, an explicit state machine, and a small, deliberate public API. @@ -20,8 +20,8 @@ and **[JobRunr](https://www.jobrunr.io/)** on Java 25. It gives you both what Quartz gives you (durable cron and interval scheduling, clustering, misfire handling) and what JobRunr gives you (fire-and-forget background jobs, a dashboard, retries) behind one small, idempotency-first API — built on virtual -threads and scoped values, with first-class PostgreSQL **and** Redis backends -held to a single shared contract test suite. If you are choosing between the +threads and scoped values, with first-class PostgreSQL, Oracle, **and** Redis +backends held to a single shared contract test suite. If you are choosing between the two, or want to consolidate both onto one library, Threadmill is the replacement. @@ -50,7 +50,7 @@ These examples use **1.0.0**. Upgrade existing installations using the ```kotlin // build.gradle.kts implementation("com.hemju.threadmill:threadmill-core:1.0.0") -implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") // or -store-redis / -store-memory +implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") // or -store-oracle / -store-redis / -store-memory implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") // optional Spring Boot integration ``` @@ -116,6 +116,7 @@ See [docs/quickstart.md](docs/quickstart.md) for a complete Spring walkthrough, | `threadmill-core` | Job model, state machine, `JobStore` SPI, serialization, engine (`ProcessingNode`, `Dispatcher`, `MaintenanceCycle`), interceptors, scheduling API. **No storage, no framework code.** | | `threadmill-store-memory` | In-memory `JobStore` for tests and local dev. Held to the same contract as the real backends. | | `threadmill-store-postgres` | PostgreSQL backend (auto-migrations, `SELECT … FOR UPDATE SKIP LOCKED` claim, per-state counter table with trigger). | +| `threadmill-store-oracle` | Oracle Database 19c+ backend (standard JDBC, indexed virtual columns as partial indexes, `FOR UPDATE SKIP LOCKED` claim, resumable migrations). | | `threadmill-store-redis` | Redis backend (reliable-fetch via Lua, per-job HASH + per-queue ZSET + counts HASH). | | `threadmill-spring-boot` | Spring Boot 4.x auto-configuration. | | `threadmill-metrics` | Micrometer integration: jobs-per-state gauges, processed/failed counters, processing-time timer. | @@ -135,6 +136,16 @@ path). Migrations are applied automatically on startup; an `emitPendingSql()` method produces pending SQL for teams that prefer Flyway/Liquibase, and `emitCleanInstallSql()` emits the full clean-install DDL. +**Oracle Database 19c+** is a first-class backend with the same model: the +body is a `CLOB`, scalar columns are indexed through virtual columns that emulate +partial indexes, and the claim uses `FOR UPDATE SKIP LOCKED`. It needs an +`AL32UTF8` database and only `CREATE TABLE`/`CREATE TRIGGER` privileges, uses +standard JDBC (bring your own driver), and resumes interrupted migrations +because Oracle commits DDL statement by statement. It has no cross-node wake +channel; other nodes pick up new work within `poll-interval`. See the +[module README](threadmill-store-oracle/README.md) and +[Oracle schema](docs/oracle-schema.md). + **Redis 7.4+** is a first-class backend and requires `noeviction`. Every multi-key state transition is a single atomic Lua script. Standalone, Sentinel, and Cluster topologies are configured through one factory path. Redis Cluster uses a @@ -230,8 +241,8 @@ process. - Job model with append-only state history, optimistic-lock versioning, relationship and result fields, and bounded size. - Centralised state machine; illegal transitions throw. -- `JobStore` SPI plus three backends (in-memory, PostgreSQL, Redis), each - held to one shared abstract contract test. +- `JobStore` SPI plus four backends (in-memory, PostgreSQL, Oracle, Redis), + each held to one shared abstract contract test. - Processing engine: virtual-thread workers, scoped-value context propagation, per-job timeout, single failure code path, circuit-breaker decay, store-outage tolerance, orphan-reclaim, poison-job quarantine. @@ -301,8 +312,11 @@ and the engine resumes automatically once the store is reachable again. ./gradlew productionCheck # release-candidate validation gauntlet ``` -The PostgreSQL and Redis tests use Testcontainers and require a running -container runtime (Docker / Podman / Colima / OrbStack). +The PostgreSQL, Oracle, and Redis tests use Testcontainers and require a +running container runtime (Docker / Podman / Colima / OrbStack). The Oracle +suite runs on Oracle 23ai Free by default; see the +[Oracle store README](threadmill-store-oracle/README.md#tests) for 21c XE and +real 19c runs. ## Examples diff --git a/docs/backend-execution-model.md b/docs/backend-execution-model.md index 17e2fd42..da00b080 100644 --- a/docs/backend-execution-model.md +++ b/docs/backend-execution-model.md @@ -130,6 +130,19 @@ database, connection pool, worker count, or queue topology can drain. A growing queue with passing concurrency invariants is a capacity signal, not by itself a same-key correctness failure. +### Oracle Path + +Oracle follows the Postgres path — the same tables, row locks, group rows, and +workflow holds in one transaction — with three mechanical differences. Oracle +rejects a row limit on a locking query, so locking cursors are index-ordered and +fetched only as far as the page budget; Oracle locks `SKIP LOCKED` rows as they +are fetched. Keyed candidates are first read through one top-n branch per key +(at most 128 keys per unproductive poll, rotated like Postgres) and then locked +by primary key with `SKIP LOCKED`, dropping rows another claimer holds. The +leapfrog rule is decided by `EXISTS` range probes bounded at each candidate's own +`current_state_at`. See the +[Oracle store README](../threadmill-store-oracle/README.md#how-the-claim-works). + ## Redis Path Redis uses server-side Lua scripts as the atomic boundary. Java may inspect diff --git a/docs/compatibility.md b/docs/compatibility.md index 4753508a..a3548fc2 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -10,7 +10,9 @@ targets 1.0.0; it is not yet a published release. ## Supported platform and storage -Java 25 is required. PostgreSQL requires 18 or later. Redis data nodes require +Java 25 is required. PostgreSQL requires 18 or later. Oracle Database requires +19c or later with the `AL32UTF8` database character set and the default +`NLS_COMP=BINARY`; the application supplies the JDBC driver. Redis data nodes require 7.4 or later and `noeviction`; validate every node that may become a primary. The exact tested dependency versions are in Gradle/npm locks and the release validation artifacts. A supported major/minimum is not evidence that every diff --git a/docs/configuration.md b/docs/configuration.md index d5b3980a..7e46d8a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,6 +74,22 @@ its own `JobStore` bean. See [postgres-schema.md](postgres-schema.md) for manual DDL, validation, and reset guidance. +## Oracle Store Properties + +Spring auto-creates an Oracle store when a `DataSource` bean exists and +`threadmill-store-oracle` is on the classpath, unless the application defines +its own `JobStore` bean or Redis is configured. The application supplies the +Oracle JDBC driver. + +| Setting | Default | Notes | +|---|---:|---| +| `threadmill.store.oracle.schema-mode` | `migrate` | `migrate`, `validate`, `none`, or `drop-and-migrate`. Auto-configured Oracle stores run this action before `OracleJobStore` is created. | +| `threadmill.store.oracle.allow-destructive-schema-reset` | `false` | Must be `true` for `drop-and-migrate`. This drops only Threadmill-owned tables and destroys stored jobs. | +| `threadmill.store.jdbc-type` | | `postgres` or `oracle`. Needed only when both JDBC store modules are on the classpath and `spring.datasource.url` does not name the database. Without it, a `jdbc:oracle:` URL selects Oracle, a `jdbc:postgresql:` URL selects PostgreSQL, and otherwise the one module present wins (PostgreSQL when both are). | + +See [oracle-schema.md](oracle-schema.md) for privileges, manual DDL, and how +interrupted migrations resume. + ## Redis Store Properties Spring can auto-create a Redis store when `threadmill.store.redis.*` is set. @@ -111,14 +127,14 @@ correctness fallback. | Setting | Default | Notes | |---|---:|---| -| `threadmill.remote-wake.enabled` | `true` | Auto-create Postgres `LISTEN`/`NOTIFY` or Redis Pub/Sub wake hints for Spring auto-configured durable stores. In-memory and custom stores stay local-wake only unless the application provides a `RemoteWakeChannel` bean. | +| `threadmill.remote-wake.enabled` | `true` | Auto-create Postgres `LISTEN`/`NOTIFY` or Redis Pub/Sub wake hints for Spring auto-configured durable stores. Oracle, in-memory, and custom stores stay local-wake only unless the application provides a `RemoteWakeChannel` bean; their other nodes pick up work within `poll-interval`. | | `threadmill.remote-wake.channel` | backend default | Optional channel override. Defaults to `threadmill_wake` for Postgres and `{threadmill}:wake` for Redis. Set this when multiple isolated Threadmill deployments share one Postgres database or Redis instance. | ## Spring Recurring Properties | Setting | Default | Notes | |---|---:|---| -| `threadmill.spring.enqueue-mode` | `after_commit` | `after_commit`, `join_transaction`, or `immediate`. `join_transaction` is Spring + Postgres only. | +| `threadmill.spring.enqueue-mode` | `after_commit` | `after_commit`, `join_transaction`, or `immediate`. `join_transaction` is Spring + PostgreSQL or Oracle only. | | `threadmill.spring.recurring-namespace` | `spring.application.name` | Namespace whose annotation-driven recurring tasks are reconciled at startup. If neither value is set, Threadmill only upserts discovered tasks and does not delete stale ones. | ### Initial and lifecycle size budgets diff --git a/docs/index.md b/docs/index.md index 5d09ea22..8247d094 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,6 +36,8 @@ - [Configuration](configuration.md) — every `threadmill.*` property. - [PostgreSQL schema](postgres-schema.md) — schema modes, manual DDL, and reset guidance. +- [Oracle schema](oracle-schema.md) — privileges, schema modes, resumable + migrations, and SQL*Plus-ready manual DDL. - [Redis topologies](redis-topologies.md) — standalone, Sentinel, Cluster. - [Operations](operations.md) — production runbook, pause / resume, monitoring. - [Troubleshooting](troubleshooting.md) — symptom → cause → fix. @@ -69,6 +71,9 @@ operational notes: - [`threadmill-store-postgres`](../threadmill-store-postgres/README.md) — the full schema, PG18-only enforcement, claim semantics, deadlock retry, connection-pool sizing. +- [`threadmill-store-oracle`](../threadmill-store-oracle/README.md) — Oracle + 19c+ requirements, virtual-column indexes, the claim path, driver and locale + notes, and how to test against 21c XE or a real 19c database. - [`threadmill-store-redis`](../threadmill-store-redis/README.md) — the full key layout, the Lua script inventory, the reliable-fetch claim, AOF durability. diff --git a/docs/operations.md b/docs/operations.md index ad72c114..2bcda066 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -137,7 +137,12 @@ stores also publish cross-node wake hints after local wake timing has already decided that work is visible. Postgres uses `LISTEN`/`NOTIFY` on `threadmill_wake` by default. Redis uses Pub/Sub on `{threadmill}:wake` by default. Set `threadmill.remote-wake.channel` when multiple isolated -Threadmill deployments share one datastore. +Threadmill deployments share one datastore. Oracle has no lightweight +notification primitive (`DBMS_ALERT` would serialize every enqueuing +transaction), so an Oracle deployment relies on polling across nodes: work +enqueued on another node starts within `threadmill.pollInterval`, while same-JVM +enqueues still wake their dispatcher immediately. Lower the poll interval if that +latency matters more than the extra idle queries. The Postgres listener holds one JDBC connection for as long as remote wake is enabled. If it uses the same application pool, size that pool with one diff --git a/docs/oracle-schema.md b/docs/oracle-schema.md new file mode 100644 index 00000000..938bc01a --- /dev/null +++ b/docs/oracle-schema.md @@ -0,0 +1,132 @@ +# Oracle Schema + +Threadmill's Oracle store targets Oracle Database 19c and later. The store +constructor checks the release, the `AL32UTF8` database character set, and the +session's `NLS_COMP=BINARY` (the default), and refuses anything else before any +job query runs. Module-level design notes — type mapping, virtual-column +indexes, the claim path — are in the +[`threadmill-store-oracle` README](../threadmill-store-oracle/README.md). + +## Privileges + +The schema user needs only: + +- `CREATE TABLE` and `CREATE TRIGGER` (the `RESOURCE` role, or the individual + privileges), and +- quota on its default tablespace. + +No `CREATE TYPE`, `DBMS_*` execute grants, Advanced Queuing, or `SELECT ANY +DICTIONARY` is required at runtime. A user without DDL rights can still run +Threadmill after a DBA applies the schema (see [Manual DDL](#manual-ddl)) and +the application uses schema mode `validate` or `none`. + +## Spring Boot Schema Modes + +When Spring Boot auto-configures `OracleJobStore` from an application +`DataSource`, it handles the Threadmill schema before constructing the store. +Applications that define their own `JobStore` bean own schema handling +themselves. + +| Property | Default | What | +|---|---|---| +| `threadmill.store.oracle.schema-mode` | `migrate` | `migrate`, `validate`, `none`, or `drop-and-migrate`. | +| `threadmill.store.oracle.allow-destructive-schema-reset` | `false` | Required for `drop-and-migrate`. | + +`migrate` applies pending migrations (and resumes an interrupted one). +`validate` fails startup unless `threadmill_schema_history` records exactly the +shipped migrations, each complete and with a matching checksum. `none` performs +no DDL or validation. `drop-and-migrate` drops Threadmill-owned tables and +recreates the schema; it destroys stored jobs and is intended only for +disposable dev/test schemas. + +When both `threadmill-store-postgres` and `threadmill-store-oracle` are on the +classpath, Spring picks the store from `threadmill.store.jdbc-type` +(`postgres` or `oracle`), else from the `spring.datasource.url` scheme, else +PostgreSQL. See [configuration](configuration.md#oracle-store-properties). + +## Migrations Are Not Atomic on Oracle + +Oracle commits every DDL statement implicitly, so a migration cannot be +applied in one transaction. `OracleMigrationRunner` compensates: + +- **Progress per statement.** `threadmill_schema_history` records, for each + migration, how many of its statements completed and whether the migration + finished. A DML statement commits together with its progress record. +- **Resume.** After a crash, the next `migrate()` continues at the first + unrecorded statement. That statement's DDL may already have committed before + the crash prevented its progress record, so for that one statement an + "already exists" error (`ORA-00955`, `ORA-01430`, `ORA-02260`, …) counts as + done. Any other failure stops the run with the statement number; fix the + cause and rerun. +- **One migrator at a time.** Migrators serialize on a row lock in + `threadmill_schema_lock`, held on a second connection because each DDL + statement would commit, and so release, a lock taken on the migrating + connection. A waiting node logs once and gives up after five minutes with an + actionable message. +- **Validation first.** Every applied migration's description and SHA-256 + checksum is checked before anything is applied; an edited shipped migration, + or a history row from a newer Threadmill, fails startup. + +The whole v1 schema is one `V1__baseline.sql`. After release, schema changes +ship as additive `V2__*.sql`, `V3__*.sql`, … files; the baseline is never +edited again. The shipped list is explicit (no classpath scanning) for +native-image friendliness. + +## Manual DDL + +The clean-install DDL is the checked-in migration: +[`V1__baseline.sql`](../threadmill-store-oracle/src/main/resources/com/hemju/threadmill/store/oracle/migrations/V1__baseline.sql). +Every statement ends with a line holding only `/`, the SQL*Plus convention, so +the file runs unchanged in SQL*Plus or SQLcl. + +To apply SQL from your own deployment system, emit it from the runner: + +```java +String install = new OracleMigrationRunner(dataSource).emitCleanInstallSql(); +String pending = new OracleMigrationRunner(dataSource).emitPendingSql(); +``` + +`emitCleanInstallSql()` returns the history-table DDL, every migration, and one +completed history row per migration. `emitPendingSql()` is read-only: it reads +`threadmill_schema_history` (prepending its DDL if the table does not exist yet) +and returns only what is missing, including the remainder of an interrupted +migration. Run the output with SQL*Plus or SQLcl, stop at the first error, and +then start the application with: + +```yaml +threadmill: + store: + oracle: + schema-mode: validate +``` + +## Reinitialization + +For local or ephemeral schemas, Spring can recreate the schema: + +```yaml +threadmill: + store: + oracle: + schema-mode: drop-and-migrate + allow-destructive-schema-reset: true +``` + +This drops only Threadmill-owned tables (`CASCADE CONSTRAINTS PURGE`, with +their indexes and trigger); the small `threadmill_schema_lock` table is kept +because the reset holds its lock. It deletes all Threadmill jobs, recurring +definitions, dedup records, queue pauses, leases, and counters. Use forward +migrations in production. + +## Operational Notes + +- **Statistics.** The claim path's hot statements carry index hints, so plans do + not depend on fresh optimizer statistics, but keep the regular automatic + statistics job enabled for the rest. +- **Temporary LOBs.** Job bodies longer than 8,000 characters are written + through temporary LOBs that the store frees after each statement; size the + `TEMP` tablespace for the concurrent write rate of large jobs. +- **No cross-node wake.** Nodes pick up work enqueued on another node within + their `pollInterval` (default 1 s); same-JVM wakes are immediate. +- **Driver.** The store uses standard JDBC. It was tested with `ojdbc11` + 23.26.3; older drivers that report no batch update counts are not supported. diff --git a/docs/quickstart.md b/docs/quickstart.md index f11eb753..c9dc0817 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -16,13 +16,14 @@ the unreleased 1.0.0 candidate; see the [release status](../README.md#status). ```kotlin implementation("com.hemju.threadmill:threadmill-spring-boot:1.0.0") implementation("com.hemju.threadmill:threadmill-store-postgres:1.0.0") +// or: implementation("com.hemju.threadmill:threadmill-store-oracle:1.0.0") + your Oracle JDBC driver // or: implementation("com.hemju.threadmill:threadmill-store-redis:1.0.0") ``` The default Spring enqueue mode is `after_commit`: returned ids are reserved before persistence, and the job insert can fail after the business transaction commits. Observe `AfterCommitEnqueueFailure`, or choose `join_transaction` with -the same PostgreSQL DataSource for atomic business/job writes. Cross-datastore +the same PostgreSQL or Oracle DataSource for atomic business/job writes. Cross-datastore atomicity requires an application-owned durable outbox. See [transaction modes](transactions.md#after_commit-default). @@ -110,7 +111,7 @@ caller's SQL transaction. Without a configured durable store or an application-provided `JobStore`, startup fails. For disposable local development only, explicitly set `threadmill.store.memory.enabled=true`; all jobs are lost when that process -stops. Configure PostgreSQL or Redis for durable work. +stops. Configure PostgreSQL, Oracle, or Redis for durable work. ```yaml threadmill: @@ -126,3 +127,10 @@ and runs pending Threadmill schema migrations by default. Use `threadmill.store.postgres.schema-mode=validate` if your deployment pipeline applies the DDL separately. See [postgres-schema.md](postgres-schema.md) for manual SQL and reset guidance. + +For Oracle Database 19c+, add `threadmill-store-oracle` plus the Oracle JDBC +driver and define the `DataSource` the same way; Spring auto-configures +`OracleJobStore` and migrates by default (`threadmill.store.oracle.schema-mode`). +If both JDBC store modules are on the classpath, a `jdbc:oracle:` datasource +URL or `threadmill.store.jdbc-type=oracle` selects Oracle. See +[oracle-schema.md](oracle-schema.md) for privileges and manual SQL. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index bb8d1a59..b74fd245 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -33,6 +33,19 @@ real-store tests, soak and correctness simulations, Javadoc, the example, dependency scanning, and artifact inspection. If it fails, fix the cause and re-run it from a clean tree — no failure is acceptable for a release build. +No freely available container image provides Oracle Database 19c, the oldest +supported Oracle release; CI covers 23ai Free and 21c XE. Run the Oracle store +suite against a real 19c database before release and record the server release +in the validation notes: + +```bash +./gradlew :threadmill-store-oracle:test -PoracleJdbcUrl=jdbc:oracle:thin:@//host:1521/SERVICE -PoracleUser=... -PoraclePassword=... +``` + +It deletes all Threadmill rows in that account's schema: use a disposable +schema. Grant it `SELECT ANY DICTIONARY` (or access to `V$SESSION`, `V$SQL`, and +`V$SQL_PLAN`) so the execution-plan tests run instead of skipping. + Then inspect artifacts: ```bash diff --git a/docs/transactions.md b/docs/transactions.md index 21633403..e190942d 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -118,8 +118,8 @@ void onDeferredEnqueueFailure(AfterCommitEnqueueFailure failure) { ``` The event is an in-process observation, not durable recovery: a process crash -can prevent its delivery. Use `join_transaction` with the same PostgreSQL -`DataSource` for atomic business and job writes, or an application-owned durable +can prevent its delivery. Use `join_transaction` with the same PostgreSQL or +Oracle `DataSource` for atomic business and job writes, or an application-owned durable outbox when crossing datastores. Handlers still require idempotency under Threadmill's at-least-once delivery guarantee. @@ -136,12 +136,13 @@ the job row are written when the method is called — and they **survive a rollback** of the surrounding business transaction. A DEBUG log line is emitted when `enqueueIfAbsent` runs inside an active transaction. If the deduplicated enqueue must roll back with the caller, use -`join_transaction` (Postgres) — there the dedup write shares the caller's +`join_transaction` (PostgreSQL or Oracle) — there the dedup write shares the caller's JDBC transaction — or restructure to a plain `enqueue()` after commit. ### `join_transaction` -Postgres + Spring can make scheduling part of the caller's SQL transaction: +PostgreSQL or Oracle + Spring can make scheduling part of the caller's SQL +transaction: ```yaml threadmill: @@ -156,16 +157,21 @@ Threadmill job visible; rollback removes both. Local worker wakeups still run only after commit so workers never race an uncommitted row. This mode is intentionally limited to the Spring auto-configured -`PostgresJobStore` using the same `DataSource` as the caller transaction. -Redis cannot join a SQL transaction; unsupported combinations fail fast at -startup. - -One deduplication edge is intentionally different in this mode. If two -transactions race on the same `(queue, dedupKey)`, the loser receives the -Postgres unique-constraint failure instead of Threadmill coalescing it into an -existing job id. Once Postgres aborts a statement inside the caller's -transaction, Threadmill cannot safely run the fallback lookup without also -owning the transaction boundary. +`PostgresJobStore` or `OracleJobStore` using the same `DataSource` as the caller +transaction. Redis cannot join a SQL transaction; unsupported combinations fail +fast at startup. + +One deduplication edge differs between the two databases in this mode. If two +transactions race on the same `(queue, dedupKey)`: + +- **PostgreSQL** — the loser receives the unique-constraint failure instead of + Threadmill coalescing it into an existing job id. Once PostgreSQL aborts a + statement inside the caller's transaction, the whole transaction is unusable, + so Threadmill cannot run the fallback lookup without owning the boundary. +- **Oracle** — the loser waits for the winner's commit and then coalesces onto + the winner's job id. Oracle rolls back only the failed statement, so + Threadmill undoes its own writes to a savepoint and runs the lookup inside the + caller's transaction. ### `immediate` diff --git a/threadmill-spring-boot/README.md b/threadmill-spring-boot/README.md index bd7f14bb..3f9fe0e4 100644 --- a/threadmill-spring-boot/README.md +++ b/threadmill-spring-boot/README.md @@ -60,10 +60,13 @@ start; `JobScheduler` verifies the handler/payload pair at enqueue time. `ThreadmillAutoConfiguration` resolves a `JobStore` bean by precedence: 1. **Explicit Redis config** (`threadmill.store.redis.*` populated) → `RedisJobStore`. -2. **DataSource present + Postgres store on classpath** → `PostgresJobStore` +2. **DataSource present + a JDBC store on classpath** → `PostgresJobStore` + (`threadmill-store-postgres`) or `OracleJobStore` (`threadmill-store-oracle`) wired from the application's existing `DataSource`. The auto-configured store applies pending Threadmill schema migrations by default before it is - created. + created. With both store modules on the classpath, `threadmill.store.jdbc-type` + (`postgres` / `oracle`) decides; otherwise the `spring.datasource.url` scheme + does, and PostgreSQL wins a tie. 3. **Explicit development opt-in** (`threadmill.store.memory.enabled=true`) → `InMemoryJobStore` with a loud warning that jobs won't survive restart. 4. **Otherwise** → startup fails with an actionable error instead of silently @@ -125,7 +128,7 @@ threadmill: enqueue-mode: immediate ``` -Use caller-transaction participation with Spring + Postgres: +Use caller-transaction participation with Spring + PostgreSQL or Oracle: ```yaml threadmill: @@ -133,8 +136,8 @@ threadmill: enqueue-mode: join_transaction ``` -`join_transaction` wires `PostgresJobStore` with a Spring transaction-bound -connection strategy. Job inserts and dedup rows commit or roll back with the +`join_transaction` wires `PostgresJobStore` or `OracleJobStore` with a Spring +transaction-bound connection strategy. Job inserts and dedup rows commit or roll back with the application transaction; local worker wakeups still fire only after commit. Redis and custom stores cannot join a SQL transaction and fail fast if this mode is requested. @@ -164,6 +167,8 @@ Durable auto-configured stores publish remote wake hints by default: - Postgres: `LISTEN`/`NOTIFY` on `threadmill_wake`. - Redis: Pub/Sub on `{threadmill}:wake`. +- Oracle: none — Oracle has no lightweight equivalent, so other nodes pick up + new work within `threadmill.poll-interval`. The listener calls `ProcessingNode.wake(queue)` on matching local dispatchers. This is a latency optimization only; polling remains the fallback when a @@ -227,6 +232,9 @@ list). The most common: | `threadmill.store.memory.enabled` | `false` | Explicitly allow volatile in-memory storage for development or tests. Without a durable store or this opt-in, startup fails. | | `threadmill.store.postgres.schema-mode` | `migrate` | `migrate`, `validate`, `none`, or `drop-and-migrate`. | | `threadmill.store.postgres.allow-destructive-schema-reset` | `false` | Required for `drop-and-migrate`; destroys stored Threadmill jobs. | +| `threadmill.store.oracle.schema-mode` | `migrate` | `migrate`, `validate`, `none`, or `drop-and-migrate`. | +| `threadmill.store.oracle.allow-destructive-schema-reset` | `false` | Required for `drop-and-migrate`; destroys stored Threadmill jobs. | +| `threadmill.store.jdbc-type` | — | `postgres` or `oracle`; only needed when both JDBC store modules are on the classpath and `spring.datasource.url` does not name the database. | | `threadmill.store.redis.mode` | `standalone` | `standalone` / `sentinel` / `cluster`. | | `threadmill.store.redis.uri` | — | Lettuce `redis://` / `rediss://` URI for standalone mode. | | `threadmill.store.redis.sentinel.data-node-username` / `.data-node-password` | — | Redis data-node ACL credentials. | diff --git a/threadmill-store-oracle/README.md b/threadmill-store-oracle/README.md new file mode 100644 index 00000000..7b1c8b28 --- /dev/null +++ b/threadmill-store-oracle/README.md @@ -0,0 +1,164 @@ +# threadmill-store-oracle + +Oracle Database 19c+ backend for the `JobStore` SPI. It passes the same shared +`AbstractJobStoreContractTest` as the in-memory, PostgreSQL, and Redis stores, +and uses standard JDBC only: applications bring their own Oracle driver. + +```kotlin +implementation("com.hemju.threadmill:threadmill-store-oracle:1.0.0") +runtimeOnly("com.oracle.database.jdbc:ojdbc11:23.26.3.0.0") // or the driver your platform mandates +``` + +```java +DataSource dataSource = ...; // host-owned pool; the store never creates or closes it +new OracleMigrationRunner(dataSource).migrate(); +JobStore store = new OracleJobStore(dataSource); +``` + +With Spring Boot, `threadmill-spring-boot` wires the store from the +application's `DataSource`; see [Oracle schema](../docs/oracle-schema.md) and +[configuration](../docs/configuration.md#oracle-store-properties). + +## Requirements, checked at startup + +`OracleJobStore`'s constructor refuses to start (with `JobEngineFatalException`) +unless: + +| Requirement | Why | +|---|---| +| Oracle Database **19c or later** | 19c is the oldest release in long-term support; the SQL uses only the 19c feature set (no `BOOLEAN`, no `IF NOT EXISTS`, no 23ai syntax). | +| Database character set **`AL32UTF8`** | Job bodies and names must round-trip every Unicode character, including 4-byte ones. `AL32UTF8` is the default since 12.2. | +| Session **`NLS_COMP=BINARY`** (the default) | Keyset predicates and the claim path's key ranges rely on binary comparisons matching index order. | +| An Oracle `DataSource` | A PostgreSQL (or other) `DataSource` fails with a message naming the product. | + +The schema user needs `CREATE TABLE`, `CREATE TRIGGER`, and tablespace quota — +nothing else (no `CREATE TYPE`, no `DBMS_*` grants, no Advanced Queuing). A +user without DDL rights can run with schema mode `validate` or `none` after a +DBA applies the emitted DDL; see [Oracle schema](../docs/oracle-schema.md). + +The B-tree key limit of an 8 KiB block (≈6,400 bytes) bounds the widest index +(`state`, `queue`, `handler_signature`, time, id ≈ 4.6 KB). Databases with +smaller default blocks are not supported. + +## Schema + +Installed by `V1__baseline.sql` under +`src/main/resources/com/hemju/threadmill/store/oracle/migrations/`, the same +tables as PostgreSQL apart from the unused `threadmill_metadata`. Type mapping: + +| Concept | Oracle type | Notes | +|---|---|---| +| Job / node / workflow ids | `RAW(16)` | Big-endian UUID bytes; RAW compares unsigned byte by byte, which equals `JobId`'s canonical natural order. | +| Timestamps | `TIMESTAMP(9)` holding UTC | Nanosecond-exact for `Instant`, bound as `LocalDateTime` at UTC, so neither the JVM nor the session time zone matters. | +| Job body, recurring payload, pause reason | `CLOB` | The body is the source of truth; scalar columns are denormalised for the hot queries. | +| Booleans | `NUMBER(1)` with a `0/1` check | `BOOLEAN` is 23ai-only. | +| Names | `VARCHAR2(n CHAR)` | Concurrency and dedup keys are `VARCHAR2(256 BYTE)`, matching their UTF-8 byte caps. | + +Oracle stores the empty string as `NULL`. The store maps it back where a value +may legitimately be empty (a recurring payload, a mutex holder), and compares +mutex holders with `DECODE`, which treats two `NULL`s as equal. + +### Partial indexes as indexed virtual columns + +Oracle has no partial indexes. Each PostgreSQL partial index becomes indexed +`VIRTUAL` columns of the form `CASE WHEN THEN END` — for +example `keyed_queue`, `keyed_key`, `keyed_at`, `keyed_id` for ENQUEUED keyed +jobs. Rows that do not match the predicate have all-`NULL` keys, which an Oracle +B-tree does not store, so each index holds only the rows its queries need. +Virtual columns (rather than expression indexes) let queries name the columns +directly and let the 19c optimizer read index order: a per-key top-n stops after +n entries instead of sorting the key's whole range. + +| Index | Serves | +|---|---| +| `threadmill_jobs_unkeyed_idx` | Unkeyed claim lane, `(queue, -priority, id)`. | +| `threadmill_jobs_keyed_idx` | Keyed claim lane: per-queue loose key scan and per-key heads in `(current_state_at, id)` order. | +| `threadmill_jobs_pending_idx`, `threadmill_jobs_exclusive_idx` | Claim-time admission (earlier pending / earlier EXCLUSIVE job in a key). | +| `threadmill_jobs_workflow_idx` | Workflow-hold outstanding counts and hold members. | +| `threadmill_jobs_scheduled_idx`, `threadmill_jobs_liveness_idx`, `threadmill_jobs_heartbeat_idx` | Promotion, orphan recovery, monitoring. | +| `threadmill_jobs_awaiting_idx` | Workflow successor promotion. | +| `threadmill_jobs_queue_age_idx` | Oldest ENQUEUED job per queue. | +| `threadmill_jobs_retention_idx`, `threadmill_jobs_state_id_idx`, `threadmill_jobs_state_page_idx`, `threadmill_jobs_search_idx`, `threadmill_jobs_handler_idx` | Retention, maintenance scans, dashboard. | +| `threadmill_concurrency_idle_idx` | Idle concurrency-group reclamation, over a `RAW` binary sort key. | + +Per-state and per-queue counts live in `threadmill_job_counts` and +`threadmill_queue_counts`, maintained by one row trigger and sharded 16 ways by +session id; only the per-state/per-queue `SUM` is meaningful. + +## How the claim works + +`claimReady` locks candidates with `FOR UPDATE SKIP LOCKED`, so contending +workers never collide and never wait: + +- **Unkeyed lane** — an index-ordered locking cursor. Oracle rejects a row limit + on a locking query (`ORA-02014`), so the store fetches only its page; Oracle + locks `SKIP LOCKED` rows as they are fetched, which a regression test pins. +- **Keyed lane** — distinct keys come from a recursive `MIN` loose scan (one + index probe per key, 128 keys per pass with a rotating per-queue cursor); each + key contributes its earliest heads through a `ROWNUM` top-n branch. Candidates + are then locked by primary key with `SKIP LOCKED`. +- **Admission** — `EXISTS` range probes bounded at each candidate's own + `current_state_at` decide the leapfrog rule without reading a key's backlog. + +Lateral joins are deliberately avoided: inside a correlated top-n lateral view +Oracle keeps a sort over each key's whole range, and a collection-driven lateral +top-n returned wrong results on a current release. The hot statements carry +index hints, and `OracleQueryPlanTest` runs them with typed binds and asserts +the executed plans under the 19c optimizer feature set. + +## Other Oracle specifics + +- **Transactions.** Every self-owned write commits explicitly and restores the + connection's auto-commit mode, so pools that hand out `autoCommit=false` + connections work. Deadlocks (`ORA-00060`) and serialization failures + (`ORA-08177`) are retried. Oracle rolls back only a failed statement, not the + transaction; the deduplicated enqueue uses a savepoint to coalesce onto a + concurrent producer's job inside a joined transaction too. +- **CLOB binding.** Values longer than 8,000 characters are written through a + temporary `Clob`, freed after the statement. Binding a long string directly + made the Oracle driver split a surrogate pair (an emoji) that straddled its + ~32 KiB chunk boundary, corrupting the body. +- **Locale.** The driver derives `NLS_LANGUAGE`, and so `NLS_SORT`, from the + client JVM locale. Text keyset pages therefore order in binary explicitly + (`NLSSORT(..., 'NLS_SORT = BINARY')` or a `RAW` sort key), so a German or + French JVM pages exactly like an English one. +- **Server time.** Leases and mutexes compare `SYS_EXTRACT_UTC(SYSTIMESTAMP)` + on the server, so a node with a skewed clock cannot steal one. +- **No remote wake.** Oracle has no lightweight equivalent of `LISTEN`/`NOTIFY` + (`DBMS_ALERT` serializes signalling transactions). Same-JVM wakes still work; + other nodes pick up new work within their `pollInterval`. + +## Migrations + +`OracleMigrationRunner` applies `V__.sql` files listed in +`SHIPPED_MIGRATIONS` (no classpath scanning). Statements end with a line holding +only `/`, so the files and the emitted SQL run unchanged in SQL*Plus or SQLcl. +Oracle commits each DDL statement, so the runner records progress after every +statement, resumes a crashed migration at the first unrecorded statement, and +serializes migrators with a row lock on `threadmill_schema_lock` held on a +second connection. Details: [Oracle schema](../docs/oracle-schema.md). + +## Tests + +```bash +./gradlew :threadmill-store-oracle:test # Oracle 23ai Free container +./gradlew :threadmill-store-oracle:test -PoracleImage=gvenzl/oracle-xe:21-slim-faststart # 21c XE (amd64; CI) +./gradlew :threadmill-store-oracle:test -PoracleJdbcUrl=jdbc:oracle:thin:@//host:1521/SVC -PoracleUser=tm_test -PoraclePassword=... +``` + +There is no freely downloadable 19c container image. CI runs 23ai Free in the +main build and 21c XE (the closest free release to 19c, without any 23ai-only +syntax) in a separate job; every session runs with +`OPTIMIZER_FEATURES_ENABLE='19.1.0'`. Before a release, run the suite against a +real 19c database with the `oracleJdbcUrl` properties above — it deletes all +Threadmill rows in that schema, so use a disposable one. The plan tests need +`SELECT ANY DICTIONARY` (or `V$SESSION`/`V$SQL`/`V$SQL_PLAN` access) and skip +otherwise. + +| Test | Covers | +|---|---| +| `OracleJobStoreContractTest` | The shared contract suite, through connections that start with `autoCommit=false`. | +| `OracleJobStoreRegressionTest` | Lock-as-you-fetch claiming, concurrent claims, keyed rotation, quarantine, CLOB supplementary characters, time-zone independence, empty strings, locale-independent paging, counters. | +| `OracleQueryPlanTest` | Executed plans of the hot and periodic statements. | +| `OracleMigrationRunnerTest` | Resume after a crash, DBA-applied SQL, concurrent migrators, edited migrations, constraints. | +| `OracleServerTest`, `OracleDeadlockRetryTest`, `OracleStatementSplitTest` | Startup gate, retry classification, statement splitting. | From 19956d1cc22ac8e18a807ce7f9d6169d9e3e9fb1 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 18:36:21 +0200 Subject: [PATCH 4/7] test(soak, simulation): add Oracle backends - threadmill-soak: OracleHarnessFixture (container or external database via -PoracleUrl / -PoracleUser / -PoraclePassword; the password is never written to run artifacts), soakOracle, Oracle in soakAll and as an opt-in endurance backend, OracleSoakTest in soakRegression (throughput, contention, queue family, recurring no-skip, container-pause recovery), harness smoke tests. - threadmill-simulation: simulateOracle, included in simulate; the run uses a Hikari pool because unpooled bursts exceed Oracle's process limit. - AGENTS.md lists the remaining Oracle follow-ups. Refs #137 --- AGENTS.md | 1 + threadmill-simulation/README.md | 17 +- threadmill-simulation/build.gradle.kts | 17 +- threadmill-simulation/gradle.lockfile | 4 +- .../threadmill/simulation/SimulationMain.java | 50 +++- threadmill-soak/README.md | 22 +- threadmill-soak/build.gradle.kts | 20 +- threadmill-soak/gradle.lockfile | 1 + .../soak/harness/HarnessPooledDataSource.java | 4 +- .../soak/harness/OracleHarnessFixture.java | 148 ++++++++++++ .../soak/harness/RunConfigWriter.java | 4 + .../soak/harness/SoakHarnessConfig.java | 9 + .../soak/harness/SoakHarnessMain.java | 6 +- .../harness/endurance/EnduranceConfig.java | 15 +- .../soak/harness/endurance/EnduranceMain.java | 9 + .../threadmill/soak/harness/package-info.java | 2 +- .../soak/harness/summary.schema.json | 2 +- .../hemju/threadmill/soak/OracleSoakTest.java | 223 ++++++++++++++++++ .../harness/HarnessQualificationTest.java | 3 + .../soak/harness/MultiProducerSmokeTest.java | 3 + .../soak/harness/NodeChurnSmokeTest.java | 3 + .../harness/RedisExternalUrlFixtureTest.java | 3 + .../soak/harness/RedisProducerOutageTest.java | 3 + .../soak/harness/RetentionChurnSmokeTest.java | 6 +- .../harness/SoakHarnessOracleSmokeTest.java | 62 +++++ .../harness/SoakHarnessPostgresSmokeTest.java | 3 + .../harness/SoakHarnessRedisSmokeTest.java | 3 + .../soak/harness/SoakHarnessRunnerTest.java | 3 + .../soak/harness/SoakHarnessSmokeTest.java | 3 + .../EnduranceOrchestratorSmokeTest.java | 3 + 30 files changed, 625 insertions(+), 27 deletions(-) create mode 100644 threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/OracleHarnessFixture.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/OracleSoakTest.java create mode 100644 threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessOracleSmokeTest.java diff --git a/AGENTS.md b/AGENTS.md index 91af3903..9f0d8152 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -653,6 +653,7 @@ These are deliberately additive and design-compatible with the current model — - **Task 3 of the v1-readiness finishing pass — per-module READMEs and full docs — landed.** All 14 module READMEs (`threadmill-core`, the three stores, `threadmill-spring-boot`, `threadmill-test-support`, `threadmill-metrics`, `threadmill-tracing`, the three dashboard modules, `threadmill-soak`, `threadmill-simulation`, `threadmill-example`) and the restructured `docs/` tree exist: `index`, `getting-started`, `quickstart` (Spring), `architecture`, `handlers`, `transactions` (deep dive — atomic boundaries per backend, handler-is-not-in-our-transaction, at-least-once + idempotency, outbox pattern), `backend-execution-model`, `configuration`, `concurrency`, `queue-topology`, `long-running-jobs`, `deduplication`, `wake-driven-pollers` (the nudge pattern: handler shape, choosing the backstop interval, what coalescing means for handler code — recurring tasks otherwise had no usage page), `operations`, `troubleshooting`, `migration`, `postgres-schema`, `redis-topologies`, `release-checklist`, plus the JobRunr / Quartz comparison pages. The Postgres README carries the full schema; the Redis README carries the full key layout and Lua script inventory. Runnable examples are compiled files under `threadmill-example/src/main/java/com/example/threadmill/`; doc snippets are maintained by hand (there is no compiled `threadmill-example/snippets/` directory). The bar is "an AI agent can use Threadmill to replace an existing job/scheduler system without reading source code." - **Task 4 of the v1-readiness finishing pass — `threadmill-simulation` module — landed.** New module, separate from `threadmill-soak` (load/performance) and `threadmill-example` (teaching). The short correctness simulation runs 50 projects with `Import` (EXCLUSIVE) and `Export` (SHARED) jobs, 400 jobs over the run (small enough to finish in seconds), random failure injection (5% exception, 0.5% hang), mid-run pause/resume, half-via-`insertAll` bulk-enqueue sample. Records JSON-lines traces under `build/simulation/`; `TraceVerifier` asserts at-least-once, concurrency exclusion (EXCLUSIVE-vs-anything, SHARED-vs-EXCLUSIVE), lock pairing, and pause-obeyed. Gradle entry points: `:threadmill-simulation:simulate` (all three backends), `simulateMemory`, `simulatePostgres`, `simulateRedis`. The Gradle task fails (non-zero exit) when any backend doesn't drain or produces a verifier violation. The worker-churn simulation lives under `com.hemju.threadmill.simulation.workerchurn` and runs through `simulateWorkerChurnPostgres` / `simulateWorkerChurnRedis` against shared local datastores, writing traces to `build/simulation/worker-churn--.jsonl` by default. The fixed process-separated nudge simulation under `com.hemju.threadmill.simulation.nudge` runs through `simulateNudgePostgres` / `simulateNudgeRedis`: a supervisor hard-kills the maintenance leader after an accepted nudge and requires a standby-process nudge run, then hard-kills a producer after its durable work write and requires a schedule-origin backstop drain. Its per-run directory contains the verified cross-process trace plus every child JVM log. - **The process-separated nudge simulation budgets its two phases independently.** Failover starts with a one-minute recurring interval and a five-minute leader poll so the two 10-second process-start allowances plus lease takeover cannot race either scheduled backstop or leader consumption. After the standby serves the accepted nudge, the supervisor performs a real timing edit to an eight-second interval for the producer-crash backstop. Ready markers publish by temp-file atomic move; every cross-process trace write drains its buffer under the file lock. Simulation work stays outside the engine namespace (`nudge_simulation_work` / `threadmill-simulation:*`) and each work-store instance keeps one datastore connection for its lifetime. +- **Oracle follow-ups (issue #137).** A test run against a real Oracle 19c database (no free image exists) is owed before relying on the store in production. Oracle is in `soakRegression`, `soakOracle`, `soakAll`, and `simulate`, but not yet in the process-separated worker-churn and nudge simulations or the default `soakEndurance` pair (`-Pbackends=oracle` works). The soak producer's outage classifier (`RecoveringProducerStore.isOutage`) recognises PostgreSQL restart states and `08xxx`, not Oracle restart errors such as `ORA-01033`/`ORA-01034`; extend it together with an Oracle producer-outage test. A cross-node wake channel for Oracle and `NCLOB`/`NVARCHAR2` support for non-`AL32UTF8` databases are deliberately out of scope until a deployment needs them. - **Batches.** The child relationship is already in the model; a `BatchCompletionInterceptor` over multiple children is the natural shape. - **External-trigger jobs.** The `PROCESSED` state is reserved; this needs an external-signal API plus an escape-hatch timeout. - **Rate limiters.** A store-side token-bucket primitive; Redis can use the standard Lua bucket pattern. diff --git a/threadmill-simulation/README.md b/threadmill-simulation/README.md index 3581dddc..fe31a106 100644 --- a/threadmill-simulation/README.md +++ b/threadmill-simulation/README.md @@ -9,20 +9,22 @@ JSON-lines traces for verification or review. ## Correctness Simulation This is the short fixed invariant verifier. It runs in seconds, uses -Testcontainers for Postgres and Redis, and fails the Gradle task when a backend +Testcontainers for Postgres, Oracle, and Redis, and fails the Gradle task when a backend does not drain or the trace verifier finds an invariant violation. ```bash ./gradlew :threadmill-simulation:simulate ``` -Runs all three backends sequentially (in-memory, Postgres 18 via -Testcontainers, Redis via Testcontainers). One command, full validation. Or -pick one: +Runs all four backends sequentially (in-memory, Postgres 18 via +Testcontainers, Oracle Database 23ai Free via Testcontainers +`gvenzl/oracle-free:23-slim-faststart`, Redis via Testcontainers). One command, +full validation. Or pick one: ```bash ./gradlew :threadmill-simulation:simulateMemory # fast, no Docker ./gradlew :threadmill-simulation:simulatePostgres # Testcontainers +./gradlew :threadmill-simulation:simulateOracle # Testcontainers ./gradlew :threadmill-simulation:simulateRedis # Testcontainers ``` @@ -79,7 +81,7 @@ object with at least `timestamp` (ISO-8601 UTC) and `event`. Common fields: { "timestamp": "…", "event": "lock_released", "jobId": "…", "lockKey": "…", "lockMode": "…" } { "timestamp": "…", "event": "queue_paused", "queue": "…" } { "timestamp": "…", "event": "queue_resumed", "queue": "…" } -{ "timestamp": "…", "event": "node_started", "nodeId": "…", "workerCount": 8, "backend": "memory|postgres|redis" } +{ "timestamp": "…", "event": "node_started", "nodeId": "…", "workerCount": 8, "backend": "memory|postgres|oracle|redis" } { "timestamp": "…", "event": "node_stopped", "nodeId": "…" } ``` @@ -103,6 +105,8 @@ This is the long-running multi-process simulation. A supervisor starts worker JVMs against one shared external datastore, enqueues jobs continuously, kills and recreates workers, and writes a JSON-lines trace under `build/simulation/worker-churn--.jsonl` by default. +It covers PostgreSQL and Redis; the Oracle store is exercised by the short +correctness simulation above and the soak suites, not by worker churn. Start local shared datastores from the repository root: @@ -149,7 +153,8 @@ when the queue does not drain within `--drain-timeout`. This fixed simulation starts real Postgres and Redis datastores with Testcontainers, then runs the maintenance leader, standby, and producers in -separate JVMs. Run both backends: +separate JVMs. It covers PostgreSQL and Redis only; Oracle is not yet part of +this simulation. Run both backends: ```bash ./gradlew :threadmill-simulation:simulateNudge diff --git a/threadmill-simulation/build.gradle.kts b/threadmill-simulation/build.gradle.kts index f071f0e5..a00e8c94 100644 --- a/threadmill-simulation/build.gradle.kts +++ b/threadmill-simulation/build.gradle.kts @@ -4,12 +4,15 @@ dependencies { implementation(project(":threadmill-core")) implementation(project(":threadmill-store-memory")) implementation(project(":threadmill-store-postgres")) + implementation(project(":threadmill-store-oracle")) implementation(project(":threadmill-store-redis")) implementation(libs.jackson.databind) implementation(libs.jackson.datatype.jsr310) implementation(libs.slf4j.api) implementation(libs.slf4j.simple) implementation(libs.postgresql.jdbc) + implementation(libs.oracle.jdbc) + implementation(libs.hikaricp) implementation(libs.lettuce.core) implementation(platform(libs.testcontainers.bom)) implementation(libs.testcontainers.postgresql) @@ -21,7 +24,7 @@ dependencies { testImplementation(libs.assertj.core) } -// Per-backend simulation tasks. The base `simulate` runs all three sequentially. +// Per-backend simulation tasks. The base `simulate` runs all four sequentially. val simulationMainClass = "com.hemju.threadmill.simulation.SimulationMain" val workerChurnMainClass = "com.hemju.threadmill.simulation.workerchurn.WorkerChurnSimulatorMain" val nudgeSimulationMainClass = "com.hemju.threadmill.simulation.nudge.NudgeSimulationMain" @@ -42,6 +45,14 @@ tasks.register("simulatePostgres") { args = listOf("--backend", "postgres") } +tasks.register("simulateOracle") { + group = "verification" + description = "Run the simulation against Oracle Database Free via Testcontainers." + classpath = sourceSets["main"].runtimeClasspath + mainClass.set(simulationMainClass) + args = listOf("--backend", "oracle") +} + tasks.register("simulateRedis") { group = "verification" description = "Run the simulation against Redis via Testcontainers." @@ -52,8 +63,8 @@ tasks.register("simulateRedis") { tasks.register("simulate") { group = "verification" - description = "Run the full simulation against all three backends." - dependsOn("simulateMemory", "simulatePostgres", "simulateRedis") + description = "Run the full simulation against all four backends." + dependsOn("simulateMemory", "simulatePostgres", "simulateOracle", "simulateRedis") } tasks.register("simulateWorkerChurnPostgres") { diff --git a/threadmill-simulation/gradle.lockfile b/threadmill-simulation/gradle.lockfile index abae9ba1..4effc60b 100644 --- a/threadmill-simulation/gradle.lockfile +++ b/threadmill-simulation/gradle.lockfile @@ -9,6 +9,8 @@ com.fasterxml.jackson:jackson-bom:2.21.5=compileClasspath,runtimeClasspath,testC com.github.docker-java:docker-java-api:3.7.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-transport-zerodep:3.7.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-transport:3.7.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.oracle.database.jdbc:ojdbc11:23.26.3.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.zaxxer:HikariCP:7.0.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath commons-codec:commons-codec:1.19.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath commons-io:commons-io:2.20.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.lettuce:lettuce-core:6.8.2.RELEASE=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath @@ -42,7 +44,7 @@ org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath org.postgresql:postgresql:42.7.12=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.reactivestreams:reactive-streams:1.0.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.rnorth.duct-tape:duct-tape:1.0.8=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.16=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.slf4j:slf4j-simple:2.0.16=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.testcontainers:testcontainers-bom:2.0.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.testcontainers:testcontainers-database-commons:2.0.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/threadmill-simulation/src/main/java/com/hemju/threadmill/simulation/SimulationMain.java b/threadmill-simulation/src/main/java/com/hemju/threadmill/simulation/SimulationMain.java index dcba431e..1abbfcf0 100644 --- a/threadmill-simulation/src/main/java/com/hemju/threadmill/simulation/SimulationMain.java +++ b/threadmill-simulation/src/main/java/com/hemju/threadmill/simulation/SimulationMain.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -12,6 +13,8 @@ import javax.sql.DataSource; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import io.lettuce.core.RedisURI; import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; @@ -24,6 +27,8 @@ import com.hemju.threadmill.core.store.JobStore; import com.hemju.threadmill.simulation.SimulationRunner.SimulationResult; import com.hemju.threadmill.store.memory.InMemoryJobStore; +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.oracle.OracleMigrationRunner; import com.hemju.threadmill.store.postgres.MigrationRunner; import com.hemju.threadmill.store.postgres.PostgresJobStore; import com.hemju.threadmill.store.redis.RedisJobStore; @@ -34,8 +39,10 @@ *

    *
  • {@code --backend memory} — fast, no Docker dependency.
    • *
  • {@code --backend postgres} — boots a {@code postgres:18-alpine} Testcontainer.
    • + *
  • {@code --backend oracle} — boots a {@code gvenzl/oracle-free:23-slim-faststart} + * Testcontainer.
    • *
  • {@code --backend redis} — boots a {@code redis:7.4-alpine} Testcontainer.
    • - *
  • {@code --backend all} (default) — runs all three sequentially.
    • + *
  • {@code --backend all} (default) — runs all four sequentially.
    • *
* *

Writes one JSON-lines trace per backend under {@code build/simulation/}, runs the @@ -57,6 +64,9 @@ public final class SimulationMain { .toFormatter(Locale.ROOT) .withZone(ZoneOffset.UTC); + private static final String ORACLE_IMAGE = "gvenzl/oracle-free:23-slim-faststart"; + private static final int ORACLE_MAX_CONNECTIONS = 32; + private SimulationMain() {} static void main(String[] args) { @@ -71,10 +81,12 @@ static void main(String[] args) { switch (backend) { case "memory" -> results.add(runMemory(config)); case "postgres" -> results.add(runPostgres(config)); + case "oracle" -> results.add(runOracle(config)); case "redis" -> results.add(runRedis(config)); case "all" -> { results.add(runMemory(config)); results.add(runPostgres(config)); + results.add(runOracle(config)); results.add(runRedis(config)); } default -> { @@ -132,6 +144,42 @@ private static RunOutcome runPostgres(SimulationConfig config) } } + @SuppressWarnings("resource") + private static RunOutcome runOracle(SimulationConfig config) + throws IOException, InterruptedException { + Path trace = traceFile("oracle"); + var container = new GenericContainer<>(DockerImageName.parse(ORACLE_IMAGE)) + .withEnv("ORACLE_PASSWORD", "threadmill") + .withEnv("APP_USER", "threadmill") + .withEnv("APP_USER_PASSWORD", "threadmill") + .withExposedPorts(1521) + .waitingFor(Wait.forLogMessage(".*DATABASE IS READY TO USE!.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(10))); + container.start(); + try { + // A real host supplies a pool; without one, a burst of short-lived + // dedicated-server sessions outruns Oracle's process limit (ORA-12516). + var pool = new HikariConfig(); + pool.setJdbcUrl("jdbc:oracle:thin:@//" + container.getHost() + ":" + + container.getMappedPort(1521) + "/FREEPDB1"); + pool.setUsername("threadmill"); + pool.setPassword("threadmill"); + pool.setMaximumPoolSize(ORACLE_MAX_CONNECTIONS); + SimulationResult result; + try (var ds = new HikariDataSource(pool); + var writer = new TraceWriter(trace)) { + new OracleMigrationRunner(ds).migrate(); + JobStore store = new OracleJobStore(ds); + var runner = new SimulationRunner(store, config, writer, "oracle"); + result = runner.run(); + } + var verifier = new TraceVerifier(trace).verify(); + return new RunOutcome("oracle", trace, result, verifier); + } finally { + container.stop(); + } + } + @SuppressWarnings("resource") private static RunOutcome runRedis(SimulationConfig config) throws IOException, InterruptedException { diff --git a/threadmill-soak/README.md b/threadmill-soak/README.md index bb6b382c..c59af055 100644 --- a/threadmill-soak/README.md +++ b/threadmill-soak/README.md @@ -9,7 +9,7 @@ Three complementary pieces: 2. The **load soak harness** — operator-driven, per-backend, per-scenario sustained load runs that produce a self-contained artifact directory rich enough for an AI agent to read cold. Tasks: `soakMemory`, `soakPostgres`, - `soakRedis`, `soakAll`. + `soakOracle`, `soakRedis`, `soakAll`. 3. The **endurance run** — the production-readiness sign-off: one harness JVM per backend (PostgreSQL **and** Redis in parallel) for hours, with node churn, live invariant verification, and a collated verdict. Task: @@ -26,7 +26,7 @@ Three complementary pieces: - **Recurring no-skip.** Defines a tight-interval recurring task and verifies it fires every interval over a multi-second window without catch-up storms. -- **Induced container-pause recovery.** Pauses the Postgres or Redis +- **Induced container-pause recovery.** Pauses the Postgres, Oracle, or Redis Testcontainer mid-run via the Docker API, asserts no progress while paused, unpauses, and asserts the engine resumes — the store-outage circuit-breaker recovery path in production. @@ -37,7 +37,9 @@ Three complementary pieces: ./gradlew :threadmill-soak:soakRegression ``` -Requires Docker (Testcontainers) for the Postgres and Redis runs. +Requires Docker (Testcontainers) for the Postgres, Oracle, and Redis runs. +The Oracle runs use `gvenzl/oracle-free:23-slim-faststart` (Oracle Database 23ai +Free); the first pull is large, so pull it once before a timed run. Numbers are reported in the test output: | Backend | Throughput (5k–10k jobs) | Recurring (5 s, 100 ms interval) | Container-pause recovery | @@ -45,6 +47,7 @@ Numbers are reported in the test output: | In-memory | ~950 jobs/sec | ~46 runs | n/a | | PostgreSQL 18 | ~270 jobs/sec | ~20 runs | pause → wait → unpause → drain | | Redis 7 (AOF) | ~250 jobs/sec | ~22 runs | pause → wait → unpause → drain | +| Oracle 23ai Free | ~900 jobs/sec | ~49 runs | pause → wait → unpause → drain | Numbers are from a developer laptop (Apple Silicon, Docker Desktop, single node, 8 workers, 32-batch claim). Treat them as a baseline; production @@ -71,8 +74,9 @@ The harness is distinct from: ```bash ./gradlew :threadmill-soak:soakMemory [-P…] ./gradlew :threadmill-soak:soakPostgres [-P…] +./gradlew :threadmill-soak:soakOracle [-P…] ./gradlew :threadmill-soak:soakRedis [-P…] -./gradlew :threadmill-soak:soakAll [-P…] # runs Postgres then Redis +./gradlew :threadmill-soak:soakAll [-P…] # runs Postgres, Oracle, then Redis ``` ### `-P` properties @@ -91,6 +95,8 @@ The harness is distinct from: | `-PfailFast=` | `true` | Abort the run on the first *definite* invariant violation (see below). | | `-PprogressInterval=` | `30s` | How often `progress.json` is rewritten. | | `-PpostgresUrl=` | unset → Testcontainers | External JDBC URL alternative. | +| `-PoracleUrl=` | unset → Testcontainers | External Oracle thin-driver URL (for example `jdbc:oracle:thin:@//host:1521/service`). Every Threadmill table in that schema is emptied before the run, so use a disposable schema. | +| `-PoracleUser=` / `-PoraclePassword=` | unset | Credentials for `-PoracleUrl`. `config.json` records the user and only whether a password was set. | | `-PredisUrl=` | unset → Testcontainers | External Redis alternative. Only the `{threadmill}:*` namespace is reset — never `FLUSHDB`. | | `-PredisTopology=` | `standalone` | `standalone`; `sentinel`/`cluster` require an external topology URL. | | `-Pforce=` | `false` | Allow overwriting an existing `-PoutputDir`. | @@ -134,7 +140,9 @@ live p99, and per-invariant status. `summary.json` supersedes it once the run ends. `TraceReplay` (in `…soak.harness.invariant`) re-verifies a finished `trace.jsonl` offline, streaming, without loading it into memory. -The Postgres harness uses a fixed connection pool (`maxConnections=80`) +The Postgres harness uses a fixed connection pool (`maxConnections=80`), the +Oracle harness one of `maxConnections=48` (Oracle refuses new dedicated-server +sessions with `ORA-12516` long before PostgreSQL runs out of backends), because the soak loop intentionally creates enough claim, completion, heartbeat, and metric traffic to make one-connection-per-operation fixtures distort the result. @@ -247,6 +255,10 @@ docker compose -f threadmill-soak/docker-compose.endurance.yml up -d Without the URL knobs each child provisions its own Testcontainer — fine for short validation runs, not recommended for hours. +Oracle is not part of the default pair; add it explicitly with +`-Pbackends=postgres,redis,oracle` (plus `-PoracleUrl` / `-PoracleUser` / +`-PoraclePassword` for an external database). + Defaults are the sign-off profile: **8 hours**, `mixed-workload`, **50 jobs/second**, **3 nodes per backend**, **node churn every 10 minutes** (`-PnodeChurn=off` to disable), `failFast=true`. All harness `-P` knobs apply; diff --git a/threadmill-soak/build.gradle.kts b/threadmill-soak/build.gradle.kts index 382df46e..ec8e3db1 100644 --- a/threadmill-soak/build.gradle.kts +++ b/threadmill-soak/build.gradle.kts @@ -4,12 +4,14 @@ dependencies { implementation(project(":threadmill-core")) implementation(project(":threadmill-store-memory")) implementation(project(":threadmill-store-postgres")) + implementation(project(":threadmill-store-oracle")) implementation(project(":threadmill-store-redis")) implementation(libs.jackson.databind) implementation(libs.jackson.datatype.jsr310) implementation(libs.slf4j.api) implementation(libs.slf4j.simple) implementation(libs.postgresql.jdbc) + implementation(libs.oracle.jdbc) implementation(libs.lettuce.core) implementation(libs.networknt.jsonschema) implementation(platform(libs.testcontainers.bom)) @@ -62,6 +64,9 @@ fun JavaExec.passSoakProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", @@ -92,6 +97,17 @@ tasks.register("soakPostgres") { passSoakProps() } +tasks.register("soakOracle") { + group = "verification" + description = + "Run a soak scenario against Oracle Database via Testcontainers " + + "(or -PoracleUrl / -PoracleUser / -PoraclePassword)." + classpath = sourceSets["main"].runtimeClasspath + mainClass.set(soakHarnessMainClass) + args = listOf("--backend", "oracle") + passSoakProps() +} + tasks.register("soakRedis") { group = "verification" description = "Run a soak scenario against Redis via Testcontainers." @@ -103,8 +119,8 @@ tasks.register("soakRedis") { tasks.register("soakAll") { group = "verification" - description = "Run the chosen scenario against Postgres then Redis sequentially." - dependsOn("soakPostgres", "soakRedis") + description = "Run the chosen scenario against Postgres, Oracle, then Redis sequentially." + dependsOn("soakPostgres", "soakOracle", "soakRedis") } tasks.register("soakEndurance") { diff --git a/threadmill-soak/gradle.lockfile b/threadmill-soak/gradle.lockfile index fe8305e1..0e9f1cdf 100644 --- a/threadmill-soak/gradle.lockfile +++ b/threadmill-soak/gradle.lockfile @@ -12,6 +12,7 @@ com.github.docker-java:docker-java-api:3.7.1=compileClasspath,runtimeClasspath,t com.github.docker-java:docker-java-transport-zerodep:3.7.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-transport:3.7.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.networknt:json-schema-validator:1.5.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.oracle.database.jdbc:ojdbc11:23.26.3.0.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath commons-codec:commons-codec:1.19.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath commons-io:commons-io:2.20.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath io.lettuce:lettuce-core:6.8.2.RELEASE=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/HarnessPooledDataSource.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/HarnessPooledDataSource.java index 7dc9f70b..38d68494 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/HarnessPooledDataSource.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/HarnessPooledDataSource.java @@ -72,7 +72,7 @@ public Connection getConnection() throws SQLException { created.decrementAndGet(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SQLException("interrupted while waiting for a pooled Postgres connection", e); + throw new SQLException("interrupted while waiting for a pooled JDBC connection", e); } } } @@ -135,7 +135,7 @@ public void close() { private void ensureOpen() throws SQLException { if (closed.get()) { - throw new SQLException("pooled Postgres datasource is closed"); + throw new SQLException("pooled JDBC datasource is closed"); } } diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/OracleHarnessFixture.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/OracleHarnessFixture.java new file mode 100644 index 00000000..8a852ba3 --- /dev/null +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/OracleHarnessFixture.java @@ -0,0 +1,148 @@ +package com.hemju.threadmill.soak.harness; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +import javax.sql.DataSource; + +import oracle.jdbc.pool.OracleDataSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import com.hemju.threadmill.core.JobEngineFatalException; +import com.hemju.threadmill.core.store.JobStore; +import com.hemju.threadmill.store.oracle.OracleJobStore; +import com.hemju.threadmill.store.oracle.OracleMigrationRunner; + +/** + * Boots a {@code gvenzl/oracle-free:23-slim-faststart} Testcontainer (or, if + * {@code -PoracleUrl=...} is given, points at an external database with + * {@code -PoracleUser} / {@code -PoraclePassword}). The Threadmill schema is + * migrated and every Threadmill row deleted before the run starts so each + * invocation begins on a clean slate. An external database's Threadmill tables + * are emptied, so point it at a disposable schema only. + */ +public final class OracleHarnessFixture implements BackendFixture { + + /** The container image used when no external database is configured. */ + public static final String IMAGE = "gvenzl/oracle-free:23-slim-faststart"; + + private static final String CONTAINER_USER = "threadmill"; + private static final String CONTAINER_PASSWORD = "threadmill"; + private static final int MAX_CONNECTIONS = 48; + + /** + * Oracle has no {@code TRUNCATE ... CASCADE} for these foreign keys, so the + * reset deletes children before parents. + */ + private static final List RESET_ORDER = List.of( + "threadmill_dedup_keys", + "threadmill_cron_task_ownership", + "threadmill_cron_task_state", + "threadmill_cron_tasks", + "threadmill_jobs", + "threadmill_nodes", + "threadmill_mutexes", + "threadmill_leases", + "threadmill_queue_pauses", + "threadmill_concurrency_groups", + "threadmill_concurrency_workflow_holds", + "threadmill_queue_counts"); + + private final GenericContainer container; + private final HarnessPooledDataSource dataSource; + private final OracleJobStore store; + + /** Start a throwaway Oracle container. */ + public OracleHarnessFixture() { + this(Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** + * Use the external database at {@code externalJdbcUrl} when present, otherwise + * start a throwaway container. + */ + @SuppressWarnings("resource") // the container is stopped in close() + public OracleHarnessFixture( + Optional externalJdbcUrl, Optional user, Optional password) { + if (externalJdbcUrl.isPresent()) { + this.container = null; + this.dataSource = pooled(externalJdbcUrl.get(), user.orElse(null), password.orElse(null)); + } else { + this.container = new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withEnv("ORACLE_PASSWORD", CONTAINER_PASSWORD) + .withEnv("APP_USER", CONTAINER_USER) + .withEnv("APP_USER_PASSWORD", CONTAINER_PASSWORD) + .withExposedPorts(1521) + .waitingFor(Wait.forLogMessage(".*DATABASE IS READY TO USE!.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(10))); + container.start(); + this.dataSource = pooled(jdbcUrl(container), CONTAINER_USER, CONTAINER_PASSWORD); + } + new OracleMigrationRunner(dataSource).migrate(); + reset(); + this.store = new OracleJobStore(dataSource); + } + + /** The thin-driver URL of the container's pluggable database. */ + public static String jdbcUrl(GenericContainer container) { + return "jdbc:oracle:thin:@//" + container.getHost() + ":" + container.getMappedPort(1521) + + "/FREEPDB1"; + } + + /** + * Delete every Threadmill row and zero the per-state counters, keeping the + * schema. Must not run while a node or producer uses the store. + */ + public void reset() { + try (Connection conn = dataSource.getConnection(); + Statement st = conn.createStatement()) { + conn.setAutoCommit(true); + for (String table : RESET_ORDER) { + st.executeUpdate("DELETE FROM " + table); + } + st.executeUpdate("UPDATE threadmill_job_counts SET job_count = 0"); + } catch (SQLException e) { + throw new JobEngineFatalException( + "could not reset Oracle tables before run: " + e.getMessage(), e); + } + } + + /** The pooled data source the store runs on; closed with this fixture. */ + public DataSource dataSource() { + return dataSource; + } + + /** The Docker id of the owned container, or empty for an external database. */ + public Optional containerId() { + return Optional.ofNullable(container).map(GenericContainer::getContainerId); + } + + @Override + public JobStore store() { + return store; + } + + @Override + public void close() { + dataSource.close(); + if (container != null && container.isRunning()) container.stop(); + } + + private static HarnessPooledDataSource pooled(String url, String user, String password) { + try { + var ds = new OracleDataSource(); + ds.setURL(url); + if (user != null) ds.setUser(user); + if (password != null) ds.setPassword(password); + return new HarnessPooledDataSource(ds, MAX_CONNECTIONS); + } catch (SQLException e) { + throw new JobEngineFatalException("could not create the Oracle data source", e); + } + } +} diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RunConfigWriter.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RunConfigWriter.java index 5f50bed4..bccf7577 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RunConfigWriter.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/RunConfigWriter.java @@ -30,6 +30,10 @@ public static Map asMap(SoakHarnessConfig c) { m.put("failFast", c.failFast()); m.put("force", c.force()); m.put("postgresUrl", c.postgresUrl().orElse(null)); + m.put("oracleUrl", c.oracleUrl().orElse(null)); + m.put("oracleUser", c.oracleUser().orElse(null)); + // Never write the Oracle password into the run artifacts. + m.put("oraclePasswordSet", c.oraclePassword().isPresent()); m.put("redisTopology", c.redisTopology()); m.put("redisUrl", c.redisUrl().orElse(null)); m.put("progressInterval", c.progressInterval().toString()); diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessConfig.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessConfig.java index 3af0cb47..87f577d5 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessConfig.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessConfig.java @@ -30,6 +30,9 @@ public record SoakHarnessConfig( String runId, boolean failFast, Optional postgresUrl, + Optional oracleUrl, + Optional oracleUser, + Optional oraclePassword, String redisTopology, Optional redisUrl, boolean force, @@ -71,6 +74,9 @@ public static SoakHarnessConfig fromSystemProperties(String backend) { boolean failFast = Boolean.parseBoolean(prop("failFast", "true")); boolean force = Boolean.parseBoolean(prop("force", "false")); Optional postgresUrl = Optional.ofNullable(prop("postgresUrl", null)); + Optional oracleUrl = Optional.ofNullable(prop("oracleUrl", null)); + Optional oracleUser = Optional.ofNullable(prop("oracleUser", null)); + Optional oraclePassword = Optional.ofNullable(prop("oraclePassword", null)); String redisTopology = prop("redisTopology", "standalone"); Optional redisUrl = Optional.ofNullable(prop("redisUrl", null)); String runId = Optional.ofNullable(prop("runId", null)) @@ -97,6 +103,9 @@ public static SoakHarnessConfig fromSystemProperties(String backend) { runId, failFast, postgresUrl, + oracleUrl, + oracleUser, + oraclePassword, redisTopology, redisUrl, force, diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessMain.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessMain.java index c89476c1..02a075d7 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessMain.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/SoakHarnessMain.java @@ -6,8 +6,8 @@ import org.slf4j.LoggerFactory; /** - * Entry point invoked by the {@code soakMemory}, {@code soakPostgres}, and - * {@code soakRedis} Gradle tasks. + * Entry point invoked by the {@code soakMemory}, {@code soakPostgres}, + * {@code soakOracle}, and {@code soakRedis} Gradle tasks. * *

Reads {@code --backend X} from the command line; everything else * (scenario, duration, rates, output dir) is read from @@ -52,6 +52,8 @@ private static BackendFixture openFixture(SoakHarnessConfig config) { return switch (config.backend()) { case "memory" -> new MemoryHarnessFixture(); case "postgres" -> new PostgresHarnessFixture(config.postgresUrl()); + case "oracle" -> + new OracleHarnessFixture(config.oracleUrl(), config.oracleUser(), config.oraclePassword()); case "redis" -> new RedisHarnessFixture(config.redisTopology(), config.redisUrl()); default -> throw new IllegalArgumentException("unknown backend: " + config.backend()); }; diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceConfig.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceConfig.java index 1855157a..30f096b9 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceConfig.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceConfig.java @@ -20,7 +20,9 @@ * *

{@code -Pbackends} exists mainly so the orchestrator can be exercised * cheaply ({@code memory,memory} in tests); production sign-off uses the - * default pair. + * default pair. {@code oracle} may be added explicitly + * ({@code -Pbackends=postgres,redis,oracle}), optionally with + * {@code -PoracleUrl} / {@code -PoracleUser} / {@code -PoraclePassword}. */ public record EnduranceConfig( List backends, @@ -34,12 +36,15 @@ public record EnduranceConfig( boolean failFast, Duration progressInterval, Optional postgresUrl, + Optional oracleUrl, + Optional oracleUser, + Optional oraclePassword, Optional redisUrl, Path outputDir, String runId, boolean force) { - private static final Set KNOWN_BACKENDS = Set.of("memory", "postgres", "redis"); + private static final Set KNOWN_BACKENDS = Set.of("memory", "postgres", "oracle", "redis"); public static EnduranceConfig fromSystemProperties() { List backends = new ArrayList<>(); @@ -73,6 +78,9 @@ public static EnduranceConfig fromSystemProperties() { boolean failFast = Boolean.parseBoolean(prop("failFast", "true")); Duration progressInterval = SoakHarnessConfig.parseDuration(prop("progressInterval", "30s")); Optional postgresUrl = Optional.ofNullable(prop("postgresUrl", null)); + Optional oracleUrl = Optional.ofNullable(prop("oracleUrl", null)); + Optional oracleUser = Optional.ofNullable(prop("oracleUser", null)); + Optional oraclePassword = Optional.ofNullable(prop("oraclePassword", null)); Optional redisUrl = Optional.ofNullable(prop("redisUrl", null)); boolean force = Boolean.parseBoolean(prop("force", "false")); String runId = Optional.ofNullable(prop("runId", null)) @@ -92,6 +100,9 @@ public static EnduranceConfig fromSystemProperties() { failFast, progressInterval, postgresUrl, + oracleUrl, + oracleUser, + oraclePassword, redisUrl, outputDir, runId, diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceMain.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceMain.java index 4968f041..a5ad9691 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceMain.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/endurance/EnduranceMain.java @@ -136,6 +136,11 @@ private static ChildRun startChild(EnduranceConfig config, String backend, Strin if ("postgres".equals(backend)) { config.postgresUrl().ifPresent(url -> addProp(command, "postgresUrl", url)); } + if ("oracle".equals(backend)) { + config.oracleUrl().ifPresent(url -> addProp(command, "oracleUrl", url)); + config.oracleUser().ifPresent(user -> addProp(command, "oracleUser", user)); + config.oraclePassword().ifPresent(password -> addProp(command, "oraclePassword", password)); + } if ("redis".equals(backend)) { config.redisUrl().ifPresent(url -> addProp(command, "redisUrl", url)); } @@ -230,6 +235,10 @@ private static void writeConfig(EnduranceConfig config) throws IOException { m.put("failFast", config.failFast()); m.put("progressInterval", config.progressInterval().toString()); m.put("postgresUrl", config.postgresUrl().orElse(null)); + m.put("oracleUrl", config.oracleUrl().orElse(null)); + m.put("oracleUser", config.oracleUser().orElse(null)); + // Never write the Oracle password into the run artifacts. + m.put("oraclePasswordSet", config.oraclePassword().isPresent()); m.put("redisUrl", config.redisUrl().orElse(null)); m.put("outputDir", config.outputDir().toString()); Files.writeString( diff --git a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/package-info.java b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/package-info.java index 79049407..da663c47 100644 --- a/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/package-info.java +++ b/threadmill-soak/src/main/java/com/hemju/threadmill/soak/harness/package-info.java @@ -5,7 +5,7 @@ * AI agent (or human) can ingest cold to answer two questions: did the engine * behave correctly under the chosen scenario, and how fast was it. The entry * points are the Gradle tasks {@code soakMemory}, {@code soakPostgres}, - * {@code soakRedis}, and {@code soakAll}, defined in this module's build. + * {@code soakOracle}, {@code soakRedis}, and {@code soakAll}, defined in this module's build. * *

Distinct from the {@code :soakRegression} JUnit task (fixed sustained * throughput, recurring no-skip, container-pause recovery) and from diff --git a/threadmill-soak/src/main/resources/com/hemju/threadmill/soak/harness/summary.schema.json b/threadmill-soak/src/main/resources/com/hemju/threadmill/soak/harness/summary.schema.json index 8d33d779..094369ca 100644 --- a/threadmill-soak/src/main/resources/com/hemju/threadmill/soak/harness/summary.schema.json +++ b/threadmill-soak/src/main/resources/com/hemju/threadmill/soak/harness/summary.schema.json @@ -8,7 +8,7 @@ "properties": { "runId": { "type": "string" }, "scenario": { "type": "string" }, - "backend": { "type": "string", "enum": ["memory", "postgres", "redis"] }, + "backend": { "type": "string", "enum": ["memory", "postgres", "oracle", "redis"] }, "config": { "type": "object", "additionalProperties": true diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/OracleSoakTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/OracleSoakTest.java new file mode 100644 index 00000000..58a875e1 --- /dev/null +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/OracleSoakTest.java @@ -0,0 +1,223 @@ +package com.hemju.threadmill.soak; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.time.Duration; +import java.time.Instant; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; + +import com.hemju.threadmill.core.ConcurrencyMode; +import com.hemju.threadmill.core.engine.ProcessingNode; +import com.hemju.threadmill.core.engine.ProcessingNodeConfig; +import com.hemju.threadmill.core.engine.QueueWeights; +import com.hemju.threadmill.core.schedule.CronTask; +import com.hemju.threadmill.core.schedule.Scheduler; +import com.hemju.threadmill.core.serialization.JsonJobSerializer; +import com.hemju.threadmill.soak.harness.OracleHarnessFixture; +import com.hemju.threadmill.store.oracle.OracleJobStore; + +/** + * Soak runs against a real Oracle Database Free ({@code gvenzl/oracle-free}) + * via Testcontainers; the Oracle counterpart of {@link PostgresSoakTest}. + * + *

    + *
  • {@code sustainedThroughput…} drives 5,000 jobs through the engine.
    • + *
  • {@code recurringNoSkip…} confirms the maintenance cycle materialises + * every interval over a five-second period.
    • + *
  • {@code clusterPauses…AcrossAnInducedStoreOutage} pauses the + * Oracle container mid-run via the Docker API, asserts no progress + * while paused, unpauses, and asserts the engine resumes.
    • + *
+ */ +@Tag("soak") +class OracleSoakTest { + + private static OracleHarnessFixture fixture; + private final JsonJobSerializer serializer = new JsonJobSerializer(); + private OracleJobStore store; + private Scheduler scheduler; + private ProcessingNode node; + + @BeforeAll + static void startContainer() { + fixture = new OracleHarnessFixture(); + } + + @AfterAll + static void stopContainer() { + if (fixture != null) fixture.close(); + } + + @BeforeEach + void freshDb() { + fixture.reset(); + store = new OracleJobStore(fixture.dataSource()); + scheduler = new Scheduler(store, serializer); + SoakWork.CountingHandler.COUNT.set(0); + } + + @AfterEach + void tearDown() { + if (node != null) node.close(); + } + + @Test + void sustainedThroughputProcessesFiveThousandJobsThroughRealOracle() { + int total = 5_000; + for (int i = 0; i < total; i++) { + scheduler.enqueue(new SoakWork.P(i), SoakWork.CountingHandler.class); + } + node = ProcessingNode.builder(store).config(fastConfig()).build(); + long started = System.nanoTime(); + node.start(); + await() + .atMost(Duration.ofMinutes(5)) + .until(() -> SoakWork.CountingHandler.COUNT.get() >= total); + long elapsedMs = (System.nanoTime() - started) / 1_000_000L; + System.out.printf( + "Soak Oracle throughput: %d jobs in %d ms (%.1f jobs/sec)%n", + total, elapsedMs, total * 1000.0 / Math.max(1, elapsedMs)); + } + + @Test + void sustainedContentionOnOneConcurrencyKeyCompletesAgainstOracle() { + int total = 100; + for (int i = 0; i < total; i++) { + var mode = i % 10 == 0 ? ConcurrencyMode.EXCLUSIVE : ConcurrencyMode.SHARED; + scheduler.enqueue( + new SoakWork.P(i), + SoakWork.CountingHandler.class, + "default", + 0, + "project:contention", + mode); + } + node = ProcessingNode.builder(store).config(fastConfig()).build(); + long started = System.nanoTime(); + node.start(); + await() + .atMost(Duration.ofMinutes(3)) + .until(() -> SoakWork.CountingHandler.COUNT.get() >= total); + long elapsedMs = (System.nanoTime() - started) / 1_000_000L; + System.out.printf( + "Soak Oracle contention: %d jobs in %d ms (%.1f jobs/sec)%n", + total, elapsedMs, total * 1000.0 / Math.max(1, elapsedMs)); + } + + @Test + void queueFamilyLaneProcessesBurstyProjectQueuesAgainstOracle() { + int total = 100; + for (int i = 0; i < total; i++) { + scheduler.enqueue( + new SoakWork.P(i), SoakWork.CountingHandler.class, "project:" + (i % 10), 0); + } + node = ProcessingNode.builder(store) + .config(fastConfig().toBuilder() + .queueFamilyDiscoveryInterval(Duration.ofMillis(50)) + .queueFamilyRetentionAfterEmpty(Duration.ofSeconds(1)) + .build()) + .lane("project:*", 8, QueueWeights.uniform()) + .build(); + long started = System.nanoTime(); + node.start(); + await() + .atMost(Duration.ofMinutes(3)) + .until(() -> SoakWork.CountingHandler.COUNT.get() >= total); + long elapsedMs = (System.nanoTime() - started) / 1_000_000L; + System.out.printf( + "Soak Oracle queue-family: %d jobs in %d ms (%.1f jobs/sec)%n", + total, elapsedMs, total * 1000.0 / Math.max(1, elapsedMs)); + } + + @Test + void recurringEveryHundredMillisHasNoSkipsAgainstOracle() { + scheduler.defineIntervalTask( + "ping", + Duration.ofMillis(100), + new SoakWork.P(0), + SoakWork.CountingHandler.class, + "default", + 0, + CronTask.MissedRunPolicy.DROP); + node = ProcessingNode.builder(store).config(fastConfig()).build(); + var start = Instant.now(); + node.start(); + Duration runFor = Duration.ofSeconds(5); + await().pollDelay(runFor).atMost(runFor.plus(Duration.ofSeconds(3))).until(() -> true); + long elapsedMs = Duration.between(start, Instant.now()).toMillis(); + // Oracle round-trips add latency per claim+promote tick; the actual no-skip + // guarantee is structural (DROP materialises exactly one instance when due). + // Here we only assert the loop is producing runs at a sustainable rate. + long expectedMin = Math.max(10L, (elapsedMs - 500) / 100L / 4); + int observed = SoakWork.CountingHandler.COUNT.get(); + System.out.printf( + "Soak Oracle recurring: %d runs in %d ms (expected >= %d)%n", + observed, elapsedMs, expectedMin); + assertThat(observed).isGreaterThanOrEqualTo((int) expectedMin); + } + + @Test + void clusterPausesAndResumesAcrossAnInducedStoreOutage() throws Exception { + // Enqueue work, then pause the Oracle container — the engine must pause its + // dispatcher (via the circuit breaker), not crash; unpause, the engine resumes. + int total = 500; + for (int i = 0; i < total; i++) { + scheduler.enqueue(new SoakWork.P(i), SoakWork.CountingHandler.class); + } + node = ProcessingNode.builder(store) + .config(fastConfig().toBuilder() + .maxConsecutiveDispatcherFailures(2) + .storeOutagePollInterval(Duration.ofMillis(200)) + .build()) + .build(); + node.start(); + // Let some jobs run before the outage so we can observe both phases. + await().atMost(Duration.ofSeconds(20)).until(() -> SoakWork.CountingHandler.COUNT.get() >= 50); + + String containerId = fixture.containerId().orElseThrow(); + var docker = DockerClientFactory.instance().client(); + docker.pauseContainerCmd(containerId).exec(); + int countAtPause = SoakWork.CountingHandler.COUNT.get(); + System.out.printf("Soak Oracle outage: paused container at %d jobs done%n", countAtPause); + try { + Thread.sleep(2_000); + } finally { + docker.unpauseContainerCmd(containerId).exec(); + } + int countAfterPause = SoakWork.CountingHandler.COUNT.get(); + // Reasonable: while paused, at most the in-flight batch's worth of work finished. + assertThat(countAfterPause - countAtPause) + .as("no large progress while the store was paused") + .isLessThan(64); + + // After unpause, the dispatcher's circuit breaker must trip back and processing resume. + await() + .atMost(Duration.ofMinutes(2)) + .until(() -> SoakWork.CountingHandler.COUNT.get() >= total); + System.out.printf( + "Soak Oracle outage: resumed; %d total processed%n", SoakWork.CountingHandler.COUNT.get()); + } + + private static ProcessingNodeConfig fastConfig() { + return ProcessingNodeConfig.builder() + .workerCount(8) + .pollInterval(Duration.ofMillis(20)) + .claimHeartbeat(Duration.ofMillis(200)) + .maintenancePollInterval(Duration.ofMillis(20)) + .heartbeatTimeout(Duration.ofSeconds(30)) + .jobTimeout(Duration.ofSeconds(15)) + .claimBatchSize(32) + .defaultMaxAttempts(1) + .retryInitialBackoff(Duration.ofMillis(50)) + .storeOutagePollInterval(Duration.ofMillis(200)) + .build(); + } +} diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/HarnessQualificationTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/HarnessQualificationTest.java index 6b306196..6c9f6cca 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/HarnessQualificationTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/HarnessQualificationTest.java @@ -34,6 +34,9 @@ void workloadDurationIsNotConsumedByAnEarlierSetupTimestamp(@TempDir Path output "duration", true, Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), "standalone", Optional.empty(), false, diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/MultiProducerSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/MultiProducerSmokeTest.java index e4262745..b4468540 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/MultiProducerSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/MultiProducerSmokeTest.java @@ -90,6 +90,9 @@ private static void clearProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/NodeChurnSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/NodeChurnSmokeTest.java index 1e9bfe12..a7bed565 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/NodeChurnSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/NodeChurnSmokeTest.java @@ -73,6 +73,9 @@ private static void clearSoakProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisExternalUrlFixtureTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisExternalUrlFixtureTest.java index 886037c4..ddb56923 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisExternalUrlFixtureTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisExternalUrlFixtureTest.java @@ -84,6 +84,9 @@ private static void clearProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java index 15e439ed..160ceb4c 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RedisProducerOutageTest.java @@ -41,6 +41,9 @@ void producerResumesAfterOutageExceedsCommandTimeout(String scenario, @TempDir P "producer-outage", true, Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), "standalone", Optional.of("redis://" + redis.getHost() + ":" + redis.getMappedPort(6379)), false, diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java index ab0fbdfe..a373eaa8 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/RetentionChurnSmokeTest.java @@ -19,7 +19,7 @@ @Tag("soak") class RetentionChurnSmokeTest { @ParameterizedTest - @ValueSource(strings = {"memory", "postgres", "redis"}) + @ValueSource(strings = {"memory", "postgres", "oracle", "redis"}) void retainedPopulationShrinksWhileRetriesAndWorkflowsComplete( String backend, @TempDir Path temporary) throws Exception { var output = temporary.resolve("run"); @@ -38,6 +38,9 @@ void retainedPopulationShrinksWhileRetriesAndWorkflowsComplete( "retention-regression", true, Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), "standalone", Optional.empty(), false, @@ -46,6 +49,7 @@ void retainedPopulationShrinksWhileRetriesAndWorkflowsComplete( try (var fixture = switch (backend) { case "postgres" -> new PostgresHarnessFixture(Optional.empty()); + case "oracle" -> new OracleHarnessFixture(); case "redis" -> new RedisHarnessFixture("standalone"); default -> new MemoryHarnessFixture(); }) { diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessOracleSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessOracleSmokeTest.java new file mode 100644 index 00000000..61f55575 --- /dev/null +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessOracleSmokeTest.java @@ -0,0 +1,62 @@ +package com.hemju.threadmill.soak.harness; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end smoke against Oracle Database Free via Testcontainers. Tagged {@code soak} + * — runs in the fixed {@code :soakRegression} task, not on every {@code check}. + */ +@Tag("soak") +final class SoakHarnessOracleSmokeTest { + + @Test + void oracleSmokeProducesPassedVerdict(@TempDir Path tempDir) throws Exception { + Path outputDir = tempDir.resolve("oracle-smoke"); + System.setProperty("threadmill.soak.scenario", "mixed-workload"); + System.setProperty("threadmill.soak.duration", "10s"); + System.setProperty("threadmill.soak.jobsPerSecond", "60"); + System.setProperty("threadmill.soak.workerCount", "4"); + System.setProperty("threadmill.soak.nodes", "1"); + System.setProperty("threadmill.soak.outputDir", outputDir.toString()); + System.setProperty("threadmill.soak.runId", "oracle-smoke"); + try { + SoakHarnessConfig config = SoakHarnessConfig.fromSystemProperties("oracle"); + OutputDir dir = new OutputDir(config.outputDir(), config.force()); + SummaryReport report; + try (BackendFixture fixture = new OracleHarnessFixture()) { + report = new SoakHarnessRunner(config, fixture, dir, "soakOracle").run(); + } + assertThat(report.verdict()).isEqualTo("passed"); + assertThat(outputDir.resolve("summary.json")).exists(); + } finally { + clearProps(); + } + } + + private static void clearProps() { + for (String k : new String[] { + "scenario", + "duration", + "jobsPerSecond", + "workerCount", + "nodes", + "outputDir", + "runId", + "failFast", + "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", + "force", + "redisTopology" + }) { + System.clearProperty("threadmill.soak." + k); + } + } +} diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessPostgresSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessPostgresSmokeTest.java index b8b408a3..7710ea2c 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessPostgresSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessPostgresSmokeTest.java @@ -51,6 +51,9 @@ private static void clearProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology" }) { diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRedisSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRedisSmokeTest.java index fb76f47d..5fa78d21 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRedisSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRedisSmokeTest.java @@ -50,6 +50,9 @@ private static void clearProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology" }) { diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRunnerTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRunnerTest.java index f766663a..5c496ba9 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRunnerTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessRunnerTest.java @@ -82,6 +82,9 @@ private static void clearSoakProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessSmokeTest.java index 30920cc1..3276271a 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/SoakHarnessSmokeTest.java @@ -107,6 +107,9 @@ private static void clearSoakSystemProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", diff --git a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/endurance/EnduranceOrchestratorSmokeTest.java b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/endurance/EnduranceOrchestratorSmokeTest.java index 36972220..40b76433 100644 --- a/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/endurance/EnduranceOrchestratorSmokeTest.java +++ b/threadmill-soak/src/test/java/com/hemju/threadmill/soak/harness/endurance/EnduranceOrchestratorSmokeTest.java @@ -106,6 +106,9 @@ private static void clearProps() { "runId", "failFast", "postgresUrl", + "oracleUrl", + "oracleUser", + "oraclePassword", "force", "redisTopology", "redisUrl", From 63c074afc39032c6287441050699ba503717ee37 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 21:43:55 +0200 Subject: [PATCH 5/7] fix(store-oracle): address review of the Oracle store - Joined write units are all-or-nothing: Oracle rolls back only the failing statement, so a caller that caught the store's error and committed also committed the unit's earlier statements (e.g. an insertAll batch before its duplicate id). Each write in a caller-owned transaction now rolls back to a savepoint on failure. - releaseMutex rejects a null holder; DECODE treated it as equal to an acquired empty holder (stored as NULL) and freed that mutex. New shared contract case for every store. - Lease and mutex durations keep nanoseconds instead of truncating sub-millisecond leases to zero. - Plan tests locate their cursor by a unique tag in V$SQL instead of the session's previous statement, which returned no plan on 21c XE in CI. - docs: the default poll interval is 500 ms. Refs #137 --- docs/oracle-schema.md | 2 +- .../SpringOracleTransactionBoundaryTest.java | 32 +++++++++++++++++ .../threadmill/store/oracle/OracleJdbc.java | 8 +++-- .../store/oracle/OracleJobStore.java | 28 ++++++++++++++- .../store/oracle/OracleQueryPlanTest.java | 36 +++++++++++++++---- .../test/AbstractJobStoreContractTest.java | 20 +++++++++++ 6 files changed, 116 insertions(+), 10 deletions(-) diff --git a/docs/oracle-schema.md b/docs/oracle-schema.md index 938bc01a..98c2c99f 100644 --- a/docs/oracle-schema.md +++ b/docs/oracle-schema.md @@ -127,6 +127,6 @@ migrations in production. through temporary LOBs that the store frees after each statement; size the `TEMP` tablespace for the concurrent write rate of large jobs. - **No cross-node wake.** Nodes pick up work enqueued on another node within - their `pollInterval` (default 1 s); same-JVM wakes are immediate. + their `pollInterval` (default 500 ms); same-JVM wakes are immediate. - **Driver.** The store uses standard JDBC. It was tested with `ojdbc11` 23.26.3; older drivers that report no batch update counts are not supported. diff --git a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java index 221c5a8b..79cf613a 100644 --- a/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java +++ b/threadmill-spring-boot/src/test/java/com/hemju/threadmill/spring/SpringOracleTransactionBoundaryTest.java @@ -8,6 +8,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneId; +import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicReference; @@ -21,7 +22,9 @@ import org.springframework.transaction.support.TransactionTemplate; import com.hemju.threadmill.core.EnqueueResult; +import com.hemju.threadmill.core.Job; import com.hemju.threadmill.core.JobId; +import com.hemju.threadmill.core.JobState; import com.hemju.threadmill.core.engine.LocalWakeBus; import com.hemju.threadmill.core.engine.ProcessingNodeConfig; import com.hemju.threadmill.core.handler.JobExecutionContext; @@ -31,6 +34,7 @@ import com.hemju.threadmill.core.schedule.CronTaskScheduleState; import com.hemju.threadmill.core.serialization.JsonJobSerializer; import com.hemju.threadmill.core.spec.JobArgument; +import com.hemju.threadmill.core.spec.JobSpec; import com.hemju.threadmill.core.store.JobStoreCapabilities; import com.hemju.threadmill.store.oracle.OracleJobStore; import com.hemju.threadmill.store.oracle.OracleMigrationRunner; @@ -108,6 +112,34 @@ void enqueueRollsBackWithCallerTransaction() { assertThat(wakes).isEmpty(); } + @Test + void aFailedJoinedWriteLeavesNothingBehindWhenTheCallerCatchesAndCommits() { + // Oracle rolls back only the failing statement. Without a savepoint per + // write unit, the batch row inserted before the duplicate id survived the + // caller's commit, breaking insertAll's all-or-none contract. + var earlier = job(); + var duplicated = job(); + + transactions.executeWithoutResult(status -> { + store.insert(earlier); + assertThatThrownBy(() -> store.insertAll(List.of(duplicated, duplicated))) + .isInstanceOf(IllegalStateException.class); + }); + + assertThat(store.findById(earlier.id())) + .as("an earlier, successful write unit") + .isPresent(); + assertThat(store.findById(duplicated.id())).as("the failed batch").isEmpty(); + assertThat(store.countsByState().get(JobState.ENQUEUED)).isEqualTo(1L); + } + + private static Job job() { + return Job.builder() + .spec(JobSpec.of( + GreetHandler.class.getName(), new JobArgument(GreetPayload.class.getName(), "{}"))) + .build(); + } + @Test void dedupRollsBackWithCallerTransaction() { AtomicReference first = new AtomicReference<>(); diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java index 85c7f154..c03b7335 100644 --- a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJdbc.java @@ -90,9 +90,13 @@ static Instant getInstant(ResultSet rs, String column) throws SQLException { return value == null ? null : value.toInstant(ZoneOffset.UTC); } - /** Seconds as an exact decimal, for {@code NUMTODSINTERVAL(?, 'SECOND')}. */ + /** + * Seconds as an exact decimal, for {@code NUMTODSINTERVAL(?, 'SECOND')}. + * Keeps nanoseconds so a positive sub-millisecond duration never binds zero. + */ static void setSeconds(PreparedStatement ps, int index, Duration value) throws SQLException { - ps.setBigDecimal(index, BigDecimal.valueOf(value.toMillis(), 3)); + ps.setBigDecimal( + index, BigDecimal.valueOf(value.getSeconds()).add(BigDecimal.valueOf(value.getNano(), 9))); } /** diff --git a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java index 5955ea30..f6588e5d 100644 --- a/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java +++ b/threadmill-store-oracle/src/main/java/com/hemju/threadmill/store/oracle/OracleJobStore.java @@ -263,11 +263,34 @@ public Optional createRemoteWakeChannel(String channelName) { private T writeTransaction(OracleConnectionWork work) throws SQLException { if (transactionBoundary.externallyManagedTransactionActive()) { - return transactionBoundary.inTransaction(work); + return transactionBoundary.inTransaction(conn -> atomically(conn, work)); } return OracleDeadlockRetry.run(() -> transactionBoundary.inTransaction(work)); } + /** + * Run one write unit inside a caller-owned transaction all-or-nothing. + * Oracle rolls back only the failing statement, not the transaction: without + * the savepoint, a caller that catches the store's exception and commits + * would also commit the unit's earlier statements, for example the rows of + * an {@code insertAll} batch before its duplicate id. PostgreSQL aborts the + * whole transaction instead, so its store needs no equivalent. + */ + private static T atomically(Connection conn, OracleConnectionWork work) + throws SQLException { + Savepoint unit = conn.setSavepoint(); + try { + return work.execute(conn); + } catch (SQLException | RuntimeException | Error failure) { + try { + conn.rollback(unit); + } catch (SQLException | RuntimeException rollbackFailure) { + failure.addSuppressed(rollbackFailure); + } + throw failure; + } + } + /** * Run one self-owned write in a transaction Threadmill commits, never the * caller's. The explicit boundary is required even for a single statement: @@ -2008,6 +2031,9 @@ public boolean tryAcquireMutex(String name, String holder, Duration leaseDuratio @Override public void releaseMutex(String name, String holder) { Names.requireName("mutex", name); + // Oracle stores an acquired empty holder as NULL and DECODE matches two + // NULLs, so a null release would free another holder's mutex. + Objects.requireNonNull(holder, "holder"); try { ownedTransaction(conn -> { try (PreparedStatement ps = conn.prepareStatement( diff --git a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java index 2a5d77ea..7d104928 100644 --- a/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java +++ b/threadmill-store-oracle/src/test/java/com/hemju/threadmill/store/oracle/OracleQueryPlanTest.java @@ -13,6 +13,7 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -247,12 +248,20 @@ private interface Binder { void bind(PreparedStatement ps) throws SQLException; } - /** Run {@code sql} with typed binds and return the plan of the executed cursor. */ + /** + * Run {@code sql} with typed binds and return the plan of the executed cursor. + * + *

A unique trailing comment gives the probe its own cursor, which is then + * found in {@code V$SQL} by that tag. Relying on the session's previous + * statement ({@code DISPLAY_CURSOR(NULL, NULL)}) returned no plan on 21c + * XE. The comment does not change optimization: hints sit at the start. + */ private static String executedPlan(String sql, Binder binder) throws SQLException { + String tag = "threadmill-plan-probe-" + UUID.randomUUID(); try (Connection conn = OracleTestDatabase.dataSource().getConnection()) { conn.setAutoCommit(false); try { - try (PreparedStatement ps = conn.prepareStatement(sql)) { + try (PreparedStatement ps = conn.prepareStatement(sql + " /* " + tag + " */")) { binder.bind(ps); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { @@ -260,11 +269,26 @@ private static String executedPlan(String sql, Binder binder) throws SQLExceptio } } } + String sqlId; + int child; + try (PreparedStatement ps = conn.prepareStatement("SELECT sql_id, child_number FROM v$sql " + + "WHERE sql_fulltext LIKE ? AND sql_fulltext NOT LIKE '%v$sql%' " + + "ORDER BY last_active_time DESC FETCH FIRST 1 ROW ONLY")) { + ps.setString(1, "%" + tag + "%"); + try (ResultSet rs = ps.executeQuery()) { + assertThat(rs.next()).as("cursor for %s in V$SQL", tag).isTrue(); + sqlId = rs.getString(1); + child = rs.getInt(2); + } + } var plan = new StringBuilder(); - try (Statement st = conn.createStatement(); - ResultSet rs = st.executeQuery("SELECT plan_table_output FROM TABLE(" - + "DBMS_XPLAN.DISPLAY_CURSOR(NULL, NULL, 'BASIC'))")) { - while (rs.next()) plan.append(rs.getString(1)).append('\n'); + try (PreparedStatement ps = conn.prepareStatement( + "SELECT plan_table_output FROM TABLE(" + "DBMS_XPLAN.DISPLAY_CURSOR(?, ?, 'BASIC'))")) { + ps.setString(1, sqlId); + ps.setInt(2, child); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) plan.append(rs.getString(1)).append('\n'); + } } assertThat(plan.toString()).as("executed-cursor plan").contains("Plan hash value"); return plan.toString(); diff --git a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java index 1ac9117f..1c040fa4 100644 --- a/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java +++ b/threadmill-test-support/src/main/java/com/hemju/threadmill/test/AbstractJobStoreContractTest.java @@ -2238,6 +2238,26 @@ void mutexLeaseSemantics() throws InterruptedException { assertThat(store.tryAcquireMutex("m", "node-c", Duration.ofSeconds(5))).isTrue(); } + @Test + @DisplayName("a null release holder never frees a mutex, including one held by an empty holder") + void releaseWithANullHolderNeverFreesAnotherHoldersMutex() { + // Stores may reject a null holder or ignore it; either way the lease must + // survive. Oracle stores an empty string as NULL, so an unguarded + // NULL-equal comparison released the empty holder's mutex. + for (String holder : List.of("node-a", "")) { + String name = "null-release-" + (holder.isEmpty() ? "empty" : holder); + assertThat(store.tryAcquireMutex(name, holder, Duration.ofMinutes(1))).isTrue(); + try { + store.releaseMutex(name, null); + } catch (RuntimeException rejected) { + // rejecting a null holder is an accepted outcome + } + assertThat(store.tryAcquireMutex(name, "other", Duration.ofMinutes(1))) + .as("mutex held by %s after a null release", holder.isEmpty() ? "\"\"" : holder) + .isFalse(); + } + } + // ================================================================ replaceJob @Test From f98ae2755b5b8e5193de2d213629d3161bbde754 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Tue, 6 Oct 2026 21:44:10 +0200 Subject: [PATCH 6/7] docs(agents): record the Oracle review regressions Refs #137 --- AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9f0d8152..b346f591 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,7 +288,7 @@ This section is the project's memory: the load-bearing decisions worth knowing b - **Text ordering is explicitly binary.** The driver derives `NLS_LANGUAGE`, and so `NLS_SORT`, from the client JVM locale (a German JVM sorts `a < Ä < B`). Comparisons follow `NLS_COMP` (binary), but `ORDER BY` follows `NLS_SORT`, so linguistic page order made keyset pages (`name > ?`) skip rows and prevented index-ordered reads. Small tables order by `NLSSORT(col, 'NLS_SORT = BINARY')`; the idle concurrency-group page reads a `RAW(2000)` virtual sort key (`idle_sort`). Never `ORDER BY` a text column in a paging query. `keysetPagesFollowBinaryOrderUnderALinguisticClientLocale` runs under a German session. - **UTC `TIMESTAMP(9)` bound as `LocalDateTime`.** Nanosecond-exact and independent of JVM and session time zones (`java.sql.Timestamp` binding would apply the JVM default zone). Server-side time is `SYS_EXTRACT_UTC(SYSTIMESTAMP)`. - **Oracle stores `''` as `NULL`.** Recurring payloads and pause reasons map `NULL` back to `""`; mutex holders compare with `DECODE`, which treats two `NULL`s as equal. -- **Statement-level rollback.** A failed statement does not abort an Oracle transaction. Upserts are update-then-insert with an `ORA-00001` retry; concurrency-group locking inserts and retries on a duplicate; the deduplicated enqueue rolls back to a savepoint and coalesces onto the concurrent winner even inside a joined Spring transaction (PostgreSQL cannot). `requestCronNudge` locks the task row (`FOR UPDATE`) so acceptance is atomic with the existence and enabled checks. +- **Statement-level rollback.** A failed statement does not abort an Oracle transaction, so every write unit joined to a caller-owned transaction runs behind a savepoint and rolls back to it on failure (`OracleJobStore.atomically`); otherwise a caller that catches the error and commits would commit the unit's earlier statements. Upserts are update-then-insert with an `ORA-00001` retry; concurrency-group locking inserts and retries on a duplicate; the deduplicated enqueue rolls back to a savepoint and coalesces onto the concurrent winner even inside a joined Spring transaction (PostgreSQL cannot). `requestCronNudge` locks the task row (`FOR UPDATE`) so acceptance is atomic with the existence and enabled checks. - **Migrations are resumable because Oracle commits DDL.** `OracleMigrationRunner` records progress per statement in `threadmill_schema_history` (`statements_applied`, `success`), resumes at the first unrecorded statement, and tolerates an "already exists" error only for that first resumed statement (its DDL may have committed before the crash). Migrators serialize on a `threadmill_schema_lock` row held on a **second** connection, because DDL on the migrating connection would commit and release it. Files use SQL*Plus `/` terminators so emitted SQL runs unchanged; leading comments are stripped from each executed statement. The single `V1__baseline.sql` already contains the full PostgreSQL V1–V11 end state; future Oracle changes ship as additive files. - **No remote wake channel.** `createRemoteWakeChannel` returns empty: `DBMS_ALERT.SIGNAL` locks the alert name until commit and would serialize enqueuing transactions, and AQ needs grants and infrastructure. Cross-node pickup falls back to `pollInterval`. - **Spring chooses between the JDBC stores from configuration, not a database round trip.** `JdbcStoreSelection` runs in conditions, before any bean exists: `threadmill.store.jdbc-type`, then the `spring.datasource.url` scheme, then the module present (PostgreSQL on a tie, preserving prior behaviour). The chosen store still verifies the server at startup. @@ -589,6 +589,8 @@ Every hard-won failure mode that has come up during development, and the test th | Oracle quarantine of an unreadable keyed member leaves the workflow hold (and the key) held | `OracleJobStoreRegressionTest.quarantinedKeyedWorkflowMemberReleasesTheWorkflowHold` + `unreadableBodyAtTheQueueHeadIsQuarantinedWithoutStallingTheQueue` | | Oracle migration crash between a committed DDL statement and its progress record wedges the schema; concurrent migrators race; edited or unknown migrations accepted | `OracleMigrationRunnerTest.interruptedMigrationResumesAndToleratesTheUnrecordedCommittedStatement` + `concurrentCleanMigrationsAreSerialized` + `migrateAndValidateRefuseAnEditedMigration` + `migrateAndValidateRefuseAnUnknownVersion` + `migrationCommitsWhenConnectionsDefaultToNonAutoCommit` + `emittedCleanInstallSqlAppliesToACleanSchemaAndValidates` | | Spring wires the wrong JDBC store when both modules are present, or Oracle `join_transaction` does not commit/roll back with the caller | `JdbcStoreSelectionTest` + `SpringOracleAutoConfigurationTest` + `SpringOracleTransactionBoundaryTest` | +| A failed Oracle write joined to a caller transaction leaves its earlier statements behind when the caller catches the error and commits (Oracle rolls back only the failing statement; PR #138 review) | `SpringOracleTransactionBoundaryTest.aFailedJoinedWriteLeavesNothingBehindWhenTheCallerCatchesAndCommits` | +| A `null` release holder frees a mutex held by an empty holder (Oracle stores `''` as `NULL`, `DECODE` matches two `NULL`s; PR #138 review) | `AbstractJobStoreContractTest.releaseWithANullHolderNeverFreesAnotherHoldersMutex` (all stores) | ### Postgres-layer improvements (engagement notes) From e8e5bedb016ec47a6c23af7a0ad87ed2f047fd25 Mon Sep 17 00:00:00 2001 From: Helmut Michael Juskewycz Date: Wed, 7 Oct 2026 07:27:08 +0200 Subject: [PATCH 7/7] build(store-oracle): lock the patched Jackson versions from main --- threadmill-store-oracle/gradle.lockfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/threadmill-store-oracle/gradle.lockfile b/threadmill-store-oracle/gradle.lockfile index 0b19d6be..093dd61a 100644 --- a/threadmill-store-oracle/gradle.lockfile +++ b/threadmill-store-oracle/gradle.lockfile @@ -2,10 +2,10 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.fasterxml.jackson.core:jackson-annotations:2.21=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson:jackson-bom:2.21.5=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.21.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.21.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.21.7=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-api:3.7.1=testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-transport-zerodep:3.7.1=testCompileClasspath,testRuntimeClasspath com.github.docker-java:docker-java-transport:3.7.1=testCompileClasspath,testRuntimeClasspath