diff --git a/.gitignore b/.gitignore index cd832456..2814f190 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ log/ note/ +bench/ # IDE .idea/ @@ -32,6 +33,7 @@ project/plugins/project/ .history .cache .lib/ +*.jar ### SBT Patch ### .bsp/ @@ -221,4 +223,6 @@ poetry.toml .ruff_cache/ # LSP config files -pyrightconfig.json \ No newline at end of file +pyrightconfig.json + +tacit_config.json \ No newline at end of file diff --git a/build.sbt b/build.sbt index ba6f7d0b..018cb8ba 100644 --- a/build.sbt +++ b/build.sbt @@ -1,19 +1,20 @@ -val scala3Version = { - val fallback = "3.8.3-RC1" - try { - val url = "https://repo.scala-lang.org/artifactory/api/storage/local-maven-nightlies/org/scala-lang/scala3-compiler_3/" - val content = scala.io.Source.fromURL(url, "UTF-8").mkString - val pattern = """"uri"\s*:\s*"/(3\.[^"]*NIGHTLY)"""".r - val versions = pattern.findAllMatchIn(content).map(_.group(1)).toList.sorted - val latest = versions.last - if (latest != fallback) println(s"[info] Use Scala 3 nightly: $latest") - latest - } catch { case _: Exception => - println(s"[warn] Failed to fetch latest nightly, using fallback: $fallback") - fallback - } -} -// val scala3Version = "3.8.4-RC1-bin-SNAPSHOT" +// val scala3Version = { +// val fallback = "3.8.3-RC1" +// try { +// val url = "https://repo.scala-lang.org/artifactory/api/storage/local-maven-nightlies/org/scala-lang/scala3-compiler_3/" +// val content = scala.io.Source.fromURL(url, "UTF-8").mkString +// val pattern = """"uri"\s*:\s*"/(3\.[^"]*NIGHTLY)"""".r +// val versions = pattern.findAllMatchIn(content).map(_.group(1)).toList.sorted +// val latest = versions.last +// if (latest != fallback) println(s"[info] Use Scala 3 nightly: $latest") +// latest +// } catch { case _: Exception => +// println(s"[warn] Failed to fetch latest nightly, using fallback: $fallback") +// fallback +// } +// } +// Need the specific version with dynamic eval published locally +val scala3Version = "3.9.0-RC1-bin-SNAPSHOT" ThisBuild / resolvers += Resolver.scalaNightlyRepository val tacitVersion = "0.1.4-SNAPSHOT" @@ -35,10 +36,22 @@ lazy val lib = project Compile / unmanagedSources / excludeFilter := "*.test.scala" || "project.scala" || "README.md", libraryDependencies ++= Seq( - "com.openai" % "openai-java" % "4.32.0", + "com.openai" % "openai-java" % "4.35.0", "io.circe" %% "circe-core" % circeVersion, "io.circe" %% "circe-parser" % circeVersion, + "org.scala-lang" %% "scala3-compiler" % scala3Version, + "org.scala-lang" %% "scala3-repl" % scala3Version, ), + // Bundle Interface.scala source as a classpath resource so the agent + // prompt can include the full capability API surface. We place it under + // `tacit/` and use a non-`.scala` extension so the incremental compiler + // doesn't re-discover it as a duplicate source on the next build. + Compile / resourceGenerators += Def.task { + val src = baseDirectory.value / "Interface.scala" + val dst = (Compile / resourceManaged).value / "tacit" / "Interface.scala.txt" + IO.copyFile(src, dst) + Seq(dst) + }.taskValue, scalacOptions ++= Seq( "-language:experimental.captureChecking", "-language:experimental.modularity", diff --git a/library/GlobalIOCap.scala b/library/GlobalIOCap.scala new file mode 100644 index 00000000..b15c4fb7 --- /dev/null +++ b/library/GlobalIOCap.scala @@ -0,0 +1,8 @@ +package tacit.library +import language.experimental.captureChecking +import caps.* + +@assumeSafe +object GlobalIOCap extends IOCapability + // This object serves as the global singleton instance of IOCapability that + // library code can use to access file/network/process capabilities. \ No newline at end of file diff --git a/library/Interface.scala b/library/Interface.scala index aff68542..4466344f 100644 --- a/library/Interface.scala +++ b/library/Interface.scala @@ -1,5 +1,7 @@ package tacit.library +import dotty.tools.repl.eval.{Eval, EvalContext, EvalResult, evalLike, evalSafeLike} + import language.experimental.captureChecking import caps.* @@ -89,7 +91,7 @@ trait ProcessPermission extends caps.SharedCapability: /** Capability gating access to standard output (`println`, `print`, `printf`). * An implicit instance is available at the REPL top level. */ @assumeSafe -class IOCapability private extends caps.SharedCapability +class IOCapability private[library] extends caps.SharedCapability // ─── Interface ────────────────────────────────────────────────────────────── @@ -128,9 +130,7 @@ class IOCapability private extends caps.SharedCapability * ``` */ @assumeSafe -trait Interface extends SharedCapability: - - val iocap: IOCapability +trait Interface: // ── File System ───────────────────────────────────────────────────── @@ -239,8 +239,10 @@ trait Interface extends SharedCapability: /** Print to the standard output stream visible to the agent. * - * If the argument is a `Classified[_]`, only the masked form - * `Classified(***)` is written here — the actual content is never shown + * If you want to show results containing classified data to the end user, + * print the classified value in a separate print call so it can be captured by + * the secure output sink. When the argument is a `Classified[_]`, only the masked form + * `Classified(***)` is written here, and the actual content is never shown * to the agent. When the host has configured a secure output sink, the * unwrapped content is additionally written to that sink, which only * the end user can read. Non-classified arguments are printed normally @@ -274,3 +276,24 @@ trait Interface extends SharedCapability: * val summary: Classified[String] = chat(secret.map(q => s"Summarize the following: $q")) * ``` */ def chat(message: Classified[String]): Classified[String] + + /** Ask the configured trusted LLM to fill the call-site placeholder with a Scala + * expression of type `T`, then compile and run it under the live REPL. + * The synthetic parameters (`bindings`, `expectedType`, `enclosingSource`) + * are populated by the compiler at the call site. + * All functions in `Interface` are in scope at the call site and can be used + * in the generated code. + * + * ``` + * val n: Int = agent[Int]("answer to the ultimate question") + * ``` + */ + @evalLike def agent[T]( + prompt: String, + bindings: Array[Eval.Binding] = Array.empty[Eval.Binding], + expectedType: String = "", + enclosingSource: String = "", + maxAttempts: Int = 3 + ): T + +end Interface diff --git a/library/impl/InterfaceImpl.scala b/library/impl/InterfaceImpl.scala index 2f9ddf96..3e85d9bb 100644 --- a/library/impl/InterfaceImpl.scala +++ b/library/impl/InterfaceImpl.scala @@ -3,13 +3,21 @@ package tacit.library import language.experimental.captureChecking import caps.* +import dotty.tools.repl.eval.{Eval, evalLike} + import java.io.{File => JFile, PrintStream, FileOutputStream} @assumeSafe abstract class InterfaceImpl( - private val config: LibraryConfig = LibraryConfig() + configJson: String ) extends Interface: + private val config = LibraryConfig.fromJson(configJson) + + // create real FileSystem by default, but allow tests to override with a virtual one + protected def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = + new RealFileSystem(root, filter, classifiedPatterns) + private val DefaultClassifiedPatterns: Set[String] = Set( ".ssh", ".gnupg", ".env", ".env.*", ".netrc", ".npmrc", ".pypirc", ".docker", ".kube", ".aws", ".azure", ".gcloud", @@ -45,19 +53,13 @@ abstract class InterfaceImpl( case _: Classified[?] => "Classified(***)" case other => other - protected def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem - export FileOps.* export ProcessOps.* export WebOps.* - // IOCapability's private constructor means user code cannot create one. - // The null sentinel is safe: IOCapability is only used as a type-level - // capability witness, never dereferenced at runtime. - val iocap: IOCapability = null.asInstanceOf[IOCapability] - private val llmOps = LlmOps(llmConfig) - export llmOps.chat + + export llmOps.* def println(x: Any)(using IOCapability): Unit = // Classified.toString already returns "Classified(***)" so the diff --git a/library/impl/LlmOps.scala b/library/impl/LlmOps.scala index 0589b401..4122eac7 100644 --- a/library/impl/LlmOps.scala +++ b/library/impl/LlmOps.scala @@ -1,8 +1,10 @@ package tacit.library import com.openai.client.OpenAIClient +import com.openai.models.ReasoningEffort import com.openai.client.okhttp.OpenAIOkHttpClient import com.openai.models.chat.completions.ChatCompletionCreateParams +import dotty.tools.repl.eval.{Eval, EvalContext, EvalResult, evalLike} class LlmOps(config: Option[LlmConfig]): @@ -24,6 +26,7 @@ class LlmOps(config: Option[LlmConfig]): val params = ChatCompletionCreateParams.builder() .model(cfg.model) .addUserMessage(message) + .reasoningEffort(ReasoningEffort.HIGH) .build() client.chat().completions().create(params) .choices().get(0).message().content().orElse("").nn @@ -34,3 +37,125 @@ class LlmOps(config: Option[LlmConfig]): def chat(message: Classified[String]): Classified[String] = requireConfig() message.map(chat) + + /** Inline-style LLM agent. Asks the LLM to fill the call-site placeholder + * with a Scala expression of the requested type, compiles and runs it + * under the live REPL, and retries with the diagnostic text on compile + * failure. The synthetic parameters (`bindings`, `expectedType`, + * `enclosingSource`) are populated by the eval rewriter at each call + * site; direct callers can leave them at their defaults. */ + @evalLike def agent[T]( + prompt: String, + bindings: Array[Eval.Binding] = Array.empty[Eval.Binding], + expectedType: String = "", + enclosingSource: String = "", + maxAttempts: Int = 3 + ): T = + requireConfig() + + @annotation.tailrec + def attempt(n: Int, prevCode: String, prevErrors: List[String]): EvalResult[T] = + val request = AgentPrompt.build( + prompt, bindings, expectedType, enclosingSource, prevCode, prevErrors) + val code = AgentPrompt.stripCodeFences(chat(request)).trim + val r = Eval.evalSafe[T](code, bindings, expectedType, enclosingSource) + if r.isSuccess || n >= maxAttempts then r + else attempt(n + 1, code, r.error.nn.errors.toList) + + attempt(1, "", Nil).get + +private object AgentPrompt: + + private val Intro = + """You are an inline Scala 3 code generator for the live REPL. + |Output ONLY a Scala expression that fills the placeholder in the user's source. + |No markdown fences, no commentary.""".stripMargin + + /** The full `Interface.scala` source, bundled as a classpath resource by the + * build, so the LLM sees the exact capability API surface available at the + * REPL call site. */ + private lazy val InterfaceReference: String = + val stream = classOf[LlmOps].getResourceAsStream("/tacit/Interface.scala.txt") + if stream != null then + try scala.io.Source.fromInputStream(stream)(using scala.io.Codec.UTF8).mkString + finally stream.close() + else "(Interface.scala source not found on classpath)" + + private val InterfaceIntro = + """You may only interact with the system through the capability-scoped API + |defined below. Do not use Java/Scala standard library APIs (java.io, + |java.nio, scala.io, sys.process, java.net, etc.) directly. All members + |of the `Interface` trait are pre-loaded into scope (via `import api.*`), + |so call them unqualified — e.g. `chat("...")`, `requestFileSystem(...)`. + |""".stripMargin + + private def interfaceSection: String = + s"""$InterfaceIntro + |```scala + |$InterfaceReference + |```""".stripMargin + + def build( + task: String, + bindings: Array[Eval.Binding], + expectedType: String, + enclosingSource: String, + prevCode: String, + prevErrors: List[String] + ): String = + List( + Intro, + interfaceSection, + typeSection(expectedType), + contextSection(enclosingSource), + bindingsSection(bindings), + s"Task: $task", + typeReminder(expectedType), + errorSection(prevCode, prevErrors) + ).filter(_.nonEmpty).mkString("\n\n") + + private def typeSection(expectedType: String): String = + if expectedType.nonEmpty then + s"""REQUIRED type of your expression: $expectedType + |Your expression MUST type-check at this exact type. Any mismatch + |will be reported as a compile error and you will be asked to retry.""".stripMargin + else + "Type of your expression: not pinned at the call site." + + private def contextSection(enclosingSource: String): String = + if enclosingSource.isEmpty then "" + else + s"""Placeholder marker: ${EvalContext.placeholder} + |Enclosing source: + |$enclosingSource""".stripMargin + + private def bindingsSection(bindings: Array[Eval.Binding]): String = + if bindings.isEmpty then "" + else s"In-scope bindings: ${bindings.iterator.map(_.name).mkString(", ")}" + + private def typeReminder(expectedType: String): String = + if expectedType.isEmpty then "" + else s"Reminder: the expression you produce must have type `$expectedType`." + + private def errorSection(prevCode: String, prevErrors: List[String]): String = + if prevErrors.isEmpty then "" + else + s"""Previous attempt: + |$prevCode + | + |Previous attempt failed with: + |${prevErrors.mkString("\n")} + | + |Either fix the cause and emit a corrected expression, or — if the + |failure looks unrecoverable — emit code that throws a descriptive + |exception so the human user sees a clear cause. Output ONLY the + |expression.""".stripMargin + + /** Strip ``` ... ``` fences around an LLM response if present. */ + def stripCodeFences(s: String): String = + val t = s.trim + if t.startsWith("```") then + t.stripPrefix("```scala").stripPrefix("```").stripPrefix("\n").stripSuffix("```").trim + else t + +end AgentPrompt diff --git a/library/impl/RealFileSystem.scala b/library/impl/RealFileSystem.scala index 2c4d7fc7..755e1dc9 100644 --- a/library/impl/RealFileSystem.scala +++ b/library/impl/RealFileSystem.scala @@ -11,12 +11,12 @@ import java.nio.file.{Files, FileVisitResult, Path, Paths, SimpleFileVisitor} import java.nio.file.attribute.BasicFileAttributes class RealFileSystem( - val root: Path, + val root: String, check: String -> Boolean = _ => true, protected val classifiedPatterns: Set[String] = Set.empty ) extends BaseFileSystem: protected val normalizedRoot: Path = - val abs = root.toAbsolutePath.normalize + val abs = Paths.get(root).toAbsolutePath.normalize if Files.exists(abs) then abs.toRealPath() else abs protected def pathCheck(relativePath: String): Boolean = check(relativePath) diff --git a/library/impl/VirtualFileSystem.scala b/library/impl/VirtualFileSystem.scala index d91b567a..60fa795c 100644 --- a/library/impl/VirtualFileSystem.scala +++ b/library/impl/VirtualFileSystem.scala @@ -8,12 +8,12 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Path, Paths} class VirtualFileSystem( - val root: Path, + val root: String, check: String -> Boolean = _ => true, initialFiles: Map[String, String] = Map.empty, protected val classifiedPatterns: Set[String] = Set.empty ) extends BaseFileSystem: - protected val normalizedRoot: Path = root.toAbsolutePath.normalize + protected val normalizedRoot: Path = Paths.get(root).toAbsolutePath.normalize protected def pathCheck(relativePath: String): Boolean = check(relativePath) private val files: TrieMap[Path, Array[Byte]] = TrieMap.empty private val directories: TrieMap[Path, Unit] = TrieMap(normalizedRoot -> ()) diff --git a/library/test/ClassifiedSuite.test.scala b/library/test/ClassifiedSuite.test.scala index f75689d0..eca08a36 100644 --- a/library/test/ClassifiedSuite.test.scala +++ b/library/test/ClassifiedSuite.test.scala @@ -67,7 +67,7 @@ class ClassifiedSuite extends munit.FunSuite: LibraryConfig(strictMode = Some(false), classifiedPaths = Some(Set("secret"))) ) { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new VirtualFileSystem(Path.of(root), filter, classifiedPatterns = classifiedPatterns) + new VirtualFileSystem(root, filter, classifiedPatterns = classifiedPatterns) }.unsafeAssumePure import interface.* @@ -239,7 +239,7 @@ class ClassifiedSuite extends munit.FunSuite: LibraryConfig(strictMode = Some(false), classifiedPaths = Some(patterns)) ) { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new VirtualFileSystem(Path.of(root), filter, classifiedPatterns = classifiedPatterns) + new VirtualFileSystem(root, filter, classifiedPatterns = classifiedPatterns) }.unsafeAssumePure test("pattern without slash matches any component") { diff --git a/library/test/LibrarySuite.test.scala b/library/test/LibrarySuite.test.scala index 271ca4c7..1976c0f2 100644 --- a/library/test/LibrarySuite.test.scala +++ b/library/test/LibrarySuite.test.scala @@ -17,7 +17,7 @@ class LibrarySuite extends munit.FunSuite: private val interface: Interface^{} = new InterfaceImpl() { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new RealFileSystem(Path.of(root), filter, classifiedPatterns) + new RealFileSystem(root, filter, classifiedPatterns) }.unsafeAssumePure import interface.* @@ -174,7 +174,7 @@ class LibrarySuite extends munit.FunSuite: LibraryConfig(strictMode = Some(false), classifiedPaths = Some(Set("secret"))) ) { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new RealFileSystem(Path.of(root), filter, classifiedPatterns) + new RealFileSystem(root, filter, classifiedPatterns) } import classifiedInterface.* @@ -219,7 +219,7 @@ class LibrarySuite extends munit.FunSuite: LibraryConfig(secureOutput = Some(secureFile.toString)) ) { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new RealFileSystem(Path.of(root), filter, classifiedPatterns) + new RealFileSystem(root, filter, classifiedPatterns) } given (IOCapability^{}) = secureInterface.iocap.unsafeAssumePure diff --git a/library/test/VirtualEnvSuite.test.scala b/library/test/VirtualEnvSuite.test.scala index 17e496a8..56e5f8b2 100644 --- a/library/test/VirtualEnvSuite.test.scala +++ b/library/test/VirtualEnvSuite.test.scala @@ -10,7 +10,7 @@ class VirtualEnvSuite extends munit.FunSuite: val interface: Interface^{} = new InterfaceImpl() { def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - new VirtualFileSystem(Path.of(root), filter, classifiedPatterns = classifiedPatterns) + new VirtualFileSystem(root, filter, classifiedPatterns = classifiedPatterns) }.unsafeAssumePure import interface.* diff --git a/src/main/scala/executor/ManagedRepl.scala b/src/main/scala/executor/ManagedRepl.scala index 7743882e..4fa08d55 100644 --- a/src/main/scala/executor/ManagedRepl.scala +++ b/src/main/scala/executor/ManagedRepl.scala @@ -36,6 +36,12 @@ object ManagedRepl: */ private def replClasspathArgs(using Context): Array[String] = val classpath = JFile(ctx.config.libraryJarPath).getAbsolutePath + + val langaugeFeatures = List( + "experimental.captureChecking", + "experimental.modularity", + ) ++ (if ctx.config.safeMode then List("experimental.safe") else Nil) + Array( "-classpath", classpath, "-color:never", @@ -45,8 +51,10 @@ object ManagedRepl: "-Yexplicit-nulls", "-Ycheck-all-patmat", "-Wsafe-init", - "-language:experimental.captureChecking", - "-language:experimental.modularity" + s"-language:${langaugeFeatures.mkString(",")}", + s"-Xrepl-eval-log-dir:${ctx.config.recordPath.getOrElse("./log")}/eval", + "-release:17", + // "-Vprint:cc" ) /** Exposes only JDK platform classes and the library JAR, keeping user code @@ -66,13 +74,15 @@ object ManagedRepl: .replace("\\", "\\\\") .replace("\"", "\\\"") s"""|import tacit.library.* + |import dotty.tools.repl.eval.{Eval, EvalContext, EvalResult, evalLike, evalSafeLike} |import caps.* - |@assumeSafe object api extends InterfaceImpl(LibraryConfig.fromJson("$jsonStr")) { - | def createFS(root: String, filter: String -> Boolean, classifiedPatterns: Set[String]): FileSystem = - | new RealFileSystem(java.nio.file.Path.of(root), filter, classifiedPatterns) - |} + |object api extends InterfaceImpl("$jsonStr") |import api.* - |@assumeSafe given IOCapability = iocap + |""".stripMargin + + // We need a separate preamble for REPL, so the first repl object will be pure. + private[executor] def libraryPreambleTracked(using Context): String = + s"""|given IOCapability = GlobalIOCap |""".stripMargin /** We swap `System.out`/`System.err` around each execution to catch output the @@ -122,9 +132,12 @@ class ManagedRepl(using Context): * preamble is a programmer bug that should surface loudly. */ def init(): this.type = + // val (output, thrown) = withOutputCapture(outputCapture, printStream): state = driver.run(libraryPreamble)(using state) - if ctx.config.safeMode then - state = driver.run("import language.experimental.safe")(using state) + state = driver.run(libraryPreambleTracked)(using state) + // For debugging preamble issues + // println(output) + // thrown.foreach(e => println(s"Preamble error: ${e.getMessage}")) this /** Execute `code` against the current state. diff --git a/src/test/scala/LibraryIntegrationSuite.scala b/src/test/scala/LibraryIntegrationSuite.scala index ff4ebb62..fb805e9e 100644 --- a/src/test/scala/LibraryIntegrationSuite.scala +++ b/src/test/scala/LibraryIntegrationSuite.scala @@ -342,7 +342,7 @@ class LibraryIntegrationSuite extends munit.FunSuite: // overrides that and allows it. val cfg = Config(libraryConfig = io.circe.Json.obj( "strictMode" -> io.circe.Json.fromBoolean(true), - "commandPermissions" -> io.circe.Json.arr(io.circe.Json.fromString("cat")) + "commandPermissions" -> io.circe.Json.arr(io.circe.Json.fromString("cat *")) )) given Context = Context(cfg, None) val result = ScalaExecutor.execute(""" @@ -547,3 +547,341 @@ class LibraryIntegrationSuite extends munit.FunSuite: """exec("echo", List("hi"))""", "no given instance of type tacit.library.ProcessPermission" ) + + // ── eval (runtime code synthesis) inside Classified.map ───────────── + // + // `Classified.map` requires a pure function `T ->{any.rd} B`. When the + // body of that lambda is `eval[B]("...")`, the outer compile only sees + // `eval[B]: B` and cannot peek inside the string. The inner compile is + // triggered when the eval call runs: it splices the body back into the + // surrounding source and re-typechecks under cc + safe mode. A body + // that captures iocap or another capability is rejected then — + // surfacing as a `RuntimeException("eval failed to compile: ...")` + // thrown from the eval call. + // + // That exception is *swallowed* by `Classified.map`'s `Try` wrapper + // (the result becomes a `Classified[Failure]`), so we cannot observe + // the rejection from stdout directly. Instead we route the failed + // `Classified` through the `secureOutput` sink: `unwrapForSecure` + // unwraps a `Failure` to the literal `">"`, + // putting the original `RuntimeException` message — including the + // inner cc diagnostic — into the secure file. A successful (i.e. + // unrejected) body would instead leak the *raw* secret to that sink, + // so the secure file is what tells us whether the inner compile + // actually rejected the body. + + /** Run `code` under a context whose secure output mirrors classified + * prints to a temp file, then return both streams together with the + * `ExecutionResult`. The temp file is deleted before returning. */ + private def runWithSecureSink(code: String): (tacit.executor.ExecutionResult, String) = + val secureFile = Files.createTempFile("eval-secure-", ".log").nn + val cfg = Config(libraryConfig = io.circe.Json.obj( + "secureOutput" -> io.circe.Json.fromString(secureFile.toString) + )) + given Context = Context(cfg, None) + try + val result = ScalaExecutor.execute(code) + val secureContent = Files.readString(secureFile).nn + (result, secureContent) + finally Files.deleteIfExists(secureFile) + + test("eval pure body inside Classified.map: transform actually runs"): + // A pure body passes cc through the eval inner compile. Beyond just + // "no error", we verify the transform actually runs by routing the + // result through the secureOutput sink, where `unwrapForSecure` + // exposes the unwrapped (transformed) value. Main stdout still + // sees only the masked form, never the original or transformed + // secret content. This also doubles as the positive control for + // the secureOutput observation pattern that the negative tests + // below rely on. + val (result, secureContent) = runWithSecureSink(""" + val secret = classify("payload-marker-OK") + val mapped = secret.map(x => eval[String]("x.reverse + \"-transformed\"")) + println(mapped) + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("Classified(***)"), + s"main stream should show masked form: ${result.output}") + assert(!result.output.contains("payload-marker-OK"), + s"main stream leaked original secret: ${result.output}") + assert(!result.output.contains("KO-rekram-daolyap"), + s"main stream leaked transformed secret: ${result.output}") + // Behaviorally verify the transform: reverse + suffix + assert(secureContent.contains("KO-rekram-daolyap-transformed"), + s"secure sink should have the reversed-and-suffixed value: $secureContent") + assert(!secureContent.contains("classified error"), + s"successful body should not produce a Failure marker: $secureContent") + + test("agent inside Classified.map compiles (@evalLike body accepted by outer cc)"): + // `agent[Int]` is `@evalLike`, so the outer compile treats it like + // `eval` — a pure call — and accepts it inside `Classified.map`'s + // `T ->{any.rd} B` lambda. No LLM is configured here, so the call + // throws at runtime, but `Classified.map`'s `Try` wrapper turns that + // into a `Classified[Failure]`; the REPL run still succeeds and the + // print shows only the masked form. + val result = ScalaExecutor.execute(""" + val ss = classify("secret") + println(ss.map(s => agent[Int]("length of s"))) + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("Classified(***)"), s"unexpected output: ${result.output}") + + test("agent inside Classified.map of a readClassified result compiles"): + // Same shape as above, but the classified value comes from the file + // system: inside `requestFileSystem`, `readClassified` (on a path + // matched by `classifiedPaths`) yields a `Classified[String]` whose + // `.map` lambda calls `agent`. The `@evalLike` agent call is still + // accepted by the outer cc — that's all this test checks; it doesn't + // exercise the agent call's behavior. With no LLM the run yields a + // `Classified[Failure]` and the print stays masked. + val tmp = Files.createTempDirectory("agent-fs-").nn + val secretFile = tmp.resolve("secret.txt") + Files.writeString(secretFile, "hunter2-marker") + val cfg = Config(libraryConfig = io.circe.Json.obj( + "classifiedPaths" -> io.circe.Json.arr(io.circe.Json.fromString("secret.txt")) + )) + given Context = Context(cfg, None) + val result = ScalaExecutor.execute(s""" + val root = "${tmp}" + val path = "${secretFile}" + requestFileSystem(root) { + val c = readClassified(path) + println(c.map(s => agent[Int]("length of s"))) + } + """) + try + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("Classified(***)"), s"unexpected output: ${result.output}") + assert(!result.output.contains("hunter2-marker"), s"secret leaked: ${result.output}") + finally + Files.deleteIfExists(secretFile) + Files.deleteIfExists(tmp) + + test("eval body using println (captures iocap) inside Classified.map is rejected"): + // The body would call api.println (the iocap-capturing one shadowed + // into scope by the preamble), so the inner compile must reject the + // splice. We verify three things: + // 1. main stdout never sees the raw secret + // 2. the secure sink reports a `` rather than the secret + // 3. that error mentions a capture-set complaint (the cc + // diagnostic surfaces verbatim through the eval exception) + val (result, secureContent) = runWithSecureSink(""" + val secret = classify("LEAK-MARKER-IOCAP-001") + val mapped = secret.map(x => eval[String]("println(x); x")) + println(mapped) + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(!result.output.contains("LEAK-MARKER-IOCAP-001"), + s"secret leaked to main stdout — eval body actually ran! ${result.output}") + assert(!secureContent.contains("LEAK-MARKER-IOCAP-001"), + s"secret leaked to secure sink — eval body actually ran! $secureContent") + assert(secureContent.contains("classified error"), + s"expected Classified to wrap a Failure, secure sink: $secureContent") + assert(secureContent.contains("eval failed to compile"), + s"expected eval inner-compile failure surfaced via secure sink: $secureContent") + val lc = secureContent.toLowerCase + assert(lc.contains("iocap") || lc.contains("capture"), + s"expected cc diagnostic about iocap/capture in secure sink: $secureContent") + + test("eval body writing file via FileSystem inside Classified.map is rejected at runtime"): + // The outer compile only sees `eval[String]: String` — it cannot + // peek inside the body string, so `secret.map(s => eval[String](...))` + // type-checks even though the body would capture the `FileSystem` + // capability that `requestFileSystem` introduced. The violation + // surfaces when the eval inner-compile runs: splicing the body back + // into the enclosing source re-typechecks it under cc + safe mode, + // `Classified.map`'s `any.rd` constraint rejects the captured + // capability, and `eval` throws `RuntimeException("eval failed to + // compile: ...")`. `Classified.map`'s `Try` swallows that into a + // `Classified[Failure]`, observed via the secure sink. The body + // never runs, so the file is never written. + val sentinel = Files.createTempDirectory("eval-fs-leak-").nn + val target = sentinel.resolve("leaked.txt") + val (result, secureContent) = runWithSecureSink(s""" + val sentinelPath = "${sentinel}" + val targetPath = "${target}" + val secret = classify("LEAK-MARKER-FS-002") + val mapped = requestFileSystem(sentinelPath) { + secret.map(s => eval[String]("access(targetPath).write(s); s")) + } + println(mapped) + """) + try + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(!Files.exists(target), s"eval body executed and wrote $target") + assert(!result.output.contains("LEAK-MARKER-FS-002"), + s"secret leaked to main stdout: ${result.output}") + assert(!secureContent.contains("LEAK-MARKER-FS-002"), + s"secret leaked to secure sink: $secureContent") + assert(secureContent.contains("eval failed to compile"), + s"expected eval inner-compile failure surfaced via secure sink: $secureContent") + val lc = secureContent.toLowerCase + assert(lc.contains("capture") || lc.contains("contextual"), + s"expected cc diagnostic about capture/contextual in secure sink: $secureContent") + finally + Files.deleteIfExists(target) + Files.deleteIfExists(sentinel) + + test("eval body calling requestFileSystem inside Classified.map is rejected"): + // `requestFileSystem` itself requires `iocap`, so capturing it from + // inside `Classified.map`'s pure lambda also violates cc — same + // shape as the println case but routed through the FS-request entry + // point. The eval body never runs; the file is never created. + val sentinel = Files.createTempDirectory("eval-iocap-leak-").nn + val target = sentinel.resolve("leaked.txt") + val (result, secureContent) = runWithSecureSink(s""" + val sentinelPath = "${sentinel}" + val targetPath = "${target}" + val secret = classify("LEAK-MARKER-IOCAP-003") + val mapped = secret.map { content => + eval[String]("requestFileSystem(sentinelPath) { access(targetPath).write(content); content }") + } + println(mapped) + """) + try + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(!Files.exists(target), s"eval body executed and wrote $target") + assert(!result.output.contains("LEAK-MARKER-IOCAP-003"), + s"secret leaked: ${result.output}") + assert(!secureContent.contains("LEAK-MARKER-IOCAP-003"), + s"secret leaked to secure sink: $secureContent") + assert(secureContent.contains("eval failed to compile"), + s"expected inner-compile failure in secure sink: $secureContent") + val lc = secureContent.toLowerCase + assert(lc.contains("iocap") || lc.contains("capture"), + s"expected cc diagnostic about iocap/capture: $secureContent") + finally + Files.deleteIfExists(target) + Files.deleteIfExists(sentinel) + + // ── eval composed with Interface functions (no Classified.map) ────── + // + // Outside the `any.rd` straitjacket of `Classified.map`, eval is just + // a runtime "compile this string in the surrounding lexical context" + // primitive. These tests exercise it together with the rest of the + // capability API, to confirm the inner compile picks up REPL bindings, + // contextual capabilities, and type pinning correctly — and that + // values cross the eval/REPL classloader boundary safely. All these + // tests run with the default Context (safe mode on). + + test("eval at top level computes a typed value"): + val result = ScalaExecutor.execute(""" + val n = 7 + eval[Int]("n * n + 1") + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("50"), s"expected 50 in output: ${result.output}") + + test("eval body resolves REPL-defined def by name"): + // Identifiers introduced by a previous statement (or earlier in the + // same block) resolve inside the body via the bindings array filled + // by the rewriter. + val result = ScalaExecutor.execute(""" + def square(j: Int) = j * j + val xs = List(2, 3, 4) + xs.map(x => eval[Int]("square(x)")) + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("4") && result.output.contains("9") && result.output.contains("16"), + s"expected List(4, 9, 16): ${result.output}") + + test("eval body inside requestFileSystem can use FileSystem capability"): + // Inside `requestFileSystem`, a `FileSystem` capability is in scope + // (a contextual function parameter). The eval body inherits that + // scope through `enclosingSource`, so `access(...).write(...)` + // resolves directly inside the body. Path and payload are bound + // to REPL `val`s so the body has no nested string-escape gymnastics. + val tmp = Files.createTempDirectory("eval-fs-ok-").nn + val file = tmp.resolve("hello.txt") + val result = ScalaExecutor.execute(s""" + val root = "${tmp}" + val target = "${file}" + val payload = "eval wrote me" + requestFileSystem(root) { + eval[String]("access(target).write(payload); access(target).read()") + } + """) + try + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("eval wrote me"), + s"expected file content in output: ${result.output}") + assert(Files.exists(file), s"file should have been written") + assertEquals(Files.readString(file).nn, "eval wrote me") + finally + Files.deleteIfExists(file) + Files.deleteIfExists(tmp) + + test("eval body inside requestExecPermission can call exec"): + // The eval body directly calls `exec`, picking up the + // `ProcessPermission` from the surrounding scope. + val result = ScalaExecutor.execute(""" + val cmd = "echo" + val arg = "eval-and-exec" + requestExecPermission(Set(cmd)) { + eval[String]("exec(cmd, List(arg)).stdout.trim") + } + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + assert(result.output.contains("eval-and-exec"), + s"expected echo output: ${result.output}") + + test("eval body composing grepRecursive through FileSystem capability"): + // Body calls grepRecursive directly and projects across user + // library types (`GrepMatch`) — exercises both contextual + // capability access and cross-classloader value handling. + val tmp = Files.createTempDirectory("eval-grep-").nn + val a = tmp.resolve("a.scala") + val b = tmp.resolve("b.scala") + val c = tmp.resolve("c.txt") + Files.writeString(a, "val needle = 1\n") + Files.writeString(b, "val something = 2\n") // no match + Files.writeString(c, "val needle = 3\n") // .txt — wrong glob + val result = ScalaExecutor.execute(s""" + val root = "${tmp}" + val pattern = "needle" + val glob = "*.scala" + requestFileSystem(root) { + eval[List[String]]("grepRecursive(root, pattern, glob).map(_.file).distinct.sorted") + } + """) + try + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + // Behavior: only a.scala matches both the pattern and the glob. + assert(result.output.contains("a.scala"), + s"a.scala (matching file) missing from output: ${result.output}") + assert(!result.output.contains("b.scala"), + s"b.scala (no match) should not appear: ${result.output}") + assert(!result.output.contains("c.txt"), + s"c.txt (wrong glob) should not appear: ${result.output}") + finally + Files.deleteIfExists(a); Files.deleteIfExists(b); Files.deleteIfExists(c) + Files.deleteIfExists(tmp) + + test("eval body type mismatch surfaces as runtime exception"): + // The expected type is pinned by `eval[Int]`, but the body returns + // a String. The inner compile rejects this at runtime — and unlike + // the Classified.map cases, there's no `Try` to swallow it, so the + // RuntimeException reaches the REPL and shows up in `output`. + // Bind the body string to a val to avoid escape gymnastics. + val result = ScalaExecutor.execute(""" + val body = "\"not an int\"" + eval[Int](body) + """) + val combined = (result.output + result.error.getOrElse("")).toLowerCase + assert(combined.contains("eval failed to compile") || combined.contains("evalcompile"), + s"expected eval inner-compile failure, got: ${result.output} / err=${result.error}") + assert(combined.contains("found") && combined.contains("required"), + s"expected Found/Required diagnostic: ${result.output}") + + test("evalSafe returns a Failure for an ill-typed body"): + // Non-throwing flavor: the diagnostic is carried as data instead. + val result = ScalaExecutor.execute(""" + val body = "\"still a string\"" + val r = Eval.evalSafe[Int](body) + r.isSuccess + """) + assert(result.success, s"execution failed: ${result.error.getOrElse(result.output)}") + // Behavior: evalSafe should report the body did not type-check. + assert(result.output.contains("false"), + s"expected EvalResult.isSuccess == false: ${result.output}") diff --git a/src/test/scala/ScalaExecutorSuite.scala b/src/test/scala/ScalaExecutorSuite.scala index ae5b7a2c..40831be3 100644 --- a/src/test/scala/ScalaExecutorSuite.scala +++ b/src/test/scala/ScalaExecutorSuite.scala @@ -119,7 +119,7 @@ class ScalaExecutorSuite extends munit.FunSuite: test("code producing large output completes"): val result = ScalaExecutor.execute("(1 to 500).toList") assert(result.success) - assert(result.output.contains("List(1, 2, 3")) + assert(result.output.replace(" ", "").replace("\n", "").contains("List(1,2,3")) test("pattern matching expression"): val result = ScalaExecutor.execute("""