Fix remote files upload issues

.changeset/public-dir-not-only-public-val.md

@@ -0,0 +1,5 @@
+---
+"@valbuild/core": patch
+---
+
+Allow storing files anywhere under `/public`, not only `/public/val`. Config validation, remote refs, and the studio file/image fields now accept any `/public/...` directory (the default remains `/public/val`). Also removed the `files` property from `SharedValConfig` — the per-schema `s.files`/`s.images` directory is now the source of truth.

.changeset/remote-gallery-upload-validation.md

@@ -0,0 +1,10 @@
+---
+"@valbuild/core": patch
+"@valbuild/server": patch
+"@valbuild/shared": patch
+"@valbuild/cli": patch
+---
+
+Remote galleries (`s.images({ remote: true })` / `s.files({ remote: true })`) now report a fixable validation error when an entry is still a local path, mirroring the single-field `s.image().remote()` / `s.file().remote()` behaviour. Running the CLI with `validate --fix` uploads the file to remote storage and rewrites the record key from the local path to the remote URL, keeping the file on disk. The gallery "untracked files" check now recognizes a remote-URL key as covering its on-disk file, so kept files are not flagged.
+
+Adds two new validation fixes: `images:upload-remote` and `files:upload-remote`.

.changeset/validation-error-code-frames.md

@@ -0,0 +1,8 @@
+---
+"@valbuild/server": patch
+"@valbuild/cli": patch
+---
+
+Validation errors from the CLI now point at the offending location in the `.val.ts` file. Each error shows a clickable `file:line:col` and a code frame (the line above, the offending line, and the line below) with carets underlining the exact source. The carets target the **value** by default and the **key** when the error is about an object/record key (`keyError`), and the output is labelled `(key)` / `(value)` accordingly. Gallery metadata errors now point at the specific record entry instead of the whole module.
+
+The module-path → source-range utility used for this (`createModulePathMap`, `getModulePathRange`, `ModulePathMap`) is now exported from `@valbuild/server`.

examples/next/app/blogs/[blog]/page.val.ts

@@ -3,7 +3,7 @@ import authorsVal from "../../../content/authors.val";
 import { linkSchema } from "../../../components/link.val";
 
 const blogSchema = s.object({
-  title: s.string().maxLength(3),
+  title: s.string(),
   content: s.richtext({
     inline: {
       a: s.route(),

examples/next/app/page.val.ts

@@ -65,7 +65,7 @@ export default c.define("/app/page.val.ts", s.router(nextAppRouter, schema), {
     link: "/blogs/blog1",
     hero: {
       title: "Content as code",
-      image: c.image("/public/val/logo_7adc7.png", {
+      image: c.image("/public/val/images/logo.png", {
         width: 944,
         height: 944,
         mimeType: "image/png",

examples/next/content/media.val.ts

@@ -9,8 +9,8 @@ export default c.define(
   }),
   {
     "/public/val/images/logo.png": {
-      width: 800,
-      height: 600,
+      width: 944,
+      height: 944,
       mimeType: "image/png",
       alt: "An example image",
     },

packages/cli/package.json

@@ -27,6 +27,7 @@
     "@valbuild/server": "workspace:*",
     "@valbuild/shared": "workspace:*",
     "chalk": "^5.6.2",
+    "chokidar": "^5.0.0",
     "cors": "^2.8.5",
     "fast-glob": "^3.3.3",
     "meow": "^9.0.0",

packages/cli/src/__fixtures__/basic/content/basic-gallery-remote-existing.val.ts

@@ -0,0 +1,20 @@
+import { c, s } from "../val.config";
+
+export default c.define(
+  "/content/basic-gallery-remote-existing.val.ts",
+  s.images({
+    directory: "/public/val/images-remote2",
+    accept: "image/*",
+    remote: true,
+  }),
+
+  {
+    "https://remote.val.build/file/p/proj123/b/01/v/1.0.0/h/abc1/f/def4/p/public/val/images-remote2/image.png":
+      {
+        width: 1,
+        height: 1,
+        mimeType: "image/png",
+        alt: null,
+      },
+  },
+);

packages/cli/src/__fixtures__/basic/content/basic-gallery-remote.val.ts

@@ -0,0 +1,19 @@
+import { c, s } from "../val.config";
+
+export default c.define(
+  "/content/basic-gallery-remote.val.ts",
+  s.images({
+    directory: "/public/val/images-remote",
+    accept: "image/*",
+    remote: true,
+  }),
+
+  {
+    "/public/val/images-remote/image.png": {
+      width: 1,
+      height: 1,
+      mimeType: "image/png",
+      alt: null,
+    },
+  },
+);

packages/cli/src/__fixtures__/basic/val.config.ts

@@ -1,5 +1,5 @@
 import { initVal } from "@valbuild/core";
 
-const { s, c } = initVal();
+const { s, c, config } = initVal();
 
-export { s, c };
+export { s, c, config };

packages/cli/src/__fixtures__/basic/val.modules.ts

@@ -0,0 +1,18 @@
+import { modules } from "@valbuild/core";
+import { config } from "./val.config";
+
+export default modules(config, [
+  { def: () => import("./content/basic-valid.val") },
+  { def: () => import("./content/basic-errors.val") },
+  { def: () => import("./content/basic-files.val") },
+  { def: () => import("./content/basic-image.val") },
+  { def: () => import("./content/basic-image-from-gallery.val") },
+  { def: () => import("./content/basic-image-from-galleries.val") },
+  { def: () => import("./content/basic-gallery.val") },
+  { def: () => import("./content/basic-gallery-2.val") },
+  { def: () => import("./content/basic-gallery-fail-on-non-unique-dir.val") },
+  { def: () => import("./content/basic-gallery-missing-tracked.val") },
+  { def: () => import("./content/basic-gallery-wrong-metadata.val") },
+  { def: () => import("./content/basic-gallery-remote.val") },
+  { def: () => import("./content/basic-gallery-remote-existing.val") },
+]);

packages/cli/src/cli.ts

@@ -28,6 +28,7 @@ async function main(): Promise<void> {
       Options:
         --root [root], -r [root] Set project root directory (default process.cwd())
         --fix  [fix]             Attempt to fix validation errors
+        --watch, -w              Re-validate on changes to val.config, val.modules and *.val files
 
 
       Command: login
@@ -43,7 +44,7 @@ async function main(): Promise<void> {
 
       Command: list-unused-files
       Description: EXPERIMENTAL.
-        List files that are in public/val but not in use by any Val module.
+        List files that are in the configured files directory (files.directory, default public/val) but not in use by any Val module.
         This is useful for cleaning up unused files.
       Options:
         --root [root], -r [root] Set project root directory (default process.cwd())
@@ -62,6 +63,10 @@ async function main(): Promise<void> {
         fix: {
           type: "boolean",
         },
+        watch: {
+          type: "boolean",
+          alias: "w",
+        },
         noEslint: {
           type: "boolean",
         },
@@ -112,9 +117,15 @@ async function main(): Promise<void> {
       if (flags.managedDir) {
         return error(`Command "validate" does not support --managedDir flag`);
       }
+      if (flags.watch && flags.fix) {
+        return error(
+          `Command "validate" does not support --watch together with --fix`,
+        );
+      }
       return validate({
         root: flags.root,
         fix: flags.fix,
+        watch: flags.watch,
       });
     default:
       return error(`Unknown command "${input.join(" ")}"`);

packages/cli/src/listUnusedFiles.ts

@@ -8,12 +8,22 @@ import {
 import { createService } from "@valbuild/server";
 import { glob } from "fast-glob";
 import path from "path";
+import { evalValConfigFile } from "./utils/evalValConfigFile";
 
 export async function listUnusedFiles({ root }: { root?: string }) {
-  const managedDir = "public/val";
   const projectRoot = root ? path.resolve(root) : process.cwd();
 
+  const valConfigFile =
+    (await evalValConfigFile(projectRoot, "val.config.ts")) ||
+    (await evalValConfigFile(projectRoot, "val.config.js"));
+  // Strip the leading "/" so it is relative to the project root (e.g. "public/val").
+  const managedDir = (valConfigFile?.files?.directory ?? "/public/val").replace(
+    /^\//,
+    "",
+  );
+
   const service = await createService(projectRoot, {});
+  const registered = new Set<ModuleFilePath>(service.getModuleFilePaths());
 
   const valFiles: string[] = await glob("**/*.val.{js,ts}", {
     ignore: ["node_modules/**"],
@@ -23,6 +33,10 @@ export async function listUnusedFiles({ root }: { root?: string }) {
   const filesUsedByVal: string[] = [];
   async function pushFilesUsedByVal(file: string) {
     const moduleId = `/${file}` as ModuleFilePath; // TODO: check if this always works? (Windows?)
+    if (!registered.has(moduleId)) {
+      // Not registered in val.modules - skip (e.g. reusable schema fragments).
+      return;
+    }
     const valModule = await service.get(moduleId, "" as ModulePath, {
       validate: true,
       source: true,

packages/cli/src/runValidation.test.ts

@@ -383,4 +383,105 @@ describe("runValidation", () => {
       service.dispose();
     }
   });
+
+  test("reports upload-remote error for local entry in a remote gallery", async () => {
+    const events: ValidationEvent[] = [];
+
+    for await (const event of runValidation({
+      root: tmpDir,
+      fix: false,
+      valFiles: ["content/basic-gallery-remote.val.ts"],
+      project: undefined,
+      remote: mockRemote,
+      fs: createDefaultValFSHost(),
+    })) {
+      events.push(event);
+    }
+
+    expect(events.at(-1)).toEqual({
+      type: "summary-errors",
+      count: expect.any(Number),
+    });
+    const errors = events.filter((e) => e.type === "validation-error");
+    expect(
+      errors.some(
+        (e) =>
+          "message" in e &&
+          (e.message as string).includes("needs to be uploaded"),
+      ),
+    ).toBe(true);
+  });
+
+  test("uploads local entry in a remote gallery and rewrites the key when fix is true", async () => {
+    const uploadRemote: IValRemote = {
+      remoteHost: DEFAULT_VAL_REMOTE_HOST,
+      getSettings: async () => ({
+        success: true,
+        data: {
+          publicProjectId: "pubproj",
+          remoteFileBuckets: [{ bucket: "01" }],
+        },
+      }),
+      uploadFile: async () => ({ success: true }),
+    };
+    // Upload requires a personal access token on disk.
+    fs.mkdirSync(path.join(tmpDir, ".val"), { recursive: true });
+    fs.writeFileSync(
+      path.join(tmpDir, ".val", "pat.json"),
+      JSON.stringify({ pat: "test-pat" }),
+    );
+
+    const events: ValidationEvent[] = [];
+    for await (const event of runValidation({
+      root: tmpDir,
+      fix: true,
+      valFiles: ["content/basic-gallery-remote.val.ts"],
+      project: "test/project",
+      remote: uploadRemote,
+      fs: createDefaultValFSHost(),
+    })) {
+      events.push(event);
+    }
+
+    expect(events.some((e) => e.type === "remote-uploaded")).toBe(true);
+    expect(events.some((e) => e.type === "fix-applied")).toBe(true);
+
+    const service = await createService(tmpDir, {}, createDefaultValFSHost());
+    try {
+      const result = await service.get(
+        "/content/basic-gallery-remote.val.ts" as ModuleFilePath,
+        "" as ModulePath,
+        { source: true, schema: true, validate: false },
+      );
+      const source = result.source as Record<string, unknown>;
+      // The local-path key is replaced by a remote URL key (file kept on disk).
+      expect(source).not.toHaveProperty("/public/val/images-remote/image.png");
+      const keys = Object.keys(source);
+      expect(keys).toHaveLength(1);
+      expect(keys[0]).toContain("pubproj");
+      expect(keys[0]).toContain("public/val/images-remote/image.png");
+    } finally {
+      service.dispose();
+    }
+  });
+
+  test("does not flag a kept-on-disk file behind a remote gallery key", async () => {
+    const events: ValidationEvent[] = [];
+
+    for await (const event of runValidation({
+      root: tmpDir,
+      fix: false,
+      valFiles: ["content/basic-gallery-remote-existing.val.ts"],
+      project: undefined,
+      remote: mockRemote,
+      fs: createDefaultValFSHost(),
+    })) {
+      events.push(event);
+    }
+
+    // The file kept on disk under the remote URL key must not be reported as
+    // untracked, and the remote URL key must not be reported as missing.
+    expect(events.at(-1)).toEqual({ type: "summary-success" });
+    expect(events.filter((e) => e.type === "validation-error")).toHaveLength(0);
+  });
 });