Download available turns to save folders

app/src/preload.js

@@ -10,26 +10,25 @@ import * as chokidar from "chokidar";
 import * as AutoLaunch from "auto-launch";
 import { RPC_INVOKE, RPC_TO_MAIN, RPC_TO_RENDERER } from "./rpcChannels.js";
 
-let watcher;
+const watchers = new Set();
 
 electron.contextBridge.exposeInMainWorld("pydtApi", {
   startChokidar: arg =>
     new Promise(resolve => {
-      if (watcher) {
-        watcher.close();
-      }
-
-      watcher = chokidar.watch(arg.path, {
+      const watcher = chokidar.watch(arg.path, {
         depth: 0,
         ignoreInitial: true,
         awaitWriteFinish: arg.awaitWriteFinish,
         usePolling: arg.awaitWriteFinish,
       });
+      watchers.add(watcher);
+      electron.ipcRenderer.send(RPC_TO_MAIN.LOG_INFO, `Watching save directory: ${arg.path}`);
 
       const changeDetected = p => {
+        electron.ipcRenderer.send(RPC_TO_MAIN.LOG_INFO, `Save change detected: ${p}`);
         electron.ipcRenderer.send(RPC_TO_MAIN.SHOW_WINDOW);
         watcher.close();
-        watcher = null;
+        watchers.delete(watcher);
         resolve(p);
       };
 

ui/app.component.html

@@ -89,6 +89,18 @@ <h5 class="modal-title">Settings</h5>
             />
             <label class="form-check-label" for="autoDownload">Automatically download turns in background</label>
           </div>
+          <div class="form-check">
+            <input
+              class="form-check-input"
+              type="checkbox"
+              name="saveDownloadedTurns"
+              [(ngModel)]="settings.saveDownloadedTurns"
+              id="saveDownloadedTurns"
+            />
+            <label class="form-check-label" for="saveDownloadedTurns">
+              Queue turns in game save folders and automatically upload completed saves
+            </label>
+          </div>
           <div class="mt-2" [ngClass]="{ 'has-error': numSaves.errors }">
             <span>Archive</span>
             <input

ui/home/home.component.ts

@@ -54,6 +54,7 @@ export class HomeComponent implements OnInit, OnDestroy {
   private sortedTurns: GameWithYourTurn[];
   private yourTurns: GameWithYourTurn[];
   private navigationSubscription: Subscription;
+  private completedGameSubscription: Subscription;
 
   ngOnInit(): void {
     void this.init();
@@ -89,6 +90,14 @@ export class HomeComponent implements OnInit, OnDestroy {
         this.updateAvailable = !!version;
       });
 
+      this.completedGameSubscription = this.turnCacheService.completedGameIds$.subscribe(gameId => {
+        this.games = this.games?.filter(game => game.gameId !== gameId) || [];
+        this.setSortedTurns();
+        this.notifyMain();
+        this.pollUrl = "";
+        void this.safeLoadGames();
+      });
+
       this.navigationSubscription = this.router.events.subscribe((e: unknown) => {
         // If reloading page reload user and games (could be changing user)
         if (e instanceof NavigationEnd) {
@@ -122,6 +131,11 @@ export class HomeComponent implements OnInit, OnDestroy {
       this.updateSub = null;
     }
 
+    if (this.completedGameSubscription) {
+      this.completedGameSubscription.unsubscribe();
+      this.completedGameSubscription = null;
+    }
+
     this.destroyed = true;
 
     window.pydtApi.ipc.removeAllListeners(RPC_TO_RENDERER.IOT_CONNECT);

ui/shared/autoDownloadSave.test.ts

@@ -0,0 +1,36 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { autoDownloadFileName, saveDownloadedTurn } from "./autoDownloadSave.ts";
+
+void test("uses the filename expected by the existing play-turn workflow", () => {
+  assert.equal(autoDownloadFileName("Civ7Save"), "(PYDT) Play This One!.Civ7Save");
+});
+
+void test("creates the configured directory and writes the downloaded data", () => {
+  const calls: Array<{ operation: string; value: unknown }> = [];
+  const data = new Uint8Array([1, 2, 3]);
+
+  const result = saveDownloadedTurn({
+    saveDir: "/games/saves",
+    saveExtension: "Civ6Save",
+    data,
+    fs: {
+      existsSync: path => {
+        calls.push({ operation: "exists", value: path });
+        return false;
+      },
+      mkdirp: path => calls.push({ operation: "mkdirp", value: path }),
+      writeFileSync: (path, contents) => calls.push({ operation: "write", value: [path, contents] }),
+    },
+    path: {
+      join: (...paths) => paths.join("/"),
+    },
+  });
+
+  assert.equal(result, "/games/saves/(PYDT) Play This One!.Civ6Save");
+  assert.deepEqual(calls, [
+    { operation: "exists", value: "/games/saves" },
+    { operation: "mkdirp", value: "/games/saves" },
+    { operation: "write", value: ["/games/saves/(PYDT) Play This One!.Civ6Save", data] },
+  ]);
+});

ui/shared/autoDownloadSave.ts

@@ -0,0 +1,31 @@
+export interface AutoDownloadFileSystem {
+  existsSync(path: string): boolean;
+  mkdirp(path: string): void;
+  writeFileSync(path: string, data: Uint8Array): void;
+}
+
+export interface AutoDownloadPath {
+  join(...paths: string[]): string;
+}
+
+export interface AutoDownloadSaveOptions {
+  saveDir: string;
+  saveExtension: string;
+  data: Uint8Array;
+  fs: AutoDownloadFileSystem;
+  path: AutoDownloadPath;
+}
+
+export const autoDownloadFileName = (saveExtension: string): string => `(PYDT) Play This One!.${saveExtension}`;
+
+export const saveDownloadedTurn = (options: AutoDownloadSaveOptions): string => {
+  if (!options.fs.existsSync(options.saveDir)) {
+    options.fs.mkdirp(options.saveDir);
+  }
+
+  const saveFile = options.path.join(options.saveDir, autoDownloadFileName(options.saveExtension));
+
+  options.fs.writeFileSync(saveFile, options.data);
+
+  return saveFile;
+};

ui/shared/pydtSettings.ts

@@ -17,6 +17,7 @@ export class PydtSettingsData {
   gameStores: { [index: string]: GameStore } = {};
   savePaths: { [index: string]: string } = {};
   autoDownload = false;
+  saveDownloadedTurns = false;
 
   constructor(
     civGames: CivGame[],

ui/shared/turnCacheService.ts

@@ -1,8 +1,11 @@
 import { Injectable, inject } from "@angular/core";
 import { BusyService, Game, GameService, GameTurnResponse } from "pydt-shared";
-import { BehaviorSubject, firstValueFrom, merge, of } from "rxjs";
-import { catchError, filter, map } from "rxjs/operators";
-import { PydtSettingsFactory } from "./pydtSettings";
+import { BehaviorSubject, firstValueFrom, merge, of, Subject } from "rxjs";
+import { catchError, filter, map, timeout } from "rxjs/operators";
+import { PydtSettingsData, PydtSettingsFactory } from "./pydtSettings";
+import { SafeMetadataLoader } from "./safeMetadataLoader";
+import { saveDownloadedTurn } from "./autoDownloadSave";
+import { RPC_TO_MAIN } from "../rpcChannels";
 
 const BASE_RETRY_BACKOFF_MS = 5000;
 const MAX_RETRY_BACKOFF_MS = 5 * 60 * 1000;
@@ -148,11 +151,15 @@ export class TurnCacheService {
   private readonly gameService = inject(GameService);
   private readonly busyService = inject(BusyService);
   private readonly pydtSettingsFactory = inject(PydtSettingsFactory);
+  private readonly metadataLoader = inject(SafeMetadataLoader);
 
   private readonly cache: TurnDownloader[] = [];
+  private readonly savedVersions = new Set<string>();
+  readonly completedGameIds$ = new Subject<string>();
 
   constructor() {
     void this.backgroundDownloader().then();
+    void this.automaticTurnQueue().then();
   }
 
   async backgroundDownloader(): Promise<void> {
@@ -173,6 +180,176 @@ export class TurnCacheService {
     }
   }
 
+  private async automaticTurnQueue(): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      await new Promise(resolve => setTimeout(resolve, 5000));
+
+      const settings = await this.pydtSettingsFactory.getSettings();
+      const td = this.cache.find(candidate => {
+        const versionKey = `${candidate.game.gameId}:${candidate.game.version}`;
+        return !this.savedVersions.has(versionKey);
+      });
+
+      if (!settings.saveDownloadedTurns || !td) {
+        continue;
+      }
+
+      try {
+        if (!td.data$.value) {
+          await td.waitForCompletion();
+        }
+
+        await this.processAutomaticTurn(td, settings);
+      } catch (err) {
+        // Keep the queue alive after temporary filesystem or network failures.
+        // eslint-disable-next-line no-console
+        console.error(`Unable to process automatic turn for ${td.game.displayName}`, err);
+      }
+    }
+  }
+
+  private async processAutomaticTurn(td: TurnDownloader, settings: PydtSettingsData): Promise<void> {
+    const data = td.data$.value;
+    const versionKey = `${td.game.gameId}:${td.game.version}`;
+
+    if (!data || this.savedVersions.has(versionKey)) {
+      return;
+    }
+
+    const metadata = await this.metadataLoader.loadMetadata();
+    const civGame = metadata?.civGames.find(x => x.id === td.game.gameType);
+
+    if (!civGame) {
+      return;
+    }
+
+    const saveDir = settings.getSavePath(civGame);
+    const saveFile = saveDownloadedTurn({
+      saveDir,
+      saveExtension: civGame.saveExtension,
+      data: data.data,
+      fs: window.pydtApi.fs,
+      path: window.pydtApi.path,
+    });
+
+    // Let the handoff write settle before watching for the save created by Civ.
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    const completedSave = await window.pydtApi.startChokidar({
+      path: saveDir,
+      awaitWriteFinish: civGame.awaitWriteFinish,
+    });
+
+    // Do not advance or rewrite the handoff after a temporary upload failure.
+    // Keep retrying the completed save so the player's work remains intact.
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      try {
+        await this.uploadCompletedTurn(td.game.gameId, completedSave);
+        break;
+      } catch (err) {
+        // eslint-disable-next-line no-console
+        console.error(`Unable to upload automatic turn for ${td.game.displayName}; retrying`, err);
+        window.pydtApi.ipc.send(
+          RPC_TO_MAIN.LOG_ERROR,
+          `Unable to upload automatic turn for ${td.game.displayName}: ${String(err)}`,
+        );
+        await new Promise(resolve => setTimeout(resolve, 30000));
+      }
+    }
+
+    const archiveDir = window.pydtApi.path.join(saveDir, "pydt-archive");
+    if (!window.pydtApi.fs.existsSync(archiveDir)) {
+      window.pydtApi.fs.mkdirp(archiveDir);
+    }
+
+    const archivedSave = window.pydtApi.path.join(
+      archiveDir,
+      `${td.game.gameId.slice(0, 8)}_${window.pydtApi.path.basename(completedSave)}`,
+    );
+    window.pydtApi.fs.renameSync(completedSave, archivedSave);
+
+    if (window.pydtApi.fs.existsSync(saveFile)) {
+      window.pydtApi.fs.unlinkSync(saveFile);
+    }
+
+    this.trimArchive(archiveDir, settings.numSaves);
+    this.savedVersions.add(versionKey);
+    this.completedGameIds$.next(td.game.gameId);
+  }
+
+  private async uploadCompletedTurn(gameId: string, saveFile: string): Promise<void> {
+    window.pydtApi.ipc.send(RPC_TO_MAIN.LOG_INFO, `Compressing completed turn: ${saveFile}`);
+    const fileData = await window.pydtApi.readFileGzipped(saveFile);
+    window.pydtApi.ipc.send(RPC_TO_MAIN.LOG_INFO, `Starting turn submission: ${gameId}`);
+    const startResp = await firstValueFrom(this.gameService.startSubmit(gameId));
+    window.pydtApi.ipc.send(RPC_TO_MAIN.LOG_INFO, `Uploading completed turn: ${gameId}`);
+
+    await new Promise<void>((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+      xhr.open("PUT", startResp.putUrl, true);
+      xhr.timeout = 60000;
+      xhr.onload = () => {
+        if (xhr.status === 200) {
+          resolve();
+        } else {
+          reject(new Error(`Turn upload returned HTTP ${xhr.status}`));
+        }
+      };
+      xhr.onerror = () => reject(new Error(`Turn upload returned HTTP ${xhr.status}`));
+      xhr.ontimeout = () => reject(new Error("Turn upload timed out"));
+      xhr.setRequestHeader("Content-Type", "application/octet-stream");
+      xhr.send(fileData as unknown as ArrayBuffer);
+    });
+
+    // The uploaded file is already durable at this point. Retry only the final
+    // confirmation if the API stalls so the save is not uploaded repeatedly.
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      try {
+        window.pydtApi.ipc.send(RPC_TO_MAIN.LOG_INFO, `Finishing turn submission: ${gameId}`);
+        await firstValueFrom(this.gameService.finishSubmit(gameId).pipe(timeout(60000)));
+        break;
+      } catch (err) {
+        const httpError = err as { status?: number; message?: string; error?: unknown };
+        const errorDetails = JSON.stringify({
+          status: httpError.status,
+          message: httpError.message,
+          error: httpError.error,
+        });
+        // eslint-disable-next-line no-console
+        console.error(`Unable to finish turn submission for ${gameId}; retrying`, err);
+        window.pydtApi.ipc.send(
+          RPC_TO_MAIN.LOG_ERROR,
+          `Unable to finish turn submission for ${gameId}: ${errorDetails}`,
+        );
+
+        if (httpError.status >= 400 && httpError.status < 500) {
+          throw err;
+        }
+
+        await new Promise(resolve => setTimeout(resolve, 30000));
+      }
+    }
+
+    window.pydtApi.ipc.send(RPC_TO_MAIN.LOG_INFO, `Turn submission completed: ${gameId}`);
+  }
+
+  private trimArchive(archiveDir: string, numSaves: number): void {
+    const files = window.pydtApi.fs
+      .readdirSync(archiveDir)
+      .flatMap(fileName => {
+        const file = window.pydtApi.path.join(archiveDir, fileName);
+        const stat = window.pydtApi.fs.statSync(file);
+        return stat.isDirectory ? [] : [{ file, time: stat.ctime.getTime() }];
+      })
+      .sort((a, b) => a.time - b.time);
+
+    while (files.length > numSaves) {
+      window.pydtApi.fs.unlinkSync(files.shift().file);
+    }
+  }
+
   updateGames(games: Game[]): void {
     const newGames = games.filter(
       x => !this.cache.some(y => x.gameId === y.game.gameId && x.version === y.game.version),
@@ -190,6 +367,7 @@ export class TurnCacheService {
 
       this.cache[i].abort();
       this.cache.splice(i, 1);
+      this.savedVersions.delete(`${dl.game.gameId}:${dl.game.version}`);
     }
   }