From 1ed5af23117379a502cbdfe3adfaa5824eb0dbff Mon Sep 17 00:00:00 2001 From: zakijss <210305895+zakijss@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:02:35 -0500 Subject: [PATCH 1/2] Add a Storage page and Clean up Storage (R1 from Startup) shows the card's free and used space and its folders largest first, measured in the background (the result is kept for the session; on a 100 GB card it takes about a minute). A opens a folder; X deletes a file or folder after asking. Nothing off the card or on its top level, nothing in .tmp_update, miyoo or miyoo354, no whole app folder (MiniMart removes apps with their boot scripts), nothing in MiniKey or MiniTop, and no key, .pem or .enc file, also inside a folder. Clean up (X on Storage) lists MiniMart's downloads, MiniTape's songs, game list caches, logs (RetroArch's, .tmp_update/logs, app logs over 1 MB), macOS leftovers and empty folders, each path in full before anything is deleted; every path is checked again just before deleting. Tests cover the delete rules, each clean-up's list, cancelling and the background measurer under -race; a 50,000-file stress run measures in 1.4 s under ARM qemu. On the device: 19.23 GB free matches df, 19 MB RSS. Requires MiniDeck 1.0.1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XNFhx3fuFPdPMoCFatrpkM --- .github/repo.json | 2 +- CHANGELOG.md | 15 +- README.md | 6 +- app/config.json | 2 +- cmd/minitop/main.go | 104 ++++-- cmd/minitop/storage.go | 678 ++++++++++++++++++++++++++++++++++++ cmd/minitop/storage_test.go | 668 +++++++++++++++++++++++++++++++++++ cmd/minitop/storage_ui.go | 412 ++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- store.json | 5 +- 11 files changed, 1861 insertions(+), 37 deletions(-) create mode 100644 cmd/minitop/storage.go create mode 100644 cmd/minitop/storage_test.go create mode 100644 cmd/minitop/storage_ui.go diff --git a/.github/repo.json b/.github/repo.json index 84b4850..b4880d8 100644 --- a/.github/repo.json +++ b/.github/repo.json @@ -1 +1 @@ -{"description": "A task manager for the Miyoo Mini Plus on Onion OS: running programs with memory and CPU, boot scripts, and a clock set over HTTPS at boot.", "topics": ["miyoo-mini-plus", "onion-os", "task-manager", "golang"]} +{"description": "Tasks and maintenance for the Miyoo Mini Plus on Onion OS: running programs with memory and CPU, boot scripts, SD card storage and clean-up, and a clock set over HTTPS at boot.", "topics": ["miyoo-mini-plus", "onion-os", "task-manager", "golang"]} diff --git a/CHANGELOG.md b/CHANGELOG.md index ee8b85c..56e7e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,24 @@ All notable changes are listed here. The format follows ## [Unreleased] +## [1.1.0] - 2026-11-02 + +### Added + +- Storage page (**R1** from Startup): free space, folder sizes largest first, and deleting a file or folder after asking. It refuses Onion's files, the card's top level, whole app folders, MiniKey and key or `.enc` files. +- Clean up (**X** on Storage): MiniMart downloads, MiniTape songs, game list caches, logs, macOS leftovers and empty folders, each listed in full before anything is deleted. + +### Changed + +- **L1**/**R1** now cycle through Running, Startup and Storage. +- Requires MiniDeck 1.0.1. + ## [1.0.0] - 2026-11-02 ### Added - The first release: running programs, startup scripts, `clocksync`, and settings to sort the programs by CPU, memory or name and to hide Onion's own. -[Unreleased]: https://github.com/zakijss/MiniTop/compare/v1.0.0...HEAD +[Unreleased]: https://github.com/zakijss/MiniTop/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/zakijss/MiniTop/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/zakijss/MiniTop/releases/tag/v1.0.0 diff --git a/README.md b/README.md index 0995545..fe1dde0 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,12 @@ # MiniTop -What's running on a Miyoo Mini Plus, and what starts at boot. It also keeps the clock right. Part of [MiniDeck](https://github.com/zakijss/MiniDeck). +Tasks and maintenance for a Miyoo Mini Plus: what's running, what starts at boot and what fills the SD card. It also keeps the clock right. Part of [MiniDeck](https://github.com/zakijss/MiniDeck). - Running programs with their memory and CPU use; **A** ends one (Onion's own processes are protected). **Y** sorts them by CPU, memory or name and can hide Onion's own. - The scripts Onion runs at boot, each turned on or off. +- Storage: free space on the card, and how much each folder takes, largest first (Roms by system, App by app). **A** opens a folder; **X** deletes the selected file or folder after asking. +- Clean up (**X** on Storage): MiniMart's downloads, MiniTape's saved songs, Onion's game list caches, logs, macOS leftovers (`._` files, `.DS_Store`, `.Trashes`) and empty folders. Each one lists exactly what it deletes before you confirm. Games, saves and settings are never part of a clean-up. - `clocksync`: a boot script that sets the clock from HTTPS servers once Wi-Fi is up, since Onion's own time sync relies on a service that no longer answers. @@ -17,7 +19,7 @@ What's running on a Miyoo Mini Plus, and what starts at boot. It also keeps the ## Using it -**L1**/**R1** switch between the running programs and the startup scripts. For daylight saving, put a zone name such as `Europe/Berlin` in `App/MiniTop/timezone`. Turn clocksync off in the startup tab. +**L1**/**R1** move between Running, Startup and Storage. Measuring the card takes a few seconds and runs in the background. The card has no trash, so a delete is final. MiniTop won't delete Onion's files, the card's top-level folders, whole app folders (remove apps in MiniMart, which also takes out their boot scripts), MiniKey's vault, or any key or `.enc` file. For daylight saving, put a zone name such as `Europe/Berlin` in `App/MiniTop/timezone`. Turn clocksync off in the startup tab. ## Install diff --git a/app/config.json b/app/config.json index 3d087b0..e03c2cb 100644 --- a/app/config.json +++ b/app/config.json @@ -2,5 +2,5 @@ "label": "MiniTop", "icon": "/mnt/SDCARD/App/MiniTop/icon.png", "launch": "launch.sh", - "description": "What's running, and what starts at boot" + "description": "Tasks and maintenance: running programs, startup, storage" } diff --git a/cmd/minitop/main.go b/cmd/minitop/main.go index 57425dd..08a119e 100644 --- a/cmd/minitop/main.go +++ b/cmd/minitop/main.go @@ -1,5 +1,6 @@ -// MiniTop shows what's running on the Miyoo (memory and CPU per program) and -// can end a program; its second tab turns startup items on and off. +// MiniTop is tasks and maintenance for the Miyoo: what's running (memory and +// CPU per program, and ending one), the startup items, turned on and off, +// and what fills the SD card, with clean-ups that free space. package main import ( @@ -22,7 +23,7 @@ func main() { log.Fatal(err) } defer scr.Close() - t := &tasks{scr: scr, input: ui.Input(), smp: newSampler(), dir: appdir.Dir()} + t := &tasks{scr: scr, input: ui.Input(), smp: newSampler(), dir: appdir.Dir(), disk: &measurer{root: sdcard()}} t.cfg = loadSettings(t.dir) t.smp.Sample() // first sample: CPU needs two t.procs = t.smp.Sample() @@ -30,33 +31,59 @@ func main() { } type tasks struct { - scr *ui.Screen - input <-chan ui.Event - smp *sampler - procs []Proc // every program, busiest first - shown []Proc // the ones listed, in Settings' order - dir string - cfg Settings - startup bool // which tab - run1 ui.List - run2 ui.List - items []StartItem + scr *ui.Screen + input <-chan ui.Event + smp *sampler + procs []Proc // every program, busiest first + shown []Proc // the ones listed, in Settings' order + dir string + cfg Settings + page int // pageRunning, pageStartup or pageStorage + run1 ui.List + run2 ui.List + run3 ui.List + items []StartItem + disk *measurer + // the Storage page's rows, for the scan they came from + rootScan *Scan + rootEnts []Entry + rootMore int + stale bool // a delete failed part way: measure again } +// The pages L1 and R1 move between, in order. +const ( + pageRunning = iota + pageStartup + pageStorage + pages +) + func mb(b int64) string { return fmt.Sprintf("%.1f MB", float64(b)/(1<<20)) } func (t *tasks) run() { tick := time.NewTicker(2 * time.Second) defer tick.Stop() for { - if t.startup { + // while the card is measured, the Storage page redraws its progress + var measuring <-chan struct{} + var redraw <-chan time.Time + switch t.page { + case pageStartup: t.drawStartup() - } else { + case pageStorage: + t.drawStorage() + if _, _, done, _ := t.disk.state(); done != nil { + measuring, redraw = done, time.After(250*time.Millisecond) + } + default: t.drawRunning() } select { + case <-measuring: + case <-redraw: case <-tick.C: - if !t.startup { + if t.page == pageRunning { t.procs = t.smp.Sample() } case ev, ok := <-t.input: @@ -70,26 +97,33 @@ func (t *tasks) run() { case ui.B, ui.Menu: return case ui.L1, ui.R1: - t.startup = !t.startup - if t.startup { - t.items = StartItems() - } + t.turn(ev.Button == ui.R1) case ui.A: - if t.startup { + switch t.page { + case pageStartup: t.toggle() - } else { + case pageStorage: + t.openStorage() + default: t.end() } case ui.X: - t.procs = t.smp.Sample() + if t.page == pageStorage { + t.cleanup() + } else { + t.procs = t.smp.Sample() + } case ui.Y: t.settings() case ui.Select: t.scr.Appearance(t.input) default: - if t.startup { + switch t.page { + case pageStartup: t.run2.Handle(ev) - } else { + case pageStorage: + t.run3.Handle(ev) + default: t.run1.Handle(ev) } } @@ -97,6 +131,22 @@ func (t *tasks) run() { } } +// turn moves to the next page (R1) or the one before (L1), wrapping +// around, and loads what the page shows. +func (t *tasks) turn(next bool) { + if next { + t.page = (t.page + 1) % pages + } else { + t.page = (t.page + pages - 1) % pages + } + switch t.page { + case pageStartup: + t.items = StartItems() + case pageStorage: + t.disk.start() + } +} + // acts reports whether the loop should act on ev. Releases never count, // and held buttons repeat only for the d-pad: a held A would turn a startup // item on and off with every repeat, a held L1 would flip the tabs. @@ -182,7 +232,7 @@ func (t *tasks) end() { func (t *tasks) drawStartup() { s := t.scr top := s.StaticFrame("Startup", "At boot: "+OnionStartApp()+" (change in Tweaks › System)", - ui.Hint{Button: "A", Label: "On / off"}, ui.Hint{Button: "L1", Label: "Running"}, ui.AppearanceHint, ui.Hint{Button: "B", Label: "Exit"}) + ui.Hint{Button: "A", Label: "On / off"}, ui.Hint{Button: "R1", Label: "Storage"}, ui.AppearanceHint, ui.Hint{Button: "B", Label: "Exit"}) var items []ui.Item for _, it := range t.items { state := "Off" diff --git a/cmd/minitop/storage.go b/cmd/minitop/storage.go new file mode 100644 index 0000000..1c0d77f --- /dev/null +++ b/cmd/minitop/storage.go @@ -0,0 +1,678 @@ +package main + +// Storage: what fills the SD card, and clean-ups that free space without +// touching games, saves or settings. measure walks the card once, in the +// background, and keeps a tree of folder sizes plus what each clean-up +// would delete; deleting anything else is one item at a time, and +// checkDelete refuses Onion's own files, keys and whole app folders. + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" +) + +// sdcard is the card's root; TASKS_SDCARD replaces it in tests. +func sdcard() string { + if d := os.Getenv("TASKS_SDCARD"); d != "" { + return d + } + return "/mnt/SDCARD" +} + +// Dir is one folder of the measured tree. Sizes are the space the files +// take on the card (whole clusters), so they add up to what the card +// reports as used. +type Dir struct { + Name string + Path string // full path + Size int64 // this folder and everything in it + Files int // files in it, at any depth + Subs []*Dir // its folders, largest first + parent *Dir +} + +// Rel is the folder's path from the card's root ("" for the root). +func (d *Dir) Rel(root string) string { + r, _ := filepath.Rel(root, d.Path) + if r == "." { + return "" + } + return filepath.ToSlash(r) +} + +// Cleanup is one kind of clean-up: what it would delete and how much. +type Cleanup struct { + ID string + Name string + About string // one line: what it is + Ask string // the question before deleting + Note string // what happens after (shown before deleting) + Paths []string + Sizes []int64 // the space each path takes + Count []int // the files each path holds (1 for a file) + Size int64 + Files int // files among the paths, at any depth +} + +func (c *Cleanup) add(p string, size int64, files int) { + c.Paths = append(c.Paths, p) + c.Sizes = append(c.Sizes, size) + c.Count = append(c.Count, files) + c.Size += size + c.Files += files +} + +// Scan is a finished measurement of the card. +type Scan struct { + Root string + Tree *Dir + Cleanups []*Cleanup + Errors int // folders that couldn't be read +} + +// Cleanup returns the clean-up with the given ID. +func (s *Scan) Cleanup(id string) *Cleanup { + for _, c := range s.Cleanups { + if c.ID == id { + return c + } + } + return nil +} + +// Progress is what a running measurement has seen so far. It is read +// from the UI while measure writes it. +type Progress struct { + Files atomic.Int64 + Bytes atomic.Int64 + where atomic.Value // the top-level folder being measured +} + +// Where is the top-level folder being measured. +func (p *Progress) Where() string { + s, _ := p.where.Load().(string) + return s +} + +// junkNames are macOS leftovers a Mac writes onto the card: folders, and +// single files (AppleDouble "._" files are matched by prefix). +var ( + junkDirs = map[string]bool{".Trashes": true, ".Spotlight-V100": true, ".fseventsd": true, ".TemporaryItems": true} + junkFiles = map[string]bool{".DS_Store": true} +) + +func junkFile(name string) bool { return junkFiles[name] || strings.HasPrefix(name, "._") } + +// systemTops are top-level folders whose empty subfolders are left alone: +// Onion, emulators and apps expect them. +var systemTops = map[string]bool{ + ".tmp_update": true, "miyoo": true, "miyoo354": true, "App": true, "Emu": true, "RApp": true, + "RetroArch": true, "Saves": true, "BIOS": true, "Themes": true, "Icons": true, +} + +// emptyOK reports whether an empty folder at rel may be offered for +// deletion: not a top-level folder, nothing in Onion's, an app's or an +// emulator's folders, not a system's folder in Roms (Roms/GBA) and not a +// box-art folder. +func emptyOK(rel string) bool { + parts := strings.Split(rel, "/") + if len(parts) < 2 || systemTops[parts[0]] { + return false + } + if parts[0] == "Roms" && len(parts) < 3 { + return false + } + return parts[len(parts)-1] != "Imgs" +} + +// protectedRe matches files that hold keys or secrets: never deleted from +// here (minideck.key, hostkey_ed25519, authorized_keys, state.enc, *.pem, +// id_ed25519). "key" must stand as its own word, so Donkey Kong is fine. +var protectedRe = regexp.MustCompile(`(?i)(^|[._ -])(host)?keys?([._ -]|$)|hostkey|authorized_keys|\.enc$|\.pem$|^id_(rsa|ecdsa|ed25519|dsa)`) + +func protectedName(name string) bool { return protectedRe.MatchString(name) } + +// logRe: the log files clean-up offers in app folders, when large. +var logRe = regexp.MustCompile(`(?i)\.log(\.old|\.[0-9])?$`) + +const bigLog = 1 << 20 + +// onDisk is the space a file takes on the card: its allocated blocks, or +// its length where the file system doesn't say. +func onDisk(fi fs.FileInfo) int64 { + if st, ok := fi.Sys().(*syscall.Stat_t); ok && st.Blocks > 0 { + return int64(st.Blocks) * 512 + } + return fi.Size() +} + +// walker measures one card. +type walker struct { + ctx context.Context + root string + prog *Progress + errors int + c map[string]*Cleanup + n int // entries since the last cancellation check +} + +func newCleanups() map[string]*Cleanup { + return map[string]*Cleanup{ + "minimart": {ID: "minimart", Name: "MiniMart downloads", About: "Thumbnails and downloads", Ask: "Clear MiniMart's downloads?", + Note: "MiniMart downloads them again when it needs them."}, + "minitape": {ID: "minitape", Name: "MiniTape songs", About: "Songs saved while playing", Ask: "Clear MiniTape's songs?", + Note: "MiniTape downloads songs again when you play them. A song that is downloading right now may stop."}, + "lists": {ID: "lists", Name: "Game list caches", About: "Onion makes them again", Ask: "Clear the game list caches?", + Note: "Onion makes each list again the next time you open that system, so the first open is slower."}, + "logs": {ID: "logs", Name: "Logs", About: "RetroArch, Onion, big app logs", Ask: "Delete the logs?", + Note: "Apps start new logs when they need them."}, + "junk": {ID: "junk", Name: "macOS leftovers", About: "._ files, .DS_Store, .Trashes", Ask: "Delete the macOS leftovers?", + Note: "A Mac writes them again when the card is next plugged into it."}, + "empty": {ID: "empty", Name: "Empty folders", About: "Outside Onion's and apps' folders", Ask: "Delete the empty folders?"}, + } +} + +// cleanupOrder is the order the Clean up screen lists them in. +var cleanupOrder = []string{"minimart", "minitape", "lists", "logs", "junk", "empty"} + +// measure walks the card at root and returns its tree and clean-ups. It +// stops with ctx's error when ctx is cancelled. Links aren't followed. +func measure(ctx context.Context, root string, prog *Progress) (*Scan, error) { + if prog == nil { + prog = &Progress{} + } + w := &walker{ctx: ctx, root: root, prog: prog, c: newCleanups()} + fi, err := os.Stat(root) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("%s is not a folder", root) + } + tree := &Dir{Name: filepath.Base(root), Path: root} + if _, err := w.dir(tree, "", false, false); err != nil { + return nil, err + } + s := &Scan{Root: root, Tree: tree, Errors: w.errors} + for _, id := range cleanupOrder { + s.Cleanups = append(s.Cleanups, w.c[id]) + } + return s, nil +} + +// dir measures d (at rel from the root) and everything in it. claimed: +// d is inside a folder a clean-up already lists whole (a macOS leftover +// folder, a cache's subfolder), so nothing in it is listed again; secret: +// inside MiniKey, where nothing is offered for clean-up. It reports +// whether d holds no files at any depth. +func (w *walker) dir(d *Dir, rel string, claimed, secret bool) (empty bool, err error) { + ents, err := os.ReadDir(d.Path) + if err != nil { + w.errors++ + if len(ents) == 0 { + return false, nil + } + } + empty = true + emptyMark := len(w.c["empty"].Paths) + cacheDir := "" + switch rel { + case "App/MiniMart/cache": + cacheDir = "minimart" + case "App/MiniTape/streams": + cacheDir = "minitape" + } + logsDir := rel == "RetroArch/.retroarch/logs" || rel == ".tmp_update/logs" || + strings.HasPrefix(rel, "RetroArch/.retroarch/logs/") || strings.HasPrefix(rel, ".tmp_update/logs/") + for _, e := range ents { + if w.n++; w.n >= 256 { + w.n = 0 + if err := w.ctx.Err(); err != nil { + return false, err + } + } + name := e.Name() + p := filepath.Join(d.Path, name) + r := name + if rel != "" { + r = rel + "/" + name + } + if rel == "" { + w.prog.where.Store(name) + } + fi, err := e.Info() + if err != nil { + continue // gone since ReadDir + } + sz := onDisk(fi) + if e.IsDir() { + sub := &Dir{Name: name, Path: p, parent: d, Size: sz} + isJunk := !claimed && !secret && junkDirs[name] + subEmpty, err := w.dir(sub, r, claimed || isJunk || cacheDir != "", secret || r == "App/MiniKey") + if err != nil { + return false, err + } + d.Subs = append(d.Subs, sub) + d.Size += sub.Size + d.Files += sub.Files + if !subEmpty { + empty = false + } + switch { + case claimed || secret: + case isJunk: + w.c["junk"].add(p, sub.Size, sub.Files) + case cacheDir != "": + w.c[cacheDir].add(p, sub.Size, sub.Files) + } + continue + } + // files (and links, which count as files: they aren't followed) + empty = false + d.Size += sz + d.Files++ + w.prog.Files.Add(1) + w.prog.Bytes.Add(sz) + if claimed || secret || protectedName(name) || !fi.Mode().IsRegular() { + continue + } + switch { + case cacheDir != "": + w.c[cacheDir].add(p, sz, 1) + case junkFile(name): + w.c["junk"].add(p, sz, 1) + case strings.HasPrefix(r, "Roms/") && strings.Count(r, "/") == 2 && strings.HasSuffix(name, "_cache6.db"): + w.c["lists"].add(p, sz, 1) + case logsDir: + w.c["logs"].add(p, sz, 1) + case strings.HasPrefix(r, "App/") && logRe.MatchString(name) && fi.Size() > bigLog: + w.c["logs"].add(p, sz, 1) + } + } + sort.SliceStable(d.Subs, func(i, j int) bool { return d.Subs[i].Size > d.Subs[j].Size }) + if empty && !claimed && !secret && emptyOK(rel) { + // one entry for the outermost empty folder, not one per level + e := w.c["empty"] + for _, s := range e.Sizes[emptyMark:] { + e.Size -= s + } + e.Paths, e.Sizes, e.Count = e.Paths[:emptyMark], e.Sizes[:emptyMark], e.Count[:emptyMark] + e.add(d.Path, d.Size, 0) + } + return empty, nil +} + +// Entry is one row of a folder's contents: a folder from the tree or a +// file read when the folder is opened. +type Entry struct { + Name string + Path string + Size int64 + Dir *Dir // nil for a file +} + +// maxEntries is how many rows a folder shows; the rest are its smallest. +const maxEntries = 2000 + +// Contents lists d's folders and files, largest first, and how many small +// files were left out to keep the list at maxEntries. +func Contents(d *Dir) ([]Entry, int) { + var out []Entry + for _, s := range d.Subs { + out = append(out, Entry{Name: s.Name, Path: s.Path, Size: s.Size, Dir: s}) + } + ents, _ := os.ReadDir(d.Path) + for _, e := range ents { + if e.IsDir() { + continue + } + fi, err := e.Info() + if err != nil { + continue + } + out = append(out, Entry{Name: e.Name(), Path: filepath.Join(d.Path, e.Name()), Size: onDisk(fi)}) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Size != out[j].Size { + return out[i].Size > out[j].Size + } + return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name) + }) + if len(out) > maxEntries { + return out[:maxEntries], len(out) - maxEntries + } + return out, 0 +} + +// errRefused wraps the reason a delete is refused. +var errRefused = errors.New("refused") + +func refuse(why string) error { return fmt.Errorf("%w: %s", errRefused, why) } + +// checkDelete reports why p must not be deleted from the Storage screen, +// or nil. It refuses anything outside the card, the card's top level +// (Onion's layout and the swap file), Onion's system folders, whole app +// folders (MiniMart removes apps with their boot scripts), MiniKey and +// MiniTop, and files that hold keys or secrets, also inside a folder. +func checkDelete(root, p string) error { + rel, err := filepath.Rel(root, p) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, "../") || filepath.IsAbs(rel) { + return refuse("it isn't on the SD card") + } + rel = filepath.ToSlash(rel) + parts := strings.Split(rel, "/") + switch { + case len(parts) == 1: + return refuse("the card's top level is Onion's layout. Open the folder and delete what's in it") + case parts[0] == ".tmp_update" || parts[0] == "miyoo" || parts[0] == "miyoo354": + return refuse("these are Onion's own files") + case parts[0] == "App" && (parts[1] == "MiniKey" || parts[1] == "MiniTop"): + return refuse(parts[1] + "'s files are kept") + case parts[0] == "App" && len(parts) == 2: + return refuse("remove apps in MiniMart, which also takes out their boot scripts") + } + fi, err := os.Lstat(p) + if err != nil { + return err + } + if protectedName(fi.Name()) { + return refuse(fi.Name() + " holds a key or a secret") + } + if !fi.IsDir() { + return nil + } + var found string + stop := errors.New("found") + filepath.WalkDir(p, func(q string, e fs.DirEntry, err error) error { + if err == nil && q != p && protectedName(e.Name()) { + found = q + return stop + } + return nil + }) + if found != "" { + r, _ := filepath.Rel(root, found) + return refuse("it holds " + filepath.ToSlash(r) + ", a key or a secret") + } + return nil +} + +// warning is an extra line for the delete confirmation, by area. +func warning(root, p string) string { + rel, _ := filepath.Rel(root, p) + switch strings.Split(filepath.ToSlash(rel), "/")[0] { + case "Saves": + return " Saved games in it are lost." + case "BIOS": + return " Games that need it stop working." + case "Roms": + return "" + case "Emu", "RApp", "RetroArch": + return " Games that use it may stop working." + } + return "" +} + +// remove deletes p (a file or a folder with everything in it). A path +// that's already gone counts as removed. +func remove(p string) error { + err := os.RemoveAll(p) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// Forget takes a deleted path out of the scan: its size leaves every +// folder above it, its folder leaves the tree, and clean-ups no longer +// list it or anything in it. +func (s *Scan) Forget(p string, size int64, files int, d *Dir) { + if d != nil { + if par := d.parent; par != nil { + for i, c := range par.Subs { + if c == d { + par.Subs = append(par.Subs[:i:i], par.Subs[i+1:]...) + break + } + } + } + size, files = d.Size, d.Files + } + for a := s.dirOf(filepath.Dir(p)); a != nil; a = a.parent { + a.Size -= size + a.Files -= files + } + for _, c := range s.Cleanups { + c.forget(p) + } +} + +// forget drops p and everything in it from c's list. +func (c *Cleanup) forget(p string) { + old := *c + c.Paths, c.Sizes, c.Count, c.Size, c.Files = nil, nil, nil, 0, 0 + for i, q := range old.Paths { + if q == p || strings.HasPrefix(q, p+string(filepath.Separator)) { + continue + } + c.add(q, old.Sizes[i], old.Count[i]) + } +} + +// dirOf finds the tree's folder for path p, or nil. +func (s *Scan) dirOf(p string) *Dir { + rel, err := filepath.Rel(s.Root, p) + if err != nil || strings.HasPrefix(rel, "..") { + return nil + } + d := s.Tree + if rel == "." { + return d + } + for _, part := range strings.Split(filepath.ToSlash(rel), "/") { + var next *Dir + for _, c := range d.Subs { + if c.Name == part { + next = c + break + } + } + if next == nil { + return nil + } + d = next + } + return d +} + +// run deletes everything c lists, calling step after each path. It +// returns how many paths were removed and the first error; it goes on +// past errors. Every path is checked again: inside the card, not the +// card's top level, not Onion's or MiniKey's (logs in .tmp_update/logs +// excepted), not a key. +func (c *Cleanup) run(root string, step func(done, total int)) (int, error) { + var first error + n := 0 + for i, p := range c.Paths { + if err := cleanupOK(root, p); err != nil { + if first == nil { + first = err + } + } else if err := remove(p); err != nil { + if first == nil { + first = err + } + } else { + n++ + } + if step != nil { + step(i+1, len(c.Paths)) + } + } + return n, first +} + +// cleanupOK re-checks a clean-up path just before it's deleted. On the +// card's top level only macOS leftovers (.DS_Store, .Trashes) may go. +func cleanupOK(root, p string) error { + rel, err := filepath.Rel(root, p) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.IsAbs(rel) { + return refuse(p + " isn't on the SD card") + } + rel = filepath.ToSlash(rel) + if !strings.Contains(rel, "/") && !junkFile(rel) && !junkDirs[rel] { + return refuse(rel + " is on the card's top level") + } + if strings.HasPrefix(rel, "App/MiniKey/") || protectedName(filepath.Base(p)) { + return refuse(rel + " is kept") + } + if strings.HasPrefix(rel, ".tmp_update/") && !strings.HasPrefix(rel, ".tmp_update/logs/") && !junkFile(filepath.Base(rel)) { + return refuse(rel + " is Onion's") + } + return nil +} + +// space returns the card's free and total bytes. +func space(root string) (free, total int64, err error) { + var st syscall.Statfs_t + if err := syscall.Statfs(root, &st); err != nil { + return 0, 0, err + } + bs := int64(st.Bsize) + return int64(st.Bavail) * bs, int64(st.Blocks) * bs, nil +} + +// size formats bytes for the screen: 512 B, 3.4 KB, 12.3 MB, 1.25 GB. +func size(b int64) string { + switch { + case b < 1<<10: + return fmt.Sprintf("%d B", b) + case b < 1<<20: + return fmt.Sprintf("%.1f KB", float64(b)/(1<<10)) + case b < 1<<30: + return fmt.Sprintf("%.1f MB", float64(b)/(1<<20)) + case b < 100<<30: + return fmt.Sprintf("%.2f GB", float64(b)/(1<<30)) + } + return fmt.Sprintf("%.0f GB", float64(b)/(1<<30)) +} + +// count formats a number with thousands separators: 12,345. +func count(n int64) string { + s := fmt.Sprint(n) + neg := strings.HasPrefix(s, "-") + s = strings.TrimPrefix(s, "-") + var b strings.Builder + for i, r := range s { + if i > 0 && (len(s)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(r) + } + if neg { + return "-" + b.String() + } + return b.String() +} + +// measurer runs one measurement at a time in the background and keeps +// the last result for the session. +type measurer struct { + root string + mu sync.Mutex + scan *Scan + err error + prog *Progress + done chan struct{} // closed when the running measurement ends; nil if none + cancel context.CancelFunc +} + +// start begins measuring unless a measurement is running or done. +func (m *measurer) start() { + m.mu.Lock() + defer m.mu.Unlock() + if m.done != nil || m.scan != nil { + return + } + ctx, cancel := context.WithCancel(context.Background()) + m.prog, m.cancel, m.err = &Progress{}, cancel, nil + done := make(chan struct{}) + m.done = done + prog := m.prog + go func() { + s, err := measure(ctx, m.root, prog) + m.mu.Lock() + if ctx.Err() == nil { + m.scan, m.err = s, err + } + m.done = nil + m.mu.Unlock() + close(done) + }() +} + +// again drops the last result and measures again. +func (m *measurer) again() { + m.stop() + m.mu.Lock() + m.scan, m.err = nil, nil + m.mu.Unlock() + m.start() +} + +// stop cancels a running measurement and waits for it to end. +func (m *measurer) stop() { + m.mu.Lock() + done, cancel := m.done, m.cancel + m.mu.Unlock() + if done != nil { + cancel() + <-done + } +} + +// state returns the finished scan (nil while measuring), the running +// measurement's progress, a channel closed when it ends, and the last +// measurement's error. +func (m *measurer) state() (*Scan, *Progress, <-chan struct{}, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.scan, m.prog, m.done, m.err +} + +// wait blocks until no measurement is running. +func (m *measurer) wait() { + if _, _, done, _ := m.state(); done != nil { + <-done + } +} + +// swapInfo returns free and total swap from /proc/meminfo, in bytes. +func swapInfo() (free, total int64) { + b, err := os.ReadFile("/proc/meminfo") + if err != nil { + return 0, 0 + } + for _, line := range strings.Split(string(b), "\n") { + k, v, _ := strings.Cut(line, ":") + var n int64 + fmt.Sscan(strings.TrimSuffix(strings.TrimSpace(v), " kB"), &n) + switch k { + case "SwapTotal": + total = n * 1024 + case "SwapFree": + free = n * 1024 + } + } + return +} diff --git a/cmd/minitop/storage_test.go b/cmd/minitop/storage_test.go new file mode 100644 index 0000000..0d889b1 --- /dev/null +++ b/cmd/minitop/storage_test.go @@ -0,0 +1,668 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/zakijss/minideck/ui" +) + +// card makes a small SD card in a temp folder: files (path -> bytes) and +// empty folders (paths ending in /). +func card(t testing.TB, files map[string]int) string { + t.Helper() + root := t.TempDir() + for p, n := range files { + full := filepath.Join(root, filepath.FromSlash(p)) + if strings.HasSuffix(p, "/") { + if err := os.MkdirAll(full, 0o755); err != nil { + t.Fatal(err) + } + continue + } + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, make([]byte, n), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +var sampleCard = map[string]int{ + "Roms/GBA/Pokemon.gba": 9000, + "Roms/GBA/Zelda.gba": 5000, + "Roms/GBA/GBA_cache6.db": 700, + "Roms/GBA/._Pokemon.gba": 100, + "Roms/GBA/Imgs/": 0, + "Roms/NES/": 0, + "Roms/PORTS/Games/Doom/doom": 3000, + "Saves/CurrentProfile/states/a.state": 2000, + "App/MiniMart/cache/ab/thumb.png": 400, + "App/MiniMart/cache/list.json": 300, + "App/MiniMart/cache/ab/._thumb.png": 50, + "App/MiniTape/streams/song.mp3": 6000, + "App/MiniRack/minirack.log": (1 << 20) + 10, + "App/MiniRack/small.log": 100, + "App/MiniKey/vault.enc": 200, + "App/MiniKey/._vault.enc": 50, + "App/MiniScale/state.enc": 100, + "App/MiniScale/miniscale": 800, + "RetroArch/.retroarch/logs/retroarch.log": 500, + ".tmp_update/logs/runtime.log": 200, + ".tmp_update/startup/clocksync.sh": 100, + ".Trashes/501/deleted.gba": 4000, + ".DS_Store": 60, + "Media/old/a/b/": 0, + "Media/music/song.mp3": 1000, + "cachefile": 8000, + "BIOS/gba_bios.bin": 1600, + "Donkey Kong.gba": 100, +} + +// diskTotal adds up the space everything under root takes, folders too, +// the same way measure counts it. +func diskTotal(t *testing.T, root string) (size int64, files int) { + t.Helper() + filepath.WalkDir(root, func(p string, e fs.DirEntry, err error) error { + if err != nil || p == root { + return err + } + fi, err := e.Info() + if err != nil { + return err + } + size += onDisk(fi) + if !e.IsDir() { + files++ + } + return nil + }) + return +} + +func rels(root string, paths []string) []string { + out := make([]string, len(paths)) + for i, p := range paths { + r, _ := filepath.Rel(root, p) + out[i] = filepath.ToSlash(r) + } + sort.Strings(out) + return out +} + +func TestMeasure(t *testing.T) { + root := card(t, sampleCard) + prog := &Progress{} + s, err := measure(context.Background(), root, prog) + if err != nil { + t.Fatal(err) + } + wantSize, wantFiles := diskTotal(t, root) + if s.Tree.Size != wantSize || s.Tree.Files != wantFiles { + t.Fatalf("tree %d bytes %d files, want %d and %d", s.Tree.Size, s.Tree.Files, wantSize, wantFiles) + } + if int(prog.Files.Load()) != wantFiles { + t.Errorf("progress counted %d files", prog.Files.Load()) + } + // every folder's size is the sum of what's in it + var check func(d *Dir) + check = func(d *Dir) { + sz, n := diskTotal(t, d.Path) + if d != s.Tree { + fi, _ := os.Lstat(d.Path) + sz += onDisk(fi) + } + if d.Size != sz || d.Files != n { + t.Errorf("%s: %d bytes %d files, want %d and %d", d.Rel(root), d.Size, d.Files, sz, n) + } + for i, c := range d.Subs { + if i > 0 && c.Size > d.Subs[i-1].Size { + t.Errorf("%s: folders not largest first", d.Rel(root)) + } + check(c) + } + } + check(s.Tree) + + want := map[string][]string{ + "minimart": {"App/MiniMart/cache/ab", "App/MiniMart/cache/list.json"}, + "minitape": {"App/MiniTape/streams/song.mp3"}, + "lists": {"Roms/GBA/GBA_cache6.db"}, + "logs": {".tmp_update/logs/runtime.log", "App/MiniRack/minirack.log", "RetroArch/.retroarch/logs/retroarch.log"}, + "junk": {".DS_Store", ".Trashes", "Roms/GBA/._Pokemon.gba"}, + "empty": {"Media/old"}, + } + for id, w := range want { + c := s.Cleanup(id) + got := rels(root, c.Paths) + if strings.Join(got, "|") != strings.Join(w, "|") { + t.Errorf("%s: %v, want %v", id, got, w) + } + var sum int64 + for _, z := range c.Sizes { + sum += z + } + if sum != c.Size || len(c.Sizes) != len(c.Paths) || len(c.Count) != len(c.Paths) { + t.Errorf("%s: sizes don't add up", id) + } + } + if s.Cleanup("junk").Files != 3 { // .DS_Store, ._Pokemon.gba, the file in .Trashes + t.Errorf("junk files %d", s.Cleanup("junk").Files) + } +} + +func TestContents(t *testing.T) { + root := card(t, sampleCard) + s, err := measure(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + ents, more := Contents(s.dirOf(filepath.Join(root, "Roms", "GBA"))) + if more != 0 || len(ents) != 5 { + t.Fatalf("%d entries, %d more", len(ents), more) + } + if ents[0].Name != "Pokemon.gba" || ents[1].Name != "Zelda.gba" { + t.Errorf("order: %v, %v", ents[0].Name, ents[1].Name) + } + for i := 1; i < len(ents); i++ { + if ents[i].Size > ents[i-1].Size { + t.Error("not largest first") + } + } +} + +func TestProtectedName(t *testing.T) { + for name, want := range map[string]bool{ + "minideck.key": true, "hostkey_ed25519": true, "authorized_keys": true, "state.enc": true, + "cacert.pem": true, "id_ed25519": true, "vault.enc": true, "keys": true, "api_key.txt": true, + "Donkey Kong.gba": false, "Monkey Island.zip": false, "keyfob": false, "Pokemon.gba": false, + "turnkey.sh": false, + } { + if got := protectedName(name); got != want { + t.Errorf("%q: %v", name, got) + } + } +} + +func TestCheckDelete(t *testing.T) { + root := card(t, sampleCard) + j := func(p string) string { return filepath.Join(root, filepath.FromSlash(p)) } + for p, ok := range map[string]bool{ + "Roms/GBA/Pokemon.gba": true, + "Roms/GBA": true, + "Saves/CurrentProfile": true, + "BIOS/gba_bios.bin": true, + "App/MiniMart/cache": true, + "App/MiniRack/minirack.log": true, + "Roms": false, // top level + "cachefile": false, // the swap file + "Donkey Kong.gba": false, // top level, even though it's a game + "Saves": false, + ".tmp_update/startup/clocksync.sh": false, + ".tmp_update/logs": false, + "App/MiniKey/vault.enc": false, + "App/MiniTop": false, + "App/MiniRack": false, // whole app: MiniMart removes it + "App/MiniScale/state.enc": false, + "App": false, + } { + err := checkDelete(root, j(p)) + if (err == nil) != ok { + t.Errorf("%s: %v", p, err) + } + if err != nil && !errors.Is(err, errRefused) { + t.Errorf("%s: not a refusal: %v", p, err) + } + } + // a folder holding a key, deeper down + os.MkdirAll(j("Media/stuff/deep"), 0o755) + os.WriteFile(j("Media/stuff/deep/minideck.key"), []byte("k"), 0o600) + if err := checkDelete(root, j("Media/stuff")); err == nil || !strings.Contains(err.Error(), "minideck.key") { + t.Errorf("folder with a key: %v", err) + } + for _, p := range []string{root, filepath.Dir(root), "/etc/passwd", j("../outside"), j("Roms/../../x")} { + if err := checkDelete(root, p); err == nil { + t.Errorf("%s: allowed", p) + } + } + // a link is deleted as a link: what it points to stays + os.Symlink(j("BIOS"), j("Media/link")) + if err := checkDelete(root, j("Media/link")); err != nil { + t.Fatal(err) + } + if err := remove(j("Media/link")); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(j("BIOS/gba_bios.bin")); err != nil { + t.Fatal("followed the link") + } +} + +func TestCleanupRun(t *testing.T) { + root := card(t, sampleCard) + s, err := measure(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + j := func(p string) string { return filepath.Join(root, filepath.FromSlash(p)) } + for _, id := range cleanupOrder { + c := s.Cleanup(id) + steps := 0 + n, err := c.run(root, func(i, total int) { steps++ }) + if err != nil || n != len(c.Paths) || steps != len(c.Paths) { + t.Errorf("%s: removed %d of %d, %d steps, %v", id, n, len(c.Paths), steps, err) + } + for _, p := range c.Paths { + if _, err := os.Lstat(p); err == nil { + t.Errorf("%s: %s still there", id, p) + } + } + } + // what must stay + for _, p := range []string{"App/MiniMart/cache", "App/MiniTape/streams", "Roms/GBA/Pokemon.gba", "Roms/NES", "Roms/GBA/Imgs", + "Saves/CurrentProfile/states/a.state", "App/MiniKey/._vault.enc", "App/MiniKey/vault.enc", "App/MiniRack/small.log", + ".tmp_update/startup/clocksync.sh", ".tmp_update/logs", "BIOS/gba_bios.bin", "cachefile", "Media/music/song.mp3"} { + if _, err := os.Lstat(j(p)); err != nil { + t.Errorf("%s was deleted", p) + } + } + // measured again, there's nothing left to clean but the empty folder + // the clean-ups left behind (Media/old had none to begin with) + s, _ = measure(context.Background(), root, nil) + for _, c := range s.Cleanups { + if c.ID != "empty" && len(c.Paths) > 0 { + t.Errorf("%s still lists %v", c.ID, rels(root, c.Paths)) + } + } + // a damaged list can't make it delete Onion's or MiniKey's files + bad := &Cleanup{} + for _, p := range []string{root, j("Saves"), j(".tmp_update/startup/clocksync.sh"), j("App/MiniKey/vault.enc"), + j("App/MiniScale/state.enc"), filepath.Dir(root), "/tmp"} { + bad.add(p, 1, 1) + } + if n, err := bad.run(root, nil); n != 0 || err == nil { + t.Errorf("bad list: removed %d, %v", n, err) + } + for _, p := range []string{"Saves", ".tmp_update/startup/clocksync.sh", "App/MiniKey/vault.enc", "App/MiniScale/state.enc"} { + if _, err := os.Lstat(j(p)); err != nil { + t.Errorf("%s was deleted by a bad list", p) + } + } +} + +func TestForget(t *testing.T) { + root := card(t, sampleCard) + s, err := measure(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + gba := s.dirOf(filepath.Join(root, "Roms", "GBA")) + roms := gba.parent + before, beforeRoot, files := roms.Size, s.Tree.Size, s.Tree.Files + if err := remove(gba.Path); err != nil { + t.Fatal(err) + } + s.Forget(gba.Path, 0, 0, gba) + if roms.Size != before-gba.Size || s.Tree.Size != beforeRoot-gba.Size || s.Tree.Files != files-gba.Files { + t.Errorf("sizes not taken off: roms %d->%d", before, roms.Size) + } + if s.dirOf(gba.Path) != nil { + t.Error("folder still in the tree") + } + if len(s.Cleanup("lists").Paths) != 0 || len(s.Cleanup("junk").Paths) != 2 { + t.Errorf("clean-ups still list deleted files: %v %v", s.Cleanup("lists").Paths, rels(root, s.Cleanup("junk").Paths)) + } + // a file + song := filepath.Join(root, "Media", "music", "song.mp3") + fi, _ := os.Lstat(song) + music := s.dirOf(filepath.Dir(song)) + m0 := music.Size + remove(song) + s.Forget(song, onDisk(fi), 1, nil) + if music.Size != m0-onDisk(fi) || music.Files != 0 { + t.Errorf("file not taken off: %d", music.Size) + } + // folders on tmpfs shrink as entries go (on FAT they don't), so allow + // for that + if want, _ := diskTotal(t, root); s.Tree.Size < want || s.Tree.Size-want > 1024 { + t.Errorf("tree %d after deletes, card %d", s.Tree.Size, want) + } +} + +func TestMeasureCancel(t *testing.T) { + files := map[string]int{} + for i := 0; i < 3000; i++ { + files[fmt.Sprintf("Roms/D%d/f%d", i%30, i)] = 1 + } + root := card(t, files) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := measure(ctx, root, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("got %v", err) + } + if _, err := measure(context.Background(), filepath.Join(root, "missing"), nil); err == nil { + t.Fatal("missing root measured") + } +} + +// The measurer runs in the background while the screen reads its +// progress; run with -race. +func TestMeasurerConcurrent(t *testing.T) { + files := map[string]int{} + for i := 0; i < 4000; i++ { + files[fmt.Sprintf("Roms/S%d/g%d.bin", i%40, i)] = 10 + } + root := card(t, files) + m := &measurer{root: root} + var wg sync.WaitGroup + stop := make(chan struct{}) + for g := 0; g < 4; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + _, p, _, _ := m.state() + if p != nil { + _ = p.Files.Load() + p.Bytes.Load() + _ = p.Where() + } + m.start() + } + }() + } + for i := 0; i < 20; i++ { + m.again() + if i%3 == 0 { + m.stop() + } + } + m.again() + m.wait() + close(stop) + wg.Wait() + m.wait() + s, _, done, err := m.state() + if s == nil || err != nil || done != nil || s.Tree.Files != 4000 { + t.Fatalf("scan %v err %v done %v", s != nil, err, done) + } +} + +// TestStress measures a big, deep card. MINITOP_STRESS=1 makes it 50,000 +// files (the default is 6,000, so CI under qemu stays quick). +func TestStress(t *testing.T) { + n := 6000 + if os.Getenv("MINITOP_STRESS") == "1" { + n = 50000 + } + root := t.TempDir() + t0 := time.Now() + for i := 0; i < n; i++ { + d := filepath.Join(root, "Roms", fmt.Sprintf("SYS%02d", i%40), fmt.Sprintf("set%03d", (i/40)%50)) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("game%05d.bin", i) + if i%10 == 0 { + name = "._" + name + } + if err := os.WriteFile(filepath.Join(d, name), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + // 300 levels deep, ending in an empty folder + deep := filepath.Join(root, "Media") + for i := 0; i < 300; i++ { + deep = filepath.Join(deep, "d") + } + if err := os.MkdirAll(deep, 0o755); err != nil { + t.Fatal(err) + } + made := time.Since(t0) + runtime.GC() + var m0, m1 runtime.MemStats + runtime.ReadMemStats(&m0) + t1 := time.Now() + s, err := measure(context.Background(), root, nil) + took := time.Since(t1) + if err != nil { + t.Fatal(err) + } + runtime.GC() + runtime.ReadMemStats(&m1) + if s.Tree.Files != n { + t.Fatalf("%d files, want %d", s.Tree.Files, n) + } + if got := len(s.Cleanup("junk").Paths); got != n/10 { + t.Errorf("junk %d, want %d", got, n/10) + } + if e := s.Cleanup("empty").Paths; len(e) != 1 || !strings.HasSuffix(e[0], filepath.Join("Media", "d")) { + t.Errorf("empty: %d paths", len(e)) + } + heap := int64(m1.HeapAlloc) - int64(m0.HeapAlloc) + runtime.KeepAlive(s) + t.Logf("%d files, 300 levels: made in %v, measured in %v (%.0f files/s), scan holds %.1f MB of heap", + n, made.Round(time.Millisecond), took.Round(time.Millisecond), float64(n)/took.Seconds(), float64(heap)/(1<<20)) + c := s.Cleanup("junk") + t2 := time.Now() + removed, err := c.run(root, nil) + if err != nil || removed != n/10 { + t.Fatalf("removed %d, %v", removed, err) + } + t.Logf("deleted %d junk files in %v", removed, time.Since(t2).Round(time.Millisecond)) + if heap > 64<<20 { + t.Errorf("the scan holds %d MB", heap>>20) + } +} + +func TestFormat(t *testing.T) { + for b, want := range map[int64]string{0: "0 B", 512: "512 B", 1536: "1.5 KB", 5 << 20: "5.0 MB", 3 << 30: "3.00 GB", 200 << 30: "200 GB"} { + if got := size(b); got != want { + t.Errorf("size(%d) = %q", b, got) + } + } + for n, want := range map[int64]string{0: "0", 999: "999", 1000: "1,000", 1234567: "1,234,567", -4200: "-4,200"} { + if got := count(n); got != want { + t.Errorf("count(%d) = %q", n, got) + } + } +} + +func headless(t *testing.T, root string, keys ...ui.Button) *tasks { + t.Helper() + t.Setenv("MINIDECK_UI_NULL", "1") + scr, err := ui.Open() + if err != nil { + t.Fatal(err) + } + in := make(chan ui.Event, len(keys)+1) + for _, b := range keys { + in <- ui.Event{Button: b, Pressed: true} + } + close(in) + return &tasks{scr: scr, input: in, dir: t.TempDir(), smp: newSampler(), procs: sample, disk: &measurer{root: root}} +} + +// R1 twice reaches Storage and starts measuring; L1 goes back to Startup. +func TestPages(t *testing.T) { + root := card(t, sampleCard) + tk := headless(t, root, ui.R1, ui.R1) + tk.run() + if tk.page != pageStorage { + t.Fatalf("page %d", tk.page) + } + tk.disk.wait() + if s, _, _, _ := tk.disk.state(); s == nil { + t.Fatal("not measured") + } + tk.drawStorage() + if len(tk.run3.Items) != len(tk.rootEnts) || tk.run3.Items[0].Label != "App" || !strings.HasPrefix(tk.run3.Items[0].Sub, "Apps · ") { + t.Errorf("rows %+v", tk.run3.Items) + } + tk.turn(true) + if tk.page != pageRunning { + t.Errorf("R1 on Storage: page %d", tk.page) + } + tk.turn(false) + tk.turn(false) + if tk.page != pageStartup { + t.Errorf("L1 twice from Running: page %d", tk.page) + } +} + +// In Roms, X on the largest folder asks, A deletes it; X on a protected +// row only explains. +func TestBrowseDelete(t *testing.T) { + root := card(t, sampleCard) + tk := headless(t, root, ui.X, ui.A, ui.B) + tk.disk.start() + tk.disk.wait() + s, _, _, _ := tk.disk.state() + roms := s.dirOf(filepath.Join(root, "Roms")) + first := roms.Subs[0] + tk.browse(s, roms) + if _, err := os.Stat(first.Path); err == nil { + t.Fatalf("%s not deleted", first.Name) + } + if s.dirOf(first.Path) != nil { + t.Error("still in the tree") + } + if want, _ := diskTotal(t, root); s.Tree.Size < want || s.Tree.Size-want > 1024 { + t.Errorf("tree %d, card %d", s.Tree.Size, want) + } + + // App/MiniRack as a whole is refused: X shows why, A closes it + tk = headless(t, root, ui.X, ui.A, ui.B) + tk.disk.start() + tk.disk.wait() + s, _, _, _ = tk.disk.state() + app := s.dirOf(filepath.Join(root, "App")) + tk.browse(s, app) + for _, d := range []string{"MiniRack", "MiniMart", "MiniTape", "MiniKey", "MiniScale"} { + if _, err := os.Stat(filepath.Join(root, "App", d)); err != nil { + t.Errorf("App/%s deleted", d) + } + } +} + +// Clean up › macOS leftovers › START › Delete removes them and measures +// the card again. +func TestCleanupScreen(t *testing.T) { + root := card(t, sampleCard) + keys := []ui.Button{} + for range indexOf("junk") { + keys = append(keys, ui.Down) + } + keys = append(keys, ui.A, ui.Start, ui.A, ui.A) + tk := headless(t, root, keys...) + tk.disk.start() + tk.disk.wait() + tk.cleanup() + for _, p := range []string{".DS_Store", ".Trashes", "Roms/GBA/._Pokemon.gba"} { + if _, err := os.Lstat(filepath.Join(root, p)); err == nil { + t.Errorf("%s still there", p) + } + } + if _, err := os.Lstat(filepath.Join(root, "App/MiniKey/._vault.enc")); err != nil { + t.Error("MiniKey's file deleted") + } + tk.disk.wait() + if s, _, _, _ := tk.disk.state(); s == nil || len(s.Cleanup("junk").Paths) != 0 { + t.Error("not measured again") + } +} + +func indexOf(id string) int { + for i, c := range cleanupOrder { + if c == id { + return i + } + } + return -1 +} + +// TestStorageScreens draws the Storage screens to PNG files for review, +// with MINIDECK_GALLERY=DIR set. +func TestStorageScreens(t *testing.T) { + dir := os.Getenv("MINIDECK_GALLERY") + if dir == "" { + t.Skip("set MINIDECK_GALLERY=DIR to draw the screens") + } + // sparse files: real-looking sizes (a few GB) that take no space + root := card(t, nil) + grow := func(p string, n int64) { + full := filepath.Join(root, filepath.FromSlash(p)) + if strings.HasSuffix(p, "/") { + os.MkdirAll(full, 0o755) + return + } + os.MkdirAll(filepath.Dir(full), 0o755) + f, err := os.Create(full) + if err != nil { + t.Fatal(err) + } + f.Truncate(n) + f.Close() + } + for k, v := range sampleCard { + grow(k, min(int64(v)*40000, 2<<30)) + } + for i := 0; i < 30; i++ { + grow(fmt.Sprintf("Roms/SYS%d/game.bin", i), int64(i+1)*3<<20) + } + shot := func(name string, f func(*tasks)) { + t.Setenv("MINIDECK_UI_PNG", filepath.Join(dir, "tasks-"+name+".png")) + scr, err := ui.Open() + if err != nil { + t.Fatal(err) + } + in := make(chan ui.Event) + close(in) + tk := &tasks{scr: scr, input: in, procs: sample, disk: &measurer{root: root}} + f(tk) + } + shot("5-storage-measuring", func(tk *tasks) { + p := &Progress{} + p.Files.Store(12345) + p.Bytes.Store(3 << 30) + p.where.Store("Roms") + tk.disk.prog, tk.disk.done = p, make(chan struct{}) + tk.drawStorage() + }) + measured := func(tk *tasks) *Scan { + tk.disk.start() + tk.disk.wait() + s, _, _, _ := tk.disk.state() + return s + } + shot("6-storage", func(tk *tasks) { measured(tk); tk.drawStorage() }) + shot("7-storage-roms", func(tk *tasks) { s := measured(tk); tk.browse(s, s.dirOf(filepath.Join(root, "Roms"))) }) + shot("8-storage-delete", func(tk *tasks) { + s := measured(tk) + g := s.dirOf(filepath.Join(root, "Roms", "GBA")) + tk.delete(s, Entry{Name: g.Name, Path: g.Path, Size: g.Size, Dir: g}) + }) + shot("9-storage-refused", func(tk *tasks) { + s := measured(tk) + tk.delete(s, Entry{Name: "MiniRack", Path: filepath.Join(root, "App", "MiniRack"), Dir: s.dirOf(filepath.Join(root, "App", "MiniRack"))}) + }) + shot("10-cleanup", func(tk *tasks) { measured(tk); tk.cleanup() }) + shot("11-cleanup-list", func(tk *tasks) { s := measured(tk); tk.cleanupList(s, s.Cleanup("junk")) }) + shot("12-cleanup-confirm", func(tk *tasks) { s := measured(tk); tk.runCleanup(s, s.Cleanup("minimart")) }) +} diff --git a/cmd/minitop/storage_ui.go b/cmd/minitop/storage_ui.go new file mode 100644 index 0000000..7eb98ae --- /dev/null +++ b/cmd/minitop/storage_ui.go @@ -0,0 +1,412 @@ +package main + +import ( + "errors" + "fmt" + "image" + "path/filepath" + "strings" + "time" + + "github.com/zakijss/minideck/ui" +) + +// topAbout describes the card's top-level folders and files on the +// Storage page. +var topAbout = map[string]string{ + "Roms": "Games", + "Saves": "Saved games and states", + "App": "Apps", + "Themes": "Themes", + "Icons": "Icon packs", + "BIOS": "BIOS files for emulators", + "Screenshots": "Screenshots", + "RetroArch": "RetroArch and its cores", + "Emu": "Emulators", + "RApp": "RetroArch apps", + "Media": "Music and videos", + ".tmp_update": "Onion OS", + "miyoo": "Miyoo system files", + "miyoo354": "Miyoo system files", + "cachefile": "Onion's swap file", + "8188fu.ko": "Wi-Fi driver", +} + +// entryItem is a list row for e; top: e is on the card's top level. +func entryItem(e Entry, top bool) ui.Item { + it := ui.Item{Label: e.Name, Detail: size(e.Size)} + if e.Dir != nil { + it.Chevron = true + it.Sub = count(int64(e.Dir.Files)) + " files" + if e.Dir.Files == 1 { + it.Sub = "1 file" + } + } else { + it.Sub = "File" + } + if about := topAbout[e.Name]; top && about != "" { + it.Sub = about + " · " + it.Sub + } + return it +} + +func entryItems(ents []Entry, more int, top bool) []ui.Item { + items := make([]ui.Item, 0, len(ents)+1) + for _, e := range ents { + items = append(items, entryItem(e, top)) + } + if more > 0 { + items = append(items, ui.Item{Label: fmt.Sprintf("And %s smaller files", count(int64(more))), Sub: "Not listed"}) + } + return items +} + +// keepSel selects the row labelled label, if it is still there. +func keepSel(l *ui.List, label string) { + for i, it := range l.Items { + if it.Label == label { + l.Sel = i + return + } + } + if l.Sel >= len(l.Items) { + l.Sel = max(0, len(l.Items)-1) + } +} + +// storageSub is the Storage page's subtitle: free space, or why it's +// unknown. +func storageSub(root string) (string, int64, int64) { + free, total, err := space(root) + if err != nil || total <= 0 { + return "Can't read the card's size", 0, 0 + } + return size(free) + " free of " + size(total), free, total +} + +// drawStorage draws the Storage page: the card's use, then its top-level +// folders and files, largest first. +func (t *tasks) drawStorage() { + s := t.scr + sub, free, total := storageSub(t.disk.root) + top := s.StaticFrame("Storage", sub, + ui.Hint{Button: "A", Label: "Open"}, ui.Hint{Button: "X", Label: "Clean up"}, ui.Hint{Button: "R1", Label: "Running"}, ui.AppearanceHint, ui.Hint{Button: "B", Label: "Exit"}) + card := image.Rect(20, top, ui.W-26, top+62) + s.Panel(card, 12) + pct := 0.0 + if total > 0 { + pct = float64(total-free) / float64(total) * 100 + } + w := s.Text(card.Min.X+16, card.Min.Y+10, fmt.Sprintf("Used %s · %.0f%%", size(total-free), pct), ui.Heading, ui.Text) + avail, _ := Memory() + sfree, stotal := swapInfo() + mem := fmt.Sprintf("RAM %.0f MB free", float64(avail)/(1<<20)) + room := card.Dx() - 32 - w - 20 + if long := mem + fmt.Sprintf(" · swap %.0f MB", float64(sfree)/(1<<20)); stotal > 0 && ui.Measure(ui.Small, long) <= room { + mem = long + } + if room > 40 { + s.TextRight(card.Max.X-16, card.Min.Y+14, ui.Fit(ui.Small, mem, room), ui.Small, ui.Muted) + } + s.Bar(image.Rect(card.Min.X+16, card.Max.Y-22, card.Max.X-16, card.Max.Y-10), pct/100, ui.Level(pct)) + area := image.Rect(20, card.Max.Y+10, ui.W-26, ui.H-58) + + scan, prog, done, err := t.disk.state() + switch { + case scan == nil && done != nil: + t.drawMeasuring(area, prog) + case scan == nil: + msg := "Not measured yet" + if err != nil { + msg = "Couldn't measure the card: " + err.Error() + } + s.TextCenter(ui.W/2, area.Min.Y+area.Dy()/2-10, ui.Fit(ui.Body, msg, area.Dx()), ui.Body, ui.Muted) + default: + if t.rootScan != scan { + t.rootScan = scan + t.rootEnts, t.rootMore = Contents(scan.Tree) + } + label := "" + if sel := t.run3.Selected(); sel != nil { + label = sel.Label + } + t.run3.Items = entryItems(t.rootEnts, t.rootMore, true) + keepSel(&t.run3, label) + t.run3.Draw(s, area) + } + s.Present() +} + +// drawMeasuring shows a running measurement's progress in r. +func (t *tasks) drawMeasuring(r image.Rectangle, p *Progress) { + s := t.scr + cy := r.Min.Y + r.Dy()/2 - 24 + s.Loading(cy, "Measuring the card…") + line := count(p.Files.Load()) + " files · " + size(p.Bytes.Load()) + if w := p.Where(); w != "" { + line += " · " + w + } + s.TextCenter(ui.W/2, cy+34, ui.Fit(ui.Small, line, r.Dx()), ui.Small, ui.Muted) +} + +// openStorage opens the selected row of the Storage page: a folder's +// contents, or the delete question for a file. +func (t *tasks) openStorage() { + scan, _, _, _ := t.disk.state() + if scan == nil || t.rootScan != scan || t.run3.Sel >= len(t.rootEnts) { + return + } + e := t.rootEnts[t.run3.Sel] + if e.Dir != nil { + t.browse(scan, e.Dir) + } else { + t.delete(scan, e) + } + t.rootScan = nil // sizes may have changed + if t.stale { + t.stale = false + t.disk.again() + } +} + +// browse lists folder d's contents, largest first. A opens a folder, X +// (or A on a file) asks to delete the selected row, B goes back. +func (t *tasks) browse(scan *Scan, d *Dir) { + l := ui.List{} + ents, more := Contents(d) + for { + label := "" + if sel := l.Selected(); sel != nil { + label = sel.Label + } + l.Items = entryItems(ents, more, false) + keepSel(&l, label) + sub := size(d.Size) + " · " + count(int64(d.Files)) + " files" + if rel := d.Rel(scan.Root); strings.Contains(rel, "/") { + sub = filepath.Dir(rel) + " · " + sub + } + top := t.scr.StaticFrame(d.Name, ui.Fit(ui.Small, sub, ui.W-60), + ui.Hint{Button: "A", Label: "Open"}, ui.Hint{Button: "X", Label: "Delete"}, ui.Hint{Button: "B", Label: "Back"}) + if len(l.Items) == 0 { + t.scr.TextCenter(ui.W/2, top+80, "Empty", ui.Body, ui.Muted) + } + l.Draw(t.scr, image.Rect(20, top, ui.W-26, ui.H-58)) + t.scr.Present() + ev, ok, frame := t.scr.Next(t.input) + if !ok { + return + } + if frame || !acts(ev) { + continue + } + switch ev.Button { + case ui.B, ui.Menu: + return + case ui.A, ui.X: + if l.Sel >= len(ents) { + continue + } + e := ents[l.Sel] + if e.Dir != nil && ev.Button == ui.A { + t.browse(scan, e.Dir) + } else { + t.delete(scan, e) + } + ents, more = Contents(d) + default: + l.Handle(ev) + } + } +} + +// reason is a refusal's text for the screen. +func reason(err error) string { + msg := strings.TrimPrefix(err.Error(), errRefused.Error()+": ") + if msg == "" { + return msg + } + return strings.ToUpper(msg[:1]) + msg[1:] + "." +} + +// delete asks before deleting e for good, unless checkDelete refuses it. +func (t *tasks) delete(scan *Scan, e Entry) { + if err := checkDelete(scan.Root, e.Path); err != nil { + if errors.Is(err, errRefused) { + t.scr.Notice(t.input, "Can't delete "+e.Name, reason(err)) + } else { + t.scr.Notice(t.input, "Can't delete "+e.Name, err.Error()) + } + return + } + what := "This file" + if e.Dir != nil { + what = "This folder and the " + count(int64(e.Dir.Files)) + " files in it" + if e.Dir.Files == 1 { + what = "This folder and the file in it" + } + } + body := fmt.Sprintf("%s (%s) are deleted for good: the card has no trash.%s", what, size(e.Size), warning(scan.Root, e.Path)) + if e.Dir == nil { + body = fmt.Sprintf("This file (%s) is deleted for good: the card has no trash.%s", size(e.Size), warning(scan.Root, e.Path)) + } + if !t.scr.Confirm(t.input, "Delete "+e.Name+"?", body, "Delete") { + return + } + t.scr.Alert("Deleting…", e.Name) + if err := remove(e.Path); err != nil { + t.stale = true // part of it may be gone: measure again + t.scr.Notice(t.input, "Couldn't delete all of it", err.Error()) + return + } + scan.Forget(e.Path, e.Size, 1, e.Dir) +} + +// next waits for input; while a measurement runs (done is not nil) it +// also returns, with got false, when it ends or a quarter second passes, +// so the screen can show progress. +func (t *tasks) next(done <-chan struct{}) (ev ui.Event, ok, got bool) { + if done == nil { + ev, ok, frame := t.scr.Next(t.input) + return ev, ok, !frame + } + select { + case ev, ok = <-t.input: + return ev, ok, true + case <-done: + case <-time.After(250 * time.Millisecond): + } + return ui.Event{}, true, false +} + +// cleanupItem is the Clean up row for c. +func cleanupItem(c *Cleanup) ui.Item { + it := ui.Item{Label: c.Name, Sub: c.About, Detail: "Nothing", Chevron: len(c.Paths) > 0} + if len(c.Paths) > 0 { + it.Detail = size(c.Size) + } + return it +} + +// cleanup is the Clean up screen (X on Storage): each clean-up with how +// much it frees. A lists what one would delete. +func (t *tasks) cleanup() { + t.disk.start() + l := ui.List{} + for { + scan, prog, done, err := t.disk.state() + top := t.scr.StaticFrame("Clean up", "Frees space; games, saves and settings stay", + ui.Hint{Button: "A", Label: "Show"}, ui.Hint{Button: "B", Label: "Back"}) + area := image.Rect(20, top, ui.W-26, ui.H-58) + switch { + case scan != nil: + l.Items = l.Items[:0] + for _, c := range scan.Cleanups { + l.Items = append(l.Items, cleanupItem(c)) + } + l.Draw(t.scr, area) + case done != nil: + t.drawMeasuring(area, prog) + default: + msg := "Not measured yet" + if err != nil { + msg = "Couldn't measure the card: " + err.Error() + } + t.scr.TextCenter(ui.W/2, area.Min.Y+area.Dy()/2-10, ui.Fit(ui.Body, msg, area.Dx()), ui.Body, ui.Muted) + } + t.scr.Present() + ev, ok, got := t.next(done) + if !ok { + return + } + if !got || !acts(ev) { + continue + } + switch ev.Button { + case ui.B, ui.Menu: + return + case ui.A: + if scan == nil || l.Sel >= len(scan.Cleanups) { + continue + } + c := scan.Cleanups[l.Sel] + if len(c.Paths) == 0 { + t.scr.Notice(t.input, c.Name, "Nothing to clean up here.") + continue + } + t.cleanupList(scan, c) + default: + l.Handle(ev) + } + } +} + +// cleanupList shows exactly what clean-up c deletes; START deletes it all +// after asking. +func (t *tasks) cleanupList(scan *Scan, c *Cleanup) { + items := make([]ui.Item, len(c.Paths)) + for i, p := range c.Paths { + rel, _ := filepath.Rel(scan.Root, p) + items[i] = ui.Item{Label: filepath.ToSlash(rel), Detail: size(c.Sizes[i])} + } + l := ui.List{Items: items} + for { + sub := fmt.Sprintf("%s items · %s", count(int64(len(c.Paths))), size(c.Size)) + top := t.scr.StaticFrame(c.Name, sub, + ui.Hint{Button: "START", Label: "Delete all"}, ui.Hint{Button: "B", Label: "Back"}) + l.Draw(t.scr, image.Rect(20, top, ui.W-26, ui.H-58)) + t.scr.Present() + ev, ok, frame := t.scr.Next(t.input) + if !ok { + return + } + if frame || !acts(ev) { + continue + } + switch ev.Button { + case ui.B, ui.Menu: + return + case ui.Start: + t.runCleanup(scan, c) + return + default: + l.Handle(ev) + } + } +} + +// runCleanup asks, deletes everything c lists with a progress line, says +// how much was freed, and measures the card again. +func (t *tasks) runCleanup(scan *Scan, c *Cleanup) { + n := len(c.Paths) + noun := "items" + if n == 1 { + noun = "item" + } + body := fmt.Sprintf("%s %s (%s) are deleted for good: the card has no trash.", count(int64(n)), noun, size(c.Size)) + if c.Note != "" { + body += " " + c.Note + } + if !t.scr.Confirm(t.input, c.Ask, body, "Delete") { + return + } + before, _, _ := space(scan.Root) + last := time.Time{} + done, err := c.run(scan.Root, func(i, total int) { + if time.Since(last) > 150*time.Millisecond || i == total { + last = time.Now() + t.scr.Alert("Deleting…", fmt.Sprintf("%s of %s", count(int64(i)), count(int64(total)))) + } + }) + after, _, _ := space(scan.Root) + freed := after - before + if freed <= 0 { + freed = c.Size + } + msg := fmt.Sprintf("Deleted %s of %s and freed %s.", count(int64(done)), count(int64(n)), size(freed)) + if err != nil { + msg += " Some were left: " + reason(err) + } + t.disk.again() + t.rootScan = nil + t.scr.Notice(t.input, "Cleaned up", msg) +} diff --git a/go.mod b/go.mod index 58b9ba7..a697fd7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/zakijss/minitop go 1.27 require ( - github.com/zakijss/minideck v1.0.0 + github.com/zakijss/minideck v1.0.1 golang.org/x/crypto/x509roots/fallback v0.0.0-20261020153306-9f2b28d4331a ) diff --git a/go.sum b/go.sum index af68850..0b62d53 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/zakijss/minideck v1.0.0 h1:pfmswsKD2o+Z9392WifQSde92saqkExavW3M0igtznM= -github.com/zakijss/minideck v1.0.0/go.mod h1:S+PxcQZzscZXyB/124oF/lSHQneBhmEEVOUdfE+ZwK4= +github.com/zakijss/minideck v1.0.1 h1:8Xh8cConmiMo9UBTQ080yaH5G3uGXks3WKpvZYGHo/Q= +github.com/zakijss/minideck v1.0.1/go.mod h1:S+PxcQZzscZXyB/124oF/lSHQneBhmEEVOUdfE+ZwK4= golang.org/x/crypto/x509roots/fallback v0.0.0-20261020153306-9f2b28d4331a h1:V5oicZXS0oqZx30gY/apgSVD7YvNBfrIOdcMnlelJY4= golang.org/x/crypto/x509roots/fallback v0.0.0-20261020153306-9f2b28d4331a/go.mod h1:HPze8vhfG6fO06AM+VSvxRm4E3+5Yk375mgrJ5M2z1E= golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= diff --git a/store.json b/store.json index 54df9c9..f3ef60f 100644 --- a/store.json +++ b/store.json @@ -1,7 +1,7 @@ { "name": "MiniTop", "folder": "MiniTop", - "desc": "What's running, and what starts at boot", + "desc": "Tasks and maintenance: running programs, startup, storage", "repo": "zakijss/MiniTop", "license": "MIT", "version": "__VERSION__", @@ -11,7 +11,8 @@ "strip": "App/", "released": "__RELEASED__", "facts": [ - "Protects Onion's own processes from being ended." + "Protects Onion's own processes from being ended.", + "Deletes files on the card only after asking, and never Onion's files, whole app folders or key files." ], "system": "Adds a boot script (.tmp_update/startup/clocksync.sh) that sets the clock from HTTPS servers after Wi-Fi is up. Turn it off in MiniTop > Startup.", "boot": [ From 2a412ec49d36c44f931c4d8d223c21bd9c945a16 Mon Sep 17 00:00:00 2001 From: zakijss <210305895+zakijss@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:27:39 -0500 Subject: [PATCH 2/2] Keep the last storage measurement; find duplicate games Storage shows the measurement kept from the last session at once (the folder tree only, in storage.json, with its age) while the card is measured again; Clean up uses only a measurement from this session. On the device the 100 GB card's tree is a 94 KB file. Duplicates (Y on Storage) groups ROMs under Roms by size, then by their first and last 64 KB, and confirms with a full SHA-1, so most files are never read. Ports, ScummVM, Imgs and .bin tracks are skipped; a file that a .cue, .m3u, .gdi or .ccd in its folder names is marked in use and can't be deleted alone, nor can the last copy. Deleting one clears that system's Onion list cache. A 20,000-file stress test with 2,000 same-size near-misses runs in about 1 s; on the device the real card took under a minute and found two FDS games stored three and two times. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XNFhx3fuFPdPMoCFatrpkM --- CHANGELOG.md | 2 + cmd/minitop/dupes.go | 333 ++++++++++++++++++++++++++++++++++++ cmd/minitop/dupes_test.go | 285 ++++++++++++++++++++++++++++++ cmd/minitop/dupes_ui.go | 222 ++++++++++++++++++++++++ cmd/minitop/main.go | 11 +- cmd/minitop/savedscan.go | 120 +++++++++++++ cmd/minitop/storage.go | 41 ++++- cmd/minitop/storage_test.go | 36 ++++ cmd/minitop/storage_ui.go | 17 +- 9 files changed, 1061 insertions(+), 6 deletions(-) create mode 100644 cmd/minitop/dupes.go create mode 100644 cmd/minitop/dupes_test.go create mode 100644 cmd/minitop/dupes_ui.go create mode 100644 cmd/minitop/savedscan.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 56e7e20..f746e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes are listed here. The format follows - Storage page (**R1** from Startup): free space, folder sizes largest first, and deleting a file or folder after asking. It refuses Onion's files, the card's top level, whole app folders, MiniKey and key or `.enc` files. - Clean up (**X** on Storage): MiniMart downloads, MiniTape songs, game list caches, logs, macOS leftovers and empty folders, each listed in full before anything is deleted. +- Storage opens at once with the last measurement (kept in `storage.json`, labelled with its age) and measures again in the background. Clean up waits for the new measurement, so it never deletes from an old list. +- Duplicates (**Y** on Storage): games under `Roms` that are byte for byte the same (same size, same first and last 64 KB, then the same SHA-1), the ones wasting most space first. X deletes one copy after asking; the last copy can't be deleted, and neither can a disc that a `.cue`, `.m3u`, `.gdi` or `.ccd` in its folder names. Ports, ScummVM games, box art and `.bin` disc tracks aren't looked at. Onion's game list for that system is refreshed. ### Changed diff --git a/cmd/minitop/dupes.go b/cmd/minitop/dupes.go new file mode 100644 index 0000000..eeb8753 --- /dev/null +++ b/cmd/minitop/dupes.go @@ -0,0 +1,333 @@ +package main + +// Duplicate games: ROM files under Roms/ that are byte for byte the same. +// Files are grouped by size, then by their first and last 64 KB, and only +// then read whole for a SHA-1, so most of the card is never read. Ports, +// ScummVM games (folders of shared engine files), box art and disc tracks +// (.bin, which .cue sheets name) are left out, and a file that a .cue, +// .m3u, .gdi or .ccd in its folder names is marked as in use: deleting it +// alone would break that game. + +import ( + "bufio" + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync/atomic" +) + +// romExts are the single-file game formats looked at, lower case. +var romExts = map[string]bool{} + +func init() { + for _, e := range strings.Fields(`gb gbc gba sgb nes fds unf sfc smc fig swc md gen smd 32x sms gg sg pce sgx + n64 z64 v64 nds a26 a52 a78 lnx j64 jag ws wsc ngp ngc vb col int vec min + pbp chd iso cso rvz zip 7z`) { + romExts["."+e] = true + } +} + +// dupSkipSystems are folders of Roms/ that are never looked in. +var dupSkipSystems = map[string]bool{"PORTS": true, "SCUMMVM": true} + +// minDupSize: smaller files aren't looked at (a few KB of homebrew). +const minDupSize = 16 << 10 + +// maxDupDepth: how deep under Roms/ files are looked for. +const maxDupDepth = 4 + +// DupFile is one copy. +type DupFile struct { + Path string + Rel string // from the card's root + InUse string // the .cue/.m3u/.gdi/.ccd in its folder that names it, if any +} + +// DupGroup is a set of identical files. +type DupGroup struct { + Size int64 + Sum string // SHA-1 + Files []DupFile +} + +// Wasted is the space the extra copies take. +func (g DupGroup) Wasted() int64 { return g.Size * int64(len(g.Files)-1) } + +// DupProgress is a running search's progress. +type DupProgress struct { + Files atomic.Int64 // ROM files found + Read atomic.Int64 // bytes read for checksums + Total atomic.Int64 // bytes to read +} + +type dupCand struct { + path, rel string + size int64 + inUse string +} + +// findDuplicates looks for identical games under root/Roms. +func findDuplicates(ctx context.Context, root string, prog *DupProgress) ([]DupGroup, error) { + if prog == nil { + prog = &DupProgress{} + } + var cands []dupCand + var walk func(dir, rel string, depth int) error + walk = func(dir, rel string, depth int) error { + if err := ctx.Err(); err != nil { + return err + } + ents, err := os.ReadDir(dir) + if err != nil { + return nil // an unreadable folder is skipped + } + used := discRefs(dir, ents) + for _, e := range ents { + name := e.Name() + if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "._") { + continue + } + p, r := filepath.Join(dir, name), rel+"/"+name + switch { + case e.IsDir(): + if depth == 0 && dupSkipSystems[strings.ToUpper(name)] || strings.EqualFold(name, "Imgs") || depth >= maxDupDepth { + continue + } + if err := walk(p, r, depth+1); err != nil { + return err + } + case e.Type().IsRegular() && depth > 0 && romExts[strings.ToLower(filepath.Ext(name))]: + fi, err := e.Info() + if err != nil || fi.Size() < minDupSize { + continue + } + prog.Files.Add(1) + cands = append(cands, dupCand{p, r, fi.Size(), used[strings.ToLower(name)]}) + } + } + return nil + } + if err := walk(filepath.Join(root, "Roms"), "Roms", 0); err != nil { + return nil, err + } + + bySize := map[int64][]dupCand{} + for _, c := range cands { + bySize[c.size] = append(bySize[c.size], c) + } + // Ends first: files of one size that differ almost always differ there. + var full [][]dupCand + for _, cs := range bySize { + if len(cs) < 2 { + continue + } + byEnds := map[string][]dupCand{} + for _, c := range cs { + k, err := endsKey(c.path, c.size) + if err != nil { + continue + } + byEnds[k] = append(byEnds[k], c) + } + for _, g := range byEnds { + if len(g) > 1 { + full = append(full, g) + for _, c := range g { + prog.Total.Add(c.size) + } + } + } + } + var out []DupGroup + for _, g := range full { + bySum := map[string][]dupCand{} + for _, c := range g { + sum, err := fileSHA1(ctx, c.path, prog) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + continue + } + bySum[sum] = append(bySum[sum], c) + } + for sum, cs := range bySum { + if len(cs) < 2 { + continue + } + dg := DupGroup{Size: cs[0].size, Sum: sum} + sort.Slice(cs, func(i, j int) bool { return cs[i].rel < cs[j].rel }) + for _, c := range cs { + dg.Files = append(dg.Files, DupFile{Path: c.path, Rel: c.rel, InUse: c.inUse}) + } + out = append(out, dg) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Wasted() != out[j].Wasted() { + return out[i].Wasted() > out[j].Wasted() + } + return out[i].Files[0].Rel < out[j].Files[0].Rel + }) + return out, nil +} + +const endBytes = 64 << 10 + +// endsKey is the file's first and last 64 KB, hashed. +func endsKey(p string, size int64) (string, error) { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + h := sha1.New() + if _, err := io.CopyN(h, f, min(size, endBytes)); err != nil { + return "", err + } + if size > endBytes { + if _, err := f.Seek(max(size-endBytes, endBytes), io.SeekStart); err != nil { + return "", err + } + if _, err := io.Copy(h, f); err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// fileSHA1 reads the whole file, stopping when ctx is cancelled. +func fileSHA1(ctx context.Context, p string, prog *DupProgress) (string, error) { + f, err := os.Open(p) + if err != nil { + return "", err + } + defer f.Close() + h := sha1.New() + buf := make([]byte, 1<<20) + for { + if err := ctx.Err(); err != nil { + return "", err + } + n, err := f.Read(buf) + h.Write(buf[:n]) + prog.Read.Add(int64(n)) + if err == io.EOF { + break + } + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +var ( + cueFileRe = regexp.MustCompile(`(?i)^\s*FILE\s+(?:"([^"]+)"|(\S+))`) + gdiNameRe = regexp.MustCompile(`(?i)(?:"([^"]+\.(?:bin|raw|iso))"|(\S+\.(?:bin|raw|iso)))`) +) + +// discRefs reads the disc sheets and playlists in a folder and returns the +// file names they name (lower case) with the sheet naming each. +func discRefs(dir string, ents []os.DirEntry) map[string]string { + out := map[string]string{} + for _, e := range ents { + name := e.Name() + ext := strings.ToLower(filepath.Ext(name)) + if !e.Type().IsRegular() || ext != ".cue" && ext != ".m3u" && ext != ".gdi" && ext != ".ccd" { + continue + } + if ext == ".ccd" { // CloneCD: the image and subchannel share its name + stem := strings.ToLower(strings.TrimSuffix(name, filepath.Ext(name))) + out[stem+".img"], out[stem+".sub"] = name, name + continue + } + b, err := readSmall(filepath.Join(dir, name), 256<<10) + if err != nil { + continue + } + sc := bufio.NewScanner(bytes.NewReader(b)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + var ref string + switch ext { + case ".cue": + if m := cueFileRe.FindStringSubmatch(line); m != nil { + ref = m[1] + m[2] + } + case ".m3u": + if line != "" && !strings.HasPrefix(line, "#") { + ref = line + } + case ".gdi": + if m := gdiNameRe.FindStringSubmatch(line); m != nil { + ref = m[1] + m[2] + } + } + if ref != "" { + out[strings.ToLower(filepath.Base(strings.ReplaceAll(ref, `\`, "/")))] = name + } + } + } + return out +} + +func readSmall(p string, limit int64) ([]byte, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, limit+1)) + if int64(len(b)) > limit { + return nil, errors.New("too big") + } + return b, err +} + +// deleteCopy deletes one copy of g, refusing the last one still on the +// card, one a disc sheet or playlist uses, and anything checkDelete +// refuses. It returns the group without it, and makes Onion re-read the +// game list of the system the copy was in. +func deleteCopy(root string, g DupGroup, i int) (DupGroup, error) { + f := g.Files[i] + if f.InUse != "" { + return g, refuse(f.InUse + " uses it: delete the whole game instead") + } + others := 0 + for j, o := range g.Files { + if j != i && fileExists(o.Path) { + others++ + } + } + if others == 0 { + return g, refuse("it's the only copy left") + } + if err := checkDelete(root, f.Path); err != nil { + return g, err + } + if err := os.Remove(f.Path); err != nil && !os.IsNotExist(err) { + return g, err + } + if parts := strings.Split(f.Rel, "/"); len(parts) > 2 { + sys := parts[1] + for _, v := range []string{"_cache6.db", "_cache2.db"} { + os.Remove(filepath.Join(root, "Roms", sys, sys+v)) + } + } + g.Files = append(g.Files[:i:i], g.Files[i+1:]...) + return g, nil +} + +func fileExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && fi.Mode().IsRegular() +} diff --git a/cmd/minitop/dupes_test.go b/cmd/minitop/dupes_test.go new file mode 100644 index 0000000..60054b1 --- /dev/null +++ b/cmd/minitop/dupes_test.go @@ -0,0 +1,285 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/zakijss/minideck/ui" +) + +// writeAt makes a file with the given contents under root. +func writeAt(t testing.TB, root, rel string, b []byte) string { + t.Helper() + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, b, 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// blob is n bytes of a pattern seeded by s. +func blob(s byte, n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = s + byte(i*7) + } + return b +} + +func TestFindDuplicates(t *testing.T) { + root := t.TempDir() + game := blob(1, 200<<10) + writeAt(t, root, "Roms/GBA/Emerald.gba", game) + writeAt(t, root, "Roms/GBA/Hacks/Emerald (copy).gba", game) + writeAt(t, root, "Roms/GBA/old/Emerald.gba", game) + // same size and the same first and last 64 KB, different in the middle + mid := append([]byte(nil), game...) + mid[100<<10] ^= 0xff + writeAt(t, root, "Roms/GBA/Emerald (rev1).gba", mid) + // identical, but in places that aren't looked at + writeAt(t, root, "Roms/PORTS/Games/A/libSDL2.zip", game) + writeAt(t, root, "Roms/SCUMMVM/monkey/data.zip", game) + writeAt(t, root, "Roms/GBA/Imgs/Emerald.zip", game) + writeAt(t, root, "Roms/Emerald.gba", game) // directly in Roms/ + writeAt(t, root, "Roms/PS/Game (Track 1).bin", game) // disc tracks aren't looked at + writeAt(t, root, "Roms/GBA/notes.txt", game) // not a game format + writeAt(t, root, "Roms/GB/tiny.gb", blob(3, 8<<10)) // too small + writeAt(t, root, "Roms/GB/tiny copy.gb", blob(3, 8<<10)) + // a multi-disc game: both discs listed by the .m3u, plus a loose copy + disc := blob(9, 300<<10) + writeAt(t, root, "Roms/PS/FF7/FF7 (Disc 1).chd", disc) + writeAt(t, root, "Roms/PS/FF7/FF7.m3u", []byte("# discs\nFF7 (Disc 1).chd\nFF7 (Disc 2).chd\n")) + writeAt(t, root, "Roms/PS/FF7 (Disc 1).chd", disc) + + groups, err := findDuplicates(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + if len(groups) != 2 { + for _, g := range groups { + t.Log(g.Files) + } + t.Fatalf("%d groups", len(groups)) + } + gba, ps := groups[0], groups[1] // most space wasted first: 2 x 200 KB, then 1 x 300 KB + if gba.Size != int64(len(game)) || len(gba.Files) != 3 || gba.Wasted() != 2*int64(len(game)) { + t.Fatalf("gba group %+v", gba) + } + for _, f := range gba.Files { + if strings.Contains(f.Rel, "rev1") { + t.Fatal("a file that differs in the middle was taken for a copy") + } + } + if len(ps.Files) != 2 || ps.Files[0].Rel != "Roms/PS/FF7 (Disc 1).chd" || ps.Files[0].InUse != "" || ps.Files[1].InUse != "FF7.m3u" { + t.Fatalf("ps group %+v", ps.Files) + } + + // deleting: the in-use copy and the last copy are refused + if _, err := deleteCopy(root, ps, 1); !errors.Is(err, errRefused) { + t.Fatalf("deleted a disc the playlist uses: %v", err) + } + cache := writeAt(t, root, "Roms/PS/PS_cache6.db", []byte("x")) + ng, err := deleteCopy(root, ps, 0) + if err != nil || len(ng.Files) != 1 || fileExists(ps.Files[0].Path) { + t.Fatalf("delete loose copy: %v %+v", err, ng) + } + if fileExists(cache) { + t.Error("Onion's list cache for the system wasn't cleared") + } + if _, err := deleteCopy(root, ng, 0); !errors.Is(err, errRefused) { + t.Fatal("deleted the last copy") + } + // a copy deleted behind its back: the remaining one is still the last + os.Remove(gba.Files[2].Path) + g2, err := deleteCopy(root, gba, 1) + if err != nil { + t.Fatal(err) + } + if _, err := deleteCopy(root, g2, 0); !errors.Is(err, errRefused) { + t.Fatal("deleted the only copy left on the card") + } +} + +func TestDiscRefs(t *testing.T) { + dir := t.TempDir() + writeAt(t, dir, "a.cue", []byte("FILE \"Game (Track 1).bin\" BINARY\n TRACK 01 MODE2/2352\nFILE track2.bin BINARY\n")) + writeAt(t, dir, "b.gdi", []byte("3\n1 0 4 2352 track01.bin 0\n2 600 0 2352 \"track 02.raw\" 0\n")) + writeAt(t, dir, "c.ccd", []byte("[CloneCD]\n")) + writeAt(t, dir, "d.m3u", []byte("#EXTM3U\nsub\\Disc 2.chd\n\n")) + ents, _ := os.ReadDir(dir) + got := discRefs(dir, ents) + want := map[string]string{"game (track 1).bin": "a.cue", "track2.bin": "a.cue", "track01.bin": "b.gdi", "track 02.raw": "b.gdi", + "c.img": "c.ccd", "c.sub": "c.ccd", "disc 2.chd": "d.m3u"} + for k, v := range want { + if got[k] != v { + t.Errorf("%q: %q, want %q", k, got[k], v) + } + } + if len(got) != len(want) { + t.Errorf("got %v", got) + } +} + +func TestFindDuplicatesCancel(t *testing.T) { + root := t.TempDir() + for i := 0; i < 4; i++ { + writeAt(t, root, fmt.Sprintf("Roms/GBA/g%d.gba", i), blob(1, 4<<20)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := findDuplicates(ctx, root, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("err %v", err) + } +} + +// TestDuplicatesStress: 20,000 games in 40 systems, with 2,000 of one +// size that differ only in the middle, and 300 real copies. +func TestDuplicatesStress(t *testing.T) { + if testing.Short() { + t.Skip("short") + } + root := t.TempDir() + base := blob(5, 64<<10+512) // ends are the same for all, middles differ + for i := 0; i < 20000; i++ { + var b []byte + switch { + case i < 2000: + b = append([]byte(nil), base...) + copy(b[64<<10-8:], fmt.Sprintf("%08d", i)) // just before the last 64 KB... still in the middle + b[(64<<10)+100] = byte(i) + b[(64<<10)+101] = byte(i >> 8) + case i < 2300: + b = blob(byte(i%250), 20<<10+i) // pairs below + default: + b = []byte(fmt.Sprintf("%020d", i)) + b = bytes.Repeat(b, (16<<10)/len(b)+1+i%7) + } + writeAt(t, root, fmt.Sprintf("Roms/S%02d/g%05d.gba", i%40, i), b) + if i >= 2000 && i < 2300 { + writeAt(t, root, fmt.Sprintf("Roms/S%02d/copy/g%05d.gba", (i+1)%40, i), b) + } + } + prog := &DupProgress{} + start := time.Now() + groups, err := findDuplicates(context.Background(), root, prog) + if err != nil { + t.Fatal(err) + } + took := time.Since(start) + if len(groups) != 300 { + t.Fatalf("%d groups, want 300", len(groups)) + } + t.Logf("%d files, %d groups, read %d of %d bytes for checksums, %v", prog.Files.Load(), len(groups), prog.Read.Load(), prog.Total.Load(), took) +} + +// The Duplicates screen, scripted: Y on Storage opens it, A opens the +// first game, X deletes the selected copy after asking (A), and the +// game leaves the list once one copy is left. +func TestDuplicatesScreen(t *testing.T) { + root := t.TempDir() + game := blob(2, 100<<10) + a := writeAt(t, root, "Roms/GB/Tetris.gb", game) + b := writeAt(t, root, "Roms/GB/dupes/Tetris.gb", game) + tk := headless(t, root, ui.R1, ui.R1, ui.Y, ui.A, ui.Down, ui.X, ui.A, ui.A, ui.B, ui.B) + tk.disk.dir = t.TempDir() + // a finished search, as when the user presses after the results show + groups, err := findDuplicates(context.Background(), root, nil) + if err != nil || len(groups) != 1 { + t.Fatalf("search: %v %d", err, len(groups)) + } + end := make(chan struct{}) + close(end) + tk.dupes = &dupSearch{groups: groups, done: true, prog: &DupProgress{}, end: end, cancel: func() {}} + done := make(chan struct{}) + go func() { tk.run(); close(done) }() + select { + case <-done: + case <-time.After(20 * time.Second): + t.Fatal("screens didn't finish") + } + if !fileExists(a) || fileExists(b) { + t.Fatalf("kept %v, deleted %v", fileExists(a), !fileExists(b)) + } + if tk.dupes == nil || len(tk.dupes.groups) != 0 { + t.Fatalf("groups left: %+v", tk.dupes) + } +} + +func TestSavedScan(t *testing.T) { + root := card(t, sampleCard) + dir := t.TempDir() + s, err := measure(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + if err := saveScan(dir, s); err != nil { + t.Fatal(err) + } + got, err := loadScan(dir, root) + if err != nil { + t.Fatal(err) + } + if !got.Saved || got.Cleanups != nil || got.Tree.Size != s.Tree.Size || got.Tree.Files != s.Tree.Files || !got.Measured.Equal(s.Measured) { + t.Fatalf("loaded %+v", got) + } + // paths and parents are rebuilt: a folder can be found and forgotten + gba := got.dirOf(filepath.Join(root, "Roms", "GBA")) + if gba == nil || gba.Path != filepath.Join(root, "Roms", "GBA") || gba.parent == nil || gba.parent.Name != "Roms" { + t.Fatalf("Roms/GBA: %+v", gba) + } + before := got.Tree.Size + got.Forget(gba.Path, 0, 0, gba) + if got.Tree.Size != before-gba.Size { + t.Fatal("forget on a loaded scan") + } + if _, err := loadScan(dir, root+"x"); err == nil { + t.Fatal("used a measurement of another card") + } + for _, bad := range []string{`{`, `{"v":2}`, `{"v":1,"root":"` + root + `","measured":"2026-01-01T00:00:00Z","tree":{"n":"x","d":[{"n":"../etc"}]}}`, + `{"v":1,"root":"` + root + `","measured":"2026-01-01T00:00:00Z","tree":{"n":"x","s":-5}}`} { + os.WriteFile(scanFile(dir), []byte(bad), 0o644) + if _, err := loadScan(dir, root); err == nil { + t.Errorf("loaded %s", bad) + } + } + if got, want := ago(time.Now().Add(-3*time.Hour-time.Minute), time.Now()), "3 h ago"; got != want { + t.Errorf("ago %q", got) + } +} + +// The measurer shows the kept measurement at once, measures again, and +// then shows (and keeps) the new one. +func TestMeasurerShowsKeptScan(t *testing.T) { + root := card(t, sampleCard) + dir := t.TempDir() + m := &measurer{root: root, dir: dir} + m.start() + m.wait() + first := m.view() + if first == nil || first.Saved { + t.Fatal("first measurement") + } + writeAt(t, root, "Roms/NES/new.nes", make([]byte, 50000)) + m2 := &measurer{root: root, dir: dir} // the next session + m2.start() + if v := m2.view(); v == nil || !v.Saved || v.Tree.Size != first.Tree.Size { + t.Fatalf("kept measurement not shown at once: %+v", v) + } + m2.wait() + if v := m2.view(); v == nil || v.Saved || v.Tree.Size <= first.Tree.Size { + t.Fatal("new measurement not shown") + } + if kept, _ := loadScan(dir, root); kept == nil || kept.Tree.Size != m2.view().Tree.Size { + t.Fatal("new measurement not kept") + } +} diff --git a/cmd/minitop/dupes_ui.go b/cmd/minitop/dupes_ui.go new file mode 100644 index 0000000..ebee57a --- /dev/null +++ b/cmd/minitop/dupes_ui.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "errors" + "fmt" + "image" + "path" + "strings" + + "github.com/zakijss/minideck/ui" +) + +// dupSearch is the duplicate search of this session. +type dupSearch struct { + groups []DupGroup + done bool + err error + prog *DupProgress + end chan struct{} + cancel context.CancelFunc +} + +func (t *tasks) startDupes() { + ctx, cancel := context.WithCancel(context.Background()) + d := &dupSearch{prog: &DupProgress{}, end: make(chan struct{}), cancel: cancel} + t.dupes = d + go func() { + g, err := findDuplicates(ctx, t.disk.root, d.prog) + t.dmu.Lock() + d.groups, d.err, d.done = g, err, true + t.dmu.Unlock() + close(d.end) + }() +} + +// dupLine is a group's second line: how many copies, and where. +func dupLine(g DupGroup) string { + var where []string + seen := map[string]bool{} + for _, f := range g.Files { + d := strings.TrimPrefix(path.Dir(f.Rel), "Roms/") + if !seen[d] { + seen[d] = true + where = append(where, d) + } + } + return fmt.Sprintf("%d copies · %s each · %s", len(g.Files), size(g.Size), strings.Join(where, ", ")) +} + +// duplicates is the Duplicates screen (Y on Storage): identical games, +// the ones wasting most space first. A opens one. +func (t *tasks) duplicates() { + if t.dupes == nil { + t.startDupes() + } + l := ui.List{} + for { + d := t.dupes + t.dmu.Lock() + done, groups, err := d.done, d.groups, d.err + t.dmu.Unlock() + var wasted int64 + for _, g := range groups { + wasted += g.Wasted() + } + sub := "Games that are byte for byte the same" + if done && len(groups) > 0 { + sub = fmt.Sprintf("%d games with extra copies · %s to free", len(groups), size(wasted)) + if len(groups) == 1 { + sub = "1 game with extra copies · " + size(wasted) + " to free" + } + } + hints := []ui.Hint{{Button: "A", Label: "Open"}, {Button: "Y", Label: "Look again"}, {Button: "B", Label: "Back"}} + if !done { + hints = []ui.Hint{{Button: "B", Label: "Stop"}} + } + top := t.scr.StaticFrame("Duplicates", sub, hints...) + area := image.Rect(20, top, ui.W-26, ui.H-58) + cy := area.Min.Y + area.Dy()/2 - 24 + switch { + case !done: + t.scr.Loading(cy, "Looking for duplicates…") + line := count(d.prog.Files.Load()) + " games found" + if tot := d.prog.Total.Load(); tot > 0 { + line += fmt.Sprintf(" · checking %s of %s", size(d.prog.Read.Load()), size(tot)) + } + t.scr.TextCenter(ui.W/2, cy+34, ui.Fit(ui.Small, line, area.Dx()), ui.Small, ui.Muted) + case err != nil: + t.scr.TextCenter(ui.W/2, cy, ui.Fit(ui.Body, "Couldn't look: "+err.Error(), area.Dx()), ui.Body, ui.Muted) + case len(groups) == 0: + t.scr.TextCenter(ui.W/2, cy, "No duplicate games", ui.Body, ui.Text) + t.scr.TextCenter(ui.W/2, cy+30, ui.Fit(ui.Small, "Ports, ScummVM games and box art aren't looked at.", area.Dx()), ui.Small, ui.Muted) + default: + l.Items = l.Items[:0] + for _, g := range groups { + l.Items = append(l.Items, ui.Item{Label: path.Base(g.Files[0].Rel), Sub: dupLine(g), Detail: size(g.Wasted()), Chevron: true}) + } + l.Sel = min(l.Sel, len(l.Items)-1) + l.Draw(t.scr, area) + } + t.scr.Present() + var wait <-chan struct{} + if !done { + wait = d.end + } + ev, ok, got := t.next(wait) + if !ok { + return + } + if !got || !acts(ev) { + continue + } + switch ev.Button { + case ui.B, ui.Menu: + if !done { // stop: the next visit looks again + d.cancel() + <-d.end + t.dupes = nil + } + return + case ui.Y: + if done { + t.startDupes() + } + case ui.A: + if done && l.Sel < len(groups) { + t.dupGroup(l.Sel) + } + default: + l.Handle(ev) + } + } +} + +// dupGroup lists the copies of one game; X deletes the selected copy +// after asking. The last copy can't be deleted. +func (t *tasks) dupGroup(gi int) { + l := ui.List{} + msg := "" + for { + t.dmu.Lock() + if gi >= len(t.dupes.groups) { + t.dmu.Unlock() + return + } + g := t.dupes.groups[gi] + t.dmu.Unlock() + l.Items = l.Items[:0] + for _, f := range g.Files { + sub := "" + if f.InUse != "" { + sub = "Used by " + f.InUse + ": part of a game" + } + l.Items = append(l.Items, ui.Item{Label: strings.TrimPrefix(f.Rel, "Roms/"), Sub: sub}) + } + l.Sel = min(l.Sel, len(l.Items)-1) + top := t.scr.StaticFrame(path.Base(g.Files[0].Rel), fmt.Sprintf("%d identical copies · %s each", len(g.Files), size(g.Size)), + ui.Hint{Button: "X", Label: "Delete this copy"}, ui.Hint{Button: "B", Label: "Back"}) + area := image.Rect(20, top, ui.W-26, ui.H-58) + if msg != "" { + t.scr.Text(area.Min.X+4, area.Max.Y-ui.LineHeight(ui.Small), ui.Fit(ui.Small, msg, area.Dx()-8), ui.Small, ui.Warn) + area.Max.Y -= ui.LineHeight(ui.Small) + 4 + } + l.Draw(t.scr, area) + t.scr.Present() + ev, ok, got := t.next(nil) + if !ok { + return + } + if !got || !acts(ev) { + continue + } + switch ev.Button { + case ui.B, ui.Menu: + return + case ui.X: + msg = "" + f := g.Files[l.Sel] + if f.InUse == "" && len(g.Files) > 1 { + var keep []string + for j, o := range g.Files { + if j != l.Sel { + keep = append(keep, strings.TrimPrefix(o.Rel, "Roms/")) + } + } + body := fmt.Sprintf("This copy (%s) is deleted for good: the card has no trash. Still on the card: %s. Saves made with this copy stay in Saves, but another copy with a different file name doesn't use them.", + size(g.Size), strings.Join(keep, ", ")) + if !t.scr.Confirm(t.input, "Delete "+path.Base(f.Rel)+"?", body, "Delete") { + break + } + } + ng, err := deleteCopy(t.disk.root, g, l.Sel) + if err != nil { + if errors.Is(err, errRefused) { + msg = reason(err) + } else { + msg = err.Error() + } + break + } + if s := t.disk.view(); s != nil { + s.Forget(f.Path, g.Size, 1, nil) + t.disk.keep(s) + } + t.rootScan = nil + t.dmu.Lock() + if len(ng.Files) < 2 { + t.dupes.groups = append(t.dupes.groups[:gi:gi], t.dupes.groups[gi+1:]...) + } else { + t.dupes.groups[gi] = ng + } + t.dmu.Unlock() + if len(ng.Files) < 2 { + t.scr.Notice(t.input, "Deleted", fmt.Sprintf("Freed %s. One copy is left: %s.", size(g.Size), strings.TrimPrefix(ng.Files[0].Rel, "Roms/"))) + return + } + default: + l.Handle(ev) + } + } +} diff --git a/cmd/minitop/main.go b/cmd/minitop/main.go index 08a119e..9ed503f 100644 --- a/cmd/minitop/main.go +++ b/cmd/minitop/main.go @@ -8,6 +8,7 @@ import ( "image" "log" "strings" + "sync" "time" "github.com/zakijss/minideck/appdir" @@ -23,7 +24,7 @@ func main() { log.Fatal(err) } defer scr.Close() - t := &tasks{scr: scr, input: ui.Input(), smp: newSampler(), dir: appdir.Dir(), disk: &measurer{root: sdcard()}} + t := &tasks{scr: scr, input: ui.Input(), smp: newSampler(), dir: appdir.Dir(), disk: &measurer{root: sdcard(), dir: appdir.Dir()}} t.cfg = loadSettings(t.dir) t.smp.Sample() // first sample: CPU needs two t.procs = t.smp.Sample() @@ -49,6 +50,8 @@ type tasks struct { rootEnts []Entry rootMore int stale bool // a delete failed part way: measure again + dupes *dupSearch + dmu sync.Mutex // guards dupes' results } // The pages L1 and R1 move between, in order. @@ -114,7 +117,11 @@ func (t *tasks) run() { t.procs = t.smp.Sample() } case ui.Y: - t.settings() + if t.page == pageStorage { + t.duplicates() + } else { + t.settings() + } case ui.Select: t.scr.Appearance(t.input) default: diff --git a/cmd/minitop/savedscan.go b/cmd/minitop/savedscan.go new file mode 100644 index 0000000..dc40136 --- /dev/null +++ b/cmd/minitop/savedscan.go @@ -0,0 +1,120 @@ +package main + +// The last measurement of the card is kept in MiniTop's folder, so the +// Storage page opens with it at once, labelled with its age, while the +// card is measured again in the background. Only the folder tree is kept: +// clean-ups always come from a measurement made this session, so nothing +// is deleted from an old list. + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/zakijss/minideck/atomicfile" +) + +// savedDir is a Dir as it is kept on the card. +type savedDir struct { + N string `json:"n"` + S int64 `json:"s"` + F int `json:"f"` + D []*savedDir `json:"d,omitempty"` +} + +type savedScan struct { + V int `json:"v"` + Root string `json:"root"` + Measured time.Time `json:"measured"` + Errors int `json:"errors"` + Tree *savedDir `json:"tree"` +} + +// maxSavedDirs: a tree with more folders than this isn't kept (the file +// would be large and slow to read on the device). +const maxSavedDirs = 60000 + +func scanFile(dir string) string { return filepath.Join(dir, "storage.json") } + +// saveScan keeps s's tree in dir. +func saveScan(dir string, s *Scan) error { + n := 0 + var conv func(d *Dir) *savedDir + conv = func(d *Dir) *savedDir { + n++ + sd := &savedDir{N: d.Name, S: d.Size, F: d.Files} + for _, c := range d.Subs { + sd.D = append(sd.D, conv(c)) + } + return sd + } + doc := savedScan{V: 1, Root: s.Root, Measured: s.Measured, Errors: s.Errors, Tree: conv(s.Tree)} + if n > maxSavedDirs { + return fmt.Errorf("%d folders: too many to keep", n) + } + b, err := json.Marshal(doc) + if err != nil { + return err + } + return atomicfile.Write(scanFile(dir), b, 0644) +} + +// loadScan reads the tree kept in dir for the card at root. A file for +// another root, in another format, or with a folder name that isn't a +// plain name is not used. +func loadScan(dir, root string) (*Scan, error) { + b, err := os.ReadFile(scanFile(dir)) + if err != nil { + return nil, err + } + var doc savedScan + if err := json.Unmarshal(b, &doc); err != nil { + return nil, err + } + if doc.V != 1 || doc.Root != root || doc.Tree == nil || doc.Measured.IsZero() { + return nil, errors.New("not a measurement of this card") + } + bad := errors.New("damaged measurement") + var conv func(sd *savedDir, parent *Dir, path string, depth int) (*Dir, error) + conv = func(sd *savedDir, parent *Dir, path string, depth int) (*Dir, error) { + if depth > 64 || sd.S < 0 || sd.F < 0 { + return nil, bad + } + d := &Dir{Name: sd.N, Path: path, Size: sd.S, Files: sd.F, parent: parent} + for _, c := range sd.D { + if c == nil || c.N == "" || c.N == "." || c.N == ".." || strings.ContainsAny(c.N, "/\x00") { + return nil, bad + } + cd, err := conv(c, d, filepath.Join(path, c.N), depth+1) + if err != nil { + return nil, err + } + d.Subs = append(d.Subs, cd) + } + return d, nil + } + tree, err := conv(doc.Tree, nil, root, 0) + if err != nil { + return nil, err + } + return &Scan{Root: root, Tree: tree, Errors: doc.Errors, Measured: doc.Measured, Saved: true}, nil +} + +// ago says how long ago t was, for people: "just now", "5 min ago", +// "3 h ago", "2 days ago". +func ago(t, now time.Time) string { + d := now.Sub(t) + switch { + case d < time.Minute: + return "just now" + case d < time.Hour: + return fmt.Sprintf("%d min ago", int(d/time.Minute)) + case d < 48*time.Hour: + return fmt.Sprintf("%d h ago", int(d/time.Hour)) + } + return fmt.Sprintf("%d days ago", int(d/(24*time.Hour))) +} diff --git a/cmd/minitop/storage.go b/cmd/minitop/storage.go index 1c0d77f..2171b8c 100644 --- a/cmd/minitop/storage.go +++ b/cmd/minitop/storage.go @@ -19,6 +19,7 @@ import ( "sync" "sync/atomic" "syscall" + "time" ) // sdcard is the card's root; TASKS_SDCARD replaces it in tests. @@ -78,6 +79,10 @@ type Scan struct { Tree *Dir Cleanups []*Cleanup Errors int // folders that couldn't be read + Measured time.Time + // Saved: the measurement kept from an earlier session (see + // savedscan.go): the tree only, no clean-ups. + Saved bool } // Cleanup returns the clean-up with the given ID. @@ -203,7 +208,7 @@ func measure(ctx context.Context, root string, prog *Progress) (*Scan, error) { if _, err := w.dir(tree, "", false, false); err != nil { return nil, err } - s := &Scan{Root: root, Tree: tree, Errors: w.errors} + s := &Scan{Root: root, Tree: tree, Errors: w.errors, Measured: time.Now()} for _, id := range cleanupOrder { s.Cleanups = append(s.Cleanups, w.c[id]) } @@ -589,8 +594,10 @@ func count(n int64) string { // the last result for the session. type measurer struct { root string + dir string // where the last measurement is kept between sessions ("": not kept) mu sync.Mutex scan *Scan + old *Scan // the kept measurement, shown until scan is done err error prog *Progress done chan struct{} // closed when the running measurement ends; nil if none @@ -604,6 +611,11 @@ func (m *measurer) start() { if m.done != nil || m.scan != nil { return } + if m.old == nil && m.dir != "" { + if s, err := loadScan(m.dir, m.root); err == nil { + m.old = s + } + } ctx, cancel := context.WithCancel(context.Background()) m.prog, m.cancel, m.err = &Progress{}, cancel, nil done := make(chan struct{}) @@ -611,9 +623,15 @@ func (m *measurer) start() { prog := m.prog go func() { s, err := measure(ctx, m.root, prog) + if err == nil && ctx.Err() == nil && m.dir != "" { + saveScan(m.dir, s) // best effort: the page works without it + } m.mu.Lock() if ctx.Err() == nil { m.scan, m.err = s, err + if err == nil { + m.old = nil + } } m.done = nil m.mu.Unlock() @@ -650,6 +668,27 @@ func (m *measurer) state() (*Scan, *Progress, <-chan struct{}, error) { return m.scan, m.prog, m.done, m.err } +// view is the measurement to show: this session's, or else the kept one. +func (m *measurer) view() *Scan { + m.mu.Lock() + defer m.mu.Unlock() + if m.scan != nil { + return m.scan + } + return m.old +} + +// keep saves s again after a delete changed it, if it is this session's +// measurement. +func (m *measurer) keep(s *Scan) { + m.mu.Lock() + cur := m.scan + m.mu.Unlock() + if s == cur && s != nil && m.dir != "" { + saveScan(m.dir, s) + } +} + // wait blocks until no measurement is running. func (m *measurer) wait() { if _, _, done, _ := m.state(); done != nil { diff --git a/cmd/minitop/storage_test.go b/cmd/minitop/storage_test.go index 0d889b1..2f37985 100644 --- a/cmd/minitop/storage_test.go +++ b/cmd/minitop/storage_test.go @@ -665,4 +665,40 @@ func TestStorageScreens(t *testing.T) { shot("10-cleanup", func(tk *tasks) { measured(tk); tk.cleanup() }) shot("11-cleanup-list", func(tk *tasks) { s := measured(tk); tk.cleanupList(s, s.Cleanup("junk")) }) shot("12-cleanup-confirm", func(tk *tasks) { s := measured(tk); tk.runCleanup(s, s.Cleanup("minimart")) }) + // the kept measurement, shown while the card is measured again + kept := t.TempDir() + shot("13-storage-kept", func(tk *tasks) { + s := measured(tk) + s.Measured = time.Now().Add(-5 * time.Hour) + saveScan(kept, s) + old, _ := loadScan(kept, root) + p := &Progress{} + p.Files.Store(4321) + tk.disk.scan, tk.disk.old, tk.disk.prog, tk.disk.done = nil, old, p, make(chan struct{}) + tk.drawStorage() + }) + game := blob(4, 300<<10) + for _, p := range []string{"Roms/GBA/Pokemon Emerald.gba", "Roms/GBA/Hacks/Pokemon Emerald.gba", "Roms/PS/FF7 (Disc 1).chd", "Roms/PS/FF7/FF7 (Disc 1).chd"} { + writeAt(t, root, p, game) + } + writeAt(t, root, "Roms/PS/FF7/FF7.m3u", []byte("FF7 (Disc 1).chd\n")) + groups, err := findDuplicates(context.Background(), root, nil) + if err != nil { + t.Fatal(err) + } + finished := func() *dupSearch { + end := make(chan struct{}) + close(end) + return &dupSearch{groups: groups, done: true, prog: &DupProgress{}, end: end, cancel: func() {}} + } + shot("14-duplicates-searching", func(tk *tasks) { + p := &DupProgress{} + p.Files.Store(18234) + p.Total.Store(2 << 30) + p.Read.Store(700 << 20) + tk.dupes = &dupSearch{prog: p, end: make(chan struct{}), cancel: func() {}} + tk.duplicates() + }) + shot("15-duplicates", func(tk *tasks) { tk.dupes = finished(); tk.duplicates() }) + shot("16-duplicates-game", func(tk *tasks) { tk.dupes = finished(); tk.dupGroup(0) }) } diff --git a/cmd/minitop/storage_ui.go b/cmd/minitop/storage_ui.go index 7eb98ae..0b4f685 100644 --- a/cmd/minitop/storage_ui.go +++ b/cmd/minitop/storage_ui.go @@ -90,7 +90,7 @@ func (t *tasks) drawStorage() { s := t.scr sub, free, total := storageSub(t.disk.root) top := s.StaticFrame("Storage", sub, - ui.Hint{Button: "A", Label: "Open"}, ui.Hint{Button: "X", Label: "Clean up"}, ui.Hint{Button: "R1", Label: "Running"}, ui.AppearanceHint, ui.Hint{Button: "B", Label: "Exit"}) + ui.Hint{Button: "A", Label: "Open"}, ui.Hint{Button: "X", Label: "Clean up"}, ui.Hint{Button: "Y", Label: "Duplicates"}, ui.Hint{Button: "B", Label: "Exit"}) card := image.Rect(20, top, ui.W-26, top+62) s.Panel(card, 12) pct := 0.0 @@ -111,7 +111,17 @@ func (t *tasks) drawStorage() { s.Bar(image.Rect(card.Min.X+16, card.Max.Y-22, card.Max.X-16, card.Max.Y-10), pct/100, ui.Level(pct)) area := image.Rect(20, card.Max.Y+10, ui.W-26, ui.H-58) - scan, prog, done, err := t.disk.state() + _, prog, done, err := t.disk.state() + scan := t.disk.view() + if scan != nil && scan.Saved { + // the kept measurement, while the card is measured again + note := "From " + ago(scan.Measured, time.Now()) + if done != nil { + note += " · measuring again: " + count(prog.Files.Load()) + " files" + } + s.Text(area.Min.X+4, area.Min.Y-2, ui.Fit(ui.Small, note, area.Dx()-8), ui.Small, ui.Muted) + area.Min.Y += ui.LineHeight(ui.Small) + 2 + } switch { case scan == nil && done != nil: t.drawMeasuring(area, prog) @@ -152,7 +162,7 @@ func (t *tasks) drawMeasuring(r image.Rectangle, p *Progress) { // openStorage opens the selected row of the Storage page: a folder's // contents, or the delete question for a file. func (t *tasks) openStorage() { - scan, _, _, _ := t.disk.state() + scan := t.disk.view() if scan == nil || t.rootScan != scan || t.run3.Sel >= len(t.rootEnts) { return } @@ -259,6 +269,7 @@ func (t *tasks) delete(scan *Scan, e Entry) { return } scan.Forget(e.Path, e.Size, 1, e.Dir) + t.disk.keep(scan) } // next waits for input; while a measurement runs (done is not nil) it