diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index df9d134..755ebc2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -16,6 +16,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + fetch-depth: 0 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 @@ -23,5 +24,12 @@ jobs: run: npm ci --ignore-scripts - name: Build the site (fails on dead links) run: npm run build + - name: Check that Chinese and English pages match + run: python3 scripts/i18n.py + - name: Warn when a page changed in one language only + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: python3 scripts/i18n.py --base "origin/$BASE_REF" - name: Scan tracked files for private identity and paths run: python3 scripts/privacy.py . diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..55fcf34 --- /dev/null +++ b/README.en.md @@ -0,0 +1,33 @@ +# ModAVD documentation site + +[简体中文](README.md) | **English** + + + +The source of the documentation site for [ModAVD](https://github.com/ModAVD/ModAVD), published at (English version: ). It is built with [VitePress](https://vitepress.dev/). + +- Pages are Markdown files under `docs/`, and Chinese is the primary language. English pages are under `docs/en/`, at the same paths as the Chinese pages. The navigation and sidebar are in `docs/.vitepress/config.mts`, and both languages use one table of pages. +- Every page has an "Edit this page on GitHub" link at the bottom; make your change and open a PR. How to write pages is described on the site under "Contributing → Documentation and this site". +- If you change a page in one language, change the other language in the same PR. If you only write one of the two, say so in the PR and the maintainer will add the other. + +## Preview locally + +You need Node.js 24. + +```powershell +npm ci --ignore-scripts +npm run dev # local preview +npm run build # builds docs/.vitepress/dist; dead links fail the build +python scripts/i18n.py # checks the Chinese and English pages and headings, same as CI +python scripts/privacy.py . # privacy scan, same as CI +``` + +## CI + +- `check.yml`: on a PR it builds the whole site (dead-link check), checks that the Chinese and English pages match (`scripts/i18n.py`: every page exists in both languages, headings have the same levels and order, site links do not cross languages, and every anchor resolves; a page changed in one language only gets a warning), and runs the privacy scan. +- `deploy.yml`: after a merge into `main` it builds the site and publishes it to GitHub Pages. +- Third-party actions are always pinned to a full 40-character commit SHA; no `pull_request_target`, no self-hosted runners. + +## License + +The documentation is published under the Apache License 2.0; see [LICENSE](LICENSE). Copyright 2026 qiyinxi. diff --git a/README.md b/README.md index 5847dc7..e0458da 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ # ModAVD 文档站 -[ModAVD(魔改 AVD)](https://github.com/ModAVD/ModAVD) 的文档站源码,发布在 。用 [VitePress](https://vitepress.dev/) 编译。 +**简体中文** | [English](README.en.md) -- 页面是 `docs/` 下的 Markdown;导航和侧边栏在 `docs/.vitepress/config.mts`。 + + +[ModAVD(魔改 AVD)](https://github.com/ModAVD/ModAVD) 的文档站源码,发布在 (英文版在 )。用 [VitePress](https://vitepress.dev/) 编译。 + +- 页面是 `docs/` 下的 Markdown,中文是主语言;英文页在 `docs/en/` 下,路径和中文页一一对应。导航和侧边栏在 `docs/.vitepress/config.mts`,两种语言用同一张页面表。 - 每页底部有「在 GitHub 上改这一页」,改完提 PR 就行。写法见站内的「参与开发 → 文档和本站」。 +- 改了一种语言的页面,在同一个 PR 里把另一种也改了;只会写一种语言的,在 PR 里说一声,由维护者补上。 ## 本地预览 @@ -13,12 +18,13 @@ npm ci --ignore-scripts npm run dev # 本地预览 npm run build # 编出 docs/.vitepress/dist,有死链会报错 +python scripts/i18n.py # 核对中英页面和标题,和 CI 一样 python scripts/privacy.py . # 隐私扫描,和 CI 一样 ``` ## CI -- `check.yml`:PR 上编译整站(查死链)、跑隐私扫描。 +- `check.yml`:PR 上编译整站(查死链)、核对中英页面(`scripts/i18n.py`:每页都有两种语言,标题的层级和顺序相同,站内链接不串语言,锚点都找得到;只改了一种语言的页面会标出提醒)、跑隐私扫描。 - `deploy.yml`:合进 `main` 后编译并发布到 GitHub Pages。 - 第三方 action 一律钉到 40 位提交 SHA;不用 `pull_request_target`,不用自托管 runner。 diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c76a5b3..f3683b7 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -1,110 +1,176 @@ -import { defineConfig } from 'vitepress' +import { defineConfig, type DefaultTheme } from 'vitepress' const repo = 'https://github.com/ModAVD/modavd.github.io' +const origin = 'https://modavd.github.io' + +type Lang = 'zh' | 'en' + +// 页面标题,两种语言放在同一张表里:新加页面时两边一起写。 +// [链接, 中文标题, 英文标题] +type Page = [link: string, zh: string, en: string] + +const guide: Page[] = [ + ['/guide/', '这是什么', 'What is ModAVD'], + ['/guide/install', '安装', 'Install'], + ['/guide/upgrade', '升级', 'Upgrade'], + ['/guide/changes', '和官方模拟器有什么不同', 'How ModAVD differs from the official emulator'], + ['/guide/feedback', '反馈问题', 'Report a problem'], + ['/guide/privacy', '隐私', 'Privacy'], + ['/guide/license', '许可证与源码', 'Licenses and source code'], +] + +const components: Page[] = [ + ['/components/manager', '管理器和命令行', 'The manager and CLI'], + ['/components/bridge', 'arm64 执行桥', 'The arm64 execution bridge'], + ['/components/ime', '宿主输入法', 'The host IME'], +] + +const develop: { group: [zh: string, en: string]; pages: Page[] }[] = [ + { + group: ['开始', 'Getting started'], + pages: [ + ['/develop/', '参与 ModAVD', 'Contributing to ModAVD'], + ['/develop/overview', '分区一览和通用要求', 'Areas and common requirements'], + ['/develop/ai', '用 AI 工具协助', 'Using AI tools'], + ], + }, + { + group: ['各分区', 'Areas'], + pages: [ + ['/develop/emulator', '模拟器本体', 'The emulator itself'], + ['/develop/build', '从源码构建', 'Building from source'], + ['/develop/bridge', 'arm64 执行桥', 'The arm64 execution bridge'], + ['/develop/manager', '管理器和命令行', 'The manager and CLI'], + ['/develop/ime', '宿主输入法', 'The host IME'], + ['/develop/package', '随包脚本', 'Bundled scripts'], + ['/develop/docs', '文档和本站', 'Documentation and this site'], + ['/develop/automas', 'AUTO-MAS 对接', 'AUTO-MAS integration'], + ['/develop/testing', '游戏实测和兼容性报告', 'Game testing and compatibility reports'], + ], + }, + { + group: ['查问题', 'Investigating problems'], + pages: [ + ['/develop/adapt-game', '适配新游戏', 'Adapting a new game'], + ['/develop/performance', '卡顿、掉帧怎么查', 'Investigating stutter and frame drops'], + ['/develop/troubleshooting', '常见问题怎么查', 'Investigating common problems'], + ], + }, +] + +const label = { + zh: { + guide: '使用', components: '组件', develop: '参与开发', download: '下载', fullPackage: 'ModAVD(整包)', + description: '基于 AOSP Android Emulator 的安卓模拟器', + }, + en: { + guide: 'Guide', components: 'Components', develop: 'Contributing', download: 'Download', fullPackage: 'ModAVD (full package)', + description: 'An Android emulator built from the AOSP Android Emulator', + }, +} + +function nav(lang: Lang): DefaultTheme.NavItem[] { + const p = lang === 'en' ? '/en' : '' + const l = label[lang] + return [ + { text: l.guide, link: `${p}/guide/`, activeMatch: `${p}/guide/` }, + { text: l.components, link: `${p}/components/manager`, activeMatch: `${p}/components/` }, + { text: l.develop, link: `${p}/develop/`, activeMatch: `${p}/develop/` }, + { + text: l.download, + items: [ + { text: l.fullPackage, link: 'https://github.com/ModAVD/ModAVD/releases' }, + { text: 'ModAVDManager', link: 'https://github.com/ModAVD/ModAVDManager/releases' }, + { text: 'ModAVDHostIME', link: 'https://github.com/ModAVD/ModAVDHostIME/releases' }, + ], + }, + ] +} + +function sidebar(lang: Lang): DefaultTheme.Sidebar { + const p = lang === 'en' ? '/en' : '' + const items = (pages: Page[]) => pages.map(([link, zh, en]) => ({ text: lang === 'en' ? en : zh, link: p + link })) + const i = lang === 'en' ? 1 : 0 + return { + [`${p}/guide/`]: [{ text: label[lang].guide, items: items(guide) }], + [`${p}/components/`]: [{ text: label[lang].components, items: items(components) }], + [`${p}/develop/`]: develop.map(({ group, pages }) => ({ text: group[i], items: items(pages) })), + } +} + +// 指向同一页的另一种语言 +function alternates(relativePath: string): [string, Record][] { + if (!relativePath.endsWith('.md') || relativePath === '404.md') return [] + const url = (path: string) => origin + '/' + path.replace(/\.md$/, '').replace(/(^|\/)index$/, '$1') + const zh = relativePath.replace(/^en\//, '') + return [ + ['link', { rel: 'alternate', hreflang: 'zh-CN', href: url(zh) }], + ['link', { rel: 'alternate', hreflang: 'en', href: url('en/' + zh) }], + ] +} export default defineConfig({ - lang: 'zh-CN', title: 'ModAVD', - description: '基于 AOSP Android Emulator 的安卓模拟器', cleanUrls: true, lastUpdated: true, // 不往外取字体、统计脚本,页面只用本站自己的文件 head: [['meta', { name: 'theme-color', content: '#3c6fb4' }]], + transformHead: ({ pageData }) => alternates(pageData.relativePath), + // 中文(根)和英文(/en/)各一份导航、侧边栏和界面文字,页面一一对应 locales: { - root: { label: '简体中文', lang: 'zh-CN' }, + root: { + label: '简体中文', + lang: 'zh-CN', + description: label.zh.description, + themeConfig: { + nav: nav('zh'), + sidebar: sidebar('zh'), + editLink: { pattern: `${repo}/edit/main/docs/:path`, text: '在 GitHub 上改这一页' }, + footer: { message: 'Android 是 Google LLC 的商标。', copyright: '文档按 Apache License 2.0 发布' }, + outline: { level: [2, 3], label: '本页内容' }, + docFooter: { prev: '上一页', next: '下一页' }, + lastUpdated: { text: '最后更新' }, + returnToTopLabel: '回到顶部', + sidebarMenuLabel: '目录', + darkModeSwitchLabel: '深色模式', + lightModeSwitchTitle: '切换到浅色', + darkModeSwitchTitle: '切换到深色', + langMenuLabel: '语言', + notFound: { title: '页面不存在', quote: '这个地址没有页面,可能是改名或挪走了。', linkText: '回首页' }, + }, + }, en: { label: 'English', lang: 'en-US', - description: 'An Android emulator built from the AOSP Android Emulator', + link: '/en/', + description: label.en.description, themeConfig: { - nav: [{ text: 'GitHub', link: 'https://github.com/ModAVD/ModAVD' }], - sidebar: { '/en/': [{ text: 'English', items: [{ text: 'Overview', link: '/en/' }] }] }, + nav: nav('en'), + sidebar: sidebar('en'), + editLink: { pattern: `${repo}/edit/main/docs/:path`, text: 'Edit this page on GitHub' }, + footer: { message: 'Android is a trademark of Google LLC.', copyright: 'Documentation is published under the Apache License 2.0' }, + outline: { level: [2, 3], label: 'On this page' }, + docFooter: { prev: 'Previous page', next: 'Next page' }, + lastUpdated: { text: 'Last updated' }, + returnToTopLabel: 'Back to top', + sidebarMenuLabel: 'Menu', + darkModeSwitchLabel: 'Appearance', + lightModeSwitchTitle: 'Switch to light theme', + darkModeSwitchTitle: 'Switch to dark theme', + langMenuLabel: 'Language', + notFound: { + title: 'PAGE NOT FOUND', + quote: 'There is no page at this address; it may have been renamed or moved.', + linkText: 'Take me home', + }, }, }, }, themeConfig: { - nav: [ - { text: '使用', link: '/guide/', activeMatch: '/guide/' }, - { text: '组件', link: '/components/manager', activeMatch: '/components/' }, - { text: '参与开发', link: '/develop/', activeMatch: '/develop/' }, - { - text: '下载', - items: [ - { text: 'ModAVD(整包)', link: 'https://github.com/ModAVD/ModAVD/releases' }, - { text: 'ModAVDManager', link: 'https://github.com/ModAVD/ModAVDManager/releases' }, - { text: 'ModAVDHostIME', link: 'https://github.com/ModAVD/ModAVDHostIME/releases' }, - ], - }, - ], - - sidebar: { - '/guide/': [ - { - text: '使用', - items: [ - { text: '这是什么', link: '/guide/' }, - { text: '安装', link: '/guide/install' }, - { text: '升级', link: '/guide/upgrade' }, - { text: '和官方模拟器有什么不同', link: '/guide/changes' }, - { text: '反馈问题', link: '/guide/feedback' }, - { text: '隐私', link: '/guide/privacy' }, - { text: '许可证与源码', link: '/guide/license' }, - ], - }, - ], - '/components/': [ - { - text: '组件', - items: [ - { text: '管理器和命令行', link: '/components/manager' }, - { text: 'arm64 执行桥', link: '/components/bridge' }, - { text: '宿主输入法', link: '/components/ime' }, - ], - }, - ], - '/develop/': [ - { - text: '开始', - items: [ - { text: '参与 ModAVD', link: '/develop/' }, - { text: '分区一览和通用要求', link: '/develop/overview' }, - { text: '用 AI 工具协助', link: '/develop/ai' }, - ], - }, - { - text: '各分区', - items: [ - { text: '模拟器本体', link: '/develop/emulator' }, - { text: '从源码构建', link: '/develop/build' }, - { text: 'arm64 执行桥', link: '/develop/bridge' }, - { text: '管理器和命令行', link: '/develop/manager' }, - { text: '宿主输入法', link: '/develop/ime' }, - { text: '随包脚本', link: '/develop/package' }, - { text: '文档和本站', link: '/develop/docs' }, - { text: 'AUTO-MAS 对接', link: '/develop/automas' }, - { text: '游戏实测和兼容性报告', link: '/develop/testing' }, - ], - }, - { - text: '查问题', - items: [ - { text: '适配新游戏', link: '/develop/adapt-game' }, - { text: '卡顿、掉帧怎么查', link: '/develop/performance' }, - { text: '常见问题怎么查', link: '/develop/troubleshooting' }, - ], - }, - ], - }, - socialLinks: [{ icon: 'github', link: 'https://github.com/ModAVD' }], - editLink: { - pattern: `${repo}/edit/main/docs/:path`, - text: '在 GitHub 上改这一页', - }, - search: { provider: 'local', options: { @@ -122,21 +188,5 @@ export default defineConfig({ }, }, }, - - footer: { - message: 'Android 是 Google LLC 的商标。', - copyright: '文档按 Apache License 2.0 发布', - }, - - outline: { level: [2, 3], label: '本页内容' }, - docFooter: { prev: '上一页', next: '下一页' }, - lastUpdated: { text: '最后更新' }, - returnToTopLabel: '回到顶部', - sidebarMenuLabel: '目录', - darkModeSwitchLabel: '深色模式', - lightModeSwitchTitle: '切换到浅色', - darkModeSwitchTitle: '切换到深色', - langMenuLabel: '语言', - notFound: { title: '页面不存在', quote: '这个地址没有页面,可能是改名或挪走了。', linkText: '回首页' }, }, }) diff --git a/docs/develop/docs.md b/docs/develop/docs.md index a5a4a84..1ddd8f9 100644 --- a/docs/develop/docs.md +++ b/docs/develop/docs.md @@ -5,7 +5,7 @@ | 内容 | 在哪改 | |---|---| | 本站(使用说明、组件介绍、参与开发) | [modavd.github.io](https://github.com/ModAVD/modavd.github.io) 仓库的 `docs/` | -| 各仓库自己的 README、`CONTRIBUTING.md` | 各自的仓库 | +| 各仓库自己的 README(中文 `README.md`、英文 `README.en.md`)、`CONTRIBUTING.md` | 各自的仓库 | | 发版说明、补丁说明(`release/`、`patches/BASE.md`)、仓库清单 | ModAVD 仓库,由发版工具导出,**不要手改**,有问题先开 issue | 每页底部有「在 GitHub 上改这一页」,点进去直接改、提 PR 就行。 @@ -18,9 +18,10 @@ npm ci --ignore-scripts npm run dev # 本地预览,改了自动刷新 npm run build # 编出静态页面,有死链会报错 +python scripts/i18n.py # 核对中英两边的页面和标题 ``` -- 页面是 `docs/` 下的 Markdown,侧边栏和导航在 `docs/.vitepress/config.mts`。新加页面要在那里挂上。 +- 页面是 `docs/` 下的 Markdown,英文页在 `docs/en/` 下,路径和中文页一一对应。侧边栏和导航在 `docs/.vitepress/config.mts`,两种语言各一份,新加页面要在两边都挂上。 - 提示框用 `::: tip`、`::: warning`、`::: danger`、`::: info`。 - 站内链接写相对路径、不带 `.md`,例如 `[安装](../guide/install)`;链到别的仓库的文件写完整的 GitHub 地址。 - 不加统计脚本、不从别的网站取字体或图片。 @@ -31,13 +32,15 @@ npm run build # 编出静态页面,有死链会报错 PR 上的 CI 会: - 编译整站,有死链就失败; +- 核对中英两边(`python scripts/i18n.py`):每页都有两种语言,标题的层级和顺序相同,站内链接不串到另一种语言,`#锚点` 都找得到;只改了一种语言的页面会标出提醒; - 跑隐私扫描(`python scripts/privacy.py .`),文档里出现本机用户目录这类路径就失败。 合进 `main` 以后自动发布到 GitHub Pages。 ## 写法 -- 中文,短句,写用户看得懂的说法。 +- 中文是主语言,英文页逐页对应。改了中文页,在同一个 PR 里把对应的英文页也改了;不会写英文的,在 PR 里说一声,由维护者补上。 +- 中文,短句,写用户看得懂的说法。英文照意思写成自然的英文,不逐字硬译。 - 写清楚是哪个版本起有的、哪个版本起变的。 - 能用表格、列表的就别写成大段。 - 命令写完整、能直接复制,路径用 `<占位符>` 标出要换的部分。 diff --git a/docs/en/components/bridge.md b/docs/en/components/bridge.md new file mode 100644 index 0000000..6db2805 --- /dev/null +++ b/docs/en/components/bridge.md @@ -0,0 +1,38 @@ +# The arm64 execution bridge + +It lets games that only have an arm64 build and use app hardening run on ModAVD's x86_64 system image. Right now it takes effect only for the 7 channel builds of *Onmyoji*. From 0.36.0 it ships with the main Release package, installed in `sdk\emulator\resources\modavd\`. + +::: info It does not touch the game +The bridge **does not modify, patch or bypass the game or its hardening libraries, and does not interfere with any integrity or anti-tamper check**. This is compatibility work on the translation layer: it makes the hooks the game installs itself execute and return correctly. Intel Houdini already works this way, and the same game runs on Houdini. We do not change a single byte of Google's translation-layer binaries. +::: + +## Why it is needed + +x86_64 images run arm64 apps through Google's translation layer: the system libraries stay native x86_64, and only the arm64 libraries that ship with the app are translated. + +A game's hardening library redirects the entry points of some functions in the system libraries to its own arm64 code. On a real device both sides are arm64, so this is fine. On an x86_64 image, the native code crashes as soon as it jumps into arm64 code, because the translation layer only supports translated code calling native code, not the other way round. The bridge fills this gap using the interfaces the translation layer itself exports. For the details, see [Develop: the arm64 execution bridge](../develop/bridge). + +## Only for games on the list + +- The list is currently the 7 channel builds of *Onmyoji*. +- For apps not on the list, after the process starts, everything the bridge changed is reverted, so they behave the same as when the bridge is not installed. + +## Installing into an instance + +- **From 0.36.0 you do not need to install it yourself**: when you start an instance with the manager `ModAVD.exe` or the CLI `tools\modavd.exe start`, both new and existing instances get the bridge automatically. If the bridge is not installed after the startup sequence finishes, it is installed; if it does not take effect, the instance is shut down and cold-booted, up to three rounds. The first boot takes 1–2 minutes longer. +- Once it is installed, start instances with AUTO-MAS as usual; the bridge stays in place. +- An instance that has only ever been started by AUTO-MAS and never by the manager is not installed automatically in this version: start it once with the manager. +- The bridge is installed in the instance's own system / vendor overlay and does not touch Google's original image. + +## If you do not want the bridge + +- When creating an instance, use `tools\modavd.exe create --no-bridge`; +- For an existing instance, run `tools\modavd.exe bridge uninstall`; after that, starts no longer install it automatically (run `install` again to restore it); +- `start --no-bridge` applies to that one start only; +- For a read-only check, use `bridge verify` (the instance must be running). + +::: warning Before rolling back to an older version +Before going back to a version earlier than 0.36.0, run `tools\modavd.exe bridge uninstall` once on every instance, and swap the package only after the result says the bridge is not installed. If you roll back without uninstalling first, every app on that instance that only has an arm64 build will fail to run. +::: + +The source is in the [`bridge/`](https://github.com/ModAVD/ModAVD/tree/main/bridge) folder of the ModAVD repository, released under AGPL-3.0-or-later. diff --git a/docs/en/components/ime.md b/docs/en/components/ime.md new file mode 100644 index 0000000..2bb0f8f --- /dev/null +++ b/docs/en/components/ime.md @@ -0,0 +1,30 @@ +# The host IME + +A headless input method, `org.modavd.ime`, installed in the emulator's Android. Its source is in [ModAVDHostIME](https://github.com/ModAVD/ModAVDHostIME). + +When you use the emulator with a window, it lets you type into Android directly with the Windows IME: Chinese, AltGr combinations and dead keys all work. Automation scripts can also type text in any language with the emulator console's `ime commit `. + +- It shows no input interface. Physical keys and `adb shell input` injection still go straight to the app, so games and automation are not affected. +- It requests no permissions, stores no data, and does not log the text that is typed. +- It needs an emulator from ModAVD 0.35.0 or newer (the `modavd_ime` channel exists from this version) and Android 11 or later. On an older emulator, installing it is harmless: it only backs off and retries the connection in the background and does not crash. + +## How it gets installed + +From 0.36.0 the package includes `components\modavd-ime.apk`, which is handled automatically when you start with the manager or the CLI: + +- It is installed when it is not installed yet, when the installed version is older than the one in the package, or when the version number is the same but the content differs. It is never downgraded; +- After a fresh install it is enabled, and the decision whether to make it the default input method is made only once: it becomes the default only if the default at that time is still the image's own keyboard (or none). After that, if you switch to another input method, it is not changed back. + +If you use a ModAVD release package, you do not need to install it manually. + +## Signing certificate of the official APK + +- Certificate SHA-256: `d18cca3c1982149349f164d28e02490b2eb694e44d56834fa02a1062eb42e823` +- Certificate subject `CN=ModAVD`, RSA 4096. +- To verify: `apksigner verify --print-certs `. The `certificate SHA-256 digest` in the output must match the value above. + +## Downloading separately + +The IME has its own version number (from v1.0.0) and is released separately on [ModAVDHostIME Releases](https://github.com/ModAVD/ModAVDHostIME/releases). The `components\modavd-ime.apk` in v1.0.0 and in the ModAVD 0.36.0 package is byte-for-byte identical. + +An APK you build yourself is signed with your own key. To keep using it, its `versionCode` must be higher than the one in the release package; otherwise the manager switches back to the official version at boot. See [Develop: the host IME](../develop/ime). diff --git a/docs/en/components/manager.md b/docs/en/components/manager.md new file mode 100644 index 0000000..0475e6b --- /dev/null +++ b/docs/en/components/manager.md @@ -0,0 +1,74 @@ +# The manager and CLI + +Since ModAVD 0.36.0 the package includes two management programs. Their source is in [ModAVDManager](https://github.com/ModAVD/ModAVDManager): + +| Program | Location | Who uses it | +|---|---|---| +| The manager `ModAVD.exe` | Package root | People: instance cards, start and stop, create, settings, screen view, logs | +| The CLI `modavd.exe` | `tools\` | Scripts, AUTO-MAS and AI assistants; `modavd mcp` is an MCP server | + +Instance files and `mas-avd.json` are written the same way as in AUTO-MAS, so instances created by either one can be started by the other. + +## The manager + +Double-click `ModAVD.exe` in the package root. It starts a local server that listens only on 127.0.0.1, then opens a window in Edge app mode (Edge comes with Windows 10 / 11); if Edge is not found, it uses the default browser. The page is embedded in the program and does not use the network. + +- Features: instance cards (with a screen thumbnail when the instance is online), start and stop (with progress), create, settings, delete, PC check, activity panel, screen view (a click is a tap; Back / Home / Recents), logs, install APK, open folder; light / dark / follow system. +- One server runs per root folder; double-clicking again only reopens the window. The log is at `\logs\modavd-ui.log`. +- After you close the window, it waits for the running tasks to finish before it exits. + +## The CLI + +``` +modavd [--root ] [--json] [--quiet] [--verbose] [args] +``` + +| Command | Description | +| --- | --- | +| `list` | All instances and their status | +| `info ` / `status ` | Detailed information / status of one instance | +| `create [--index N] [--title T] [--memory 6G] [--cpu N] [--data-gb N] [--fps 30\|60] [--no-bridge]` | Create an instance. It includes the arm64 execution bridge by default, installed on the first boot | +| `set [--title T] [--memory M] [--cpu N] [--fps F] [--data-gb N]` | Change settings; they take effect on the next start. `--data-gb` can only grow the disk; game data is untouched | +| `delete --yes` | Delete a shut-down instance together with its game data | +| `start [--window] [--timeout 300] [--no-bridge]` | Start, then wait until boot is complete and the post-boot setup is done | +| `stop [--force]`, `stop --all` | Shut down: first ask Android for a clean shutdown; if that fails, sync + kill. `--force` is sync + force-kill | +| `restart [--window] [--timeout 300] [--force] [--no-bridge]` | Shut down, then start | +| `bridge install\|uninstall\|verify` | Install, uninstall or check the arm64 execution bridge | +| `check [--memory M]` | PC check before starting | +| `window show\|hide`, `install `, `shell `, `adb -- ` | Window, install an APK, guest command, adb passthrough | +| `servers`, `version`, `mcp`, `help [command]` | | + +- Write `` as `3` or `mas_3` (0–9, except 5). Write memory as `6144`, `6144M` or `6G`; there are only four levels: 3 / 4 / 5 / 6 GB. Cores are 2 / 4 / 6, frame rate 30 / 60. +- Root folder: `--root`; if it is not given, the environment variable `MODAVD_ROOT`; otherwise the folder containing `modavd.exe` or its parent. +- adb uses only the root folder's private server (`adbServerPort` in `mas-avd.json`) and never touches the default 5037. +- Shutdown syncs first and then stops, so it does not leave 0-byte files behind. + +For machine-readable output (`--json`), exit codes and the full option reference, see [the ModAVDManager README](https://github.com/ModAVD/ModAVDManager/blob/main/README.en.md#command-line). + +## MCP + +``` +modavd mcp --root D:\ModAVD +``` + +Tools: `list_instances`, `get_instance`, `check_host`, `create_instance`, `update_instance`, `delete_instance`, `start_instance` (with progress), `stop_instance`, `set_window_visible`, `shell`, `install_apk`. + +To register it in Claude Code (in PowerShell, `--` must be quoted, or it gets swallowed): + +```powershell +claude mcp add --transport stdio --scope user modavd '--' D:\ModAVD\tools\modavd.exe mcp --root D:\ModAVD +``` + +Or in a project's `.mcp.json`: + +```json +{"mcpServers": {"modavd": {"command": "D:\\ModAVD\\tools\\modavd.exe", "args": ["mcp", "--root", "D:\\ModAVD"]}}} +``` + +## Downloading separately + +The manager has its own version number (from v0.1.0) and is released separately on [ModAVDManager Releases](https://github.com/ModAVD/ModAVDManager/releases). + +- Normally just use the copy that comes in the ModAVD package; it is replaced together with the package when you upgrade. +- A separately downloaded package can go in any folder; point it at the ModAVD root folder with `--root` or `MODAVD_ROOT`. Do not extract it into the ModAVD root folder and overwrite the files from the package. +- After downloading, check the hash against `SHA256SUMS.txt` in the release. diff --git a/docs/en/develop/adapt-game.md b/docs/en/develop/adapt-game.md new file mode 100644 index 0000000..f2f6171 --- /dev/null +++ b/docs/en/develop/adapt-game.md @@ -0,0 +1,67 @@ +# Adapting a new game + +To get a game that currently will not start or runs badly working on ModAVD, work in the order below. Adaptation has no separate code folder: after you find the cause, the change lands in one place, either the execution bridge, the emulator itself or the automation script. Specific cases investigated before are in [How to investigate common problems](./troubleshooting). + +## Preparation + +- Use the latest ModAVD, create a new dedicated instance, and log in with a test account. Adapting a game means installing, uninstalling and force-killing over and over, so do not use the instance you normally use to run games unattended. Old versions lack a lot; for example, 0.32 does not have the arm64 execution bridge yet. +- For adb, use the private port from `mas-avd.json`: `adb -P -s emulator- ...`. Commands below that need root are written as `shell su 0 `. +- The hardware is the same as for [testing on real games](./testing#hardware), plus disk space for this game. + +## Step 1: Find out about the game {#step1} + +- Package name, channel (official server, Bilibili server, the various app stores) and game version. Find the package name with `shell pm list packages | findstr `. +- Whether it is arm64-only: `shell dumpsys package | findstr primaryCpuAbi`. `arm64-v8a` has to go through translation; `x86_64` or `x86` runs natively. +- Whether it uses app hardening or anti-cheat: unpack the APK and look at the library names under `lib/`. +- Whether an automation script already exists (the MAA series, M9A, SRC and the like). This decides whether the game can enter the pre-release regression tests later. + +## Step 2: Reproduce and collect evidence {#step2} + +- Before launching, run `logcat -c`, then `logcat -b all -v threadtime > 启动.txt`, then launch the game, and stop once it reproduces. +- If it crashed, take the newest file in `/data/tombstones/`: `shell su 0 ls -t /data/tombstones`. +- The emulator logs are in `logs\` in the root folder. `收集问题信息.bat` (collects a problem report) packs the logs together. +- Look mainly at these: + - `Fatal signal`: which library and which thread it crashed in; + - `FATAL EXCEPTION`; + - `ANR in`; + - `VK_ERROR` and `DEVICE_LOST` in the emulator log. +- Note exactly which step it gets stuck on, and whether it happens every time or only sometimes. + +## Step 3: Find what can be changed, by cause {#step3} + +Try approaches that need no recompiling first, and settle the direction before you touch code. + +| Cause | Try first (no recompiling) | Where to change it if you really have to | +|---|---|---| +| arm64-only; crashes in the translation layer or exits right after launch | Make the execution bridge take effect for it temporarily: first make sure the instance has the bridge installed, then run `shell su 0 setprop modavd.gate.target `, then cold-start the game. This property replaces the whole list, is not persistent, and is gone when the guest restarts. Check logcat for `shim-ctor OK` and whether the game gets through | If it gets through: send a PR that adds the package name to `kDefaultGateTargets` in `bridge/companion/jni/native_to_guest_companion.cc`. If it still fails: this is the deep end of the translation layer and the bridge; open an issue with the tombstone and the launch logcat | +| Graphics: black screen, garbled picture, Vulkan errors, device lost | Two environment-variable switches that are off by default (`ANDROID_EMU_VK_HIDE_REBAR=1`, `ANDROID_EMU_VK_ASTC_BC3=1`; see [How ModAVD differs from the official emulator](../guide/changes#graphics)); change the resolution (two levels, 720p / 1080p); while investigating, you can uninstall the bridge with `modavd bridge uninstall` to compare, and uninstalling the bridge restores the GL transport to what it was | The emulator's gfxstream patches; see [the emulator itself](./emulator) | +| Stutter and frame drops | Use [How to investigate stutter and frame drops](./performance) to find out which thread and which stage the bottleneck is in; then compare a few existing switches: GL transport asg / pipe, `ANDROID_EMU_VK_HIDE_REBAR=1`, and the Vulkan feature flags | If the bottleneck is the graphics round trip: change the emulator's gfxstream. If it is in translation: open an issue to discuss it. If it is in the game's own main thread: accept it, or lower the quality | +| The game detects the emulator or root, or kicks you out after you enter the game | — | Do not do anything. ModAVD does not bypass the game's protection and does not change game files. Note the exact message text and open an issue describing the situation | +| Network request timeout, hot update fails | Switch networks; check the game's data folder for 0-byte files (left behind by a hard kill) | Usually not something to change in ModAVD | +| The game runs, but the automation script cannot recognize the screen or taps inaccurately | Check the script's screenshot method, input method and resolution settings | Change it on the script side (for example, add a resource overlay for ModAVD to it), not in ModAVD | + +## Step 4: Delivery {#step4} + +- First open an issue titled "Adapt: \ (\)", describing what you found in the first three steps and how you plan to change it. +- Adding to the bridge list: the PR changes only `kDefaultGateTargets` and comes with this evidence: + - the launch logcat before and after adding to the list; + - a game that was already on the list (*Onmyoji*) still works as usual; + - a game not on the list still works as usual, and the log has the line saying the changes were reverted. + + The maintainer rebuilds the payload and pins its hash; it ships with the next version. +- Graphics changes: submit patches as described in [the emulator itself](./emulator#patches). +- You are not asked to run the full set of regression tests; the maintainer builds and tests on their own machine. + +## Step 5: After it works {#after} + +- Add a note in the pinned "兼容性报告" (compatibility report) issue: the game, channel, version, how you got it running, and roughly what frame rate it gets. +- If the game has an automation script and tasks that do not use many resources, the maintainer will add it to the pre-release regression tests and record a frame-rate baseline. + +## Rules + +Use a test account; do not modify game files or bypass the game's protection; do not pull gacha or buy anything; stop when you hit an update prompt; sync before you shut down the emulator. + +## Suggested models + +- Steps 1 and 2, and the classification in step 3 (reading logcat and the emulator logs): Sonnet 5.5 or GPT-6.1 Sol; +- Finding the root cause in the translation layer, the bridge or graphics problems: Opus 5.5 or GPT-6 Astra, with reasoning set high. diff --git a/docs/en/develop/ai.md b/docs/en/develop/ai.md new file mode 100644 index 0000000..9a5c5ff --- /dev/null +++ b/docs/en/develop/ai.md @@ -0,0 +1,27 @@ +# Using AI tools + +You can use AI tools to write code and investigate problems. There are only two rules: + +- **Have the AI test its own changes**: compile, run the unit tests, and if you can start an instance, start one and look at the result. State in the PR what you tested and what the result was. +- **Do not give accounts, keys or private paths to AI tools.** + +## Choosing a model by task + +| Task | Claude | GPT (Codex) | +|---|---|---| +| The emulator itself, the execution bridge, root causes of crashes | Opus 5.5, reasoning set high | GPT-6 Astra, High or Extra High | +| Ordinary features and UI of the manager, the IME, CI, scripts | Sonnet 5.5 | GPT-6.1 Sol, Medium | +| Anything security-related, such as package updates, signing, release workflows and permissions | Sonnet 5.5 writes, Opus 5.5 reviews, a person reviews last | GPT-6.1 Sol writes, GPT-6 Astra reviews, a person reviews last | +| Running tests, reading logs, watching progress, writing docs | Sonnet 5.5 | GPT-6.1 Terra | +| Tidying logs, bulk reformatting, checking lists | Haiku 5.5 | GPT-6 Luna | +| Review before merging a PR | Opus 5.5, not the same session that wrote the code | GPT-6 Astra, likewise not the same session that wrote the code | + +- Model names are as of October 2026. Both vendors update quickly; after a new generation, choose by the same tiers: give hard work to the strongest model with reasoning set high; give work with clear rules to a mid-tier model; give pure shuffling to a cheap model. +- On the GPT side, GPT-6.1 Sol is close to Astra and a good deal cheaper, which suits long coding sessions; when the budget is tight, the emulator itself and the bridge can also use Sol with Extra High. +- More specific advice for each area is in [Areas](./overview) and in the "Suggested models" section of each area's page. + +## Prompting the AI + +- Tell it the rules to follow directly: sync before you shut down the emulator; use the private port for adb and never touch 5037; do not change game files or bypass the game's protection; do not pull gacha or buy anything in the game. +- Have it gather evidence before it draws conclusions. The misjudgments in the past (taking a fork deadlock for a slow network, taking files damaged by a hard kill for a CDN that could not be reached) all came from guessing without looking at the actual scene. See [How to investigate common problems](./troubleshooting). +- When comparing performance, have it run alternating rounds; a single run of each does not count. diff --git a/docs/en/develop/automas.md b/docs/en/develop/automas.md new file mode 100644 index 0000000..fcee584 --- /dev/null +++ b/docs/en/develop/automas.md @@ -0,0 +1,31 @@ +# AUTO-MAS integration + +**What to change**: the part of [AUTO-MAS](https://github.com/AUTO-MAS-Project/AUTO-MAS) that manages ModAVD instances (creating instances, start and stop, updating the package). It is planned to change from AUTO-MAS's own Python implementation to calling the `modavd` CLI. The change goes to the AUTO-MAS repository, and AUTO-MAS's maintainers decide whether to accept it. + +## Environment + +Follow AUTO-MAS's own development guidelines (Python environment, generated frontend files, test requirements). + +## Hardware + +In addition to AUTO-MAS's own development environment. If you need to start instances, it is best to also install a game for real runs. + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 4 cores, 8 threads | +| Memory | 32 GB | 16 GB | +| Free disk space | 100 GB | 50 GB | + +## Integration points + +- stdout is a single line of JSON and the log goes to stderr; collect the two separately; +- go by `error.code`; +- the first boot installs the execution bridge and restarts one or two more times, so allow a longer time limit; +- shut down with `modavd stop`, which syncs first and then stops; +- adb uses only the root folder's private server and never touches 5037. + +The contract for the commands and the machine-readable output is in [the ModAVDManager README](https://github.com/ModAVD/ModAVDManager/blob/main/README.en.md#command-line). + +## Suggested models + +Sonnet 5.5 or GPT-6.1 Sol writes; for subprocesses, time limits and mutual exclusion during updates, Opus 5.5 or GPT-6 Astra reviews. diff --git a/docs/en/develop/bridge.md b/docs/en/develop/bridge.md new file mode 100644 index 0000000..78f5ea7 --- /dev/null +++ b/docs/en/develop/bridge.md @@ -0,0 +1,87 @@ +# The arm64 execution bridge + +**What to change**: the layer that lets games that only have an arm64 build (currently the 7 channel builds of *Onmyoji*) run on the x86_64 image. It is in the [`bridge/`](https://github.com/ModAVD/ModAVD/tree/main/bridge) folder of the ModAVD repository: + +| Directory | Output | What it does | +| --- | --- | --- | +| `nbshim/` | `libmodavd_bridge.so` | Our own native bridge entry. `ro.dalvik.vm.native.bridge` points to it; `libnativebridge` opens it and gets `NativeBridgeItf`. It opens the real translation layer, copies the real struct over byte by byte at the length for its version (v6 = 144, v7 = 160, v8 = 168 bytes), forwards the calls one by one, and brings up the companion library | +| `companion/` | `libmodavd_gate.so` | The bridge itself. Using the interfaces the translation layer exports itself (`WrapGuestFunctionImpl`, `RunGuestCall`, `GuestMapShadow`, `MakeTrampolineCallable`), it adds from the outside the path "native x86 code calls a translated arm64 function through a rewritten GOT slot" | +| `nbshim/modavd_bridge.rc` | — | Re-registers the arm64 `binfmt_misc` for the new property value (the one that comes with the image only recognizes `libndk_translation.so`; once the property is changed, arm64 executables silently stop running); on instances that have been remounted, it loads zram at boot | + +Two generations of the translation layer are supported: ndk_translation 0.2.3 on the API 34 image (this is the one ModAVD uses) and Berberis 16 on API 36.1. Symbols are resolved by name from the images that are already mapped, and are not tied to a build-id. + +::: danger Hard rules +Do not modify, patch or bypass any file of the game itself or of its protection libraries, and do not interfere with any integrity or anti-tamper check. Not a single byte of Google's translation-layer binaries is changed. +::: + +## Why it crashes + +Google's native bridge keeps ART and the system libraries native x86_64 and translates only the app's own arm64 libraries, so one process is split into two halves. + +To watch dex reads, the game's hardening library (arm64, translated) makes the GNU_RELRO of the host `libartbase.so` writable, rewrites the GOT slots of functions such as `read` / `mmap` to point to its own arm64 hooks, and then makes it read-only again. On a real device both sides are arm64, and everything works up to this point. + +On an x86_64 image, the native x86 ART runs `call [GOT slot]` and jumps into an arm64 code page. The translation layer maps guest code pages as non-executable on the host side, so the instruction fetch faults (`SIGSEGV`, `SEGV_ACCERR`, and the fault address is RIP). The translation layer's own signal handler has no branch for "native code jumped into a guest page" and treats it as a fatal error. + +## Only for apps on the list + +- The list is comma-separated package names, compiled into the companion library (`kDefaultGateTargets` in `companion/jni/native_to_guest_companion.cc`). +- If the property `modavd.gate.target` is set, it replaces the whole list; `setprop modavd.gate.target none` turns it off. The property is not persistent and is gone when the guest restarts, which suits trying out a new game temporarily (see [Adapting a new game](./adapt-game)). +- For apps not on the list, after the process is forked, the forwarding callbacks and the slots the companion library changed are all reverted, so they behave the same as when the bridge is not installed. + +## Signal order + +The bridge works through SIGSEGV: the instruction-fetch fault raised when native code jumps into a guest page through a rewritten GOT slot is the bridge's entry point. This fault must reach us first, or another handler will treat it as a real crash, for example WebView's crashpad. + +The shim hands the companion library's decision function to ART through `getSignalHandler(SIGSEGV)` of the native bridge interface. ART registers it with libsigchain as a special handler, placed before all `sigaction()` handlers. It recognizes only our own trap and lets everything else through. + +## Environment and hardware + +- Android NDK r27c, about 2.2 GB; if you only compile, 4 cores and 8 GB of memory are enough. +- The build is deterministic: the same source built in different folders gives outputs that differ only in the build-id. + +Verification needs a running instance with *Onmyoji* installed (6 GB of memory and 6 cores for testing; the instance is about 26 GB): + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 6 cores, 12 threads (the instance gets 6 cores); 4 cores, 8 threads also works, but the instance can only get 4 cores and the frame rate is lower | +| Memory | 32 GB | 16 GB | +| Free disk space | 50 GB | 35 GB | + +## Build + +```powershell +$env:ANDROID_NDK_ROOT = '' +pwsh -File bridge\scripts\build-companion.ps1 [-OutDir ] +pwsh -File bridge\scripts\build-nbshim.ps1 [-OutDir ] +``` + +- The outputs go under `%TEMP%\modavd-nbbridge\` by default. +- **Never add `-ftls-model=initial-exec`.** Both libraries are loaded with dlopen. A library with initial-exec TLS relocations is rejected by Bionic's linker, the whole native bridge fails to load, and the device loses all arm64 support. `build-nbshim.ps1` checks after building and reports an error if there are such relocations. +- The same source built in different folders differs only in `.note.gnu.build-id` (the build-id contains the source path); with that section removed, the outputs are byte-identical. So it is normal that the hash of your own build does not match the one in the Release; the SHA-256 of the three files in the Release is written in `modavd-manifest.json` in the root of the main package. + +## Installing into an instance + +What does the work is `tools\modavd-bridge.ps1` in the package (its source is in `package/` in the ModAVD repository); the `bridge` command of the manager and the CLI both call it: + +- The instance must boot with a writable system image. The emulator from 0.36.0 does this when it sees the boot marker `modavd-writable-system` in the instance folder. +- The two `.so` files go into `/system/lib64` (the shim has to be there: `libnativebridge` opens it in the linker's "system" namespace), and the `.rc` goes into `/system/etc/init/`. +- Append `ro.dalvik.vm.native.bridge=libmodavd_bridge.so` to `/vendor/build.prop`. Back it up before changing it; check it after writing and before rebooting; run `sync` before rebooting. +- Install in the order files first, property last; uninstall in the order property first, then files. The property is the only switch. +- All of this is written to the instance's own system / vendor overlay and does not touch Google's original image. +- When the guest needs a restart, always shut down and cold boot; do not hot-restart inside the guest (a hot restart hangs on WHPX, see [How to investigate common problems](./troubleshooting#system-level)). + +For how users install and uninstall the bridge, see [Components: the arm64 execution bridge](../components/bridge). + +## Verification + +- You need a logged-in instance with a game on the list installed; install the bridge with the manager or `modavd bridge install`; +- Check logcat for `shim-ctor OK`; for a game not on the list, the log must have the line saying the changes were reverted; +- Every version must be run on real games. + +## Delivery + +Open an issue first to discuss the change. It directly affects whether games can start. The maintainer rebuilds the payload and pins its hash; it ships with the next version. + +## Suggested models + +Opus 5.5 or GPT-6 Astra. With native code, the ABI, the dynamic linker and GOT rewriting, one mistake means the game will not start, and it also touches the hard rules above. diff --git a/docs/en/develop/build.md b/docs/en/develop/build.md new file mode 100644 index 0000000..fbe6c62 --- /dev/null +++ b/docs/en/develop/build.md @@ -0,0 +1,137 @@ +# Building from source + +The examples below use `0.36.0` (build number mas-36). For other versions, use that version's repository manifests and patches; do not mix source and binaries from different versions. + +The matching source bundle is in the [0.36.0 Release](https://github.com/ModAVD/ModAVD/releases/tag/v0.36.0). Its `SOURCES.txt` explains where each source file comes from and what it is for. From 0.32.1 the version is written `0..

` (tag `v0.32.1`), and N is the build number mas-N: the manifest is still called `manifest-masN.xml`, and the patch folders are still named masN. + +For the hardware requirements, see [the emulator itself](./emulator#hardware). + +## Environment + +- Windows x86_64; Visual Studio 2022 Build Tools **17.14**, with the C++ desktop build tools, ATL, MFC, and the Windows 10 SDK 19041 / Windows 11 SDK 22621 installed. +- Turn on Windows Developer Mode, so that Git and repo can create symbolic links. +- Python 3.12, Git for Windows, and the [repo tool](https://gerrit.googlesource.com/git-repo/). Put Python, Git and repo on the terminal PATH. +- Use a short source root folder, and leave enough space for all the AOSP projects, the prebuilt libraries, the intermediate build files and the outputs. Below, `$src` stands for the source folder you choose, and `$modavd` for the location of the ModAVD repository. + +The commands below run in PowerShell 7. On Windows, repo can be used through `repo.cmd`, which calls its Python launch script; for upstream's notes, see `external/qemu/android/docs/WINDOWS-DEV.md` in the source. + +Set up your own Git commit identity. Turn off line-ending conversion and turn on symbolic links and long paths; the settings below affect only the current terminal and its child processes: + +```powershell +$env:GIT_CONFIG_COUNT = '3' +$env:GIT_CONFIG_KEY_0 = 'core.autocrlf' +$env:GIT_CONFIG_VALUE_0 = 'false' +$env:GIT_CONFIG_KEY_1 = 'core.symlinks' +$env:GIT_CONFIG_VALUE_1 = 'true' +$env:GIT_CONFIG_KEY_2 = 'core.longpaths' +$env:GIT_CONFIG_VALUE_2 = 'true' +``` + +## Getting the exact version + +First clone the ModAVD repository and check out the release tag of that version. The release tag of mas30 predates `manifests/`; to build mas30 you can check out the mas32 tag, which also contains the manifest and patches for mas30. Note the absolute location of the repository folder: + +```powershell +git clone https://github.com/ModAVD/ModAVD.git +Set-Location ModAVD +git checkout v0.36.0 +$modavd = (Get-Location).Path +``` + +The manifests are in `manifests/`, and the source bundle contains the same files: + +- `manifest-mas36.xml` records the full target commits used at build time (there is one `manifest-.xml` per version). +- `manifest-aosp-base.xml` replaces the three patch repositories with the public AOSP base; all other projects stay the same. + +::: warning Do not run repo sync directly on the target manifest +The three local commits in qemu, gfxstream and Crashpad are not on the AOSP servers. Use the target manifest as the reference for the version, run `repo init` / `repo sync` with the matching public base manifest, and then replay the patches to get the target source tree. +::: + +Run this in the short-path source folder you chose (first set `$src` to the absolute path of this folder): + +```powershell +New-Item -ItemType Directory -Path $src +Set-Location $src +repo init -u https://android.googlesource.com/platform/manifest -b emu-main-dev --depth=1 +if ($LASTEXITCODE) { throw 'repo init failed' } +Copy-Item "$modavd/manifests/manifest-aosp-base.xml" .repo/manifests/manifest-aosp-base.xml +repo init -u https://android.googlesource.com/platform/manifest -b emu-main-dev -m manifest-aosp-base.xml --depth=1 +if ($LASTEXITCODE) { throw 'pinned repo init failed' } +repo sync -c --no-clone-bundle -j4 +if ($LASTEXITCODE) { throw 'repo sync failed' } +``` + +The prebuilt tools and libraries in the manifest are also build inputs and should be synced along with the rest. Do not substitute the heads of moving branches for the pinned revisions. + +## Replaying the patches + +For the base, the folders, the number of patches and the license of each patch, see [patches/BASE.md](https://github.com/ModAVD/ModAVD/blob/main/patches/BASE.md). Apply them in file-name order with `git am --keep-cr`, which keeps the CR bytes the patches already contain: + +```powershell +$series = @( + @{ Repo = 'external/qemu'; Patches = 'qemu/mas36' }, + @{ Repo = 'hardware/google/gfxstream'; Patches = 'gfxstream/mas35' }, + @{ Repo = 'external/crashpad'; Patches = 'crashpad/module-version22' } +) +foreach ($s in $series) { + $repo = Join-Path $src $s.Repo + git -C $repo switch -c modavd-mas36 + if ($LASTEXITCODE) { throw "branch failed: $($s.Repo)" } + $patches = @(Get-ChildItem (Join-Path "$modavd/patches" $s.Patches) -Filter '*.patch' | + Sort-Object Name | ForEach-Object FullName) + git -C $repo am --keep-cr @patches + if ($LASTEXITCODE) { throw "patch replay failed: $($s.Repo)" } +} +``` + +The authorship of the patches has been unified. Replaying produces different commit IDs, so compare the source trees instead of requiring the local commit IDs to equal the private commits in the target manifest. `git -C rev-parse 'HEAD^{tree}'` should give the same values as the table below (`patches/series.json` in the ModAVD repository also records these trees): + +| Repository | mas36 target tree | +| --- | --- | +| qemu | `5483e1bf05ebb3cd7549aed2adcde998bf1c4703` | +| gfxstream | `51288846bdcb9054d899ffc8458d6744d6206e32` | +| Crashpad | `0a3c056231f5257034514ae8d83192de388351f0` | + +You can also skip the replay and fetch tag `v0.36.0` from the already-patched repositories [ModAVDQemu](https://github.com/ModAVD/ModAVDQemu), [ModAVDGfxstream](https://github.com/ModAVD/ModAVDGfxstream) and [ModAVDCrashpad](https://github.com/ModAVD/ModAVDCrashpad); the source trees are the ones in the table above: + +```powershell +$tags = @( + @{ Repo = 'external/qemu'; Url = 'https://github.com/ModAVD/ModAVDQemu.git' }, + @{ Repo = 'hardware/google/gfxstream'; Url = 'https://github.com/ModAVD/ModAVDGfxstream.git' }, + @{ Repo = 'external/crashpad'; Url = 'https://github.com/ModAVD/ModAVDCrashpad.git' } +) +foreach ($t in $tags) { + $repo = Join-Path $src $t.Repo + git -C $repo fetch $t.Url v0.36.0 + if ($LASTEXITCODE) { throw "fetch failed: $($t.Repo)" } + git -C $repo switch -c modavd-mas36 FETCH_HEAD + if ($LASTEXITCODE) { throw "branch failed: $($t.Repo)" } +} +``` + +All other projects should stay at the revisions recorded in the target manifest. + +## Build + +Start from the synced source root, and use this version's prebuilt libraries and tools. The command below skips cleaning, rebuilding the prebuilt libraries and the test tasks, does a release build, and keeps the incremental build folder. + +```powershell +$env:PYTHONUTF8 = '1' +$env:CMAKE_BUILD_PARALLEL_LEVEL = '12' # no more than 16; lower it according to the available memory +$env:PATH = "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer;$env:PATH" +$out = Join-Path $src 'objs' +$dist = Join-Path $src 'dist' +Set-Location (Join-Path $src 'external/qemu') +python android/build/python/cmake.py --out $out --dist $dist ` + --task-disable clean --task-disable removeleftover --task-disable prebuilts ` + --task-disable ctest --task-disable emugentest --task-disable genentriestest ` + --task-disable coveragereport --task-disable packagesamples ` + --task-disable zipintegrationtests --task-disable integrationtest +if ($LASTEXITCODE) { throw 'build failed' } +``` + +The ready-to-use emulator folder is at `$out/distribution/emulator/`, and the distribution archive is at `$dist/`. + +To install it as your own SDK, copy `emulator/` in full, and set `Pkg.BuildId` in its `source.properties` to the matching build number, for example `mas-36`. What is built here is the emulator program; adb, the system images, the instances and the game data are managed separately. + +This flow uses the AOSP prebuilt dependencies pinned by the manifest. If you need to modify, rebuild or relink third-party libraries such as Qt and GLib, the source bundle has the matching source, patches and build files; handle them according to `SOURCES.txt` and the project notes in `LICENSES/NOTICE.txt` of the main package. diff --git a/docs/en/develop/docs.md b/docs/en/develop/docs.md new file mode 100644 index 0000000..f70de7f --- /dev/null +++ b/docs/en/develop/docs.md @@ -0,0 +1,54 @@ +# Documentation and this site + +## Where the documentation is + +| Content | Where to change it | +|---|---| +| This site (user guide, component introductions, contributing) | `docs/` in the [modavd.github.io](https://github.com/ModAVD/modavd.github.io) repository | +| The README of each repository (Chinese `README.md`, English `README.en.md`), `CONTRIBUTING.md` | Each repository itself | +| Release notes, patch notes (`release/`, `patches/BASE.md`), the repository list | The ModAVD repository. They are exported by the release tool, so **do not edit them by hand**; if you find a problem, open an issue first | + +Every page has "Edit this page on GitHub" at the bottom. Click through, edit directly, and send a PR. + +## Previewing this site locally + +You need Node.js 24. In the repository root: + +```powershell +npm ci --ignore-scripts +npm run dev # local preview, refreshes automatically on change +npm run build # builds the static pages; dead links cause an error +python scripts/i18n.py # checks the pages and headings of the Chinese and English sides +``` + +- The pages are Markdown under `docs/`. The English pages are under `docs/en/`, with paths matching the Chinese pages one to one. The sidebar and navigation are in `docs/.vitepress/config.mts`, one copy per language, so a new page must be added to both. +- Callouts use `::: tip`, `::: warning`, `::: danger` and `::: info`. +- Links inside the site are relative paths without `.md`, for example `[Installation](../guide/install)`. For files in other repositories, write the full GitHub address. +- Do not add analytics scripts, and do not load fonts or images from other websites. +- For image and icon changes, open an issue first with the source and the license. + +## Checks + +CI on a PR will: + +- build the whole site, and fail on dead links; +- check the Chinese and English sides against each other (`python scripts/i18n.py`): every page exists in both languages, headings have the same levels and order, links inside the site do not lead to the other language, and every `#anchor` can be found. A page changed in only one language is flagged as a reminder; +- run the privacy scan (`python scripts/privacy.py .`), which fails if the docs contain paths such as a local user directory. + +After a merge into `main`, the site is published to GitHub Pages automatically. + +## Writing style + +- Chinese is the main language, and the English pages match it page by page. If you change a Chinese page, change the matching English page in the same PR. If you cannot write English, say so in the PR and the maintainer will add it. +- In Chinese, use short sentences and wording that users understand. In English, write natural English that follows the meaning, not word for word. +- Say from which version something exists and from which version it changed. +- Use tables and lists where you can instead of long paragraphs. +- Write commands in full so they can be copied as they are, and mark the parts of a path to replace with ``. + +## Hardware + +No limit. Anything that can run Node.js and Python is fine. + +## Suggested models + +Sonnet 5.5 or GPT-6.1 Terra; for bulk formatting changes, Haiku 5.5 or GPT-6 Luna. diff --git a/docs/en/develop/emulator.md b/docs/en/develop/emulator.md new file mode 100644 index 0000000..4d3af83 --- /dev/null +++ b/docs/en/develop/emulator.md @@ -0,0 +1,57 @@ +# The emulator itself + +**What you change**: the patches on three repositories of the AOSP emulator: qemu, gfxstream (graphics) and crashpad. The patches for each version are in `patches///` of the ModAVD repository, and the matching manifests are in `manifests/`. + +## Environment + +For details, see [Building from source](./build): + +- Visual Studio 2022 Build Tools 17.14: C++ desktop build tools, ATL, MFC, Windows 10 SDK 19041 / Windows 11 SDK 22621; +- Python 3.12, Git for Windows and the [repo tool](https://gerrit.googlesource.com/git-repo/), all on PATH; +- Turn on Windows Developer Mode (Git and repo need to create symbolic links); +- Turn off line-ending conversion in Git and turn on symbolic links and long paths (`core.autocrlf=false`, `core.symlinks=true`, `core.longpaths=true`); +- Pick a short source directory and leave plenty of space: source plus build output is about **124 GB**, and an SSD is recommended. + +## Hardware + +| | Recommended | Minimum | +|---|---|---| +| CPU | 16 cores, 32 threads: a full build takes about 1.5 hours | 8 cores, 16 threads: a full build is estimated at over 3 hours (estimated from the core count, not measured) | +| RAM | 32 GB: after building you can start an instance directly and test games | 16 GB: lower the build parallelism and do not run instances while building (estimate) | +| Free disk space | 200 GB, SSD: includes build output, test instances and one game | 140 GB: source plus output is about 124 GB, plus one instance with no games installed | + +Incremental builds are much faster than full builds. + +## Getting the source and changing patches {#patches} + +1. Open an issue first, saying what to change and why. +2. Get the source: in the ModAVD repository directory, run `python scripts/checks.py patches --work `. It shallow-fetches the AOSP base from android.googlesource.com, replays the patches of the current version in `patches/`, and creates one repository per patch series, such as `qemu-mas36`. + + You can also clone the already patched repositories directly: [ModAVDQemu](https://github.com/ModAVD/ModAVDQemu), [ModAVDGfxstream](https://github.com/ModAVD/ModAVDGfxstream) and [ModAVDCrashpad](https://github.com/ModAVD/ModAVDCrashpad). The current version is on the `release/0.36` branch, and the source tree is the same as the replayed one, for example `git clone -b release/0.36 https://github.com/ModAVD/ModAVDQemu.git`. The full history is about 400 MB; add `--depth 100` if you only want the most recent commits. +3. Go into the repository you want to change and **first change the commit identity to your own**, because the script sets the maintainer's identity on these repositories: + + ``` + git config user.name "" + git config user.email "" + ``` + +4. Commit your changes and export them with `git format-patch -`. Attach the `.patch` files to the issue, or open a PR that only adds `incoming//*.patch`. + - The patch is exactly what will be taken in: based on the current series, with your noreply e-mail as the author and no `Co-authored-by`. Do not put test scripts into the patch itself; put them in the PR description or in `incoming//tests/`. CI checks these points. + - The maintainer will say in the PR what needs to change, and you update it yourself. Once it is reviewed, it is merged, and merged means accepted: the maintainer takes it into the next release unchanged with `git am`, and your credit is kept. At release time the patch is merged into a new series in `patches/`, and this copy in `incoming/` is deleted. +5. You do not need to know how to build or run regression tests. After the review, the maintainer builds and tests on their own computer. + +Replaying patches alone needs no build environment. To build an emulator that runs, follow [Building from source](./build). + +## Verification + +- After building, start an instance on a machine with WHPX and a Vulkan driver, and run real games; +- Graphics-related changes need to be checked on different GPUs. The maintainer has only one configuration, AMD CPU with NVIDIA GPU, so this part especially lacks people; +- For changes to the disk I/O path or the shutdown flow (block devices, disk writes, anything related to sync, the data disk format, guest writeback parameters), the maintainer will run the hard-kill regression. + +## Upstream + +General fixes may later be sent to google/gfxstream or AOSP Gerrit. Upstream requires the author to sign the Google CLA personally; you will be asked first. + +## Suggested models + +Opus 5.5 or GPT-6 Astra, with high reasoning effort (choose High or Extra High for GPT). Root causes of crashes, WHPX, and gfxstream synchronization problems all take long trial and error in a very large codebase, and a wrong judgment is costly. A human reviews the change in the end. diff --git a/docs/en/develop/ime.md b/docs/en/develop/ime.md new file mode 100644 index 0000000..67306ec --- /dev/null +++ b/docs/en/develop/ime.md @@ -0,0 +1,45 @@ +# The host IME + +**What you change**: the [ModAVDHostIME](https://github.com/ModAVD/ModAVDHostIME) repository. It holds the headless input method in the guest, `org.modavd.ime`, which receives text typed with the Windows IME through the emulator's `modavd_ime` channel. It also holds the test APK and the channel protocol description `docs/protocol-v1.md`. + +For the directory layout, the key points of the protocol and how to use the test APK, see the repository's [README](https://github.com/ModAVD/ModAVDHostIME/blob/main/README.en.md). + +## Environment + +- JDK: Temurin 25.0.4.1+1 (`javac --release 17`); +- Android SDK: build-tools 36.1.0 and platforms android-36 (revision 2), using aapt, aapt2, d8, zipalign and apksigner; point `ANDROID_HOME` at it; +- No Gradle. `build.ps1` runs only on Windows; the protocol self-test needs only javac and works on any system; +- To build an unsigned APK that is byte-identical to the released one, set the time zone to Central European Time (`TZ=CET-1`) when building. + +## Hardware + +- Building only: the JDK and the two Android SDK components take about 1 GB together; 4 cores and 8 GB of RAM are enough; +- Verification needs one instance with no games installed: + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 4 cores, 8 threads | +| RAM | 16 GB | 16 GB (8 GB for only building) | +| Free disk space | 30 GB | 20 GB | + +## Signing + +- The official build is signed only by the maintainer on the maintainer's machine, and the official key never goes into CI; +- A build you make yourself uses an automatically generated development key (`out\keys\dev.p12`), and keys never go into git; +- When installed on a ModAVD instance, it is replaced by the official build unless your `versionCode` is higher; when the signatures differ, the manager uninstalls first and then installs. + +## Verification + +You need an emulator from 0.35 or later that has the `modavd_ime` channel. Install the test APK, drive input with the `ime` command of the emulator console, and check the result. + +## Protocol + +Changing the protocol means changing both the host and the guest side: the host side is in ModAVD's emulator patches. You may only add new types and not change the format of existing ones; please discuss it in an issue first. + +## Versioning + +Numbered separately since v1.0.0. The tag is `v1.x.y`, and the APK's `versionName` is the same as the numeric part of the tag. When the APK's content changes, `versionCode` goes up by one. + +## Suggested models + +Sonnet 5.5 or GPT-6.1 Sol. The code is small and the rules are clear. diff --git a/docs/en/develop/index.md b/docs/en/develop/index.md new file mode 100644 index 0000000..1cb6c49 --- /dev/null +++ b/docs/en/develop/index.md @@ -0,0 +1,54 @@ +# Contributing to ModAVD + +ModAVD has only one maintainer, so replies can take a few days. For bigger changes, please open an issue to discuss them first. A big PR that was not discussed may not be merged. + +To see what environment and hardware each area needs, and which models to pick when you use AI tools, see [Areas and common requirements](./overview). + +## What is needed most: try it on your computer + +The maintainer has tested on only one computer: Windows 11, AMD CPU, NVIDIA discrete GPU. What is missing most is testing on these configurations: Intel CPU, AMD or Intel GPU, laptops with dual GPUs, Windows 10, and 16 GB of RAM. + +1. Install by following [Installation](../guide/install), add the instance in AUTO-MAS, start it, and run one round of daily tasks. +2. Double-click `复制配置信息.bat` (copies your system and game info) in the folder (it is in the package from 0.35.0). Paste the output into the pinned "兼容性报告" (compatibility report) issue in Issues, fill in the last three lines, and add one sentence on whether it runs. + +This output does not contain your user name, computer name, full paths or game accounts, so you can post it publicly as it is. For details, see [Testing on real games and compatibility reports](./testing). + +## Reporting problems + +- Open an issue and attach the problem report made by `收集问题信息.bat` (collects a problem report). For what you can post publicly and how to send crash dumps privately, see [Reporting problems](../guide/feedback). +- Before you paste logs, replace `C:\Users\` in paths with `%USERPROFILE%`. +- Problems with AUTO-MAS's interface, tasks or scheduling belong to [AUTO-MAS](https://github.com/AUTO-MAS-Project/AUTO-MAS/issues). +- Do not report security problems publicly. Send an e-mail to qiyinxi@foxmail.com. + +## Changing documentation and code + +- Say so in the issue before you start. If an issue is already assigned to someone, that person is working on it. +- Open PRs from your own fork. CI for external PRs only runs after the maintainer approves it. +- If something on this site or in the READMEs of the repositories is wrong or unclear, send a PR directly. See [Documentation and this site](./docs). +- Bundled scripts, `package/`: you can send PRs, and once merged they ship with the next release. For the rules to follow, see [Bundled scripts](./package). +- The arm64 execution bridge, `bridge/`: open an issue to discuss changes first. It directly affects whether games can start, so every release needs real game regression runs on an instance with the bridge installed. +- The manager and CLI are in [ModAVDManager](https://github.com/ModAVD/ModAVDManager), the host IME is in [ModAVDHostIME](https://github.com/ModAVD/ModAVDHostIME), and the pre-release regression tests (end-to-end tests) are in [ModAVDE2E](https://github.com/ModAVD/ModAVDE2E). Take code issues and PRs there. The rules for credit and licensing are the same as on this page. +- `patches/`, `manifests/` and `release/` in the ModAVD repository are exported by the release tool (`tools/pubrel/`). PRs do not change them directly. If you find a problem, open an issue first. +- The release and build tools `tools/` and the CI checks `scripts/` in the ModAVD repository (the release tool uses this copy too): open an issue to discuss changes first. +- New or changed workflows: pin third-party actions to a full 40-character commit SHA, do not use `pull_request_target`, and do not use self-hosted runners. `mirrorchyan_*.yml` is maintained by MirrorChyan itself, so keep it as it is. +- Build CI is not accepted. Source plus build output is about 124 GB, a full build on a 16-core machine takes about 1.5 hours, and GitHub's free machines cannot do that. A self-hosted runner would let external PRs run arbitrary commands on that machine. To help verify the build, build on your own computer by following [Building from source](./build) and post the result in an issue. + +To submit emulator patches, see [Emulator](./emulator). + +## Credit and licensing + +- Sign with your own name or a pseudonym, and use a GitHub noreply address as your e-mail (`+@users.noreply.github.com`, which you can see in GitHub's e-mail settings). Do not write `Co-authored-by`. Your credit is kept after your change is accepted, and the release notes will thank you. +- Your changes come in under the license of the files they change; see [Licenses and source code](../guide/license). The copyright line of a new file carries your name or pseudonym. You do not need to sign a CLA, and `Signed-off-by` is optional. +- You may use AI tools, but the AI must test the change itself, and you are responsible for it. The rules for AI tools are in [AGENTS.md](https://github.com/ModAVD/ModAVD/blob/main/AGENTS.md) at the root of the ModAVD repository; Codex, Claude Code and others read it on their own. See [Using AI tools](./ai). +- General fixes may later be sent to the Google upstream (AOSP, google/gfxstream). Upstream requires the author to sign the Google CLA personally. You will be asked first, and if you do not want to, the change stays in ModAVD only. +- For image and icon changes, open an issue first with the source and the license. Fan art and images with an unclear source are not accepted. + +## When you test yourself + +- Shut down instances in AUTO-MAS or the manager when you can. In your own script, run `adb -s shell sync` first, then `adb -s emu kill`. Without sync, files the game just wrote can become 0 bytes. +- For force-kill tests, use a newly created instance and a test account, not an instance logged in to your main account. +- Do not use adb on the default port 5037. It fights with the adb that other emulators bring along over the same server. Take the port from `scriptAdbServerPort` in `mas-avd.json` (default 20049), the same one AUTO-MAS gives to scripts. + +## Other + +If you want to help sort issues (reproduce, merge duplicates, add labels), ask the maintainer for Triage permission. diff --git a/docs/en/develop/manager.md b/docs/en/develop/manager.md new file mode 100644 index 0000000..0155721 --- /dev/null +++ b/docs/en/develop/manager.md @@ -0,0 +1,57 @@ +# The manager and CLI + +**What you change**: the [ModAVDManager](https://github.com/ModAVD/ModAVDManager) repository. + +- The manager `ModAVD.exe`: the interface for people, with instance cards, start and stop, create, settings, screen view and logs; +- The CLI `modavd.exe` and its MCP server: for scripts, AUTO-MAS and AI assistants. + +The code structure (which part of AUTO-MAS's code each package under `internal/` corresponds to), the full CLI documentation, the machine-readable output and the exit codes are in the repository's [README](https://github.com/ModAVD/ModAVDManager/blob/main/README.en.md). For an introduction for users, see [Components: the manager and CLI](../components/manager). + +## Environment + +- Go **1.26.5** (the toolchain in `go.mod`), with `GOTOOLCHAIN=local`, standard library only; +- PowerShell 7; +- The pages are in `internal/ui/web/`, in plain JS / CSS with no front-end build step. To change styles, start a fake backend with `ModAVD.exe --dev`; you do not need to start the emulator. + +## Hardware + +- Only building, running unit tests, and changing the interface with `--dev`: 4 cores, 8 GB of RAM and 2 GB of disk are enough; +- Testing on a real instance means starting one instance with no games installed: + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 4 cores, 8 threads | +| RAM | 16 GB | 16 GB (8 GB for only building and unit tests) | +| Free disk space | 30 GB | 20 GB | + +## Build + +```powershell +pwsh -NoProfile -File build.ps1 # -> out\ModAVD.exe, out\tools\modavd.exe +pwsh -NoProfile -File build.ps1 -Test # run go vet and go test first +``` + +It uses `-trimpath -buildvcs=false` and does not write the version number with `-X`. The same source gives byte-identical results on any machine and in any directory, so you can check against the published hash. + +## Verification + +- `gofmt -l .` prints nothing, and `go vet ./...` and `go test ./...` pass; +- Some tests run AUTO-MAS's original logic with Python 3 and compare it with Go's output byte by byte, so Python 3 must be installed; +- Testing on a real instance needs a ModAVD root folder: extract the release package, then fetch the Google components; +- Use newly created instances and a private adb port for these tests; +- Do not turn on the live-test switch (`MODAVD_TEST_LIVE=1`) against a root folder that holds important instances. + +CI runs the format check, `go vet`, `go test`, `build.ps1`, the privacy scan and the author check on PRs. + +## Contract + +The envelope of the `--json` output, the error codes and the `protocol` version are what AUTO-MAS depends on. Adding fields is fine. To change a meaning or remove a field, add one to `Protocol` in `internal/buildinfo` and open an issue first. + +## Versioning + +The manager has been numbered and released separately since v0.1.0 (the two exe files of v0.1.0 are byte-identical to those in the ModAVD 0.36.0 package). When the ModAVD package is released, it pins the manager version and hash at that time, and the copy inside the package changes together with the package. The manager does not update itself: from 0.2.0, `modavd update` updates the whole ModAVD root folder (from GitHub or MirrorChyan, going online only when you run it by hand), and a manager downloaded on its own has to be replaced by hand. + +## Suggested models + +- Interface and ordinary features: Sonnet 5.5 or GPT-6.1 Sol; +- Boot flow, concurrency and locks, process management and package updates, which are things that easily hide subtle problems and involve security: Opus 5.5 or GPT-6 Astra to write or review. diff --git a/docs/en/develop/overview.md b/docs/en/develop/overview.md new file mode 100644 index 0000000..987ce5d --- /dev/null +++ b/docs/en/develop/overview.md @@ -0,0 +1,56 @@ +# Areas and common requirements + +ModAVD is the AOSP Android Emulator plus a set of patches, with WHPX acceleration and gfxstream rendering, and a few components written by the project. The areas below are sorted by development area. Each area's page says what hardware and environment it needs, how to verify, and which model to suggest when you use AI tools (both Claude and GPT are listed). + +For how to take part (credit, licensing, how to submit), [Contributing to ModAVD](./) is authoritative. + +## Repositories and areas + +| Area | Repository / directory | Main language | Difficulty | What GitHub CI can verify | Suggested models (Claude / GPT) | +|---|---|---|---|---|---| +| [Emulator](./emulator) (qemu / gfxstream / crashpad patches) | `patches/` and `manifests/` of [ModAVD](https://github.com/ModAVD/ModAVD); the patched source is in [ModAVDQemu](https://github.com/ModAVD/ModAVDQemu), [ModAVDGfxstream](https://github.com/ModAVD/ModAVDGfxstream) and [ModAVDCrashpad](https://github.com/ModAVD/ModAVDCrashpad) (`release/0.36`) | C / C++ | High | Only replays the patches and compares the source tree; does not build | Opus 5.5 / GPT-6 Astra, with high reasoning effort | +| [arm64 execution bridge](./bridge) | `bridge/` of ModAVD | C++ (NDK) | High | Nothing | Opus 5.5 / GPT-6 Astra | +| [The manager and CLI](./manager) | [ModAVDManager](https://github.com/ModAVD/ModAVDManager) | Go; the pages are plain JS / CSS | Medium | Unit tests, build, reproducible-build check | Sonnet 5.5 / GPT-6.1 Sol; Opus 5.5 / GPT-6 Astra for the boot flow, concurrency and package updates | +| [The host IME](./ime) | [ModAVDHostIME](https://github.com/ModAVD/ModAVDHostIME) | Java (Android) | Medium to low | Protocol self-test, APK build, release check | Sonnet 5.5 / GPT-6.1 Sol | +| [Bundled scripts](./package) | `package/` of ModAVD | PowerShell, batch | Medium to low | Privacy scan | Sonnet 5.5 / GPT-6.1 Sol | +| Pre-release regression tests (end-to-end tests) | [ModAVDE2E](https://github.com/ModAVD/ModAVDE2E) | PowerShell, Python | Medium | Syntax parsing, privacy scan; the regression run itself is not run | Sonnet 5.5 / GPT-6.1 Sol | +| [Documentation and this site](./docs) | This site's repository [modavd.github.io](https://github.com/ModAVD/modavd.github.io), and the READMEs of the repositories | Markdown | Low | Builds the site, checks dead links, privacy scan | Sonnet 5.5 / GPT-6.1 Terra; Haiku 5.5 / GPT-6 Luna for mechanical changes | +| [AUTO-MAS integration](./automas) | [AUTO-MAS](https://github.com/AUTO-MAS-Project/AUTO-MAS) | Python, front end | Medium | AUTO-MAS's own CI | Sonnet 5.5 / GPT-6.1 Sol; Opus 5.5 / GPT-6 Astra for review | +| [Testing on real games, compatibility reports](./testing) | The pinned "兼容性报告" (compatibility report) issue | — | Low to medium | Nothing | Sonnet 5.5 / GPT-6.1 Terra for reading logs; Haiku 5.5 / GPT-6 Luna for tidying up | +| [Making a new game work](./adapt-game) (won't start, runs poorly) | Once the cause is found, it ends up in the bridge, the emulator itself or the automation scripts | Depends on the cause | Medium to high | Nothing | Sonnet 5.5 / GPT-6.1 Sol for sorting logs; Opus 5.5 / GPT-6 Astra for root causes in the translation layer and graphics | + +## Requirements for all areas + +- **System**: Windows 11 x64 (Windows 10 is tested less, and reports are welcome), virtualization turned on in the BIOS, and Windows Hypervisor Platform (WHPX) turned on. +- **GPU**: Install a GPU driver with Vulkan (a discrete GPU or a fairly recent integrated GPU). Without a usable GPU, only software rendering is possible, and games will use up the CPU. +- **CPU, RAM, disk**: These differ a lot by area. What matters is whether you build the emulator itself, run instances and install games. They are written under "Hardware" on each area's page. The minimum configuration runs but is slow; the recommended one is what the maintainer finds comfortable. +- **Common tools**: + - PowerShell 7 (development scripts); the bundled scripts must also work on Windows PowerShell 5.1; + - Python 3.12; + - Git for Windows. +- **Shutting down the emulator**: Run `adb -s shell sync` first, then `adb -s emu kill`, or use the shutdown of the manager or the CLI. Without sync, files the guest just wrote can become 0 bytes. +- **adb port**: Do not use the adb server on the default port 5037. It fights with the adb that other emulators bring along. Use the private port in `mas-avd.json` in the ModAVD root folder. +- **Test data**: Use newly created instances and test accounts. Do not run tests such as force-kill on an instance logged in to your main account. +- **Submitting**: + - Open PRs from your own fork; + - sign with your own name or a pseudonym; + - CI for external PRs only runs after the maintainer approves it. +- **Using AI tools**: You may. The AI must test the change itself: build, run the unit tests, and if it can start an instance, start one and look at the result. State in the PR what was tested and with what result. Do not give accounts, keys or private paths to AI tools. The rules for AI tools are in [AGENTS.md](https://github.com/ModAVD/ModAVD/blob/main/AGENTS.md) at the root of the ModAVD repository; Codex, Claude Code and others read it on their own. See [Using AI tools](./ai). + +### What it takes to run an instance + +Verification in several areas needs a running instance. The numbers come from ModAVD's documentation and from measurements on the maintainer's machine: + +- **CPU**: x86_64, with virtualization turned on in the BIOS (Intel VT-x / AMD SVM). Android can use at most half of the computer's logical processors as virtual CPUs, so an instance with 4 cores needs 8 threads, and 6 cores need 12 threads. +- **RAM**: Instance memory can be 3 / 4 / 5 / 6 GB (AUTO-MAS defaults to 6 GB). At boot the computer needs about another 1.5 GB free, and the emulator itself takes about 0.6–0.75 GB more. +- **Disk**: On an NTFS drive. The components and the emulator itself take about 5 GB together (6.5 GB must be free when fetching the Google components). The drive of each instance needs 10 GB or more free on its first boot; a new instance with no games installed actually takes only about 1.1 GB. An instance with games installed takes tens of GB; see [Testing on real games](./testing). + +## What is in progress + +Look here before you start, to avoid collisions: + +- AUTO-MAS plans to switch to the `modavd` CLI for managing instances and updating the package (the changes go to the AUTO-MAS repository); +- ModAVDManager: v0.1.0 is released; v0.2.0 (finishes initialization in the background when creating an instance, `modavd update`) is not released yet; 0.3.0 (components and problem reporting, platform layer) is being worked on in main; +- ModAVDHostIME: v1.0.0 is released. + +If what you want to do overlaps with these, say so first in an issue in the matching repository. diff --git a/docs/en/develop/package.md b/docs/en/develop/package.md new file mode 100644 index 0000000..2e4c78c --- /dev/null +++ b/docs/en/develop/package.md @@ -0,0 +1,42 @@ +# Bundled scripts + +**What you change**: the scripts that ship in the ModAVD package, in [`package/`](https://github.com/ModAVD/ModAVD/tree/main/package) of the ModAVD repository. The three `.bat` files go in the root of the package and the four `.ps1` files in the package's `tools\` folder. They are byte-identical to the files shipped in the package. + +| File | What it does | In the package since | +| --- | --- | --- | +| `补全组件.bat` (fetches the Google components) → `fetch-components.ps1` | First shows Google's Android Software Development Kit License Agreement. After you agree, it downloads adb and the system image from the official channels, checks the SHA-1 and installs them into `sdk\`. It can resume downloads and falls back to the next mirror in order | 37.2.10-mas30 | +| `收集问题信息.bat` (collects a problem report) → `collect-info.ps1` | Makes a problem report on the desktop; see [Reporting problems](../guide/feedback) | 37.2.10-mas30 | +| `复制配置信息.bat` (copies your system and game info) → `report-config.ps1` | Puts the computer's configuration and the game packages installed in each instance into one block of text and copies it to the clipboard, to paste into the compatibility report issue | 0.35.0 | +| `modavd-bridge.ps1` | Installs, removes or checks the arm64 execution bridge on an instance; see [Development: arm64 execution bridge](./bridge#installing-into-an-instance) | 0.36.0 | + +## Writing rules + +- They must also run on the Windows PowerShell 5.1 that ships with Windows, because it is the default on users' machines: + - save `.ps1` files as UTF-8 with BOM; + - use only ASCII in `.bat` files, with CRLF line endings; + - `.gitattributes` makes this directory store files as they are, with no line-ending conversion. When you send a PR, keep the file's original line endings and BOM: the bytes in the repository are the bytes in the release package. +- No hard-coded local paths. adb goes only through the private port in `mas-avd.json` in the ModAVD root folder and never touches 5037. +- Before restarting or shutting down, run `adb shell sync` first, then end the emulator. +- Output that users paste into public issues must not contain user names, computer names, full paths or game accounts. + +## Hardware + +To run them for real on an installed ModAVD, you need to start an instance: + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 4 cores, 8 threads | +| RAM | 16 GB | 16 GB | +| Free disk space | 30 GB | 20 GB | + +## Verification + +Run each script with PowerShell 5.1 on Windows 10 and 11, and under different system languages. + +## Delivery + +Open an issue to say so first, then send a PR. Once merged, it ships with the next Release, and the release notes will say so. The license is AGPL-3.0-or-later. + +## Suggested models + +Sonnet 5.5 or GPT-6.1 Sol; for mechanical checks such as line endings and encoding, Haiku 5.5 or GPT-6 Luna. diff --git a/docs/en/develop/performance.md b/docs/en/develop/performance.md new file mode 100644 index 0000000..b160c1c --- /dev/null +++ b/docs/en/develop/performance.md @@ -0,0 +1,44 @@ +# Investigating stutter and frame drops + +This is how the problems in *Onmyoji*, *Honkai: Star Rail* and *Arknights: Endfield* were found. Developers' machines are usually well equipped, so what is worth investigating is what adding cores or memory does not fix. + +## 1. First confirm what you are measuring + +- Confirm that you really reached the screen you want to measure. Measure menus, loading screens, battles and the open world separately. +- For battles, cut out each fight using the automation script's log and compute from that. With a 30-second window, menus and loading screens are mixed in, and what you compute is not the battle frame rate. +- Note how many other emulators are open on the same machine. With too many open, all cores drop their clock speed and the frame rate falls across the board, so read the conclusion together with this number. +- When comparing, use the same scene, the same graphics quality and the same instance configuration. + +## 2. Measure the frame rate + +- In the guest logcat, `EGL_emulation` prints a line `app_time_stats: avg=… min=… max=… count=…` every second. +- Take only the lines of the game's main process: the one whose parent is `zygote64` and whose name is the package name. +- Add up `count` and divide by the number of seconds to get the frame rate. `max` is the slowest frame in that second. + +## 3. See whether the bottleneck is in the guest or the host + +**Guest**: `shell top -H -p ` shows the CPU of each thread. Look mainly at the main thread (called `UnityMain` in Unity games) and the render thread. + +- One thread uses up a whole core while the other cores are mostly idle: a single-thread bottleneck. Adding cores or memory does not help. +- The main thread is very busy, but the frame rate does not change with the CPU frequency: it is most likely waiting (for locks, memory access, replies from the host), not computing. + +**Host**: Use Task Manager or Process Explorer to see how much the virtual CPU threads and the render thread of the qemu process each take, then look at the GPU 3D usage. If the render thread is full, the bottleneck is on the graphics side. + +## 4. Dig deeper + +- What the main thread is doing and waiting for: in the guest, record for 20 to 30 seconds with `simpleperf record -g -t ` and look at the hot spots and call chains (for example, many `futex` waits). +- Stuck or frozen picture: `shell su 0 debuggerd -b ` gets the stacks of all threads, so you can see who is waiting for whom. +- Graphics round trips: if the render thread spends most of its time on synchronous round trips between the guest and the host, see whether they can be batched or the synchronous calls replaced. +- Slowest at loading and right after entering a battle: look at the guest's memory swap-out and disk reads and writes. The main thread is most likely waiting then, not computing. + +## 5. Write down the conclusion + +Write down which scene, how many frames, which thread was full, which switch you changed, and the before and after numbers. Run the comparison in alternating rounds, several times, and check whether the difference between groups is larger than the fluctuation within a group. One run each does not count. + +## Examples found earlier + +| Game | Symptom | Cause found | Result | +|---|---|---|---| +| *Onmyoji* | Courtyard 44–48 fps; menus about 44 fps, and the loading screen when entering or leaving a battle stalls for about 1 second | Courtyard: GL transport goes through a pipe. Menus and loading screens: the game's main thread is fully loaded in user mode, running translated arm64 code; the rest of the guest's CPU is 70% idle, and the host render thread is only at 20–30% | After switching GL transport to asg, the courtyard runs at 60 fps. Menus and loading screens are translation overhead, and adding cores does not help | +| *Honkai: Star Rail* | Battles 40–48 fps; the first 30 seconds of the first battle only 7–16 fps | Battles: the main thread and the render thread each use up one core and are limited by single-core speed; going to 8 cores or adding memory does not raise it. First battle: the main thread is only at 31%, waiting for loading and memory swap-out | The battle frame rate can only be raised through translation efficiency, single-core frequency or lower graphics quality; for the first-battle stretch, swap-out has to be reduced | +| *Arknights: Endfield* | While running tasks 12.8 fps; title screen 14.6 fps | Running tasks: the render thread spends 79% of its time on synchronous round trips between the guest and the host, with about 159 descriptor allocations per frame. Title screen: host-visible video memory (ReBAR) made uploads take the slow path | After turning on Vulkan batched descriptor updates, 30.5 fps; after setting `ANDROID_EMU_VK_HIDE_REBAR=1`, the title screen runs at 60.7 fps | diff --git a/docs/en/develop/testing.md b/docs/en/develop/testing.md new file mode 100644 index 0000000..131a82e --- /dev/null +++ b/docs/en/develop/testing.md @@ -0,0 +1,54 @@ +# Game testing and compatibility reports + +## The easiest way to help + +Install on your own computer by following [Installation](../guide/install), run one round of daily tasks in AUTO-MAS, and then: + +1. Double-click `复制配置信息.bat` (copies your system and game info); +2. Paste the output into the pinned "兼容性报告" (compatibility report) issue in the Issues of the ModAVD repository, and fill in the last three lines; +3. Add one sentence on whether it runs. + +These configurations are missing most: Intel CPU, AMD or Intel GPU, laptops with dual GPUs, Windows 10, and 16 GB of RAM. For compatibility reports, low-end machines are in fact the most wanted. As long as you meet the [requirements](../guide/#requirements), you are welcome to test. + +## Hardware + +When you run game tests yourself: + +| | Recommended | Minimum | +|---|---|---| +| CPU | 8 cores, 16 threads | 4 cores, 8 threads | +| RAM | 32 GB | 16 GB | +| Free disk space | 30–80 GB per game | Same | + +What each game's instance looks like in practice on the maintainer's machine: + +| Game | RAM of the test instance | Disk used by the instance | +|---|---|---| +| *Onmyoji* | 6 GB | 26 GB | +| *Arknights* | 4 GB | 38 GB | +| *Arknights: Endfield* | 8 GB | 56 GB | +| *Reverse: 1999* | 6 GB | 67 GB | +| *Honkai: Star Rail* | 6 GB | 77 GB | + +The frame rate is strongly affected by other emulators open at the same time. + +## Rules + +- Do not pull gacha, buy anything or use stamina items; +- Operate in the game with scripts only; +- Stop when the game shows an update prompt; +- Use a test account, and run sync before shutting down the emulator. + +## The maintainer's pre-release regression tests + +- On the maintainer's machine, real tasks are run with each game's automation scripts (MAA, M9A, SRC, MaaEnd, MaaYYs), not by leaving the game unattended on the title screen; +- Games always run on instances with the bridge installed and routed by package name, the same as users' instances; +- The items are picked by what changed in this version: a change to the emulator itself runs the full set, and a change only to the manager, the IME or the bundled scripts runs just the few relevant items; +- The frame rate is compared with the baseline of the previous release, with a tolerance for each game. When more than 2 other emulators run on the same machine, it is only recorded and not judged; +- It needs a logged-in test account, 32 GB or more of RAM, and few other emulators open on the same machine. + +A game that has an automation script and tasks that do not consume resources can be added to the regression tests once it has been made to work; see [Making a new game work](./adapt-game#after). + +## Suggested models + +Sonnet 5.5 or GPT-6.1 Terra to run tests and read logs; Haiku 5.5 or GPT-6 Luna to tidy up logs and sum up numbers. diff --git a/docs/en/develop/troubleshooting.md b/docs/en/develop/troubleshooting.md new file mode 100644 index 0000000..31c6a0b --- /dev/null +++ b/docs/en/develop/troubleshooting.md @@ -0,0 +1,129 @@ +# Investigating common problems + +Below are problems that were actually investigated before. Each entry gives the symptom, how it was investigated, what was found, and the result. For a new problem, you can usually find a similar way to investigate here. For stutter and frame drops, see also [Investigating stutter and frame drops](./performance). + +## Commonly used ways to collect evidence + +- **What a thread is doing**: + - `top -H -p ` shows the CPU of each thread; + - `/proc//task//stat` and `wchan` show a thread's state and which kernel function it is stopped in; + - `TracerPid` in `/proc//status` shows whether someone else is ptrace-ing it; + - `/proc//syscall` shows which system call it is stuck in and with what arguments. +- **Thread stacks**: `shell su 0 debuggerd -b `, which does not kill the process. +- **Sampling and tracing**: + - `simpleperf record -g -t `, which can also follow a given system call; + - kernel tracefs (`/sys/kernel/tracing`) to follow signals; + - uprobe / uretprobe to see the return value of a library function. +- **Network**: + - the emulator console's `emu network capture start ` / `stop` captures all of the guest's traffic; + - `ss -tni` shows the receive backlog (Recv-Q) of each connection. +- **Disk**: `/proc/diskstats`; `wchar` and `syscw` in `/proc//io`; the read and write counts of the qemu process on the host. +- **Host**: Process Explorer shows the CPU of each qemu thread. When the emulator crashes, take the crash dump and read the stack with the symbols of the matching version. +- **For games with protection, prefer passively reading `/proc`**, and do not casually ptrace the game process. + +## Hangs and freezes + +### Works sometimes at startup; when it fails, ANR after about 20 seconds (*Onmyoji*) + +- **How it was investigated**: At the ANR, the whole machine's CPU was only 4.6%, so the main thread was not busy but halted. Passively reading `/proc` showed that the main thread's state was `t` (stopped by tracing), `wchan` was `ptrace_stop`, and `TracerPid` was not 0. Following it, what halted the thread was a child process of the game: WebView's crash handler, whose command line has `--trace-parent-with-exception`, which means it had decided the parent process crashed. +- **What was found**: The bridge is entered through a deliberate segmentation fault. WebView's crash handling is native code, and the signal handler it installs got to the very front. It took the bridge's segmentation fault for a crash, so it ptraced the parent process and hung there. Whether you hit this depends entirely on timing, so it works sometimes and fails sometimes. +- **Result**: Switched to the "first right to handle" that the native bridge interface already has (`getSignalHandler`). It comes before all `sigaction` handlers, so no one can get in front of it, however late they install theirs. + +### Stuck at "正在获取音频列表..." ("Fetching the audio list...") and not moving (*Onmyoji*, about one cold boot in five) + +- **How it was investigated**: All 139 threads of the game were asleep, and a 60-second packet capture had only one packet, so it was not the network. The main logic thread was reading a pipe, waiting for the child process it had forked itself. The child process was stopped on a futex whose lock address fell inside the translation layer's internal data. The same lock was free in the parent process and locked in the child's copy. +- **What was found**: The translation layer is not fork-safe. At the moment of the fork, this lock was held by another thread of the parent process, and in the child process nobody would ever release it. This is the classic multi-threaded fork deadlock. +- **Result**: In the companion library, `pthread_atfork` carries these locks across the fork. After the fix, there were 0 hangs in 15 consecutive cold boots. + +### The picture freezes after about twenty minutes (*Honkai: Star Rail*) + +- **How it was investigated**: `debuggerd -b` showed the render thread waiting in `dequeueBuffer` for a free buffer the whole time, and logcat had `Too many open files`. The game process's fd count grew linearly with the number of frames, one per frame, and ran out at about 32,000 frames, which matched the time of the freeze. +- **What was found**: The guest Vulkan driver bundled with the image leaks one fd per frame. After the fds ran out, the callback that sends the buffer back failed to unpack on the app side, and the buffer was never returned. +- **Result**: Worked around by setting the property `debug.angle.feature_overrides_disabled=supportsSwapchainMaintenance1` after boot. The emulator does not need to change. + +### Occasional deadlock while loading (*Arknights: Endfield*; introduced by the first version of the patch for the crash below) + +- **How it was investigated**: `debuggerd -b` showed the render thread waiting for a lock, and several other threads waiting for replies from the host, one of which was holding that lock. +- **What was found**: With `VulkanQueueSubmitWithCommands` on, the host executes commands one by one in global sequence-number order. The render thread took a sequence number first and then waited for the lock, while the thread holding the lock was waiting for a reply with a later sequence number. The host had to execute the render thread's command first, so they waited for each other. +- **Result**: The rule is: **do not hold a lock across a round trip to the host**. The second version of the patch only looks up the table under the lock, and sends to the host after leaving the lock. + +## Crashes + +### Crashes within 1 second of entering the game, every time (*Onmyoji*; the bridge was built for this) + +- **What was found**: The game's protection library changes some GOT slots in the native x86 code to jump into translated arm64 code. On a real device these are all arm64 and fine. The translation layer supports only translated code calling native code and not the reverse, so it crashes every time. +- **Result**: The bridge uses the interface the translation layer itself exports to add the missing "native calls translated" path. + - It hooks the translation layer's own import table (the entry that actually takes effect is its `mprotect`), because loading the libraries of an arm64 app goes through the translation layer's own loader and not through `libnativeloader`; + - It never modifies the game or its protection library. + +### Random crashes; the tombstone shows SIGSEGV with pc 0 and a different thread name each time (*Arknights: Endfield*) + +- **How it was investigated**: This is a secondary crash scene. Tracing signals with tracefs gave this chain: the game's own assertion → `SIGTRAP` → crash-reporting SDK → translation layer → the reporting SDK takes a page fault on a host thread → turned into SIGSEGV. It also confirmed that the anti-cheat sent no signal. The assertion was `ret == VK_SUCCESS`, and a uretprobe then caught `vkBindImageMemory` returning -2. +- **What was found**: In the guest Vulkan driver bundled with the image, `BindImageMemory` looks up the handle table without taking a lock. When another thread happens to grow the table, the lookup misses and returns -2. This is a data race, and the upstream code is the same section. +- **Result**: Patched the guest driver to take the lock before looking up the table. There were 0 assertions in 10 cold boots, and no new tombstone in a 3-hour long run. + +### The emulator itself crashes (during the 0.36 release regression tests) + +- **How it was investigated**: In the upgrade test, qemu crashed when the instance restarted and adb reconnected. Took the crash dump, read the stack with the symbols of that version, then disassembled, and located an instruction that reads data from near a null pointer. +- **What was found**: When unregistering the network listener, it did not first check whether the listener node existed. +- **Result**: Check for null before unregistering. After the fix, the whole regression suite was rerun. + +## Display + +### Flicker and occasional all-black screenshots (NVIDIA GPUs, 2%–5% of frames) + +- **How it was investigated**: Put probes on four paths (window, shared memory, adb screenshot, screen recording) and counted all-black frames frame by frame. All the paths turned out to be affected, so the problem is in the host's read-back step. +- **What was found**: When the NVIDIA driver requires a "dedicated allocation", gfxstream did not give one, and that part of the frame buffer read back as all 0. +- **Result**: After the fix, the number of all-black frames is 0 on every path. Submitted upstream. + +## Network, downloads and data + +### "网络请求超时" ("network request timed out") is shown, but the network is fine (*Honkai: Star Rail*) + +- **How it was investigated**: Looked for 0-byte files in the game data directory, and captured packets with the emulator console to confirm that the hot-update requests and downloads were normal. +- **What was found**: An earlier shutdown without sync had turned the hot-update manifest that had just been written into 0 bytes. The game threw an exception when parsing the empty file, but the interface showed it as a network error. +- **Result**: With the game stopped, write the content of `_cache` in the same directory back into the real file. From now on, always sync before shutting down. + +### Slow downloads, under 8 MB/s (*Arknights*) + +- **How it was investigated**: + - Compared in three layers: the host straight to the CDN at about 100 MB/s, a single connection on the emulator's network card at 80-odd MB/s, and the game itself at only 5.9 MB/s. + - `ss` showed a receive backlog of several MB on each connection, which means the data had already reached the guest and the game was reading slowly. + - `/proc//io` and `/proc/diskstats`: the amount written to disk was about 8 times the download size, all in small blocks. The download thread was stopped in `jbd2_log_wait_commit`, that is, it was doing fsync while downloading. + - No single qemu thread was maxed out. +- **What was found**: The bottleneck is the round-trip cost of each write request on the data disk. Each guest write turns into about 4 writes on the host. This way of writing is fine on a real device's flash, but on the emulator it drops to only 6 MB/s. +- **Result**: Not fully fixed yet. The direction is to lower the round-trip cost of each write. This kind of change touches the disk I/O path, so the hard-kill regression has to be run first. + +### Files become 0 bytes or revert to an old version after a hard kill + +- **How it was investigated**: Several ways of writing files (delete first then rename, rename over the old file, overwrite directly) were written under different writeback parameters and killed right after writing, to count how soon after a write a kill loses data. +- **Result**: This is where the "sync before shutdown" rule comes from. At boot the emulator also shortens the guest's writeback period, which narrows the window for data loss but cannot eliminate it. + +## System level + +### The guest freezes after a warm reboot (WHPX) + +- **How it was investigated**: The guest kernel log had a soft lockup: one virtual CPU did not respond to inter-processor interrupts, and the other cores were stuck waiting for it. +- **Result**: Everywhere the guest needs to restart now does a cold boot (shut down, then start), not `adb reboot`. + +### Memory use is higher than other emulators + +- **How it was investigated**: Measured in three parts, then compared with other emulators under the same conditions: how much the guest actually uses, how big the guest memory block on the host is, and how much the emulator itself takes on top (about 0.65–0.75 GB). +- **Result**: + - Turned on free page reporting for the memory balloon, so memory freed in the guest goes back to Windows; + - Made the graphics staging buffer grow on demand, so each instance takes about 140 MB less when idle. + +### Slow cold boot; adding cores or raising the clock speed does not help (*Arknights: Endfield*) + +- **How it was investigated**: + - First look at the host power plan: if "Maximum processor state" is 99%, turbo boost is off. + - Then run several ways of pinning cores in alternating rounds, several times each, and check whether the difference between groups is larger than the fluctuation within a group. +- **What was found**: Pinning to any group of cores made no difference. After turbo boost was turned on, the clock speed was about 20% higher, but cold boot was only about 2% faster. The main thread looks busy but is actually waiting (for memory access, the translation cache or other waits); it is not a lack of compute. +- **Result**: Nothing needs to change for core pinning. The next step is to find out what the main thread is waiting for. + +## A few lessons + +- **Collect evidence first, then draw conclusions.** In the past, a fork deadlock was taken for a slow network, a file broken by a hard kill was taken for a CDN that could not be reached, and a frame rate mixed with menus was taken for the battle frame rate. All of these were guesses made without collecting evidence. +- **Run comparisons in alternating rounds**, several times, and check whether the difference between groups is larger than the fluctuation within a group. One run each does not count. +- **When the problem is in a component bundled with the guest image** (the guest Vulkan driver, ANGLE), first look for a property or a guest-side workaround. The emulator does not necessarily have to change. +- **For changes that touch disk I/O or the shutdown flow, run the hard-kill regression first.** diff --git a/docs/en/guide/changes.md b/docs/en/guide/changes.md new file mode 100644 index 0000000..5524b95 --- /dev/null +++ b/docs/en/guide/changes.md @@ -0,0 +1,79 @@ +# How ModAVD differs from the official emulator + +Below are the changes that affect users in the current version (`0.36.0`, build mas-36, based on Google's AOSP Android Emulator 37.2.10). For the individual patches, see [`patches/`](https://github.com/ModAVD/ModAVD/tree/main/patches) in the ModAVD repository. For the changes in each version, see the [Release notes](https://github.com/ModAVD/ModAVD/releases). + +## Nothing is sent out + +- It does not upload usage statistics (`-metrics-collection` is treated as `-no-metrics`). +- It does not upload crash reports and does not pop up a question. Crash dumps stay on this machine, and the latest 5 are kept. +- It does not check for updates at boot and does not download feature flags (it uses the local cache or the copy shipped in the package). + +For details see [Privacy](./privacy). + +## Survives a force-kill + +- The data disk is handed to Android write-through, so force-killing the emulator process does not corrupt the virtual disk's (qcow2) metadata. Our hard-kill tests have not produced any corruption. Files that were still not written to disk in the last few seconds before shutdown can still be lost. +- Fixed the crash handler (crashpad_handler) exiting at random. Before the fix, once it had exited, the emulator hung the next time it crashed and left no dump. + +## Uses fewer resources + +- It supports free page reporting for the memory balloon: memory that Android frees goes back to Windows. What it saves is physical memory actually in use, not the commit charge, and the effect depends on the load. +- A new instance no longer takes up the full data disk limit temporarily on its first boot. When the drive holding the data disk is NTFS or ReFS, creating the disk for the first time only requires 10 GB free (the official build requires 1.2 times the limit: 77 GB for a 64 GB limit). +- Without `-logcat`, Android logs are no longer forwarded to the host. +- The graphics staging buffer grows on demand from 16 MB (the limit is still 128 MB) instead of allocating 128 MB at boot. On NVIDIA GPUs each idle instance uses about 140 MB less memory. + +## Disk + +- When Android submits several small reads or writes at once (64 KiB in total at most), they are merged into one block before being handed to Windows. This cuts the system calls per disk write from about 4 to 1. +- The data disk can be enlarged without touching game data: when the configured data disk is larger than the existing one, only the virtual disk is enlarged before boot (it is never shrunk), and Android expands it online after boot. In this case the official build either ignores the change or wipes the whole instance. +- A new data disk gets at most 1,048,576 inodes (the default for a 16 GB partition). For a 64 GB data disk the inode table goes from 1 GiB to 256 MiB, so a new instance uses about 0.7 GB less disk. Existing disks are unchanged. +- The resize inode of a new data disk is laid out as the kernel's online resize requires, so disks created at 16 GB from now on can be enlarged fully (before, they could only grow to about 30 GB). + +## Windows Hypervisor Platform (WHPX) + +- Fixed several problems where the guest clock (TSC) went out of sync around pause, resume and loading a snapshot. +- The local APIC is emulated by the Windows hypervisor by default. +- The virtual CPU limit goes from 6 to half the computer's logical processors. + +## Network + +- The buffer of each TCP connection goes from 8 KiB to 128 KiB. +- On Windows all network connections are polled. Before, only the first 128 were looked at. +- Connections on the host side have a short keepalive, so they close sooner when the peer is lost. +- Fixed a null pointer read, which made the whole emulator exit, when unregistering a connection that was never registered. It was hit when adb reconnected after an instance restart, and the official code has it too. + +## Graphics (gfxstream) {#graphics} + +- Fixed about 2%–5% of frames being read back all black on NVIDIA GPUs (flicker, and occasional all-black screenshots). Submitted upstream. +- Fixed render threads contending for the OpenGL context. Before, when a game switched videos, the YUV conversion shader could fail to compile and the whole emulator would exit. +- When a program in Android adds extension structures that the host does not recognize (ray tracing, mesh shaders and so on) to a Vulkan query, the emulator no longer exits. *Arknights: Endfield* used to make the emulator exit as soon as it created a Vulkan device. +- Fixed some compatibility problems in Vulkan descriptor template updates and OpenGL ES compute shaders, and a crash and several video memory leaks in the hardware video decoding (CUDA) path. +- Added two environment variable switches, both off by default: + - `ANDROID_EMU_VK_HIDE_REBAR=1`: reports the host's "video memory and host-visible" memory (NVIDIA ReBAR) to Android as video memory only; + - `ANDROID_EMU_VK_ASTC_BC3=1`: when emulating ASTC textures, decompresses them to BC3 (experimental). + +## Interface and logs + +- The side toolbar is cut down to power, volume up and down, back and home (plus minimize and close). Set the environment variable `ANDROID_EMU_FULL_TOOLBAR=1` to get the full toolbar back. +- Each log line is written immediately, and standard error is no longer buffered in blocks, so the last lines before a crash are not lost. +- The program icon and window icon are now ModAVD's own icon (fan art, see [Licenses and source code](./license#icon-and-artwork)). + +## Input method + +- Added a host IME channel (the goldfish pipe service `modavd_ime`) and the console command `ime`. Together with the guest IME in Android, they let you type Chinese and other text into Android directly in the emulator window with a Windows input method. +- From 0.36.0 the guest IME comes with the package (`components\modavd-ime.apk`) and is installed automatically when you start with the manager or the CLI. When it is not installed, the emulator behaves as before. See [The host IME](../components/ime). + +## arm64 execution bridge + +- When the instance directory has the boot marker `modavd-writable-system`, the instance boots with a writable system image no matter who starts it. +- From 0.36.0 the arm64 execution bridge puts this marker there when it is installed into an instance and removes it when uninstalled. So an instance with the bridge installed is started as usual by AUTO-MAS, and the bridge is still there. Instances without the marker behave as before. See [The arm64 execution bridge](../components/bridge). + +## Removed files + +Compared with the files that an official build installs, the Release package does not have these (the emulator does not need them to run): + +- `bin64\` (Cygwin e2fsprogs) and `drivers\` (the USB passthrough driver); +- the virtual camera scenes, the macro samples and the Pixel 10 skin, and the FlatBuffers development files; +- the ARM and 32-bit x86 `qemu-system`. + +So only x86_64 system images can run, and there is no `-usb-passthrough` and no virtual scene camera. diff --git a/docs/en/guide/feedback.md b/docs/en/guide/feedback.md new file mode 100644 index 0000000..a291b1d --- /dev/null +++ b/docs/en/guide/feedback.md @@ -0,0 +1,54 @@ +# Report a problem + +Open an issue in the [Issues](https://github.com/ModAVD/ModAVD/issues) of the ModAVD repository. Before you do, search for whether someone has already reported the same problem. + +- For problems with AUTO-MAS's interface, tasks or scheduling, please go to [AUTO-MAS](https://github.com/AUTO-MAS-Project/AUTO-MAS/issues). +- Do not post security problems publicly. Send an email to qiyinxi@foxmail.com. + +## 1. Collect a problem report + +1. **Do not shut down the emulator after something goes wrong.** If the instance is still running, the collection also records the screen and state at that moment. +2. Double-click `收集问题信息.bat` (collects a problem report) in the folder. After about a minute, a `魔改AVD问题信息-