[core, lua, sql] YAML as static data source; enums codegen; schema validation

.editorconfig

@@ -80,6 +80,18 @@ indent_brace_style = allman # This is unfortunately ignored in Visual Studio
 [Makefile]
 indent_style = tab
 
-[*.yml]
+[*.{yml,yaml}]
 indent_style = space
 indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.json]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

.github/workflows/pr_checks.yml

@@ -29,7 +29,7 @@ jobs:
       # $GITHUB_WORKSPACE is the workspace in the container (/__w/server/server)
       - run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -82,7 +82,7 @@ jobs:
 
       - run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         if: ${{ steps.check-changes.outputs.lua_changes == 'true' }}
         with:
           fetch-depth: 0
@@ -93,6 +93,10 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install --upgrade -r ./tools/requirements.txt
 
+      - name: Generate codegen Lua enums
+        if: ${{ steps.check-changes.outputs.lua_changes == 'true' }}
+        run: python -m tools.codegen "$RUNNER_TEMP/codegen-out"
+
       - name: Run Lua Language Server
         id: run_lls
         if: ${{ steps.check-changes.outputs.lua_changes == 'true' }}
@@ -129,6 +133,110 @@ jobs:
           echo "Lua Language Server failed. See summary for details."
           exit 1
 
+  Schema_Validation:
+    needs: Sanity_Checks
+    name: Data validation
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Download Changed Files
+        uses: actions/download-artifact@v4
+        with:
+          name: changed-files
+
+      - name: Check for data/ or codegen changes
+        id: changed
+        run: |
+          mapfile -t changed_files < changed-files.txt
+          touched=false
+          for f in "${changed_files[@]}"; do
+            if [[ $f == data/* || $f == tools/codegen/* ]]; then
+              touched=true
+              break
+            fi
+          done
+          echo "touched=$touched" >> "$GITHUB_OUTPUT"
+          echo "Touched: $touched"
+
+      - uses: actions/checkout@v6
+        if: steps.changed.outputs.touched == 'true'
+
+      - name: Setup Python
+        if: steps.changed.outputs.touched == 'true'
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - name: Install Python dependencies
+        if: steps.changed.outputs.touched == 'true'
+        run: python -m pip install jinja2 jsonschema pyyaml
+
+      - name: Run codegen --validate
+        if: steps.changed.outputs.touched == 'true'
+        run: |
+          mkdir -p "$RUNNER_TEMP/codegen-out"
+          python -m tools.codegen "$RUNNER_TEMP/codegen-out" --validate
+
+  Formatting:
+    needs: Sanity_Checks
+    name: Data formatting
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Download Changed Files
+        uses: actions/download-artifact@v4
+        with:
+          name: changed-files
+
+      - name: Filter data/ changed files
+        id: changed
+        run: |
+          mapfile -t changed_files < changed-files.txt
+          data_files=()
+          for f in "${changed_files[@]}"; do
+            if [[ $f == data/* ]]; then
+              data_files+=("$f")
+            fi
+          done
+          echo "files=${data_files[*]}" >> "$GITHUB_OUTPUT"
+          echo "Changed data files: ${data_files[*]}"
+
+      - uses: actions/checkout@v6
+        if: steps.changed.outputs.files != ''
+
+      - name: Cache prettier results
+        if: steps.changed.outputs.files != ''
+        uses: actions/cache@v5
+        with:
+          path: /tmp/prettier-cache
+          key: prettier-cache-${{ hashFiles('data/**/*.yaml', 'data/**/*.json') }}
+          restore-keys: prettier-cache-
+
+      - name: Run prettier
+        if: steps.changed.outputs.files != ''
+        run: |
+          set +e
+          FAILED=$(npx --yes prettier@3 --cache --cache-strategy content --cache-location /tmp/prettier-cache --list-different ${{ steps.changed.outputs.files }})
+          set -e
+          if [ -z "$FAILED" ]; then
+            echo "All files use Prettier code style!"
+            exit 0
+          fi
+          echo "::group::Files needing reformat"
+          echo "$FAILED"
+          echo "::endgroup::"
+          echo "::group::Diff"
+          for f in $FAILED; do
+            diff -u "$f" <(npx --yes prettier@3 "$f") || true
+          done
+          echo "::endgroup::"
+          echo "Run \`npx prettier --write data\` locally to fix."
+          exit 1
+
   Clang_Tidy:
     needs: Sanity_Checks
     name: Clang Tidy

.github/workflows/runner_build.yml

@@ -190,6 +190,10 @@ jobs:
         with:
           python-version: '3.12'
 
+      - name: Install Python codegen deps
+        if: ${{ env.SKIP_BUILD != 'true' }}
+        run: python -m pip install jinja2 jsonschema pyyaml
+
       - name: Enable modules
         if: ${{ inputs.build_modules && env.SKIP_BUILD != 'true' }}
         run: |

.github/workflows/runner_test.yml

@@ -126,6 +126,9 @@ jobs:
           echo "${{ runner.os == 'Windows' && '.venv/Scripts' || '.venv/bin' }}" >> $GITHUB_PATH
           echo "VIRTUAL_ENV=$(pwd)/.venv" >> $GITHUB_ENV
 
+      - name: Generate codegen Lua enums
+        run: python -m tools.codegen "$RUNNER_TEMP/codegen-out"
+
       - name: Download artifact
         if: ${{ github.event_name == 'pull_request' }}
         uses: actions/download-artifact@v4

.gitignore

@@ -266,3 +266,7 @@ __pycache__
 sanity_checks_summary.md
 changed-files.txt
 clang_tidy_summary.md
+
+# YAML codegen
+*.codegen.json
+*.codegen.lua

.prettierignore

@@ -0,0 +1,7 @@
+*.codegen.json
+*.codegen.lua
+
+build/
+build*/
+cmake-*/
+out/

.prettierrc

@@ -0,0 +1,15 @@
+{
+  "tabWidth": 2,
+  "useTabs": false,
+  "printWidth": 120,
+  "endOfLine": "lf",
+  "overrides": [
+    {
+      "files": ["**/*.yaml", "**/*.yml"],
+      "options": {
+        "singleQuote": false,
+        "bracketSpacing": true
+      }
+    }
+  ]
+}

.vscode/extensions.json

@@ -11,6 +11,8 @@
         "ms-vscode.cpptools-extension-pack",
         "ms-vscode.cpptools",
         "jeff-hykin.better-cpp-syntax",
-        "bierner.github-markdown-preview"
+        "bierner.github-markdown-preview",
+        "esbenp.prettier-vscode",
+        "redhat.vscode-yaml"
     ]
 }

.vscode/settings.json

@@ -355,6 +355,13 @@
         "editor.defaultFormatter": "ms-python.black-formatter",
         "editor.formatOnSave": true
     },
+    "[yaml]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[json]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "yaml.format.enable": false,
     "files.associations": {
         "algorithm": "cpp",
         "any": "cpp",

CMakeLists.txt

@@ -177,6 +177,38 @@ if (NOT IPC_EXIT_CODE EQUAL 0)
     message(FATAL_ERROR "Failed to generate IPC stubs")
 endif()
 
+execute_process(
+    COMMAND ${Python_EXECUTABLE} -c "import jinja2, jsonschema, yaml"
+    RESULT_VARIABLE CODEGEN_DEPS_EXIT_CODE
+    OUTPUT_QUIET
+    ERROR_QUIET
+)
+if(NOT CODEGEN_DEPS_EXIT_CODE EQUAL 0)
+    message(FATAL_ERROR "Run: ${Python_EXECUTABLE} -m pip install -r tools/requirements.txt")
+endif()
+
+add_custom_target(data_codegen
+    COMMAND ${Python_EXECUTABLE} -m tools.codegen ${CMAKE_BINARY_DIR}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    BYPRODUCTS
+        ${CMAKE_BINARY_DIR}/generated/.codegen.stamp
+        ${CMAKE_BINARY_DIR}/generated/data/all.h
+    USES_TERMINAL
+    VERBATIM
+)
+
+if(NOT EXISTS ${CMAKE_BINARY_DIR}/generated/.codegen.stamp)
+    message(STATUS "Bootstrapping data codegen")
+    execute_process(
+        COMMAND ${Python_EXECUTABLE} -m tools.codegen ${CMAKE_BINARY_DIR}
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+        RESULT_VARIABLE CODEGEN_EXIT_CODE
+    )
+    if(NOT CODEGEN_EXIT_CODE EQUAL 0)
+        message(FATAL_ERROR "Bootstrap codegen failed")
+    endif()
+endif()
+
 include_directories(${CMAKE_BINARY_DIR}/generated) # Globally include the build/generated directory
 
 add_subdirectory(src)

data/enums/effect_overwrite.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=../schemas/enums/_enum.schema.json
+
+meta:
+  cpp:
+    underlying: uint8_t
+  lua:
+    table: xi.effectOverwrite
+
+values:
+  equal_higher: 0 # only overwrite if equal or higher (tier, power)
+  higher: 1 # only overwrite if strictly higher
+  never: 2
+  always: 3
+  ignore_duplicate: 4
+  tier_higher: 5 # only overwrite if tier is higher (regardless of power)

data/enums/element.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=../schemas/enums/_enum.schema.json
+
+meta:
+  cpp:
+    underlying: uint8_t
+  lua:
+    table: xi.element
+
+values:
+  none: 0
+  fire: 1
+  ice: 2
+  wind: 3
+  earth: 4
+  thunder: 5
+  water: 6
+  light: 7
+  dark: 8

data/enums/status_effect_flag.yaml

@@ -0,0 +1,42 @@
+# yaml-language-server: $schema=../schemas/enums/_enum.schema.json
+
+meta:
+  flags: true
+  cpp:
+    underlying: uint32_t
+  lua:
+    table: xi.effectFlag
+
+values:
+  none: 0x00000000
+  dispelable: 0x00000001
+  erasable: 0x00000002
+  attack: 0x00000004
+  empathy: 0x00000008
+  damage: 0x00000010
+  death: 0x00000020
+  magic_begin: 0x00000040
+  magic_end: 0x00000080
+  on_zone: 0x00000100
+  no_loss_message: 0x00000200
+  invisible: 0x00000400
+  detectable: 0x00000800
+  no_rest: 0x00001000
+  prevent_action: 0x00002000
+  waltzable: 0x00004000
+  food: 0x00008000
+  song: 0x00010000
+  roll: 0x00020000
+  synth_support: 0x00040000
+  confrontation: 0x00080000
+  logout: 0x00100000
+  bloodpact: 0x00200000
+  on_jobchange: 0x00400000
+  no_cancel: 0x00800000
+  influence: 0x01000000
+  offline_tick: 0x02000000
+  aura: 0x04000000
+  hide_timer: 0x08000000
+  on_zone_pathos: 0x10000000
+  always_expiring: 0x20000000
+  on_attack: 0x40000000

data/schemas/_meta.schema.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "_meta.schema.json",
+  "title": "Data-file codegen metadata",
+  "description": "Optional top-level `meta:` block in data yamls. Carries directives that drive codegen.",
+  "type": "object",
+  "unevaluatedProperties": false,
+  "properties": {
+    "enum": {
+      "type": "object",
+      "unevaluatedProperties": false,
+      "required": ["cpp"],
+      "description": "Emit a typed enum derived from a top-level section of the data file.",
+      "properties": {
+        "cpp": {
+          "type": "object",
+          "unevaluatedProperties": false,
+          "required": ["underlying", "class"],
+          "properties": {
+            "underlying": { "type": "string" },
+            "class": { "type": "string", "pattern": "^[A-Z][A-Za-z0-9]*$" },
+            "case": { "type": "string", "enum": ["pascal", "screaming"], "default": "pascal" }
+          }
+        },
+        "lua": {
+          "type": "object",
+          "unevaluatedProperties": false,
+          "properties": {
+            "table": { "type": "string", "pattern": "^xi\\.[A-Za-z][A-Za-z0-9_]*$" }
+          }
+        },
+        "section": {
+          "type": "string",
+          "description": "Which top-level YAML section to walk. Defaults to the filename."
+        },
+        "name_from": {
+          "type": "string",
+          "description": "Dot-path inside each entry to a string that gets slugged into the enum value name (e.g., `name.en`). When set, the YAML key is treated as the integer id."
+        }
+      }
+    }
+  }
+}