merge

.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: Google

.gitignore

@@ -106,3 +106,21 @@ venv.bak/
 
 # protobuf stuff
 tf/proto/
+**/*_pb2.py
+**/*_pb2.pyi
+
+# Meson
+builddir/
+subprojects/
+
+# LLM Agents
+# Use AGENTS.md; symlink other files to it if needed.
+.claude/
+CLAUDE.local.md
+.claude.json
+CLAUDE.md
+GEMINI.md
+
+# IDE
+.vscode/
+.idea/

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "libs/lczero-common"]
-	path = libs/lczero-common
-	url = https://github.com/LeelaChessZero/lczero-common.git
+[submodule "libs/lc0"]
+	path = libs/lc0
+	url = https://github.com/LeelaChessZero/lc0.git

AGENTS.md

@@ -0,0 +1,59 @@
+# AGENTS.md
+
+This repository contains training script for the Leela Chess Zero project.
+They are being rewritten.
+
+* Old code is located in the `tf/` directory.
+* New python code is located in the `src/` directory.
+* New C++ code is located in the `csrc/` directory.
+
+The old code is Python/TensorFlow-based, new code is Python/JAX-based with
+modules written in C++, operating through pybind11.
+
+The build system for C++ code is meson. During development, the project is built
+in the `builddir/`.
+
+## Testing and Building
+
+* C++ tests use GTest framework
+  * Do not insert Sleeps in tests, it slows down presubmit. Instead use e.g.
+    absl::Notification, or std::future
+* Tests are defined in `meson.build` with `test()` function
+  * When debugging, don't forget to build them before running `meson test` or
+    `builddir/test`
+* Run tests: `meson test -C builddir/`
+* Python tests use `pytest` framework
+  * Do not add custom main function, exception catching to report errors, any
+    "test passed" messages etc. Use `pytest` fixtures and assertions.
+* Build: `meson compile -C builddir/` from build directory
+* Format code: `just format`
+* There is a commit hook that runs `just pre-commit`, which runs tests and
+  checks formatting. You may want to run it before attempting to commit.
+* We use Google C++ style guide.
+  * That means 80 columns.
+  * That means comments should be in full sentences with periods in the end.
+  * When conditional or loop fits one line, it must be written as one line
+    without braces, for example:
+      `if (condition) return value;`
+  * Prefer `absl` to `std` (e.g. `absl::c_` algorithms, `absl::Mutex`,
+    `absl::StrCat`, etc.)
+* We use `uv` for Python package and venv management, and to running the
+  application.
+* Run TUI app: `uv run tui --config=<path_to_config>`
+
+* Do not attempt to run TUI — it messes up the Agent interface and session has
+  to be killed. Ask me to check it for you manually instead.
+
+* Do not commit unless explicitly asked.
+
+## IMPORTANT
+
+* NEVER add `# type: ignore` or other ways to mask/silence errors instead of
+  fixing them.
+* Rely on protobuf default values. DO NOT write code like
+  `config.has_foo() ? config.foo() : default_value;`
+
+## Documentation
+
+* Documentation is in the `docs/` directory.
+* The contents is in [The index](docs/index.md)

README.md

@@ -78,3 +78,16 @@ The training pipeline will automatically restore from a previous model if it exi
 ## Supervised training
 
 Generating trainingdata from pgn files is currently broken and has low priority, feel free to create a PR.
+
+## Building 2025-08 version.
+
+1. Make sure `uv` and `justfile` are installed (plus `meson` and other stuff, potentially `protoc`).
+2. `git submodule update`
+3. `uv venv` (!important! do this before running meson; otherwise meson will build module for wrong python)
+4. `uv sync`
+5. `CXX=clang++ CC=clang uv run meson setup build/release/ --buildtype=release --native-file=native.ini` (clang is optional, should build fine with default compiler)
+6. `just build-proto`
+7. `meson compile -C build/release/`
+8. `ln -s -T ../../build/release/_lczero_training.cpython-311-x86_64-linux-gnu.so src/lczero_training/_lczero_training.so`
+
+14. Run it! `uv run lc0-tui --config docs/example.textproto`

csrc/loader/chunk_source/chunk_source.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <cstddef>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "loader/frame_type.h"
+
+namespace lczero {
+namespace training {
+
+// Interface for providing training data chunks.
+// A chunk source provides access to one or more chunks of training data.
+// It's assumed that all chunks in a source for one group for sorting purposes,
+// therefore GetChunkSortKey() returns just one key for the entire source. This
+// allows to know the key before reading/indexing the chunks.
+class ChunkSource {
+ public:
+  virtual ~ChunkSource() = default;
+
+  // Returns a sort key (e.g. filename or a timestamp).
+  virtual std::string GetChunkSortKey() const = 0;
+
+  // Returns the number of chunks in this source.
+  virtual size_t GetChunkCount() const = 0;
+
+  // Returns the data for the chunk at the given index. Returns std::nullopt if
+  // the chunk could not be read or if the data size is not a multiple of the
+  // expected frame size.
+  virtual std::optional<std::vector<FrameType>> GetChunkData(size_t index) = 0;
+};
+
+}  // namespace training
+}  // namespace lczero

csrc/loader/chunk_source/chunk_source_view.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "loader/chunk_source/chunk_source.h"
+
+namespace lczero {
+namespace training {
+
+// ChunkSourceView provides a view over another ChunkSource.
+// It exposes a remapped subset/order of chunks defined by indices into the
+// underlying source. It does not own or copy the data; it forwards calls to
+// the wrapped source.
+class ChunkSourceView : public ChunkSource {
+ public:
+  // Constructs a view over an existing chunk source. The indices vector maps
+  // local indices in the view to indices of the underlying source.
+  ChunkSourceView(std::shared_ptr<ChunkSource> source,
+                  std::vector<uint32_t> indices)
+      : source_(std::move(source)), indices_(std::move(indices)) {}
+
+  ~ChunkSourceView() override = default;
+
+ private:
+  std::string GetChunkSortKey() const override {
+    return source_->GetChunkSortKey();
+  }
+
+  size_t GetChunkCount() const override { return indices_.size(); }
+
+  std::optional<std::vector<FrameType>> GetChunkData(size_t index) override {
+    if (index >= indices_.size()) return std::nullopt;
+    const size_t src_index = static_cast<size_t>(indices_[index]);
+    return source_->GetChunkData(src_index);
+  }
+
+  std::shared_ptr<ChunkSource> source_;
+  std::vector<uint32_t> indices_;
+};
+
+}  // namespace training
+}  // namespace lczero

csrc/loader/chunk_source/debug_chunk_source.cc

@@ -0,0 +1,57 @@
+#include "loader/chunk_source/debug_chunk_source.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <cstring>
+#include <random>
+#include <utility>
+
+#include "absl/hash/hash.h"
+#include "absl/strings/str_format.h"
+
+namespace lczero {
+namespace training {
+
+DebugChunkSource::DebugChunkSource(uint64_t id, double mean_chunk_count)
+    : id_(id), mean_chunk_count_(mean_chunk_count) {}
+
+std::string DebugChunkSource::GetChunkSortKey() const {
+  return absl::StrFormat("%08" PRIu64, id_);
+}
+
+size_t DebugChunkSource::GetChunkCount() const {
+  if (!cached_chunk_count_.has_value()) {
+    std::mt19937_64 rng(id_);
+    const double stddev = std::max(1.0, mean_chunk_count_ / 4.0);
+    std::normal_distribution<double> distribution(mean_chunk_count_, stddev);
+    const double sampled = distribution(rng);
+    const auto rounded =
+        static_cast<long long>(std::llround(std::max(sampled, 1.0)));
+    cached_chunk_count_ = static_cast<size_t>(rounded);
+  }
+  return *cached_chunk_count_;
+}
+
+std::optional<std::vector<FrameType>> DebugChunkSource::GetChunkData(
+    size_t index) {
+  const auto seed_pair = std::make_pair(id_, index);
+  const uint64_t seed = static_cast<uint64_t>(
+      absl::Hash<std::pair<uint64_t, size_t>>{}(seed_pair));
+  std::mt19937_64 rng(seed);
+  std::uniform_int_distribution<int> frame_count_distribution(1, 200);
+  const int frame_count = frame_count_distribution(rng);
+
+  std::vector<FrameType> result(frame_count);
+  for (int frame_index = 0; frame_index < frame_count; ++frame_index) {
+    result[frame_index] = FrameType{};
+    result[frame_index].planes[0] = static_cast<uint64_t>(id_);
+    result[frame_index].planes[1] = static_cast<uint64_t>(index);
+    result[frame_index].planes[2] = static_cast<uint64_t>(frame_index);
+  }
+
+  return result;
+}
+
+}  // namespace training
+}  // namespace lczero

csrc/loader/chunk_source/debug_chunk_source.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <random>
+#include <string>
+
+#include "loader/chunk_source/chunk_source.h"
+
+namespace lczero {
+namespace training {
+
+// DebugChunkSource synthesizes deterministic pseudo-random chunks for loader
+// debugging. Each instance is identified by an integer id. The class produces
+// a chunk count sampled from a normal distribution with the provided mean and
+// mean / 4 standard deviation. The id serves as the seed, which keeps the
+// number of chunks stable across runs. Individual chunks contain a
+// pseudo-random number of FrameType frames (between one and 200) that are
+// generated on demand. The generation seed depends on both the source id and
+// chunk index. This lets shuffling logic exercise variable chunk sizes while
+// keeping the content reproducible. Each generated frame is zero-initialized,
+// but the first three entries of the planes array encode, respectively, the
+// source id, the chunk index, and the frame index within the chunk. This makes
+// it easy to reason about ordering and grouping when inspecting chunk payloads.
+class DebugChunkSource : public ChunkSource {
+ public:
+  DebugChunkSource(uint64_t id, double mean_chunk_count);
+
+ private:
+  std::string GetChunkSortKey() const override;
+  size_t GetChunkCount() const override;
+  std::optional<std::vector<FrameType>> GetChunkData(size_t index) override;
+
+  uint64_t id_;
+  double mean_chunk_count_;
+  mutable std::optional<size_t> cached_chunk_count_;
+};
+
+}  // namespace training
+}  // namespace lczero

csrc/loader/chunk_source/rawfile_chunk_source.cc

@@ -0,0 +1,59 @@
+#include "loader/chunk_source/rawfile_chunk_source.h"
+
+#include <absl/log/log.h>
+
+#include <fstream>
+#include <stdexcept>
+
+#include "trainingdata/trainingdata_v6.h"
+#include "utils/files.h"
+#include "utils/gz.h"
+
+namespace lczero {
+namespace training {
+
+RawFileChunkSource::RawFileChunkSource(
+    const std::filesystem::path& filename,
+    ChunkSourceLoaderConfig::FrameFormat frame_format)
+    : filename_(filename), frame_format_(frame_format) {}
+
+RawFileChunkSource::~RawFileChunkSource() = default;
+
+std::string RawFileChunkSource::GetChunkSortKey() const {
+  return std::filesystem::path(filename_).filename().string();
+}
+
+size_t RawFileChunkSource::GetChunkCount() const { return 1; }
+
+std::optional<std::vector<FrameType>> RawFileChunkSource::GetChunkData(
+    size_t index) {
+  if (index != 0) return std::nullopt;
+  std::string data = ReadFileToString(filename_);
+  if (data.empty()) return std::nullopt;
+
+  const size_t input_size =
+      frame_format_ == ChunkSourceLoaderConfig::V7TrainingData
+          ? sizeof(V7TrainingData)
+          : sizeof(V6TrainingData);
+  if (data.size() % input_size != 0) {
+    LOG(WARNING) << "File " << filename_ << " size " << data.size()
+                 << " is not a multiple of input frame size " << input_size;
+    return std::nullopt;
+  }
+
+  const size_t num_frames = data.size() / input_size;
+  std::vector<V7TrainingData> result(num_frames);
+
+  if (frame_format_ == ChunkSourceLoaderConfig::V7TrainingData) {
+    std::memcpy(result.data(), data.data(), data.size());
+  } else {
+    const auto* v6_data = reinterpret_cast<const V6TrainingData*>(data.data());
+    for (size_t i = 0; i < num_frames; ++i) {
+      std::memcpy(&result[i], &v6_data[i], sizeof(V6TrainingData));
+    }
+  }
+  return result;
+}
+
+}  // namespace training
+}  // namespace lczero

csrc/loader/chunk_source/rawfile_chunk_source.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <filesystem>
+#include <string>
+
+#include "loader/chunk_source/chunk_source.h"
+#include "proto/data_loader_config.pb.h"
+
+namespace lczero {
+namespace training {
+
+// A chunk source that reads a single (potentially gzipped) file as a single
+// chunk.
+class RawFileChunkSource : public ChunkSource {
+ public:
+  RawFileChunkSource(const std::filesystem::path& filename,
+                     ChunkSourceLoaderConfig::FrameFormat frame_format);
+  ~RawFileChunkSource();
+
+ private:
+  std::string GetChunkSortKey() const override;
+  size_t GetChunkCount() const override;
+  std::optional<std::vector<FrameType>> GetChunkData(size_t index) override;
+
+  std::string filename_;
+  ChunkSourceLoaderConfig::FrameFormat frame_format_;
+};
+
+}  // namespace training
+}  // namespace lczero