From a20b0ac747c594aac10733a132dfec8c4f4d5fca Mon Sep 17 00:00:00 2001 From: Liam Stevens Date: Sun, 14 Dec 2025 12:11:40 +1000 Subject: [PATCH 1/2] ci(release): add goreleaser and release workflow (M5-11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automated release process: .goreleaser.yml: - Build binaries for Linux, macOS, and Windows (amd64 + arm64) - Create archives with README, LICENSE, and docs - Generate SHA-256 checksums - Auto-generate changelog from conventional commits - Optional Homebrew tap support .github/workflows/release.yml: - Trigger on version tags (v*) - Run tests and lint before release - Use goreleaser to build and publish - Verify release artifacts work on Linux and macOS To create a release: ```bash git tag v0.1.0 git push origin v0.1.0 ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/release.yml | 92 +++++++++++++++++++++++++ .goreleaser.yml | 126 ++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..496334d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,92 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + test: + name: Test before release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Run tests + run: go test -race ./... + + - name: Run integration tests + run: go test -tags=integration -race -timeout 5m ./... + + lint: + name: Lint before release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: latest + + release: + name: Release + runs-on: ubuntu-latest + needs: [test, lint] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Uncomment if using Homebrew tap + # HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} + + verify: + name: Verify release artifacts + runs-on: ${{ matrix.os }} + needs: release + strategy: + matrix: + include: + - os: ubuntu-latest + artifact: git-los_*_linux_x86_64.tar.gz + - os: macos-latest + artifact: git-los_*_darwin_x86_64.tar.gz + + steps: + - name: Download release + uses: robinraju/release-downloader@v1.9 + with: + latest: false + tag: ${{ github.ref_name }} + fileName: ${{ matrix.artifact }} + extract: true + + - name: Verify binary works + run: | + chmod +x git-los + ./git-los version + ./git-los --help diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..e33a938 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,126 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj + +project_name: git-los + +before: + hooks: + - go mod tidy + - go generate ./... + +builds: + - id: git-los + binary: git-los + main: ./cmd/git-los + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X main.version={{.Version}} + - -X main.commit={{.Commit}} + - -X main.date={{.Date}} + +archives: + - id: default + format: tar.gz + # Use zip for Windows + format_overrides: + - goos: windows + format: zip + name_template: >- + {{ .ProjectName }}_ + {{- .Version }}_ + {{- .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + files: + - README.md + - LICENSE* + - CONTRIBUTING.md + - docs/* + +checksum: + name_template: 'checksums.txt' + algorithm: sha256 + +snapshot: + name_template: "{{ incpatch .Version }}-next" + +changelog: + sort: asc + use: github + groups: + - title: Features + regexp: "^.*feat[(\\w)]*:+.*$" + order: 0 + - title: Bug fixes + regexp: "^.*fix[(\\w)]*:+.*$" + order: 1 + - title: Documentation + regexp: "^.*docs[(\\w)]*:+.*$" + order: 2 + - title: Other + order: 999 + filters: + exclude: + - '^test:' + - '^chore:' + - Merge pull request + - Merge branch + +release: + github: + owner: valent-au + name: git-los + draft: false + prerelease: auto + name_template: "{{.Tag}}" + header: | + ## git-los {{.Tag}} + + Git LFS custom transfer agent with native GCS and S3 support. + + ### Installation + + Download the appropriate archive for your platform below, extract it, and add the binary to your PATH. + + ### Quick Start + + ```bash + # Configure git-los as your LFS transfer agent + git config --global lfs.standalonetransferagent git-los + git config --global lfs.customtransfer.git-los.path git-los + git config --global lfs.customtransfer.git-los.args agent + + # Set your backend URL + git config lfs.url "gs://your-bucket/lfs" + # or + git config lfs.url "s3://your-bucket/lfs" + ``` + + footer: | + **Full Changelog**: https://github.com/valent-au/git-los/compare/{{ .PreviousTag }}...{{ .Tag }} + +brews: + - name: git-los + repository: + owner: valent-au + name: homebrew-tap + token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" + directory: Formula + homepage: https://github.com/valent-au/git-los + description: Git LFS custom transfer agent with native GCS and S3 support + license: Apache-2.0 + skip_upload: auto + test: | + system "#{bin}/git-los", "version" + install: | + bin.install "git-los" From e26d71c2b0614871d9383a6b10502df3dbd730ab Mon Sep 17 00:00:00 2001 From: Liam Stevens Date: Sun, 14 Dec 2025 12:40:40 +1000 Subject: [PATCH 2/2] docs: add user documentation (M5-07) (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive user documentation: - docs/installation.md - Platform-specific installation guide - docs/configuration.md - Backend setup and all config options - docs/troubleshooting.md - Common issues and solutions - docs/examples/game-dev.md - Game development workflow with binary assets - docs/examples/ml-datasets.md - ML dataset versioning and management Updated README.md to link to new documentation sections: - User Guides (install, config, troubleshooting) - Example Workflows (game-dev, ML) - Development (contributing, dev guide) - Specifications (existing spec docs) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 --- README.md | 18 ++ docs/configuration.md | 278 +++++++++++++++++++++++ docs/examples/game-dev.md | 327 +++++++++++++++++++++++++++ docs/examples/ml-datasets.md | 421 +++++++++++++++++++++++++++++++++++ docs/installation.md | 177 +++++++++++++++ docs/troubleshooting.md | 363 ++++++++++++++++++++++++++++++ 6 files changed, 1584 insertions(+) create mode 100644 docs/configuration.md create mode 100644 docs/examples/game-dev.md create mode 100644 docs/examples/ml-datasets.md create mode 100644 docs/installation.md create mode 100644 docs/troubleshooting.md diff --git a/README.md b/README.md index b095d19..5c079d1 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,24 @@ git push # LFS objects upload to your bucket via git-los ## Documentation +### User Guides + +- [Installation](./docs/installation.md) — Installing and setting up git-los +- [Configuration](./docs/configuration.md) — Backend setup and options +- [Troubleshooting](./docs/troubleshooting.md) — Common issues and solutions + +### Example Workflows + +- [Game Development](./docs/examples/game-dev.md) — Managing textures, models, and audio +- [ML Datasets](./docs/examples/ml-datasets.md) — Dataset versioning and model management + +### Development + +- [Contributing](./CONTRIBUTING.md) — How to contribute to git-los +- [Development Guide](./docs/development.md) — Building and testing locally + +### Specifications + See the [spec](./spec/README.md) directory for detailed specifications: - [Architecture](./spec/architecture.md) — System components and their interactions diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..e8a98bb --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,278 @@ +# Configuration Guide + +This guide covers all configuration options for git-los, including cloud backend setup, daemon settings, and advanced options. + +## Quick Start + +### Google Cloud Storage (GCS) + +```bash +# Set up Application Default Credentials +gcloud auth application-default login + +# Configure a repository +cd your-repo +git-los remote add origin gs://your-bucket/lfs +``` + +### Amazon S3 + +```bash +# Set up AWS credentials +aws configure + +# Configure a repository +cd your-repo +git-los remote add origin s3://your-bucket/lfs +``` + +## Backend URL Format + +git-los uses URL-style identifiers for storage backends: + +| Backend | URL Format | Example | +|---------|------------|---------| +| GCS | `gs://bucket/prefix` | `gs://my-project-lfs/objects` | +| S3 | `s3://bucket/prefix` | `s3://my-lfs-bucket/data` | + +The prefix is optional but recommended for organizing LFS objects. + +## Cloud Credentials + +### Google Cloud Storage + +git-los uses Application Default Credentials (ADC) in this order: + +1. `GOOGLE_APPLICATION_CREDENTIALS` environment variable pointing to a service account key +2. User credentials from `gcloud auth application-default login` +3. Service account attached to GCE/GKE instances +4. Workload Identity on GKE + +**For development:** +```bash +gcloud auth application-default login +``` + +**For CI/CD:** +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json +``` + +**Required IAM permissions:** +- `storage.objects.create` - Upload objects +- `storage.objects.get` - Download objects +- `storage.objects.delete` - Delete objects (optional) + +### Amazon S3 + +git-los uses standard AWS credential chain: + +1. Environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` +2. AWS credentials file (`~/.aws/credentials`) +3. IAM role attached to EC2/ECS instances +4. Web Identity Token for EKS + +**For development:** +```bash +aws configure +``` + +**For CI/CD:** +```bash +export AWS_ACCESS_KEY_ID=your-key-id +export AWS_SECRET_ACCESS_KEY=your-secret-key +export AWS_REGION=us-east-1 +``` + +**Required IAM permissions:** +```json +{ + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::your-bucket/lfs/*" +} +``` + +## Git Configuration + +### Repository-Level Settings + +Configure in your repository's `.git/config`: + +```gitconfig +[lfs] + url = gs://your-bucket/lfs +``` + +Or use the CLI: +```bash +git-los remote add origin gs://your-bucket/lfs +``` + +### Global Settings + +Configure in `~/.gitconfig`: + +```gitconfig +[lfs] + standalonetransferagent = git-los + +[lfs "customtransfer.git-los"] + path = git-los + args = agent + concurrent = true + direction = both +``` + +These are automatically set by `git-los install --global`. + +## Daemon Configuration + +The daemon provides connection pooling and credential caching. + +### Starting the Daemon + +```bash +# Start in foreground (for debugging) +git-los daemon start --foreground + +# Start in background (normal operation) +git-los daemon start + +# Check status +git-los daemon status +``` + +### Daemon Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--foreground` | Run in foreground | `false` | +| `--log-level` | Log level (debug, info, warn, error) | `info` | +| `--log-file` | Log file path | `stderr` | +| `--max-concurrent` | Max concurrent transfers | `8` | + +### Socket Location + +The daemon socket is created at: +1. `$GIT_LOS_SOCKET` if set +2. `$XDG_RUNTIME_DIR/git-los/daemon.sock` +3. `~/.cache/git-los/daemon.sock` + +### Log Files + +Daemon logs are written to: +- Default: `stderr` (when running in foreground) +- Configured: Use `--log-file /path/to/daemon.log` + +Logs are rotated automatically: +- Max size: 10MB per file +- Max backups: 3 files +- Compression: gzip + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `GIT_LOS_SOCKET` | Override daemon socket path | +| `GIT_LOS_DEBUG` | Enable debug logging (`1` or `true`) | +| `GOOGLE_APPLICATION_CREDENTIALS` | GCS service account key path | +| `AWS_ACCESS_KEY_ID` | AWS access key | +| `AWS_SECRET_ACCESS_KEY` | AWS secret key | +| `AWS_REGION` | AWS region | + +## Advanced Options + +### Custom S3 Endpoints + +For S3-compatible storage (MinIO, etc.): + +```bash +export AWS_ENDPOINT_URL=http://localhost:9000 +git-los remote add origin s3://bucket/prefix +``` + +### Concurrent Transfers + +Adjust the number of concurrent transfers: + +```gitconfig +[lfs "customtransfer.git-los"] + concurrent = true +``` + +### Retry Configuration + +git-los automatically retries failed transfers with exponential backoff: +- Max attempts: 3 +- Initial wait: 1 second +- Max wait: 30 seconds + +Retryable errors include: +- Network timeouts +- Server errors (5xx) +- Rate limiting + +Non-retryable errors include: +- Authentication failures (401, 403) +- Not found (404) +- Bad requests (400) + +## Configuration Precedence + +git-los reads configuration in this order (later overrides earlier): + +1. Built-in defaults +2. System git config (`/etc/gitconfig`) +3. Global git config (`~/.gitconfig`) +4. Repository git config (`.git/config`) +5. Environment variables +6. Command-line flags + +## Troubleshooting Configuration + +### Verify Backend URL + +```bash +git config lfs.url +# or +git-los remote list +``` + +### Check Credentials + +**GCS:** +```bash +gcloud auth application-default print-access-token +``` + +**S3:** +```bash +aws sts get-caller-identity +``` + +### Test Connection + +```bash +# Upload a test file +echo "test" > /tmp/test.txt +git-los upload --oid $(sha256sum /tmp/test.txt | cut -d' ' -f1) --path /tmp/test.txt +``` + +### View Daemon Logs + +```bash +git-los daemon status +# Check log file location, then: +tail -f /path/to/daemon.log +``` + +## Next Steps + +- [Troubleshoot common issues](./troubleshooting.md) +- [Game development workflow](./examples/game-dev.md) +- [ML dataset management](./examples/ml-datasets.md) diff --git a/docs/examples/game-dev.md b/docs/examples/game-dev.md new file mode 100644 index 0000000..d231328 --- /dev/null +++ b/docs/examples/game-dev.md @@ -0,0 +1,327 @@ +# Game Development Workflow + +This guide demonstrates using git-los for game development with large binary assets like textures, models, and audio files. + +## Overview + +Game projects often include: +- **Textures** (PNG, TGA, PSD) - 10MB to 100MB+ each +- **3D Models** (FBX, OBJ, GLTF) - 1MB to 500MB each +- **Audio** (WAV, OGG) - 10MB to 1GB each +- **Video** (MP4, MOV) - 100MB to 10GB each + +git-los with cloud storage provides a cost-effective solution compared to hosted LFS services. + +## Initial Setup + +### 1. Create a Cloud Bucket + +**GCS:** +```bash +# Create bucket in your nearest region +gsutil mb -l us-central1 gs://my-game-lfs + +# Enable versioning (optional, for recovery) +gsutil versioning set on gs://my-game-lfs +``` + +**S3:** +```bash +# Create bucket +aws s3 mb s3://my-game-lfs --region us-east-1 + +# Enable versioning (optional) +aws s3api put-bucket-versioning \ + --bucket my-game-lfs \ + --versioning-configuration Status=Enabled +``` + +### 2. Configure git-los + +```bash +# Install git-los globally +git-los install --global + +# Start the daemon +git-los daemon start +``` + +### 3. Initialize Your Game Repository + +```bash +cd my-game-project + +# Initialize git and git-lfs +git init +git lfs install + +# Configure git-los backend +git-los remote add origin gs://my-game-lfs/objects +# or +git-los remote add origin s3://my-game-lfs/objects +``` + +## Tracking Game Assets + +### Recommended .gitattributes + +Create a `.gitattributes` file optimized for game development: + +```gitattributes +# Textures +*.png filter=lfs diff=lfs merge=lfs -text +*.tga filter=lfs diff=lfs merge=lfs -text +*.psd filter=lfs diff=lfs merge=lfs -text +*.tif filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +*.dds filter=lfs diff=lfs merge=lfs -text +*.exr filter=lfs diff=lfs merge=lfs -text +*.hdr filter=lfs diff=lfs merge=lfs -text + +# 3D Models +*.fbx filter=lfs diff=lfs merge=lfs -text +*.obj filter=lfs diff=lfs merge=lfs -text +*.gltf filter=lfs diff=lfs merge=lfs -text +*.glb filter=lfs diff=lfs merge=lfs -text +*.blend filter=lfs diff=lfs merge=lfs -text +*.max filter=lfs diff=lfs merge=lfs -text +*.ma filter=lfs diff=lfs merge=lfs -text +*.mb filter=lfs diff=lfs merge=lfs -text + +# Audio +*.wav filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.flac filter=lfs diff=lfs merge=lfs -text +*.aif filter=lfs diff=lfs merge=lfs -text +*.aiff filter=lfs diff=lfs merge=lfs -text + +# Video +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.mov filter=lfs diff=lfs merge=lfs -text +*.avi filter=lfs diff=lfs merge=lfs -text +*.mkv filter=lfs diff=lfs merge=lfs -text + +# Game Engine Packages +*.unitypackage filter=lfs diff=lfs merge=lfs -text +*.uasset filter=lfs diff=lfs merge=lfs -text +*.umap filter=lfs diff=lfs merge=lfs -text + +# Archives (pre-built assets) +*.zip filter=lfs diff=lfs merge=lfs -text +*.7z filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text + +# Fonts +*.ttf filter=lfs diff=lfs merge=lfs -text +*.otf filter=lfs diff=lfs merge=lfs -text +``` + +### Initialize Tracking + +```bash +# Add the .gitattributes +git add .gitattributes +git commit -m "Configure LFS tracking for game assets" +``` + +## Daily Workflow + +### Adding New Assets + +```bash +# Add a new texture +cp ~/Downloads/hero_texture.png assets/textures/ + +# Stage and commit +git add assets/textures/hero_texture.png +git commit -m "Add hero character texture" + +# Push (LFS objects go to cloud storage) +git push +``` + +### Updating Existing Assets + +```bash +# Artist updates a model file +git add assets/models/character.fbx +git commit -m "Update character model with new animations" +git push +``` + +### Pulling Changes + +```bash +# Pull code and LFS pointers +git pull + +# LFS objects are fetched automatically +# Or manually fetch all: +git lfs pull +``` + +## Team Collaboration + +### Sharing the Repository + +1. **Share git repository** (GitHub, GitLab, etc.) +2. **Share cloud bucket access:** + +**GCS:** +```bash +# Grant team members access +gcloud storage buckets add-iam-policy-binding gs://my-game-lfs \ + --member=user:teammate@example.com \ + --role=roles/storage.objectUser +``` + +**S3:** +```bash +# Create an IAM policy for the team +# Then attach to team members' IAM users/roles +``` + +### New Team Member Setup + +```bash +# Clone the repository +git clone https://github.com/your-team/my-game-project +cd my-game-project + +# Install git-los +git-los install + +# Authenticate with cloud +gcloud auth application-default login # GCS +# or +aws configure # S3 + +# Fetch LFS objects +git lfs pull +``` + +## Advanced Workflows + +### Partial Clone (Save Bandwidth) + +For artists who only need specific assets: + +```bash +# Clone without LFS objects +GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/team/game + +# Fetch only what you need +cd game +git lfs pull --include="assets/textures/*" +``` + +### Build Server Setup + +For CI/CD pipelines: + +```bash +# In your CI config +- name: Setup git-los + run: | + go install github.com/valent-au/git-los/cmd/git-los@latest + git-los install + +- name: Fetch LFS objects + run: git lfs pull + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCS_KEY }} +``` + +### Cleaning Up Old Versions + +If you've enabled bucket versioning: + +```bash +# GCS: Set lifecycle policy to delete old versions +gsutil lifecycle set lifecycle.json gs://my-game-lfs + +# lifecycle.json +{ + "rule": [{ + "action": {"type": "Delete"}, + "condition": { + "numNewerVersions": 3, + "isLive": false + } + }] +} +``` + +## Cost Estimation + +### Storage Costs + +| Provider | Storage | Egress | +|----------|---------|--------| +| GCS (Standard) | ~$0.020/GB/month | ~$0.12/GB | +| S3 (Standard) | ~$0.023/GB/month | ~$0.09/GB | + +### Example: Small Indie Game + +- **Assets:** 50GB total +- **Team:** 5 developers +- **Monthly downloads:** 200GB egress + +**Estimated monthly cost:** +- Storage: $1.00-1.15 +- Egress: $18-24 +- **Total: ~$20-25/month** + +### Tips for Reducing Costs + +1. **Use nearline/infrequent access** for archived branches +2. **Enable CDN** (Cloud CDN / CloudFront) for frequently accessed objects +3. **Compress assets** before tracking (where lossless compression helps) +4. **Use regional buckets** close to your team + +## Project Structure Example + +``` +my-game/ +├── .gitattributes # LFS tracking rules +├── .gitignore +├── assets/ +│ ├── textures/ # 2D textures (LFS) +│ ├── models/ # 3D models (LFS) +│ ├── audio/ # Sound effects & music (LFS) +│ └── video/ # Cutscenes (LFS) +├── src/ # Game code (regular git) +├── docs/ # Documentation (regular git) +└── tools/ # Build tools (regular git) +``` + +## Troubleshooting + +### Large Push Taking Too Long + +```bash +# Check progress +git-los daemon status + +# Increase concurrency for faster uploads +git config lfs.concurrenttransfers 4 +``` + +### Missing Assets After Clone + +```bash +# Fetch all LFS objects +git lfs fetch --all +git lfs checkout +``` + +### Authentication Issues + +See the [troubleshooting guide](../troubleshooting.md#authentication-errors). + +## Next Steps + +- Review [configuration options](../configuration.md) +- Set up [CI/CD for your game](../configuration.md#build-server-setup) +- Explore [ML dataset workflows](./ml-datasets.md) if you have training data diff --git a/docs/examples/ml-datasets.md b/docs/examples/ml-datasets.md new file mode 100644 index 0000000..ed2788d --- /dev/null +++ b/docs/examples/ml-datasets.md @@ -0,0 +1,421 @@ +# Machine Learning Dataset Management + +This guide demonstrates using git-los for managing large ML datasets, model weights, and training artifacts. + +## Overview + +ML projects often include: +- **Training datasets** - Images, videos, text corpora (GB to TB) +- **Model weights** - Checkpoint files, saved models (100MB to 10GB+) +- **Preprocessed data** - NumPy arrays, tensors, embeddings +- **Evaluation results** - Metrics, visualizations, logs + +git-los provides versioned, reproducible dataset management with cloud-native performance. + +## Initial Setup + +### 1. Create a Cloud Bucket + +**GCS (recommended for Google Cloud ML):** +```bash +# Create bucket in a region near your training infrastructure +gsutil mb -l us-central1 -c standard gs://my-ml-data + +# Enable versioning for experiment reproducibility +gsutil versioning set on gs://my-ml-data +``` + +**S3 (recommended for AWS SageMaker):** +```bash +# Create bucket +aws s3 mb s3://my-ml-data --region us-east-1 + +# Enable versioning +aws s3api put-bucket-versioning \ + --bucket my-ml-data \ + --versioning-configuration Status=Enabled +``` + +### 2. Configure git-los + +```bash +# Install git-los +git-los install --global + +# Start the daemon +git-los daemon start +``` + +### 3. Initialize Your ML Repository + +```bash +cd my-ml-project + +# Initialize git and git-lfs +git init +git lfs install + +# Configure git-los backend +git-los remote add origin gs://my-ml-data/lfs +``` + +## Tracking ML Assets + +### Recommended .gitattributes + +```gitattributes +# Datasets +*.csv filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.tfrecord filter=lfs diff=lfs merge=lfs -text + +# Images (training data) +data/**/*.jpg filter=lfs diff=lfs merge=lfs -text +data/**/*.jpeg filter=lfs diff=lfs merge=lfs -text +data/**/*.png filter=lfs diff=lfs merge=lfs -text + +# NumPy/Pickle +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text + +# Model weights +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.hdf5 filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text + +# Checkpoints +checkpoints/**/* filter=lfs diff=lfs merge=lfs -text +models/**/*.bin filter=lfs diff=lfs merge=lfs -text + +# Archives +*.tar filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text + +# Embeddings +*.vec filter=lfs diff=lfs merge=lfs -text +embeddings/**/* filter=lfs diff=lfs merge=lfs -text +``` + +## Dataset Versioning Workflow + +### Organizing Your Project + +``` +my-ml-project/ +├── .gitattributes +├── data/ +│ ├── raw/ # Original unprocessed data (LFS) +│ ├── processed/ # Preprocessed features (LFS) +│ └── splits/ # Train/val/test splits (LFS) +├── models/ +│ ├── checkpoints/ # Training checkpoints (LFS) +│ └── final/ # Final model weights (LFS) +├── notebooks/ # Jupyter notebooks (regular git) +├── src/ # Training code (regular git) +├── configs/ # Experiment configs (regular git) +├── requirements.txt # Dependencies (regular git) +└── README.md +``` + +### Dataset Version Workflow + +**1. Initial dataset commit:** +```bash +# Add raw data +cp -r ~/datasets/imagenet data/raw/imagenet +git add data/raw/ +git commit -m "Add ImageNet training data v1" +git push +``` + +**2. Create processed features:** +```bash +# Run preprocessing +python src/preprocess.py --input data/raw --output data/processed + +# Commit processed data +git add data/processed/ +git commit -m "Add preprocessed features v1" +git push +``` + +**3. Tag a dataset version:** +```bash +git tag -a data-v1.0 -m "ImageNet processed v1.0" +git push --tags +``` + +### Experiment Reproducibility + +**1. Create experiment branch:** +```bash +git checkout -b experiment/resnet50-baseline +``` + +**2. Train and save checkpoints:** +```bash +# Training saves checkpoints to models/checkpoints/ +python src/train.py --config configs/resnet50.yaml + +# Commit best model +git add models/checkpoints/best.pt +git commit -m "ResNet50 baseline: 76.1% accuracy" +``` + +**3. Tag successful experiments:** +```bash +git tag -a exp-resnet50-v1 -m "ResNet50 baseline 76.1%" +git push --tags +``` + +## Training Infrastructure Integration + +### Colab/Jupyter Setup + +```python +# Install git-los in Colab +!pip install git-lfs +!go install github.com/valent-au/git-los/cmd/git-los@latest +!git-los install + +# Authenticate +from google.colab import auth +auth.authenticate_user() + +# Clone with LFS +!git clone https://github.com/your-org/ml-project +!cd ml-project && git lfs pull +``` + +### Docker Training Environment + +```dockerfile +FROM python:3.10 + +# Install git-los +RUN apt-get update && apt-get install -y git git-lfs +RUN go install github.com/valent-au/git-los/cmd/git-los@latest +RUN git-los install --global + +# Your training code +COPY . /app +WORKDIR /app +``` + +### CI/CD Pipeline + +```yaml +# GitHub Actions example +name: Train Model + +on: + push: + branches: [main] + paths: + - 'configs/**' + - 'src/**' + +jobs: + train: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: false # We'll use git-los instead + + - name: Setup git-los + run: | + go install github.com/valent-au/git-los/cmd/git-los@latest + git-los install + + - name: Fetch training data + run: git lfs pull --include="data/processed/*" + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCS_KEY }} + + - name: Train model + run: python src/train.py + + - name: Upload model + run: | + git add models/checkpoints/ + git commit -m "Training results from CI" + git push +``` + +## Large Dataset Strategies + +### Partial Dataset Clones + +For very large datasets, fetch only what you need: + +```bash +# Clone without LFS objects +GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/org/ml-project + +# Fetch only training data (not validation/test) +cd ml-project +git lfs pull --include="data/processed/train/*" + +# Or fetch by file type +git lfs pull --include="*.parquet" +``` + +### Dataset Sharding + +For TB-scale datasets, organize by shards: + +``` +data/ +├── train/ +│ ├── shard-0000.parquet +│ ├── shard-0001.parquet +│ └── ... +└── metadata.json +``` + +Then fetch specific shards: +```bash +git lfs pull --include="data/train/shard-000*.parquet" +``` + +### Model Weight Management + +For large models (multiple GB), consider: + +```bash +# Track only final models, not all checkpoints +echo "models/checkpoints/epoch-*.pt" >> .gitignore +echo "models/final/*.pt filter=lfs diff=lfs merge=lfs -text" >> .gitattributes +``` + +## Cost Optimization + +### Storage Tiers + +**GCS:** +```bash +# Move old dataset versions to nearline storage +gsutil rewrite -s nearline gs://my-ml-data/lfs/objects/sha256/old-data/* +``` + +**S3:** +```bash +# Configure lifecycle policy for intelligent tiering +aws s3api put-bucket-lifecycle-configuration \ + --bucket my-ml-data \ + --lifecycle-configuration file://lifecycle.json +``` + +### Estimated Costs + +| Dataset Size | Storage/Month | Transfer (1 team member) | +|--------------|---------------|--------------------------| +| 100GB | $2-3 | $9-12 | +| 1TB | $20-23 | $90-120 | +| 10TB | $200-230 | $900-1200 | + +**Tips:** +- Use transfer acceleration for faster uploads +- Set up CDN for frequently accessed datasets +- Archive old experiment branches + +## Best Practices + +### 1. Version Your Data + +```bash +# Always tag stable dataset versions +git tag data-v1.0.0 +git tag data-v1.1.0 # Added new training examples +``` + +### 2. Document Dataset Changes + +```bash +git commit -m "Add 10k new training images + +- Source: manual annotation sprint +- Classes affected: class_a, class_b +- Quality: manually verified +" +``` + +### 3. Separate Code and Data Commits + +```bash +# Good: separate commits +git add src/ +git commit -m "Improve data augmentation pipeline" + +git add data/processed/ +git commit -m "Regenerate processed data with new augmentation" + +# Bad: mixing code and data changes +git add . +git commit -m "Updates" +``` + +### 4. Use .gitignore Wisely + +```gitignore +# Ignore temporary training artifacts +*.tmp +__pycache__/ +.ipynb_checkpoints/ + +# Ignore local-only data +data/local/ + +# Keep important files +!data/.gitkeep +!models/.gitkeep +``` + +## Troubleshooting + +### Slow Dataset Uploads + +```bash +# Enable parallel uploads +git config lfs.concurrenttransfers 8 + +# Or upload in batches +git lfs push --object-id sha256:abc123... +``` + +### Out of Memory During LFS Operations + +```bash +# Process files one at a time +git lfs fetch --include="data/train/shard-0001.parquet" +``` + +### Credentials in Training Environment + +```bash +# For GCS on GCE/GKE +# Service account is attached automatically + +# For S3 on EC2 +# IAM role is attached automatically + +# For local development +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json +# or +aws configure +``` + +## Next Steps + +- Set up [CI/CD for automated training](../configuration.md) +- Review [troubleshooting tips](../troubleshooting.md) +- Explore [game dev workflow](./game-dev.md) for multimedia assets diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..0757b2b --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,177 @@ +# Installation Guide + +This guide covers installing git-los on various platforms and configuring it for use with Git LFS. + +## Prerequisites + +Before installing git-los, ensure you have: + +- **Git** version 2.27 or later +- **Git LFS** version 2.9 or later +- Cloud credentials configured (see [Configuration](./configuration.md)) + +### Installing Git LFS + +If you don't have Git LFS installed: + +**macOS (Homebrew):** +```bash +brew install git-lfs +git lfs install +``` + +**Ubuntu/Debian:** +```bash +sudo apt-get install git-lfs +git lfs install +``` + +**Windows:** +Download from [git-lfs.com](https://git-lfs.com/) or use: +```powershell +winget install GitHub.GitLFS +git lfs install +``` + +## Installing git-los + +### From Binary Releases (Recommended) + +Download the latest release for your platform from [GitHub Releases](https://github.com/valent-au/git-los/releases). + +**macOS/Linux:** +```bash +# Download and extract (replace VERSION and PLATFORM) +curl -LO https://github.com/valent-au/git-los/releases/download/vVERSION/git-los_VERSION_PLATFORM.tar.gz +tar -xzf git-los_VERSION_PLATFORM.tar.gz + +# Move to a directory in your PATH +sudo mv git-los /usr/local/bin/ + +# Verify installation +git-los version +``` + +**Windows:** +```powershell +# Download from releases page and extract +# Move git-los.exe to a directory in your PATH + +# Verify installation +git-los version +``` + +### From Source + +Requires Go 1.21 or later: + +```bash +go install github.com/valent-au/git-los/cmd/git-los@latest +``` + +The binary will be installed to `$GOPATH/bin` (usually `~/go/bin`). Ensure this directory is in your PATH. + +### Using Homebrew (macOS) + +```bash +# Add the tap (one-time) +brew tap valent-au/tap + +# Install git-los +brew install git-los +``` + +## Global Installation + +After installing the binary, configure git to use git-los as the LFS transfer agent: + +```bash +# Install globally (recommended for most users) +git-los install --global +``` + +This command: +1. Registers git-los as a custom transfer agent +2. Configures git-lfs to use git-los for transfers +3. Sets up the daemon for credential caching + +To verify the installation: +```bash +git config --global --get lfs.standalonetransferagent +# Should output: git-los +``` + +## Per-Repository Installation + +If you prefer to configure git-los for specific repositories only: + +```bash +cd your-repo +git-los install +``` + +## Uninstalling + +To remove the git-los configuration: + +```bash +# Remove global configuration +git-los uninstall --global + +# Or remove from a specific repository +cd your-repo +git-los uninstall +``` + +To fully uninstall, also remove the binary: + +**macOS/Linux:** +```bash +sudo rm /usr/local/bin/git-los +``` + +**Homebrew:** +```bash +brew uninstall git-los +``` + +## Verifying Installation + +1. **Check git-los is accessible:** + ```bash + git-los version + ``` + +2. **Check git configuration:** + ```bash + git config --global lfs.standalonetransferagent + git config --global lfs.customtransfer.git-los.path + ``` + +3. **Test with a repository:** + ```bash + cd your-repo + git lfs env + # Should show git-los as the transfer agent + ``` + +## Updating + +### Binary Installation +Download the new release and replace the existing binary. + +### Go Install +```bash +go install github.com/valent-au/git-los/cmd/git-los@latest +``` + +### Homebrew +```bash +brew upgrade git-los +``` + +## Next Steps + +- [Configure your cloud backend](./configuration.md) +- [Set up your first repository](./examples/game-dev.md) +- [Review troubleshooting tips](./troubleshooting.md) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..d867bb5 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,363 @@ +# Troubleshooting Guide + +This guide helps you diagnose and resolve common issues with git-los. + +## Quick Diagnostics + +Run these commands to gather diagnostic information: + +```bash +# Check git-los version +git-los version + +# Check daemon status +git-los daemon status + +# Check git LFS configuration +git lfs env + +# Verify cloud credentials +gcloud auth application-default print-access-token # GCS +aws sts get-caller-identity # S3 +``` + +## Common Issues + +### Authentication Errors + +#### "401 Unauthorized" or "403 Forbidden" + +**Symptoms:** +- Push/pull fails with authentication errors +- Error message mentions unauthorized or forbidden access + +**Solutions:** + +**For GCS:** +```bash +# Re-authenticate with Google Cloud +gcloud auth application-default login + +# Or verify your service account key +echo $GOOGLE_APPLICATION_CREDENTIALS +cat $GOOGLE_APPLICATION_CREDENTIALS | jq .client_email +``` + +**For S3:** +```bash +# Verify credentials are configured +aws configure list + +# Test credentials +aws sts get-caller-identity + +# Check bucket access +aws s3 ls s3://your-bucket/lfs/ +``` + +**Common causes:** +- Expired credentials +- Missing IAM permissions +- Wrong bucket or project +- Service account not activated + +--- + +#### "credentials not found" + +**Symptoms:** +- Error mentions missing credentials +- Works in one environment but not another + +**Solutions:** + +**For GCS:** +```bash +# Set up application default credentials +gcloud auth application-default login + +# Or point to a service account key +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json +``` + +**For S3:** +```bash +# Run the configuration wizard +aws configure + +# Or set environment variables +export AWS_ACCESS_KEY_ID=your-key +export AWS_SECRET_ACCESS_KEY=your-secret +export AWS_REGION=your-region +``` + +--- + +### Network Errors + +#### "connection refused" or "timeout" + +**Symptoms:** +- Transfer fails after waiting +- Intermittent failures + +**Solutions:** + +1. **Check network connectivity:** + ```bash + # GCS + curl -I https://storage.googleapis.com + + # S3 + curl -I https://s3.amazonaws.com + ``` + +2. **Check proxy settings:** + ```bash + echo $HTTP_PROXY + echo $HTTPS_PROXY + ``` + +3. **Verify firewall allows outbound HTTPS (port 443)** + +4. **Retry the operation** - git-los automatically retries transient failures + +--- + +#### "DNS resolution failed" + +**Symptoms:** +- Cannot resolve storage service hostname + +**Solutions:** + +1. **Check DNS configuration:** + ```bash + nslookup storage.googleapis.com + nslookup s3.amazonaws.com + ``` + +2. **Try a different DNS server:** + ```bash + # Temporary test with Google DNS + nslookup storage.googleapis.com 8.8.8.8 + ``` + +--- + +### Transfer Errors + +#### "checksum mismatch" + +**Symptoms:** +- Download completes but verification fails +- Error mentions checksum or hash mismatch + +**Solutions:** + +1. **Retry the download** - may be corrupted in transit + +2. **Check if the object exists and is complete:** + ```bash + # GCS + gsutil stat gs://bucket/lfs/objects/sha256/... + + # S3 + aws s3api head-object --bucket bucket --key lfs/objects/sha256/... + ``` + +3. **Re-upload the object** from a working copy + +--- + +#### "object not found" (404) + +**Symptoms:** +- Download fails with "not found" error +- Object was expected to exist + +**Solutions:** + +1. **Verify the LFS URL is correct:** + ```bash + git config lfs.url + ``` + +2. **Check if object exists in the bucket:** + ```bash + # GCS + gsutil ls gs://bucket/lfs/objects/sha256/ab/cd/... + + # S3 + aws s3 ls s3://bucket/lfs/objects/sha256/ab/cd/... + ``` + +3. **Push the missing object:** + ```bash + git lfs push --all origin + ``` + +--- + +### Daemon Issues + +#### "cannot connect to daemon" + +**Symptoms:** +- Operations fail immediately +- Error mentions socket connection + +**Solutions:** + +1. **Check if daemon is running:** + ```bash + git-los daemon status + ``` + +2. **Start the daemon:** + ```bash + git-los daemon start + ``` + +3. **Check socket path:** + ```bash + ls -la ~/.cache/git-los/daemon.sock + # or + ls -la $XDG_RUNTIME_DIR/git-los/daemon.sock + ``` + +4. **Clean up stale socket and restart:** + ```bash + rm ~/.cache/git-los/daemon.sock + git-los daemon start + ``` + +--- + +#### "daemon not responding" + +**Symptoms:** +- Daemon appears running but operations hang +- Status shows daemon active but transfers timeout + +**Solutions:** + +1. **Stop and restart the daemon:** + ```bash + git-los daemon stop + git-los daemon start + ``` + +2. **Check daemon logs:** + ```bash + git-los daemon status + # Note the log file path, then: + tail -f /path/to/daemon.log + ``` + +3. **Run in foreground for debugging:** + ```bash + git-los daemon stop + git-los daemon start --foreground --log-level debug + ``` + +--- + +### Git LFS Integration Issues + +#### "git-los: command not found" + +**Symptoms:** +- Git LFS operations fail +- Error says git-los is not found + +**Solutions:** + +1. **Verify git-los is installed:** + ```bash + which git-los + git-los version + ``` + +2. **Ensure git-los is in PATH:** + ```bash + # Add to your shell profile (~/.bashrc, ~/.zshrc, etc.) + export PATH="$PATH:$HOME/go/bin" + ``` + +3. **Reconfigure git:** + ```bash + git-los install --global + ``` + +--- + +#### "not a valid LFS pointer" + +**Symptoms:** +- Files show as LFS pointers instead of content +- Git operations fail with pointer errors + +**Solutions:** + +1. **Verify LFS is tracking the files:** + ```bash + git lfs track + cat .gitattributes + ``` + +2. **Fetch LFS objects:** + ```bash + git lfs fetch --all + git lfs checkout + ``` + +3. **Check remote URL:** + ```bash + git config lfs.url + ``` + +--- + +## Debug Mode + +Enable debug logging for more information: + +```bash +# Set debug mode +export GIT_LOS_DEBUG=1 + +# Or use --log-level flag +git-los daemon start --foreground --log-level debug +``` + +## Getting Help + +If you're still having issues: + +1. **Check existing issues:** [GitHub Issues](https://github.com/valent-au/git-los/issues) + +2. **Gather diagnostic info:** + ```bash + git-los version + git lfs env + git-los daemon status + ``` + +3. **Create a new issue** with: + - Description of the problem + - Error messages (with sensitive info redacted) + - Steps to reproduce + - Diagnostic information + +## Common Error Messages Reference + +| Error | Likely Cause | Solution | +|-------|--------------|----------| +| `401 Unauthorized` | Invalid or expired credentials | Re-authenticate | +| `403 Forbidden` | Missing permissions | Check IAM roles/policies | +| `404 Not Found` | Object doesn't exist | Push missing objects | +| `connection refused` | Network issue or daemon not running | Check network, start daemon | +| `checksum mismatch` | Corrupted transfer | Retry download | +| `socket permission denied` | Wrong socket permissions | Delete socket, restart daemon | +| `timeout` | Slow network or large file | Retry, check network |