From 2c0f6797c1163d5631f32266b8a734845a931720 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Fri, 10 Apr 2026 13:49:20 +0200 Subject: [PATCH] add FCOS + k3s single-node deployment for Hetzner butane config provisions FCOS with k3s, nftables firewall, sysctl/ulimit tuning, and a first-boot chain that clones the repo and builds the kuri image. k3s manifests handle yugabyte statefulset, kuri deployment with init-container wallet guard, and secretGenerator-based secrets management. tested on EPYC 7502P / 256GB / 3.84TB NVMe with 1 TiB mixed workload. --- .github/workflows/build-image.yml | 43 +++ .gitignore | 1 + deploy/hetzner-test/QUICKSTART.md | 129 +++++++++ deploy/hetzner-test/config.bu | 303 ++++++++++++++++++++ deploy/hetzner-test/install.sh | 33 +++ deploy/hetzner-test/k3s/configmap.yaml | 32 +++ deploy/hetzner-test/k3s/kuri.yaml | 200 +++++++++++++ deploy/hetzner-test/k3s/kustomization.yaml | 32 +++ deploy/hetzner-test/k3s/namespace.yaml | 4 + deploy/hetzner-test/k3s/secrets.env.example | 12 + deploy/hetzner-test/k3s/secrets.yaml | 16 ++ deploy/hetzner-test/k3s/yugabyte.yaml | 113 ++++++++ 12 files changed, 918 insertions(+) create mode 100644 .github/workflows/build-image.yml create mode 100644 deploy/hetzner-test/QUICKSTART.md create mode 100644 deploy/hetzner-test/config.bu create mode 100755 deploy/hetzner-test/install.sh create mode 100644 deploy/hetzner-test/k3s/configmap.yaml create mode 100644 deploy/hetzner-test/k3s/kuri.yaml create mode 100644 deploy/hetzner-test/k3s/kustomization.yaml create mode 100644 deploy/hetzner-test/k3s/namespace.yaml create mode 100644 deploy/hetzner-test/k3s/secrets.env.example create mode 100644 deploy/hetzner-test/k3s/secrets.yaml create mode 100644 deploy/hetzner-test/k3s/yugabyte.yaml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000..ca97b7b --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,43 @@ +name: Build container image + +on: + push: + branches: [main, deploy/*] + pull_request: + branches: [main] + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: docker/login-action@v3 + if: github.event_name == 'push' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha,prefix= + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ github.event_name == 'push' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 diff --git a/.gitignore b/.gitignore index e00fe5a..46fb608 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ gwcfg kuri settings.env .env +secrets.env *~ .idea/ diff --git a/deploy/hetzner-test/QUICKSTART.md b/deploy/hetzner-test/QUICKSTART.md new file mode 100644 index 0000000..dd2b6c3 --- /dev/null +++ b/deploy/hetzner-test/QUICKSTART.md @@ -0,0 +1,129 @@ +# filecoin-gateway on Hetzner (FCOS + k3s) + +## provision + +```bash +scp deploy/hetzner-test/{config.bu,install.sh} root@RESCUE_IP:/tmp/ +ssh root@RESCUE_IP + +curl -sLO https://github.com/coreos/butane/releases/latest/download/butane-x86_64-unknown-linux-gnu +mv butane-x86_64-unknown-linux-gnu /usr/local/bin/butane && chmod +x /usr/local/bin/butane + +export SSH_PUBKEY="ssh-ed25519 AAAA... you@host" +envsubst '$SSH_PUBKEY' < /tmp/config.bu | butane --strict > /tmp/config.ign +bash /tmp/install.sh /tmp/config.ign +reboot +``` + +## first boot + +FCOS boots, layers packages (tmux, btop, rclone, bc), reboots once, +then installs k3s, clones repo, builds image, imports into k3s containerd. + +```bash +ssh core@SERVER_IP + +# check first-boot services (may need to wait for the package reboot) +journalctl -u rpm-ostree-overlay.service +journalctl -u install-k3s.service +journalctl -u setup-fgw.service +kubectl get nodes +``` + +## configure + +```bash +# create wallet + CIDGravity account + staging config +podman run --rm -it \ + -v /var/mnt/fgw/config:/config:Z \ + -v /var/mnt/fgw/wallet:/root/.ribswallet:Z \ + localhost/fgw:local ./gwcfg -f /config/settings.env + +# create wallet secret from generated keys +kubectl create secret generic wallet-keys \ + --from-file=/var/mnt/fgw/wallet/ \ + -n filecoin-gateway + +# create secrets.env from template +cd /opt/filecoin-gateway +cp deploy/hetzner-test/k3s/secrets.env.example deploy/hetzner-test/k3s/secrets.env +# edit with CIDGravity token from gwcfg output +$EDITOR deploy/hetzner-test/k3s/secrets.env + +# edit configmap if needed (EXTERNAL_LOCALWEB_URL, fallback providers, etc) +$EDITOR deploy/hetzner-test/k3s/configmap.yaml +``` + +## deploy + +```bash +kubectl apply -k deploy/hetzner-test/k3s/ + +kubectl -n filecoin-gateway get pods -w +kubectl -n filecoin-gateway logs -f deployment/kuri +``` + +## DNS (required for deal-making) + +kuri's builtin autocert provisions a Let's Encrypt cert on port 443, +which SPs use to pull CAR data. point a DNS A record at the server IP +with no proxy (no cloudflare orange cloud): + +``` +fgw-test.yourdomain.com → A → SERVER_IP +``` + +set `EXTERNAL_LOCALWEB_URL=https://fgw-test.yourdomain.com` in the configmap. + +## test + +```bash +# S3 is accessible via localhost port-forward +kubectl -n filecoin-gateway port-forward deployment/kuri 8078:8078 & + +curl -s -o /dev/null -w '%{http_code}' http://localhost:8078/healthz + +echo "hello" | rclone rcat fgw:test/hello.txt +rclone cat fgw:test/hello.txt + +# upload test workload +./test-workload-gen.sh /var/mnt/testdata/upload +rclone copy /var/mnt/testdata/upload fgw:testbucket/ --transfers 8 --progress +``` + +## operations + +```bash +# logs +kubectl -n filecoin-gateway logs deployment/kuri --tail=50 +kubectl -n filecoin-gateway logs statefulset/yugabyte --tail=50 + +# dashboard (ssh tunnel) +ssh -L 9010:localhost:9010 core@SERVER_IP +# then http://localhost:9010 + +# restart kuri +kubectl -n filecoin-gateway rollout restart deployment/kuri + +# update config +$EDITOR deploy/hetzner-test/k3s/configmap.yaml +kubectl apply -k deploy/hetzner-test/k3s/ +kubectl -n filecoin-gateway rollout restart deployment/kuri + +# update image (when maintainers ship a release, pull from ghcr instead) +cd /opt/filecoin-gateway && git pull +podman build . -t fgw:local +podman save localhost/fgw:local | sudo k3s ctr images import - +kubectl -n filecoin-gateway rollout restart deployment/kuri +``` + +## storage / redundancy + +this deployment uses k3s `local-path` provisioner -- PVCs are +directories on the host NVMe with no replication or snapshots. + +**storage redundancy is the operator's responsibility.** consider: +- RAID1 for OS disk, separate data disk(s) for block groups +- external block storage with snapshots (e.g. hetzner volumes) +- YugabyteDB is single-replica here; production should be RF=3 + diff --git a/deploy/hetzner-test/config.bu b/deploy/hetzner-test/config.bu new file mode 100644 index 0000000..8d19b9b --- /dev/null +++ b/deploy/hetzner-test/config.bu @@ -0,0 +1,303 @@ +variant: fcos +version: "1.6.0" + +# filecoin-gateway -- FCOS + k3s single-node deployment +# target: Hetzner dedicated, tested on EPYC 7502P, 256GB RAM, 1x 3.84TB NVMe +# +# prerequisites: +# - butane (https://coreos.github.io/butane/) +# - envsubst (part of gettext, usually pre-installed) +# +# install from hetzner rescue: +# export SSH_PUBKEY="ssh-ed25519 AAAA... you@host" +# envsubst '$SSH_PUBKEY' < config.bu | butane --strict > config.ign +# ./install.sh config.ign +# reboot +# +# single-drive layout -- everything on the OS NVMe: +# /var/mnt/fgw gateway data (YugabyteDB + block groups) +# /var/mnt/testdata test workload (source + downloads + results) +# +# storage redundancy is the operator's responsibility. this config +# assumes a single NVMe with no RAID. for production, consider: +# - RAID1 for OS, separate data disk(s) for block groups +# - or external block storage with snapshots (e.g. hetzner volumes) +# - YugabyteDB in single-replica mode has no WAL redundancy +# +# FCOS root is ~3.5 GB; the rest of the drive is available via +# /var which lives on the same partition. we use /var/mnt/ subdirs +# to keep data organized. +# +# post-boot: +# ssh core@ +# kubectl get nodes # should show Ready + +passwd: + users: + - name: core + ssh_authorized_keys: + - "${SSH_PUBKEY}" + +storage: + # no extra disks to partition -- single NVMe, FCOS manages root + # /var lives on the root partition with all available space + + directories: + # gateway data -- root-owned, containers run as root internally + - path: /var/mnt/fgw + mode: 0755 + - path: /var/mnt/fgw/config + mode: 0755 + - path: /var/mnt/fgw/wallet + mode: 0755 + - path: /var/mnt/fgw/fgw + mode: 0755 + - path: /var/mnt/fgw/ipfs + mode: 0755 + - path: /var/mnt/fgw/yb + mode: 0755 + # test data -- core-owned, generated by core user + - path: /var/mnt/testdata + mode: 0755 + user: + name: core + group: + name: core + # repo -- core-owned, git operations by core user + - path: /opt/filecoin-gateway + mode: 0755 + user: + name: core + group: + name: core + + files: + # disable zincati auto-updates during testing + - path: /etc/zincati/config.d/90-disable-auto-updates.toml + contents: + inline: | + [updates] + enabled = false + + # raise file descriptor limits (F13: YB needs high nofile) + - path: /etc/security/limits.d/90-nofile.conf + contents: + inline: | + * soft nofile 1048576 + * hard nofile 1048576 + + - path: /etc/sysctl.d/90-fgw.conf + contents: + inline: | + fs.file-max = 2097152 + fs.nr_open = 1048576 + net.core.somaxconn = 65535 + + - path: /var/home/core/.bashrc + append: + - inline: | + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + + - path: /etc/sysconfig/nftables-fgw.nft + contents: + inline: | + table inet filter { + chain input { + type filter hook input priority 0; policy drop; + ct state established,related accept + iif lo accept + meta l4proto icmp accept + meta l4proto ipv6-icmp accept + tcp dport 22 accept + # localweb TLS (SPs fetch CARs) + tcp dport 443 accept + # k3s / podman / CNI bridge traffic + ip saddr 10.42.0.0/16 accept + ip saddr 10.43.0.0/16 accept + ip saddr 10.88.0.0/16 accept + ip saddr 172.16.0.0/12 accept + iifname "cni*" accept + iifname "flannel*" accept + iifname "podman*" accept + iifname "veth*" accept + # k3s API (localhost only) + tcp dport 6443 ip saddr 127.0.0.0/8 accept + } + chain forward { + type filter hook forward priority 0; policy accept; + } + chain output { + type filter hook output priority 0; policy accept; + } + } + + - path: /etc/sysconfig/nftables.conf + overwrite: true + contents: + inline: | + include "/etc/sysconfig/nftables-fgw.nft" + + - path: /usr/local/bin/install-k3s.sh + mode: 0755 + contents: + inline: | + #!/bin/bash + set -euo pipefail + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 --disable traefik" sh - + systemctl daemon-reload + systemctl start k3s + until [ -f /etc/rancher/k3s/k3s.yaml ]; do sleep 2; done + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + until /usr/local/bin/k3s kubectl get nodes 2>/dev/null | grep -q ' Ready'; do sleep 2; done + /usr/local/bin/k3s kubectl create namespace filecoin-gateway --dry-run=client -o yaml | \ + /usr/local/bin/k3s kubectl apply -f - + + # rclone config for testing + - path: /var/home/core/.config/rclone/rclone.conf + mode: 0644 + user: + name: core + group: + name: core + contents: + inline: | + [fgw] + type = s3 + provider = Other + endpoint = http://127.0.0.1:8078 + acl = private + no_check_bucket = true + list_version = 2 + + # setup script -- runs after first boot + - path: /usr/local/bin/setup-fgw.sh + mode: 0755 + contents: + inline: | + #!/bin/bash + set -euo pipefail + + REPO=/opt/filecoin-gateway + DATA=/var/mnt/fgw + + # podman is available immediately on FCOS (no socket needed) + + # clone repo (into existing dir that has .env from ignition) + cd "$REPO" + if [ ! -d ".git" ]; then + git init + git remote add origin https://github.com/CIDgravity/filecoin-gateway.git + fi + git fetch origin main + git checkout -f main 2>/dev/null || git checkout -b main origin/main + + # build image + if ! podman image exists localhost/fgw:local 2>/dev/null; then + podman build . -t fgw:local + fi + + # create data dirs + mkdir -p "$DATA"/{fgw,wallet,ipfs,yb,config} + + # stamp completion + touch "$REPO/.setup-complete" + + # import image into k3s containerd + podman save localhost/fgw:local | sudo k3s ctr images import - + + echo "setup complete. next steps:" + echo " # run gwcfg to create wallet + config:" + echo " podman run --rm -it -v $DATA/config:/config:Z -v $DATA/wallet:/root/.ribswallet:Z localhost/fgw:local ./gwcfg -f /config/settings.env" + echo " # create wallet secret:" + echo " kubectl create secret generic wallet-keys --from-file=$DATA/wallet/ -n filecoin-gateway" + echo " # create secrets.env from template:" + echo " cp $REPO/deploy/hetzner-test/k3s/secrets.env.example $REPO/deploy/hetzner-test/k3s/secrets.env" + echo " \$EDITOR $REPO/deploy/hetzner-test/k3s/secrets.env" + echo " # deploy:" + echo " kubectl apply -k $REPO/deploy/hetzner-test/k3s/" + +systemd: + units: + # nftables firewall + - name: nftables.service + enabled: true + + - name: firewalld.service + mask: true + + - name: k3s.service + dropins: + - name: 90-nofile.conf + contents: | + [Service] + LimitNOFILE=1048576 + + # layer diagnostic/operational packages on first boot. + # requires a reboot to take effect (no --apply-live; that flag + # is best-effort per FCOS docs and unreliable across updates). + # the install-k3s and setup-fgw units will run after reboot. + - name: rpm-ostree-overlay.service + enabled: true + contents: | + [Unit] + Description=Layer packages via rpm-ostree + After=network-online.target + Wants=network-online.target + ConditionPathExists=!/var/lib/.rpm-ostree-overlay-done + + [Service] + Type=oneshot + ExecStart=/usr/bin/rpm-ostree install --allow-inactive tmux btop rclone bc + ExecStartPost=/usr/bin/touch /var/lib/.rpm-ostree-overlay-done + ExecStartPost=/usr/bin/systemctl reboot + RemainAfterExit=true + + [Install] + WantedBy=multi-user.target + + # install k3s on first boot + - name: install-k3s.service + enabled: true + contents: | + [Unit] + Description=Install k3s + After=network-online.target + Wants=network-online.target + ConditionPathExists=!/usr/local/bin/k3s + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/install-k3s.sh + RemainAfterExit=true + + [Install] + WantedBy=multi-user.target + + # one-shot setup after boot + # note: no SELinux relabel service needed -- podman :Z volume + # flag handles relabeling automatically + - name: setup-fgw.service + enabled: true + contents: | + [Unit] + Description=Setup filecoin-gateway + After=network-online.target install-k3s.service + Wants=network-online.target + Requires=install-k3s.service + ConditionPathExists=!/opt/filecoin-gateway/.setup-complete + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/setup-fgw.sh + RemainAfterExit=true + User=core + Group=core + # no supplementary groups needed -- podman is rootless + Environment=PATH=/usr/local/bin:/usr/bin:/bin + + [Install] + WantedBy=multi-user.target + + # NM wait-online times out on Hetzner BMC USB NIC + - name: NetworkManager-wait-online.service + enabled: false diff --git a/deploy/hetzner-test/install.sh b/deploy/hetzner-test/install.sh new file mode 100755 index 0000000..3ce458c --- /dev/null +++ b/deploy/hetzner-test/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# install FCOS on hetzner baremetal from rescue mode +set -euo pipefail + +OS_DISK=$(ls /dev/disk/by-path/*-nvme-1 2>/dev/null | head -1) +[ -n "$OS_DISK" ] || { echo "error: no NVMe found in /dev/disk/by-path/"; exit 1; } +IGN_FILE="${1:?usage: $0 }" +[ -f "$IGN_FILE" ] || { echo "error: $IGN_FILE not found"; exit 1; } + +REAL_DISK=$(readlink -f "$OS_DISK") +echo "target: $OS_DISK -> $REAL_DISK" +lsblk "$REAL_DISK" + +read -p "this will ERASE $REAL_DISK. continue? [y/N] " -n 1 -r +echo +[[ $REPLY =~ ^[Yy]$ ]] || exit 1 + +if ! command -v podman &>/dev/null; then + echo "installing podman..." + apt-get update && apt-get install -y podman +fi + +echo "installing FCOS..." +podman run --pull=always --privileged --rm --pid=host \ + --network=host \ + -v /dev:/dev -v /run/udev:/run/udev \ + -v "$(dirname "$(readlink -f "$IGN_FILE")"):/data" \ + quay.io/coreos/coreos-installer:release install \ + "$OS_DISK" \ + --ignition-file "/data/$(basename "$IGN_FILE")" \ + --stream stable + +echo "done. run 'reboot' to boot into FCOS" diff --git a/deploy/hetzner-test/k3s/configmap.yaml b/deploy/hetzner-test/k3s/configmap.yaml new file mode 100644 index 0000000..38ee7f8 --- /dev/null +++ b/deploy/hetzner-test/k3s/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fgw-config + namespace: filecoin-gateway +data: + settings.env: | + RIBS_YUGABYTE_SQL_HOST=yugabyte.filecoin-gateway.svc.cluster.local + RIBS_YUGABYTE_SQL_PORT=5433 + RIBS_YUGABYTE_SQL_DB=filecoingw + RIBS_YUGABYTE_CQL_HOSTS=yugabyte.filecoin-gateway.svc.cluster.local + RIBS_YUGABYTE_CQL_PORT=9042 + RIBS_YUGABYTE_CQL_KEYSPACE=filecoingw + RIBS_S3API_BIND_ADDR=:8078 + RIBS_S3API_REGION=EU + EXTERNAL_LOCALWEB_BUILTIN_SERVER=true + EXTERNAL_LOCALWEB_SERVER_PORT=443 + EXTERNAL_LOCALWEB_SERVER_TLS=true + EXTERNAL_LOCALWEB_PATH=/root/.ribsdata/cardata + EXTERNAL_LOCALWEB_URL=https://fgw.example.com + RIBS_PROMETHEUS_PORT=2112 + # deal-making: adjust to taste + RIBS_MINIMUM_REPLICA_COUNT=5 + RIBS_MAXIMUM_REPLICA_COUNT=10 + RIBS_MINIMUM_RETRIEVABLE_COUNT=3 + RIBS_RETRIEVABLE_REPAIR_THRESHOLD=2 + RIBS_RUN_SP_CRAWLER=true + RIBS_ENABLE_PARALLEL_WRITES=false + # F10: if CIDGravity free-tier GBAP returns non-200, uncomment + # these to use fallback providers directly: + # RIBS_DEAL_FALLBACK_PROVIDERS=f02620,f01249 + # RIBS_DEAL_FALLBACK_PROVIDERS_ONLY=true diff --git a/deploy/hetzner-test/k3s/kuri.yaml b/deploy/hetzner-test/k3s/kuri.yaml new file mode 100644 index 0000000..959fdef --- /dev/null +++ b/deploy/hetzner-test/k3s/kuri.yaml @@ -0,0 +1,200 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fgw-data + namespace: filecoin-gateway +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 2Ti +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ipfs-data + namespace: filecoin-gateway +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: kuri-s3 + namespace: filecoin-gateway +spec: + selector: + app: kuri + ports: + - name: s3 + port: 8078 + targetPort: 8078 +--- +apiVersion: v1 +kind: Service +metadata: + name: kuri-metrics + namespace: filecoin-gateway +spec: + selector: + app: kuri + ports: + - name: metrics + port: 2112 + targetPort: 2112 +--- +apiVersion: v1 +kind: Service +metadata: + name: kuri-webui + namespace: filecoin-gateway +spec: + selector: + app: kuri + ports: + - name: webui + port: 9010 + targetPort: 9010 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kuri + namespace: filecoin-gateway +spec: + replicas: 1 + strategy: + type: Recreate # single writer, can't run two instances + selector: + matchLabels: + app: kuri + template: + metadata: + labels: + app: kuri + spec: + initContainers: + # source settings.env + secrets into a merged env file + # that kuri can load. this avoids the env_file format issues + # and keeps secrets out of the configmap. + - name: merge-config + image: docker.io/library/busybox:latest + command: + - sh + - -c + - | + # merge configmap + secrets into a single env file + cp /config-ro/settings.env /config-rw/settings.env + echo "" >> /config-rw/settings.env + echo "CIDGRAVITY_API_TOKEN=$(cat /secrets/CIDGRAVITY_API_TOKEN)" >> /config-rw/settings.env + echo "RIBS_YUGABYTE_SQL_USER=$(cat /secrets/RIBS_YUGABYTE_SQL_USER)" >> /config-rw/settings.env + echo "RIBS_YUGABYTE_SQL_PASS=$(cat /secrets/RIBS_YUGABYTE_SQL_PASS)" >> /config-rw/settings.env + + # copy wallet keys to regular dir (k8s secret symlinks have + # 0777 which lotus rejects -- must be 0600) + mkdir -p /wallet-rw + found=0 + for f in /wallet-ro/*; do + [ -f "$f" ] || continue + cp "$f" "/wallet-rw/$(basename $f)" + found=1 + done + + # F23 guard: refuse to start with empty wallet. + # kuri silently regenerates a new (unfunded) wallet on + # empty dir, which leads to silent state divergence. + if [ "$found" -eq 0 ]; then + echo "FATAL: wallet-keys secret is empty. refusing to start." + echo "create it with: kubectl create secret generic wallet-keys --from-file=/path/to/wallet/ -n filecoin-gateway" + exit 1 + fi + chmod 0600 /wallet-rw/* + chmod 0700 /wallet-rw + volumeMounts: + - name: config-ro + mountPath: /config-ro + - name: config-rw + mountPath: /config-rw + - name: secrets + mountPath: /secrets + - name: wallet-keys + mountPath: /wallet-ro + - name: wallet-rw + mountPath: /wallet-rw + containers: + - name: kuri + image: localhost/fgw:local + command: + - sh + - -c + - | + set -a + . /app/config/settings.env + set +a + ./kuri init + exec ./kuri daemon + ports: + - containerPort: 8078 + name: s3 + - containerPort: 443 + hostPort: 443 + name: localweb + - containerPort: 2112 + name: metrics + - containerPort: 9010 + name: webui + volumeMounts: + - name: fgw-data + mountPath: /root/.ribsdata + - name: wallet-rw + mountPath: /root/.ribswallet + - name: ipfs-data + mountPath: /root/.ipfs + - name: config-rw + mountPath: /app/config + readOnly: true + resources: + requests: + cpu: "4" + memory: 4Gi + limits: + cpu: "32" + memory: 64Gi + readinessProbe: + httpGet: + path: /healthz + port: 8078 + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 8078 + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 5 + volumes: + - name: fgw-data + persistentVolumeClaim: + claimName: fgw-data + - name: ipfs-data + persistentVolumeClaim: + claimName: ipfs-data + - name: wallet-keys + secret: + secretName: wallet-keys + - name: config-ro + configMap: + name: fgw-config + - name: config-rw + emptyDir: {} + - name: wallet-rw + emptyDir: {} + - name: secrets + secret: + secretName: fgw-secrets diff --git a/deploy/hetzner-test/k3s/kustomization.yaml b/deploy/hetzner-test/k3s/kustomization.yaml new file mode 100644 index 0000000..637aa1f --- /dev/null +++ b/deploy/hetzner-test/k3s/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: filecoin-gateway + +resources: + - namespace.yaml + - configmap.yaml + - yugabyte.yaml + - kuri.yaml + +# fgw-secrets is generated from a local .env file that is NOT checked +# into git. create it before running kustomize: +# +# cp secrets.env.example secrets.env +# $EDITOR secrets.env # fill in real values +# +# kustomize will create the secret on apply and update it on re-apply, +# but the name is kept stable (no hash suffix) so kuri doesn't need +# a rollout on every apply. +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: + - name: fgw-secrets + envs: + - secrets.env + +# wallet-keys secret MUST be created out-of-band -- it contains +# binary key files, not env vars. see secrets.env.example for +# the kubectl create command. the init container will refuse to +# start if wallet-keys is empty (F23 guard). diff --git a/deploy/hetzner-test/k3s/namespace.yaml b/deploy/hetzner-test/k3s/namespace.yaml new file mode 100644 index 0000000..52715c8 --- /dev/null +++ b/deploy/hetzner-test/k3s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: filecoin-gateway diff --git a/deploy/hetzner-test/k3s/secrets.env.example b/deploy/hetzner-test/k3s/secrets.env.example new file mode 100644 index 0000000..7cba8d5 --- /dev/null +++ b/deploy/hetzner-test/k3s/secrets.env.example @@ -0,0 +1,12 @@ +# copy to secrets.env and fill in real values. +# secrets.env is consumed by kustomize secretGenerator and must NOT +# be committed to git. +# +# wallet keys are binary and must be created separately: +# kubectl create secret generic wallet-keys \ +# --from-file=/var/mnt/fgw/wallet/ \ +# -n filecoin-gateway + +CIDGRAVITY_API_TOKEN=CHANGE_ME +RIBS_YUGABYTE_SQL_USER=yugabyte +RIBS_YUGABYTE_SQL_PASS=yugabyte diff --git a/deploy/hetzner-test/k3s/secrets.yaml b/deploy/hetzner-test/k3s/secrets.yaml new file mode 100644 index 0000000..446e95f --- /dev/null +++ b/deploy/hetzner-test/k3s/secrets.yaml @@ -0,0 +1,16 @@ +# THIS FILE IS NOT USED BY KUSTOMIZE. +# +# secrets are managed two ways: +# +# 1. fgw-secrets (env vars): generated by kustomize from secrets.env +# cp secrets.env.example secrets.env && $EDITOR secrets.env +# +# 2. wallet-keys (binary key files): created out-of-band +# kubectl create secret generic wallet-keys \ +# --from-file=/var/mnt/fgw/wallet/ \ +# -n filecoin-gateway +# +# WARNING: do not add this file or a generic Secret template to +# kustomization.yaml resources. applying a Secret with `data: {}` +# will silently overwrite real secrets, causing kuri to regenerate +# the wallet with a new (unfunded) address (F23). diff --git a/deploy/hetzner-test/k3s/yugabyte.yaml b/deploy/hetzner-test/k3s/yugabyte.yaml new file mode 100644 index 0000000..3121d77 --- /dev/null +++ b/deploy/hetzner-test/k3s/yugabyte.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: yb-data + namespace: filecoin-gateway +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 100Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: yugabyte + namespace: filecoin-gateway +spec: + clusterIP: # regular service, always resolvable + selector: + app: yugabyte + ports: + - name: ysql + port: 5433 + targetPort: 5433 + - name: ycql + port: 9042 + targetPort: 9042 + - name: master-web + port: 7000 + targetPort: 7000 + - name: tserver-web + port: 9000 + targetPort: 9000 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: yugabyte + namespace: filecoin-gateway +spec: + serviceName: yugabyte + replicas: 1 + selector: + matchLabels: + app: yugabyte + template: + metadata: + labels: + app: yugabyte + spec: + containers: + - name: yugabyte + image: docker.io/yugabytedb/yugabyte:2024.2.5.0-b59 + command: + - bin/yugabyted + - start + - --background=false + - --advertise_address=$(POD_IP) + - --tserver_flags=rocksdb_max_background_compactions=8,rocksdb_max_background_flushes=4,db_block_cache_size_bytes=536870912,tablet_server_svc_queue_length=10000 + - --master_flags=default_memory_limit_to_ram_ratio=0.25 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: YSQL_DB + value: filecoingw + - name: YCQL_KEYSPACE + value: filecoingw + ports: + - containerPort: 5433 + name: ysql + - containerPort: 9042 + name: ycql + - containerPort: 7000 + name: master-web + - containerPort: 9000 + name: tserver-web + volumeMounts: + - name: yb-data + mountPath: /root/var + resources: + requests: + cpu: "2" + memory: 2Gi + limits: + cpu: "16" + memory: 8Gi + readinessProbe: + exec: + command: + - bash + - -c + - > + bin/ysqlsh -h $(hostname -i) -U yugabyte -tAc 'select 1' -d yugabyte && + bin/ycqlsh $(hostname -i) -e 'describe keyspaces' + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 10 + livenessProbe: + exec: + command: + - bash + - -c + - bin/ysqlsh -h $(hostname -i) -U yugabyte -tAc 'select 1' -d yugabyte + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + volumes: + - name: yb-data + persistentVolumeClaim: + claimName: yb-data