From 20651a9c743088b1148cd5e47f97be7c3bbbe5ac Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Sun, 4 Oct 2026 18:27:42 -0700 Subject: [PATCH] chore(storage): remove democratic-csi leftovers Nothing on any cluster uses democratic-csi any more (no CSIDriver, StorageClass, PV, Helm release or namespace; the last orphaned Released PV on nprd was deleted). Drop the democratic_csi_* variables, the three deploy_democratic_csi_* null_resources (count 0, never in state), the democratic_csi_config output, the install/values-generation scripts and the setup doc. Point docs at TrueNAS CSI. --- CHANGELOG.md | 3 + README.md | 6 +- docs/ARCHITECTURE.md | 3 +- docs/CLOUDNATIVEPG_SETUP.md | 4 +- docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md | 1120 ------------------- docs/DEPLOYMENT_GUIDE.md | 2 +- docs/OPS_NOTES.md | 5 +- docs/README.md | 1 - docs/STORAGE_CLASS_DEFAULT.md | 93 +- docs/TROUBLESHOOTING.md | 244 +--- docs/TRUENAS_CSI_MIGRATION.md | 2 + docs/TRUENAS_CSI_MIGRATION_PRIORITY.md | 4 +- docs/TRUENAS_CSI_MULTI_NODE.md | 10 +- scripts/README.md | 19 - scripts/generate-helm-values-from-tfvars.sh | 213 ---- scripts/install-democratic-csi.sh | 195 ---- terraform/main.tf | 403 ------- terraform/outputs.tf | 15 - terraform/terraform.tfvars.example | 17 - terraform/variables.tf | 65 -- 20 files changed, 44 insertions(+), 2380 deletions(-) delete mode 100644 docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md delete mode 100755 scripts/generate-helm-values-from-tfvars.sh delete mode 100755 scripts/install-democratic-csi.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e2a30..32f7826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Docs no longer claim `~/.kube/.yaml` are RKE2 admin kubeconfigs (they were overwritten by Rancher-proxied ones); break-glass is `~/.kube/-rke2.yaml` - poc-apps wildcard certificate copy re-synced: gitops-core `cert-sync` uses direct RKE2 admin credentials for all clusters and fails loudly ([gitops-core#6](https://github.com/DataKnifeAI/gitops-core/pull/6)) +### Removed +- **democratic-csi**: `democratic_csi_*` variables, the `deploy_democratic_csi_{nprd,prd,poc}_apps` resources (never in state; count was 0), the `democratic_csi_config` output, `scripts/install-democratic-csi.sh`, `scripts/generate-helm-values-from-tfvars.sh` and `docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md`. Nothing on any cluster used the driver; TrueNAS CSI is the only storage driver. Remove the `democratic_csi_*` entries from your local `terraform.tfvars` + ### Known issues - Terraform Rancher API token expired; registration modules read a hard-coded `/home/lee/git/rancher-deploy/config/.rancher-api-token` - RKE2 leaf certs on all control-plane nodes expire 2027-01-08 (poc-apps 2027-01-15) — rotate per [docs/RKE2_CERT_ROTATION.md](docs/RKE2_CERT_ROTATION.md) diff --git a/README.md b/README.md index 82fc25e..6d7856c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Also automated when enabled in `terraform.tfvars`: - **cert-manager** + **Rancher** on the manager cluster - Downstream registration into Rancher -- **Democratic CSI** (TrueNAS NFS) and/or official **TrueNAS CSI** on app clusters +- Official **TrueNAS CSI** (NFS) on app clusters - **Envoy Gateway** + **kube-vip** LoadBalancer IPs on app clusters - Operators: CloudNativePG, MongoDB Community, OpenSearch, GitHub ARC - Optional **Palworld operator** (default: **prd-apps** only) @@ -92,8 +92,8 @@ Full walkthrough: [docs/DEPLOYMENT_GUIDE.md](docs/DEPLOYMENT_GUIDE.md). | SSH keys & recovery | [SSH_AND_ACCESS.md](docs/SSH_AND_ACCESS.md) | | Cluster access, SSO & break-glass | [CLUSTER_ACCESS_AND_SSO.md](docs/CLUSTER_ACCESS_AND_SSO.md) | | DNS | [DNS_CONFIGURATION.md](docs/DNS_CONFIGURATION.md) | -| TrueNAS / Democratic CSI | [DEMOCRATIC_CSI_TRUENAS_SETUP.md](docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md) | -| TrueNAS CSI migration | [TRUENAS_CSI_MIGRATION.md](docs/TRUENAS_CSI_MIGRATION.md) | +| TrueNAS CSI | [TRUENAS_CSI_MULTI_NODE.md](docs/TRUENAS_CSI_MULTI_NODE.md) | +| TrueNAS CSI migration (done) | [TRUENAS_CSI_MIGRATION.md](docs/TRUENAS_CSI_MIGRATION.md) | | Troubleshooting | [TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) | | Ops notes | [OPS_NOTES.md](docs/OPS_NOTES.md) | | Example homelab diagram | [examples/homelab/index.html](examples/homelab/index.html) | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4b50f0f..353dc4e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -50,9 +50,8 @@ Break-glass paths (Rancher local user, RKE2 admin kubeconfigs) bypass Authentik. - **VM disks**: Proxmox datastore from `clusters.*.storage` (example: `local-vm-zfs`). - **Persistent volumes** (apps clusters): - - **Democratic CSI** → storage class `truenas-nfs` (vars: `democratic_csi_*`) - **Official TrueNAS CSI** → `truenas-csi-nfs` (vars: `truenas_csi_*`; SCALE 25.10+) - - Migration between drivers: [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md) + - democratic-csi (`truenas-nfs`) was retired; migration history: [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md) For a visual of an example two-node Proxmox homelab (hosts, MD1420 shelves, bonds, Ceph/ZFS, TrueNAS CSI, guest clusters), see [../examples/homelab/index.html](../examples/homelab/index.html). diff --git a/docs/CLOUDNATIVEPG_SETUP.md b/docs/CLOUDNATIVEPG_SETUP.md index 7c5925f..ec286b4 100644 --- a/docs/CLOUDNATIVEPG_SETUP.md +++ b/docs/CLOUDNATIVEPG_SETUP.md @@ -491,13 +491,13 @@ kubectl logs -n cnpg-system deployment/cnpg-controller-manager ```bash # Verify storage class exists -kubectl get storageclass truenas-nfs +kubectl get storageclass truenas-csi-nfs # Check PVC binding kubectl describe pvc my-postgres-cluster-1 # Verify TrueNAS connectivity -kubectl get pods -n democratic-csi +kubectl get pods -n truenas-csi ``` ### Replication Issues diff --git a/docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md b/docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md deleted file mode 100644 index 553626f..0000000 --- a/docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md +++ /dev/null @@ -1,1120 +0,0 @@ -# Democratic CSI with TrueNAS and Rancher Setup Guide - -## Overview - -Democratic CSI is a unified Container Storage Interface (CSI) driver that supports multiple storage backends, including TrueNAS (both SCALE and CORE). This guide covers setting up democratic-csi with TrueNAS in your Rancher-managed RKE2 clusters. - -## Prerequisites - -### TrueNAS Requirements -- TrueNAS SCALE or TrueNAS CORE running and accessible -- NFS or iSCSI service enabled on TrueNAS -- A dataset/pool configured for Kubernetes storage -- User credentials with appropriate permissions -- Network connectivity from Kubernetes nodes to TrueNAS - -### Rancher/RKE2 Requirements -- RKE2 cluster managed by Rancher -- `kubectl` access to the cluster -- Helm 3.x installed (for installation) -- Sufficient permissions to create namespaces, service accounts, and cluster roles - -## Architecture - -``` -RKE2 Cluster -├── Democratic CSI Driver -│ ├── Controller Pod (manages volumes) -│ └── Node Pods (one per worker node) -├── Storage Classes -│ ├── NFS Storage Class -│ └── iSCSI Storage Class (optional) -└── TrueNAS Backend - ├── NFS Shares - └── iSCSI Targets -``` - -## Step 1: Prepare TrueNAS - -### 1.1 Create Dataset for Kubernetes - -1. **Log into TrueNAS Web UI** -2. **Create a dataset:** - - Storage → Pools → Add Dataset - - Name: `k8s-storage` (or your preferred name, e.g., `RKE2`) - - Type: Filesystem - - Share Type: Generic - - Enable compression (recommended: lz4) - - Enable deduplication (optional, requires more RAM) - -### 1.2 Configure User Permissions - -The TrueNAS user needs specific permissions to create and manage datasets via the API. - -#### Minimum Required Permissions - -**Quick Answer:** -- User with **write access** to dataset -- Ability to **create and delete** child datasets -- **API key** for that user -- **Network access** to TrueNAS API (port 443) - -#### Detailed Requirements - -**1. Dataset Permissions** - -The user needs to be able to: -- ✅ **Create** datasets under parent dataset (for PVCs) -- ✅ **Delete** datasets under parent dataset (when PVCs are deleted) -- ✅ **Read** dataset properties -- ✅ **Modify** dataset properties (optional, for quotas/compression) - -**How to Grant:** -```bash -# Option 1: Ownership (simplest and recommended) -ssh root@your-truenas-host -chown -R csi-user:csi-user /mnt/pool/dataset - -# Option 2: ACLs (more flexible, TrueNAS SCALE) -setfacl -R -m u:csi-user:rwx /mnt/pool/dataset -setfacl -R -d -m u:csi-user:rwx /mnt/pool/dataset -``` - -**2. Verify Permissions** - -Test if the user can create and delete datasets: -```bash -# SSH to TrueNAS -ssh root@your-truenas-host - -# Test if user can create datasets -sudo -u csi-user zfs create pool/dataset/test-permission-check -sudo -u csi-user zfs destroy pool/dataset/test-permission-check - -# If successful: ✅ User has sufficient permissions -# If fails: Grant permissions using chown or ACLs as shown above -``` - -**3. Permission Levels Comparison** - -| Level | Permissions | Limitations | Use Case | -|-------|-------------|-------------|----------| -| **Level 1: Minimum** | Dataset access only | Cannot create NFS shares via API | Basic functionality, manual NFS management | -| **Level 2: Recommended** | Dataset + NFS management | May require admin/root-level access | Full automated functionality | -| **Level 3: Production** | Dedicated user with minimal permissions | Requires careful setup | Production environments (best practice) | - -**Recommended for Production:** Use Level 3 - dedicated user with ownership of dataset only. - -#### API Access - -The user needs: -- ✅ **API key** created in TrueNAS (System → API Keys) -- ✅ **HTTPS access** to TrueNAS API (port 443) -- ✅ **Network connectivity** from RKE2 nodes to TrueNAS - -**API Endpoints Used:** -- `POST /api/v2.0/pool/dataset` - Create datasets -- `DELETE /api/v2.0/pool/dataset/id/{id}` - Delete datasets -- `GET /api/v2.0/pool/dataset/id/{id}` - Read dataset info -- `GET /api/v2.0/pool/dataset` - List datasets - -**Note:** API keys inherit the user's permissions, so if the user can create datasets, the API key will work. - -### 1.3 Create API Key - -1. **Login to TrueNAS UI**: `https://your-truenas-host` -2. **Navigate**: **System → API Keys → Add** -3. **User**: Select the user with dataset permissions (e.g., `csi-user` or `rke2`) -4. **Name**: `k8s-csi` (or your preferred name) -5. **Generate key** and **copy it immediately** (you won't see it again) -6. **Add to Terraform variables**: - ```hcl - # terraform/terraform.tfvars - truenas_api_key = "1-xxxxxxxxxxxxx" - ``` - -**Important Notes:** -- Use the IP address that NFS service is bound to (not just any IP) -- Verify NFS service binding: Services → NFS → Edit → Check "Bind IP Addresses" -- API Port: `443` (HTTPS) or `80` (HTTP) - -**Test API Access:** -```bash -API_KEY="your-api-key" -TRUENAS_HOST="your-truenas-host" - -# Test dataset query -curl -k -H "Authorization: Bearer ${API_KEY}" \ - "https://${TRUENAS_HOST}/api/v2.0/pool/dataset/id/SAS%2FRKE2" - -# Test dataset creation -curl -k -X POST \ - -H "Authorization: Bearer ${API_KEY}" \ - -H "Content-Type: application/json" \ - -d '{"name": "SAS/RKE2/test-api", "type": "FILESYSTEM"}' \ - "https://${TRUENAS_HOST}/api/v2.0/pool/dataset" - -# Clean up test dataset -curl -k -X DELETE \ - -H "Authorization: Bearer ${API_KEY}" \ - "https://${TRUENAS_HOST}/api/v2.0/pool/dataset/id/SAS%2FRKE2%2Ftest-api" -``` - -### 1.4 Configure NFS Share - -1. **Enable NFS Service:** - - Services → NFS → Enable - - **Important:** Check "Bind IP Addresses" configuration - - Ensure NFS service is bound to the IP address you'll use in CSI configuration - - If using multiple IPs, bind to all required IPs or use `0.0.0.0` for all interfaces - -2. **Create NFS Share:** - - Sharing → Unix Shares (NFS) → Add - - Path: `/mnt//k8s-storage` (or your dataset path) - - Description: `Kubernetes Storage` - - Enable: ✓ - - Network: `10.0.0.0/24` (your cluster subnet, e.g., `192.168.14.0/24`) - - Authorized Networks: `10.0.0.0/24` (your cluster subnet) - - Maproot User: `root` (or user with dataset ownership) - - Maproot Group: `wheel` - - Save - -3. **Verify NFS Service Binding:** - - **Critical:** Test NFS connectivity from a cluster node before configuring CSI - ```bash - # From a cluster node, test NFS port accessibility: - timeout 3 bash -c 'cat < /dev/null > /dev/tcp/192.168.9.10/2049' && echo 'Port 2049 accessible' || echo 'Port 2049 NOT accessible' - - # Test manual mount with default options: - sudo mkdir -p /tmp/test-nfs - sudo mount -t nfs 192.168.9.10:/mnt/pool/k8s-storage /tmp/test-nfs - ls -la /tmp/test-nfs - sudo umount /tmp/test-nfs - sudo rmdir /tmp/test-nfs - ``` - - **If mount fails with "Connection refused":** - - NFS service may not be bound to that IP address - - Check TrueNAS NFS service configuration - - Update bind IP addresses or use correct IP in CSI configuration - -## Step 2: Install Democratic CSI - -### 2.1 Automated Installation via Terraform (Recommended) - -Democratic CSI is **automatically deployed** at the end of your Terraform plan if TrueNAS is configured: - -1. **Configure Democratic CSI / TrueNAS in Terraform** (`terraform/terraform.tfvars`): - ```hcl - install_democratic_csi = true - democratic_csi_host = "truenas.example.com" - democratic_csi_api_key = "your-api-key-here" - democratic_csi_dataset = "/mnt/SAS/RKE2" - democratic_csi_user = "rke2" - democratic_csi_protocol = "https" - democratic_csi_port = 443 - democratic_csi_allow_insecure = false - democratic_csi_storage_class_name = "truenas-nfs" - democratic_csi_storage_class_default = true - ``` - - > Older docs used `truenas_*` / `csi_*` names. Current variables are `democratic_csi_*` (see `terraform/variables.tf`). For the official TrueNAS CSI driver, use `install_truenas_csi` / `truenas_csi_*` instead — see [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md). - -2. **Generate Helm values from Terraform variables**: - ```bash - ./scripts/generate-helm-values-from-tfvars.sh - ``` - -3. **Run Terraform**: - ```bash - cd terraform - terraform apply - ``` - - The storage class will be created automatically at the end of the deployment. - -**Resource Dependencies:** -``` -VMs → RKE2 Clusters → Rancher → Downstream Registration → Kubeconfigs → Democratic CSI -``` - -The democratic-csi resource depends on: -- `null_resource.merge_kubeconfigs` (kubeconfigs ready) -- `module.rke2_nprd_apps` (nprd-apps cluster ready) - -### 2.2 Secrets Management - -#### Architecture - -``` -terraform/terraform.tfvars (source of truth) - ↓ -scripts/generate-helm-values-from-tfvars.sh - ↓ -helm-values/democratic-csi-truenas.yaml (generated) - ↓ -Helm installation uses generated values -``` - -#### Workflow - -1. **Edit secrets** in `terraform/terraform.tfvars` -2. **Generate Helm values**: `./scripts/generate-helm-values-from-tfvars.sh` -3. **Install/Update**: Terraform will deploy automatically, or run Helm manually - -#### Security - -- ✅ `terraform/terraform.tfvars` - gitignored (contains secrets) -- ✅ `helm-values/democratic-csi-truenas.yaml` - gitignored (auto-generated) -- ✅ Example files tracked in git (no secrets) - -#### Terraform Variables - -All Democratic CSI variables are defined in `terraform/variables.tf`: - -- `install_democratic_csi` - Enable install on apps clusters -- `democratic_csi_host` - TrueNAS hostname -- `democratic_csi_api_key` - API key (sensitive) -- `democratic_csi_dataset` - Dataset path (e.g. `/mnt/SAS/RKE2`) -- `democratic_csi_user` - Username (reference) -- `democratic_csi_protocol` - Protocol (https/http) -- `democratic_csi_port` - API port -- `democratic_csi_allow_insecure` - Allow self-signed certs -- `democratic_csi_storage_class_name` - Storage class name (default `truenas-nfs`) -- `democratic_csi_storage_class_default` - Make it default - -#### Updating Configuration - -To update TrueNAS configuration: - -1. Edit `terraform/terraform.tfvars` -2. Run `./scripts/generate-helm-values-from-tfvars.sh` to regenerate Helm values -3. Run `terraform apply` (if using Terraform) or reinstall via Helm - -**Note:** The Helm values file is **auto-generated** from `terraform.tfvars`. Do not edit it manually - it will be overwritten. - -### 2.3 Manual Installation via Rancher UI - -If you prefer manual installation or need to install after Terraform: - -1. **Navigate to Cluster:** - - Cluster Management → nprd-apps → Explore Cluster - -2. **Install via Apps & Marketplace:** - - Apps & Marketplace → Charts - - Search for: `democratic-csi` - - Click: `democratic-csi` (from `democratic-csi` repository) - - Click: Install - -3. **Configure Installation:** - - Namespace: `democratic-csi` (create new) - - Release Name: `democratic-csi` - - Version: Latest stable (e.g., `v1.1.0`) - -4. **Configure Values:** - -1. **Navigate to Cluster:** - - Cluster Management → nprd-apps → Explore Cluster - -2. **Install via Apps & Marketplace:** - - Apps & Marketplace → Charts - - Search for: `democratic-csi` - - Click: `democratic-csi` (from `democratic-csi` repository) - - Click: Install - -3. **Configure Installation:** - - Namespace: `democratic-csi` (create new) - - Release Name: `democratic-csi` - - Version: Latest stable (e.g., `v1.1.0`) - -4. **Configure Values:** - -```yaml -# CSI Driver Configuration (required by chart) -csiDriver: - name: truenas-nfs # Must be unique per cluster - enabled: true - attachRequired: true - podInfoOnMount: true - -# Driver Configuration -driver: - config: - driver: freenas-api-nfs # Use freenas-api-nfs for TrueNAS API-based NFS - # HTTP connection to TrueNAS API - httpConnection: - protocol: "https" # or "http" - host: "truenas.example.com" # Your TrueNAS hostname/IP - port: 443 # or 80 for HTTP - apiKey: "your-api-key-here" # From Step 1.3 - allowInsecure: false # Set to true if using self-signed cert - # ZFS dataset configuration - zfs: - datasetParentName: "pool/k8s-storage" # ZFS dataset name (without /mnt/ prefix) - detachedSnapshotsDatasetParentName: "pool/k8s-storage-snapshots" - datasetEnableQuotas: true - datasetEnableReservation: false - datasetPermissionsMode: "0777" - datasetPermissionsUser: 0 - datasetPermissionsGroup: 0 - # NFS share configuration - # IMPORTANT: Use shareHost (not server) for freenas-api-nfs driver - nfs: - shareHost: "truenas.example.com" # NFS server hostname/IP - shareAlldirs: false - shareAllowedHosts: [] - shareAllowedNetworks: [] - shareMaprootUser: root - shareMaprootGroup: root - shareMapallUser: "" - shareMapallGroup: "" - -# Storage Classes -storageClasses: - - name: truenas-nfs - default: true - reclaimPolicy: Delete - volumeBindingMode: Immediate - allowVolumeExpansion: true - parameters: - fsType: "nfs" - parentDataset: "pool/k8s-storage" # ZFS dataset name (without /mnt/ prefix) - nfsServer: "truenas.example.com" # NFS server for volume context - nfsVersion: "4" # Recommended NFS version -``` - -5. **Click Install** - -### 2.4 Manual Installation via Helm CLI (Alternative) - -```bash -# Add democratic-csi Helm repository -helm repo add democratic-csi https://democratic-csi.github.io/charts/ -helm repo update - -# Set your kubeconfig -export KUBECONFIG=~/.kube/nprd-apps.yaml - -# Create namespace -kubectl create namespace democratic-csi - -# Install democratic-csi -helm install democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - --set csiDriver.name=truenas-nfs \ - --set csiDriver.enabled=true \ - --set driver.config.driver=truenas-nfs \ - --set config.truenas.host=truenas.example.com \ - --set config.truenas.apiKey=your-api-key \ - --set config.truenas.protocol=https \ - --set config.truenas.port=443 \ - --set config.truenas.dataset=/mnt/pool/k8s-storage \ - --set storageClasses[0].name=truenas-nfs \ - --set storageClasses[0].default=true -``` - -### 2.5 Manual Installation via Script - -If you prefer using the installation script: - -```bash -# Generate Helm values first -./scripts/generate-helm-values-from-tfvars.sh - -# Install via script -export KUBECONFIG=~/.kube/nprd-apps.yaml -./scripts/install-democratic-csi.sh -``` - -## Step 3: Verify Installation - -### 3.1 Check Pods - -```bash -kubectl get pods -n democratic-csi -``` - -Expected output: -``` -NAME READY STATUS RESTARTS AGE -democratic-csi-controller-0 2/2 Running 0 2m -democratic-csi-node-xxxxx 2/2 Running 0 2m -democratic-csi-node-yyyyy 2/2 Running 0 2m -democratic-csi-node-zzzzz 2/2 Running 0 2m -``` - -### 3.2 Check Storage Classes - -```bash -kubectl get storageclass -``` - -Expected output: -``` -NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE -truenas-nfs truenas-nfs.csi.k8s.io Delete Immediate true 2m -``` - -### 3.3 Check CSI Driver - -```bash -kubectl get csidriver -``` - -Expected output: -``` -NAME ATTACHREQUIRED PODINFOONMOUNT STORAGECAPACITY TOKENREQUESTS REQUIRESREPUBLISH AGE -truenas-nfs false false false false 2m -``` - -## Step 4: Create Test PVC - -### 4.1 Create PVC Manifest - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: test-pvc - namespace: default -spec: - accessModes: - - ReadWriteMany # NFS supports ReadWriteMany - storageClassName: truenas-nfs - resources: - requests: - storage: 10Gi -``` - -### 4.2 Apply and Verify - -```bash -kubectl apply -f test-pvc.yaml -kubectl get pvc -kubectl describe pvc test-pvc -``` - -Expected output: -``` -NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -test-pvc Bound pvc-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 10Gi RWX truenas-nfs 30s -``` - -### 4.3 Verify Volume on TrueNAS - -1. **Check TrueNAS UI:** - - Storage → Pools → k8s-storage - - You should see a new dataset: `pvc-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` - -2. **Check via API:** - ```bash - curl -k -H "Authorization: Bearer YOUR_API_KEY" \ - https://truenas.example.com/api/v2.0/pool/dataset/id/k8s-storage%2Fpvc-xxxxxxxx - ``` - -## Step 5: Configure Storage Classes in Rancher - -### 5.1 Create Storage Class via Rancher UI - -1. **Navigate to Cluster:** - - Cluster Management → nprd-apps → Storage → Storage Classes - -2. **Create Storage Class:** - - Click: Create - - Name: `truenas-nfs-fast` (or your preferred name) - - Provisioner: `truenas-nfs.csi.k8s.io` - - Reclaim Policy: `Delete` - - Volume Binding Mode: `Immediate` - - Allow Volume Expansion: ✓ - -3. **Add Parameters:** - ``` - parentDataset: /mnt/pool/k8s-storage - fsType: nfs - ``` - -4. **Set as Default** (optional): ✓ - -5. **Click Create** - -### 5.2 Multiple Storage Classes (Optional) - -You can create multiple storage classes for different use cases: - -```yaml -# Fast storage (SSD pool) -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: truenas-nfs-fast -provisioner: truenas-nfs.csi.k8s.io -parameters: - parentDataset: /mnt/ssd-pool/k8s-storage - fsType: nfs -reclaimPolicy: Delete -volumeBindingMode: Immediate -allowVolumeExpansion: true ---- -# Standard storage (HDD pool) -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: truenas-nfs-standard -provisioner: truenas-nfs.csi.k8s.io -parameters: - parentDataset: /mnt/hdd-pool/k8s-storage - fsType: nfs -reclaimPolicy: Delete -volumeBindingMode: Immediate -allowVolumeExpansion: true -``` - -## Step 6: Use Storage in Applications - -### 6.1 Example: Deploy Application with PVC - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-with-storage -spec: - replicas: 2 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx:latest - volumeMounts: - - name: storage - mountPath: /usr/share/nginx/html - volumes: - - name: storage - persistentVolumeClaim: - claimName: test-pvc -``` - -### 6.2 Example: StatefulSet - -```yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mysql -spec: - serviceName: mysql - replicas: 1 - selector: - matchLabels: - app: mysql - template: - metadata: - labels: - app: mysql - spec: - containers: - - name: mysql - image: mysql:8.0 - env: - - name: MYSQL_ROOT_PASSWORD - value: "password" - volumeMounts: - - name: data - mountPath: /var/lib/mysql - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: truenas-nfs - resources: - requests: - storage: 20Gi -``` - -## Step 7: Monitoring and Troubleshooting - -### 7.1 Check CSI Driver Logs - -```bash -# Controller logs -kubectl logs -n democratic-csi -l app=democratic-csi-controller - -# Node logs -kubectl logs -n democratic-csi -l app=democratic-csi-node -``` - -### 7.2 Common Issues - -#### Issue: PVC stuck in Pending - -**Symptoms:** -``` -kubectl get pvc -NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -test-pvc Pending truenas-nfs 5m -``` - -**Troubleshooting:** -```bash -# Check PVC events -kubectl describe pvc test-pvc - -# Check CSI driver pods -kubectl get pods -n democratic-csi - -# Check storage class -kubectl get storageclass truenas-nfs -o yaml - -# Verify TrueNAS connectivity from nodes -kubectl run -it --rm debug --image=busybox --restart=Never -- sh -# Inside pod: -ping truenas.example.com -``` - -#### Issue: Volume Creation Fails - -**Check:** -1. TrueNAS API key is valid -2. Dataset path exists on TrueNAS -3. Network connectivity from nodes to TrueNAS -4. NFS service is enabled on TrueNAS -5. Firewall rules allow NFS traffic (port 2049) - -#### Issue: Mount Failures - -**Check:** -1. NFS client tools installed on nodes: - ```bash - # On each RKE2 node - sudo apt-get install -y nfs-common - ``` - -2. NFS share permissions on TrueNAS -3. Network connectivity (ports 111, 2049) - -#### Issue: Permission Denied When Creating PVC - -**Symptom**: PVC creation fails with "Permission denied" errors - -**Check:** -```bash -# Verify user can create datasets -ssh root@your-truenas-host -sudo -u csi-user zfs create pool/dataset/test-permission - -# Check dataset ownership -ls -ld /mnt/SAS/RKE2 - -# Check ACLs (TrueNAS SCALE) -getfacl /mnt/SAS/RKE2 -``` - -**Fix:** -```bash -# Grant ownership -chown -R csi-user:csi-user /mnt/pool/dataset - -# Or use ACLs (TrueNAS SCALE) -setfacl -R -m u:csi-user:rwx /mnt/pool/dataset -setfacl -R -d -m u:csi-user:rwx /mnt/pool/dataset -``` - -#### Issue: API Authentication Failed - -**Symptom**: Controller logs show API authentication errors - -**Check:** -- API key is correct -- User account is active -- API key hasn't expired -- Network connectivity to TrueNAS - -**Fix:** -- Create new API key in TrueNAS UI -- Verify user permissions -- Test API access with curl (see Step 1.3) -- Update `terraform.tfvars` and regenerate Helm values - -#### Issue: CSI Node Pods Not Scheduling on Server Nodes - -**Symptom**: CSI node pods only on worker nodes, PVCs fail to mount on server nodes - -**Root Cause:** -RKE2 server nodes have taints that prevent normal pods from scheduling: -- `node-role.kubernetes.io/control-plane:NoSchedule` -- `node-role.kubernetes.io/etcd:NoExecute` -- `CriticalAddonsOnly` - -**Solution:** -Add tolerations to CSI node daemonset (already included in Helm values): -```yaml -# helm-values/democratic-csi-truenas.yaml -node: - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/etcd - operator: Exists - effect: NoExecute - - key: CriticalAddonsOnly - operator: Exists -``` - -This ensures CSI node pods can run on server nodes to handle volume mounts for pods scheduled there. - -### 7.3 Enable Debug Logging - -Update democratic-csi values: - -```yaml -driver: - config: - driver: freenas-api-nfs - logLevel: debug -``` - -Or via Helm: - -```bash -helm upgrade democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - --set driver.config.logLevel=debug -``` - -## Step 8: Production Considerations - -### 8.1 Security - -1. **Use HTTPS for TrueNAS API:** - ```yaml - config: - truenas: - protocol: "https" - allowInsecure: false - ``` - -2. **Restrict API Key Permissions:** - - Create dedicated user for CSI - - Grant minimal required permissions - -3. **Network Security:** - - Use VLANs to isolate storage traffic - - Configure firewall rules - - Consider VPN for remote access - -### 8.2 Performance - -1. **Dataset Configuration:** - - Use SSD pools for high-performance workloads - - Enable compression (lz4 recommended) - - Consider deduplication for similar data - -2. **NFS Tuning:** - - Adjust NFS version (v4 recommended) - - Configure appropriate timeouts - - Consider NFS over RDMA for high-performance - -3. **Storage Class Selection:** - - Use fast storage classes for databases - - Use standard storage classes for backups/archives - -### 8.3 Backup Strategy - -1. **TrueNAS Snapshots:** - - Configure periodic snapshots of k8s-storage dataset - - Use TrueNAS replication for off-site backups - -2. **Kubernetes Backup:** - - Use Velero for application-level backups - - Backup PVCs and application data - -### 8.4 High Availability - -1. **Multiple TrueNAS Systems:** - - Configure multiple storage classes pointing to different TrueNAS systems - - Use Rancher storage class management for failover - -2. **NFS Redundancy:** - - Use TrueNAS High Availability (HA) if available - - Configure multiple NFS shares for redundancy - -## Step 9: Integration with Rancher - -### 9.1 Make Storage Class Available in Rancher Projects - -1. **Navigate to Project:** - - Cluster Management → nprd-apps → Projects/Namespaces → Select Project - -2. **Resource Quotas:** - - Set storage quotas per project - - Limit PVC count and total storage - -### 9.2 Use in Rancher Workloads - -1. **Deploy Workload:** - - Workloads → Deploy → Configure → Volumes → Add Volume - - Select: Use a Persistent Volume Claim - - Choose: Create a new Persistent Volume Claim - - Storage Class: `truenas-nfs` - - Size: Specify desired size - - Access Mode: ReadWriteMany (for NFS) - -2. **Deploy and Verify:** - - Workload should start with persistent storage - - Data persists across pod restarts - -## Configuration Reference - -### Complete Helm Values Example - -```yaml -# democratic-csi Helm values -# CSI Driver Configuration (required by chart) -csiDriver: - name: truenas-nfs # Must be unique per cluster - enabled: true - attachRequired: true - podInfoOnMount: true - -# Driver Configuration -driver: - config: - driver: freenas-api-nfs # Use freenas-api-nfs for TrueNAS API-based NFS - httpConnection: - protocol: "https" - host: "truenas.example.com" - port: 443 - apiKey: "your-api-key-here" - allowInsecure: false - zfs: - datasetParentName: "pool/k8s-storage" # ZFS dataset name (without /mnt/ prefix) - detachedSnapshotsDatasetParentName: "pool/k8s-storage-snapshots" - datasetEnableQuotas: true - datasetEnableReservation: false - datasetPermissionsMode: "0777" - datasetPermissionsUser: 0 - datasetPermissionsGroup: 0 - # NFS share configuration - # IMPORTANT: Use shareHost (not server) for freenas-api-nfs driver - nfs: - shareHost: "truenas.example.com" # NFS server hostname/IP - shareAlldirs: false - shareAllowedHosts: [] - shareAllowedNetworks: [] - shareMaprootUser: root - shareMaprootGroup: root - shareMapallUser: "" - shareMapallGroup: "" - -# Storage Classes -storageClasses: - - name: truenas-nfs - default: true - reclaimPolicy: Delete - volumeBindingMode: Immediate - allowVolumeExpansion: true - parameters: - fsType: "nfs" - parentDataset: "pool/k8s-storage" # ZFS dataset name (without /mnt/ prefix) - nfsServer: "192.168.9.10" # Must match IP that NFS service is bound to - nfsVersion: "4" - # Using default mount options (no override needed in most cases) - # If custom mount options needed, use array format: - # mountOptions: - # - noatime - # - nfsvers=4 - - - name: truenas-nfs-fast - default: false - reclaimPolicy: Delete - volumeBindingMode: Immediate - allowVolumeExpansion: true - parameters: - fsType: "nfs" - parentDataset: "/mnt/ssd-pool/k8s-storage-fast" - -# Controller settings -controller: - replicas: 1 - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - -# Node settings -node: - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 256Mi -``` - -## Troubleshooting - -### PVC Mount Failures - -See [TROUBLESHOOTING.md](TROUBLESHOOTING.md#storagepvc-mount-issues) for detailed troubleshooting of: -- NFS service binding issues (Connection refused) -- Mount option configuration -- Storage class parameter updates -- CSI node pod scheduling - -### Key Learnings - -1. **Always verify NFS service binding** - TrueNAS NFS service may only be bound to specific IP addresses. Test manual mounts before configuring CSI. - -2. **Default mount options work fine** - No need to override with `nolock` or custom mount flags unless specifically required. If manual mount works with defaults, CSI will also work. - -3. **Storage class parameters are immutable** - Cannot update storage class parameters after creation. Must delete and recreate storage class to change parameters. - -4. **Test connectivity first** - Always test NFS connectivity from cluster nodes before deploying CSI: - ```bash - # Test port accessibility - timeout 3 bash -c 'cat < /dev/null > /dev/tcp/192.168.9.10/2049' - - # Test manual mount - sudo mount -t nfs 192.168.9.10:/mnt/pool/k8s-storage /tmp/test - ``` - -## Security Considerations - -### What Democratic CSI Does - -1. **Creates datasets** for each PVC (e.g., `pool/dataset/pvc-xxxxx`) -2. **Creates NFS shares** for each dataset (if using NFS) -3. **Deletes datasets** when PVCs are deleted -4. **Deletes NFS shares** when volumes are removed - -### Minimum Permissions Needed - -- ✅ **Dataset operations** on parent dataset and children -- ✅ **NFS share operations** (if automated, otherwise manual setup) -- ✅ **API access** to TrueNAS management interface - -### What's NOT Needed - -- ❌ Full pool access -- ❌ System administration -- ❌ Access to other datasets -- ❌ Shell/SSH access (API only) - -### Security Best Practices - -1. **Use HTTPS for TrueNAS API** (set `truenas_protocol = "https"` in tfvars) -2. **Create dedicated user** with minimal permissions (not root) -3. **Restrict API key scope** to dataset management only -4. **Use VLANs** to isolate storage traffic -5. **Configure firewall rules** to limit access to necessary ports -6. **Rotate API keys** periodically -7. **Monitor API access logs** in TrueNAS - -## Terraform Integration - -### Automatic Deployment - -When TrueNAS is configured in `terraform.tfvars`, Terraform will: - -1. Generate Helm values from Terraform variables -2. Install democratic-csi at the end of the plan -3. Create and configure the storage class -4. Set it as default (if configured) - -### Resource Dependencies - -``` -VMs → RKE2 Clusters → Rancher → Downstream Registration → Kubeconfigs → Democratic CSI -``` - -The democratic-csi resource depends on: -- `null_resource.merge_kubeconfigs` (kubeconfigs ready) -- `module.rke2_nprd_apps` (nprd-apps cluster ready) - -### Updating Configuration - -To update TrueNAS configuration: - -1. Edit `terraform/terraform.tfvars` -2. Run `terraform apply` -3. Terraform will regenerate Helm values and update the deployment - -Or manually: -1. Edit `terraform/terraform.tfvars` -2. Run `./scripts/generate-helm-values-from-tfvars.sh` -3. Reinstall via Helm: `helm upgrade democratic-csi ...` - -## Common Commands - -```bash -# Set kubeconfig -export KUBECONFIG=~/.kube/nprd-apps.yaml - -# Check CSI status -kubectl get pods -n democratic-csi -kubectl get storageclass -kubectl get csidriver - -# View logs -kubectl logs -n democratic-csi -l app=democratic-csi-controller -kubectl logs -n democratic-csi -l app=democratic-csi-node - -# List PVCs -kubectl get pvc --all-namespaces - -# Delete test PVC -kubectl delete pvc test-pvc - -# Verify TrueNAS connectivity from nodes -WORKER_IP=$(kubectl get nodes -o wide | grep worker | head -1 | awk '{print $6}') -ssh ubuntu@$WORKER_IP "timeout 3 bash -c 'cat < /dev/null > /dev/tcp/192.168.9.10/2049' && echo 'Port 2049 accessible' || echo 'Port 2049 NOT accessible'" - -# Test manual mount from node -ssh ubuntu@$WORKER_IP "sudo mkdir -p /tmp/test-nfs && sudo mount -t nfs 192.168.9.10:/mnt/SAS/RKE2 /tmp/test-nfs 2>&1 && echo 'Mount successful!' && ls -la /tmp/test-nfs | head -5 && sudo umount /tmp/test-nfs && sudo rmdir /tmp/test-nfs || echo 'Mount failed'" -``` - -## Additional Resources - -- **Democratic CSI Documentation:** https://github.com/democratic-csi/democratic-csi -- **TrueNAS Documentation:** https://www.truenas.com/docs/ -- **Rancher Storage Documentation:** https://rancher.com/docs/rancher/v2.7/en/storage/ -- **Kubernetes CSI Documentation:** https://kubernetes-csi.github.io/docs/ -- **Troubleshooting Guide:** [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - -## Troubleshooting Checklist - -- [ ] TrueNAS NFS service is enabled and running -- [ ] NFS share is configured and accessible -- [ ] User has write access to dataset (ownership or ACLs) -- [ ] User can create/delete child datasets (test with `zfs create/destroy`) -- [ ] API key is valid and has proper permissions -- [ ] Network connectivity from nodes to TrueNAS (ports 111, 2049, 443) -- [ ] Dataset path exists on TrueNAS -- [ ] `nfs-common` package installed on all nodes -- [ ] CSI driver pods are running (including on server nodes if needed) -- [ ] Storage class is created and default (if desired) -- [ ] PVC can be created and bound -- [ ] Pods can mount volumes successfully -- [ ] Helm values file is generated from Terraform variables - -## Next Steps - -1. ✅ Install democratic-csi (via Terraform or manually) -2. ✅ Configure TrueNAS backend (dataset, permissions, API key) -3. ✅ Create storage classes -4. ✅ Test with sample PVC -5. ✅ Deploy applications using persistent storage -6. ✅ Configure backups and monitoring -7. ✅ Document your specific configuration diff --git a/docs/DEPLOYMENT_GUIDE.md b/docs/DEPLOYMENT_GUIDE.md index 1c66b40..485343c 100644 --- a/docs/DEPLOYMENT_GUIDE.md +++ b/docs/DEPLOYMENT_GUIDE.md @@ -76,7 +76,7 @@ The script will: - RKE2 installation (NPRD Apps): ~5-10 minutes - RKE2 installation (PRD Apps): ~5-10 minutes - RKE2 installation (POC Apps): ~5-10 minutes -- Democratic CSI deployment (all apps clusters): ~2-3 minutes per cluster +- TrueNAS CSI deployment (all apps clusters): ~2-3 minutes per cluster - **Total: 50-70 minutes** (fully automated, no manual steps) ### 3. Monitor Progress diff --git a/docs/OPS_NOTES.md b/docs/OPS_NOTES.md index bb0e402..19ec987 100644 --- a/docs/OPS_NOTES.md +++ b/docs/OPS_NOTES.md @@ -18,8 +18,8 @@ Short operational truths that do not belong in the README. ## Storage -- **Democratic CSI** (`democratic_csi_*`, class `truenas-nfs`) is the common default path. -- **Official TrueNAS CSI** (`truenas_csi_*`, class `truenas-csi-nfs`) is optional and can coexist during migration; old Democratic volumes are not mountable by the new driver. See [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md) and [TRUENAS_CSI_MULTI_NODE.md](TRUENAS_CSI_MULTI_NODE.md). +- **Official TrueNAS CSI** (`truenas_csi_*`, driver `csi.truenas.io`, class `truenas-csi-nfs`) is the only CSI driver on the app clusters. See [TRUENAS_CSI_MULTI_NODE.md](TRUENAS_CSI_MULTI_NODE.md). +- **democratic-csi removed (2026-10-04).** No driver, StorageClass, PV, Helm release or namespace left on any cluster; the last orphaned `Released` PV on nprd (`pvc-25d4289c…`, old Loki ingester) was deleted (its dataset `/mnt/SAS/RKE2/pvc-25d4289c-38a0-4c20-96ef-35d2cd91356d` is still on TrueNAS). The `democratic_csi_*` variables, resources, output and scripts are gone from this repo. Migration history: [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md). - Apps-cluster nodes get label `topology.truenas.io/pool=` via RKE2 `node-label` at bootstrap, plus a post-kubeconfig `null_resource` that labels all nodes. Manager nodes are not labeled (CSI is apps-side). ## Version pins (RKE2 / Rancher / OS) @@ -87,7 +87,6 @@ Example layout (mgmt `vmbr0`/`bond0`, storage jumbo `vmbr1`/`bond1`, aux `vmbr2` | `terraform/terraform.tfvars` | API tokens, passwords (gitignored) | | `.keys/` | SSH deploy keys (gitignored) | | `config/` | Tokens (incl. `.rancher-api-token`), registry pull secrets (gitignored) | -| `helm-values/democratic-csi-truenas.yaml` | Generated CSI values (gitignored) | | `~/.config/rancher-saml/` | Rancher SAML SP cert/key + Authentik IdP metadata (outside the repo) | | `~/.rancher/cli2.json` | Rancher CLI token cache (`rancher token delete all` to clear) | | `~/.kube/*.yaml`, `~/.kube/config*` | Kubeconfigs; RKE2 admin ones are cluster-admin | diff --git a/docs/README.md b/docs/README.md index ffd4d8d..94403d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,7 +30,6 @@ Guides for deploying and operating Rancher / RKE2 on Proxmox with this repo. | Doc | Topic | |-----|--------| -| [DEMOCRATIC_CSI_TRUENAS_SETUP.md](DEMOCRATIC_CSI_TRUENAS_SETUP.md) | Democratic CSI + TrueNAS | | [STORAGE_CLASS_DEFAULT.md](STORAGE_CLASS_DEFAULT.md) | Default storage class | | [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md) | Democratic → official TrueNAS CSI | | [TRUENAS_CSI_MIGRATION_PRIORITY.md](TRUENAS_CSI_MIGRATION_PRIORITY.md) | Migration ordering | diff --git a/docs/STORAGE_CLASS_DEFAULT.md b/docs/STORAGE_CLASS_DEFAULT.md index ba05f8f..c83e136 100644 --- a/docs/STORAGE_CLASS_DEFAULT.md +++ b/docs/STORAGE_CLASS_DEFAULT.md @@ -2,84 +2,43 @@ ## Current Configuration -The `truenas-nfs` storage class is configured as **default** in your Helm values: +On the app clusters the default StorageClass is **`truenas-csi-nfs`** (official TrueNAS CSI driver, `csi.truenas.io`). Terraform sets it with `truenas_csi_storage_class_default = true` in `terraform.tfvars` (the variable defaults to `false`). -```yaml -storageClasses: - - name: truenas-nfs - default: true # ← This makes it the default -``` +prd-apps also has two non-default classes from the same driver: `truenas-csi-nfs-no-mapall` and `truenas-csi-nfs-postgres`. ## What "Default" Means When a storage class is marked as **default**: -- ✅ PVCs created **without** specifying `storageClassName` will automatically use it -- ✅ Rancher UI will show it as the default option -- ✅ It's marked with the annotation: `storageclass.kubernetes.io/is-default-class: "true"` - -## Important: Only One Default Allowed +- PVCs created **without** `storageClassName` use it +- Rancher UI shows it as the default option +- It carries the annotation `storageclass.kubernetes.io/is-default-class: "true"` -⚠️ **Kubernetes only allows ONE default storage class at a time.** +## Only One Default Allowed -If you already have a default storage class in your cluster, you have two options: +Kubernetes expects exactly **one** default StorageClass. RKE2 doesn't ship a default one on these clusters, so `truenas-csi-nfs` is the only one. -### Option 1: Replace Existing Default (Recommended) - -Make `truenas-nfs` the new default: +To move the default to another class: ```bash -export KUBECONFIG=~/.kube/nprd-apps.yaml +export KUBECONFIG=~/.kube/nprd-apps-rke2.yaml # Find current default kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' -# Remove default from existing storage class (replace with actual name) -kubectl patch storageclass -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' - -# Set truenas-nfs as default -kubectl patch storageclass truenas-nfs -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}' +# Remove default from the current class, then set it on the new one +kubectl patch storageclass truenas-csi-nfs -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' +kubectl patch storageclass -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}' ``` -### Option 2: Keep Existing Default, Make truenas-nfs Non-Default - -If you want to keep your current default storage class: - -1. **Before installation**, edit `helm-values/democratic-csi-truenas.yaml`: - ```yaml - storageClasses: - - name: truenas-nfs - default: false # ← Change to false - ``` - -2. **Or after installation**, remove the default annotation: - ```bash - kubectl patch storageclass truenas-nfs -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' - ``` +Change `truenas_csi_storage_class_default` in `terraform.tfvars` too, or the next Terraform apply puts it back. ## Check Current Default Storage Class ```bash -export KUBECONFIG=~/.kube/nprd-apps.yaml - -# List all storage classes with default status kubectl get storageclass - -# Find which one is default -kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' ``` -## RKE2 Default Storage Class - -RKE2 typically comes with a **local-path** storage class that may be set as default. This is fine for development but not ideal for production workloads that need shared storage. - -**Recommendation:** Make `truenas-nfs` the default for production workloads that need: -- Shared storage across nodes -- Persistent data that survives pod restarts -- NFS-backed volumes - -## Using Non-Default Storage Class - -Even if `truenas-nfs` is not the default, you can still use it by explicitly specifying it: +## Using a Non-Default Storage Class ```yaml apiVersion: v1 @@ -87,30 +46,12 @@ kind: PersistentVolumeClaim metadata: name: my-pvc spec: - storageClassName: truenas-nfs # ← Explicitly specify + storageClassName: truenas-csi-nfs-postgres accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 10Gi ``` -## Verification After Installation - -After installing democratic-csi, verify the default: - -```bash -kubectl get storageclass truenas-nfs -o yaml | grep -A 1 "is-default-class" -``` - -Should show: -```yaml -storageclass.kubernetes.io/is-default-class: "true" -``` - -## Summary - -- ✅ **Configured as default** in your Helm values -- ⚠️ **May conflict** with existing default storage class -- 🔧 **Installation script** will warn you if there's a conflict -- 📝 **You can change it** before or after installation +democratic-csi (`truenas-nfs`) was removed on 2026-10-04; see [TRUENAS_CSI_MIGRATION.md](TRUENAS_CSI_MIGRATION.md) for the history. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 43510ba..bcbe14a 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -953,245 +953,19 @@ See [DNS_CONFIGURATION.md](DNS_CONFIGURATION.md) for complete DNS troubleshootin ## Storage/PVC Mount Issues -### Issue: PVCs stuck in ContainerCreating, mount failures +App clusters use the official TrueNAS CSI driver (`csi.truenas.io`, StorageClass `truenas-csi-nfs`; namespace `truenas-csi`). democratic-csi was removed on 2026-10-04; its troubleshooting notes are in git history. -**Symptoms:** -- PVCs are `Bound` but pods remain in `ContainerCreating` -- CSI node pod logs show `mount.nfs: Connection refused` -- CSI node pod logs show `mount.nfs: rpc.statd is not running` -- Pod events show `MountVolume.MountDevice failed` - -**Root Causes & Solutions:** - -#### 1. NFS Service Not Bound to Correct IP Address - -**Symptom:** -``` -mount.nfs: Connection refused -executing mount command: mount -t nfs -o defaults 192.168.x.x:/mnt/path ... -``` - -**Root Cause:** -TrueNAS NFS service may only be bound to specific IP addresses. If CSI tries to mount from a different IP, it will fail with "Connection refused". - -**Solution:** -1. **Verify NFS service binding on TrueNAS:** - - TrueNAS Web UI → Services → NFS → Edit - - Check "Bind IP Addresses" configuration - - Ensure NFS service is bound to the IP address configured in CSI - -2. **Test NFS connectivity manually before configuring CSI:** - ```bash - # From a cluster node, test both IPs: - export KUBECONFIG=~/.kube/nprd-apps.yaml - WORKER_IP=$(kubectl get nodes -o wide | grep worker | head -1 | awk '{print $6}') - - # Test port accessibility - ssh ubuntu@$WORKER_IP "timeout 3 bash -c 'cat < /dev/null > /dev/tcp/192.168.9.10/2049' && echo 'Port 2049 accessible' || echo 'Port 2049 NOT accessible'" - - # Test manual mount with default options - ssh ubuntu@$WORKER_IP "sudo mkdir -p /tmp/test-nfs && sudo mount -t nfs 192.168.9.10:/mnt/SAS/RKE2 /tmp/test-nfs 2>&1 && echo 'Mount successful!' && ls -la /tmp/test-nfs | head -5 && sudo umount /tmp/test-nfs && sudo rmdir /tmp/test-nfs || echo 'Mount failed'" - ``` - -3. **Update Terraform configuration with correct IP:** - ```hcl - # terraform/terraform.tfvars - democratic_csi_host = "192.168.9.10" # Use IP that NFS service is bound to - ``` - -4. **Regenerate Helm values and upgrade CSI:** - ```bash - ./scripts/generate-helm-values-from-tfvars.sh - - # Upgrade apps clusters as needed - export KUBECONFIG=~/.kube/nprd-apps.yaml - helm upgrade democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml \ - --wait --timeout 5m - - export KUBECONFIG=~/.kube/prd-apps.yaml - helm upgrade democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml \ - --wait --timeout 5m - ``` - -5. **Recreate storage class with correct IP:** - ```bash - # Storage classes cannot be updated, must delete and recreate - export KUBECONFIG=~/.kube/nprd-apps.yaml - kubectl patch storageclass truenas-nfs -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}' - kubectl delete storageclass truenas-nfs - - # Helm upgrade will recreate storage class with correct IP - helm upgrade democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml --wait - ``` - -**Prevention:** -- Always verify NFS service binding on TrueNAS before configuring CSI -- Test manual NFS mounts from cluster nodes to verify connectivity -- Confirm which IP addresses the NFS service is bound to - -#### 2. Default Mount Options Work Fine (No Override Needed) - -**Symptom:** -``` -mount.nfs: rpc.statd is not running but is required for remote locking. -mount.nfs: Either use '-o nolock' to keep locks local, or start statd. -``` - -**Root Cause:** -Attempting to add `nolock` mount option override when default mount options work fine. The `rpc.statd` warning may appear but doesn't necessarily prevent mounts from working. - -**Solution:** -- **Use default mount options** - Democratic CSI with default NFS mount options works correctly -- No need to override with `nolock` or `vers=4` in most cases -- If manual mount works with default options, CSI will also work with defaults - -**Configuration:** -```yaml -# helm-values/democratic-csi-truenas.yaml -storageClasses: - - name: truenas-nfs - default: true - parameters: - fsType: "nfs" - nfsServer: "192.168.9.10" - nfsVersion: "4" - # Using default mount options (no override) - # mountOptions: [] # Optional - leave empty or omit -``` - -**Note:** If you need custom mount options, they must be an array format: -```yaml -mountOptions: - - noatime - - nfsvers=4 -``` - -#### 3. Storage Class Parameters Cannot Be Updated - -**Symptom:** -``` -Error: UPGRADE FAILED: cannot patch "truenas-nfs" with kind StorageClass: -StorageClass.storage.k8s.io "truenas-nfs" is invalid: parameters: -Forbidden: updates to parameters are forbidden. -``` - -**Root Cause:** -Kubernetes storage class parameters are immutable - they cannot be changed after creation. - -**Solution:** -1. **Delete existing storage class:** - ```bash - export KUBECONFIG=~/.kube/nprd-apps.yaml - kubectl patch storageclass truenas-nfs -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}' - kubectl delete storageclass truenas-nfs - ``` - -2. **Upgrade Helm to recreate with new parameters:** - ```bash - helm upgrade democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml --wait - ``` - -3. **Verify new storage class:** - ```bash - kubectl get storageclass truenas-nfs -o yaml | grep -A 10 "parameters:" - ``` - -**Prevention:** -- Test configuration thoroughly before deploying to production -- Use separate storage class names for testing vs production -- Verify TrueNAS IP address before initial deployment - -#### 4. Verifying PVC Mount Success - -**Test Steps:** ```bash -# 1. Create test PVC and pod -cat < /mnt/pvc/test.txt && cat /mnt/pvc/test.txt && sleep 3600'] - volumeMounts: - - name: pvc-volume - mountPath: /mnt/pvc - volumes: - - name: pvc-volume - persistentVolumeClaim: - claimName: test-pvc -EOF - -# 2. Check PVC and pod status -kubectl get pvc test-pvc -kubectl get pod test-pvc-pod - -# 3. Verify storage access -kubectl exec test-pvc-pod -- cat /mnt/pvc/test.txt - -# 4. Check CSI node pod logs for mount issues -NODE_POD=$(kubectl get pods -n democratic-csi -o wide | grep worker | head -1 | awk '{print $1}') -kubectl logs -n democratic-csi $NODE_POD -c csi-driver --tail=50 | grep -E "(mount|error|Error)" -``` - -**Expected Results:** -- PVC status: `Bound` -- Pod status: `Running` (1/1 Ready) -- Storage access: File can be read/written -- CSI logs: No mount errors - -### Issue: CSI Node Pods Not Scheduling on Server Nodes - -**Symptom:** -- CSI node pods only on worker nodes -- PVCs fail to mount when pods scheduled on server nodes - -**Root Cause:** -RKE2 server nodes have taints that prevent normal pods from scheduling: -- `node-role.kubernetes.io/control-plane:NoSchedule` -- `node-role.kubernetes.io/etcd:NoExecute` -- `CriticalAddonsOnly` - -**Solution:** -Add tolerations to CSI node daemonset (already included in Helm values): -```yaml -# helm-values/democratic-csi-truenas.yaml -node: - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/etcd - operator: Exists - effect: NoExecute - - key: CriticalAddonsOnly - operator: Exists +kubectl get csidrivers +kubectl get storageclass +kubectl -n truenas-csi get pods -o wide +kubectl describe pvc -n # provisioning events +kubectl get volumeattachments | grep # attach state ``` -This ensures CSI node pods can run on server nodes to handle volume mounts for pods scheduled there. +- TrueNAS CSI pods restart when the TrueNAS API stalls; mounts recover on their own. See [OPS_NOTES.md](OPS_NOTES.md#known-issues-2026-10-04). +- Multi-node / topology setup: [TRUENAS_CSI_MULTI_NODE.md](TRUENAS_CSI_MULTI_NODE.md). +- NFS mounts fail with `Connection refused`: check the TrueNAS NFS service is bound to the IP the driver uses (`truenas_csi_host`). Mounts run inside the `truenas-csi` node pods; the hosts don't need `nfs-common`. ## Related Documentation diff --git a/docs/TRUENAS_CSI_MIGRATION.md b/docs/TRUENAS_CSI_MIGRATION.md index 8c9cab0..277e4e2 100644 --- a/docs/TRUENAS_CSI_MIGRATION.md +++ b/docs/TRUENAS_CSI_MIGRATION.md @@ -1,5 +1,7 @@ # Migration: Democratic CSI → TrueNAS CSI +> **Done.** Every workload runs on TrueNAS CSI and democratic-csi was removed from all clusters and from this repo on 2026-10-04. Kept as history and as a template for future volume moves. + ## Can we create new PVCs and remount? **Yes.** The migration flow is: create new PVC with `truenas-csi-nfs` → copy data → update the workload to use the new PVC → delete the old PVC. The pod is updated to reference the new PVC; Kubernetes recreates the pod with the new volume mounted. diff --git a/docs/TRUENAS_CSI_MIGRATION_PRIORITY.md b/docs/TRUENAS_CSI_MIGRATION_PRIORITY.md index 6056073..7615974 100644 --- a/docs/TRUENAS_CSI_MIGRATION_PRIORITY.md +++ b/docs/TRUENAS_CSI_MIGRATION_PRIORITY.md @@ -1,6 +1,8 @@ # TrueNAS CSI Migration – Workload Inventory & Priority -All workloads below use **Democratic CSI** (`truenas-nfs`). Migrate to **TrueNAS CSI** (`truenas-csi-nfs`) in the order shown. +> **Done.** All workloads below were migrated and democratic-csi was removed on 2026-10-04. Kept as history. + +All workloads below used **Democratic CSI** (`truenas-nfs`) and were migrated to **TrueNAS CSI** (`truenas-csi-nfs`) in the order shown. --- diff --git a/docs/TRUENAS_CSI_MULTI_NODE.md b/docs/TRUENAS_CSI_MULTI_NODE.md index f517357..cacd5a9 100644 --- a/docs/TRUENAS_CSI_MULTI_NODE.md +++ b/docs/TRUENAS_CSI_MULTI_NODE.md @@ -23,14 +23,6 @@ After changing args, roll the controller/node pods, clear stuck `VolumeAttachmen ## Alternatives -### 1. Use Democratic CSI (truenas-nfs) for flexible scheduling - -Democratic CSI works on any node. Use it for workloads that need scheduling flexibility: - -```yaml -storageClassName: truenas-nfs # Democratic CSI -``` - -### 2. Hard-delete patch (legacy / optional) +### Hard-delete patch (legacy / optional) Older local workaround before upstream mode flags: remove the node check in `ControllerPublishVolume` and rebuild a patched image. Prefer the upstream `--mode=controller` / `--mode=node` deploy flags instead; keep this only if you cannot run a mode-split deploy. diff --git a/scripts/README.md b/scripts/README.md index 785ce4d..309bf06 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -34,13 +34,6 @@ GitHub Actions Runner Controller installation and configuration: - **`suggest-github-app-name.sh`** - Suggest unique GitHub App names to avoid conflicts - **`generate-jwt.sh`** - Generate JWT for GitHub App authentication -### Storage Setup Scripts - -Storage and CSI driver installation: - -- **`install-democratic-csi.sh`** - Install Democratic CSI driver for TrueNAS -- **`generate-helm-values-from-tfvars.sh`** - Generate Helm values from Terraform variables - ### Database Setup Scripts Database operator installation: @@ -102,17 +95,6 @@ rancher token delete all # clear the cache / force re-login ./scripts/complete-arc-setup.sh ``` -### Storage Setup - -```bash -# Generate Helm values from Terraform -./scripts/generate-helm-values-from-tfvars.sh - -# Install Democratic CSI -export KUBECONFIG=~/.kube/nprd-apps.yaml -./scripts/install-democratic-csi.sh -``` - ### Database Setup ```bash @@ -142,6 +124,5 @@ Some scripts require: ## Related Documentation - **[../docs/GITHUB_ARC_SETUP.md](../docs/GITHUB_ARC_SETUP.md)** - Complete GitHub ARC setup guide -- **[../docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md](../docs/DEMOCRATIC_CSI_TRUENAS_SETUP.md)** - TrueNAS storage setup guide - **[../docs/RANCHER_API_TOKEN_CREATION.md](../docs/RANCHER_API_TOKEN_CREATION.md)** - Rancher API token documentation - **[../docs/CLUSTER_ACCESS_AND_SSO.md](../docs/CLUSTER_ACCESS_AND_SSO.md)** - kubectl / Rancher SSO access and break-glass diff --git a/scripts/generate-helm-values-from-tfvars.sh b/scripts/generate-helm-values-from-tfvars.sh deleted file mode 100755 index 6ed1bb4..0000000 --- a/scripts/generate-helm-values-from-tfvars.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/bin/bash - -# Generate Helm values file from Terraform variables -# This script reads terraform.tfvars and generates helm-values/democratic-csi-truenas.yaml - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -TERRAFORM_DIR="$REPO_ROOT/terraform" -VALUES_FILE="$REPO_ROOT/helm-values/democratic-csi-truenas.yaml" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -echo -e "${GREEN}Generating Helm values from Terraform variables...${NC}" -echo "" - -# Check if terraform.tfvars exists -TFVARS_FILE="$TERRAFORM_DIR/terraform.tfvars" -if [ ! -f "$TFVARS_FILE" ]; then - echo -e "${RED}Error: terraform.tfvars not found at $TFVARS_FILE${NC}" - exit 1 -fi - -# Extract values from terraform.tfvars using grep and sed -# Note: This is a simple parser - assumes format: key = "value" or key = value -extract_value() { - local key="$1" - local default="$2" - local value=$(grep -E "^${key}\s*=" "$TFVARS_FILE" | sed -E 's/^[^=]*=\s*["'\'']?([^"'\'']*)["'\'']?/\1/' | head -1) - if [ -z "$value" ]; then - echo "$default" - else - echo "$value" - fi -} - -# Extract Democratic CSI configuration -TRUENAS_HOST=$(extract_value "democratic_csi_host" "") -TRUENAS_API_KEY=$(extract_value "democratic_csi_api_key" "") -TRUENAS_DATASET=$(extract_value "democratic_csi_dataset" "") -TRUENAS_USER=$(extract_value "democratic_csi_user" "rke2") -TRUENAS_PROTOCOL=$(extract_value "democratic_csi_protocol" "https") -TRUENAS_PORT=$(extract_value "democratic_csi_port" "443") -TRUENAS_ALLOW_INSECURE=$(extract_value "democratic_csi_allow_insecure" "false") -CSI_STORAGE_CLASS_NAME=$(extract_value "democratic_csi_storage_class_name" "truenas-nfs") -CSI_STORAGE_CLASS_DEFAULT=$(extract_value "democratic_csi_storage_class_default" "true") - -# Validate required values -if [ -z "$TRUENAS_HOST" ] || [ -z "$TRUENAS_API_KEY" ] || [ -z "$TRUENAS_DATASET" ]; then - echo -e "${RED}Error: Missing required Democratic CSI configuration in terraform.tfvars${NC}" - echo "Required: democratic_csi_host, democratic_csi_api_key, democratic_csi_dataset" - exit 1 -fi - -# Convert boolean string to YAML boolean -if [ "$TRUENAS_ALLOW_INSECURE" = "true" ] || [ "$TRUENAS_ALLOW_INSECURE" = "1" ]; then - TRUENAS_ALLOW_INSECURE_YAML="true" -else - TRUENAS_ALLOW_INSECURE_YAML="false" -fi - -if [ "$CSI_STORAGE_CLASS_DEFAULT" = "true" ] || [ "$CSI_STORAGE_CLASS_DEFAULT" = "1" ]; then - CSI_STORAGE_CLASS_DEFAULT_YAML="true" -else - CSI_STORAGE_CLASS_DEFAULT_YAML="false" -fi - -# Convert mountpoint path to ZFS dataset name (remove /mnt/ prefix) -# e.g., /mnt/SAS/RKE2 -> SAS/RKE2 -TRUENAS_DATASET_NAME=$(echo "$TRUENAS_DATASET" | sed 's|^/mnt/||') -TRUENAS_DATASET_SNAPSHOTS="${TRUENAS_DATASET_NAME}-snapshots" - -# Generate Helm values file -cat > "$VALUES_FILE" < /dev/null; then - echo "Error: kubectl not found" - exit 1 -fi -echo "✓ kubectl found" - -if ! command -v helm &> /dev/null; then - echo "Error: helm not found" - exit 1 -fi -echo "✓ helm found" - -# Set kubeconfig -if [ -f "$HOME/.kube/nprd-apps.yaml" ]; then - export KUBECONFIG="$HOME/.kube/nprd-apps.yaml" - echo "✓ Using kubeconfig: ~/.kube/nprd-apps.yaml" -else - echo "⚠ Using default kubeconfig" -fi - -# Verify cluster access -if ! kubectl cluster-info &> /dev/null; then - echo "Error: Cannot access Kubernetes cluster" - exit 1 -fi -echo "✓ Cluster access verified" -echo "" - -# Add Helm repository -echo -e "${YELLOW}Adding Helm repository...${NC}" -helm repo add democratic-csi https://democratic-csi.github.io/charts/ 2>/dev/null || echo "Repository already added" -helm repo update -echo "✓ Helm repository ready" -echo "" - -# Create namespace -echo -e "${YELLOW}Creating namespace...${NC}" -kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - -echo "✓ Namespace created" -echo "" - -# Install democratic-csi -echo -e "${YELLOW}Installing democratic-csi...${NC}" -echo "Using values file: $ACTUAL_VALUES_FILE" -echo "" - -helm upgrade --install "$RELEASE_NAME" democratic-csi/democratic-csi \ - --namespace "$NAMESPACE" \ - -f "$VALUES_FILE" \ - --wait \ - --timeout 10m - -echo "" -echo -e "${GREEN}✓ Democratic CSI installed${NC}" -echo "" - -# Wait for pods -echo -e "${YELLOW}Waiting for pods to be ready...${NC}" -kubectl wait --for=condition=ready pod -l app=democratic-csi-controller -n "$NAMESPACE" --timeout=5m || true -kubectl wait --for=condition=ready pod -l app=democratic-csi-node -n "$NAMESPACE" --timeout=5m || true - -# Verify installation -echo "" -echo -e "${YELLOW}Verifying installation...${NC}" -echo "" -echo "Pods:" -kubectl get pods -n "$NAMESPACE" -echo "" -echo "Storage Classes:" -kubectl get storageclass -echo "" - -# Check for default storage class conflicts -DEFAULT_SC=$(kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}') -if [ -n "$DEFAULT_SC" ]; then - echo -e "${YELLOW}Current default storage class: ${DEFAULT_SC}${NC}" - if [ "$DEFAULT_SC" != "truenas-nfs" ]; then - echo -e "${YELLOW}⚠️ Warning: Another storage class is already default${NC}" - echo "To make truenas-nfs the default, run:" - echo " kubectl patch storageclass $DEFAULT_SC -p '{\"metadata\": {\"annotations\": {\"storageclass.kubernetes.io/is-default-class\": \"false\"}}}'" - echo " kubectl patch storageclass truenas-nfs -p '{\"metadata\": {\"annotations\": {\"storageclass.kubernetes.io/is-default-class\": \"true\"}}}'" - else - echo -e "${GREEN}✓ truenas-nfs is the default storage class${NC}" - fi -else - echo -e "${GREEN}✓ truenas-nfs is the default storage class${NC}" -fi - -echo "" -echo "CSI Drivers:" -kubectl get csidriver - -# Test PVC -echo "" -read -p "Create a test PVC? (y/n) " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]]; then - echo -e "${YELLOW}Creating test PVC...${NC}" - kubectl apply -f - </dev/null; then - echo "ERROR: Cannot access nprd-apps cluster" - exit 1 - fi - - echo "✓ Cluster access verified" - - # Add Helm repository - echo "Adding Helm repository..." - helm repo add democratic-csi https://democratic-csi.github.io/charts/ 2>/dev/null || echo "Repository already added" - helm repo update - - # Create namespace - echo "Creating namespace..." - kubectl create namespace democratic-csi --dry-run=client -o yaml | kubectl apply -f - - - # Install democratic-csi - echo "Installing democratic-csi..." - helm upgrade --install democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml \ - --wait \ - --timeout 10m - - echo "✓ Democratic CSI installed" - - # Wait for pods to be ready - echo "Waiting for pods to be ready..." - kubectl wait --for=condition=ready pod -l app=democratic-csi-controller -n democratic-csi --timeout=5m || true - kubectl wait --for=condition=ready pod -l app=democratic-csi-node -n democratic-csi --timeout=5m || true - - # Verify storage class - echo "" - echo "Verifying storage class..." - if kubectl get storageclass ${var.democratic_csi_storage_class_name} &>/dev/null; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' created" - - # Check if it's default - IS_DEFAULT=$(kubectl get storageclass ${var.democratic_csi_storage_class_name} -o jsonpath='{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}' 2>/dev/null || echo "") - if [ "$IS_DEFAULT" = "true" ]; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' is set as default" - elif [ "${var.democratic_csi_storage_class_default}" = "true" ]; then - echo "Setting storage class as default..." - # Remove default from any existing default storage class - EXISTING_DEFAULT=$(kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' 2>/dev/null || echo "") - if [ -n "$EXISTING_DEFAULT" ] && [ "$EXISTING_DEFAULT" != "${var.democratic_csi_storage_class_name}" ]; then - echo " Removing default from: $EXISTING_DEFAULT" - kubectl patch storageclass "$EXISTING_DEFAULT" -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' || true - fi - # Set as default - kubectl patch storageclass ${var.democratic_csi_storage_class_name} -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}' || true - echo "✓ Storage class set as default" - fi - else - echo "⚠ Storage class '${var.democratic_csi_storage_class_name}' not found" - echo " This may be normal if Helm installation is still in progress" - fi - - echo "" - echo "Storage Classes:" - kubectl get storageclass - - echo "" - echo "Democratic CSI Pods:" - kubectl get pods -n democratic-csi - - echo "" - echo "==========================================" - echo "✓ Democratic CSI deployment complete" - echo "==========================================" - EOT - } - - provisioner "local-exec" { - when = destroy - on_failure = continue - command = <<-EOT - echo "==========================================" - echo "Removing Democratic CSI" - echo "==========================================" - - export KUBECONFIG="$HOME/.kube/nprd-apps.yaml" - - if kubectl get namespace democratic-csi &>/dev/null; then - echo "Uninstalling democratic-csi..." - helm uninstall democratic-csi --namespace democratic-csi 2>/dev/null || true - - echo "Deleting namespace..." - kubectl delete namespace democratic-csi --timeout=2m 2>/dev/null || true - - echo "✓ Democratic CSI removed" - else - echo "✓ Namespace already removed" - fi - EOT - } - - depends_on = [ - null_resource.merge_kubeconfigs, - module.rke2_nprd_apps - ] - - triggers = { - democratic_csi_host = var.democratic_csi_host - democratic_csi_api_key = sha256(var.democratic_csi_api_key) - democratic_csi_dataset = var.democratic_csi_dataset - democratic_csi_storage_class = var.democratic_csi_storage_class_name - democratic_csi_storage_default = tostring(var.democratic_csi_storage_class_default) - helm_values_file = filemd5("${path.root}/../scripts/generate-helm-values-from-tfvars.sh") - } -} - -resource "null_resource" "deploy_democratic_csi_prd_apps" { - count = var.install_democratic_csi && var.democratic_csi_host != "" && var.democratic_csi_api_key != "" ? 1 : 0 - - provisioner "local-exec" { - command = <<-EOT - set -e - - echo "==========================================" - echo "Deploying Democratic CSI with TrueNAS to PRD Apps Cluster" - echo "==========================================" - - # Generate Helm values from Terraform variables - echo "Generating Helm values from terraform.tfvars..." - cd "${path.root}/.." - ./scripts/generate-helm-values-from-tfvars.sh - - # Set kubeconfig to prd-apps cluster - export KUBECONFIG="$HOME/.kube/prd-apps.yaml" - - # Verify cluster access - if ! kubectl cluster-info &>/dev/null; then - echo "ERROR: Cannot access prd-apps cluster" - exit 1 - fi - - echo "✓ Cluster access verified" - - # Add Helm repository - echo "Adding Helm repository..." - helm repo add democratic-csi https://democratic-csi.github.io/charts/ 2>/dev/null || echo "Repository already added" - helm repo update - - # Create namespace - echo "Creating namespace..." - kubectl create namespace democratic-csi --dry-run=client -o yaml | kubectl apply -f - - - # Install democratic-csi - echo "Installing democratic-csi..." - helm upgrade --install democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml \ - --wait \ - --timeout 10m - - echo "✓ Democratic CSI installed" - - # Wait for pods to be ready - echo "Waiting for pods to be ready..." - kubectl wait --for=condition=ready pod -l app=democratic-csi-controller -n democratic-csi --timeout=5m || true - kubectl wait --for=condition=ready pod -l app=democratic-csi-node -n democratic-csi --timeout=5m || true - - # Verify storage class - echo "" - echo "Verifying storage class..." - if kubectl get storageclass ${var.democratic_csi_storage_class_name} &>/dev/null; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' created" - - # Check if it's default - IS_DEFAULT=$(kubectl get storageclass ${var.democratic_csi_storage_class_name} -o jsonpath='{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}' 2>/dev/null || echo "") - if [ "$IS_DEFAULT" = "true" ]; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' is set as default" - elif [ "${var.democratic_csi_storage_class_default}" = "true" ]; then - echo "Setting storage class as default..." - # Remove default from any existing default storage class - EXISTING_DEFAULT=$(kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' 2>/dev/null || echo "") - if [ -n "$EXISTING_DEFAULT" ] && [ "$EXISTING_DEFAULT" != "${var.democratic_csi_storage_class_name}" ]; then - echo " Removing default from: $EXISTING_DEFAULT" - kubectl patch storageclass "$EXISTING_DEFAULT" -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' || true - fi - # Set as default - kubectl patch storageclass ${var.democratic_csi_storage_class_name} -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}' || true - echo "✓ Storage class set as default" - fi - else - echo "⚠ Storage class '${var.democratic_csi_storage_class_name}' not found" - echo " This may be normal if Helm installation is still in progress" - fi - - echo "" - echo "Storage Classes:" - kubectl get storageclass - - echo "" - echo "Democratic CSI Pods:" - kubectl get pods -n democratic-csi - - echo "" - echo "==========================================" - echo "✓ Democratic CSI deployment complete" - echo "==========================================" - EOT - } - - provisioner "local-exec" { - when = destroy - on_failure = continue - command = <<-EOT - echo "==========================================" - echo "Removing Democratic CSI from PRD Apps Cluster" - echo "==========================================" - - export KUBECONFIG="$HOME/.kube/prd-apps.yaml" - - if kubectl get namespace democratic-csi &>/dev/null; then - echo "Uninstalling democratic-csi..." - helm uninstall democratic-csi --namespace democratic-csi 2>/dev/null || true - - echo "Deleting namespace..." - kubectl delete namespace democratic-csi --timeout=2m 2>/dev/null || true - - echo "✓ Democratic CSI removed" - else - echo "✓ Namespace already removed" - fi - EOT - } - - depends_on = [ - null_resource.merge_kubeconfigs, - module.rke2_prd_apps - ] - - triggers = { - democratic_csi_host = var.democratic_csi_host - democratic_csi_api_key = sha256(var.democratic_csi_api_key) - democratic_csi_dataset = var.democratic_csi_dataset - democratic_csi_storage_class = var.democratic_csi_storage_class_name - democratic_csi_storage_default = tostring(var.democratic_csi_storage_class_default) - helm_values_file = filemd5("${path.root}/../scripts/generate-helm-values-from-tfvars.sh") - } -} - -resource "null_resource" "deploy_democratic_csi_poc_apps" { - count = var.install_democratic_csi && var.democratic_csi_host != "" && var.democratic_csi_api_key != "" ? 1 : 0 - - provisioner "local-exec" { - command = <<-EOT - set -e - - echo "==========================================" - echo "Deploying Democratic CSI with TrueNAS to POC Apps Cluster" - echo "==========================================" - - # Generate Helm values from Terraform variables - echo "Generating Helm values from terraform.tfvars..." - cd "${path.root}/.." - ./scripts/generate-helm-values-from-tfvars.sh - - # Set kubeconfig to poc-apps cluster - export KUBECONFIG="$HOME/.kube/poc-apps.yaml" - - # Verify cluster access - if ! kubectl cluster-info &>/dev/null; then - echo "ERROR: Cannot access poc-apps cluster" - exit 1 - fi - - echo "✓ Cluster access verified" - - # Add Helm repository - echo "Adding Helm repository..." - helm repo add democratic-csi https://democratic-csi.github.io/charts/ 2>/dev/null || echo "Repository already added" - helm repo update - - # Create namespace - echo "Creating namespace..." - kubectl create namespace democratic-csi --dry-run=client -o yaml | kubectl apply -f - - - # Install democratic-csi - echo "Installing democratic-csi..." - helm upgrade --install democratic-csi democratic-csi/democratic-csi \ - --namespace democratic-csi \ - -f helm-values/democratic-csi-truenas.yaml \ - --wait \ - --timeout 10m - - echo "✓ Democratic CSI installed" - - # Wait for pods to be ready - echo "Waiting for pods to be ready..." - kubectl wait --for=condition=ready pod -l app=democratic-csi-controller -n democratic-csi --timeout=5m || true - kubectl wait --for=condition=ready pod -l app=democratic-csi-node -n democratic-csi --timeout=5m || true - - # Verify storage class - echo "" - echo "Verifying storage class..." - if kubectl get storageclass ${var.democratic_csi_storage_class_name} &>/dev/null; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' created" - - # Check if it's default - IS_DEFAULT=$(kubectl get storageclass ${var.democratic_csi_storage_class_name} -o jsonpath='{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}' 2>/dev/null || echo "") - if [ "$IS_DEFAULT" = "true" ]; then - echo "✓ Storage class '${var.democratic_csi_storage_class_name}' is set as default" - elif [ "${var.democratic_csi_storage_class_default}" = "true" ]; then - echo "Setting storage class as default..." - # Remove default from any existing default storage class - EXISTING_DEFAULT=$(kubectl get storageclass -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}' 2>/dev/null || echo "") - if [ -n "$EXISTING_DEFAULT" ] && [ "$EXISTING_DEFAULT" != "${var.democratic_csi_storage_class_name}" ]; then - echo " Removing default from: $EXISTING_DEFAULT" - kubectl patch storageclass "$EXISTING_DEFAULT" -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "false"}}}' || true - fi - # Set as default - kubectl patch storageclass ${var.democratic_csi_storage_class_name} -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}' || true - echo "✓ Storage class set as default" - fi - else - echo "⚠ Storage class '${var.democratic_csi_storage_class_name}' not found" - echo " This may be normal if Helm installation is still in progress" - fi - - echo "" - echo "Storage Classes:" - kubectl get storageclass - - echo "" - echo "Democratic CSI Pods:" - kubectl get pods -n democratic-csi - - echo "" - echo "==========================================" - echo "✓ Democratic CSI deployment complete" - echo "==========================================" - EOT - } - - provisioner "local-exec" { - when = destroy - on_failure = continue - command = <<-EOT - echo "==========================================" - echo "Removing Democratic CSI from POC Apps Cluster" - echo "==========================================" - - export KUBECONFIG="$HOME/.kube/poc-apps.yaml" - - if kubectl get namespace democratic-csi &>/dev/null; then - echo "Uninstalling democratic-csi..." - helm uninstall democratic-csi --namespace democratic-csi 2>/dev/null || true - - echo "Deleting namespace..." - kubectl delete namespace democratic-csi --timeout=2m 2>/dev/null || true - - echo "✓ Democratic CSI removed" - else - echo "✓ Namespace already removed" - fi - EOT - } - - depends_on = [ - null_resource.merge_kubeconfigs, - module.rke2_poc_apps - ] - - triggers = { - democratic_csi_host = var.democratic_csi_host - democratic_csi_api_key = sha256(var.democratic_csi_api_key) - democratic_csi_dataset = var.democratic_csi_dataset - democratic_csi_storage_class = var.democratic_csi_storage_class_name - democratic_csi_storage_default = tostring(var.democratic_csi_storage_class_default) - helm_values_file = filemd5("${path.root}/../scripts/generate-helm-values-from-tfvars.sh") - } -} - # ============================================================================ # TRUENAS CSI DRIVER DEPLOYMENT (Official driver from truenas/truenas-csi) # Requires TrueNAS SCALE 25.10+. Deploys to nprd-apps, prd-apps, poc-apps diff --git a/terraform/outputs.tf b/terraform/outputs.tf index ad3cad6..9b71059 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,21 +1,6 @@ # Terraform Outputs # These outputs can be used by other tools or scripts -output "democratic_csi_config" { - description = "Democratic CSI configuration" - value = { - host = var.democratic_csi_host - dataset = var.democratic_csi_dataset - user = var.democratic_csi_user - protocol = var.democratic_csi_protocol - port = var.democratic_csi_port - allow_insecure = var.democratic_csi_allow_insecure - storage_class = var.democratic_csi_storage_class_name - is_default = var.democratic_csi_storage_class_default - } - sensitive = false -} - output "truenas_csi_config" { description = "TrueNAS CSI configuration" value = { diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index b794a0b..38ed375 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -280,23 +280,6 @@ poc_apps_cluster_aliases = ["poc.example.com"] # Alias for poc apps cluster a # Set to one of the manager node IPs (usually the primary node IP) rancher_manager_ip = "192.168.1.100" -# ============================================================================ -# DEMOCRATIC CSI CONFIGURATION (community driver: democratic-csi) -# ============================================================================ - -install_democratic_csi = true - -democratic_csi_host = "truenas.example.com" -democratic_csi_api_key = "your-truenas-api-key-here" -democratic_csi_dataset = "/mnt/SAS/RKE2" -democratic_csi_user = "rke2" -democratic_csi_protocol = "https" -democratic_csi_port = 443 -democratic_csi_allow_insecure = false - -democratic_csi_storage_class_name = "truenas-nfs" -democratic_csi_storage_class_default = true - # ============================================================================ # TRUENAS CSI CONFIGURATION (official driver: truenas/truenas-csi) # ============================================================================ diff --git a/terraform/variables.tf b/terraform/variables.tf index 0157d85..2b1307a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -280,71 +280,6 @@ variable "downstream_cluster_id" { default = "" # Now fetched dynamically from Rancher API } -# ============================================================================ -# DEMOCRATIC CSI CONFIGURATION (community driver: democratic-csi) -# ============================================================================ - -variable "install_democratic_csi" { - description = "Install Democratic CSI with TrueNAS backend" - type = bool - default = true -} - -variable "democratic_csi_host" { - description = "TrueNAS hostname or IP for Democratic CSI" - type = string - default = "" -} - -variable "democratic_csi_api_key" { - description = "TrueNAS API key for Democratic CSI (obtain from TrueNAS: System → API Keys)" - type = string - sensitive = true - default = "" -} - -variable "democratic_csi_dataset" { - description = "TrueNAS dataset path for NFS (e.g., /mnt/SAS/RKE2)" - type = string - default = "" -} - -variable "democratic_csi_user" { - description = "TrueNAS username for API access (reference only)" - type = string - default = "" -} - -variable "democratic_csi_protocol" { - description = "TrueNAS API protocol (https or http)" - type = string - default = "https" -} - -variable "democratic_csi_port" { - description = "TrueNAS API port" - type = number - default = 443 -} - -variable "democratic_csi_allow_insecure" { - description = "Allow insecure TLS (self-signed certs)" - type = bool - default = false -} - -variable "democratic_csi_storage_class_name" { - description = "Storage class name for Democratic CSI" - type = string - default = "truenas-nfs" -} - -variable "democratic_csi_storage_class_default" { - description = "Make Democratic CSI storage class the default" - type = bool - default = true -} - # ============================================================================ # TRUENAS CSI CONFIGURATION (official driver: truenas/truenas-csi) # ============================================================================