From a4483f87014829d7e8d02ec6f7a7c74bf6b9b9cc Mon Sep 17 00:00:00 2001
From: Alexander Marshalov <_@marshalov.org>
Date: Mon, 10 Jun 2024 15:10:27 +0200
Subject: [PATCH] Add docs and example for multi-level cluster setup

---
 config/examples/multi_level_cluster.yaml | 128 +++++++++++++++++++++++
 docs/resources/vmcluster.md              |   6 ++
 2 files changed, 134 insertions(+)
 create mode 100644 config/examples/multi_level_cluster.yaml

diff --git a/config/examples/multi_level_cluster.yaml b/config/examples/multi_level_cluster.yaml
new file mode 100644
index 000000000..f7d2cd012
--- /dev/null
+++ b/config/examples/multi_level_cluster.yaml
@@ -0,0 +1,128 @@
+# Minimal example for multi-level cluster setup with operator
+# (https://docs.victoriametrics.com/cluster-victoriametrics/#multi-level-cluster-setup)
+# There are two clusters: blue and green. Each cluster has its own storage, select and insert nodes.
+# And there is a shared VMCluster that uses VMSelect and VMInsert nodes from blue and green clusters through clusterNative.
+# The shared cluster also has a VMAgent that sends data to the shared insert nodes and VMAuth that selects data from the shared select nodes.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMCluster
+metadata:
+  name: blue
+  namespace: monitoring
+spec:
+  retentionPeriod: "1"
+  vmstorage:
+    replicaCount: 3
+  vminsert:
+    replicaCount: 1
+    extraArgs:
+      clusternativeListenAddr: ":8400"
+    serviceSpec:
+      useAsDefault: true
+      spec:
+        ports:
+          - name: clusternative
+            port: 8400
+            protocol: TCP
+            targetPort: 8400
+  vmselect:
+    replicaCount: 1
+    extraArgs:
+      clusternativeListenAddr: ":8401"
+    serviceSpec:
+      useAsDefault: true
+      spec:
+        ports:
+          - name: clusternative
+            port: 8401
+            protocol: TCP
+            targetPort: 8401
+
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMCluster
+metadata:
+  name: green
+  namespace: monitoring
+spec:
+  retentionPeriod: "1"
+  vmstorage:
+    replicaCount: 3
+  vminsert:
+    replicaCount: 1
+    extraArgs:
+      clusternativeListenAddr: ":8400"
+    serviceSpec:
+      useAsDefault: true
+      spec:
+        ports:
+          - name: clusternative
+            port: 8400
+            protocol: TCP
+            targetPort: 8400
+  vmselect:
+    replicaCount: 1
+    extraArgs:
+      clusternativeListenAddr: ":8401"
+    serviceSpec:
+      useAsDefault: true
+      spec:
+        ports:
+          - name: clusternative
+            port: 8401
+            protocol: TCP
+            targetPort: 8401
+
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMCluster
+metadata:
+  name: shared
+  namespace: monitoring
+spec:
+  retentionPeriod: "1"
+  vmselect:
+    replicaCount: 1
+    extraArgs:
+      storageNode: "blue/vmselect-blue-0.vmselect-blue,green/vmselect-green-0.vmselect-green"
+  vminsert:
+    replicaCount: 1
+    extraArgs:
+      storageNode: "blue/vminsert-blue-0.vminsert-blue,green/vminsert-green-0.vminsert-green"
+
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAgent
+metadata:
+  name: shared
+  namespace: monitoring
+spec:
+  selectAllByDefault: true
+  remoteWrite:
+    - url: "http://vminsert-shared:8480/insert/0/prometheus/api/v1/write"
+    - url: "http://vminsert-shared:8480/insert/0/prometheus/api/v1/write"
+
+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMAuth
+metadata:
+  name: shared
+  namespace: monitoring
+spec:
+  userSelector: {}
+  userNamespaceSelector:
+    matchLabels:
+      kubernetes.io/metadata.name: monitoring
+  ingress:
+    class_name: nginx
+    host: vm.k8s.orb.local # specify your ingress host here
+  unauthorizedAccessConfig:
+    - src_paths: [".*"]
+      url_prefix: ["http://vmselect-shared:8481/select/0"]
diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md
index 69564e76c..f1957c1fd 100644
--- a/docs/resources/vmcluster.md
+++ b/docs/resources/vmcluster.md
@@ -630,6 +630,12 @@ Possible configuration options for backup crd can be found at [link](../api.md#v
 
 Also see VMCLuster example spec [here](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmcluster_with_backuper.yaml).
 
+## Multi-level cluster setup
+
+You can construct a [multi-level cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#multi-level-cluster-setup) with several `VMCluster` resources.
+
+See minimal configuration [example here](https://github.com/VictoriaMetrics/operator/blob/master/config/examples/vmagent-stream-aggr.yaml) for that.
+
 ## Examples
 
 ### Minimal example without persistence