feat: Add a data importer for bringing cloud data into onPrem

charts/currents/templates/_common.tpl

@@ -166,6 +166,19 @@ Create the name of the service account to use
 {{- end }}
 {{- end -}}
 
+{{/*
+ClickHouse restore mode.
+
+Suppresses change-stream-driven ClickHouse sync while an organization's
+ClickHouse data is loaded from an external export (scripts/org-import).
+*/}}
+{{- define "currents.clickhouseRestoreModeEnv" -}}
+{{- if (.Values.maintenance).clickhouseRestoreMode }}
+- name: CURRENTS_CLICKHOUSE_RESTORE_MODE
+  value: "true"
+{{- end }}
+{{- end -}}
+
 {{- define "currents.URLConfigEnv" -}}
 - name: GITLAB_REDIRECT_URL
   value: {{ printf "%s/integrations/gitlab/callback" (include "currents.url" (dict "context" . "input" .Values.currents.domains.recordApiHost)) }}

charts/currents/templates/_helpers.tpl

@@ -40,6 +40,13 @@ Create webhooks name and version as used by the chart label.
 {{- printf "%s-%s" (include "currents.fullname" .) .Values.webhooks.name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Create toolbox name as used by the chart label.
+*/}}
+{{- define "currents.toolbox.fullname" -}}
+{{- printf "%s-%s" (include "currents.fullname" .) .Values.toolbox.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{- define "currents.url" -}}
 {{- if .context.Values.currents.domains.https -}}
 {{- printf "https://%s" .input -}}

charts/currents/templates/changestreams/deployment.yaml

@@ -47,6 +47,7 @@ spec:
               value: "onprem"
             {{- include "currents.connectionConfigEnv" . | nindent 12 }}
             {{- include "currents.URLConfigEnv" . | nindent 12 }}
+            {{- include "currents.clickhouseRestoreModeEnv" . | nindent 12 }}
             {{- with (concat .Values.global.env .Values.changestreams.env) }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/currents/templates/toolbox/pod.yaml

@@ -0,0 +1,98 @@
+{{- if .Values.toolbox.enabled }}
+{{/*
+Toolbox pod — a gated, manually-driven pod for the org import (ENG-912/913).
+
+Single container on the onPrem scheduler image, which now carries BOTH the org-import
+CLIs (baked in at /app/packages/scheduler/dist/orgImport) and `mongorestore`. So the whole
+import runs here with nothing kubectl-cp'd: the operator copies in a small download.json
+and runs `currents-import`.
+
+It runs `sleep infinity` and does nothing on its own.
+*/}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ include "currents.toolbox.fullname" . }}
+  labels:
+    {{- include "currents.labels" (dict "context" . "component" .Values.toolbox.name) | nindent 4 }}
+  {{- with .Values.global.podAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  # A bare Pod, not a Job: the import is operator-driven and must never auto-retry.
+  restartPolicy: Never
+  {{- with .Values.global.imagePullSecrets }}
+  imagePullSecrets:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  serviceAccountName: {{ include "currents.serviceAccountName" . }}
+  {{- /* fsGroup makes the PVC writable by the non-root container user. */}}
+  {{- with .Values.global.securityContext }}
+  securityContext:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.global.priorityClassName }}
+  priorityClassName: {{ . }}
+  {{- end }}
+  containers:
+    - name: toolbox
+      {{- with .Values.global.containerSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      # The onPrem scheduler image: node + @currents/* + the org-import CLIs + mongorestore.
+      image: {{ include "currents.image" (dict "context" . "imageRoot" .Values.scheduler.image) }}
+      imagePullPolicy: {{ default .Values.global.imagePullPolicy .Values.scheduler.image.pullPolicy }}
+      # Override the image's pm2 entrypoint — nothing runs on its own here.
+      command: ["sleep", "infinity"]
+      env:
+        - name: CURRENTS_ENV
+          value: "onprem"
+        # node/mongorestore write config under $HOME; the root FS is read-only, so point it
+        # at the PVC.
+        - name: HOME
+          value: /data
+        {{- /* A bulk merge INSERT … SELECT into test_metric_v2 (+ its rollup MVs) can run for
+               minutes with no bytes on the socket; raise the client's request timeout past the
+               30s default so it doesn't trip mid-insert. Matches ch-import's max_execution_time. */}}
+        - name: CLICKHOUSE_REQUEST_TIMEOUT_MS
+          value: {{ .Values.toolbox.clickhouseRequestTimeoutMs | default 3600000 | int64 | quote }}
+        {{- include "currents.connectionConfigEnv" . | nindent 8 }}
+        {{- /* The importer refuses a ClickHouse import unless it sees this — safe only
+               because the same helper sets it on the change-streams deployment. */}}
+        {{- include "currents.clickhouseRestoreModeEnv" . | nindent 8 }}
+        {{- with (concat .Values.global.env .Values.toolbox.env) }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
+      volumeMounts:
+        - name: data
+          mountPath: /data
+        # Writable /tmp for the local merge FIFOs (the transformed BSON flows through the
+        # kernel pipe, not to disk). Needed because the root FS is read-only.
+        - name: tmp
+          mountPath: /tmp
+      {{- with .Values.toolbox.resources }}
+      resources:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  volumes:
+    - name: data
+      persistentVolumeClaim:
+        claimName: {{ include "currents.toolbox.fullname" . }}-data
+    - name: tmp
+      emptyDir:
+        medium: Memory
+  {{- with .Values.toolbox.nodeSelector | default .Values.global.nodeSelector }}
+  nodeSelector:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.toolbox.tolerations | default .Values.global.tolerations }}
+  tolerations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.toolbox.affinity | default .Values.global.affinity }}
+  affinity:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/currents/templates/toolbox/pvc.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.toolbox.enabled }}
+{{/*
+Scratch space for the toolbox pod.
+
+For an org import this holds the export artifact (*.bson, *.native,
+manifest.json) plus the exactly-once import state file, so size it at roughly
+1.5x the export's manifest.totals.exportedBytes + clickhouseTotals.exportedBytes.
+
+It deliberately outlives individual exec sessions: the ClickHouse import records
+which tables it has already inserted in /data/.import-state.json, and that record
+is what prevents a re-run from double-counting the AggregatingMergeTree rollups.
+Deleting this volume mid-import destroys that guard.
+*/}}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ include "currents.toolbox.fullname" . }}-data
+  labels:
+    {{- include "currents.labels" (dict "context" . "component" .Values.toolbox.name) | nindent 4 }}
+  annotations:
+    # Keep the volume (and the import state file) if the release is uninstalled
+    # while an import is in flight.
+    helm.sh/resource-policy: keep
+spec:
+  accessModes:
+    - {{ .Values.toolbox.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.toolbox.persistence.size | quote }}
+{{- if .Values.toolbox.persistence.volumeName }}
+  volumeName: {{ .Values.toolbox.persistence.volumeName }}
+{{- end }}
+{{- if .Values.toolbox.persistence.storageClass }}
+  storageClassName: "{{ .Values.toolbox.persistence.storageClass }}"
+{{- end }}
+{{- end }}

charts/currents/values.yaml

@@ -665,6 +665,43 @@ webhooks:
   affinity: {}
 # END Webhooks Configuration
 
+# Toolbox Configuration
+# An optional, manually-driven pod for one-off operational work (data import,
+# ad-hoc maintenance). It runs `sleep infinity` and does nothing on its own —
+# an operator execs in and drives it, then disables it again.
+toolbox:
+  # -- Create the toolbox pod and its scratch PVC. Leave disabled for normal
+  # installs — with this off, nothing in this section renders.
+  enabled: false
+  name: toolbox
+  # -- Scratch space for artifacts and the import state file. Size it at ~1.5x
+  # the export's total bytes (manifest totals.exportedBytes +
+  # clickhouseTotals.exportedBytes).
+  # @default -- See [values.yaml] for default values
+  persistence:
+    accessMode: ReadWriteOnce
+    size: 20Gi
+    storageClass: ""
+    ## if volumeName is set, use this existing PersistentVolume
+    # volumeName:
+  # -- Additional environment variables for toolbox containers.
+  env: []
+  # -- ClickHouse client per-request socket timeout (ms) for the import.
+  clickhouseRequestTimeoutMs: 3600000
+  # -- Resource limits for the toolbox containers. A large import is IO-bound;
+  # give it enough memory to stream comfortably.
+  resources: {}
+  # -- [Node selector] for the toolbox pod
+  # @default -- `{}` (defaults to global.nodeSelector)
+  nodeSelector: {}
+  # -- [Tolerations] for use with node taints
+  # @default -- `[]` (defaults to global.tolerations)
+  tolerations: []
+  # -- Assign custom [affinity] rules to the pod
+  # @default -- `{}` (defaults to the global.affinity preset)
+  affinity: {}
+# END Toolbox Configuration
+
 # Service Account Configuration
 # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
 serviceAccount:
@@ -709,3 +746,26 @@ redis:
   sysctl:
     resourcesPreset: "none"
 # END Redis Configuration
+
+# Maintenance Configuration
+# Special-purpose switches for one-off operational procedures. Everything here is
+# expected to be OFF during normal operation — turn it on for the duration of a
+# procedure, then turn it back off.
+maintenance:
+  # -- Suppress change-stream-driven ClickHouse sync while an organization's
+  # ClickHouse data is restored from an external export (see scripts/org-import
+  # in the currents repo). Enable it together with `toolbox.enabled` for a
+  # Mongo + ClickHouse restore, then turn both off again.
+  #
+  # Without it, restoring documents into Mongo makes change-streams re-derive
+  # ClickHouse rows on top of the ones the import writes directly, permanently
+  # double-counting the hourly materialized views.
+  #
+  # Leave this OFF for a Mongo-only restore: there, change-streams re-deriving
+  # the restored documents is exactly how ClickHouse gets populated.
+  #
+  # NOTE: while this is on, NO org gets new ClickHouse metrics — the install
+  # keeps recording to Mongo while dashboards quietly stop updating. It is not
+  # a setting to leave enabled.
+  clickhouseRestoreMode: false
+# END Maintenance Configuration

docs/README.md

@@ -8,6 +8,7 @@ The Currents Helm Chart is stateless, so depends on being connected to stateful
 
 - [🚀 Start Here: EKS Quickstart](./eks/quickstart.md)
 - [EKS Upgrade Guide](./eks/upgrading.md)
+- [Importing Organization Data (Cloud → Self-Hosted Migration)](./org-data-import.md)
 - [Development Guide](./developer-guide/README.md)
 - [Support Policy](./support.md)
 - [Configuration Reference](configuration.md)

docs/configuration.md

@@ -213,6 +213,15 @@ The following table lists the configurable parameters of the `currents` chart an
 | webhooks.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] |
 | webhooks.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints |
 | webhooks.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the deployment |
+| toolbox.enabled | bool | `false` | Create the toolbox pod and its scratch PVC. Leave disabled for normal installs — with this off, nothing in this section renders. |
+| toolbox.name | string | `"toolbox"` |  |
+| toolbox.persistence | object | See [values.yaml] for default values | Scratch space for artifacts and the import state file. Size it at ~1.5x the export's total bytes (manifest totals.exportedBytes + clickhouseTotals.exportedBytes). |
+| toolbox.env | list | `[]` | Additional environment variables for toolbox containers. |
+| toolbox.clickhouseRequestTimeoutMs | int | `3600000` | ClickHouse client per-request socket timeout (ms) for the import. |
+| toolbox.resources | object | `{}` | Resource limits for the toolbox containers. A large import is IO-bound; give it enough memory to stream comfortably. |
+| toolbox.nodeSelector | object | `{}` (defaults to global.nodeSelector) | [Node selector] for the toolbox pod |
+| toolbox.tolerations | list | `[]` (defaults to global.tolerations) | [Tolerations] for use with node taints |
+| toolbox.affinity | object | `{}` (defaults to the global.affinity preset) | Assign custom [affinity] rules to the pod |
 | serviceAccount.create | bool | `true` | Specifies whether a service account should be created |
 | serviceAccount.name | string | If not set and create is true, a name is generated using the fullname template | The name of the service account to use. |
 | serviceAccount.annotations | object | `{}` | Optional additional annotations to add to the Service Account. Templates are allowed for both keys and values. |
@@ -228,5 +237,6 @@ The following table lists the configurable parameters of the `currents` chart an
 | redis.metrics.resourcesPreset | string | `"none"` |  |
 | redis.volumePermissions.resourcesPreset | string | `"none"` |  |
 | redis.sysctl.resourcesPreset | string | `"none"` |  |
+| maintenance.clickhouseRestoreMode | bool | `false` | Suppress change-stream-driven ClickHouse sync while an organization's ClickHouse data is restored from an external export (see scripts/org-import in the currents repo). Enable it together with `toolbox.enabled` for a Mongo + ClickHouse restore, then turn both off again.  Without it, restoring documents into Mongo makes change-streams re-derive ClickHouse rows on top of the ones the import writes directly, permanently double-counting the hourly materialized views.  Leave this OFF for a Mongo-only restore: there, change-streams re-deriving the restored documents is exactly how ClickHouse gets populated.  NOTE: while this is on, NO org gets new ClickHouse metrics — the install keeps recording to Mongo while dashboards quietly stop updating. It is not a setting to leave enabled. |
 
 [values.yaml]: https://github.com/currents-dev/helm-charts/blob/main/charts/currents/values.yaml