Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
@@ -1,25 +1,16 @@
# Cargo build configuration for dfe-transform-elastic
#
# Target platforms: Linux x86_64 and aarch64 only.
# SIMD matters here: simd-json, memchr and aho-corasick all dispatch on it.
# Why these targets: docs/performance.md.

# x86_64: AVX2, BMI1/2, FMA -- Haswell and later.
[target.x86_64-unknown-linux-gnu]
rustflags = ["-C", "target-cpu=x86-64-v3"]

# aarch64: NEON is baseline, so generic already has it.
#
# mold rather than the default bfd, because a PGO-instrumented build of this
# binary overflows AArch64's 128 MB direct branch and bfd cannot reach past it
# (hyperi-io/hyperi-ci#142). mold inserts the range-extension thunks.
#
# Not lld, which hyperi-ci installs only under its version-suffixed name, so
# gcc's `-fuse-ld=lld` finds no `ld.lld` and collect2 fails the link outright.
# mold: a PGO build overflows AArch64's direct branch range under bfd, and lld is not on the runners (hyperi-io/hyperi-ci#142).
[target.aarch64-unknown-linux-gnu]
linker = "aarch64-linux-gnu-gcc"
rustflags = ["-C", "target-cpu=generic", "-C", "link-arg=-fuse-ld=mold"]

# Untargeted local builds get the build host's own features. The per-target
# sections above win whenever `--target` is passed, which is what CI does.
# Untargeted local builds. CI passes `--target`, so the sections above win there.
[build]
rustflags = ["-C", "target-cpu=native"]
27 changes: 6 additions & 21 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@
name: CI

on:
# No nightly schedule: no test here is #[ignore]d, so the full tier is the core
# tier every push already runs (docs/testing.md).
# No nightly schedule: no test is #[ignore]d, so core is the full tier (docs/testing.md).
push:
branches: ["**"]
paths-ignore:
Expand Down Expand Up @@ -70,25 +69,18 @@ jobs:
ci:
uses: hyperi-io/hyperi-ci/.github/workflows/rust-ci.yml@main
with:
# No runner-quality: one dfe-transforms rustc peaks near 14 GiB, which a
# 16 GiB 8x16 runner kills, so Quality stays on the org's GH_RUNNER_RUST.
# `both` resolves to the release channel; `internal` is the spike
# channel and would demote a release build silently.
# No runner-quality: a 16 GiB runner kills a 14 GiB rustc. `internal` would demote a release to the spike channel.
publish-target: both
tag: ${{ inputs.tag || '' }}
from-head: ${{ inputs.from-head || '' }}
bump: ${{ inputs.bump || 'auto' }}
skip-optimize: ${{ inputs.skip-optimize || '' }}
release-unoptimized: ${{ inputs.release-unoptimized || '' }}
optimize-tier: ${{ inputs.optimize-tier || '' }}
# A PGO+BOLT release builds the binary three times, ending in a single-threaded LTO pass; one build took over three hours.
# A PGO and BOLT release builds the binary three times.
build-timeout-minutes: 360
test-tier: ${{ inputs.test-tier || 'core' }}
# Least privilege: the org carries 31+ secrets and this pipeline uses seven.
# Only names declared in rust-ci.yml's `workflow_call.secrets` can be
# passed, so an addition there is needed before one can be added here.
# No crates.io token: the package has binary targets and publishes no
# crate, so the preflight never asks for one.
# Least privilege: only the secrets this pipeline uses. No crates.io token, nothing is published there.
secrets:
GITLEAKS_GH_ACTIONS_KEY: ${{ secrets.GITLEAKS_GH_ACTIONS_KEY }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
Expand All @@ -98,15 +90,8 @@ jobs:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}

# The engine offers `sources.yaml` as its source catalogue and content-pins it
# the way it pins dfe-schemas, so a release has to carry the file (issue #18).
# hyperi-ci has no release-asset stanza and this needs no change there: the
# upload lives in this repo's own workflow.
#
# NOT triggered on `release: published` -- semantic-release creates the
# release with a token whose events cannot start another workflow run, so
# that trigger would never fire. This gates on the same events that can
# publish and then proves a release for THIS commit exists before uploading.
# The engine pins `sources.yaml` from the release, so a release carries it (issue #18).
# Not on `release: published`: semantic-release's token cannot start a workflow run.
catalogue-asset:
name: Source catalogue asset
needs: ci
Expand Down
10 changes: 2 additions & 8 deletions .github/workflows/sync-elastic-variants.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,7 @@
# License: BUSL-1.1
# Copyright: (c) 2026 HYPERI PTY LIMITED
#
# dfe-schemas checks every elastic catalogue entry against that file, and the contract this
# service publishes names its variants but not the intakes each one accepts.
#
# Auth: a GitHub App installation token (vars.GH_APP_CLIENT_ID + secrets.GH_APP_PRIVATE_KEY),
# scoped to dfe-schemas with contents + pull-requests write, which covers the branch push
# and opening, editing or closing the PR.
# Optional var: DFE_SCHEMAS_REPO - repo as owner/name (default: hyperi-io/dfe-schemas)
# Why: CONTRIBUTING.md, "The catalogue and the detect tables". Optional var DFE_SCHEMAS_REPO (owner/name).

name: Sync elastic variants

Expand All @@ -36,7 +30,7 @@ concurrency:
jobs:
push-to-dfe-schemas:
name: Open or update the dfe-schemas sync PR
# scripts/sources.py reads with PyYAML, which the native runner image's python3 carries and the vanilla default may not.
# The native runner's python3 has PyYAML, which scripts/sources.py needs.
runs-on: ${{ vars.GH_RUNNER_PYTHON || 'ubuntu-latest' }}
timeout-minutes: 10
steps:
Expand Down
15 changes: 2 additions & 13 deletions .gitleaks.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@
# License: BUSL-1.1
# Copyright: (c) 2026 HYPERI PTY LIMITED
#
# Extends the CI base configuration (ci/.gitleaks.toml)
#
# The CI runs gitleaks to detect secrets in your code. This file lets you
# exclude false positives specific to your project.
#
# Documentation: https://github.com/gitleaks/gitleaks#configuration

[extend]
Expand All @@ -19,23 +14,17 @@ useDefault = true
[allowlist]
description = "Project-specific false positive allowlist"

# Paths to exclude from scanning (regex patterns)
paths = [
# Git submodules (have their own scanning)
'''(^|/)ci/''',
'''(^|/)ai/''',
# Vendor sample logs -- synthetic ids that trip the entropy rules.
# Sample logs: invented ids trip the entropy rules.
'''(^|/)tests/fixtures/''',
'''(^|/)tests/pipelines/''',
]

# Regex patterns for known false positives
regexes = [
# Meraki event-type literal carried through from the Elastic pipeline.
'''8021x_eap_failure''',
# Confluent's PUBLIC signing-key fingerprint, pinned by the generated
# Dockerfile and carried into docs/deployment-contract.json. Publishing it
# is the security control -- the build verifies the downloaded key against
# it before trusting the apt repo.
# Confluent's public signing-key fingerprint, which the Dockerfile verifies.
'''CBBB821E8FAF364F79835C438B1DA6120C2BF624''',
]
45 changes: 9 additions & 36 deletions .hyperi-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,81 +5,54 @@
# License: BUSL-1.1
# Copyright: (c) 2026 HYPERI PTY LIMITED

# Repo category. Matches the `classification` custom property on the org.
# Values: internal, product, fork, general-oss
classification: product

# Project language (auto-detected if not specified)
# Values: python, typescript, golang, rust, bash, none
language: rust

# Quality checks
quality:
enabled: true
ignore:
# GHSA-w9wp-h8wv-79jx / CVE-2026-48504: opentelemetry_sdk 0.31.0 is in the
# lock through scalo, in the affected range (<= 0.32.0, fixed in 0.32.1).
# The flaw is in BaggagePropagator::extract_with_context, which parses an
# inbound W3C Baggage header before enforcing its size limits. Neither this
# repo nor scalo registers a Baggage or other text map propagator, so it is
# never called. Recorded as not reachable in the DFE threat model (dfe-infra
# docs/THREAT-MODEL.md, accepted risks). It is a GitHub advisory with no
# RustSec entry, so only osv-scanner sees it. scalo holds 0.31 because
# metrics-exporter-opentelemetry 0.2.1 requires opentelemetry ^0.31;
# tracked in hyperi-io/scalo-rs#281.
- tool: osv-scanner
id: GHSA-w9wp-h8wv-79jx
reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator registered in dfe-transform-elastic or scalo. Not reachable per dfe-infra docs/THREAT-MODEL.md accepted risks. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281."

# Testing
test:
enabled: true
coverage: true
rust:
# Required, not auto: the memory and cascade tests read process-wide state, and only nextest gives each test a process of its own.
# Required: the memory and cascade tests need a process each (docs/testing.md).
nextest: true

# Building
build:
enabled: true
strategies:
- native
rust:
glibc_max: "2.41"
# librdkafka and OpenSSL link statically from hyperi-ci's prefix, so the
# image installs no C library packages.
# librdkafka and OpenSSL link statically, so the image installs no C libraries.
static_libs: true
# The arm64 leg needs a larger runner for MEMORY, not cores. `dfe-transforms`
# carries one module per data stream and holds 13.15 GiB in a single rustc at
# opt-level 1; the hosted `ubuntu-24.04-arm` is 2 vCPU and 8 GiB on a private
# repo, so the build was OOM-killed after 12 minutes. A repo-level
# `GH_RUNNER_ARM64` points this repo at a 16-core, 64 GiB runner (issue #27).
# arm64 builds on the larger runner `GH_RUNNER_ARM64` names, for memory (issue #27).
targets:
- x86_64-unknown-linux-gnu
- aarch64-unknown-linux-gnu
# One rustc here peaks near 14 GiB, and clippy --all-targets builds the lib and its test target at once, so a 24 GiB runner fits one job.
# One rustc here peaks near 14 GiB.
memory_per_job_gib: 14
# The cost lands on a RELEASE and never on a PR: the build half only runs
# behind the `Publish: true` trailer, a `hyperi-ci publish` dispatch, or a
# dispatch with `optimize-tier: release`, so the day-to-day loop is untouched.
# Release builds only, never a PR (CONTRIBUTING.md, "CI is slow here").
optimize:
pgo:
enabled: true
# 600s of load, 300s per source. The prefix holds it there over the duration_secs hyperi-ci exports.
# The prefix holds the load at 600 s over the duration_secs hyperi-ci exports.
workload_cmd: "env PGO_WORKLOAD_DURATION_SECS=600 bash scripts/pgo-workload.sh"
# Built before the workload clock starts, so the script's own cargo build finds it fresh.
# Built before the workload clock starts.
workload_setup_cmd: "cargo build --release -p pgo-driver --features driver"
# hyperi-ci's timeout is this plus 600s, which covers the broker pull as well as the load.
# hyperi-ci's timeout is this plus 600 s, which covers the broker pull.
duration_secs: 900
bolt:
enabled: true

# Publishing (on release). The stack pins every app by GHCR image digest, so
# a release must land an image, and the thin chart assembled from the emitted
# contract ships beside it.
release:
enabled: true
# Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes.
# Keep `library` at the scalo version in Cargo.toml (CONTRIBUTING.md, "Generated artefacts").
helm:
enabled: true
contract: emit
Expand Down
Loading
Loading