diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 0e61403..a70e7ba 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -17,130 +17,145 @@ export default withMermaid(defineConfig({ siteTitle: 'Edwin Philip', nav: [ { text: 'Home', link: '/' }, - { text: 'About & Experience', link: '/about' }, - { text: 'Knowledge Base', link: '/guides/' }, - { text: 'Networking', link: '/guides/networking/' }, - { text: 'Linux Internals', link: '/guides/linux-internals/' }, - { text: 'Kubernetes', link: '/guides/kubernetes/' }, - { text: 'Terraform', link: '/guides/terraform/' }, - { text: 'GCP Fundamentals', link: '/guides/gcp/' }, - { text: 'GKE Architecture', link: '/guides/gke/' }, - { text: 'GKE Operations', link: '/guides/gke/' } + { text: 'About me', link: '/about' }, + { text: 'GitHub', link: 'https://github.com/EdwinPhilip' }, + { text: 'LinkedIn', link: 'https://www.linkedin.com/in/edwin-philip-sudo/' } ], sidebar: { - '/guides/networking/': [ - { text: 'Networking Fundamentals', link: '/guides/networking/' }, - { text: 'Complete Module 1', items: [ - { text: '1. HTTPS request lifecycle', link: '/guides/networking/q1' }, - { text: '2. TCP fundamentals', link: '/guides/networking/q2' }, - { text: '3. DNS internals', link: '/guides/networking/q3' }, - { text: '4. TLS and PKI', link: '/guides/networking/q4' }, - { text: '5. Load balancers and proxies', link: '/guides/networking/q5' }, - { text: '6. CIDR, routing and NAT', link: '/guides/networking/q6' }, - { text: '7. Kubernetes and GKE networking', link: '/guides/networking/q7' }, - { text: '8. Production troubleshooting', link: '/guides/networking/q8' }, - ] } - ], - '/guides/linux-internals/': [ - { text: 'Linux Internals', link: '/guides/linux-internals/' }, - { text: 'Complete Module 2', items: [ - { text: '1. Processes & scheduling', link: '/guides/linux-internals/q1' }, - { text: '2. Virtual memory', link: '/guides/linux-internals/q2' }, - { text: '3. Cgroups & OOM', link: '/guides/linux-internals/q3' }, - { text: '4. CPU quotas & PSI', link: '/guides/linux-internals/q4' }, - { text: '5. Storage & durability', link: '/guides/linux-internals/q5' }, - { text: '6. File descriptors & epoll', link: '/guides/linux-internals/q6' }, - { text: '7. Incident investigation', link: '/guides/linux-internals/q7' }, + '/guides/': [ + { text: 'Getting Started', collapsed: false, items: [ + { text: 'Knowledge Base', link: '/guides/' } + ] }, + { text: 'Module 1 · Networking Fundamentals', link: '/guides/networking/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/networking/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. HTTPS request lifecycle', link: '/guides/networking/q1' }, + { text: '2. TCP fundamentals', link: '/guides/networking/q2' }, + { text: '3. DNS internals', link: '/guides/networking/q3' }, + { text: '4. TLS and PKI', link: '/guides/networking/q4' }, + { text: '5. Load balancers and proxies', link: '/guides/networking/q5' }, + { text: '6. CIDR, routing and NAT', link: '/guides/networking/q6' }, + { text: '7. Kubernetes and GKE networking', link: '/guides/networking/q7' }, + { text: '8. Production troubleshooting', link: '/guides/networking/q8' }, + ] }, + ] }, + { text: 'Module 2 · Linux Internals', link: '/guides/linux-internals/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/linux-internals/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. Processes & scheduling', link: '/guides/linux-internals/q1' }, + { text: '2. Virtual memory', link: '/guides/linux-internals/q2' }, + { text: '3. Cgroups & OOM', link: '/guides/linux-internals/q3' }, + { text: '4. CPU quotas & PSI', link: '/guides/linux-internals/q4' }, + { text: '5. Storage & durability', link: '/guides/linux-internals/q5' }, + { text: '6. File descriptors & epoll', link: '/guides/linux-internals/q6' }, + { text: '7. Incident investigation', link: '/guides/linux-internals/q7' }, + ] }, { text: 'Hands-on labs', link: '/guides/linux-internals/labs' }, - { text: '35 difficult follow-ups', link: '/guides/linux-internals/follow-ups' } - ] } - ], - '/guides/kubernetes/': [ - { text: 'Kubernetes Architecture & Operations', link: '/guides/kubernetes/' }, - { text: 'Complete Module 3', items: [ - { text: '1. Architecture', link: '/guides/kubernetes/q1' }, - { text: '2. Deployment lifecycle', link: '/guides/kubernetes/q2' }, - { text: '3. etcd and Raft', link: '/guides/kubernetes/q3' }, - { text: '4. Controllers and operators', link: '/guides/kubernetes/q4' }, - { text: '5. Scheduler', link: '/guides/kubernetes/q5' }, - { text: '6. Workload controllers', link: '/guides/kubernetes/q6' }, - { text: '7. Resources and QoS', link: '/guides/kubernetes/q7' }, - { text: '8. Probes and lifecycle', link: '/guides/kubernetes/q8' }, - { text: '9. CSI and storage', link: '/guides/kubernetes/q9' }, - { text: '10. Autoscaling', link: '/guides/kubernetes/q10' }, - { text: '11. Security and admission', link: '/guides/kubernetes/q11' }, - { text: '12. Upgrades and troubleshooting', link: '/guides/kubernetes/q12' }, + { text: '35 difficult follow-ups', link: '/guides/linux-internals/follow-ups' }, + ] }, + { text: 'Module 3 · Kubernetes Architecture', link: '/guides/kubernetes/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/kubernetes/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. Architecture', link: '/guides/kubernetes/q1' }, + { text: '2. Deployment lifecycle', link: '/guides/kubernetes/q2' }, + { text: '3. etcd and Raft', link: '/guides/kubernetes/q3' }, + { text: '4. Controllers and operators', link: '/guides/kubernetes/q4' }, + { text: '5. Scheduler', link: '/guides/kubernetes/q5' }, + { text: '6. Workload controllers', link: '/guides/kubernetes/q6' }, + { text: '7. Resources and QoS', link: '/guides/kubernetes/q7' }, + { text: '8. Probes and lifecycle', link: '/guides/kubernetes/q8' }, + { text: '9. CSI and storage', link: '/guides/kubernetes/q9' }, + { text: '10. Autoscaling', link: '/guides/kubernetes/q10' }, + { text: '11. Security and admission', link: '/guides/kubernetes/q11' }, + { text: '12. Upgrades and troubleshooting', link: '/guides/kubernetes/q12' }, + ] }, { text: 'Hands-on labs', link: '/guides/kubernetes/labs' }, - { text: '42 difficult follow-ups', link: '/guides/kubernetes/follow-ups' } - ] } - ], - '/guides/terraform/': [ - { text: 'Terraform Architecture, AWS and GCP', link: '/guides/terraform/' }, - { text: 'Complete Module 4', items: [ - { text: '1. Terraform fundamentals', link: '/guides/terraform/q1' }, - { text: '2. Dependency graph and planning', link: '/guides/terraform/q2' }, - { text: '3. HCL and resource identity', link: '/guides/terraform/q3' }, - { text: '4. State and remote backends', link: '/guides/terraform/q4' }, - { text: '5. Modules and providers', link: '/guides/terraform/q5' }, - { text: '6. Terraform on AWS', link: '/guides/terraform/q6' }, - { text: '7. Terraform on Google Cloud', link: '/guides/terraform/q7' }, - { text: '8. Import and refactoring', link: '/guides/terraform/q8' }, - { text: '9. Drift and recovery', link: '/guides/terraform/q9' }, - { text: '10. Security, secrets and policy', link: '/guides/terraform/q10' }, - { text: '11. CI/CD and Atlantis', link: '/guides/terraform/q11' }, - { text: '12. Enterprise platform design', link: '/guides/terraform/q12' }, + { text: '42 difficult follow-ups', link: '/guides/kubernetes/follow-ups' }, + ] }, + { text: 'Module 4 · Terraform', link: '/guides/terraform/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/terraform/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. Terraform fundamentals', link: '/guides/terraform/q1' }, + { text: '2. Dependency graph and planning', link: '/guides/terraform/q2' }, + { text: '3. HCL and resource identity', link: '/guides/terraform/q3' }, + { text: '4. State and remote backends', link: '/guides/terraform/q4' }, + { text: '5. Modules and providers', link: '/guides/terraform/q5' }, + { text: '6. Terraform on AWS', link: '/guides/terraform/q6' }, + { text: '7. Terraform on Google Cloud', link: '/guides/terraform/q7' }, + { text: '8. Import and refactoring', link: '/guides/terraform/q8' }, + { text: '9. Drift and recovery', link: '/guides/terraform/q9' }, + { text: '10. Security, secrets and policy', link: '/guides/terraform/q10' }, + { text: '11. CI/CD and Atlantis', link: '/guides/terraform/q11' }, + { text: '12. Enterprise platform design', link: '/guides/terraform/q12' }, + ] }, { text: 'Hands-on labs', link: '/guides/terraform/labs' }, - { text: '45 difficult follow-ups', link: '/guides/terraform/follow-ups' } - ] } - ], - '/guides/gcp/': [ - { text: 'GCP Fundamentals and Production Engineering', link: '/guides/gcp/' }, - { text: 'Complete Module 5', items: [ - { text: '1. Resource hierarchy', link: '/guides/gcp/q1' }, - { text: '2. IAM and workload identity', link: '/guides/gcp/q2' }, - { text: '3. VPC, Shared VPC and firewall', link: '/guides/gcp/q3' }, - { text: '4. NAT, DNS and load balancing', link: '/guides/gcp/q4' }, - { text: '5. Compute Engine and MIGs', link: '/guides/gcp/q5' }, - { text: '6. Storage, artifacts and data', link: '/guides/gcp/q6' }, - { text: '7. Observability and SLOs', link: '/guides/gcp/q7' }, - { text: '8. Architecture and recovery', link: '/guides/gcp/q8' }, + { text: '45 difficult follow-ups', link: '/guides/terraform/follow-ups' }, + ] }, + { text: 'Module 5 · Google Cloud', link: '/guides/gcp/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/gcp/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. Resource hierarchy', link: '/guides/gcp/q1' }, + { text: '2. IAM and workload identity', link: '/guides/gcp/q2' }, + { text: '3. VPC, Shared VPC and firewall', link: '/guides/gcp/q3' }, + { text: '4. NAT, DNS and load balancing', link: '/guides/gcp/q4' }, + { text: '5. Compute Engine and MIGs', link: '/guides/gcp/q5' }, + { text: '6. Storage, artifacts and data', link: '/guides/gcp/q6' }, + { text: '7. Observability and SLOs', link: '/guides/gcp/q7' }, + { text: '8. Architecture and recovery', link: '/guides/gcp/q8' }, + ] }, { text: 'Hands-on labs', link: '/guides/gcp/labs' }, - { text: '40 challenging follow-ups', link: '/guides/gcp/follow-ups' } - ] } - ], - '/guides/gke/': [ - { text: 'GKE Architecture & Production Operations', link: '/guides/gke/' }, - { text: 'Complete Module 6', items: [ - { text: '1. Architecture and control plane', link: '/guides/gke/q1' }, - { text: '2. Node pools and scheduling', link: '/guides/gke/q2' }, - { text: '3. VPC-native networking', link: '/guides/gke/q3' }, - { text: '4. Ingress, Gateway and NEGs', link: '/guides/gke/q4' }, - { text: '5. Security and workload identity', link: '/guides/gke/q5' }, - { text: '6. CSI storage and StatefulSets', link: '/guides/gke/q6' }, - { text: '7. Autoscaling and QoS', link: '/guides/gke/q7' }, - { text: '8. Upgrades and incidents', link: '/guides/gke/q8' }, + { text: '40 challenging follow-ups', link: '/guides/gcp/follow-ups' }, + ] }, + { text: 'Module 6 · GKE Operations', link: '/guides/gke/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/gke/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. Architecture and control plane', link: '/guides/gke/q1' }, + { text: '2. Node pools and scheduling', link: '/guides/gke/q2' }, + { text: '3. VPC-native networking', link: '/guides/gke/q3' }, + { text: '4. Ingress, Gateway and NEGs', link: '/guides/gke/q4' }, + { text: '5. Security and workload identity', link: '/guides/gke/q5' }, + { text: '6. CSI storage and StatefulSets', link: '/guides/gke/q6' }, + { text: '7. Autoscaling and QoS', link: '/guides/gke/q7' }, + { text: '8. Upgrades and incidents', link: '/guides/gke/q8' }, + ] }, { text: 'Hands-on labs', link: '/guides/gke/labs' }, - { text: '40 challenging follow-ups', link: '/guides/gke/follow-ups' } - ] } - ], - '/guides/': [ - { text: 'Learning Guides', items: [ - { text: 'All guides', link: '/guides/' }, - { text: 'Networking Fundamentals', link: '/guides/networking/' }, - { text: 'Linux Internals', link: '/guides/linux-internals/' }, - { text: 'Kubernetes Architecture', link: '/guides/kubernetes/' }, - { text: 'Terraform Architecture', link: '/guides/terraform/' }, - { text: 'GCP Fundamentals', link: '/guides/gcp/' } - ] } + { text: '40 challenging follow-ups', link: '/guides/gke/follow-ups' }, + ] }, + { text: 'Module 7 · Cloud Reliability', link: '/guides/reliability/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/reliability/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. SLIs, SLOs and error budgets', link: '/guides/reliability/q1' }, + { text: '2. High availability and zones', link: '/guides/reliability/q2' }, + { text: '3. Backpressure and retries', link: '/guides/reliability/q3' }, + { text: '4. Transaction correctness', link: '/guides/reliability/q4' }, + { text: '5. Multi-region and disaster recovery', link: '/guides/reliability/q5' }, + { text: '6. Peak capacity and load testing', link: '/guides/reliability/q6' }, + { text: '7. Cache and data resilience', link: '/guides/reliability/q7' }, + { text: '8. Incident command and game days', link: '/guides/reliability/q8' }, + ] }, + { text: 'Seven hands-on labs', link: '/guides/reliability/labs' }, + { text: '40 difficult follow-ups', link: '/guides/reliability/follow-ups' }, + ] }, + { text: 'Module 8 · Platform Engineering', link: '/guides/platform-engineering/', collapsed: true, items: [ + { text: 'Overview', link: '/guides/platform-engineering/' }, + { text: 'Chapters', collapsed: false, items: [ + { text: '1. CI/CD and trusted builds', link: '/guides/platform-engineering/q1' }, + { text: '2. GitOps and Argo CD', link: '/guides/platform-engineering/q2' }, + { text: '3. Progressive delivery', link: '/guides/platform-engineering/q3' }, + { text: '4. Supply-chain security', link: '/guides/platform-engineering/q4' }, + { text: '5. Identities and secrets', link: '/guides/platform-engineering/q5' }, + { text: '6. Developer platforms', link: '/guides/platform-engineering/q6' }, + { text: '7. Platform observability', link: '/guides/platform-engineering/q7' }, + { text: '8. Multi-cluster governance', link: '/guides/platform-engineering/q8' }, + ] }, + { text: 'Eight hands-on labs', link: '/guides/platform-engineering/labs' }, + { text: '40 difficult follow-ups', link: '/guides/platform-engineering/follow-ups' }, + ] }, ] }, - socialLinks: [ - { icon: 'github', link: 'https://github.com/EdwinPhilip' }, - { icon: 'linkedin', link: 'https://www.linkedin.com/in/edwin-philip-sudo/' } - ], search: { provider: 'local' }, footer: { message: 'Practical engineering knowledge, openly shared.', copyright: '© 2026 Edwin Philip' }, editLink: { pattern: 'https://github.com/EdwinPhilip/EdwinPhilip.github.io/edit/main/docs/:path', text: 'Suggest an improvement' }, - outline: { level: [2, 3] } + outline: { level: [2, 3], label: 'On this page' } } })) \ No newline at end of file diff --git a/docs/guides/index.md b/docs/guides/index.md index f903458..fe5cb40 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -30,8 +30,16 @@ Eight comprehensive chapters covering GCP hierarchy, identity and federation, Sh Eight in-depth chapters on GKE control plane, Standard/Autopilot, node scheduling, VPC-native networking/eBPF, Ingress/Gateway/NEGs, workload identity, CSI, autoscaling, upgrades and incident recovery. Includes [eight labs](/guides/gke/labs) and [40 difficult follow-ups](/guides/gke/follow-ups). +### [Cloud Architecture, Reliability & System Design · Module 7](/guides/reliability/) + +Eight in-depth chapters covering SLIs/SLOs, high availability, failure domains, backpressure, durable transaction correctness, multi-region disaster recovery, capacity, data-layer resilience and incident leadership. Includes [40 difficult follow-ups](/guides/reliability/follow-ups) and [seven hands-on labs](/guides/reliability/labs). + +### [CI/CD, GitOps & Platform Engineering · Module 8](/guides/platform-engineering/) + +Eight chapters on trusted builds, GitOps reconciliation, progressive delivery, supply-chain security, federated identity, internal developer platforms, observability and multi-cluster governance. Includes [40 difficult follow-ups](/guides/platform-engineering/follow-ups) and [eight hands-on labs](/guides/platform-engineering/labs). + ## Planned -Reliability, CI/CD and platform engineering, and Python data processing. +Module 9: Python coding and data processing. [About the author](/about) diff --git a/docs/guides/platform-engineering/follow-ups.md b/docs/guides/platform-engineering/follow-ups.md new file mode 100644 index 0000000..6a44a6f --- /dev/null +++ b/docs/guides/platform-engineering/follow-ups.md @@ -0,0 +1,245 @@ +# Module 8 · 40 Difficult CI/CD, GitOps and Platform Engineering Follow-ups + +Questions focus on system guarantees, threat models and production trade-offs. Answer aloud before reading the model response. + +## 1. How can a green CI run deploy an untested artifact? + +*Area: CI/CD* + +**Model answer:** If the deploy job fetches a mutable tag, rebuilds source with different inputs or references the wrong branch, running bytes can differ from what CI tested. Bind tests, build provenance and deployment manifests to one immutable digest. Verify the deployed digest and preserve the source revision and builder identity in the release record. Green checks are evidence about particular inputs, not arbitrary later artifacts. + +## 2. What is the difference between continuous delivery and continuous deployment? + +*Area: CI/CD* + +**Model answer:** Continuous delivery keeps changes releasable and supports controlled promotion, which can include a human authorization step. Continuous deployment automatically promotes eligible changes to production according to policy. Both require tested artifacts, observability and recovery. The decision to automate production promotion should follow risk, business SLO and change compatibility, not a simplistic team-maturity label. + +## 3. Why is caching dependencies across trusted and untrusted CI jobs dangerous? + +*Area: CI/CD* + +**Model answer:** An untrusted PR may poison a shared writable cache with modified build outputs or dependencies later consumed by a privileged release job. Partition caches by trust boundary, validate lockfiles and avoid promoting untrusted build artifacts directly. Faster builds aren't worth compromising artifact identity. A trusted release should use verified inputs on isolated runners or safely scoped caches. + +## 4. Should the CI runner hold cluster-admin credentials to run deployment commands? + +*Area: CI/CD* + +**Model answer:** Usually no. It increases blast radius of a compromised PR script or runner. CI should publish an immutable artifact and approved release metadata; a cluster-scoped controller or tightly constrained deploy identity handles application of desired state. If direct deployment is justified, isolate the job and enforce environment approvals, OIDC, precise RBAC and audit logs. + +## 5. What are meaningful CI/CD performance metrics? + +*Area: CI/CD* + +**Model answer:** Measure time to first useful failure, queue wait, build/test duration, flake rate, reliable deployment lead time, change failure rate and safe recovery time. A fast pipeline that produces unreviewed releases or hides flaky failures isn't an improvement. Segment by service and test suite, then optimize bottlenecks rather than minimizing all tests indiscriminately. + +## 6. Does Argo CD Synced mean users can successfully place an order? + +*Area: GitOps* + +**Model answer:** No. Synced means desired and live manifests agree under diff rules. It doesn't prove the gateway routes correctly, dependencies are healthy or the business operation committed. Use readiness, synthetic user journeys, transaction outcome metrics and SLOs separately. A release gate should validate both reconciler state and customer-visible behavior. + +## 7. What happens when GitOps selfHeal is enabled and an SRE manually fixes production? + +*Area: GitOps* + +**Model answer:** The reconciler may restore the Git version of fields it owns, undoing the emergency action. Define a break-glass procedure that updates authorized desired state or pauses the correct controller owner. With ApplicationSet-managed Applications, modifying the generated Application alone may not persist. Audit the override and reconcile drift after recovery. + +## 8. Is enabling prune on every application safe? + +*Area: GitOps* + +**Model answer:** No. Prune allows deletion of managed resources that disappear from desired manifests; accidental Git deletion or bad generator output can cascade into destructive changes. Classify critical resources, use review and admission controls, plan data retention, and evaluate prune protections. A synced controller isn't authorized to delete resources whose lifecycle is owned elsewhere. + +## 9. Why might Argo CD and HPA fight over the same Deployment? + +*Area: GitOps* + +**Model answer:** If GitOps repeatedly writes spec.replicas to a fixed value while HPA changes it through the scale subresource, reconciliation can oscillate. Choose one owner for the field or configure a precise supported diff/field-management strategy. Also avoid blanket ignores that conceal real drift. Ownership contracts are as important as the Git layout. + +## 10. How can a GitOps controller recover if its own namespace is broken? + +*Area: GitOps* + +**Model answer:** Document and test an independent bootstrap/recovery path with tightly controlled credentials, backups of desired configuration, and compatible CRDs/controller versions. Avoid a circular dependency where Argo CD can only restore itself if Argo CD already works. Recover the control plane and admission prerequisites before expecting application sync to resume. + +## 11. What's the practical difference between blue-green and canary deployment? + +*Area: Progressive delivery* + +**Model answer:** Blue-green runs two stacks and switches an entry route; it offers rapid code cutback but requires spare capacity and compatibility with stateful dependencies. Canary exposes a progressively larger fraction of traffic to a candidate and can use observed metrics to abort. Both need safe schema evolution and external side-effect handling. Choose based on blast radius and routing support. + +## 12. Why might a 10% canary receive far more or less than 10% of requests? + +*Area: Progressive delivery* + +**Model answer:** Without weighted traffic-routing integration, a controller may approximate canary exposure with replica counts. Connection-level routing, sticky sessions, long-lived streams and uneven request rates can change actual proportions. Verify ingress/service-mesh behavior and observe request/version labels before interpreting comparative metrics. Weight labels alone are not proof of user exposure. + +## 13. Can you automatically rollback a database migration with the image rollback? + +*Area: Progressive delivery* + +**Model answer:** Not safely by default. New writes may rely on schema or data conversions incompatible with old binaries, and reversing migrations can destroy information. Design expand-contract schema transitions, mixed-version compatibility windows and explicit forward repair. Declare abort rules in advance and treat database recovery independently from the traffic switch. + +## 14. How do you avoid a false-positive canary abort on low traffic? + +*Area: Progressive delivery* + +**Model answer:** Require a minimum eligible sample size and appropriate observation window, compare baseline and candidate cohorts, and separate critical correctness signals from noisy percentile fluctuations. A catastrophic business correctness violation may still require immediate abort despite low samples. Use uncertainty-aware policy rather than an unqualified percentage threshold applied to three requests. + +## 15. What can a rollout AnalysisTemplate detect that readiness probes cannot? + +*Area: Progressive delivery* + +**Model answer:** It can evaluate external metrics and checks such as error ratio, p99, synthetic transaction outcomes or custom business invariants over time. Readiness mainly determines whether an individual Pod is eligible for traffic. Analysis can decide whether to continue, pause or abort fleet-level exposure, but depends on correct metrics, traffic sampling and controller behavior. + +## 16. Does signing a container image guarantee it contains safe code? + +*Area: Supply chain* + +**Model answer:** No. It authenticates a signer/digest relationship under a trust model; a compromised or malicious trusted builder can still sign harmful code. Combine review, isolated build, dependency controls, provenance, SBOM, scanning, least-privilege identity and runtime admission. Policy must specify which signer/builder and source revision are acceptable. + +## 17. What does an SBOM prove, and what does it not prove? + +*Area: Supply chain* + +**Model answer:** An SBOM inventories declared or discovered components and helps with vulnerability response and license review. It doesn't by itself prove the artifact was built from reviewed source, that the inventory is complete, or that no runtime exploit exists. Link the SBOM to an immutable digest and trusted build provenance, and verify important dependencies or build steps separately. + +## 18. Why is pull_request_target risky when untrusted code is checked out and executed? + +*Area: Supply chain* + +**Model answer:** That event can run in a privileged base-repository context; if the workflow executes attacker-controlled PR content with sensitive tokens or secrets, it crosses a dangerous trust boundary. Use lower-privilege pull_request verification, avoid secret access for untrusted scripts, and isolate any approval or labeling automation from executing PR code. + +## 19. Is a floating Docker image tag suitable as an immutable release identifier? + +*Area: Supply chain* + +**Model answer:** No. A tag can be reassigned, and replicas may pull different bytes at different times. Promote a digest tied to reviewed source, trusted provenance and an authorized release record. An emergency rollback should select a known digest, not assume an old human-readable tag still points to the same artifact. + +## 20. A dependency installs a malicious script in CI. What should incident response do beyond reverting package.json? + +*Area: Supply chain* + +**Model answer:** Freeze release promotion, inspect build logs and artifact provenance, isolate or replace compromised runners, rotate exposed credentials, inventory affected digests and deployed runtimes, then rebuild from vetted dependencies. Reverting source alone doesn't revoke stolen tokens or replace running compromised artifacts. Track affected supply-chain scope with evidence. + +## 21. Does id-token: write in GitHub Actions itself grant production cloud permissions? + +*Area: Identity* + +**Model answer:** No. It permits the job to request an OIDC identity token; the cloud identity provider's trust policy and IAM permissions determine what that token can access. Still, a broad cloud trust relationship can be dangerous. Constrain repository, ref, environment, audience and target role, and grant permission only to narrowly scoped jobs. + +## 22. Can a Kubernetes namespace alone be a strong boundary between hostile tenants? + +*Area: Identity* + +**Model answer:** Not necessarily. Namespaces scope API objects and RBAC but share a Kubernetes control plane and often nodes, CNI and privileged controllers. For hostile tenants, consider dedicated clusters/accounts plus policy enforcement, workload hardening, admission, network isolation and cloud IAM boundaries. Choose isolation strength based on threat model, not organizational convenience. + +## 23. Why should a normal HTTP workload disable ServiceAccount API token mounting? + +*Area: Identity* + +**Model answer:** If it never calls the Kubernetes API, exposing an API credential offers a compromised process an unnecessary escalation path. Disable automount or use narrowly scoped projected tokens only where needed. Cloud workload identity and Kubernetes API RBAC have separate mechanics; validate the deployed federation configuration before assuming one token does everything. + +## 24. What is the difference between runtime workload identity and CI identity? + +*Area: Identity* + +**Model answer:** CI is a short-lived build/release principal authorized to read source, publish artifacts or request controlled promotion. Runtime workload identity is tied to a deployed service and grants only the APIs required during business execution. Reusing a production deployment role inside all app Pods expands lateral movement and undermines auditability. + +## 25. What happens if a trusted admission webhook becomes unavailable? + +*Area: Identity* + +**Model answer:** For matching API requests, its failurePolicy determines whether the request is rejected or can proceed without that webhook's evaluation. Fail closed may protect security but block deployments; fail open may preserve availability but weaken enforcement. Scope webhooks carefully, run HA backends, use short timeouts and rehearse outage behavior. Never remove all security controls reflexively. + +## 26. What distinguishes an internal developer platform from an internal developer portal? + +*Area: Platform product* + +**Model answer:** A portal is one possible interface. The platform includes APIs, automation, golden paths, templates, security guardrails, operations, documentation and support. A portal with manually fulfilled tickets is not effective self-service. Measure time to usable environment, self-service completion and developer pain rather than portal page views. + +## 27. When is a golden path harmful? + +*Area: Platform product* + +**Model answer:** When the approved path cannot satisfy legitimate workload requirements and exceptions are opaque or slow, teams create shadow infrastructure. Offer reliable defaults for common cases, a documented governed escape hatch and a feedback mechanism. Reduce complexity for users without pretending unusual stateful or compliance-sensitive services are interchangeable. + +## 28. How would you model provisioning as an asynchronous platform API? + +*Area: Platform product* + +**Model answer:** Return stable resource identity and status immediately, reconcile desired state idempotently, and expose conditions such as Ready, Blocked and Failed with observedGeneration. Use finalizers for external cleanup, bounded retries and audit trails. Never report 'environment created' as 'environment usable' until service checks pass; handle partial resource creation explicitly. + +## 29. How can self-service create a cloud cost leak? + +*Area: Platform product* + +**Model answer:** Unbounded previews, orphaned storage or abandoned load balancers may persist after team tasks end. Enforce ownership, TTL, quotas, cost attribution, environment-class limits and cleanup reconciliation with alerts on age. A cleanup controller failure must be observable and recoverable; forcibly deleting finalizers may leave infrastructure behind. + +## 30. What evidence shows the platform reduces developer toil? + +*Area: Platform product* + +**Model answer:** Measure time from request to first deploy, successful self-service task completion, support requests per team, repeat incidents, wait time, changes requiring manual approval and user-reported friction. Combine quantitative metrics with interviews and workflow observation. Adoption counts alone can rise even while total engineer time lost increases. + +## 31. Why shouldn't you place a unique commit SHA in every Prometheus metric label? + +*Area: Observability* + +**Model answer:** It creates high-cardinality time series, multiplying storage, query cost and operational overhead as revisions accumulate. Use stable dimensions such as service, environment and status on metrics; keep unique IDs as trace/log attributes or exemplars when supported. Correlation remains possible without turning every deployment into a new metric universe. + +## 32. Is deployment frequency a valid performance score for an individual engineer? + +*Area: Observability* + +**Model answer:** No. DORA-style metrics describe delivery systems and organizational bottlenecks, not individual value. Optimizing individual counts rewards splitting changes or risky deployment. Use consistent team/system definitions, pair frequency with change failure and recovery, and corroborate with developer experience and product outcomes. + +## 33. What are distinct SLOs for a platform versus an application? + +*Area: Observability* + +**Model answer:** An application SLO measures customer outcomes, such as correct requests and latency. A platform SLO measures developer journeys, such as successful environment creation, artifact promotion or recovery time. A platform outage might block deployments while running apps remain healthy, or vice versa. Both need well-defined denominators and ownership. + +## 34. Can one green trace prove the rollout is healthy? + +*Area: Observability* + +**Model answer:** No. Sampling might miss rare failures, and one trace represents one operation. Combine correctly aggregated request metrics, business outcome counters, event lag, synthetic checks and targeted traces to explain failures. Traces answer 'where did this request go', while SLI accounting answers 'how often do eligible requests succeed'. + +## 35. What should be logged during a release incident without leaking secrets? + +*Area: Observability* + +**Model answer:** Record release digest, source revision, environment, workflow/run ID, gate outcome, target controller state, correlation IDs, failure reason and timestamps. Avoid raw credentials, request bodies with personal data and unbounded debug output. Establish access and retention policy. Logs should support investigation without exposing customers or deployment authority. + +## 36. How would you prevent Terraform and Argo CD from fighting over the same resource? + +*Area: Governance* + +**Model answer:** Define an ownership graph: Terraform owns cloud networks/IAM and usually cluster foundations, Argo CD owns Kubernetes workload manifests, and operators own their custom-resource-managed children. Exchange identifiers/contracts deliberately and keep field ownership explicit. Concurrent writers can oscillate or destroy state even when each tool acts according to its own desired configuration. + +## 37. What is the right rollback process after a policy change breaks every cluster? + +*Area: Governance* + +**Model answer:** Freeze conflicting automation, identify which Git/Terraform/admission policy owns the failure, preserve current state and find the narrowest safe reversal. Ensure the recovery identity is independent of the broken policy path. Apply the authorized fix, then reconcile emergency drift into version control and test the policy against representative workloads before rollout. + +## 38. Should platform teams centralize every production deployment approval? + +*Area: Governance* + +**Model answer:** Not automatically. A permanent manual gate can become a bottleneck and reduce accountability. Automate low-risk well-tested changes with policy and progressive delivery, while reserving explicit risk-based approval for high-blast-radius operations or incidents. Audit who can bypass and provide a clear escalation path for exceptional deployments. + +## 39. How do you decide whether to operate one shared or several dedicated clusters? + +*Area: Governance* + +**Model answer:** Evaluate trust requirements, failure domains, compliance, team autonomy, scale and operational overhead. A shared cluster is efficient but can concentrate privileged-controller, networking and noisy-neighbor risks. Dedicated clusters improve isolation at additional cost. Make the choice by threat model and service SLO, not solely by number of teams. + +## 40. What makes a platform incident response senior-level? + +*Area: Governance* + +**Model answer:** Name an incident commander, determine actual affected developer/customer journeys, assign technical workstreams, constrain concurrent risky changes, make a reversible mitigation and verify business outcomes. Then reconcile Git/IaC drift, rotate emergency access, and complete a learning-focused review with tested preventive actions. Senior leadership improves the system rather than simply knowing the most kubectl commands. + +[Return to Module 8](./) · [Hands-on labs](./labs) diff --git a/docs/guides/platform-engineering/index.md b/docs/guides/platform-engineering/index.md new file mode 100644 index 0000000..dc6ab52 --- /dev/null +++ b/docs/guides/platform-engineering/index.md @@ -0,0 +1,49 @@ +# CI/CD, GitOps & Platform Engineering + +**Module 8.** Eight comprehensive senior-level chapters on secure delivery architecture, pull-based deployment reconciliation, progressive rollout, artifact supply-chain security, federated identity, developer platforms, observability and multi-cluster governance. + +::: info Public learning material +All company, repository, marketplace and cloud examples are hypothetical. Manifest fragments require appropriate controllers, versions, policies and environment-specific values. This module does not disclose or describe a particular employer's systems. +::: + +## Prerequisites + +Git, CI basics, containers, Kubernetes workload lifecycle, networking, IAM fundamentals, Terraform and a basic understanding of service-level objectives. + +## Learning objectives + +- Trace source review through reproducible or verifiable build, immutable digest and controlled promotion. +- Explain Argo CD reconciliation and handle drift, pruning and emergency intervention. +- Select canary, blue-green, rolling or feature-flag rollout strategies and business-aware gates. +- Threat-model build runners, dependencies, tokens, SBOMs, signatures and provenance. +- Isolate human, CI, CD and runtime identities using short-lived credentials and least privilege. +- Design usable self-service contracts, golden paths and safe cleanup. +- Define platform user-journey SLOs, DORA-style delivery trends and telemetry cost limits. +- Govern multi-cluster change ownership, upgrades, disaster recovery and incident response. + +## Eight chapters + +1. [CI/CD architecture, trust boundaries and immutable artifacts](./q1) +2. [GitOps, Argo CD, drift and ownership](./q2) +3. [Progressive delivery and controlled rollout analysis](./q3) +4. [Software supply-chain security, SBOM and provenance](./q4) +5. [Identity, secrets and environment isolation](./q5) +6. [Internal developer platforms and self-service contracts](./q6) +7. [Platform observability and engineering effectiveness](./q7) +8. [Multi-cluster governance, recovery and technical leadership](./q8) + +## Practice and sequence + +Begin with Q1-Q4 to establish the trusted delivery chain; continue with Q5-Q6 for security and developer experience; finish with Q7-Q8 for operating a platform as a product. Work through [eight labs](./labs) and [40 difficult follow-up questions with complete answers](./follow-ups). + +## Authoritative references + +- [GitHub Actions security](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) +- [Argo CD user guide](https://argo-cd.readthedocs.io/en/stable/user-guide/) +- [Argo Rollouts analysis](https://argoproj.github.io/argo-rollouts/features/analysis/) +- [SLSA software supply chain framework](https://slsa.dev/) +- [Kubernetes Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) +- [CNCF platform engineering resources](https://tag-app-delivery.cncf.io/whitepapers/platforms/) +- [DORA](https://dora.dev/) + +[← All engineering guides](/guides/) · [Start Q1 →](./q1) diff --git a/docs/guides/platform-engineering/labs.md b/docs/guides/platform-engineering/labs.md new file mode 100644 index 0000000..7c0d74e --- /dev/null +++ b/docs/guides/platform-engineering/labs.md @@ -0,0 +1,100 @@ +# Module 8 · CI/CD, GitOps and Platform Engineering Labs + +These exercises should be run in a **disposable repository and Kubernetes sandbox**, not with production credentials. For risky scenarios, use a tabletop simulation instead of a live failure. + +## Lab 1: Design a trust-separated CI and CD pipeline + +Create a repository with a pull-request workflow running lint/unit tests, a protected-main workflow building an immutable artifact, and a separate promotion PR updating a manifest digest. + +```bash +git rev-parse HEAD +npm ci +npm test +docker build -t catalog-test:local . +docker image inspect catalog-test:local --format '{{.Id}}' +``` + +**Expected:** the test job cannot deploy, source revision and artifact digest are traceable, and the production update requires an approved change. **Safety:** no real cloud secrets in a PR job. + +## Lab 2: Show GitOps drift and reconcile behavior + +Install Argo CD only in a sandbox. Apply a sample Application with automated sync and explicit `prune: false`. Make an authorized manual change to a managed field, then compare behavior with self-heal disabled and enabled. + +```bash +argocd app get catalog-preview +argocd app diff catalog-preview +argocd app history catalog-preview +kubectl get application catalog-preview -n argocd -o yaml +``` + +**Expected:** Synced and Healthy are different concepts; drift correction follows sync policy and owner behavior. Record how to suspend the right ApplicationSet owner during an emergency. **Safety:** do not experiment with automatic prune on shared namespaces. + +## Lab 3: Progressive canary with rollback criteria + +In a sandbox with Argo Rollouts installed, create an AnalysisTemplate that checks an instrumented error or success counter and pauses at 10% traffic. Inject failures only into the canary version. Verify that the controller stops or aborts according to the template. + +```bash +kubectl argo rollouts get rollout catalog -n marketplace --watch +kubectl argo rollouts history catalog -n marketplace +``` + +**Expected:** candidate error signal produces an observable gate result. Verify actual traffic split and sample size, not just a configured `setWeight`. **Safety:** use test data and test routing only. + +## Lab 4: Test OIDC trust boundaries on paper and in a sandbox + +Draft two federated trust policies: one incorrectly accepts all repository branches; the other limits deployment to one repository and protected production environment. Evaluate whether a fork PR, arbitrary branch and approved main environment can assume the role. + +**Expected:** untrusted runs are denied. Cloud role permissions remain least privilege even after the identity is authenticated. Do not issue or persist real long-lived production keys to prove the concept. + +## Lab 5: Track SBOM, provenance and digest identity + +Build a toy container using pinned dependencies. Generate an SBOM with an approved local tool, capture the image digest and create a mock provenance statement linking builder/source/digest. Change one build input and compare identity. + +```bash +git rev-parse HEAD +docker image inspect catalog-test:local --format '{{json .RepoDigests}}' +``` + +**Expected:** a tag label alone cannot prove immutable content; signature and provenance verification require trusted signer/builder policy. A green scanner does not guarantee safe source. **Safety:** no secret environment variables in provenance/logs. + +## Lab 6: Model an internal platform API with honest status + +Define a fictional `ServiceEnvironment` contract with owner, TTL, resource class and exposure. In a design exercise, enumerate reconcile outcomes: Pending, Ready, Blocked, Deleting and Failed. Describe finalizer handling when DNS creation succeeds but volume provisioning fails. + +```yaml +apiVersion: platform.example.com/v1alpha1 +kind: ServiceEnvironment +metadata: {name: test-preview} +spec: + owner: catalog-team + class: ephemeral + ttlHours: 8 + exposure: internal +``` + +**Expected:** stable identity, idempotent reconcile, useful status conditions and safe cleanup. This API is hypothetical and won't work without a CRD/controller. + +## Lab 7: Build a platform user-journey SLI + +Create synthetic event records for environment-request start, validation, reconciliation, readiness and user-access verification. Compute the fraction of eligible environments usable within ten minutes. Compare that with controller uptime. + +**Expected:** the user-journey SLI can be poor when controller health is green; identify step latency with traces. Avoid high-cardinality request IDs as always-on Prometheus labels. + +## Lab 8: Multi-cluster recovery tabletop + +Simulate a bad organization-wide policy that breaks new Pod admission while existing workloads run. Identify which controller owns the change, separate production from nonproduction, preserve audit logs and choose a narrowly scoped recovery. + +```bash +kubectl get validatingwebhookconfigurations +kubectl get pods -A -o wide +kubectl get events -A --sort-by=.lastTimestamp +kubectl auth can-i create deployments -n marketplace +``` + +**Expected:** a documented independent break-glass path, controlled rollback or forward-fix, recovery of developer workflows and reconciliation into Git/IaC. Never disable all security admission in production as a lab. + +## Lab completion criteria + +For every lab write down the trust boundary, desired behavior, observables, failure mode, stop condition and recovery procedure. Distinguish **pipeline success**, **GitOps reconciliation** and **correct customer/developer outcome**. + +[Module 8 index](./) · [40 challenging follow-ups](./follow-ups) diff --git a/docs/guides/platform-engineering/q1.md b/docs/guides/platform-engineering/q1.md new file mode 100644 index 0000000..a313d25 --- /dev/null +++ b/docs/guides/platform-engineering/q1.md @@ -0,0 +1,75 @@ +# Q1 · CI/CD Architecture, Reproducible Builds and Delivery Boundaries + +> **Interviewer:** Design a secure and fast pipeline for 60 independently deployed microservices. What must CI prove, what does CD own, and how do you ensure the production artifact is the exact code that passed tests? + +## First principles and system architecture + +Continuous integration verifies a specific source revision and produces evidence that it can be built and tested. Continuous delivery promotes an approved **immutable artifact** into a controlled environment; continuous deployment automates that promotion subject to policy. CI success is not equivalent to business success in production. Draw trust boundaries around untrusted contribution, privileged release and cloud deployment. + +```mermaid +flowchart TD + PR["Reviewed source revision"] --> VALIDATE["Lint, unit, contract, integration tests"] + VALIDATE --> BUILD["Isolated deterministic builder"] + BUILD --> SBOM["SBOM, scans, provenance"] + SBOM --> SIGN["Immutable digest and attestation"] + SIGN --> REG["Artifact registry"] + REG --> CHANGE["Promotion Git PR / release record"] + CHANGE --> APPROVE["Environment approval and policy"] + APPROVE --> CD["Deployment controller"] + CD --> CHECK["Smoke, SLO and business outcome verification"] +``` + +### Control boundaries + +- **Source integrity:** protected branch, code review, CODEOWNERS, required checks, dependency update policy. Forked PR code must not receive deployment credentials. +- **Build isolation:** pin runtime and lockfile, use ephemeral runners where possible, restrict writeable shared caches, record build input and artifact identity. +- **Release authority:** only a narrowly authorized job or pull-based reconciler may deploy a reviewed digest to production. Do not use floating image tags as a trusted release identity. +- **Verification:** compare deployed digest to CI record; capture release version in traces; roll back according to pre-agreed health and business criteria. + +### GitHub Actions CI example + +```yaml +name: Verify +on: + pull_request: +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: {node-version: "20", cache: npm} + - run: npm ci + - run: npm test + - run: npm run build +``` + +This is an instructional CI job, not an attested production release pipeline. For real deployment, pin third-party actions to reviewed commit SHAs, isolate privileges and add organization-appropriate provenance, approvals and digest publication. + +## Build once, promote by digest + +```bash +docker build --pull -t registry.example.com/catalog:build-1234 . +docker push registry.example.com/catalog:build-1234 +# Resolve immutable sha256 digest from registry/build output. +kubectl get deploy catalog-api -n marketplace \ + -o jsonpath='{.spec.template.spec.containers[*].image}' +``` + +Tags can be mutated; a digest binds the deployment to bytes. Dependency lockfiles alone do not guarantee bit-for-bit reproducibility because builders, timestamps, operating-system packages and network downloads may vary. Separate **reproducibility** (same outputs) from **provenance** (trusted evidence of how outputs were produced). + +## Test strategy and feedback loop + +Use fast unit/static checks for every commit, interface/contract tests for compatibility, ephemeral integration environments for critical dependencies and a small number of valuable end-to-end user journeys. Measure queue wait, test flake rate, time to first useful failure and test failure classification. Optimizations must not let an untrusted cache contaminate privileged builds. + +### Hypothetical incident + +The `stable` Docker tag is repointed after CI tests commit A. Production later pulls commit B, which never passed release tests. Compare deployed RepoDigests with the build attestation, halt promotions and restore the known tested digest. Fix the promotion pipeline to record and enforce immutable identities. + +### Senior trade-offs and spoken answer + +> “I would isolate three identities: the reviewed source revision, the immutable artifact digest, and the approved environment deployment record. CI verifies the source and builds once on trusted infrastructure; CD promotes that exact digest with least privilege and observed SLO gates. I would optimize feedback speed without sacrificing trust boundaries or the ability to audit exactly what code is running.” + +**References:** [GitHub Actions hardening](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) · [SLSA](https://slsa.dev/) diff --git a/docs/guides/platform-engineering/q2.md b/docs/guides/platform-engineering/q2.md new file mode 100644 index 0000000..dd70880 --- /dev/null +++ b/docs/guides/platform-engineering/q2.md @@ -0,0 +1,92 @@ +# Q2 · GitOps Reconciliation, Argo CD, Drift and Ownership + +> **Interviewer:** Explain what Argo CD actually reconciles. How does GitOps differ from simply running kubectl in CI? What happens when an SRE fixes a production incident manually but GitOps keeps reverting the change? + +## Desired state and data plane + +GitOps is a control-loop architecture, not a synonym for Git-hosted YAML. A reviewed configuration source defines desired state; a controller observes live state, computes the difference and converges where allowed. CI should produce and attest the artifact; the delivery repository references an immutable image digest; Argo CD applies declarative desired state using controlled cluster credentials. + +```mermaid +sequenceDiagram + participant D as Developer + participant CI as CI build + participant R as Registry + participant G as Config Git + participant A as Argo CD + participant K as Kubernetes API + D->>CI: Merge reviewed commit + CI->>R: Push immutable digest + CI->>G: Propose promotion PR + G-->>A: Approved change observed + A->>K: Diff rendered desired and live + A->>K: Sync permitted resources + K-->>A: Watch state and health +``` + +## Meaning of status + +**Synced** means the configured comparison rules find desired and observed resources aligned. It does not prove the application can complete business transactions. **Healthy** depends on resource-specific assessments and can miss downstream correctness. **Prune** enables deletion when a formerly managed resource is removed from Git; **selfHeal** attempts to correct drift initiated outside Git. Without an explicit sync trigger and self-heal, a manual patch can remain. + +Argo CD operates with reconciliation intervals, webhooks and caches, so changes are not universally instantaneous. ApplicationSet controllers can recreate or override Application specifications. The emergency suspension mechanism must target the actual owner, not only a generated child object. + +## Application example + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: catalog-prod + namespace: argocd +spec: + project: catalog + source: + repoURL: https://github.com/example-org/deployment-config.git + targetRevision: main + path: apps/catalog/prod + destination: + server: https://kubernetes.default.svc + namespace: marketplace + syncPolicy: + automated: + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=false +``` + +This needs the Argo CD CRDs/controller, a legitimate `catalog` AppProject, repo access and destination authorization. For a large team, scope sources, clusters, namespaces and resources with AppProjects and Kubernetes RBAC. Do not operate every team from a single unrestricted application credential. + +## Who owns replicas, fields and secrets? + +If HPA controls `spec.replicas` but GitOps insists on a fixed value, two controllers can repeatedly fight. Avoid unmanaged overlapping writers; adopt a defined ownership policy or targeted diff ignore when warranted. Operators also mutate annotations, generated ConfigMaps and CRD status. Ignore only known legitimate fields, not the entire spec. Do not commit plaintext production secrets; use an approved encryption or external secrets mechanism, and authorize the controller narrowly. + +```mermaid +flowchart TD + SRC["Approved config Git"] --> ARGO["Argo reconciler"] + ARGO --> API["Kubernetes API"] + HPA["Autoscaler"] --> API + OP["Operator / admission controller"] --> API + API --> LIVE["Live object"] + LIVE -.->|"Observed changes"| ARGO + OWN["Field ownership contract"] -.-> ARGO + OWN -.-> HPA + OWN -.-> OP +``` + +## Hypothetical incident: emergency fix reverts + +A production operator increases replicas or reverts an image manually while `selfHeal` remains active. The GitOps controller restores the broken Git version. The solution is a documented break-glass process: suspend the correct automated owner when authorized, approve the emergency revision in Git, or isolate the release path for rollback. Record the temporary override and reconcile it after recovery. Audit log and Git diff must explain what actually happened. + +```bash +argocd app get catalog-prod +argocd app diff catalog-prod +argocd app history catalog-prod +kubectl get application catalog-prod -n argocd -o yaml +kubectl get applicationsets -n argocd +``` + +## Spoken answer + +> “GitOps is declarative reconciliation from a reviewed desired-state source, separate from the software build. The controller observes Git and live resources and applies allowed changes, with policies for self-heal and pruning. I'd distinguish Synced from real application health and define field ownership alongside HPA and operators. During an incident, the emergency process must change the source of truth or safely suspend the correct reconciler so it cannot immediately undo the fix.” + +**References:** [Argo CD sync policy](https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/) · [Argo CD projects](https://argo-cd.readthedocs.io/en/stable/user-guide/projects/) diff --git a/docs/guides/platform-engineering/q3.md b/docs/guides/platform-engineering/q3.md new file mode 100644 index 0000000..5b9401e --- /dev/null +++ b/docs/guides/platform-engineering/q3.md @@ -0,0 +1,105 @@ +# Q3 · Progressive Delivery: Canary, Blue-Green, Analysis and Cutback + +> **Interviewer:** A release looks good in staging but fails under a subset of production payments. Design controlled rollout with objective abort decisions. Why does rolling back application code not always recover the system? + +## Release changes are experiments with blast radius + +A rollout strategy defines **who receives the new behavior, how rapidly the population grows, how the candidate is measured, and what happens when it fails**. A readiness probe can establish that a Pod responds; it cannot establish transaction correctness. A metric gate should compare candidate and stable traffic, account for sample size, and inspect business outcomes. + +| Strategy | Strength | Limitation | +| --- | --- | --- | +| Rolling | Native and simple | Coarse traffic control; mixed versions | +| Blue-green | Fast frontend switch | Double capacity and backend-state compatibility | +| Canary | Gradual exposure | Routing/metrics integration and statistical noise | +| Feature flags | Decouple exposure from deploy | Runtime flags, debt and audit | + +```mermaid +flowchart TD + IMAGE["Tested artifact digest"] --> STAGE["Start 1-5% canary"] + STAGE --> MET{"Enough data and healthy?"} + MET -->|No| AB["Abort, route to stable"] + MET -->|Yes| TEN["Increase to 10%"] + TEN --> G2{"SLO and transaction checks pass?"} + G2 -->|No| AB + G2 -->|Yes| HALF["50% + hold"] + HALF --> G3{"Healthy?"} + G3 -->|No| AB + G3 -->|Yes| PROM["Promote 100%"] +``` + +### Argo Rollouts canary example + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: {name: catalog, namespace: marketplace} +spec: + replicas: 8 + selector: + matchLabels: {app: catalog} + template: + metadata: + labels: {app: catalog} + spec: + containers: + - name: catalog + image: registry.example.com/catalog@sha256:REPLACE_WITH_REAL_DIGEST + ports: [{containerPort: 8080}] + strategy: + canary: + steps: + - setWeight: 10 + - pause: {duration: 5m} + - analysis: + templates: + - templateName: catalog-slo + - setWeight: 50 + - pause: {} +``` + +This requires an installed controller, a real AnalysisTemplate and a compatible routing integration. Without explicit traffic routing support, percentage rollout may approximate weights through replica counts rather than providing exact traffic shares. Long-lived sessions and uneven request costs can invalidate naive assumptions about traffic split. + +### Objective gates + +Define before deployment: +- Candidate error ratio versus stable control and absolute SLO burn. +- Endpoint-specific p95/p99 with appropriate histogram aggregation. +- Duplicate business effects, failed writes, stale reads and event lag. +- Sample count and statistical confidence. A five-minute hold with three requests proves little. +- Resource saturation, queue length, node and database health. +- Operator override, abort action, escalation and maximum duration. + +### Code rollback is not state rollback + +An old image may not understand a new database schema. Even if the service is rolled back, external payments, emitted events and irreversible migrations remain. Use **expand-contract** migration: first add backward-compatible fields, deploy dual-compatible application versions, migrate data, then remove legacy fields only after all consumers are updated. Version event contracts for mixed workloads. + +```mermaid +sequenceDiagram + participant C as Controller + participant R as Router + participant M as Metrics + participant S as Stable pods + participant N as New pods + C->>R: Route 10% to new + R->>S: Stable share + R->>N: Canary share + M-->>C: Candidate errors above threshold + C->>R: Abort new selection + Note over C: Rollback image cannot undo prior database or payment writes +``` + +## Failure drill + +A checkout canary returns duplicate payment callbacks only after a delayed response. Conventional HTTP error rate remains low. Gate on idempotency violations and per-operation reconciliation, abort candidate routing, preserve audit records, and reconcile affected transaction IDs. Fix retry semantics, not merely Pod count. + +```bash +kubectl argo rollouts get rollout catalog -n marketplace --watch +kubectl argo rollouts abort catalog -n marketplace +kubectl argo rollouts history catalog -n marketplace +``` + +**Spoken answer** + +> “I'd choose a progressive strategy with pre-agreed abort gates for latency, errors and business correctness. I'd validate that the ingress actually enforces the traffic split and that the sample is meaningful. If the candidate fails, cut new traffic back to stable while reconciling side effects. I would never promise a one-command rollback of database schemas or external transactions; compatibility and compensating procedures must be part of the release design.” + +**References:** [Argo Rollouts analysis](https://argoproj.github.io/argo-rollouts/features/analysis/) · [Canary deployment](https://argoproj.github.io/argo-rollouts/features/canary/) diff --git a/docs/guides/platform-engineering/q4.md b/docs/guides/platform-engineering/q4.md new file mode 100644 index 0000000..6056ea1 --- /dev/null +++ b/docs/guides/platform-engineering/q4.md @@ -0,0 +1,78 @@ +# Q4 · Software Supply Chain, SBOM, Signing and Provenance + +> **Interviewer:** How do you prevent a compromised dependency, GitHub Actions workflow or build runner from delivering malicious code to production? Distinguish source review, signatures, SBOMs, provenance and deployment enforcement. + +## Threat model the artifact path + +An attacker may control untrusted PR code, package registry releases, an action dependency, build cache, CI token, signer or registry permissions. A vulnerability scanner does not prove the released bytes match reviewed source. A signature proves who signed a digest under its trust model, **not** that the contents are benign. Provenance records structured build claims; it is valuable only if the builder/issuer and verification policy are trustworthy. + +```mermaid +flowchart LR + SRC["Protected reviewed commit"] --> BUILD["Isolated trusted builder"] + DEPS["Pinned dependencies"] --> BUILD + BUILD --> ART["Immutable image digest"] + BUILD --> SBOM["SBOM inventory"] + BUILD --> ATT["Signed provenance"] + ART --> VERIFY["Policy at promotion/admission"] + ATT --> VERIFY + SBOM --> VERIFY + VERIFY --> DEPLOY["Production"] +``` + +### Defense in depth + +1. Protected branches, CODEOWNERS and reviewed releases with minimized bypass permissions. +2. No high-privilege token exposure to scripts from untrusted forks. Avoid dangerous pull_request_target checkout behavior. +3. Pin third-party CI actions to reviewed full commit SHAs. Update through controlled review. +4. Lock direct and transitive dependencies where ecosystem supports it. Watch for package takeover and install scripts. +5. Prefer ephemeral isolated runners. Segregate caches and network access by trust level. +6. Use OIDC short-lived cloud credentials scoped to repo, environment, branch and audience. No static account keys in CI. +7. Generate SBOM, provenance attestations and immutable artifact digests. +8. Enforce expected signer/builder, source revision and artifact digest before deployment. +9. Apply runtime restrictions and vulnerability prioritization based on exposure and exploitability. +10. Preserve auditable release records and a tested incident inventory by digest. + +### GitHub Actions least privilege example + +```yaml +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: echo "Run isolated verification" +``` + +For a cloud deployment job, configure `id-token: write` only when needed and restrict the cloud trust conditions. This event-less workflow fragment is illustrative and not independently runnable as a full Actions workflow. + +### Admission policy concept + +```text +permit production promotion when + artifact digest is immutable and exists + AND approved source revision matches provenance + AND builder/issuer identity is trusted + AND security exception policy is satisfied + AND production approval recorded +``` + +Use Sigstore/Cosign, trusted attestation verification and admission controls where appropriate. Avoid treating signature validation as a substitute for review or least privilege. + +### Supply-chain incident + +A recently updated package's install hook reads tokens from the CI environment. Freeze new promotions, revoke exposed identities, isolate affected runners, enumerate builds by dependency version/digest, rebuild with clean trusted infrastructure and verify deployed identities. Deleting a tag will not revoke stolen credentials or remove an already running artifact. Communicate scope based on evidence rather than asserting all deployments are compromised. + +```bash +git log --oneline -n 5 +git diff HEAD~1 -- package-lock.json +kubectl get deployments -A \ + -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,IMAGES:.spec.template.spec.containers[*].image' +``` + +### Spoken answer + +> “I'd secure the chain from a reviewed commit to immutable runtime bytes. That means isolated builders, pinned dependencies and actions, OIDC instead of long-lived credentials, artifact digests, SBOM and trusted provenance. Deployment should verify the source and build identity, not only a tag or a green scan. I would also keep a digest-based inventory so we can contain a compromised build and rotate affected credentials quickly.” + +**References:** [SLSA specification](https://slsa.dev/spec/) · [GitHub artifact attestations](https://docs.github.com/en/actions/concepts/security/artifact-attestations) · [GitHub Actions security](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions) diff --git a/docs/guides/platform-engineering/q5.md b/docs/guides/platform-engineering/q5.md new file mode 100644 index 0000000..9c9e599 --- /dev/null +++ b/docs/guides/platform-engineering/q5.md @@ -0,0 +1,78 @@ +# Q5 · Identity, Secrets, Multi-Environment Isolation and Deployment Authority + +> **Interviewer:** Multiple teams deploy to separate staging and production clusters on AWS or Google Cloud. How should CI, GitOps and workloads authenticate? Why is a namespace or cluster-admin ServiceAccount insufficient as a security model? + +## Distinguish four identities + +**Human identity:** authenticated SSO principals/groups with reviewed production access. +**CI builder identity:** may read source and publish approved artifacts, but generally cannot deploy production directly. +**CD controller identity:** can reconcile a bounded set of namespaces and resource kinds in the target cluster. +**Runtime workload identity:** uses the minimum Kubernetes and cloud API permissions its application needs. + +```mermaid +flowchart TD + HUMAN["SSO + reviewed approvals"] --> GIT["Protected deployment Git"] + CI["Build runner"] --> OIDC["OIDC federation / short lived token"] + OIDC --> REG["Scoped artifact publication"] + GIT --> ARGO["GitOps controller"] + ARGO --> RBAC["Namespace/project-scoped K8s RBAC"] + RBAC --> K["Target cluster"] + K --> SA["Workload ServiceAccount"] + SA --> FED["Cloud workload identity federation"] + FED --> API["Least-privilege cloud API"] +``` + +### No static secrets in manifests + +Base64 encoding Kubernetes Secrets isn't encryption. Secrets can leak through Git history, CI logs, `kubectl get` output, environment dumps and backups. Use managed secret storage, encryption at rest, narrow RBAC and a rotation model. Prefer external secret references where operationally justified; understand that synchronization controllers often materialize Kubernetes Secret objects, so at-rest and access control still matter. + +### Cloud federation specifics + +- GitHub Actions OIDC: trust the exact repository/branch/environment and audience claims; grant `id-token: write` only to the intended job. +- AWS IRSA/EKS Pod Identity and GKE Workload Identity Federation have different trust and admission semantics. Model cloud IAM and Kubernetes RBAC separately. +- A Pod can be allowed to call an object-store API without being allowed to read cluster Secrets. +- An ingress controller, secrets operator and Argo CD may each require different permissions; do not share one privileged cloud identity. + +### RBAC sample + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: rollout-reader, namespace: marketplace} +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: rollout-reader, namespace: marketplace} +subjects: + - kind: ServiceAccount + name: release-observer + namespace: marketplace +roleRef: {kind: Role, name: rollout-reader, apiGroup: rbac.authorization.k8s.io} +``` + +Provision the referenced ServiceAccount separately. Reading Deployments doesn't permit deploying or accessing Secrets. Workload creation and `pods/exec` also create indirect privilege paths, so assess effective permissions rather than RBAC labels alone. + +## Environment isolation + +Use accounts/projects and clusters/namespace boundaries appropriate for trust: staging must not possess credentials that can mutate production. Separate remote state, artifact promotion rights, encryption keys and audit trails. Namespace isolation alone cannot contain a compromised cluster-admin controller or shared node privilege escape. Consider quotas, Pod Security Admission, NetworkPolicy and cloud IAM. + +## Failure scenario: OIDC trust too broad + +A CI workflow accepts tokens from any branch of a repository and can impersonate a production cloud role. An attacker with permission to push a development branch obtains deployment authority. First revoke/limit role trust, inspect cloud audit logs for issued sessions, rotate any downstream credentials, and test exact subject/audience/environment conditions. Avoid reintroducing permanent static keys as a workaround. + +```bash +kubectl auth can-i get deployments -n marketplace \ + --as=system:serviceaccount:marketplace:release-observer +kubectl get serviceaccounts -n marketplace +kubectl get rolebindings -n marketplace +``` + +### Spoken answer + +> “I'd separate people, CI, CD and runtime identity. CI publishes signed immutable artifacts; a constrained deployment controller reconciles reviewed configuration. Each workload has its own short-lived cloud identity and minimal Kubernetes RBAC. Secrets remain outside public Git and are rotated. I would enforce environment boundaries at both cloud IAM and cluster scope so a compromised test workflow cannot become production administration.” + +**References:** [GitHub OIDC cloud deployments](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers) · [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) diff --git a/docs/guides/platform-engineering/q6.md b/docs/guides/platform-engineering/q6.md new file mode 100644 index 0000000..907605d --- /dev/null +++ b/docs/guides/platform-engineering/q6.md @@ -0,0 +1,90 @@ +# Q6 · Internal Developer Platforms, Golden Paths and Self-Service APIs + +> **Interviewer:** Engineers wait several days for an environment or an ingress rule. Design an internal developer platform that reduces toil without becoming another central bottleneck. + +## Platform as a product + +Platform engineering creates supported capabilities and clear contracts for users of the platform, usually application engineers. An internal developer portal is a user interface; it is not the entire platform. The platform comprises APIs, templates, policy, automation, documentation, observability and a support model. The objective is to shorten safe time-to-value, not maximize the number of Kubernetes abstractions exposed. + +```mermaid +flowchart TD + DEV["Product engineer"] --> PORTAL["Portal / CLI / API"] + PORTAL --> CATALOG["Golden path catalog"] + CATALOG --> CONTRACT["Typed service/environment contract"] + CONTRACT --> POLICY["Security, quotas and ownership validation"] + POLICY --> PR["Reviewable infrastructure and deploy PRs"] + PR --> PROV["Trusted Terraform / GitOps reconciliation"] + PROV --> CLOUD["Cloud + Kubernetes resources"] + CLOUD --> STATUS["Status, cost, SLO and support diagnostics"] + STATUS --> DEV +``` + +### Define the platform API before choosing tools + +```yaml +apiVersion: platform.example.com/v1alpha1 +kind: ServiceEnvironment +metadata: + name: catalog-preview +spec: + owner: product-catalog + class: ephemeral + region: europe-region-a + exposure: internal + ttlHours: 24 + resources: + cpuRequest: 500m + memoryRequest: 512Mi +``` + +This is a **hypothetical platform custom resource**, not a built-in Kubernetes API. A real implementation needs CRD schema, RBAC, reconciler, status conditions, failure cleanup, cost attribution and idempotent creation/deletion. A developer should not be able to request arbitrary privileged host mounts through the contract. + +### Golden paths versus paved roads + +Offer a recommended safe default for common service types (HTTP API, async worker, scheduled job) including CI, observability, resources, identity and rollout policy. Provide documented escape hatches for exceptional workloads, backed by review and clear ownership. Mandatory golden paths that cannot meet product needs incentivize shadow infrastructure. + +### Onboarding flow + +```mermaid +sequenceDiagram + participant D as Developer + participant P as Platform API + participant G as Git configuration + participant C as Infrastructure controllers + D->>P: Request preview environment + P->>P: Validate ownership, quota, TTL + P->>G: Propose auditable change + G->>C: Approved desired state + C->>C: Reconcile namespace, policies, service + C-->>P: Conditions: Ready/Blocked/Failed + P-->>D: URL, cost and troubleshooting links +``` + +### API design and error behavior + +An asynchronous API should return a resource identity and status, not pretend an environment is ready when provisioning is still in progress. Use `observedGeneration`, conditions, stable request IDs, explicit reconciliation backoff and finalizers for external cleanup. Handle partial failures: namespace exists but database provisioning failed, or DNS creation succeeded while access-policy setup failed. Retry idempotently; do not leak abandoned preview resources. + +## Product metrics + +Measure median and p90 time from request to usable environment; support ticket volume; time-to-first-deploy; developer task completion rate; security exceptions; platform availability; cost per environment; and adoption among eligible teams. Add qualitative usability research. “Services onboarded” is a vanity metric if teams still maintain hand-written deployment YAML. + +### Trade-offs + +Backstage, a custom CLI, Crossplane, Terraform modules and Kubernetes operators solve different problems. Choose based on supported workflow and team maintenance capacity. A small, well-maintained set of templates can be more useful than an enormous portal with fragile automation. + +## Failure scenario + +A preview environment cleanup controller is down for a week. TTL resources remain running and cost spikes. First restore cleanup safely, inspect finalizer conditions and downstream ownership, then reconcile stale resources through controlled deletion. Post-incident, alert on aged environments and quantify unowned cost. + +```bash +kubectl get customresourcedefinitions +kubectl get events -A --sort-by=.lastTimestamp +kubectl get namespaces -L platform.example.com/owner +# Query controller status and preview environment age via platform metrics +``` + +### Spoken answer + +> “I'd treat the platform as a product. I'd identify the few repeated developer journeys that cause the most wait time, define typed self-service contracts and automate them through reviewed infrastructure and GitOps workflows. I'd expose honest asynchronous status and safe cleanup, provide default security and cost boundaries, and measure time-to-usable-environment rather than just portal adoption. Exceptions need a governed path so the platform enables rather than blocks teams.” + +**References:** [CNCF platform engineering white paper](https://tag-app-delivery.cncf.io/whitepapers/platforms/) · [Kubernetes operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) diff --git a/docs/guides/platform-engineering/q7.md b/docs/guides/platform-engineering/q7.md new file mode 100644 index 0000000..1fbd4fc --- /dev/null +++ b/docs/guides/platform-engineering/q7.md @@ -0,0 +1,71 @@ +# Q7 · Observability, Engineering Effectiveness and Platform SLOs + +> **Interviewer:** Your internal platform has 95% uptime, but developers report slow deployments and frequent confusing errors. What would you measure, and how would you prove that the platform improves engineering productivity? + +## User journeys define platform reliability + +A successful platform API response isn't enough. Measure whether a developer can **create an environment, deploy a reviewed change, observe it, recover it and delete it** within agreed targets. Platform SLOs are separate from product runtime SLOs: a deployment control plane can be degraded while existing applications continue serving. + +| Platform journey | SLI example | Reliability question | +| --- | --- | --- | +| Preview environment | Successful usable environment within 10 minutes | Can developer start work? | +| CI feedback | Valid source check completes under agreed wait | Can developer learn quickly? | +| CD promotion | Approved artifact reaches expected environment | Can team ship safely? | +| Rollback/cutback | Recovery to known-good behavior within deadline | Can team recover? | +| Developer docs | Task-completion success and support contact rate | Is self-service understandable? | + +Never blindly maximize deployment frequency. Use DORA metrics such as lead time for changes, deployment frequency, change failure rate, and recovery time as contextual trends; measure over consistent definitions and avoid using them as individual performance rankings. Focus on customer impact, not reward for gaming events. + +## Three observability signals, one correlation model + +Metrics quantify rate, errors, duration and saturation. Logs record structured state transitions with trace/release/owner identifiers and explicit error reasons. Distributed traces isolate where time is spent across services. Correlation requires consistent trace context and time alignment, but traces may sample low-volume failures away; preserve critical error events independently. + +```mermaid +flowchart TD + USER["Developer intent: deploy revision"] --> PIPE["CI events and artifact identity"] + PIPE --> GIT["Promotion and approval event"] + GIT --> ARGO["GitOps reconciliation and sync"] + ARGO --> KUBE["Kubernetes workload lifecycle"] + KUBE --> BIZ["Smoke checks and runtime SLO"] + PIPE --> TEL["Metrics / logs / traces"] + GIT --> TEL + ARGO --> TEL + KUBE --> TEL + BIZ --> TEL + TEL --> VIEW["Per-release timeline and platform SLI"] +``` + +### Useful cardinality and cost controls + +Avoid placing unique user IDs, commit hashes or PR IDs directly on every high-frequency Prometheus time series. Use those as trace/log attributes or in exemplars with careful retention. Metrics should aggregate on stable service, environment, status and outcome labels. Limit log payloads, redact tokens and personal data, apply trace sampling deliberately, and allocate observability storage cost by usage. + +### PromQL example: CI request success + +```promql +sum(rate(platform_operation_total{operation="create_preview",outcome="succeeded"}[30m])) +/ +clamp_min(sum(rate(platform_operation_total{operation="create_preview"}[30m])), 1e-9) +``` + +This requires that `platform_operation_total` counts each eligible operation exactly once when reaching a terminal outcome; a metric increment at both start and finish corrupts the denominator. Separately track still-pending operations and oldest pending age. + +### Observability architecture trade-offs + +A single shared metrics cluster can be a failure domain and a cost bottleneck; complete per-tenant isolation makes fleet-wide detection harder. Set retention, cardinality budgets, query limits and access policy. Prefer consistent OpenTelemetry semantic conventions where supported, but don't assume a migration is free or that sampling preserves every distributed trace. + +## Incident scenario + +A deployment platform reports Sync=True, but 30% of teams cannot reach newly deployed preview URLs. Correlate Git desired revision, Argo health, target ingress registration, DNS propagation and synthetic user-journey checks. A green controller metric is insufficient. Add a post-deploy URL reachability SLI and scoped alerting; avoid an expensive full end-to-end test on every reconcile event. + +```bash +argocd app get catalog-preview +kubectl get pods,svc,ingress -n catalog-preview +kubectl get endpointslices -n catalog-preview +kubectl get events -n catalog-preview --sort-by=.lastTimestamp +``` + +## Spoken answer + +> “I'd treat developers as platform customers and measure the success and duration of their actual journeys, not only controller availability. I'd correlate source revision, build artifact, GitOps sync, Kubernetes readiness and user-facing checks using consistent identifiers. Then I'd combine SLOs with DORA-style delivery metrics and qualitative feedback, taking care not to create misleading individual performance scores or runaway telemetry costs. The platform is successful when safe changes and recovery become predictably easier.” + +**References:** [Google SRE monitoring](https://sre.google/sre-book/monitoring-distributed-systems/) · [DORA research](https://dora.dev/) · [OpenTelemetry](https://opentelemetry.io/docs/) diff --git a/docs/guides/platform-engineering/q8.md b/docs/guides/platform-engineering/q8.md new file mode 100644 index 0000000..49e2d21 --- /dev/null +++ b/docs/guides/platform-engineering/q8.md @@ -0,0 +1,90 @@ +# Q8 · Multi-Cluster Platform Governance, Recovery and Technical Leadership + +> **Interviewer:** Your company has ten Kubernetes clusters, dozens of teams and multiple cloud providers. Design ownership, disaster recovery, infrastructure change control and production incident processes for a reliable internal platform. + +## Start with responsibility boundaries + +Separate shared foundation teams from application ownership. Platform teams own supported service contracts, GitOps controllers, runner infrastructure, policies, cluster upgrades and incident tooling. Product teams own business SLOs, application correctness and compatibility. The boundary is negotiated through contracts and documented escalation, not implied by who can type `kubectl`. + +```mermaid +flowchart TD + TEAM["Product teams"] --> API["Approved platform service contracts"] + API --> GIT["Protected configuration + modules"] + GIT --> CI["Policy, tests, cost and impact check"] + CI --> PLAN["Risk-based review / promotion"] + PLAN --> ARGO["Per-cluster GitOps controllers"] + ARGO --> C1["Cluster A"] + ARGO --> C2["Cluster B"] + ARGO --> C3["Cluster C"] + FND["Foundation: networking, IAM, clusters"] --> C1 + FND --> C2 + FND --> C3 + OBS["Fleet inventory, SLOs, audit and incident command"] -.-> C1 + OBS -.-> C2 + OBS -.-> C3 +``` + +### Platform architecture decisions + +- **Fleet boundaries:** isolate production from nonproduction using sufficiently strong cloud/project, account and cluster boundaries. Classify shared versus dedicated clusters by blast radius and compliance. +- **GitOps ownership:** each cluster has explicit application destinations, AppProjects and workload trust policy. Bootstrap documented recovery credentials because the GitOps controller can itself fail. +- **Infrastructure management:** Terraform owns cloud resources, GitOps owns Kubernetes manifests, operators own their CRDs. Avoid two tools owning the same field/resource. Define cross-layer contracts. +- **Secrets:** OIDC/federation, workload identity, rotation and breach inventory; do not replicate global production credentials across all clusters. +- **Upgrades:** release channels/version-skew checks, compatibility matrix for CSI/CNI/webhooks, canary node pools, PDB/readiness and rollback/forward-recovery procedures. +- **Disaster recovery:** backup external state and platform desired configuration, test clean-room reconstruction and keep control-plane recovery paths independent of the broken control plane. +- **Cost:** attribute nodes, idle preview namespaces, observability and data egress to owners; combine cost controls with safety headroom. + +## Change risk and blast radius + +A one-line edit to a shared cloud firewall rule can be more dangerous than 100 application updates. Approval should depend on potential reach, reversibility, security, storage/data-loss risk and SLO impact, not YAML size. + +### Model incident: cluster fails after a policy change + +```mermaid +sequenceDiagram + participant I as Incident commander + participant G as Git / infra change log + participant C as Cluster services + participant P as Cloud provider + I->>C: Inspect customer impact and Pod/endpoint states + I->>G: Identify latest policy/network change + I->>P: Check effective routes/firewall and authorization + I->>C: Apply authorized narrow remediation + C-->>I: User-journey checks and telemetry recover + I->>G: Reconcile emergency state into desired config + Note over I,G: Follow up with policy tests and owner review +``` + +Do not delete all NetworkPolicies or force-unlock every Terraform backend as a first step. Preserve state and identify which control plane owns the change. If GitOps self-heal is active, address the desired configuration or suspend the correct owner under break-glass controls. + +### Safe incident operating model + +1. Declare severity and command roles from actual customer/developer impact. +2. Assign investigative workstreams by failure boundary. +3. Keep one timeline of evidence, changes and decisions. +4. Freeze risky concurrent writes to the affected infrastructure. +5. Apply a narrow, reversible mitigation with a named owner. +6. Verify application outcome, identity security and workload health. +7. Reconcile emergency drift, rotate emergency access and document follow-ups. +8. Run a game day to prove the fix and rollback/failover mechanisms. + +```bash +kubectl config get-contexts +kubectl get nodes -o wide +kubectl get events -A --sort-by=.lastTimestamp +kubectl get validatingwebhookconfigurations +kubectl get applications -A +terraform plan -refresh-only +``` + +The `terraform plan -refresh-only` command should be used only with correct workspace, credentials and backend state; it is read-focused but can still trigger provider reads and expose sensitive information. Don't apply destructive changes as part of discovery. + +## Team success measures + +Track platform task lead time, SLO burn, production incident recovery time, change failure rate, module/template adoption, policy escape hatches, ownership coverage, cost allocation and recurring support toil. Prioritize improvements based on user pain and risk; a platform that centralizes approvals without creating self-service may harm delivery. + +## Spoken answer + +> “I'd establish clear ownership across cloud foundations, GitOps, application teams and security. Each cluster should have least-privilege delivery authority, audited configuration and a rehearsed recovery path independent of the healthy day-to-day control plane. I'd assess changes by blast radius, not line count, and use canary upgrades and policy tests. During an incident I'd identify the failed boundary, contain it safely, reconcile state afterward and turn repeated support work into durable platform capabilities.” + +**References:** [CNCF platform engineering](https://tag-app-delivery.cncf.io/whitepapers/platforms/) · [Kubernetes multi-tenancy](https://kubernetes.io/docs/concepts/security/multi-tenancy/) · [Google SRE incident management](https://sre.google/sre-book/managing-incidents/) diff --git a/docs/guides/reliability/follow-ups.md b/docs/guides/reliability/follow-ups.md new file mode 100644 index 0000000..a3e52a4 --- /dev/null +++ b/docs/guides/reliability/follow-ups.md @@ -0,0 +1,245 @@ +# Module 7 · 40 Difficult Cloud Reliability Follow-ups + +Answer each aloud before reading the response. All incidents and services are hypothetical; provider-specific details must be checked against deployed versions. + +## 1. Why is 99.9% availability not always equivalent to 43.2 minutes of downtime in 30 days? + +*Area: SLOs* + +**Model answer:** 43.2 minutes follows only a time-based model across a 30-day interval. A request-based SLO allows 0.1% bad eligible requests, so outage impact depends on demand distribution. A two-minute peak outage can burn more than the whole monthly request budget, while a longer low-traffic problem can burn little. Define numerator, denominator and time window explicitly before converting budgets. + +## 2. A service returns 200 while silently dropping one in 100 queued bids. Is its availability SLO met? + +*Area: SLOs* + +**Model answer:** Its HTTP transport availability may appear healthy, but the business operation correctness objective is violated. Define a success event only after durable acceptance or a truthful rejection, and track the accepted operation ID through downstream completion when the API promises that outcome. Use reconciliation to detect missing events and ensure retries cannot double-apply financial side effects. + +## 3. Would you page every time a service records an error? + +*Area: SLOs* + +**Model answer:** No. Page on actionable customer impact and meaningful error-budget burn, using multiwindow checks to separate acute from persistent problems. Low-volume services require care because one failure can create misleading percentage spikes; pair ratios with eligible volume and severity. Route nonurgent risk trends to a ticket instead of exhausting responders with noise. + +## 4. What is the danger of averaging p99 latencies across Kubernetes Pods? + +*Area: SLOs* + +**Model answer:** Percentiles are not linearly aggregatable. Averaging per-Pod p99 values can hide a high-traffic Pod with poor tail latency and yield a number that is not the fleet's true p99. Export compatible latency histograms or distributions, aggregate observations first, then compute the desired quantile. Segment by operation and region where user experience differs. + +## 5. How should an error budget affect release policy? + +*Area: SLOs* + +**Model answer:** Define policy with product owners: high burn may call for reducing risky changes, prioritizing remediation or requiring stronger verification. It need not mean a blanket freeze, especially when a reliability fix is the change. Analyze which changes consume the budget and whether the service objective remains aligned with customer value. Treat it as a risk-governance mechanism, not an engineer scorecard. + +## 6. If two replicas each have 99.9% uptime, is their combined availability automatically 99.9999%? + +*Area: High availability* + +**Model answer:** Only under a simplified assumption that failures are independent and either replica alone has adequate capacity. Shared cloud control plane, DNS, database, rollout policy and zone-level dependencies create correlated failures. Inspect full dependency and traffic routing graphs, then validate with fault injection. Mathematical independence is a modeling hypothesis, not a property conferred by having two Pods. + +## 7. Does distributing six Pods over three zones guarantee surviving a zone failure? + +*Area: High availability* + +**Model answer:** Not by itself. The database, ingress, caches and remaining two zones must still be functional and have sufficient headroom. Hard topology spread may also block recovery if capacity is missing. Measure actual user transaction success during zone isolation and verify the scheduler, node autoscaler, load balancer and database failover assumptions, not just Pod distribution. + +## 8. What is the difference between a PDB and a multi-zone availability strategy? + +*Area: High availability* + +**Model answer:** A PodDisruptionBudget governs certain voluntary evictions using the Kubernetes Eviction API. It cannot prevent abrupt node or zone loss and does not directly enforce all Deployment rollout actions. Multi-zone design addresses fault-domain isolation and headroom; graceful Pod termination and PDBs reduce planned-maintenance impact. All are complementary, none is a complete guarantee. + +## 9. Why can an aggressive load-balancer health check cause an outage? + +*Area: High availability* + +**Model answer:** An overly strict dependency-aware check may mark every otherwise functional application Pod unhealthy when a shared dependency blips. Traffic can be rejected or routed unexpectedly under all-unhealthy behavior. Separate local liveness, readiness and user-journey health; use hysteresis and realistic thresholds; understand fail-open/fail-closed semantics of the specific load balancer. + +## 10. How would you choose between multi-zone and multi-region? + +*Area: High availability* + +**Model answer:** Start with quantified business RTO, RPO, geographic latency, compliance and failure-domain requirements. Multi-zone is often much simpler for regional availability, while multi-region adds replication, global routing, split-brain prevention, cost and operational complexity. Choose the least complex design meeting an explicit failure model, and demonstrate recovery with exercises. + +## 11. Why are three retries at each of three service layers dangerous? + +*Area: Timeouts and backpressure* + +**Model answer:** Nested retries amplify attempts multiplicatively. Even modest per-hop retries can create dozens of downstream requests per original call under correlated failure, increasing saturation and preventing recovery. Prefer one well-owned retry boundary, deadline propagation, attempt caps and an overall retry budget. Measure retries per user operation, not just per proxy. + +## 12. A payment POST times out after 3 seconds. Is it safe to resend? + +*Area: Timeouts and backpressure* + +**Model answer:** A timeout only proves the caller did not receive confirmation by its deadline. The provider may have committed the charge. Use a persisted idempotency key, query operation status or perform a provider-supported reconciliation before retry. Never equate transport failure with business failure; limit retries and keep an auditable operation state machine. + +## 13. How does a circuit breaker differ from a rate limiter? + +*Area: Timeouts and backpressure* + +**Model answer:** A rate limiter restricts admission based on a configured budget such as requests per second or burst tokens. A circuit breaker stops or reduces calls when a dependency is judged unhealthy or resource limits are breached, often with a controlled half-open probe. A concurrency limiter instead bounds in-flight operations. Combine mechanisms thoughtfully because each protects a different bottleneck. + +## 14. What failure can an unbounded queue cause even when no requests are dropped? + +*Area: Timeouts and backpressure* + +**Model answer:** The queue can grow until memory pressure, timeouts or unacceptable latency cause a larger outage. Retaining every request is not equivalent to serving it correctly within a deadline. Bound admission and queue age, shed work with clear overload semantics, and reserve capacity for critical requests. Prioritize oldest-item age and end-to-end success alongside queue depth. + +## 15. Why can a downstream service still fail after its original outage ends? + +*Area: Timeouts and backpressure* + +**Model answer:** Queued work and synchronized client retries create a recovery storm. The dependency may have recovered but immediately becomes saturated again, causing repeated circuit openings and timeouts. Use jittered backoff, retry budgets, staged worker concurrency and backlog-drain monitoring. Recovery design is part of reliability, not only failure detection. + +## 16. Does a transactional outbox make delivery exactly once end to end? + +*Area: Transactions* + +**Model answer:** No. It atomically records the business mutation and event intent, but publishing can repeat if a worker crashes between sending and recording delivery progress. The consumer needs deduplication/idempotent processing, and side effects across other systems require independent correctness controls. The outbox removes one dual-write failure window; it does not solve all distributed transaction semantics. + +## 17. How do you prevent double acceptance when two requests share an idempotency key? + +*Area: Transactions* + +**Model answer:** Enforce a unique constraint within the authoritative transaction and associate the key with a request fingerprint and stored final outcome. Concurrent inserts must serialize or conflict safely; a duplicate with different parameters should be rejected rather than reuse an unrelated outcome. Define retention and retry semantics. An in-memory cache key alone cannot guarantee durability or race safety. + +## 18. Can you decide bid winners from a stale cache? + +*Area: Transactions* + +**Model answer:** Not when correctness requires the current authoritative ordering and committed state. Caches or search replicas may lag, so they are useful for display or discovery but not necessarily for accepting or ordering bids. Use a single authoritative serialization point or a proven consistency model, then project committed results asynchronously to read replicas. Expose freshness semantics on read paths. + +## 19. If two regions both accept writes during a partition, what breaks? + +*Area: Transactions* + +**Model answer:** Without a defined global ordering/consensus or conflict-resolution model, both sides can commit incompatible outcomes. Last-write-wins can discard a legitimately acknowledged financial action. Establish a single writer or proven multi-writer semantics with fencing and an explicit correctness contract. During partition, some writes may need to be rejected to preserve consistency. + +## 20. Why not use two-phase commit for every cross-service operation? + +*Area: Transactions* + +**Model answer:** Global coordination increases coupling, lock duration and sensitivity to partial failure, and not all external services support a shared transaction coordinator. A saga with idempotent local transactions and compensations can be appropriate, but compensation is not always possible for irreversible actions. Choose based on business invariants rather than one universal distributed transaction recipe. + +## 21. Why isn't a cross-region database replica a complete backup? + +*Area: Disaster recovery* + +**Model answer:** Logical corruption, accidental deletes and ransomware can replicate to the standby. Backups need separate recovery points, retention, access controls and proven restore procedures. Measure whether a clean-room restore can recreate the application and required keys, not only whether replication is current. RPO and recovery scope should include dependent systems and configurations. + +## 22. What is the difference between RTO and detection time? + +*Area: Disaster recovery* + +**Model answer:** RTO is the target time from a specified disruptive event to restoration of the defined business service. Detection, decision, fencing, recovery, route convergence and validation all consume that budget. Measuring only automated failover duration understates customer recovery time. Build a recovery timeline that includes business correctness and manual dependencies. + +## 23. Can low DNS TTL guarantee instant region failover? + +*Area: Disaster recovery* + +**Model answer:** No. Recursive/client caching, stale resolvers, connection pools, existing TCP sessions and application endpoint configuration can outlive the advertised TTL. DNS changes also do not fence the old database writer. Validate actual convergence across networks, enforce safe authority changes and design clients to reconnect appropriately. + +## 24. What is a split-brain failure and how do you prevent it? + +*Area: Disaster recovery* + +**Model answer:** Split brain means more than one node or region believes it is authorized to mutate the same authoritative state. Prevent it through robust leader selection, quorum/fencing and write-authority controls before promoting a standby. Network partitions can make an old leader unreachable to some clients while still alive. Don't rely only on removing it from DNS. + +## 25. When is active-active multi-region the wrong choice? + +*Area: Disaster recovery* + +**Model answer:** When the business tolerates an RTO of hours or authoritative writes require simple ordering, multi-writer active-active can impose disproportionate cost and failure modes. Evaluate latency benefit against conflict semantics, data sovereignty, change complexity and operational support. A well-tested active-passive design can deliver better real reliability than an untested sophisticated setup. + +## 26. How do you distinguish request rate from concurrency? + +*Area: Capacity* + +**Model answer:** RPS measures arrivals per time; concurrency measures simultaneous in-flight work. Little's Law links them under stable conditions as average concurrency equals throughput times average time in system. A slow dependency can increase concurrency without higher arrival rate, filling queues and pools. Size with distributions and p99 rather than only average counts. + +## 27. Why might adding application replicas make latency worse? + +*Area: Capacity* + +**Model answer:** New replicas multiply connection pools, retries and requests against shared databases, caches or NAT ports. They also consume node memory and can warm caches simultaneously. Identify the tightest downstream capacity bound before scaling and apply admission/concurrency limits. Additional replicas help only if some component has additional useful capacity to provide. + +## 28. Does CPU-based HPA scale quickly enough for a predictable thirty-second traffic surge? + +*Area: Capacity* + +**Model answer:** Often not by itself: metrics, reconcile intervals, Pod scheduling, node provisioning, image pulls and readiness take time. If demand is predictable, pre-scale safely and keep node headroom. Use request/queue signals as appropriate and test the entire time-to-serving path. Reactive scaling alone cannot create instantaneous capacity. + +## 29. How would you calculate safe worker concurrency for a database-limited queue? + +*Area: Capacity* + +**Model answer:** Start from sustainable transaction throughput, per-operation cost, database connection limits and other consumers. Bound worker inflight requests and inspect p95/p99, lock contention and queue oldest age under representative load. A backlog target divided by desired completion time gives an approximate required service rate, not proof that the database can deliver it. + +## 30. What is the difference between a load test and a resilience experiment? + +*Area: Capacity* + +**Model answer:** A load test evaluates throughput, latency and saturation under traffic scenarios. A resilience experiment injects a specific fault while verifying business and recovery objectives, often under representative load. Combine them to expose failure behavior during peaks. Both need stop conditions, safe synthetic data, observable expected results and rollback. + +## 31. How do you avoid a cache stampede after a hot key expires? + +*Area: Data layer* + +**Model answer:** Use TTL jitter, singleflight/coalesced refresh, selective prewarming and controlled fallback concurrency. Stale-while-revalidate is safe only for data whose freshness tolerance is explicit. Rate-limit miss storms before the database saturates, and monitor hot-key frequency and cache misses. A distributed lock can itself become an availability dependency. + +## 32. Does ReadWriteOnce volume access guarantee one writer at the database level? + +*Area: Data layer* + +**Model answer:** No. RWO is primarily a storage attachment/access mode constraint, often scoped to a node, not a transaction or database fencing mechanism. Multiple processes may still access the filesystem, and failover may have ambiguous old-writer status. Use storage fencing, proven database replication/leadership and durable transaction protocols. + +## 33. Can an application serve catalog data stale during a database outage? + +*Area: Data layer* + +**Model answer:** Possibly, if product owners define a freshness bound and the endpoint is not used for authoritative actions. Mark freshness/version as appropriate, cap stale exposure and fall back selectively. Never report a successful monetary transaction based only on a stale cache. Separate read-degraded capability from write correctness. + +## 34. What makes a database failover create a second incident? + +*Area: Data layer* + +**Model answer:** Client pools may all reconnect at once, retries can amplify load, DNS or writer endpoints may converge slowly and read replicas can lag. Apply jittered reconnection, bounded pools, workload admission and correctness reconciliation. Test mixed connection states and ensure the previous writer is safely fenced before accepting new writes. + +## 35. Why is a snapshot not necessarily an application-consistent backup? + +*Area: Data layer* + +**Model answer:** A snapshot can capture blocks without ensuring in-memory transactions, WAL checkpoints or multi-volume coordination were consistent. Coordinate application/database backup semantics, test restore and measure RPO/RTO. Validate encryption keys, retention and integrity as well as whether a restored database passes application invariants. + +## 36. How do you prioritize changes during an ambiguous zone failure and bad deployment? + +*Area: Incidents* + +**Model answer:** Separate hypotheses and identify blast radius by release, zone, route and dependency. An incident commander assigns owners and records one change at a time where possible. Choose reversible mitigations with quantified capacity and transaction risks, then verify impact with user-facing SLIs. Avoid randomly restarting unrelated systems and losing diagnostic evidence. + +## 37. Is a rollback always the right incident mitigation? + +*Area: Incidents* + +**Model answer:** No. If the old version is incompatible with a deployed schema or external side effect, rollback can worsen corruption or outages. Evaluate the whole state transition first: image, database, queue messages, infrastructure routes and feature flags. Sometimes a forward-compatible hotfix or traffic isolation is safer. Record the decision and recovery conditions. + +## 38. What belongs in a blameless postmortem? + +*Area: Incidents* + +**Model answer:** Impact and affected user operations, factual timeline, contributing technical and organizational conditions, detection gaps, mitigation reasoning, verification, and prioritized preventive actions with owners. Avoid a single human-error conclusion. Test whether the actions genuinely prevent or reduce recurrence and close them when verified. + +## 39. What proves a game day was successful? + +*Area: Incidents* + +**Model answer:** A clearly stated failure hypothesis, scoped fault injection, expected customer SLO/RTO outcome, stop condition, observed metrics, recovery timing and evidence that business state remained correct. Merely restoring all Pods or having an experiment finish is insufficient. Document surprises and update runbooks, architecture assumptions and monitoring. + +## 40. How do you balance reliability spending against feature delivery? + +*Area: Incidents* + +**Model answer:** Use customer SLOs, error budgets, recovery requirements, incident history and marginal risk reduction to prioritize investments. Compare architecture maintenance and operational cost, not only cloud invoices. A second region might be less useful than fixing a shared database/identity failure or improving release controls. Make trade-offs explicit and review them with stakeholders. + +[Return to Module 7](./) · [Practice labs](./labs) diff --git a/docs/guides/reliability/index.md b/docs/guides/reliability/index.md new file mode 100644 index 0000000..8ee2fb6 --- /dev/null +++ b/docs/guides/reliability/index.md @@ -0,0 +1,47 @@ +# Cloud Architecture, Reliability & System Design + +**Module 7.** An eight-chapter field guide for senior platform engineers designing and operating reliable cloud services. It covers user-facing SLOs, failure domains, cascading failures, consistency, multi-region recovery, capacity, caching and incident leadership. + +::: info Public learning material +All marketplace/SaaS examples are fictional and designed for learning. Provider limits, API behavior, monitoring schema and feature support depend on the deployed version and service configuration. No example represents a verified private infrastructure deployment. +::: + +## Prerequisites + +Linux and container fundamentals; networking and TLS; Kubernetes workloads and scheduling; database transactions; familiarity with one cloud provider. + +## Learning objectives + +- Define business-outcome SLIs, realistic SLOs and error-budget policies. +- Design around node, zone, region, shared-dependency and human-error failure domains. +- Prevent cascading failure using deadlines, bulkheads, bounded queues, backpressure and safe retries. +- Preserve financial/event correctness through idempotency, authoritative storage and an outbox. +- Choose active-passive or active-active recovery only after defining RPO/RTO and fencing. +- Forecast peak capacity and include autoscaling delay plus downstream saturation. +- Design cache degradation that never violates authoritative write semantics. +- Lead an evidence-driven incident, verify recovery and invest in tested corrective actions. + +## Eight chapters + +1. [SLIs, SLOs, error budgets and reliability governance](./q1) +2. [High availability, multi-zone design and failure domains](./q2) +3. [Deadlines, retries, circuit breaking and backpressure](./q3) +4. [Transaction correctness, consistency and event delivery](./q4) +5. [Multi-region architecture, disaster recovery and RPO/RTO](./q5) +6. [Peak capacity, autoscaling and load testing](./q6) +7. [Data-layer reliability, caching and degradation](./q7) +8. [Incident leadership, game days and cost trade-offs](./q8) + +## Practice and sequence + +Read Q1-Q2 to define what must work, Q3-Q4 to preserve behavior under partial failure, Q5-Q7 to build resilient infrastructure/data paths, and Q8 to practice operating the design. Then complete [seven labs](./labs) and rehearse [40 challenging follow-ups with model answers](./follow-ups). + +## Authoritative references + +- [Google SRE book](https://sre.google/sre-book/table-of-contents/) +- [Google SRE workbook: alerting on SLOs](https://sre.google/workbook/alerting-on-slos/) +- [AWS Well-Architected Reliability Pillar](https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html) +- [Kubernetes scheduling and topology](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) +- [PostgreSQL transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) + +[← All engineering guides](/guides/) · [Start Q1 →](./q1) diff --git a/docs/guides/reliability/labs.md b/docs/guides/reliability/labs.md new file mode 100644 index 0000000..4eff6ec --- /dev/null +++ b/docs/guides/reliability/labs.md @@ -0,0 +1,88 @@ +# Module 7 · Reliability and Cloud Architecture Labs + +All tasks use a **sandbox or explicitly authorized isolated environment**. Never send synthetic financial operations to real payment endpoints. Each lab has a hypothesis, expected observation and rollback/stop condition. + +## Lab 1: Define a user-journey SLI and calculate error budget + +**Hypothesis:** HTTP 2xx rate can miss incorrect business outcomes. Create toy data with `eligible`, `accepted`, `durable`, `latency_ms`, `region`, and `version` fields. Define a good bid as eligible + one correct durable outcome within 500 ms. Calculate good/eligible across a 30-day rolling window and compare against 99.9% and 99.95% targets. + +```python +eligible = 100_000 +good = 99_930 +slo = 0.999 +bad = eligible - good +allowed_bad = eligible * (1 - slo) +print({"bad": bad, "allowed_bad": allowed_bad, "budget_left": allowed_bad - bad}) +``` + +**Expected:** 70 bad requests versus a 100-request 99.9% budget leaves 30 events. State why a 30-day request budget is not automatically a minute-based outage limit. **Safety:** synthetic data only; avoid personal identifiers in metrics. + +## Lab 2: Simulate a dependency timeout and retry storm + +Use a disposable HTTP service that intermittently sleeps beyond client timeout. Drive 100 original requests and record attempted upstream calls. Repeat with a maximum of two attempts, full jitter and a total request deadline. + +```bash +# Illustrative from an isolated client with a running test server: +curl --connect-timeout 1 --max-time 2 -v http://127.0.0.1:8080/slow +``` + +**Expected:** Retry amplification can exceed client request rate, while bounded retries cap additional work. **Stop condition:** do not overwhelm a shared environment. Document how a payment side effect would require idempotency rather than blind POST retries. + +## Lab 3: Validate multi-zone failure headroom + +List replicas and zone placement in a sandbox cluster. Calculate whether remaining zones would accommodate the removed zone's workload based on Pod requests and node allocatable capacity. + +```bash +kubectl get nodes -L topology.kubernetes.io/zone +kubectl get pods -A -o wide +kubectl get pdb -A +kubectl describe deploy catalog-api -n marketplace +``` + +**Expected:** Replica spread is not equivalent to guaranteed surviving-zone capacity. Don't cordon or drain production nodes. In a local sandbox, simulate an unavailable zone using test node affinity and verify Pending events. + +## Lab 4: Exercise a transaction outbox with duplicate delivery + +Create a toy SQL schema with a unique `idempotency_key`, a bid table and an outbox table. Within one transaction insert business outcome and outbox record; crash the relay after publishing but before recording progress. + +**Expected:** Publication repeats; consumer deduplication prevents duplicate business effects. Assert exactly one durable bid record for each operation key. **Safety:** use an ephemeral database, never real transaction records. + +## Lab 5: Plan regional failover with explicit fencing + +Write a tabletop timeline: detect primary failure, stop/fence old writes, inspect replication lag, promote replica, move routing, warm service, reconcile ambiguous operations and verify user journeys. + +```text +RTO target: 30 minutes +RPO target: 5 minutes +Measured lag before promotion: 90 seconds +Routing convergence and app warmup: 4 minutes +Question: was the old primary fenced before promotion? +``` + +**Expected:** Show measured worst-case RPO/RTO separately. DNS TTL alone is not fencing. **Safety:** do not fail over live databases as an educational exercise. + +## Lab 6: Load-test bottleneck and backpressure + +In a local test stack, vary arrival rate and record request throughput, p99, queue oldest age, downstream pool utilization and rejected requests. Add bounded queues and concurrency limits. + +**Expected:** Throughput reaches a knee; unbounded queueing raises p99 and timeouts. Controlled shedding can preserve latency for admitted work. Verify error responses communicate overload truthfully rather than fake success. + +## Lab 7: Run an incident game-day tabletop + +Use a fictional event: database primary restart during zone loss and application rollout. Assign incident commander, application, infrastructure and communications roles. Require one hypothesis per owner, a timeline, reversible mitigation and user/transaction verification. + +```bash +# Read-only observations on an approved sandbox cluster: +kubectl get pods -A -o wide +kubectl get events -A --sort-by=.lastTimestamp +kubectl rollout history deploy/catalog-api -n marketplace +kubectl get endpointslices -n marketplace +``` + +**Expected:** Recovery decision considers data state, zone headroom and rollback/schema compatibility. End with a learning review and measurable corrective actions. + +## Lab completion criteria + +For each exercise record: **fault hypothesis, measurement plan, observed evidence, impact, containment, recovery, follow-up and safety boundary**. Do not accept a green infrastructure status as proof that a user transaction is correct. + +[Module 7 index](./) · [40 challenging follow-ups](./follow-ups) diff --git a/docs/guides/reliability/q1.md b/docs/guides/reliability/q1.md new file mode 100644 index 0000000..4996491 --- /dev/null +++ b/docs/guides/reliability/q1.md @@ -0,0 +1,93 @@ +# Q1 · SLIs, SLOs, Error Budgets and Reliability Governance + +> **Interviewer:** Define meaningful reliability objectives for a multi-service marketplace. How do you measure reliability without hiding failed business transactions behind HTTP 200 responses? What should a team do when the error budget is nearly exhausted? + +## First principles + +Reliability is the probability that a system performs its intended function under stated conditions during a time interval. Availability, latency, durability, correctness and freshness are different qualities. A Kubernetes Pod being Ready is an infrastructure signal, not proof that an auction bid was accepted correctly. + +An **SLI** is a quantitative measure. An **SLO** is the target for that measure over an explicit window. An **SLA** is a contractual commitment with consequences. An **error budget** is the allowed amount of SLO misses, not a license to ignore failures. + +### User-journey SLIs + +| Journey | Eligible events | Good events | +| --- | --- | --- | +| View listing | Valid listing-view attempts at edge | Correct response under latency threshold | +| Place bid | Authenticated valid bid submissions | Exactly one authorized durable outcome within deadline | +| Checkout | Eligible checkout initiations | Correct accepted or explained outcome within deadline | +| Search freshness | Recently updated items | Changes visible within agreed lag | + +Clearly define denominators, exclusions, client cancellations, duplicate retries and low-traffic behavior. A synthetic GET /health should never be the only SLI for a transaction API. + +```mermaid +flowchart LR + C["Clients / synthetic probes"] --> E["Edge and request IDs"] + E --> A["Application outcome events"] + A --> M["Metrics, traces, structured logs"] + M --> S["SLI calculation and error-budget ledger"] + S --> P["Policy: release / investigate / escalate"] + A --> D["Durable transaction store"] +``` + +## Availability and budget arithmetic + +For a 30-day rolling window, 99.9% request availability permits approximately 0.1% bad eligible requests. If measured as time, 0.1% of 30 days is 43.2 minutes, but time and request-based budgets are not interchangeable. A heavy traffic outage can consume a request budget in minutes. + +```text +error_ratio = bad_eligible_events / all_eligible_events +budget = 1 - SLO_target +remaining_budget = budget - observed_error_ratio +burn_rate = observed_error_ratio / budget +``` + +Example: with a 99.95% request SLO, the allowed bad fraction is 0.0005. Over one million eligible requests the budget is 500 bad requests. A burst with 600 errors exhausts that budget, even if it lasted only two minutes. Define the window and denominator before reporting burn. + +### PromQL example + +Assume application instrumentation exports **mutually exclusive** counters for eligible requests and correct successful outcomes: + +```promql +sum(rate(api_requests_total{route="/bid",eligible="true",outcome="good"}[5m])) +/ +clamp_min( + sum(rate(api_requests_total{route="/bid",eligible="true"}[5m])), + 1e-9 +) +``` + +This query is illustrative. The label contract is owned by the instrumentation and must avoid duplicated counting, unbounded user IDs and accidental omission of failures that never reached the application. For latency SLOs, use histogram buckets aggregated with the appropriate threshold rather than averaging per-Pod percentiles. + +## Alerting + +Prefer multiwindow, multi-burn-rate paging: fast burn on short and longer windows catches acute incidents, slow burn catches sustained degradation. Alerting on raw CPU alone is less directly linked to the user experience. Separate page-level actionability from tickets for capacity trends. Account for low-volume routes so a single request does not create misleading statistical certainty. + +```mermaid +flowchart TD + O["Objective: 99.9% good requests"] --> B["Allowed bad fraction: 0.1%"] + B --> FAST["Fast burn: page on acute impact"] + B --> SLOW["Slow burn: ticket or page per policy"] + FAST --> TRIAGE["Check regional breakdown and deployment changes"] + SLOW --> REVIEW["Capacity and reliability work"] + TRIAGE --> SAFE["Mitigate, verify real user outcome"] +``` + +## Production failure exercise + +A gateway returns 200 to 100% of calls, while the bid-processing worker silently drops 3% of accepted messages. The HTTP availability chart is green. The business-correctness SLI must track accepted bid IDs through a durable outcome and alert on missing transitions past a bounded deadline. Reconciliation should detect missing records and correct them idempotently. A frontend response cannot establish business completion unless the contract specifically guarantees it. + +### Commands and evidence + +```bash +kubectl get deploy,pods -n marketplace +kubectl rollout history deploy/bidding-api -n marketplace +kubectl top pods -n marketplace +# Query request-success and business-outcome counters through monitoring +``` + +## Trade-offs and spoken answer + +An unattainable 100% SLO increases cost and encourages hiding failures; a permissive SLO ignores real harm. Negotiate targets by user journey, impact and cost. Segment by operation and region without creating hundreds of unowned objectives. Tie budget policy to release risk, not an automatic blanket release freeze. + +> “I'd define reliability from the user's successful operation, not infrastructure health. For a bid API I'd measure eligible accepted-and-durable outcomes within a deadline, plus latency. I'd set a rolling-window SLO, derive the allowed error budget, and page on meaningful burn rates. If the budget falls, we'd investigate the source of bad events, reduce risky changes where justified and invest in prevention. I'd verify instrumentation can detect incorrect 200 responses and lost asynchronous work. An SLO is a decision tool as well as a dashboard.” + +**References:** [Google SRE SLOs](https://sre.google/sre-book/service-level-objectives/) · [Alerting on SLOs](https://sre.google/workbook/alerting-on-slos/) diff --git a/docs/guides/reliability/q2.md b/docs/guides/reliability/q2.md new file mode 100644 index 0000000..7c10e93 --- /dev/null +++ b/docs/guides/reliability/q2.md @@ -0,0 +1,90 @@ +# Q2 · High Availability, Failure Domains and Dependency Mapping + +> **Interviewer:** Design an available catalog and checkout API within one cloud region. What survives a node or availability-zone failure? Which parts of a three-zone Kubernetes design remain single points of failure? + +## Define the failure model + +High availability means satisfying a defined service objective despite specified failures. Enumerate risks before adding replicas: process crash, node loss, rack/zone loss, regional network disruption, certificate expiry, bad deployment, shared database outage, credential failure and control-plane impairment. Independent replicas do not remove common-mode dependencies. + +```mermaid +flowchart TD + CLIENT["Clients"] --> EDGE["Regional/global entry"] + EDGE --> LB["Healthy-backend routing"] + subgraph REG["One region"] + Z1["Zone A: app replicas"] + Z2["Zone B: app replicas"] + Z3["Zone C: app replicas"] + end + LB --> Z1 + LB --> Z2 + LB --> Z3 + Z1 --> DB["Primary transactional DB"] + Z2 --> DB + Z3 --> DB + DB --> REP["Synchronous/async replica as configured"] + Z1 --> CACHE["Shared cache"] + Z2 --> CACHE + Z3 --> CACHE +``` + +The diagram is hypothetical. A database primary in one zone is a failure dependency even if its compute clients span three zones. Managed databases may automate failover, but the RPO/RTO and application reconnection behavior must be tested. + +### Availability math: not all failures are independent + +If two independent components both must be up and each has availability 99.9%, their simple series availability is 0.999 × 0.999 = 99.8001%. If two genuinely independent redundant instances can each serve the full workload, the probability both fail under the simplified model is (0.001)^2. Real zones share region-wide networks, IAM, DNS, deployments and operator actions. Correlated failure invalidates naive multiplication. + +### Deployment and topology controls + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: {name: catalog-api, namespace: marketplace} +spec: + replicas: 6 + selector: + matchLabels: {app: catalog-api} + template: + metadata: + labels: {app: catalog-api} + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: {app: catalog-api} + containers: + - name: app + image: registry.example.com/catalog@sha256:REPLACE_WITH_REAL_DIGEST + resources: + requests: {cpu: 250m, memory: 256Mi} + readinessProbe: + httpGet: {path: /ready, port: 8080} + periodSeconds: 5 +``` + +Illustrative image digest placeholder is not deployable. Hard spread constraints can leave Pods Pending if a zone lacks eligible nodes. Balance strict spread against available capacity. Use PDBs for voluntary evictions; they do not prevent involuntary zone loss or guarantee deployment rollout availability. + +### Dependency and blast-radius inventory + +Record for each dependency: ownership, zones/regions, write/read semantics, maximum tolerable outage, backup, failover trigger, recovery path, and whether it shares credentials or control plane with the primary service. For checkout, include payment-provider connectivity, idempotency storage, queue and fraud service. A cached listing may serve stale data; a final monetary transaction needs a stricter correctness contract. + +```bash +kubectl get pods -n marketplace -o wide +kubectl get pdb -n marketplace +kubectl describe deploy catalog-api -n marketplace +kubectl get endpointslices -n marketplace +kubectl get nodes -L topology.kubernetes.io/zone +``` + +## Failure scenario: zone A disappears + +Check endpoints removed from routing, database primary failover, surviving-zone headroom and rate of retries from displaced clients. Do not shift 100% of regional traffic to two zones if they were provisioned with zero spare capacity. A failed node's application sessions disappear; the replacement needs reconnect/resume logic. Test failure *while* a deployment is rolling, not only during steady state. + +### Senior trade-offs + +Single-region multi-zone is simpler and usually has lower write latency than multi-region active-active. Multi-region introduces conflict handling, replication lag, global routing and substantially greater operational cost. Use explicit business recovery requirements to justify it. + +> “I'd begin with the failure domains the service must withstand. Multiple zones remove individual node and zone dependency only if routing, replicas, storage and remaining capacity are independently viable. I'd map shared databases, identity, DNS and rollout pipelines as common failure points. Then I'd prove the design with controlled zone failure tests and measure customer success, not just Pod readiness.” + +**References:** [Google SRE addressing cascading failures](https://sre.google/sre-book/addressing-cascading-failures/) · [Kubernetes topology spread](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) diff --git a/docs/guides/reliability/q3.md b/docs/guides/reliability/q3.md new file mode 100644 index 0000000..169a503 --- /dev/null +++ b/docs/guides/reliability/q3.md @@ -0,0 +1,74 @@ +# Q3 · Timeouts, Retries, Circuit Breakers, Backpressure and Bulkheads + +> **Interviewer:** A downstream payment provider becomes slow. Explain how bad timeout and retry policies can take down otherwise healthy APIs. Design a bounded failure-handling strategy. + +## Budget the request end to end + +A 2-second client deadline is not compatible with four synchronous internal calls each configured with a 2-second timeout and three retries. Specify absolute request deadlines, reserve time for serialization and response, and propagate cancellation through all components. Not every library propagates deadline or cancellation automatically. + +```mermaid +sequenceDiagram + participant C as Client + participant API as API gateway + participant APP as Checkout + participant PAY as Payment provider + C->>API: Request, 2s deadline + API->>APP: Propagated deadline + APP->>PAY: Attempt 1, bounded timeout + PAY--xAPP: Timeout + Note over APP: Is retry safe and within deadline? + APP->>PAY: Attempt 2 after jitter, if budget allows + PAY-->>APP: Valid response + APP-->>API: Durable result + API-->>C: One final outcome +``` + +### Timeouts by phase + +Connection timeout covers establishment; TLS/handshake has its own behavior; read/header/body timeouts govern later phases; total deadline governs the overall operation. Idle connection timeout is different from end-to-end request deadline. Set them to reflect latency distributions and resource budgets, not arbitrary copied constants. + +### Retry math + +If every original request can make three attempts, 2,000 incoming RPS can become 6,000 upstream attempts at peak. Nested retries multiply further. Retry *only* transient, recoverable failure classes, with exponential backoff and full jitter, per-hop attempt caps, a shared deadline and a global retry budget. A timeout after sending a payment is ambiguous: the remote side may have committed it. Use a persisted idempotency key and reconcile outcomes before replaying. + +```python +import random +def next_delay(attempt: int, base: float = 0.1, cap: float = 3.0) -> float: + return random.uniform(0.0, min(cap, base * 2 ** attempt)) +``` + +This illustrates delay generation only. Production retry code also needs a deadline, retryable-error predicate, idempotency policy and cancellation. + +### Circuit breakers and bulkheads + +A breaker uses closed/open/half-open states to avoid repeatedly calling an unhealthy dependency. It protects resources but can reject real customers. A bulkhead isolates concurrency pools or queues so a failing dependency cannot consume all request workers. Queue length must be bounded, with deliberate overload responses. + +```mermaid +stateDiagram-v2 + [*] --> Closed + Closed --> Open: failures/latency cross threshold + Open --> HalfOpen: cooldown elapsed + HalfOpen --> Closed: probe succeeds + HalfOpen --> Open: probe fails +``` + +### Retry storms, load shedding and fairness + +When a payment provider recovers, queued work plus client retries may re-saturate it. Apply admission control before expensive processing and return explicit overload status with retry hints where protocol supports it. Preserve fairness between tenants or workloads if one caller can consume shared capacity. Rate limits protect one layer; concurrency limits protect in-flight work. + +## Incident drill + +Checkout p99 moves from 150 ms to 7 seconds while CPU is low. Distributed traces show 90% of time in external calls, thread pools near capacity, retry attempts tripled. Mitigate by reducing attempts, enforcing a bulkhead and preserving idempotent finalization. Evaluate whether temporarily degrading a noncritical fraud enrichment is legally/business permissible before bypassing anything. + +```bash +kubectl top pods -n checkout +kubectl logs deploy/checkout-api -n checkout --since=10m +# Examine upstream requests, retries, deadlines and queue metrics +# Sample correlation using request IDs in a tracing backend +``` + +**Observability:** attempts per original operation, breaker state transitions, pending/active call count, rejected concurrency, deadline exceeded, server-side committed outcome, and downstream latency distribution. + +> “I'd give each request a total deadline and partition it among dependencies. Retrying a failed call is a new load event, so I'd use bounded budgets, jitter and retryable error classification. For checkout I'd also require idempotency keys because a timeout doesn't prove the side effect failed. Bulkheads, bounded queues and circuit breakers limit cascading failure, while metrics show whether we're protecting or merely shifting the bottleneck.” + +**References:** [Google SRE overload and cascading failures](https://sre.google/sre-book/addressing-cascading-failures/) · [AWS Builders Library timeouts and retries](https://aws.amazon.com/builders-library/timeouts-retries-and-backoff-with-jitter/) diff --git a/docs/guides/reliability/q4.md b/docs/guides/reliability/q4.md new file mode 100644 index 0000000..4927f9d --- /dev/null +++ b/docs/guides/reliability/q4.md @@ -0,0 +1,70 @@ +# Q4 · Transaction Correctness, Consistency and Event Delivery + +> **Interviewer:** A bidder clicks Submit twice after a timeout. The API, database and event queue disagree briefly. Design a system that avoids double-processing and can recover from ambiguous outcomes. + +## Availability is not correctness + +A system may be available for reads while writes need consensus or a single authoritative owner. Business correctness includes preventing duplicate effects, preserving order where required, and returning truthful status to callers. Clarify invariants before choosing technology: e.g. each unique bid request ID has one durable result, only valid bids participate, and the winner is computed from committed history. + +### Idempotent write pattern + +Require a client-generated or server-issued operation ID, store its result alongside the business transaction under a unique constraint, and return the same result on replay. Enforce the uniqueness at the authoritative transactional boundary rather than only in a best-effort cache. + +```sql +CREATE TABLE bid_requests ( + idempotency_key TEXT PRIMARY KEY, + listing_id BIGINT NOT NULL, + outcome TEXT NOT NULL, + committed_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +-- This table is schematic; actual bid validation and transaction +-- isolation must be designed together with ledger writes. +``` + +### Transactional outbox + +Without an outbox, this sequence is unsafe: commit database change, crash before publishing event. Publishing first is also unsafe: event delivered but database transaction rolls back. Write the business mutation and outbox record in *one database transaction*, then asynchronously publish and mark delivery progress. Consumers must deduplicate because publication can repeat. + +```mermaid +sequenceDiagram + participant API as API + participant DB as Transaction DB + participant RELAY as Outbox relay + participant BUS as Event bus + participant CON as Consumer + API->>DB: BEGIN + API->>DB: Insert bid + idempotency result + outbox row + API->>DB: COMMIT + DB-->>API: Durable acknowledgment + RELAY->>DB: Poll committed unsent rows + RELAY->>BUS: Publish with event ID + BUS->>CON: Deliver (possibly repeated) + CON->>CON: Deduplicate / idempotently apply + RELAY->>DB: Mark delivered or checkpoint +``` + +### Ordering, concurrency and isolation + +Two concurrent bids on one listing may need serialization at a per-listing authority, an optimistic version check or a suitable database transaction isolation/locking strategy. Global event ordering is expensive and not usually required. Partition order by aggregate ID only if the broker guarantees it and the business operation needs it. “Exactly once” can refer to a narrow broker processing model but is not a universal end-to-end business guarantee across a database and external services. + +### Consistency design choices + +- Strong writes with one owner simplify bid correctness but create a write availability trade-off during partition. +- Eventually consistent caches/search indexes improve read scale but must expose freshness bounds and not decide authoritative winners. +- Sagas coordinate multi-service workflows with compensations, but not every side effect is reversible (e.g. money movement). +- Replication lag affects reads after writes; route critical confirmation to authoritative committed state or enforce read-your-writes semantics where supported. + +## Incident simulation + +A broker outage lasts 10 minutes; bidding API commits successfully and outbox backlog grows. Customers see accepted results; downstream notifications are delayed. SLOs must separately capture durable acceptance and event freshness. On recovery, the relay uses bounded concurrency, deduplication and backlog-aware autoscaling. Do not publish uncommitted rows or replay blindly. + +```bash +kubectl get pods -n marketplace -l app=outbox-relay +kubectl logs -n marketplace deploy/outbox-relay --since=20m +# DB: query committed pending outbox count, oldest age, duplicate IDs +# Broker: inspect lag, retry and dead-letter metrics +``` + +> “I'd write down the transaction invariants before selecting a queue. For repeated bid submissions I'd use an idempotency key backed by a database uniqueness constraint and store the final outcome. I'd use a transactional outbox to close the commit-versus-publish gap, and make consumers idempotent because delivery is at least once in many practical setups. Caches and events can be eventually consistent, but the authoritative bid decision needs a well-defined consistency boundary.” + +**References:** [Microservices.io transactional outbox](https://microservices.io/patterns/data/transactional-outbox.html) · [PostgreSQL transaction isolation](https://www.postgresql.org/docs/current/transaction-iso.html) diff --git a/docs/guides/reliability/q5.md b/docs/guides/reliability/q5.md new file mode 100644 index 0000000..cebd68a --- /dev/null +++ b/docs/guides/reliability/q5.md @@ -0,0 +1,90 @@ +# Q5 · Multi-Region Design, Disaster Recovery and RPO/RTO + +> **Interviewer:** Your marketplace runs in one European region and product leadership wants regional-failure protection. Compare active-passive and active-active designs. How do you establish a realistic RPO and RTO and rehearse failover? + +## Begin with business continuity objectives + +**RTO** is the target maximum time to restore a service after a qualifying disruption. **RPO** is the target maximum acceptable age of recoverable data, often expressed as time. Neither guarantees recovery unless regularly verified. Distinguish restoration of the UI, read-only catalog, checkout authorization and authoritative bid submission because business requirements may differ. + +A failure taxonomy matters: zonal failure, full region outage, operator deletion, corrupted database, credential compromise and globally deployed defective code. Multi-region compute alone cannot repair corruption replicated to both regions. + +## Architecture choices + +| Pattern | Strength | Trade-offs | +| --- | --- | --- | +| Single region, multiple zones | Simpler, usually low write latency | Region remains failure domain | +| Active-passive, warm standby | Clear primary for writes, controllable failover | Standby cost, readiness/replication lag, failover time | +| Active-active reads, primary writes | Low-latency regional reads | Consistency and read freshness choices | +| Multi-writer active-active | Regional write availability | Conflict handling, fencing, global consistency, cost | + +```mermaid +flowchart TD + G["Global traffic routing / health policy"] --> A["Primary region"] + G -.->|"Failover only after decision"| B["Standby region"] + subgraph AREG["Primary"] + A --> APIA["Active API and workers"] + APIA --> DBA["Authoritative database writer"] + end + subgraph BREG["Standby"] + B --> APIB["Warm service capacity"] + APIB --> DBB["Replica / restored database"] + end + DBA -->|"Asynchronous replication: nonzero lag"| DBB + S["Independent backups"] -.-> DBA + S -.-> DBB +``` + +The diagram is illustrative. A standby replica is not a reliable backup against logical corruption or accidental deletion that replicates. + +## Failover is a coordinated state transition + +1. Detect and confirm genuine primary failure using multiple signals. +2. Decide who is authorized to initiate failover and how to avoid false positives. +3. **Fence old writers** or otherwise guarantee there is only one authoritative writer. +4. Confirm standby data consistency and measured lag. +5. Promote the standby if permitted by database semantics. +6. Switch client routing, secrets/configuration and supporting workers. +7. Resume idempotent processing; reconcile ambiguous transactions. +8. Verify real customer journeys and data invariants. +9. Document return-to-primary plan and avoid automatic failback without data reconciliation. + +```mermaid +sequenceDiagram + participant M as Detection/incident lead + participant A as Primary writer + participant B as Standby DB + participant R as Global router + participant C as Clients + M->>M: Confirm impact and eligible failover + M->>A: Fence writes (or establish primary is unavailable) + M->>B: Inspect lag / promote safely + B-->>M: New write authority ready + M->>R: Shift eligible traffic + R-->>C: Send new connections to standby + M->>M: Reconcile ambiguous writes and monitor +``` + +### Split-brain danger + +Two regions independently accepting writes against the same logical listing can produce incompatible accepted bids. DNS failover isn't a consensus algorithm: DNS TTLs, caches and existing sessions also delay traffic shifts. If the old region returns, it must not resume writes until leadership is explicitly resolved. A multi-writer database demands a proven conflict/ordering model; last-write-wins is usually inappropriate for financial or auction invariants. + +## Recovery testing + +Create a game day in a sandbox or preapproved production slice: isolate one region, inject DNS/routing failure, simulate a dead primary, verify backup restore to a new environment and measure outcome. Report separately **detection time, decision time, database promotion, routing, application warmup and business reconciliation**. Test credential/KMS reachability from the standby region. + +```bash +# Read-only investigation examples; replace synthetic names +kubectl get deploy,pods -A -o wide +kubectl get events -A --sort-by=.lastTimestamp +# Query provider database replication lag and backup status +# Verify DNS resolution from multiple networks and client regions +dig api.example.com A +``` + +### When *not* to choose multi-region + +If the stated RTO is hours and the product tolerates some stale catalog data, expensive active-active operation may add more failure modes than it solves. RPO zero plus low-latency global writes demands an architecture with corresponding consensus/coordination costs. Establish recovery economics instead of promising “zero downtime” from adding a second region. + +> “I'd start with per-journey RTO and RPO and name the failures we must survive. A warm standby with one authoritative writer is often easier to reason about than multi-writer active-active. I would account for replication lag, fencing, route convergence, dependencies and ambiguous writes. Then I'd rehearse promotion and restoration, measure actual recovery against targets and verify transaction correctness, not just that both regions contain running Pods.” + +**References:** [Google SRE disaster recovery](https://sre.google/sre-book/availability-table/) · [AWS reliability pillar](https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html) diff --git a/docs/guides/reliability/q6.md b/docs/guides/reliability/q6.md new file mode 100644 index 0000000..c05ab83 --- /dev/null +++ b/docs/guides/reliability/q6.md @@ -0,0 +1,79 @@ +# Q6 · Capacity Planning, Load Testing, Queues and Peak Events + +> **Interviewer:** Traffic quadruples in the last minute of a time-bounded event. How do you capacity plan an API and workers when autoscaling and new nodes both take time? + +## Model demand instead of picking replica counts + +Capture ordinary, peak and pathological conditions: request rate, concurrency, per-request CPU time, memory working set, downstream calls, new connections, file descriptors, queue backlog and data growth. For synchronous services apply Little's Law, L = λ × W, only where throughput and residence time are measured consistently in stable conditions. E.g. 2,000 RPS at 0.1 seconds mean about 200 average in-flight requests, *not* 200 pods. + +### Resource budget + +```text +capacity_per_pod = sustainable RPS at target p99 and error rate +steady_replicas = ceil(expected_RPS / capacity_per_pod) +failure_headroom = enough replicas/nodes to survive stated zone loss +effective_capacity = min(API, DB, cache, queue, NAT, upstream provider) +``` + +One bottleneck governs the useful end-to-end capacity. Doubling Pods can exhaust a database connection pool and worsen throughput. CPU HPA is reactive and may miss waiting/blocked workflows; queue-depth scaling can miss variation in processing cost. + +```mermaid +flowchart TD + TRAFFIC["Traffic forecast + live signals"] --> PRE["Pre-warm minimum capacity"] + PRE --> APPS["Application replicas"] + TRAFFIC --> HPA["HPA/queue scaler"] + HPA --> APPS + APPS --> SCHED["Scheduler"] + SCHED --> NODES["Provision nodes / pull images"] + APPS --> DB["Database pool + writes"] + APPS --> CACHE["Caches"] + APPS --> OUT["External dependency / NAT"] + DB --> LIMIT["End-to-end capacity ceiling"] + CACHE --> LIMIT + OUT --> LIMIT +``` + +## Autoscaling latency budget + +Scale-out includes metric sample age, controller sync, ReplicaSet creation, scheduling, cloud node provisioning, image pull, initialization, readiness and load-balancer registration. For known deadline-based traffic peaks, scheduled pre-scaling and node headroom may be justified. Preserve an intentional scale-down stabilization window to avoid oscillating before the next burst. + +### Load test dimensions + +- Baseline steady state: sustained representative mix. +- Step load: detect saturation knee and transient queues. +- Spike: sudden arrivals and TLS/connect churn. +- Soak: memory/FD leaks and cache growth. +- Failure under load: zone loss or database restart. +- Recovery: backlog drain and retry storm. +- Write correctness: unique accepted operations, idempotency replays, consumer lag. + +A load test that sends only GET /health or runs from one source IP may miss the true workload. Use synthetic safe data and a production-like dependency topology without generating real financial effects. + +## Queue mathematics and fairness + +For a queue with arrival rate λ and service capacity μ, backlog grows while λ > μ. More workers help only if downstream capacity permits. Bound queue size, set message age SLOs, allocate dead-letter paths, and use backpressure. A fair scheduler can prevent a single tenant's large export from starving interactive purchases. + +```promql +# Illustrative backlog time: queued_items / processed_items_per_second +sum(worker_queue_depth) +/ +clamp_min(sum(rate(worker_processed_total[5m])), 0.001) +``` + +This is an *approximation* that assumes roughly constant arrival and processing cost; it is not a guaranteed time-to-drain during changing traffic. For different message classes, split by queue and measure oldest-item age directly. + +## Practical validation + +```bash +kubectl get hpa -A +kubectl describe hpa catalog-api -n marketplace +kubectl top pods -n marketplace +kubectl get pods -A --field-selector=status.phase=Pending +kubectl get nodes -L topology.kubernetes.io/zone +``` + +Correlate request p99, success rate, queue depth, accepted/rejected connections, database saturation and node provisioning latency. Validate with traffic that matches your event profile. + +> “I would load-test sustainable throughput at the required p99, then budget for dependent systems and failure headroom. For a predictable spike I'd pre-provision enough nodes and warm application replicas rather than expect HPA to react instantly. I'd add admission control and bounded queues to prevent overload, watch the next bottleneck such as database or NAT, and test both spike onset and recovery with business-correctness assertions.” + +**References:** [Kubernetes HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) · [Google SRE handling overload](https://sre.google/sre-book/handling-overload/) diff --git a/docs/guides/reliability/q7.md b/docs/guides/reliability/q7.md new file mode 100644 index 0000000..57522e4 --- /dev/null +++ b/docs/guides/reliability/q7.md @@ -0,0 +1,81 @@ +# Q7 · Data-Layer Reliability, Caching and Graceful Degradation + +> **Interviewer:** The cache cluster fails during peak catalog browsing, then the database's primary restarts. How do you avoid a thundering herd and preserve correct writes while keeping noncritical read paths useful? + +## Map consistency by operation + +| Operation | Possible consistency | Key failure concern | +| --- | --- | --- | +| Product image/static asset | CDN eventual refresh | Stale/missing content | +| Catalog search | Eventually consistent index | Freshness lag | +| Listing read | Cache backed by authoritative store | Stale TTL, stampede | +| Bid acceptance | Transactionally authoritative | Concurrent writes, duplicate effects | +| Payment state | Strong idempotent state machine | Ambiguous external outcomes | + +The correct cache strategy depends on acceptable staleness and failure semantics. A cache must not become the sole source of authoritative transaction history without a durability design. + +## Cache-aside request flow + +```mermaid +sequenceDiagram + participant C as Client + participant A as Catalog API + participant K as Cache + participant D as Authoritative DB + C->>A: Read listing + A->>K: Get key + alt Hit + K-->>A: Cached value + else Miss + A->>D: Query listing + D-->>A: Current value + A->>K: Set with TTL + jitter + end + A-->>C: Response with freshness semantics +``` + +### Stampede controls + +If one popular key expires during an auction event, thousands of requests may hit the DB. Prefer suitable combination of TTL jitter, singleflight/request coalescing, stale-while-revalidate for safely stale content, bounded cache fills and prewarming. These mechanisms have failure trade-offs: a distributed lock service might become another bottleneck; excessive stale serving can violate bid correctness. + +### Database saturation and failover + +Database failover can invalidate connections, rotate writer endpoints and trigger pool reconnection bursts. Set client reconnect jitter, pool ceilings and query deadlines. Reader replicas may lag and should not answer read-after-write-critical operations without an explicit consistency mechanism. Transactions need appropriate isolation and retry semantics when a leader changes. + +```mermaid +flowchart TD + REQ["Read request"] --> CLASS{"Authoritative or cache-safe?"} + CLASS -->|Authoritative| DB["Current transactional state"] + CLASS -->|Cache safe| C["CDN/cache"] + C -->|Healthy| RESP["Serve with version/freshness metadata"] + C -->|Failed| LIMIT["Coalesced, bounded DB fallback"] + LIMIT --> DB + DB --> RESP + DB -->|Overloaded| DEG["Graceful degradation for eligible reads"] +``` + +## Graceful degradation examples + +- Catalog images: serve an appropriate cached variant. +- Search: show cached results with clear freshness expectations. +- Recommendations: skip personalization if nonessential. +- Bid submission: reject clearly or queue only if the product contract supports delayed acceptance; never falsely return accepted when not committed. +- Checkout: don't fake a successful payment because an upstream timed out. + +### Operational proof + +```bash +kubectl get endpointslices -n marketplace +kubectl top pods -n marketplace +# Inspect application connection-pool busy/wait metrics +# Compare DB primary/replica lag and failover events +# Inspect cache hit/miss, hot-key and eviction patterns +``` + +### Failure scenario + +A Redis-like cache restarts; cache miss ratio approaches 100%, DB pool fills, API p99 rises, retries increase work. First contain blast radius: shed optional read load, cap DB concurrency and enable safe stale content while cache warms. Validate that committed bidding remains authoritative. In the postmortem assess TTL policy, cache-warm strategy, dependent pool sizes and disaster tests. + +> “I'd define which data may be stale and which must be authoritative. A cache failure shouldn't make every reader stampede onto the database; I'd use coalescing, jitter and bounded fallback, with stale responses only for operations where correctness permits it. During database failover I'd protect connection pools and retry only safe operations. I would rather return a clear temporary failure for an uncommitted bid than a misleading success.” + +**References:** [Google SRE data integrity](https://sre.google/sre-book/data-integrity/) · [PostgreSQL high availability](https://www.postgresql.org/docs/current/high-availability.html) diff --git a/docs/guides/reliability/q8.md b/docs/guides/reliability/q8.md new file mode 100644 index 0000000..f19c2a6 --- /dev/null +++ b/docs/guides/reliability/q8.md @@ -0,0 +1,64 @@ +# Q8 · Incident Command, Game Days, Error-Budget Policy and Cost Trade-offs + +> **Interviewer:** A rollout and a cloud-zone disturbance happen together. The platform is intermittently failing and three teams want different fixes. How do you lead the incident, prove recovery, and make the system safer? + +## Incident leadership model + +Identify incident commander (IC), operations/technical lead, communications owner and scribe according to team size. One person may combine roles in a small incident, but decision rights should remain clear. Declare severity from customer/business impact rather than a universal CPU threshold. Preserve an event timeline, decisions and mitigation owners. + +```mermaid +flowchart TD + ALERT["Customer-impacting symptom"] --> SCOPE["Scope by user journey, zone, release and dependency"] + SCOPE --> OWNER["IC assigns focused investigations"] + OWNER --> H1["App + transaction consistency"] + OWNER --> H2["Cloud routing + node/zone health"] + OWNER --> H3["Recent deployment / policy changes"] + H1 --> DEC["Evidence and impact briefing"] + H2 --> DEC + H3 --> DEC + DEC --> MIT["Reversible mitigation with clear owner"] + MIT --> VERIFY["Verify SLO + correct business outcomes"] + VERIFY --> REVIEW["Learning review and tracked actions"] +``` + +### Triage sequence + +1. Confirm actual blast radius and correctness risks. Can users submit bids twice? Are accepted results durable? +2. Compare current errors and p95/p99 by route, region, release, pod and dependency. +3. Correlate feature flags, Git/rollout events, cloud advisories and recent changes. +4. Make the smallest reversible mitigation: halt promotion, revert a known-bad release, isolate unhealthy endpoints, shed optional work. +5. Ensure surviving zones can serve redirected traffic before shifting load. +6. Verify user journey, data reconciliation and recovery of queue lag. +7. Avoid random simultaneous changes that destroy the ability to attribute improvement. + +### Emergency rollback may not reverse data mutations + +An image rollback does not roll back schema migrations, external payments, Terraform applies or published events. Record the cutback procedure per dependency. Prefer expand-and-contract schema evolution and idempotent recovery. Re-run testing with a realistic mixed-version state. + +## Sample read-only runbook + +```bash +kubectl rollout status deploy/catalog-api -n marketplace +kubectl rollout history deploy/catalog-api -n marketplace +kubectl get pods -n marketplace -o wide +kubectl get events -n marketplace --sort-by=.lastTimestamp +kubectl get endpointslices -n marketplace +kubectl get pdb -n marketplace +# Review tracing, LB target health, database and cloud region signals +``` + +### Game days + +Experiment on one failure assumption at a time: kill a node, refuse database connections, disable a zone in a disposable environment, expire a test certificate, simulate partial network partition, and restore from backup. A passing experiment must include SLO outcome, time-to-detect, time-to-mitigate, rollback, safety guardrails and corrected documentation. Never initiate destructive tests in production without explicit authorization and a tested stop mechanism. + +## Learning review versus blame + +A high-quality review includes impact, timeline, contributing technical and organizational factors, why safeguards didn't detect it earlier, which assumptions failed and prioritized follow-ups with owners and due dates. “Engineer should be more careful” is not a useful system improvement. Examples: rollout controller should block on elevated error budget burn; restore tests should run quarterly; tenant retry budget should be enforced in SDK defaults. + +## Reliability and cost + +Availability improvements have diminishing returns and opportunity costs. Evaluate cost per unit of risk reduction, engineering maintenance, failure correlation and data correctness. A second region may be unjustified if the required RTO is hours; a multi-zone database failover test may reduce more risk at lower cost. Present trade-offs, not “always multi-region”. + +> “I'd appoint clear incident roles, define the affected user operation and assign investigations by failure boundary. I'd stabilize with a narrow reversible change, track ambiguous transactions separately, and verify success at the user and data layer. After recovery I'd conduct a blameless technical review, identify which assumptions and guardrails failed, and test the preventive actions through game days. Architecture changes need to earn their operational cost against agreed RTO, RPO and SLO objectives.” + +**References:** [Google SRE incident management](https://sre.google/sre-book/managing-incidents/) · [Google SRE postmortem culture](https://sre.google/sre-book/postmortem-culture/)