From 265a5dbf3274f20c7305bca0e11e6c40711ab4cb Mon Sep 17 00:00:00 2001 From: Chad Ferman Date: Fri, 10 Apr 2026 10:19:14 -0500 Subject: [PATCH 1/2] ci: add Dependabot config and pin ShellCheck action to v2.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automated dependency management via Dependabot for GitHub Actions and Python pre-commit hooks. Replace unstable @master reference with stable v2.0.0 release for ludeeus/action-shellcheck to prevent unexpected breaking changes in CI workflows. Changes: - Add .github/dependabot.yml for weekly dependency updates - Update pr-validation.yml: ludeeus/action-shellcheck@master → @2.0.0 - Update shell-script-testing.yml: ludeeus/action-shellcheck@master → @2.0.0 Co-Authored-By: Claude Sonnet 4.5 --- .github/dependabot.yml | 46 ++++++++++++++++++++++ .github/workflows/pr-validation.yml | 2 +- .github/workflows/shell-script-testing.yml | 2 +- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9061361 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,46 @@ +version: 2 +updates: + # GitHub Actions version updates + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + labels: + - "dependencies" + - "github-actions" + commit-message: + prefix: "ci" + include: "scope" + open-pull-requests-limit: 5 + reviewers: + - "cferman" + # Group all GitHub Actions updates into a single PR + groups: + github-actions: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + # Pre-commit hooks updates + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + labels: + - "dependencies" + - "python" + commit-message: + prefix: "chore" + include: "scope" + open-pull-requests-limit: 3 + # Only update if there are requirements files or pre-commit config changes + ignore: + # Ignore patch updates for stability unless security-related + - dependency-name: "*" + update-types: ["version-update:semver-patch"] diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 938bad7..c6edfb9 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -132,7 +132,7 @@ jobs: uses: actions/checkout@v4 - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master + uses: ludeeus/action-shellcheck@2.0.0 with: severity: error ignore_paths: node_modules vendor diff --git a/.github/workflows/shell-script-testing.yml b/.github/workflows/shell-script-testing.yml index 3b1d27f..8e65a94 100644 --- a/.github/workflows/shell-script-testing.yml +++ b/.github/workflows/shell-script-testing.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master + uses: ludeeus/action-shellcheck@2.0.0 with: severity: warning ignore_paths: | From c7c874aecf4481c2cf13b68c284e2f592fc177ba Mon Sep 17 00:00:00 2001 From: Chad Ferman Date: Fri, 10 Apr 2026 14:50:15 -0500 Subject: [PATCH 2/2] refactor: reorganize testing and CI infrastructure into top-level tests/ directory Move all testing, validation, and CI-related scripts and manifests from scripts/ and openshift/dr-testing/ into a new top-level tests/ directory for better organization and separation of concerns. New structure: - tests/scripts/ - Test and validation scripts (DR, RTO/RPO, validation) - tests/hooks/ - Pre-commit and CI hooks - tests/openshift/ - OpenShift DR testing CronJob manifests - tests/README.md - Comprehensive testing documentation scripts/ now contains only operational scripts: - AAP scaling and cluster management - EFM integration hooks - Shared operational libraries Changes (30 files): - Moved 6 test scripts to tests/scripts/ - Moved 2 CI hooks to tests/hooks/ - Moved 6 DR testing manifests to tests/openshift/dr-testing/ - Updated 15 documentation files with new paths - Updated .pre-commit-config.yaml with new hook paths - Updated CLAUDE.md and README.md repository structure diagrams - Created tests/README.md with complete testing documentation All file moves preserve complete git history via 'git mv'. Co-Authored-By: Claude Sonnet 4.5 --- .pre-commit-config.yaml | 4 +- CLAUDE.md | 34 ++++----- CONTRIBUTING.md | 3 +- README.md | 10 +-- docs/INDEX.md | 14 ++-- docs/aap-containerized-quickstart.md | 4 +- docs/architecture.md | 2 +- docs/component-testing-results.md | 2 +- docs/dr-replication-implementation-status.md | 18 ++--- docs/dr-testing-guide.md | 6 +- docs/dr-testing-implementation-summary.md | 10 +-- docs/quick-start-guide.md | 4 +- docs/scripts-hooks-and-cicd.md | 28 ++++---- reports/dr-architecture-validation-report.md | 2 +- scripts/README.md | 62 +++++------------ tests/README.md | 69 +++++++++++++++++++ .../hooks/check-script-permissions.sh | 0 .../hooks/validate-openshift-manifests.sh | 0 .../openshift}/dr-testing/README.md | 0 .../dr-testing/configmap-dr-scripts.yaml | 8 +-- .../dr-testing/cronjob-dr-test.yaml | 0 .../openshift}/dr-testing/kustomization.yaml | 0 .../dr-testing/pvc-test-results.yaml | 0 .../openshift}/dr-testing/serviceaccount.yaml | 0 .../scripts}/dr-failover-test.sh | 0 .../scripts}/generate-dr-report.sh | 0 {scripts => tests/scripts}/measure-rto-rpo.sh | 0 .../scripts}/run-ci-checks-locally.sh | 0 .../scripts}/test-split-brain-prevention.sh | 0 .../scripts}/validate-aap-data.sh | 0 30 files changed, 163 insertions(+), 117 deletions(-) create mode 100644 tests/README.md rename {scripts => tests}/hooks/check-script-permissions.sh (100%) rename {scripts => tests}/hooks/validate-openshift-manifests.sh (100%) rename {openshift => tests/openshift}/dr-testing/README.md (100%) rename {openshift => tests/openshift}/dr-testing/configmap-dr-scripts.yaml (66%) rename {openshift => tests/openshift}/dr-testing/cronjob-dr-test.yaml (100%) rename {openshift => tests/openshift}/dr-testing/kustomization.yaml (100%) rename {openshift => tests/openshift}/dr-testing/pvc-test-results.yaml (100%) rename {openshift => tests/openshift}/dr-testing/serviceaccount.yaml (100%) rename {scripts => tests/scripts}/dr-failover-test.sh (100%) rename {scripts => tests/scripts}/generate-dr-report.sh (100%) rename {scripts => tests/scripts}/measure-rto-rpo.sh (100%) rename {scripts => tests/scripts}/run-ci-checks-locally.sh (100%) rename {scripts => tests/scripts}/test-split-brain-prevention.sh (100%) rename {scripts => tests/scripts}/validate-aap-data.sh (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 474428a..6699542 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -108,7 +108,7 @@ repos: - id: openshift-manifest-validate name: Validate OpenShift manifests - entry: scripts/hooks/validate-openshift-manifests.sh + entry: tests/hooks/validate-openshift-manifests.sh language: script files: \.(yaml|yml)$ pass_filenames: true @@ -116,7 +116,7 @@ repos: - id: script-executable name: Ensure scripts are executable - entry: scripts/hooks/check-script-permissions.sh + entry: tests/hooks/check-script-permissions.sh language: script files: ^scripts/.*\.sh$ pass_filenames: true diff --git a/CLAUDE.md b/CLAUDE.md index 8ff4e2b..35af94e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,10 +49,12 @@ See `docs/architecture.md` for complete details. ├── scripts/ # Operational automation scripts │ ├── lib/ # Shared libraries (logging, scaling) │ ├── scale-aap-*.sh # AAP scaling (OpenShift) -│ ├── dr-*.sh # DR orchestration and testing -│ ├── validate-*.sh # Validation and integrity checks +│ ├── start/stop-*.sh # AAP cluster management (RHEL) │ └── efm-*.sh # EFM integration hooks -├── openshift/dr-testing/ # DR testing CronJob manifests +├── tests/ # Testing and CI infrastructure +│ ├── scripts/ # Test and validation scripts +│ ├── hooks/ # Pre-commit and CI hooks +│ └── openshift/ # DR testing CronJob manifests └── reports/ # Deployment validation reports ``` @@ -97,25 +99,25 @@ kustomize build aap-deploy/openshift/ **Local CI checks:** ```bash -./scripts/run-ci-checks-locally.sh +./tests/scripts/run-ci-checks-locally.sh ``` ### DR Testing **Run automated DR failover test:** ```bash -./scripts/dr-failover-test.sh --dc1-context --dc2-context +./tests/scripts/dr-failover-test.sh --dc1-context --dc2-context ``` **Measure RTO/RPO:** ```bash -./scripts/measure-rto-rpo.sh --dc1-context --dc2-context +./tests/scripts/measure-rto-rpo.sh --dc1-context --dc2-context ``` **Validate AAP data integrity:** ```bash -./scripts/validate-aap-data.sh create-baseline -./scripts/validate-aap-data.sh validate +./tests/scripts/validate-aap-data.sh create-baseline +./tests/scripts/validate-aap-data.sh validate ``` ### AAP Cluster Management (OpenShift) @@ -247,15 +249,15 @@ See `.cursor/skills/ansible-redhat-cop-practices/` for complete guidelines. - **Split-brain prevention:** Fencing logic ensures only one primary active **DR test workflow:** -1. Create baseline: `./scripts/validate-aap-data.sh create-baseline ` -2. Trigger failover: `./scripts/dr-failover-test.sh --dc1-context --dc2-context ` -3. Measure metrics: `./scripts/measure-rto-rpo.sh` (tracks timestamps, calculates RTO/RPO) -4. Validate data: `./scripts/validate-aap-data.sh validate ` -5. Generate report: `./scripts/generate-dr-report.sh --latest` +1. Create baseline: `./tests/scripts/validate-aap-data.sh create-baseline ` +2. Trigger failover: `./tests/scripts/dr-failover-test.sh --dc1-context --dc2-context ` +3. Measure metrics: `./tests/scripts/measure-rto-rpo.sh` (tracks timestamps, calculates RTO/RPO) +4. Validate data: `./tests/scripts/validate-aap-data.sh validate ` +5. Generate report: `./tests/scripts/generate-dr-report.sh --latest` **Automated testing:** -- OpenShift CronJob: `openshift/dr-testing/cronjob-dr-test.yaml` -- Results stored in PVC: `openshift/dr-testing/pvc-test-results.yaml` +- OpenShift CronJob: `tests/openshift/dr-testing/cronjob-dr-test.yaml` +- Results stored in PVC: `tests/openshift/dr-testing/pvc-test-results.yaml` See `docs/dr-testing-guide.md` for complete framework. @@ -268,7 +270,7 @@ GitHub Actions workflows (`.github/workflows/`): **Pre-commit hooks** (`.pre-commit-config.yaml`): - Trailing whitespace, YAML syntax, ShellCheck, markdownlint, secret detection -- Custom hooks: `hooks/check-script-permissions.sh`, `hooks/validate-openshift-manifests.sh` +- Custom hooks: `tests/hooks/check-script-permissions.sh`, `tests/hooks/validate-openshift-manifests.sh` ## Common Tasks diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2009bd..463b4e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,8 @@ Before contributing, ensure you have: - `/docs/` for general documentation - `/aap-deploy/` for AAP deployment docs - `/db-deploy/` for database deployment docs - - `/scripts/` for script documentation + - `/scripts/` for operational script documentation + - `/tests/` for testing and CI documentation ### Formatting diff --git a/README.md b/README.md index 1416167..667afad 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,12 @@ EDB_Testing/ ├── scripts/ # Operational automation scripts │ ├── lib/ # Shared libraries (logging, scaling) │ ├── scale-aap-*.sh # AAP scaling scripts -│ ├── dr-*.sh # DR orchestration -│ └── validate-*.sh # Validation scripts -├── openshift/ # OpenShift-specific resources -│ └── dr-testing/ # DR testing CronJob +│ ├── efm-*.sh # EFM integration hooks +│ └── start/stop-*.sh # AAP cluster management (RHEL) +├── tests/ # Testing and CI infrastructure +│ ├── scripts/ # Test and validation scripts +│ ├── hooks/ # Pre-commit and CI hooks +│ └── openshift/ # DR testing CronJob manifests └── .github/ # CI/CD workflows └── workflows/ # GitHub Actions ``` diff --git a/docs/INDEX.md b/docs/INDEX.md index d6650ba..53c1978 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -129,7 +129,7 @@ Choose based on your requirements: - Scaling AAP up/down: See [scale-aap-up.sh](../scripts/scale-aap-up.sh), [scale-aap-down.sh](../scripts/scale-aap-down.sh) - Monitoring replication: See [monitor-efm-scripts.sh](../scripts/monitor-efm-scripts.sh) - DR failover: See [efm-orchestrated-failover.sh](../scripts/efm-orchestrated-failover.sh) -- Data validation: See [validate-aap-data.sh](../scripts/validate-aap-data.sh) +- Data validation: See [validate-aap-data.sh](../tests/scripts/validate-aap-data.sh) --- @@ -144,10 +144,10 @@ Choose based on your requirements: | **[efm-orchestrated-failover.sh](../scripts/efm-orchestrated-failover.sh)** | Full DR failover orchestration | Called by EFM (post-promotion) | | **[efm-aap-failover-wrapper.sh](../scripts/efm-aap-failover-wrapper.sh)** | EFM integration hook | Called by EFM with failover context | | **[monitor-efm-scripts.sh](../scripts/monitor-efm-scripts.sh)** | Monitor EFM failover events | `./monitor-efm-scripts.sh` (CronJob) | -| **[dr-failover-test.sh](../scripts/dr-failover-test.sh)** | Automated DR testing framework | See [DR Testing Guide](dr-testing-guide.md) | -| **[validate-aap-data.sh](../scripts/validate-aap-data.sh)** | AAP data integrity validation | `./validate-aap-data.sh ` | -| **[measure-rto-rpo.sh](../scripts/measure-rto-rpo.sh)** | RTO/RPO measurement with milestones | `./measure-rto-rpo.sh start ` | -| **[generate-dr-report.sh](../scripts/generate-dr-report.sh)** | DR test report generation | `./generate-dr-report.sh ` | +| **[dr-failover-test.sh](../tests/scripts/dr-failover-test.sh)** | Automated DR testing framework | See [DR Testing Guide](dr-testing-guide.md) | +| **[validate-aap-data.sh](../tests/scripts/validate-aap-data.sh)** | AAP data integrity validation | `./validate-aap-data.sh ` | +| **[measure-rto-rpo.sh](../tests/scripts/measure-rto-rpo.sh)** | RTO/RPO measurement with milestones | `./measure-rto-rpo.sh start ` | +| **[generate-dr-report.sh](../tests/scripts/generate-dr-report.sh)** | DR test report generation | `./generate-dr-report.sh ` | **Script Documentation:** - **[Scripts README](../scripts/README.md)** ⭐ - Quick reference for all scripts @@ -175,7 +175,7 @@ Choose based on your requirements: **Testing:** - [Component Testing Results](component-testing-results.md) - Script validation (macOS/CRC) - [AAP Deployment Validation](aap-deployment-validation-crc.md) - End-to-end validation -- [run-ci-checks-locally.sh](../scripts/run-ci-checks-locally.sh) - Run CI checks before pushing +- [run-ci-checks-locally.sh](../tests/scripts/run-ci-checks-locally.sh) - Run CI checks before pushing --- @@ -199,7 +199,7 @@ Choose based on your requirements: **Security considerations:** - [Pre-commit Secret Detection](../.pre-commit-config.yaml#L89-L98) - `detect-secrets` integration -- [RBAC Configuration](../openshift/dr-testing/serviceaccount.yaml) - DR testing ServiceAccount +- [RBAC Configuration](../tests/openshift/dr-testing/serviceaccount.yaml) - DR testing ServiceAccount - [EFM Security](enterprisefailovermanager.md#security) - EFM permissions and VIP management **Planned Documentation:** diff --git a/docs/aap-containerized-quickstart.md b/docs/aap-containerized-quickstart.md index 66879b8..d6e8f97 100644 --- a/docs/aap-containerized-quickstart.md +++ b/docs/aap-containerized-quickstart.md @@ -291,7 +291,7 @@ curl -k https://aap.example.com/api/v2/ping/ ./scripts/manual-failback-dc1.sh # Measure RTO/RPO -./scripts/measure-rto-rpo.sh +./tests/scripts/measure-rto-rpo.sh ``` --- @@ -412,7 +412,7 @@ sudo dnf install -y postgres_exporter node_exporter ```bash # Add to crontab for quarterly testing -0 2 * * 6 /path/to/scripts/dr-failover-test.sh quarterly-$(date +%Y-Q%q) +0 2 * * 6 /path/to/tests/scripts/dr-failover-test.sh quarterly-$(date +%Y-Q%q) ``` ### Document Your Deployment diff --git a/docs/architecture.md b/docs/architecture.md index 8177751..f280d96 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -482,7 +482,7 @@ script.post.promotion=/usr/edb/efm-4.x/bin/efm-orchestrated-failover.sh %h %s %a - [`scale-aap-up.sh`](../scripts/scale-aap-up.sh) - Scale AAP to operational state in target datacenter - [`scale-aap-down.sh`](../scripts/scale-aap-down.sh) - Scale AAP to zero in inactive datacenter - [`efm-orchestrated-failover.sh`](../scripts/efm-orchestrated-failover.sh) - Full DR failover orchestration -- [`validate-aap-data.sh`](../scripts/validate-aap-data.sh) - Post-failover data validation +- [`validate-aap-data.sh`](../tests/scripts/validate-aap-data.sh) - Post-failover data validation - [`monitor-efm-scripts.sh`](../scripts/monitor-efm-scripts.sh) - EFM integration monitoring **Runbook:** diff --git a/docs/component-testing-results.md b/docs/component-testing-results.md index 327440a..6dbc459 100644 --- a/docs/component-testing-results.md +++ b/docs/component-testing-results.md @@ -188,7 +188,7 @@ Requires multi-cluster environment or access to remote cluster for full failover ### Fix #1: macOS Date Compatibility -**File:** `/scripts/measure-rto-rpo.sh` +**File:** `/tests/scripts/measure-rto-rpo.sh` **Changes:** diff --git a/docs/dr-replication-implementation-status.md b/docs/dr-replication-implementation-status.md index ac79b67..d4bf9c4 100644 --- a/docs/dr-replication-implementation-status.md +++ b/docs/dr-replication-implementation-status.md @@ -42,7 +42,7 @@ Following the replication architecture validation (score: 7.1/10), this document - `/scripts/scale-aap-up.sh` - Added database role validation **Files Created:** -- `/scripts/test-split-brain-prevention.sh` - Automated test script +- `/tests/scripts/test-split-brain-prevention.sh` - Automated test script - `/docs/split-brain-prevention.md` - Comprehensive documentation ### Changes Made @@ -77,7 +77,7 @@ fi #### 2. Test Script -Created `/scripts/test-split-brain-prevention.sh` with 4 test cases: +Created `/tests/scripts/test-split-brain-prevention.sh` with 4 test cases: 1. Database role detection verification 2. Safety code presence validation 3. Replica scenario simulation (manual test) @@ -85,7 +85,7 @@ Created `/scripts/test-split-brain-prevention.sh` with 4 test cases: **Usage:** ```bash -./scripts/test-split-brain-prevention.sh +./tests/scripts/test-split-brain-prevention.sh ``` #### 3. Documentation @@ -171,7 +171,7 @@ The split-brain check is now active in: **Objective:** Execute comprehensive failover testing to validate documented RTO/RPO targets **Deliverables:** -1. `/scripts/dr-failover-test.sh` - Automated failover drill script +1. `/tests/scripts/dr-failover-test.sh` - Automated failover drill script 2. `/docs/failover-test-results.md` - Test report template 3. Quarterly testing schedule 4. Measured actual RTO/RPO values @@ -199,7 +199,7 @@ The split-brain check is now active in: - Measure time to AAP availability 3. **Validation:** - - Run `/scripts/validate-aap-data.sh` (to be created) + - Run `/tests/scripts/validate-aap-data.sh` (to be created) - Verify no data loss - Confirm AAP job execution @@ -380,12 +380,12 @@ spec: - Send notifications to relevant teams 2. **Begin GAP-REP-002 Implementation:** - - Create `/scripts/dr-failover-test.sh` - - Create `/scripts/validate-aap-data.sh` + - Create `/tests/scripts/dr-failover-test.sh` + - Create `/tests/scripts/validate-aap-data.sh` - Document test procedures 3. **Validate Split-Brain Prevention:** - - Execute `/scripts/test-split-brain-prevention.sh` + - Execute `/tests/scripts/test-split-brain-prevention.sh` - Document results - Add to weekly health check @@ -414,7 +414,7 @@ spec: - **Baseline Validation:** `/reports/dr-replication-validation-report.md` - **Split-Brain Documentation:** `/docs/split-brain-prevention.md` - **Scale AAP Script:** `/scripts/scale-aap-up.sh` -- **Test Script:** `/scripts/test-split-brain-prevention.sh` +- **Test Script:** `/tests/scripts/test-split-brain-prevention.sh` - **DR Scenarios:** `/docs/dr-scenarios.md` - **EFM Integration:** `/docs/enterprisefailovermanager.md` diff --git a/docs/dr-testing-guide.md b/docs/dr-testing-guide.md index e79911a..6985fc7 100644 --- a/docs/dr-testing-guide.md +++ b/docs/dr-testing-guide.md @@ -118,7 +118,7 @@ Result: ✅ PASSED | **ServiceAccount** | RBAC permissions for test execution | - | | **PVC** | Persistent storage for test results | 5Gi storage | -**Location:** `/openshift/dr-testing/` +**Location:** `/tests/openshift/dr-testing/` ### 3. Test Phases @@ -358,7 +358,7 @@ Recovery Point Objective (RPO): **1. Configure cluster contexts:** -Edit `/openshift/dr-testing/kustomization.yaml`: +Edit `/tests/openshift/dr-testing/kustomization.yaml`: ```yaml configMapGenerator: @@ -379,7 +379,7 @@ oc create secret generic dr-test-kubeconfig \ **3. Deploy CronJob:** ```bash -cd openshift/dr-testing +cd tests/openshift/dr-testing oc apply -k . ``` diff --git a/docs/dr-testing-implementation-summary.md b/docs/dr-testing-implementation-summary.md index 2abd6eb..ec7fe88 100644 --- a/docs/dr-testing-implementation-summary.md +++ b/docs/dr-testing-implementation-summary.md @@ -41,14 +41,14 @@ Successfully implemented a comprehensive, production-ready disaster recovery tes | **PVC** | Test results storage (5Gi) | ✅ Complete | | **Kustomization** | Declarative deployment | ✅ Complete | -**Location:** `/openshift/dr-testing/` +**Location:** `/tests/openshift/dr-testing/` ### ✅ Documentation (2 guides) | Document | Pages | Purpose | Status | |----------|-------|---------|--------| | **dr-testing-guide.md** | 25+ | Comprehensive usage guide | ✅ Complete | -| **openshift/dr-testing/README.md** | 8+ | OpenShift deployment guide | ✅ Complete | +| **tests/openshift/dr-testing/README.md** | 8+ | OpenShift deployment guide | ✅ Complete | **Total:** ~10,000 words of documentation @@ -268,7 +268,7 @@ RTO: 287.4 seconds **Deploy quarterly automation:** ```bash -cd /Users/cferman/Documents/GitHub/EDB_Testing/openshift/dr-testing +cd /Users/cferman/Documents/GitHub/EDB_Testing/tests/openshift/dr-testing # Update cluster contexts in kustomization.yaml vim kustomization.yaml @@ -337,7 +337,7 @@ done ✅ **Kubernetes manifest validation:** ```bash -cd openshift/dr-testing +cd tests/openshift/dr-testing kustomize build . | kubectl apply --dry-run=client -f - ``` @@ -521,7 +521,7 @@ scripts/ ### OpenShift manifests (6 files) ``` -openshift/dr-testing/ +tests/openshift/dr-testing/ ├── cronjob-dr-test.yaml ✅ ├── serviceaccount.yaml ✅ ├── configmap-dr-scripts.yaml ✅ diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md index 8b402c8..cf9bcfe 100644 --- a/docs/quick-start-guide.md +++ b/docs/quick-start-guide.md @@ -616,10 +616,10 @@ oc apply -f monitoring/prometheus/servicemonitor.yaml ```bash # Schedule quarterly DR tests -oc apply -f openshift/dr-testing/cronjob-dr-test.yaml +oc apply -f tests/openshift/dr-testing/cronjob-dr-test.yaml # Run manual DR test -./scripts/dr-failover-test.sh --test-id manual-test-001 +./tests/scripts/dr-failover-test.sh --test-id manual-test-001 ``` **See:** [docs/dr-testing-guide.md](dr-testing-guide.md) diff --git a/docs/scripts-hooks-and-cicd.md b/docs/scripts-hooks-and-cicd.md index 107b224..6815416 100644 --- a/docs/scripts-hooks-and-cicd.md +++ b/docs/scripts-hooks-and-cicd.md @@ -12,19 +12,19 @@ The repository includes automation for code quality, testing, and validation: ## Pre-Commit Hooks -Location: `scripts/hooks/` +Location: `tests/hooks/` ### check-script-permissions.sh **Purpose:** Ensures all shell scripts have executable permissions before committing. -**Location:** `scripts/hooks/check-script-permissions.sh` +**Location:** `tests/hooks/check-script-permissions.sh` **Usage:** ```bash # Called automatically by pre-commit framework # Or manually: -./scripts/hooks/check-script-permissions.sh script1.sh script2.sh +./tests/hooks/check-script-permissions.sh script1.sh script2.sh ``` **What It Checks:** @@ -54,7 +54,7 @@ repos: hooks: - id: check-script-permissions name: Check script permissions - entry: scripts/hooks/check-script-permissions.sh + entry: tests/hooks/check-script-permissions.sh language: script files: \.sh$ ``` @@ -65,13 +65,13 @@ repos: **Purpose:** Validates Kubernetes/OpenShift YAML manifests using `kubeval`. -**Location:** `scripts/hooks/validate-openshift-manifests.sh` +**Location:** `tests/hooks/validate-openshift-manifests.sh` **Usage:** ```bash # Called automatically by pre-commit framework # Or manually: -./scripts/hooks/validate-openshift-manifests.sh manifest1.yaml manifest2.yaml +./tests/hooks/validate-openshift-manifests.sh manifest1.yaml manifest2.yaml ``` **What It Validates:** @@ -115,7 +115,7 @@ repos: hooks: - id: validate-k8s-manifests name: Validate Kubernetes manifests - entry: scripts/hooks/validate-openshift-manifests.sh + entry: tests/hooks/validate-openshift-manifests.sh language: script files: \.(yaml|yml)$ ``` @@ -128,15 +128,15 @@ repos: **Purpose:** Runs comprehensive quality checks locally before pushing code. -**Location:** `scripts/run-ci-checks-locally.sh` +**Location:** `tests/scripts/run-ci-checks-locally.sh` **Usage:** ```bash # Run all CI checks -./scripts/run-ci-checks-locally.sh +./tests/scripts/run-ci-checks-locally.sh # Simulates GitHub Actions validation -cd /path/to/repo && ./scripts/run-ci-checks-locally.sh +cd /path/to/repo && ./tests/scripts/run-ci-checks-locally.sh ``` **Checks Performed:** @@ -320,14 +320,14 @@ repos: # Check script permissions - id: check-script-permissions name: Check script permissions - entry: scripts/hooks/check-script-permissions.sh + entry: tests/hooks/check-script-permissions.sh language: script files: \.sh$ # Validate Kubernetes manifests - id: validate-k8s-manifests name: Validate Kubernetes manifests - entry: scripts/hooks/validate-openshift-manifests.sh + entry: tests/hooks/validate-openshift-manifests.sh language: script files: \.(yaml|yml)$ @@ -433,7 +433,7 @@ jobs: sudo mv kubeval /usr/local/bin/ - name: Run CI checks - run: ./scripts/run-ci-checks-locally.sh + run: ./tests/scripts/run-ci-checks-locally.sh - name: Run pre-commit hooks run: pre-commit run --all-files @@ -525,7 +525,7 @@ brew install pre-commit 2. **Test locally before pushing:** ```bash - ./scripts/run-ci-checks-locally.sh + ./tests/scripts/run-ci-checks-locally.sh ``` 3. **Fix shellcheck warnings:** diff --git a/reports/dr-architecture-validation-report.md b/reports/dr-architecture-validation-report.md index af359b5..bf2e46e 100644 --- a/reports/dr-architecture-validation-report.md +++ b/reports/dr-architecture-validation-report.md @@ -713,7 +713,7 @@ From `/docs/dr-scenarios.md`: - Inconsistencies between DC1 and DC2 undetected **Files Affected:** -- Missing: `/scripts/validate-aap-data.sh` +- Missing: `/tests/scripts/validate-aap-data.sh` **Resolution:** - Create data validation script diff --git a/scripts/README.md b/scripts/README.md index 4341b71..e42c63d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -23,15 +23,6 @@ Scripts for managing AAP on RHEL servers: - **[`start-aap-cluster.sh`](start-aap-cluster.sh)** - Start all AAP systemd services - **[`stop-aap-cluster.sh`](stop-aap-cluster.sh)** - Stop all AAP systemd services -### Disaster Recovery Testing - -Automated DR testing and validation: - -- **[`dr-failover-test.sh`](dr-failover-test.sh)** - End-to-end automated DR failover test -- **[`measure-rto-rpo.sh`](measure-rto-rpo.sh)** - RTO/RPO measurement and tracking -- **[`validate-aap-data.sh`](validate-aap-data.sh)** - AAP data integrity validation -- **[`generate-dr-report.sh`](generate-dr-report.sh)** - Generate DR test reports - ### EFM Integration Scripts for EDB Failover Manager integration: @@ -41,20 +32,6 @@ Scripts for EDB Failover Manager integration: - **[`monitor-efm-scripts.sh`](monitor-efm-scripts.sh)** - Monitor EFM script execution - **Configuration**: [`efm.properties.sample`](efm.properties.sample) - Sample EFM configuration -### Testing & Quality Assurance - -Scripts for testing and validation: - -- **[`test-split-brain-prevention.sh`](test-split-brain-prevention.sh)** - Validate split-brain prevention logic -- **[`run-ci-checks-locally.sh`](run-ci-checks-locally.sh)** - Run CI checks before pushing - -### Git Hooks - -Pre-commit hooks for code quality (in `hooks/`): - -- **[`hooks/check-script-permissions.sh`](hooks/check-script-permissions.sh)** - Ensure scripts are executable -- **[`hooks/validate-openshift-manifests.sh`](hooks/validate-openshift-manifests.sh)** - Validate Kubernetes YAML - ### Shared Libraries Reusable code libraries (in `lib/`): @@ -62,6 +39,16 @@ Reusable code libraries (in `lib/`): - **[`lib/aap-scaling.sh`](lib/aap-scaling.sh)** - Common AAP scaling functions - **[`lib/logging.sh`](lib/logging.sh)** - Standardized logging functions +## Testing and CI + +All testing, validation, and CI/CD infrastructure has been moved to the **[`tests/`](../tests/)** directory: + +- **DR Testing Scripts**: `tests/scripts/` - Failover testing, RTO/RPO measurement, data validation +- **CI Hooks**: `tests/hooks/` - Pre-commit hooks for code quality +- **OpenShift Testing**: `tests/openshift/dr-testing/` - CronJob-based automated testing + +See **[`tests/README.md`](../tests/README.md)** for complete testing documentation. + ## Quick Start ### Scale AAP Up (OpenShift) @@ -71,24 +58,9 @@ Reusable code libraries (in `lib/`): ./scripts/scale-aap-up.sh ``` -### Run a DR Failover Test - -```bash -# Full automated DR test -./scripts/dr-failover-test.sh \ - --dc1-context \ - --dc2-context -``` - -### Validate AAP Data +### Run DR Testing -```bash -# Create baseline -./scripts/validate-aap-data.sh create-baseline - -# Validate against baseline -./scripts/validate-aap-data.sh validate -``` +DR testing scripts have moved to `tests/scripts/`. See **[`tests/README.md`](../tests/README.md)**. ## Prerequisites @@ -112,11 +84,11 @@ Comprehensive guides are available in the `docs/` directory: # 1. Scale up AAP in standby DC ./scripts/scale-aap-up.sh -# 2. Validate data integrity -./scripts/validate-aap-data.sh validate +# 2. Validate data integrity (see tests/) +./tests/scripts/validate-aap-data.sh validate -# 3. Generate DR report -./scripts/generate-dr-report.sh --latest +# 3. Generate DR report (see tests/) +./tests/scripts/generate-dr-report.sh --latest ``` ### EFM Integration Setup @@ -136,7 +108,7 @@ sudo -u efm /usr/edb/efm-4.x/bin/efm-aap-failover-wrapper.sh test standby test t ```bash # Run all quality checks before committing -./scripts/run-ci-checks-locally.sh +./tests/scripts/run-ci-checks-locally.sh ``` ## Support diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..8e625fe --- /dev/null +++ b/tests/README.md @@ -0,0 +1,69 @@ +# Tests and CI Infrastructure + +This directory contains all testing, validation, and CI/CD related scripts and manifests. + +## Directory Structure + +``` +tests/ +├── scripts/ # Test and validation scripts +│ ├── dr-failover-test.sh +│ ├── measure-rto-rpo.sh +│ ├── validate-aap-data.sh +│ ├── test-split-brain-prevention.sh +│ ├── generate-dr-report.sh +│ └── run-ci-checks-locally.sh +├── hooks/ # Pre-commit and CI hooks +│ ├── check-script-permissions.sh +│ └── validate-openshift-manifests.sh +├── openshift/ # OpenShift test manifests +│ └── dr-testing/ # DR testing CronJob and resources +└── lib/ # Shared test libraries (if needed) +``` + +## Test Scripts + +### DR Testing +- **dr-failover-test.sh** - Automated DR failover testing +- **measure-rto-rpo.sh** - RTO/RPO measurement and reporting +- **validate-aap-data.sh** - AAP data integrity validation +- **generate-dr-report.sh** - DR test report generation + +### Unit/Integration Tests +- **test-split-brain-prevention.sh** - Split-brain prevention testing +- **run-ci-checks-locally.sh** - Local CI pipeline validation + +## CI Hooks + +Pre-commit hooks integrated with `.pre-commit-config.yaml`: +- **check-script-permissions.sh** - Verify script executability +- **validate-openshift-manifests.sh** - Kubernetes manifest validation + +## OpenShift Testing + +CronJob-based automated DR testing deployed to OpenShift clusters. See `openshift/dr-testing/README.md` for details. + +## Usage + +**Run local CI checks:** +```bash +./tests/scripts/run-ci-checks-locally.sh +``` + +**Run DR failover test:** +```bash +./tests/scripts/dr-failover-test.sh --dc1-context --dc2-context +``` + +**Measure RTO/RPO:** +```bash +./tests/scripts/measure-rto-rpo.sh --dc1-context --dc2-context +``` + +**Validate AAP data:** +```bash +./tests/scripts/validate-aap-data.sh create-baseline +./tests/scripts/validate-aap-data.sh validate +``` + +See individual script documentation for detailed usage. diff --git a/scripts/hooks/check-script-permissions.sh b/tests/hooks/check-script-permissions.sh similarity index 100% rename from scripts/hooks/check-script-permissions.sh rename to tests/hooks/check-script-permissions.sh diff --git a/scripts/hooks/validate-openshift-manifests.sh b/tests/hooks/validate-openshift-manifests.sh similarity index 100% rename from scripts/hooks/validate-openshift-manifests.sh rename to tests/hooks/validate-openshift-manifests.sh diff --git a/openshift/dr-testing/README.md b/tests/openshift/dr-testing/README.md similarity index 100% rename from openshift/dr-testing/README.md rename to tests/openshift/dr-testing/README.md diff --git a/openshift/dr-testing/configmap-dr-scripts.yaml b/tests/openshift/dr-testing/configmap-dr-scripts.yaml similarity index 66% rename from openshift/dr-testing/configmap-dr-scripts.yaml rename to tests/openshift/dr-testing/configmap-dr-scripts.yaml index c5bcf0c..e9cb2a3 100644 --- a/openshift/dr-testing/configmap-dr-scripts.yaml +++ b/tests/openshift/dr-testing/configmap-dr-scripts.yaml @@ -9,18 +9,18 @@ metadata: component: scripts data: dr-failover-test.sh: | - # Script content would be injected here from /scripts/dr-failover-test.sh + # Script content would be injected here from /tests/scripts/dr-failover-test.sh # In production, build this into a container image instead echo "DR test script - see container image" validate-aap-data.sh: | - # Script content from /scripts/validate-aap-data.sh + # Script content from /tests/scripts/validate-aap-data.sh echo "Data validation script - see container image" measure-rto-rpo.sh: | - # Script content from /scripts/measure-rto-rpo.sh + # Script content from /tests/scripts/measure-rto-rpo.sh echo "RTO/RPO measurement script - see container image" generate-dr-report.sh: | - # Script content from /scripts/generate-dr-report.sh + # Script content from /tests/scripts/generate-dr-report.sh echo "Report generator script - see container image" diff --git a/openshift/dr-testing/cronjob-dr-test.yaml b/tests/openshift/dr-testing/cronjob-dr-test.yaml similarity index 100% rename from openshift/dr-testing/cronjob-dr-test.yaml rename to tests/openshift/dr-testing/cronjob-dr-test.yaml diff --git a/openshift/dr-testing/kustomization.yaml b/tests/openshift/dr-testing/kustomization.yaml similarity index 100% rename from openshift/dr-testing/kustomization.yaml rename to tests/openshift/dr-testing/kustomization.yaml diff --git a/openshift/dr-testing/pvc-test-results.yaml b/tests/openshift/dr-testing/pvc-test-results.yaml similarity index 100% rename from openshift/dr-testing/pvc-test-results.yaml rename to tests/openshift/dr-testing/pvc-test-results.yaml diff --git a/openshift/dr-testing/serviceaccount.yaml b/tests/openshift/dr-testing/serviceaccount.yaml similarity index 100% rename from openshift/dr-testing/serviceaccount.yaml rename to tests/openshift/dr-testing/serviceaccount.yaml diff --git a/scripts/dr-failover-test.sh b/tests/scripts/dr-failover-test.sh similarity index 100% rename from scripts/dr-failover-test.sh rename to tests/scripts/dr-failover-test.sh diff --git a/scripts/generate-dr-report.sh b/tests/scripts/generate-dr-report.sh similarity index 100% rename from scripts/generate-dr-report.sh rename to tests/scripts/generate-dr-report.sh diff --git a/scripts/measure-rto-rpo.sh b/tests/scripts/measure-rto-rpo.sh similarity index 100% rename from scripts/measure-rto-rpo.sh rename to tests/scripts/measure-rto-rpo.sh diff --git a/scripts/run-ci-checks-locally.sh b/tests/scripts/run-ci-checks-locally.sh similarity index 100% rename from scripts/run-ci-checks-locally.sh rename to tests/scripts/run-ci-checks-locally.sh diff --git a/scripts/test-split-brain-prevention.sh b/tests/scripts/test-split-brain-prevention.sh similarity index 100% rename from scripts/test-split-brain-prevention.sh rename to tests/scripts/test-split-brain-prevention.sh diff --git a/scripts/validate-aap-data.sh b/tests/scripts/validate-aap-data.sh similarity index 100% rename from scripts/validate-aap-data.sh rename to tests/scripts/validate-aap-data.sh