diff --git a/README.md b/README.md index 844af5d..aaa596a 100644 --- a/README.md +++ b/README.md @@ -186,12 +186,12 @@ cat test-suite.log Current code coverage metrics from GTest suite: ``` -Lines......: 83.4% (1753 of 2102 lines) -Functions..: 97.8% (87 of 89 functions) -Branches...: 77.9% (1003 of 1288 branches) +Lines......: 90.5% (1903 of 2102 lines) +Functions..: 100.0% (89 of 89 functions) +Branches...: 84.3% (1086 of 1288 branches) ``` -**Coverage Target**: Maintain >80% line coverage, >95% function coverage +**Coverage Target**: Maintain >90% line coverage, 100% function coverage, >80% branches coverage ### L2 Functional Tests @@ -203,13 +203,35 @@ cd test/functional-tests ``` **Test Status**: -1. ✅ Lock and Exit - Completed -2. ✅ Lock and Wait - Completed -3. 🔄 Minidump Upload - TBD -4. 🔄 Coredump Upload - TBD -5. 🔄 Ratelimit - TBD -6. 🔄 Startup Cleanup - TBD -7. 🔄 OptOut - TBD + +| Module | Feature File | Status | +|--------|-------------|--------| +| Argument parsing (TC-008) | `arg_parsing.feature` | ✅ Complete | +| Failure return codes | `failure_return.feature` | ✅ Complete | +| Reboot flag / log output | `reboot_and_log_scenario.feature` | ✅ Complete | +| Config & path selection (TC-006/007/009) | `config_and_path.feature` | ✅ Complete | +| Dump detection & platform baseline (TC-019–028) | `config_checks_and_baseline.feature` | ✅ Complete | +| Unsupported device types (TC-004/005/017/018) | `unsupported_device_types.feature` | ✅ Complete | +| Broadband archive behaviour (TC-067) | `broadband_env.feature` | ✅ Complete | +| No dumps exit (NODMP-01..04) | `no_dumps_exit.feature` | ✅ Complete | +| Upload deferral (TC-035/036/037) | `upload_deferral.feature` | ✅ Complete | +| Lock and exit (TC-012/013/015/085) | `test_lock_and_exit.feature` | ✅ Complete | +| Lock and wait | `test_lock_and_wait.feature` | ✅ Complete | +| Lock lifecycle (TC-011/014/016) | `lock_lifecycle.feature` | ✅ Complete | +| Signal lock cleanup (SIG-01/02) | `signal_lock_cleanup.feature` | ✅ Complete | +| Startup cleanup batch (TC-041–047) | `cleanup_batch.feature` | ✅ Complete | +| Rate limiting — block (TC-049/051/055) | `ratelimit.feature` | ✅ Complete | +| Rate limiting — allow (TC-048/050/052–054) | `ratelimit.feature` | ✅ Complete | +| Telemetry opt-out (TC-038/039/040) | `telemetry_optout.feature` | ✅ Complete | +| Crash telemetry (TC-072/073) | `crash_telemetry.feature` | ✅ Complete | +| Scanner / filename sanitisation (TC-057–059/064) | `scanner_behaviour.feature` | ✅ Complete | +| Dump processing (TC-060/071) | `dump_processing.feature` | ✅ Complete | +| Archive naming (TC-061/062/063) | `archive_naming.feature` | ✅ Complete | +| Archive content (TC-065/066/075/078–080) | `archive_content.feature` | ✅ Complete | +| Single successful upload (TC-081) | `upload.feature` | ✅ Complete | +| Upload retry (TC-082/083) | `upload_retry.feature` | ✅ Complete | + +**Coverage**: 66 / 66 applicable TCs implemented (19 not applicable — see `test/L2_TESTS.md`) ## Rate Limiting & Recovery Mode diff --git a/c_sourcecode/src/upload/upload.c b/c_sourcecode/src/upload/upload.c index 22b5d09..742fef7 100644 --- a/c_sourcecode/src/upload/upload.c +++ b/c_sourcecode/src/upload/upload.c @@ -291,7 +291,7 @@ int upload_file(const char *filepath, const char *url, const char *dump_name, co } else { - CRASHUPLOAD_INFO("S3 %s Upload is successful\n", filepath); + CRASHUPLOAD_INFO("Upload is successful with TLS1.2 for %s\n", filepath); if (t2_enabled) { t2CountNotify("SYS_INFO_S3CoreUploaded", 1); diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000..38557b7 --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,43 @@ +schema: spec-driven + +context: | + Repo: crashupload, an RDK crash dump discovery, packaging, and upload system with two active implementation paths: legacy shell scripts and a newer C binary. + + Use this context strategy to keep SDLC artifact generation accurate and token-efficient: + - Read OpenSpec docs first: openspec/project.md, openspec/runtime/**, openspec/pipeline/**, openspec/subsystems/**, openspec/integrations/**, openspec/diagrams/**, openspec/gaps/**. + - Read documentation second: README.md, c_sourcecode/README.md, unittest/README.md, docs/migration/**, and other repo README files relevant to the touched area. + - Read implementation third, but only the narrow slice needed for the task: uploadDumps.sh, runDumpUpload.sh, uploadDumpsUtils.sh, c_sourcecode/src/**, c_sourcecode/include/**, c_sourcecode/common/**, src/inotify-minidump-watcher.c, and related service files. + - Read verification and delivery context only when relevant: run_ut.sh, run_l2.sh, cov_build.sh, rdk_build.sh, test/L2_TESTS.md, unittest/**, test/functional-tests/**, and .github/workflows/*.yml. + + Repository reality to preserve in artifacts: + - Runtime is event/timer/script driven, not a single long-running daemon. + - uploadDumps.sh is the top-level dispatcher. + - mediaclient is binary-first when /usr/bin/crashupload is present; broadband and extender still rely on the legacy shell path. + - C implementation lives under c_sourcecode/ and legacy shell behavior remains authoritative for some device types. + - L1 means unit-test coverage centered on unittest/ and ./run_ut.sh. + - L2 means functional coverage centered on test/functional-tests/, test/L2_TESTS.md, and ./run_l2.sh after ./cov_build.sh --l2-test. + - CI context comes from .github/workflows/*.yml plus build/test entry scripts at repo root. + + Context minimization rules: + - Prefer the existing OpenSpec artifacts as the summary layer instead of re-reading large code trees. + - Pull only the docs and code slices directly related to the subsystem being changed. + - Do not load all docs or all source files by default; start with the nearest README or OpenSpec doc, then drill into exact code paths. + - Separate shell-path behavior, C-path behavior, test impact, and CI impact instead of merging them into a generic summary. + +rules: + proposal: + - Start from openspec/project.md and only add the specific OpenSpec/doc/code files needed for the affected subsystem. + - State whether the change touches shell runtime, C runtime, test harness, CI, documentation, or a combination. + - Keep repository background concise; prefer explicit file anchors over broad repo summaries. + design: + - Prefer openspec/runtime, openspec/pipeline, openspec/subsystems, and openspec/diagrams before raw implementation files. + - Pull docs/migration/** and relevant README files when they add architecture, deployment, or migration detail. + - Call out device-type routing differences and any shell-versus-C divergence. + specs: + - Use code-truth for behavioral claims: uploadDumps.sh, runDumpUpload.sh, c_sourcecode/src/**, service units, and workflows when behavior depends on them. + - Separate L1 unit-test expectations from L2 functional-test expectations. + - Mention CI behavior only if workflows, build scripts, or test scripts are affected. + tasks: + - Generate file-scoped, validation-oriented tasks. + - Reference the smallest sufficient context set for each task. + - Include concrete verification commands when applicable, choosing from ./cov_build.sh, ./run_ut.sh, ./cov_build.sh --l2-test, and ./run_l2.sh. diff --git a/openspec/integrations/client-integration-model.md b/openspec/integrations/client-integration-model.md new file mode 100644 index 0000000..865a370 --- /dev/null +++ b/openspec/integrations/client-integration-model.md @@ -0,0 +1,76 @@ +# Client and Integration Model + +## External Interaction Surfaces + +## 1) CLI / command interfaces +### uploadDumps.sh +- Primary external entry for services and helper watchers. +- Accepts positional arguments passed through to selected runtime path. + +### crashupload binary +- Fact: current main expects positional flags where argv[2] determines dump mode (0 minidump, 1 coredump) and optional argv[3]=secure, argv[4]=wait_for_lock. +- Inference: practical CLI contract is script-oriented rather than end-user friendly options. + +### inotify-minidump-watcher +- Usage includes directory, command, args, and file patterns. +- Executes configured command when matching file is created. + +## 2) System integration points +- systemd units and timer/path triggers. +- Filesystem directories as crash intake source and local spool. +- Marker/flag files in /tmp and /opt governing policy and control flow. + +## 3) Library/API integrations +- Property/config: getDevicePropertyData, getIncludePropertyData, GetPartnerId, GetModelNum, etc. +- Policy/config APIs: RFC parameter read/write, RBUS privacy parameter reads. +- Telemetry: T2 events for counters and split values. +- Upload/security: uploadutil and TLS/curl stack. + +## Request Lifecycle: Trigger to Completion +1. Trigger source invokes uploadDumps.sh. +2. Dispatcher selects C or legacy path. +3. Runtime initializes config/platform and lock. +4. Runtime verifies eligibility (dumps present, deferral/privacy/reboot/rate-limit rules). +5. Runtime discovers and normalizes artifacts. +6. Runtime archives payload with metadata/log context. +7. Runtime performs upload workflow. +8. Runtime records outcome markers and cleanup. +9. Runtime exits with status consumed by invoker policy. + +## Intended Caller Workflows +### Systemd boot workflow +- Timer fires after boot window, invoking minidump processing path via uploadDumps.sh. + +### Filesystem event workflow +- Path/inotify detects dump creation, invokes upload script. + +### Manual operator workflow +- invoke script or binary for diagnostics/retry scenarios. + +## Configuration and Environment Dependencies +- Files: + - /etc/device.properties + - /etc/include.properties + - /etc/breakpad-logmapper.conf +- Runtime dirs: + - /opt/minidumps, /minidumps, /var/lib/systemd/coredump, /opt/secure/* +- Runtime flags/markers: + - /tmp/.uploadMinidumps, /tmp/.uploadCoredumps + - /tmp/.minidump_upload_timestamps, /tmp/.deny_dump_uploads_till + - /tmp/set_crash_reboot_flag + - /opt/.upload_on_startup + +## Rate-Limit State Semantics +- /tmp/.deny_dump_uploads_till: global cooldown gate; if active, upload processing is blocked regardless of dump type. +- /tmp/.minidump_upload_timestamps: minidump-only rolling counter used to trigger cooldown when upload volume exceeds threshold. + +## Device-Side vs External Integration Boundaries +- Device-side only: + - file scanning, archive composition, local policy gates, cleanup. +- Externally integrated: + - RFC/RBUS data sources, telemetry backplane, crash portal endpoint/signed URL service, S3 upload endpoint. + +## Integration Risks for OpenSpec Planning +- Fact: behavior differs strongly by device type and runtime branch. +- Fact: some integration paths are stubs or unsupported in specific branches (for example broadband in upload.c). +- Inference: future changes should specify per-device/path applicability and rollback strategy. diff --git a/openspec/runtime/execution-model.md b/openspec/runtime/execution-model.md new file mode 100644 index 0000000..5828c48 --- /dev/null +++ b/openspec/runtime/execution-model.md @@ -0,0 +1,94 @@ +# Runtime Execution Model + +## Execution Modes +### Mode 1: Mediaclient preferred C path +- Fact: uploadDumps.sh selects crashupload binary if present. +- Fact: binary may fall back to legacy via script if exit code indicates failure and dispatcher chooses fallback. + +### Mode 2: Broadband/extender legacy path +- Fact: uploadDumps.sh directly invokes runDumpUpload.sh for broadband/extender. +- Fact: upload.c currently marks broadband branch unsupported for C upload_process. + +### Mode 3: Manual/diagnostic invocation +- Inference: crashupload can be executed directly with argument contract expected by main (argv[2] dump type flag; optional secure/wait_for_lock flags). +- Unknown: direct CLI contract stability across releases. + +## Detailed C Binary Flow (Observed) +1. Preconditions and argument check +- argc must be >= 3. +- argv[2] determines dump type branch: 0 minidump, 1 coredump. + +2. Locking and signal behavior +- lock file path set by dump type. +- SIGTERM handler removes lock file based on active mode. +- lock mode optional wait_for_lock causes blocking lock behavior. + +3. Initialization +- t2 init, config load, platform init, core log creation. + +4. Prerequisite gate +- dump presence check in expected directory. +- mediaclient deferral to 480 seconds uptime threshold. +- optional coredump completion wait logic tied to mutex flag file. + +5. Privacy gate +- mediaclient reads RBUS privacy mode. +- DO_NOT_SHARE skips upload, but cleanup path still executes. + +6. Startup cleanup and scan +- cleanup_batch runs before and after processing. +- scanner finds candidate files and ensures size stability. + +7. Per-dump processing +- sanitize/rename/telemetry extraction. +- metadata naming with mac/date/box/model/process context. +- archive creation with selected sidecar files. + +8. Rate limit and reboot checks +- skip upload if reboot flag present. +- global deny-window (`/tmp/.deny_dump_uploads_till`) is evaluated before dump-type specific checks and can block both minidump and coredump uploads while active. +- minidump upload-counter check (`/tmp/.minidump_upload_timestamps`) is only evaluated for minidump runs. +- when blocked, pending dumps in the active working directory can be removed. + +9. Upload loop +- sequential upload per archive. +- break on first upload error. + +10. Cleanup and termination +- cleanup batch rerun. +- release lock, uninit telemetry, exit with ret code. + +## Concurrency and Process Model +- Fact: single-process, mostly sequential processing. +- Fact: no explicit multithreading in processing path. +- Fact: inter-process concurrency controlled by flock lock files. +- Inference: throughput scales by trigger frequency and batch size rather than parallel upload workers. + +## State and File Semantics +### Locking +- /tmp/.uploadMinidumps +- /tmp/.uploadCoredumps + +### Rate limit and deny windows +- /tmp/.minidump_upload_timestamps +- /tmp/.deny_dump_uploads_till +- policy note: deny-window is global cooldown; minidump timestamp file is minidump-only counter state. + +### Startup/cleanup markers +- /tmp/.on_startup_dumps_cleaned_up_ +- /opt/.upload_on_startup + +### Reboot/flow flags +- /tmp/set_crash_reboot_flag +- /tmp/coredump_mutex_release (checked in prerequisites path) + +## Failure Handling Semantics +- Fact: lock contention in non-wait mode exits quickly. +- Fact: scan failures/no dumps route to cleanup and often zero exit. +- Fact: upload failure breaks remaining upload loop. +- Fact: cleanup is best-effort and often non-fatal. +- Unknown: upstream service restart policies and retry cadence on non-zero exits. + +## Execution Truth vs Expectations +- Fact: several comments indicate optimized 7-step conceptual flow, but implementation includes TODOs and mixed maturity. +- Inference: OpenSpec work should baseline on observed behavior rather than intended design commentary. diff --git a/openspec/subsystems/subsystem-analysis.md b/openspec/subsystems/subsystem-analysis.md new file mode 100644 index 0000000..4ef130e --- /dev/null +++ b/openspec/subsystems/subsystem-analysis.md @@ -0,0 +1,83 @@ +# Subsystem Responsibility Analysis + +## Subsystem Map +1. Trigger and orchestration subsystem + - systemd units, timer/path, optional inotify watcher, uploadDumps.sh dispatcher. + +2. Initialization and config subsystem + - system_init, config_manager, platform. + +3. Intake and qualification subsystem + - prerequisites, lock_manager, scanner. + +4. Packaging subsystem + - archive, file utilities, log mapping in scanner. + +5. Transport subsystem + - upload module with endpoint discovery, metadata POST, S3 PUT. + +6. Policy subsystem + - ratelimit, privacy mode check, reboot flag behavior. + +7. Operations and observability subsystem + - logger and telemetry interface wrappers. + +## Subsystem A: Trigger and orchestration +- Facts: + - systemd timer and path units invoke uploadDumps.sh. + - uploadDumps.sh chooses C vs legacy runtime branch and fallback. +- Inferences: + - This layer is the primary operational policy switch for migration strategy. +- Unknowns: + - exact service installation/enabling matrix per platform SKU. + +## Subsystem B: Initialization and config +- Facts: + - config_manager derives mode from argv and device properties. + - platform subsystem builds identifier set (mac/model/sha1) with fallbacks. +- Risks: + - mixed defaults and TODOs can hide platform-specific behavior gaps. + +## Subsystem C: Intake and qualification +- Facts: + - lock files ensure single active worker per dump type. + - prerequisites include dump-presence check and optional deferred upload. + - scanner enforces max file count and file-size stability gate. +- Inference: + - stability gate mitigates racing with producer write completion. + +## Subsystem D: Packaging +- Facts: + - archive_create_smart creates tar.gz and can include mapped process logs. + - filename normalization logic handles long names and container delimiter cleanup. +- Unknown: + - full fidelity vs legacy script for all edge-case filename conventions. + +## Subsystem E: Transport +- Facts: + - endpoint resolution path includes RFC and device property fallback. + - upload is two-step (metadata then object upload) with telemetry markers. +- Inference: + - control plane and data plane are split, enabling signed URL workflows. + +## Subsystem F: Policy (rate limit/privacy/reboot) +- Facts: + - deny-window and upload timestamp files in /tmp govern upload throttling. + - deny-window (`/tmp/.deny_dump_uploads_till`) is a global cooldown gate and can block both minidump and coredump uploads while active. + - upload timestamp counter (`/tmp/.minidump_upload_timestamps`) is only used for minidump rate-threshold evaluation. + - reboot flag can skip upload to avoid conflict during reboot transitions. + - privacy DO_NOT_SHARE bypasses upload for mediaclient path. +- Assumption: + - product-level privacy compliance also depends on paths outside this component. + +## Subsystem G: Observability +- Facts: + - component emits logs and T2 counters/values. + - telemetry calls are feature-flag/build-flag dependent wrappers. +- Unknown: + - production log routing and marker governance ownership. + +## Cross-Cutting Concerns +- Device type branching significantly changes behavior. +- Build flags (RFC_API_ENABLED, RBUS_API_ENABLED, T2_EVENT_ENABLED, L2_TEST) alter effective runtime semantics. +- Legacy and C implementations coexist and must be jointly reasoned during change planning. diff --git a/test/L2_TESTS.md b/test/L2_TESTS.md new file mode 100644 index 0000000..631081a --- /dev/null +++ b/test/L2_TESTS.md @@ -0,0 +1,203 @@ +# L2 Test Coverage — CrashUpload C Implementation + +**Reference:** Original test cases defined in `uploadDumps_TestCases.md` (85 TCs for the +shell-script implementation of `uploadDumps.sh`). + +**Purpose:** This document cross-references every TC against the C implementation +(`c_sourcecode/`) to determine applicability, and maps each applicable TC to any existing +L2 functional test in `test/functional-tests/tests/`. + +**Build requirement:** L2 tests require a binary compiled with `-DL2_TEST`: +```bash +sh cov_build.sh --l2-test +``` + +--- + +## Legend + +| Symbol | Meaning | +|--------|---------| +| ✅ Yes | TC is fully applicable to the C implementation | +| ⚠️ Partial | TC is partially applicable; C behavior differs from shell in some aspects | +| ❌ No | TC is not applicable; feature is shell-specific with no C equivalent | +| ✅ Implemented | One or more L2 functional tests cover this TC | +| 🔲 Not Yet | TC is applicable but no L2 functional test exists yet | +| — | Not applicable (N/A row) | + +--- + +## Test Case Table + +| TC-ID | Test Case Name | Category | Applicable to C Code | Reason if N/A / Notes | Implemented | Test File :: Function | +|-------|----------------|----------|----------------------|-----------------------|-------------|-----------------------| +| TC-001 | Source `device.properties` on startup | Config & Init | ❌ No | Shell `source device.properties` sets global vars; C uses `rf_hal_property_get()` internal API — no file sourcing | — | — | +| TC-002 | Source `include.properties` on startup | Config & Init | ❌ No | Shell global variable setup from `include.properties`; C uses compile-time / runtime config API | — | — | +| TC-003 | Source `t2Shared_api.sh` T2 library | Config & Init | ❌ No | Shell sources T2 helper script; C links directly to T2 library via `t2_event_s()` — no script dependency | — | — | +| TC-004 | Broadband device type path initialisation | Config & Init | ✅ Yes | `config_init_load()` in `config_manager.c` sets `minidump_path=/minidumps` and `core_path=/var/lib/systemd/coredump` for broadband (normal mode) | ✅ Implemented | `test_unsupported_devicetypes.py` :: `test_broadband_device_type_no_dumps_exits_0`, `test_broadband_minidump_detection_in_core_path` | +| TC-005 | Extender device type path initialisation | Config & Init | ✅ Yes | `config_init_load()` sets `DEVICE_TYPE_EXTENDER` paths (`minidump_path=/opt/minidumps`, `core_path=/var/lib/systemd/coredump`) | ✅ Implemented | `test_unsupported_devicetypes.py` :: `test_extender_device_type_no_dumps_exits_0`, `test_extender_minidump_detection_in_core_path` | +| TC-006 | Secure mode path selection | Config & Init | ✅ Yes | `argv[3]=="secure"` causes `config_init_load()` to select `/opt/secure/minidumps` & `/opt/secure/corefiles` | ✅ Implemented | `test_config_and_path.py` :: `test_secure_mode_selects_secure_minidump_path`, `test_secure_mode_selects_secure_coredump_path` | +| TC-007 | Normal (non-secure) mode path selection | Config & Init | ✅ Yes | Default paths (`minidump_path=/opt/minidumps`, `core_path=/var/lib/systemd/coredump`) used when `argv[3] != "secure"` | ✅ Implemented | `test_config_and_path.py` :: `test_normal_mode_selects_normal_minidump_path` | +| TC-008 | Insufficient arguments → exit(1) | Config & Init | ✅ Yes | `argc < 3` check in `main.c` prints "Number of parameter is less" and calls `exit(1)` | ✅ Implemented | `test_crashupload_arg_parsing.py` :: `test_no_args_exits_with_1`, `test_one_arg_exits_with_1`, `test_two_args_exits_with_1` | +| TC-009 | Dump type selection (minidump / coredump) | Config & Init | ✅ Yes | `argv[2]=="0"` → `DUMP_TYPE_MINIDUMP`; `"1"` → `DUMP_TYPE_COREDUMP` in `config_init_load()` | ✅ Implemented | `test_config_and_path.py` :: `test_coredump_mode_ignores_minidump_files`, `test_minidump_mode_ignores_coredump_files` | +| TC-010 | TLS v1.2 curl flag | Upload | ❌ No | Shell passes `--tlsv1.2` on CLI curl invocation; C `upload.c` uses libcurl `CURLOPT_SSLVERSION` — different implementation path | — | — | +| TC-011 | First instance acquires process lock | Lock Mechanism | ✅ Yes | `acquire_process_lock_or_exit()` in `lock_manager.c` uses `flock(LOCK_EX\|LOCK_NB)` to acquire lock file | ✅ Implemented | `test_lock_lifecycle.py` :: `test_first_instance_acquires_minidump_lock`, `test_first_instance_acquires_coredump_lock` | +| TC-012 | Second instance exits immediately (lock held) | Lock Mechanism | ✅ Yes | Second `flock(LOCK_NB)` call fails → `ERR_LOCK_HELD` → binary prints "already working" and exits | ✅ Implemented | `test_lock_and_exit.py` :: `test_multiple_instance_prevention_minidump`, `test_multiple_instance_prevention_coredump` | +| TC-013 | Wait-for-lock mode blocks until holder exits | Lock Mechanism | ✅ Yes | `argv[4]=="wait_for_lock"` → `acquire_process_lock_or_wait()` loops on blocking `flock()` until previous holder exits | ✅ Implemented | `test_lock_and_wait.py` :: `test_wait_for_lock_minidump`, `test_wait_for_lock_coredump` | +| TC-014 | Lock file removed on clean exit | Lock Mechanism | ✅ Yes | `lock_release()` calls `flock(LOCK_UN)`, closes fd, and `unlink()`s lock file at `cleanup:` label in `main.c` | ✅ Implemented | `test_lock_lifecycle.py` :: `test_lock_removed_on_clean_exit_minidump`, `test_lock_removed_on_clean_exit_coredump` | +| TC-015 | SIGTERM handler removes lock file | Lock Mechanism | ✅ Yes | `handle_signal()` in `main.c` calls `unlink(MINIDUMP_LOCK_FILE)` or `unlink(COREDUMP_LOCK_FILE)` based on `lock_dir_prefix` | ✅ Implemented | `test_signal_lock_cleanup.py` :: `test_sigterm_removes_minidump_lock`, `test_sigterm_removes_coredump_lock` | +| TC-016 | SIGKILL — lock file persists (uncatchable) | Lock Mechanism | ⚠️ Partial | SIGKILL cannot be caught; C handles SIGTERM/SIGINT only. A test can verify lock file remains after `kill -9`, confirming expected uncatchable-signal behavior | ✅ Implemented | `test_lock_lifecycle.py` :: `test_sigkill_lock_file_persists_minidump`, `test_sigkill_lock_file_persists_coredump` | +| TC-017 | Detect minidumps at `/opt/minidumps` (broadband) | Dump Detection | ✅ Yes | `directory_has_pattern(config->minidump_path, ".dmp")` in `prerequisites.c` scans for `.dmp` files | ✅ Implemented | `test_unsupported_devicetypes.py` :: `test_broadband_minidump_detection_in_core_path` | +| TC-018 | Detect minidumps at `/opt/minidumps` (extender) | Dump Detection | ✅ Yes | Same `directory_has_pattern()` call with `DEVICE_TYPE_EXTENDER` paths set by `config_init_load()` | ✅ Implemented | `test_unsupported_devicetypes.py` :: `test_extender_minidump_detection_in_core_path` | +| TC-019 | Detect coredumps at `/opt/coredumps` | Dump Detection | ✅ Yes | `directory_has_pattern(config->core_path, "_core")` in `prerequisites.c` | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_coredump_detection_normal_path` | +| TC-020 | Detect minidump `.dmp` file at secure path | Dump Detection | ✅ Yes | `directory_has_pattern("/opt/secure/minidumps", ".dmp")` returns 1 when `.dmp` file is present | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_minidump_detection_secure_path` | +| TC-021 | Detect coredump `_core` file at secure path | Dump Detection | ✅ Yes | `directory_has_pattern("/opt/secure/corefiles", "_core")` returns 1 when `_core` file is present | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_coredump_detection_secure_path` | +| TC-022 | No dump files found → `exit(0)` | Dump Detection | ✅ Yes | `directory_has_pattern()` returns 0 → `NO_DUMPS_FOUND=5` → `prerequisites_wait()` fails → `goto cleanup` → `exit(0)` | ✅ Implemented | `test_no_dumps_exit.py` :: `test_empty_minidump_dir_exits_with_0`, `test_empty_corefiles_dir_exits_with_0`, `test_wrong_extension_file_exits_with_0`, `test_nonexistent_dump_dir_exits_with_0`; `test_crashupload_failure_return.py` :: `test_prerequisites_no_dumps_exits_with_0` | +| TC-023 | MAC address read and normalised | Device Info | ✅ Yes | `GetEstbMac()` in `platform.c` reads from `MAC_FILE` (`/tmp/.macAddress`); `NormalizeMac()` strips `:` and uppercases | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_mac_address_read_and_normalised` | +| TC-024 | MAC address retrieved from hardware when file empty | Device Info | ✅ Yes | `GetEstbMac()` falls back to hardware interface query when `/tmp/.macAddress` is empty or missing | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_mac_fallback_when_file_missing_or_empty` | +| TC-025 | Model number retrieved from device | Device Info | ⚠️ Partial | `getModelNum()` in `platform.c` uses `common_device_api` internally; differs from shell `getDeviceDetails.sh` invocation but functionally equivalent | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_model_number_retrieved` | +| TC-026 | Model number fallback path | Device Info | ⚠️ Partial | `getModelNum()` C fallback logic exists; maps to a different code path than shell fallback | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_model_number_retrieved` | +| TC-027 | Model number from `getDeviceDetails.sh` | Device Info | ❌ No | Shell-specific: calls `/lib/rdk/getDeviceDetails.sh`; C uses direct `common_device_api` library call — no shell script invocation | — | — | +| TC-028 | SHA1 firmware hash retrieval | Device Info | ⚠️ Partial | `getSHA1()` in `platform.c` reads firmware hash via C API; shell calls `getSHA1` command — same result, different mechanism | ✅ Implemented | `test_config_checks_and_baseline.py` :: `test_sha1_firmware_hash_from_version_txt` | +| TC-029 | Partner ID via `getpartnerid.sh` | Device Info | ❌ No | Shell-specific: invokes `getpartnerid.sh`; C reads partner ID directly via `rf_hal_property_get()` — no script invocation | — | — | +| TC-030 | Partner ID from account management file | Device Info | ❌ No | Shell `grep`s account file for partner ID; no equivalent file-grep logic in C implementation | — | — | +| TC-031 | Wait for network connectivity before upload | Network / Prerequisites | ❌ No | Network-wait loop is an unimplemented TODO in `prerequisites_wait()` (`prerequisites.c`); the C binary does not perform any network check before uploading | — | — | +| TC-032 | Network becomes available → processing proceeds | Network / Prerequisites | ❌ No | Depends on the same unimplemented network-wait loop as TC-031 | — | — | +| TC-033 | Network timeout → abort | Network / Prerequisites | ❌ No | Depends on the same unimplemented network-wait loop as TC-031; no timeout path exists | — | — | +| TC-034 | Broadband network via `network_commn_status` | Network / Prerequisites | ❌ No | `network_commn_status` is a pure shell function; no direct equivalent in C implementation | — | — | +| TC-035 | Defer upload when uptime < 480 s | Upload Deferral | ✅ Yes | `defer_upload_if_needed()` in `prerequisites.c` reads `UPTIME_FILE` and sleeps for `DEVICE_TYPE_MEDIACLIENT` when uptime < 480 s | ✅ Implemented | `test_upload_deferral.py` :: `test_deferred_when_uptime_below_threshold` | +| TC-036 | No deferral when uptime ≥ 480 s | Upload Deferral | ✅ Yes | `defer_upload_if_needed()` skips sleep when uptime ≥ 480 s; L2_TEST build uses `/opt/uptime` for controlled values | ✅ Implemented | `test_upload_deferral.py` :: `test_no_deferral_when_uptime_above_threshold` | +| TC-037 | Reboot flag present → skip upload, exit(0) | Upload Deferral | ✅ Yes | `filePresentCheck("/tmp/set_crash_reboot_flag")` returns true inside `defer_upload_if_needed()` → `ret=0` → `goto cleanup` → `exit(0)` | ✅ Implemented | `test_reboot_and_log_scenario.py` :: `test_reboot_flag_present_skips_upload_exits_0` | +| TC-038 | RFC opt-out flag set → skip upload | Telemetry Opt-Out | ✅ Yes | `get_opt_out_status()` in `config_manager.c` queries RFC value; returns opt-out when flag is set | ✅ Implemented | `test_t2_optout.py` :: `test_rfc_optout_set_mediaclient_exits_0` | +| TC-039 | Opt-out file present → skip upload | Telemetry Opt-Out | ✅ Yes | `get_opt_out_status()` additionally checks for presence of opt-out override file on filesystem | ✅ Implemented | `test_t2_optout.py` :: `test_optout_file_absent_upload_not_blocked` | +| TC-040 | Opt-out check only for `MEDIACLIENT` device type | Telemetry Opt-Out | ✅ Yes | `get_opt_out_status()` check is gated on `DEVICE_TYPE_MEDIACLIENT`; other device types bypass this check | ✅ Implemented | `test_t2_optout.py` :: `test_optout_check_bypassed_for_nonmediaclient` | +| TC-041 | On-startup cleanup removes oldest dump files | Cleanup | ✅ Yes | `cleanup_batch()` in `cleanup_batch.c` mtime-sorts files; removes oldest beyond limit; uses `ON_STARTUP_DUMPS_CLEANED_UP_BASE` flag | ✅ Implemented | `test_cleanup_batch.py` :: `test_startup_cleanup_deletes_non_dump_files` | +| TC-042 | Old dump files deleted to enforce limit | Cleanup | ✅ Yes | Files older than newest `MAX_CORE_FILES=4` are deleted during cleanup pass | ✅ Implemented | `test_cleanup_batch.py` :: `test_stale_archive_files_older_than_2_days_deleted` | +| TC-043 | First-run cleanup flag file created | Cleanup | ✅ Yes | `ON_STARTUP_DUMPS_CLEANED_UP_BASE` flag file is created after first-run cleanup completes | ✅ Implemented | `test_cleanup_batch.py` :: `test_first_run_flag_created` | +| TC-044 | Subsequent runs skip first-run cleanup pass | Cleanup | ✅ Yes | Flag file presence detected → `cleanup_batch()` skips first-run cleanup path on re-entry | ✅ Implemented | `test_cleanup_batch.py` :: `test_subsequent_run_skips_startup_cleanup` | +| TC-045 | `MAX_CORE_FILES=4` limit enforced | Cleanup | ✅ Yes | Only the 4 newest files are retained; all older files are deleted by `cleanup_batch()` | ✅ Implemented | `test_cleanup_batch.py` :: `test_max_core_files_limit_enforced` | +| TC-046 | Empty dump directory handled gracefully | Cleanup | ✅ Yes | `cleanup_batch()` handles `opendir()` returning NULL or empty directory without error | ✅ Implemented | `test_cleanup_batch.py` :: `test_empty_dir_handled_gracefully` | +| TC-047 | Upload-on-startup mode (minidump-on-bootup) | Cleanup | ⚠️ Partial | Upload-on-startup flow exists in `main.c` via `minidump-on-bootup-upload.service`; exact behaviour may differ from shell on-startup path | ✅ Implemented | `test_cleanup_batch.py` :: `test_upload_on_startup_flag_removed_for_coredump_mode` | +| TC-048 | Upload count ≤ 10 → ALLOW_UPLOAD | Rate Limiting | ✅ Yes | `is_upload_limit_reached()` in `ratelimit.c` counts timestamp file lines; ≤ 10 entries → returns `ALLOW_UPLOAD` | ✅ Implemented | `test_ratelimit_allow.py` :: `test_upload_allowed_when_count_at_or_below_limit` | +| TC-049 | Upload count > 10 within window → STOP_UPLOAD | Rate Limiting | ✅ Yes | > 10 lines within `RECOVERY_DELAY_SECONDS` window → `is_upload_limit_reached()` returns `STOP_UPLOAD` | ✅ Implemented | `test_ratelimit.py` :: `test_upload_blocked_when_count_exceeds_10` | +| TC-050 | Minidump counter check is minidump-only | Rate Limiting | ✅ Yes | `is_upload_limit_reached()` reads `/tmp/.minidump_upload_timestamps` only for `DUMP_TYPE_MINIDUMP`; coredump path bypasses this counter check when deny-window is not active | ✅ Implemented | `test_ratelimit_allow.py` :: `test_coredump_not_rate_limited_by_minidump_counter` | +| TC-051 | Recovery time not yet reached → global cooldown blocks uploads | Rate Limiting | ✅ Yes | `is_recovery_time_reached()` reads deny-till timestamp from `/tmp/.deny_dump_uploads_till`; if still inside window, both minidump and coredump runs are blocked | ✅ Implemented | `test_ratelimit.py` :: `test_upload_blocked_when_deny_file_active`, `ratelimit_gtest.cpp` :: `RatelimitCheckUnified_CoredumpType_RecoveryWindow_BlockUpload` | +| TC-052 | Recovery time reached → uploads unblocked | Rate Limiting | ✅ Yes | `is_recovery_time_reached()` returns `true` after window expires → uploads resume normally | ✅ Implemented | `test_ratelimit_allow.py` :: `test_recovery_time_expired_unblocks_upload` | +| TC-053 | Timestamp written to rate limit log after upload | Rate Limiting | ✅ Yes | `set_time()` in `ratelimit.c` appends current timestamp entry to rate limit log file | ✅ Implemented | `test_ratelimit_allow.py` :: `test_no_deny_file_allows_upload_to_proceed` | +| TC-054 | Rate limit counter resets after recovery period | Rate Limiting | ✅ Yes | Timestamps older than `RECOVERY_DELAY_SECONDS` are not counted; counter effectively resets after recovery period | ✅ Implemented | `test_ratelimit_allow.py` :: `test_rate_limit_resets_after_recovery_period` | +| TC-055 | Timestamp written in truncated integer format | Rate Limiting | ✅ Yes | `set_time()` writes truncated (integer) timestamp — fractional seconds stripped before writing | ✅ Implemented | `test_ratelimit.py` :: `test_set_time_writes_integer_format_timestamp` | +| TC-056 | Timestamp suppressed for non-production builds | Rate Limiting | ❌ No | Shell silences timestamp writes when `BUILD_TYPE` is non-prod; C `ratelimit.c` always writes timestamp regardless of build type | — | — | +| TC-057 | Filename sanitisation preserves container delimiter | Dump Processing | ✅ Yes | `sanitize_filename_preserve_container()` in `scanner.c` preserves the `<#=#>` delimiter and container name suffix intact | ✅ Implemented | `test_scanner_behaviour.py` :: `test_container_delimiter_preserved_in_sanitization` | +| TC-058 | Special characters in filename replaced with `_` | Dump Processing | ✅ Yes | Non-alphanumeric chars (excluding `-`, `.`, `_`) are replaced with `_` by `sanitize_filename_preserve_container()` | ✅ Implemented | `test_scanner_behaviour.py` :: `test_forbidden_chars_dropped_from_filename` | +| TC-059 | Container name preserved after sanitisation | Dump Processing | ✅ Yes | Text after `<#=#>` delimiter is kept verbatim through the sanitisation pass | ✅ Implemented | `test_scanner_behaviour.py` :: `test_container_name_preserved_with_forbidden_chars` | +| TC-060 | Skip existing `.tgz` archive files | Dump Processing | ✅ Yes | Dump iteration loop in `main.c` skips files whose extension matches `.tgz` to avoid re-processing already-archived dumps | ✅ Implemented | `test_dump_processing.py` :: `test_existing_tgz_not_re_archived` | +| TC-061 | Archive filename includes MAC + timestamp + pname + version | Dump Processing | ✅ Yes | Output archive named `___.tgz`; assembled in `main.c` / `archive.c` | ✅ Implemented | `test_file_ext_check.py` :: `test_archive_filename_contains_required_fields` | +| TC-062 | Archive filename truncated at 135 characters | Dump Processing | ✅ Yes | `trim_process_name_in_path()` in `main.c` enforces 135-character filename limit for archive names | ✅ Implemented | `test_file_ext_check.py` :: `test_archive_filename_truncated_at_135_chars` | +| TC-063 | `mpeos-main` process name mapped correctly | Dump Processing | ✅ Yes | `mpeos-main` recognised and mapped to its standardised archive name during dump naming in `main.c` | ✅ Implemented | `test_file_ext_check.py` :: `test_mpeos_main_uses_mtime_not_crashts` | +| TC-064 | Dump filename components parsed correctly | Dump Processing | ✅ Yes | `extract_pname()` and `extract_appname()` in `scanner.c` parse process name and application name from dump filename | ✅ Implemented | `test_scanner_behaviour.py` :: `test_dump_filename_components_parsed_correctly` | +| TC-065 | Archive created with dump file and associated logs | Archive Creation | ✅ Yes | `archive_create_smart()` in `archive.c` produces a `.tgz` containing the dump file and all mapped log files | ✅ Implemented | `test_archive_content.py` :: `test_archive_created_for_dump` | +| TC-066 | Archive contains all required files | Archive Creation | ✅ Yes | `archive_create_smart()` verifies each required file is added; returns error if mandatory file missing | ✅ Implemented | `test_archive_content.py` :: `test_archive_contains_required_members` | +| TC-067 | Broadband-specific log archive behaviour | Archive Creation | ⚠️ Partial | Broadband archive logic exists in `archive.c` but the set of included logs differs from the shell implementation | ✅ Implemented | `test_broadband_env.py` :: `test_broadband_minidump_archive_not_created` | +| TC-068 | `/tmp` free-space check before archiving | Archive Creation | ❌ No | Shell checks `/tmp` free space before creating archive; C `archive_create_smart()` performs no disk-space pre-check | — | — | +| TC-069 | Archive retry when `/tmp` disk is full | Archive Creation | ❌ No | Shell retries archive creation using a fallback `copy_log_files_tmp_dir` path; C has no equivalent retry mechanism | — | — | +| TC-070 | Temporary directory cleanup after archive | Archive Creation | ❌ No | Shell-specific temp directory copy-and-cleanup pattern (`copy_log_files_tmp_dir`); not present in C | — | — | +| TC-071 | Zero-size dump file skipped | Dump Processing | ✅ Yes | `stat()`-based size check in `scanner.c` / `archive.c` detects and skips zero-byte dump files | ✅ Implemented | `test_dump_processing.py` :: `test_zero_size_dump_handled_gracefully` | +| TC-072 | Process crash telemetry event sent | Crash Telemetry | ✅ Yes | `t2_event_s(T2_EVENT_PROCESS_CRASH, ...)` called in `scanner.c` when a process crash dump is processed | ✅ Implemented | `test_telemetry.py` :: `test_process_crash_telemetry_path_exercised` | +| TC-073 | Container crash telemetry event sent | Crash Telemetry | ✅ Yes | `t2_event_s(T2_EVENT_CONTAINER_CRASH, ...)` called in `scanner.c` when a container crash dump is detected | ✅ Implemented | `test_telemetry.py` :: `test_container_crash_telemetry_path_exercised` | +| TC-074 | Telemetry on tarball-retry (isTgz) detection | Crash Telemetry | ❌ No | Shell-specific: `isTgz` checks if re-upload of a previous tarball is happening and emits telemetry; no equivalent in C | — | — | +| TC-075 | Log files mapped for crashed process | Log File Mapping | ✅ Yes | `lookup_log_files_for_proc()` in `scanner.c` parses `LOGMAPPER_FILE_PATH` to find log files associated with the crashed process | ✅ Implemented | `test_archive_content.py` :: `test_log_files_mapped_for_crashed_process` | +| TC-076 | Log file lines capped at 500 (production build) | Log File Mapping | ❌ No | Shell truncates log files to 500 lines for prod `BUILD_TYPE`; C adds entire log file to archive without line-count limits | — | — | +| TC-077 | Log file lines capped at 5000 (non-production build) | Log File Mapping | ❌ No | Shell truncates log files to 5000 lines for non-prod builds; C has no build-type-based line-count limit | — | — | +| TC-078 | Missing log file handled gracefully | Log File Mapping | ✅ Yes | `scanner.c` checks file existence (`filePresentCheck()`) before adding to archive; missing log file is skipped with a warning log | ✅ Implemented | `test_archive_content.py` :: `test_missing_log_file_handled_gracefully` | +| TC-079 | `crashed_url.txt` generated with upload URL | Log File Mapping | ✅ Yes | `crashed_url.txt` written with the S3 upload URL in `scanner.c` / `archive.c` before archive is finalised | ✅ Implemented | `test_archive_content.py` :: `test_crashed_url_file_included_in_archive` | +| TC-080 | All associated log files added to archive | Log File Mapping | ✅ Yes | `archive_add_file()` called for each log file returned by `lookup_log_files_for_proc()`; all mapped logs included | ✅ Implemented | `test_archive_content.py` :: `test_all_mapped_log_files_added_to_archive` | +| TC-081 | Upload succeeds on first attempt | Upload | ✅ Yes | `upload_file()` in `upload.c` returns `UPLOAD_SUCCESS` on first successful libcurl transfer | ✅ Implemented | `test_single_dump_upload.py` :: `TestUpload::test_single_successful_upload` | +| TC-082 | Upload retried up to 3 times on failure | Upload | ✅ Yes | `MAX_RETRIES=3` in `upload.c`; `/tmp/cu_fail_n=2` forces iterations 1–2 to fail; iteration 3 succeeds via real server | ✅ Implemented | `test_upload_retry.py` :: `TestUploadSuccessAfterRetries::test_upload_succeeds_after_retries` | +| TC-083 | Upload permanently fails after 3 retries → error logged | Upload | ✅ Yes | After 3 consecutive failures `upload_file()` logs error and returns failure code to caller | ✅ Implemented | `test_upload_retry.py` :: `TestUploadPermanentFailure::test_upload_fails_after_max_retries` | +| TC-084 | Fallback to alternative upload path | Upload | ❌ No | Fallback upload path is an explicit `TODO: SUPPORT NOT AVAILABLE` stub in `upload.c`; the C implementation does not have a fallback path | — | — | +| TC-085 | Single instance lock prevents duplicate execution | Lock Mechanism | ✅ Yes | `acquire_process_lock_or_exit()` prevents a second concurrent instance — identical mechanism to TC-012 | ✅ Implemented | `test_lock_and_exit.py` :: `test_multiple_instance_prevention_minidump`, `test_multiple_instance_prevention_coredump` | + +--- + +## Coverage Statistics + +| Metric | Count | +|--------|-------| +| Total TCs in `uploadDumps_TestCases.md` | 85 | +| ✅ Applicable to C implementation | 60 | +| ⚠️ Partially applicable | 10 | +| ❌ Not applicable (shell-only) | 15 | +| **Applicable TCs with L2 tests** | **66** out of 66 applicable | +| **Applicable TCs without L2 tests** | **0** | + +> **Key counts:** 85 total TCs − 19 not-applicable = **66 applicable**. 66 implemented + 0 pending = 66 ✓ + +### Applicable TCs — Coverage Breakdown by Category + +> Columns: **Total** = TC count in category | **✅ Yes** = full C equivalent | **⚠️ Part.** = partial C equivalent | **❌ No** = not applicable (no C equivalent) | **Impl.** = have L2 tests + +| Category | Total | ✅ Yes | ⚠️ Part. | ❌ No | Impl. | +|----------|-----------|--------|------------|-------|-------------| +| Config & Init | 9 | 6 | 0 | 3 | 6 (TC-004, TC-005, TC-006, TC-007, TC-008, TC-009) | +| Lock Mechanism | 7 | 6 | 1 | 0 | 7 (TC-011, TC-012, TC-013, TC-014, TC-015, TC-016, TC-085) | +| Dump Detection | 6 | 6 | 0 | 0 | 6 (TC-017, TC-018, TC-019, TC-020, TC-021, TC-022) | +| Device Info | 8 | 2 | 3 | 3 | 5 (TC-023, TC-024, TC-025, TC-026, TC-028) | +| Network / Prerequisites | 4 | 0 | 0 | 4 | 0 | +| Upload Deferral | 3 | 3 | 0 | 0 | 3 (TC-035, TC-036, TC-037) | +| Telemetry Opt-Out | 3 | 3 | 0 | 0 | 3 (TC-038, TC-039, TC-040) | +| Cleanup | 7 | 6 | 1 | 0 | 7 (TC-041, TC-042, TC-043, TC-044, TC-045, TC-046, TC-047) | +| Rate Limiting | 9 | 8 | 0 | 1 | 8 (TC-048, TC-049, TC-050, TC-051, TC-052, TC-053, TC-054, TC-055) | +| Dump Processing & Naming | 9 | 9 | 0 | 0 | 9 (TC-057, TC-058, TC-059, TC-060, TC-061, TC-062, TC-063, TC-064, TC-071) | +| Archive Creation | 6 | 2 | 1 | 3 | 3 (TC-065, TC-066, TC-067) | +| Crash Telemetry | 3 | 2 | 0 | 1 | 2 (TC-072, TC-073) | +| Log File Mapping | 6 | 4 | 0 | 2 | 4 (TC-075, TC-078, TC-079, TC-080) | +| Upload | 5 | 3 | 0 | 2 | 3 (TC-081, TC-082, TC-083) | +| **Total** | **85** | **60** | **6** | **19** | **66** | + +--- + +## Extra L2 Tests (No Matching TC in `uploadDumps_TestCases.md`) + +These tests were added to cover C-specific behaviour not captured in the original shell-oriented TCs. +TC-IDs prefixed with **`C-TC-`** denote C-implementation-specific test cases. + +| TC-ID | Test File :: Function | Description | +|-------|-----------------------|-------------| +| C-TC-001 | `test_crashupload_failure_return.py` :: `test_system_initialize_failure_exits_with_1` | Verifies `system_initialize()` failure (unwritable `/opt/logs`) causes `exit(1)` | +| C-TC-002 | `test_reboot_and_log_scenario.py` :: `test_binary_produces_log_output` | Verifies binary always emits log output to stdout (logger init / exit) regardless of dump presence | + +--- + +## Not-Applicable TCs — Summary + +| TC-IDs | Reason | +|--------|--------| +| TC-001, TC-002, TC-003 | Shell `source` of `.properties` / `.sh` files; C uses internal `rf_hal_property_get()` API | +| TC-010 | Shell passes `--tlsv1.2` on CLI curl; C uses libcurl `CURLOPT_SSLVERSION` | +| TC-027 | Shell invokes `getDeviceDetails.sh`; C uses `common_device_api` library call | +| TC-029, TC-030 | Shell invokes `getpartnerid.sh` / greps account file; C uses `rf_hal_property_get()` | +| TC-034 | `network_commn_status` is a pure shell function; no C equivalent | +| TC-031, TC-032, TC-033 | Network-wait loop is an unimplemented TODO stub in `prerequisites_wait()` (`prerequisites.c`); the C binary performs no network check — these TCs describe behaviour that does not exist in the C implementation | +| TC-084 | Fallback upload path is an explicit `TODO: SUPPORT NOT AVAILABLE` stub in `upload.c`; no fallback mechanism exists in the C implementation | +| TC-056 | Shell suppresses rate-limit timestamps for non-prod `BUILD_TYPE`; C always writes timestamps | +| TC-068, TC-069, TC-070 | Shell `/tmp` disk-space check + `copy_log_files_tmp_dir` retry; not implemented in C | +| TC-074 | Shell `isTgz` tarball-retry detection telemetry; no C equivalent | +| TC-076, TC-077 | Shell per-build-type log line-count caps (500 / 5000); C copies full log files | + +--- + +## C Source ↔ TC Mapping Reference + +| C Source File | Key Functions | Related TCs | +|---------------|---------------|-------------| +| `config/config_manager.c` | `config_init_load()`, `get_opt_out_status()` | TC-004–009, TC-038–040 | +| `init/system_init.c` | `system_initialize()` | TC-004 (init failure path) | +| `platform/platform.c` | `GetEstbMac()`, `NormalizeMac()`, `getModelNum()`, `getSHA1()` | TC-023–028 | +| `utils/prerequisites.c` | `prerequisites_wait()`, `directory_has_pattern()`, `defer_upload_if_needed()` | TC-017–022, TC-031–037 | +| `lock/lock_manager.c` | `acquire_process_lock_or_exit()`, `acquire_process_lock_or_wait()`, `lock_release()` | TC-011–016, TC-085 | +| `ratelimit/ratelimit.c` | `is_upload_limit_reached()`, `is_recovery_time_reached()`, `set_time()` | TC-048–055 | +| `scanner/scanner.c` | `sanitize_filename_preserve_container()`, `extract_pname()`, `lookup_log_files_for_proc()` | TC-057–059, TC-064, TC-071–073, TC-075, TC-078–080 | +| `archive/archive.c` | `archive_create_smart()`, `archive_add_file()` | TC-060–067, TC-071, TC-079–080 | +| `upload/upload.c` | `upload_file()` | TC-081–083 | +| `cleanup/cleanup_batch.c` | `cleanup_batch()` | TC-041–047 | +| `main.c` | Signal handler, dump loop, `trim_process_name_in_path()` | TC-008–009, TC-015–016, TC-060–063 | diff --git a/test/L2_TODO.md b/test/L2_TODO.md new file mode 100644 index 0000000..8897afe --- /dev/null +++ b/test/L2_TODO.md @@ -0,0 +1,162 @@ +# L2 Test TODO — CrashUpload C Implementation + +Tracks which applicable TCs still need L2 functional tests. + +**Reference:** `L2_TESTS.md` for full TC table and coverage statistics. + +--- + +## ✅ Applicable + Implemented — 65 TCs + +| TC-ID | Test Case Name | +|-------|----------------| +| TC-004 | Broadband device type path initialisation | +| TC-005 | Extender device type path initialisation | +| TC-006 | Secure mode path selection | +| TC-007 | Normal (non-secure) mode path selection | +| TC-008 | Insufficient arguments → exit(1) | +| TC-009 | Dump type selection (minidump / coredump) | +| TC-011 | First instance acquires process lock | +| TC-012 | Second instance exits immediately (lock held) | +| TC-013 | Wait-for-lock mode blocks until holder exits | +| TC-014 | Lock file removed on clean exit | +| TC-015 | SIGTERM handler removes lock file | +| TC-016 | SIGKILL — lock file persists (uncatchable) | +| TC-017 | Detect minidumps at `/opt/minidumps` (broadband) | +| TC-018 | Detect minidumps at `/opt/minidumps` (extender) | +| TC-019 | Detect coredumps at `/opt/coredumps` | +| TC-020 | Detect minidump `.dmp` file at secure path | +| TC-021 | Detect coredump `_core` file at secure path | +| TC-022 | No dump files found → exit(0) | +| TC-023 | MAC address read and normalised | +| TC-024 | MAC address retrieved from hardware when file empty | +| TC-025 | Model number retrieved from device | +| TC-026 | Model number fallback path | +| TC-028 | SHA1 firmware hash retrieval | +| TC-035 | Defer upload when uptime < 480 s | +| TC-036 | No deferral when uptime ≥ 480 s | +| TC-037 | Reboot flag present → skip upload, exit(0) | +| TC-038 | RFC opt-out flag set → skip upload | +| TC-039 | Opt-out file present → skip upload | +| TC-040 | Opt-out check only for `MEDIACLIENT` device type | +| TC-041 | On-startup cleanup removes oldest dump files | +| TC-042 | Old dump files deleted to enforce limit | +| TC-043 | First-run cleanup flag file created | +| TC-044 | Subsequent runs skip first-run cleanup pass | +| TC-045 | `MAX_CORE_FILES=4` limit enforced | +| TC-046 | Empty dump directory handled gracefully | +| TC-047 | Upload-on-startup mode — `/opt/.upload_on_startup` removed on coredump run | +| TC-048 | Upload count ≤ 10 → ALLOW_UPLOAD | +| TC-049 | Upload count > 10 within window → STOP_UPLOAD | +| TC-050 | Minidump counter check is minidump-only | +| TC-051 | Recovery time not yet reached → uploads still blocked | +| TC-052 | Recovery time expired → uploads unblocked | +| TC-054 | Rate-limit counter reset after recovery period | +| TC-057 | Filename sanitisation preserves container delimiter | +| TC-058 | Special characters in filename replaced with `_` | +| TC-059 | Container name preserved after sanitisation | +| TC-060 | Skip existing `.tgz` archive files | +| TC-061 | Archive filename includes MAC + timestamp + pname + version | +| TC-062 | Archive filename truncated at 135 characters | +| TC-063 | `mpeos-main` process name mapped correctly | +| TC-064 | Dump filename components parsed correctly | +| TC-065 | Archive created with dump file and associated logs | +| TC-066 | Archive contains all required files | +| TC-067 | Broadband-specific log archive behaviour | +| TC-071 | Zero-size dump file skipped | +| TC-072 | Process crash telemetry event sent | +| TC-073 | Container crash telemetry event sent | +| TC-075 | Log files mapped for crashed process | +| TC-078 | Missing log file handled gracefully | +| TC-079 | `crashed_url.txt` generated with upload URL | +| TC-080 | All associated log files added to archive | +| TC-053 | Rate-limit check passes when no deny file present (is_recovery_time_reached) | Rate Limiting | +| TC-055 | set_time() writes timestamps in integer format (no fractional seconds) | Rate Limiting | +| TC-081 | Upload succeeds on first attempt | Upload | +| TC-082 | Upload retried up to 3 times on failure | Upload | +| TC-083 | Upload permanently fails after 3 retries → error logged | Upload | +| TC-085 | Single instance lock prevents duplicate execution | + +--- + +## 🔲 Applicable + Not Implemented — 0 TCs + +*(All applicable TCs now have L2 test coverage.)* + +--- + +## ❌ Not Applicable — 19 TCs + +These TCs cover shell-specific behaviour or unimplemented C stubs with no equivalent in the C implementation. +See `L2_TESTS.md` "Not-Applicable TCs — Summary" for the reason each was excluded. + +### Config & Init — 3 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-001 | Source `device.properties` on startup | Shell `source` sets global vars; C uses `rf_hal_property_get()` API internally | +| TC-002 | Source `include.properties` on startup | Shell global variable setup; C uses compile-time / runtime config API | +| TC-003 | Source `t2Shared_api.sh` T2 library | Shell sources T2 helper script; C links T2 library directly | + +### Network / Prerequisites — 4 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-031 | Wait for network connectivity before upload | Network-wait loop is an unimplemented TODO stub in `prerequisites_wait()` | +| TC-032 | Network becomes available → processing proceeds | Depends on same unimplemented network-wait loop as TC-031 | +| TC-033 | Network timeout → abort | Depends on same unimplemented network-wait loop; no timeout path exists | +| TC-034 | Broadband network via `network_commn_status` | Pure shell function; no direct equivalent in C | + +### Device Info — 3 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-027 | Model number from `getDeviceDetails.sh` | Shell-specific; C uses `common_device_api` library call directly | +| TC-029 | Partner ID via `getpartnerid.sh` | Shell-specific; C reads partner ID via `rf_hal_property_get()` | +| TC-030 | Partner ID from account management file | Shell `grep`s account file; no equivalent file-grep logic in C | + +### Rate Limiting — 1 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-056 | Timestamp suppressed for non-production builds | Shell silences timestamp writes for non-prod; C `ratelimit.c` always writes regardless of build type | + +### Upload — 2 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-010 | TLS v1.2 curl flag | Shell passes `--tlsv1.2` on CLI curl; C uses libcurl `CURLOPT_SSLVERSION` | +| TC-084 | Fallback to alternative upload path | Fallback path is `TODO: SUPPORT NOT AVAILABLE` stub in `upload.c`; feature not implemented | + +### Archive Creation — 3 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-068 | `/tmp` free-space check before archiving | Shell pre-checks `/tmp` disk space; C `archive_create_smart()` has no disk-space check | +| TC-069 | Archive retry when `/tmp` disk is full | Shell retries via `copy_log_files_tmp_dir` fallback; C has no retry mechanism | +| TC-070 | Temporary directory cleanup after archive | Shell-specific temp directory copy-and-cleanup pattern; not present in C | + +### Crash Telemetry — 1 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-074 | Telemetry on tarball-retry (`isTgz`) detection | Shell-specific `isTgz` re-upload detection; no equivalent in C | + +### Log File Mapping — 2 + +| TC-ID | Test Case Name | Reason | +|-------|----------------|--------| +| TC-076 | Log file lines capped at 500 (production build) | Shell truncates to 500 lines for prod; C archives entire log without line-count limits | +| TC-077 | Log file lines capped at 5000 (non-production build) | Shell truncates to 5000 lines for non-prod; C has no build-type-based line-count limit | + +--- + +## Summary + +| Status | Count | +|--------|-------| +| Total TCs in `uploadDumps_TestCases.md` | 85 | +| ❌ Not Applicable | 19 | +| ✅ Applicable — total | 66 | +|    ✅ Implemented | 66 | +|    🔲 Pending | 0 | diff --git a/test/functional-tests/features/ratelimit.feature b/test/functional-tests/features/ratelimit.feature new file mode 100644 index 0000000..53b0112 --- /dev/null +++ b/test/functional-tests/features/ratelimit.feature @@ -0,0 +1,101 @@ +########################################################################## +# If not stated otherwise in this file or this component's LICENSE +# file the following copyright and licenses apply: +# +# Copyright 2025 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +# TC-048 / TC-049 / TC-050 / TC-051 / TC-052 / TC-053 / TC-055 +Feature: Rate limiting — allow and block upload paths + + ratelimit_check_unified() enforces two independent limits: + 1. Deny-window check (is_recovery_time_reached): reads /tmp/.deny_dump_uploads_till. + 2. Per-minidump upload-count check (is_upload_limit_reached): reads + /tmp/.minidump_upload_timestamps (counter check only for minidump mode). + When either check returns BLOCK, remove_pending_dumps() cleans the working dir. + + Background: + Given the crashupload binary exists + And /opt/logs directory and core_log.txt exist + And a .dmp file is present so prerequisites_wait succeeds + And /tmp/set_crash_reboot_flag is set to prevent actual network uploads + + # TC-048 + Scenario: Upload count at or below limit (10 entries) allows upload to proceed + Given /tmp/.minidump_upload_timestamps contains exactly 10 lines + And /tmp/.deny_dump_uploads_till does not exist + When the binary runs in minidump mode + Then is_upload_limit_reached returns ALLOW_UPLOAD + And ratelimit_check_unified returns ALLOW_UPLOAD + And /tmp/.deny_dump_uploads_till is NOT created + And the binary exits with return code 0 + + # TC-049 + Scenario: Upload count exceeding 10 within the window triggers rate-limit block + Given /tmp/.minidump_upload_timestamps has 11 lines where the first timestamp is within 600 s + When the binary runs in minidump mode + Then is_upload_limit_reached returns STOP_UPLOAD + And ratelimit_check_unified returns RATELIMIT_BLOCK + And remove_pending_dumps deletes all .dmp and .tgz files from the working directory + And /tmp/.deny_dump_uploads_till is created with a future timestamp + And the binary exits with return code 0 + + # TC-050 + Scenario: Rate-limit upload-count check is skipped for coredump mode when deny-window is inactive + Given /tmp/.minidump_upload_timestamps has 11 lines (would block a minidump run) + And /tmp/.deny_dump_uploads_till does not exist + When the binary runs in coredump mode + Then the upload-count else-branch returns ALLOW_UPLOAD immediately + And /tmp/.deny_dump_uploads_till is NOT created by the rate limiter + And the binary exits with return code 0 + + # TC-051 + Scenario: Active deny-window file blocks upload regardless of count + Given /tmp/.deny_dump_uploads_till contains a timestamp 1 hour in the future + When the binary runs in minidump mode + Then is_recovery_time_reached returns false (still within deny window) + And ratelimit_check_unified returns RATELIMIT_BLOCK + And the working directory contains no .dmp or .tgz files after the run + And the binary exits with return code 0 + + # TC-053 + Scenario: No deny file present — is_recovery_time_reached passes and upload is allowed + Given /tmp/.deny_dump_uploads_till does not exist + And /tmp/.minidump_upload_timestamps does not exist + When the binary runs in minidump mode + Then is_recovery_time_reached returns ALLOW_UPLOAD (file absent) + And the rate limiter does not block the upload + And the binary exits with return code 0 + + # TC-055 + Scenario: set_time() writes the deny-until timestamp as a plain integer (no fractional seconds) + Given the rate limit has triggered (count > 10 within window) + When the binary writes /tmp/.deny_dump_uploads_till + Then the file contains a single integer Unix timestamp with no decimal point + + # TC-052 + Scenario: Expired deny-window file is ignored and upload proceeds + Given /tmp/.deny_dump_uploads_till contains a timestamp 1 hour in the past + When the binary runs in minidump mode + Then is_recovery_time_reached returns ALLOW_UPLOAD (window expired) + And the binary exits with return code 0 + + # TC-054 + Scenario: Rate-limit counter resets after the recovery period elapses + Given /tmp/.minidump_upload_timestamps has 11 lines where the first timestamp is older than 600 s + When the binary runs in minidump mode + Then is_upload_limit_reached sees the window as expired + And ALLOW_UPLOAD is returned + And the binary exits with return code 0 diff --git a/test/functional-tests/tests/README.md b/test/functional-tests/tests/README.md index 765e526..519a8cd 100644 --- a/test/functional-tests/tests/README.md +++ b/test/functional-tests/tests/README.md @@ -1,43 +1,122 @@ -# Functional Test Implementation +# Functional Test Implementations -This directory contains pytest-based functional tests that execute the Gherkin scenarios defined in [features/](../features/). +This directory contains the pytest-based L2 functional test suite for the +crashupload C binary. Each `.py` file exercises one or more TC-IDs from +[uploadDumps_TestCases.md](../../uploadDumps_TestCases.md) by running the +compiled binary directly against a set of controlled filesystem conditions. -## Overview +Gherkin-format specifications for all tests are in [../features/](../features/). -Tests are implemented using **pytest-bdd**, which maps Gherkin steps to Python functions: +--- -```python -@given('a valid core dump file exists') -def create_dump_file(context): - # Implementation +## Running the Tests -@when('the upload process runs') -def run_upload(context): - # Implementation +```bash +# From the repository root inside the native-platform container: +sh run_l2.sh + +# Individual test file +pytest -v -s test/functional-tests/tests/test_archive_content.py -@then('the dump file is uploaded to the server') -def verify_upload(context): - # Assert expected behavior +# All tests with JSON report +pytest -v -s --json-report --json-report-summary \ + test/functional-tests/tests/ ``` -## Running Tests +The binary under test is located at +`c_sourcecode/src/crashupload` (built by `sh cov_build.sh --l2-test`). +The `CRASHUPLOAD_BINARY` environment variable overrides the path. -```bash -# Run all functional tests -pytest test/functional-tests/tests/ +--- -# Run specific feature -pytest test/functional-tests/tests/test_upload.py +## Test File Index -# Verbose output -pytest -v test/functional-tests/tests/ -``` +### Argument Parsing & Init + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_crashupload_arg_parsing.py` | TC-008 | Validates `argc < 3` exits with code 1 | +| `test_crashupload_failure_return.py` | FAIL-01/02 | `system_initialize` failure → exit 1; no-dumps → exit 0 | +| `test_reboot_and_log_scenario.py` | REBOOT-01, LOG-01 | Reboot flag skips upload; binary always emits log output | + +### Configuration & Path Selection + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_config_and_path.py` | TC-006, TC-007, TC-009 | Secure/normal path selection; minidump vs coredump mode | +| `test_config_checks_and_baseline.py` | TC-019–021, TC-023–026, TC-028 | Dump detection and platform metadata (MAC, model, SHA1) | +| `test_unsupported_devicetypes.py` | TC-004, TC-005, TC-017, TC-018 | Broadband/extender device type exit behaviour | +| `test_broadband_env.py` | TC-067 | Broadband mode produces no .tgz | + +### No Dumps / Prerequisites + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_no_dumps_exit.py` | NODMP-01..04 | Empty/missing/wrong-extension dump dir → exit 0 | +| `test_upload_deferral.py` | TC-035, TC-036, TC-037 | Uptime-based deferral (L2_TEST build reads `/opt/uptime`) | + +### Lock Mechanism + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_lock_and_exit.py` | TC-012, TC-013, TC-015, TC-085 | Second instance exits when lock is held | +| `test_lock_and_wait.py` | TC-011 (wait mode) | `wait_for_lock` argument waits for lock release | +| `test_lock_lifecycle.py` | TC-011, TC-014, TC-016 | Lock acquired, removed on clean exit, persists after SIGKILL | +| `test_signal_lock_cleanup.py` | SIG-01, SIG-02 | SIGTERM unlinks the process lock file | + +### Cleanup & Rate Limiting + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_cleanup_batch.py` | TC-041–047 | On-startup cleanup, stale archive removal, MAX_CORE_FILES | +| `test_ratelimit.py` | TC-049, TC-051, TC-055 | Rate-limit block path; deny-window; integer timestamp format | +| `test_ratelimit_allow.py` | TC-048, TC-050, TC-052, TC-053, TC-054 | Rate-limit allow path; coredump bypass; expired window | + +### Telemetry + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_t2_optout.py` | TC-038, TC-039, TC-040 | RFC/file opt-out suppresses upload on mediaclient | +| `test_telemetry.py` | TC-072, TC-073 | Process and container crash telemetry log-mapper paths | + +### Dump Scanning, Processing & Archiving + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_scanner_behaviour.py` | TC-057, TC-058, TC-059, TC-064 | Filename sanitisation and process-name extraction | +| `test_dump_processing.py` | TC-060, TC-071 | Pre-existing .tgz passthrough; zero-size dump handling | +| `test_file_ext_check.py` | TC-061, TC-062, TC-063 | Archive filename metadata fields, truncation, mpeos-main mtime | +| `test_archive_content.py` | TC-065, TC-066, TC-075, TC-078, TC-079, TC-080 | Archive creation and log-file bundling | + +### Upload + +| File | TC-IDs | Description | +|------|--------|-------------| +| `test_single_dump_upload.py` | TC-081 | End-to-end successful upload to mock S3 | +| `test_upload_retry.py` | TC-082, TC-083 | Retry succeeds on 3rd attempt; all retries exhausted | + +--- + +## Shared Utilities + +`testUtility.py` provides: +- `binary_path` fixture — resolves `CRASHUPLOAD_BINARY` env var or default build path +- `cleanup_pytest_cache` fixture — removes `.pytest_cache` after each test +- `create_dummy_dump()`, `stash_dir_dumps()`, `restore_stashed_dumps()` +- `hold_lock_and_release()`, `wait_for_path()` +- Constants: `SECURE_MINIDUMP_PATH`, `UPLOADED_CRASHES_DIR`, `DENY_UPLOADS_FILE`, … + +`conftest.py` provides: +- Session-level pass/fail counters +- `_TC_MAP` (test function name → TC-ID) used to emit a coverage report +- `pytest_sessionfinish` hook that prints a TC coverage summary + +--- -## Test Structure +## TC Coverage -- Each `.py` file implements steps for corresponding `.feature` files -- Test fixtures in `conftest.py` provide common setup/teardown -- Step definitions are reusable across multiple scenarios -- Tests run against actual compiled binaries for integration validation +- **Applicable TCs**: 66 / 85 +- **Implemented**: 66 / 66 (100%) +- **Not applicable** (shell-only / unimplemented stubs): 19 -See [features/](../features/) for test specifications in Gherkin format. +See [../../L2_TESTS.md](../../L2_TESTS.md) for the full applicability matrix. diff --git a/test/functional-tests/tests/test_ratelimit_allow.py b/test/functional-tests/tests/test_ratelimit_allow.py new file mode 100644 index 0000000..1871884 --- /dev/null +++ b/test/functional-tests/tests/test_ratelimit_allow.py @@ -0,0 +1,446 @@ +########################################################################## +# If not stated otherwise in this file or this component's LICENSE +# file the following copyright and licenses apply: +# +# Copyright 2025 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +""" +test_ratelimit_allow.py — Rate-limiting allow-path functional tests. + +Background +---------- +`ratelimit_check_unified()` in `ratelimit.c` enforces two checks in sequence: + + Step 1 — deny-window check (is_recovery_time_reached): + Reads `/tmp/.deny_dump_uploads_till`. + • If absent OR expired → ALLOW_UPLOAD + • If active (future timestamp) → STOP_UPLOAD for both minidump and coredump + + Step 2 — per-minidump count check (is_upload_limit_reached): + Reads `/tmp/.minidump_upload_timestamps`. + • File absent OR line count ≤ 10 → ALLOW_UPLOAD + • Line count > 10 AND first timestamp within RECOVERY_DELAY_SEC → STOP_UPLOAD + This check is ONLY performed when dump_type == DUMP_TYPE_MINIDUMP. + For DUMP_TYPE_COREDUMP the `else { status = ALLOW_UPLOAD; }` branch fires. + +When both checks return ALLOW_UPLOAD the function returns ALLOW_UPLOAD (1). +main.c does NOT call `remove_pending_dumps()` and proceeds to the upload loop. +The upload loop is bypassed by the reboot flag (is_box_rebooting → exit(0)). + +Observable when upload is NOT blocked: + • Binary exits 0 (via reboot-flag path, not rate-limit path). + • DENY_UPLOADS_FILE (/tmp/.deny_dump_uploads_till) is NOT created by the + rate-limiter (set_time(RECOVERY_TIME) is only called on RATELIMIT_BLOCK). + +TC-048 Upload count ≤ 10 → ALLOW_UPLOAD + ───────────────────────────────────────── + `/tmp/.minidump_upload_timestamps` is pre-populated with exactly 10 lines + (boundary condition: the rate-limiter should NOT trigger at 10 or fewer). + is_upload_limit_reached() returns ALLOW_UPLOAD → rate limit not triggered. + +TC-050 Minidump upload-count check is minidump-only + ───────────────────────────────────────────────────── + Even when `/tmp/.minidump_upload_timestamps` has 11 lines (which would block + a minidump run), a coredump run (argv[2]="1") skips the count check entirely + inside ratelimit_check_unified() and is allowed when no deny-window is active. +""" + +import os +import subprocess +import time +from pathlib import Path + +import pytest + +from testUtility import ( + cleanup_pytest_cache, + binary_path, + create_dummy_dump, + stash_dir_dumps, + restore_stashed_dumps, + DEFAULT_LOG_PATH, + CORE_LOG_FILE, + SECURE_MINIDUMP_PATH, + SECURE_COREDUMP_PATH, + MINIDUMP_LOCK_FILE, + COREDUMP_LOCK_FILE, + DENY_UPLOADS_FILE, + MINIDUMP_TIMESTAMPS_FILE, + RECOVERY_DELAY_SEC, + REBOOT_FLAG_FILE, + ON_STARTUP_CLEANED_UP_BASE, +) + +_ON_STARTUP_FLAG_MINI = f"{ON_STARTUP_CLEANED_UP_BASE}_0" +_ON_STARTUP_FLAG_CORE = f"{ON_STARTUP_CLEANED_UP_BASE}_1" + + +# --------------------------------------------------------------------------- +# Module-level helpers +# --------------------------------------------------------------------------- + +def _ensure_system_init_prereqs(): + """Create the log directory and core_log.txt required by system_initialize().""" + os.makedirs(DEFAULT_LOG_PATH, exist_ok=True) + Path(CORE_LOG_FILE).touch(exist_ok=True) + + +def _cleanup_tgz(directory: str) -> None: + """Remove .tgz files left by the binary in *directory*.""" + if os.path.isdir(directory): + for f in Path(directory).glob("*.tgz"): + f.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + +class TestRateLimitAllow: + """ + TC-048 and TC-050 — rate-limit allow paths. + + These tests verify that the binary is NOT blocked by the rate limiter and + proceeds normally (to the reboot-flag exit) when the count threshold is + not exceeded (TC-048) or when the dump type is coredump (TC-050). + """ + + def test_upload_allowed_when_count_at_or_below_limit( + self, binary_path, cleanup_pytest_cache + ): + """TC-048: timestamp file has exactly 10 lines → ALLOW_UPLOAD, no rate-limit block. + + `is_upload_limit_reached()` checks `line_cnt <= 10` — the boundary is + inclusive. 10 entries must NOT trigger a block. + + Setup: + • DENY_UPLOADS_FILE absent — deny-window check passes. + • MINIDUMP_TIMESTAMPS_FILE contains exactly 10 recent timestamps. + • One .dmp file planted in SECURE_MINIDUMP_PATH. + • REBOOT_FLAG_FILE set so the upload loop exits cleanly without + a network call. + + Assertions: + • exit(0) + • DENY_UPLOADS_FILE was NOT created by the rate limiter. + (set_time(DENY_UPLOADS_FILE, RECOVERY_TIME) is only called on + RATELIMIT_BLOCK, never on ALLOW_UPLOAD.) + """ + _ensure_system_init_prereqs() + os.makedirs(SECURE_MINIDUMP_PATH, exist_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + + # Ensure clean rate-limit state + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + + # 10 lines — boundary value: must NOT trigger block + now_ts = int(time.time()) + timestamps = "\n".join([str(now_ts - i * 10) for i in range(10)]) + "\n" + Path(MINIDUMP_TIMESTAMPS_FILE).write_text(timestamps) + + stashed = stash_dir_dumps(SECURE_MINIDUMP_PATH, ".dmp") + dump_path = create_dummy_dump(SECURE_MINIDUMP_PATH, "tc048_allow.dmp") + Path(REBOOT_FLAG_FILE).touch() + try: + result = subprocess.run( + [binary_path, "", "0", "secure"], + capture_output=True, text=True, timeout=60, + ) + assert result.returncode == 0, ( + f"TC-048: expected exit(0) when count ≤ 10, got {result.returncode}\n" + f"stdout={result.stdout}\nstderr={result.stderr}" + ) + # Rate limiter must NOT have fired — deny file must be absent + assert not os.path.exists(DENY_UPLOADS_FILE), ( + "TC-048: DENY_UPLOADS_FILE was created even though upload count " + f"was ≤ 10 — rate limiter should have returned ALLOW_UPLOAD, " + "not RATELIMIT_BLOCK" + ) + finally: + Path(dump_path).unlink(missing_ok=True) + _cleanup_tgz(SECURE_MINIDUMP_PATH) + restore_stashed_dumps(stashed) + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + Path(REBOOT_FLAG_FILE).unlink(missing_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + Path(_ON_STARTUP_FLAG_MINI).unlink(missing_ok=True) + + def test_coredump_not_rate_limited_by_minidump_counter( + self, binary_path, cleanup_pytest_cache + ): + """TC-050: coredump run bypasses the minidump count check → ALLOW_UPLOAD. + + `ratelimit_check_unified(DUMP_TYPE_COREDUMP)` reaches the `else` branch + and sets `status = ALLOW_UPLOAD` without reading MINIDUMP_TIMESTAMPS_FILE. + + A minidump run with the same precondition (11 entries, within window) + WOULD be blocked (TC-049 verifies this). This test confirms that + coredump ignores the minidump counter file itself. + + Setup: + • DENY_UPLOADS_FILE absent — deny-window check passes. + • MINIDUMP_TIMESTAMPS_FILE contains 11 recent timestamps (would block + a minidump run, but must be ignored for coredump). + • One `_core` file planted in SECURE_COREDUMP_PATH. + • REBOOT_FLAG_FILE set so the upload loop exits cleanly. + + Assertions: + • exit(0) + • DENY_UPLOADS_FILE was NOT created by the rate limiter. + (This case validates the minidump counter bypass when deny-window + is absent; active deny-window behavior is validated separately.) + """ + _ensure_system_init_prereqs() + os.makedirs(SECURE_COREDUMP_PATH, exist_ok=True) + Path(COREDUMP_LOCK_FILE).unlink(missing_ok=True) + + # Ensure clean deny-window state + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + + # 11 lines within window — sufficient to block a minidump run + now_ts = int(time.time()) + first_ts = now_ts - 60 # 1 minute ago, well within 600s window + lines = [str(first_ts)] + [str(now_ts)] * 10 # 11 total + Path(MINIDUMP_TIMESTAMPS_FILE).write_text("\n".join(lines) + "\n") + + stashed = stash_dir_dumps(SECURE_COREDUMP_PATH, "_core") + dump_path = create_dummy_dump(SECURE_COREDUMP_PATH, "proc_tc050_core.prog") + Path(REBOOT_FLAG_FILE).touch() + try: + result = subprocess.run( + [binary_path, "", "1", "secure"], # "1" → DUMP_TYPE_COREDUMP + capture_output=True, text=True, timeout=60, + ) + assert result.returncode == 0, ( + f"TC-050: expected exit(0) for coredump run ignoring minidump " + f"counter, got {result.returncode}\n" + f"stdout={result.stdout}\nstderr={result.stderr}" + ) + # In this no-deny-window setup, counter-based rate limiting must not + # fire for coredump mode. + assert not os.path.exists(DENY_UPLOADS_FILE), ( + "TC-050: DENY_UPLOADS_FILE was created for a coredump run — " + "ratelimit_check_unified(COREDUMP) should bypass the minidump " + "counter check when deny-window is absent" + ) + finally: + Path(dump_path).unlink(missing_ok=True) + _cleanup_tgz(SECURE_COREDUMP_PATH) + restore_stashed_dumps(stashed) + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + Path(REBOOT_FLAG_FILE).unlink(missing_ok=True) + Path(COREDUMP_LOCK_FILE).unlink(missing_ok=True) + Path(_ON_STARTUP_FLAG_CORE).unlink(missing_ok=True) + + def test_recovery_time_expired_unblocks_upload( + self, binary_path, cleanup_pytest_cache + ): + """TC-052: Expired deny-window timestamp → ALLOW_UPLOAD; deny file not refreshed. + + DENY_UPLOADS_FILE is pre-written with a timestamp 700 seconds in the past + (beyond the 600-second RECOVERY_DELAY_SEC window). + is_recovery_time_reached() reads the file, evaluates `now > deny_until` → ALLOW_UPLOAD. + + Because ALLOW_UPLOAD is returned, ratelimit_check_unified does NOT call + set_time(DENY_UPLOADS_FILE, RECOVERY_TIME) — that call only happens on RATELIMIT_BLOCK. + DENY_UPLOADS_FILE therefore retains its original past timestamp throughout the run. + + Note: REBOOT_FLAG_FILE is deliberately absent. is_box_rebooting() is checked + BEFORE ratelimit_check_unified() in main.c; with the reboot flag set the binary + would exit before reaching the ratelimit code and nothing would be proven. + The upload loop is reached and fails (no network in the test container) — the + binary exits non-zero. Exit code is NOT the primary assertion here. + + Primary assertion: + DENY_UPLOADS_FILE still contains the original past timestamp after the run + (i.e. set_time() was NOT called), proving is_recovery_time_reached() returned + ALLOW_UPLOAD for the expired timestamp — not STOP_UPLOAD. + """ + _ensure_system_init_prereqs() + os.makedirs(SECURE_MINIDUMP_PATH, exist_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + + # Past timestamp — 700 s ago, beyond the 600-second recovery window + past_ts = int(time.time()) - 700 + Path(DENY_UPLOADS_FILE).write_text(f"{past_ts}\n") + + # 10 timestamped entries so that the count check also returns ALLOW_UPLOAD + now_ts = int(time.time()) + Path(MINIDUMP_TIMESTAMPS_FILE).write_text( + "\n".join([str(now_ts - i * 10) for i in range(10)]) + "\n" + ) + + stashed = stash_dir_dumps(SECURE_MINIDUMP_PATH, ".dmp") + dump_path = create_dummy_dump(SECURE_MINIDUMP_PATH, "tc052_allow.dmp") + # No REBOOT_FLAG_FILE — must reach ratelimit_check_unified() + try: + subprocess.run( + [binary_path, "", "0", "secure"], + capture_output=True, text=True, timeout=60, + ) + # Exit code not asserted — upload fails in the test container (no network). + # The meaningful assertion is the deny-file content below. + assert os.path.exists(DENY_UPLOADS_FILE), ( + "TC-052: DENY_UPLOADS_FILE was unexpectedly removed during the run" + ) + recorded_ts = int(Path(DENY_UPLOADS_FILE).read_text().strip()) + now_after = int(time.time()) + assert recorded_ts <= now_after, ( + f"TC-052: DENY_UPLOADS_FILE was refreshed to a future timestamp " + f"({recorded_ts} > now {now_after}) — is_recovery_time_reached() " + "must have returned STOP_UPLOAD for the already-expired timestamp " + "instead of ALLOW_UPLOAD." + ) + assert recorded_ts == past_ts, ( + f"TC-052: DENY_UPLOADS_FILE timestamp changed from {past_ts} to " + f"{recorded_ts} — set_time(DENY_UPLOADS_FILE, RECOVERY_TIME) should " + "not be called on the ALLOW_UPLOAD path." + ) + finally: + Path(dump_path).unlink(missing_ok=True) + _cleanup_tgz(SECURE_MINIDUMP_PATH) + restore_stashed_dumps(stashed) + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + Path(REBOOT_FLAG_FILE).unlink(missing_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + Path(_ON_STARTUP_FLAG_MINI).unlink(missing_ok=True) + + def test_rate_limit_resets_after_recovery_period( + self, binary_path, cleanup_pytest_cache + ): + """TC-054: Timestamps file with expired window → counter reset (file unlinked). + + MINIDUMP_TIMESTAMPS_FILE is pre-written with 11 entries whose FIRST line + is a timestamp 700 seconds in the past (beyond RECOVERY_DELAY_SEC = 600 s). + + Inside is_upload_limit_reached(): + line_cnt = 11 > 10 + (now - first_crash_time) = 700 >= RECOVERY_DELAY_SEC (600) + → calls unlink(MINIDUMP_TIMESTAMPS_FILE) ← counter RESET + → returns ALLOW_UPLOAD + + Because the function returns ALLOW_UPLOAD, ratelimit_check_unified does NOT + call set_time(DENY_UPLOADS_FILE, RECOVERY_TIME) and DENY_UPLOADS_FILE is + never created. + + Note: REBOOT_FLAG_FILE is absent for the same reason as TC-052 — reboot + flag would shortcut past the ratelimit check. Upload fails (no network) + and exit code is not the primary assertion. + + Primary assertions: + 1. MINIDUMP_TIMESTAMPS_FILE was DELETED (by unlink() inside the C function), + proving the expired-window counter-reset path was taken. + 2. DENY_UPLOADS_FILE was NOT created, proving RATELIMIT_BLOCK was not returned. + """ + _ensure_system_init_prereqs() + os.makedirs(SECURE_MINIDUMP_PATH, exist_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + + # No deny window — is_recovery_time_reached() returns ALLOW immediately + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + + # 11 entries; first timestamp is 700 s ago → window expired → counter resets + first_ts = int(time.time()) - 700 + now_ts = int(time.time()) + lines = [str(first_ts)] + [str(now_ts)] * 10 # 11 total + Path(MINIDUMP_TIMESTAMPS_FILE).write_text("\n".join(lines) + "\n") + + stashed = stash_dir_dumps(SECURE_MINIDUMP_PATH, ".dmp") + dump_path = create_dummy_dump(SECURE_MINIDUMP_PATH, "tc054_reset.dmp") + # No REBOOT_FLAG_FILE — must reach ratelimit_check_unified() + try: + subprocess.run( + [binary_path, "", "0", "secure"], + capture_output=True, text=True, timeout=60, + ) + # Exit code not asserted — upload fails in the test container (no network). + assert not os.path.exists(MINIDUMP_TIMESTAMPS_FILE), ( + "TC-054: MINIDUMP_TIMESTAMPS_FILE still present after the run — " + "is_upload_limit_reached() should have called unlink() when the " + f"first timestamp was older than RECOVERY_DELAY_SEC ({RECOVERY_DELAY_SEC} s)." + ) + assert not os.path.exists(DENY_UPLOADS_FILE), ( + "TC-054: DENY_UPLOADS_FILE was created, indicating RATELIMIT_BLOCK " + "was returned — the expired-window reset path should return ALLOW_UPLOAD " + "without creating the deny file." + ) + finally: + Path(dump_path).unlink(missing_ok=True) + _cleanup_tgz(SECURE_MINIDUMP_PATH) + restore_stashed_dumps(stashed) + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + Path(REBOOT_FLAG_FILE).unlink(missing_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + Path(_ON_STARTUP_FLAG_MINI).unlink(missing_ok=True) + + # ------------------------------------------------------------------ + # TC-053: is_recovery_time_reached() returns ALLOW_UPLOAD when + # deny file is absent (no rate-limit block on clean state) + # ------------------------------------------------------------------ + def test_no_deny_file_allows_upload_to_proceed( + self, binary_path, cleanup_pytest_cache + ): + """ + TC-053 — is_recovery_time_reached() returns ALLOW_UPLOAD when deny file absent. + + When /tmp/.deny_dump_uploads_till does NOT exist: + - stat(deny_file) fails → is_recovery_time_reached() returns ALLOW_UPLOAD + - ratelimit_check_unified() calls is_upload_limit_reached() + - No timestamps file → line_cnt = 0 ≤ 10 → returns ALLOW_UPLOAD + - Binary falls through to the upload loop (fails — no network in test + container). set_time(DENY_UPLOADS_FILE, RECOVERY_TIME) is NEVER called + on the ALLOW_UPLOAD path. + + Primary assertion: + DENY_UPLOADS_FILE is NOT created after the run — proves the rate-limiter + did not fire and set_time(RECOVERY_TIME) was not called. + """ + _ensure_system_init_prereqs() + os.makedirs(SECURE_MINIDUMP_PATH, exist_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + + # Clean rate-limit state: no deny file, no timestamps file + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + + stashed = stash_dir_dumps(SECURE_MINIDUMP_PATH, ".dmp") + dump_path = create_dummy_dump(SECURE_MINIDUMP_PATH, "tc053_allow.dmp") + # No REBOOT_FLAG_FILE — must reach ratelimit_check_unified() + try: + subprocess.run( + [binary_path, "", "0", "secure"], + capture_output=True, text=True, timeout=60, + ) + # Exit code not asserted — upload fails in test container (no network). + assert not os.path.exists(DENY_UPLOADS_FILE), ( + "TC-053: DENY_UPLOADS_FILE was created even though no deny file " + "existed and no count limit was active — " + "is_recovery_time_reached() must return ALLOW_UPLOAD when the file " + "is absent; set_time(RECOVERY_TIME) must not be called on the " + "clean allow path" + ) + finally: + Path(dump_path).unlink(missing_ok=True) + _cleanup_tgz(SECURE_MINIDUMP_PATH) + restore_stashed_dumps(stashed) + Path(DENY_UPLOADS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_TIMESTAMPS_FILE).unlink(missing_ok=True) + Path(MINIDUMP_LOCK_FILE).unlink(missing_ok=True) + Path(_ON_STARTUP_FLAG_MINI).unlink(missing_ok=True) diff --git a/uploadDumps_TestCases.md b/test/uploadDumps_TestCases.md similarity index 100% rename from uploadDumps_TestCases.md rename to test/uploadDumps_TestCases.md diff --git a/unittest/ratelimit_gtest.cpp b/unittest/ratelimit_gtest.cpp index d136cb7..6ad8d8f 100644 --- a/unittest/ratelimit_gtest.cpp +++ b/unittest/ratelimit_gtest.cpp @@ -298,7 +298,7 @@ TEST_F(RateLimitTest, RatelimitCheckUnified_NoFiles_AllowUpload) { TEST_F(RateLimitTest, RatelimitCheckUnified_RecoveryNotReached_BlockUpload) { // Set recovery time in future - time_t future = time(NULL) + 600; + long future = (long)time(NULL) + 600L; CreateDenyFile("/tmp/.deny_dump_uploads_till", future); int ret = ratelimit_check_unified(DUMP_TYPE_MINIDUMP); @@ -317,12 +317,22 @@ TEST_F(RateLimitTest, RatelimitCheckUnified_UploadLimitReached_BlockUpload) { } TEST_F(RateLimitTest, RatelimitCheckUnified_CoredumpType_AllowUpload) { - // Coredump should always allow (no rate limiting) + // Coredump is allowed when no global deny-window is active. int ret = ratelimit_check_unified(DUMP_TYPE_COREDUMP); EXPECT_EQ(ret, ALLOW_UPLOAD); } +TEST_F(RateLimitTest, RatelimitCheckUnified_CoredumpType_RecoveryWindow_BlockUpload) { + // The recovery window is global, so coredumps are blocked while it is active. + long future = (long)time(NULL) + 600L; + CreateDenyFile("/tmp/.deny_dump_uploads_till", future); + + int ret = ratelimit_check_unified(DUMP_TYPE_COREDUMP); + + EXPECT_EQ(ret, RATELIMIT_BLOCK); +} + TEST_F(RateLimitTest, RatelimitCheckUnified_BothChecksPassed_AllowUpload) { // Recovery time passed and upload limit not reached CreateTimestampFile("/tmp/.minidump_upload_timestamps", 5);