diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c0443fd..f9d927e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,8 @@ repos: 01-introduction-horizontal-scalability/ecsdemo-.*/kubernetes/.*| 01-introduction-horizontal-scalability/ecsdemo-.*/copilot/.*| 08-cloud-networking-vpc/cloudformation/.*| - 09-distributed-file-systems/cloudformation\.yaml + 09-distributed-file-systems/cloudformation\.yaml| + 10-databases/.*/cloudformation\.yaml )$ - id: check-json - id: check-added-large-files diff --git a/.secrets.baseline b/.secrets.baseline index 37b3553..b131906 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -190,14 +190,14 @@ "filename": "06-security-https-oauth2-keycloak/README.md", "hashed_secret": "44b5ba9d33e8bcaefe1d26c4665f20a90fb84b5a", "is_verified": false, - "line_number": 665 + "line_number": 668 }, { "type": "Secret Keyword", "filename": "06-security-https-oauth2-keycloak/README.md", "hashed_secret": "2be88ca4242c76e8253ac62474851065032d6833", "is_verified": false, - "line_number": 699 + "line_number": 702 } ], "08-cloud-networking-vpc/LAB-MACOS.md": [ @@ -217,7 +217,43 @@ "is_verified": false, "line_number": 108 } + ], + "10-databases/mysql/docker-compose.yml": [ + { + "type": "Secret Keyword", + "filename": "10-databases/mysql/docker-compose.yml", + "hashed_secret": "e63a0ab2f8e7ecde486b42ebfec16d4434840af4", + "is_verified": false, + "line_number": 6 + } + ], + "10-databases/mysql/setup.sh": [ + { + "type": "Secret Keyword", + "filename": "10-databases/mysql/setup.sh", + "hashed_secret": "7667ad761bd61125e6eb84aafc63fe3bb914ab5b", + "is_verified": false, + "line_number": 81 + } + ], + "10-databases/visualizer/docker-compose.yml": [ + { + "type": "Secret Keyword", + "filename": "10-databases/visualizer/docker-compose.yml", + "hashed_secret": "e63a0ab2f8e7ecde486b42ebfec16d4434840af4", + "is_verified": false, + "line_number": 8 + } + ], + "10-databases/visualizer/setup.sh": [ + { + "type": "Secret Keyword", + "filename": "10-databases/visualizer/setup.sh", + "hashed_secret": "7667ad761bd61125e6eb84aafc63fe3bb914ab5b", + "is_verified": false, + "line_number": 54 + } ] }, - "generated_at": "2026-03-28T05:23:05Z" + "generated_at": "2026-04-14T01:35:11Z" } diff --git a/10-databases/LAB.md b/10-databases/LAB.md new file mode 100644 index 0000000..e4c9da4 --- /dev/null +++ b/10-databases/LAB.md @@ -0,0 +1,63 @@ +# Lab 10: Database Scalability + +## Main Lab (Interactive Visualizer) + +The primary lab for this module is an interactive web visualizer +connected to a live MySQL primary-replica cluster. It covers three +scalability mechanisms through animated diagrams, real SQL execution, +and a built-in SQL console. + +| Lab | What It Covers | Time | +| --- | --- | --- | +| [Interactive Visualizer](visualizer/LAB-VISUALIZER.md) | Replication, ACID transactions, indexing -- all via browser | ~45 min | + +```bash +cd 10-databases/visualizer +./setup.sh +# Open http://localhost:8081 +``` + +## Optional Labs (CLI-based, deeper dive) + +For students who want to go deeper with other database paradigms, +three optional CLI labs use the same university enrollment data model: + +| Lab | Database | Scalability Mechanisms | Time | +| --- | --- | --- | --- | +| [10A: MySQL](mysql/LAB-MYSQL.md) | MySQL 8 (Relational) | GTID replication, ACID transactions, indexing | ~30 min | +| [10B: MongoDB](mongodb/LAB-MONGODB.md) | MongoDB 7 (Document) | Replica set, read/write concerns, denormalization | ~30 min | +| [10C: Cassandra](cassandra/LAB-CASSANDRA.md) | Cassandra 4.1 (Wide-column) | Multi-node ring, tunable consistency, partition keys | ~30 min | + +## Scalability Mechanisms Compared + +| Mechanism | MySQL | MongoDB | Cassandra | +| --- | --- | --- | --- | +| **Replication** | Primary + replica (GTID) | 3-node replica set | 3-node ring (RF=3) | +| **Failover** | Manual promotion | Automatic election | No single point of failure | +| **Consistency** | ACID transactions | Tunable write/read concern | Tunable CL (ONE/QUORUM/ALL) | +| **Schema** | Rigid, normalized | Flexible, denormalized | Query-driven, partition keys | +| **Scaling reads** | Add read replicas | Read from secondaries | Read from any node | +| **Scaling writes** | Vertical only | Sharding (manual) | Add nodes to ring | + +## Shared Data Model + +All labs use the same university enrollment scenario: + +- **Students** -- 10 students with name, email, major +- **Courses** -- 4 courses with code, title, capacity +- **Enrollments** -- student-to-course relationships + +This lets you compare how the same data is modeled differently in +each paradigm (tables with foreign keys vs embedded documents vs +partition-key-driven tables). + +## Environment Options + +Each lab supports two environments: + +| Environment | What You Need | Setup | +| --- | --- | --- | +| **Local** | Docker Desktop | `./setup.sh` in the lab directory | +| **EC2** | Browser + SSH | Upload `cloudformation.yaml` via AWS Console | + +Run one lab at a time to avoid port conflicts and resource contention. diff --git a/10-databases/README.md b/10-databases/README.md index f65f6c8..b907520 100644 --- a/10-databases/README.md +++ b/10-databases/README.md @@ -1,44 +1,270 @@ -# Module 10 — Databases +# Database Scalability: Replication, Consistency, and Indexing -## Topics Covered +![MySQL](https://img.shields.io/badge/MySQL-%234479A1.svg?style=for-the-badge&logo=mysql&logoColor=white) +![MongoDB](https://img.shields.io/badge/MongoDB-%2347A248.svg?style=for-the-badge&logo=mongodb&logoColor=white) +![Cassandra](https://img.shields.io/badge/Cassandra-%231287B1.svg?style=for-the-badge&logo=apachecassandra&logoColor=white) +![Docker](https://img.shields.io/badge/Docker-%232496ED.svg?style=for-the-badge&logo=docker&logoColor=white) -- DBMS responsibilities and managed services -- SQL vs NoSQL tradeoffs -- CAP Theorem and consistency models -- Data storage types (relational, document, key-value, graph, columnar, time series) -- ACID vs BASE semantics -- Database scaling strategies (indexing, materialized views, vertical scaling, - caching, replication, sharding, denormalization) -- Amazon RDS (6 engines, Multi-AZ, read replicas) -- Amazon DynamoDB (keys, indexes, partition key flow, read consistency) -- Architecture patterns (three-tier, serverless) +## Overview + +This hands-on lab explores three fundamental database scalability +mechanisms: replication (distributing reads across nodes), ACID +transactions (guaranteeing consistency under concurrency), and indexing +(optimizing query performance on large tables). Students interact with +a live MySQL primary-replica cluster through an interactive web +visualizer that animates real SQL operations, measures latency, and +includes a built-in SQL console. Optional CLI labs extend the same +concepts to MongoDB (document store) and Cassandra (wide-column store). + +## Lab Instructions + +The main lab runs in the browser. Follow the step-by-step instructions +in **[visualizer/LAB-VISUALIZER.md](visualizer/LAB-VISUALIZER.md)**. + +| Environment | Requirements | Setup | +| --- | --- | --- | +| **Local** (Docker Desktop) | Docker Desktop + browser | `cd visualizer && ./setup.sh` | +| **EC2** (AWS Academy) | Browser only | Upload `visualizer/cloudformation.yaml` via AWS Console | + +For optional CLI-based labs with MongoDB and Cassandra, see +[LAB.md](LAB.md). + +## Learning Objectives + +- Configure and verify MySQL GTID-based primary-replica replication +- Observe replication lag and understand how read replicas scale reads +- Execute ACID transactions and see atomicity in action (commit vs + rollback) +- Use EXPLAIN to compare query execution plans with and without indexes +- Run SQL directly against primary and replica nodes via the built-in + console +- Compare how MySQL, MongoDB, and Cassandra each implement replication, + consistency, and schema design (optional labs) + +## Prerequisites + +- **Docker Desktop** installed and running (includes Docker Compose) +- A web browser (Chrome, Firefox, or Safari) +- Basic SQL knowledge (SELECT, INSERT, UPDATE) +- No cloud account required for the main lab + +## Architecture + +The visualizer connects to a live MySQL primary-replica cluster. Every +button click and console query executes real SQL against real databases. + +```mermaid +graph LR + subgraph Docker ["Docker Compose Network"] + subgraph Viz ["Visualizer"] + WEB["Browser
(localhost:8081)"] + API["Python API Bridge
(server.py)"] + end + subgraph DB ["MySQL Cluster"] + PRIMARY[("Primary
port 3306
read-write")] + REPLICA[("Replica
port 3307
read-only")] + end + end + + WEB -->|"HTTP"| API + API -->|"INSERT / UPDATE"| PRIMARY + API -->|"SELECT / SHOW STATUS"| REPLICA + PRIMARY -.->|"binlog
replication"| REPLICA +``` + +**Data flow:** + +1. The browser sends HTTP requests to the Python API bridge +1. The API executes real SQL on the primary (writes) or replica (reads) +1. The primary replicates changes to the replica via GTID-based binlog +1. The browser animates the data flow with measured latency + +## Lab Structure + +```text +10-databases/ +├── README.md # This file (lab overview) +├── LAB.md # Full lab index with comparison table +├── visualizer/ # Main lab (interactive browser) +│ ├── LAB-VISUALIZER.md # Step-by-step instructions (5 tasks) +│ ├── docker-compose.yml # MySQL primary + replica + visualizer +│ ├── setup.sh / cleanup.sh # Start and stop environment +│ ├── Dockerfile # Python API bridge image +│ ├── server.py # API bridge (proxies SQL to MySQL) +│ ├── index.html # Interactive UI with SVG diagram +│ ├── app.js # Animation engine and SQL console +│ └── style.css # Dark theme styling +├── mysql/ # Optional: MySQL CLI lab +│ ├── LAB-MYSQL.md # 4 tasks (replication, ACID, indexing) +│ ├── docker-compose.yml # MySQL primary + replica +│ ├── cloudformation.yaml # EC2 deployment template +│ ├── setup.sh / cleanup.sh +│ └── init/primary-init.sql # Schema and seed data +├── mongodb/ # Optional: MongoDB CLI lab +│ ├── LAB-MONGODB.md # 4 tasks (replica set, consistency) +│ ├── docker-compose.yml # 3-node replica set +│ ├── cloudformation.yaml +│ ├── setup.sh / cleanup.sh +│ └── init/rs-init.js # Replica set initialization +├── cassandra/ # Optional: Cassandra CLI lab +│ ├── LAB-CASSANDRA.md # 4 tasks (ring, consistency levels) +│ ├── docker-compose.yml # 3-node cluster (RF=3) +│ ├── cloudformation.yaml +│ ├── setup.sh / cleanup.sh +│ └── init/schema.cql # Keyspace and table creation +└── presentation/ # Module slide deck + └── index.html +``` + +## Quick Start + +```bash +cd visualizer +./setup.sh +``` + +Open [http://localhost:8081](http://localhost:8081). The visualizer +has three tabs (Replication, Consistency, Schema & Indexing) and a +SQL console at the bottom. Follow +[LAB-VISUALIZER.md](visualizer/LAB-VISUALIZER.md) for the guided +walkthrough. + +## Tasks Overview + +### Main Lab (Interactive Visualizer, ~45 min) + +| Task | Topic | What You Do | +| --- | --- | --- | +| 1. Explore the Environment | Setup | Inspect schema, verify replication, use SQL console | +| 2. Replication | Scaling reads | Write to primary, read from replica, observe lag | +| 3. ACID Transactions | Consistency | Transfer enrollment atomically, trigger rollback | +| 4. Indexing | Query optimization | EXPLAIN with/without index, compare rows scanned | +| 5. Free Exploration | SQL console | Run arbitrary queries, compare primary vs replica | + +### Optional CLI Labs (~30 min each) + +| Lab | Database | Scalability Mechanisms | +| --- | --- | --- | +| [10A: MySQL](mysql/LAB-MYSQL.md) | MySQL 8 | GTID replication, ACID, indexing | +| [10B: MongoDB](mongodb/LAB-MONGODB.md) | MongoDB 7 | Replica set, read/write concerns, denormalization | +| [10C: Cassandra](cassandra/LAB-CASSANDRA.md) | Cassandra 4.1 | Multi-node ring, tunable consistency, partition keys | + +## Cleanup + +```bash +cd visualizer +./cleanup.sh +``` + +## Troubleshooting + +| Issue | Cause | Fix | +| --- | --- | --- | +| Port 8081 in use | Another service running | Change port in `visualizer/docker-compose.yml` | +| SQL Console shows error | Blocked SQL command | Some DDL commands are blocked for safety | +| Replica shows `--` in sidebar | Replication not configured | Re-run `./setup.sh` | +| EXPLAIN shows `rows: 1` | Table stats stale | Run `ANALYZE TABLE access_log;` in the console | +| Animation stuck | Previous operation running | Wait for it to finish or refresh the page | +| Port 3306 in use | Local MySQL running | Stop local MySQL or change port in docker-compose | +| Cassandra OOM crash | Not enough Docker memory | Allocate at least 4 GB RAM in Docker Desktop | + +## Key Concepts + +| Concept | Description | +| --- | --- | +| **Replication** | Copying data from a primary to replicas via binary log | +| **Read replica** | A read-only copy that offloads SELECT queries from the primary | +| **Replication lag** | Delay between a write on primary and its appearance on replica | +| **ACID** | Atomicity, Consistency, Isolation, Durability -- transaction guarantees | +| **Atomicity** | A transaction either fully succeeds (COMMIT) or fully fails (ROLLBACK) | +| **BASE** | Basically Available, Soft state, Eventually consistent -- NoSQL trade-off | +| **Full table scan** | MySQL reads every row to find matches -- slow on large tables | +| **Composite index** | An index on multiple columns for multi-column WHERE clauses | +| **EXPLAIN** | Shows MySQL's query plan: index used, rows examined | +| **Read-write split** | Send writes to primary, reads to replicas | +| **Partition key** | Determines data distribution across nodes (Cassandra, DynamoDB) | +| **Write concern** | How many nodes must acknowledge a write (MongoDB) | +| **Consistency level** | How many replicas must respond for a read/write (Cassandra) | ## Presentation Interactive reveal.js presentation in `presentation/index.html`. See `shared/presentation/README.md` for the template API reference. -### Quick Start - ```bash -# Open locally (macOS) open presentation/index.html - -# Or serve with any HTTP server -python3 -m http.server 8080 -# Then visit http://localhost:8080/presentation/ ``` -### Features +31 slides with animated SVG diagrams, bilingual EN/ES toggle, +dark/light mode, interactive expandable cards, and official AWS +architecture icons. + +## How This Relates to Scalable Systems Design + +**Replication is the most common first step in scaling a relational +database.** Adding read replicas distributes SELECT queries across +multiple nodes, but all writes still go to one primary. At companies +like Facebook, MySQL read replicas serve billions of reads per day +while a single primary cluster handles writes. When even the primary +becomes a bottleneck, sharding distributes writes -- but that is a +fundamentally harder problem. + +**ACID transactions prevent data corruption under concurrency.** When +two users try to book the last seat on a flight simultaneously, only +one succeeds. Without atomicity, a transfer that debits one account +but fails to credit another causes money to vanish. ACID is +non-negotiable for banking, inventory, and healthcare systems. NoSQL +databases trade ACID for BASE (Basically Available, Soft state, +Eventually consistent) to achieve higher throughput and availability. + +**Indexing is the highest-impact optimization for query performance.** +A single composite index can turn a 10,000-row scan into a 24-row +lookup -- a 400x improvement with no application code changes. But +indexes are not free: they consume storage, slow down writes, and must +match your query patterns. Over-indexing is as harmful as +under-indexing. + +**Connection to earlier labs:** The caching patterns from Lab 11 sit +in front of databases -- a Redis cache with 90% hit rate reduces +database load by 10x. The load balancing from Lab 03 distributes +application traffic, but read-write splitting requires the load +balancer to route based on query type. The security patterns from +Lab 06 apply to database access control -- connection credentials, +network isolation (Lab 08 VPC), and encryption at rest. + +## Conclusions + +After completing this lab, you should take away these lessons: + +1. **Replication scales reads, not writes.** Adding replicas distributes + SELECT queries across multiple nodes, but all writes still go to one + primary. This is the most common first step in scaling a database. + +2. **ACID prevents data corruption under concurrency.** The enrollment + transfer either fully succeeds or fully rolls back. Without + atomicity, partial failures leave the database in an inconsistent + state. + +3. **Indexes are the highest-impact optimization.** A composite index + turned a 10,000-row scan into a 24-row lookup. But indexes slow + down writes and consume storage -- the right index depends on your + query patterns. -- 31 slides with animated SVG diagrams (ByteByteGo-style) -- Bilingual EN/ES with toggle -- Dark/light mode with persistence -- Interactive expandable cards on every slide -- Official AWS architecture icons -- Click-to-highlight architecture diagrams -- Particle canvas background +4. **Different databases make different trade-offs.** MySQL provides + ACID with manual failover. MongoDB provides automatic failover with + tunable consistency. Cassandra provides zero-downtime writes with + tunable consistency levels. No universally best database exists -- + only the right one for your requirements. -## Lab +## Next Steps -Lab content to be defined. +- [Module 11 -- Caching](../11-caching/) -- learn how Redis reduces + database load through caching patterns +- [Module 12 -- Proxies](../12-proxies/) -- explore how reverse proxies + complement database read-write splitting +- [MySQL Documentation](https://dev.mysql.com/doc/refman/8.4/en/) -- + deep dive into replication, InnoDB, and query optimization +- [MongoDB Manual](https://www.mongodb.com/docs/manual/) -- replica + sets, read preferences, and aggregation pipeline +- [Cassandra Documentation](https://cassandra.apache.org/doc/latest/) + -- ring architecture, consistency levels, and data modeling diff --git a/10-databases/cassandra/LAB-CASSANDRA.md b/10-databases/cassandra/LAB-CASSANDRA.md new file mode 100644 index 0000000..a7516d5 --- /dev/null +++ b/10-databases/cassandra/LAB-CASSANDRA.md @@ -0,0 +1,382 @@ +# Lab 10C: Cassandra Multi-node, Consistency, and Partition Keys + +![Cassandra](https://img.shields.io/badge/Cassandra-%231287B1.svg?style=for-the-badge&logo=apachecassandra&logoColor=white) +![Docker](https://img.shields.io/badge/Docker-%232496ED.svg?style=for-the-badge&logo=docker&logoColor=white) + +## Overview + +This lab explores three database scalability mechanisms using Apache +Cassandra: multi-node cluster replication with fault tolerance, tunable +consistency levels (ONE, QUORUM, ALL), and partition key design for +distributed data. Students work with a university enrollment dataset +across a three-node Cassandra ring with replication factor 3. + +## Learning Objectives + +- Deploy and verify a 3-node Cassandra cluster with RF=3 +- Test fault tolerance by stopping a node and verifying reads still work +- Compare consistency levels and observe write failures when nodes are down +- Design effective partition keys and understand the impact of bad + partition key choices + +## Prerequisites + +- **Docker Desktop** installed and running (allocate at least 4 GB RAM) +- Basic SQL familiarity (CQL is similar to SQL) +- No cloud account required (Option A) or AWS Academy credentials + (Option B) + +## Choose Your Environment + +| Environment | What You Need | Setup | +| --- | --- | --- | +| **Option A: Local** | Docker Desktop + terminal | `./setup.sh` | +| **Option B: EC2** | Browser + SSH client | Upload `cloudformation.yaml` via AWS Console | + +### Option A: Local Setup + +```bash +cd 10-databases/cassandra +chmod +x setup.sh cleanup.sh +./setup.sh +``` + +Setup takes 2-4 minutes (Cassandra nodes join sequentially). +Then skip to **Task 1** below. + +### Option B: EC2 Setup (AWS Academy) + +1. Download `cloudformation.yaml` from this directory +2. In the AWS Console, go to **CloudFormation** > **Create stack** +3. Upload the template, name it `lab10c-cassandra`, click **Submit** +4. Wait ~5 minutes (Cassandra takes longer to start) +5. Find the **PublicIP** in Outputs, SSH in: + +```bash +chmod 400 labsuser.pem +ssh -i labsuser.pem ec2-user@YOUR_PUBLIC_IP +ls ~/LAB_READY +cd ~/system-design-course/10-databases/cassandra +``` + +--- + +## Task 1: Verify the Cluster + +### Step 1.1: Check cluster status + +```bash +docker exec cass1 nodetool status +``` + +Expected output shows 3 nodes with status `UN` (Up/Normal): + +```text +Datacenter: dc1 +=============== +Status=Up/Down +|/ State=Normal/Leaving/Joining/Moving +-- Address Load Tokens ... State ... Rack +UN 172.x.x.x xxx KiB 16 ... Normal ... rack1 +UN 172.x.x.x xxx KiB 16 ... Normal ... rack2 +UN 172.x.x.x xxx KiB 16 ... Normal ... rack3 +``` + +### Step 1.2: Verify seed data + +```bash +docker exec cass1 cqlsh -e " +USE university; +SELECT COUNT(*) FROM students; +SELECT COUNT(*) FROM courses; +" +``` + +Expected: 10 students, 4 courses. + +### Step 1.3: Query students by partition key (major) + +```bash +docker exec cass1 cqlsh -e " +USE university; +SELECT name, email FROM students WHERE major = 'Computer Science'; +" +``` + +This query is efficient because `major` is the partition key. + +> **Question:** Why does Cassandra require you to query by partition key? +> +> **Hint:** Data is distributed across nodes based on the partition key +> hash. Without it, Cassandra must scan all nodes. + +--- + +## Task 2: Replication and Fault Tolerance + +### Step 2.1: Insert data on cass1 + +```bash +docker exec cass1 cqlsh -e " +USE university; +INSERT INTO courses (code, title, capacity, enrolled) +VALUES ('ENG101', 'English Composition', 40, 0); +" +``` + +### Step 2.2: Read from cass2 + +```bash +docker exec cass2 cqlsh -e " +USE university; +SELECT * FROM courses WHERE code = 'ENG101'; +" +``` + +The data is available on cass2 because RF=3 replicates to all nodes. + +### Step 2.3: Stop a node + +```bash +docker stop cass3 +``` + +### Step 2.4: Verify reads still work + +```bash +docker exec cass1 cqlsh -e " +USE university; +CONSISTENCY ONE; +SELECT * FROM courses WHERE code = 'ENG101'; +" +``` + +The query succeeds because CL=ONE only needs one replica to respond, +and two nodes are still running. + +### Step 2.5: Verify writes still work + +```bash +docker exec cass1 cqlsh -e " +USE university; +CONSISTENCY ONE; +INSERT INTO courses (code, title, capacity, enrolled) +VALUES ('ART101', 'Art History', 20, 0); +SELECT * FROM courses WHERE code = 'ART101'; +" +``` + +> **Question:** With RF=3 and one node down, what is the maximum +> consistency level that still allows reads and writes? +> +> **Hint:** QUORUM requires `(RF/2) + 1` nodes. With RF=3, that is 2. + +--- + +## Task 3: Tunable Consistency Levels + +### Step 3.1: Read with CL=QUORUM (2 of 3 nodes must respond) + +```bash +docker exec cass1 cqlsh -e " +USE university; +CONSISTENCY QUORUM; +SELECT * FROM courses WHERE code = 'CS101'; +" +``` + +This works because 2 of 3 nodes are up (cass3 is still stopped). + +### Step 3.2: Attempt CL=ALL with a node down + +```bash +docker exec cass1 cqlsh -e " +USE university; +CONSISTENCY ALL; +SELECT * FROM courses WHERE code = 'CS101'; +" +``` + +Expected error: + +```text +NoHostAvailable: +``` + +CL=ALL requires all 3 replicas to respond, but cass3 is down. + +### Step 3.3: Restart the stopped node + +```bash +docker start cass3 +``` + +Wait about 30 seconds for the node to rejoin, then retry: + +```bash +docker exec cass1 cqlsh -e " +USE university; +CONSISTENCY ALL; +SELECT * FROM courses WHERE code = 'CS101'; +" +``` + +Now CL=ALL succeeds because all 3 nodes are up. + +### Step 3.4: Verify the stopped node caught up + +Check that data written while cass3 was down is now present: + +```bash +docker exec cass3 cqlsh -e " +USE university; +CONSISTENCY ONE; +SELECT * FROM courses WHERE code = 'ART101'; +" +``` + +Cassandra's hinted handoff delivered the missed writes. + +> **Question:** When would you use CL=ALL in production? +> +> **Hint:** Almost never -- it sacrifices availability. CL=QUORUM +> gives strong consistency while tolerating one node failure. + +--- + +## Task 4: Partition Key Design + +### Step 4.1: Insert data with a BAD partition key + +The `access_log_bad` table uses `log_type` as partition key. All rows +go to the same partition: + +```bash +docker exec cass1 cqlsh -e " +USE university; +INSERT INTO access_log_bad (log_type, accessed_at, student_name, resource) +VALUES ('web', toTimestamp(now()), 'Alice', 'resource-1'); +INSERT INTO access_log_bad (log_type, accessed_at, student_name, resource) +VALUES ('web', toTimestamp(now()), 'Bob', 'resource-2'); +INSERT INTO access_log_bad (log_type, accessed_at, student_name, resource) +VALUES ('web', toTimestamp(now()), 'Carol', 'resource-3'); +" +``` + +### Step 4.2: Insert data with a GOOD partition key + +The `access_log_good` table uses `day` as partition key. Data spreads +across days: + +```bash +docker exec cass1 cqlsh -e " +USE university; +INSERT INTO access_log_good (day, accessed_at, student_name, resource) +VALUES ('2025-01-15', toTimestamp(now()), 'Alice', 'resource-1'); +INSERT INTO access_log_good (day, accessed_at, student_name, resource) +VALUES ('2025-01-16', toTimestamp(now()), 'Bob', 'resource-2'); +INSERT INTO access_log_good (day, accessed_at, student_name, resource) +VALUES ('2025-01-17', toTimestamp(now()), 'Carol', 'resource-3'); +" +``` + +### Step 4.3: Compare partition distribution + +```bash +docker exec cass1 nodetool tablestats university.access_log_bad 2>/dev/null \ + | grep -E "Number of partitions|Compacted partition" +docker exec cass1 nodetool tablestats university.access_log_good 2>/dev/null \ + | grep -E "Number of partitions|Compacted partition" +``` + +The bad table has 1 partition (all data on one node = hotspot). +The good table has 3 partitions (data distributed across nodes). + +### Step 4.4: Query efficiency + +Querying by partition key is fast (single-node lookup): + +```bash +docker exec cass1 cqlsh -e " +USE university; +SELECT * FROM access_log_good WHERE day = '2025-01-15'; +" +``` + +Querying without partition key requires a full cluster scan: + +```bash +docker exec cass1 cqlsh -e " +USE university; +SELECT * FROM access_log_good WHERE student_name = 'Alice' +ALLOW FILTERING; +" +``` + +The `ALLOW FILTERING` keyword is a red flag -- it means Cassandra must +scan all partitions. In production with millions of rows, this query +would be unacceptably slow. + +> **Question:** Why is a single-value partition key (like `log_type = +> 'web'`) bad at scale? +> +> **Hint:** All data lands on the same set of replica nodes. Those +> nodes become hotspots while others sit idle. + +--- + +## Cleanup + +```bash +./cleanup.sh +``` + +For EC2, also delete the CloudFormation stack in the AWS Console. + +## Troubleshooting + +| Issue | Cause | Fix | +| --- | --- | --- | +| `nodetool status` shows DN | Node still starting | Wait 30-60 seconds, retry | +| `NoHostAvailable` error | Not enough replicas for CL | Lower consistency level or start the stopped node | +| `Cannot achieve consistency` | Node down + CL too high | Use CL=ONE or restart the node | +| Cassandra OOM crash | Not enough Docker memory | Allocate at least 4 GB RAM in Docker Desktop | +| `ALLOW FILTERING` timeout | Full scan on large data | Design queries around partition keys instead | + +## Key Concepts + +| Concept | Description | +| --- | --- | +| **Ring topology** | Cassandra distributes data across nodes in a hash ring | +| **Replication Factor** | Number of copies of each piece of data (RF=3 means 3 copies) | +| **Consistency Level** | How many replicas must respond for a read/write to succeed | +| **ONE** | Only 1 replica needed -- fastest, lowest consistency | +| **QUORUM** | Majority of replicas needed -- strong consistency, tolerates failures | +| **ALL** | All replicas needed -- strongest consistency, zero fault tolerance | +| **Partition Key** | Determines which node stores the data -- critical for performance | +| **Hinted Handoff** | Mechanism to deliver missed writes to a node that was temporarily down | + +## Conclusions + +1. **Cassandra is designed for fault tolerance.** With RF=3, losing + one node has zero impact on reads and writes at CL=ONE or QUORUM. + This is fundamentally different from MySQL where losing the primary + halts all writes. + +2. **Consistency is a dial, not a switch.** CL=ONE is fast but risks + stale reads. CL=QUORUM is the sweet spot for most production + workloads. CL=ALL gives maximum consistency but any single node + failure makes the entire system unavailable. + +3. **Partition key design determines everything.** A bad partition key + creates hotspots and forces expensive full-cluster scans. A good + partition key distributes data evenly and makes queries hit a + single node. In Cassandra, you design the schema around your + queries, not the other way around. + +## Next Steps + +- [Lab 10A -- MySQL](../mysql/LAB-MYSQL.md) -- compare with relational + replication and ACID transactions +- [Lab 10B -- MongoDB](../mongodb/LAB-MONGODB.md) -- compare with + document store replication and consistency diff --git a/10-databases/cassandra/cleanup.sh b/10-databases/cassandra/cleanup.sh new file mode 100755 index 0000000..9edaa93 --- /dev/null +++ b/10-databases/cassandra/cleanup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Cleaning up Cassandra lab environment ===" +echo "" + +echo "Stopping containers..." +docker compose down -v --remove-orphans 2>/dev/null || true + +echo "" +echo "=== Cleanup complete ===" diff --git a/10-databases/cassandra/cloudformation.yaml b/10-databases/cassandra/cloudformation.yaml new file mode 100644 index 0000000..54871ba --- /dev/null +++ b/10-databases/cassandra/cloudformation.yaml @@ -0,0 +1,78 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + Lab 10C - Cassandra Multi-node, Consistency, and Partition Keys. + Launches a t3.large EC2 instance with Docker and Docker Compose + pre-installed. Clones the course repository and runs the Cassandra lab + setup automatically. SSH access enabled. Uses t3.large because + Cassandra 3-node clusters need more memory than t3.medium. + +Parameters: + LatestAmiId: + Description: Amazon Linux 2023 AMI (auto-resolved via SSM) + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 + +Resources: + LabSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Lab 10C Cassandra - SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + Description: SSH access + + LabInstance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t3.large + ImageId: !Ref LatestAmiId + KeyName: vockey + SecurityGroupIds: + - !GetAtt LabSecurityGroup.GroupId + Tags: + - Key: Name + Value: lab10c-cassandra + UserData: + Fn::Base64: | + #!/bin/bash + set -ex + + dnf update -y + dnf install -y docker git + systemctl enable docker + systemctl start docker + usermod -aG docker ec2-user + + ARCH=$(uname -m) + mkdir -p /usr/local/lib/docker/cli-plugins + + curl -fsSL \ + "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${ARCH}" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + + su - ec2-user -c ' + git clone https://github.com/gamaware/system-design-course.git + cd system-design-course/10-databases/cassandra + chmod +x setup.sh cleanup.sh + ./setup.sh + ' + + touch /home/ec2-user/LAB_READY + +Outputs: + InstanceId: + Description: EC2 instance ID + Value: !Ref LabInstance + + PublicIP: + Description: Public IP address (use for SSH) + Value: !GetAtt LabInstance.PublicIp + + SSHCommand: + Description: SSH command (use the labsuser.pem key from AWS Details) + Value: !Sub >- + ssh -i labsuser.pem ec2-user@${LabInstance.PublicIp} diff --git a/10-databases/cassandra/docker-compose.yml b/10-databases/cassandra/docker-compose.yml new file mode 100644 index 0000000..476dbe5 --- /dev/null +++ b/10-databases/cassandra/docker-compose.yml @@ -0,0 +1,67 @@ +services: + cass1: + image: cassandra:4.1 + container_name: cass1 + ports: + - "9042:9042" + environment: + CASSANDRA_CLUSTER_NAME: university-cluster + CASSANDRA_DC: dc1 + CASSANDRA_RACK: rack1 + CASSANDRA_SEEDS: cass1 + CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch + volumes: + - cass1-data:/var/lib/cassandra + healthcheck: + test: ["CMD-SHELL", "cqlsh -e 'SELECT now() FROM system.local' || exit 1"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 60s + + cass2: + image: cassandra:4.1 + container_name: cass2 + environment: + CASSANDRA_CLUSTER_NAME: university-cluster + CASSANDRA_DC: dc1 + CASSANDRA_RACK: rack2 + CASSANDRA_SEEDS: cass1 + CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch + volumes: + - cass2-data:/var/lib/cassandra + depends_on: + cass1: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "cqlsh -e 'SELECT now() FROM system.local' || exit 1"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 60s + + cass3: + image: cassandra:4.1 + container_name: cass3 + environment: + CASSANDRA_CLUSTER_NAME: university-cluster + CASSANDRA_DC: dc1 + CASSANDRA_RACK: rack3 + CASSANDRA_SEEDS: cass1 + CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch + volumes: + - cass3-data:/var/lib/cassandra + depends_on: + cass2: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "cqlsh -e 'SELECT now() FROM system.local' || exit 1"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 60s + +volumes: + cass1-data: + cass2-data: + cass3-data: diff --git a/10-databases/cassandra/init/schema.cql b/10-databases/cassandra/init/schema.cql new file mode 100644 index 0000000..01ddde5 --- /dev/null +++ b/10-databases/cassandra/init/schema.cql @@ -0,0 +1,81 @@ +-- Create keyspace with replication factor 3 +CREATE KEYSPACE IF NOT EXISTS university + WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 3}; + +USE university; + +-- Students table (partition by major for efficient queries) +CREATE TABLE IF NOT EXISTS students ( + major text, + student_id uuid, + name text, + email text, + PRIMARY KEY (major, student_id) +); + +-- Courses table (partition by code) +CREATE TABLE IF NOT EXISTS courses ( + code text PRIMARY KEY, + title text, + capacity int, + enrolled int +); + +-- Enrollments by student (good partition key) +CREATE TABLE IF NOT EXISTS enrollments_by_student ( + student_id uuid, + course_code text, + course_title text, + enrolled_at timestamp, + PRIMARY KEY (student_id, course_code) +); + +-- Access log with BAD partition key (all data in one partition) +CREATE TABLE IF NOT EXISTS access_log_bad ( + log_type text, + accessed_at timestamp, + student_name text, + resource text, + PRIMARY KEY (log_type, accessed_at) +); + +-- Access log with GOOD partition key (distributed by date) +CREATE TABLE IF NOT EXISTS access_log_good ( + day text, + accessed_at timestamp, + student_name text, + resource text, + PRIMARY KEY (day, accessed_at) +); + +-- Seed students +INSERT INTO students (major, student_id, name, email) + VALUES ('Computer Science', uuid(), 'Alice Johnson', 'alice@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Mathematics', uuid(), 'Bob Smith', 'bob@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Computer Science', uuid(), 'Carol Davis', 'carol@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Physics', uuid(), 'David Lee', 'david@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Computer Science', uuid(), 'Eva Martinez', 'eva@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Mathematics', uuid(), 'Frank Wilson', 'frank@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Biology', uuid(), 'Grace Kim', 'grace@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Computer Science', uuid(), 'Henry Brown', 'henry@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Physics', uuid(), 'Iris Chen', 'iris@university.edu'); +INSERT INTO students (major, student_id, name, email) + VALUES ('Mathematics', uuid(), 'Jack Taylor', 'jack@university.edu'); + +-- Seed courses +INSERT INTO courses (code, title, capacity, enrolled) + VALUES ('CS101', 'Intro to Programming', 30, 4); +INSERT INTO courses (code, title, capacity, enrolled) + VALUES ('CS201', 'Data Structures', 25, 2); +INSERT INTO courses (code, title, capacity, enrolled) + VALUES ('MATH101', 'Calculus I', 35, 2); +INSERT INTO courses (code, title, capacity, enrolled) + VALUES ('PHYS101', 'Physics I', 30, 2); diff --git a/10-databases/cassandra/setup.sh b/10-databases/cassandra/setup.sh new file mode 100755 index 0000000..90da70c --- /dev/null +++ b/10-databases/cassandra/setup.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Lab 10C: Cassandra Multi-node, Consistency, and Partition Keys ===" +echo "" + +# Check prerequisites +echo "Checking prerequisites..." + +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed." + echo "Install Docker Desktop from https://www.docker.com/products/docker-desktop/" + exit 1 +fi + +if ! docker info &> /dev/null 2>&1; then + echo "ERROR: Docker daemon is not running. Start Docker Desktop first." + exit 1 +fi + +if ! docker compose version &> /dev/null 2>&1; then + echo "ERROR: Docker Compose is not available." + exit 1 +fi + +echo " Docker: $(docker --version)" +echo " Docker Compose: $(docker compose version --short)" +echo "" + +# Start Cassandra cluster (nodes start sequentially due to depends_on) +echo "Starting 3-node Cassandra cluster..." +echo " (This takes 2-4 minutes -- each node must join the ring)" +docker compose up -d + +# Wait for all nodes to be healthy +for node in cass1 cass2 cass3; do + echo "" + echo "Waiting for $node to join the cluster..." + i=0 + while [ "$i" -lt 180 ]; do + i=$((i + 1)) + if docker exec "$node" cqlsh -e "SELECT now() FROM system.local" > /dev/null 2>&1; then + echo " $node: ready" + break + fi + if [ "$i" -eq 180 ]; then + echo "ERROR: $node did not start within 3 minutes." + docker compose logs "$node" | tail -20 + exit 1 + fi + sleep 1 + done +done + +# Check cluster status +echo "" +echo "Cluster status:" +docker exec cass1 nodetool status 2>/dev/null | grep -E "^(UN|DN|Datacenter|=)" || true + +# Load schema and seed data +echo "" +echo "Loading schema and seed data..." +docker cp init/schema.cql cass1:/tmp/schema.cql +docker exec cass1 cqlsh -f /tmp/schema.cql 2>/dev/null + +echo " Schema created and data seeded." + +echo "" +echo "=== Environment ready ===" +echo "" +echo "Connect to Cassandra:" +echo " docker exec -it cass1 cqlsh" +echo "" +echo "Check cluster health:" +echo " docker exec cass1 nodetool status" +echo "" +echo "Follow the instructions in LAB-CASSANDRA.md for the full walkthrough." diff --git a/10-databases/mongodb/LAB-MONGODB.md b/10-databases/mongodb/LAB-MONGODB.md new file mode 100644 index 0000000..3dd5f05 --- /dev/null +++ b/10-databases/mongodb/LAB-MONGODB.md @@ -0,0 +1,336 @@ +# Lab 10B: MongoDB Replica Set, Consistency, and Schema Design + +![MongoDB](https://img.shields.io/badge/MongoDB-%2347A248.svg?style=for-the-badge&logo=mongodb&logoColor=white) +![Docker](https://img.shields.io/badge/Docker-%232496ED.svg?style=for-the-badge&logo=docker&logoColor=white) + +## Overview + +This lab explores three database scalability mechanisms using MongoDB: +replica set replication with automatic failover, tunable read/write +concerns for consistency control, and schema design trade-offs between +normalized and denormalized documents. Students work with a university +enrollment dataset across a three-node MongoDB replica set. + +## Learning Objectives + +- Deploy and verify a MongoDB replica set with three nodes +- Observe replication by writing to the primary and reading from + secondaries +- Compare read and write concern levels and their consistency guarantees +- Contrast normalized (multi-collection lookups) vs denormalized + (embedded) schema design using explain plans + +## Prerequisites + +- **Docker Desktop** installed and running +- Basic familiarity with JSON documents +- No cloud account required (Option A) or AWS Academy credentials + (Option B) + +## Choose Your Environment + +| Environment | What You Need | Setup | +| --- | --- | --- | +| **Option A: Local** | Docker Desktop + terminal | `./setup.sh` | +| **Option B: EC2** | Browser + SSH client | Upload `cloudformation.yaml` via AWS Console | + +### Option A: Local Setup + +```bash +cd 10-databases/mongodb +chmod +x setup.sh cleanup.sh +./setup.sh +``` + +Then skip to **Task 1** below. + +### Option B: EC2 Setup (AWS Academy) + +1. Download `cloudformation.yaml` from this directory +2. In the AWS Console, go to **CloudFormation** > **Create stack** +3. Upload the template, name it `lab10b-mongodb`, click **Submit** +4. Wait ~3 minutes, find the **PublicIP** in Outputs +5. SSH in: + +```bash +chmod 400 labsuser.pem +ssh -i labsuser.pem ec2-user@YOUR_PUBLIC_IP +ls ~/LAB_READY +cd ~/system-design-course/10-databases/mongodb +``` + +--- + +## Task 1: Verify the Replica Set + +### Step 1.1: Check replica set status + +```bash +docker exec mongo1 mongosh --quiet --eval "rs.status().members.forEach( + m => print(m.name + ' -> ' + m.stateStr) +)" +``` + +Expected output shows one PRIMARY and two SECONDARY nodes. + +### Step 1.2: Verify seed data + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +print('Students: ' + db.students.countDocuments()); +print('Courses: ' + db.courses.countDocuments()); +print('Enrollments: ' + db.enrollments.countDocuments()); +" +``` + +Expected: 10 students, 4 courses, 10 enrollments. + +> **Question:** What happens if the PRIMARY node goes down? +> +> **Hint:** MongoDB replica sets automatically elect a new primary +> from the remaining secondaries. + +--- + +## Task 2: Replication in Action + +### Step 2.1: Write to the primary + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +db.students.insertOne({ + name: 'Zoe Adams', + email: 'zoe@university.edu', + major: 'Engineering' +}); +print('Inserted on primary.'); +" +``` + +### Step 2.2: Read from a secondary + +```bash +docker exec mongo2 mongosh --quiet --eval " +db.getMongo().setReadPref('secondary'); +use('university'); +const zoe = db.students.findOne({ name: 'Zoe Adams' }); +print('Found on secondary: ' + zoe.name + ' (' + zoe.major + ')'); +" +``` + +The document replicated from mongo1 (primary) to mongo2 (secondary). + +### Step 2.3: Verify secondaries reject direct writes + +```bash +docker exec mongo2 mongosh --quiet --eval " +use('university'); +try { + db.students.insertOne({ name: 'Test', email: 'test@u.edu', major: 'X' }); +} catch(e) { + print('Error: ' + e.message); +} +" +``` + +Expected: error about not being primary. + +> **Question:** How does MongoDB replication differ from MySQL +> replication? +> +> **Hint:** Think about automatic failover. MySQL requires manual +> promotion; MongoDB does it automatically via election. + +--- + +## Task 3: Tunable Consistency + +### Step 3.1: Write with w:1 (acknowledge from primary only) + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +const start = Date.now(); +db.courses.insertOne( + { code: 'BIO101', title: 'Biology I', capacity: 30, enrolled: 0 }, + { writeConcern: { w: 1 } } +); +print('w:1 took ' + (Date.now() - start) + 'ms'); +" +``` + +### Step 3.2: Write with w:majority (acknowledge from majority) + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +const start = Date.now(); +db.courses.insertOne( + { code: 'CHEM101', title: 'Chemistry I', capacity: 25, enrolled: 0 }, + { writeConcern: { w: 'majority' } } +); +print('w:majority took ' + (Date.now() - start) + 'ms'); +" +``` + +Compare the two timings. `w:majority` waits for 2 of 3 nodes to +acknowledge, so it takes longer but guarantees the write survives a +single node failure. + +### Step 3.3: Read with different concerns + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +// Read from local -- fastest, may read uncommitted data +let result = db.courses.find({ code: 'BIO101' }) + .readConcern('local').toArray(); +print('readConcern local: ' + result.length + ' result(s)'); + +// Read majority -- only returns data committed to majority +result = db.courses.find({ code: 'CHEM101' }) + .readConcern('majority').toArray(); +print('readConcern majority: ' + result.length + ' result(s)'); +" +``` + +> **Question:** When would you use `w:1` instead of `w:majority`? +> +> **Hint:** Think about use cases where speed matters more than +> durability -- logging, analytics, non-critical data. + +--- + +## Task 4: Schema Design -- Normalized vs Denormalized + +### Step 4.1: Query normalized data (3 collections + lookup) + +Fetch a student with their enrolled courses using `$lookup` (join): + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +const result = db.students.aggregate([ + { \$match: { name: 'Alice Johnson' } }, + { \$lookup: { + from: 'enrollments', + localField: '_id', + foreignField: 'studentId', + as: 'enrollments' + }}, + { \$lookup: { + from: 'courses', + localField: 'enrollments.courseId', + foreignField: '_id', + as: 'courses' + }}, + { \$project: { name: 1, 'courses.code': 1, 'courses.title': 1 } } +]).toArray(); +printjson(result); +" +``` + +### Step 4.2: Query denormalized data (single collection) + +The same information, but embedded in a single document: + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +const result = db.students_denormalized.findOne( + { name: 'Alice Johnson' }, + { name: 1, enrollments: 1 } +); +printjson(result); +" +``` + +### Step 4.3: Compare with explain + +```bash +docker exec mongo1 mongosh --quiet --eval " +use('university'); +// Normalized: aggregation explain +const norm = db.students.explain('executionStats').aggregate([ + { \$match: { name: 'Alice Johnson' } }, + { \$lookup: { + from: 'enrollments', + localField: '_id', + foreignField: 'studentId', + as: 'enrollments' + }} +]); +print('Normalized stages: ' + norm.stages.length); +print('Docs examined: ' + norm.stages[0].\$cursor.executionStats.totalDocsExamined); + +// Denormalized: simple find explain +const denorm = db.students_denormalized.find( + { name: 'Alice Johnson' } +).explain('executionStats'); +print('Denormalized docs examined: ' + + denorm.executionStats.totalDocsExamined); +" +``` + +The denormalized query examines fewer documents because all data is +in a single collection -- no joins needed. + +> **Question:** What is the downside of denormalization? +> +> **Hint:** If a course title changes, you must update it in every +> student document that embeds it, not just one place. + +--- + +## Cleanup + +```bash +./cleanup.sh +``` + +For EC2, also delete the CloudFormation stack in the AWS Console. + +## Troubleshooting + +| Issue | Cause | Fix | +| --- | --- | --- | +| `MongoServerError: not primary` | Writing to a secondary | Connect to mongo1 (the initial primary) | +| Replica set shows no primary | Election not complete | Wait 10-15 seconds, re-check `rs.status()` | +| `readPref` has no effect | Must call before query | Call `db.getMongo().setReadPref()` first | +| `$lookup` returns empty arrays | Data not seeded | Re-run `setup.sh` | +| Container exits immediately | Port conflict | Check `docker ps` for conflicts on 27017-27019 | + +## Key Concepts + +| Concept | Description | +| --- | --- | +| **Replica Set** | A group of MongoDB nodes that maintain the same data, with automatic failover | +| **Primary** | The node that accepts all write operations | +| **Secondary** | Nodes that replicate data from the primary; can serve reads | +| **Write Concern** | How many nodes must acknowledge a write before it is considered successful | +| **Read Concern** | What level of data consistency is guaranteed for read operations | +| **$lookup** | MongoDB's aggregation operator for joining data across collections | +| **Denormalization** | Embedding related data in a single document to avoid joins | + +## Conclusions + +1. **Replica sets provide automatic failover.** Unlike MySQL where + promoting a replica is manual, MongoDB automatically elects a new + primary when one fails. + +2. **Write concern is a speed vs safety trade-off.** `w:1` is fast + but risks data loss if the primary crashes before replicating. + `w:majority` is slower but survives single-node failures. + +3. **Denormalization trades write complexity for read speed.** Embedding + related data eliminates joins and reduces query time, but updates + become harder when the same data appears in multiple documents. + +## Next Steps + +- [Lab 10A -- MySQL](../mysql/LAB-MYSQL.md) -- compare with relational + replication and ACID transactions +- [Lab 10C -- Cassandra](../cassandra/LAB-CASSANDRA.md) -- compare + with wide-column store and partition key design diff --git a/10-databases/mongodb/cleanup.sh b/10-databases/mongodb/cleanup.sh new file mode 100755 index 0000000..395152b --- /dev/null +++ b/10-databases/mongodb/cleanup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Cleaning up MongoDB lab environment ===" +echo "" + +echo "Stopping containers..." +docker compose down -v --remove-orphans 2>/dev/null || true + +echo "" +echo "=== Cleanup complete ===" diff --git a/10-databases/mongodb/cloudformation.yaml b/10-databases/mongodb/cloudformation.yaml new file mode 100644 index 0000000..31eabab --- /dev/null +++ b/10-databases/mongodb/cloudformation.yaml @@ -0,0 +1,77 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + Lab 10B - MongoDB Replica Set, Consistency, and Schema Design. + Launches a t3.medium EC2 instance with Docker and Docker Compose + pre-installed. Clones the course repository and runs the MongoDB lab + setup automatically. SSH access enabled. + +Parameters: + LatestAmiId: + Description: Amazon Linux 2023 AMI (auto-resolved via SSM) + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 + +Resources: + LabSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Lab 10B MongoDB - SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + Description: SSH access + + LabInstance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t3.medium + ImageId: !Ref LatestAmiId + KeyName: vockey + SecurityGroupIds: + - !GetAtt LabSecurityGroup.GroupId + Tags: + - Key: Name + Value: lab10b-mongodb + UserData: + Fn::Base64: | + #!/bin/bash + set -ex + + dnf update -y + dnf install -y docker git + systemctl enable docker + systemctl start docker + usermod -aG docker ec2-user + + ARCH=$(uname -m) + mkdir -p /usr/local/lib/docker/cli-plugins + + curl -fsSL \ + "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${ARCH}" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + + su - ec2-user -c ' + git clone https://github.com/gamaware/system-design-course.git + cd system-design-course/10-databases/mongodb + chmod +x setup.sh cleanup.sh + ./setup.sh + ' + + touch /home/ec2-user/LAB_READY + +Outputs: + InstanceId: + Description: EC2 instance ID + Value: !Ref LabInstance + + PublicIP: + Description: Public IP address (use for SSH) + Value: !GetAtt LabInstance.PublicIp + + SSHCommand: + Description: SSH command (use the labsuser.pem key from AWS Details) + Value: !Sub >- + ssh -i labsuser.pem ec2-user@${LabInstance.PublicIp} diff --git a/10-databases/mongodb/docker-compose.yml b/10-databases/mongodb/docker-compose.yml new file mode 100644 index 0000000..075f23a --- /dev/null +++ b/10-databases/mongodb/docker-compose.yml @@ -0,0 +1,47 @@ +services: + mongo1: + image: mongo:7 + container_name: mongo1 + ports: + - "27017:27017" + command: mongod --replSet rs0 --bind_ip_all + volumes: + - mongo1-data:/data/db + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 3s + retries: 10 + + mongo2: + image: mongo:7 + container_name: mongo2 + ports: + - "27018:27017" + command: mongod --replSet rs0 --bind_ip_all + volumes: + - mongo2-data:/data/db + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 3s + retries: 10 + + mongo3: + image: mongo:7 + container_name: mongo3 + ports: + - "27019:27017" + command: mongod --replSet rs0 --bind_ip_all + volumes: + - mongo3-data:/data/db + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + interval: 5s + timeout: 3s + retries: 10 + +volumes: + mongo1-data: + mongo2-data: + mongo3-data: diff --git a/10-databases/mongodb/init/rs-init.js b/10-databases/mongodb/init/rs-init.js new file mode 100644 index 0000000..1831746 --- /dev/null +++ b/10-databases/mongodb/init/rs-init.js @@ -0,0 +1,94 @@ +// Initialize replica set and seed university data + +rs.initiate({ + _id: "rs0", + members: [ + { _id: 0, host: "mongo1:27017", priority: 2 }, + { _id: 1, host: "mongo2:27017", priority: 1 }, + { _id: 2, host: "mongo3:27017", priority: 1 } + ] +}); + +// Wait for THIS node to become primary +let attempts = 0; +while (attempts < 60) { + try { + const hello = db.adminCommand({ hello: 1 }); + if (hello.isWritablePrimary) { + print("This node is now PRIMARY."); + break; + } + } catch (e) { + // ignore errors during election + } + sleep(1000); + attempts++; +} +if (attempts >= 60) { + print("WARNING: Timed out waiting for primary election."); +} + +// Switch to university database +const db = db.getSiblingDB("university"); + +// Seed students +db.students.insertMany([ + { name: "Alice Johnson", email: "alice@university.edu", major: "Computer Science" }, + { name: "Bob Smith", email: "bob@university.edu", major: "Mathematics" }, + { name: "Carol Davis", email: "carol@university.edu", major: "Computer Science" }, + { name: "David Lee", email: "david@university.edu", major: "Physics" }, + { name: "Eva Martinez", email: "eva@university.edu", major: "Computer Science" }, + { name: "Frank Wilson", email: "frank@university.edu", major: "Mathematics" }, + { name: "Grace Kim", email: "grace@university.edu", major: "Biology" }, + { name: "Henry Brown", email: "henry@university.edu", major: "Computer Science" }, + { name: "Iris Chen", email: "iris@university.edu", major: "Physics" }, + { name: "Jack Taylor", email: "jack@university.edu", major: "Mathematics" } +]); + +// Seed courses +db.courses.insertMany([ + { code: "CS101", title: "Intro to Programming", capacity: 30, enrolled: 4 }, + { code: "CS201", title: "Data Structures", capacity: 25, enrolled: 2 }, + { code: "MATH101", title: "Calculus I", capacity: 35, enrolled: 2 }, + { code: "PHYS101", title: "Physics I", capacity: 30, enrolled: 2 } +]); + +// Seed enrollments (normalized -- separate collection) +const students = db.students.find().toArray(); +const courses = db.courses.find().toArray(); + +db.enrollments.insertMany([ + { studentId: students[0]._id, courseId: courses[0]._id, enrolledAt: new Date() }, + { studentId: students[2]._id, courseId: courses[0]._id, enrolledAt: new Date() }, + { studentId: students[4]._id, courseId: courses[0]._id, enrolledAt: new Date() }, + { studentId: students[7]._id, courseId: courses[0]._id, enrolledAt: new Date() }, + { studentId: students[0]._id, courseId: courses[1]._id, enrolledAt: new Date() }, + { studentId: students[2]._id, courseId: courses[1]._id, enrolledAt: new Date() }, + { studentId: students[1]._id, courseId: courses[2]._id, enrolledAt: new Date() }, + { studentId: students[5]._id, courseId: courses[2]._id, enrolledAt: new Date() }, + { studentId: students[3]._id, courseId: courses[3]._id, enrolledAt: new Date() }, + { studentId: students[8]._id, courseId: courses[3]._id, enrolledAt: new Date() } +]); + +// Seed denormalized collection (embedded documents for comparison) +db.students_denormalized.insertMany([ + { + name: "Alice Johnson", + email: "alice@university.edu", + major: "Computer Science", + enrollments: [ + { code: "CS101", title: "Intro to Programming", enrolledAt: new Date() }, + { code: "CS201", title: "Data Structures", enrolledAt: new Date() } + ] + }, + { + name: "Bob Smith", + email: "bob@university.edu", + major: "Mathematics", + enrollments: [ + { code: "MATH101", title: "Calculus I", enrolledAt: new Date() } + ] + } +]); + +print("University database seeded successfully."); diff --git a/10-databases/mongodb/setup.sh b/10-databases/mongodb/setup.sh new file mode 100755 index 0000000..296fc49 --- /dev/null +++ b/10-databases/mongodb/setup.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Lab 10B: MongoDB Replica Set, Consistency, and Schema Design ===" +echo "" + +# Check prerequisites +echo "Checking prerequisites..." + +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed." + echo "Install Docker Desktop from https://www.docker.com/products/docker-desktop/" + exit 1 +fi + +if ! docker info &> /dev/null 2>&1; then + echo "ERROR: Docker daemon is not running. Start Docker Desktop first." + exit 1 +fi + +if ! docker compose version &> /dev/null 2>&1; then + echo "ERROR: Docker Compose is not available." + exit 1 +fi + +echo " Docker: $(docker --version)" +echo " Docker Compose: $(docker compose version --short)" +echo "" + +# Start MongoDB nodes +echo "Starting 3-node MongoDB replica set..." +docker compose up -d + +echo "" +echo "Waiting for all MongoDB nodes to be healthy..." + +for node in mongo1 mongo2 mongo3; do + i=0 + while [ "$i" -lt 60 ]; do + i=$((i + 1)) + if docker exec "$node" mongosh --quiet --eval "db.adminCommand('ping')" 2>/dev/null | grep -q "ok"; then + echo " $node: ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "ERROR: $node did not start within 60 seconds." + docker compose logs "$node" + exit 1 + fi + sleep 1 + done +done + +# Initialize replica set and seed data +echo "" +echo "Initializing replica set and seeding data..." +docker cp init/rs-init.js mongo1:/tmp/rs-init.js +if ! docker exec mongo1 mongosh --quiet --file /tmp/rs-init.js 2>/dev/null; then + echo "ERROR: Replica set initialization failed." + docker compose logs mongo1 | tail -20 + exit 1 +fi + +# Wait for replica set to stabilize +echo " Waiting for primary election..." +i=0 +while [ "$i" -lt 30 ]; do + i=$((i + 1)) + if docker exec mongo1 mongosh --quiet --eval "rs.status().members.find(m => m.stateStr === 'PRIMARY') ? 'ok' : ''" 2>/dev/null | grep -q "ok"; then + break + fi + sleep 1 +done + +# Verify replica set +rs_status=$(docker exec mongo1 mongosh --quiet --eval " +const s = rs.status(); +s.members.forEach(m => print(m.name + ': ' + m.stateStr)); +" 2>/dev/null || true) + +echo " Replica set members:" +echo "$rs_status" | while IFS= read -r line; do + if [ "$line" != "" ]; then + echo " $line" + fi +done + +echo "" +echo "=== Environment ready ===" +echo "" +echo "Connect to primary:" +echo " docker exec -it mongo1 mongosh" +echo "" +echo "Follow the instructions in LAB-MONGODB.md for the full walkthrough." diff --git a/10-databases/mysql/LAB-MYSQL.md b/10-databases/mysql/LAB-MYSQL.md new file mode 100644 index 0000000..23ea4d3 --- /dev/null +++ b/10-databases/mysql/LAB-MYSQL.md @@ -0,0 +1,367 @@ +# Lab 10A: MySQL Replication, ACID, and Indexing + +![MySQL](https://img.shields.io/badge/MySQL-%234479A1.svg?style=for-the-badge&logo=mysql&logoColor=white) +![Docker](https://img.shields.io/badge/Docker-%232496ED.svg?style=for-the-badge&logo=docker&logoColor=white) + +## Overview + +This lab explores three database scalability mechanisms using MySQL: +GTID-based replication (primary-replica), ACID transactions with +rollback behavior, and query optimization through indexing. Students +work with a university enrollment dataset across a two-node MySQL +cluster running in Docker. + +## Learning Objectives + +- Configure and verify MySQL GTID-based primary-replica replication +- Observe replication lag and understand its implications +- Execute ACID transactions and observe rollback on constraint violations +- Use EXPLAIN to analyze query execution plans before and after indexing + +## Prerequisites + +- **Docker Desktop** installed and running +- Basic SQL knowledge (SELECT, INSERT, UPDATE) +- No cloud account required (Option A) or AWS Academy credentials + (Option B) + +## Choose Your Environment + +| Environment | What You Need | Setup | +| --- | --- | --- | +| **Option A: Local** | Docker Desktop + terminal | `./setup.sh` | +| **Option B: EC2** | Browser + SSH client | Upload `cloudformation.yaml` via AWS Console | + +Both options run the same Docker containers and the same 4 tasks. + +### Option A: Local Setup + +```bash +cd 10-databases/mysql +chmod +x setup.sh cleanup.sh +./setup.sh +``` + +Then skip to **Task 1** below. + +### Option B: EC2 Setup (AWS Academy) + +1. Download `cloudformation.yaml` from this directory +1. In the AWS Console, go to **CloudFormation** > **Create stack** +1. Upload the template, name it `lab10a-mysql`, click **Submit** +1. Wait ~3 minutes for the stack to complete +1. Find the **PublicIP** in the Outputs tab +1. SSH in: + + ```bash + chmod 400 labsuser.pem + ssh -i labsuser.pem ec2-user@YOUR_PUBLIC_IP + ``` + +1. Wait for setup to complete: + + ```bash + ls ~/LAB_READY + cd ~/system-design-course/10-databases/mysql + ``` + +Then continue with **Task 1** below. + +--- + +## Task 1: Verify the Cluster + +Start the environment and confirm both MySQL nodes are running with +active replication. + +### Step 1.1: Check containers + +```bash +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" +``` + +Expected output shows `mysql-primary` and `mysql-replica` both healthy. + +### Step 1.2: Connect to the primary + +```bash +docker exec -it mysql-primary mysql -u root -prootpass university +``` + +Verify the seed data: + +```sql +SELECT COUNT(*) AS total_students FROM students; +SELECT COUNT(*) AS total_enrollments FROM enrollments; +``` + +Expected: 10 students, 10 enrollments. Type `exit` to leave. + +### Step 1.3: Check replication status + +```bash +docker exec mysql-replica mysql -u root -prootpass \ + -e "SHOW REPLICA STATUS\G" 2>/dev/null | grep -E \ + "Replica_IO_Running|Replica_SQL_Running|Seconds_Behind" +``` + +Expected output: + +```text + Replica_IO_Running: Yes + Replica_SQL_Running: Yes + Seconds_Behind_Source: 0 +``` + +> **Question:** What do the IO and SQL threads do in MySQL replication? +> +> **Hint:** One fetches the binary log from the primary, the other +> replays it locally. + +--- + +## Task 2: Replication in Action + +Write data to the primary and observe it appear on the replica. + +### Step 2.1: Insert on the primary + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +INSERT INTO students (name, email, major) +VALUES ('Zoe Adams', 'zoe@university.edu', 'Engineering'); +" +``` + +### Step 2.2: Read from the replica + +```bash +docker exec mysql-replica mysql -u root -prootpass university -e " +SELECT student_id, name, major FROM students WHERE name = 'Zoe Adams'; +" +``` + +The row should appear. Replication propagated the write automatically. + +### Step 2.3: Check replication lag + +```bash +docker exec mysql-replica mysql -u root -prootpass \ + -e "SHOW REPLICA STATUS\G" 2>/dev/null \ + | grep "Seconds_Behind_Source" +``` + +Expected: `Seconds_Behind_Source: 0` (near-instant in a local setup). + +### Step 2.4: Verify the replica is read-only + +```bash +docker exec mysql-replica mysql -u root -prootpass university -e " +INSERT INTO students (name, email, major) +VALUES ('Test User', 'test@university.edu', 'Test'); +" +``` + +Expected error: + +```text +ERROR 1290 (HY000): The MySQL server is running with the +--super-read-only option so it cannot execute this statement +``` + +> **Question:** Why would you configure a replica as read-only? +> +> **Hint:** Think about what happens if clients accidentally write +> to the replica instead of the primary. + +--- + +## Task 3: ACID Transactions + +Test atomicity and isolation by transferring a student between courses +inside a transaction. + +### Step 3.1: Check current enrollment counts + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +SELECT c.code, c.title, c.enrolled +FROM courses c ORDER BY c.code; +" +``` + +### Step 3.2: Successful transaction (transfer enrollment) + +Transfer student Alice (ID 1) from CS101 to PHYS101: + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +START TRANSACTION; + +DELETE FROM enrollments WHERE student_id = 1 AND course_id = 1; +UPDATE courses SET enrolled = enrolled - 1 WHERE course_id = 1; + +INSERT INTO enrollments (student_id, course_id) VALUES (1, 4); +UPDATE courses SET enrolled = enrolled + 1 WHERE course_id = 4; + +COMMIT; +" +``` + +Verify the counts changed: + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +SELECT c.code, c.enrolled FROM courses c WHERE c.code IN ('CS101', 'PHYS101'); +" +``` + +Expected: CS101 has 3, PHYS101 has 3. + +### Step 3.3: Failed transaction (constraint violation) + +Try to enroll Alice in CS201 twice (violates unique constraint): + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +START TRANSACTION; +INSERT INTO enrollments (student_id, course_id) VALUES (1, 2); +INSERT INTO enrollments (student_id, course_id) VALUES (1, 2); +COMMIT; +" 2>&1 || true +``` + +The second INSERT fails due to the unique constraint. Check that Alice +has exactly one enrollment in CS201: + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +SELECT COUNT(*) AS alice_cs201 +FROM enrollments WHERE student_id = 1 AND course_id = 2; +" +``` + +> **Question:** What ACID property ensures that both the DELETE and +> INSERT in step 3.2 either both succeed or both fail? +> +> **Hint:** Think about what happens if the server crashes between +> the DELETE and the INSERT. + +--- + +## Task 4: Indexing for Query Performance + +Compare query performance with and without an index using the +`access_log` table (10,000 rows). + +### Step 4.1: Update table statistics and run a query without an index + +First, update MySQL's statistics so EXPLAIN shows accurate row counts: + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +ANALYZE TABLE access_log; +" +``` + +Now run EXPLAIN on a query with no index: + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +EXPLAIN SELECT * FROM access_log +WHERE student_id = 3 AND resource = 'resource-10'\G +" +``` + +Look at the `rows` field. Without an index, MySQL scans all ~10,000 +rows (full table scan). The `type` shows `ALL` (full scan). + +### Step 4.2: Add a composite index + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +CREATE INDEX idx_student_resource ON access_log (student_id, resource); +" +``` + +### Step 4.3: Re-run the same query with EXPLAIN + +```bash +docker exec mysql-primary mysql -u root -prootpass university -e " +EXPLAIN SELECT * FROM access_log +WHERE student_id = 3 AND resource = 'resource-10'\G +" +``` + +Compare the `rows` field. With the index, MySQL scans far fewer rows +(typically under 100 instead of 10,000). + +### Step 4.4: Verify the index replicated + +```bash +docker exec mysql-replica mysql -u root -prootpass university -e " +SHOW INDEX FROM access_log WHERE Key_name = 'idx_student_resource'\G +" +``` + +The index exists on the replica too -- DDL changes replicate +automatically. + +> **Question:** Why does over-indexing hurt write performance? +> +> **Hint:** Every INSERT and UPDATE must also update all indexes on +> that table. + +--- + +## Cleanup + +```bash +./cleanup.sh +``` + +For EC2, also delete the CloudFormation stack: + +1. Go to **CloudFormation** in the AWS Console +2. Select `lab10a-mysql`, click **Delete** + +## Troubleshooting + +| Issue | Cause | Fix | +| --- | --- | --- | +| `Can't connect to MySQL server` | Container not ready | Wait 30s, retry | +| Replica shows `Connecting` | Primary not reachable | Check `docker compose logs mysql-replica` | +| `Seconds_Behind_Source: NULL` | Replication not started | Re-run `setup.sh` | +| `super-read-only` on INSERT | Writing to replica | Connect to primary (port 3306) instead | +| EXPLAIN shows full scan after index | Wrong column order | Ensure index matches query WHERE clause | + +## Key Concepts + +| Concept | Description | +| --- | --- | +| **GTID Replication** | Global Transaction IDs let replicas track exactly which transactions they have applied | +| **Primary-Replica** | One node handles writes, replicas handle reads for horizontal read scaling | +| **Replication Lag** | Delay between a write on the primary and its appearance on replicas | +| **ACID** | Atomicity, Consistency, Isolation, Durability -- guarantees for relational transactions | +| **EXPLAIN** | Shows the query execution plan: which indexes are used and how many rows are scanned | +| **Composite Index** | An index on multiple columns, effective when queries filter on those columns together | + +## Conclusions + +1. **Replication scales reads, not writes.** Adding replicas lets you + distribute SELECT queries but all writes still go to one primary. + +2. **ACID transactions prevent partial updates.** The enrollment + transfer either fully succeeds or fully rolls back -- no + inconsistent state. + +3. **Indexes dramatically reduce query cost.** A composite index + turned a 10,000-row scan into a targeted lookup. But each index + adds overhead to writes. + +## Next Steps + +- [Lab 10B -- MongoDB](../mongodb/LAB-MONGODB.md) -- compare with + document store replication and tunable consistency +- [Lab 10C -- Cassandra](../cassandra/LAB-CASSANDRA.md) -- compare + with wide-column store and partition key design diff --git a/10-databases/mysql/cleanup.sh b/10-databases/mysql/cleanup.sh new file mode 100755 index 0000000..a1ff050 --- /dev/null +++ b/10-databases/mysql/cleanup.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Cleaning up MySQL lab environment ===" +echo "" + +echo "Stopping containers..." +docker compose down -v --remove-orphans 2>/dev/null || true + +echo "Removing volumes..." +docker volume rm mysql_primary-data mysql_replica-data 2>/dev/null || true + +echo "" +echo "=== Cleanup complete ===" diff --git a/10-databases/mysql/cloudformation.yaml b/10-databases/mysql/cloudformation.yaml new file mode 100644 index 0000000..f4d2197 --- /dev/null +++ b/10-databases/mysql/cloudformation.yaml @@ -0,0 +1,81 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + Lab 10A - MySQL Replication, ACID, and Indexing. + Launches a t3.medium EC2 instance with Docker and Docker Compose + pre-installed. Clones the course repository and runs the MySQL lab + setup automatically. SSH access enabled. + +Parameters: + LatestAmiId: + Description: Amazon Linux 2023 AMI (auto-resolved via SSM) + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 + +Resources: + LabSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Lab 10A MySQL - SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + Description: SSH access + + LabInstance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t3.medium + ImageId: !Ref LatestAmiId + KeyName: vockey + SecurityGroupIds: + - !GetAtt LabSecurityGroup.GroupId + Tags: + - Key: Name + Value: lab10a-mysql + UserData: + Fn::Base64: | + #!/bin/bash + set -ex + + # Install Docker from Amazon Linux repos + dnf update -y + dnf install -y docker git + systemctl enable docker + systemctl start docker + usermod -aG docker ec2-user + + # Install Docker Compose plugin + ARCH=$(uname -m) + mkdir -p /usr/local/lib/docker/cli-plugins + + curl -fsSL \ + "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${ARCH}" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + + # Clone the repo and run the lab as ec2-user + su - ec2-user -c ' + git clone https://github.com/gamaware/system-design-course.git + cd system-design-course/10-databases/mysql + chmod +x setup.sh cleanup.sh + ./setup.sh + ' + + # Signal that setup is complete + touch /home/ec2-user/LAB_READY + +Outputs: + InstanceId: + Description: EC2 instance ID + Value: !Ref LabInstance + + PublicIP: + Description: Public IP address (use for SSH) + Value: !GetAtt LabInstance.PublicIp + + SSHCommand: + Description: SSH command (use the labsuser.pem key from AWS Details) + Value: !Sub >- + ssh -i labsuser.pem ec2-user@${LabInstance.PublicIp} diff --git a/10-databases/mysql/docker-compose.yml b/10-databases/mysql/docker-compose.yml new file mode 100644 index 0000000..aff5eb9 --- /dev/null +++ b/10-databases/mysql/docker-compose.yml @@ -0,0 +1,51 @@ +services: + mysql-primary: + image: mysql:8 + container_name: mysql-primary + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: university + ports: + - "3306:3306" + volumes: + - ./init/primary-init.sql:/docker-entrypoint-initdb.d/01-init.sql + - primary-data:/var/lib/mysql + command: + - --server-id=1 + - --log-bin=mysql-bin + - --gtid-mode=ON + - --enforce-gtid-consistency=ON + - --binlog-format=ROW + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpass"] + interval: 5s + timeout: 3s + retries: 10 + + mysql-replica: + image: mysql:8 + container_name: mysql-replica + environment: + MYSQL_ROOT_PASSWORD: rootpass + ports: + - "3307:3306" + volumes: + - replica-data:/var/lib/mysql + command: + - --server-id=2 + - --log-bin=mysql-bin + - --gtid-mode=ON + - --enforce-gtid-consistency=ON + - --binlog-format=ROW + depends_on: + mysql-primary: + condition: service_healthy + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpass"] + interval: 5s + timeout: 3s + retries: 10 + +volumes: + primary-data: + replica-data: diff --git a/10-databases/mysql/init/primary-init.sql b/10-databases/mysql/init/primary-init.sql new file mode 100644 index 0000000..1d03c82 --- /dev/null +++ b/10-databases/mysql/init/primary-init.sql @@ -0,0 +1,84 @@ +-- University enrollment schema and seed data +CREATE DATABASE IF NOT EXISTS university; +USE university; + +CREATE TABLE students ( + student_id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + email VARCHAR(100) NOT NULL UNIQUE, + major VARCHAR(50), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE courses ( + course_id INT AUTO_INCREMENT PRIMARY KEY, + code VARCHAR(10) NOT NULL UNIQUE, + title VARCHAR(100) NOT NULL, + capacity INT NOT NULL DEFAULT 30, + enrolled INT NOT NULL DEFAULT 0 +); + +CREATE TABLE enrollments ( + enrollment_id INT AUTO_INCREMENT PRIMARY KEY, + student_id INT NOT NULL, + course_id INT NOT NULL, + enrolled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (student_id) REFERENCES students(student_id), + FOREIGN KEY (course_id) REFERENCES courses(course_id), + UNIQUE KEY unique_enrollment (student_id, course_id) +); + +-- Seed students +INSERT INTO students (name, email, major) VALUES +('Alice Johnson', 'alice@university.edu', 'Computer Science'), +('Bob Smith', 'bob@university.edu', 'Mathematics'), +('Carol Davis', 'carol@university.edu', 'Computer Science'), +('David Lee', 'david@university.edu', 'Physics'), +('Eva Martinez', 'eva@university.edu', 'Computer Science'), +('Frank Wilson', 'frank@university.edu', 'Mathematics'), +('Grace Kim', 'grace@university.edu', 'Biology'), +('Henry Brown', 'henry@university.edu', 'Computer Science'), +('Iris Chen', 'iris@university.edu', 'Physics'), +('Jack Taylor', 'jack@university.edu', 'Mathematics'); + +-- Seed courses +INSERT INTO courses (code, title, capacity, enrolled) VALUES +('CS101', 'Intro to Programming', 30, 4), +('CS201', 'Data Structures', 25, 2), +('MATH101', 'Calculus I', 35, 2), +('PHYS101', 'Physics I', 30, 2); + +-- Seed enrollments +INSERT INTO enrollments (student_id, course_id) VALUES +(1, 1), (3, 1), (5, 1), (8, 1), +(1, 2), (3, 2), +(2, 3), (6, 3), +(4, 4), (9, 4); + +-- Create a large table for indexing exercise (no index on purpose) +CREATE TABLE access_log ( + log_id INT AUTO_INCREMENT PRIMARY KEY, + student_id INT NOT NULL, + resource VARCHAR(100) NOT NULL, + accessed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Insert 10,000 rows for indexing demo +DELIMITER // +CREATE PROCEDURE seed_access_log() +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < 10000 DO + INSERT INTO access_log (student_id, resource, accessed_at) + VALUES ( + FLOOR(1 + RAND() * 10), + CONCAT('resource-', FLOOR(1 + RAND() * 50)), + DATE_SUB(NOW(), INTERVAL FLOOR(RAND() * 365) DAY) + ); + SET i = i + 1; + END WHILE; +END // +DELIMITER ; + +CALL seed_access_log(); +DROP PROCEDURE seed_access_log; diff --git a/10-databases/mysql/setup.sh b/10-databases/mysql/setup.sh new file mode 100755 index 0000000..8d69d42 --- /dev/null +++ b/10-databases/mysql/setup.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Lab 10A: MySQL Replication, ACID, and Indexing ===" +echo "" + +# Check prerequisites +echo "Checking prerequisites..." + +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed." + echo "Install Docker Desktop from https://www.docker.com/products/docker-desktop/" + exit 1 +fi + +if ! docker info &> /dev/null 2>&1; then + echo "ERROR: Docker daemon is not running. Start Docker Desktop first." + exit 1 +fi + +if ! docker compose version &> /dev/null 2>&1; then + echo "ERROR: Docker Compose is not available." + exit 1 +fi + +echo " Docker: $(docker --version)" +echo " Docker Compose: $(docker compose version --short)" +echo "" + +# Start MySQL primary and replica +echo "Starting MySQL primary and replica..." +docker compose up -d --build + +echo "" +echo "Waiting for MySQL primary to be healthy..." +i=0 +while [ "$i" -lt 90 ]; do + i=$((i + 1)) + if docker exec mysql-primary mysqladmin ping -u root -prootpass 2>/dev/null | grep -q "alive"; then + echo " MySQL primary: ready" + break + fi + if [ "$i" -eq 90 ]; then + echo "ERROR: MySQL primary did not start within 90 seconds." + docker compose logs mysql-primary + exit 1 + fi + sleep 1 +done + +echo "Waiting for MySQL replica to be healthy..." +i=0 +while [ "$i" -lt 90 ]; do + i=$((i + 1)) + if docker exec mysql-replica mysqladmin ping -u root -prootpass 2>/dev/null | grep -q "alive"; then + echo " MySQL replica: ready" + break + fi + if [ "$i" -eq 90 ]; then + echo "ERROR: MySQL replica did not start within 90 seconds." + docker compose logs mysql-replica + exit 1 + fi + sleep 1 +done + +# Configure replication +echo "" +echo "Configuring GTID-based replication..." + +# Create replication user on primary +docker exec mysql-primary mysql -u root -prootpass \ + -e "CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'replpass'; GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; FLUSH PRIVILEGES;" \ + 2>/dev/null + +# Point replica to primary +docker exec mysql-replica mysql -u root -prootpass \ + -e "STOP REPLICA; CHANGE REPLICATION SOURCE TO SOURCE_HOST='mysql-primary', SOURCE_USER='repl', SOURCE_PASSWORD='replpass', SOURCE_AUTO_POSITION=1, GET_SOURCE_PUBLIC_KEY=1; START REPLICA;" \ + 2>/dev/null + +# Set replica to read-only after replication is configured +docker exec mysql-replica mysql -u root -prootpass \ + -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" \ + 2>/dev/null + +# Wait for replication to catch up +echo " Waiting for replication to initialize..." +i=0 +while [ "$i" -lt 30 ]; do + i=$((i + 1)) + if docker exec mysql-replica mysql -u root -prootpass \ + -e "SHOW REPLICA STATUS\G" 2>/dev/null | grep -q "Replica_IO_Running: Yes"; then + break + fi + sleep 1 +done + +# Verify replication +replica_status=$(docker exec mysql-replica mysql -u root -prootpass \ + -e "SHOW REPLICA STATUS\G" 2>/dev/null || true) + +if echo "$replica_status" | grep -q "Replica_IO_Running: Yes"; then + echo " Replication IO thread: running" +else + echo " WARNING: Replication IO thread not running yet" +fi + +if echo "$replica_status" | grep -q "Replica_SQL_Running: Yes"; then + echo " Replication SQL thread: running" +else + echo " WARNING: Replication SQL thread not running yet" +fi + +echo "" +echo "=== Environment ready ===" +echo "" +echo "Connect to primary:" +echo " docker exec -it mysql-primary mysql -u root -prootpass university" +echo "" +echo "Connect to replica:" +echo " docker exec -it mysql-replica mysql -u root -prootpass university" +echo "" +echo "Follow the instructions in LAB-MYSQL.md for the full walkthrough." diff --git a/10-databases/visualizer/Dockerfile b/10-databases/visualizer/Dockerfile new file mode 100644 index 0000000..629fcb4 --- /dev/null +++ b/10-databases/visualizer/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.13-slim + +WORKDIR /app + +RUN pip install --no-cache-dir pymysql==1.1.1 cryptography==44.0.3 + +COPY . . + +EXPOSE 8080 + +CMD ["python", "server.py"] diff --git a/10-databases/visualizer/LAB-VISUALIZER.md b/10-databases/visualizer/LAB-VISUALIZER.md new file mode 100644 index 0000000..7c36822 --- /dev/null +++ b/10-databases/visualizer/LAB-VISUALIZER.md @@ -0,0 +1,522 @@ +# Lab 10: Database Scalability (Interactive Visualizer) + +![MySQL](https://img.shields.io/badge/MySQL-%234479A1.svg?style=for-the-badge&logo=mysql&logoColor=white) +![Docker](https://img.shields.io/badge/Docker-%232496ED.svg?style=for-the-badge&logo=docker&logoColor=white) + +## Overview + +This is the main lab for Module 10. You will use an interactive web +visualizer connected to a live MySQL primary-replica cluster to +explore three database scalability mechanisms: replication, ACID +consistency, and query optimization through indexing. Every button +click executes real SQL against real databases -- the animated +diagrams show actual data flow with measured latency. + +The visualizer includes a built-in SQL console where you can run +arbitrary queries against either the primary or replica node and +see the results in real time. + +## Learning Objectives + +- Understand how primary-replica replication distributes read workloads +- Observe replication lag and understand its impact on data consistency +- Execute ACID transactions and see atomicity in action (commit vs + rollback) +- Use EXPLAIN to compare query execution plans with and without indexes +- Run SQL directly against primary and replica nodes to verify behavior +- Connect these mechanisms to real-world scaling strategies (read + replicas, transactional safety, query optimization) + +## Prerequisites + +- A web browser (Chrome, Firefox, or Safari) +- **Option A (Local):** Docker Desktop installed and running +- **Option B (EC2):** AWS Academy credentials (no local tools required) + +## Choose Your Environment + +| Environment | What You Need | Setup | +| --- | --- | --- | +| **Option A: Local** | Docker Desktop + browser | `./setup.sh` | +| **Option B: EC2** | Browser only | Upload `cloudformation.yaml` via AWS Console | + +Both options run the same MySQL cluster and visualizer. Choose one. + +### Option A: Local Setup (Docker Desktop) + +```bash +cd 10-databases/visualizer +chmod +x setup.sh cleanup.sh +./setup.sh +``` + +Open [http://localhost:8081](http://localhost:8081) in your browser. +Then skip to **Task 1** below. + +### Option B: EC2 Setup (AWS Academy) + +This option runs the visualizer on an EC2 instance -- no local tools +required beyond a web browser. + +#### Step 1: Start the Learner Lab + +Log in to your AWS Academy Learner Lab course: + +1. Go to **Modules** and click **Launch AWS Academy Learner Lab** +1. Click **Start Lab** and wait for the AWS indicator to turn **green** +1. Click the **AWS** link (green dot) to open the AWS Management Console + +#### Step 2: Download the CloudFormation template + +Download `cloudformation.yaml` from the course repository to your +computer: + +```text +https://raw.githubusercontent.com/gamaware/system-design-course/main/10-databases/visualizer/cloudformation.yaml +``` + +#### Step 3: Create the CloudFormation stack + +In the AWS Console: + +1. Navigate to **CloudFormation** (search for it in the top bar) +1. Click **Create stack** > **With new resources (standard)** +1. Select **Upload a template file**, click **Choose file**, and + upload `cloudformation.yaml` +1. Click **Next** +1. Enter the stack name: `lab10-visualizer` +1. Click **Next** twice (skip Configure stack options) +1. On the Review page, scroll down and click **Submit** + +#### Step 4: Wait for the stack to complete + +The stack takes about 3-5 minutes to create. Watch the progress in +the **Events** tab. + +When complete, click the **Outputs** tab to find: + +- **PublicIP** -- the EC2 instance address +- **VisualizerURL** -- the URL to open in your browser + +#### Step 5: Open the visualizer + +Open the **VisualizerURL** from the Outputs tab in your browser: + +```text +http://YOUR_PUBLIC_IP:8081 +``` + +The visualizer should load with the three tabs and sidebar showing +database state. If the page does not load, wait another minute for +the setup to finish. + +Then continue with **Task 1** below. The visualizer works identically +whether running locally or on EC2. + +#### EC2 Cleanup + +When done with the lab, delete the CloudFormation stack: + +1. Go to **CloudFormation** in the AWS Console +1. Select `lab10-visualizer` +1. Click **Delete** and confirm + +This terminates the EC2 instance and removes all resources. + +--- + +## Architecture + +The visualizer connects to a live MySQL primary-replica cluster. +All operations are real SQL queries executed against real databases. + +```mermaid +graph LR + subgraph Docker ["Docker Compose Network"] + subgraph Viz ["Visualizer"] + WEB["Browser
(localhost:8081)"] + API["Python API
(server.py)"] + end + subgraph DB ["MySQL Cluster"] + PRIMARY[("Primary
port 3306
read-write")] + REPLICA[("Replica
port 3307
read-only")] + end + end + + WEB -->|"HTTP"| API + API -->|"INSERT / UPDATE"| PRIMARY + API -->|"SELECT / SHOW STATUS"| REPLICA + PRIMARY -.->|"binlog replication"| REPLICA +``` + +## Lab Structure + +```text +10-databases/visualizer/ +├── LAB-VISUALIZER.md # This file (lab instructions) +├── docker-compose.yml # MySQL primary + replica + visualizer +├── setup.sh # Start environment and configure replication +├── cleanup.sh # Tear down all containers and volumes +├── Dockerfile # Python API bridge image +├── server.py # API bridge (proxies SQL to MySQL nodes) +├── index.html # Interactive visualizer UI +├── app.js # Animation engine and interaction logic +└── style.css # Dark theme styling +``` + +--- + +## Task 1: Explore the Environment + +### Step 1.1: Verify the cluster + +After running `./setup.sh`, open +[http://localhost:8081](http://localhost:8081). The sidebar on the +right shows live database state: + +- **IO Thread / SQL Thread**: Both should show `Yes` (replication + active) +- **Lag (sec)**: Should be `0` (replica is caught up) +- **Primary Rows / Replica Rows**: Both show `10` (same data) +- **Courses**: 4 courses with enrollment counts + +### Step 1.2: Open the SQL Console + +At the bottom of the page, click **SQL Console** to expand it. +Select **Primary** and run: + +```sql +SHOW TABLES; +``` + +You should see: `students`, `courses`, `enrollments`, `access_log`. + +Now select **Replica** and run the same query. The replica has +identical tables -- it received them through replication. + +### Step 1.3: Inspect the schema + +In the SQL Console (Primary), run: + +```sql +DESCRIBE students; +``` + +Note the columns: `student_id` (auto-increment PK), `name`, `email` +(unique), `major`, `created_at`. This is a relational schema with +enforced constraints -- the database rejects invalid data at the +storage layer, not just in application code. + +> **Question:** What is the difference between enforcing uniqueness in +> the application layer vs the database layer? +> +> **Hint:** Think about what happens when two requests try to insert +> the same email simultaneously. Only the database can guarantee +> atomicity across concurrent connections. + +--- + +## Task 2: Replication -- Scaling Reads + +Replication lets you distribute read queries across multiple nodes. +The primary handles all writes; replicas receive changes through the +binary log and can serve read traffic independently. + +### Step 2.1: Write to primary, read from replica + +In the **Replication** tab: + +1. Enter a student name (e.g., `Maria Lopez`) and select a major +1. Click **Write & Read** + +Watch the animation: + +1. Blue arrow: INSERT goes to the Primary node +1. Purple arrow: SELECT reads from the Replica node +1. Latency pills show real timing for each operation + +The result panel shows **REPLICATED** -- the data appeared on the +replica within milliseconds. + +### Step 2.2: Verify via SQL Console + +Switch to **Replica** in the SQL Console and run: + +```sql +SELECT student_id, name, major FROM students ORDER BY student_id DESC LIMIT 3; +``` + +You should see the student you just created. The replica received the +INSERT through replication without any action from you. + +### Step 2.3: Try writing to the replica + +In the SQL Console, select **Replica** and run: + +```sql +INSERT INTO students (name, email, major) VALUES ('Test', 'test@u.edu', 'X'); +``` + +Expected error: `The MySQL server is running with the +--super-read-only option`. Replicas are read-only by design -- this +prevents split-brain scenarios where two nodes accept conflicting +writes. + +### Step 2.4: Check replication status + +In the SQL Console (Replica), run: + +```sql +SHOW REPLICA STATUS\G +``` + +Look for `Replica_IO_Running: Yes`, `Replica_SQL_Running: Yes`, and +`Seconds_Behind_Source: 0`. + +> **Question:** A social media platform has 100,000 read requests per +> second but only 1,000 writes. How does replication help? +> +> **Hint:** Add read replicas to handle the 100:1 read-to-write ratio. +> Each replica independently serves reads, so 10 replicas reduce each +> node's load to ~10,000 reads/second. But all writes still go to one +> primary -- replication scales reads, not writes. + +--- + +## Task 3: ACID Transactions -- Consistency Under Concurrency + +ACID (Atomicity, Consistency, Isolation, Durability) guarantees that +database transactions are reliable. The key property here is +**atomicity**: a transaction either fully succeeds or fully rolls +back. No partial state exists. + +### Step 3.1: Successful transfer + +Switch to the **Consistency (ACID)** tab. + +The controls show a student enrollment transfer: move Student 1 from +CS101 to PHYS101. + +1. Set Student ID to `1`, From Course to `CS101`, To Course to `PHYS101` +1. Click **Transfer Enrollment** + +Watch the animation step through: + +1. `BEGIN` -- start the transaction +1. `DELETE enrollment (CS101)` -- remove the old enrollment +1. `UPDATE CS101 enrolled-1` -- decrement the count +1. `INSERT enrollment (PHYS101)` -- add the new enrollment +1. `UPDATE PHYS101 enrolled+1` -- increment the count +1. `COMMIT` -- make all changes permanent + +The result shows **COMMITTED**. Check the sidebar: CS101 dropped from +4 to 3, PHYS101 went from 2 to 3. + +### Step 3.2: Trigger a rollback + +Now try to transfer Student 1 to PHYS101 again (they are already +enrolled there): + +1. Same settings: Student ID `1`, From `CS101`, To `PHYS101` +1. Click **Transfer Enrollment** + +The INSERT fails with **CONSTRAINT VIOLATION** (duplicate enrollment). +The result shows **ROLLED BACK** -- the DELETE and UPDATE that happened +before the failed INSERT were also undone. The counts in the sidebar +remain unchanged. + +### Step 3.3: Verify atomicity via SQL Console + +In the SQL Console (Primary), run: + +```sql +SELECT c.code, c.enrolled FROM courses ORDER BY code; +``` + +Confirm the counts match what the sidebar shows. The failed +transaction left zero trace in the database. + +> **Question:** A banking system transfers $100 from Account A to +> Account B. The debit from A succeeds, but the credit to B fails +> (network error). What happens without ACID? +> +> **Hint:** Without atomicity, the $100 disappears -- debited from A +> but never credited to B. With ACID, the entire transaction rolls +> back, and Account A keeps its money. This is why financial systems +> require ACID compliance. + +--- + +## Task 4: Schema Design -- Indexing for Performance + +As tables grow, query performance depends on whether the database can +find rows efficiently. Without an index, MySQL must scan every row +(full table scan). With the right index, it jumps directly to +matching rows. + +### Step 4.1: EXPLAIN without an index + +Switch to the **Schema & Indexing** tab. + +The controls show a query: find access log entries for Student 3 +accessing resource-10. + +1. Set Student ID to `3`, Resource to `resource-10` +1. Click **Run EXPLAIN** + +The result shows **Rows scanned: ~9,894** in red, with Key: **NONE +(full scan)**. MySQL examined nearly all 10,000 rows to find a +handful of matches. Check the sidebar: Indexes shows **None**. + +### Step 4.2: Add a composite index + +Click **Add Index**. This creates a composite index on +`(student_id, resource)`. + +Check the sidebar: Indexes now shows **idx_student_resource**. + +### Step 4.3: EXPLAIN with the index + +Click **Run EXPLAIN** again (same query). + +The result now shows **Rows scanned: ~24** in green, with Key: +**idx_student_resource**. MySQL used the index to jump directly to +matching rows -- a ~400x improvement. + +### Step 4.4: Verify via SQL Console + +In the SQL Console (Primary), compare: + +```sql +EXPLAIN SELECT * FROM access_log WHERE student_id = 3 AND resource = 'resource-10'; +``` + +Look at the `type` column: `ref` (index lookup) instead of `ALL` +(full scan). The `rows` column confirms the improvement. + +### Step 4.5: Understand the trade-off + +Drop the index: click **Drop Index**. + +Now run an INSERT in the SQL Console (Primary): + +```sql +INSERT INTO access_log (student_id, resource) +VALUES (1, 'resource-99'); +``` + +Note the latency. Now add the index back (click **Add Index**) and +run the same INSERT again. With the index, writes take slightly +longer because MySQL must update both the table and the index. + +> **Question:** An e-commerce site has a `products` table with 10 +> million rows. Searches by `category` are slow. Should you add an +> index on `category`? +> +> **Hint:** If the table has 50 categories with roughly equal +> distribution, an index on `category` returns ~200,000 rows per +> lookup. The index helps, but a composite index like +> `(category, price)` would be far more selective if queries also +> filter by price range. + +--- + +## Task 5: Free Exploration + +Use the SQL Console to explore on your own. Here are some queries to +try: + +**Check which students are enrolled in which courses:** + +```sql +SELECT s.name, c.code, c.title +FROM enrollments e +JOIN students s ON e.student_id = s.student_id +JOIN courses c ON e.course_id = c.course_id +ORDER BY s.name; +``` + +**Compare primary vs replica data:** + +Run the same SELECT on both Primary and Replica to verify they +return identical results. + +**Test isolation by running concurrent queries:** + +In one SQL Console query, insert a student. In the sidebar, watch +the Primary Rows counter increment and -- a moment later -- the +Replica Rows counter catch up. + +**Reset the database:** + +Click the **Reset DB** button in the top-right corner to restore +the original data (10 students, 4 courses, 10 enrollments, no +custom indexes). + +--- + +## Cleanup + +```bash +./cleanup.sh +``` + +## Troubleshooting + +| Issue | Cause | Fix | +| --- | --- | --- | +| Port 8081 in use | Another service running | Change port in docker-compose.yml | +| SQL Console shows error | Blocked SQL command | Some DDL commands are blocked for safety | +| Replica shows `--` in sidebar | Replication not configured | Re-run `./setup.sh` | +| EXPLAIN shows `rows: 1` | Table stats stale | Run `ANALYZE TABLE access_log;` in the console | +| Animation stuck | Previous operation still running | Wait for it to finish or refresh the page | + +## Key Concepts + +| Concept | Description | +| --- | --- | +| **Replication** | Copying data from a primary to one or more replicas via binary log | +| **Read replica** | A read-only copy that offloads SELECT queries from the primary | +| **Replication lag** | Delay between a write on primary and its appearance on replica | +| **ACID** | Atomicity, Consistency, Isolation, Durability -- transaction guarantees | +| **Atomicity** | A transaction either fully succeeds (COMMIT) or fully fails (ROLLBACK) | +| **Full table scan** | MySQL reads every row to find matches -- slow on large tables | +| **Composite index** | An index on multiple columns that speeds up multi-column WHERE clauses | +| **EXPLAIN** | Shows MySQL's query execution plan: index used, rows examined | +| **Read-write split** | Send writes to primary, reads to replicas -- the most common scaling pattern | + +## Conclusions + +1. **Replication scales reads, not writes.** Adding replicas + distributes SELECT queries across multiple nodes, but all writes + still go to one primary. This is the most common first step in + scaling a relational database. + +2. **ACID prevents data corruption under concurrency.** The + enrollment transfer either fully succeeds or fully rolls back. + Without atomicity, partial failures leave the database in an + inconsistent state -- money disappears, inventory goes negative, + seats are double-booked. + +3. **Indexes are the highest-impact optimization for query + performance.** A single composite index turned a 10,000-row scan + into a 24-row lookup -- a 400x improvement. But indexes are not + free: they consume storage and slow down writes. The right index + depends on your query patterns. + +4. **These three mechanisms compose.** A production system uses all + three: replication for read scaling, ACID for correctness, and + indexes for performance. They are not alternatives -- they solve + different problems at different layers. + +## Next Steps (Optional) + +For deeper hands-on work with other database paradigms, try the +optional labs: + +- [Lab 10A -- MySQL CLI](../mysql/LAB-MYSQL.md) -- same mechanisms + via the command line +- [Lab 10B -- MongoDB](../mongodb/LAB-MONGODB.md) -- replica sets, + tunable consistency, denormalized schema +- [Lab 10C -- Cassandra](../cassandra/LAB-CASSANDRA.md) -- multi-node + ring, consistency levels, partition key design diff --git a/10-databases/visualizer/app.js b/10-databases/visualizer/app.js new file mode 100644 index 0000000..8c1f5fb --- /dev/null +++ b/10-databases/visualizer/app.js @@ -0,0 +1,589 @@ +'use strict'; + +const STEP_ARROW_MS = 300; +const STEP_PAUSE_MS = 500; +const POLL_INTERVAL_MS = 3000; +const MAX_LOG_ENTRIES = 100; + +const PATTERN_DESCRIPTIONS = { + replication: + 'Write to primary, read from replica. Observe replication lag in real time.', + consistency: + 'Transfer enrollment between courses inside an ACID transaction. Observe commit or rollback.', + schema: + 'Run EXPLAIN to see query plans. Add/drop indexes and compare rows scanned.', +}; + +const EXPLANATIONS = { + replication: [ + { + title: '1. INSERT on Primary', + detail: + 'The new student row is written to the primary MySQL instance. ' + + 'The primary records the change in its binary log (binlog).', + }, + { + title: '2. SELECT on Replica', + detail: + 'The replica applies binlog events via its IO and SQL threads. ' + + 'We immediately read the row from the replica to check replication.', + }, + { + title: '3. Check Replication Lag', + detail: + 'SHOW REPLICA STATUS reveals Seconds_Behind_Source. ' + + 'In this local setup it is typically 0, but under heavy load it grows.', + }, + ], + consistency: [ + { + title: '1. BEGIN Transaction', + detail: + 'MySQL starts a transaction. All subsequent statements are ' + + 'isolated from other connections until COMMIT or ROLLBACK.', + }, + { + title: '2. DELETE + UPDATE (source course)', + detail: + 'Remove the enrollment and decrement the enrolled count. ' + + 'If the student is not enrolled, no rows are affected.', + }, + { + title: '3. INSERT + UPDATE (target course)', + detail: + 'Add the new enrollment and increment the count. ' + + 'If a unique constraint is violated, the whole transaction rolls back.', + }, + { + title: '4. COMMIT or ROLLBACK', + detail: + 'On success, all changes are made permanent atomically. ' + + 'On failure, the database reverts to its state before BEGIN.', + }, + ], + schema: [ + { + title: '1. EXPLAIN (Query Plan)', + detail: + 'MySQL shows how it would execute the query. The "rows" field ' + + 'shows how many rows it expects to examine. Full scan = ~10,000.', + }, + { + title: '2. Add/Drop Index', + detail: + 'A composite index on (student_id, resource) lets MySQL jump ' + + 'directly to matching rows. Rows examined drops from ~10,000 to ~20.', + }, + ], +}; + +let animating = false; +let pollTimer = null; + +function $(sel) { return document.querySelector(sel); } +function $$(sel) { return document.querySelectorAll(sel); } +function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } + +async function apiPost(url, body) { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + return res.json(); +} + +async function apiGet(url) { + const res = await fetch(url); + return res.json(); +} + +function setButtonsDisabled(d) { + $$('button').forEach(b => { b.disabled = d; }); +} + +function clearAnimations() { + $$('.arrow').forEach(a => { + a.classList.remove('arrow-active', 'arrow-success', 'arrow-op', 'arrow-error'); + }); + $$('.travel-dot').forEach(d => d.setAttribute('opacity', '0')); + $$('.latency-group').forEach(g => { + g.setAttribute('opacity', '0'); + const t = g.querySelector('.latency-label'); + if (t) t.textContent = ''; + }); + $$('.node-badge').forEach(b => b.setAttribute('opacity', '0')); +} + +// --- Arrow Animation --- + +async function animateArrow(arrowId, dotId, latencyId, ms, cls) { + const arrow = $(`#${arrowId}`); + const dot = $(`#${dotId}`); + const latGroup = $(`#${latencyId}`); + + if (arrow) { + arrow.classList.add('arrow-active', cls || 'arrow-op'); + } + + // Animate dot along arrow path + if (dot && arrow) { + const x1 = parseFloat(arrow.getAttribute('x1')); + const y1 = parseFloat(arrow.getAttribute('y1')); + const x2 = parseFloat(arrow.getAttribute('x2')); + const y2 = parseFloat(arrow.getAttribute('y2')); + dot.setAttribute('opacity', '1'); + const steps = 20; + for (let i = 0; i <= steps; i++) { + const t = i / steps; + dot.setAttribute('cx', x1 + (x2 - x1) * t); + dot.setAttribute('cy', y1 + (y2 - y1) * t); + await sleep(STEP_ARROW_MS / steps); + } + dot.setAttribute('opacity', '0'); + } + + // Show latency + if (latGroup && ms !== undefined) { + latGroup.setAttribute('opacity', '1'); + const label = latGroup.querySelector('.latency-label'); + if (label) label.textContent = `${ms.toFixed(1)}ms`; + } +} + +// --- Event Log --- + +function logEvent(step) { + const log = $('#event-log'); + const entry = document.createElement('div'); + entry.className = 'log-entry'; + + const time = new Date().toLocaleTimeString('en-US', { hour12: false }); + const targetCls = step.target === 'primary' ? 'primary' : 'replica'; + const resultCls = step.result === 'OK' || step.result === 'FOUND' || + step.result === 'COMMITTED' ? 'ok' : 'error'; + + entry.innerHTML = + `${time}` + + `${step.action}` + + `${step.target}` + + `${step.result}` + + `${step.latency_ms.toFixed(1)}ms`; + + log.prepend(entry); + while (log.children.length > MAX_LOG_ENTRIES) { + log.removeChild(log.lastChild); + } +} + +// --- Explanation Panel --- + +function showExplanation(tab) { + const panel = $('#explanation-panel'); + const container = $('#explanation-steps'); + const items = EXPLANATIONS[tab] || []; + + if (items.length === 0) { + panel.classList.add('hidden'); + return; + } + + container.innerHTML = items.map((item, i) => + `
+

${item.title}

+

${item.detail}

+
` + ).join(''); + panel.classList.remove('hidden'); +} + +function highlightExpStep(index) { + $$('.explanation-step').forEach((el, i) => { + el.classList.toggle('active', i === index); + }); +} + +// --- Result Panel --- + +function showResult(status, latency, data, cls) { + const panel = $('#result-panel'); + const statusEl = $('#result-status'); + const latencyEl = $('#result-latency'); + const dataEl = $('#result-data'); + + statusEl.textContent = status; + statusEl.className = `result-status ${cls || ''}`; + latencyEl.textContent = `${latency.toFixed(1)}ms total`; + dataEl.textContent = JSON.stringify(data, null, 2); + panel.classList.remove('hidden'); +} + +// --- Sidebar Update --- + +async function updateSidebar() { + try { + const state = await apiGet('/api/db/state'); + + $('#stat-io').textContent = state.replica?.io_running || '--'; + $('#stat-io').className = 'stat-value ' + + (state.replica?.io_running === 'Yes' ? 'ok' : 'warn'); + + $('#stat-sql').textContent = state.replica?.sql_running || '--'; + $('#stat-sql').className = 'stat-value ' + + (state.replica?.sql_running === 'Yes' ? 'ok' : 'warn'); + + const lag = state.replica?.lag; + $('#stat-lag').textContent = lag !== null && lag !== undefined ? lag : '--'; + $('#stat-lag').className = 'stat-value ' + (lag === 0 ? 'ok' : 'warn'); + + $('#stat-primary-rows').textContent = state.primary?.students || '--'; + $('#stat-replica-rows').textContent = state.replica?.students || '--'; + $('#stat-log-rows').textContent = state.primary?.access_log_rows || '--'; + + // Courses + const courseList = $('#course-list'); + courseList.innerHTML = (state.primary?.courses || []).map(c => + `
  • + ${c.code} + ${c.enrolled} enrolled +
    • ` + ).join(''); + + // Indexes + const indexes = state.primary?.indexes || []; + const unique = [...new Set(indexes)]; + $('#index-list').textContent = unique.length > 0 ? unique.join(', ') : 'None'; + } catch { + // silently retry next poll + } +} + +// --- Replication Action --- + +async function doReplication() { + if (animating) return; + animating = true; + setButtonsDisabled(true); + clearAnimations(); + showExplanation('replication'); + + const name = $('#repl-name').value || 'Test Student'; + const major = $('#repl-major').value; + const email = name.toLowerCase().replace(/\s+/g, '.') + + Math.floor(Math.random() * 9999) + '@university.edu'; + + const result = await apiPost('/api/replication/write', { name, email, major }); + + if (result.error) { + showResult('ERROR', 0, result, 'rolled-back'); + animating = false; + setButtonsDisabled(false); + return; + } + + for (const step of result.steps) { + const idx = step.seq - 1; + highlightExpStep(idx); + logEvent(step); + + if (step.action === 'INSERT') { + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, 'arrow-op'); + } else if (step.action === 'SELECT') { + const cls = step.result === 'FOUND' ? 'arrow-success' : 'arrow-error'; + await animateArrow('arrow-app-replica', 'dot-app-replica', + 'latency-app-replica', step.latency_ms, cls); + } + await sleep(STEP_PAUSE_MS); + } + + const lastStep = result.steps[result.steps.length - 1]; + showResult( + lastStep.data?.lag_seconds === 0 ? 'REPLICATED' : 'LAG DETECTED', + result.total_ms, + result.steps.map(s => ({ action: s.action, result: s.result, ms: s.latency_ms })), + lastStep.data?.lag_seconds === 0 ? 'committed' : 'rolled-back' + ); + + await updateSidebar(); + animating = false; + setButtonsDisabled(false); +} + +// --- Consistency Action --- + +async function doTransfer() { + if (animating) return; + animating = true; + setButtonsDisabled(true); + clearAnimations(); + showExplanation('consistency'); + + const studentId = $('#tx-student').value; + const fromCourse = $('#tx-from').value; + const toCourse = $('#tx-to').value; + + const result = await apiPost('/api/consistency/transfer', { + student_id: studentId, from_course: fromCourse, to_course: toCourse, + }); + + if (result.error) { + showResult('ERROR', 0, result, 'rolled-back'); + animating = false; + setButtonsDisabled(false); + return; + } + + let expIdx = 0; + for (const step of result.steps) { + logEvent(step); + + if (step.action === 'BEGIN') { + highlightExpStep(0); + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, 'arrow-op'); + } else if (step.action.startsWith('DELETE') || step.action.startsWith('UPDATE')) { + if (expIdx < 1) expIdx = 1; + highlightExpStep(expIdx); + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, 'arrow-op'); + } else if (step.action.startsWith('INSERT')) { + expIdx = 2; + highlightExpStep(2); + const cls = step.result === 'OK' ? 'arrow-success' : 'arrow-error'; + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, cls); + } else if (step.action === 'COMMIT' || step.action === 'ROLLBACK') { + highlightExpStep(3); + const cls = step.action === 'COMMIT' ? 'arrow-success' : 'arrow-error'; + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, cls); + } + await sleep(STEP_PAUSE_MS / 2); + } + + const committed = result.outcome === 'COMMITTED'; + showResult( + result.outcome, + result.total_ms, + result.steps.map(s => ({ action: s.action, result: s.result, ms: s.latency_ms })), + committed ? 'committed' : 'rolled-back' + ); + + await updateSidebar(); + animating = false; + setButtonsDisabled(false); +} + +// --- Schema Action --- + +async function doExplain() { + if (animating) return; + animating = true; + setButtonsDisabled(true); + clearAnimations(); + showExplanation('schema'); + + const studentId = $('#idx-student').value; + const resource = $('#idx-resource').value; + + const result = await apiPost('/api/schema/explain', { + student_id: studentId, resource, + }); + + highlightExpStep(0); + for (const step of result.steps) { + logEvent(step); + await animateArrow('arrow-app-primary', 'dot-app-primary', + 'latency-app-primary', step.latency_ms, 'arrow-op'); + await sleep(STEP_PAUSE_MS); + } + + const plan = result.steps[0]?.data || {}; + const rows = plan.rows || plan.row || '?'; + const keyUsed = plan.key || 'NONE (full scan)'; + + showResult( + `Rows scanned: ${rows} | Key: ${keyUsed}`, + result.total_ms, + plan, + rows > 1000 ? 'rolled-back' : 'committed' + ); + + await updateSidebar(); + animating = false; + setButtonsDisabled(false); +} + +async function doAddIndex() { + const result = await apiPost('/api/schema/add-index', {}); + logEvent({ action: 'CREATE INDEX', target: 'primary', + result: result.result, latency_ms: result.latency_ms }); + await updateSidebar(); +} + +async function doDropIndex() { + const result = await apiPost('/api/schema/drop-index', {}); + logEvent({ action: 'DROP INDEX', target: 'primary', + result: result.result, latency_ms: result.latency_ms }); + await updateSidebar(); +} + +// --- Reset --- + +async function doReset() { + const result = await apiPost('/api/db/reset', {}); + logEvent({ action: 'RESET DB', target: 'primary', + result: result.result || 'OK', latency_ms: 0 }); + await updateSidebar(); + $('#result-panel').classList.add('hidden'); + $('#explanation-panel').classList.add('hidden'); + clearAnimations(); +} + +// --- Init --- + +function initTabs() { + $$('.tab').forEach(tab => { + tab.addEventListener('click', () => { + $$('.tab').forEach(t => t.classList.remove('active')); + tab.classList.add('active'); + const target = tab.dataset.tab; + $$('.controls').forEach(p => { + p.classList.toggle('active', p.id === `controls-${target}`); + }); + const desc = $('#pattern-description'); + if (desc && PATTERN_DESCRIPTIONS[target]) { + desc.textContent = PATTERN_DESCRIPTIONS[target]; + } + showExplanation(target); + clearAnimations(); + $('#result-panel').classList.add('hidden'); + }); + }); +} + +// --- SQL Console --- + +let consoleHistory = []; +let consoleHistoryIndex = -1; + +async function doSqlExec() { + const input = $('#console-input'); + const query = input.value.trim(); + if (!query) return; + + const target = document.querySelector('input[name="sql-target"]:checked').value; + const output = $('#console-output'); + + // Add to history + consoleHistory.unshift(query); + consoleHistoryIndex = -1; + + // Show query + const queryDiv = document.createElement('div'); + queryDiv.className = 'console-query'; + queryDiv.textContent = `mysql(${target})> ${query}`; + output.appendChild(queryDiv); + + const result = await apiPost('/api/sql/exec', { query, target }); + + if (result.error) { + const errDiv = document.createElement('div'); + errDiv.className = 'console-error'; + errDiv.textContent = `ERROR: ${result.error}`; + output.appendChild(errDiv); + } else if (result.columns) { + // Build HTML table + const table = document.createElement('table'); + table.className = 'console-result-table'; + const thead = document.createElement('thead'); + const headerRow = document.createElement('tr'); + result.columns.forEach(col => { + const th = document.createElement('th'); + th.textContent = col; + headerRow.appendChild(th); + }); + thead.appendChild(headerRow); + table.appendChild(thead); + + const tbody = document.createElement('tbody'); + const maxRows = Math.min(result.rows.length, 20); + for (let i = 0; i < maxRows; i++) { + const tr = document.createElement('tr'); + result.columns.forEach(col => { + const td = document.createElement('td'); + td.textContent = result.rows[i][col] ?? 'NULL'; + tr.appendChild(td); + }); + tbody.appendChild(tr); + } + table.appendChild(tbody); + output.appendChild(table); + + const meta = document.createElement('div'); + meta.className = 'console-meta'; + const truncated = result.rows.length > 20 ? ` (showing 20 of ${result.rows.length})` : ''; + meta.textContent = + `${result.row_count} row(s)${truncated} in ${result.latency_ms}ms`; + output.appendChild(meta); + } else { + const metaDiv = document.createElement('div'); + metaDiv.className = 'console-meta'; + metaDiv.textContent = + `${result.affected_rows} row(s) affected in ${result.latency_ms}ms`; + output.appendChild(metaDiv); + } + + // Log it + logEvent({ + action: query.split(' ').slice(0, 2).join(' '), + target, + result: result.error ? 'ERROR' : 'OK', + latency_ms: result.latency_ms || 0, + }); + + output.scrollTop = output.scrollHeight; + input.value = ''; + await updateSidebar(); +} + +function initConsole() { + $('#console-submit').addEventListener('click', doSqlExec); + $('#console-input').addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + doSqlExec(); + } else if (e.key === 'ArrowUp' && consoleHistory.length > 0) { + consoleHistoryIndex = Math.min( + consoleHistoryIndex + 1, consoleHistory.length - 1 + ); + $('#console-input').value = consoleHistory[consoleHistoryIndex]; + } else if (e.key === 'ArrowDown') { + consoleHistoryIndex = Math.max(consoleHistoryIndex - 1, -1); + $('#console-input').value = + consoleHistoryIndex >= 0 ? consoleHistory[consoleHistoryIndex] : ''; + } + }); +} + +function initButtons() { + $('#repl-write').addEventListener('click', doReplication); + $('#tx-transfer').addEventListener('click', doTransfer); + $('#idx-explain').addEventListener('click', doExplain); + $('#idx-add').addEventListener('click', doAddIndex); + $('#idx-drop').addEventListener('click', doDropIndex); + $('#btn-reset-db').addEventListener('click', doReset); +} + +function startPolling() { + updateSidebar(); + pollTimer = setInterval(updateSidebar, POLL_INTERVAL_MS); +} + +document.addEventListener('DOMContentLoaded', () => { + initTabs(); + initButtons(); + initConsole(); + startPolling(); + showExplanation('replication'); +}); diff --git a/10-databases/visualizer/cleanup.sh b/10-databases/visualizer/cleanup.sh new file mode 100755 index 0000000..c47e822 --- /dev/null +++ b/10-databases/visualizer/cleanup.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Cleaning up visualizer environment ===" +docker compose down -v --remove-orphans 2>/dev/null || true +echo "=== Cleanup complete ===" diff --git a/10-databases/visualizer/cloudformation.yaml b/10-databases/visualizer/cloudformation.yaml new file mode 100644 index 0000000..173b925 --- /dev/null +++ b/10-databases/visualizer/cloudformation.yaml @@ -0,0 +1,92 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + Lab 10 - Database Scalability Visualizer. + Launches a t3.medium EC2 instance with Docker and Docker Compose + pre-installed. Clones the course repository, runs the MySQL + primary-replica cluster and visualizer automatically. SSH and + visualizer web access enabled. + +Parameters: + LatestAmiId: + Description: Amazon Linux 2023 AMI (auto-resolved via SSM) + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 + +Resources: + LabSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Lab 10 Database Visualizer - SSH and web access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + Description: SSH access + - IpProtocol: tcp + FromPort: 8081 + ToPort: 8081 + CidrIp: 0.0.0.0/0 + Description: Database Visualizer web UI + + LabInstance: + Type: AWS::EC2::Instance + Properties: + InstanceType: t3.medium + ImageId: !Ref LatestAmiId + KeyName: vockey + SecurityGroupIds: + - !GetAtt LabSecurityGroup.GroupId + Tags: + - Key: Name + Value: lab10-db-visualizer + UserData: + Fn::Base64: | + #!/bin/bash + set -ex + + # Install Docker from Amazon Linux repos + dnf update -y + dnf install -y docker git + systemctl enable docker + systemctl start docker + usermod -aG docker ec2-user + + # Install Docker Compose plugin + ARCH=$(uname -m) + mkdir -p /usr/local/lib/docker/cli-plugins + + curl -fsSL \ + "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${ARCH}" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose + chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + + # Clone the repo and run the visualizer as ec2-user + su - ec2-user -c ' + git clone https://github.com/gamaware/system-design-course.git + cd system-design-course/10-databases/visualizer + chmod +x setup.sh cleanup.sh + ./setup.sh + ' + + # Signal that setup is complete + touch /home/ec2-user/LAB_READY + +Outputs: + InstanceId: + Description: EC2 instance ID + Value: !Ref LabInstance + + PublicIP: + Description: Public IP address (use for SSH and visualizer) + Value: !GetAtt LabInstance.PublicIp + + SSHCommand: + Description: SSH command (use the labsuser.pem key from AWS Details) + Value: !Sub >- + ssh -i labsuser.pem ec2-user@${LabInstance.PublicIp} + + VisualizerURL: + Description: Database Visualizer web UI + Value: !Sub >- + http://${LabInstance.PublicIp}:8081 diff --git a/10-databases/visualizer/docker-compose.yml b/10-databases/visualizer/docker-compose.yml new file mode 100644 index 0000000..f0663ff --- /dev/null +++ b/10-databases/visualizer/docker-compose.yml @@ -0,0 +1,68 @@ +services: + mysql-primary: + image: mysql:8 + container_name: mysql-primary + ports: + - "3306:3306" + environment: + MYSQL_ROOT_PASSWORD: rootpass + MYSQL_DATABASE: university + volumes: + - ../mysql/init/primary-init.sql:/docker-entrypoint-initdb.d/01-init.sql + - primary-data:/var/lib/mysql + command: + - --server-id=1 + - --log-bin=mysql-bin + - --gtid-mode=ON + - --enforce-gtid-consistency=ON + - --binlog-format=ROW + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpass"] + interval: 5s + timeout: 3s + retries: 10 + + mysql-replica: + image: mysql:8 + container_name: mysql-replica + ports: + - "3307:3306" + environment: + MYSQL_ROOT_PASSWORD: rootpass + volumes: + - replica-data:/var/lib/mysql + command: + - --server-id=2 + - --log-bin=mysql-bin + - --gtid-mode=ON + - --enforce-gtid-consistency=ON + - --binlog-format=ROW + depends_on: + mysql-primary: + condition: service_healthy + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpass"] + interval: 5s + timeout: 3s + retries: 10 + + visualizer: + build: . + container_name: db-visualizer + ports: + - "8081:8080" + environment: + - MYSQL_PRIMARY_HOST=mysql-primary + - MYSQL_REPLICA_HOST=mysql-replica + - MYSQL_USER=root + - MYSQL_PASS=rootpass + - MYSQL_DB=university + depends_on: + mysql-primary: + condition: service_healthy + mysql-replica: + condition: service_healthy + +volumes: + primary-data: + replica-data: diff --git a/10-databases/visualizer/index.html b/10-databases/visualizer/index.html new file mode 100644 index 0000000..0a98130 --- /dev/null +++ b/10-databases/visualizer/index.html @@ -0,0 +1,283 @@ + + + + + + Database Scalability Visualizer — Lab 10 + + + +
    +
    + + + +
    + +
    + +
    + Write to primary, read from replica. Observe replication lag in real time. +
    + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + replication + + + + + + + + + + + + + + + + + + + + + + + + + + + + + </> + + Visualizer + + + + + + + + + + + + + + + + Primary + MySQL :3306 + + + + + + + + + + + + + + + RO + + Replica + MySQL :3307 + + + + + + +
    + + + + +
    + +
    +
    + + +
    +
    + + +
    + +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    + + +
    +
    + + +
    +
    + + +
    + + + +
    +
    + + + +
    + + +
    + +
    +
    +
    +

    Event Log

    +
    +
    + +
    +
    + SQL Console +
    +
    + + +
    +
    +
    + + +
    +
    +
    +
    +
    +
    + + + + diff --git a/10-databases/visualizer/server.py b/10-databases/visualizer/server.py new file mode 100644 index 0000000..1ba7138 --- /dev/null +++ b/10-databases/visualizer/server.py @@ -0,0 +1,509 @@ +"""Database scalability visualizer API bridge — MySQL primary-replica.""" + +import contextlib +import json +import os +import time +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn + +import pymysql + +PRIMARY_HOST = os.environ.get("MYSQL_PRIMARY_HOST", "mysql-primary") +REPLICA_HOST = os.environ.get("MYSQL_REPLICA_HOST", "mysql-replica") +MYSQL_USER = os.environ.get("MYSQL_USER", "root") +MYSQL_PASS = os.environ.get("MYSQL_PASS", "rootpass") +MYSQL_DB = os.environ.get("MYSQL_DB", "university") + + +def get_conn(host): + """Return a MySQL connection to the specified host.""" + return pymysql.connect( + host=host, user=MYSQL_USER, password=MYSQL_PASS, + database=MYSQL_DB, cursorclass=pymysql.cursors.DictCursor, + autocommit=True, + ) + + +def step_entry(seq, action, target, result, latency_ms, data=None): + """Build a single step trace entry.""" + entry = { + "seq": seq, "action": action, "target": target, + "result": result, "latency_ms": round(latency_ms, 2), + } + if data is not None: + entry["data"] = data + return entry + + +# ---- Replication Tab ---- + +def replication_write(body): + """Insert a student on the primary and read from replica.""" + name = body.get("name", "Test Student") + email = body.get("email", "test@university.edu") + major = body.get("major", "Computer Science") + steps = [] + total_start = time.perf_counter() + seq = 1 + + # Write to primary + conn = get_conn(PRIMARY_HOST) + try: + t0 = time.perf_counter() + with conn.cursor() as cur: + cur.execute( + "INSERT INTO students (name, email, major) VALUES (%s, %s, %s)", + (name, email, major), + ) + new_id = cur.lastrowid + t1 = time.perf_counter() + steps.append(step_entry(seq, "INSERT", "primary", "OK", + (t1 - t0) * 1000, {"student_id": new_id})) + seq += 1 + finally: + conn.close() + + # Read from replica + conn = get_conn(REPLICA_HOST) + try: + t0 = time.perf_counter() + with conn.cursor() as cur: + cur.execute("SELECT * FROM students WHERE student_id = %s", (new_id,)) + row = cur.fetchone() + t1 = time.perf_counter() + found = row is not None + steps.append(step_entry( + seq, "SELECT", "replica", + "FOUND" if found else "NOT YET REPLICATED", + (t1 - t0) * 1000, row, + )) + seq += 1 + + # Check replication lag + t0 = time.perf_counter() + with conn.cursor() as cur: + cur.execute("SHOW REPLICA STATUS") + status = cur.fetchone() + t1 = time.perf_counter() + lag = status.get("Seconds_Behind_Source", "N/A") if status else "N/A" + steps.append(step_entry(seq, "SHOW REPLICA STATUS", "replica", "OK", + (t1 - t0) * 1000, {"lag_seconds": lag})) + finally: + conn.close() + + total_ms = (time.perf_counter() - total_start) * 1000 + return {"pattern": "replication", "steps": steps, "total_ms": round(total_ms, 2)} + + +def replication_status(_body): + """Get current replication status.""" + conn = get_conn(REPLICA_HOST) + try: + with conn.cursor() as cur: + cur.execute("SHOW REPLICA STATUS") + status = cur.fetchone() + if not status: + return {"error": "Replication not configured"} + return { + "io_running": status.get("Replica_IO_Running"), + "sql_running": status.get("Replica_SQL_Running"), + "lag_seconds": status.get("Seconds_Behind_Source"), + "source_host": status.get("Source_Host"), + } + finally: + conn.close() + + +# ---- Consistency Tab (ACID Transactions) ---- + +def consistency_transfer(body): + """Transfer enrollment between courses in a transaction.""" + student_id = int(body.get("student_id", 1)) + from_course = body.get("from_course", "CS101") + to_course = body.get("to_course", "PHYS101") + steps = [] + total_start = time.perf_counter() + seq = 1 + + conn = get_conn(PRIMARY_HOST) + conn.autocommit(False) + try: + t0 = time.perf_counter() + conn.begin() + t1 = time.perf_counter() + steps.append(step_entry(seq, "BEGIN", "primary", "OK", (t1 - t0) * 1000)) + seq += 1 + + with conn.cursor() as cur: + # Get course IDs + cur.execute("SELECT course_id FROM courses WHERE code = %s", (from_course,)) + from_row = cur.fetchone() + cur.execute("SELECT course_id FROM courses WHERE code = %s", (to_course,)) + to_row = cur.fetchone() + + if not from_row or not to_row: + conn.rollback() + return {"error": f"Course not found: {from_course} or {to_course}"} + + from_id = from_row["course_id"] + to_id = to_row["course_id"] + + # Delete old enrollment + t0 = time.perf_counter() + cur.execute("DELETE FROM enrollments WHERE student_id = %s AND course_id = %s", + (student_id, from_id)) + affected = cur.rowcount + t1 = time.perf_counter() + steps.append(step_entry(seq, f"DELETE enrollment ({from_course})", "primary", + "OK" if affected > 0 else "NO ROWS", + (t1 - t0) * 1000, {"rows_affected": affected})) + seq += 1 + + # Update from-course count + t0 = time.perf_counter() + cur.execute("UPDATE courses SET enrolled = enrolled - 1 WHERE course_id = %s AND enrolled > 0", + (from_id,)) + t1 = time.perf_counter() + steps.append(step_entry(seq, f"UPDATE {from_course} enrolled-1", "primary", "OK", + (t1 - t0) * 1000)) + seq += 1 + + # Insert new enrollment + t0 = time.perf_counter() + try: + cur.execute("INSERT INTO enrollments (student_id, course_id) VALUES (%s, %s)", + (student_id, to_id)) + t1 = time.perf_counter() + steps.append(step_entry(seq, f"INSERT enrollment ({to_course})", "primary", "OK", + (t1 - t0) * 1000)) + except pymysql.IntegrityError as exc: + t1 = time.perf_counter() + conn.rollback() + steps.append(step_entry(seq, f"INSERT enrollment ({to_course})", "primary", + "CONSTRAINT VIOLATION", (t1 - t0) * 1000, + {"error": str(exc)})) + steps.append(step_entry(seq + 1, "ROLLBACK", "primary", "OK", 0)) + total_ms = (time.perf_counter() - total_start) * 1000 + return {"pattern": "consistency", "steps": steps, + "total_ms": round(total_ms, 2), "outcome": "ROLLED BACK"} + seq += 1 + + # Update to-course count + t0 = time.perf_counter() + cur.execute("UPDATE courses SET enrolled = enrolled + 1 WHERE course_id = %s", + (to_id,)) + t1 = time.perf_counter() + steps.append(step_entry(seq, f"UPDATE {to_course} enrolled+1", "primary", "OK", + (t1 - t0) * 1000)) + seq += 1 + + # COMMIT + t0 = time.perf_counter() + conn.commit() + t1 = time.perf_counter() + steps.append(step_entry(seq, "COMMIT", "primary", "OK", (t1 - t0) * 1000)) + + total_ms = (time.perf_counter() - total_start) * 1000 + return {"pattern": "consistency", "steps": steps, + "total_ms": round(total_ms, 2), "outcome": "COMMITTED"} + except Exception as exc: + conn.rollback() + return {"error": str(exc)} + finally: + conn.close() + + +# ---- Schema Tab (Indexing) ---- + +def schema_explain(body): + """Run EXPLAIN on a query with optional index creation.""" + student_id = int(body.get("student_id", 3)) + resource = body.get("resource", "resource-10") + steps = [] + total_start = time.perf_counter() + seq = 1 + + conn = get_conn(PRIMARY_HOST) + try: + with conn.cursor() as cur: + # Run EXPLAIN + t0 = time.perf_counter() + cur.execute( + "EXPLAIN SELECT * FROM access_log WHERE student_id = %s AND resource = %s", + (student_id, resource), + ) + plan = cur.fetchone() + t1 = time.perf_counter() + steps.append(step_entry(seq, "EXPLAIN", "primary", "OK", + (t1 - t0) * 1000, plan)) + seq += 1 + + # Run actual query with timing + t0 = time.perf_counter() + cur.execute( + "SELECT COUNT(*) as cnt FROM access_log WHERE student_id = %s AND resource = %s", + (student_id, resource), + ) + result = cur.fetchone() + t1 = time.perf_counter() + steps.append(step_entry(seq, "SELECT COUNT(*)", "primary", "OK", + (t1 - t0) * 1000, result)) + + total_ms = (time.perf_counter() - total_start) * 1000 + return {"pattern": "schema", "steps": steps, "total_ms": round(total_ms, 2)} + finally: + conn.close() + + +def schema_add_index(_body): + """Add composite index on access_log.""" + conn = get_conn(PRIMARY_HOST) + try: + with conn.cursor() as cur: + t0 = time.perf_counter() + try: + cur.execute( + "CREATE INDEX idx_student_resource ON access_log (student_id, resource)" + ) + result = "CREATED" + except pymysql.err.OperationalError: + result = "ALREADY EXISTS" + t1 = time.perf_counter() + return {"action": "CREATE INDEX", "result": result, + "latency_ms": round((t1 - t0) * 1000, 2)} + finally: + conn.close() + + +def schema_drop_index(_body): + """Drop the composite index.""" + conn = get_conn(PRIMARY_HOST) + try: + with conn.cursor() as cur: + t0 = time.perf_counter() + try: + cur.execute("DROP INDEX idx_student_resource ON access_log") + result = "DROPPED" + except pymysql.err.OperationalError: + result = "NOT FOUND" + t1 = time.perf_counter() + return {"action": "DROP INDEX", "result": result, + "latency_ms": round((t1 - t0) * 1000, 2)} + finally: + conn.close() + + +# ---- Database State ---- + +def db_state(_body): + """Gather current database state for sidebar.""" + state = {"primary": {}, "replica": {}} + + # Primary stats + conn = get_conn(PRIMARY_HOST) + try: + with conn.cursor() as cur: + cur.execute("SELECT COUNT(*) as cnt FROM students") + state["primary"]["students"] = cur.fetchone()["cnt"] + cur.execute("SELECT code, enrolled FROM courses ORDER BY code") + state["primary"]["courses"] = cur.fetchall() + cur.execute("SELECT COUNT(*) as cnt FROM access_log") + state["primary"]["access_log_rows"] = cur.fetchone()["cnt"] + cur.execute("SHOW INDEX FROM access_log WHERE Key_name != 'PRIMARY'") + state["primary"]["indexes"] = [r["Key_name"] for r in cur.fetchall()] + finally: + conn.close() + + # Replica lag + conn = get_conn(REPLICA_HOST) + try: + with conn.cursor() as cur: + cur.execute("SHOW REPLICA STATUS") + rs = cur.fetchone() + if rs: + state["replica"]["io_running"] = rs.get("Replica_IO_Running") + state["replica"]["sql_running"] = rs.get("Replica_SQL_Running") + state["replica"]["lag"] = rs.get("Seconds_Behind_Source") + cur.execute("SELECT COUNT(*) as cnt FROM students") + state["replica"]["students"] = cur.fetchone()["cnt"] + finally: + conn.close() + + return state + + +def db_reset(_body): + """Reset database to initial state.""" + conn = get_conn(PRIMARY_HOST) + try: + with conn.cursor() as cur: + cur.execute("DELETE FROM enrollments") + cur.execute("DELETE FROM students WHERE student_id > 10") + cur.execute("DELETE FROM courses WHERE code NOT IN ('CS101','CS201','MATH101','PHYS101')") + cur.execute("""UPDATE courses SET enrolled = CASE code + WHEN 'CS101' THEN 4 WHEN 'CS201' THEN 2 + WHEN 'MATH101' THEN 2 WHEN 'PHYS101' THEN 2 END""") + # Re-insert original enrollments + cur.execute("""INSERT INTO enrollments (student_id, course_id) VALUES + (1,1),(3,1),(5,1),(8,1),(1,2),(3,2),(2,3),(6,3),(4,4),(9,4)""") + with contextlib.suppress(pymysql.err.OperationalError): + cur.execute("DROP INDEX idx_student_resource ON access_log") + return {"result": "OK", "message": "Database reset to initial state"} + finally: + conn.close() + + +ALLOWED_SQL = frozenset({ + "SELECT", "INSERT", "UPDATE", "DELETE", + "EXPLAIN", "SHOW", "DESCRIBE", "DESC", + "ANALYZE", "START", "COMMIT", "ROLLBACK", +}) + + +def sql_exec(body): + """Execute arbitrary SQL against primary or replica.""" + query = body.get("query", "").strip() + target = body.get("target", "primary") + if not query: + return {"error": "Empty query"} + + # Reject multi-statement queries + if ";" in query.rstrip(";"): + return {"error": "Multi-statement queries are not allowed"} + + # Allowlist: only permit known safe SQL commands + first_word = query.split()[0].upper() if query.split() else "" + if first_word not in ALLOWED_SQL: + return {"error": f"Command not allowed: {first_word}. " + f"Permitted: {', '.join(sorted(ALLOWED_SQL))}"} + + host = REPLICA_HOST if target == "replica" else PRIMARY_HOST + conn = get_conn(host) + try: + t0 = time.perf_counter() + with conn.cursor() as cur: + cur.execute(query) + if cur.description: + columns = [d[0] for d in cur.description] + rows = cur.fetchall() + t1 = time.perf_counter() + return { + "columns": columns, "rows": rows, + "row_count": len(rows), + "latency_ms": round((t1 - t0) * 1000, 2), + "target": target, "query": query, + } + t1 = time.perf_counter() + return { + "affected_rows": cur.rowcount, + "latency_ms": round((t1 - t0) * 1000, 2), + "target": target, "query": query, + } + except pymysql.MySQLError as exc: + return {"error": str(exc), "target": target, "query": query} + finally: + conn.close() + + +ROUTES = { + ("GET", "/api/db/state"): db_state, + ("POST", "/api/db/reset"): db_reset, + ("POST", "/api/replication/write"): replication_write, + ("GET", "/api/replication/status"): replication_status, + ("POST", "/api/consistency/transfer"): consistency_transfer, + ("POST", "/api/schema/explain"): schema_explain, + ("POST", "/api/schema/add-index"): schema_add_index, + ("POST", "/api/schema/drop-index"): schema_drop_index, + ("POST", "/api/sql/exec"): sql_exec, +} + +MIME_TYPES = { + ".html": "text/html", ".css": "text/css", + ".js": "application/javascript", ".svg": "image/svg+xml", +} + + +class Handler(SimpleHTTPRequestHandler): + """HTTP request handler for the database visualizer.""" + + def end_headers(self): + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + super().end_headers() + + def do_OPTIONS(self): # noqa: N802 + self.send_response(200) + self.end_headers() + + def do_GET(self): # noqa: N802 + path = self.path.split("?")[0] + handler = ROUTES.get(("GET", path)) + if handler: + self._send_json(handler(None)) + return + if path == "/": + path = "/index.html" + self._serve_static(path) + + def do_POST(self): # noqa: N802 + path = self.path.split("?")[0] + handler = ROUTES.get(("POST", path)) + if not handler: + self._send_json({"error": "Not found"}, 404) + return + content_length = int(self.headers.get("Content-Length", 0)) + body = {} + if content_length > 0: + raw = self.rfile.read(content_length) + try: + body = json.loads(raw.decode()) + except (json.JSONDecodeError, UnicodeDecodeError): + self._send_json({"error": "Invalid JSON body"}, 400) + return + try: + result = handler(body) + self._send_json(result) + except Exception as exc: # noqa: BLE001 + self._send_json({"error": str(exc)}, 500) + + def _send_json(self, data, status=200): + payload = json.dumps(data, default=str).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def _serve_static(self, path): + filename = path.lstrip("/") + resolved = os.path.realpath(filename) + if not resolved.startswith(os.path.realpath(os.getcwd())): + self._send_json({"error": "Forbidden"}, 403) + return + ext = os.path.splitext(filename)[1] + mime = MIME_TYPES.get(ext, "application/octet-stream") + try: + with open(resolved, "rb") as f: + content = f.read() + self.send_response(200) + self.send_header("Content-Type", mime) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + except FileNotFoundError: + self._send_json({"error": "Not found"}, 404) + + def log_message(self, format, *args): # noqa: A002 + pass + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + +if __name__ == "__main__": + print("Database scalability visualizer listening on :8080") + server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler) # noqa: S104 + server.serve_forever() diff --git a/10-databases/visualizer/setup.sh b/10-databases/visualizer/setup.sh new file mode 100755 index 0000000..1e56f92 --- /dev/null +++ b/10-databases/visualizer/setup.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +echo "=== Database Scalability Visualizer ===" +echo "" + +if ! command -v docker &> /dev/null; then + echo "ERROR: Docker is not installed." + exit 1 +fi + +if ! docker info &> /dev/null 2>&1; then + echo "ERROR: Docker daemon is not running." + exit 1 +fi + +if ! docker compose version &> /dev/null 2>&1; then + echo "ERROR: Docker Compose is not available." + exit 1 +fi + +echo "Starting MySQL primary, replica, and visualizer..." +docker compose up -d --build + +echo "" +echo "Waiting for MySQL to be healthy..." +i=0 +while [ "$i" -lt 90 ]; do + i=$((i + 1)) + if docker exec mysql-primary mysqladmin ping -u root -prootpass 2>/dev/null | grep -q "alive" \ + && docker exec mysql-replica mysqladmin ping -u root -prootpass 2>/dev/null | grep -q "alive"; then + echo " MySQL primary: ready" + echo " MySQL replica: ready" + break + fi + if [ "$i" -eq 90 ]; then + echo "ERROR: MySQL did not start within 90 seconds." + exit 1 + fi + sleep 1 +done + +# Configure replication +echo "" +echo "Configuring replication..." +docker exec mysql-primary mysql -u root -prootpass \ + -e "CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'replpass'; GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; FLUSH PRIVILEGES;" \ + 2>/dev/null + +docker exec mysql-replica mysql -u root -prootpass \ + -e "STOP REPLICA; CHANGE REPLICATION SOURCE TO SOURCE_HOST='mysql-primary', SOURCE_USER='repl', SOURCE_PASSWORD='replpass', SOURCE_AUTO_POSITION=1, GET_SOURCE_PUBLIC_KEY=1; START REPLICA;" \ + 2>/dev/null + +docker exec mysql-replica mysql -u root -prootpass \ + -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" \ + 2>/dev/null + +echo " Replication configured." + +echo "" +echo "=== Visualizer ready ===" +echo "" +echo "Open http://localhost:8081 in your browser." +echo "" +echo "Three tabs: Replication | Consistency (ACID) | Schema & Indexing" diff --git a/10-databases/visualizer/style.css b/10-databases/visualizer/style.css new file mode 100644 index 0000000..f105e05 --- /dev/null +++ b/10-databases/visualizer/style.css @@ -0,0 +1,615 @@ +:root { + --color-bg: #0f172a; + --color-bg-surface: #1e293b; + --color-bg-elevated: #334155; + --color-text: #e2e8f0; + --color-text-muted: #94a3b8; + --color-border: #475569; + --color-primary: #3b82f6; + --color-replica: #8b5cf6; + --color-success: #22c55e; + --color-warning: #f97316; + --color-error: #ef4444; + --font-sans: system-ui, -apple-system, sans-serif; + --font-mono: 'SF Mono', 'Fira Code', monospace; + --radius: 8px; + --radius-lg: 12px; + --sidebar-width: 280px; + --tab-height: 52px; + --bottom-bar-height: 200px; +} + +*, +*::before, +*::after { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +html { + min-width: 900px; +} + +body { + font-family: var(--font-sans); + background: var(--color-bg); + color: var(--color-text); + line-height: 1.5; + min-height: 100vh; + display: flex; + flex-direction: column; +} + +h2 { + font-size: 1.2rem; + font-weight: 600; +} + +h3 { + font-size: 0.9rem; + color: var(--color-text-muted); + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 0.5rem; + font-weight: 600; +} + +/* --- Tab Bar --- */ +.tab-bar { + display: flex; + justify-content: space-between; + height: var(--tab-height); + background: var(--color-bg-surface); + border-bottom: 1px solid var(--color-border); + padding: 6px 1rem; + align-items: center; +} + +.tab-group { + display: flex; + gap: 4px; +} + +.tab { + padding: 8px 18px; + border: 1px solid var(--color-border); + border-radius: var(--radius); + background: transparent; + color: var(--color-text-muted); + cursor: pointer; + font-size: 0.85rem; + font-weight: 500; + transition: all 0.15s ease; +} + +.tab:hover { + background: var(--color-bg-elevated); + color: var(--color-text); +} + +.tab.active { + background: var(--color-primary); + color: #fff; + border-color: var(--color-primary); +} + +/* --- Layout --- */ +.layout { + display: flex; + flex: 1; + overflow: hidden; +} + +.main-area { + flex: 1; + display: flex; + flex-direction: column; + padding: 1rem; + gap: 1rem; + overflow-y: auto; +} + +.sidebar { + width: var(--sidebar-width); + background: var(--color-bg-surface); + border-left: 1px solid var(--color-border); + padding: 1rem; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 1rem; +} + +/* --- Pattern Description --- */ +.pattern-description { + padding: 8px 1rem; + background: var(--color-bg-surface); + border-bottom: 1px solid var(--color-border); + color: var(--color-text-muted); + font-size: 0.85rem; +} + +/* --- SVG Diagram --- */ +.diagram-container { + background: var(--color-bg-surface); + border-radius: var(--radius-lg); + border: 1px solid var(--color-border); + padding: 1rem; +} + +.diagram-container svg { + width: 100%; + height: auto; +} + +.arrow { + opacity: 0; + transition: opacity 0.3s; +} + +.arrow-active { + opacity: 1; +} + +.arrow-success { + stroke: var(--color-success); +} + +.arrow-op { + stroke: var(--color-primary); +} + +.arrow-error { + stroke: var(--color-error); +} + +.node-pulse { + animation: pulse 0.6s ease-in-out; +} + +@keyframes pulse { + 0% { transform: scale(1); } + 50% { transform: scale(1.03); } + 100% { transform: scale(1); } +} + +.latency-label { + fill: #94a3b8; + font-size: 11px; + font-family: var(--font-mono); +} + +.travel-dot { + transition: opacity 0.2s; +} + +/* --- Controls Panel --- */ +.controls-panel { + background: var(--color-bg-surface); + border-radius: var(--radius-lg); + border: 1px solid var(--color-border); + padding: 1rem; +} + +.controls { + display: none; + gap: 1rem; + align-items: end; + flex-wrap: wrap; +} + +.controls.active { + display: flex; +} + +.control-group { + display: flex; + flex-direction: column; + gap: 4px; +} + +.control-group label { + font-size: 0.8rem; + color: var(--color-text-muted); + font-weight: 500; +} + +.control-group input, +.control-group select { + padding: 6px 10px; + border-radius: var(--radius); + border: 1px solid var(--color-border); + background: var(--color-bg); + color: var(--color-text); + font-size: 0.85rem; + font-family: var(--font-mono); + width: 160px; +} + +/* --- Buttons --- */ +.btn { + padding: 8px 16px; + border-radius: var(--radius); + border: 1px solid transparent; + cursor: pointer; + font-size: 0.85rem; + font-weight: 500; + transition: all 0.15s ease; +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-primary { + background: var(--color-primary); + color: #fff; +} + +.btn-primary:hover:not(:disabled) { + background: #2563eb; +} + +.btn-secondary { + background: var(--color-bg-elevated); + color: var(--color-text); + border-color: var(--color-border); +} + +.btn-danger { + background: var(--color-error); + color: #fff; +} + +.btn-reset { + background: transparent; + color: var(--color-text-muted); + border: 1px solid var(--color-border); + font-size: 0.8rem; +} + +.btn-reset:hover { + color: var(--color-error); + border-color: var(--color-error); +} + +/* --- Result Panel --- */ +.result-panel { + background: var(--color-bg-surface); + border-radius: var(--radius-lg); + border: 1px solid var(--color-border); + padding: 1rem; +} + +.result-panel.hidden { + display: none; +} + +.result-header { + display: flex; + justify-content: space-between; + margin-bottom: 0.5rem; +} + +.result-status { + font-weight: 600; + font-size: 0.85rem; +} + +.result-status.committed { + color: var(--color-success); +} + +.result-status.rolled-back { + color: var(--color-error); +} + +.result-latency { + color: var(--color-text-muted); + font-size: 0.85rem; + font-family: var(--font-mono); +} + +.result-data { + font-family: var(--font-mono); + font-size: 0.8rem; + color: var(--color-text); + background: var(--color-bg); + border-radius: var(--radius); + padding: 0.75rem; + max-height: 150px; + overflow-y: auto; + white-space: pre-wrap; +} + +/* --- Explanation Panel --- */ +.explanation-panel { + background: var(--color-bg-surface); + border-radius: var(--radius-lg); + border: 1px solid var(--color-border); + padding: 1rem; +} + +.explanation-panel.hidden { + display: none; +} + +.explanation-title { + margin-bottom: 0.75rem; + color: var(--color-text); + font-size: 1rem; +} + +.explanation-steps { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.explanation-step { + padding: 0.75rem; + background: var(--color-bg); + border-radius: var(--radius); + border-left: 3px solid var(--color-primary); +} + +.explanation-step.active { + border-left-color: var(--color-success); + background: rgba(34, 197, 94, 0.05); +} + +.explanation-step h4 { + font-size: 0.85rem; + margin-bottom: 4px; +} + +.explanation-step p { + font-size: 0.8rem; + color: var(--color-text-muted); +} + +/* --- Sidebar --- */ +.sidebar-section { + padding-bottom: 1rem; + border-bottom: 1px solid var(--color-border); +} + +.sidebar-section:last-child { + border-bottom: none; +} + +.stat-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.5rem; +} + +.stat { + display: flex; + flex-direction: column; +} + +.stat-label { + font-size: 0.75rem; + color: var(--color-text-muted); +} + +.stat-value { + font-size: 1rem; + font-weight: 600; + font-family: var(--font-mono); +} + +.stat-value.ok { + color: var(--color-success); +} + +.stat-value.warn { + color: var(--color-warning); +} + +.course-list { + list-style: none; + display: flex; + flex-direction: column; + gap: 4px; +} + +.course-item { + display: flex; + justify-content: space-between; + font-size: 0.8rem; + padding: 4px 8px; + background: var(--color-bg); + border-radius: var(--radius); +} + +.course-code { + font-family: var(--font-mono); + font-weight: 600; +} + +.course-enrolled { + color: var(--color-text-muted); +} + +/* --- Bottom Bar (Event Log) --- */ +.bottom-bar { + background: var(--color-bg-surface); + border-top: 1px solid var(--color-border); + padding: 0.75rem 1rem; + max-height: var(--bottom-bar-height); +} + +.event-log { + font-family: var(--font-mono); + font-size: 0.75rem; + max-height: 150px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 2px; +} + +.log-entry { + display: flex; + gap: 0.5rem; + padding: 2px 6px; + border-radius: 3px; +} + +.log-entry:nth-child(odd) { + background: rgba(255, 255, 255, 0.02); +} + +.log-time { + color: var(--color-text-muted); + min-width: 70px; +} + +.log-action { + color: var(--color-primary); + min-width: 100px; +} + +.log-target { + min-width: 70px; +} + +.log-target.primary { + color: var(--color-primary); +} + +.log-target.replica { + color: var(--color-replica); +} + +.log-result { + min-width: 60px; +} + +.log-result.ok { + color: var(--color-success); +} + +.log-result.error { + color: var(--color-error); +} + +.log-latency { + color: var(--color-text-muted); + margin-left: auto; +} + +/* --- Bottom Split --- */ +.bottom-split { + display: flex; + gap: 1rem; + height: 100%; +} + +.event-log-container { + flex: 1; + min-width: 0; +} + +/* --- SQL Console --- */ +.sql-console { + flex: 1; + min-width: 0; +} + +.sql-console details { + height: 100%; +} + +.sql-console summary { + cursor: pointer; + font-size: 0.9rem; + font-weight: 600; + color: var(--color-text-muted); + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 0.5rem; +} + +.console-body { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.console-target { + display: flex; + gap: 1rem; + font-size: 0.8rem; + color: var(--color-text-muted); +} + +.console-target label { + display: flex; + align-items: center; + gap: 4px; + cursor: pointer; +} + +.console-output { + font-family: var(--font-mono); + font-size: 0.75rem; + background: var(--color-bg); + border-radius: var(--radius); + padding: 0.5rem; + max-height: 100px; + overflow-y: auto; + white-space: pre-wrap; + color: var(--color-text); +} + +.console-input-row { + display: flex; + gap: 0.5rem; +} + +.console-input-row input { + flex: 1; + padding: 6px 10px; + border-radius: var(--radius); + border: 1px solid var(--color-border); + background: var(--color-bg); + color: var(--color-text); + font-family: var(--font-mono); + font-size: 0.8rem; +} + +.console-result-table { + border-collapse: collapse; + font-size: 0.7rem; + width: 100%; +} + +.console-result-table th, +.console-result-table td { + border: 1px solid var(--color-border); + padding: 2px 6px; + text-align: left; +} + +.console-result-table th { + background: var(--color-bg-elevated); + color: var(--color-text-muted); +} + +.console-query { + color: var(--color-primary); +} + +.console-error { + color: var(--color-error); +} + +.console-meta { + color: var(--color-text-muted); + font-size: 0.7rem; +} diff --git a/eslint.config.js b/eslint.config.js index 5d3f669..3cd588f 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,6 +1,10 @@ export default [ { - ignores: ['01-introduction-horizontal-scalability/**', 'node_modules/**'], + ignores: [ + '01-introduction-horizontal-scalability/**', + 'node_modules/**', + '10-databases/mongodb/init/**', + ], }, { files: ['**/*.js'],