Clean repo

.github/workflows/build.yml

@@ -27,6 +27,11 @@ jobs:
         go-version: '1.22'
         cache: false
 
+    - name: Run Go Vet
+      # Enforce standard Go idioms and detect subtle concurrency/shadowing bugs
+      run: go vet ./...
+      shell: bash
+
     - name: Run Unit & Race Tests
       # The -race flag ensures your lock-free atomic pointers are safe
       # Note: We omit -race and set CGO_ENABLED=0 on macOS due to a known Apple dyld LC_UUID linker bug

.gitignore

@@ -0,0 +1,24 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+network.test.exe
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+cpu.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+# Local DB files
+*.wal
+*.sst

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mrigank Bhatnagar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,24 @@
+.PHONY: build test bench chaos clean
+
+# Build the Forest Database Engine server
+build:
+	go build -o forest_server ./cmd/server
+
+# Run the standard unit test suite with race detection
+test:
+	go vet ./...
+	go test -v -race ./...
+
+# Run the performance benchmarks to profile zero-allocation read/write paths
+bench:
+	go test -bench=. -benchmem ./internal/network/...
+
+# Execute the Chaos Engineering test script
+chaos: build
+	bash ./scripts/chaos_test.sh
+
+# Clean up build artifacts and database files
+clean:
+	rm -f forest_server
+	rm -f *.wal *.sst
+	rm -f cpu.out network.test.exe

README.md

@@ -28,6 +28,12 @@ BenchmarkParseHeader-16   206667542   5.822 ns/op   0 B/op   0 allocs/op
 ```
 *Result: `5.8ns` parsing latency with **zero heap allocations** per request, allowing the engine to sustain massive throughput without triggering GC pauses.*
 
+**CPU Profiling (pprof):**
+
+![CPU Flame Graph proving zero GC overhead and async WAL syncing](./assets/cpu-flamegraph.png)
+
+*The flame graph above demonstrates the engine under heavy concurrent write load. Notice the complete absence of `mallocgc` (Garbage Collection) blocks on the TCP read/write paths, and the heavily isolated `engine.(*WAL).syncLoop` handling disk persistence entirely asynchronously.*
+
 ---
 
 ## System Architecture
@@ -82,8 +88,8 @@ Forest is built to survive catastrophic failure. You can run the Chaos Engineeri
 This script compiles the server, blasts it with concurrent writes, abruptly kills the process, and validates that the WAL recovers 100% of the acknowledged data on reboot.
 
 ```bash
-git clone [https://github.com/Awesome06/forest.git](https://github.com/Awesome06/forest.git)
-cd forest
+git clone [https://github.com/Awesome06/Forest_DatabaseEngine.git](https://github.com/Awesome06/Forest_DatabaseEngine.git)
+cd Forest_DatabaseEngine
 chmod +x ./scripts/chaos_test.sh
 ./scripts/chaos_test.sh
 ```
@@ -99,4 +105,4 @@ go build -o bin/forest ./cmd/server
 ## Author
 **Mrigank Bhatnagar** 
 * GitHub: [@Awesome06](https://github.com/Awesome06)
-* Connect regarding distributed systems, data engineering, and backend infrastructure.
+* Connect regarding distributed systems, data engineering, and backend infrastructure.

cmd/server/main.go

@@ -1,6 +1,11 @@
+// Package main is the entry point for the Forest Database Engine server.
+// It initializes the storage engine, sets up the TCP network listener,
+// and manages graceful shutdown procedures.
 package main
 
 import (
+	"flag"
+	"fmt"
 	"log"
 	"os"
 	"os/signal"
@@ -11,25 +16,55 @@ import (
 )
 
 func main() {
-	db, err := engine.NewDB("forest.wal")
+	// Parse command line flags for server configuration.
+	portFlag := flag.Int("port", 9000, "TCP port to listen on for incoming client connections")
+	walDirFlag := flag.String("wal-dir", "data/wal", "Directory path for Write-Ahead Log (WAL) files")
+	// sstDirFlag is provided to match the CLI usage from the README.
+	// Currently unused by engine.NewDB, but available for future use.
+	_ = flag.String("sst-dir", "data/sst", "Directory path for Sorted String Table (SSTable) files")
+	flag.Parse()
+
+	// Ensure the WAL directory exists before initializing the engine.
+	if err := os.MkdirAll(*walDirFlag, 0755); err != nil {
+		log.Fatalf("failed to create WAL directory: %v", err)
+	}
+
+	walPath := fmt.Sprintf("%s/forest.wal", *walDirFlag)
+
+	// Initialize the storage engine.
+	dbEngine, err := engine.NewDB(walPath)
 	if err != nil {
-		log.Fatalf("Failed to initialize engine: %v", err)
+		log.Fatalf("failed to initialize storage engine at %s: %v", walPath, err)
 	}
 
-	server := network.NewServer("127.0.0.1:9000", db)
+	// Initialize the TCP network server.
+	addr := fmt.Sprintf("127.0.0.1:%d", *portFlag)
+	tcpServer := network.NewServer(addr, dbEngine)
 
-	// Handle graceful shutdown
+	// Handle graceful shutdown via OS signals.
 	go func() {
 		sigChan := make(chan os.Signal, 1)
 		signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
 		<-sigChan
-
-		log.Println("Shutting down database...")
-		db.Close()
+
+		log.Println("Received termination signal. Shutting down database...")
+
+		// Stop accepting new TCP connections.
+		if err := tcpServer.Stop(); err != nil {
+			log.Printf("error stopping TCP server: %v", err)
+		}
+
+		// Flush remaining MemTables and close WAL.
+		if err := dbEngine.Close(); err != nil {
+			log.Printf("error closing storage engine: %v", err)
+		}
+
+		log.Println("Database shut down successfully.")
 		os.Exit(0)
 	}()
 
-	if err := server.Start(); err != nil {
-		log.Fatalf("Server failed: %v", err)
+	// Start accepting connections. This call blocks indefinitely.
+	if err := tcpServer.Start(); err != nil {
+		log.Fatalf("TCP server encountered a fatal error: %v", err)
 	}
 }

internal/engine/bloom.go

@@ -2,18 +2,22 @@ package engine
 
 import (
 	"encoding/binary"
+	"fmt"
 	"hash/fnv"
 	"math"
 )
 
-// BloomFilter is a custom bit-array bloom filter.
+// BloomFilter is a highly optimized bit-array used to probabilistically test whether a key 
+// is present in an SSTable. It mathematically guarantees no false negatives, allowing us to 
+// bypass expensive disk seeks for keys that definitively do not exist.
 type BloomFilter struct {
-	bits []uint64 // The underlying bit array
-	k    uint8    // Number of hash functions
-	m    uint64   // Total number of bits
+	bits []uint64 // The underlying bit array packed into 64-bit words for fast bitwise math.
+	k    uint8    // The optimal number of hash functions applied per key.
+	m    uint64   // The total number of bits in the filter.
 }
 
-// NewBloomFilter calculates the optimal number of bits and hash functions.
+// NewBloomFilter calculates the optimal bit array size (m) and hash functions (k) 
+// for an expected number of elements (n) and a target false positive probability (p).
 func NewBloomFilter(n int, p float64) *BloomFilter {
 	if n <= 0 {
 		n = 1
@@ -22,20 +26,23 @@ func NewBloomFilter(n int, p float64) *BloomFilter {
 		p = 0.01 // default to 1% false positive rate
 	}
 
-	// m = ceil((n * log(p)) / log(1 / pow(2, log(2))))
+	// m = - (n * ln(p)) / (ln(2)^2)
+	// We allocate the exact mathematical minimum number of bits required.
 	mFloat := math.Ceil((float64(n) * math.Log(p)) / math.Log(1.0/math.Pow(2.0, math.Log(2.0))))
 	m := uint64(mFloat)
 	if m == 0 {
 		m = 64
 	}
 
-	// k = round((m / n) * log(2))
+	// k = (m / n) * ln(2)
+	// Balances the bit array saturation so we don't overwrite too many bits per insert.
 	kFloat := math.Round((float64(m) / float64(n)) * math.Log(2.0))
 	k := uint8(kFloat)
 	if k == 0 {
 		k = 1
 	}
 
+	// Pack bits into 64-bit words. The +63 ensures we round up to the nearest word boundary.
 	numUint64s := (m + 63) / 64
 	return &BloomFilter{
 		bits: make([]uint64, numUint64s),
@@ -44,41 +51,52 @@ func NewBloomFilter(n int, p float64) *BloomFilter {
 	}
 }
 
-// hash generates two 32-bit hashes from a single 64-bit FNV hash.
+// hash generates two distinct 32-bit hashes from a single 64-bit FNV hash.
+// This implements the Kirsch-Mitzenmacher optimization, which proves mathematically 
+// that we can simulate 'k' independent hash functions using just two base hashes: 
+// h_i = h1 + i * h2. This drastically cuts down CPU hashing overhead.
 func hash(key []byte) (uint32, uint32) {
 	h := fnv.New64a()
 	h.Write(key)
 	sum := h.Sum64()
+	// Split the 64-bit hash into two 32-bit halves.
 	return uint32(sum), uint32(sum >> 32)
 }
 
-// Add inserts a key into the Bloom Filter using Kirsch-Mitzenmacher double-hashing.
+// Add sets 'k' bits in the array for the given key.
 func (bf *BloomFilter) Add(key []byte) {
 	h1, h2 := hash(key)
 	for i := uint8(0); i < bf.k; i++ {
+		// Calculate the compound hash for this iteration.
 		idx := (uint64(h1) + uint64(i)*uint64(h2)) % bf.m
+
+		// Map the bit index to a specific uint64 word and flip the specific bit.
 		wordIdx := idx / 64
 		bitIdx := idx % 64
 		bf.bits[wordIdx] |= (1 << bitIdx)
 	}
 }
 
-// Contains checks if a key might be in the set.
+// Contains checks if all 'k' bits for the key are set. 
+// If any bit is 0, the key absolutely does NOT exist (saving a disk read).
+// If all bits are 1, the key MIGHT exist (subject to false positive rate 'p').
 func (bf *BloomFilter) Contains(key []byte) bool {
 	h1, h2 := hash(key)
 	for i := uint8(0); i < bf.k; i++ {
 		idx := (uint64(h1) + uint64(i)*uint64(h2)) % bf.m
 		wordIdx := idx / 64
 		bitIdx := idx % 64
+
+		// If the specific bit is zero via bitwise AND, this key was never added.
 		if (bf.bits[wordIdx] & (1 << bitIdx)) == 0 {
 			return false
 		}
 	}
 	return true
 }
 
-// Encode serializes the Bloom Filter to a byte slice for SSTable storage.
-// Format: [k (1B)] | [m (8B)] | [bit array...]
+// Encode serializes the Bloom Filter state to a raw byte slice for SSTable persistence.
+// Layout: k (1B) | m (8B) | bit array (m/8 B)
 func (bf *BloomFilter) Encode() []byte {
 	buf := make([]byte, 1+8+(len(bf.bits)*8))
 	buf[0] = bf.k
@@ -92,10 +110,10 @@ func (bf *BloomFilter) Encode() []byte {
 	return buf
 }
 
-// DecodeBloomFilter reconstructs a BloomFilter from a byte slice.
-func DecodeBloomFilter(data []byte) *BloomFilter {
+// DecodeBloomFilter reconstructs a BloomFilter from a byte slice loaded from disk.
+func DecodeBloomFilter(data []byte) (*BloomFilter, error) {
 	if len(data) < 9 {
-		return nil
+		return nil, fmt.Errorf("bloom filter data too short: expected at least 9 bytes, got %d", len(data))
 	}
 
 	k := data[0]
@@ -107,7 +125,7 @@ func DecodeBloomFilter(data []byte) *BloomFilter {
 	offset := 9
 	for i := 0; i < int(numUint64s); i++ {
 		if offset+8 > len(data) {
-			break
+			return nil, fmt.Errorf("bloom filter data truncated: missing bytes")
 		}
 		bits[i] = binary.BigEndian.Uint64(data[offset : offset+8])
 		offset += 8
@@ -117,5 +135,5 @@ func DecodeBloomFilter(data []byte) *BloomFilter {
 		bits: bits,
 		k:    k,
 		m:    m,
-	}
+	}, nil
 }

internal/engine/bloom_test.go

@@ -63,10 +63,13 @@ func TestBloomFilter_EncodeDecode(t *testing.T) {
 	bf.Add([]byte("test_key"))
 
 	encoded := bf.Encode()
-	decoded := DecodeBloomFilter(encoded)
+	decoded, err := DecodeBloomFilter(encoded)
 
+	if err != nil {
+		t.Fatalf("failed to decode bloom filter: %v", err)
+	}
 	if decoded == nil {
-		t.Fatalf("failed to decode bloom filter")
+		t.Fatalf("failed to decode bloom filter: returned nil without error")
 	}
 	if decoded.k != bf.k {
 		t.Errorf("k mismatch: got %d, want %d", decoded.k, bf.k)