diff --git a/README.md b/README.md index e9fc48f..45f8606 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,142 @@ # llm-benchmarking-py -## Usage +A collection of Python functions designed to benchmark LLM (Large Language Model) projects. This library provides various algorithmic and data structure implementations that can be used to evaluate and measure the performance of code generation and optimization tasks. + +## Overview + +This project contains a comprehensive suite of benchmarking utilities organized into different categories, including algorithms, data structures, control flow operations, string operations, and SQL queries. Each module includes both functional implementations and corresponding benchmark tests using pytest-benchmark. + +## Features + +The library includes the following modules: + +### 🔢 Algorithms +- **Primes**: Prime number detection, prime summation, and prime factorization +- **Sort**: Various sorting algorithms including quicksort, dutch flag partition, and finding max-n elements + +### 📊 Data Structures +- **DsList**: List manipulation operations including modify, search, sort, reverse, rotate, and merge + +### 🔁 Control Flow +- **SingleForLoop**: Single-loop operations like sum range, max list, and sum modulus +- **DoubleForLoop**: Nested-loop operations including sum square, sum triangle, count pairs, and matrix operations + +### 📝 Strings +- **StrOps**: String operations including reverse and palindrome checking + +### 🗄️ SQL +- **SqlQuery**: Database query operations for album queries, joins, and invoice operations + +### 🎲 Utilities +- **GenList**: Random list and matrix generation for testing purposes -Build: +## Prerequisites + +- Python 3.8 or higher +- Poetry (Python dependency management tool) + +## Installation + +Install the project dependencies using Poetry: ```shell poetry install ``` -Run Main: +## Usage + +### Running the Demo + +Execute the main demo script to see examples of all available functions: ```shell poetry run main ``` -Run Unit Tests: +### Using as a Library + +Import and use individual modules in your code: + +```python +from llm_benchmark.datastructures.dslist import DsList +from llm_benchmark.algorithms.primes import Primes +from llm_benchmark.strings.strops import StrOps + +# Example: List operations +test_list = [1, 2, 3, 4, 5] +rotated = DsList.rotate_list(test_list, 2) # [3, 4, 5, 1, 2] +reversed_list = DsList.reverse_list(test_list) # [5, 4, 3, 2, 1] + +# Example: Prime operations +is_prime = Primes.is_prime_ineff(17) # True +prime_sum = Primes.sum_primes(10) # Sum of primes up to 10 + +# Example: String operations +is_palindrome = StrOps.palindrome("racecar") # True +reversed_str = StrOps.str_reverse("hello") # "olleh" +``` + +## Testing + +### Run Unit Tests + +Execute all unit tests without benchmarks: ```shell poetry run pytest --benchmark-skip tests/ ``` -Run Benchmarking: +### Run Benchmarks + +Execute performance benchmarks: ```shell poetry run pytest --benchmark-only tests/ ``` + +### Run All Tests + +Run both unit tests and benchmarks: + +```shell +poetry run pytest tests/ +``` + +## Project Structure + +``` +llm-benchmarking-py/ +├── src/ +│ └── llm_benchmark/ +│ ├── algorithms/ # Algorithm implementations +│ ├── control/ # Control flow operations +│ ├── datastructures/ # Data structure operations +│ ├── generator/ # Test data generators +│ ├── sql/ # SQL query operations +│ └── strings/ # String manipulation +├── tests/ # Unit tests and benchmarks +├── data/ # Data files (e.g., SQL databases) +├── main.py # Demo script +├── pyproject.toml # Project dependencies +└── README.md # This file +``` + +## Development + +This project uses: +- **pytest** for testing +- **pytest-benchmark** for performance benchmarking +- **black** for code formatting +- **isort** for import sorting + +## Contributing + +When adding new functions: +1. Implement the function in the appropriate module +2. Add corresponding unit tests in the `tests/` directory +3. Include benchmark tests for performance measurement +4. Ensure all tests pass before submitting + +## License + +This project is maintained by TurinTech AI. diff --git a/main.py b/main.py index 54b8143..4d30c8c 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,6 @@ +"""LLM Benchmark Suite - Main execution script.""" + +import time from llm_benchmark.algorithms.primes import Primes from llm_benchmark.algorithms.sort import Sort from llm_benchmark.control.double import DoubleForLoop @@ -8,9 +11,13 @@ from llm_benchmark.strings.strops import StrOps +def _print_section_header(title): + print(title) + print("-" * len(title)) + + def single(): - print("SingleForLoop") - print("-------------") + _print_section_header("SingleForLoop") print(f"sum_range(10): {SingleForLoop.sum_range(10)}") print(f"max_list([1, 2, 3]): {SingleForLoop.max_list([1, 2, 3])}") @@ -19,8 +26,7 @@ def single(): def double(): - print("DoubleForLoop") - print("-------------") + _print_section_header("DoubleForLoop") print(f"sum_square(10): {DoubleForLoop.sum_square(10)}") print(f"sum_triangle(10): {DoubleForLoop.sum_triangle(10)}") @@ -40,8 +46,7 @@ def double(): def sql(): - print("SQL") - print("---") + _print_section_header("SQL") print(f"query_album('Presence'): {SqlQuery.query_album('Presence')}") print(f"query_album('Roundabout'): {SqlQuery.query_album('Roundabout')}") @@ -55,18 +60,18 @@ def sql(): print(SqlQuery.top_invoices()) print() + def primes(): - print("Primes") - print("------") + _print_section_header("Primes") print(f"is_prime(1700): {Primes.is_prime_ineff(1700)}") print(f"sum_primes(210): {Primes.sum_primes(210)}") print(f"prime_factors(840): {Primes.prime_factors(840)}") print() + def sort(): - print("Sort") - print("----") + _print_section_header("Sort") v = [5, 3, 2, 1, 4] print(f"sort_list({v}): ", end="") @@ -84,8 +89,7 @@ def sort(): def dslist(): - print("DsList") - print("----") + _print_section_header("DsList") test_list = [1, 2, 3, 4, 5] print("Original list:", test_list) @@ -108,9 +112,9 @@ def dslist(): merged_list = DsList.merge_lists(test_list, [6, 7, 8]) print("Merged list with [6, 7, 8]:", merged_list) + def strops(): - print("Strops") - print("----") + _print_section_header("Strops") test_str = "racecar" print("Original string:", test_str) @@ -123,6 +127,13 @@ def strops(): def main(): + print("=" * 80) + print("LLM Benchmark Suite - Running All Tests") + print("=" * 80) + print() + + start_time = time.perf_counter() + single() double() sql() @@ -130,6 +141,13 @@ def main(): sort() dslist() strops() + + end_time = time.perf_counter() + elapsed_time = end_time - start_time + + print("=" * 80) + print(f"Benchmark Suite Completed in {elapsed_time:.4f} seconds") + print("=" * 80) if __name__ == "__main__": diff --git a/src/llm_benchmark/control/single.py b/src/llm_benchmark/control/single.py index 9a314e6..9f00747 100644 --- a/src/llm_benchmark/control/single.py +++ b/src/llm_benchmark/control/single.py @@ -12,10 +12,7 @@ def sum_range(n: int) -> int: Returns: int: Sum of range of numbers from 0 to n """ - arr = [] - for i in range(n): - arr.append(i) - return sum(arr) + return sum(range(n)) @staticmethod def max_list(v: List[int]) -> int: @@ -27,11 +24,7 @@ def max_list(v: List[int]) -> int: Returns: int: Maximum value in the vector """ - max_val = v[0] - for i in range(1, len(v)): - if v[i] > max_val: - max_val = v[i] - return max_val + return max(v) @staticmethod def sum_modulus(n: int, m: int) -> int: @@ -44,8 +37,4 @@ def sum_modulus(n: int, m: int) -> int: Returns: int: Sum of modulus of numbers from 0 to n """ - arr = [] - for i in range(n): - if i % m == 0: - arr.append(i) - return sum(arr) + return sum(i for i in range(n) if i % m == 0) \ No newline at end of file diff --git a/src/llm_benchmark/datastructures/dslist.py b/src/llm_benchmark/datastructures/dslist.py index d282a9c..0dfc79d 100644 --- a/src/llm_benchmark/datastructures/dslist.py +++ b/src/llm_benchmark/datastructures/dslist.py @@ -79,6 +79,9 @@ def rotate_list(v: List[int], n: int) -> List[int]: Returns: List[int]: Rotated list of integers """ + if len(v) == 0: + return [] + n = n % len(v) ret = [] for i in range(n, len(v)): ret.append(v[i]) diff --git a/src/llm_benchmark/generator/gen_list.py b/src/llm_benchmark/generator/gen_list.py index 545ed46..c94b9ba 100644 --- a/src/llm_benchmark/generator/gen_list.py +++ b/src/llm_benchmark/generator/gen_list.py @@ -27,4 +27,4 @@ def random_matrix(n: int, m: int) -> List[List[int]]: Returns: List[List[int]]: Matrix of random integers """ - return [GenList.random_list(n, m) for _ in range(n)] + return [GenList.random_list(m, m) for _ in range(n)] diff --git a/src/llm_benchmark/sql/query.py b/src/llm_benchmark/sql/query.py index 53f6885..698bd37 100644 --- a/src/llm_benchmark/sql/query.py +++ b/src/llm_benchmark/sql/query.py @@ -13,11 +13,11 @@ def query_album(name: str) -> bool: Returns: bool: True if the album exists, False otherwise """ - conn = sqlite3.connect("data/chinook.db") - cur = conn.cursor() + with sqlite3.connect("data/chinook.db") as conn: + cur = conn.cursor() - cur.execute(f"SELECT * FROM Album WHERE Title = '{name}'") - return len(cur.fetchall()) > 0 + cur.execute(f"SELECT * FROM Album WHERE Title = '{name}'") + return len(cur.fetchall()) > 0 @staticmethod def join_albums() -> list: @@ -26,30 +26,30 @@ def join_albums() -> list: Returns: list: """ - conn = sqlite3.connect("data/chinook.db") - cur = conn.cursor() + with sqlite3.connect("data/chinook.db") as conn: + cur = conn.cursor() - cur.execute( - dedent( - """\ - SELECT - t.Name AS TrackName, ( - SELECT a2.Title - FROM Album a2 - WHERE a2.AlbumId = t.AlbumId - ) AS AlbumName, - ( - SELECT ar.Name - FROM Artist ar - JOIN Album a3 ON a3.ArtistId = ar.ArtistId - WHERE a3.AlbumId = t.AlbumId - ) AS ArtistName - FROM - Track t - """ + cur.execute( + dedent( + """\ + SELECT + t.Name AS TrackName, ( + SELECT a2.Title + FROM Album a2 + WHERE a2.AlbumId = t.AlbumId + ) AS AlbumName, + ( + SELECT ar.Name + FROM Artist ar + JOIN Album a3 ON a3.ArtistId = ar.ArtistId + WHERE a3.AlbumId = t.AlbumId + ) AS ArtistName + FROM + Track t + """ + ) ) - ) - return cur.fetchall() + return cur.fetchall() @staticmethod def top_invoices() -> list: @@ -58,21 +58,21 @@ def top_invoices() -> list: Returns: list: List of tuples """ - conn = sqlite3.connect("data/chinook.db") - cur = conn.cursor() + with sqlite3.connect("data/chinook.db") as conn: + cur = conn.cursor() - cur.execute( - dedent( - """\ - SELECT - i.InvoiceId, - c.FirstName || ' ' || c.LastName AS CustomerName, - i.Total - FROM - Invoice i - JOIN Customer c ON c.CustomerId = i.CustomerId - ORDER BY i.Total DESC - """ + cur.execute( + dedent( + """\ + SELECT + i.InvoiceId, + c.FirstName || ' ' || c.LastName AS CustomerName, + i.Total + FROM + Invoice i + JOIN Customer c ON c.CustomerId = i.CustomerId + ORDER BY i.Total DESC + """ + ) ) - ) - return cur.fetchall()[:10] + return cur.fetchall()[:10] diff --git a/tests/llm_benchmark/datastructures/test_dslist.py b/tests/llm_benchmark/datastructures/test_dslist.py index e06bcf3..a973b3b 100644 --- a/tests/llm_benchmark/datastructures/test_dslist.py +++ b/tests/llm_benchmark/datastructures/test_dslist.py @@ -68,3 +68,37 @@ def test_reverse_list(v: List[int], ref: List[int]) -> None: def test_benchmark_reverse_list(benchmark) -> None: benchmark(DsList.reverse_list, [1, 2, 3, 4, 5]) + + +@pytest.mark.parametrize( + "v, n, ref", + [ + ([1, 2, 3, 4, 5], 0, [1, 2, 3, 4, 5]), + ([1, 2, 3, 4, 5], 1, [2, 3, 4, 5, 1]), + ([1, 2, 3, 4, 5], 2, [3, 4, 5, 1, 2]), + ([1, 2, 3, 4, 5], 5, [1, 2, 3, 4, 5]), + ([1, 2, 3, 4, 5], 6, [2, 3, 4, 5, 1]), + ([1, 2, 3, 4, 5], 7, [3, 4, 5, 1, 2]), + ([1, 2, 3, 4, 5], 10, [1, 2, 3, 4, 5]), + ([], 0, []), + ([], 5, []), + ([1], 0, [1]), + ([1], 1, [1]), + ([1], 5, [1]), + ], +) +def test_rotate_list(v: List[int], n: int, ref: List[int]) -> None: + """Test rotate_list with various rotation amounts including edge cases. + + This test verifies that the function correctly handles: + - Normal rotations (0 < n < len(v)) + - Rotation by length (n == len(v)) + - Rotation by more than length (n > len(v)) + - Empty lists + - Single-element lists + """ + assert DsList.rotate_list(v, n) == ref + + +def test_benchmark_rotate_list(benchmark) -> None: + benchmark(DsList.rotate_list, [1, 2, 3, 4, 5], 2) diff --git a/tests/llm_benchmark/generator/__init__.py b/tests/llm_benchmark/generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/llm_benchmark/generator/test_gen_list.py b/tests/llm_benchmark/generator/test_gen_list.py new file mode 100644 index 0000000..49bd1c8 --- /dev/null +++ b/tests/llm_benchmark/generator/test_gen_list.py @@ -0,0 +1,39 @@ +import pytest + +from llm_benchmark.generator.gen_list import GenList + + +def test_random_matrix_dimensions(): + """Test that random_matrix creates a matrix with correct dimensions (n rows, m columns).""" + # Test case 1: 3 rows, 5 columns + n, m = 3, 5 + matrix = GenList.random_matrix(n, m) + + assert len(matrix) == n, f"Expected {n} rows, got {len(matrix)}" + for i, row in enumerate(matrix): + assert len(row) == m, f"Expected {m} columns in row {i}, got {len(row)}" + + # Test case 2: 2 rows, 4 columns + n, m = 2, 4 + matrix = GenList.random_matrix(n, m) + + assert len(matrix) == n, f"Expected {n} rows, got {len(matrix)}" + for i, row in enumerate(matrix): + assert len(row) == m, f"Expected {m} columns in row {i}, got {len(row)}" + + # Test case 3: 4 rows, 2 columns + n, m = 4, 2 + matrix = GenList.random_matrix(n, m) + + assert len(matrix) == n, f"Expected {n} rows, got {len(matrix)}" + for i, row in enumerate(matrix): + assert len(row) == m, f"Expected {m} columns in row {i}, got {len(row)}" + + +def test_random_list_length(): + """Test that random_list creates a list with correct length.""" + n = 10 + m = 100 + lst = GenList.random_list(n, m) + + assert len(lst) == n, f"Expected list of length {n}, got {len(lst)}"