[Fuzzer] Clean code

test-db/Dockerfile

@@ -1,4 +1,3 @@
-
 FROM python:3.10-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -10,9 +9,12 @@ COPY . /app
 
 RUN pip install -r requirements.txt
 
-RUN chmod +x /app/main.py && ln -s /app/main.py /usr/bin/test-db
+# Create /usr/bin/test-db to point to main.py using Python
+RUN echo '#!/bin/sh\nexec python /app/main.py "$@"' > /usr/bin/test-db \
+    && chmod +x /usr/bin/test-db
+
+ENTRYPOINT ["/usr/bin/test-db"]
 
-ENTRYPOINT ["test-db"]
 
 #build image with: docker build -t fuzzer-db .
 #then run : docker run --rm -v $(pwd)/shared:/data --network host fuzzer-db -r 100

test-db/README.md

@@ -1,38 +1,40 @@
 # catASTrophe
 Automated Bug Detection in the Database Engines SQLite3.26.0
 
-# Requirements
-To proceed with the scripts, please make sure you installed all the following requirements by typing this command
-```
-pip install -r requirements.txt
-```
-
 # Project
 
 **Objectives**:
 
 1. Evaluates the reliability of SQLite3.26.0 by detecting crashes and logic bugs. 
 2. Build our own SQL generator to produce intersting queries i.e. likely to trigger bugs (crashes OR incorrect results)
 
-**Evaluation**:
 
-1. Bug-finding capability
-2. Characteristics of the generated SQL queries
-3. Code Coverage
-4. Performance
 
 # User Guide
 Description of the command lines
 
-To run the script type the following command
+**Since we are using docker compose up and docker exec [...] our code is not runnable with docker run using a DockerFile. We therefore provide the following commands to properly run our code.**
+
+
+## Requirements
+To proceed with the scripts, please make sure you installed all the following requirements by typing this command
+```
+pip install -r requirements.txt
+```
+
+## Run the Tool
+To run the tool, type the following command
 ```
-python3 main.py
+python main.py
 ```
 
-### Arguments
-One can add argument like ``-v`` or ``--version`` with one of the following values
+## Arguments
+#### Number of runs
+``-r`` or ``--runs`` with a numerical value.
+It will launch the fuzzer for this number of iterations. Default is 100.
+Example run:
 ```
-/usr/bin/sqlite3-3.26.0
-/usr/bin/sqlite3-3.39.4
+python main.py -r 100
 ```
-It will only launch the tester with the selected SQL engine version, otherwise it will test both by default.
+#### Gcov
+Add the flag ``--gcov`` to run the code with gcov enabled.

test-db/config.py

@@ -1,10 +1,7 @@
 SQL_CLAUSES = ["SELECT", "FROM", "JOIN", "WHERE", "GROUP BY", "ORDER BY", "LIMIT", "HAVING", "UPDATE", "DELETE", "MAX", "MIN", "SUM", "COUNT", "AVG"]
-#TODO: check that all clauses apply to our versions of SQLite
 
-# VERSIONS = ["/usr/bin/sqlite3-3.26.0", "/usr/bin/sqlite3-3.39.4"]
 VERSIONS = ["3.26.0", "3.39.4"]
 
-
 BUG_TYPES = {'crash': "CRASH", 'logic': "LOGIC"}
 
 TABLES_HEADER = {
@@ -17,9 +14,4 @@
     "kpop_song_rankings": {"year": "INT", "time": "INT", "rank": "INT", "song_title": "TEXT", "artist": "TEXT", "album": "TEXT"}
 }
 
-NUMERIC_COLS = ["height", "weight", "year", "rank", "time"]
-
-IGNORABLE_ERRORS = [
-    "a GROUP BY clause is required before HAVING",
-    # Add more known, ignorable SQLite errors here
-]
+NUMERIC_COLS = ["height", "weight", "year", "rank", "time"]

test-db/database_gen.py

@@ -1,5 +1,4 @@
 import random
-import pandas as pd
 
 from utils import create_table, load_csv, generate_insert
 from config import TABLES_HEADER, TABLES_HEADER_WITH_TYPE

test-db/docker-compose.yml

@@ -9,26 +9,4 @@ services:
     volumes:
       - ./shared:/data
     stdin_open: true
-    tty: true
-
-
-
-# version: '3'
-# services:
-#   sqlite3-3.26.0:
-#     image: theosotr/sqlite3-test:3.26.0
-#     container_name: sqlite3-3.26.0
-#     working_dir: /home/test/sqlite
-#     volumes:
-#       - ./shared:/data
-#     stdin_open: true
-#     tty: true
-
-#   sqlite3-3.39.4:
-#     image: theosotr/sqlite3-test:3.39.4
-#     container_name: sqlite3-3.39.4
-#     working_dir: /home/test/sqlite
-#     volumes:
-#       - ./shared:/data
-#     stdin_open: true
-#     tty: true
+    tty: true

test-db/docker_utils.py

@@ -29,6 +29,16 @@ def initialize_database_in_container(version, init_sql_path, db_path='/data/test
 
     print(f"Initializing the database using {binary}...")
 
+    try:
+        subprocess.run(
+            ["docker", "exec", container_name, "rm", "-f", db_path],
+            check=True
+        )
+        print("Removed existing test.db in container.")
+    except subprocess.CalledProcessError as e:
+        print("⚠️ Could not remove existing test.db. Might not exist yet.")
+
+
     db_file = os.path.join(os.getcwd(), 'shared', 'test.db')
     if os.path.exists(db_file):
         os.remove(db_file)
@@ -45,13 +55,12 @@ def initialize_database_in_container(version, init_sql_path, db_path='/data/test
         with open(init_sql_path, 'r', encoding='utf-8') as f:
             sql_script = f.read()
 
-        result = subprocess.run(
+        subprocess.run(
             command, input=sql_script, text=True, capture_output=True, check=True
         )
 
         print("Database initialized successfully.")
-        # if result.stdout:
-        #     print("SQLite output:", result.stdout)
+
 
     except subprocess.CalledProcessError as e:
         print("Failed to initialize database in container:")

test-db/main.py

@@ -11,20 +11,7 @@
 from database_gen import DatabaseGenerator
 
 from config import VERSIONS, SQL_CLAUSES, BUG_TYPES
-from utils import update_count_clauses, get_freq_clauses, get_expression_depth, get_validity
-
-from config import IGNORABLE_ERRORS
-
-
-
-def is_ignorable_error(stderr_output: str) -> bool:
-    ignorable_errors = [
-        "a GROUP BY clause is required before HAVING",
-        "no such column: nan",
-        "HAVING clause on a non-aggregate query"
-    ]
-    return any(ignorable_error in stderr_output for ignorable_error in ignorable_errors)
-
+from utils import update_count_clauses, get_freq_clauses, get_expression_depth, get_validity, is_ignorable_error
 
 def main(versions, test_flag, runs):
     sql_clauses_count = {clause: [] for clause in SQL_CLAUSES}
@@ -42,7 +29,7 @@ def main(versions, test_flag, runs):
     results = {}
     total_queries = 0
     start_time = time.time()
-    #execution_times = []
+
 
     # Generate one query at a time and run it across all versions
     for _ in range(runs):
@@ -62,37 +49,25 @@ def main(versions, test_flag, runs):
         if not test_flag:
             for version in versions:
                 initialize_database_in_container(version, database)
-                runner = QueryRunner(version)
-                #start_time = time.time()
-                bug_type, result = runner.run(query_sql)
-                #end_time = time.time()
-                #execution_times.append(end_time - start_time)
-
-                print(f"--- SQLite {version} Output ---")
-                #print(result)
-                results[version] = result
-
-
+                runner = QueryRunner(version="3.39.4", use_gcov=args.gcov)
 
 
-                # if output_3260 != output_3394:
-                #     return "LOGIC_BUG", (output_3260, output_3394)
-
+                bug_type, result = runner.run(query_sql)
 
+                results[version] = result
 
                 if bug_type:
                     if is_ignorable_error(result) :
-                        print("⚠️ Skipping ignorable error GROUP BY before HAVING or column:nan.")
+                        print("Skipping ignorable error GROUP BY before HAVING or column:nan.")
                         continue
                     else:
                         print("Bug detected!")
-                        print("HERE 1")
+                        print(result.std)
+
                         recorder.report_bug(query_sql, version, bug_type, stderr_output=result)
                 else:
                     partitioning = runner.run_partitioning(query, result, database)
-                    if not partitioning:
-                        print("HERE 2")
-                        #recorder.report_bug(query_sql, version, BUG_TYPES['crash'])
+
 
             total_queries += 1
             print("Current iteration: ", total_queries)
@@ -103,22 +78,21 @@ def main(versions, test_flag, runs):
             if results[v0] != results[v1]:
                 print("HERE 3")
                 recorder.report_bug(query_sql, v0+v1, BUG_TYPES['logic'], stderr_output=results[v0])
-                print(f"\n❗ Output mismatch between {v0} and {v1}")
+                print(f"\nLOGIC BUG: Output mismatch between {v0} and {v1}")
                 print(f"{v0}:\n{results[v0]}")
                 print(f"{v1}:\n{results[v1]}")
             else:
-                print(f"✅ Output is consistent across {v0} and {v1}")
+                print(f"--- Output is consistent across {v0} and {v1} ---")
 
     elapsed_time = time.time() - start_time
-    queries_per_minute = (total_queries / elapsed_time) * 60
+    queries_per_minute = ((total_queries / elapsed_time) * 60)*2 #*2 for the number of versions the query was executed
 
     # Final stats
     print(f"\nPerformance: {queries_per_minute:.2f} queries/min")
     print(f"\nFrequency per clauses: {get_freq_clauses(sql_clauses_count)}")
     print(f"Average Expression Depth: {sum(expression_depth) / len(expression_depth)}")
     print(f"Query Validity: {sum(query_validity) / len(query_validity)}")
-    #avg_time = sum(execution_times) / len(execution_times) if execution_times else 0
-    #print(f"✅ Average Execution Time per Query: {avg_time:.4f} seconds")
+
 
 
 if __name__ == "__main__":
@@ -129,5 +103,6 @@ def main(versions, test_flag, runs):
     )
     parser.add_argument("-t", "--test", default=False, help="Print the queries but do not run on docker, use -t True")
     parser.add_argument("-r", "--runs", type=int, default=100, help="Provide the number of runs you want, e.g. -r 100000")
+    parser.add_argument("--gcov", action="store_true", help="Enable gcov code coverage reporting")    
     args = parser.parse_args()
     main(args.version, args.test, args.runs)

test-db/querie_gen.py

@@ -3,19 +3,14 @@
 # Will use the techniques found in the resources of the instructions
 
 # Pivoted Query Synthesis (PQS) : https://www.youtube.com/watch?v=0aeDyXgzo04
-# 100 000 queries for each generated database
 
 # Query Partitioning : https://dl.acm.org/doi/10.1145/3428279
 
-# Query Plan Guidance : https://ieeexplore.ieee.org/document/10172874
-
-# Equivalent Expression Transformation : https://www.usenix.org/system/files/osdi24-jiang.pdf
-
 # We are going to use the 2 datasets kpop_idols and kpop_ranking for query generation
 
 import random
 import math
-from sqlglot import exp, select
+from sqlglot import exp
 
 operators = ['=', '!=', '<', '>', 'LIKE']
 
@@ -76,7 +71,7 @@ def get_random_assignment(self, pivot):
         if value is None:
             new_value = exp.Null()
         elif isinstance(value, str):
-            new_value = exp.Literal.string(value[::-1])  # Reverse string
+            new_value = exp.Literal.string(value[::-1])
         elif isinstance(value, (int, float)):
             new_value = exp.Literal.number(str(value + random.randint(-5, 5)))
         else:
@@ -85,27 +80,23 @@ def get_random_assignment(self, pivot):
         return exp.EQ(this=exp.Column(this=col), expression=new_value)
 
     def generate_aggregate_query(self, pivot, table_name):
-        # Filter numeric columns
         numeric_cols = [col for col, val in pivot.items() if isinstance(val, (int, float)) and not math.isnan(val)]
         if not numeric_cols:
-            return self.generate_select(pivot, table_name)  # Fallback
+            return self.generate_select(pivot, table_name)
 
         col = random.choice(numeric_cols)
         col_expr = exp.Column(this=col)
         func = random.choice([exp.Max, exp.Min, exp.Sum, exp.Avg, exp.Count])
 
-        # Create aggregate expression
         agg_expr = func(this=col_expr)
 
         query = exp.select(agg_expr).from_(table_name)
 
-        # Optional GROUP BY (on a non-agg column)
         non_agg_cols = [c for c in pivot.keys() if c != col]
         if non_agg_cols and random.random() < 0.5:
             group_col = random.choice(non_agg_cols)
             query = query.group_by(exp.Column(this=group_col))
 
-        # Optional HAVING
         if random.random() < 0.3:
             query = query.having(
                 exp.GT(this=agg_expr.copy(), expression=exp.Literal.number(random.randint(1, 100)))
@@ -131,39 +122,20 @@ def generate_select(self, pivot, table_name):
         if random.random() < 0.3:
             having_cond = self.get_condition(random.choice(list(pivot.values())), random.choice(list(pivot.keys())))
             query = query.having(having_cond)
-
-        # weirdness = random.random()
-        # if weirdness < 0.01:
-        #     query = select("1 = 1 AND 1 = 0").from_(table_name)
-        # elif weirdness < 0.02:
-        #     query = select("COUNT(SELECT * FROM table)")
-        # elif weirdness < 0.03:
-        #     query = select("name AS age", "age").from_(table_name).order_by("age")
-        # elif weirdness < 0.04:
-        #     query = select().from_(table_name).select(
-        #         exp.EQ(this=exp.Column(this="weight"), expression=exp.Literal.number("-9223372036854775809"))
-        #     )
-        # elif weirdness < 0.05:
-        #     query = select().from_(table_name).select(
-        #         exp.EQ(this=exp.Column(this="weight"), expression=exp.Literal.number("1E28475"))
-        #     )
-
         return query
 
     def generate_update(self, pivot, table_name):
         assignment = self.get_random_assignment(pivot)        
         where_expr = self.generate_where_clause(pivot)
         if where_expr is None:
-            where_expr = exp.TRUE  # Redundant now, but safe
+            where_expr = exp.TRUE
         query = exp.Update().table(table_name).set_(assignment).where(where_expr)
         return query
 
     def generate_delete(self, pivot, table_name):
         where_expr = self.generate_where_clause(pivot)
         if where_expr is None:
             where_expr = exp.TRUE
-
-        print(f"TAble Name: {table_name}")
         query = exp.delete(table=table_name).where(where_expr)
         return query
 
@@ -177,9 +149,4 @@ def generate_query_for_pivot(self, pivot, table_name):
         elif choice < 0.75:
             return self.generate_update(pivot, table_name)
         else:
-            return self.generate_delete(pivot, table_name)
-
-
-    def generate_query(self):
-        # TODO: implement
-        return random.random()
+            return self.generate_delete(pivot, table_name)