Time to put it on master?

bring2lite/classes/gui.py

@@ -98,6 +98,9 @@ def start_processing_wal(self):
             self.walp = WALParser()
 
             for i in tqdm(self.wals):
+                if os.stat(i).st_size == 0:
+                    tqdm.write(f"Wal file empty: {i}")
+                    return
                 if i[0:-4] in self.sqlites:
                     self.walp.parse(i, self.output, self.format, True)
                 else:

bring2lite/classes/parser.py

@@ -105,9 +105,12 @@ def _extract_cells(self, page, num_of_cells_in_frame, schema_offset):
             content_length = payload_length - header_length
             current_page_cell_content = page[cell_offset + current_index: cell_offset + current_index + content_length] + overflow_content
 
-            tempresult = self._typeHelper(cell_types, current_page_cell_content)
-
-            res.append(tempresult)
+            try:
+                tempresult = self._typeHelper(cell_types, current_page_cell_content)
+                res.append(tempresult)
+            except ValueError as e:
+                self.logger.debug(f"exception while parsing cells at offset {cell_offset+current_index}: {e}", e)
+                break
 
         self.logger.debug("end parsing cells")
         return res
@@ -132,7 +135,10 @@ def _extract_overflow_pages(self, page, first_overflow_page_offset, size_to_extr
                 result = result + p[4: 4 + size_to_extract]
                 return result
 
+            visitedPages = set()
             while next_page:
+                visitedPages.add(next_page)
+
                 if not self.is_wal:
                     f.seek(self.page_size * (next_page))
                 else:
@@ -142,6 +148,12 @@ def _extract_overflow_pages(self, page, first_overflow_page_offset, size_to_extr
                     next_page = unpack('>I', p[:4])[0]
                 except error:
                     break
+
+                # Loop detection
+                if next_page in visitedPages:
+                    self.logger.debug("Loop detected in overflow_page refs")
+                    break;
+
                 if next_page:
                     result += (p[4: self.page_size - 4])
                 else:
@@ -187,30 +199,44 @@ def _typeHelper(self, types, data):
             if t == 0:
                 cell_data.append(['NULL', 'NULL'])
             elif t == 1:
+                if len(data[index:])<1:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["8bit", unpack('>b', data[index:index + 1])[0]]
                 cell_data.append(d)
                 index += 1
             elif t == 2:
+                if len(data[index:])<2:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["16bit", unpack('>h', data[index:index + 2])[0]]
                 cell_data.append(d)
                 index += 2
             elif t == 3:
+                if len(data[index:])<3:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["24bit", int(binascii.hexlify(data[index:index + 3]), 16)]
                 cell_data.append(d)
                 index += 3
             elif t == 4:
+                if len(data[index:])<4:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["32bit", unpack('>i', data[index:index + 4])[0]]
                 cell_data.append(d)
                 index += 4
             elif t == 5:
+                if len(data[index:])<6:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["48bit", int(binascii.hexlify(data[index:index + 6]), 16)]
                 cell_data.append(d)
                 index += 6
             elif t == 6:
+                if len(data[index:])<8:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["64bit", unpack('>q', data[index:index + 8])[0]]
                 cell_data.append(d)
                 index += 8
             elif t == 7:
+                if len(data[index:])<8:
+                    raise ValueError('Short data:  ' + binascii.hexlify(data).decode('ASCII'))
                 d = ["64bitf", unpack('>d', data[index:index + 8])[0]]
                 cell_data.append(d)
                 index += 8

bring2lite/classes/potentially_parser.py

@@ -32,13 +32,12 @@ def parse_page(self, page, filename="", is_first_page=False, is_wal=False, is_tr
 
         self.start_of_cell_content_area = pageheader[3]
 
-        unalloc_content = None
         if self.is_trunk_page:
             unalloc_content = self._extract_trunk_page_content(page)
         else:
             unalloc_content = self._extract_unalloc_content(page, schema_offset)
-            if not unalloc_content:
-                return []
+        if not unalloc_content:
+            return []
 
 
         self.result = []
@@ -100,6 +99,9 @@ def parse_page(self, page, filename="", is_first_page=False, is_wal=False, is_tr
             except error:
                 return self.result
 
+            if (current_index + content_length <1):
+                self.logger.debug(f"Unable to make progress in Potential: {cell_offset}, {current_index + content_length}")
+                stop = True
             cell_offset += current_index + content_length
 
             if cell_offset == len(unalloc_content):
@@ -168,6 +170,8 @@ def _extract_trunk_page_content(self, page):
                 break
 
         tmp_result = page[end_of_cell_pointer_array: self.page_size]
+        if not tmp_result:
+            return None
 
         start_of_content = 0
         offset_tester = unpack('>B', tmp_result[start_of_content: start_of_content + 1])[0]

bring2lite/classes/report_generator.py

@@ -1,81 +1,99 @@
 import os
 import hashlib
+import csv
 from tqdm import tqdm
-from colorama import *
-import binascii
-from tkinter import *
-from tkinter.filedialog import askopenfilename
-from tkinter.filedialog import askdirectory
+
+
+def create_path_if_it_doesnt_exist(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+
 
 class ReportGenerator:
     def __init__(self):
         self.my_path = ""
 
-    def generateReport(self, path, filename, data, format="CSV", schema=["No schema found"]):
+    def generateReport(self, path, filename, data, schema=["No schema found"]):
+        """
+        Alternate results writer, using a CSV module to avoid not escaping comma's e.d.
+        :param path: path to directory to write the log files to
+        :param filename: name of the logfile (without .log postfix)
+        :param data: a list of lists containing carved records
+        :param schema: possibly found a database schema to be added on top of the log file
+        :return: None
+        """
         if data is None:
             return
 
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-        # if format:
-        #     with open(path+filename+'.csv', 'w', newline='') as f:
-        #         writer = csv.writer(f)
-        #         writer.writerows(data)
-        if data:
-            out = ""
-            for datatype in schema:
-                out += str(datatype) + ","
-            out += "\n"
-
-            for frame in data:
-                if isinstance(frame, list):
-                    for y in frame:
-                        if self.is_text(y[0]):
-                            try:
-                                out += str(y[1].decode('utf-8')) + ","
-                            except UnicodeDecodeError:
-                                out +=str(y[1]) + ","
-                                continue
-                        else:
-                            out += str(y[1]) + ","
-                out += "\n"
-            out += "++++++++++++++++++++++++++++\n"
-            try:
-                with open(path + "/" + filename + '.log', "a") as f:
-                    f.write(out)
-            except UnicodeEncodeError:
-                tqdm.write("can not write the record because of unicode errors")
-
-            self.print_hash(path + "/" + filename + '.log')
+        create_path_if_it_doesnt_exist(path)
+
+        # Iterator to create CSV from
+        out = []
+        # If a schema is found, add the column types to the top of the log file:
+        if schema:
+            out.append(schema)
+        else:
+            out.append(['no schema found'])
+        # A 'frame' is actually a possible carved record
+        # Represented as a list where each element is a list containing the type of
+        # column value and the column value itself
+        for frame in data:
+            csv_row = []
+            if isinstance(frame, list):
+                for column in frame:
+                    # If the carved record column is of type 'TEXT' try to decode it
+                    if self.is_text(column[0]):
+                        try:
+                            csv_row.append(column[1].decode('utf-8'))
+                        except UnicodeDecodeError:
+                            csv_row.append(column[1])
+                    # Otherwise just add the value as it is:
+                    else:
+                        csv_row.append(column[1])
+            out.append(csv_row)
+
+        # Write results
+        self.write_to_csv(filename, path, out)
+
+    def write_to_csv(self, filename, path, out):
+        """
+        Write the contents of out to the filename by using the csv writer module
+        :param filename: name of the file to write to
+        :param path: path where the file will be written to
+        :param out: the contents to be written to a file (must be an iterator like a list with lists)
+        :return: none
+        """
+        file_out = f'{path}/{filename}.log'
+        if os.path.exists(file_out):
+            tqdm.write(f"Logfile {filename} already exists! Overwriting the results.")
+        try:
+            with open(file_out, 'w', newline='') as csvfile:
+                csv_writer = csv.writer(csvfile)
+                csv_writer.writerows(out)
+
+        except UnicodeEncodeError:
+            tqdm.write("can not write the record because of unicode errors")
+
+        self.print_hash(file_out)
 
     def generate_schema_report(self, path, filename, data, csv):
         if data is None:
             return
+        create_path_if_it_doesnt_exist(path)
 
-        if not os.path.exists(path):
-            os.makedirs(path)
+        out = []
+        for key, value in data.items():
+            if isinstance(value, list):
+                out.append(value)
 
-        out = ""
-        with open(path + "/" + filename + '.log', "a") as f:
-            for key, value in data.items():
-                # out += str(key) + ", "
-                if isinstance(value, list):
-                    for y in value:
-                        out += str(y) + ", "
-                out += "\n"
-            out += "++++++++++++++++++++++++++++\n"
+        # Write results
+        self.write_to_csv(filename, path, out)
 
-            f.write(out)
-
-        self.print_hash(path + "/" + filename + '.log')
 
     def generate_freeblock_report(self, path, filename, freeblocks):
         if freeblocks is None:
             return
-
-        if not os.path.exists(path):
-            os.makedirs(path)
+        create_path_if_it_doesnt_exist(path)
 
         with open(path + "/" + filename + '.log', "a") as f:
             for solutions in freeblocks:
@@ -97,4 +115,4 @@ def is_text(self, tester):
     def print_hash(self, filename):
         with open(filename, "rb") as f:
             d = f.read()
-            tqdm.write("sha-256: " + filename + '\t => \t' + str(hashlib.sha256(d).hexdigest()))
+            tqdm.write("sha-256: " + filename + '\t => \t' + str(hashlib.sha256(d).hexdigest()))