Use mozinstall to extract builds (#76)

benchtester/BenchTester.py

@@ -7,15 +7,23 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import sys
-import os
 import argparse
+import mercurial, mercurial.ui, mercurial.hg, mercurial.commands
+import os
+import re
 import sqlite3
 import subprocess
-import mercurial, mercurial.ui, mercurial.hg, mercurial.commands
+import sys
 import time
 
+# Database version, bump this when incompatible DB changes are made
+gVersion = 1
+
 gTableSchemas = [
+  # benchtester_version - the database version, can be used for upgrade scripts
+  '''CREATE TABLE IF NOT EXISTS
+      "benchtester_version" ("version" INTEGER NOT NULL UNIQUE)''',
+
   # Builds - info on builds we have tests for
   '''CREATE TABLE IF NOT EXISTS
       "benchtester_builds" ("id" INTEGER PRIMARY KEY NOT NULL,
@@ -35,12 +43,26 @@
       "benchtester_datapoints" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
                                 "name" VARCHAR NOT NULL UNIQUE)''',
 
+  # Procs - names of processes
+  '''CREATE TABLE IF NOT EXISTS
+      "benchtester_procs" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+                           "name" VARCHAR NOT NULL UNIQUE)''',
+
+  # Checkpoints - names of checkpoints
+  '''CREATE TABLE IF NOT EXISTS
+      "benchtester_checkpoints" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+                                 "name" VARCHAR NOT NULL UNIQUE)''',
+
   # Data - datapoints from tests
   '''CREATE TABLE IF NOT EXISTS
       "benchtester_data" ("test_id" INTEGER NOT NULL,
                           "datapoint_id" INTEGER NOT NULL,
+                          "checkpoint_id" INTEGER NOT NULL,
+                          "proc_id" INTEGER NOT NULL,
+                          "iteration" INTEGER NOT NULL,
                           "value" INTEGER NOT NULL,
-                          "meta" VARCHAR)''',
+                          "units" INTEGER NOT NULL,
+                          "kind" INTEGER NOT NULL)''',
 
   # Some default indexes
   '''CREATE INDEX IF NOT EXISTS test_lookup ON benchtester_tests ( name, build_id DESC )''',
@@ -139,6 +161,95 @@ def load_module(self, modname):
 
     return True
 
+  @staticmethod
+  def map_process_names(process_names):
+    # Normalize the process names.
+    # Given: [ "Main", "Web Content (123)", "Web Content (345)", "Web Content (678)" ]
+    # Mapping: [ "Main" => "Main",
+    #            "Web Content (123)" => "Web Content",
+    #            "Web Content (345)" => "Web Content 2",
+    #            "Web Content (678)" => "Web Content 3"
+    #          ]
+    proc_name_counts = {}
+    proc_name_mapping = {}
+
+    for full_process_name in process_names:
+      # Drop the pid portion of process name
+      process_re = r'(.*)\s+\(\d+\)'
+      m = re.match(process_re, full_process_name)
+      if m:
+        proc_name = m.group(1)
+        if proc_name in proc_name_counts:
+          proc_name_counts[proc_name] += 1
+          proc_name_mapping[full_process_name] = "%s %d" % (proc_name, proc_name_counts[proc_name])
+        else:
+          # Leave the first entry w/o a number
+          proc_name_counts[proc_name] = 1
+          proc_name_mapping[full_process_name] = proc_name
+      else:
+        proc_name_mapping[full_process_name] = full_process_name
+
+    return proc_name_mapping
+
+  def insert_results(self, test_id, results):
+    # - results is an array of iterations
+    # - iterations is an array of checkpoints
+    # - checkpoint is a dict with: label, reports
+    # - reports is a dict of processes
+    cur = self.sqlite.cursor()
+
+    for x, iteration in enumerate(results):
+      iternum = x + 1
+      for checkpoint in iteration:
+        label = checkpoint['label']
+
+        # insert checkpoint name, get checkpoint_id
+        cur.execute("SELECT id FROM benchtester_checkpoints WHERE name = ?", (label, ))
+        row = cur.fetchone()
+        checkpoint_id = row[0] if row else None
+        if checkpoint_id is None:
+          cur.execute("INSERT INTO benchtester_checkpoints(name) VALUES (?)", (label, ))
+          checkpoint_id = cur.lastrowid
+
+        proc_name_mapping = self.map_process_names(checkpoint['reports'])
+        for process_name, reports in checkpoint['reports'].iteritems():
+          # reports is a dictionary of datapoint_name: { val, unit, kind }
+          process_name = proc_name_mapping[process_name]
+
+          # insert process name, get process_id
+          cur.execute("SELECT id FROM benchtester_procs WHERE name = ?", (process_name, ))
+          row = cur.fetchone()
+          process_id = row[0] if row else None
+          if process_id is None:
+            cur.execute("INSERT INTO benchtester_procs(name) VALUES (?)", (process_name, ))
+            process_id = cur.lastrowid
+
+          # insert datapoint names
+          insertbegin = time.time()
+          self.info("Inserting %u datapoints into DB" % len(reports))
+          cur.executemany("INSERT OR IGNORE INTO `benchtester_datapoints`(name) "
+                          "VALUES (?)",
+                          ( [ k ] for k in reports.iterkeys() ))
+          self.sqlite.commit()
+          self.info("Filled datapoint names in %.02fs" % (time.time() - insertbegin))
+
+          # insert datapoint values
+          insertbegin = time.time()
+          cur.executemany("INSERT INTO `benchtester_data` "
+                          "SELECT ?, p.id, ?, ?, ?, ?, ?, ? FROM `benchtester_datapoints` p "
+                          "WHERE p.name = ?",
+                          ( [ test_id,
+                              checkpoint_id,
+                              process_id,
+                              iternum,
+                              dp['val'],
+                              dp['unit'],
+                              dp['kind'],
+                              name ]
+                            for name, dp in reports.iteritems() if dp ))
+          self.sqlite.commit()
+          self.info("Filled datapoint values in %.02fs" % (time.time() - insertbegin))
+
   # datapoints a list of the format [ [ "key", value, "meta"], ... ].
   # Duplicate keys are allowed. Value is numeric and required, meta is an
   # optional string (see db format)
@@ -165,27 +276,11 @@ def add_test_results(self, testname, datapoints, succeeded=True):
 
         if datapoints:
           testid = cur.fetchone()[0]
-          insertbegin = time.time()
-          self.info("Inserting %u datapoints into DB" % len(datapoints))
-          cur.executemany("INSERT OR IGNORE INTO `benchtester_datapoints`(name) "
-                          "VALUES (?)",
-                          ([ datapoint[0] ] for datapoint in datapoints))
-          self.sqlite.commit()
-          self.info("Filled datapoint names in %.02fs" % (time.time() - insertbegin))
-          insertbegin = time.time()
-          # If val is a list, it is interpreted as [ value, meta ]
-          cur.executemany("INSERT INTO `benchtester_data` "
-                          "SELECT ?, p.id, ?, ? FROM `benchtester_datapoints` p "
-                          "WHERE p.name = ?",
-                          ( [ testid,
-                              dp[1],
-                              dp[2] if len(dp) > 2 else None,
-                              dp[0] ]
-                            for dp in datapoints ))
-          self.sqlite.commit()
-          self.info("Filled datapoint values in %.02fs" % (time.time() - insertbegin))
+          self.insert_results(testid, datapoints)
       except Exception, e:
         self.error("Failed to insert data into sqlite, got '%s': %s" % (type(e), e))
+        import traceback
+        traceback.print_exc()
         self.sqlite.rollback()
         return False
     return True
@@ -264,11 +359,28 @@ def _open_db(self):
       self.sqlitedb = self.args['sqlitedb'] = None
       return False
     try:
+      db_exists = os.path.exists(self.args['sqlitedb'])
+
       sql_path = os.path.abspath(self.args['sqlitedb'])
       self.sqlite = sqlite3.connect(sql_path, timeout=900)
       cur = self.sqlite.cursor()
+
+      if db_exists:
+        # make sure the version matches
+        cur.execute("SELECT `version` FROM `benchtester_version` WHERE `version` = ?", [ gVersion ])
+        row = cur.fetchone()
+        version = row[0] if row else None
+        if version != gVersion:
+          self.error("Incompatible versions: %s is version %s, current version is %s" % (self.args['sqlitedb'], version, gVersion))
+          self.sqlitedb = self.args['sqlitedb'] = None
+          return False
+
       for schema in gTableSchemas:
         cur.execute(schema)
+
+      if not db_exists:
+        cur.execute("INSERT INTO `benchtester_version` (`version`) VALUES (?)", [ gVersion ])
+
       # Create/update build ID
       cur.execute("SELECT `time`, `id` FROM `benchtester_builds` WHERE `name` = ?", [ self.buildname ])
       buildrow = cur.fetchone()

benchtester/BuildGetter.py

@@ -22,6 +22,7 @@
 import urllib2
 
 import mozdownload
+import mozinstall
 
 PUSHLOG_BRANCH_MAP = {
   'mozilla-inbound': 'integration/mozilla-inbound',
@@ -152,6 +153,7 @@ def __init__(self, scraper_args, directory=None,
 
     self._branch = None
     self._extracted = directory
+    self._install_dir = None
     self._cleanup_dir = False
     self._prepared = False
     self._revision = None
@@ -202,14 +204,6 @@ def __init__(self, scraper_args, directory=None,
 
     self._valid = True
 
-  @staticmethod
-  def extract_build(src, dstdir):
-    """Extracts the given build to the given directory."""
-
-    # cross-platform FIXME, this is hardcoded to tar at the moment
-    with tarfile.open(src, mode='r:*') as tar:
-      tar.extractall(path=dstdir)
-
   def prepare(self):
     """
     Prepares the build for testing.
@@ -228,7 +222,7 @@ def prepare(self):
     self._scraperTarget = self._scraper.filename
 
     _stat("Extracting build")
-    self.extract_build(self._scraper.filename, self._extracted)
+    self._install_dir = mozinstall.install(self._scraper.filename, self._extracted)
 
     self._prepared = True
     self._scraper = None
@@ -242,7 +236,7 @@ def cleanup(self):
       os.remove(self._scraperTarget)
 
       # remove the extracted archive
-      shutil.rmtree(os.path.join(self._extracted, "firefox"))
+      mozinstall.uninstall(self._install_dir)
 
     # remove the temp directory that was created
     if self._cleanup_dir:
@@ -259,8 +253,7 @@ def get_valid(self):
   def get_binary(self):
     if not self._prepared:
       raise Exception("Build is not prepared")
-    # FIXME More hard-coded linux stuff
-    return os.path.join(self._extracted, "firefox", "firefox")
+    return mozinstall.get_binary(self._install_dir, "firefox")
 
   def get_buildtime(self):
     return self._timestamp

benchtester/MarionetteTest.py

@@ -25,6 +25,9 @@ def __init__(self, parent):
     parent.add_argument('--gecko_log',
                         help="Logfile for gecko output. Defaults to 'gecko.log'",
                         default=None)
+    parent.add_argument('--process_count',
+                         help="Number of e10s processes to use",
+                         default=1)
     self.name = "MarionetteTest"
     self.parent = parent
 
@@ -34,6 +37,8 @@ def setup(self):
     self.endurance_results = None
     self.port = int(self.parent.args['marionette_port'])
     self.gecko_log = self.parent.args['gecko_log']
+    self.process_count = int(self.parent.args['process_count'])
+    self.info("Process Count: %d " % self.process_count)
 
     return True
 
@@ -61,6 +66,14 @@ def run_test(self, testname, testvars={}):
       "browser.tabs.remote.autostart": e10s,
       "browser.tabs.remote.autostart.1": e10s,
       "browser.tabs.remote.autostart.2": e10s,
+      "browser.tabs.remote.autostart.3": e10s,
+      "browser.tabs.remote.autostart.4": e10s,
+      "browser.tabs.remote.autostart.5": e10s,
+      "browser.tabs.remote.autostart.6": e10s,
+      "dom.ipc.processCount": self.process_count,
+
+      # prevent "You're using e10s!" dialog from showing up
+      "browser.displayedE10SNotice": 1000,
 
       # We're not testing flash memory usage. Also: it likes to crash in VNC sessions.
       "plugin.disable": True,
@@ -128,23 +141,7 @@ def run_test(self, testname, testvars={}):
 
     self.endurance_results = runner.testvars.get("results", [])
 
-    results = list()
-    for x in range(len(self.endurance_results)):
-      iteration = self.endurance_results[x]
-      for checkpoint in iteration:
-        iternum = x + 1
-        label = checkpoint['label']
-        # TODO(ER): Handle all process entries
-        for memtype,memval in checkpoint['memory']['Main'].items():
-          if type(memval) is dict:
-            prefix = memval['unit'] + ":"
-            memval = memval['val']
-          else:
-            prefix = ""
-          datapoint = [ "%s%s" % (prefix, memtype), memval, "%s:%u" % (label, iternum) ]
-          results.append(datapoint)
-
-    if not self.tester.add_test_results(testname, results, not failures):
+    if not self.tester.add_test_results(testname, self.endurance_results, not failures):
       return self.error("Failed to save test results")
     if failures:
       return self.error("%u failures occured during test run" % failures)