diff --git a/benchtester/BenchTester.py b/benchtester/BenchTester.py index f75a2f8..d9ceaea 100644 --- a/benchtester/BenchTester.py +++ b/benchtester/BenchTester.py @@ -7,15 +7,23 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -import sys -import os import argparse +import mercurial, mercurial.ui, mercurial.hg, mercurial.commands +import os +import re import sqlite3 import subprocess -import mercurial, mercurial.ui, mercurial.hg, mercurial.commands +import sys import time +# Database version, bump this when incompatible DB changes are made +gVersion = 1 + gTableSchemas = [ + # benchtester_version - the database version, can be used for upgrade scripts + '''CREATE TABLE IF NOT EXISTS + "benchtester_version" ("version" INTEGER NOT NULL UNIQUE)''', + # Builds - info on builds we have tests for '''CREATE TABLE IF NOT EXISTS "benchtester_builds" ("id" INTEGER PRIMARY KEY NOT NULL, @@ -35,12 +43,26 @@ "benchtester_datapoints" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "name" VARCHAR NOT NULL UNIQUE)''', + # Procs - names of processes + '''CREATE TABLE IF NOT EXISTS + "benchtester_procs" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + "name" VARCHAR NOT NULL UNIQUE)''', + + # Checkpoints - names of checkpoints + '''CREATE TABLE IF NOT EXISTS + "benchtester_checkpoints" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + "name" VARCHAR NOT NULL UNIQUE)''', + # Data - datapoints from tests '''CREATE TABLE IF NOT EXISTS "benchtester_data" ("test_id" INTEGER NOT NULL, "datapoint_id" INTEGER NOT NULL, + "checkpoint_id" INTEGER NOT NULL, + "proc_id" INTEGER NOT NULL, + "iteration" INTEGER NOT NULL, "value" INTEGER NOT NULL, - "meta" VARCHAR)''', + "units" INTEGER NOT NULL, + "kind" INTEGER NOT NULL)''', # Some default indexes '''CREATE INDEX IF NOT EXISTS test_lookup ON benchtester_tests ( name, build_id DESC )''', @@ -139,6 +161,95 @@ def load_module(self, modname): return True + @staticmethod + def map_process_names(process_names): + # Normalize the process names. + # Given: [ "Main", "Web Content (123)", "Web Content (345)", "Web Content (678)" ] + # Mapping: [ "Main" => "Main", + # "Web Content (123)" => "Web Content", + # "Web Content (345)" => "Web Content 2", + # "Web Content (678)" => "Web Content 3" + # ] + proc_name_counts = {} + proc_name_mapping = {} + + for full_process_name in process_names: + # Drop the pid portion of process name + process_re = r'(.*)\s+\(\d+\)' + m = re.match(process_re, full_process_name) + if m: + proc_name = m.group(1) + if proc_name in proc_name_counts: + proc_name_counts[proc_name] += 1 + proc_name_mapping[full_process_name] = "%s %d" % (proc_name, proc_name_counts[proc_name]) + else: + # Leave the first entry w/o a number + proc_name_counts[proc_name] = 1 + proc_name_mapping[full_process_name] = proc_name + else: + proc_name_mapping[full_process_name] = full_process_name + + return proc_name_mapping + + def insert_results(self, test_id, results): + # - results is an array of iterations + # - iterations is an array of checkpoints + # - checkpoint is a dict with: label, reports + # - reports is a dict of processes + cur = self.sqlite.cursor() + + for x, iteration in enumerate(results): + iternum = x + 1 + for checkpoint in iteration: + label = checkpoint['label'] + + # insert checkpoint name, get checkpoint_id + cur.execute("SELECT id FROM benchtester_checkpoints WHERE name = ?", (label, )) + row = cur.fetchone() + checkpoint_id = row[0] if row else None + if checkpoint_id is None: + cur.execute("INSERT INTO benchtester_checkpoints(name) VALUES (?)", (label, )) + checkpoint_id = cur.lastrowid + + proc_name_mapping = self.map_process_names(checkpoint['reports']) + for process_name, reports in checkpoint['reports'].iteritems(): + # reports is a dictionary of datapoint_name: { val, unit, kind } + process_name = proc_name_mapping[process_name] + + # insert process name, get process_id + cur.execute("SELECT id FROM benchtester_procs WHERE name = ?", (process_name, )) + row = cur.fetchone() + process_id = row[0] if row else None + if process_id is None: + cur.execute("INSERT INTO benchtester_procs(name) VALUES (?)", (process_name, )) + process_id = cur.lastrowid + + # insert datapoint names + insertbegin = time.time() + self.info("Inserting %u datapoints into DB" % len(reports)) + cur.executemany("INSERT OR IGNORE INTO `benchtester_datapoints`(name) " + "VALUES (?)", + ( [ k ] for k in reports.iterkeys() )) + self.sqlite.commit() + self.info("Filled datapoint names in %.02fs" % (time.time() - insertbegin)) + + # insert datapoint values + insertbegin = time.time() + cur.executemany("INSERT INTO `benchtester_data` " + "SELECT ?, p.id, ?, ?, ?, ?, ?, ? FROM `benchtester_datapoints` p " + "WHERE p.name = ?", + ( [ test_id, + checkpoint_id, + process_id, + iternum, + dp['val'], + dp['unit'], + dp['kind'], + name ] + for name, dp in reports.iteritems() if dp )) + self.sqlite.commit() + self.info("Filled datapoint values in %.02fs" % (time.time() - insertbegin)) + # datapoints a list of the format [ [ "key", value, "meta"], ... ]. # Duplicate keys are allowed. Value is numeric and required, meta is an # optional string (see db format) @@ -165,27 +276,11 @@ def add_test_results(self, testname, datapoints, succeeded=True): if datapoints: testid = cur.fetchone()[0] - insertbegin = time.time() - self.info("Inserting %u datapoints into DB" % len(datapoints)) - cur.executemany("INSERT OR IGNORE INTO `benchtester_datapoints`(name) " - "VALUES (?)", - ([ datapoint[0] ] for datapoint in datapoints)) - self.sqlite.commit() - self.info("Filled datapoint names in %.02fs" % (time.time() - insertbegin)) - insertbegin = time.time() - # If val is a list, it is interpreted as [ value, meta ] - cur.executemany("INSERT INTO `benchtester_data` " - "SELECT ?, p.id, ?, ? FROM `benchtester_datapoints` p " - "WHERE p.name = ?", - ( [ testid, - dp[1], - dp[2] if len(dp) > 2 else None, - dp[0] ] - for dp in datapoints )) - self.sqlite.commit() - self.info("Filled datapoint values in %.02fs" % (time.time() - insertbegin)) + self.insert_results(testid, datapoints) except Exception, e: self.error("Failed to insert data into sqlite, got '%s': %s" % (type(e), e)) + import traceback + traceback.print_exc() self.sqlite.rollback() return False return True @@ -264,11 +359,28 @@ def _open_db(self): self.sqlitedb = self.args['sqlitedb'] = None return False try: + db_exists = os.path.exists(self.args['sqlitedb']) + sql_path = os.path.abspath(self.args['sqlitedb']) self.sqlite = sqlite3.connect(sql_path, timeout=900) cur = self.sqlite.cursor() + + if db_exists: + # make sure the version matches + cur.execute("SELECT `version` FROM `benchtester_version` WHERE `version` = ?", [ gVersion ]) + row = cur.fetchone() + version = row[0] if row else None + if version != gVersion: + self.error("Incompatible versions: %s is version %s, current version is %s" % (self.args['sqlitedb'], version, gVersion)) + self.sqlitedb = self.args['sqlitedb'] = None + return False + for schema in gTableSchemas: cur.execute(schema) + + if not db_exists: + cur.execute("INSERT INTO `benchtester_version` (`version`) VALUES (?)", [ gVersion ]) + # Create/update build ID cur.execute("SELECT `time`, `id` FROM `benchtester_builds` WHERE `name` = ?", [ self.buildname ]) buildrow = cur.fetchone() diff --git a/benchtester/BuildGetter.py b/benchtester/BuildGetter.py index 3192315..c5c7302 100644 --- a/benchtester/BuildGetter.py +++ b/benchtester/BuildGetter.py @@ -22,6 +22,7 @@ import urllib2 import mozdownload +import mozinstall PUSHLOG_BRANCH_MAP = { 'mozilla-inbound': 'integration/mozilla-inbound', @@ -152,6 +153,7 @@ def __init__(self, scraper_args, directory=None, self._branch = None self._extracted = directory + self._install_dir = None self._cleanup_dir = False self._prepared = False self._revision = None @@ -202,14 +204,6 @@ def __init__(self, scraper_args, directory=None, self._valid = True - @staticmethod - def extract_build(src, dstdir): - """Extracts the given build to the given directory.""" - - # cross-platform FIXME, this is hardcoded to tar at the moment - with tarfile.open(src, mode='r:*') as tar: - tar.extractall(path=dstdir) - def prepare(self): """ Prepares the build for testing. @@ -228,7 +222,7 @@ def prepare(self): self._scraperTarget = self._scraper.filename _stat("Extracting build") - self.extract_build(self._scraper.filename, self._extracted) + self._install_dir = mozinstall.install(self._scraper.filename, self._extracted) self._prepared = True self._scraper = None @@ -242,7 +236,7 @@ def cleanup(self): os.remove(self._scraperTarget) # remove the extracted archive - shutil.rmtree(os.path.join(self._extracted, "firefox")) + mozinstall.uninstall(self._install_dir) # remove the temp directory that was created if self._cleanup_dir: @@ -259,8 +253,7 @@ def get_valid(self): def get_binary(self): if not self._prepared: raise Exception("Build is not prepared") - # FIXME More hard-coded linux stuff - return os.path.join(self._extracted, "firefox", "firefox") + return mozinstall.get_binary(self._install_dir, "firefox") def get_buildtime(self): return self._timestamp diff --git a/benchtester/MarionetteTest.py b/benchtester/MarionetteTest.py index c25d207..b7c8962 100644 --- a/benchtester/MarionetteTest.py +++ b/benchtester/MarionetteTest.py @@ -25,6 +25,9 @@ def __init__(self, parent): parent.add_argument('--gecko_log', help="Logfile for gecko output. Defaults to 'gecko.log'", default=None) + parent.add_argument('--process_count', + help="Number of e10s processes to use", + default=1) self.name = "MarionetteTest" self.parent = parent @@ -34,6 +37,8 @@ def setup(self): self.endurance_results = None self.port = int(self.parent.args['marionette_port']) self.gecko_log = self.parent.args['gecko_log'] + self.process_count = int(self.parent.args['process_count']) + self.info("Process Count: %d " % self.process_count) return True @@ -61,6 +66,14 @@ def run_test(self, testname, testvars={}): "browser.tabs.remote.autostart": e10s, "browser.tabs.remote.autostart.1": e10s, "browser.tabs.remote.autostart.2": e10s, + "browser.tabs.remote.autostart.3": e10s, + "browser.tabs.remote.autostart.4": e10s, + "browser.tabs.remote.autostart.5": e10s, + "browser.tabs.remote.autostart.6": e10s, + "dom.ipc.processCount": self.process_count, + + # prevent "You're using e10s!" dialog from showing up + "browser.displayedE10SNotice": 1000, # We're not testing flash memory usage. Also: it likes to crash in VNC sessions. "plugin.disable": True, @@ -128,23 +141,7 @@ def run_test(self, testname, testvars={}): self.endurance_results = runner.testvars.get("results", []) - results = list() - for x in range(len(self.endurance_results)): - iteration = self.endurance_results[x] - for checkpoint in iteration: - iternum = x + 1 - label = checkpoint['label'] - # TODO(ER): Handle all process entries - for memtype,memval in checkpoint['memory']['Main'].items(): - if type(memval) is dict: - prefix = memval['unit'] + ":" - memval = memval['val'] - else: - prefix = "" - datapoint = [ "%s%s" % (prefix, memtype), memval, "%s:%u" % (label, iternum) ] - results.append(datapoint) - - if not self.tester.add_test_results(testname, results, not failures): + if not self.tester.add_test_results(testname, self.endurance_results, not failures): return self.error("Failed to save test results") if failures: return self.error("%u failures occured during test run" % failures) diff --git a/benchtester/checkpoint.js b/benchtester/checkpoint.js index 67fa3b6..025e766 100644 --- a/benchtester/checkpoint.js +++ b/benchtester/checkpoint.js @@ -13,7 +13,7 @@ function createCheckpoint(aLabel) { var result = { label: aLabel, timestamp: new Date(), - memory: {}, + reports: {}, }; var knownHeap = {}; @@ -26,40 +26,18 @@ function createCheckpoint(aLabel) { aProcess = "Main" } - var unitname; - switch (aUnits) { - // Old builds had no units field and assumed bytes - case undefined: - case Ci.nsIMemoryReporter.UNITS_BYTES: - break; - case Ci.nsIMemoryReporter.UNITS_COUNT: - unitname = "cnt"; - break; - case Ci.nsIMemoryReporter.UNITS_PERCENTAGE: - unitname = "pct"; - break; - default: - // Unhandled - return; + if (!result['reports'][aProcess]) { + result['reports'][aProcess] = {} } - // For types with non-bytes units the value is - // { 'unit': 'percent', 'val': 1234 } - // For bytes it is just a number, so as not to bloat output (we end up - // exporting 11k+ reporters on newer builds) - if (!result['memory'][aProcess]) { - result['memory'][aProcess] = {} - } - - if (result['memory'][aProcess][aPath]) { - if (unitname) - result['memory'][aProcess][aPath]['val'] += aAmount; - else - result['memory'][aProcess][aPath] += aAmount; - } else if (unitname) { - result['memory'][aProcess][aPath] = { 'unit': unitname, 'val': aAmount }; + if (result['reports'][aProcess][aPath]) { + result['reports'][aProcess][aPath]['val'] += aAmount; } else { - result['memory'][aProcess][aPath] = aAmount; + result['reports'][aProcess][aPath] = { + 'unit': aUnits, + 'val': aAmount, + 'kind': aKind + }; } if (aKind !== undefined && aKind == Ci.nsIMemoryReporter.KIND_HEAP @@ -78,11 +56,14 @@ function createCheckpoint(aLabel) { */ function onFinish(aClosure) { // Calculate heap-unclassified for each process - var keys = Object.keys(result['memory']); + var keys = Object.keys(result['reports']); for (var idx = 0; idx < keys.length; idx++) { let proc = keys[idx]; - result['memory'][proc]['explicit/heap-unclassified'] = - result['memory'][proc]['heap-allocated'] - knownHeap[proc]; + result['reports'][proc]['explicit/heap-unclassified'] = { + 'unit': Ci.nsIMemoryReporter.UNITS_BYTES, + 'val': result['reports'][proc]['heap-allocated']['val'] - knownHeap[proc], + 'kind': Ci.nsIMemoryReporter.KIND_HEAP + }; } marionetteScriptFinished(result); @@ -92,6 +73,8 @@ function createCheckpoint(aLabel) { var memMgr = Cc["@mozilla.org/memory-reporter-manager;1"]. getService(Ci.nsIMemoryReporterManager); + // NB: |memMgr.getReports| was added in Fx28, we do not support releases + // prior to that. memMgr.getReports(addReport, null, onFinish, null, /* anonymize */ false); } diff --git a/benchtester/test_memory_usage.py b/benchtester/test_memory_usage.py index 840d0eb..192947b 100644 --- a/benchtester/test_memory_usage.py +++ b/benchtester/test_memory_usage.py @@ -245,6 +245,7 @@ def open_and_focus(self): """ page_to_load = self._urls[self._pages_loaded % len(self._urls)] tabs_loaded = len(self._tabs) + is_new_tab = False if tabs_loaded < self._maxTabs and tabs_loaded <= self._pages_loaded: full_tab_list = self.marionette.window_handles @@ -255,8 +256,7 @@ def open_and_focus(self): {'anonid': 'tabs-newtab-button'})) newtab_button.click() - # Janky workaround to make sure the tab is loaded - time.sleep(0.25) + self.wait_for_condition(lambda mn: len(mn.window_handles) == tabs_loaded + 1) # NB: The tab list isn't sorted, so we do a set diff to determine # which is the new tab @@ -266,6 +266,8 @@ def open_and_focus(self): self._tabs.append(new_tabs[0]) tabs_loaded += 1 + is_new_tab = True + tab_idx = self._pages_loaded % self._maxTabs tab = self._tabs[tab_idx] @@ -283,6 +285,16 @@ def open_and_focus(self): self.marionette.navigate(page_to_load) self.logger.debug("loaded!") + # On e10s the tab handle can change after actually loading content + if is_new_tab: + # First build a set up w/o the current tab + old_tabs = set(self._tabs) + old_tabs.remove(tab) + # Perform a set diff to get the (possibly) new handle + [new_tab] = set(self.marionette.window_handles) - old_tabs + # Update the tab list at the current index to preserve the tab ordering + self._tabs[tab_idx] = new_tab + # give the page time to settle time.sleep(self._perTabPause) diff --git a/create_graph_json.py b/create_graph_json.py index 44a7ee0..fd31e4b 100755 --- a/create_graph_json.py +++ b/create_graph_json.py @@ -41,25 +41,111 @@ "nodeize": "/", "dump": True, "series": { - "MaxMemoryV2": {"datapoint": "Iteration 5/TabsOpen/explicit"}, - "MaxMemorySettledV2": {"datapoint": "Iteration 5/TabsOpenSettled/explicit"}, - "MaxMemoryForceGCV2": {"datapoint": "Iteration 5/TabsOpenForceGC/explicit"}, - "MaxMemoryResidentV2": {"datapoint": "Iteration 5/TabsOpen/resident"}, - "MaxMemoryResidentSettledV2": {"datapoint": "Iteration 5/TabsOpenSettled/resident"}, - "MaxMemoryResidentForceGCV2": {"datapoint": "Iteration 5/TabsOpenForceGC/resident"}, - "StartMemoryV2": {"datapoint": "Iteration 1/Start/explicit"}, - "StartMemoryResidentV2": {"datapoint": "Iteration 1/Start/resident"}, - "StartMemorySettledV2": {"datapoint": "Iteration 1/StartSettled/explicit"}, - "StartMemoryResidentSettledV2": {"datapoint": "Iteration 1/StartSettled/resident"}, - "EndMemoryV2": {"datapoint": "Iteration 5/TabsClosed/explicit"}, - "EndMemoryResidentV2": {"datapoint": "Iteration 5/TabsClosed/resident"}, - "EndMemorySettledV2": {"datapoint": "Iteration 5/TabsClosedSettled/explicit"}, - "EndMemoryForceGCV2": {"datapoint": "Iteration 5/TabsClosedForceGC/explicit"}, - "EndMemoryResidentSettledV2": {"datapoint": "Iteration 5/TabsClosedSettled/resident"}, - "EndMemoryResidentForceGCV2": {"datapoint": "Iteration 5/TabsClosedForceGC/resident"}, - "MaxHeapUnclassifiedV2": {"datapoint": "Iteration 5/TabsOpenSettled/explicit/heap-unclassified"}, + "MaxMemoryV2": { + "datapoint": [ + "Iteration 5/TabsOpen/Main/explicit", + "Iteration 5/TabsOpen/explicit", + ] + }, + "MaxMemorySettledV2": { + "datapoint": [ + "Iteration 5/TabsOpenSettled/Main/explicit", + "Iteration 5/TabsOpenSettled/explicit", + ] + }, + "MaxMemoryForceGCV2": { + "datapoint": [ + "Iteration 5/TabsOpenForceGC/Main/explicit", + "Iteration 5/TabsOpenForceGC/explicit", + ] + }, + "MaxMemoryResidentV2": { + "datapoint": [ + "Iteration 5/TabsOpen/Main/resident", + "Iteration 5/TabsOpen/resident", + ] + }, + "MaxMemoryResidentSettledV2": { + "datapoint": [ + "Iteration 5/TabsOpenSettled/Main/resident", + "Iteration 5/TabsOpenSettled/resident", + ] + }, + "MaxMemoryResidentForceGCV2": { + "datapoint": [ + "Iteration 5/TabsOpenForceGC/Main/resident", + "Iteration 5/TabsOpenForceGC/resident", + ] + }, + "StartMemoryV2": { + "datapoint": [ + "Iteration 1/Start/Main/explicit", + "Iteration 1/Start/explicit", + ] + }, + "StartMemoryResidentV2": { + "datapoint": [ + "Iteration 1/Start/Main/resident", + "Iteration 1/Start/resident", + ] + }, + "StartMemorySettledV2": { + "datapoint": [ + "Iteration 1/StartSettled/Main/explicit", + "Iteration 1/StartSettled/explicit", + ] + }, + "StartMemoryResidentSettledV2": { + "datapoint": [ + "Iteration 1/StartSettled/Main/resident", + "Iteration 1/StartSettled/resident", + ] + }, + "EndMemoryV2": { + "datapoint": [ + "Iteration 5/TabsClosed/Main/explicit", + "Iteration 5/TabsClosed/explicit", + ] + }, + "EndMemoryResidentV2": { + "datapoint": [ + "Iteration 5/TabsClosed/Main/resident", + "Iteration 5/TabsClosed/resident", + ] + }, + "EndMemorySettledV2": { + "datapoint": [ + "Iteration 5/TabsClosedSettled/Main/explicit", + "Iteration 5/TabsClosedSettled/explicit", + ] + }, + "EndMemoryForceGCV2": { + "datapoint": [ + "Iteration 5/TabsClosedForceGC/Main/explicit", + "Iteration 5/TabsClosedForceGC/explicit", + ] + }, + "EndMemoryResidentSettledV2": { + "datapoint": [ + "Iteration 5/TabsClosedSettled/Main/resident", + "Iteration 5/TabsClosedSettled/resident", + ] + }, + "EndMemoryResidentForceGCV2": { + "datapoint": [ + "Iteration 5/TabsClosedForceGC/Main/resident", + "Iteration 5/TabsClosedForceGC/resident", + ] + }, + "MaxHeapUnclassifiedV2": { + "datapoint": [ + "Iteration 5/TabsOpenSettled/Main/explicit/heap-unclassified", + "Iteration 5/TabsOpenSettled/explicit/heap-unclassified", + ] + }, "MaxJSV2": { "datapoint": [ + "Iteration 5/TabsOpenSettled/Main/js-main-runtime", # As of Jul 2012 "Iteration 5/TabsOpenSettled/js-main-runtime", # Pre-Jul 2012 @@ -72,13 +158,34 @@ }, "MaxImagesV2": { "datapoint": [ + "Iteration 5/TabsOpenSettled/Main/explicit/images", "Iteration 5/TabsOpenSettled/explicit/images", # Old ~FF4 reporters "Iteration 5/TabsOpenSettled/images", # Brief period in may 2011 before heap-used became explicit "Iteration 5/TabsOpenSettled/heap-used/images" ] - } + }, + + "Web Content MaxMemoryV2": {"datapoint": "Iteration 5/TabsOpen/Web Content/explicit"}, + "Web Content MaxMemorySettledV2": {"datapoint": "Iteration 5/TabsOpenSettled/Web Content/explicit"}, + "Web Content MaxMemoryForceGCV2": {"datapoint": "Iteration 5/TabsOpenForceGC/Web Content/explicit"}, + "Web Content MaxMemoryResidentV2": {"datapoint": "Iteration 5/TabsOpen/Web Content/resident"}, + "Web Content MaxMemoryResidentSettledV2": {"datapoint": "Iteration 5/TabsOpenSettled/Web Content/resident"}, + "Web Content MaxMemoryResidentForceGCV2": {"datapoint": "Iteration 5/TabsOpenForceGC/Web Content/resident"}, + "Web Content StartMemoryV2": {"datapoint": "Iteration 1/Start/Web Content/explicit"}, + "Web Content StartMemoryResidentV2": {"datapoint": "Iteration 1/Start/Web Content/resident"}, + "Web Content StartMemorySettledV2": {"datapoint": "Iteration 1/StartSettled/Web Content/explicit"}, + "Web Content StartMemoryResidentSettledV2": {"datapoint": "Iteration 1/StartSettled/Web Content/resident"}, + "Web Content EndMemoryV2": {"datapoint": "Iteration 5/TabsClosed/Web Content/explicit"}, + "Web Content EndMemoryResidentV2": {"datapoint": "Iteration 5/TabsClosed/Web Content/resident"}, + "Web Content EndMemorySettledV2": {"datapoint": "Iteration 5/TabsClosedSettled/Web Content/explicit"}, + "Web Content EndMemoryForceGCV2": {"datapoint": "Iteration 5/TabsClosedForceGC/Web Content/explicit"}, + "Web Content EndMemoryResidentSettledV2": {"datapoint": "Iteration 5/TabsClosedSettled/Web Content/resident"}, + "Web Content EndMemoryResidentForceGCV2": {"datapoint": "Iteration 5/TabsClosedForceGC/Web Content/resident"}, + "Web Content MaxHeapUnclassifiedV2": {"datapoint": "Iteration 5/TabsOpenSettled/Web Content/explicit/heap-unclassified"}, + "Web Content MaxJSV2": {"datapoint": "Iteration 5/TabsOpenSettled/Web Content/js-main-runtime"}, + "Web Content MaxImagesV2": {"datapoint": "Iteration 5/TabsOpenSettled/Web Content/explicit/images"} } }, "Android-ARMv6": { @@ -89,8 +196,20 @@ } } +# Mapping of unit values to names +unit_map = { + 0: 'bytes', + 1: 'cnt', + # 2 => UNITS_COUNT_CUMULATIVE, currently this isn't handled + 3: 'pct' +} + # Reuse default tests for android, but s/Iteration 5/Iteration 1/ for k, v in gTests['Slimtest-TalosTP5-Slow']['series'].iteritems(): + # Only use the "Main" entries as a template + if "Web Content" in k: + continue + if type(v['datapoint']) is list: out = [] for x in v['datapoint']: @@ -143,8 +262,6 @@ def error(msg): builds = cur.fetchall() hg_ui = None hg_repo = None - - def build_sort(build_a, build_b): global hg_repo, hg_ui if build_a['time'] != build_b['time']: @@ -282,33 +399,38 @@ def _findNode(nodes, datapoint, nodeize): nodeize = False # Pull all data for latest run of this test on this build - allrows = cur.execute('''SELECT p.name AS datapoint, d.value, d.meta - FROM benchtester_data d, benchtester_datapoints p - WHERE test_id = ? AND p.id = d.datapoint_id + allrows = cur.execute('''SELECT dp.name AS datapoint, + c.name AS checkpoint, + p.name AS process, + d.iteration, d.value, d.units, d.kind + FROM benchtester_data d, + benchtester_datapoints dp, + benchtester_procs p, + benchtester_checkpoints c + WHERE test_id = ? AND dp.id = d.datapoint_id + AND c.id = d.checkpoint_id + AND p.id = d.proc_id ''', [testdata[testname]['id']]) + # NB: For now kind is ignored + # Sort data, splitting it up into nodes if requested. Calculate the value # of each node - either a sum of its childnodes, or its explicit value if # given. The idea is to reduce the amount of data juggling the frontend # needs to do. for row in allrows: - # If the datapoint begins with "AAA:..." then the datapoint has - # non-bytes units, and we include _units and strip the prefix datapoint = row['datapoint'] - units = datapoint.find(':', 0, 4) - if units != -1: - (units, datapoint) = datapoint.split(':', 1) - else: - units = 'bytes' + units = unit_map.get(row['units']) + if not units: + print("skipping unhandled unit %s for %s" % (row['units'], datapoint)) + continue - # The 'meta' field in the db holds "CheckpointName:Iteration". Prefix - # these on to the reporter name, e.g. "Iteration 1/MaxMem/" so - # they fit nicely into a tree. - meta = row['meta'].split(':') - datapoint = "Iteration %u/%s/%s" % (int(meta[1]), meta[0], datapoint) + # Prefix the reporter name, e.g. "Iteration 1/StartSettled/Main/" so + # that it fits nicely into a tree. + datapoint = "Iteration %u/%s/%s/%s" % (row['iteration'], row['checkpoint'], row['process'], datapoint) if nodeize: - # Note that we perserve null values as 'none', to differentiate missing + # Note that we preserve null values as 'none', to differentiate missing # data from values of 0 cursor = testdata[testname]['nodes'] thisnode = datapoint.split(nodeize) diff --git a/html/about_memory_worker.js b/html/about_memory_worker.js index 720a83c..ffe3c7b 100644 --- a/html/about_memory_worker.js +++ b/html/about_memory_worker.js @@ -51,7 +51,8 @@ onmessage = function(aEvent) { // @param {aPath} The node path. // @param {aData} The data node. // @param {aReports} The array of report entries that is being built. -function checkpointToAboutMemory(aPath, aData, aReports) { +// @param {aProcess} The process this report is for. +function checkpointToAboutMemory(aPath, aData, aReports, aProcess) { function defval(aObj) { if (typeof(aObj) == 'number') { return aObj; @@ -99,7 +100,7 @@ function checkpointToAboutMemory(aPath, aData, aReports) { // This is a leaf node. var report = { description: "", - process: "Main Process", + process: aProcess + " Process", amount: defval(aData), units: units(aData), path: aPath, @@ -113,7 +114,12 @@ function checkpointToAboutMemory(aPath, aData, aReports) { var node; while (node = childern.shift()) { var nodePath = aPath != "" ? aPath + '/' + node : node; - checkpointToAboutMemory(nodePath, aData[node], aReports); + var process = aProcess; + if (!process) { + process = nodePath; + nodePath = ""; + } + checkpointToAboutMemory(nodePath, aData[node], aReports, process); } } diff --git a/html/slimyet.js b/html/slimyet.js index d45fd40..deb7b91 100644 --- a/html/slimyet.js +++ b/html/slimyet.js @@ -523,7 +523,7 @@ var gReleaseLookup = function() { // /data/areweslimyet.json and comments below. These are exported from the full // test database by create_graph_json.py var gSeries = { - "Resident Memory" : { + "Main Resident Memory" : { 'StartMemoryResidentV2': "RSS: Fresh start", 'StartMemoryResidentSettledV2': "RSS: Fresh start [+30s]", 'MaxMemoryResidentV2': "RSS: After TP5", @@ -533,7 +533,7 @@ var gSeries = { 'EndMemoryResidentSettledV2': "RSS: After TP5, tabs closed [+30s]", 'EndMemoryResidentForceGCV2': "RSS: After TP5, tabs closed [+30s, forced GC]" }, - "Explicit Memory" : { + "Main Explicit Memory" : { 'StartMemoryV2': "Explicit: Fresh start", 'StartMemorySettledV2': "Explicit: Fresh start [+30s]", 'MaxMemoryV2': "Explicit: After TP5", @@ -543,10 +543,35 @@ var gSeries = { 'EndMemorySettledV2': "Explicit: After TP5, tabs closed [+30s]", 'EndMemoryForceGCV2': "Explicit: After TP5, tabs closed [+30s, forced GC]" }, - "Miscellaneous Measurements" : { + "Main Miscellaneous Measurements" : { 'MaxHeapUnclassifiedV2': "Heap Unclassified: After TP5 [+30s]", 'MaxJSV2': "JS: After TP5 [+30s]", 'MaxImagesV2': "Images: After TP5 [+30s]" + }, + "Web Content Resident Memory" : { + 'Web Content StartMemoryResidentV2': "RSS: Fresh start", + 'Web Content StartMemoryResidentSettledV2': "RSS: Fresh start [+30s]", + 'Web Content MaxMemoryResidentV2': "RSS: After TP5", + 'Web Content MaxMemoryResidentSettledV2': "RSS: After TP5 [+30s]", + 'Web Content MaxMemoryResidentForceGCV2': "RSS: After TP5 [+30s, forced GC]", + 'Web Content EndMemoryResidentV2': "RSS: After TP5, tabs closed", + 'Web Content EndMemoryResidentSettledV2': "RSS: After TP5, tabs closed [+30s]", + 'Web Content EndMemoryResidentForceGCV2': "RSS: After TP5, tabs closed [+30s, forced GC]" + }, + "Web Content Explicit Memory" : { + 'Web Content StartMemoryV2': "Explicit: Fresh start", + 'Web Content StartMemorySettledV2': "Explicit: Fresh start [+30s]", + 'Web Content MaxMemoryV2': "Explicit: After TP5", + 'Web Content MaxMemorySettledV2': "Explicit: After TP5 [+30s]", + 'Web Content MaxMemoryForceGCV2': "Explicit: After TP5 [+30s, forced GC]", + 'Web Content EndMemoryV2': "Explicit: After TP5, tabs closed", + 'Web Content EndMemorySettledV2': "Explicit: After TP5, tabs closed [+30s]", + 'Web Content EndMemoryForceGCV2': "Explicit: After TP5, tabs closed [+30s, forced GC]" + }, + "Web Content Miscellaneous Measurements" : { + 'Web Content MaxHeapUnclassifiedV2': "Heap Unclassified: After TP5 [+30s]", + 'Web Content MaxJSV2': "JS: After TP5 [+30s]", + 'Web Content MaxImagesV2': "Images: After TP5 [+30s]" } }; @@ -556,6 +581,9 @@ var gHgBaseUrl = 'https://hg.mozilla.org/integration/mozilla-inbound'; // prepend 'Android' to series names. if (gQueryVars['mobile']) { for (var series in gSeries) { + if (series.startswith('Web Content')) + continue; + for (var dp in gSeries[series]) { gSeries[series]['Android'+dp] = gSeries[series][dp].replace("After TP5", "After tabs"); delete gSeries[series][dp]; @@ -817,8 +845,8 @@ function memoryTreeNode(target, data, select, path, depth) { // TODO Use 'mixed' units as an indicator of container nodes instead of hard // coding. - var showVal = depth >= 2; - var showPct = depth >= 3; + var showVal = depth >= 3; + var showPct = depth >= 4; // if select is passed as "a/b/c", split it so it is an array if (typeof(select) == "string") { diff --git a/slimtest_config.py b/slimtest_config.py index 3174667..0cf8dd9 100644 --- a/slimtest_config.py +++ b/slimtest_config.py @@ -33,6 +33,7 @@ 'vars': { 'test': [ 'benchtester', 'test_memory_usage.py' ], + 'e10s': True, 'proxyPort': 3128, } }, diff --git a/tests/benchtester/test_bench_tester.py b/tests/benchtester/test_bench_tester.py new file mode 100644 index 0000000..aa3be53 --- /dev/null +++ b/tests/benchtester/test_bench_tester.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import unittest + +# Janky hack to work around not having modules setup +sys.path.insert(0, "../../benchtester") +from BenchTester import BenchTester + +class BenchTesterTest(unittest.TestCase): + + def test_process_name_mapping(self): + # Test one w/ pid, one w/o + proc_names_list = [ + "Main", + "Web Content (1234)" + ] + + expected_mappings = { + "Main": "Main", + "Web Content (1234)": "Web Content" + } + + proc_name_mappings = BenchTester.map_process_names(proc_names_list) + self.assertEqual(expected_mappings, proc_name_mappings) + + # Test multiple of one type + proc_names_list = [ + "Main", + "Web Content (1234)", + "Web Content (2345)", + "Web Content (3456)" + ] + + expected_mappings = { + "Main": "Main", + "Web Content (1234)": "Web Content", + "Web Content (2345)": "Web Content 2", + "Web Content (3456)": "Web Content 3" + } + + proc_name_mappings = BenchTester.map_process_names(proc_names_list) + self.assertEqual(expected_mappings, proc_name_mappings) + + # Test multiple of several types + proc_names_list = [ + "Main", + "Web Content (1234)", + "Web Content (2345)", + "Web Content (3456)", + "GMP (1234)", + "GMP (2345)" + ] + + expected_mappings = { + "Main": "Main", + "Web Content (1234)": "Web Content", + "Web Content (2345)": "Web Content 2", + "Web Content (3456)": "Web Content 3", + "GMP (1234)": "GMP", + "GMP (2345)": "GMP 2" + } + + proc_name_mappings = BenchTester.map_process_names(proc_names_list) + self.assertEqual(expected_mappings, proc_name_mappings) + + # Test with a dictionary + proc_names_dict = { + "Main": [], + "Web Content (1234)": [], + "Web Content (2345)": [], + "Web Content (3456)": [], + "GMP (1234)": [], + "GMP (2345)": [] + } + + proc_name_mappings = BenchTester.map_process_names(proc_names_dict) + self.assertEqual(expected_mappings, proc_name_mappings) + + # Test with different pid orderings + proc_names_dict = { + "Main": [], + "Web Content (2345)": [], + "Web Content (1234)": [], + "Web Content (3456)": [], + "GMP (2345)": [], + "GMP (1234)": [] + } + + expected_mappings = { + "Main": "Main", + "Web Content (2345)": "Web Content", + "Web Content (1234)": "Web Content 2", + "Web Content (3456)": "Web Content 3", + "GMP (2345)": "GMP", + "GMP (1234)": "GMP 2" + } + + proc_name_mappings = BenchTester.map_process_names(proc_names_dict) + self.assertEqual(expected_mappings, proc_name_mappings) + + +if __name__ == '__main__': + unittest.main() diff --git a/util/update_database.py b/util/update_database.py index ad8a731..03558fe 100755 --- a/util/update_database.py +++ b/util/update_database.py @@ -89,6 +89,9 @@ print("Database is already the newest format!") sys.exit(1) +# Explicitly set the new version to 0. +cur.execute("INSERT INTO `benchtester_version` (`version`) VALUES (?)", [ 0 ]) + # Copy all non-excluded tests # (this was added so I could drop the obsolete Slimtest-TalosTP5 test from old DBs) diff --git a/util/update_database_v0_v1.py b/util/update_database_v0_v1.py new file mode 100755 index 0000000..644e017 --- /dev/null +++ b/util/update_database_v0_v1.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright © 2012 Mozilla Corporation + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + + +# Converts a database using the older unversioned format to the v1 format. + +import os +import re +import sqlite3 +import sys +import time + +sys.path.append(os.path.join('.', 'benchtester')) + +# We need gTableSchemas to create the new database +try: + import BenchTester +except: + sys.stderr.write("Couldn't find benchtester in current directory. Run me from the root!\n"); + sys.exit(1); + + +# memory report 'kind' constants +KIND_NONHEAP = 0 +KIND_HEAP = 1 +KIND_OTHER = 2 + +# memory report 'units' constants +UNITS_BYTES = 0 +UNITS_COUNT = 1 +UNITS_COUNT_CUMULATIVE = 2 +UNITS_PERCENTAGE = 3 + +if len(sys.argv) < 2: + sys.stderr.write("Usage: %s \n" % (sys.argv[0],)); + sys.stderr.write(" will create a new database named .new in the\n"); + sys.stderr.write(" newer format. The optional second parameter is one or\n"); + sys.stderr.write(" more tests (by name) to omit from the new database.\n"); + sys.exit(1); + +if not os.path.exists(sys.argv[1]): + sys.stderr.write("Database '%s' does not exist" % (sys.argv[1],)) + sys.exit(1) + +newdb = sys.argv[1] + '.new' +print("Creating %s..." % (newdb,)) +sql = sqlite3.connect(newdb, timeout=900) +sql.row_factory = sqlite3.Row +cur = sql.cursor() +for schema in BenchTester.gTableSchemas: + print(schema) + cur.execute(schema) + +# This will speed things up significantly at the expense of ~4GiB memory usage +cur.execute('''PRAGMA cache_size = -4000000''') +cur.execute('''PRAGMA temp_store = 2''') +# The new database is empty if we don't reach COMMIT, so we don't particularly +# care if we corrupt it. This also significantly speeds up the operation. +cur.execute('''PRAGMA journal_mode = OFF''') +cur.execute('''PRAGMA synchronous = OFF''') + +# Open old db +print("Opening %s..." % (sys.argv[1],)) +cur.execute('''ATTACH DATABASE ? AS old''', [ sys.argv[1] ]) + +print("Counting rows...") +cur.execute('SELECT COUNT(*) FROM old.benchtester_tests') +totalrows = cur.fetchone()[0] +print("%u total tests" % totalrows) + +# +# Determine format of old DB +# + +db_version = None +try: + cur.execute('SELECT * FROM old.benchtester_version LIMIT 1') + db_version = cur.fetchone()['version'] +except sqlite3.OperationalError: + db_version = 0 + +if db_version == BenchTester.gVersion: + print("Database is up to date, version = %s" % db_version) + sys.exit(1) +elif db_version != 0 or BenchTester.gVersion != 1: + print("This script currently only handles 0 => 1") + sys.exit(1) +else: + print("Upgrading db version from %s to %s" % (db_version, BenchTester.gVersion)) + +starttime = time.time() + +# Set the DB version +cur.execute('INSERT INTO benchtester_version(version) VALUES ( ? )', (BenchTester.gVersion, )) + +# Add the benchtester_checkpoints +cur.execute('SELECT DISTINCT meta FROM old.benchtester_data') +checkpoints = set([ row['meta'].split(':')[0] for row in cur.fetchall() ]) +cur.executemany('INSERT INTO benchtester_checkpoints(name) ' + 'VALUES (?)', ( [ checkpoint ] for checkpoint in checkpoints )) + +print("[%.02fs] Inserted %d checkpoints" % ((time.time() - starttime), len(checkpoints))) + +# Add an entry for Main in benchtester_procs +cur.execute('INSERT INTO benchtester_procs(name) VALUES ( ? )', ('Main', )) + +# Fill in the datapoints table +cur.execute('SELECT DISTINCT name AS datapoint ' + 'FROM old.benchtester_datapoints d ') + +# Given an old datapoint name, returns [ newname, units ] +def splitunits(dp): + units = UNITS_BYTES + match = re.match(r'(cnt|pct):(.*)', dp) + + if match: + dp = match.group(2) + units = UNITS_COUNT if match.group(1) == 'cnt' else UNITS_PERCENTAGE + + return [ dp, units ] + +datapoints = set(( splitunits(row['datapoint'])[0] for row in cur.fetchall() )) + +print("[%.02fs] Selected %d datapoints" % ((time.time() - starttime), len(datapoints))) + +# Insert all datapoint names +cur.executemany('INSERT OR IGNORE INTO benchtester_datapoints(name) ' + 'VALUES (?)', + ( [ dp ] for dp in datapoints )) + +print("[%.02fs] Inserted %d datapoints" % ((time.time() - starttime), len(datapoints))) + +# Copy the builds table +cur.execute('INSERT INTO benchtester_builds(id, name, time) ' + 'SELECT id, name, time from old.benchtester_builds ') + +print("[%.02fs] Copied benchtester_builds" % (time.time() - starttime)) + +# Copy the tests table +cur.execute('INSERT INTO benchtester_tests(id, name, time, build_id, successful) ' + 'SELECT id, name, time, build_id, successful FROM old.benchtester_tests') + +print("[%.02fs] Copied benchtester_tests" % (time.time() - starttime)) + +# Fill in the new benchtester_data table +data = cur.execute('SELECT d.test_id, p.name AS datapoint, d.value, d.meta ' + 'FROM old.benchtester_data d ' + 'JOIN old.benchtester_datapoints p ' + 'ON d.datapoint_id = p.id ') + +def splitmeta(meta): + return meta.split(':') + +def rowify(row): + dp, units = splitunits(row['datapoint']) + checkpoint, iteration = splitmeta(row['meta']) + proc_id = 1 # there's just the Main process in version 0 + + # we just say kind is heap if under explicit, other if not + kind = KIND_HEAP if dp.startswith('explicit') else KIND_OTHER + + return [ row['test_id'], + proc_id, + int(iteration), + row['value'], + units, + kind, + dp, + checkpoint ] + +# Insert data +cur.executemany('INSERT INTO benchtester_data(test_id,datapoint_id,checkpoint_id,proc_id,iteration,value,units,kind) ' + 'SELECT ?, p.id, c.id, ?, ?, ?, ?, ? ' + 'FROM benchtester_datapoints p, ' + ' benchtester_checkpoints c ' + 'WHERE p.name = ? AND c.name = ?', + ( rowify(row) for row in data.fetchall() )) + +print("[%.02fs] Inserted benchtester_data" % (time.time() - starttime)) + +sql.commit() + +print("[%.02fs] Committed everything" % (time.time() - starttime)) +