From 5bf7cffd2dc60f156d32e2166fe8f652169d6e17 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 12 Sep 2019 19:37:44 +0100
Subject: [PATCH 01/51] Start work on a decision task / taskgraph

---
 tools/ci/commands.json  |  10 +++
 tools/ci/taskgraph.py   | 146 ++++++++++++++++++++++++++++++++++++
 tools/ci/tasks/test.yml | 159 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 315 insertions(+)
 create mode 100644 tools/ci/taskgraph.py
 create mode 100644 tools/ci/tasks/test.yml

diff --git a/tools/ci/commands.json b/tools/ci/commands.json
index 841fd855c80568..994187ec1a7054 100644
--- a/tools/ci/commands.json
+++ b/tools/ci/commands.json
@@ -24,5 +24,15 @@
       "requests",
       "pygithub"
     ]
+  },
+  "taskgraph": {
+    "path": "taskgraph.py",
+    "script": "run",
+    "help": "Build the taskgraph",
+    "virtualenv": true,
+    "install": [
+      "requests",
+      "pyyaml"
+    ]
   }
 }
diff --git a/tools/ci/taskgraph.py b/tools/ci/taskgraph.py
new file mode 100644
index 00000000000000..7207ec5710973c
--- /dev/null
+++ b/tools/ci/taskgraph.py
@@ -0,0 +1,146 @@
+import json
+import os
+import re
+from copy import deepcopy
+
+import six
+import yaml
+from six import iteritems
+
+here = os.path.dirname(__file__)
+
+def load_task_file(path):
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def update_recursive(data, update_data):
+    for key, value in iteritems(update_data):
+        if key not in data:
+            data[key] = value
+        else:
+            initial_value = data[key]
+            if isinstance(value, dict):
+                if not isinstance(initial_value, dict):
+                    import pdb
+                    pdb.set_trace()
+                    raise ValueError
+                update_recursive(initial_value, value)
+            elif isinstance(value, list):
+                if not isinstance(initial_value, list):
+                    import pdb
+                    pdb.set_trace()
+                    raise ValueError
+                initial_value.extend(value)
+            else:
+                data[key] = value
+
+
+def resolve_use(task_data, templates):
+    rv = {}
+    if "use" in task_data:
+        for template_name in task_data["use"]:
+            update_recursive(rv, deepcopy(templates[template_name]))
+    update_recursive(rv, task_data)
+    rv.pop("use", None)
+    return rv
+
+
+def resolve_name(task_data, default_name):
+    if "name" not in task_data:
+        task_data["name"] = default_name
+    return task_data
+
+
+def resolve_chunks(task_data):
+    if "chunks" not in task_data:
+        return [task_data]
+    rv = []
+    total_chunks = task_data["chunks"]
+    for i in range(1, total_chunks + 1):
+        chunk_data = deepcopy(task_data)
+        chunk_data["chunks"] = {"id": i,
+                                "total": total_chunks}
+        rv.append(chunk_data)
+    return rv
+
+
+def replace_vars(input_string, variables):
+    variable_re = re.compile(r"(?<!\\)\${([^}]+)}")
+
+    def replacer(m):
+        var = m.group(1).split(".")
+        repl = variables
+        for part in var:
+            try:
+                repl = repl[part]
+            except KeyError:
+                import pdb
+                pdb.set_trace()
+        return str(repl)
+
+    return variable_re.sub(replacer, input_string)
+
+
+def sub_variables(data, variables):
+    if isinstance(data, six.string_types):
+        return replace_vars(data, variables)
+    if isinstance(data, list):
+        return [sub_variables(item, variables) for item in data]
+    if isinstance(data, dict):
+        return {key: sub_variables(value, variables)
+                for key, value in iteritems(data)}
+    return data
+
+
+def substitute_variables(task):
+    variables = {"vars": task.get("vars", {}),
+                 "chunks": task.get("chunks", {})}
+
+    return sub_variables(task, variables)
+
+
+def expand_maps(task):
+    name = task.keys()[0]
+    if name != "$map":
+        return [task]
+
+    map_data = task["$map"]
+    assert set(map_data.keys()) == set(["for", "do"])
+    rv = []
+    for for_data in map_data["for"]:
+        do_values = expand_maps(map_data["do"])
+        for do_data in do_values:
+            task_data = deepcopy(for_data)
+            assert len(do_data.keys()) == 1
+            name = do_data.keys()[0]
+            update_recursive(task_data, deepcopy(do_data[name]))
+            rv.append({name: task_data})
+    return rv
+
+
+def load_tasks(tasks_data):
+    map_resolved_tasks = {}
+    tasks = []
+
+    for task in tasks_data["tasks"]:
+        assert len(task.keys()) == 1
+        for task in expand_maps(task):
+            assert len(task.keys()) == 1
+            name = task.keys()[0]
+            data = task[name]
+            new_name = sub_variables(name, {"vars": data.get("vars", {})})
+            assert new_name not in map_resolved_tasks
+            map_resolved_tasks[new_name] = substitute_variables(data)
+
+    for task_default_name, data in iteritems(map_resolved_tasks):
+        task = resolve_use(data, tasks_data["templates"])
+        task = resolve_name(task, task_default_name)
+        tasks.extend(resolve_chunks(task))
+
+    return [substitute_variables(task) for task in tasks]
+
+
+def run(venv, **kwargs):
+    tasks_data = load_task_file(os.path.join(here, "tasks/test.yml"))
+    print(json.dumps(load_tasks(tasks_data), indent=2))
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
new file mode 100644
index 00000000000000..da69e425467b32
--- /dev/null
+++ b/tools/ci/tasks/test.yml
@@ -0,0 +1,159 @@
+templates:
+  wpt-base:
+    image: harjgam/web-platform-tests:0.33
+    maxRunTime: 7200,
+    artifacts:
+      public/results:
+        path: /home/test/artifacts
+        type: directory
+
+  wpt-testharness:
+    chunks: 12
+    vars:
+      test-type: testharness
+
+  wpt-reftest:
+    chunks: 6
+    vars:
+      test-type: reftest
+
+  wpt-wdspec:
+    chunks: 1
+    vars:
+      test-type: wdspec
+
+  wpt-run:
+    name: wpt-${vars.browser}-${vars.channel}-${vars.suite}-chunk-${chunks.id}
+    options:
+      xvfb: true
+      oom-killer: true
+      hosts: true
+      browser: ${vars.browser}
+      channel: ${vars.channel}
+    command: >-
+      ./tools/ci/taskcluster-run.py
+      ${vars.browser}
+      --
+      --channel=${vars.channel}
+      --log-wptreport=../artifacts/wpt_report.json
+      --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+      --no-fail-on-unexpected
+      --this-chunk=${chunks.id}
+      --total-chunks=${chunks.total}
+
+  trigger-master:
+    trigger:
+      branch:
+        - master
+
+  trigger-push:
+    trigger:
+      branch:
+        - triggers/${vars.browser}_${vars.channel}
+
+  trigger-daily:
+    trigger:
+      branch:
+        - epochs/daily
+
+  trigger-weekly:
+    trigger:
+      branch:
+        - epochs/weekly
+
+  trigger-pr:
+    trigger:
+      pull-request:
+
+  browser-firefox:
+    require:
+      - download-firefox-${vars.channel}
+
+  browser-chrome: {}
+
+
+tasks:
+  - $map:
+      for:
+        - vars:
+            suite: testharness
+        - vars:
+            suite: reftest
+        - vars:
+            suite: wdspec
+      do:
+        $map:
+          for:
+            - vars:
+                browser: firefox
+                channel: nightly
+              use:
+                - trigger-master
+                - trigger-push
+            - vars:
+                browser: firefox
+                channel: beta
+              use:
+                - trigger-weekly
+                - trigger-push
+            - vars:
+                browser: firefox
+                channel: stable
+              use:
+                - trigger-daily
+                - trigger-push
+            - vars:
+                browser: chrome
+                channel: dev
+              use:
+                - trigger-master
+                - trigger-push
+            - vars:
+                browser: chrome
+                channel: beta
+              use:
+                - trigger-weekly
+                - trigger-push
+            - vars:
+                browser: chrome
+                channel: stable
+              use:
+                - trigger-daily
+                - trigger-push
+          do:
+            ${vars.browser}-${vars.channel}-${vars.suite}:
+              use:
+                - wpt-base
+                - wpt-run
+                - browser-${vars.browser}
+                - wpt-${vars.suite}
+
+  - $map:
+      for:
+        - vars:
+            channel: nightly
+        - vars:
+            channel: beta
+        - vars:
+            channel: stable
+      do:
+        download-firefox-${vars.channel}:
+          use:
+            - wpt-base
+          command: "./wpt download --channel=${vars.channel} firefox browser"
+
+  - lint:
+      use:
+        - wpt-base
+        - trigger-master
+        - trigger-pr
+      command: "wpt-lint --all"
+
+  - update-built:
+      use:
+        - wpt-base
+        - trigger-pr
+      schedule-if:
+        run-job: update_built
+      command: "tools/ci/ci_built_diff.sh"
+

From b235c47bf4f267a678df1e90f6546365c396f517 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 17 Sep 2019 08:54:02 +0900
Subject: [PATCH 02/51] More wip

---
 tools/ci/commands.json  |  11 ++++
 tools/ci/decision.py    | 118 ++++++++++++++++++++++++++++++++++++++++
 tools/ci/taskgraph.py   |  12 ++--
 tools/ci/tasks/test.yml |   3 +-
 4 files changed, 139 insertions(+), 5 deletions(-)
 create mode 100644 tools/ci/decision.py

diff --git a/tools/ci/commands.json b/tools/ci/commands.json
index 994187ec1a7054..b846eb5ecb2116 100644
--- a/tools/ci/commands.json
+++ b/tools/ci/commands.json
@@ -34,5 +34,16 @@
       "requests",
       "pyyaml"
     ]
+  },
+  "decision": {
+    "path": "decision.py",
+    "script": "run",
+    "help": "Run the decision task",
+    "virtualenv": true,
+    "install": [
+        "requests",
+        "pyyaml",
+        "taskcluster"
+    ]
   }
 }
diff --git a/tools/ci/decision.py b/tools/ci/decision.py
new file mode 100644
index 00000000000000..b5f1446af535a6
--- /dev/null
+++ b/tools/ci/decision.py
@@ -0,0 +1,118 @@
+import json
+import os
+import taskgraph
+from six import iteritems
+from six.moves.urllib import request
+
+
+QUEUE_BASE = "https://queue.taskcluster.net/v1/task"
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def get_triggers(event):
+    # Set some variables that we use to get the commits on the current branch
+    ref_prefix = "refs/heads/"
+    pull_request = "pull_request" in event
+    branch = None
+    if not pull_request and "ref" in event:
+        branch = event["ref"]
+        if branch.startswith(ref_prefix):
+            branch = branch[len(ref_prefix):]
+
+    return pull_request, branch
+
+
+def fetch_event_data():
+    try:
+        task_id = os.environ["TASK_ID"]
+    except KeyError:
+        print("WARNING: Missing TASK_ID environment variable")
+        # For example under local testing
+        return None
+
+    resp = request("%s/%s" % (QUEUE_BASE, task_id))
+
+    task_data = json.load(resp)
+    event_data = task_data.get("extra", {}).get("github_event")
+    if event_data is not None:
+        return json.loads(event_data)
+
+
+def filter_triggers(event, all_tasks):
+    is_pr, branch = get_triggers(event)
+    triggered = {}
+    for name, task in iteritems(all_tasks):
+        if "trigger" in task:
+            if is_pr and "pull-request" in task["trigger"]:
+                triggered[name] = task
+            elif branch is not None and "branch" in task["trigger"]:
+                for trigger_branch in task["trigger"]["branch"]:
+                    if (trigger_branch == branch or
+                        trigger_branch.endswith("*") and branch.startswith(trigger_branch[:-1])):
+                        triggered[name] = task
+    return triggered
+
+
+def get_run_jobs():
+    import jobs
+    paths = jobs.get_paths()
+    return jobs.get_jobs(paths)
+
+
+def filter_schedule_if(tasks):
+    scheduled = {}
+    run_jobs = None
+    for name, task in iteritems(tasks):
+        if "schedule-if" in task:
+            if "run-job" in task["schedule-if"]:
+                if run_jobs is None:
+                    run_jobs = get_run_jobs()
+                if any(item in run_jobs for item in task["schedule-if"]):
+                    scheduled[name] = task
+        else:
+            scheduled[name] = task
+    return scheduled
+
+
+class TaskNode(object):
+    def __init__(self, data):
+        self.data = data
+        self.children = []
+
+    def append(self, other):
+        self.children.append(other)
+
+def build_task_graph(tasks, all_task):
+    task_id_map = {}
+
+    def add_task(task_name, task):
+        required_ids = []
+        if "require" in task:
+            for required_name in task["require"]:
+                if required_name not in task_id_map:
+                    required_id = add_task(required_name,
+                                           all_tasks.get(required_name))
+                    
+
+    for task_name in tasks:
+        add_task(task)
+
+
+def run(venv, **kwargs):
+    if "TASK_EVENT" in os.environ:
+        event = json.loads(os.environ["TASK_EVENT"])
+    else:
+        event = fetch_event_data()
+
+    all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
+
+    print(all_tasks)
+
+    triggered_tasks = filter_triggers(event, all_tasks)
+    scheduled_tasks = filter_schedule_if(triggered_tasks)
+
+    task_graph = build_task_graph(all_tasks, triggered_tasks)
+
+
diff --git a/tools/ci/taskgraph.py b/tools/ci/taskgraph.py
index 7207ec5710973c..d61c48b9372388 100644
--- a/tools/ci/taskgraph.py
+++ b/tools/ci/taskgraph.py
@@ -74,7 +74,7 @@ def replacer(m):
         for part in var:
             try:
                 repl = repl[part]
-            except KeyError:
+            except Exception:
                 import pdb
                 pdb.set_trace()
         return str(repl)
@@ -138,9 +138,13 @@ def load_tasks(tasks_data):
         task = resolve_name(task, task_default_name)
         tasks.extend(resolve_chunks(task))
 
-    return [substitute_variables(task) for task in tasks]
+    tasks = [substitute_variables(task_data) for task_data in tasks]
+    return {task["name"]: task for task in tasks}
+
+
+def load_tasks_from_path(path):
+    return load_tasks(load_task_file(path))
 
 
 def run(venv, **kwargs):
-    tasks_data = load_task_file(os.path.join(here, "tasks/test.yml"))
-    print(json.dumps(load_tasks(tasks_data), indent=2))
+    print(json.dumps(load_tasks_from_path(os.path.join(here, "tasks/test.yml")), indent=2))
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index da69e425467b32..3ded4f1cdb8a3f 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -154,6 +154,7 @@ tasks:
         - wpt-base
         - trigger-pr
       schedule-if:
-        run-job: update_built
+        run-job:
+          - update_built
       command: "tools/ci/ci_built_diff.sh"
 

From 6c2b7098ffcffe65a9675d2cf19781ca0bc3c331 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Mon, 23 Sep 2019 19:34:48 +0100
Subject: [PATCH 03/51] Get as far as being able to print tasks to schedule

---
 tools/ci/decision.py    | 113 +++++++++++++++++++++++++++++++++-------
 tools/ci/taskgraph.py   |   9 ++--
 tools/ci/tasks/test.yml |  62 +++++++++++++++++++++-
 3 files changed, 161 insertions(+), 23 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index b5f1446af535a6..18d0df49acd8ac 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -1,9 +1,11 @@
 import json
 import os
-import taskgraph
+
+import taskcluster
 from six import iteritems
 from six.moves.urllib import request
 
+from . import taskgraph
 
 QUEUE_BASE = "https://queue.taskcluster.net/v1/task"
 
@@ -76,15 +78,87 @@ def filter_schedule_if(tasks):
     return scheduled
 
 
-class TaskNode(object):
-    def __init__(self, data):
-        self.data = data
-        self.children = []
-
-    def append(self, other):
-        self.children.append(other)
-
-def build_task_graph(tasks, all_task):
+def get_fetch_rev(event):
+    is_pr, _ = get_triggers(event)
+    if is_pr:
+        return event["pull_request"]["merge_commit_sha"]
+    else:
+        return event["after"]
+
+
+def build_full_command(event, task):
+    cmd_args = {
+        "task_name": task["name"],
+        "repo_url": event["repository"]["url"],
+        "fetch_rev": get_fetch_rev(event),
+        "task_cmd": task["command"]
+    }
+
+    options = task.get("options", {})
+    options_args = []
+    if options.get("oom-killer"):
+        options_args.append("--oom-killer")
+    if options.get("xvfb"):
+        options_args.append("--xvfb")
+    if not options.get("hosts"):
+        options_args.append("--no-hosts")
+    if not options.get("hosts"):
+        options_args.append("--no-hosts")
+    if options.get("checkout"):
+        options_args.append("--checkout=%s" % options["checkout"])
+
+    cmd_args["options_str"] = "\n".join("  %s" % item for item in options_args)
+
+    return ["/bin/bash",
+            "--login",
+            "-c",
+            """
+~/start.sh
+  %(repo_url)s
+  %(fetch_rev)s;
+cd web-platform-tests;
+./tools/ci/run_tc.py
+  %(options_str)s
+  %(task_cmd)s;
+""" % cmd_args]
+
+
+def create_tc_task(event, task, required_task_ids):
+    command = build_full_command(event, task)
+    worker_type = ("wpt-docker-worker"
+                   if event["repository"]["full_name"] == 'web-platform-tests/wpt'
+                   else "github-worker")
+    task_id = taskcluster.slugId()
+    task_data = {
+        "taskGroupId": "", # TODO
+        "created": taskcluster.fromNowJSON(""),
+        "deadline": taskcluster.fromNowJSON(task["deadline"]),
+        "provisionerId": task["provisionerId"],
+        "workerType": worker_type,
+        "metadata": {
+            "name": task["name"],
+            "description": task.get("description", ""),
+            "owner": "%s@users.noreply.github.com" % event["sender"]["login"],
+            "source": event["repository"]["url"]
+        },
+        "payload": {
+            "artifacts": task.get("artifacts"),
+            "command": command,
+            "image": task.get("image"),
+            "maxRunTime": task.get("maxRunTime"),
+            "env": task.get("env", []),
+        },
+        "extras": {
+            "github_event": json.dumps(event)
+        }
+    }
+    if required_task_ids:
+        task_data["dependencies"] = required_task_ids
+        task_data["requires"] = "all-completed"
+    return task_id, task_data
+
+
+def build_task_graph(event, all_tasks, tasks):
     task_id_map = {}
 
     def add_task(task_name, task):
@@ -92,13 +166,16 @@ def add_task(task_name, task):
         if "require" in task:
             for required_name in task["require"]:
                 if required_name not in task_id_map:
-                    required_id = add_task(required_name,
-                                           all_tasks.get(required_name))
-                    
+                    add_task(required_name,
+                             all_tasks[required_name])
+                required_ids.append(task_id_map[required_name][0])
+        task_id, task_data = create_tc_task(event, task, required_ids)
+        task_id_map[task_name] = (task_id, task_data)
 
-    for task_name in tasks:
-        add_task(task)
+    for task_name, task in iteritems(tasks):
+        add_task(task_name, task)
 
+    return task_id_map
 
 def run(venv, **kwargs):
     if "TASK_EVENT" in os.environ:
@@ -108,11 +185,11 @@ def run(venv, **kwargs):
 
     all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
 
-    print(all_tasks)
+    print(json.dumps(all_tasks, indent=2))
 
     triggered_tasks = filter_triggers(event, all_tasks)
     scheduled_tasks = filter_schedule_if(triggered_tasks)
 
-    task_graph = build_task_graph(all_tasks, triggered_tasks)
-
+    task_id_map = build_task_graph(event, all_tasks, triggered_tasks)
 
+    print(json.dumps(task_id_map, indent=2))
diff --git a/tools/ci/taskgraph.py b/tools/ci/taskgraph.py
index d61c48b9372388..012962dfab8733 100644
--- a/tools/ci/taskgraph.py
+++ b/tools/ci/taskgraph.py
@@ -75,8 +75,8 @@ def replacer(m):
             try:
                 repl = repl[part]
             except Exception:
-                import pdb
-                pdb.set_trace()
+                # Don't substitute
+                return m.group(0)
         return str(repl)
 
     return variable_re.sub(replacer, input_string)
@@ -130,11 +130,12 @@ def load_tasks(tasks_data):
             name = task.keys()[0]
             data = task[name]
             new_name = sub_variables(name, {"vars": data.get("vars", {})})
-            assert new_name not in map_resolved_tasks
+            if new_name in map_resolved_tasks:
+                raise ValueError("Got duplicate task name %s" % new_name)
             map_resolved_tasks[new_name] = substitute_variables(data)
 
     for task_default_name, data in iteritems(map_resolved_tasks):
-        task = resolve_use(data, tasks_data["templates"])
+        task = resolve_use(data, tasks_data["components"])
         task = resolve_name(task, task_default_name)
         tasks.extend(resolve_chunks(task))
 
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 3ded4f1cdb8a3f..d497f350fb1a43 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -1,5 +1,7 @@
-templates:
+components:
   wpt-base:
+    provisionerId: aws-provisioner-v1
+    deadline: "24 hours"
     image: harjgam/web-platform-tests:0.33
     maxRunTime: 7200,
     artifacts:
@@ -73,6 +75,7 @@ templates:
 
 
 tasks:
+  # Run full suites on push
   - $map:
       for:
         - vars:
@@ -127,7 +130,63 @@ tasks:
                 - wpt-run
                 - browser-${vars.browser}
                 - wpt-${vars.suite}
+              description: >-
+                A subset of WPT's "${vars.suite}" tests (chunk number ${chunks.id}
+                of ${chunks.total}), run in the ${vars.channel} release of
+                ${vars.browser}.
 
+  - $map:
+      for:
+        - vars:
+            browser: firefox
+            channel: nightly
+        - vars:
+            browser: chrome
+            channel: dev
+      do:
+        wpt-${vars.browser}-${vars.channel}-stability:
+          use:
+            - wpt-base
+            - browser-${vars.browser}
+          description: >-
+            Verify that all tests affected by a pull request are stable
+            when executed in ${vars.browser}.
+          command: >-
+            ./tools/ci/taskcluster-run.py
+            --commit-range base_head
+            ${vars.browser}
+            --
+            --channel=${vars.channel}
+            --verify
+          wpt-${vars.browser}-${vars.channel}-results:
+            description: >-
+              Collect results for all tests affected by a pull request in
+              ${vars.browser}.
+            command: >-
+              ./tools/ci/taskcluster-run.py
+              --commit-range base_head
+              ${vars.browser}
+              --
+              --channel=${vars.channel}
+              --no-fail-on-unexpected
+              --log-wptreport=../artifacts/wpt_report.json
+              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+
+          wpt-${vars.browser}-${vars.channel}-results-without-changes:
+            options:
+              checkout: base_head
+            description: >-
+              Collect results for all tests affected by a pull request in
+              ${vars.browser} but without the changes in the PR.
+            command: >-
+              ./tools/ci/taskcluster-run.py
+              --commit-range task_head
+              ${vars.browser}
+              --
+              --channel=${vars.channel}
+              --no-fail-on-unexpected
+              --log-wptreport=../artifacts/wpt_report.json
+              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
   - $map:
       for:
         - vars:
@@ -148,6 +207,7 @@ tasks:
         - trigger-master
         - trigger-pr
       command: "wpt-lint --all"
+      description:
 
   - update-built:
       use:

From f3d4483a32d87c0a8f2fa27013a35e0d8397613f Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 15:10:06 +0100
Subject: [PATCH 04/51] Add unittests

---
 tools/ci/decision.py    | 11 +++++++++++
 tools/ci/tasks/test.yml | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 18d0df49acd8ac..8a78593bb75ba5 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -109,6 +109,16 @@ def build_full_command(event, task):
 
     cmd_args["options_str"] = "\n".join("  %s" % item for item in options_args)
 
+    install_str = "";
+    install_packages = task.get("install")
+    if install_packages:
+        install_items = ["apt update -qqy"]
+        install_items.extend("apt install -qqy %s" % item
+                             for item in install_packages)
+        install_str = "\n".join("sudo %s;" % item for item in install_items)
+
+    cmd_args["install_cmd"] = install_str
+
     return ["/bin/bash",
             "--login",
             "-c",
@@ -116,6 +126,7 @@ def build_full_command(event, task):
 ~/start.sh
   %(repo_url)s
   %(fetch_rev)s;
+%(install_str)s
 cd web-platform-tests;
 ./tools/ci/run_tc.py
   %(options_str)s
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index d497f350fb1a43..198b2e51ab20f0 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -218,3 +218,22 @@ tasks:
           - update_built
       command: "tools/ci/ci_built_diff.sh"
 
+  - tools/unittests (Python 2):
+      description: >-
+        Unit tests for tools running under Python 2.7, excluding wptrunner
+      command: tools/ci/ci_tools_unittest.sh
+      env:
+        - TOXENV=py27;
+        - HYPOTHESIS_PROFILE=ci;
+        - PY_COLORS=0;
+
+  - tools/unittests (Python 3):
+      description: >-
+        Unit tests for tools running under Python 3, excluding wptrunner
+      command: tools/ci/ci_tools_unittest.sh
+      env:
+        - TOXENV=py36;
+        - HYPOTHESIS_PROFILE=ci;
+        - PY_COLORS=0;
+      install:
+        - python3-pip

From 1f3be2a9239ce8cf2cf526ca461ded9f8d471722 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 15:10:42 +0100
Subject: [PATCH 05/51] Make it possible to schedule tasks

---
 tools/ci/commands.json |  1 +
 tools/ci/decision.py   | 65 ++++++++++++++++++++++++++++++++----------
 2 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/tools/ci/commands.json b/tools/ci/commands.json
index b846eb5ecb2116..3d731c30fa70d9 100644
--- a/tools/ci/commands.json
+++ b/tools/ci/commands.json
@@ -37,6 +37,7 @@
   },
   "decision": {
     "path": "decision.py",
+    "parser": "get_parser",
     "script": "run",
     "help": "Run the decision task",
     "virtualenv": true,
diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 8a78593bb75ba5..4efb5b29588b78 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -1,12 +1,15 @@
+import argparse
 import json
 import os
+from collections import OrderedDict
 
 import taskcluster
-from six import iteritems
+from six import iteritems, itervalues
 from six.moves.urllib import request
 
 from . import taskgraph
 
+TC_ROOT = "https://taskcluster.net"
 QUEUE_BASE = "https://queue.taskcluster.net/v1/task"
 
 
@@ -39,7 +42,7 @@ def fetch_event_data():
     task_data = json.load(resp)
     event_data = task_data.get("extra", {}).get("github_event")
     if event_data is not None:
-        return json.loads(event_data)
+        return event_data
 
 
 def filter_triggers(event, all_tasks):
@@ -59,7 +62,7 @@ def filter_triggers(event, all_tasks):
 
 def get_run_jobs():
     import jobs
-    paths = jobs.get_paths()
+    paths = jobs.get_paths(revish=None)
     return jobs.get_jobs(paths)
 
 
@@ -134,18 +137,19 @@ def build_full_command(event, task):
 """ % cmd_args]
 
 
-def create_tc_task(event, task, required_task_ids):
+def create_tc_task(event, task, taskgroup_id, required_task_ids):
     command = build_full_command(event, task)
     worker_type = ("wpt-docker-worker"
                    if event["repository"]["full_name"] == 'web-platform-tests/wpt'
                    else "github-worker")
     task_id = taskcluster.slugId()
     task_data = {
-        "taskGroupId": "", # TODO
+        "taskGroupId": taskgroup_id,
         "created": taskcluster.fromNowJSON(""),
         "deadline": taskcluster.fromNowJSON(task["deadline"]),
         "provisionerId": task["provisionerId"],
         "workerType": worker_type,
+        "priority": "lowest",
         "metadata": {
             "name": task["name"],
             "description": task.get("description", ""),
@@ -159,7 +163,7 @@ def create_tc_task(event, task, required_task_ids):
             "maxRunTime": task.get("maxRunTime"),
             "env": task.get("env", []),
         },
-        "extras": {
+        "extra": {
             "github_event": json.dumps(event)
         }
     }
@@ -170,7 +174,8 @@ def create_tc_task(event, task, required_task_ids):
 
 
 def build_task_graph(event, all_tasks, tasks):
-    task_id_map = {}
+    task_id_map = OrderedDict()
+    taskgroup_id = os.environ.get("TASK_ID", taskcluster.slugId())
 
     def add_task(task_name, task):
         required_ids = []
@@ -180,7 +185,7 @@ def add_task(task_name, task):
                     add_task(required_name,
                              all_tasks[required_name])
                 required_ids.append(task_id_map[required_name][0])
-        task_id, task_data = create_tc_task(event, task, required_ids)
+        task_id, task_data = create_tc_task(event, task, taskgroup_id, required_ids)
         task_id_map[task_name] = (task_id, task_data)
 
     for task_name, task in iteritems(tasks):
@@ -188,19 +193,49 @@ def add_task(task_name, task):
 
     return task_id_map
 
-def run(venv, **kwargs):
-    if "TASK_EVENT" in os.environ:
-        event = json.loads(os.environ["TASK_EVENT"])
+
+def create_tasks(queue, task_id_map):
+    for (task_id, task_data) in itervalues(task_id_map):
+        queue.createTask(task_id, task_data)
+
+
+def get_event(**kwargs):
+    if "event_path" in kwargs:
+        with open(kwargs["event_path"]) as f:
+            event_str = f.read()
+    elif "TASK_EVENT" in os.environ:
+        event_str = os.environ["TASK_EVENT"]
     else:
-        event = fetch_event_data()
+        event_str = fetch_event_data()
+    if not event_str:
+        raise ValueError("Can't find GitHub event definition; for local testing pass --event-path")
+    return json.loads(event_str)
 
-    all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
 
-    print(json.dumps(all_tasks, indent=2))
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--event-path",
+                        help="Path to file containing serialized GitHub event")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Don't actually create the tasks, just output the tasks that "
+                        "would be created")
+    return parser
+
+
+def run(venv, **kwargs):
+    auth = taskcluster.Auth({'rootUrl': TC_ROOT}, taskcluster.optionsFromEnvironment())
+    queue = taskcluster.Queue({'rootUrl': TC_ROOT})
+
+    event = get_event(**kwargs)
+
+    all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
 
     triggered_tasks = filter_triggers(event, all_tasks)
     scheduled_tasks = filter_schedule_if(triggered_tasks)
 
     task_id_map = build_task_graph(event, all_tasks, triggered_tasks)
 
-    print(json.dumps(task_id_map, indent=2))
+    if not kwargs["dry_run"]:
+        create_tasks(queue, task_id_map)
+    else:
+        print(json.dumps(task_id_map, indent=2))

From 3aeca509274d5b8b390a29cdf06680f6df9da61c Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 19:12:35 +0100
Subject: [PATCH 06/51] Add .taskcluster.yml

---
 .taskcluster.yml        | 393 +++++-----------------------------------
 tools/ci/decision.py    |  27 +--
 tools/ci/tasks/test.yml |  93 +++++++++-
 3 files changed, 149 insertions(+), 364 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 61cc38c9bd1d05..cfb548c542f387 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -4,351 +4,52 @@ policy:
 tasks:
   $let:
     event_str: {$json: {$eval: event}}
+    scope_suffix:
+      $if: 'tasks_for == "github-push"'
+      then:
+        $let:
+          branch:
+            $if: "event.ref[:11] == 'refs/heads/'"
+            then: "${event.ref[11:]}"
+        in: "branch:${branch}"
+      else: "pull-request"
   in:
-    $flattenDeep:
-      - $if: tasks_for == "github-push"
-        then:
-          $map:
-            $flatten:
-              $match: {
-                event.ref == "refs/heads/master": [{name: firefox, channel: nightly}, {name: chrome, channel: dev}],
-                event.ref == "refs/heads/epochs/daily": [{name: firefox, channel: stable}, {name: chrome, channel: stable}],
-                event.ref == "refs/heads/epochs/weekly": [{name: firefox, channel: beta}, {name: chrome, channel: beta}],
-                event.ref == "refs/heads/triggers/chrome_stable": [{name: chrome, channel: stable}],
-                event.ref == "refs/heads/triggers/chrome_beta": [{name: chrome, channel: beta}],
-                event.ref == "refs/heads/triggers/chrome_dev": [{name: chrome, channel: dev}],
-                event.ref == "refs/heads/triggers/firefox_stable": [{name: firefox, channel: stable}],
-                event.ref == "refs/heads/triggers/firefox_beta": [{name: firefox, channel: beta}],
-                event.ref == "refs/heads/triggers/firefox_nightly": [{name: firefox, channel: nightly}]
-                }
-          each(browser):
-            $map:
-              - [testharness, 1, 15]
-              - [testharness, 2, 15]
-              - [testharness, 3, 15]
-              - [testharness, 4, 15]
-              - [testharness, 5, 15]
-              - [testharness, 6, 15]
-              - [testharness, 7, 15]
-              - [testharness, 8, 15]
-              - [testharness, 9, 15]
-              - [testharness, 10, 15]
-              - [testharness, 11, 15]
-              - [testharness, 12, 15]
-              - [testharness, 13, 15]
-              - [testharness, 14, 15]
-              - [testharness, 15, 15]
-              - [reftest, 1, 10]
-              - [reftest, 2, 10]
-              - [reftest, 3, 10]
-              - [reftest, 4, 10]
-              - [reftest, 5, 10]
-              - [reftest, 6, 10]
-              - [reftest, 7, 10]
-              - [reftest, 8, 10]
-              - [reftest, 9, 10]
-              - [reftest, 10, 10]
-              - [wdspec, 1, 1]
-            each(chunk):
-              taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
-              taskGroupId: {$eval: 'as_slugid("task group")'}
-              created: {$fromNow: ''}
-              deadline: {$fromNow: '24 hours'}
-              provisionerId: aws-provisioner-v1
-              workerType:
-                $if: event.repository.full_name == 'web-platform-tests/wpt'
-                then:
-                  wpt-docker-worker
-                else:
-                  github-worker
-              metadata:
-                name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
-                description: >-
-                  A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
-                  of ${chunk[2]}), run in the ${browser.channel} release of
-                  ${browser.name}.
-                owner: ${event.pusher.email}
-                source: ${event.repository.url}
-              payload:
-                image: harjgam/web-platform-tests:0.33
-                maxRunTime: 7200
-                artifacts:
-                  public/results:
-                    path: /home/test/artifacts
-                    type: directory
-                command:
-                  - /bin/bash
-                  - --login
-                  - -c
-                  - set -ex;
-                    echo "wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}";
-                    ~/start.sh
-                      ${event.repository.url}
-                      ${event.ref};
-                    cd ~/web-platform-tests;
-                    sudo cp tools/certs/cacert.pem
-                        /usr/local/share/ca-certificates/cacert.crt;
-                    sudo update-ca-certificates;
-                    ./tools/ci/run_tc.py
-                      --checkout=${event.after}
-                      --oom-killer
-                      --hosts
-                      --browser=${browser.name}
-                      --channel=${browser.channel}
-                      --xvfb
-                      run-all
-                      ./tools/ci/taskcluster-run.py
-                        ${browser.name}
-                        --
-                        --channel=${browser.channel}
-                        --log-wptreport=../artifacts/wpt_report.json
-                        --log-wptscreenshot=../artifacts/wpt_screenshot.txt
-                        --no-fail-on-unexpected
-                        --test-type=${chunk[0]}
-                        --this-chunk=${chunk[1]}
-                        --total-chunks=${chunk[2]};
-              extra:
-                github_event: "${event_str}"
-
-      - $if: tasks_for == "github-pull-request"
-        # PR tasks that run the tests in various configurations
-        then:
-          # Taskcluster responds to a number of events issued by the GitHub API
-          # which should not trigger re-validation.
-          $if: event.action in ['opened', 'reopened', 'synchronize']
+    - $if: '(tasks_for == "github-push" && event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly"]) || tasks_for == "github-pull-request"'
+      then:
+        taskId: "${ownTaskId}"
+        taskGroupId: "${ownTaskId}"
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '24 hours'}
+        provisionerId: aws-provisioner-v1
+        workerType:
+          $if: event.repository.full_name == 'web-platform-tests/wpt'
           then:
-            $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
-            each(browser):
-              $map:
-                # This is the main place to define new stability checks
-                - name: wpt-${browser.name}-${browser.channel}-stability
-                  checkout: task_head
-                  diff_base: base_head
-                  description: >-
-                    Verify that all tests affected by a pull request are stable
-                    when executed in ${browser.name}.
-                  extra_args: '--verify'
-                - name: wpt-${browser.name}-${browser.channel}-results
-                  checkout: task_head
-                  diff_base: base_head
-                  description: >-
-                    Collect results for all tests affected by a pull request in
-                    ${browser.name}.
-                  extra_args: >-
-                    --no-fail-on-unexpected
-                    --log-wptreport=../artifacts/wpt_report.json
-                    --log-wptscreenshot=../artifacts/wpt_screenshot.txt
-                - name: wpt-${browser.name}-${browser.channel}-results-without-changes
-                  checkout: base_head
-                  diff_base: task_head
-                  description: >-
-                    Collect results for all tests affected by a pull request in
-                    ${browser.name} but without the changes in the PR.
-                  extra_args: >-
-                    --no-fail-on-unexpected
-                    --log-wptreport=../artifacts/wpt_report.json
-                    --log-wptscreenshot=../artifacts/wpt_screenshot.txt
-              each(operation):
-                taskId: {$eval: 'as_slugid(operation.name)'}
-                taskGroupId: {$eval: 'as_slugid("task group")'}
-                created: {$fromNow: ''}
-                deadline: {$fromNow: '24 hours'}
-                provisionerId: aws-provisioner-v1
-                workerType:
-                  $if: event.repository.full_name == 'web-platform-tests/wpt'
-                  then:
-                    wpt-docker-worker
-                  else:
-                    github-worker
-                metadata:
-                  name: ${operation.name}
-                  description: ${operation.description}
-                  owner: ${event.pull_request.user.login}@users.noreply.github.com
-                  source: ${event.repository.url}
-                payload:
-                  image: harjgam/web-platform-tests:0.33
-                  maxRunTime: 7200
-                  artifacts:
-                    public/results:
-                      path: /home/test/artifacts
-                      type: directory
-                  # Fetch the GitHub-provided merge commit (rather than the pull
-                  # request branch) so that the tasks simulate the behavior of the
-                  # submitted patch after it is merged. Using the merge commit also
-                  # simplifies detection of modified files because the first parent
-                  # of the merge commit can consistently be used to summarize the
-                  # changes.
-                  command:
-                    - /bin/bash
-                    - --login
-                    - -c
-                    - set -ex;
-                      echo "${operation.name}";
-                      ~/start.sh
-                        ${event.repository.clone_url}
-                        refs/pull/${event.number}/merge;
-                      cd web-platform-tests;
-                      ./tools/ci/run_tc.py
-                        --checkout=${operation.checkout}
-                        --oom-killer
-                        --browser=${browser.name}
-                        --channel=${browser.channel}
-                        --xvfb
-                        stability
-                        ./tools/ci/taskcluster-run.py
-                          --commit-range ${operation.diff_base}
-                          ${browser.name}
-                          --
-                          --channel=${browser.channel}
-                          ${operation.extra_args};
-                extra:
-                  github_event: "${event_str}"
-
-      - $map:
-          # This is the main point to define new CI checks other than stability checks
-        - name: lint
-          description: >-
-            Lint for wpt-specific requirements
-          script: >-
-            ./tools/ci/run_tc.py \
-              --no-hosts \
-              lint \
-             ./wpt lint --all
-          conditions:
-            push
-            pull-request
-        - name: update built tests
-          description: >-
-            Ensure test suites that require a build step are updated
-          script: >-
-            ./tools/ci/run_tc.py \
-             --no-hosts \
-             update_built \
-             tools/ci/ci_built_diff.sh
-          conditions:
-            pull-request
-        - name: tools/ unittests (Python 2)
-          description: >-
-            Unit tests for tools running under Python 2.7, excluding wptrunner
-          script: >-
-            export TOXENV=py27;
-            export HYPOTHESIS_PROFILE=ci;
-            export PY_COLORS=0;
-            ./tools/ci/run_tc.py \
-              tools_unittest \
-              tools/ci/ci_tools_unittest.sh
-          conditions:
-            push
-            pull-request
-        - name: tools/ unittests (Python 3)
-          description: >-
-            Unit tests for tools running under Python 3, excluding wptrunner
-          script: >-
-            export TOXENV=py36;
-            export HYPOTHESIS_PROFILE=ci;
-            export PY_COLORS=0;
-            sudo apt update -qqy;
-            sudo apt install -qqy python3-pip;
-            ./tools/ci/run_tc.py \
-              tools_unittest \
-              tools/ci/ci_tools_unittest.sh
-          conditions:
-            push
-            pull-request
-        - name: tools/wpt/ tests
-          description: >-
-            Integration tests for wpt commands
-          script: >-
-            export TOXENV=py27;
-            sudo apt update -qqy;
-            sudo apt install -qqy libnss3-tools;
-            ./tools/ci/run_tc.py \
-              --oom-killer \
-              --browser=firefox \
-              --browser=chrome \
-              --channel=experimental \
-              --xvfb \
-              wpt_integration \
-              tools/ci/ci_wpt.sh
-          conditions:
-            pull-request
-        - name: resources/ tests
-          description: >-
-            Tests for testharness.js and other files in resources/
-          script: >-
-            export TOXENV=py27;
-            ./tools/ci/run_tc.py \
-              --browser=firefox \
-              --channel=experimental \
-              --xvfb \
-              resources_unittest \
-              tools/ci/ci_resources_unittest.sh
-          conditions:
-            pull-request
-        - name: infrastructure/ tests
-          description: >-
-            Smoketests for wptrunner
-          script: >-
-            sudo apt update -qqy;
-            sudo apt install -qqy libnss3-tools libappindicator1 fonts-liberation;
-            ./tools/ci/run_tc.py \
-              --oom-killer \
-              --browser=firefox \
-              --browser=chrome \
-              --channel=experimental \
-              --no-hosts \
-              --xvfb \
-              wptrunner_infrastructure \
-              tools/ci/ci_wptrunner_infrastructure.sh
-          conditions:
-            pull-request
-        each(operation):
-          # Note: jsone doesn't short-circuit evaluation so all parts of the conditional are evaluated
-          # Accessing properties using the [] notation allows them to evaluate as null in case they're undefined
-          # TODO: Allow running pushes on branches other than master
-          - $if: ("push" in operation.conditions && tasks_for == "github-push" && event['ref'] == "refs/heads/master") || ("pull-request" in operation.conditions && tasks_for == "github-pull-request" && event['action'] in ['opened', 'reopened', 'synchronize'])
-            then:
-              $let:
-                checkout_ref:
-                  $if: tasks_for == "github-push"
-                  then:
-                    ${event.ref}
-                  else:
-                    refs/pull/${event.number}/merge
-              in:
-                taskId: {$eval: 'as_slugid(operation.name)'}
-                taskGroupId: {$eval: 'as_slugid("task group")'}
-                created: {$fromNow: ''}
-                deadline: {$fromNow: '24 hours'}
-                provisionerId: aws-provisioner-v1
-                workerType:
-                  $if: event.repository.full_name == 'web-platform-tests/wpt'
-                  then:
-                    wpt-docker-worker
-                  else:
-                    github-worker
-                metadata:
-                  name: ${operation.name}
-                  description: ${operation.description}
-                  owner: ${event.sender.login}@users.noreply.github.com
-                  source: ${event.repository.url}
-                payload:
-                  image: harjgam/web-platform-tests:0.33
-                  maxRunTime: 7200
-                  artifacts:
-                    public/results:
-                      path: /home/test/artifacts
-                      type: directory
-                  command:
-                    - /bin/bash
-                    - --login
-                    - -c
-                    - set -ex;
-                      echo "${operation.name}";
-                      ~/start.sh
-                        ${event.repository.clone_url}
-                        ${checkout_ref};
-                      cd ~/web-platform-tests;
-                      ${operation.script};
-                extra:
-                  github_event: "${event_str}"
+            wpt-docker-worker
+          else:
+            github-worker
+        metadata:
+          name: "wpt Decision Task"
+          description: "The task that creates all of the other tasks in the task graph"
+          owner: ${event.pusher.email}
+          source: ${event.repository.url}
+        payload:
+          image: harjgam/web-platform-tests:0.33
+          maxRunTime: 7200
+          artifacts:
+            public/results:
+              path: /home/test/artifacts
+              type: directory
+          command:
+            - /bin/bash
+            - --login
+            - -c
+            - set -ex;
+              ~/start.sh
+                ${event.repository.url}
+                ${event.ref};
+              cd ~/web-platform-tests;
+              ./wpt decision
+        scopes:
+          - assume:repo:github.com/web-platform-tests/wpt:${scope_suffix}
+        extra:
+          github_event: "${event_str}"
diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 4efb5b29588b78..8fd2c15cd9030b 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -60,20 +60,21 @@ def filter_triggers(event, all_tasks):
     return triggered
 
 
-def get_run_jobs():
+def get_run_jobs(event):
     import jobs
-    paths = jobs.get_paths(revish=None)
+    paths = jobs.get_paths(revish="%s..%s" % (event["before"],
+                                              event["after"]))
     return jobs.get_jobs(paths)
 
 
-def filter_schedule_if(tasks):
+def filter_schedule_if(event, tasks):
     scheduled = {}
     run_jobs = None
     for name, task in iteritems(tasks):
         if "schedule-if" in task:
             if "run-job" in task["schedule-if"]:
                 if run_jobs is None:
-                    run_jobs = get_run_jobs()
+                    run_jobs = get_run_jobs(event)
                 if any(item in run_jobs for item in task["schedule-if"]):
                     scheduled[name] = task
         else:
@@ -94,7 +95,8 @@ def build_full_command(event, task):
         "task_name": task["name"],
         "repo_url": event["repository"]["url"],
         "fetch_rev": get_fetch_rev(event),
-        "task_cmd": task["command"]
+        "task_cmd": task["command"],
+        "install_str": "",
     }
 
     options = task.get("options", {})
@@ -109,18 +111,20 @@ def build_full_command(event, task):
         options_args.append("--no-hosts")
     if options.get("checkout"):
         options_args.append("--checkout=%s" % options["checkout"])
+    for browser in options.get("browser", []):
+        options_args.append("--browser=%s" % browser)
+    if options.get("checkout"):
+        options_args.append("--checkout=%s" % options["checkout"])
+
 
     cmd_args["options_str"] = "\n".join("  %s" % item for item in options_args)
 
-    install_str = "";
     install_packages = task.get("install")
     if install_packages:
         install_items = ["apt update -qqy"]
         install_items.extend("apt install -qqy %s" % item
                              for item in install_packages)
-        install_str = "\n".join("sudo %s;" % item for item in install_items)
-
-    cmd_args["install_cmd"] = install_str
+        cmd_args["install_str"] = "\n".join("sudo %s;" % item for item in install_items)
 
     return ["/bin/bash",
             "--login",
@@ -223,7 +227,6 @@ def get_parser():
 
 
 def run(venv, **kwargs):
-    auth = taskcluster.Auth({'rootUrl': TC_ROOT}, taskcluster.optionsFromEnvironment())
     queue = taskcluster.Queue({'rootUrl': TC_ROOT})
 
     event = get_event(**kwargs)
@@ -231,9 +234,9 @@ def run(venv, **kwargs):
     all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
 
     triggered_tasks = filter_triggers(event, all_tasks)
-    scheduled_tasks = filter_schedule_if(triggered_tasks)
+    scheduled_tasks = filter_schedule_if(event, triggered_tasks)
 
-    task_id_map = build_task_graph(event, all_tasks, triggered_tasks)
+    task_id_map = build_task_graph(event, all_tasks, scheduled_tasks)
 
     if not kwargs["dry_run"]:
         create_tasks(queue, task_id_map)
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 198b2e51ab20f0..e1f484791e4292 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -73,6 +73,17 @@ components:
 
   browser-chrome: {}
 
+  tox-python2:
+    env:
+      - TOXENV=py27
+      - PY_COLORS=0
+
+  tox-python3:
+    env:
+      - TOXENV=py36
+      - PY_COLORS=0
+    install:
+      - python3-pip
 
 tasks:
   # Run full suites on push
@@ -221,19 +232,89 @@ tasks:
   - tools/unittests (Python 2):
       description: >-
         Unit tests for tools running under Python 2.7, excluding wptrunner
+      use:
+        - wpt-base
+        - trigger-pr
+        - tox-python2
       command: tools/ci/ci_tools_unittest.sh
       env:
-        - TOXENV=py27;
-        - HYPOTHESIS_PROFILE=ci;
-        - PY_COLORS=0;
+        - HYPOTHESIS_PROFILE=ci
+      schedule-if:
+        run-job:
+          - tools_unittest
 
   - tools/unittests (Python 3):
       description: >-
         Unit tests for tools running under Python 3, excluding wptrunner
+      use:
+        - wpt-base
+        - trigger-pr
+        - tox-python3
       command: tools/ci/ci_tools_unittest.sh
       env:
-        - TOXENV=py36;
         - HYPOTHESIS_PROFILE=ci;
-        - PY_COLORS=0;
+      schedule-if:
+        run-job:
+          - tools_unittest
+
+  - tools/wpt tests:
+      description: >-
+        Integration tests for wpt commands
+      use:
+        - wpt-base
+        - trigger-pr
+        - tox-python2
+      command: tools/ci/ci_wpt.sh
       install:
-        - python3-pip
+        - libnss3-tools
+      options:
+        oom-killer: true
+        browser:
+          - firefox
+          - chrome
+        channel: experimental
+        xvfb: true
+      schedule-if:
+        run-job:
+          - wpt_integration
+
+  - resources/ tests:
+      description: >-
+        Tests for testharness.js and other files in resources/
+      use:
+        - wpt-base
+        - trigger-pr
+        - tox-python2
+      command: tools/ci/ci_resources_unittest.sh
+      options:
+        browser:
+          - firefox
+        channel: experimental
+        xvfb: true
+      schedule-if:
+        run-job:
+          - resources_unittest
+
+  - infrastructure/ tests:
+      description: >-
+        Smoketests for wptrunner
+      use:
+        - wpt-base
+        - trigger-pr
+        - tox-python2
+      command: tools/ci/ci_wpt.sh
+      install:
+        - libnss3-tools
+        - libappindicator1
+        - fonts-liberation
+      options:
+        oom-killer: true
+        browser:
+          - firefox
+          - chrome
+        channel: experimental
+        xvfb: true
+        hosts: false
+      schedule-if:
+        run-job:
+          - wptrunner_infrastructure

From 36b64d0db464f91c8d7fc56294b0b18e14c99439 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 19:26:43 +0100
Subject: [PATCH 07/51] Add install-certificates, remove run-job logic in
 tc_run

---
 tools/ci/decision.py    |  3 +-
 tools/ci/run_tc.py      | 70 +++++++-------------------------------
 tools/ci/tasks/test.yml | 75 ++++++++++++++++++++++++-----------------
 3 files changed, 60 insertions(+), 88 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 8fd2c15cd9030b..a4f5837f5a8efd 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -115,7 +115,8 @@ def build_full_command(event, task):
         options_args.append("--browser=%s" % browser)
     if options.get("checkout"):
         options_args.append("--checkout=%s" % options["checkout"])
-
+    if options.get("install-certificates"):
+        options_args.append("--install-certificates")
 
     cmd_args["options_str"] = "\n".join("  %s" % item for item in options_args)
 
diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py
index ea4a1ac1a6ac4a..2f3b2a209482bb 100755
--- a/tools/ci/run_tc.py
+++ b/tools/ci/run_tc.py
@@ -99,8 +99,10 @@ def get_parser():
                    help="Start xvfb")
     p.add_argument("--checkout",
                    help="Revision to checkout before starting job")
-    p.add_argument("job",
-                   help="Name of the job associated with the current event")
+    p.add_argument("--install-certificates", action="store_true", default=None,
+                   help="Install web-platform.test certificates to UA store")
+    p.add_argument("--no-install-certificates", action="store_false", default=None,
+                   help="Install web-platform.test certificates to UA store")
     p.add_argument("script",
                    help="Script to run for the job")
     p.add_argument("script_args",
@@ -123,6 +125,12 @@ def checkout_revision(rev):
     subprocess.check_call(["git", "checkout", "--quiet", rev])
 
 
+def install_certificates():
+    subprocess.check_call(["sudo", "cp", "tools/certs/cacert.pem",
+                           "/usr/local/share/ca-certificates/cacert.crt"])
+    subprocess.check_call(["update-ca-certificates"])
+
+
 def install_chrome(channel):
     if channel in ("experimental", "dev", "nightly"):
         deb_archive = "google-chrome-unstable_current_amd64.deb"
@@ -150,29 +158,6 @@ def start_xvfb():
     start(["sudo", "fluxbox", "-display", os.environ["DISPLAY"]])
 
 
-def get_extra_jobs(event):
-    body = None
-    jobs = set()
-    if "commits" in event and event["commits"]:
-        body = event["commits"][0]["message"]
-    elif "pull_request" in event:
-        body = event["pull_request"]["body"]
-
-    if not body:
-        return jobs
-
-    regexp = re.compile(r"\s*tc-jobs:(.*)$")
-
-    for line in body.splitlines():
-        m = regexp.match(line)
-        if m:
-            items = m.group(1)
-            for item in items.split(","):
-                jobs.add(item.strip())
-            break
-    return jobs
-
-
 def set_variables(event):
     # Set some variables that we use to get the commits on the current branch
     ref_prefix = "refs/heads/"
@@ -193,23 +178,13 @@ def set_variables(event):
         os.environ["GITHUB_BRANCH"] = branch
 
 
-def include_job(job):
-    # Special case things that unconditionally run on pushes,
-    # assuming a higher layer is filtering the required list of branches
-    if (os.environ["GITHUB_PULL_REQUEST"] == "false" and
-        job == "run-all"):
-        return True
-
-    jobs_str = run([os.path.join(root, "wpt"),
-                    "test-jobs"], return_stdout=True)
-    print(jobs_str)
-    return job in set(jobs_str.splitlines())
-
-
 def setup_environment(args):
     if args.hosts_file:
         make_hosts_file()
 
+    if args["install_certificates"]:
+        install_certificates()
+
     if "chrome" in args.browser:
         assert args.channel is not None
         install_chrome(args.channel)
@@ -277,25 +252,6 @@ def main():
 
     setup_repository()
 
-    extra_jobs = get_extra_jobs(event)
-
-    job = args.job
-
-    print("Job %s" % job)
-
-    run_if = [(lambda: job == "all", "job set to 'all'"),
-              (lambda:"all" in extra_jobs, "Manually specified jobs includes 'all'"),
-              (lambda:job in extra_jobs, "Manually specified jobs includes '%s'" % job),
-              (lambda:include_job(job), "CI required jobs includes '%s'" % job)]
-
-    for fn, msg in run_if:
-        if fn():
-            print(msg)
-            break
-    else:
-        print("Job not scheduled for this push")
-        return
-
     # Run the job
     setup_environment(args)
     os.chdir(root)
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index e1f484791e4292..b57103c4d0530a 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -24,12 +24,16 @@ components:
     vars:
       test-type: wdspec
 
-  wpt-run:
-    name: wpt-${vars.browser}-${vars.channel}-${vars.suite}-chunk-${chunks.id}
+  run-options:
     options:
       xvfb: true
       oom-killer: true
       hosts: true
+      install-certificates: true
+
+  wpt-run:
+    name: wpt-${vars.browser}-${vars.channel}-${vars.suite}-chunk-${chunks.id}
+    options:
       browser: ${vars.browser}
       channel: ${vars.channel}
     command: >-
@@ -138,6 +142,7 @@ tasks:
             ${vars.browser}-${vars.channel}-${vars.suite}:
               use:
                 - wpt-base
+                - run-options
                 - wpt-run
                 - browser-${vars.browser}
                 - wpt-${vars.suite}
@@ -158,6 +163,7 @@ tasks:
         wpt-${vars.browser}-${vars.channel}-stability:
           use:
             - wpt-base
+            - run-options
             - browser-${vars.browser}
           description: >-
             Verify that all tests affected by a pull request are stable
@@ -169,35 +175,44 @@ tasks:
             --
             --channel=${vars.channel}
             --verify
-          wpt-${vars.browser}-${vars.channel}-results:
-            description: >-
-              Collect results for all tests affected by a pull request in
-              ${vars.browser}.
-            command: >-
-              ./tools/ci/taskcluster-run.py
-              --commit-range base_head
-              ${vars.browser}
-              --
-              --channel=${vars.channel}
-              --no-fail-on-unexpected
-              --log-wptreport=../artifacts/wpt_report.json
-              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
 
-          wpt-${vars.browser}-${vars.channel}-results-without-changes:
-            options:
-              checkout: base_head
-            description: >-
-              Collect results for all tests affected by a pull request in
-              ${vars.browser} but without the changes in the PR.
-            command: >-
-              ./tools/ci/taskcluster-run.py
-              --commit-range task_head
-              ${vars.browser}
-              --
-              --channel=${vars.channel}
-              --no-fail-on-unexpected
-              --log-wptreport=../artifacts/wpt_report.json
-              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+        wpt-${vars.browser}-${vars.channel}-results:
+          use:
+            - wpt-base
+            - run-options
+            - browser-${vars.browser}
+          description: >-
+            Collect results for all tests affected by a pull request in
+            ${vars.browser}.
+          command: >-
+            ./tools/ci/taskcluster-run.py
+            --commit-range base_head
+            ${vars.browser}
+            --
+            --channel=${vars.channel}
+            --no-fail-on-unexpected
+            --log-wptreport=../artifacts/wpt_report.json
+            --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+
+        wpt-${vars.browser}-${vars.channel}-results-without-changes:
+          use:
+            - wpt-base
+            - run-options
+            - browser-${vars.browser}
+          options:
+            checkout: base_head
+          description: >-
+            Collect results for all tests affected by a pull request in
+            ${vars.browser} but without the changes in the PR.
+          command: >-
+            ./tools/ci/taskcluster-run.py
+            --commit-range task_head
+            ${vars.browser}
+            --
+            --channel=${vars.channel}
+            --no-fail-on-unexpected
+            --log-wptreport=../artifacts/wpt_report.json
+            --log-wptscreenshot=../artifacts/wpt_screenshot.txt
   - $map:
       for:
         - vars:

From f9d6a0fc13c20c39d405f85c5136cecca8035266 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 21:34:09 +0100
Subject: [PATCH 08/51] Don't set taskgroupId

---
 .taskcluster.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index cfb548c542f387..05014338062ee8 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -16,8 +16,6 @@ tasks:
   in:
     - $if: '(tasks_for == "github-push" && event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly"]) || tasks_for == "github-pull-request"'
       then:
-        taskId: "${ownTaskId}"
-        taskGroupId: "${ownTaskId}"
         created: {$fromNow: ''}
         deadline: {$fromNow: '24 hours'}
         provisionerId: aws-provisioner-v1

From eaad4681df3553f28c3680ae52540d3c2bb1f02e Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Tue, 24 Sep 2019 21:55:55 +0100
Subject: [PATCH 09/51] Fix some TC errors

---
 .taskcluster.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 05014338062ee8..60398b814d2546 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -11,10 +11,21 @@ tasks:
           branch:
             $if: "event.ref[:11] == 'refs/heads/'"
             then: "${event.ref[11:]}"
+            else: "${event.ref}"
         in: "branch:${branch}"
       else: "pull-request"
+    run_task:
+      $if: 'tasks_for == "github-push"'
+      then:
+        $if: 'event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly"]'
+        then: true
+        else: false
+      else:
+        $if: 'tasks_for == "github-pull-request"'
+        then: true
+        else: false
   in:
-    - $if: '(tasks_for == "github-push" && event.ref in ["refs/heads/master", "refs/heads/epochs/daily", "refs/heads/epochs/weekly", "refs/heads/triggers/chrome_stable", "refs/heads/triggers/chrome_beta", "refs/heads/triggers/chrome_dev", "refs/heads/triggers/firefox_stable", "refs/heads/triggers/firefox_beta", "refs/heads/triggers/firefox_nightly"]) || tasks_for == "github-pull-request"'
+    - $if: run_task
       then:
         created: {$fromNow: ''}
         deadline: {$fromNow: '24 hours'}
@@ -44,7 +55,7 @@ tasks:
             - set -ex;
               ~/start.sh
                 ${event.repository.url}
-                ${event.ref};
+                ${event.after};
               cd ~/web-platform-tests;
               ./wpt decision
         scopes:

From 762c0a2ee2ebf8e804b4f49a4ba36656f86bebe8 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 15:29:17 +0100
Subject: [PATCH 10/51] Try scopes based on repo name

---
 .taskcluster.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 60398b814d2546..2321f287d783b6 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -4,7 +4,7 @@ policy:
 tasks:
   $let:
     event_str: {$json: {$eval: event}}
-    scope_suffix:
+    scopes:
       $if: 'tasks_for == "github-push"'
       then:
         $let:
@@ -12,8 +12,8 @@ tasks:
             $if: "event.ref[:11] == 'refs/heads/'"
             then: "${event.ref[11:]}"
             else: "${event.ref}"
-        in: "branch:${branch}"
-      else: "pull-request"
+        in: "assume:repo:github.com/${event.repository.full_name}:branch:${branch}"
+      else: "assume:repo:github.com/${event.repository.full_name}:pull-request"
     run_task:
       $if: 'tasks_for == "github-push"'
       then:
@@ -59,6 +59,6 @@ tasks:
               cd ~/web-platform-tests;
               ./wpt decision
         scopes:
-          - assume:repo:github.com/web-platform-tests/wpt:${scope_suffix}
+          - scopes
         extra:
           github_event: "${event_str}"

From 9b343bb151b02ab1d7dcbcd7c198c415c086168a Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 15:49:37 +0100
Subject: [PATCH 11/51] Actually use scopes variable

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 2321f287d783b6..42499c43abc850 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -59,6 +59,6 @@ tasks:
               cd ~/web-platform-tests;
               ./wpt decision
         scopes:
-          - scopes
+          - ${scopes}
         extra:
           github_event: "${event_str}"

From daeb27afb233bb57e7b30a3e3d54a2ceff533c7d Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 15:54:30 +0100
Subject: [PATCH 12/51] Support the tc-jobs: syntax to specify different jobs

---
 tools/ci/decision.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index a4f5837f5a8efd..5bd07bceb5d237 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -1,6 +1,7 @@
 import argparse
 import json
 import os
+import re
 from collections import OrderedDict
 
 import taskcluster
@@ -64,7 +65,32 @@ def get_run_jobs(event):
     import jobs
     paths = jobs.get_paths(revish="%s..%s" % (event["before"],
                                               event["after"]))
-    return jobs.get_jobs(paths)
+    path_jobs = jobs.get_jobs(paths)
+    all_jobs = path_jobs | get_extra_jobs(event)
+    return all_jobs
+
+
+def get_extra_jobs(event):
+    body = None
+    jobs = set()
+    if "commits" in event and event["commits"]:
+        body = event["commits"][0]["message"]
+    elif "pull_request" in event:
+        body = event["pull_request"]["body"]
+
+    if not body:
+        return jobs
+
+    regexp = re.compile(r"\s*tc-jobs:(.*)$")
+
+    for line in body.splitlines():
+        m = regexp.match(line)
+        if m:
+            items = m.group(1)
+            for item in items.split(","):
+                jobs.add(item.strip())
+            break
+    return jobs
 
 
 def filter_schedule_if(event, tasks):
@@ -75,7 +101,7 @@ def filter_schedule_if(event, tasks):
             if "run-job" in task["schedule-if"]:
                 if run_jobs is None:
                     run_jobs = get_run_jobs(event)
-                if any(item in run_jobs for item in task["schedule-if"]):
+                if "all" in run_jobs or any(item in run_jobs for item in task["schedule-if"]):
                     scheduled[name] = task
         else:
             scheduled[name] = task

From a2291fdbc945758e7d3adf432deaa5d53c6a961c Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 16:17:56 +0100
Subject: [PATCH 13/51] Fixup logging

---
 .taskcluster.yml     |  2 +-
 tools/ci/decision.py | 33 ++++++++++++++++++++++++++++-----
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 42499c43abc850..3a6c94efb15426 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -57,7 +57,7 @@ tasks:
                 ${event.repository.url}
                 ${event.after};
               cd ~/web-platform-tests;
-              ./wpt decision
+              ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
         scopes:
           - ${scopes}
         extra:
diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 5bd07bceb5d237..cb24b3be6fe97c 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -2,6 +2,7 @@
 import json
 import os
 import re
+import logging
 from collections import OrderedDict
 
 import taskcluster
@@ -17,6 +18,10 @@
 here = os.path.abspath(os.path.dirname(__file__))
 
 
+logging.basicConfig()
+logger = logging.getLogger()
+
+
 def get_triggers(event):
     # Set some variables that we use to get the commits on the current branch
     ref_prefix = "refs/heads/"
@@ -34,7 +39,7 @@ def fetch_event_data():
     try:
         task_id = os.environ["TASK_ID"]
     except KeyError:
-        print("WARNING: Missing TASK_ID environment variable")
+        logger.warning("Missing TASK_ID environment variable")
         # For example under local testing
         return None
 
@@ -57,6 +62,7 @@ def filter_triggers(event, all_tasks):
                 for trigger_branch in task["trigger"]["branch"]:
                     if (trigger_branch == branch or
                         trigger_branch.endswith("*") and branch.startswith(trigger_branch[:-1])):
+                        logger.info("Triggers include task %s" % name)
                         triggered[name] = task
     return triggered
 
@@ -67,6 +73,7 @@ def get_run_jobs(event):
                                               event["after"]))
     path_jobs = jobs.get_jobs(paths)
     all_jobs = path_jobs | get_extra_jobs(event)
+    logger.info("Including jobs %s" % ", ".join(all_jobs))
     return all_jobs
 
 
@@ -103,8 +110,10 @@ def filter_schedule_if(event, tasks):
                     run_jobs = get_run_jobs(event)
                 if "all" in run_jobs or any(item in run_jobs for item in task["schedule-if"]):
                     scheduled[name] = task
+                    logger.info("Scheduled tasks include %s" % name)
         else:
             scheduled[name] = task
+            logger.info("Scheduled tasks include %s" % name)
     return scheduled
 
 
@@ -231,16 +240,24 @@ def create_tasks(queue, task_id_map):
 
 
 def get_event(**kwargs):
-    if "event_path" in kwargs:
-        with open(kwargs["event_path"]) as f:
-            event_str = f.read()
+    if kwargs["event_path"] is not None:
+        try:
+            with open(kwargs["event_path"]) as f:
+                event_str = f.read()
+        except IOError:
+            logger.error("Missing event file at path %s" % kwargs["event_path"])
+            rause
     elif "TASK_EVENT" in os.environ:
         event_str = os.environ["TASK_EVENT"]
     else:
         event_str = fetch_event_data()
     if not event_str:
         raise ValueError("Can't find GitHub event definition; for local testing pass --event-path")
-    return json.loads(event_str)
+    try:
+        return json.loads(event_str)
+    except ValueError:
+        logger.error("Event was not valid JSON")
+        raise
 
 
 def get_parser():
@@ -250,6 +267,8 @@ def get_parser():
     parser.add_argument("--dry-run", action="store_true",
                         help="Don't actually create the tasks, just output the tasks that "
                         "would be created")
+    parser.add_argument("--tasks-path",
+                        help="Path to file in which to write payload for all scheduled tasks")
     return parser
 
 
@@ -269,3 +288,7 @@ def run(venv, **kwargs):
         create_tasks(queue, task_id_map)
     else:
         print(json.dumps(task_id_map, indent=2))
+
+    if kwargs["tasks_path"]:
+        with open(kwargs["tasks_path"], "w") as f:
+            json.dump(task_id_map, f, indent=2)

From 3b2a0263b3396e83d9f06737488596540e9f4782 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 16:20:23 +0100
Subject: [PATCH 14/51] Use urlopen correctly

---
 tools/ci/decision.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index cb24b3be6fe97c..1f83d09fac7b32 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -7,7 +7,7 @@
 
 import taskcluster
 from six import iteritems, itervalues
-from six.moves.urllib import request
+from six.moves.urllib.request import urlopen
 
 from . import taskgraph
 
@@ -43,7 +43,7 @@ def fetch_event_data():
         # For example under local testing
         return None
 
-    resp = request("%s/%s" % (QUEUE_BASE, task_id))
+    resp = urlopen("%s/%s" % (QUEUE_BASE, task_id))
 
     task_data = json.load(resp)
     event_data = task_data.get("extra", {}).get("github_event")

From 88490e12260bb03f65f2689b40c56edd14bd1ca8 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 16:56:45 +0100
Subject: [PATCH 15/51] Fix handling of do blocks

---
 tools/ci/taskgraph.py   |   6 +-
 tools/ci/tasks/test.yml | 124 ++++++++++++++++++++--------------------
 2 files changed, 66 insertions(+), 64 deletions(-)

diff --git a/tools/ci/taskgraph.py b/tools/ci/taskgraph.py
index 012962dfab8733..1a62ea8eefe94d 100644
--- a/tools/ci/taskgraph.py
+++ b/tools/ci/taskgraph.py
@@ -109,8 +109,10 @@ def expand_maps(task):
     assert set(map_data.keys()) == set(["for", "do"])
     rv = []
     for for_data in map_data["for"]:
-        do_values = expand_maps(map_data["do"])
-        for do_data in do_values:
+        do_items = map_data["do"]
+        if not isinstance(do_items, list):
+            do_items = expand_maps(do_items)
+        for do_data in do_items:
             task_data = deepcopy(for_data)
             assert len(do_data.keys()) == 1
             name = do_data.keys()[0]
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index b57103c4d0530a..e939fc33d588b7 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -139,17 +139,17 @@ tasks:
                 - trigger-daily
                 - trigger-push
           do:
-            ${vars.browser}-${vars.channel}-${vars.suite}:
-              use:
-                - wpt-base
-                - run-options
-                - wpt-run
-                - browser-${vars.browser}
-                - wpt-${vars.suite}
-              description: >-
-                A subset of WPT's "${vars.suite}" tests (chunk number ${chunks.id}
-                of ${chunks.total}), run in the ${vars.channel} release of
-                ${vars.browser}.
+            - ${vars.browser}-${vars.channel}-${vars.suite}:
+                use:
+                  - wpt-base
+                  - run-options
+                  - wpt-run
+                  - browser-${vars.browser}
+                  - wpt-${vars.suite}
+                description: >-
+                  A subset of WPT's "${vars.suite}" tests (chunk number ${chunks.id}
+                  of ${chunks.total}), run in the ${vars.channel} release of
+                  ${vars.browser}.
 
   - $map:
       for:
@@ -160,59 +160,59 @@ tasks:
             browser: chrome
             channel: dev
       do:
-        wpt-${vars.browser}-${vars.channel}-stability:
-          use:
-            - wpt-base
-            - run-options
-            - browser-${vars.browser}
-          description: >-
-            Verify that all tests affected by a pull request are stable
-            when executed in ${vars.browser}.
-          command: >-
-            ./tools/ci/taskcluster-run.py
-            --commit-range base_head
-            ${vars.browser}
-            --
-            --channel=${vars.channel}
-            --verify
+        - wpt-${vars.browser}-${vars.channel}-stability:
+            use:
+              - wpt-base
+              - run-options
+              - browser-${vars.browser}
+            description: >-
+              Verify that all tests affected by a pull request are stable
+              when executed in ${vars.browser}.
+            command: >-
+              ./tools/ci/taskcluster-run.py
+              --commit-range base_head
+              ${vars.browser}
+              --
+              --channel=${vars.channel}
+              --verify
 
-        wpt-${vars.browser}-${vars.channel}-results:
-          use:
-            - wpt-base
-            - run-options
-            - browser-${vars.browser}
-          description: >-
-            Collect results for all tests affected by a pull request in
-            ${vars.browser}.
-          command: >-
-            ./tools/ci/taskcluster-run.py
-            --commit-range base_head
-            ${vars.browser}
-            --
-            --channel=${vars.channel}
-            --no-fail-on-unexpected
-            --log-wptreport=../artifacts/wpt_report.json
-            --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+        - wpt-${vars.browser}-${vars.channel}-results:
+            use:
+              - wpt-base
+              - run-options
+              - browser-${vars.browser}
+            description: >-
+              Collect results for all tests affected by a pull request in
+              ${vars.browser}.
+            command: >-
+              ./tools/ci/taskcluster-run.py
+              --commit-range base_head
+              ${vars.browser}
+              --
+              --channel=${vars.channel}
+              --no-fail-on-unexpected
+              --log-wptreport=../artifacts/wpt_report.json
+              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
 
-        wpt-${vars.browser}-${vars.channel}-results-without-changes:
-          use:
-            - wpt-base
-            - run-options
-            - browser-${vars.browser}
-          options:
-            checkout: base_head
-          description: >-
-            Collect results for all tests affected by a pull request in
-            ${vars.browser} but without the changes in the PR.
-          command: >-
-            ./tools/ci/taskcluster-run.py
-            --commit-range task_head
-            ${vars.browser}
-            --
-            --channel=${vars.channel}
-            --no-fail-on-unexpected
-            --log-wptreport=../artifacts/wpt_report.json
-            --log-wptscreenshot=../artifacts/wpt_screenshot.txt
+        - wpt-${vars.browser}-${vars.channel}-results-without-changes:
+            use:
+              - wpt-base
+              - run-options
+              - browser-${vars.browser}
+            options:
+              checkout: base_head
+            description: >-
+              Collect results for all tests affected by a pull request in
+              ${vars.browser} but without the changes in the PR.
+            command: >-
+              ./tools/ci/taskcluster-run.py
+              --commit-range task_head
+              ${vars.browser}
+              --
+              --channel=${vars.channel}
+              --no-fail-on-unexpected
+              --log-wptreport=../artifacts/wpt_report.json
+              --log-wptscreenshot=../artifacts/wpt_screenshot.txt
   - $map:
       for:
         - vars:

From 0178f822057a3fa251cfe65e6f0a7964fb48b2aa Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 17:35:41 +0100
Subject: [PATCH 16/51] Fix some run options

---
 tools/ci/decision.py    | 4 ++--
 tools/ci/taskgraph.py   | 4 ----
 tools/ci/tasks/test.yml | 4 ++--
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 1f83d09fac7b32..5aa48dff47113e 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -142,8 +142,8 @@ def build_full_command(event, task):
         options_args.append("--xvfb")
     if not options.get("hosts"):
         options_args.append("--no-hosts")
-    if not options.get("hosts"):
-        options_args.append("--no-hosts")
+    else:
+        options_args.append("--hosts")
     if options.get("checkout"):
         options_args.append("--checkout=%s" % options["checkout"])
     for browser in options.get("browser", []):
diff --git a/tools/ci/taskgraph.py b/tools/ci/taskgraph.py
index 1a62ea8eefe94d..106e6b0ca37f9e 100644
--- a/tools/ci/taskgraph.py
+++ b/tools/ci/taskgraph.py
@@ -22,14 +22,10 @@ def update_recursive(data, update_data):
             initial_value = data[key]
             if isinstance(value, dict):
                 if not isinstance(initial_value, dict):
-                    import pdb
-                    pdb.set_trace()
                     raise ValueError
                 update_recursive(initial_value, value)
             elif isinstance(value, list):
                 if not isinstance(initial_value, list):
-                    import pdb
-                    pdb.set_trace()
                     raise ValueError
                 initial_value.extend(value)
             else:
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index e939fc33d588b7..6b0d6fb42dc716 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -34,7 +34,8 @@ components:
   wpt-run:
     name: wpt-${vars.browser}-${vars.channel}-${vars.suite}-chunk-${chunks.id}
     options:
-      browser: ${vars.browser}
+      browser:
+        - ${vars.browser}
       channel: ${vars.channel}
     command: >-
       ./tools/ci/taskcluster-run.py
@@ -163,7 +164,6 @@ tasks:
         - wpt-${vars.browser}-${vars.channel}-stability:
             use:
               - wpt-base
-              - run-options
               - browser-${vars.browser}
             description: >-
               Verify that all tests affected by a pull request are stable

From bddc80d6b2fe7e8f5895e024dbf4ddd0a6a21b57 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 18:16:26 +0100
Subject: [PATCH 17/51] Update worker selector

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 3a6c94efb15426..d0d4ca491b6db8 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -31,7 +31,7 @@ tasks:
         deadline: {$fromNow: '24 hours'}
         provisionerId: aws-provisioner-v1
         workerType:
-          $if: event.repository.full_name == 'web-platform-tests/wpt'
+          $if: event.repository.full_name == 'web-platform-tests/wpt' || event.repository.full_name == 'jgraham/web-platform-tests'
           then:
             wpt-docker-worker
           else:

From e879fe7d881832b6ed7f94b92812d2510998ec3f Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 18:23:25 +0100
Subject: [PATCH 18/51] Use correct worker type

---
 .taskcluster.yml     | 2 +-
 tools/ci/decision.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index d0d4ca491b6db8..d7a0f507f75307 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -31,7 +31,7 @@ tasks:
         deadline: {$fromNow: '24 hours'}
         provisionerId: aws-provisioner-v1
         workerType:
-          $if: event.repository.full_name == 'web-platform-tests/wpt' || event.repository.full_name == 'jgraham/web-platform-tests'
+          $if: event.repository.full_name in ['web-platform-tests/wpt', 'jgraham/web-platform-tests']
           then:
             wpt-docker-worker
           else:
diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 5aa48dff47113e..77f0d91b620ec4 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -180,7 +180,7 @@ def build_full_command(event, task):
 def create_tc_task(event, task, taskgroup_id, required_task_ids):
     command = build_full_command(event, task)
     worker_type = ("wpt-docker-worker"
-                   if event["repository"]["full_name"] == 'web-platform-tests/wpt'
+                   if event["repository"]["full_name"] in ['web-platform-tests/wpt', "jgraham/web-platform-tests"]
                    else "github-worker")
     task_id = taskcluster.slugId()
     task_data = {

From effa18688bb9ee2f9c43aa27023e967b7624a6c1 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 18:29:15 +0100
Subject: [PATCH 19/51] Change scopes

---
 .taskcluster.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index d7a0f507f75307..c82579bd2a63a4 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -12,8 +12,8 @@ tasks:
             $if: "event.ref[:11] == 'refs/heads/'"
             then: "${event.ref[11:]}"
             else: "${event.ref}"
-        in: "assume:repo:github.com/${event.repository.full_name}:branch:${branch}"
-      else: "assume:repo:github.com/${event.repository.full_name}:pull-request"
+        in: "repo:github.com/${event.repository.full_name}:branch:${branch}"
+      else: "repo:github.com/${event.repository.full_name}:pull-request"
     run_task:
       $if: 'tasks_for == "github-push"'
       then:

From d6120a0db335617887289bdff423da0215d3fecb Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Wed, 25 Sep 2019 18:32:24 +0100
Subject: [PATCH 20/51] Revert "Change scopes"

This reverts commit effa18688bb9ee2f9c43aa27023e967b7624a6c1.
---
 .taskcluster.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index c82579bd2a63a4..d7a0f507f75307 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -12,8 +12,8 @@ tasks:
             $if: "event.ref[:11] == 'refs/heads/'"
             then: "${event.ref[11:]}"
             else: "${event.ref}"
-        in: "repo:github.com/${event.repository.full_name}:branch:${branch}"
-      else: "repo:github.com/${event.repository.full_name}:pull-request"
+        in: "assume:repo:github.com/${event.repository.full_name}:branch:${branch}"
+      else: "assume:repo:github.com/${event.repository.full_name}:pull-request"
     run_task:
       $if: 'tasks_for == "github-push"'
       then:

From 41bdcee20dd86c38b3e5d30512f52eea383d37f7 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 10:53:02 +0100
Subject: [PATCH 21/51] Fix typo

---
 tools/ci/decision.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 77f0d91b620ec4..d9cf8ff452a5ed 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -189,7 +189,6 @@ def create_tc_task(event, task, taskgroup_id, required_task_ids):
         "deadline": taskcluster.fromNowJSON(task["deadline"]),
         "provisionerId": task["provisionerId"],
         "workerType": worker_type,
-        "priority": "lowest",
         "metadata": {
             "name": task["name"],
             "description": task.get("description", ""),
@@ -246,7 +245,7 @@ def get_event(**kwargs):
                 event_str = f.read()
         except IOError:
             logger.error("Missing event file at path %s" % kwargs["event_path"])
-            rause
+            raise
     elif "TASK_EVENT" in os.environ:
         event_str = os.environ["TASK_EVENT"]
     else:

From ef6b09a080cb7d4763b4aebd8ba025d3f07afebf Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 14:15:36 +0100
Subject: [PATCH 22/51] Set schedulerId

---
 tools/ci/decision.py    | 1 +
 tools/ci/tasks/test.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index d9cf8ff452a5ed..f6fa710574b098 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -188,6 +188,7 @@ def create_tc_task(event, task, taskgroup_id, required_task_ids):
         "created": taskcluster.fromNowJSON(""),
         "deadline": taskcluster.fromNowJSON(task["deadline"]),
         "provisionerId": task["provisionerId"],
+        "schedulerId": task["schedulerId"],
         "workerType": worker_type,
         "metadata": {
             "name": task["name"],
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 6b0d6fb42dc716..28c1cc230c5ea1 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -1,6 +1,7 @@
 components:
   wpt-base:
     provisionerId: aws-provisioner-v1
+    schedulerId: taskcluster-github
     deadline: "24 hours"
     image: harjgam/web-platform-tests:0.33
     maxRunTime: 7200,

From ba1b477ba53fde3b0a670390eb88f517013d8163 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 14:47:31 +0100
Subject: [PATCH 23/51] Use taskcluster client to get event data

---
 tools/ci/decision.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index f6fa710574b098..a589c6e57ef284 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -7,13 +7,9 @@
 
 import taskcluster
 from six import iteritems, itervalues
-from six.moves.urllib.request import urlopen
 
 from . import taskgraph
 
-TC_ROOT = "https://taskcluster.net"
-QUEUE_BASE = "https://queue.taskcluster.net/v1/task"
-
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -35,7 +31,7 @@ def get_triggers(event):
     return pull_request, branch
 
 
-def fetch_event_data():
+def fetch_event_data(queue):
     try:
         task_id = os.environ["TASK_ID"]
     except KeyError:
@@ -43,9 +39,8 @@ def fetch_event_data():
         # For example under local testing
         return None
 
-    resp = urlopen("%s/%s" % (QUEUE_BASE, task_id))
+    task_data = queue.task(task_id)
 
-    task_data = json.load(resp)
     event_data = task_data.get("extra", {}).get("github_event")
     if event_data is not None:
         return event_data
@@ -239,7 +234,7 @@ def create_tasks(queue, task_id_map):
         queue.createTask(task_id, task_data)
 
 
-def get_event(**kwargs):
+def get_event(queue, **kwargs):
     if kwargs["event_path"] is not None:
         try:
             with open(kwargs["event_path"]) as f:
@@ -250,7 +245,7 @@ def get_event(**kwargs):
     elif "TASK_EVENT" in os.environ:
         event_str = os.environ["TASK_EVENT"]
     else:
-        event_str = fetch_event_data()
+        event_str = fetch_event_data(queue)
     if not event_str:
         raise ValueError("Can't find GitHub event definition; for local testing pass --event-path")
     try:
@@ -273,9 +268,9 @@ def get_parser():
 
 
 def run(venv, **kwargs):
-    queue = taskcluster.Queue({'rootUrl': TC_ROOT})
+    queue = taskcluster.Queue({'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']})
 
-    event = get_event(**kwargs)
+    event = get_event(queue, **kwargs)
 
     all_tasks = taskgraph.load_tasks_from_path(os.path.join(here, "tasks/test.yml"))
 

From df0cd50b9a45ada6c95d76056cfcfc5d2dc90764 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:01:21 +0100
Subject: [PATCH 24/51] Don't start subshell

---
 .taskcluster.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index d7a0f507f75307..9f816f61105b6d 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -49,15 +49,11 @@ tasks:
               path: /home/test/artifacts
               type: directory
           command:
-            - /bin/bash
-            - --login
-            - -c
-            - set -ex;
-              ~/start.sh
+            - ~/start.sh
                 ${event.repository.url}
                 ${event.after};
-              cd ~/web-platform-tests;
-              ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
+            - cd ~/web-platform-tests;
+            - ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
         scopes:
           - ${scopes}
         extra:

From 4c57fe82ebda4fa96c4a89702cbfb805ee55aa70 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:08:19 +0100
Subject: [PATCH 25/51] Add taskclusterProxy feature

---
 .taskcluster.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 9f816f61105b6d..a0c4023000b8b7 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -54,6 +54,8 @@ tasks:
                 ${event.after};
             - cd ~/web-platform-tests;
             - ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
+          features :
+            taskclusterProxy: true
         scopes:
           - ${scopes}
         extra:

From 3f488e62af9afbd931544c8b1d2937283c1acca7 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:09:59 +0100
Subject: [PATCH 26/51] Revert "Don't start subshell"

This reverts commit df0cd50b9a45ada6c95d76056cfcfc5d2dc90764.
---
 .taskcluster.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index a0c4023000b8b7..722ac9cd419dfb 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -49,11 +49,15 @@ tasks:
               path: /home/test/artifacts
               type: directory
           command:
-            - ~/start.sh
+            - /bin/bash
+            - --login
+            - -c
+            - set -ex;
+              ~/start.sh
                 ${event.repository.url}
                 ${event.after};
-            - cd ~/web-platform-tests;
-            - ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
+              cd ~/web-platform-tests;
+              ./wpt decision --tasks-path=/home/test/artifacts/tasks.json
           features :
             taskclusterProxy: true
         scopes:

From dda0ffc40ebb6e2ad59c5eb4aab31207f67cbd40 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:19:50 +0100
Subject: [PATCH 27/51] Fix some broken task definitions

---
 tools/ci/decision.py    | 17 +++++++++--------
 tools/ci/tasks/test.yml | 20 +++++++++++---------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index a589c6e57ef284..9960b2da9475e6 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -279,11 +279,12 @@ def run(venv, **kwargs):
 
     task_id_map = build_task_graph(event, all_tasks, scheduled_tasks)
 
-    if not kwargs["dry_run"]:
-        create_tasks(queue, task_id_map)
-    else:
-        print(json.dumps(task_id_map, indent=2))
-
-    if kwargs["tasks_path"]:
-        with open(kwargs["tasks_path"], "w") as f:
-            json.dump(task_id_map, f, indent=2)
+    try:
+        if not kwargs["dry_run"]:
+            create_tasks(queue, task_id_map)
+        else:
+            print(json.dumps(task_id_map, indent=2))
+    finally:
+        if kwargs["tasks_path"]:
+            with open(kwargs["tasks_path"], "w") as f:
+                json.dump(task_id_map, f, indent=2)
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 28c1cc230c5ea1..cbcb0f9de63e01 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -4,7 +4,7 @@ components:
     schedulerId: taskcluster-github
     deadline: "24 hours"
     image: harjgam/web-platform-tests:0.33
-    maxRunTime: 7200,
+    maxRunTime: 7200
     artifacts:
       public/results:
         path: /home/test/artifacts
@@ -81,13 +81,13 @@ components:
 
   tox-python2:
     env:
-      - TOXENV=py27
-      - PY_COLORS=0
+      TOXENV: py27
+      PY_COLORS: 0
 
   tox-python3:
     env:
-      - TOXENV=py36
-      - PY_COLORS=0
+      TOXENV: py36
+      PY_COLORS: 0
     install:
       - python3-pip
 
@@ -233,6 +233,8 @@ tasks:
         - wpt-base
         - trigger-master
         - trigger-pr
+      description: >-
+        Lint for wpt-specific requirements
       command: "wpt-lint --all"
       description:
 
@@ -246,15 +248,15 @@ tasks:
       command: "tools/ci/ci_built_diff.sh"
 
   - tools/unittests (Python 2):
-      description: >-
-        Unit tests for tools running under Python 2.7, excluding wptrunner
       use:
         - wpt-base
         - trigger-pr
         - tox-python2
+      description: >-
+        Unit tests for tools running under Python 2.7, excluding wptrunner
       command: tools/ci/ci_tools_unittest.sh
       env:
-        - HYPOTHESIS_PROFILE=ci
+        HYPOTHESIS_PROFILE: ci
       schedule-if:
         run-job:
           - tools_unittest
@@ -268,7 +270,7 @@ tasks:
         - tox-python3
       command: tools/ci/ci_tools_unittest.sh
       env:
-        - HYPOTHESIS_PROFILE=ci;
+        HYPOTHESIS_PROFILE: ci
       schedule-if:
         run-job:
           - tools_unittest

From 0c4f8a4ccdf19a64abf77d39d39e053bf419458a Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:27:39 +0100
Subject: [PATCH 28/51] More schema fixups

---
 tools/ci/decision.py    | 2 +-
 tools/ci/tasks/test.yml | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 9960b2da9475e6..cde4bff7cf9c4d 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -196,7 +196,7 @@ def create_tc_task(event, task, taskgroup_id, required_task_ids):
             "command": command,
             "image": task.get("image"),
             "maxRunTime": task.get("maxRunTime"),
-            "env": task.get("env", []),
+            "env": task.get("env", {}),
         },
         "extra": {
             "github_event": json.dumps(event)
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index cbcb0f9de63e01..88523f5989f6c4 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -236,7 +236,6 @@ tasks:
       description: >-
         Lint for wpt-specific requirements
       command: "wpt-lint --all"
-      description:
 
   - update-built:
       use:

From ff0ed8d032fbe63fd64698cc7824f6a9df290276 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:52:55 +0100
Subject: [PATCH 29/51] Fixup commands

---
 tools/ci/decision.py    | 6 +++---
 tools/ci/tasks/test.yml | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index cde4bff7cf9c4d..bd1daab28fd55c 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -148,7 +148,7 @@ def build_full_command(event, task):
     if options.get("install-certificates"):
         options_args.append("--install-certificates")
 
-    cmd_args["options_str"] = "\n".join("  %s" % item for item in options_args)
+    cmd_args["options_str"] = "\n".join("  %s \\" % item for item in options_args)
 
     install_packages = task.get("install")
     if install_packages:
@@ -166,8 +166,8 @@ def build_full_command(event, task):
   %(fetch_rev)s;
 %(install_str)s
 cd web-platform-tests;
-./tools/ci/run_tc.py
-  %(options_str)s
+./tools/ci/run_tc.py \
+%(options_str)s \
   %(task_cmd)s;
 """ % cmd_args]
 
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 88523f5989f6c4..098e48715fd8fe 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -226,7 +226,7 @@ tasks:
         download-firefox-${vars.channel}:
           use:
             - wpt-base
-          command: "./wpt download --channel=${vars.channel} firefox browser"
+          command: "./wpt install --download-only --channel=${vars.channel} firefox browser"
 
   - lint:
       use:

From 72aae6f19e624cf0d12eb054c519b2947d029567 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 15:53:30 +0100
Subject: [PATCH 30/51] Add support for just downloading browser

---
 tools/wpt/browser.py | 84 ++++++++++++++++++++++++++++++++++++++------
 tools/wpt/install.py |  2 ++
 2 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py
index ca86c55d45f065..d11a7f546c5b6b 100644
--- a/tools/wpt/browser.py
+++ b/tools/wpt/browser.py
@@ -43,6 +43,11 @@ class Browser(object):
     def __init__(self, logger):
         self.logger = logger
 
+    @abstractmethod
+    def download(self, dest=None, channel=None):
+        """Download a package or installer for the browser"""
+        return NotImplemented
+
     @abstractmethod
     def install(self, dest=None):
         """Install the browser."""
@@ -113,11 +118,7 @@ def platform_string_geckodriver(self):
 
         return "%s%s" % (self.platform, bits)
 
-    def install(self, dest=None, channel="nightly"):
-        """Install Firefox."""
-
-        import mozinstall
-
+    def download(self, dest=None, channel="nightly"):
         product = {
             "nightly": "firefox-nightly-latest-ssl",
             "beta": "firefox-beta-latest-ssl",
@@ -172,6 +173,15 @@ def install(self, dest=None, channel="nightly"):
         with open(installer_path, "wb") as f:
             f.write(resp.content)
 
+        return installer_path
+
+    def install(self, dest=None, channel="nightly"):
+        """Install Firefox."""
+
+        import mozinstall
+
+        installer_path = self.download(dest, channel)
+
         try:
             mozinstall.install(installer_path, dest)
         except mozinstall.mozinstall.InstallError:
@@ -419,7 +429,7 @@ class FirefoxAndroid(Browser):
     product = "firefox_android"
     requirements = "requirements_firefox.txt"
 
-    def install(self, dest=None, channel=None):
+    def download(self, dest=None, channel=None):
         if dest is None:
             dest = os.pwd
 
@@ -444,6 +454,9 @@ def install(self, dest=None, channel=None):
 
         return apk_path
 
+    def install(self, dest=None, channel=None):
+        return self.download(dest, channel)
+
     def install_prefs(self, binary, dest=None, channel=None):
         fx_browser = Firefox(self.logger)
         return fx_browser.install_prefs(binary, dest, channel)
@@ -470,6 +483,9 @@ class Chrome(Browser):
     product = "chrome"
     requirements = "requirements_chrome.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -624,6 +640,9 @@ class ChromeAndroidBase(Browser):
     def __init__(self, logger):
         super(ChromeAndroidBase, self).__init__(logger)
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -709,6 +728,9 @@ class ChromeiOS(Browser):
     product = "chrome_ios"
     requirements = "requirements_chrome_ios.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -742,6 +764,9 @@ def binary(self):
         self.logger.warning("Unable to find the browser binary.")
         return None
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -811,6 +836,9 @@ class EdgeChromium(Browser):
     edgedriver_name = "msedgedriver"
     requirements = "requirements_edge_chromium.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -905,6 +933,9 @@ class Edge(Browser):
     product = "edge"
     requirements = "requirements_edge.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -936,6 +967,9 @@ class InternetExplorer(Browser):
     product = "ie"
     requirements = "requirements_ie.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -961,6 +995,9 @@ class Safari(Browser):
     product = "safari"
     requirements = "requirements_safari.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -1020,17 +1057,33 @@ def platform_components(self):
 
         return (platform, extension, decompress)
 
-    def install(self, dest=None, channel="nightly"):
-        """Install latest Browser Engine."""
+    def _get(self, channel="nightly"):
         if channel != "nightly":
             raise ValueError("Only nightly versions of Servo are available")
+
+        platform, extension, _ = self.platform_components()
+        url = "https://download.servo.org/nightly/%s/servo-latest%s" % (platform, extension)
+        return get(url)
+
+    def download(self, dest=None, channel="nightly"):
         if dest is None:
             dest = os.pwd
 
-        platform, extension, decompress = self.platform_components()
-        url = "https://download.servo.org/nightly/%s/servo-latest%s" % (platform, extension)
+        resp = self._get(dest, channel)
+        _, extension, _ = self.platform_components()
 
-        decompress(get(url).raw, dest=dest)
+        with open(os.path.join(dest, "servo-latest%s" % (extension,)), "w") as f:
+            f.write(resp.content)
+
+    def install(self, dest=None, channel="nightly"):
+        """Install latest Browser Engine."""
+        if dest is None:
+            dest = os.pwd
+
+        _, _, decompress = self.platform_components()
+
+        resp = self._get(dest, channel)
+        decompress(resp.raw, dest=dest)
         path = find_executable("servo", os.path.join(dest, "servo"))
         st = os.stat(path)
         os.chmod(path, st.st_mode | stat.S_IEXEC)
@@ -1066,6 +1119,9 @@ class Sauce(Browser):
     product = "sauce"
     requirements = "requirements_sauce.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -1088,6 +1144,9 @@ class WebKit(Browser):
     product = "webkit"
     requirements = "requirements_webkit.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
@@ -1110,6 +1169,9 @@ class Epiphany(Browser):
     product = "epiphany"
     requirements = "requirements_epiphany.txt"
 
+    def download(self, dest=None, channel=None):
+        raise NotImplementedError
+
     def install(self, dest=None, channel=None):
         raise NotImplementedError
 
diff --git a/tools/wpt/install.py b/tools/wpt/install.py
index 8215dfe09161ad..1cacc34d3c452b 100644
--- a/tools/wpt/install.py
+++ b/tools/wpt/install.py
@@ -42,6 +42,8 @@ def get_parser():
                         'the latest available development release. For WebDriver installs, '
                         'we attempt to select an appropriate, compatible, version for the '
                         'latest browser release on the selected channel.')
+    parser.add_argument('--download-only', action="store_true",
+                        help="Download the selected component but don't install it")
     parser.add_argument('-d', '--destination',
                         help='filesystem directory to place the component')
     return parser

From 33c5fb88ac627945ce4125c5324ea5f88c471e97 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 16:10:42 +0100
Subject: [PATCH 31/51] Fixup run_tc command

---
 tools/ci/decision.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index bd1daab28fd55c..35d3f7bd316098 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -148,7 +148,7 @@ def build_full_command(event, task):
     if options.get("install-certificates"):
         options_args.append("--install-certificates")
 
-    cmd_args["options_str"] = "\n".join("  %s \\" % item for item in options_args)
+    cmd_args["options_str"] = " ".join("%s" % item for item in options_args)
 
     install_packages = task.get("install")
     if install_packages:
@@ -161,14 +161,12 @@ def build_full_command(event, task):
             "--login",
             "-c",
             """
-~/start.sh
-  %(repo_url)s
+~/start.sh \
+  %(repo_url)s \
   %(fetch_rev)s;
 %(install_str)s
 cd web-platform-tests;
-./tools/ci/run_tc.py \
-%(options_str)s \
-  %(task_cmd)s;
+./tools/ci/run_tc.py %(options_str)s -- %(task_cmd)s;
 """ % cmd_args]
 
 

From 5ad6d7ec5e7db591b66757284ff564b08385d780 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 16:29:15 +0100
Subject: [PATCH 32/51] Fixup run_tc changes

---
 tools/ci/run_tc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py
index 2f3b2a209482bb..5ab06060440b7c 100755
--- a/tools/ci/run_tc.py
+++ b/tools/ci/run_tc.py
@@ -182,7 +182,7 @@ def setup_environment(args):
     if args.hosts_file:
         make_hosts_file()
 
-    if args["install_certificates"]:
+    if args.install_certificates:
         install_certificates()
 
     if "chrome" in args.browser:

From d2eb25637020d3d6dba7ecfdf9fa09856ecfbcac Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 16:39:06 +0100
Subject: [PATCH 33/51] More command fixup

---
 tools/ci/run_tc.py      |  2 +-
 tools/ci/tasks/test.yml | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py
index 5ab06060440b7c..99c33242749de9 100755
--- a/tools/ci/run_tc.py
+++ b/tools/ci/run_tc.py
@@ -128,7 +128,7 @@ def checkout_revision(rev):
 def install_certificates():
     subprocess.check_call(["sudo", "cp", "tools/certs/cacert.pem",
                            "/usr/local/share/ca-certificates/cacert.crt"])
-    subprocess.check_call(["update-ca-certificates"])
+    subprocess.check_call(["sudo", "update-ca-certificates"])
 
 
 def install_chrome(channel):
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 098e48715fd8fe..fa974d7c861482 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -235,7 +235,7 @@ tasks:
         - trigger-pr
       description: >-
         Lint for wpt-specific requirements
-      command: "wpt-lint --all"
+      command: "./wpt lint --all"
 
   - update-built:
       use:
@@ -244,7 +244,7 @@ tasks:
       schedule-if:
         run-job:
           - update_built
-      command: "tools/ci/ci_built_diff.sh"
+      command: "./tools/ci/ci_built_diff.sh"
 
   - tools/unittests (Python 2):
       use:
@@ -253,7 +253,7 @@ tasks:
         - tox-python2
       description: >-
         Unit tests for tools running under Python 2.7, excluding wptrunner
-      command: tools/ci/ci_tools_unittest.sh
+      command: ./tools/ci/ci_tools_unittest.sh
       env:
         HYPOTHESIS_PROFILE: ci
       schedule-if:
@@ -267,7 +267,7 @@ tasks:
         - wpt-base
         - trigger-pr
         - tox-python3
-      command: tools/ci/ci_tools_unittest.sh
+      command: ./tools/ci/ci_tools_unittest.sh
       env:
         HYPOTHESIS_PROFILE: ci
       schedule-if:
@@ -281,7 +281,7 @@ tasks:
         - wpt-base
         - trigger-pr
         - tox-python2
-      command: tools/ci/ci_wpt.sh
+      command: ./tools/ci/ci_wpt.sh
       install:
         - libnss3-tools
       options:
@@ -302,7 +302,7 @@ tasks:
         - wpt-base
         - trigger-pr
         - tox-python2
-      command: tools/ci/ci_resources_unittest.sh
+      command: ./tools/ci/ci_resources_unittest.sh
       options:
         browser:
           - firefox
@@ -319,7 +319,7 @@ tasks:
         - wpt-base
         - trigger-pr
         - tox-python2
-      command: tools/ci/ci_wpt.sh
+      command: ./tools/ci/ci_wpt.sh
       install:
         - libnss3-tools
         - libappindicator1

From ec5526fe3e130513e52b29c60032820b5d3768ee Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 17:18:45 +0100
Subject: [PATCH 34/51] Fixup sending channel into run_tc

---
 tools/ci/decision.py    | 2 ++
 tools/ci/tasks/test.yml | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 35d3f7bd316098..e219e304fd076c 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -143,6 +143,8 @@ def build_full_command(event, task):
         options_args.append("--checkout=%s" % options["checkout"])
     for browser in options.get("browser", []):
         options_args.append("--browser=%s" % browser)
+    if options.get("channel"):
+        options_args.append("--channel=%s" % options["channel"])
     if options.get("checkout"):
         options_args.append("--checkout=%s" % options["checkout"])
     if options.get("install-certificates"):
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index fa974d7c861482..d2fc6c20552845 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -226,7 +226,7 @@ tasks:
         download-firefox-${vars.channel}:
           use:
             - wpt-base
-          command: "./wpt install --download-only --channel=${vars.channel} firefox browser"
+          command: "./wpt install --download-only --destination /home/test/artifacts/ --channel=${vars.channel} firefox browser"
 
   - lint:
       use:

From ba932188f80cb39e1c9b57e6cd04a3625eec8637 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 17:29:20 +0100
Subject: [PATCH 35/51] Fix wpt install --download-only

---
 tools/wpt/install.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tools/wpt/install.py b/tools/wpt/install.py
index 1cacc34d3c452b..24915f0c98d0be 100644
--- a/tools/wpt/install.py
+++ b/tools/wpt/install.py
@@ -75,21 +75,22 @@ def run(venv, **kwargs):
             raise argparse.ArgumentError(None,
                                          "No --destination argument, and no default for the environment")
 
-    install(browser, kwargs["component"], destination, channel)
+    install(browser, kwargs["component"], destination, channel,
+            download_only=kwargs["download_only"])
 
 
-def install(name, component, destination, channel="nightly", logger=None):
+def install(name, component, destination, channel="nightly", logger=None, download_only=False):
     if logger is None:
         import logging
         logger = logging.getLogger("install")
 
-    if component == 'webdriver':
-        method = 'install_webdriver'
-    else:
-        method = 'install'
+    prefix = "download" if download_only else "install"
+    suffix = "_webdriver" if component == 'webdriver' else ""
+
+    method = prefix + suffix
 
     subclass = getattr(browser, name.title())
     sys.stdout.write('Now installing %s %s...\n' % (name, component))
     path = getattr(subclass(logger), method)(dest=destination, channel=channel)
     if path:
-        sys.stdout.write('Binary installed as %s\n' % (path,))
+        sys.stdout.write('Binary %s as %s\n' % ("downloaded" if download_only else "installed", path,))

From d560c17787612ddb65ef6f56df08616d30460a5e Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 18:57:31 +0100
Subject: [PATCH 36/51] Make an example change to wptrunner

---
 tools/wptrunner/wptrunner/wptrunner.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/wptrunner/wptrunner/wptrunner.py b/tools/wptrunner/wptrunner/wptrunner.py
index 8dcdcdebe154fc..19133c4c689e74 100644
--- a/tools/wptrunner/wptrunner/wptrunner.py
+++ b/tools/wptrunner/wptrunner/wptrunner.py
@@ -1,5 +1,7 @@
 from __future__ import print_function, unicode_literals
 
+# Example change
+
 import json
 import os
 import sys

From 55a5f463101f868c69d1dfeda2b870e4e367cd35 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 19:02:21 +0100
Subject: [PATCH 37/51] Don't use .pusher for PRs

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 722ac9cd419dfb..dbbace76d1e0bd 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -39,7 +39,7 @@ tasks:
         metadata:
           name: "wpt Decision Task"
           description: "The task that creates all of the other tasks in the task graph"
-          owner: ${event.pusher.email}
+          owner: "${event.sender.login}@users.noreply.github.com"
           source: ${event.repository.url}
         payload:
           image: harjgam/web-platform-tests:0.33

From fb41522f9eda65b64f606beeae4a895e5454c060 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 19:12:50 +0100
Subject: [PATCH 38/51] Use clone_url

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index dbbace76d1e0bd..e8156cec6b49b0 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -40,7 +40,7 @@ tasks:
           name: "wpt Decision Task"
           description: "The task that creates all of the other tasks in the task graph"
           owner: "${event.sender.login}@users.noreply.github.com"
-          source: ${event.repository.url}
+          source: ${event.repository.clone_url}
         payload:
           image: harjgam/web-platform-tests:0.33
           maxRunTime: 7200

From d4a8dd3aef74c14359bf7aade06f38ff201b9355 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Thu, 26 Sep 2019 19:20:04 +0100
Subject: [PATCH 39/51] Really use clone_url

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index e8156cec6b49b0..b5100e0beba00b 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -54,7 +54,7 @@ tasks:
             - -c
             - set -ex;
               ~/start.sh
-                ${event.repository.url}
+                ${event.repository.clone_url}
                 ${event.after};
               cd ~/web-platform-tests;
               ./wpt decision --tasks-path=/home/test/artifacts/tasks.json

From a12614943125599fa68811e0def8b8568ed38597 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 11:42:23 +0100
Subject: [PATCH 40/51] Ensure we only run for correct actions

---
 .taskcluster.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index b5100e0beba00b..aba05b913723c3 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -22,7 +22,10 @@ tasks:
         else: false
       else:
         $if: 'tasks_for == "github-pull-request"'
-        then: true
+        then:
+          $if: event.action in ['opened', 'reopened', 'synchronize']
+          then: true
+          else: false
         else: false
   in:
     - $if: run_task

From 8eb39e4be9a475a4425c934538bf421206a1ee8a Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 13:55:03 +0100
Subject: [PATCH 41/51] More clone_url fixes and better logging

---
 tools/ci/decision.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index e219e304fd076c..d49128ef14d7ad 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -64,8 +64,11 @@ def filter_triggers(event, all_tasks):
 
 def get_run_jobs(event):
     import jobs
-    paths = jobs.get_paths(revish="%s..%s" % (event["before"],
-                                              event["after"]))
+    revish="%s..%s" % (event["before"],
+                       event["after"])
+    logger.info("Looking for changes in range %s" % revish)
+    paths = jobs.get_paths(revish=revish)
+    logger.info("Found changes in paths:%s" % "\n".join(paths))
     path_jobs = jobs.get_jobs(paths)
     all_jobs = path_jobs | get_extra_jobs(event)
     logger.info("Including jobs %s" % ", ".join(all_jobs))
@@ -123,7 +126,7 @@ def get_fetch_rev(event):
 def build_full_command(event, task):
     cmd_args = {
         "task_name": task["name"],
-        "repo_url": event["repository"]["url"],
+        "repo_url": event["repository"]["clone_url"],
         "fetch_rev": get_fetch_rev(event),
         "task_cmd": task["command"],
         "install_str": "",
@@ -189,7 +192,7 @@ def create_tc_task(event, task, taskgroup_id, required_task_ids):
             "name": task["name"],
             "description": task.get("description", ""),
             "owner": "%s@users.noreply.github.com" % event["sender"]["login"],
-            "source": event["repository"]["url"]
+            "source": event["repository"]["clone_url"]
         },
         "payload": {
             "artifacts": task.get("artifacts"),

From e1e66e3d16f0a514e1c74f68a6858ab594a1b6ed Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 14:32:13 +0100
Subject: [PATCH 42/51] Fix handling of revs for PRs

---
 tools/ci/decision.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index d49128ef14d7ad..85ffe1380e52d5 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -1,8 +1,9 @@
 import argparse
 import json
+import logging
 import os
 import re
-import logging
+import subprocess
 from collections import OrderedDict
 
 import taskcluster
@@ -64,7 +65,9 @@ def filter_triggers(event, all_tasks):
 
 def get_run_jobs(event):
     import jobs
-    revish="%s..%s" % (event["before"],
+    revish="%s..%s" % (event["pull_request"]["base"]["sha"]
+                       if "pull_request" in event
+                       else event["before"],
                        event["after"])
     logger.info("Looking for changes in range %s" % revish)
     paths = jobs.get_paths(revish=revish)
@@ -118,7 +121,19 @@ def filter_schedule_if(event, tasks):
 def get_fetch_rev(event):
     is_pr, _ = get_triggers(event)
     if is_pr:
-        return event["pull_request"]["merge_commit_sha"]
+        # Try to get the actual rev so that all non-decision tasks are pinned to that
+        ref = "refs/pull/%s/merge" % event["pull_request"]["number"]
+        try:
+            output = subprocess.check_output(["git", "ls-remote", "origin", ref])
+        except subprocess.CalledProcessError:
+            import traceback
+            logger.error(join(traceback.format_exc())
+            logger.error("Failed to get merge commit sha2")
+            return ref
+        if not output:
+            logger.error("Failed to get merge commit")
+            return ref
+        return output.split()[0]
     else:
         return event["after"]
 

From b85d91ffe47ca5063dc4d5f0d339daa7bea63653 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 14:44:05 +0100
Subject: [PATCH 43/51] Typo

---
 tools/ci/decision.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 85ffe1380e52d5..aa4dc3295ac107 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -127,7 +127,7 @@ def get_fetch_rev(event):
             output = subprocess.check_output(["git", "ls-remote", "origin", ref])
         except subprocess.CalledProcessError:
             import traceback
-            logger.error(join(traceback.format_exc())
+            logger.error(traceback.format_exc())
             logger.error("Failed to get merge commit sha2")
             return ref
         if not output:

From 1278177e145932160a9b30de158bae05ab422e85 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 14:53:55 +0100
Subject: [PATCH 44/51] Fixup scheduling

---
 tools/ci/decision.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index aa4dc3295ac107..92a6f3bf9167a9 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -58,8 +58,8 @@ def filter_triggers(event, all_tasks):
                 for trigger_branch in task["trigger"]["branch"]:
                     if (trigger_branch == branch or
                         trigger_branch.endswith("*") and branch.startswith(trigger_branch[:-1])):
-                        logger.info("Triggers include task %s" % name)
                         triggered[name] = task
+    logger.info("Triggers match tasks:\n%s" % "\n".join(triggered.keys()))
     return triggered
 
 
@@ -109,12 +109,11 @@ def filter_schedule_if(event, tasks):
             if "run-job" in task["schedule-if"]:
                 if run_jobs is None:
                     run_jobs = get_run_jobs(event)
-                if "all" in run_jobs or any(item in run_jobs for item in task["schedule-if"]):
+                if "all" in run_jobs or any(item in run_jobs for item in task["schedule-if"]["run-job"]):
                     scheduled[name] = task
-                    logger.info("Scheduled tasks include %s" % name)
         else:
             scheduled[name] = task
-            logger.info("Scheduled tasks include %s" % name)
+    logger.info("Scheduling rules match tasks:\n%s" % "\n".join(scheduled.keys()))
     return scheduled
 
 

From 2744c1dd4a8db23163aeca0fc55d7ac353cbe087 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 15:21:41 +0100
Subject: [PATCH 45/51] Setup hosts for more jobs

---
 tools/ci/decision.py    | 6 +++---
 tools/ci/tasks/test.yml | 4 +++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index 92a6f3bf9167a9..c59ca6ecb92bfd 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -59,7 +59,7 @@ def filter_triggers(event, all_tasks):
                     if (trigger_branch == branch or
                         trigger_branch.endswith("*") and branch.startswith(trigger_branch[:-1])):
                         triggered[name] = task
-    logger.info("Triggers match tasks:\n%s" % "\n".join(triggered.keys()))
+    logger.info("Triggers match tasks:\n * %s" % "\n * ".join(triggered.keys()))
     return triggered
 
 
@@ -74,7 +74,7 @@ def get_run_jobs(event):
     logger.info("Found changes in paths:%s" % "\n".join(paths))
     path_jobs = jobs.get_jobs(paths)
     all_jobs = path_jobs | get_extra_jobs(event)
-    logger.info("Including jobs %s" % ", ".join(all_jobs))
+    logger.info("Including jobs:\n * %s" % "\n * ".join(all_jobs))
     return all_jobs
 
 
@@ -113,7 +113,7 @@ def filter_schedule_if(event, tasks):
                     scheduled[name] = task
         else:
             scheduled[name] = task
-    logger.info("Scheduling rules match tasks:\n%s" % "\n".join(scheduled.keys()))
+    logger.info("Scheduling rules match tasks:\n * %s" % "\n * ".join(scheduled.keys()))
     return scheduled
 
 
diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index d2fc6c20552845..41fb0d208b1e60 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -291,6 +291,7 @@ tasks:
           - chrome
         channel: experimental
         xvfb: true
+        hosts: true
       schedule-if:
         run-job:
           - wpt_integration
@@ -308,6 +309,7 @@ tasks:
           - firefox
         channel: experimental
         xvfb: true
+        hosts: true
       schedule-if:
         run-job:
           - resources_unittest
@@ -331,7 +333,7 @@ tasks:
           - chrome
         channel: experimental
         xvfb: true
-        hosts: false
+        hosts: true
       schedule-if:
         run-job:
           - wptrunner_infrastructure

From b5d66681b7104e29a77b2b47f30f55132e6b1be8 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 15:21:54 +0100
Subject: [PATCH 46/51] Change tests to reflect new code

---
 tools/ci/tests/test_run_tc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/ci/tests/test_run_tc.py b/tools/ci/tests/test_run_tc.py
index a72dcfe5cfdd51..ad416ab1164bb0 100644
--- a/tools/ci/tests/test_run_tc.py
+++ b/tools/ci/tests/test_run_tc.py
@@ -2,7 +2,7 @@
 
 from six import iteritems
 
-from tools.ci import run_tc
+from tools.ci import decision
 
 
 @pytest.mark.parametrize("msg,expected", [
@@ -30,4 +30,4 @@ def sub(obj):
 
     event = sub(event)
 
-    assert run_tc.get_extra_jobs(event) == expected
+    assert decision.get_extra_jobs(event) == expected

From aa78849bf9da0cac347b78e4191623f32294913b Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 16:46:02 +0100
Subject: [PATCH 47/51] Fix pyflakes errors

---
 tools/ci/decision.py | 8 ++++----
 tools/ci/run_tc.py   | 1 -
 tools/tox.ini        | 2 ++
 tools/wpt/browser.py | 1 +
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tools/ci/decision.py b/tools/ci/decision.py
index c59ca6ecb92bfd..2bbe92ea57ce70 100644
--- a/tools/ci/decision.py
+++ b/tools/ci/decision.py
@@ -65,10 +65,10 @@ def filter_triggers(event, all_tasks):
 
 def get_run_jobs(event):
     import jobs
-    revish="%s..%s" % (event["pull_request"]["base"]["sha"]
-                       if "pull_request" in event
-                       else event["before"],
-                       event["after"])
+    revish = "%s..%s" % (event["pull_request"]["base"]["sha"]
+                         if "pull_request" in event
+                         else event["before"],
+                         event["after"])
     logger.info("Looking for changes in range %s" % revish)
     paths = jobs.get_paths(revish=revish)
     logger.info("Found changes in paths:%s" % "\n".join(paths))
diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py
index 99c33242749de9..94064a5dabd1ea 100755
--- a/tools/ci/run_tc.py
+++ b/tools/ci/run_tc.py
@@ -38,7 +38,6 @@
 import argparse
 import json
 import os
-import re
 import subprocess
 import sys
 try:
diff --git a/tools/tox.ini b/tools/tox.ini
index 52e749e4bdc4c7..b5e6e553cb5fce 100644
--- a/tools/tox.ini
+++ b/tools/tox.ini
@@ -10,6 +10,8 @@ deps =
   hypothesis
   # `requests` is required by `update_pr_preview.py`
   requests
+  taskcluster
+  yaml
 
 commands = pytest {posargs}
 
diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py
index d11a7f546c5b6b..574b103fc91448 100644
--- a/tools/wpt/browser.py
+++ b/tools/wpt/browser.py
@@ -181,6 +181,7 @@ def install(self, dest=None, channel="nightly"):
         import mozinstall
 
         installer_path = self.download(dest, channel)
+        filename = os.path.basename(installer_path)
 
         try:
             mozinstall.install(installer_path, dest)

From 38b77c69f0a8a3e38af1244d92cf0f65596086f7 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 16:53:53 +0100
Subject: [PATCH 48/51] Fix yaml name

---
 tools/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/tox.ini b/tools/tox.ini
index b5e6e553cb5fce..336747aa3a54dc 100644
--- a/tools/tox.ini
+++ b/tools/tox.ini
@@ -11,7 +11,7 @@ deps =
   # `requests` is required by `update_pr_preview.py`
   requests
   taskcluster
-  yaml
+  pyyaml
 
 commands = pytest {posargs}
 

From 9e826da56170ba39f0d82fe646e11e483776b07d Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 16:57:36 +0100
Subject: [PATCH 49/51] Fix infra tests

---
 tools/ci/tasks/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/ci/tasks/test.yml b/tools/ci/tasks/test.yml
index 41fb0d208b1e60..f812bf9a972d13 100644
--- a/tools/ci/tasks/test.yml
+++ b/tools/ci/tasks/test.yml
@@ -321,7 +321,7 @@ tasks:
         - wpt-base
         - trigger-pr
         - tox-python2
-      command: ./tools/ci/ci_wpt.sh
+      command: ./tools/ci/ci_wptrunner_infrastructure.sh
       install:
         - libnss3-tools
         - libappindicator1
@@ -333,7 +333,7 @@ tasks:
           - chrome
         channel: experimental
         xvfb: true
-        hosts: true
+        hosts: false
       schedule-if:
         run-job:
           - wptrunner_infrastructure

From 98902b1399503b091d8e1fd53ed8edc89cdcb967 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Fri, 27 Sep 2019 17:05:58 +0100
Subject: [PATCH 50/51] Fix location of Fx install

---
 tools/wpt/browser.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py
index 574b103fc91448..6baab590aec00b 100644
--- a/tools/wpt/browser.py
+++ b/tools/wpt/browser.py
@@ -118,6 +118,18 @@ def platform_string_geckodriver(self):
 
         return "%s%s" % (self.platform, bits)
 
+    def _get_dest(self, dest, channel):
+        if dest is None:
+            # os.getcwd() doesn't include the venv path
+            dest = os.path.join(os.getcwd(), "_venv")
+
+        dest = os.path.join(dest, "browsers", channel)
+
+        if not os.path.exists(dest):
+            os.makedirs(dest)
+
+        return dest
+
     def download(self, dest=None, channel="nightly"):
         product = {
             "nightly": "firefox-nightly-latest-ssl",
@@ -134,21 +146,15 @@ def download(self, dest=None, channel="nightly"):
         }
         os_key = (self.platform, uname[4])
 
+        if dest is None:
+            dest = self._get_dest(None, channel)
+
         if channel not in product:
             raise ValueError("Unrecognised release channel: %s" % channel)
 
         if os_key not in os_builds:
             raise ValueError("Unsupported platform: %s %s" % os_key)
 
-        if dest is None:
-            # os.getcwd() doesn't include the venv path
-            dest = os.path.join(os.getcwd(), "_venv")
-
-        dest = os.path.join(dest, "browsers", channel)
-
-        if not os.path.exists(dest):
-            os.makedirs(dest)
-
         url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel],
                                                                              os_builds[os_key])
         self.logger.info("Downloading Firefox from %s" % url)
@@ -177,9 +183,10 @@ def download(self, dest=None, channel="nightly"):
 
     def install(self, dest=None, channel="nightly"):
         """Install Firefox."""
-
         import mozinstall
 
+        dest = self._get_dest(dest, channel)
+
         installer_path = self.download(dest, channel)
         filename = os.path.basename(installer_path)
 

From 273d9b9f8ac6ecfb7fead91b54657aa7ff2fa948 Mon Sep 17 00:00:00 2001
From: James Graham <james@hoppipolla.co.uk>
Date: Mon, 30 Sep 2019 13:31:15 +0100
Subject: [PATCH 51/51] Add some taskgraph documentation

---
 tools/ci/README.md | 232 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 232 insertions(+)
 create mode 100644 tools/ci/README.md

diff --git a/tools/ci/README.md b/tools/ci/README.md
new file mode 100644
index 00000000000000..1d8d68d33eb3f1
--- /dev/null
+++ b/tools/ci/README.md
@@ -0,0 +1,232 @@
+# Taskgraph Setup
+
+The taskgraph is built from a yaml file. This file has two top-level
+properties: `components` and `tasks`. The full list of tasks is
+defined by the `tasks` object; each task is an object with a single
+property representing the task with the corresponding value an object
+representing the task properties. Each task requires the following
+top-level properties:
+
+* `provisionerId`: String. Name of TaskCluster provisioner
+* `schedulerId`: String. Name of TaskCluster scheduler
+* `deadline`: String. Time until the task expires
+* `image`: String. Name of docker image to use for task
+* `maxRunTime`: Number. Maximum time in seconds for which the task can
+  run.
+* `artifacts`: Object. List of artifacts and directories to upload; see
+  TaskCluster documentation.
+* `command`: String. Command to run. This is automatically wrapped in a
+  run_tc command
+* `options`: Optional Object. Options to pass into run_tc
+  - xvfb: Boolean. Enable Xvfb for run
+  - oom-killer: Boolean. Enable xvfb for run
+  - hosts: Boolean. Update hosts file with wpt hosts before run
+  - install-certificates: Boolean. Install wpt certs into OS
+    certificate store for run
+  - browser: List. List of browser names for run
+  - channel: String. Browser channel for run
+* `trigger`: Object. Conditions on which to consider task. One or more
+  of following properties:
+  - branch: List. List of branch names on which to trigger.
+  - pull-request: No value. Trigger for pull request actions
+* `schedule-if`: Optional Object. Conditions on which task should be
+  scheduled given it meets the trigger conditions.
+  - `run-job`: List. Job names for which this task should be considered,
+    matching the output from ./wpt jobs
+* `env`: Optional Object. Environment variables to set when running task.
+* `require`: Optional list. List of task names that must be complete
+  before the current task is scheduled.
+* `description`: String. Task description.
+* `name`: Optional String. Name to use for the task overriding the
+  property name. This is usful in combination with substitutions
+  described below.
+
+## Task Expansions
+
+Using the above syntax it's possble to describe each task
+directly. But typically in a taskgraph there are many common
+properties between tasks so it's tedious and error prone to repeat
+information that's common to multiple tasks. Therefore the taskgraph
+format provides several mechanisms to reuse partial task definitions
+across multiple tasks.
+
+### Components
+
+The other top-level property in the taskgraph format is
+`components`. The value of this property is an object containing named
+partial task definitions. Each task definition may contain a property called
+`use` which is a list of components to use as the basis for the task
+definition. The components list is evaluated in-order. If a property
+is not previously defined in the output it is added to the output. If
+it was previously defined, the value is updated according to the type:
+ * Strings and numbers are replaced with a new value
+ * Lists are extended with the additional values
+ * Objects are updated recursively following the above rules
+This means that types must always match between components and the
+final value.
+
+For example
+```
+components:
+  example-1:
+    list_prop:
+      - first
+      - second
+    object_prop:
+      key1: value1
+      key2: base_value
+  example-2:
+    list_prop:
+      - third
+      - fourth
+    object_prop:
+      key3:
+        - value3-1
+
+tasks:
+  - example-task:
+      use:
+        - example-1
+        - example-2
+      object_prop:
+        key2: value2
+        key3:
+         - value3-2
+```
+
+will evaluate to the following task:
+
+```
+example-task:
+  list_prop:
+    - first
+    - second
+    - third
+    - fourth
+  object_prop:
+    key1: value1
+    key2: value2
+    key3:
+      - value3-1
+      - value3-2
+```
+
+Note that components cannot currently define `use` properties of their own.
+
+## Substitutions
+
+Components and tasks can define a property `vars` that holds variables
+which are later substituted into the task definition using the syntax
+`${vars.property-name}`. For example:
+
+```
+components:
+  generic-component:
+    prop: ${vars.value}
+
+tasks:
+  - first:
+      use:
+        - generic-component
+      vars:
+        value: value1
+  - second:
+      use:
+        - generic-component
+      vars:
+        value: value2
+```
+
+Results in the following tasks:
+
+```
+first:
+  prop: value1
+second:
+  prop: value2
+```
+
+## Maps
+
+Instead of defining a task directly, an item in the tasks property may
+be an object with a single property `$map`. This object itself has two
+child properties; `for` and `do`. The value of `for` is a list of
+objects, and the value of `do` is either an object or a list of
+objects. For each object in the `for` property, a set of tasks is
+created by taking a copy of that object for each task in the `do`
+property, updating the object with the properties from the
+corresponding `do` object, using the same rules as for components
+above, and then processing as for a normal task. `$map` rules can also
+be nested.
+
+For example
+
+```
+components: {}
+tasks:
+  $map:
+    for:
+      - vars:
+          example: value1
+      - vars:
+          example: value2
+    do:
+      example-${vars.example}
+        prop: ${vars.example}
+```
+
+Results in the tasks
+
+```
+example-value1:
+  prop: value1
+example-value2:
+  prop: value2
+```
+
+Note that in combination with `$map`, variable substitutions are
+applied *twice*; once after the `$map` is evaluated and once after the
+`use` statements are evaluated.
+
+## Chunks
+
+A common requirements for tasks is that they are "chunked" into N
+partial tasks. This is handled specially in the syntax. A top level
+property `chunks` can be used to define the number of individual
+chunks to create for a specific task. Each chunked task is created
+with a `chunks` property set to an object containing an `id` property
+containing the one-based index of the chunk an a `total` property
+containing the total number of chunks. These can be substituted into
+the task definition using the same syntax as for `vars` above
+e.g. `${chunks.id}`. Note that because task names must be unique, it's
+common to specify a `name` property on the task that will override the
+property name e.g.
+
+```
+components: {}
+tasks:
+  - chunked-task:
+      chunks:2
+      command: "task-run --chunk=${chunks.id} --totalChunks=${chunks.total}"
+      name: task-chunk-${chunks.id}
+```
+
+creates tasks:
+
+```
+task-chunk-1:
+  command: "task-run --chunk=1 --totalChunks=2"
+task-chunk-2:
+  command: "task-run --chunk=2 --totalChunks=2"
+```
+
+# Overall processing model
+
+The overall processing model for tasks is as follows:
+ * Evaluate maps
+ * Perform subsitutions
+ * Evaluate use statements
+ * Exapnd chunks
+ * Perform subsitutions
+
+At each point after maps are evaluated tasks must have a unique name.