diff --git a/gantry/__main__.py b/gantry/__main__.py
index 68acc60..744fbce 100644
--- a/gantry/__main__.py
+++ b/gantry/__main__.py
@@ -28,6 +28,7 @@ async def apply_migrations(db: aiosqlite.Connection):
         # and not inadvertently added to the migrations folder
         ("001_initial.sql", 1),
         ("002_spec_index.sql", 2),
+        ("003_oom_retry.sql", 3),
     ]
 
     # apply migrations that have not been applied
diff --git a/gantry/clients/gitlab.py b/gantry/clients/gitlab.py
index 1e94e55..812b268 100644
--- a/gantry/clients/gitlab.py
+++ b/gantry/clients/gitlab.py
@@ -1,3 +1,5 @@
+import urllib.parse
+
 import aiohttp
 
 
@@ -6,26 +8,43 @@ def __init__(self, base_url: str, api_token: str):
         self.base_url = base_url
         self.headers = {"PRIVATE-TOKEN": api_token}
 
-    async def _request(self, url: str, response_type: str) -> dict | str:
+    async def _request(self, method: str, url: str, response_type: str) -> dict | str:
         """
         Helper for requests to the Gitlab API.
 
         args:
+            method: HTTP method (GET, POST)
             url: the url to request
-            response_type: the type of response to expect (json or text)
+            response_type: the type of response to expect (json, text by default)
 
         returns: the response from Gitlab in the specified format
         """
 
         async with aiohttp.ClientSession() as session:
-            async with session.get(url, headers=self.headers) as resp:
+            async with session.request(method, url, headers=self.headers) as resp:
                 if response_type == "json":
                     return await resp.json()
-                if response_type == "text":
-                    return await resp.text()
+                return await resp.text()
 
     async def job_log(self, gl_id: int) -> str:
         """Given a job id, returns the log from that job"""
 
         url = f"{self.base_url}/jobs/{gl_id}/trace"
-        return await self._request(url, "text")
+        return await self._request("get", url, "text")
+
+    async def start_pipeline(self, ref: str) -> dict:
+        """Given a ref, starts a pipeline"""
+        url = f"{self.base_url}/pipeline?ref={urllib.parse.quote(ref)}"
+        return await self._request("POST", url, "json")
+
+    async def job_oom(self, gl_id: int) -> bool:
+        """Given a job id, returns if the job was OOM killed"""
+
+        url = f"""{self.base_url}/jobs/{gl_id}/artifacts/
+                jobs_scratch_dir/user_data/oom-info"""
+        try:
+            await self._request("get", url, "text")
+            return True
+        # when gitlab can't find an artifact, it returns HTTP 400
+        except aiohttp.client_exceptions.ClientPayloadError:
+            return False
diff --git a/gantry/clients/prometheus/job.py b/gantry/clients/prometheus/job.py
index 9f9d7ed..e6f595c 100644
--- a/gantry/clients/prometheus/job.py
+++ b/gantry/clients/prometheus/job.py
@@ -53,6 +53,9 @@ async def get_annotations(self, gl_id: int, time: float) -> dict:
                     "annotation_metrics_spack_job_spec_compiler_version"
                 ],
                 "stack": annotations["annotation_metrics_spack_ci_stack_name"],
+                "retry_count": int(
+                    annotations.get("annotation_metrics_spack_job_retry_count", 0)
+                ),
             }
         except KeyError as e:
             # if any of the annotations are missing, raise an error
diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py
index 7908672..228d0e0 100644
--- a/gantry/routes/collection.py
+++ b/gantry/routes/collection.py
@@ -9,6 +9,7 @@
 from gantry.clients.prometheus import PrometheusClient
 from gantry.clients.prometheus.util import IncompleteData
 from gantry.models import Job
+from gantry.routes.prediction import RETRY_COUNT_LIMIT
 
 MB_IN_BYTES = 1_000_000
 BUILD_STAGE_REGEX = r"^stage-\d+$"
@@ -16,23 +17,84 @@
 logger = logging.getLogger(__name__)
 
 
+async def handle_pipeline(
+    payload: dict,
+    db_conn: aiosqlite.Connection,
+    gitlab: GitlabClient,
+    prometheus: PrometheusClient,
+) -> None | bool:
+    """
+    Sends any failed jobs from a pipeline to fetch_job.
+    If any of the failed jobs were OOM killed, the pipeline will be recreated.
+
+    args:
+        payload: a dictionary containing the information from the Gitlab pipeline hook
+        db: an active aiosqlite connection
+        gitlab: gitlab client
+        prometheus: prometheus client
+
+    returns: True if the pipeline was recreated, else None
+    """
+
+    if payload["object_attributes"]["status"] != "failed":
+        return
+
+    ref = payload["object_attributes"]["ref"]
+    failed_jobs = [
+        # imitate the payload from the job hook, which fetch_jobs expects
+        {
+            "build_status": job["status"],
+            "build_id": job["id"],
+            "build_started_at": job["started_at"],
+            "build_finished_at": job["finished_at"],
+            "ref": ref,
+            "build_stage": job["stage"],
+            "runner": job["runner"],
+        }
+        for job in payload["builds"]
+        if job["status"] == "failed"
+    ]
+
+    retry_pipeline = False
+
+    for job in failed_jobs:
+        # insert every potentially oomed job
+        # if a job has been retried RETRY_COUNT_LIMIT times, oomed will be False
+        # start_pipeline will be called if any of the failed_jobs fit the criteria
+        # the same check is performed on the prediction side, and won't re-bump memory
+        oomed = await fetch_job(job, db_conn, gitlab, prometheus, from_pipeline=True)
+
+        # fetch_job can return None or (job_id: int, oomed: bool)
+        if oomed and oomed[1]:
+            retry_pipeline = True
+
+    # once all jobs are collected/discarded, retry the pipeline if needed
+    if retry_pipeline:
+        await gitlab.start_pipeline(ref)
+        return retry_pipeline
+
+
 async def fetch_job(
     payload: dict,
     db_conn: aiosqlite.Connection,
     gitlab: GitlabClient,
     prometheus: PrometheusClient,
-) -> None:
+    from_pipeline: bool = False,
+) -> tuple[int, bool] | None:
     """
+    Collects a job's information from Prometheus and inserts into db.
+    Warnings about missing data will be logged; check uncaught exceptions.
     Fetches a job's information from Prometheus and inserts it into the database.
-    If there is data missing at any point, the function will still return so the webhook
-    responds as expected. If an exception is thrown, that behavior was unanticipated by
-    this program and should be investigated.
 
     args:
-        payload: a dictionary containing the information from the Gitlab job hook
+        payload: a dictionary containing the information from the gitlab job hook
         db: an active aiosqlite connection
+        gitlab: gitlab client
+        prometheus: prometheus client
+        from_pipeline: if the job was called from a pipeline handler
 
-    returns: None in order to accommodate a 200 response for the webhook.
+    returns: if data was inserted,
+                a tuple of the job id and if the job was OOM killed, else None
     """
 
     job = Job(
@@ -45,7 +107,12 @@ async def fetch_job(
 
     # perform checks to see if we should collect data for this job
     if (
+        # successful jobs should not come from a handle_pipeline call
         job.status != "success"
+        and from_pipeline is False
+        # we don't want to collect failed jobs that aren't from a handle_pipeline call
+        or job.status != "failed"
+        and from_pipeline is True
         # if the stage is not stage-NUMBER, it's not a build job
         or not re.match(BUILD_STAGE_REGEX, payload["build_stage"])
         # some jobs don't have runners..?
@@ -57,6 +124,9 @@ async def fetch_job(
     ):
         return
 
+    # track if job was OOM killed and needs to be retried
+    oomed = False
+
     try:
         # all code that makes HTTP requests should be in this try block
 
@@ -68,6 +138,18 @@ async def fetch_job(
             return
 
         annotations = await prometheus.job.get_annotations(job.gl_id, job.midpoint)
+        # check if failed job was OOM killed,
+        # return early if it wasn't because we don't care about it anymore
+        # do not retry if the job has already been retried RETRY_COUNT_LIMIT times
+        if job.status == "failed":
+            if (
+                await gitlab.job_oom(job.gl_id)
+                and annotations["retry_count"] < RETRY_COUNT_LIMIT
+            ):
+                oomed = True
+            else:
+                return
+
         resources, node_hostname = await prometheus.job.get_resources(
             annotations["pod"], job.midpoint
         )
@@ -90,6 +172,7 @@ async def fetch_job(
             "gitlab_id": job.gl_id,
             "job_status": job.status,
             "ref": job.ref,
+            "oomed": oomed,
             **annotations,
             **resources,
             **usage,
@@ -99,8 +182,7 @@ async def fetch_job(
     # job and node will get saved at the same time to make sure
     # we don't accidentally commit a node without a job
     await db_conn.commit()
-
-    return job_id
+    return (job_id, oomed)
 
 
 async def fetch_node(
diff --git a/gantry/routes/prediction.py b/gantry/routes/prediction.py
index e8d696a..824bb57 100644
--- a/gantry/routes/prediction.py
+++ b/gantry/routes/prediction.py
@@ -19,6 +19,8 @@
     "openmp",
     "hdf5",
 }
+MEM_BUMP_FACTOR = 1.2
+RETRY_COUNT_LIMIT = 3
 
 
 async def predict(db: aiosqlite.Connection, spec: dict) -> dict:
@@ -33,6 +35,11 @@ async def predict(db: aiosqlite.Connection, spec: dict) -> dict:
         CPU in millicore, mem in MB
     """
 
+    # check if the memory limit needs to be increased
+    alloc_oom = await check_oom(db, spec)
+    if alloc_oom:
+        return {"variables": alloc_oom}
+
     sample = await get_sample(db, spec)
     predictions = {}
     if not sample:
@@ -56,12 +63,7 @@ async def predict(db: aiosqlite.Connection, spec: dict) -> dict:
         logger.warning(f"Warning: Memory request for {spec} is below 10MB")
         predictions["mem_request"] = DEFAULT_MEM_REQUEST
 
-    # convert predictions to k8s friendly format
-    for k, v in predictions.items():
-        if k.startswith("cpu"):
-            predictions[k] = k8s.convert_cores(v)
-        elif k.startswith("mem"):
-            predictions[k] = k8s.convert_bytes(v)
+    predictions = k8s.convert_allocations(predictions)
 
     return {
         "variables": {
@@ -115,7 +117,8 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l
         # within this combo, variants included
         query = f"""
         SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs
-        WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in filters.keys())}
+        WHERE ref='develop' AND job_status='success'
+        AND {' AND '.join(f'{param}=?' for param in filters.keys())}
         ORDER BY end DESC LIMIT {IDEAL_SAMPLE}
         """
 
@@ -155,7 +158,8 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l
 
         query = f"""
         SELECT cpu_mean, cpu_max, mem_mean, mem_max FROM jobs
-        WHERE ref='develop' AND {' AND '.join(f'{param}=?' for param in filters.keys())}
+        WHERE ref='develop' AND job_status='success'
+        AND {' AND '.join(f'{param}=?' for param in filters.keys())}
         AND {' AND '.join(exp_variant_conditions)}
         ORDER BY end DESC LIMIT {IDEAL_SAMPLE}
         """
@@ -164,3 +168,59 @@ async def select_sample(query: str, filters: dict, extra_params: list = []) -> l
             return sample
 
     return []
+
+
+async def check_oom(db: aiosqlite.Connection, spec: dict) -> dict:
+    """
+    Check if the spec's last build was OOM killed and bump
+    the prediction if necessary.
+
+    args:
+        spec: see predict
+    returns:
+        dict of variables for the k8s job
+    """
+
+    # look for an exact match of the spec that has been OOM killed
+    query = """
+    SELECT cpu_mean, cpu_max, mem_mean, mem_limit, retry_count FROM jobs
+    WHERE pkg_name=? AND pkg_version=? AND pkg_variants=?
+    AND compiler_name=? AND compiler_version=? AND arch=? AND oomed=1
+    ORDER BY end DESC LIMIT 1
+    """
+
+    async with db.execute(
+        query,
+        (
+            spec["pkg_name"],
+            spec["pkg_version"],
+            spec["pkg_variants"],
+            spec["compiler_name"],
+            spec["compiler_version"],
+            spec["arch"],
+        ),
+    ) as cursor:
+        res = await cursor.fetchall()
+
+    if not res:
+        return {}
+
+    # use the last build's resource usage as a baseline
+    # using mem_limit instead of max to ensure it's increased by the bump factor
+    variables = {
+        "KUBERNETES_CPU_REQUEST": res[0][0],
+        "KUBERNETES_CPU_LIMIT": res[0][1],
+        "KUBERNETES_MEMORY_REQUEST": res[0][2],
+        "KUBERNETES_MEMORY_LIMIT": res[0][3],
+        "GANTRY_RETRY_COUNT": res[0][4],
+    }
+
+    if variables["GANTRY_RETRY_COUNT"] < RETRY_COUNT_LIMIT:
+        # only bump the memory if it's been a certain amount
+        # the build will likely fail but this is to prevent infinite retries
+        variables["KUBERNETES_MEMORY_LIMIT"] = (
+            variables["KUBERNETES_MEMORY_LIMIT"] * MEM_BUMP_FACTOR
+        )
+        variables["GANTRY_RETRY_COUNT"] += 1
+
+    return k8s.convert_allocations(variables)
diff --git a/gantry/tests/defs/collection.py b/gantry/tests/defs/collection.py
index d419d79..dce51fd 100644
--- a/gantry/tests/defs/collection.py
+++ b/gantry/tests/defs/collection.py
@@ -18,20 +18,53 @@
     "runner": {"description": "aws"},
 }
 
+FAILED_JOB = {
+    "build_status": "failed",
+    "build_stage": "stage-1",
+    "build_id": 9892514,  # not used in testing unless it already exists in the db
+    "build_started_at": "2024-01-24 17:24:06 UTC",
+    "build_finished_at": "2024-01-24 17:47:00 UTC",
+    "ref": "pr42264_bugfix/mathomp4/hdf5-appleclang15",
+    "runner": {"description": "aws"},
+}
+
+
 # used to compare successful insertions
 # run SELECT * FROM table_name WHERE id = 1; from python sqlite api and grab fetchone() result
-INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485)
+INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485, 0, 0)
 INSERTED_NODE = (1, 'ec253b04-b1dc-f08b-acac-e23df83b3602', 'ip-192-168-86-107.ec2.internal', 24.0, 196608000000.0, 'amd64', 'linux', 'i3en.6xlarge')
 
 # these were obtained by executing the respective queries to Prometheus and capturing the JSON output
 # or the raw output of PrometheusClient._query
-VALID_ANNOTATIONS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_annotations', 'annotation_gitlab_ci_job_id': '9892514', 'annotation_metrics_spack_ci_stack_name': 'e4s', 'annotation_metrics_spack_job_spec_arch': 'linux-ubuntu20.04-x86_64_v3', 'annotation_metrics_spack_job_spec_compiler_name': 'gcc', 'annotation_metrics_spack_job_spec_compiler_version': '11.4.0', 'annotation_metrics_spack_job_spec_pkg_name': 'gmsh', 'annotation_metrics_spack_job_spec_pkg_version': '4.8.4', 'annotation_metrics_spack_job_spec_variants': '+alglib~cairo+cgns+compression~eigen~external+fltk+gmp~hdf5~ipo+med+metis+mmg+mpi+netgen+oce~opencascade~openmp~petsc~privateapi+shared~slepc+tetgen+voropp build_system=cmake build_type=Release generator=make', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0'}, 'value': [1706117733, '1']}]}}
+VALID_ANNOTATIONS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_annotations', 'annotation_gitlab_ci_job_id': '9892514', 'annotation_metrics_spack_ci_stack_name': 'e4s', 'annotation_metrics_spack_job_spec_arch': 'linux', 'annotation_metrics_spack_job_spec_compiler_name': 'gcc', 'annotation_metrics_spack_job_spec_compiler_version': '11.4.0', 'annotation_metrics_spack_job_retry_count': '0', 'annotation_metrics_spack_job_spec_pkg_name': 'gmsh', 'annotation_metrics_spack_job_spec_pkg_version': '4.8.4', 'annotation_metrics_spack_job_spec_variants': '+alglib~cairo+cgns+compression~eigen~external+fltk+gmp~hdf5~ipo+med+metis+mmg+mpi+netgen+oce~opencascade~openmp~petsc~privateapi+shared~slepc+tetgen+voropp build_system=cmake build_type=Release generator=make', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0'}, 'value': [1706117733, '1']}]}}
 VALID_RESOURCE_REQUESTS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'cpu', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'core'}, 'value': [1706117733, '0.75']}, {'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '2000000000']}]}}
 VALID_RESOURCE_LIMITS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_limits', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '48000000000']}]}}
 VALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '2785280'], [1706117116, '2785280'], [1706117117, '2785280'], [1706117118, '2785280'], [1706117119, '2785280'], [1706117120, '2785280'], [1706117121, '2785280'], [1706117122, '2785280'], [1706117123, '2785280'], [1706117124, '2785280'], [1706117125, '2785280'], [1706117126, '2785280'], [1706117127, '2785280'], [1706117128, '2785280'], [1706117129, '2785280'], [1706117130, '2785280'], [1706118416, '594620416'], [1706118417, '594620416'], [1706118418, '594620416'], [1706118419, '594620416'], [1706118420, '594620416']]}]}}
 VALID_CPU_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'container': 'build', 'cpu': 'total', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117145, '0.2483743618267752'], [1706117146, '0.25650526138466395'], [1706117147, '0.26463616094255266'], [1706117148, '0.2727670605004414'], [1706117149, '0.28089796005833007'], [1706117150, '0.2890288596162188'], [1706117151, '0.2971597591741076'], [1706117357, '3.7319005481816236'], [1706117358, '3.7319005481816236'], [1706117359, '3.7319005481816236'], [1706117360, '3.7319005481816245'], [1706117361, '3.7319005481816245'], [1706118420, '4.128116379389054']]}]}}
 VALID_NODE_INFO = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_info', 'container': 'kube-state-metrics', 'container_runtime_version': 'containerd://1.7.2', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'internal_ip': '192.168.86.107', 'job': 'kube-state-metrics', 'kernel_version': '5.10.205-195.804.amzn2.x86_64', 'kubelet_version': 'v1.27.9-eks-5e0fdde', 'kubeproxy_version': 'v1.27.9-eks-5e0fdde', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'os_image': 'Amazon Linux 2', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'provider_id': 'aws:///us-east-1c/i-0fe9d9c99fdb3631d', 'service': 'kube-prometheus-stack-kube-state-metrics', 'system_uuid': 'ec253b04-b1dc-f08b-acac-e23df83b3602'}, 'value': [1706117733, '1']}]}}
 VALID_NODE_LABELS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_labels', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'label_beta_kubernetes_io_arch': 'amd64', 'label_beta_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_beta_kubernetes_io_os': 'linux', 'label_failure_domain_beta_kubernetes_io_region': 'us-east-1', 'label_failure_domain_beta_kubernetes_io_zone': 'us-east-1c', 'label_k8s_io_cloud_provider_aws': 'ceb9f9cc8e47252a6f7fe7d6bded2655', 'label_karpenter_k8s_aws_instance_category': 'i', 'label_karpenter_k8s_aws_instance_cpu': '24', 'label_karpenter_k8s_aws_instance_encryption_in_transit_supported': 'true', 'label_karpenter_k8s_aws_instance_family': 'i3en', 'label_karpenter_k8s_aws_instance_generation': '3', 'label_karpenter_k8s_aws_instance_hypervisor': 'nitro', 'label_karpenter_k8s_aws_instance_local_nvme': '15000', 'label_karpenter_k8s_aws_instance_memory': '196608', 'label_karpenter_k8s_aws_instance_network_bandwidth': '25000', 'label_karpenter_k8s_aws_instance_pods': '234', 'label_karpenter_k8s_aws_instance_size': '6xlarge', 'label_karpenter_sh_capacity_type': 'spot', 'label_karpenter_sh_initialized': 'true', 'label_karpenter_sh_provisioner_name': 'glr-x86-64-v4', 'label_kubernetes_io_arch': 'amd64', 'label_kubernetes_io_hostname': 'ip-192-168-86-107.ec2.internal', 'label_kubernetes_io_os': 'linux', 'label_node_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_spack_io_pipeline': 'true', 'label_spack_io_x86_64': 'v4', 'label_topology_ebs_csi_aws_com_zone': 'us-east-1c', 'label_topology_kubernetes_io_region': 'us-east-1', 'label_topology_kubernetes_io_zone': 'us-east-1c', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'service': 'kube-prometheus-stack-kube-state-metrics'}, 'value': [1706117733, '1']}]}}
-
+# TODO FIX THIS WHEN WE KNOW THE RIGHT METHOD
+OOM_KILLED = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_status_last_terminated_reason', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'container': 'build', 'reason': 'OOMKilled'}, 'value': [1706117733, '1']}]}}
+NOT_OOM_KILLED = {'status': 'success', 'data': {'resultType': 'vector', 'result': []}}
 # modified version of VALID_MEMORY_USAGE to make the mean/stddev 0
 INVALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '0']]}]}}
+
+# pipeline definitions
+SUCCESSFUL_PIPELINE = {
+    "object_attributes": {
+        "status": "success",
+        "ref": "pr42264_bugfix/mathomp4/hdf5-appleclang15",
+    }
+}
+
+FAILED_PIPELINE = {
+    "object_attributes": {
+        "status": "failed",
+        "ref": "pr42264_bugfix/mathomp4/hdf5-appleclang15",
+    },
+    "builds": [
+        {"id": 9892514, "stage": "stage-1", "status": "failed", "runner": {"description": "aws"}, "started_at": "2024-01-24 17:24:06 UTC", "finished_at": "2024-01-24 17:47:00 UTC"},
+    ]
+}
+
+INSERTED_OOM_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'failed', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485, 1, 0)
diff --git a/gantry/tests/defs/prediction.py b/gantry/tests/defs/prediction.py
index 7b697c4..3806fb8 100644
--- a/gantry/tests/defs/prediction.py
+++ b/gantry/tests/defs/prediction.py
@@ -34,3 +34,13 @@
         "KUBERNETES_MEMORY_REQUEST": "2000M",
     },
 }
+
+OOM_PREDICTION = {
+    "variables": {
+        "KUBERNETES_CPU_REQUEST": "14779m",
+        "KUBERNETES_CPU_LIMIT": "12001m",
+        "KUBERNETES_MEMORY_REQUEST": "9358M",
+        "KUBERNETES_MEMORY_LIMIT": "76800M",
+        "GANTRY_RETRY_COUNT": 1,
+    },
+}
diff --git a/gantry/tests/sql/insert_job.sql b/gantry/tests/sql/insert_job.sql
index 3008da6..359fa1e 100644
--- a/gantry/tests/sql/insert_job.sql
+++ b/gantry/tests/sql/insert_job.sql
@@ -1 +1 @@
-INSERT INTO jobs VALUES(1,'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z',2,1706117046,1706118420,9892514,'success','pr42264_bugfix/mathomp4/hdf5-appleclang15','gmsh','4.8.4','{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',16,0.75,NULL,4.12532286694540495,3.15805864677520409,11.6038107294648877,0.248374361826775191,3.34888880339475214,2000000000.0,48000000000.0,1649868862.72588062,999763968.0,5679742976.0,2785280.0,1378705563.21018671);
\ No newline at end of file
+INSERT INTO jobs VALUES(1,'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z',2,1706117046,1706118420,9892514,'success','pr42264_bugfix/mathomp4/hdf5-appleclang15','gmsh','4.8.4','{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}','gcc','11.4.0','linux','e4s',16,0.75,NULL,4.12532286694540495,3.15805864677520409,11.6038107294648877,0.248374361826775191,3.34888880339475214,2000000000.0,48000000000.0,1649868862.72588062,999763968.0,5679742976.0,2785280.0,1378705563.21018671, 0, 0);
\ No newline at end of file
diff --git a/gantry/tests/sql/insert_samples.sql b/gantry/tests/sql/insert_samples.sql
index d017ebe..edc57f9 100644
--- a/gantry/tests/sql/insert_samples.sql
+++ b/gantry/tests/sql/insert_samples.sql
@@ -1,6 +1,11 @@
 INSERT INTO nodes VALUES(6789,'ec2c47a0-7e9b-cfa3-9ad4-ac227ade598d','ip-192-168-202-150.ec2.internal',32.0,131072000000.0,'amd64','linux','m5.8xlarge');
-INSERT INTO jobs VALUES(6781,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi1',6789,1708919572.983000041,1708924744.811000108,101502092,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,9.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9652098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419);
-INSERT INTO jobs VALUES(6782,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi2',6789,1708919572.983000041,1708924744.811000108,101502093,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,10.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9958098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419);
-INSERT INTO jobs VALUES(6783,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi3',6789,1708919572.983000041,1708924744.811000108,101502094,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,11.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9158098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419);
-INSERT INTO jobs VALUES(6784,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi4',6789,1708919572.983000041,1708924744.811000108,101502095,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,12.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9758098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419);
-INSERT INTO jobs VALUES(6785,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi5',6789,1708919572.983000041,1708924744.811000108,101502096,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,13.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9358098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419);
+INSERT INTO jobs VALUES(6781,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi1',6789,1708919572.983000041,1708924744.811000108,101502092,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux','e4s',12,12.0,NULL,9.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9652098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0, 0);
+INSERT INTO jobs VALUES(6782,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi2',6789,1708919572.983000041,1708924744.811000108,101502093,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux','e4s',12,12.0,NULL,10.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9958098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0, 0);
+INSERT INTO jobs VALUES(6783,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi3',6789,1708919572.983000041,1708924744.811000108,101502094,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux','e4s',12,12.0,NULL,11.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9158098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0, 0);
+INSERT INTO jobs VALUES(6784,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi4',6789,1708919572.983000041,1708924744.811000108,101502095,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux','e4s',12,12.0,NULL,12.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9758098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0, 0);
+INSERT INTO jobs VALUES(6785,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi5',6789,1708919572.983000041,1708924744.811000108,101502096,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux','e4s',12,12.0,NULL,13.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9358098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0, 0);
+
+-- oom detection:
+-- end time is modified to ensure it's the most recent spec
+-- arch is different so the other prediction tests don't get matched and get bumped
+INSERT INTO jobs VALUES(6786,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi6',6789,1708919572.983000041,1708924746.811000108,101502097,'failure','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','oom-arch','e4s',12,12.0,NULL,14.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9358098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 1, 0);
diff --git a/gantry/tests/test_collection.py b/gantry/tests/test_collection.py
index 926a161..1f3a103 100644
--- a/gantry/tests/test_collection.py
+++ b/gantry/tests/test_collection.py
@@ -1,14 +1,17 @@
+import copy
+
 import pytest
 
 from gantry.clients.gitlab import GitlabClient
 from gantry.clients.prometheus import PrometheusClient
-from gantry.routes.collection import fetch_job, fetch_node
+from gantry.routes.collection import fetch_job, fetch_node, handle_pipeline
+from gantry.routes.prediction import RETRY_COUNT_LIMIT
 from gantry.tests.defs import collection as defs
 
 # mapping of prometheus request shortcuts
 # to raw values that would be returned by resp.json()
 
-# note: the ordering of this dict indicated the order of the calls
+# note: the ordering of this dict indicated the order of the calls.
 # if the order in which PrometheusClient._query is called changes,
 # this dict must be updated
 PROMETHEUS_REQS = {
@@ -23,12 +26,17 @@
 
 
 @pytest.fixture
-async def gitlab(mocker):
+async def gitlab(mocker, request):
     """Returns GitlabClient with some default (mocked) behavior"""
 
     # mock the request to the gitlab api
     # default is to return normal log that wouldn't be detected as a ghost job
     mocker.patch.object(GitlabClient, "_request", return_value=defs.VALID_JOB_LOG)
+
+    # Optionally mock the start_pipeline method if needed
+    if getattr(request, "param", {}).get("with_restart", False):
+        mocker.patch.object(GitlabClient, "start_pipeline", return_value=None)
+
     return GitlabClient("", "")
 
 
@@ -126,6 +134,20 @@ async def test_job_node_inserted(db_conn, gitlab, prometheus):
     assert node == defs.INSERTED_NODE
 
 
+async def test_failed_job(db_conn, gitlab, prometheus):
+    """Tests condition check for failed/success in fetch_job."""
+
+    # successful jobs from pipeline should not be inserted
+    await fetch_job(defs.VALID_JOB, db_conn, gitlab, prometheus, from_pipeline=True)
+    async with db_conn.execute("SELECT * FROM jobs") as cursor:
+        assert await cursor.fetchone() is None
+
+    # failed jobs not from the pipeline should not be inserted
+    await fetch_job(defs.FAILED_JOB, db_conn, gitlab, prometheus, from_pipeline=False)
+    async with db_conn.execute("SELECT * FROM jobs") as cursor:
+        assert await cursor.fetchone() is None
+
+
 async def test_node_exists(db_conn, prometheus):
     """Tests that fetch_node returns the existing node id when the node
     is already in the database"""
@@ -143,3 +165,62 @@ async def test_node_exists(db_conn, prometheus):
         await db_conn.executescript(f.read())
 
     assert await fetch_node(db_conn, prometheus, None, None) == 2
+
+
+@pytest.mark.parametrize("gitlab", [{"with_restart": True}], indirect=True)
+async def test_handle_pipeline(db_conn, gitlab, prometheus):
+    """Tests the behavior of handle_pipeline with different pipeline
+    and job statuses."""
+
+    p = PROMETHEUS_REQS.copy()
+
+    # successful pipeline
+    assert (
+        await handle_pipeline(defs.SUCCESSFUL_PIPELINE, db_conn, gitlab, prometheus)
+        is None
+    )
+
+    # pipeline failed and not oomed
+    p_list = list(p.values())
+    # insert a prometheus response indicating the job was not oom killed
+    p_list.insert(1, defs.NOT_OOM_KILLED)
+    prometheus._query.side_effect = p_list
+    assert (
+        await handle_pipeline(defs.FAILED_PIPELINE, db_conn, gitlab, prometheus) is None
+    )
+
+    # job oom killed, but over retry limit
+    p_list = list(p.values())
+    # modify the annotations
+    # deepcopy so the original annotations are not modified
+    p_list[0] = copy.deepcopy(p_list[0])
+    p_list[0]["data"]["result"][0]["metric"] |= {
+        "annotation_metrics_spack_job_retry_count": str(RETRY_COUNT_LIMIT)
+    }
+    p_list.insert(1, defs.OOM_KILLED)
+    prometheus._query.side_effect = p_list
+    # handle_pipeline should not allow a retry because the retry count is over the limit
+    # however, if another job was oomed but not over the limit, it should be retried
+    assert (
+        await handle_pipeline(defs.FAILED_PIPELINE, db_conn, gitlab, prometheus) is None
+    )
+
+    # pipeline failed, one job was oomed the other was not
+    p_list = list(p.values())
+    # after verifying job was not oomed, go onto the next job to insert annotations
+    p_list[1:1] = [defs.NOT_OOM_KILLED, p["job_annotations"], defs.OOM_KILLED]
+    prometheus._query.side_effect = p_list
+    # duplicate the same job so it calls fetch_job twice
+    pipeline = defs.FAILED_PIPELINE.copy()
+    pipeline["builds"].append(pipeline["builds"][0])
+    assert await handle_pipeline(pipeline, db_conn, gitlab, prometheus)
+
+    # verify that OOM job was inserted
+    async with db_conn.execute("SELECT * FROM jobs WHERE id=?", (1,)) as cursor:
+        job = await cursor.fetchone()
+    assert job == defs.INSERTED_OOM_JOB
+
+
+# TODO test if OOM status is correct
+# TODO test start_pipeline response is json not None
+# test retry limit on collect side
diff --git a/gantry/tests/test_prediction.py b/gantry/tests/test_prediction.py
index c321ea8..dab36b3 100644
--- a/gantry/tests/test_prediction.py
+++ b/gantry/tests/test_prediction.py
@@ -96,21 +96,52 @@ def test_invalid_specs():
     assert parse_alloc_spec("hi") == {}
 
     # missing package
-    assert parse_alloc_spec("@29.2 +json+native+treesitter%gcc@12.3.0") == {}
+    assert (
+        parse_alloc_spec("@29.2 +json+native+treesitter arch=x86_64%gcc@12.3.0") == {}
+    )
 
     # missing compiler
-    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter") == {}
+    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter arch=x86_64") == {}
 
     # variants not spaced correctly
-    assert parse_alloc_spec("emacs@29.2+json+native+treesitter%gcc@12.3.0") == {}
+    assert (
+        parse_alloc_spec("emacs@29.2+json+native+treesitter arch=x86_64%gcc@12.3.0")
+        == {}
+    )
 
     # missing compiler version
-    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter%gcc@") == {}
-    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter%gcc") == {}
+    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter arch=x86_64%gcc@") == {}
+    assert parse_alloc_spec("emacs@29.2 +json+native+treesitter arch=x86_64%gcc") == {}
 
     # missing package version
-    assert parse_alloc_spec("emacs@ +json+native+treesitter%gcc@12.3.0") == {}
-    assert parse_alloc_spec("emacs+json+native+treesitter%gcc@12.3.0") == {}
+    assert (
+        parse_alloc_spec("emacs@ +json+native+treesitter arch=x86_64%gcc@12.3.0") == {}
+    )
+    assert parse_alloc_spec("emacs+json+native+treesitter arch=x86_64%gcc@12.3.0") == {}
 
     # invalid variants
-    assert parse_alloc_spec("emacs@29.2 this_is_not_a_thing%gcc@12.3.0") == {}
+    assert (
+        parse_alloc_spec("emacs@29.2 this_is_not_a_thing arch=x86_64%gcc@12.3.0") == {}
+    )
+
+
+async def test_oom(db_conn_inserted):
+    """Tests that the prediction is based on the highest OOM sample."""
+
+    # differentiate the arch so the non-oom tests above don't think they were oom killed
+    build = defs.NORMAL_BUILD.copy()
+    build["arch"] = "oom-arch"
+
+    # test allocation was bumped up by 20%
+    assert await prediction.predict(db_conn_inserted, build) == defs.OOM_PREDICTION
+
+    # simulate job having been retried 3x
+    await db_conn_inserted.execute(
+        "UPDATE jobs SET retry_count=3, mem_limit=76800000000 WHERE id=6786"
+    )
+
+    last_prediction = defs.OOM_PREDICTION.copy()
+    last_prediction["variables"]["GANTRY_RETRY_COUNT"] = 3
+
+    # it shouldn't change
+    assert await prediction.predict(db_conn_inserted, build) == last_prediction
diff --git a/gantry/util/k8s.py b/gantry/util/k8s.py
index 470f945..47d7ad4 100644
--- a/gantry/util/k8s.py
+++ b/gantry/util/k8s.py
@@ -1,10 +1,19 @@
 BYTES_TO_MEGABYTES = 1 / 1_000_000
 CORES_TO_MILLICORES = 1_000
 
-# these functions convert the predictions to k8s friendly format
 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
 
 
+def convert_allocations(allocations: dict) -> dict:
+    """converts the allocations to k8s friendly format"""
+    for k, v in allocations.items():
+        if "cpu" in k.lower():
+            allocations[k] = convert_cores(v)
+        elif "mem" in k.lower():
+            allocations[k] = convert_bytes(v)
+    return allocations
+
+
 def convert_bytes(bytes: float) -> str:
     """bytes to megabytes"""
     return str(int(round(bytes * BYTES_TO_MEGABYTES))) + "M"
diff --git a/gantry/views.py b/gantry/views.py
index b23931f..45f5488 100644
--- a/gantry/views.py
+++ b/gantry/views.py
@@ -5,7 +5,7 @@
 
 from aiohttp import web
 
-from gantry.routes.collection import fetch_job
+from gantry.routes.collection import fetch_job, handle_pipeline
 from gantry.routes.prediction import predict
 from gantry.util.spec import parse_alloc_spec
 
@@ -14,7 +14,7 @@
 
 
 @routes.post("/v1/collect")
-async def collect_job(request: web.Request) -> web.Response:
+async def collect(request: web.Request) -> web.Response:
     try:
         payload = await request.json()
     except json.decoder.JSONDecodeError:
@@ -23,18 +23,22 @@ async def collect_job(request: web.Request) -> web.Response:
     if request.headers.get("X-Gitlab-Token") != os.environ["GITLAB_WEBHOOK_TOKEN"]:
         return web.Response(status=401, text="invalid token")
 
-    if request.headers.get("X-Gitlab-Event") != "Job Hook":
-        logger.error(f"invalid event type {request.headers.get('X-Gitlab-Event')}")
-        # return 200 so gitlab doesn't disable the webhook -- this is not fatal
-        return web.Response(status=200)
-
-    # will return immediately, but will not block the event loop
-    # allowing fetch_job to run in the background
-    asyncio.ensure_future(
-        fetch_job(
-            payload, request.app["db"], request.app["gitlab"], request.app["prometheus"]
-        )
+    hook_type = request.headers.get("X-Gitlab-Event")
+    job_args = (
+        payload,
+        request.app["db"],
+        request.app["gitlab"],
+        request.app["prometheus"],
     )
+    if hook_type == "Job Hook":
+        # using ensure_future because it doesn't block the event loop
+        # and returns immediately, allowing jobs to run in the background
+        asyncio.ensure_future(fetch_job(*job_args))
+    elif hook_type == "Pipeline Hook":
+        asyncio.ensure_future(handle_pipeline(*job_args))
+    else:
+        # this is not fatal, but we should log it
+        logger.error(f"invalid event type {hook_type}")
 
     return web.Response(status=200)
 
diff --git a/migrations/003_oom_retry.sql b/migrations/003_oom_retry.sql
new file mode 100644
index 0000000..fc3d22b
--- /dev/null
+++ b/migrations/003_oom_retry.sql
@@ -0,0 +1,2 @@
+ALTER TABLE jobs ADD COLUMN oomed BOOLEAN DEFAULT FALSE;
+ALTER TABLE jobs ADD COLUMN retry_count INTEGER DEFAULT 0;