Add MVTEC LOCO training/eval scripts and results

eval_mvtec_loco_all.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$ROOT_DIR"
+
+PYTHON_BIN="${PYTHON_BIN:-python}"
+
+MVTEC_LOCO_PATH="${MVTEC_LOCO_PATH:-../../datasets/mvtec_loco_anomaly_detection}"
+
+if [[ -n "${OUTPUT_DIR:-}" ]]; then
+  OUT="$OUTPUT_DIR"
+else
+  OUT="output/1"
+  if [[ -d output ]]; then
+    mapfile -t out_nums < <(find output -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | grep -E '^[0-9]+$' | sort -n)
+    if [[ ${#out_nums[@]} -gt 0 ]]; then
+      last_idx=$(( ${#out_nums[@]} - 1 ))
+      OUT="output/${out_nums[$last_idx]}"
+    fi
+  fi
+fi
+
+ANOMALY_MAPS_DIR="${ANOMALY_MAPS_DIR:-$OUT/anomaly_maps/mvtec_loco}"
+METRICS_DIR="${METRICS_DIR:-$OUT/metrics/mvtec_loco}"
+
+if [[ ! -d "$MVTEC_LOCO_PATH" ]]; then
+  echo "ERROR: MVTEC_LOCO_PATH not found: $MVTEC_LOCO_PATH" >&2
+  exit 1
+fi
+
+if [[ ! -d "$ANOMALY_MAPS_DIR" ]]; then
+  echo "ERROR: ANOMALY_MAPS_DIR not found: $ANOMALY_MAPS_DIR" >&2
+  exit 1
+fi
+
+if [[ ! -f "mvtec_loco_ad_evaluation/evaluate_experiment.py" ]]; then
+  echo "ERROR: missing mvtec_loco_ad_evaluation/evaluate_experiment.py" >&2
+  exit 1
+fi
+
+echo "Using --dataset_base_dir: $MVTEC_LOCO_PATH"
+echo "Using --anomaly_maps_dir: $ANOMALY_MAPS_DIR"
+echo "Using --output_dir base: $METRICS_DIR"
+
+mapfile -t subdatasets < <(find "$MVTEC_LOCO_PATH" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort)
+if [[ ${#subdatasets[@]} -eq 0 ]]; then
+  echo "ERROR: no subdataset dirs found under: $MVTEC_LOCO_PATH" >&2
+  exit 1
+fi
+
+for subdataset in "${subdatasets[@]}"; do
+  test_maps_dir="$ANOMALY_MAPS_DIR/$subdataset/test"
+  if [[ ! -d "$test_maps_dir" ]]; then
+    echo "SKIP: anomaly maps not found: $test_maps_dir" >&2
+    continue
+  fi
+
+  echo "=== Evaluating: object_name=$subdataset ==="
+  "$PYTHON_BIN" mvtec_loco_ad_evaluation/evaluate_experiment.py \
+    --dataset_base_dir "$MVTEC_LOCO_PATH" \
+    --anomaly_maps_dir "$ANOMALY_MAPS_DIR" \
+    --output_dir "$METRICS_DIR/$subdataset" \
+    --object_name "$subdataset"
+done

mvtec_loco_ad_evaluation/LICENSE.txt

@@ -0,0 +1,11 @@
+Copyright 2022 MVTec Software GmbH
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

mvtec_loco_ad_evaluation/README.md

@@ -0,0 +1,93 @@
+# Evaluate your experiments on the MVTec LOCO AD dataset
+
+The evaluation scripts can be used to assess the performance of a method on the MVTec Logical Costraints Anomaly Detection (MVTec LOCO AD) dataset.
+Given a directory with anomaly maps, the scripts compute the area under the sPRO curve for anomaly localization. 
+Additionally, the area under the ROC curve for anomaly classification is computed.
+
+## Installation.
+Our evaluation scripts require a python 3.7 installation as well as the following
+packages:
+- numpy
+- tifffile
+- pillow
+- tabulate
+- tqdm
+
+For Linux, we provide an conda environment file. It can be used to create a new conda environment with all required packages readily installed:
+```
+conda env create --name mvtec_loco_eval --file=conda_environment.yml
+conda activate mvtec_loco_eval
+```
+
+## Evaluating a single experiment.
+The evaluation script requires an anomaly map to be present for each test sample in our dataset in `.tiff` format. 
+Anomaly maps must contain real-valued anomaly scores and their size must match the one of the corresponding dataset images. 
+Anomaly maps must all share the same base directory and adhere to the following folder structure: 
+`<anomaly_maps_dir>/<object_name>/test/<defect_name>/<image_id>.tiff`
+
+To evaluate a single experiment on one of the dataset objects, the script `evaluate_experiment.py` can be used.  
+It requires the following user arguments:
+- `object_name`: Name of the dataset object to be evaluated.
+- `dataset_base_dir`: Base directory that contains the MVTec LOCO AD dataset.
+- `anomaly_maps_dir`: Base directory that contains the corresponding anomaly maps.
+- `output_dir`: Directory to store evaluation results as `.json` files.
+
+A possible example call to this script would be:
+```
+python evaluate_experiment.py \
+    --object_name pushpins \
+    --dataset_base_dir 'path/to/dataset/' \
+    --anomaly_maps_dir 'path/to/anomaly_maps/' \
+    --output_dir 'metrics/'
+```
+
+The evaluation script computes the area under the sPRO curve up to a limited false positive rate as described in our paper. 
+The integration limits are specified by the variable `MAX_FPRS`.
+
+## Evaluate multiple experiments
+
+If more than one experiment should be evaluated simultaneously, the script `evaluate_multiple_experiments.py` can be used. 
+Multiple directories conatining anomaly maps should be specified in a `config.json` file with the following structure:
+```
+{
+    "exp_base_dir": "/path/to/all/experiments/",
+    "anomaly_maps_dirs": {
+    "experiment_id_1": "eg/model_1/anomaly_maps/",
+    "experiment_id_2": "eg/model_2/anomaly_maps/",
+    "experiment_id_3": "eg/model_3/anomaly_maps/",
+    "...": "..."
+    }
+}
+```
+- `exp_base_dir`: Base directory that contains all experimental results for each evaluated method.
+- `anomaly_maps_dirs`: Dictionary that contains an identifier for each evaluated experiment and the location of its anomaly maps relative to the `exp_base_dir`.
+
+The evaluation is run by calling `evaluate_multiple_experiments.py` with the following user arguments:
+- `dataset_base_dir`: Base directory that contains the MVTec LOCO dataset.
+- `experiment_configs`: Path to the above `config.json` that contains all experiments to be evaluated.
+- `output_dir`: Directory to store evaluation results as `.json` files.
+
+A possible example call to this script would be:
+```
+  python evaluate_multiple_experiments.py \
+    --dataset_base_dir 'path/to/dataset/' \
+    --experiment_configs 'configs.json' \
+    --output_dir 'metrics/'
+```
+
+## Visualize the evaluation results.
+After running `evaluate_experiment.py` or `evaluate_multiple_experiments.py`, the script `print_metrics.py`  can be used to visualize all computed metrics in a table. 
+In total, three tables are printed to the standard output. The first two tables display the performance for the structural and logical anomalies, respectively. 
+The third table shows the mean performance over both anomaly types.
+
+The script requires the following user arguments:
+- `metrics_folder`: The base directory that contains the computed metrics for each evaluated method. This directory is usually identical to the output directory specified in `evaluate_experiment.py` or `evaluate_multiple_experiments.py`.
+- `metric_type`: Select either `localization` or `classification`. When selecting `localization`,
+the AUC-sPRO results for the pixelwise localization of anomalies is shown. When selecting `classification`, the image level AUC-ROC for anomaly classification is shown.
+- `integration_limit`: The integration limit until which the area under the sPRO curve is computed. This parameter is only applicable when `metric_type` is set to `localization`.
+
+# License
+The license agreement for our evaluation code is found in the accompanying
+`LICENSE.txt` file.
+
+The version of this evaluation script is: 2.0

mvtec_loco_ad_evaluation/conda_environment.yml

@@ -0,0 +1,78 @@
+name: mvtec_loco_eval
+channels:
+  - anaconda
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=1_gnu
+  - blosc=1.21.0=h9c3ff4c_0
+  - brotli=1.0.9=h7f98852_6
+  - brotli-bin=1.0.9=h7f98852_6
+  - brunsli=0.1=h9c3ff4c_0
+  - bzip2=1.0.8=h7f98852_4
+  - c-ares=1.18.1=h7f98852_0
+  - ca-certificates=2021.10.8=ha878542_0
+  - cfitsio=3.470=hb418390_7
+  - charls=2.2.0=h2531618_0
+  - colorama=0.4.4=pyh9f0ad1d_0
+  - freetype=2.10.4=h5ab3b9f_0
+  - giflib=5.2.1=h36c2ea0_2
+  - imagecodecs=2021.8.26=py37h4cda21f_0
+  - jpeg=9e=h7f98852_0
+  - jxrlib=1.1=h7f98852_2
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.19.2=h08a2579_4
+  - lcms2=2.12=hddcbb42_0
+  - ld_impl_linux-64=2.36.1=hea4e1c9_2
+  - lerc=3.0=h9c3ff4c_0
+  - libaec=1.0.6=h9c3ff4c_0
+  - libblas=3.9.0=13_linux64_openblas
+  - libbrotlicommon=1.0.9=h7f98852_6
+  - libbrotlidec=1.0.9=h7f98852_6
+  - libbrotlienc=1.0.9=h7f98852_6
+  - libcblas=3.9.0=13_linux64_openblas
+  - libcurl=7.81.0=h494985f_0
+  - libdeflate=1.8=h7f98852_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=11.2.0=h1d223b6_13
+  - libgfortran-ng=11.2.0=h69a702a_13
+  - libgfortran5=11.2.0=h5c6108e_13
+  - libgomp=11.2.0=h1d223b6_13
+  - liblapack=3.9.0=13_linux64_openblas
+  - libnghttp2=1.47.0=he49606f_0
+  - libnsl=2.0.0=h7f98852_0
+  - libopenblas=0.3.18=pthreads_h8fe5266_0
+  - libpng=1.6.37=h21135ba_2
+  - libssh2=1.10.0=ha35d2d1_2
+  - libstdcxx-ng=11.2.0=he4da1e4_13
+  - libtiff=4.2.0=h85742a9_0
+  - libwebp=1.2.2=h55f646e_0
+  - libwebp-base=1.2.2=h7f98852_1
+  - libzlib=1.2.11=h36c2ea0_1013
+  - libzopfli=1.0.3=h9c3ff4c_0
+  - lz4-c=1.9.3=h9c3ff4c_1
+  - ncurses=6.3=h9c3ff4c_0
+  - numpy=1.21.5=py37hf2998dd_0
+  - olefile=0.46=py37_0
+  - openjpeg=2.4.0=hb52868f_1
+  - openssl=3.0.0=h7f98852_2
+  - pillow=8.0.0=py37h9a89aac_0
+  - pip=22.0.4=pyhd8ed1ab_0
+  - python=3.7.12=hf930737_100_cpython
+  - python_abi=3.7=2_cp37m
+  - readline=8.1=h46c0cb4_0
+  - setuptools=60.9.3=py37h89c1867_0
+  - snappy=1.1.8=he1b5a44_3
+  - sqlite=3.37.0=h9cd32fc_0
+  - tabulate=0.8.9=pyhd8ed1ab_0
+  - tifffile=2021.7.2=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - tqdm=4.63.0=pyhd8ed1ab_0
+  - wheel=0.37.1=pyhd8ed1ab_0
+  - xz=5.2.5=h516909a_1
+  - zfp=0.5.5=h9c3ff4c_8
+  - zlib=1.2.11=h36c2ea0_1013
+  - zstd=1.4.9=ha95c52a_0