diff --git a/eval_mvtec_loco_all.sh b/eval_mvtec_loco_all.sh new file mode 100755 index 0000000..21e7e98 --- /dev/null +++ b/eval_mvtec_loco_all.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python}" + +MVTEC_LOCO_PATH="${MVTEC_LOCO_PATH:-../../datasets/mvtec_loco_anomaly_detection}" + +if [[ -n "${OUTPUT_DIR:-}" ]]; then + OUT="$OUTPUT_DIR" +else + OUT="output/1" + if [[ -d output ]]; then + mapfile -t out_nums < <(find output -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | grep -E '^[0-9]+$' | sort -n) + if [[ ${#out_nums[@]} -gt 0 ]]; then + last_idx=$(( ${#out_nums[@]} - 1 )) + OUT="output/${out_nums[$last_idx]}" + fi + fi +fi + +ANOMALY_MAPS_DIR="${ANOMALY_MAPS_DIR:-$OUT/anomaly_maps/mvtec_loco}" +METRICS_DIR="${METRICS_DIR:-$OUT/metrics/mvtec_loco}" + +if [[ ! -d "$MVTEC_LOCO_PATH" ]]; then + echo "ERROR: MVTEC_LOCO_PATH not found: $MVTEC_LOCO_PATH" >&2 + exit 1 +fi + +if [[ ! -d "$ANOMALY_MAPS_DIR" ]]; then + echo "ERROR: ANOMALY_MAPS_DIR not found: $ANOMALY_MAPS_DIR" >&2 + exit 1 +fi + +if [[ ! -f "mvtec_loco_ad_evaluation/evaluate_experiment.py" ]]; then + echo "ERROR: missing mvtec_loco_ad_evaluation/evaluate_experiment.py" >&2 + exit 1 +fi + +echo "Using --dataset_base_dir: $MVTEC_LOCO_PATH" +echo "Using --anomaly_maps_dir: $ANOMALY_MAPS_DIR" +echo "Using --output_dir base: $METRICS_DIR" + +mapfile -t subdatasets < <(find "$MVTEC_LOCO_PATH" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort) +if [[ ${#subdatasets[@]} -eq 0 ]]; then + echo "ERROR: no subdataset dirs found under: $MVTEC_LOCO_PATH" >&2 + exit 1 +fi + +for subdataset in "${subdatasets[@]}"; do + test_maps_dir="$ANOMALY_MAPS_DIR/$subdataset/test" + if [[ ! -d "$test_maps_dir" ]]; then + echo "SKIP: anomaly maps not found: $test_maps_dir" >&2 + continue + fi + + echo "=== Evaluating: object_name=$subdataset ===" + "$PYTHON_BIN" mvtec_loco_ad_evaluation/evaluate_experiment.py \ + --dataset_base_dir "$MVTEC_LOCO_PATH" \ + --anomaly_maps_dir "$ANOMALY_MAPS_DIR" \ + --output_dir "$METRICS_DIR/$subdataset" \ + --object_name "$subdataset" +done diff --git a/mvtec_loco_ad_evaluation/LICENSE.txt b/mvtec_loco_ad_evaluation/LICENSE.txt new file mode 100644 index 0000000..d10b7dc --- /dev/null +++ b/mvtec_loco_ad_evaluation/LICENSE.txt @@ -0,0 +1,11 @@ +Copyright 2022 MVTec Software GmbH + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mvtec_loco_ad_evaluation/README.md b/mvtec_loco_ad_evaluation/README.md new file mode 100644 index 0000000..15793f9 --- /dev/null +++ b/mvtec_loco_ad_evaluation/README.md @@ -0,0 +1,93 @@ +# Evaluate your experiments on the MVTec LOCO AD dataset + +The evaluation scripts can be used to assess the performance of a method on the MVTec Logical Costraints Anomaly Detection (MVTec LOCO AD) dataset. +Given a directory with anomaly maps, the scripts compute the area under the sPRO curve for anomaly localization. +Additionally, the area under the ROC curve for anomaly classification is computed. + +## Installation. +Our evaluation scripts require a python 3.7 installation as well as the following +packages: +- numpy +- tifffile +- pillow +- tabulate +- tqdm + +For Linux, we provide an conda environment file. It can be used to create a new conda environment with all required packages readily installed: +``` +conda env create --name mvtec_loco_eval --file=conda_environment.yml +conda activate mvtec_loco_eval +``` + +## Evaluating a single experiment. +The evaluation script requires an anomaly map to be present for each test sample in our dataset in `.tiff` format. +Anomaly maps must contain real-valued anomaly scores and their size must match the one of the corresponding dataset images. +Anomaly maps must all share the same base directory and adhere to the following folder structure: +`//test//.tiff` + +To evaluate a single experiment on one of the dataset objects, the script `evaluate_experiment.py` can be used. +It requires the following user arguments: +- `object_name`: Name of the dataset object to be evaluated. +- `dataset_base_dir`: Base directory that contains the MVTec LOCO AD dataset. +- `anomaly_maps_dir`: Base directory that contains the corresponding anomaly maps. +- `output_dir`: Directory to store evaluation results as `.json` files. + +A possible example call to this script would be: +``` +python evaluate_experiment.py \ + --object_name pushpins \ + --dataset_base_dir 'path/to/dataset/' \ + --anomaly_maps_dir 'path/to/anomaly_maps/' \ + --output_dir 'metrics/' +``` + +The evaluation script computes the area under the sPRO curve up to a limited false positive rate as described in our paper. +The integration limits are specified by the variable `MAX_FPRS`. + +## Evaluate multiple experiments + +If more than one experiment should be evaluated simultaneously, the script `evaluate_multiple_experiments.py` can be used. +Multiple directories conatining anomaly maps should be specified in a `config.json` file with the following structure: +``` +{ + "exp_base_dir": "/path/to/all/experiments/", + "anomaly_maps_dirs": { + "experiment_id_1": "eg/model_1/anomaly_maps/", + "experiment_id_2": "eg/model_2/anomaly_maps/", + "experiment_id_3": "eg/model_3/anomaly_maps/", + "...": "..." + } +} +``` +- `exp_base_dir`: Base directory that contains all experimental results for each evaluated method. +- `anomaly_maps_dirs`: Dictionary that contains an identifier for each evaluated experiment and the location of its anomaly maps relative to the `exp_base_dir`. + +The evaluation is run by calling `evaluate_multiple_experiments.py` with the following user arguments: +- `dataset_base_dir`: Base directory that contains the MVTec LOCO dataset. +- `experiment_configs`: Path to the above `config.json` that contains all experiments to be evaluated. +- `output_dir`: Directory to store evaluation results as `.json` files. + +A possible example call to this script would be: +``` + python evaluate_multiple_experiments.py \ + --dataset_base_dir 'path/to/dataset/' \ + --experiment_configs 'configs.json' \ + --output_dir 'metrics/' +``` + +## Visualize the evaluation results. +After running `evaluate_experiment.py` or `evaluate_multiple_experiments.py`, the script `print_metrics.py` can be used to visualize all computed metrics in a table. +In total, three tables are printed to the standard output. The first two tables display the performance for the structural and logical anomalies, respectively. +The third table shows the mean performance over both anomaly types. + +The script requires the following user arguments: +- `metrics_folder`: The base directory that contains the computed metrics for each evaluated method. This directory is usually identical to the output directory specified in `evaluate_experiment.py` or `evaluate_multiple_experiments.py`. +- `metric_type`: Select either `localization` or `classification`. When selecting `localization`, +the AUC-sPRO results for the pixelwise localization of anomalies is shown. When selecting `classification`, the image level AUC-ROC for anomaly classification is shown. +- `integration_limit`: The integration limit until which the area under the sPRO curve is computed. This parameter is only applicable when `metric_type` is set to `localization`. + +# License +The license agreement for our evaluation code is found in the accompanying +`LICENSE.txt` file. + +The version of this evaluation script is: 2.0 diff --git a/mvtec_loco_ad_evaluation/conda_environment.yml b/mvtec_loco_ad_evaluation/conda_environment.yml new file mode 100644 index 0000000..0a176aa --- /dev/null +++ b/mvtec_loco_ad_evaluation/conda_environment.yml @@ -0,0 +1,78 @@ +name: mvtec_loco_eval +channels: + - anaconda + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=1_gnu + - blosc=1.21.0=h9c3ff4c_0 + - brotli=1.0.9=h7f98852_6 + - brotli-bin=1.0.9=h7f98852_6 + - brunsli=0.1=h9c3ff4c_0 + - bzip2=1.0.8=h7f98852_4 + - c-ares=1.18.1=h7f98852_0 + - ca-certificates=2021.10.8=ha878542_0 + - cfitsio=3.470=hb418390_7 + - charls=2.2.0=h2531618_0 + - colorama=0.4.4=pyh9f0ad1d_0 + - freetype=2.10.4=h5ab3b9f_0 + - giflib=5.2.1=h36c2ea0_2 + - imagecodecs=2021.8.26=py37h4cda21f_0 + - jpeg=9e=h7f98852_0 + - jxrlib=1.1=h7f98852_2 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.19.2=h08a2579_4 + - lcms2=2.12=hddcbb42_0 + - ld_impl_linux-64=2.36.1=hea4e1c9_2 + - lerc=3.0=h9c3ff4c_0 + - libaec=1.0.6=h9c3ff4c_0 + - libblas=3.9.0=13_linux64_openblas + - libbrotlicommon=1.0.9=h7f98852_6 + - libbrotlidec=1.0.9=h7f98852_6 + - libbrotlienc=1.0.9=h7f98852_6 + - libcblas=3.9.0=13_linux64_openblas + - libcurl=7.81.0=h494985f_0 + - libdeflate=1.8=h7f98852_0 + - libedit=3.1.20191231=he28a2e2_2 + - libev=4.33=h516909a_1 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=11.2.0=h1d223b6_13 + - libgfortran-ng=11.2.0=h69a702a_13 + - libgfortran5=11.2.0=h5c6108e_13 + - libgomp=11.2.0=h1d223b6_13 + - liblapack=3.9.0=13_linux64_openblas + - libnghttp2=1.47.0=he49606f_0 + - libnsl=2.0.0=h7f98852_0 + - libopenblas=0.3.18=pthreads_h8fe5266_0 + - libpng=1.6.37=h21135ba_2 + - libssh2=1.10.0=ha35d2d1_2 + - libstdcxx-ng=11.2.0=he4da1e4_13 + - libtiff=4.2.0=h85742a9_0 + - libwebp=1.2.2=h55f646e_0 + - libwebp-base=1.2.2=h7f98852_1 + - libzlib=1.2.11=h36c2ea0_1013 + - libzopfli=1.0.3=h9c3ff4c_0 + - lz4-c=1.9.3=h9c3ff4c_1 + - ncurses=6.3=h9c3ff4c_0 + - numpy=1.21.5=py37hf2998dd_0 + - olefile=0.46=py37_0 + - openjpeg=2.4.0=hb52868f_1 + - openssl=3.0.0=h7f98852_2 + - pillow=8.0.0=py37h9a89aac_0 + - pip=22.0.4=pyhd8ed1ab_0 + - python=3.7.12=hf930737_100_cpython + - python_abi=3.7=2_cp37m + - readline=8.1=h46c0cb4_0 + - setuptools=60.9.3=py37h89c1867_0 + - snappy=1.1.8=he1b5a44_3 + - sqlite=3.37.0=h9cd32fc_0 + - tabulate=0.8.9=pyhd8ed1ab_0 + - tifffile=2021.7.2=pyhd8ed1ab_0 + - tk=8.6.12=h27826a3_0 + - tqdm=4.63.0=pyhd8ed1ab_0 + - wheel=0.37.1=pyhd8ed1ab_0 + - xz=5.2.5=h516909a_1 + - zfp=0.5.5=h9c3ff4c_8 + - zlib=1.2.11=h36c2ea0_1013 + - zstd=1.4.9=ha95c52a_0 diff --git a/mvtec_loco_ad_evaluation/evaluate_experiment.py b/mvtec_loco_ad_evaluation/evaluate_experiment.py new file mode 100644 index 0000000..e9307b4 --- /dev/null +++ b/mvtec_loco_ad_evaluation/evaluate_experiment.py @@ -0,0 +1,484 @@ +""" +Evaluate a single experiment on a single object of the MVTec LOCO AD dataset. + +For more information, see: python main.py --help or have a look at the README. +""" + +__author__ = "Kilian Batzner, Paul Bergmann, Michael Fauser, David Sattlegger" +__copyright__ = "2022, MVTec Software GmbH" + +import argparse +import glob +import json +import os +from typing import Optional, Iterable + +import numpy as np +from tqdm import tqdm + +from src.aggregation import MetricsAggregator, ThresholdMetrics +from src.image import GroundTruthMap, AnomalyMap, DefectsConfig +from src.util import get_auc_for_max_fpr, listdir, set_niceness, \ + compute_classification_auc_roc + +TIFF_EXTS = ['.tif', '.tiff', '.TIF', '.TIFF'] +OBJECT_NAMES = ['breakfast_box', 'juice_bottle', 'pushpins', 'screw_bag', + 'splicing_connectors'] + +# The AU-sPRO is only computed up to a certain integration limit. Here, you +# can specify the integration limits that should be evaluated. +MAX_FPRS = [0.01, 0.05, 0.1, 0.3, 1.] + + +def parse_arguments(): + """ + Parse user arguments for the evaluation of a method on the MVTec LOCO AD + dataset. + + returns: + Parsed user arguments. + """ + parser = argparse.ArgumentParser() + + parser.add_argument( + '--object_name', + choices=OBJECT_NAMES, + help='Name of the dataset object to be evaluated.') + + parser.add_argument( + '--dataset_base_dir', + help='Path to the directory that contains the dataset images of the' + ' MVTec LOCO AD dataset.') + + parser.add_argument( + '--anomaly_maps_dir', + required=True, + help='Path to the anomaly maps directory of the evaluated method.') + + parser.add_argument( + '--output_dir', + default=None, + help='Path to the directory to store evaluation results. If no output' + ' directory is specified, the results are not written to drive.') + + parser.add_argument( + '--num_parallel_workers', + default=None, + type=optional_int, + help='If None (default), nothing will be parallelized across CPUs.' + ' Otherwise, the value denotes the number of CPUs to use for' + ' parallelism. A value of 1 will result in suboptimal performance' + ' compared to None.') + + parser.add_argument( + '--curve_max_distance', + default=0.001, + type=float, + help='Maximum distance between two points on the overall FPR-sPRO' + ' curve. Will be used for selecting anomaly thresholds.' + ' Decrease this value to increase the overall accuracy of' + ' results.') + + parser.add_argument( + '--niceness', + type=int, + default=19, + choices=list(range(20)), + help='UNIX niceness of all evaluation processes.') + + args = parser.parse_args() + + return args + + +def main(): + args = parse_arguments() + + set_niceness(args.niceness) + + # Read the defects config file of the evaluated object. + defects_config_path = os.path.join( + args.dataset_base_dir, args.object_name, 'defects_config.json') + with open(defects_config_path) as defects_config_file: + defects_list = json.load(defects_config_file) + defects_config = DefectsConfig.create_from_list(defects_list) + + # Read the ground truth maps and the anomaly maps. + gt_dir = os.path.join( + args.dataset_base_dir, args.object_name, 'ground_truth') + anomaly_maps_test_dir = os.path.join( + args.anomaly_maps_dir, args.object_name, 'test') + gt_maps, anomaly_maps = read_maps( + gt_dir=gt_dir, + anomaly_maps_test_dir=anomaly_maps_test_dir, + defects_config=defects_config) + + # Collect relevant metrics based on the ground truth and anomaly maps. + metrics_aggregator = MetricsAggregator( + gt_maps=gt_maps, + anomaly_maps=anomaly_maps, + parallel_workers=args.num_parallel_workers, + parallel_niceness=args.niceness) + metrics = metrics_aggregator.run( + curve_max_distance=args.curve_max_distance) + + # Fetch the anomaly localization results. + localization_results = get_auc_spro_results( + metrics=metrics, + anomaly_maps_test_dir=anomaly_maps_test_dir) + + # Fetch the image-level anomaly detection results. + classification_results = get_image_level_detection_metrics( + gt_maps=gt_maps, + anomaly_maps=anomaly_maps) + + # Create the dict to write to metrics.json. + results = { + 'localization': localization_results, + 'classification': classification_results + } + + # Write the results to the output directory. + if args.output_dir is not None: + print(f'Writing results to {args.output_dir}') + os.makedirs(args.output_dir, exist_ok=True) + results_path = os.path.join(args.output_dir, 'metrics.json') + with open(results_path, 'w') as results_file: + json.dump(results, results_file, indent=4, sort_keys=True) + + +def read_maps(gt_dir: str, + anomaly_maps_test_dir: str, + defects_config: DefectsConfig): + """Read the ground truth and the anomaly maps.""" + print('Reading ground truth and corresponding anomaly maps...') + gt_maps = [] + anomaly_maps = [] + + # Search for available relative paths to ground truth maps and to + # anomaly maps. + gt_map_rel_paths = set(get_available_gt_map_rel_paths(gt_dir)) + anomaly_rel_paths = list( + get_available_test_image_rel_paths(anomaly_maps_test_dir)) + anomaly_rel_paths_no_ext = [os.path.splitext(p)[0] + for p in anomaly_rel_paths] + # Check that there are no duplicates with different file endings. + assert len(set(anomaly_rel_paths_no_ext)) == len(anomaly_rel_paths_no_ext) + + # Every ground truth image must have an anomaly image. + skipped_gt_maps = gt_map_rel_paths.difference(anomaly_rel_paths_no_ext) + if len(skipped_gt_maps) > 0: + raise OSError( + 'These ground truth maps do not have corresponding anomaly' + f' maps: {sorted(skipped_gt_maps)}') + + # For every relative path, read the ground truth (if available) and the + # anomaly map. + for rel_path, rel_path_no_ext in tqdm( + zip(anomaly_rel_paths, anomaly_rel_paths_no_ext), + total=len(anomaly_rel_paths)): + anomaly_map_path = os.path.join(anomaly_maps_test_dir, rel_path) + anomaly_map = AnomalyMap.read_from_tiff(anomaly_map_path) + anomaly_maps.append(anomaly_map) + + if rel_path_no_ext in gt_map_rel_paths: + gt_map_path = os.path.join(gt_dir, rel_path_no_ext) + gt_map = GroundTruthMap.read_from_png_dir( + png_dir=gt_map_path, + defects_config=defects_config) + gt_maps.append(gt_map) + else: + # This must be a good image. Hence, the path must start with + # "good/". + if not rel_path.startswith('good/'): + raise OSError( + f'Anomaly map {rel_path} has no corresponding ground' + ' truth map, so it must be a good image. Good images' + ' must have a relative path starting with "good/" ' + f' (relative to the test dir at {anomaly_maps_test_dir})' + ) + gt_maps.append(None) + + return gt_maps, anomaly_maps + + +def get_available_gt_map_rel_paths(gt_dir: str) -> Iterable[str]: + """Search for available relative paths to ground truth maps. + + Note that ground truth maps are represented as a directory containing + .png files (one for each channel). + + The returned paths are the relative paths to the directories. + """ + for defect_type_name in listdir(gt_dir): + file_complaint_str = ('Ground truth directory must not contain files' + ' except for the .pngs nested in the single' + ' image directories.') + + # Get the logical_anomalies and structural_anomalies subdirectories. + defect_type_dir = os.path.join(gt_dir, defect_type_name) + if not os.path.isdir(defect_type_dir): + raise OSError(file_complaint_str) + + # Raise if the directory name is not what we expect. + valid_defect_type_names = ['logical_anomalies', 'structural_anomalies'] + if defect_type_name not in valid_defect_type_names: + raise OSError( + f'Subdirectory of ground truth maps' + f' directory has name {defect_type_name}, but should be' + f' "logical_anomalies" or "structural_anomalies".') + + # Get the list of all non-empty image directories (000, 001, etc.). + for image_dir_name in listdir(defect_type_dir): + image_dir_path = os.path.join(defect_type_dir, image_dir_name) + # Image dir must be a dir. + if not os.path.isdir(image_dir_path): + raise OSError(file_complaint_str) + # Image dir must contain pngs. + if len(glob.glob(os.path.join(image_dir_path, '*.png'))) == 0: + raise OSError( + f'Ground truth directory of single image' + f' at {image_dir_path} does not contain any pngs.') + + # Yield the relative paths to the image subdirectory. + yield os.path.join(defect_type_name, image_dir_name) + + +def get_available_test_image_rel_paths(test_images_dir: str) -> Iterable[str]: + """Search for available relative paths to test anomaly maps.""" + + for defect_dir_name in listdir(test_images_dir): + defect_dir = os.path.join(test_images_dir, defect_dir_name) + if not os.path.isdir(defect_dir): + continue + + for file_name in listdir(defect_dir): + _, ext = os.path.splitext(file_name) + if ext not in TIFF_EXTS: + continue + yield os.path.join(defect_dir_name, file_name) + + +def get_auc_spro_results(metrics: ThresholdMetrics, + anomaly_maps_test_dir: str): + """Compute AUC sPRO values for all images, images in subdirectories and + defect names. + """ + # Compute the AUC sPRO for logical and structural anomalies. + auc_spro = get_auc_spros_per_subdir( + metrics=metrics, + anomaly_maps_test_dir=anomaly_maps_test_dir, + add_good_images=True) + + # Compute the mean performance over logical and structural anomalies. + mean_spros = dict() + for limit in auc_spro['structural_anomalies'].keys(): + auc_spro_structural = auc_spro['structural_anomalies'][limit] + auc_spro_logical = auc_spro['logical_anomalies'][limit] + mean = 0.5 * (auc_spro_structural + auc_spro_logical) + mean_spros[limit] = mean + auc_spro['mean'] = mean_spros + + return {'auc_spro': auc_spro} + + +def get_auc_spros_per_defect_type(metrics: ThresholdMetrics, + defects_config: DefectsConfig, + add_good_images): + """Compute the AUC sPRO for images by defect type. + + For each defect type, all ground truth maps that contain at least one + channel with this type will be considered. For the FPR computation, all + channels of each image will be used. For the sPRO computation, however, + only channels with the respective defect type will be used. + + If add_good_images is True, all good images in `metrics` will be added for + the computation of each defect type's AUC sPRO value. + """ + aucs_per_defect_type = {} + defect_configs = defects_config.pixel_value_to_entry.values() + defect_names = set(c.defect_name for c in defect_configs) + + for defect_name in defect_names: + # Collect all anomaly maps, where the corresponding ground truth + # maps has at least one channel with the defect type corresponding to + # defect_name. + reduced_anomaly_maps = [] + for anomaly_map, gt_map in zip(metrics.anomaly_maps, metrics.gt_maps): + # Optionally add good images as well. + if gt_map is None: + if add_good_images: + reduced_anomaly_maps.append(anomaly_map) + continue + + # Get the defect types in the ground truth map. + defect_names_in_gt_map = \ + set(c.defect_config.defect_name for c in gt_map.channels) + + if defect_name in defect_names_in_gt_map: + reduced_anomaly_maps.append(anomaly_map) + + # Reduce the threshold metrics and compute the AUC sPRO value. + reduced_metrics = metrics.reduce_to_images(reduced_anomaly_maps) + aucs_per_defect_type[defect_name] = get_auc_spros_for_metrics( + metrics=reduced_metrics, + filter_defect_names_for_spro=[defect_name]) + return aucs_per_defect_type + + +def get_auc_spros_per_subdir(metrics: ThresholdMetrics, + anomaly_maps_test_dir, + add_good_images): + """Compute the AUC sPRO for images in subdirectories (usually "good", + "structural_anomalies" and "logical_anomalies"). + + If add_good_images is True, the images in the "good" subdirectory will be + added to the images of each subdirectory for computing the corresponding + AUC sPRO value. Hence, the result dict will not contain a "good" key. + """ + aucs_per_subdir = {} + subdir_names = listdir(anomaly_maps_test_dir) + + good_images = [] + if add_good_images: + # Include the good images for each subdir. + assert 'good' in subdir_names + good_subdir = os.path.join(anomaly_maps_test_dir, 'good') + good_subdir = os.path.realpath(good_subdir) + good_images = [ + a for a in metrics.anomaly_maps + if os.path.realpath(a.file_path).startswith(good_subdir)] + # Regardless of add_good_images, we cannot compute an AUC sPRO value only + # for the good images. + if 'good' in subdir_names: + subdir_names.remove('good') + + for subdir_name in subdir_names: + subdir = os.path.join(anomaly_maps_test_dir, subdir_name) + subdir = os.path.realpath(subdir) + # Get all anomaly maps in here. + subdir_anomaly_maps = [ + a for a in metrics.anomaly_maps + if os.path.realpath(a.file_path).startswith(subdir)] + if add_good_images: + subdir_anomaly_maps += good_images + + subdir_metrics = metrics.reduce_to_images(subdir_anomaly_maps) + + aucs_per_subdir[subdir_name] = get_auc_spros_for_metrics( + subdir_metrics) + return aucs_per_subdir + + +def get_auc_spros_for_metrics( + metrics: ThresholdMetrics, + filter_defect_names_for_spro: Optional[Iterable[str]] = None): + """Compute AUC sPRO values for a given ThresholdMetrics instance. + + Args: + metrics: The ThresholdMetrics instance. + filter_defect_names_for_spro: If not None, only the sPRO values from + defect names in this sequence will be used. Does not affect the + computation of FPRs! + """ + auc_spros = {} + for max_fpr in MAX_FPRS: + try: + fp_rates = metrics.get_fp_rates() + except ZeroDivisionError: + auc = None + else: + mean_spros = metrics.get_mean_spros( + filter_defect_names=filter_defect_names_for_spro) + auc = get_auc_for_max_fpr(fprs=fp_rates, + y_values=mean_spros, + max_fpr=max_fpr, + scale_to_one=True) + auc_spros[max_fpr] = auc + return auc_spros + + +def get_image_level_detection_metrics(gt_maps, anomaly_maps): + """Main function for computing all image-level anomaly detection results.""" + per_image_results = get_image_level_detection_metrics_per_image( + gt_maps, anomaly_maps) + auc_roc_classification = get_image_level_detection_metrics_aggregated( + per_image_results) + return auc_roc_classification + + +def get_image_level_detection_metrics_per_image(gt_maps, anomaly_maps): + """Computes the image-level anomaly scores for an anomaly map. + """ + per_image_results = [] + for gt_map, anomaly_map in zip(gt_maps, anomaly_maps): + # Determine the image type (good, logical, or structural). + parent_dir_path, _ = os.path.split(anomaly_map.file_path) + _, image_type = os.path.split(parent_dir_path) + + image_results = { + 'file_path': anomaly_map.file_path, + 'gt_contains_defect': gt_map is not None, + 'anomaly_scores': {}, + 'image_type': image_type + } + # Compute the image-level anomaly score by taking the maximum of the + # anomaly map. + image_level_score = float(np.max(anomaly_map.np_array)) + image_results['anomaly_scores']['max'] = image_level_score + per_image_results.append(image_results) + + return per_image_results + + +def get_image_level_detection_metrics_aggregated(per_image_results): + """Compute aggregated image-level anomaly detection metrics like the AUC-ROC + + Args: + per_image_results: should be the result of + get_image_level_detection_metrics_per_image + """ + agg_results = {'auc_roc': {}} + # Iterate through the different image-level score computation methods. + + # Collect image-level scores of good and anomalous images. + anomaly_scores = \ + {'good': [], 'structural_anomalies': [], 'logical_anomalies': []} + + # Iterate through the images and collect the information required for + # computing the AUC-ROC. + for image_result in per_image_results: + anomaly_scores[image_result['image_type']].append( + image_result['anomaly_scores']['max']) + + # Compute auc-roc for structural and logical anomalies separately. + auc_roc_structural = \ + compute_classification_auc_roc( + anomaly_scores['good'], anomaly_scores['structural_anomalies']) + + auc_roc_logical = \ + compute_classification_auc_roc( + anomaly_scores['good'], anomaly_scores['logical_anomalies']) + + agg_results['auc_roc']['logical_anomalies'] = auc_roc_logical + agg_results['auc_roc']['structural_anomalies'] = auc_roc_structural + agg_results['auc_roc']['mean'] = \ + 0.5 * (auc_roc_logical + auc_roc_structural) + return agg_results + + +def optional_int(str_value: Optional[str]) -> Optional[int]: + """Helper function for parsing optional integer arguments with argparse.""" + if str_value is None or str_value.lower() == 'none': + return None + else: + try: + return int(str_value) + except ValueError: + raise argparse.ArgumentTypeError( + 'An optional integer argument was given the value' + f' {str_value}, but the value must be None or an integer.') + + +if __name__ == '__main__': + main() diff --git a/mvtec_loco_ad_evaluation/evaluate_multiple_experiments.py b/mvtec_loco_ad_evaluation/evaluate_multiple_experiments.py new file mode 100644 index 0000000..9d35aa1 --- /dev/null +++ b/mvtec_loco_ad_evaluation/evaluate_multiple_experiments.py @@ -0,0 +1,110 @@ +""" +Run the evaluation script for multiple experiments. + +This is a wrapper around evaluate_experiment.py which is called for each +experiment specified in the config file passed to this script. + +Run `print_metrics.py` afterwards in order to print the results. + +For usage, see: python main.py --help or have a look at the README file. +""" + +__author__ = "Kilian Batzner, Paul Bergmann, Michael Fauser, David Sattlegger" +__copyright__ = "2022, MVTec Software GmbH" + +import argparse +import json +import subprocess +from os import path + +from evaluate_experiment import OBJECT_NAMES + + +def parse_user_arguments(): + """ + Parse user arguments. + + Returns: Parsed arguments. + """ + parser = argparse.ArgumentParser(description="""Parse user arguments.""") + + parser.add_argument( + '--experiment_configs', + default='experiment_configs.json', + help='Path to the config file that contains the locations of all' + ' experiments that should be evaluated.') + + parser.add_argument( + '--dataset_base_dir', + required=True, + help='Path to the directory that contains the dataset images of the' + ' MVTec LOCO dataset.') + + parser.add_argument( + '--output_dir', + default='./metrics/', + help='Path to write evaluation results to.') + + parser.add_argument( + '--curve_max_distance', + default=0.001, + type=float, + help='Maximum distance between two points on the overall FPR-sPRO' + ' curve. Will be used for selecting anomaly thresholds.' + ' Decrease this value to increase the overall accuracy of' + ' results.') + + parser.add_argument( + '--dry_run', + choices=['True', 'False'], + default='False', + help='If set to \'True\', the script is run without' + ' perfoming the actual evalutions. Instead, the' + ' experiments to be evaluated are simply printed' + ' to the standard output.') + + return parser.parse_args() + + +def main(): + """ + Run the evaluation script for multiple experiments. + """ + # Parse user arguments. + args = parse_user_arguments() + + # Read the experiment configurations to be evaluated. + with open(args.experiment_configs) as file: + experiment_configs = json.load(file) + + # Call the evaluation script for each experiment separately. + for experiment_id in experiment_configs['anomaly_maps_dirs']: + print(f"\n=== Evaluate experiment: {experiment_id} ===") + for object_name in OBJECT_NAMES: + print(f"\nEvaluate object: {object_name}") + + # Anomaly maps for this experiment are located in this directory. + anomaly_maps_dir = path.join( + experiment_configs['exp_base_dir'], + experiment_configs['anomaly_maps_dirs'][experiment_id]) + + output_dir = path.join(args.output_dir, experiment_id, object_name) + + # Set up python call for the evaluation script. + call = ['python', 'evaluate_experiment.py', + '--object_name', object_name, + '--dataset_base_dir', args.dataset_base_dir, + '--anomaly_maps_dir', anomaly_maps_dir, + '--output_dir', output_dir, + '--num_parallel_workers', str(4), + '--curve_max_distance', str(args.curve_max_distance), + '--niceness', str(19)] + + if args.dry_run == 'True': + print(f"Would call: {call}\n") + else: + subprocess.run(call, check=True) + + +if __name__ == '__main__': + main() diff --git a/mvtec_loco_ad_evaluation/experiment_configs.json b/mvtec_loco_ad_evaluation/experiment_configs.json new file mode 100644 index 0000000..3beb064 --- /dev/null +++ b/mvtec_loco_ad_evaluation/experiment_configs.json @@ -0,0 +1,8 @@ +{ + "exp_base_dir": "/path/to/all/experiments/", + "anomaly_maps_dirs": { + "experiment_id_1": "method_1/anomaly_maps/", + "experiment_id_2": "method_2/anomaly_maps/", + "experiment_id_3": "method_3/anomaly_maps/" + } +} \ No newline at end of file diff --git a/mvtec_loco_ad_evaluation/print_metrics.py b/mvtec_loco_ad_evaluation/print_metrics.py new file mode 100644 index 0000000..9bd6e22 --- /dev/null +++ b/mvtec_loco_ad_evaluation/print_metrics.py @@ -0,0 +1,174 @@ +""" +Print the key metrics of multiple experiments to the standard output. + +For more information, see: python main.py --help or have a look at the README. +""" + +__author__ = "Kilian Batzner, Paul Bergmann, Michael Fauser, David Sattlegger" +__copyright__ = "2022, MVTec Software GmbH" + +import argparse +import json +import os +from os.path import join + +import numpy as np +from tabulate import tabulate + +from evaluate_experiment import OBJECT_NAMES + + +def parse_user_arguments(): + """ + Parse user arguments. + + Returns: Parsed arguments. + """ + parser = argparse.ArgumentParser(description="""Parse user arguments.""") + + parser.add_argument( + '--metrics_folder', + default="./metrics/", + help='Path to the folder that contains the evaluation results.') + + parser.add_argument( + '--metric_type', + default='localization', + choices=['localization', 'classification'], + help='Print results for either anomaly localization or' + ' classification.') + + parser.add_argument( + '--integration_limit', + default='0.05', + help='For anomaly localization, report results at this integration' + ' limit. Note that the value specified here also needs to be' + ' present in the respective \'metrics.json\' file. Therefore,' + ' it also needs to be specified as an integration limit in ' + ' \'evaluate_experiment.py\' during evaluation.') + + return parser.parse_args() + + +def extract_table_rows(metrics_folder, metric_type, integration_limit, mode): + """ + Extract all rows to create a table that displays a given metric for each + evaluated experiment. + + Args: + metrics_folder: Base folder that contains evaluation results. + metric_type: Type of the metric to be extracted. Choose between + 'localization' and 'classification'. + integration_limit: For anomaly localization, fetch results for a certain + integration limit. + mode: Fetch metrics for logical anomalies, structural + anomalies, or the mean of both values. + + Returns: + List of table rows. Each row contains the experiment name and the + extracted metrics for each evaluated object as well as the mean + performance. + """ + assert metric_type in ['localization', 'classification'] + assert mode in ['logical_anomalies', 'structural_anomalies', 'mean'] + + # check whether metrics_folder is already an experiment, e.g. the + # output_dir specified in evaluate_experiment.py + is_experiment_folder = False + + for obj in OBJECT_NAMES: + if os.path.exists(join(metrics_folder, obj)): + is_experiment_folder = True + break + + # Iterate each evaluated experiment. + if not is_experiment_folder: + exp_ids = os.listdir(metrics_folder) + else: + exp_ids = [os.path.split(metrics_folder)[-1]] + + rows = [] + for exp_id in exp_ids: + + # Each row starts with the name of the experiment. + row = [exp_id] + metric_values = [] + + # Fetch metrics for each dataset object. + skipped = False + for obj in OBJECT_NAMES: + + # Open the metrics file. + if not is_experiment_folder: + metrics_file = \ + join(metrics_folder, exp_id, obj, 'metrics.json') + else: + metrics_file = \ + join(metrics_folder, obj, 'metrics.json') + + # Skip this object if no numbers exist for it. + if not os.path.exists(metrics_file): + print(f"Warning: {metrics_file} does not exist.") + row.append('-') + skipped = True + continue + + with open(metrics_file) as file: + metrics = json.load(file) + + # Fetch the desired metric and store it in the table row. + if metric_type == 'localization': + metric = metrics[metric_type]['auc_spro'][mode][integration_limit] + else: + metric = metrics[metric_type]['auc_roc'][mode] + metric_values.append(metric) + row.append(np.round(metric, decimals=3)) + + # Compute mean performance if no object had to be skipped. + if not skipped: + mean_performance = np.mean(metric_values) + row.append(np.round(mean_performance, decimals=3)) + else: + row.append('-') + + rows.append(row) + + # only one experiment + if is_experiment_folder: + break + + return rows + + +def main(): + """ + Print the key metrics of multiple experiments to the standard output. + """ + # Parse user arguments. + args = parse_user_arguments() + + for mode in ['logical_anomalies', 'structural_anomalies', 'mean']: + + # Create the table rows. One row for each experiment. + rows_pro_structural = extract_table_rows( + metrics_folder=args.metrics_folder, + metric_type=args.metric_type, + integration_limit=args.integration_limit, + mode=mode) + + # Print localization result table. + + if args.metric_type == 'localization': + info_string = f"\nAUC sPRO ({args.metric_type} -- {mode})" + info_string += f" Integration limit: {args.integration_limit}" + else: + info_string = f"\nAUC ROC ({args.metric_type} -- {mode})" + print(info_string) + + print(tabulate(rows_pro_structural, + headers=['Experiment'] + OBJECT_NAMES + ['Mean'], + tablefmt='fancy_grid')) + + +if __name__ == "__main__": + main() diff --git a/mvtec_loco_ad_evaluation/src/aggregation.py b/mvtec_loco_ad_evaluation/src/aggregation.py new file mode 100644 index 0000000..5bfe61a --- /dev/null +++ b/mvtec_loco_ad_evaluation/src/aggregation.py @@ -0,0 +1,538 @@ +"""Wrapper around metrics.py for dynamically refining thresholds. + +The important class for users of this module is MetricsAggregator. +""" + +from typing import Sequence, Optional, Callable, Iterable + +import numpy as np + +from src.image import GroundTruthMap, AnomalyMap, get_file_path_repr +from src.metrics import get_fp_tn_areas_per_image, get_fp_rates +from src.metrics import get_spros_of_defects_of_images +from src.util import get_auc_for_max_fpr, take, flatten_2d +from src.util import get_sorted_nested_arrays, concat_nested_arrays + + +def binary_refinement(init_queries: Sequence, + init_values: Sequence, + max_distance: float, + get_values: Callable[[Sequence], Sequence], + min_queries_per_step: int, + max_queries_per_step: int, + max_steps: int): + """Refine queries to maximize the resolution of a list of values. + + At each step, a query will be added between adjacent queries if the + distance between the corresponding values is larger than max_distance. + If there are still queries needed to reach min_queries_per_step, multiple + queries will be inserted into intervals that have a large distance. + + Args: + init_queries: The initial *sorted* queries. May be sorted in ascending + or descending order. Can be of any type, but must support + np.linspace(q1, q2, ...) for queries q1 and q2 and must support + comparisons between q1 and q2. Must not contain duplicate queries. + init_values: The initial values. Can be of any type, but + np.linalg.norm(v1 - v2) must be a scalar for values v1 and v2. + max_distance: The refinement will stop when all distances between + adjacent values are smaller or equal to max_distance. + get_values: A function that returns a list of values for a list of + queries. This function will be used for refinement. + min_queries_per_step: get_values will be called with at least this + many queries per step. + max_queries_per_step: get_values will be called with at most this + many queries per step. + max_steps: The maximum number of calls of get_values(). + + Returns: + A list with the final refined queries. + A list with the final refined values. + """ + # Input validation. + assert len(init_queries) == len(init_values) + assert len(set(init_queries)) == len(init_queries) + pairwise_less = [init_queries[i] < init_queries[i + 1] + for i in range(len(init_queries) - 1)] + if all(pairwise_less): + ascending = True + elif not any(pairwise_less): + ascending = False + else: + raise AssertionError + + # Get all adjacent queries, whose corresponding values are further apart + # than max_distance. As a secondary condition, the maximum tolerated area + # between two points is derived from max_distance. The area is defined by + # the triangle with side lengths delta_x and delta_y. Points with an area + # smaller than the tolerated area are not considered as candidates for + # further refinement of the curve. + candidates = [] + max_area = 0.25 * max_distance**2 + for i in range(len(init_queries) - 1): + distance = np.linalg.norm(init_values[i] - init_values[i + 1]) + + # the area of the triangle defined by delta_x and delta_y + area = 0.5 * np.prod(init_values[i] - init_values[i + 1]) + + if distance <= max_distance or area <= max_area: + continue + query_left, query_right = init_queries[i], init_queries[i + 1] + candidates.append((distance, (query_left, query_right))) + if len(candidates) == 0 or max_steps < 1: + return init_queries, init_values + + # Sort the candidate query intervals by distance. + candidates = sorted(candidates, key=lambda c: c[0], reverse=True) + print(f'Max Distance between points: {candidates[0][0]}') + candidates = candidates[:max_queries_per_step] + + # Distribute the remaining queries to reach min_queries_per_step. + # Distribute them weighted by each candidate's relative distance. + num_remaining = max(min_queries_per_step - len(candidates), 0) + total_distance_remaining = sum(distance for distance, _ in candidates) + queries = [] + for distance, (query_left, query_right) in candidates: + additional = distance / total_distance_remaining * num_remaining + additional = int(np.round(additional)) + num_interval_queries = 1 + additional + interval_queries = np.linspace(query_left, query_right, + num=2 + num_interval_queries)[1:-1] + queries.extend(interval_queries) + num_remaining -= additional + total_distance_remaining -= distance + + # Query the function. + queried_values = get_values(queries) + + # Merge the new queries and values with the old queries and values. + all_queries = list(init_queries) + list(queries) + all_values = list(init_values) + list(queried_values) + sort_indices = np.argsort(all_queries) + all_queries = [all_queries[i] for i in sort_indices] + all_values = [all_values[i] for i in sort_indices] + + if not ascending: + all_queries = list(reversed(all_queries)) + all_values = list(reversed(all_values)) + + # Start a new refinement step. + return binary_refinement(init_queries=all_queries, + init_values=all_values, + max_distance=max_distance, + get_values=get_values, + min_queries_per_step=min_queries_per_step, + max_queries_per_step=max_queries_per_step, + max_steps=max_steps - 1) + + +class ThresholdMetrics: + """Collection of metrics obtained for a list of anomaly thresholds and + images. + """ + + def __init__(self, + gt_maps: Sequence[Optional[GroundTruthMap]], + anomaly_maps: Sequence[AnomalyMap], + anomaly_thresholds: np.ndarray, + spros_of_defects_of_images: Sequence[Sequence[np.ndarray]], + fp_areas_per_image: Sequence[np.array], + tn_areas_per_image: Sequence[np.array]): + """ + Args: + gt_maps: Sequence of GroundTruthMap or None entries with the + same length and ordering as anomaly_maps. Use None for "good" + images without ground truth annotations. + anomaly_maps: Must have the same length and ordering as + gt_maps. + anomaly_thresholds: Thresholds for obtaining binary anomaly maps. + Must be sorted in descending order so that the first + threshold corresponds to a sPRO of 0 and an FPR of 0! + spros_of_defects_of_images: See + metrics.get_spros_of_defects_of_images(...). + fp_areas_per_image: See metrics.get_fp_tn_areas_per_image(...) + tn_areas_per_image: See metrics.get_fp_tn_areas_per_image(...) + """ + + # Input validation. + assert len({len(gt_maps), + len(anomaly_maps), + len(spros_of_defects_of_images), + len(fp_areas_per_image), + len(tn_areas_per_image)}) == 1 + + # Thresholds must be sorted in descending order. + assert np.array_equal(np.sort(anomaly_thresholds)[::-1], + anomaly_thresholds) + + for per_threshold_array in (flatten_2d(spros_of_defects_of_images) + + list(fp_areas_per_image) + + list(tn_areas_per_image)): + assert isinstance(per_threshold_array, np.ndarray) + assert len(per_threshold_array) == len(anomaly_thresholds) + + self.gt_maps = gt_maps + self.anomaly_maps = anomaly_maps + self.anomaly_thresholds = anomaly_thresholds + self.spros_of_defects_of_images = spros_of_defects_of_images + self.fp_areas_per_image = fp_areas_per_image + self.tn_areas_per_image = tn_areas_per_image + + def merge_with(self, other: 'ThresholdMetrics'): + """Merge this collection of metrics with another one. + + The thresholds of both instances may differ, but they must have been + computed on the same collection of ground truth and anomaly maps. + Note that these image classes do not implement __eq__ at the moment, + so while the lists themselves (gt_maps and anomaly_maps) may be + different objects, the image objects must be identical for self and + other. + + Returns: + The merged ThresholdMetrics instance. The input instances will + not be modified. The anomaly thresholds will be sorted such + that the first threshold corresponds to a sPRO of 0 and an FPR + of 0 and the last threshold to a sPRO of 1 and an FPR of 1. The + entries in the metrics attributes will be ordered accordingly. + """ + + # Thresholds may differ, but must have been computed on the same + # collection of ground truth and anomaly maps + assert self.gt_maps == other.gt_maps + assert self.anomaly_maps == other.anomaly_maps + + # Merge the 1-D numpy arrays. + anomaly_thresholds = np.concatenate([self.anomaly_thresholds, + other.anomaly_thresholds]) + # Merge the nested 1-D numpy arrays. + fp_areas_per_image = concat_nested_arrays(self.fp_areas_per_image, + other.fp_areas_per_image) + tn_areas_per_image = concat_nested_arrays(self.tn_areas_per_image, + other.tn_areas_per_image) + spros_of_defects_of_images = concat_nested_arrays( + self.spros_of_defects_of_images, + other.spros_of_defects_of_images, + nest_level=2) + + # Sort the anomaly thresholds in descending order so that the first + # threshold corresponds to a sPRO of 0 and an FPR of 0. + sort_indices = np.argsort(anomaly_thresholds)[::-1] + anomaly_thresholds = anomaly_thresholds[sort_indices] + fp_areas_per_image = get_sorted_nested_arrays( + nested_arrays=fp_areas_per_image, + sort_indices=sort_indices) + tn_areas_per_image = get_sorted_nested_arrays( + nested_arrays=tn_areas_per_image, + sort_indices=sort_indices) + spros_of_defects_of_images = get_sorted_nested_arrays( + nested_arrays=spros_of_defects_of_images, + sort_indices=sort_indices, + nest_level=2) + + return ThresholdMetrics( + gt_maps=self.gt_maps, + anomaly_maps=self.anomaly_maps, + anomaly_thresholds=anomaly_thresholds, + spros_of_defects_of_images=spros_of_defects_of_images, + fp_areas_per_image=fp_areas_per_image, + tn_areas_per_image=tn_areas_per_image) + + def reduce_to_images(self, take_anomaly_maps: Sequence[AnomalyMap]): + """Return a new ThresholdMetrics instance that only contains metrics + for the given anomaly maps. + + The order of the anomaly maps is the same as take_anomaly_maps, + which must be a subset of self.anomaly_maps. + """ + take_indices = [self.anomaly_maps.index(image) + for image in take_anomaly_maps] + return ThresholdMetrics( + gt_maps=take(self.gt_maps, take_indices), + anomaly_maps=take(self.anomaly_maps, take_indices), + anomaly_thresholds=self.anomaly_thresholds, + spros_of_defects_of_images=take(self.spros_of_defects_of_images, + take_indices), + fp_areas_per_image=take(self.fp_areas_per_image, take_indices), + tn_areas_per_image=take(self.tn_areas_per_image, take_indices)) + + def get_fp_rates(self): + """Compute FPRs for this instance's anomaly thresholds and images. + + Raises: + ZeroDivisionError if there is no defect-free pixel in any of the + images. This would result in a zero division when computing the + FPR for any anomaly threshold. + """ + return get_fp_rates(fp_areas_per_image=self.fp_areas_per_image, + tn_areas_per_image=self.tn_areas_per_image) + + def get_mean_spros(self, + filter_defect_names: Optional[Iterable[str]] = None): + """Compute the mean sPROs per threshold across all defects. + + Args: + filter_defect_names: If not None, only the sPRO values from defect + names in this sequence will be used. + + Returns: + A 1-D numpy array containing the mean sPRO values averaged + across all defects in self.spros_of_defect_of_images. + """ + spros_of_defects = [] + + # Flatten the list of lists of 1-D numpy arrays. + for gt_image, image_spros in zip(self.gt_maps, + self.spros_of_defects_of_images): + if gt_image is None: + assert len(image_spros) == 0 + continue + for gt_channel, spros in zip(gt_image.channels, image_spros): + defect_name = gt_channel.defect_config.defect_name + if (filter_defect_names is None + or defect_name in filter_defect_names): + spros_of_defects.append(spros) + + if len(spros_of_defects) > 0: + return np.mean(spros_of_defects, axis=0) + else: + return None + + def get_per_image_results_dicts(self, auc_max_fprs: Iterable[float]): + """Yield dicts containing per-image metrics for json output. + + Args: + auc_max_fprs: Maximum FPR values for computing AUC sPRO values. + """ + for anomaly_map, fp_areas, tn_areas, spros_of_defects in zip( + self.anomaly_maps, + self.fp_areas_per_image, + self.tn_areas_per_image, + self.spros_of_defects_of_images): + + # Set the values that exist for "good" images as well. + spros_of_defects = [spros.tolist() for spros in spros_of_defects] + results_dict = { + 'path_full': anomaly_map.file_path, + 'path_short': get_file_path_repr(anomaly_map.file_path), + 'fp_areas': fp_areas.tolist(), + 'tn_areas': tn_areas.tolist(), + 'spros_of_defects': spros_of_defects + } + + # Set the values that exist only for anomalous images. + mean_spros = None + if len(spros_of_defects) > 0: + mean_spros = np.mean(spros_of_defects, axis=0) + results_dict['mean_spros'] = mean_spros.tolist() + + # If there are no defect-free pixels, FP+TN will always be zero. + # Then, we cannot compute the FPR. + if np.sum(fp_areas + tn_areas) > 0: + fp_rates = get_fp_rates(fp_areas_per_image=[fp_areas], + tn_areas_per_image=[tn_areas]) + results_dict['fp_rates'] = fp_rates.tolist() + + if mean_spros is not None: + # Compute the AUC sPRO values for different max FPRs. + auc_spros = {} + for max_fpr in auc_max_fprs: + auc = get_auc_for_max_fpr(fprs=fp_rates, + y_values=mean_spros, + max_fpr=max_fpr, + scale_to_one=True) + auc_spros[max_fpr] = auc + results_dict['auc_spros'] = auc_spros + + yield results_dict + + +class MetricsAggregator: + """Compute sPRO and FPR values for given ground truth and anomaly maps. + + The thresholds are computed dynamically by adding thresholds until the + FPR-sPRO curve has a high resolution. + + Attributes (in addition to init arguments): + threshold_metrics: The threshold metrics computed by the last run(...) + call. + """ + + def __init__(self, + gt_maps: Sequence[Optional[GroundTruthMap]], + anomaly_maps: Sequence[AnomalyMap], + parallel_workers: Optional[int] = None, + parallel_niceness: int = 19): + """ + Args: + gt_maps: Sequence of GroundTruthMap or None entries with the + same length and ordering as anomaly_maps. Use None for "good" + images without ground truth annotations. + anomaly_maps: Must have the same length and ordering as + gt_maps. + parallel_workers: If None (default), nothing will be parallelized + across CPUs. Otherwise, the value denotes the number of CPUs to + use for parallelism. A value of 1 will result in suboptimal + performance compared to None. + parallel_niceness: Niceness of child processes. Only applied in the + parallelized setting. + """ + self.gt_maps = gt_maps + self.anomaly_maps = anomaly_maps + self.parallel_workers = parallel_workers + self.parallel_niceness = parallel_niceness + self.threshold_metrics: Optional[ThresholdMetrics] = None + + def run(self, curve_max_distance: float): + """Iteratively refine the anomaly thresholds and collect metrics. + + Args: + curve_max_distance: Maximum distance between two points on the + overall FPR-sPRO curve. Will be used for selecting anomaly + thresholds. + + Returns: + The resulting ThresholdMetrics instance. + """ + initial_thresholds = self._get_initial_thresholds() + initial_values = self._refinement_callback(initial_thresholds) + binary_refinement(init_queries=initial_thresholds, + init_values=initial_values, + max_distance=curve_max_distance, + get_values=self._refinement_callback, + min_queries_per_step=20, + max_queries_per_step=100, + max_steps=10) + return self.threshold_metrics + + def _refinement_callback(self, anomaly_thresholds: Sequence): + # Sort the anomaly thresholds, but remember how to undo the sorting + # for returning the points to binary_refinement in the right order. + sort_indices = np.argsort(anomaly_thresholds)[::-1] + unsort_indices = np.empty_like(sort_indices) + unsort_indices[sort_indices] = np.arange(len(sort_indices)) + anomaly_thresholds_sorted = np.take(anomaly_thresholds, sort_indices) + + spros_of_defects_of_images = get_spros_of_defects_of_images( + gt_maps=self.gt_maps, + anomaly_maps=self.anomaly_maps, + anomaly_thresholds=anomaly_thresholds_sorted, + parallel_workers=self.parallel_workers, + parallel_niceness=self.parallel_niceness) + + fp_areas_per_image, tn_areas_per_image = \ + get_fp_tn_areas_per_image( + gt_maps=self.gt_maps, + anomaly_maps=self.anomaly_maps, + anomaly_thresholds=anomaly_thresholds_sorted, + parallel_workers=self.parallel_workers, + parallel_niceness=self.parallel_niceness) + + # Store the raw per-image metrics in a ThresholdMetrics instance. + threshold_metrics_delta = ThresholdMetrics( + gt_maps=self.gt_maps, + anomaly_maps=self.anomaly_maps, + anomaly_thresholds=np.array(anomaly_thresholds_sorted), + spros_of_defects_of_images=spros_of_defects_of_images, + fp_areas_per_image=fp_areas_per_image, + tn_areas_per_image=tn_areas_per_image) + + # Compute metrics across images. + mean_spros = threshold_metrics_delta.get_mean_spros() + fp_rates = threshold_metrics_delta.get_fp_rates() + + # Update the threshold metrics collection. + if self.threshold_metrics is None: + self.threshold_metrics = threshold_metrics_delta + else: + self.threshold_metrics = self.threshold_metrics.merge_with( + threshold_metrics_delta) + + result_values_sorted = [np.array([spro, fp_rate]) + for spro, fp_rate in zip(mean_spros, fp_rates)] + result_values = np.take(result_values_sorted, unsort_indices, axis=0) + return result_values + + def _get_initial_thresholds(self, num_thresholds=50, epsilon=1e-6): + """Returns initial anomaly thresholds for refining a sPRO curve. + + The thresholds are sorted in descending order. The first threshold is + the maximum of all anomaly scores in self.anomaly_maps, plus a given + epsilon. The last threshold is the minimum of all anomaly scores, minus + a given epsilon. Thus, the first threshold corresponds to an FPR of 0 + and a sPRO of 0, while the last threshold corresponds to an FPR of 1 and + a sPRO of 1. + + The thresholds in between are selected by sorting the anomaly scores + and picking scores at equidistant indices. If the number of anomaly + scores is large, this is done on a random subset of the anomaly scores. + + Args: + num_thresholds: The length of the list of anomaly thresholds + returned. + epsilon: A small value to add to the maximum and subtract from the + minimum of anomaly scores to reach an FPR and sPRO of 0 or 1, + respectively. + + Returns: + A list of floats sorted in descending order. + """ + # sampled_scores should contain at most 10 million values to prevent + # memory overflow and keep sorting (see below) fast. + max_num_scores = 10_000_000 + # Therefore, we possibly need to sample random anomaly scores from each + # anomaly map. + num_images = len(self.anomaly_maps) + num_scores_per_image = np.prod(self.anomaly_maps[0].np_array.shape) + if num_images * num_scores_per_image <= max_num_scores: + # We don't need to subsample. This corresponds to sampling + # all scores (given that we sample without replacement). + num_sampled_per_image = num_scores_per_image + else: + # We need to subsample. + num_sampled_per_image = int(np.floor(max_num_scores / num_images)) + sampled_scores = [] + + # Iterate through the images and keep track of the maximum and minimum + # of all anomaly scores. + some_score = self.anomaly_maps[0].np_array[0, 0] + min_score, max_score = some_score, some_score + for anomaly_map in self.anomaly_maps: + min_score = min(min_score, np.min(anomaly_map.np_array)) + max_score = max(max_score, np.max(anomaly_map.np_array)) + + # Sample scores from this image. + flat_scores = anomaly_map.np_array.flatten() + assert len(flat_scores) == num_scores_per_image + sampled_scores_image = np.random.choice(flat_scores, + size=num_sampled_per_image, + replace=False) + sampled_scores.append(sampled_scores_image) + + min_threshold = min_score - epsilon + max_threshold = max_score + epsilon + + # Sort the sampled scores. + sampled_scores = np.concatenate(sampled_scores).flatten() + sampled_scores.sort() + + # From the sampled scores, take values at equidistant indices. + equidistant_indices = np.linspace(0, len(sampled_scores) - 1, + num=num_thresholds, + endpoint=True, + dtype=int) + equidistant_indices = equidistant_indices[1:-1] + equiheight_scores = sampled_scores[equidistant_indices] + + # Combine the minimum, the maximum and the equidistant anomaly scores + # to form the list of thresholds, sorted in descending order. + thresholds = equiheight_scores.tolist()[::-1] + thresholds = [max_threshold] + thresholds + [min_threshold] + unique_thresholds = [] + seen = set() + for t in thresholds: + if t in seen: + continue + unique_thresholds.append(t) + seen.add(t) + thresholds = unique_thresholds + return thresholds diff --git a/mvtec_loco_ad_evaluation/src/image.py b/mvtec_loco_ad_evaluation/src/image.py new file mode 100644 index 0000000..1607b2a --- /dev/null +++ b/mvtec_loco_ad_evaluation/src/image.py @@ -0,0 +1,265 @@ +"""Classes for handling ground truth and anomaly maps.""" + +import glob +import os +from functools import lru_cache +from typing import Sequence, Optional, Mapping, Iterable, Tuple, Union, Any + +import numpy as np +import tifffile +from PIL import Image + + +def get_file_path_repr(file_path: Optional[str]) -> str: + if file_path is None: + return 'no file path' + else: + parent_dir_path, file_name = os.path.split(file_path) + _, parent_dir = os.path.split(parent_dir_path) + return f'.../{parent_dir}/{file_name}' + + +class DefectConfig: + def __init__(self, + defect_name: str, + pixel_value: int, + saturation_threshold: Union[int, float], + relative_saturation: bool): + # Input validation. + assert 1 <= pixel_value <= 255 + if relative_saturation: + assert isinstance(saturation_threshold, float) + assert 0. < saturation_threshold <= 1. + else: + assert isinstance(saturation_threshold, int) + + self.defect_name = defect_name + self.pixel_value = pixel_value + self.saturation_threshold = saturation_threshold + self.relative_saturation = relative_saturation + + def __repr__(self): + return f'DefectConfig({self.__dict__})' + + +class DefectsConfig: + def __init__(self, entries: Sequence[DefectConfig]): + # Create a pixel_value -> entry mapping for faster lookup. + self.pixel_value_to_entry = {e.pixel_value: e for e in entries} + + @property + def entries(self): + return tuple(self.pixel_value_to_entry.values()) + + @classmethod + def create_from_list(cls, defects_list: Sequence[Mapping[str, Any]]): + entries = [] + for defect_config in defects_list: + entry = DefectConfig( + defect_name=defect_config['defect_name'], + pixel_value=defect_config['pixel_value'], + saturation_threshold=defect_config['saturation_threshold'], + relative_saturation=defect_config['relative_saturation']) + entries.append(entry) + return DefectsConfig(entries=entries) + + +class GroundTruthChannel: + """A channel of a ground truth map. + + Corresponds to exactly one defect in a ground truth map. Must not be used + to represent a defect-free image. + """ + + def __init__(self, + bool_array: np.ndarray, + defect_config: DefectConfig): + """ + Args: + bool_array: A 2-D numpy array with dtype np.bool_. A True value + indicates an anomalous pixel. + defect_config: The DefectConfig for this channel's defect type. + """ + + # Input validation. + # numpy dtypes need to be checked with == instead of `is`, see + # https://stackoverflow.com/a/26921882/2305095 + # We want np.bool_ for a fast computation of unions, intersections etc. + assert len(bool_array.shape) == 2 and bool_array.dtype == np.bool_ + + self.bool_array = bool_array + self.defect_config = defect_config + + def get_defect_area(self): + return np.sum(self.bool_array) + + def get_saturation_area(self): + defect_area = self.get_defect_area() + if self.defect_config.relative_saturation: + return int(self.defect_config.saturation_threshold * defect_area) + else: + return np.minimum(self.defect_config.saturation_threshold, + defect_area) + + @classmethod + def create_from_integer_array(cls, + np_array: np.ndarray, + defects_config: DefectsConfig): + """Create a new GroundTruthChannel from an integer array. + + Args: + np_array: A 2-D array with exactly one distinct positive value. All + non-positive entries must be zero and correspond to defect-free + pixels. + defects_config: The defects configuration for the dataset object + being evaluated. + """ + assert np.issubdtype(np_array.dtype, np.integer) + + # Ensure that each channel has exactly one unique positive integer. + sorted_unique = sorted(np.unique(np_array)) + if len(sorted_unique) == 1: + defect_id = sorted_unique[0] + else: + zero, defect_id = sorted_unique + assert zero == 0 + assert defect_id > 0 + # Cast np.uint8 etc. to int. + defect_id = int(defect_id) + + # Convert to bool for faster logical operations with anomaly maps. + bool_array = np_array.astype(np.bool_) + + # Look up the defect config for this defect id. + defect_config = defects_config.pixel_value_to_entry[defect_id] + return GroundTruthChannel(bool_array=bool_array, + defect_config=defect_config) + + +class GroundTruthMap: + """A ground truth map for an anomalous image. + + Each channel corresponds to one defect in the image. + + Use GroundTruthMap.read_from_tiff(...) to read a GroundTruthMap from a + .tiff file. + + If defect_id_to_name is None, it is constructed based on the defect ids in + the channels, using defect_id -> str(defect_id). + """ + + def __init__(self, + channels: Sequence[GroundTruthChannel], + file_path: Optional[str] = None): + + # Input validation. + assert len(channels) > 0 + # Ensure that each channel has the same size. + first_shape = channels[0].bool_array.shape + assert set(c.bool_array.shape for c in channels) == {first_shape} + # Check whether some channels have larger saturation thresholds than + # defect areas. + for i_channel, channel in enumerate(channels): + if channel.defect_config.relative_saturation: + continue + threshold = channel.defect_config.saturation_threshold + defect_area = channel.get_defect_area() + if threshold > defect_area: + print(f'WARNING: Channel {i_channel + 1} (1=first) of ground' + f' truth image {get_file_path_repr(file_path)} has a' + f' defect area of {defect_area}, but a saturation' + f' threshold of {threshold}. Corresponding defect' + f' config: {channel.defect_config}') + + self.channels = tuple(channels) + self.file_path = file_path + + @property + def size(self): + return self.channels[0].bool_array.shape + + def get_or_over_channels(self) -> np.ndarray: + """Combine the channels with a logical OR operation. + + Returns a numpy array of type np.bool_. + """ + channels_np = tuple(c.bool_array for c in self.channels) + return np.sum(channels_np, axis=0).astype(bool) + + @classmethod + def read_from_png_dir(cls, + png_dir: str, + defects_config: DefectsConfig): + """Read a GroundTruthMap from a directory containing one .png per + channel. + """ + gt_channels = [] + for png_path in sorted(glob.glob(os.path.join(png_dir, '*.png'))): + image = Image.open(png_path) + np_array = np.array(image) + gt_channel = GroundTruthChannel.create_from_integer_array( + np_array=np_array, + defects_config=defects_config) + gt_channels.append(gt_channel) + + return cls(channels=gt_channels, file_path=png_dir) + + +class AnomalyMap: + """An anomaly map generated by a model. + + Use AnomalyMap.read_from_tiff(...) to read an AnomalyMap from a + .tiff file. + """ + + def __init__(self, + np_array: np.ndarray, + file_path: Optional[str] = None): + """ + Args: + np_array: A 2-D numpy array containing the real-valued anomaly + scores. + file_path: (optional) file path of the image. Not used for I/O. + """ + + assert len(np_array.shape) == 2 + + self.np_array = np_array + self.file_path = file_path + + def __repr__(self): + return f'AnomalyMap({get_file_path_repr(self.file_path)})' + + @property + def size(self): + return self.np_array.shape + + def get_binary_image(self, anomaly_threshold: float): + """Return the binary anomaly map based on a given threshold. + + The result is a 2-D numpy array with dtype np.bool_. + """ + return self.get_binary_images( + anomaly_thresholds=[anomaly_threshold])[0] + + def get_binary_images(self, anomaly_thresholds: Iterable[float]): + """Return binary anomaly maps based on given thresholds. + + The result is a 3-D numpy array with dtype np.bool_. The first + dimension has the same length as the anomaly_thresholds. + """ + return self._get_binary_images( + anomaly_thresholds=tuple(anomaly_thresholds)) + + @lru_cache(maxsize=3) + def _get_binary_images(self, anomaly_thresholds: Tuple[float, ...]): + thresholds = [[[t]] for t in anomaly_thresholds] + return np.greater(self.np_array[np.newaxis, :, :], thresholds) + + @classmethod + def read_from_tiff(cls, tiff_path: str): + """Read an AnomalyMap from a TIFF-file.""" + np_array = tifffile.imread(tiff_path) + assert len(np_array.shape) == 2 + return cls(np_array=np_array, + file_path=tiff_path) diff --git a/mvtec_loco_ad_evaluation/src/metrics.py b/mvtec_loco_ad_evaluation/src/metrics.py new file mode 100644 index 0000000..15a139c --- /dev/null +++ b/mvtec_loco_ad_evaluation/src/metrics.py @@ -0,0 +1,367 @@ +"""Metrics computed on a single image, but many anomaly thresholds. + +At the bottom, there are two functions for computing sPRO values and false +positive rates efficiently for many images and many anomaly thresholds: +- get_spros_of_defects_of_images(...) +- get_fp_tn_areas_per_image(...) +""" + +from concurrent.futures import ProcessPoolExecutor +from typing import Sequence, Optional, MutableMapping + +import numpy as np +from tqdm import tqdm + +from src.image import AnomalyMap, GroundTruthMap, GroundTruthChannel +from src.util import set_niceness + + +def get_spro(gt_channel: GroundTruthChannel, + anomaly_map: AnomalyMap, + anomaly_threshold: float) -> float: + """Compute the saturated PRO metric for a single ground truth channel + (i.e. defect) and a single threshold. + + Only use this function for testing and understanding. Do not use it + repeatedly for different anomaly thresholds. Use get_spros(...) for that. + """ + binary_anomaly_map = anomaly_map.get_binary_image(anomaly_threshold) + tp = np.logical_and(binary_anomaly_map, gt_channel.bool_array) + tp_area = np.sum(tp) + saturation_area = gt_channel.get_saturation_area() + return np.minimum(tp_area / saturation_area, 1.) + + +def get_spros_for_thresholds(gt_channel: GroundTruthChannel, + anomaly_map: AnomalyMap, + anomaly_thresholds: Sequence[float]): + """Compute the saturated PRO metric for a single ground truth channel + (i.e. defect) and multiple thresholds. + + Returns: + A 1-D numpy array with the same length as anomaly_thresholds + containing the sPRO values. + """ + tp_areas = get_tp_areas_for_thresholds( + gt_channel=gt_channel, + anomaly_map=anomaly_map, + anomaly_thresholds=anomaly_thresholds) + saturation_area = gt_channel.get_saturation_area() + return np.minimum(tp_areas / saturation_area, 1.) + + +def get_spros_per_defect_for_thresholds(gt_map: Optional[GroundTruthMap], + anomaly_map: AnomalyMap, + anomaly_thresholds: Sequence[float]): + """Compute the saturated PRO metric for a single ground truth map + (containing multiple defects / channels) and multiple thresholds. + + Returns: + A tuple of 1-D numpy arrays. The length of the tuple is given by + the number of channels in the ground truth map. Each numpy array + has the same length as anomaly_thresholds and contains the sPRO + values for the respective channel. If gt_map is None, the + returned tuple is empty. + """ + if gt_map is None: + return [] + + assert anomaly_map.np_array.shape == gt_map.size + spros_per_defect = [] + for channel in gt_map.channels: + spros = get_spros_for_thresholds(gt_channel=channel, + anomaly_map=anomaly_map, + anomaly_thresholds=anomaly_thresholds) + spros_per_defect.append(spros) + return tuple(spros_per_defect) + + +def get_tp_areas_for_thresholds(gt_channel: GroundTruthChannel, + anomaly_map: AnomalyMap, + anomaly_thresholds: Sequence[float]): + """Compute the true positive areas for a single ground truth channel + (i.e. defect) and multiple thresholds. + + Returns: + A 1-D numpy array with the same length as anomaly_thresholds + containing the true positive areas. + """ + binary_anomaly_maps = anomaly_map.get_binary_images(anomaly_thresholds) + tps = np.logical_and(binary_anomaly_maps, gt_channel.bool_array) + tp_areas = np.sum(tps, axis=(1, 2)) + return tp_areas + + +def get_fp_areas_for_thresholds(gt_map: Optional[GroundTruthMap], + anomaly_map: AnomalyMap, + anomaly_thresholds: Sequence[float]): + """Compute the false positive areas for a single ground truth map + (containing multiple defects / channels) and multiple thresholds. + + Set gt_map to None for "good" images without ground truth annotations. + + Needs the whole GT maps to make sure that we do not mark a pixel a false + positive that would be a true positive in another channel. + + A false positive pixel is a pixel that is defect-free in all channels of + the ground truth map, but is a positive in the anomaly map. + + Returns: + A 1-D numpy array with the same length as anomaly_thresholds + containing the false positive areas. + """ + + binary_anomaly_maps = anomaly_map.get_binary_images(anomaly_thresholds) + + fp_areas: np.ndarray + if gt_map is None: + # This is a good image. All positive pixels are false positives. + fp_areas = np.sum(binary_anomaly_maps, axis=(1, 2)) + else: + # False positives do not depend on a single channel, like true + # positives. Only pixels that are defect-free in all ground truth + # channels can be a false positive. + gt_combined = gt_map.get_or_over_channels() + fps = np.logical_and(binary_anomaly_maps, + np.logical_not(gt_combined)) + fp_areas = np.sum(fps, axis=(1, 2)) + return fp_areas + + +def get_tn_areas_for_thresholds(gt_map: Optional[GroundTruthMap], + anomaly_map: AnomalyMap, + anomaly_thresholds: Sequence[float], + fp_areas: Optional[np.ndarray] = None): + """Compute the true negative areas for a single ground truth map + (containing multiple defects / channels) and multiple thresholds. + + Set gt_map to None for "good" images without ground truth annotations. + + A true negative pixel is a pixel that is defect-free in all channels of + the ground truth map and is a negative in the anomaly map. + + The true negative area plus the false positive area equals the number of + pixels that are defect-free in *all* channels of the ground truth map, + see get_fp_areas_for_thresholds(...). + + The computation can be sped up significantly by setting fp_areas to the + result of get_fp_areas_for_thresholds(...)! + + Returns: + A 1-D numpy array with the same length as anomaly_thresholds + containing the true negative areas. + """ + + binary_anomaly_maps = anomaly_map.get_binary_images(anomaly_thresholds) + + tn_areas: np.ndarray + if gt_map is None: + # This is a good image. All negative pixels are true negatives. + tn_areas = np.sum(np.logical_not(binary_anomaly_maps), axis=(1, 2)) + else: + # True negatives are pixels that have a negative prediction and are + # marked as being defect-free in all channels of a ground truth map. + gt_combined = gt_map.get_or_over_channels() + + if fp_areas is not None: + # Compute the true negatives based on the false positives and the + # total defect-free area. + no_defect_area = np.sum(np.logical_not(gt_combined)) + tn_areas = no_defect_area - fp_areas + else: + # NOT prediction=True AND NOT defect=True can be replaced with + # NOT (prediction=True OR defect=True) + tns = np.logical_not(np.logical_or(binary_anomaly_maps, + gt_combined)) + tn_areas = np.sum(tns, axis=(1, 2)) + return tn_areas + + +def _get_spros_per_defect_for_thresholds_kwargs(kwargs: MutableMapping): + if 'niceness' in kwargs: + set_niceness(kwargs['niceness']) + del kwargs['niceness'] + return get_spros_per_defect_for_thresholds(**kwargs) + + +def get_spros_of_defects_of_images( + gt_maps: Sequence[Optional[GroundTruthMap]], + anomaly_maps: Sequence[AnomalyMap], + anomaly_thresholds: Sequence[float], + parallel_workers: Optional[int] = None, + parallel_niceness: int = 19): + """Compute the saturated PRO values for several images and anomaly + thresholds, possibly in parallel. + + Args: + gt_maps: Sequence of GroundTruthMap or None entries with the same + length and ordering as anomaly_maps. Use None for "good" images + without ground truth annotations. + anomaly_maps: Must have the same length and ordering as gt_maps. + anomaly_thresholds: Thresholds for obtaining binary anomaly maps. + parallel_workers: If None (default), nothing will be parallelized + across CPUs. Otherwise, the value denotes the number of CPUs to use + for parallelism. A value of 1 will result in suboptimal performance + compared to None. + parallel_niceness: Niceness of child processes. Only applied in the + parallelized setting. + + Returns: + A list of tuples of numpy arrays. The outer list will have the same + length as gt_maps and anomaly_maps. The length of each inner + tuple is given by the number of defects per image. The length of + each numpy array is given by the number of anomaly thresholds. + "good" images will have an empty inner tuple. + """ + assert len(gt_maps) == len(anomaly_maps) + + # Construct the kwargs for each call to get_spros_per_defect_for_thresholds + # via _get_spros_per_defect_for_thresholds_kwargs. + kwargs_list = [] + for gt_map, anomaly_map in zip(gt_maps, anomaly_maps): + kwargs = { + 'gt_map': gt_map, + 'anomaly_map': anomaly_map, + 'anomaly_thresholds': anomaly_thresholds + } + if parallel_workers is not None: + kwargs['niceness'] = parallel_niceness + kwargs_list.append(kwargs) + + if parallel_workers is None: + print(f'Computing mean sPROs for {len(anomaly_thresholds)} anomaly' + f' thresholds...') + spros_of_defects_of_images = [ + _get_spros_per_defect_for_thresholds_kwargs(kwargs) + for kwargs in tqdm(kwargs_list)] + else: + print(f'Computing mean sPROs for {len(anomaly_thresholds)} anomaly' + f' thresholds in parallel on {parallel_workers} CPUs...') + pool = ProcessPoolExecutor(max_workers=parallel_workers) + spros_of_defects_of_images = pool.map( + _get_spros_per_defect_for_thresholds_kwargs, + kwargs_list) + spros_of_defects_of_images = list(spros_of_defects_of_images) + return spros_of_defects_of_images + + +def _get_fp_areas_for_thresholds_kwargs(kwargs: MutableMapping): + if 'niceness' in kwargs: + set_niceness(kwargs['niceness']) + del kwargs['niceness'] + return get_fp_areas_for_thresholds(**kwargs) + + +def get_fp_tn_areas_per_image( + gt_maps: Sequence[Optional[GroundTruthMap]], + anomaly_maps: Sequence[AnomalyMap], + anomaly_thresholds: Sequence[float], + parallel_workers: Optional[int] = None, + parallel_niceness: int = 19): + """Compute the false positive and the true negative areas for several + images and anomaly thresholds, possibly in parallel. + + Args: + gt_maps: Sequence of GroundTruthMap or None entries with the same + length and ordering as anomaly_maps. Use for "good" images + without ground truth annotations. + anomaly_maps: Must have the same length and ordering as gt_maps. + anomaly_thresholds: Thresholds for obtaining binary anomaly maps. + parallel_workers: If None (default), nothing will be parallelized + across CPUs. Otherwise, the value denotes the number of CPUs to use + for parallelism. A value of 1 will result in suboptimal performance + compared to None. + parallel_niceness: Niceness of child processes. Only applied in the + parallelized setting. + + Returns: + A list of 1-D numpy arrays. The list has the same length as gt_maps + and anomaly_maps. It contains the false positive areas for each + image. Each numpy array has the same length as anomaly_thresholds. + A list of 1-D numpy arrays. The list has the same length as gt_maps + and anomaly_maps. It contains the true negative areas for each + image. Each numpy array has the same length as anomaly_thresholds. + """ + assert len(gt_maps) == len(anomaly_maps) + + # Construct the kwargs for each call to get_fp_areas_for_thresholds via + # _get_fp_areas_for_thresholds_kwargs. + kwargs_list = [] + for gt_map, anomaly_map in zip(gt_maps, anomaly_maps): + kwargs = { + 'gt_map': gt_map, + 'anomaly_map': anomaly_map, + 'anomaly_thresholds': anomaly_thresholds + } + if parallel_workers is not None: + kwargs['niceness'] = parallel_niceness + kwargs_list.append(kwargs) + + # For each anomaly threshold, compute the FP areas per image. + if parallel_workers is None: + print(f'Computing FPRs for {len(anomaly_thresholds)} anomaly' + f' thresholds...') + fp_areas_per_image = [ + _get_fp_areas_for_thresholds_kwargs(kwargs) + for kwargs in tqdm(kwargs_list)] + else: + print(f'Computing FPRs for {len(anomaly_thresholds)} anomaly' + f' thresholds in parallel on {parallel_workers} CPUs...') + pool = ProcessPoolExecutor(max_workers=parallel_workers) + fp_areas_per_image = pool.map( + _get_fp_areas_for_thresholds_kwargs, + kwargs_list) + fp_areas_per_image = list(fp_areas_per_image) + + # For each anomaly threshold, compute the TN areas per image. + tn_areas_per_image = [] + for gt_map, anomaly_map, fp_areas in zip( + gt_maps, anomaly_maps, fp_areas_per_image): + # For each image, there is only one FP area and one TN area per + # anomaly threshold. + tn_areas = get_tn_areas_for_thresholds( + gt_map=gt_map, + anomaly_map=anomaly_map, + anomaly_thresholds=anomaly_thresholds, + fp_areas=fp_areas) + tn_areas_per_image.append(tn_areas) + return fp_areas_per_image, tn_areas_per_image + + +def get_fp_rates(fp_areas_per_image: Sequence[np.ndarray], + tn_areas_per_image: Sequence[np.ndarray]): + """Compute false positive rates based on the results of + get_fp_tn_areas_per_image(...). + + Args: + fp_areas_per_image: See get_fp_tn_areas_per_image(...). + tn_areas_per_image: See get_fp_tn_areas_per_image(...). + + Returns: + A 1-D numpy array with the same length as each array in + fp_areas_per_image and tn_areas_per_image. For each + anomaly threshold, it contains the FPR computed over all images. + + Raises: + ZeroDivisionError if there is no defect-free pixel in any of the + images. This would result in a zero division when computing the + FPR for any anomaly threshold. + """ + total_fp_areas = np.zeros_like(fp_areas_per_image[0], dtype=np.int64) + total_tn_areas = np.zeros_like(fp_areas_per_image[0], dtype=np.int64) + for fp_areas, tn_areas in zip(fp_areas_per_image, tn_areas_per_image): + assert len(fp_areas) == len(tn_areas) + total_fp_areas += fp_areas + total_tn_areas += tn_areas + + # If there is no defect-free pixel in any of the images, there cannot be + # false positives or true negatives. Then, TN+FP will be zero, regardless + # of the anomaly threshold. Otherwise, TN+FP will be positive, regardless + # of the anomaly threshold. + # Therefore, we prevent division by zero by checking if the sum of TN+FP + # is zero for any of the thresholds. + if total_tn_areas[0] + total_fp_areas[0] == 0: + assert np.sum(total_fp_areas + total_tn_areas) == 0 + raise ZeroDivisionError + fp_rates = total_fp_areas / (total_tn_areas + total_fp_areas) + return fp_rates diff --git a/mvtec_loco_ad_evaluation/src/util.py b/mvtec_loco_ad_evaluation/src/util.py new file mode 100644 index 0000000..f9ab274 --- /dev/null +++ b/mvtec_loco_ad_evaluation/src/util.py @@ -0,0 +1,312 @@ +"""Collection of utility functions.""" +import os +import platform +from bisect import bisect +from typing import Iterable, Sequence, List, Callable + +import numpy as np + + +def is_dict_order_stable(): + """Returns true, if and only if dicts always iterate in the same order.""" + if platform.python_implementation() == 'CPython': + required_minor = 6 + else: + required_minor = 7 + major, minor, _ = platform.python_version_tuple() + assert major == '3' and all(s.isdigit() for s in minor) + return int(minor) >= required_minor + + +def listdir(path, sort=True, include_hidden=False): + file_names = os.listdir(path) + if sort: + file_names = sorted(file_names) + if not include_hidden: + file_names = [f for f in file_names if not f.startswith('.')] + return file_names + + +def set_niceness(niceness): + # Same as os.nice, but takes an absolute niceness instead of an increment. + current_niceness = os.nice(0) + niceness_increment = niceness - current_niceness + # Regular users are not allowed to decrease the niceness. Doing so would + # raise an exception, even if the resulting niceness would be positive. + niceness_increment = max(0, niceness_increment) + return os.nice(niceness_increment) + + +def take(seq: Sequence, indices: Iterable[int]) -> List: + return [seq[i] for i in indices] + + +def flatten_2d(seq: Sequence[Sequence]) -> List: + return [elem for innerseq in seq for elem in innerseq] + + +def get_sorted_nested_arrays(nested_arrays, sort_indices, nest_level=1): + return map_nested(nested_objects=nested_arrays, + fun=lambda a: a[sort_indices], + nest_level=nest_level) + + +def concat_nested_arrays(head_arrays: Sequence, + tail_arrays: Sequence, + nest_level=1): + """Concatenate numpy arrays nested in a sequence (of sequences ... + of sequences). + + Args: + head_arrays: Sequence (of sequences ... of sequences) of numpy arrays. + The lengths of the nested numpy arrays may differ. + tail_arrays: Sequence (of sequences ... of sequences) of numpy arrays. + Must have the same structure as head_arrays. + The lengths of the nested numpy arrays may differ. + nest_level: Number of sequence levels. 1 means there is a sequence of + arrays. 2 means there is a sequence of sequences of arrays. + Must be >= 1. + + Returns: + A sequence (of sequences ... of sequences) of numpy arrays with the + same structure as head_arrays and tail_arrays containing the + concatenated arrays. + """ + + # Zip the heads and tails at the deepest level. + head_tail_arrays = zip_nested(head_arrays, tail_arrays, + nest_level=nest_level) + + def concat(args): + head, tail = args + return np.concatenate([head, tail]) + + return map_nested(nested_objects=head_tail_arrays, + fun=concat, + nest_level=nest_level) + + +def map_nested(nested_objects: Sequence, fun: Callable, nest_level=1): + """Apply a function to objects nested in a sequence (of sequences ... + of sequences). + + Args: + nested_objects: Sequence (of sequences ... of sequences) of objects. + fun: Function to call for each object. + nest_level: Number of sequence levels. 1 means there is a sequence of + objects. 2 means there is a sequence of sequences of objects. + Must be >= 1. + + Returns: + A list (of lists ... of lists) of mapped objects. This list has the + same structure as nested_objects. Each item is the result of + applying fun to the corresponding nested object. + """ + assert 1 <= nest_level + if nest_level == 1: + return [fun(o) for o in nested_objects] + else: + # Go one level deeper. + mapped = [] + for lower_nested_objects in nested_objects: + # Map the nested sequence of objects. + lower_mapped = map_nested( + nested_objects=lower_nested_objects, + fun=fun, + nest_level=nest_level - 1 + ) + mapped.append(lower_mapped) + return mapped + + +def zip_nested(*seqs: Sequence, nest_level=1): + """Zip sequences (of sequences ... of sequences) of objects at the deepest + level. + + Args: + seqs: Sequences (of sequences ... of sequences) of objects. + All sequences must have the same structure (length, length of + descending sequences etc.). + nest_level: Number of sequence levels. 1 means each sequence is a + sequence of objects. 2 means each is a sequence of sequences of + objects. Must be >= 1. + + Returns: + A list (of lists ... of lists) of tuples containing the zipped objects. + This list has the same structure as each sequence in seqs. + """ + assert 1 <= nest_level + # All sequences must have the same length. + seq_length = len(seqs[0]) + assert set(len(seq) for seq in seqs) == {seq_length} + + if nest_level == 1: + return list(zip(*seqs)) + else: + # Zip one level deeper. + zipped = [] + for i in range(seq_length): + # Get the i-th sequence in every sequence of sequences. + nested_seqs = [seq[i] for seq in seqs] + # Zip the i-th sequences. + zipped_nested = zip_nested(*nested_seqs, nest_level=nest_level - 1) + zipped.append(zipped_nested) + return zipped + + +def get_auc_for_max_fpr(fprs, y_values, max_fpr, scale_to_one=True): + """Compute AUCs for varying maximum FPRs.""" + assert fprs[0] == 0 and fprs[-1] == 1 + auc = trapz(x=fprs, + y=y_values, + x_max=max_fpr) + if scale_to_one: + auc /= max_fpr + return auc + + +def trapz(x, y, x_max=None): + """ + This function calculates the definit integral of a curve given by + x- and corresponding y-values. In contrast to, e.g., 'numpy.trapz()', + this function allows to define an upper bound to the integration range by + setting a value x_max. + + Points that do not have a finite x or y value will be ignored with a + warning. + + Args: + x: Samples from the domain of the function to integrate + Need to be sorted in ascending order. May contain the same value + multiple times. In that case, the order of the corresponding + y values will affect the integration with the trapezoidal rule. + y: Values of the function corresponding to x values. + x_max: Upper limit of the integration. The y value at max_x will be + determined by interpolating between its neighbors. Must not lie + outside the range of x. + + Returns: + Area under the curve. + """ + + x = np.array(x) + y = np.array(y) + finite_mask = np.logical_and(np.isfinite(x), np.isfinite(y)) + if not finite_mask.all(): + print("""WARNING: Not all x and y values passed to trapezoid(...) + are finite. Will continue with only the finite values.""") + x = x[finite_mask] + y = y[finite_mask] + + # Introduce a correction term if max_x is not an element of x. + correction = 0. + if x_max is not None: + if x_max not in x: + # Get the insertion index that would keep x sorted after + # np.insert(x, ins, x_max). + ins = bisect(x, x_max) + # x_max must be between the minimum and the maximum, so the + # insertion_point cannot be zero or len(x). + assert 0 < ins < len(x) + + # Calculate the correction term which is the integral between + # the last x[ins-1] and x_max. Since we do not know the exact value + # of y at x_max, we interpolate between y[ins] and y[ins-1]. + y_interp = y[ins - 1] + ((y[ins] - y[ins - 1]) * + (x_max - x[ins - 1]) / + (x[ins] - x[ins - 1])) + correction = 0.5 * (y_interp + y[ins - 1]) * (x_max - x[ins - 1]) + + # Cut off at x_max. + mask = x <= x_max + x = x[mask] + y = y[mask] + + # Return area under the curve using the trapezoidal rule. + return np.sum(0.5 * (y[1:] + y[:-1]) * (x[1:] - x[:-1])) + correction + + +def compute_classification_roc( + anomaly_scores_ok, + anomaly_scores_nok): + """ + Compute the ROC curve for anomaly classification on the image level. + + Args: + anomaly_scores_ok: List of real-valued anomaly scores of anomaly-free + samples. + anomaly_scores_nok: List of real-valued anomaly scores of anomalous + samples. + + Returns: + fprs: List of false positive rates. + tprs: List of correspoding true positive rates. + """ + # Merge anomaly scores into a single list, keeping track of the GT label. + # 0 = anomaly-free. 1 = anomalous. + anomaly_scores = [] + + anomaly_scores.extend([(x, 0) for x in anomaly_scores_ok]) + anomaly_scores.extend([(x, 1) for x in anomaly_scores_nok]) + + # Sort anomaly scores. + anomaly_scores = sorted(anomaly_scores, key=lambda x: x[0]) + + # Fetch the number of ok and nok samples. + num_scores = len(anomaly_scores) + num_nok = len(anomaly_scores_nok) + num_ok = len(anomaly_scores_ok) + + # Initially, every NOK sample is correctly classified as anomalous + # (tpr = 1.0), and every OK sample is incorrectly classified as anomalous + # (fpr = 1.0). + fprs = [1.0] + tprs = [1.0] + + # Keep track of the current number of false and true positive predictions. + num_fp = num_ok + num_tp = num_nok + + # Compute new true and false positive rates when successively increasing + # the threshold. Add points to the curve only when anomaly scores change. + prev_score = None + for i, (score, label) in enumerate(anomaly_scores): + if label == 0: + num_fp -= 1 + else: + num_tp -= 1 + + if (prev_score is None) or (score != prev_score) or ( + i == num_scores - 1): + fprs.append(num_fp / num_ok) + tprs.append(num_tp / num_nok) + prev_score = score + + # Return (FPR, TPR) pairs in increasing order. + fprs = fprs[::-1] + tprs = tprs[::-1] + + return fprs, tprs + + +def compute_classification_auc_roc( + anomaly_scores_ok, + anomaly_scores_nok): + """ + Compute the area under the ROC curve for anomaly classification. + + Args: + anomaly_scores_ok: List of real-valued anomaly scores of anomaly-free + samples. + anomaly_scores_nok: List of real-valued anomaly scores of anomalous + samples. + + Returns: + auc_roc: Area under the ROC curve. + """ + # Compute the ROC curve. + fprs, tprs = \ + compute_classification_roc(anomaly_scores_ok, anomaly_scores_nok) + + # Integrate its area. + return trapz(fprs, tprs) diff --git a/output/1/metrics/mvtec_loco/breakfast_box/metrics.json b/output/1/metrics/mvtec_loco/breakfast_box/metrics.json new file mode 100644 index 0000000..ac557d0 --- /dev/null +++ b/output/1/metrics/mvtec_loco/breakfast_box/metrics.json @@ -0,0 +1,34 @@ +{ + "classification": { + "auc_roc": { + "logical_anomalies": 0.8577840774864163, + "mean": 0.8643495550830774, + "structural_anomalies": 0.8709150326797386 + } + }, + "localization": { + "auc_spro": { + "logical_anomalies": { + "0.01": 0.31521530050056024, + "0.05": 0.5610771569853442, + "0.1": 0.6340104034451396, + "0.3": 0.6881612296912394, + "1.0": 0.8578198461499188 + }, + "mean": { + "0.01": 0.41612954859441526, + "0.05": 0.6486749008869972, + "0.1": 0.7160204429448767, + "0.3": 0.7692738960184431, + "1.0": 0.8990842342894558 + }, + "structural_anomalies": { + "0.01": 0.5170437966882703, + "0.05": 0.7362726447886503, + "0.1": 0.7980304824446136, + "0.3": 0.8503865623456469, + "1.0": 0.940348622428993 + } + } + } +} \ No newline at end of file diff --git a/output/1/metrics/mvtec_loco/juice_bottle/metrics.json b/output/1/metrics/mvtec_loco/juice_bottle/metrics.json new file mode 100644 index 0000000..7215eba --- /dev/null +++ b/output/1/metrics/mvtec_loco/juice_bottle/metrics.json @@ -0,0 +1,34 @@ +{ + "classification": { + "auc_roc": { + "logical_anomalies": 0.999475576865448, + "mean": 0.9991153348338104, + "structural_anomalies": 0.998755092802173 + } + }, + "localization": { + "auc_spro": { + "logical_anomalies": { + "0.01": 0.7533507221713671, + "0.05": 0.9448955814330564, + "0.1": 0.9723211710271423, + "0.3": 0.9906869133661499, + "1.0": 0.997194145766232 + }, + "mean": { + "0.01": 0.7188354364547923, + "0.05": 0.934457757479168, + "0.1": 0.966429600703532, + "0.3": 0.9881690707043218, + "1.0": 0.9963936253741217 + }, + "structural_anomalies": { + "0.01": 0.6843201507382175, + "0.05": 0.9240199335252797, + "0.1": 0.9605380303799216, + "0.3": 0.9856512280424936, + "1.0": 0.9955931049820113 + } + } + } +} \ No newline at end of file diff --git a/output/1/metrics/mvtec_loco/pushpins/metrics.json b/output/1/metrics/mvtec_loco/pushpins/metrics.json new file mode 100644 index 0000000..a20d2ea --- /dev/null +++ b/output/1/metrics/mvtec_loco/pushpins/metrics.json @@ -0,0 +1,34 @@ +{ + "classification": { + "auc_roc": { + "logical_anomalies": 0.9744386048733875, + "mean": 0.9697295905025374, + "structural_anomalies": 0.9650205761316871 + } + }, + "localization": { + "auc_spro": { + "logical_anomalies": { + "0.01": 0.801947378457531, + "0.05": 0.9567564232469552, + "0.1": 0.9779051221939025, + "0.3": 0.9925682746783943, + "1.0": 0.997770188586086 + }, + "mean": { + "0.01": 0.6875402421405222, + "0.05": 0.8560792925090542, + "0.1": 0.8940897060275753, + "0.3": 0.9316666576145333, + "1.0": 0.9689940667164954 + }, + "structural_anomalies": { + "0.01": 0.5731331058235135, + "0.05": 0.7554021617711533, + "0.1": 0.8102742898612482, + "0.3": 0.8707650405506723, + "1.0": 0.9402179448469048 + } + } + } +} \ No newline at end of file diff --git a/output/1/metrics/mvtec_loco/screw_bag/metrics.json b/output/1/metrics/mvtec_loco/screw_bag/metrics.json new file mode 100644 index 0000000..ca2934c --- /dev/null +++ b/output/1/metrics/mvtec_loco/screw_bag/metrics.json @@ -0,0 +1,34 @@ +{ + "classification": { + "auc_roc": { + "logical_anomalies": 0.5114275457700131, + "mean": 0.7018353242644548, + "structural_anomalies": 0.8922431027588964 + } + }, + "localization": { + "auc_spro": { + "logical_anomalies": { + "0.01": 0.14242798959526642, + "0.05": 0.5048525192770085, + "0.1": 0.655448339977079, + "0.3": 0.7581164208495348, + "1.0": 0.8985977955768583 + }, + "mean": { + "0.01": 0.3668483379301123, + "0.05": 0.6644346283291267, + "0.1": 0.7692685394649253, + "0.3": 0.8420500858658126, + "1.0": 0.9345342283969826 + }, + "structural_anomalies": { + "0.01": 0.5912686862649581, + "0.05": 0.8240167373812448, + "0.1": 0.8830887389527717, + "0.3": 0.9259837508820906, + "1.0": 0.9704706612171071 + } + } + } +} \ No newline at end of file diff --git a/output/1/metrics/mvtec_loco/splicing_connectors/metrics.json b/output/1/metrics/mvtec_loco/splicing_connectors/metrics.json new file mode 100644 index 0000000..202900b --- /dev/null +++ b/output/1/metrics/mvtec_loco/splicing_connectors/metrics.json @@ -0,0 +1,34 @@ +{ + "classification": { + "auc_roc": { + "logical_anomalies": 0.9413320883909119, + "mean": 0.9630535874480511, + "structural_anomalies": 0.9847750865051904 + } + }, + "localization": { + "auc_spro": { + "logical_anomalies": { + "0.01": 0.6754597991604356, + "0.05": 0.8714011082323736, + "0.1": 0.920679108226321, + "0.3": 0.957520732109652, + "1.0": 0.9830360861229218 + }, + "mean": { + "0.01": 0.7080500987505104, + "0.05": 0.8911055305049624, + "0.1": 0.9305460298082108, + "0.3": 0.9597699522128427, + "1.0": 0.9848496832293774 + }, + "structural_anomalies": { + "0.01": 0.7406403983405851, + "0.05": 0.9108099527775513, + "0.1": 0.9404129513901005, + "0.3": 0.9620191723160334, + "1.0": 0.9866632803358331 + } + } + } +} \ No newline at end of file diff --git a/output/1/metrics/mvtec_loco/summary.md b/output/1/metrics/mvtec_loco/summary.md new file mode 100644 index 0000000..102858b --- /dev/null +++ b/output/1/metrics/mvtec_loco/summary.md @@ -0,0 +1,49 @@ +# MVTec LOCO 评测结果汇总 + +- metrics_dir: `output/1/metrics/mvtec_loco` +- objects: 5 +- format: percent(0-100) + +## Image-level (AUC ROC) + +| object | mean | logical_anomalies | structural_anomalies | +| --- | ---: | ---: | ---: | +| breakfast_box | 86.43 | 85.78 | 87.09 | +| juice_bottle | 99.91 | 99.95 | 99.88 | +| pushpins | 96.97 | 97.44 | 96.50 | +| screw_bag | 70.18 | 51.14 | 89.22 | +| splicing_connectors | 96.31 | 94.13 | 98.48 | +| avg | 89.96 | 85.69 | 94.23 | + +## Localization (AU-sPRO mean) + +| object | @0.01 | @0.05 | @0.1 | @0.3 | @1.0 | +| --- | ---: | ---: | ---: | ---: | ---: | +| breakfast_box | 41.61 | 64.87 | 71.60 | 76.93 | 89.91 | +| juice_bottle | 71.88 | 93.45 | 96.64 | 98.82 | 99.64 | +| pushpins | 68.75 | 85.61 | 89.41 | 93.17 | 96.90 | +| screw_bag | 36.68 | 66.44 | 76.93 | 84.21 | 93.45 | +| splicing_connectors | 70.81 | 89.11 | 93.05 | 95.98 | 98.48 | +| avg | 57.95 | 79.90 | 85.53 | 89.82 | 95.68 | + +## Localization (AU-sPRO logical_anomalies) + +| object | @0.01 | @0.05 | @0.1 | @0.3 | @1.0 | +| --- | ---: | ---: | ---: | ---: | ---: | +| breakfast_box | 31.52 | 56.11 | 63.40 | 68.82 | 85.78 | +| juice_bottle | 75.34 | 94.49 | 97.23 | 99.07 | 99.72 | +| pushpins | 80.19 | 95.68 | 97.79 | 99.26 | 99.78 | +| screw_bag | 14.24 | 50.49 | 65.54 | 75.81 | 89.86 | +| splicing_connectors | 67.55 | 87.14 | 92.07 | 95.75 | 98.30 | +| avg | 53.77 | 76.78 | 83.21 | 87.74 | 94.69 | + +## Localization (AU-sPRO structural_anomalies) + +| object | @0.01 | @0.05 | @0.1 | @0.3 | @1.0 | +| --- | ---: | ---: | ---: | ---: | ---: | +| breakfast_box | 51.70 | 73.63 | 79.80 | 85.04 | 94.03 | +| juice_bottle | 68.43 | 92.40 | 96.05 | 98.57 | 99.56 | +| pushpins | 57.31 | 75.54 | 81.03 | 87.08 | 94.02 | +| screw_bag | 59.13 | 82.40 | 88.31 | 92.60 | 97.05 | +| splicing_connectors | 74.06 | 91.08 | 94.04 | 96.20 | 98.67 | +| avg | 62.13 | 83.01 | 87.85 | 91.90 | 96.67 | diff --git a/output/1/trainings/mvtec_loco/breakfast_box/autoencoder_final.pth b/output/1/trainings/mvtec_loco/breakfast_box/autoencoder_final.pth new file mode 100644 index 0000000..818e8e0 Binary files /dev/null and b/output/1/trainings/mvtec_loco/breakfast_box/autoencoder_final.pth differ diff --git a/output/1/trainings/mvtec_loco/breakfast_box/student_final.pth b/output/1/trainings/mvtec_loco/breakfast_box/student_final.pth new file mode 100644 index 0000000..357edcf Binary files /dev/null and b/output/1/trainings/mvtec_loco/breakfast_box/student_final.pth differ diff --git a/output/1/trainings/mvtec_loco/breakfast_box/teacher_final.pth b/output/1/trainings/mvtec_loco/breakfast_box/teacher_final.pth new file mode 100644 index 0000000..168e382 Binary files /dev/null and b/output/1/trainings/mvtec_loco/breakfast_box/teacher_final.pth differ diff --git a/output/1/trainings/mvtec_loco/juice_bottle/autoencoder_final.pth b/output/1/trainings/mvtec_loco/juice_bottle/autoencoder_final.pth new file mode 100644 index 0000000..4225ded Binary files /dev/null and b/output/1/trainings/mvtec_loco/juice_bottle/autoencoder_final.pth differ diff --git a/output/1/trainings/mvtec_loco/juice_bottle/student_final.pth b/output/1/trainings/mvtec_loco/juice_bottle/student_final.pth new file mode 100644 index 0000000..b9fed75 Binary files /dev/null and b/output/1/trainings/mvtec_loco/juice_bottle/student_final.pth differ diff --git a/output/1/trainings/mvtec_loco/juice_bottle/teacher_final.pth b/output/1/trainings/mvtec_loco/juice_bottle/teacher_final.pth new file mode 100644 index 0000000..168e382 Binary files /dev/null and b/output/1/trainings/mvtec_loco/juice_bottle/teacher_final.pth differ diff --git a/output/1/trainings/mvtec_loco/pushpins/autoencoder_final.pth b/output/1/trainings/mvtec_loco/pushpins/autoencoder_final.pth new file mode 100644 index 0000000..eed65c6 Binary files /dev/null and b/output/1/trainings/mvtec_loco/pushpins/autoencoder_final.pth differ diff --git a/output/1/trainings/mvtec_loco/pushpins/student_final.pth b/output/1/trainings/mvtec_loco/pushpins/student_final.pth new file mode 100644 index 0000000..4d6ca4b Binary files /dev/null and b/output/1/trainings/mvtec_loco/pushpins/student_final.pth differ diff --git a/output/1/trainings/mvtec_loco/pushpins/teacher_final.pth b/output/1/trainings/mvtec_loco/pushpins/teacher_final.pth new file mode 100644 index 0000000..168e382 Binary files /dev/null and b/output/1/trainings/mvtec_loco/pushpins/teacher_final.pth differ diff --git a/output/1/trainings/mvtec_loco/screw_bag/autoencoder_final.pth b/output/1/trainings/mvtec_loco/screw_bag/autoencoder_final.pth new file mode 100644 index 0000000..3fb988d Binary files /dev/null and b/output/1/trainings/mvtec_loco/screw_bag/autoencoder_final.pth differ diff --git a/output/1/trainings/mvtec_loco/screw_bag/student_final.pth b/output/1/trainings/mvtec_loco/screw_bag/student_final.pth new file mode 100644 index 0000000..ca5aef2 Binary files /dev/null and b/output/1/trainings/mvtec_loco/screw_bag/student_final.pth differ diff --git a/output/1/trainings/mvtec_loco/screw_bag/teacher_final.pth b/output/1/trainings/mvtec_loco/screw_bag/teacher_final.pth new file mode 100644 index 0000000..168e382 Binary files /dev/null and b/output/1/trainings/mvtec_loco/screw_bag/teacher_final.pth differ diff --git a/output/1/trainings/mvtec_loco/splicing_connectors/autoencoder_final.pth b/output/1/trainings/mvtec_loco/splicing_connectors/autoencoder_final.pth new file mode 100644 index 0000000..551e683 Binary files /dev/null and b/output/1/trainings/mvtec_loco/splicing_connectors/autoencoder_final.pth differ diff --git a/output/1/trainings/mvtec_loco/splicing_connectors/student_final.pth b/output/1/trainings/mvtec_loco/splicing_connectors/student_final.pth new file mode 100644 index 0000000..0206013 Binary files /dev/null and b/output/1/trainings/mvtec_loco/splicing_connectors/student_final.pth differ diff --git a/output/1/trainings/mvtec_loco/splicing_connectors/teacher_final.pth b/output/1/trainings/mvtec_loco/splicing_connectors/teacher_final.pth new file mode 100644 index 0000000..168e382 Binary files /dev/null and b/output/1/trainings/mvtec_loco/splicing_connectors/teacher_final.pth differ diff --git a/summarize_mvtec_loco_metrics.py b/summarize_mvtec_loco_metrics.py new file mode 100755 index 0000000..4822f8c --- /dev/null +++ b/summarize_mvtec_loco_metrics.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from statistics import mean +from typing import Any + + +DEFAULT_MAX_FPRS = ["0.01", "0.05", "0.1", "0.3", "1.0"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Summarize MVTec LOCO evaluation metrics.json files into a Markdown table." + ) + parser.add_argument( + "--metrics_dir", + default="output/1/metrics/mvtec_loco", + help="Directory containing per-object subdirs with metrics.json.", + ) + parser.add_argument( + "--output_md", + default=None, + help="Output markdown path (default: /summary.md).", + ) + parser.add_argument( + "--raw", + action="store_true", + default=False, + help="Output raw values in [0,1] instead of percent.", + ) + parser.add_argument( + "--digits", + type=int, + default=2, + help="Number of decimal digits to keep.", + ) + parser.add_argument( + "--max_fprs", + default=",".join(DEFAULT_MAX_FPRS), + help=f"Comma-separated max FPRs for AU-sPRO columns (default: {','.join(DEFAULT_MAX_FPRS)}).", + ) + return parser.parse_args() + + +def fmt(value: float, *, as_percent: bool, digits: int) -> str: + scaled = value * 100.0 if as_percent else value + return f"{scaled:.{digits}f}" + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> None: + args = parse_args() + + metrics_dir = Path(args.metrics_dir) + if not metrics_dir.exists(): + raise SystemExit(f"ERROR: metrics_dir not found: {metrics_dir}") + + output_md = Path(args.output_md) if args.output_md else (metrics_dir / "summary.md") + max_fprs = [s.strip() for s in str(args.max_fprs).split(",") if s.strip()] + if not max_fprs: + raise SystemExit("ERROR: --max_fprs is empty") + as_percent = not args.raw + + metrics_paths = sorted(metrics_dir.glob("*/metrics.json")) + if not metrics_paths: + raise SystemExit(f"ERROR: no metrics.json found under: {metrics_dir}") + + rows_classification: list[dict[str, Any]] = [] + rows_localization_by_group: dict[str, list[dict[str, Any]]] = { + "mean": [], + "logical_anomalies": [], + "structural_anomalies": [], + } + + for p in metrics_paths: + obj = p.parent.name + data = load_json(p) + + auc_roc = data["classification"]["auc_roc"] + auc_spro = data["localization"]["auc_spro"] + + rows_classification.append( + { + "object": obj, + "mean": float(auc_roc["mean"]), + "logical_anomalies": float(auc_roc["logical_anomalies"]), + "structural_anomalies": float(auc_roc["structural_anomalies"]), + } + ) + + for group_name, group_values in rows_localization_by_group.items(): + if group_name not in auc_spro: + raise SystemExit(f"ERROR: missing auc_spro[{group_name}] in {p}") + group_dict = auc_spro[group_name] + row_loc: dict[str, Any] = {"object": obj} + for k in max_fprs: + if k not in group_dict: + raise SystemExit(f"ERROR: missing auc_spro.{group_name}[{k}] in {p}") + row_loc[k] = float(group_dict[k]) + group_values.append(row_loc) + + # Macro averages + rows_classification_sorted = sorted(rows_classification, key=lambda r: r["object"]) + + rows_classification_sorted.append( + { + "object": "avg", + "mean": mean(r["mean"] for r in rows_classification_sorted), + "logical_anomalies": mean(r["logical_anomalies"] for r in rows_classification_sorted), + "structural_anomalies": mean(r["structural_anomalies"] for r in rows_classification_sorted), + } + ) + + rows_localization_sorted_by_group: dict[str, list[dict[str, Any]]] = {} + for group_name, group_rows in rows_localization_by_group.items(): + group_sorted = sorted(group_rows, key=lambda r: r["object"]) + avg_loc: dict[str, Any] = {"object": "avg"} + for k in max_fprs: + avg_loc[k] = mean(r[k] for r in group_sorted) + group_sorted.append(avg_loc) + rows_localization_sorted_by_group[group_name] = group_sorted + + # Render markdown + lines: list[str] = [] + lines.append("# MVTec LOCO 评测结果汇总") + lines.append("") + lines.append(f"- metrics_dir: `{metrics_dir}`") + lines.append(f"- objects: {len(metrics_paths)}") + lines.append(f"- format: {'percent(0-100)' if as_percent else 'raw(0-1)'}") + lines.append("") + + lines.append("## Image-level (AUC ROC)") + lines.append("") + lines.append("| object | mean | logical_anomalies | structural_anomalies |") + lines.append("| --- | ---: | ---: | ---: |") + for r in rows_classification_sorted: + lines.append( + "| {object} | {mean} | {logical} | {struct} |".format( + object=r["object"], + mean=fmt(r["mean"], as_percent=as_percent, digits=args.digits), + logical=fmt(r["logical_anomalies"], as_percent=as_percent, digits=args.digits), + struct=fmt(r["structural_anomalies"], as_percent=as_percent, digits=args.digits), + ) + ) + lines.append("") + + lines.append("## Localization (AU-sPRO mean)") + lines.append("") + def render_loc_table(group_name: str) -> None: + headers = ["object"] + [f"@{k}" for k in max_fprs] + lines.append("| " + " | ".join(headers) + " |") + lines.append("| " + " | ".join(["---"] + ["---:"] * len(max_fprs)) + " |") + for r in rows_localization_sorted_by_group[group_name]: + cells = [r["object"]] + [ + fmt(r[k], as_percent=as_percent, digits=args.digits) for k in max_fprs + ] + lines.append("| " + " | ".join(cells) + " |") + + render_loc_table("mean") + lines.append("") + + lines.append("## Localization (AU-sPRO logical_anomalies)") + lines.append("") + render_loc_table("logical_anomalies") + lines.append("") + + lines.append("## Localization (AU-sPRO structural_anomalies)") + lines.append("") + render_loc_table("structural_anomalies") + lines.append("") + + output_md.parent.mkdir(parents=True, exist_ok=True) + output_md.write_text("\n".join(lines), encoding="utf-8") + print(f"Wrote: {output_md}") + + +if __name__ == "__main__": + main() diff --git a/train_mvtec_loco_all.sh b/train_mvtec_loco_all.sh new file mode 100755 index 0000000..4a38ac5 --- /dev/null +++ b/train_mvtec_loco_all.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +PYTHON_BIN="${PYTHON_BIN:-python}" +DATASET="mvtec_loco" + +MODEL_SIZE="${MODEL_SIZE:-medium}" +WEIGHTS="${WEIGHTS:-models/teacher_medium.pth}" +IMAGENET_TRAIN_PATH="${IMAGENET_TRAIN_PATH:-../../datasets/ImageNet2012/train}" +MVTEC_LOCO_PATH="${MVTEC_LOCO_PATH:-../../datasets/mvtec_loco_anomaly_detection}" + +if [[ ! -d "$MVTEC_LOCO_PATH" ]]; then + echo "ERROR: MVTEC_LOCO_PATH not found: $MVTEC_LOCO_PATH" >&2 + exit 1 +fi + +if [[ ! -f "$WEIGHTS" ]]; then + echo "ERROR: weights file not found: $WEIGHTS" >&2 + exit 1 +fi + +if [[ -n "${OUTPUT_DIR:-}" ]]; then + OUT="$OUTPUT_DIR" +else + base="output" + mkdir -p "$base" + n=1 + while [[ -e "$base/$n" ]]; do + n=$((n + 1)) + done + OUT="$base/$n" +fi + +echo "Using --output_dir: $OUT" +echo "Scanning subdatasets under: $MVTEC_LOCO_PATH" + +mapfile -t subdatasets < <(find "$MVTEC_LOCO_PATH" -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | sort) +if [[ ${#subdatasets[@]} -eq 0 ]]; then + echo "ERROR: no subdataset dirs found under: $MVTEC_LOCO_PATH" >&2 + exit 1 +fi + +for subdataset in "${subdatasets[@]}"; do + echo "=== Training: dataset=$DATASET subdataset=$subdataset model_size=$MODEL_SIZE ===" + "$PYTHON_BIN" efficientad.py \ + --dataset "$DATASET" \ + --subdataset "$subdataset" \ + --model_size "$MODEL_SIZE" \ + --weights "$WEIGHTS" \ + --imagenet_train_path "$IMAGENET_TRAIN_PATH" \ + --mvtec_loco_path "$MVTEC_LOCO_PATH" \ + --output_dir "$OUT" +done