diff --git a/analysis/analysis_import_export.py b/analysis/analysis_import_export.py index 21a65de1b..c1df93b3b 100644 --- a/analysis/analysis_import_export.py +++ b/analysis/analysis_import_export.py @@ -5,7 +5,13 @@ from django.contrib.auth.models import User from django.core.serializers import serialize -from analysis.models import Analysis, GenomeBuild, AnnotationVersion, AnalysisEdge, AnalysisTemplateType +from analysis.models import ( + Analysis, + AnalysisEdge, + AnalysisTemplateType, + AnnotationVersion, + GenomeBuild, +) from analysis.models.nodes.node_utils import reload_analysis_nodes from analysis.serializers import AnalysisNodeSerializer, AnalysisSerializer diff --git a/analysis/analysis_templates.py b/analysis/analysis_templates.py index a1286e06f..deb0a48cc 100644 --- a/analysis/analysis_templates.py +++ b/analysis/analysis_templates.py @@ -1,18 +1,26 @@ import logging import re from dataclasses import dataclass -from typing import Optional, List +from typing import Optional from auditlog.context import disable_auditlog from django.contrib.auth.models import User -from analysis.models import Analysis, AnalysisNode, AnalysisTemplate, AnalysisTemplateRun, \ - AnalysisTemplateRunArgument, SampleAnalysisTemplateRun, CohortAnalysisTemplateRun, AutoLaunchAnalysisTemplate +from analysis.models import ( + Analysis, + AnalysisNode, + AnalysisTemplate, + AnalysisTemplateRun, + AnalysisTemplateRunArgument, + AutoLaunchAnalysisTemplate, + CohortAnalysisTemplateRun, + SampleAnalysisTemplateRun, +) from analysis.models.nodes.node_utils import get_toposorted_nodes, reload_analysis_nodes from analysis.related_analyses import get_related_analysis_details_for_samples from genes.models import ActiveSampleGeneList from library.guardian_utils import add_public_group_read_permission -from snpdb.models import Sample, GenomeBuild, Cohort +from snpdb.models import Cohort, GenomeBuild, Sample def run_analysis_template(analysis_template: AnalysisTemplate, @@ -123,7 +131,7 @@ def match(self) -> bool: return self.enrichment_kit_match and self.sample_regex_match -def get_auto_launch_analysis_template_matches(user, sample_enrichment_kit_name, sample_name) -> List[AutoLaunchAnalysisTemplateMatch]: +def get_auto_launch_analysis_template_matches(user, sample_enrichment_kit_name, sample_name) -> list[AutoLaunchAnalysisTemplateMatch]: matches = [] templates_qs = AnalysisTemplate.filter_for_user(user) for auto_launch in AutoLaunchAnalysisTemplate.objects.filter(template__in=templates_qs): diff --git a/analysis/apps.py b/analysis/apps.py index 9e77271e0..267afc8ba 100644 --- a/analysis/apps.py +++ b/analysis/apps.py @@ -9,14 +9,21 @@ class AnalysisConfig(AppConfig): def ready(self): # pylint: disable=import-outside-toplevel,unused-import # imported to activate receivers - from analysis.signals import analysis_search, analysis_health_check from analysis.models import VariantTag - from analysis.signals.signal_handlers import variant_tag_create, variant_tag_delete, \ - handle_vcf_import_success, handle_active_sample_gene_list_created - from analysis.signals.source_data_invalidation import handle_sample_pre_delete, \ - handle_cohort_pre_delete, handle_trio_pre_delete, handle_pedigree_pre_delete, \ - handle_quad_pre_delete + from analysis.signals.signal_handlers import ( + handle_active_sample_gene_list_created, + handle_vcf_import_success, + variant_tag_create, + variant_tag_delete, + ) + from analysis.signals.source_data_invalidation import ( + handle_cohort_pre_delete, + handle_pedigree_pre_delete, + handle_quad_pre_delete, + handle_sample_pre_delete, + handle_trio_pre_delete, + ) from genes.models import ActiveSampleGeneList from pedigree.models import Pedigree from snpdb.models import Cohort, Quad, Sample, Trio diff --git a/analysis/forms/forms.py b/analysis/forms/forms.py index 587ab634c..509952bea 100644 --- a/analysis/forms/forms.py +++ b/analysis/forms/forms.py @@ -4,16 +4,30 @@ from collections import defaultdict from crispy_forms.bootstrap import FieldWithButtons, StrictButton -from crispy_forms.layout import Layout, Field +from crispy_forms.layout import Field, Layout from dal import forward from django import forms from django.core.exceptions import ValidationError from django.forms import inlineformset_factory from django.forms.widgets import TextInput -from analysis.models import Analysis, NodeGraphType, FilterNodeItem, AnalysisTemplate, AnalysisTemplateVersion, \ - AnalysisNode, CandidateStatus, AutoLaunchAnalysisTemplate -from analysis.models.enums import SNPMatrix, AnalysisTemplateType, TrioSample, QuadSample, AnalysisType +from analysis.models import ( + Analysis, + AnalysisNode, + AnalysisTemplate, + AnalysisTemplateVersion, + AutoLaunchAnalysisTemplate, + CandidateStatus, + FilterNodeItem, + NodeGraphType, +) +from analysis.models.enums import ( + AnalysisTemplateType, + AnalysisType, + QuadSample, + SNPMatrix, + TrioSample, +) from analysis.models.models_karyomapping import KaryomappingGene from analysis.models.nodes.node_types import get_nodes_by_classification from annotation.models.models import AnnotationVersion, VariantAnnotationVersion @@ -23,7 +37,7 @@ from library.guardian_utils import assign_permission_to_user_and_groups from seqauto.models import EnrichmentKit from snpdb.forms import GenomeBuildAutocompleteForwardMixin, UserSettingsGenomeBuildMixin -from snpdb.models import CustomColumnsCollection, VariantGridColumn, Trio, UserSettings +from snpdb.models import CustomColumnsCollection, Trio, UserSettings, VariantGridColumn from uicore.utils.form_helpers import form_helper_horizontal diff --git a/analysis/forms/forms_nodes.py b/analysis/forms/forms_nodes.py index 6888b613b..cb7fa04ad 100644 --- a/analysis/forms/forms_nodes.py +++ b/analysis/forms/forms_nodes.py @@ -3,13 +3,13 @@ from dal import forward from django import forms from django.forms.models import fields_for_model -from django.forms.widgets import TextInput, HiddenInput +from django.forms.widgets import HiddenInput, TextInput from django.utils.text import slugify from django_starfield import Stars from analysis import models -from analysis.models import AnalysisNode, AnalysisTemplateType, Analysis, MOINode -from analysis.models.nodes.analysis_node import NodeVCFFilter, NodeAlleleFrequencyFilter +from analysis.models import Analysis, AnalysisNode, AnalysisTemplateType, MOINode +from analysis.models.nodes.analysis_node import NodeAlleleFrequencyFilter, NodeVCFFilter from analysis.models.nodes.filters.conservation_node import ConservationNode from analysis.models.nodes.filters.damage_node import DamageNode from analysis.models.nodes.filters.gene_list_node import GeneListNode @@ -23,24 +23,27 @@ from analysis.models.nodes.filters.venn_node import VennNode from analysis.models.nodes.sources.all_variants_node import AllVariantsNode from analysis.models.nodes.sources.classifications_node import ClassificationsNode -from analysis.models.nodes.sources.cohort_node import CohortNode, CohortNodeZygosityFiltersCollection, \ - CohortNodeZygosityFilter +from analysis.models.nodes.sources.cohort_node import ( + CohortNode, + CohortNodeZygosityFilter, + CohortNodeZygosityFiltersCollection, +) from analysis.models.nodes.sources.pedigree_node import PedigreeNode -from analysis.models.nodes.sources.sample_node import SampleNode from analysis.models.nodes.sources.quad_node import QuadNode +from analysis.models.nodes.sources.sample_node import SampleNode from analysis.models.nodes.sources.trio_node import TrioNode from annotation.models import VariantAnnotation from annotation.pathogenicity_predictions import TOOLS from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.hgvs import get_hgvs_variant_coordinate, get_hgvs_variant, HGVSException -from genes.models import GeneListCategory, CustomTextGeneList, GeneList, PanelAppPanel +from genes.hgvs import HGVSException, get_hgvs_variant, get_hgvs_variant_coordinate +from genes.models import CustomTextGeneList, GeneList, GeneListCategory, PanelAppPanel from library.django_utils.autocomplete_utils import ModelSelect2, ModelSelect2Multiple from library.forms import NumberInput from library.utils import sha256sum_str from ontology.models import OntologyTerm from patients.models_enums import GnomADPopulation from snpdb.forms import GenomeBuildAutocompleteForwardMixin -from snpdb.models import GenomicInterval, Sample, VCFFilter, Tag, Lab +from snpdb.models import GenomicInterval, Lab, Sample, Tag, VCFFilter from snpdb.models.models_genome import Contig # Can use this for ModelForm.exclude to only use node specific fields diff --git a/analysis/grid_export.py b/analysis/grid_export.py index 99cd3cf41..10d5eeea4 100644 --- a/analysis/grid_export.py +++ b/analysis/grid_export.py @@ -2,11 +2,12 @@ import operator import re from collections import Counter +from collections.abc import Iterator from io import StringIO -from typing import Iterator, Optional +from typing import Optional -from vcf import Writer, Reader -from vcf.model import _Substitution, _Record, make_calldata_tuple, _Call +from vcf import Reader, Writer +from vcf.model import _Call, _Record, _Substitution, make_calldata_tuple from analysis.grids import ExportVariantGrid from analysis.models import AnalysisNode @@ -16,7 +17,7 @@ from library.django_utils.jqgrid_view import grid_export_csv from library.utils import StashFile from patients.models_enums import Zygosity -from snpdb.models import Sample, ColumnVCFInfo, VCFInfoTypes +from snpdb.models import ColumnVCFInfo, Sample, VCFInfoTypes from snpdb.vcf_export_utils import get_vcf_header_from_contigs diff --git a/analysis/grids.py b/analysis/grids.py index d08417140..99aee8b65 100644 --- a/analysis/grids.py +++ b/analysis/grids.py @@ -2,24 +2,39 @@ import operator import time from collections import defaultdict +from collections.abc import Callable from functools import reduce -from typing import Optional, Any, Callable +from typing import Any, Optional import pandas as pd from auditlog.models import LogEntry from django.conf import settings from django.contrib.postgres.aggregates import StringAgg from django.core.exceptions import PermissionDenied -from django.db.models import Max, F, Q, QuerySet +from django.db.models import F, Max, Q, QuerySet from django.db.models.functions import Substr from django.shortcuts import get_object_or_404 from django.urls.base import reverse from django.utils.functional import SimpleLazyObject -from analysis.models import Analysis, AnalysisNode, NodeCount, NodeStatus, AnalysisTemplate, GroupOperation, \ - CandidateSearchRun, CandidateSearchType, Candidate, CandidateStatus, AnalysisType +from analysis.models import ( + Analysis, + AnalysisNode, + AnalysisTemplate, + AnalysisType, + Candidate, + CandidateSearchRun, + CandidateSearchType, + CandidateStatus, + GroupOperation, + NodeCount, + NodeStatus, +) from analysis.models.models_karyomapping import KaryomappingAnalysis -from analysis.models.nodes.analysis_node import get_extra_filters_q, NodeColumnSummaryCacheCollection +from analysis.models.nodes.analysis_node import ( + NodeColumnSummaryCacheCollection, + get_extra_filters_q, +) from analysis.views.analysis_permissions import get_node_subclass_or_404 from annotation.models import HumanProteinAtlasAnnotation from genes.grids import GeneListGenesColumns @@ -28,18 +43,29 @@ from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from library.pandas_jqgrid import DataFrameJqGrid from library.unit_percent import get_allele_frequency_formatter -from library.utils import update_dict_of_dict_values, JsonDataType, sha256sum_str +from library.utils import JsonDataType, sha256sum_str, update_dict_of_dict_values from ontology.grids import AbstractOntologyGenesGrid -from ontology.models import OntologyTermRelation, GeneDiseaseClassification, OntologyVersion +from ontology.models import GeneDiseaseClassification, OntologyTermRelation, OntologyVersion from patients.models_enums import Zygosity -from snpdb.grid_columns.custom_columns import get_custom_column_fields_override_and_sample_position, \ - get_variantgrid_extra_annotate -from snpdb.grid_columns.grid_sample_columns import get_available_format_columns, \ - get_variantgrid_zygosity_annotation_kwargs +from snpdb.grid_columns.custom_columns import ( + get_custom_column_fields_override_and_sample_position, + get_variantgrid_extra_annotate, +) +from snpdb.grid_columns.grid_sample_columns import ( + get_available_format_columns, + get_variantgrid_zygosity_annotation_kwargs, +) from snpdb.grids import AbstractVariantGrid -from snpdb.models import VariantGridColumn, UserGridConfig, VCFFilter, Sample, CohortGenotype, ProcessingStatus +from snpdb.models import ( + CohortGenotype, + ProcessingStatus, + Sample, + UserGridConfig, + VariantGridColumn, + VCFFilter, +) from snpdb.models.models_genome import GenomeBuild -from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder, CellData +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder class VariantGrid(AbstractVariantGrid): @@ -230,7 +256,7 @@ def get_grid_genotype_columns_and_overrides(cohorts, visibility, # Some legacy data (Missing data in FreeBayes before PythonKnownVariantsImporter v12) has -2147483647 for # empty values (what CyVCF2 returns using format()) @see https://github.com/SACGF/variantgrid/issues/59 MISSING_VALUES = [CohortGenotype.MISSING_NUMBER_VALUE, CohortGenotype.MISSING_FT_VALUE, -2147483648] - packed_data_replace.update({mv: VariantGrid.GENOTYPE_COLUMNS_MISSING_VALUE for mv in MISSING_VALUES}) + packed_data_replace.update(dict.fromkeys(MISSING_VALUES, VariantGrid.GENOTYPE_COLUMNS_MISSING_VALUE)) # We now have separate aliases for packed data, so each cohort handled separately sample_cohort_index = {} diff --git a/analysis/management/commands/analyses_process_incomplete_nodes.py b/analysis/management/commands/analyses_process_incomplete_nodes.py index b55da7fa6..a78b64364 100644 --- a/analysis/management/commands/analyses_process_incomplete_nodes.py +++ b/analysis/management/commands/analyses_process_incomplete_nodes.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand -from analysis.models import NodeStatus, AnalysisNode +from analysis.models import AnalysisNode, NodeStatus from analysis.models.models_analysis import Analysis from analysis.models.nodes.node_utils import update_analysis diff --git a/analysis/management/commands/analysis_create_default_templates.py b/analysis/management/commands/analysis_create_default_templates.py index cedefc668..b2e764faa 100644 --- a/analysis/management/commands/analysis_create_default_templates.py +++ b/analysis/management/commands/analysis_create_default_templates.py @@ -5,8 +5,8 @@ from django.core.management.base import BaseCommand from analysis.analysis_import_export import analysis_import -from analysis.models import AnalysisTemplateType, AnalysisTemplate, AnalysisTemplateVersion -from library.guardian_utils import admin_bot, add_public_group_read_permission +from analysis.models import AnalysisTemplate, AnalysisTemplateType, AnalysisTemplateVersion +from library.guardian_utils import add_public_group_read_permission, admin_bot from snpdb.models import GenomeBuild diff --git a/analysis/management/commands/fix_analysis_permissions.py b/analysis/management/commands/fix_analysis_permissions.py index 2b0ac9e33..27ee73e1d 100644 --- a/analysis/management/commands/fix_analysis_permissions.py +++ b/analysis/management/commands/fix_analysis_permissions.py @@ -1,6 +1,7 @@ from django.core.management.base import BaseCommand from analysis.models.models_analysis import Analysis + # Analysis templates didn't set permissions correctly until fix on October 19th 2020 # Need to fix legacy data created in systems before this fix: https://github.com/SACGF/variantgrid/issues/84 # Can be removed once environments fixed diff --git a/analysis/management/commands/fix_legacy_phenotype_nodes.py b/analysis/management/commands/fix_legacy_phenotype_nodes.py index e2630123b..cbc6bbc64 100644 --- a/analysis/management/commands/fix_legacy_phenotype_nodes.py +++ b/analysis/management/commands/fix_legacy_phenotype_nodes.py @@ -9,7 +9,7 @@ from django.db import IntegrityError from analysis.models import PhenotypeNodeOntologyTerm -from ontology.models import OntologyTerm, OntologyService +from ontology.models import OntologyService, OntologyTerm class Command(BaseCommand): diff --git a/analysis/management/commands/fix_liftover_existing_variant_tags.py b/analysis/management/commands/fix_liftover_existing_variant_tags.py index 1ac987ab2..2465c9697 100644 --- a/analysis/management/commands/fix_liftover_existing_variant_tags.py +++ b/analysis/management/commands/fix_liftover_existing_variant_tags.py @@ -5,7 +5,7 @@ from library.utils import iter_fixed_chunks from snpdb.clingen_allele import populate_clingen_alleles_for_variants from snpdb.liftover import create_liftover_pipelines -from snpdb.models import GenomeBuild, ImportSource, Variant, Allele +from snpdb.models import Allele, GenomeBuild, ImportSource, Variant class Command(BaseCommand): diff --git a/analysis/management/commands/fix_variant_tag_permissions.py b/analysis/management/commands/fix_variant_tag_permissions.py index 3557ce47a..d3022c750 100644 --- a/analysis/management/commands/fix_variant_tag_permissions.py +++ b/analysis/management/commands/fix_variant_tag_permissions.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand -from guardian.shortcuts import get_groups_with_perms, get_group_perms, assign_perm +from guardian.shortcuts import assign_perm, get_group_perms, get_groups_with_perms from analysis.models.models_analysis import Analysis from analysis.models.models_variant_tag import VariantTag diff --git a/analysis/management/commands/profile_analysis_nodes.py b/analysis/management/commands/profile_analysis_nodes.py index 68e41ae4b..52ab8f22b 100644 --- a/analysis/management/commands/profile_analysis_nodes.py +++ b/analysis/management/commands/profile_analysis_nodes.py @@ -29,13 +29,12 @@ import socket import sys import time -import traceback from datetime import datetime from django.conf import settings from django.core.management.base import BaseCommand, CommandError from django.db import connection -from django.db.models import F, Q +from django.db.models import Q from django.db.models.functions import Substr as DjSubstr from analysis.models import Analysis @@ -45,7 +44,6 @@ from snpdb.models import Cohort, Sample, Trio, Variant, VariantCollection from snpdb.models.models_enums import ProcessingStatus - CSV_FIELDS = [ "source", "analysis_id", diff --git a/analysis/models.py b/analysis/models.py index 32288046d..0a8c8a6b1 100644 --- a/analysis/models.py +++ b/analysis/models.py @@ -1,28 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from analysis.models.gene_counts import NodeGenesCountCollection, NodeGenesCount -from analysis.models.models_analysis import Analysis, AnalysisLock, AnalysisNodeCountConfiguration, \ - AnalysisNodeCountConfigRecord, AnalysisVariable, AnalysisTemplate, AnalysisTemplateVersion, AnalysisTemplateRun, \ - AnalysisTemplateRunArgument, CohortAnalysisTemplateRun, SampleAnalysisTemplateRun -from analysis.models.models_karyomapping import KaryomappingAnalysis, KaryomappingGene, GenomeKaryomappingCounts, \ - ContigKaryomappingCounts -from analysis.models.models_variant_tag import VariantTagsImport, ImportedVariantTag, VariantTag -from analysis.models.mutational_signatures import MutationalSignatureCalculator, MutationalSignature, \ - MutationalSignatureMinimisationResult, MutationalSignatureMutationCount -from analysis.models.nodes.analysis_node import NodeTask, NodeWiki, AnalysisNodeAlleleSource, NodeVersion, NodeCache, \ - NodeCount, NodeColumnSummaryCacheCollection, NodeColumnSummaryData, NodeVCFFilter, NodeAlleleFrequencyFilter, \ - NodeAlleleFrequencyRange, AnalysisClassification -from analysis.models.nodes.filters.filter_node import FilterNodeItem -from analysis.models.nodes.filters.gene_list_node import GeneListNodeGeneList, GeneListNodePanelAppPanel -from analysis.models.nodes.filters.moi_node import MOINodeOntologyTerm, MOINodeModeOfInheritance, MOINodeSubmitter -from analysis.models.nodes.filters.phenotype_node import PhenotypeNodeOntologyTerm -from analysis.models.nodes.filters.population_node import PopulationNodeGnomADPopulation -from analysis.models.nodes.filters.selected_in_parent_node import NodeVariant -from analysis.models.nodes.filters.tag_node import TagNodeTag -from analysis.models.nodes.filters.venn_node import VennNodeCache -from analysis.models.nodes.node_types import NodeGraphType -from analysis.models.nodes.sources.all_variants_node import AllVariantsNode -from analysis.models.nodes.sources.classifications_node import ClassificationsNodeLab -from analysis.models.nodes.sources.cohort_node import CohortNode, CohortNodeZygosityFiltersCollection, \ - CohortNodeZygosityFilter # pylint: enable=unused-import diff --git a/analysis/models/models_analysis.py b/analysis/models/models_analysis.py index 4b0a792cc..677f409cc 100644 --- a/analysis/models/models_analysis.py +++ b/analysis/models/models_analysis.py @@ -1,31 +1,37 @@ from collections import defaultdict from functools import cached_property -from typing import Union, Optional +from typing import Optional, Union from auditlog.context import disable_auditlog from auditlog.models import LogEntry from auditlog.registry import auditlog from django.apps import apps from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.db import models -from django.db.models import Model, Q, Count, Max, QuerySet -from django.db.models.deletion import SET_NULL, CASCADE, SET_DEFAULT, PROTECT, ProtectedError +from django.db.models import Count, Max, Model, Q, QuerySet +from django.db.models.deletion import CASCADE, PROTECT, SET_DEFAULT, SET_NULL, ProtectedError from django.db.models.signals import pre_delete from django.dispatch import receiver from django.urls.base import reverse from django.utils import timezone from django_extensions.db.models import TimeStampedModel -from analysis.models.enums import AnalysisType, AnalysisTemplateType +from analysis.models.enums import AnalysisTemplateType, AnalysisType from annotation.models import AnnotationVersion, InvalidAnnotationVersionError from genes.models import CanonicalTranscriptCollection from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin from library.guardian_utils import admin_bot, assign_permission_to_user_and_groups from library.preview_request import PreviewModelMixin from seqauto.models import EnrichmentKit -from snpdb.models import CustomColumnsCollection, CustomColumn, \ - UserSettings, AbstractNodeCountSettings, Sample, Cohort +from snpdb.models import ( + AbstractNodeCountSettings, + Cohort, + CustomColumn, + CustomColumnsCollection, + Sample, + UserSettings, +) from snpdb.models.models_enums import BuiltInFilters from snpdb.models.models_genome import GenomeBuild @@ -235,11 +241,11 @@ def get_samples(self) -> list[Sample]: samples = set() for node in self.analysisnode_set.filter(analysisnode_parent__isnull=True).select_subclasses(): samples.update(node.get_samples_from_node_only_not_ancestors()) - return list(sorted(samples)) + return sorted(samples) def clone(self, user: User = None): """ user - if provided, assign new ownership """ - from analysis.models import AnalysisNode, AnalysisEdge + from analysis.models import AnalysisEdge, AnalysisNode from analysis.models.nodes.node_utils import get_toposorted_nodes # No point documenting all this copying diff --git a/analysis/models/models_candidate_search.py b/analysis/models/models_candidate_search.py index 96a2743d5..9b27aee91 100644 --- a/analysis/models/models_candidate_search.py +++ b/analysis/models/models_candidate_search.py @@ -18,13 +18,13 @@ from model_utils.models import TimeStampedModel from analysis.models import Analysis -from annotation.models import ClinVar, AnnotationVersion +from annotation.models import AnnotationVersion, ClinVar from classification.models import Classification from library.django_utils import count_values_qs from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin from library.utils.django_utils import get_cached_project_git_hash from patients.models_enums import Zygosity -from snpdb.models import Variant, Sample, ProcessingStatus +from snpdb.models import ProcessingStatus, Sample, Variant class CandidateSearchType(models.TextChoices): diff --git a/analysis/models/models_karyomapping.py b/analysis/models/models_karyomapping.py index 43d97b47d..0a7f010f3 100644 --- a/analysis/models/models_karyomapping.py +++ b/analysis/models/models_karyomapping.py @@ -1,5 +1,5 @@ import operator -from collections import defaultdict, Counter +from collections import Counter, defaultdict from functools import reduce from django.contrib.auth.models import User @@ -9,7 +9,9 @@ from django.urls.base import reverse from django_extensions.db.models import TimeStampedModel -from annotation.annotation_version_querysets import get_variant_queryset_for_latest_annotation_version +from annotation.annotation_version_querysets import ( + get_variant_queryset_for_latest_annotation_version, +) from genes.models import GeneSymbol from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin from patients.models_enums import Zygosity diff --git a/analysis/models/models_variant_tag.py b/analysis/models/models_variant_tag.py index ec152aecb..725ea3f83 100644 --- a/analysis/models/models_variant_tag.py +++ b/analysis/models/models_variant_tag.py @@ -1,15 +1,15 @@ from typing import Union -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.db import models -from django.db.models import CASCADE, SET_NULL, PROTECT, Q, QuerySet, Count, Max +from django.db.models import CASCADE, PROTECT, SET_NULL, Count, Max, Q, QuerySet from django_extensions.db.models import TimeStampedModel from analysis.models.enums import TagLocation from analysis.models.models_analysis import Analysis from analysis.models.nodes.analysis_node import AnalysisNode from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin -from snpdb.models import Variant, GenomeBuild, Tag, VariantAllele, Allele +from snpdb.models import Allele, GenomeBuild, Tag, Variant, VariantAllele class VariantTagsImport(TimeStampedModel): diff --git a/analysis/models/nodes/analysis_node.py b/analysis/models/nodes/analysis_node.py index 8c996bfb6..ee9b57c26 100644 --- a/analysis/models/nodes/analysis_node.py +++ b/analysis/models/nodes/analysis_node.py @@ -3,10 +3,11 @@ import logging import operator from collections import defaultdict +from collections.abc import Sequence from functools import cached_property, reduce from random import random from time import time -from typing import Sequence, Optional +from typing import Optional from auditlog.context import disable_auditlog from auditlog.models import AuditlogHistoryField, LogEntry @@ -17,32 +18,56 @@ from django.core.cache import cache from django.core.exceptions import FieldError from django.db import connection, models -from django.db.models import Value, IntegerField, QuerySet +from django.db.models import IntegerField, QuerySet, Value from django.db.models.aggregates import Count from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query_utils import Q from django.db.models.signals import post_delete from django.dispatch import receiver from django.utils import timezone -from django_dag.models import node_factory, edge_factory +from django_dag.models import edge_factory, node_factory from django_extensions.db.models import TimeStampedModel from model_utils.managers import InheritanceManager -from analysis.exceptions import NonFatalNodeError, NodeParentErrorsException, NodeConfigurationException, \ - NodeParentNotReadyException, NodeNotFoundException, NodeOutOfDateException -from analysis.models.enums import GroupOperation, NodeStatus, NodeColors, NodeErrorSource, AnalysisTemplateType +from analysis.exceptions import ( + NodeConfigurationException, + NodeNotFoundException, + NodeOutOfDateException, + NodeParentErrorsException, + NodeParentNotReadyException, + NonFatalNodeError, +) +from analysis.models.enums import ( + AnalysisTemplateType, + GroupOperation, + NodeColors, + NodeErrorSource, + NodeStatus, +) from analysis.models.models_analysis import Analysis from analysis.models.nodes.node_counts import get_extra_filters_q, get_node_counts_and_labels_dict from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version from classification.models import Classification from library.constants import DAY_SECS, MINUTE_SECS from library.django_utils import thread_safe_unique_together_get_or_create -from library.log_utils import report_event, log_traceback -from library.utils import format_percent, add_exception_note +from library.log_utils import log_traceback, report_event +from library.utils import add_exception_note, format_percent from library.utils.database_utils import queryset_to_sql from library.utils.django_utils import get_model_content_type_dict -from snpdb.models import BuiltInFilters, Sample, Variant, VCFFilter, Wiki, Cohort, VariantCollection, \ - ProcessingStatus, GenomeBuild, AlleleSource, Contig, SampleFilePath +from snpdb.models import ( + AlleleSource, + BuiltInFilters, + Cohort, + Contig, + GenomeBuild, + ProcessingStatus, + Sample, + SampleFilePath, + Variant, + VariantCollection, + VCFFilter, + Wiki, +) from snpdb.variant_collection import write_sql_to_variant_collection diff --git a/analysis/models/nodes/cohort_mixin.py b/analysis/models/nodes/cohort_mixin.py index e79bba4c6..aff9ee20b 100644 --- a/analysis/models/nodes/cohort_mixin.py +++ b/analysis/models/nodes/cohort_mixin.py @@ -6,11 +6,11 @@ from django.db.models import Q from analysis.models.enums import GroupOperation -from analysis.models.nodes.analysis_node import NodeVCFFilter, NodeAlleleFrequencyFilter +from analysis.models.nodes.analysis_node import NodeAlleleFrequencyFilter, NodeVCFFilter from annotation.annotation_versions import get_lowest_unannotated_variant_id from patients.models_enums import Zygosity from snpdb.archive import DataArchivedError -from snpdb.models import VCFFilter, Cohort, Sample, CohortGenotypeCollection +from snpdb.models import Cohort, CohortGenotypeCollection, Sample, VCFFilter from upload.models import UploadedVCF @@ -74,7 +74,7 @@ def _get_cohorts_and_sample_visibility(self): visibility = {} if cohort := self._get_cohort(): cohorts = [cohort] - visibility = {s: cohort.has_genotype for s in cohort.get_samples()} + visibility = dict.fromkeys(cohort.get_samples(), cohort.has_genotype) return cohorts, visibility @property @@ -162,7 +162,7 @@ def get_cohort_and_arg_q_dict(self) -> tuple[Cohort, dict[Optional[str], dict[st arg_q_dict[vc.variant_collection_alias] = {str(q_vc): q_vc} else: missing = [Zygosity.UNKNOWN_ZYGOSITY, Zygosity.MISSING] - sample_zygosities_dict = {s: missing for s in cohort.get_samples()} + sample_zygosities_dict = dict.fromkeys(cohort.get_samples(), missing) q_sub = cgc.get_zygosity_q(sample_zygosities_dict, exclude=True) q_and.append(q_sub) q_and.extend(self._get_q_and_list()) diff --git a/analysis/models/nodes/filter_child.py b/analysis/models/nodes/filter_child.py index 6d692193e..df71ea45e 100644 --- a/analysis/models/nodes/filter_child.py +++ b/analysis/models/nodes/filter_child.py @@ -4,7 +4,7 @@ from analysis.models.nodes.filters.gene_list_node import GeneListNode from analysis.models.nodes.node_utils import update_analysis from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.models import GeneListCategory, CustomTextGeneList +from genes.models import CustomTextGeneList, GeneListCategory from snpdb.models import VariantGridColumn diff --git a/analysis/models/nodes/filters/damage_node.py b/analysis/models/nodes/filters/damage_node.py index db25b92ff..06d9068d4 100644 --- a/analysis/models/nodes/filters/damage_node.py +++ b/analysis/models/nodes/filters/damage_node.py @@ -10,8 +10,12 @@ from analysis.models.nodes.analysis_node import AnalysisNode from annotation.models.damage_enums import ( - PathogenicityImpact, ALoFTPrediction, AlphaMissensePrediction, - ClinPredPrediction, MetaRNNPrediction, PrimateAIPrediction, + ALoFTPrediction, + AlphaMissensePrediction, + ClinPredPrediction, + MetaRNNPrediction, + PathogenicityImpact, + PrimateAIPrediction, ) from annotation.models.models import VariantAnnotation from annotation.pathogenicity_predictions import TOOLS, TOOLS_BY_PRED_FIELD diff --git a/analysis/models/nodes/filters/filter_node.py b/analysis/models/nodes/filters/filter_node.py index 659834385..774a814e4 100644 --- a/analysis/models/nodes/filters/filter_node.py +++ b/analysis/models/nodes/filters/filter_node.py @@ -8,7 +8,7 @@ from analysis.models.nodes.analysis_node import AnalysisNode, NodeAuditLogMixin from library.jqgrid.jqgrid import JqGrid, format_operation -from snpdb.models import VariantGridColumn, Variant +from snpdb.models import Variant, VariantGridColumn # TODO: This node has quite a few redundant operations - e.g. it will filter the queryset diff --git a/analysis/models/nodes/filters/gene_list_node.py b/analysis/models/nodes/filters/gene_list_node.py index fb50aa460..bce5cab4f 100644 --- a/analysis/models/nodes/filters/gene_list_node.py +++ b/analysis/models/nodes/filters/gene_list_node.py @@ -5,7 +5,7 @@ from auditlog.registry import auditlog from django.db import models from django.db.models import Q -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.signals import post_delete from django.dispatch.dispatcher import receiver from rest_framework.exceptions import NotFound @@ -16,12 +16,18 @@ from analysis.models.nodes.gene_coverage_mixin import GeneCoverageMixin from annotation.models import VariantTranscriptAnnotation from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.models import GeneList, CustomTextGeneList, SampleGeneList, \ - ActiveSampleGeneList, PanelAppPanel, PanelAppPanelLocalCache +from genes.models import ( + ActiveSampleGeneList, + CustomTextGeneList, + GeneList, + PanelAppPanel, + PanelAppPanelLocalCache, + SampleGeneList, +) from genes.models_enums import PanelAppConfidence from genes.panel_app import get_panel_app_local_cache from pathtests.models import PathologyTestVersion -from snpdb.models import Sample, Contig +from snpdb.models import Contig, Sample from snpdb.models.models_enums import ImportStatus @@ -110,7 +116,7 @@ def _get_sorted_gene_names(self): gene_names_set = set() for gene_list in self.get_gene_lists(): gene_names_set.update(gene_list.get_gene_names()) - return list(sorted(gene_names_set)) + return sorted(gene_names_set) def _get_gene_list_names(self) -> list[str]: gene_list_names = [] diff --git a/analysis/models/nodes/filters/intersection_node.py b/analysis/models/nodes/filters/intersection_node.py index 677f9bfee..26d1a2481 100644 --- a/analysis/models/nodes/filters/intersection_node.py +++ b/analysis/models/nodes/filters/intersection_node.py @@ -13,8 +13,14 @@ from analysis.models.nodes.analysis_node import AnalysisNode from genes.hgvs import get_hgvs_variant -from snpdb.models import GenomicIntervalsCollection, GenomicInterval, Sample, \ - VCFBedIntersection, Cohort, VariantCollection +from snpdb.models import ( + Cohort, + GenomicInterval, + GenomicIntervalsCollection, + Sample, + VariantCollection, + VCFBedIntersection, +) from snpdb.models.models_variant import Variant from snpdb.variants_to_vcf import write_qs_to_vcf_file_sort_alphabetically diff --git a/analysis/models/nodes/filters/merge_node.py b/analysis/models/nodes/filters/merge_node.py index cc7dd78dc..a98fb427c 100644 --- a/analysis/models/nodes/filters/merge_node.py +++ b/analysis/models/nodes/filters/merge_node.py @@ -1,5 +1,5 @@ import operator -from collections import defaultdict, Counter +from collections import Counter, defaultdict from functools import cached_property, reduce from typing import Optional diff --git a/analysis/models/nodes/filters/moi_node.py b/analysis/models/nodes/filters/moi_node.py index ca0ffa952..1c9224ff8 100644 --- a/analysis/models/nodes/filters/moi_node.py +++ b/analysis/models/nodes/filters/moi_node.py @@ -6,12 +6,12 @@ from auditlog.registry import auditlog from django.db import models -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query_utils import Q from analysis.models.nodes.analysis_node import AnalysisNode, NodeAuditLogMixin from analysis.models.nodes.cohort_mixin import AncestorSampleMixin -from annotation.models import VariantTranscriptAnnotation, OntologyTerm +from annotation.models import OntologyTerm, VariantTranscriptAnnotation from genes.models import GeneSymbol from ontology.models import GeneDiseaseClassification, OntologyTermRelation from patients.models_enums import Zygosity diff --git a/analysis/models/nodes/filters/phenotype_node.py b/analysis/models/nodes/filters/phenotype_node.py index ed240cd3d..a5f48bb12 100644 --- a/analysis/models/nodes/filters/phenotype_node.py +++ b/analysis/models/nodes/filters/phenotype_node.py @@ -7,11 +7,11 @@ from auditlog.registry import auditlog from cache_memoize import cache_memoize from django.db import models -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query_utils import Q from analysis.models.nodes.analysis_node import AnalysisNode, NodeAuditLogMixin -from annotation.models import VariantTranscriptAnnotation, OntologyTerm +from annotation.models import OntologyTerm, VariantTranscriptAnnotation from genes.models import GeneSymbol from library.constants import DAY_SECS from patients.models import Patient diff --git a/analysis/models/nodes/filters/population_node.py b/analysis/models/nodes/filters/population_node.py index cb21c87bd..4db08784b 100644 --- a/analysis/models/nodes/filters/population_node.py +++ b/analysis/models/nodes/filters/population_node.py @@ -14,7 +14,7 @@ from classification.enums import ClinicalSignificance from classification.models.classification import Classification from library.constants import MINUTE_SECS -from patients.models_enums import SimpleZygosity, GnomADPopulation +from patients.models_enums import GnomADPopulation, SimpleZygosity from snpdb.models import VariantZygosityCountCollection @@ -53,7 +53,7 @@ def population_database_fields(self) -> list[str]: def has_filtering_allele_frequency(self) -> bool: try: return self.analysis.annotation_version.variant_annotation_version.gnomad_major_version >= 4 - except AttributeError as e: + except AttributeError: return False def modifies_parents(self): diff --git a/analysis/models/nodes/filters/tag_node.py b/analysis/models/nodes/filters/tag_node.py index 95d81c102..1491e5fce 100644 --- a/analysis/models/nodes/filters/tag_node.py +++ b/analysis/models/nodes/filters/tag_node.py @@ -3,7 +3,7 @@ from auditlog.registry import auditlog from django.db import models -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query_utils import Q from analysis.models.enums import TagNodeMode diff --git a/analysis/models/nodes/filters/tissue_node.py b/analysis/models/nodes/filters/tissue_node.py index 90378e72a..6db4fa6fd 100644 --- a/analysis/models/nodes/filters/tissue_node.py +++ b/analysis/models/nodes/filters/tissue_node.py @@ -7,8 +7,11 @@ from analysis.models import GroupOperation from analysis.models.nodes.analysis_node import AnalysisNode -from annotation.models.models import HumanProteinAtlasTissueSample, \ - HumanProteinAtlasAnnotation, VariantTranscriptAnnotation +from annotation.models.models import ( + HumanProteinAtlasAnnotation, + HumanProteinAtlasTissueSample, + VariantTranscriptAnnotation, +) from annotation.models.models_enums import DetectedHumanProteinAtlasAbundance from genes.models import Gene, GeneSymbol from genes.models_enums import AnnotationConsortium diff --git a/analysis/models/nodes/filters/venn_node.py b/analysis/models/nodes/filters/venn_node.py index d7307c5d0..fad511ac5 100644 --- a/analysis/models/nodes/filters/venn_node.py +++ b/analysis/models/nodes/filters/venn_node.py @@ -7,14 +7,14 @@ import celery from auditlog.registry import auditlog from django.db import models -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query_utils import Q from django.db.models.signals import post_delete from django.dispatch import receiver from analysis.models.enums import SetOperations from analysis.models.nodes.analysis_node import AnalysisNode, NodeStatus, NodeVersion -from snpdb.models import VariantCollection, ProcessingStatus, VariantCollectionRecord +from snpdb.models import ProcessingStatus, VariantCollection, VariantCollectionRecord class VennNode(AnalysisNode): diff --git a/analysis/models/nodes/node_utils.py b/analysis/models/nodes/node_utils.py index 7312ec6a1..6a07cd118 100644 --- a/analysis/models/nodes/node_utils.py +++ b/analysis/models/nodes/node_utils.py @@ -9,7 +9,7 @@ from django.utils import timezone from toposort import toposort -from analysis.models import Analysis, NodeStatus, NodeColors +from analysis.models import Analysis, NodeColors, NodeStatus from analysis.models.nodes.analysis_node import AnalysisEdge, NodeVersion from analysis.tasks.node_update_tasks import delete_analysis_old_node_versions from library.utils import add_exception_note diff --git a/analysis/models/nodes/sources/_stats_cache.py b/analysis/models/nodes/sources/_stats_cache.py index 8c03f84a0..2c4a64046 100644 --- a/analysis/models/nodes/sources/_stats_cache.py +++ b/analysis/models/nodes/sources/_stats_cache.py @@ -6,7 +6,8 @@ uses the same canonical_filter_key helper to populate filter-keyed rows. A silent mismatch between the two would produce cache misses forever. """ -from typing import Iterable, Optional, Union +from collections.abc import Iterable +from typing import Optional, Union from django.core.exceptions import ObjectDoesNotExist diff --git a/analysis/models/nodes/sources/all_variants_node.py b/analysis/models/nodes/sources/all_variants_node.py index b302849a9..c05fb27b4 100644 --- a/analysis/models/nodes/sources/all_variants_node.py +++ b/analysis/models/nodes/sources/all_variants_node.py @@ -4,7 +4,7 @@ from auditlog.registry import auditlog from django.db import models -from django.db.models import Q, CASCADE, SET_NULL +from django.db.models import CASCADE, SET_NULL, Q from analysis.models.nodes.analysis_node import AnalysisNode, NodeAuditLogMixin from analysis.models.nodes.zygosity_count_node import AbstractZygosityCountNode diff --git a/analysis/models/nodes/sources/classifications_node.py b/analysis/models/nodes/sources/classifications_node.py index b2dab8633..fb7286ef8 100644 --- a/analysis/models/nodes/sources/classifications_node.py +++ b/analysis/models/nodes/sources/classifications_node.py @@ -2,7 +2,7 @@ from auditlog.registry import auditlog from django.db import models -from django.db.models import Q, CASCADE +from django.db.models import CASCADE, Q from analysis.models.nodes.analysis_node import AnalysisNode from classification.enums import ClinicalSignificance diff --git a/analysis/models/nodes/sources/cohort_node.py b/analysis/models/nodes/sources/cohort_node.py index 731d50c24..525f1b953 100644 --- a/analysis/models/nodes/sources/cohort_node.py +++ b/analysis/models/nodes/sources/cohort_node.py @@ -4,18 +4,20 @@ from auditlog.registry import auditlog from django.db import models from django.db.models import Q -from django.db.models.deletion import SET_NULL, CASCADE -from django.db.models.expressions import Value, F -from django.db.models.functions import Concat, Substr, Length, Replace +from django.db.models.deletion import CASCADE, SET_NULL +from django.db.models.expressions import F, Value +from django.db.models.functions import Concat, Length, Replace, Substr -from analysis.models import GroupOperation, AnalysisNode +from analysis.models import AnalysisNode, GroupOperation from analysis.models.nodes.cohort_mixin import CohortMixin from analysis.models.nodes.sources._stats_cache import ( - get_cached_label_count_for_cohort, get_handler_for_node, UNCACHEABLE, + UNCACHEABLE, + get_cached_label_count_for_cohort, + get_handler_for_node, ) from analysis.models.nodes.zygosity_count_node import AbstractZygosityCountNode -from patients.models_enums import Zygosity, SimpleZygosity -from snpdb.models import Cohort, CohortSample, VariantsType, CohortGenotypeCollection +from patients.models_enums import SimpleZygosity, Zygosity +from snpdb.models import Cohort, CohortGenotypeCollection, CohortSample, VariantsType class AbstractCohortBasedNode(CohortMixin, AnalysisNode): diff --git a/analysis/models/nodes/sources/pedigree_node.py b/analysis/models/nodes/sources/pedigree_node.py index 3cfb17461..7c548f09c 100644 --- a/analysis/models/nodes/sources/pedigree_node.py +++ b/analysis/models/nodes/sources/pedigree_node.py @@ -7,10 +7,12 @@ from analysis.models.nodes.sources import AbstractCohortBasedNode from analysis.models.nodes.sources._stats_cache import ( - get_cached_label_count_for_cohort, get_handler_for_node, UNCACHEABLE, + UNCACHEABLE, + get_cached_label_count_for_cohort, + get_handler_for_node, ) from patients.models_enums import Zygosity -from pedigree.models import Pedigree, PedigreeInheritance, CohortSamplePedFileRecord +from pedigree.models import CohortSamplePedFileRecord, Pedigree, PedigreeInheritance class PedigreeNode(AbstractCohortBasedNode): diff --git a/analysis/models/nodes/sources/quad_node.py b/analysis/models/nodes/sources/quad_node.py index c69ec5ae8..5e971c077 100644 --- a/analysis/models/nodes/sources/quad_node.py +++ b/analysis/models/nodes/sources/quad_node.py @@ -10,7 +10,7 @@ from django.db.models.deletion import SET_NULL from django.db.models.query_utils import Q -from analysis.models.enums import QuadInheritance, NodeErrorSource, AnalysisTemplateType +from analysis.models.enums import AnalysisTemplateType, NodeErrorSource, QuadInheritance from analysis.models.nodes.sources import AbstractCohortBasedNode from analysis.models.nodes.sources.trio_node import ( AbstractTrioInheritance, @@ -21,7 +21,7 @@ from annotation.models.models import VariantTranscriptAnnotation from library.constants import DAY_SECS from patients.models_enums import Zygosity -from snpdb.models import Quad, Sample, Contig +from snpdb.models import Contig, Quad, Sample class AbstractQuadInheritance(ABC): @@ -297,7 +297,7 @@ def get_errors(self, include_parent_errors=True, flat=False): errors = super().get_errors(include_parent_errors=include_parent_errors) if self.analysis.template_type != AnalysisTemplateType.TEMPLATE: if quad_errors := self.get_quad_inheritance_errors(self.quad, self.inheritance): - errors.extend(((NodeErrorSource.CONFIGURATION, e) for e in quad_errors)) + errors.extend((NodeErrorSource.CONFIGURATION, e) for e in quad_errors) if flat: errors = self.flatten_errors(errors) return errors @@ -465,7 +465,7 @@ def _get_cohorts_and_sample_visibility_for_node(self): if self.quad: cohort = self.quad.cohort cohorts = [cohort] - visibility = {s: cohort.has_genotype for s in self.quad.get_samples()} + visibility = dict.fromkeys(self.quad.get_samples(), cohort.has_genotype) return cohorts, visibility def _get_proband_sample_for_node(self) -> Optional[Sample]: diff --git a/analysis/models/nodes/sources/trio_node.py b/analysis/models/nodes/sources/trio_node.py index d04f33ce9..a6b65fc23 100644 --- a/analysis/models/nodes/sources/trio_node.py +++ b/analysis/models/nodes/sources/trio_node.py @@ -10,15 +10,17 @@ from django.db.models.deletion import SET_NULL from django.db.models.query_utils import Q -from analysis.models.enums import TrioInheritance, NodeErrorSource, AnalysisTemplateType +from analysis.models.enums import AnalysisTemplateType, NodeErrorSource, TrioInheritance from analysis.models.nodes.sources import AbstractCohortBasedNode from analysis.models.nodes.sources._stats_cache import ( - get_cached_label_count_for_cohort, get_handler_for_node, UNCACHEABLE, + UNCACHEABLE, + get_cached_label_count_for_cohort, + get_handler_for_node, ) from annotation.models.models import VariantTranscriptAnnotation from library.constants import DAY_SECS -from patients.models_enums import Zygosity, Sex -from snpdb.models import Trio, Sample, Contig +from patients.models_enums import Sex, Zygosity +from snpdb.models import Contig, Sample, Trio def _build_family_zyg_q(cohort_genotype_collection, sample_zyg_require: list[tuple]) -> Q: @@ -328,7 +330,7 @@ def get_errors(self, include_parent_errors=True, flat=False): # Allow template to configure anything w/o checks if self.analysis.template_type != AnalysisTemplateType.TEMPLATE: if trio_errors := self.get_trio_inheritance_errors(self.trio, self.inheritance): - errors.extend(((NodeErrorSource.CONFIGURATION, e) for e in trio_errors)) + errors.extend((NodeErrorSource.CONFIGURATION, e) for e in trio_errors) if flat: errors = self.flatten_errors(errors) return errors @@ -537,7 +539,7 @@ def _get_cohorts_and_sample_visibility_for_node(self): if self.trio: cohort = self.trio.cohort cohorts = [cohort] - visibility = {s: cohort.has_genotype for s in self.trio.get_samples()} + visibility = dict.fromkeys(self.trio.get_samples(), cohort.has_genotype) return cohorts, visibility def _get_proband_sample_for_node(self) -> Optional[Sample]: diff --git a/analysis/models/nodes/zygosity_count_node.py b/analysis/models/nodes/zygosity_count_node.py index 307140080..19e7cc90c 100644 --- a/analysis/models/nodes/zygosity_count_node.py +++ b/analysis/models/nodes/zygosity_count_node.py @@ -4,7 +4,7 @@ from typing import Optional from django.db import models -from django.db.models import Q, Model +from django.db.models import Model, Q class AbstractZygosityCountNode(Model): diff --git a/analysis/related_analyses.py b/analysis/related_analyses.py index ccfb6493e..aa124fadd 100644 --- a/analysis/related_analyses.py +++ b/analysis/related_analyses.py @@ -1,7 +1,7 @@ from collections import defaultdict from analysis.models.nodes.analysis_node import Analysis -from analysis.models.nodes.sources import SampleNode, CohortNode, TrioNode, PedigreeNode +from analysis.models.nodes.sources import CohortNode, PedigreeNode, SampleNode, TrioNode from analysis.models.nodes.sources.quad_node import QuadNode diff --git a/analysis/serializers.py b/analysis/serializers.py index f9477ffe3..18bb49fc3 100644 --- a/analysis/serializers.py +++ b/analysis/serializers.py @@ -1,21 +1,48 @@ from rest_framework import serializers -from analysis.models import AnalysisVariable, FilterNode, FilterNodeItem, PhenotypeNode, BuiltInFilterNode, \ - ClassificationsNode, DamageNode, VennNode, ZygosityNode, TrioNode, CohortNode, PedigreeNode, \ - TissueNode, SelectedInParentNode, SampleNode, PopulationNode, PopulationNodeGnomADPopulation, GeneListNode, \ - IntersectionNode, GeneListNodeGeneList, Analysis, AlleleFrequencyNode, AllVariantsNode, TagNode, MergeNode, \ - PhenotypeNodeOntologyTerm, CandidateSearchRun +from analysis.models import ( + AlleleFrequencyNode, + AllVariantsNode, + Analysis, + AnalysisVariable, + BuiltInFilterNode, + CandidateSearchRun, + ClassificationsNode, + CohortNode, + DamageNode, + FilterNode, + FilterNodeItem, + GeneListNode, + GeneListNodeGeneList, + IntersectionNode, + MergeNode, + PedigreeNode, + PhenotypeNode, + PhenotypeNodeOntologyTerm, + PopulationNode, + PopulationNodeGnomADPopulation, + SampleNode, + SelectedInParentNode, + TagNode, + TissueNode, + TrioNode, + VennNode, + ZygosityNode, +) from analysis.models.models_variant_tag import VariantTag -from analysis.models.nodes.analysis_node import NodeAlleleFrequencyRange, NodeAlleleFrequencyFilter, AnalysisNode, \ - NodeWiki +from analysis.models.nodes.analysis_node import ( + AnalysisNode, + NodeAlleleFrequencyFilter, + NodeAlleleFrequencyRange, + NodeWiki, +) from analysis.models.nodes.filters.conservation_node import ConservationNode -from genes.models import GeneList from genes.serializers import GeneListSerializer from library.django_utils import get_model_fields from library.django_utils.django_rest_utils import DynamicFieldsModelSerializer from ontology.models import OntologyTerm from ontology.serializers import OntologyTermSerializer -from snpdb.serializers import UserSerializer, TimestampField +from snpdb.serializers import TimestampField, UserSerializer class NodeAlleleFrequencyRangeSerializer(serializers.ModelSerializer): diff --git a/analysis/signals/analysis_health_check.py b/analysis/signals/analysis_health_check.py index f820b70db..55ea93997 100644 --- a/analysis/signals/analysis_health_check.py +++ b/analysis/signals/analysis_health_check.py @@ -1,7 +1,7 @@ from django.dispatch import receiver from analysis.models import Analysis -from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckRecentActivity +from library.health_check import HealthCheckRecentActivity, HealthCheckRequest, health_check_signal from variantgrid.perm_path import get_visible_url_names diff --git a/analysis/signals/analysis_search.py b/analysis/signals/analysis_search.py index 7cee74d19..389d42b35 100644 --- a/analysis/signals/analysis_search.py +++ b/analysis/signals/analysis_search.py @@ -1,7 +1,7 @@ import re from analysis.models import Analysis -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver ANALYSIS_PREFIX_PATTERN = re.compile(r"^an(\d+)$") diff --git a/analysis/signals/signal_handlers.py b/analysis/signals/signal_handlers.py index 078d86cad..89e4625a6 100644 --- a/analysis/signals/signal_handlers.py +++ b/analysis/signals/signal_handlers.py @@ -3,10 +3,16 @@ import celery from django.db import transaction -from analysis.tasks.auto_analysis_tasks import auto_run_analyses_for_vcf, auto_run_analyses_for_sample, \ - reload_auto_analyses_for_vcf -from analysis.tasks.variant_tag_tasks import variant_tag_created_task, variant_tag_deleted_in_analysis_task, \ - analysis_tag_nodes_set_dirty +from analysis.tasks.auto_analysis_tasks import ( + auto_run_analyses_for_sample, + auto_run_analyses_for_vcf, + reload_auto_analyses_for_vcf, +) +from analysis.tasks.variant_tag_tasks import ( + analysis_tag_nodes_set_dirty, + variant_tag_created_task, + variant_tag_deleted_in_analysis_task, +) from snpdb.models import ImportStatus diff --git a/analysis/tasks/abstract_candidate_search_task.py b/analysis/tasks/abstract_candidate_search_task.py index bf3b16934..8a4d0227a 100644 --- a/analysis/tasks/abstract_candidate_search_task.py +++ b/analysis/tasks/abstract_candidate_search_task.py @@ -2,7 +2,7 @@ from celery import Task -from analysis.models import CandidateSearchRun, Candidate +from analysis.models import Candidate, CandidateSearchRun from library.log_utils import get_traceback, log_traceback from snpdb.models import ProcessingStatus @@ -22,7 +22,7 @@ def run(self, candidate_search_run_id): if records: Candidate.objects.bulk_create(records, batch_size=1000) candidate_search_run.status = ProcessingStatus.SUCCESS - except Exception as e: + except Exception: log_traceback() candidate_search_run.error_exception = get_traceback() candidate_search_run.status = ProcessingStatus.ERROR diff --git a/analysis/tasks/analysis_grid_export_tasks.py b/analysis/tasks/analysis_grid_export_tasks.py index bd45d39d0..c85c9bcc5 100644 --- a/analysis/tasks/analysis_grid_export_tasks.py +++ b/analysis/tasks/analysis_grid_export_tasks.py @@ -10,13 +10,13 @@ from analysis.analysis_templates import get_cohort_analysis, get_sample_analysis from analysis.grid_export import node_grid_get_export_iterator -from analysis.models import AnalysisTemplate, NodeStatus, CohortNode, SampleNode +from analysis.models import AnalysisTemplate, CohortNode, NodeStatus, SampleNode from analysis.tasks.node_update_tasks import wait_for_node from library.django_utils import FakeRequest from library.guardian_utils import admin_bot from library.log_utils import log_traceback -from library.utils import name_from_filename, sha256sum_str, mk_path_for_file -from snpdb.models import Cohort, Sample, CachedGeneratedFile +from library.utils import mk_path_for_file, name_from_filename, sha256sum_str +from snpdb.models import CachedGeneratedFile, Cohort, Sample def get_annotated_download_files_cgf(generator, pk) -> dict[str, Optional[CachedGeneratedFile]]: diff --git a/analysis/tasks/analysis_update_tasks.py b/analysis/tasks/analysis_update_tasks.py index 3a949485f..dfc7f5c43 100644 --- a/analysis/tasks/analysis_update_tasks.py +++ b/analysis/tasks/analysis_update_tasks.py @@ -10,7 +10,7 @@ from django.db import transaction from django.utils import timezone -from analysis.models import AnalysisEdge, NodeStatus, NodeColors, NodeTask, AnalysisNode +from analysis.models import AnalysisEdge, AnalysisNode, NodeColors, NodeStatus, NodeTask from analysis.models.nodes.analysis_node import NodeCache, NodeVersion from analysis.models.nodes.node_utils import get_nodes_by_id from analysis.tasks.node_update_tasks import MAX_NODE_ATTEMPTS diff --git a/analysis/tasks/auto_analysis_tasks.py b/analysis/tasks/auto_analysis_tasks.py index b3abc7a7b..9b4cdf7af 100644 --- a/analysis/tasks/auto_analysis_tasks.py +++ b/analysis/tasks/auto_analysis_tasks.py @@ -31,7 +31,7 @@ def auto_run_analyses_for_vcf(vcf_id: int, analysis_description: str, skip_alrea @celery.shared_task def reload_auto_analyses_for_vcf(vcf_id: int): """ On VCF re-import, reload nodes in any existing auto-analyses so node counts stay fresh. """ - from analysis.models import SampleAnalysisTemplateRun, CohortAnalysisTemplateRun + from analysis.models import CohortAnalysisTemplateRun, SampleAnalysisTemplateRun vcf = VCF.objects.get(pk=vcf_id) diff --git a/analysis/tasks/karyomapping_tasks.py b/analysis/tasks/karyomapping_tasks.py index cc2de8836..768591840 100644 --- a/analysis/tasks/karyomapping_tasks.py +++ b/analysis/tasks/karyomapping_tasks.py @@ -2,7 +2,11 @@ import celery -from analysis.models.models_karyomapping import KaryotypeBins, GenomeKaryomappingCounts, ContigKaryomappingCounts +from analysis.models.models_karyomapping import ( + ContigKaryomappingCounts, + GenomeKaryomappingCounts, + KaryotypeBins, +) from snpdb.models import Trio from snpdb.models.models_enums import ImportStatus diff --git a/analysis/tasks/mutational_signatures_task.py b/analysis/tasks/mutational_signatures_task.py index a9444203b..77e81654f 100644 --- a/analysis/tasks/mutational_signatures_task.py +++ b/analysis/tasks/mutational_signatures_task.py @@ -5,8 +5,12 @@ from django.conf import settings from analysis.models.enums import MinimisationResultType -from analysis.models.mutational_signatures import MutationalSignatureCalculator, \ - MutationalSignature, MutationalSignatureMinimisationResult, MutationalSignatureMutationCount +from analysis.models.mutational_signatures import ( + MutationalSignature, + MutationalSignatureCalculator, + MutationalSignatureMinimisationResult, + MutationalSignatureMutationCount, +) from library.genomics.calculate_cancer_mutation_signatures import MutationSignatures, invert_muttype from snpdb.models import Sample, Variant from snpdb.models.models_enums import ImportStatus diff --git a/analysis/tasks/node_update_tasks.py b/analysis/tasks/node_update_tasks.py index 89fd29637..98a3b17a2 100644 --- a/analysis/tasks/node_update_tasks.py +++ b/analysis/tasks/node_update_tasks.py @@ -6,17 +6,28 @@ from celery.canvas import Signature from celery.contrib.abortable import AbortableTask from celery.result import AsyncResult -from django.db.models import OuterRef, Subquery, F, Q -from django.db.utils import OperationalError, IntegrityError +from django.db.models import F, OuterRef, Q, Subquery +from django.db.utils import IntegrityError, OperationalError from django.utils import timezone -from analysis.exceptions import NodeConfigurationException, NodeParentErrorsException, CeleryTasksObsoleteException, \ - NodeOutOfDateException -from analysis.models.nodes.analysis_node import AnalysisNode, NodeStatus, NodeVersion, NodeCache, NodeTask, NodeColors +from analysis.exceptions import ( + CeleryTasksObsoleteException, + NodeConfigurationException, + NodeOutOfDateException, + NodeParentErrorsException, +) +from analysis.models.nodes.analysis_node import ( + AnalysisNode, + NodeCache, + NodeColors, + NodeStatus, + NodeTask, + NodeVersion, +) from eventlog.models import create_event from library.constants import MINUTE_SECS from library.enums.log_level import LogLevel -from library.log_utils import log_traceback, get_traceback +from library.log_utils import get_traceback, log_traceback from snpdb.models import ProcessingStatus CREATE_AND_LAUNCH_TASK = "analysis.tasks.analysis_update_tasks.create_and_launch_analysis_tasks" diff --git a/analysis/tasks/reanalysis_tasks.py b/analysis/tasks/reanalysis_tasks.py index 3c1c5f1ed..f70673eb8 100644 --- a/analysis/tasks/reanalysis_tasks.py +++ b/analysis/tasks/reanalysis_tasks.py @@ -4,7 +4,7 @@ from django.utils.timesince import timesince from analysis.grids import AnalysesColumns -from analysis.models import Analysis, Candidate, AnalysisNode +from analysis.models import Analysis, AnalysisNode, Candidate from analysis.tasks.abstract_candidate_search_task import AbstractCandidateSearchTask from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version from annotation.models import AnnotationVersion diff --git a/analysis/tasks/variant_tag_tasks.py b/analysis/tasks/variant_tag_tasks.py index 6302f485a..79ebb0372 100644 --- a/analysis/tasks/variant_tag_tasks.py +++ b/analysis/tasks/variant_tag_tasks.py @@ -1,12 +1,12 @@ import celery from django.db.models import Q -from analysis.models import VariantTag, Analysis, TagNode +from analysis.models import Analysis, TagNode, VariantTag from analysis.models.nodes.node_utils import update_analysis from library.guardian_utils import admin_bot from snpdb.clingen_allele import populate_clingen_alleles_for_variants from snpdb.liftover import create_liftover_pipelines -from snpdb.models import ImportSource, VariantAllele, Tag +from snpdb.models import ImportSource, Tag, VariantAllele def analysis_tag_nodes_set_dirty(analysis: Analysis, tag: Tag, visible: bool): diff --git a/analysis/templatetags/related_analyses_tags.py b/analysis/templatetags/related_analyses_tags.py index ac15fd28f..64b7978ea 100644 --- a/analysis/templatetags/related_analyses_tags.py +++ b/analysis/templatetags/related_analyses_tags.py @@ -4,18 +4,21 @@ from functools import reduce from django.conf import settings -from django.db.models import Q, Model +from django.db.models import Model, Q from django.template import Library from analysis.forms import get_analysis_template_form_for_variables_only_of_class -from analysis.models import MutationalSignature, Analysis, AnalysisTemplate +from analysis.models import Analysis, AnalysisTemplate, MutationalSignature from analysis.models.models_karyomapping import KaryomappingAnalysis -from analysis.related_analyses import get_related_analysis_details_for_samples, \ - get_related_analysis_details_for_cohort, \ - get_related_analysis_details_for_pedigree, get_related_analysis_details_for_quad, \ - get_related_analysis_details_for_trio +from analysis.related_analyses import ( + get_related_analysis_details_for_cohort, + get_related_analysis_details_for_pedigree, + get_related_analysis_details_for_quad, + get_related_analysis_details_for_samples, + get_related_analysis_details_for_trio, +) from pedigree.models import Pedigree -from snpdb.models import Cohort, Trio, Quad +from snpdb.models import Cohort, Trio register = Library() diff --git a/analysis/tests/test_clone_nodes.py b/analysis/tests/test_clone_nodes.py index 28b913099..853b8690b 100644 --- a/analysis/tests/test_clone_nodes.py +++ b/analysis/tests/test_clone_nodes.py @@ -1,18 +1,36 @@ from django.contrib.auth.models import User from django.test import TestCase, override_settings -from analysis.models import Analysis, CohortNode, CohortNodeZygosityFiltersCollection, FilterNode, FilterNodeItem, \ - SampleNode, GeneListNode, IntersectionNode, PhenotypeNode, PopulationNode, TagNode, VariantTag, \ - AlleleFrequencyNode, NodeAlleleFrequencyFilter, NodeVCFFilter, TrioNode, MOINode, MergeNode, PedigreeNode -from annotation.fake_annotation import get_fake_annotation_version, create_fake_variants -from annotation.models import VariantGeneOverlap, AnnotationRun +from analysis.models import ( + AlleleFrequencyNode, + Analysis, + CohortNode, + CohortNodeZygosityFiltersCollection, + FilterNode, + FilterNodeItem, + GeneListNode, + IntersectionNode, + MergeNode, + MOINode, + NodeAlleleFrequencyFilter, + NodeVCFFilter, + PedigreeNode, + PhenotypeNode, + PopulationNode, + SampleNode, + TagNode, + TrioNode, + VariantTag, +) +from annotation.fake_annotation import create_fake_variants, get_fake_annotation_version +from annotation.models import AnnotationRun, VariantGeneOverlap from annotation.tests.test_data_fake_genes import create_fake_transcript_version from genes.models import GeneList, GeneListGeneSymbol from ontology.models import OntologyTerm from patients.models_enums import GnomADPopulation from pedigree.models import PedigreeInheritance -from snpdb.models import GenomeBuild, ImportStatus, Variant, GenomicInterval, Tag -from snpdb.tests.utils.fake_cohort_data import create_fake_trio, create_fake_pedigree +from snpdb.models import GenomeBuild, GenomicInterval, ImportStatus, Tag, Variant +from snpdb.tests.utils.fake_cohort_data import create_fake_pedigree, create_fake_trio @override_settings(ANALYSIS_NODE_CACHE_Q=False) diff --git a/analysis/tests/test_cohort_stats_cache.py b/analysis/tests/test_cohort_stats_cache.py index 951c7799f..bb6d9a785 100644 --- a/analysis/tests/test_cohort_stats_cache.py +++ b/analysis/tests/test_cohort_stats_cache.py @@ -15,9 +15,9 @@ from analysis.models.enums import TrioInheritance from analysis.models.nodes.sources._stats_cache import ( + UNCACHEABLE, NoFilterHandler, TrioInheritanceHandler, - UNCACHEABLE, get_filter_keys_to_precompute_for_cohort, ) from annotation.fake_annotation import get_fake_annotation_version diff --git a/analysis/tests/test_explicit_pk_substitution_baseline.py b/analysis/tests/test_explicit_pk_substitution_baseline.py index 39a174abb..f11f119d3 100644 --- a/analysis/tests/test_explicit_pk_substitution_baseline.py +++ b/analysis/tests/test_explicit_pk_substitution_baseline.py @@ -33,7 +33,9 @@ from patients.models_enums import Zygosity from snpdb.models import GenomeBuild, Variant from snpdb.models.models_cohort import ( - CohortGenotype, CohortGenotypeCollection, CohortGenotypeCommonFilterVersion, + CohortGenotype, + CohortGenotypeCollection, + CohortGenotypeCommonFilterVersion, ) from snpdb.models.models_enums import CohortGenotypeCollectionType from snpdb.tests.utils.fake_cohort_data import create_fake_trio diff --git a/analysis/tests/test_models.py b/analysis/tests/test_models.py index 6f41ec2a1..e56e96449 100644 --- a/analysis/tests/test_models.py +++ b/analysis/tests/test_models.py @@ -4,10 +4,18 @@ from django.test import TestCase, override_settings from django.utils import timezone -from analysis.models import Analysis, AnalysisLock, SampleNode, GeneListNode, ZygosityNode +from analysis.models import Analysis, AnalysisLock, GeneListNode, SampleNode, ZygosityNode from annotation.fake_annotation import get_fake_annotation_version from library.guardian_utils import assign_permission_to_user_and_groups -from snpdb.models import GenomeBuild, VCF, Sample, Cohort, CohortSample, CohortGenotypeCollection, ImportStatus +from snpdb.models import ( + VCF, + Cohort, + CohortGenotypeCollection, + CohortSample, + GenomeBuild, + ImportStatus, + Sample, +) @override_settings(ANALYSIS_NODE_CACHE_Q=False) diff --git a/analysis/tests/test_node_filter_bugs.py b/analysis/tests/test_node_filter_bugs.py index a456ffb31..d3e4c0f57 100644 --- a/analysis/tests/test_node_filter_bugs.py +++ b/analysis/tests/test_node_filter_bugs.py @@ -21,12 +21,11 @@ from django.test import TestCase, override_settings -from analysis.models import Analysis, AllVariantsNode +from analysis.models import AllVariantsNode, Analysis from analysis.models.enums import GroupOperation from analysis.models.nodes.filters.population_node import PopulationNode from analysis.tests.utils import AnalysisSetupMixin - # --------------------------------------------------------------------------- # Bug 1 & 2: AbstractZygosityCountNode — max_count=0 falsy check # --------------------------------------------------------------------------- diff --git a/analysis/tests/test_scheduler.py b/analysis/tests/test_scheduler.py index c43cbab71..2aa471175 100644 --- a/analysis/tests/test_scheduler.py +++ b/analysis/tests/test_scheduler.py @@ -19,10 +19,20 @@ from analysis.models.nodes.filters.gene_list_node import GeneListNode from analysis.models.nodes.filters.venn_node import VennNode, VennNodeCache, venn_cache_count from analysis.tasks import analysis_update_tasks -from analysis.tasks.analysis_update_tasks import lease_ready_nodes, _node_launch_signature, \ - _node_ready_to_lease, create_and_launch_analysis_tasks -from analysis.tasks.node_update_tasks import next_backoff, _backoff_node, reschedule_stalled_analyses, \ - update_node_task, node_cache_task, MAX_NODE_ATTEMPTS +from analysis.tasks.analysis_update_tasks import ( + _node_launch_signature, + _node_ready_to_lease, + create_and_launch_analysis_tasks, + lease_ready_nodes, +) +from analysis.tasks.node_update_tasks import ( + MAX_NODE_ATTEMPTS, + _backoff_node, + next_backoff, + node_cache_task, + reschedule_stalled_analyses, + update_node_task, +) from analysis.tests.utils import AnalysisSetupMixin from snpdb.models import ProcessingStatus, VariantCollection diff --git a/analysis/tests/test_source_node_archive.py b/analysis/tests/test_source_node_archive.py index eaecc5de6..5dbac361f 100644 --- a/analysis/tests/test_source_node_archive.py +++ b/analysis/tests/test_source_node_archive.py @@ -12,7 +12,11 @@ from analysis.models.nodes.cohort_mixin import CohortMixin from snpdb.archive import DataArchivedError from snpdb.models import GenomeBuild -from snpdb.tests.utils.fake_cohort_data import create_fake_cohort, create_fake_trio, create_fake_quad +from snpdb.tests.utils.fake_cohort_data import ( + create_fake_cohort, + create_fake_quad, + create_fake_trio, +) class SourceNodeArchiveToleranceTests(TestCase): diff --git a/analysis/tests/test_sub_cohort_any_sample_called_vc.py b/analysis/tests/test_sub_cohort_any_sample_called_vc.py index fdf977b65..236376bef 100644 --- a/analysis/tests/test_sub_cohort_any_sample_called_vc.py +++ b/analysis/tests/test_sub_cohort_any_sample_called_vc.py @@ -17,10 +17,16 @@ from annotation.fake_annotation import get_fake_annotation_version from snpdb.models import GenomeBuild, Variant, VariantCollection from snpdb.models.models_cohort import ( - CohortGenotype, CohortGenotypeCollection, CohortVersion, SubCohortVariantCollection, + CohortGenotype, + CohortGenotypeCollection, + CohortVersion, + SubCohortVariantCollection, ) -from snpdb.tasks.sub_cohort_tasks import build_sub_cohort_any_sample_called_vc_task, delete_old_cohort_versions from snpdb.models.models_enums import ProcessingStatus +from snpdb.tasks.sub_cohort_tasks import ( + build_sub_cohort_any_sample_called_vc_task, + delete_old_cohort_versions, +) from snpdb.tests.utils.fake_cohort_data import create_fake_trio from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant diff --git a/analysis/tests/test_trio_node.py b/analysis/tests/test_trio_node.py index 26792634d..5baa3e18c 100644 --- a/analysis/tests/test_trio_node.py +++ b/analysis/tests/test_trio_node.py @@ -6,7 +6,7 @@ from analysis.models import Analysis, TrioNode from analysis.models.enums import TrioInheritance from annotation.fake_annotation import get_fake_annotation_version -from snpdb.models import GenomeBuild, Variant, BuiltInFilters +from snpdb.models import BuiltInFilters, GenomeBuild, Variant from snpdb.models.models_cohort import CohortGenotype, CohortGenotypeCollection from snpdb.tests.utils.fake_cohort_data import create_fake_trio from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant diff --git a/analysis/tests/test_urls.py b/analysis/tests/test_urls.py index 1751ceede..60afcdf5f 100644 --- a/analysis/tests/test_urls.py +++ b/analysis/tests/test_urls.py @@ -7,14 +7,26 @@ from django.urls.base import reverse from django.utils import timezone -from analysis.models import Analysis, SampleNode, KaryomappingAnalysis, GeneListNode, GeneListNodeGeneList +from analysis.models import ( + Analysis, + GeneListNode, + GeneListNodeGeneList, + KaryomappingAnalysis, + SampleNode, +) from analysis.models.enums import SNPMatrix from annotation.fake_annotation import get_fake_annotation_version from genes.models import GeneList -from library.django_utils.unittest_utils import prevent_request_warnings, URLTestCase +from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings from library.guardian_utils import assign_permission_to_user_and_groups from snpdb.models import Variant -from snpdb.models.models_cohort import Cohort, Trio, CohortSample, CohortGenotypeCollection, CohortGenotype +from snpdb.models.models_cohort import ( + Cohort, + CohortGenotype, + CohortGenotypeCollection, + CohortSample, + Trio, +) from snpdb.models.models_enums import ImportStatus from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_vcf import VCF, Sample diff --git a/analysis/tests/test_zygosity_nodes.py b/analysis/tests/test_zygosity_nodes.py index bc1b13e4e..dbb9f3a57 100644 --- a/analysis/tests/test_zygosity_nodes.py +++ b/analysis/tests/test_zygosity_nodes.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import User from django.test import TestCase, override_settings -from analysis.models import Analysis, CohortNode, AnalysisNode, AllVariantsNode +from analysis.models import AllVariantsNode, Analysis, AnalysisNode, CohortNode from annotation.fake_annotation import get_fake_annotation_version from snpdb.models import GenomeBuild from snpdb.tests.utils.fake_cohort_data import create_fake_trio diff --git a/analysis/urls.py b/analysis/urls.py index b7dbb9fed..a1e772b37 100644 --- a/analysis/urls.py +++ b/analysis/urls.py @@ -1,9 +1,27 @@ -from analysis.grids import AnalysesGrid, NodeColumnSummaryGrid, AnalysisTemplatesGrid, \ - NodeOntologyGenesGrid, NodeGeneDiseaseClassificationGenesGrid, \ - NodeGeneListGenesColumns, AnalysisLogEntryColumns, CandidateSearchRunColumns, CandidateColumns, AnalysesColumns, \ - AnalysisNodeIssuesColumns, KaryomappingAnalysesColumns, NodeTissueExpressionGenesColumns, \ - NodeTissueUniProtGenesColumns -from analysis.views import views, views_json, views_grid, views_karyomapping, views_autocomplete, views_candidate_search +from analysis.grids import ( + AnalysesColumns, + AnalysesGrid, + AnalysisLogEntryColumns, + AnalysisNodeIssuesColumns, + AnalysisTemplatesGrid, + CandidateColumns, + CandidateSearchRunColumns, + KaryomappingAnalysesColumns, + NodeColumnSummaryGrid, + NodeGeneDiseaseClassificationGenesGrid, + NodeGeneListGenesColumns, + NodeOntologyGenesGrid, + NodeTissueExpressionGenesColumns, + NodeTissueUniProtGenesColumns, +) +from analysis.views import ( + views, + views_autocomplete, + views_candidate_search, + views_grid, + views_json, + views_karyomapping, +) from library.django_utils.jqgrid_view import JQGridView from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/analysis/views/node_json_view.py b/analysis/views/node_json_view.py index a344b95ac..e4c73b349 100644 --- a/analysis/views/node_json_view.py +++ b/analysis/views/node_json_view.py @@ -4,7 +4,7 @@ from django.views.generic.base import View from rest_framework.status import HTTP_200_OK -from analysis.exceptions import NonFatalNodeError, NodeNotFoundException +from analysis.exceptions import NodeNotFoundException, NonFatalNodeError from analysis.models.nodes.analysis_node import AnalysisNode from library.log_utils import log_traceback diff --git a/analysis/views/nodes/gene_list_node_view.py b/analysis/views/nodes/gene_list_node_view.py index 9aa316e80..ad8fe696e 100644 --- a/analysis/views/nodes/gene_list_node_view.py +++ b/analysis/views/nodes/gene_list_node_view.py @@ -5,7 +5,7 @@ from analysis.forms.forms_nodes import GeneListNodeForm from analysis.models.nodes.filters.gene_list_node import GeneListNode from analysis.views.nodes.gene_coverage_node_view import GeneCoverageNodeView -from genes.models import GeneListCategory, GeneList, SampleGeneList, PanelAppServer, PanelAppPanel +from genes.models import GeneList, GeneListCategory, PanelAppPanel, PanelAppServer, SampleGeneList class GeneListNodeView(GeneCoverageNodeView): diff --git a/analysis/views/nodes/node_view.py b/analysis/views/nodes/node_view.py index daebe5e2d..9b659f9e8 100644 --- a/analysis/views/nodes/node_view.py +++ b/analysis/views/nodes/node_view.py @@ -4,7 +4,7 @@ from django.views.generic.edit import UpdateView from analysis.exceptions import NonFatalNodeError -from analysis.forms import GraphTypeChoiceForm, ColumnSummaryForm, SNPMatrixForm +from analysis.forms import ColumnSummaryForm, GraphTypeChoiceForm, SNPMatrixForm from analysis.grids import VariantGrid from analysis.models import AnalysisTemplateType from analysis.models.nodes.node_utils import update_analysis diff --git a/analysis/views/nodes/node_views.py b/analysis/views/nodes/node_views.py index 11124d827..20d609a8a 100644 --- a/analysis/views/nodes/node_views.py +++ b/analysis/views/nodes/node_views.py @@ -4,12 +4,30 @@ from django.http.response import HttpResponse from analysis.exceptions import NonFatalNodeError -from analysis.forms.forms_nodes import AllVariantsNodeForm, BuiltInFilterNodeForm, \ - ClassificationsNodeForm, DamageNodeForm, FilterNodeForm, IntersectionNodeForm, \ - PedigreeNodeForm, PhenotypeNodeForm, PopulationNodeForm, QuadNodeForm, TagNodeForm, TissueNodeForm, TrioNodeForm, \ - VennNodeForm, ZygosityNodeForm, CohortNodeForm, AlleleFrequencyNodeForm, SelectedInParentNodeForm, MergeNodeForm, \ - MOINodeForm, ConservationNodeForm -from analysis.models import TagNode, OntologyTerm, MOINode +from analysis.forms.forms_nodes import ( + AlleleFrequencyNodeForm, + AllVariantsNodeForm, + BuiltInFilterNodeForm, + ClassificationsNodeForm, + CohortNodeForm, + ConservationNodeForm, + DamageNodeForm, + FilterNodeForm, + IntersectionNodeForm, + MergeNodeForm, + MOINodeForm, + PedigreeNodeForm, + PhenotypeNodeForm, + PopulationNodeForm, + QuadNodeForm, + SelectedInParentNodeForm, + TagNodeForm, + TissueNodeForm, + TrioNodeForm, + VennNodeForm, + ZygosityNodeForm, +) +from analysis.models import MOINode, OntologyTerm, TagNode from analysis.models.enums import SetOperations from analysis.models.nodes.filters.allele_frequency_node import AlleleFrequencyNode from analysis.models.nodes.filters.built_in_filter_node import BuiltInFilterNode @@ -33,11 +51,11 @@ from analysis.models.nodes.sources.trio_node import TrioNode from analysis.views.nodes.node_view import NodeView from analysis.views.views_json import get_sample_patient_gene_disease_data -from snpdb.models.models_user_settings import UserSettings from classification.models.classification import Classification from classification.views.classification_datatables import ClassificationColumns from library.django_utils import highest_pk from library.jqgrid.jqgrid import JqGrid +from snpdb.models.models_user_settings import UserSettings from snpdb.models.models_variant import Variant diff --git a/analysis/views/views.py b/analysis/views/views.py index f54ed35ff..39c9f6191 100644 --- a/analysis/views/views.py +++ b/analysis/views/views.py @@ -14,7 +14,7 @@ from django.contrib import messages from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.models import User -from django.core.exceptions import PermissionDenied, EmptyResultSet +from django.core.exceptions import EmptyResultSet, PermissionDenied from django.db.models import Count from django.forms.models import inlineformset_factory from django.http.response import HttpResponse, HttpResponseRedirect, JsonResponse @@ -28,40 +28,96 @@ from htmlmin.decorators import not_minified_response from analysis import forms -from analysis.analysis_templates import populate_analysis_from_template_run, get_auto_launch_analysis_template_matches -from analysis.exceptions import NonFatalNodeError, NodeOutOfDateException -from analysis.forms import SelectGridColumnForm, UserTrioWizardForm, UserQuadWizardForm, VCFLocusFilterForm, \ - AnalysisChoiceForm, AnalysisTemplateTypeChoiceForm, AnalysisTemplateVersionForm, AnalysisTemplateForm, \ - AnalysisTemplateAutoLaunchForm, AutoLaunchFormSet +from analysis.analysis_templates import ( + get_auto_launch_analysis_template_matches, + populate_analysis_from_template_run, +) +from analysis.exceptions import NodeOutOfDateException, NonFatalNodeError +from analysis.forms import ( + AnalysisChoiceForm, + AnalysisTemplateAutoLaunchForm, + AnalysisTemplateForm, + AnalysisTemplateTypeChoiceForm, + AnalysisTemplateVersionForm, + AutoLaunchFormSet, + SelectGridColumnForm, + UserQuadWizardForm, + UserTrioWizardForm, + VCFLocusFilterForm, +) from analysis.graphs.column_boxplot_graph import ColumnBoxplotGraph from analysis.grids import VariantGrid -from analysis.models import AnalysisNode, NodeGraphType, VariantTag, TagNode, AnalysisVariable, AnalysisTemplate, \ - AnalysisTemplateRun, AnalysisLock, Analysis -from analysis.models.enums import AnalysisTemplateType, SNPMatrix, MinimisationResultType, NodeStatus, TrioSample, QuadSample +from analysis.models import ( + Analysis, + AnalysisLock, + AnalysisNode, + AnalysisTemplate, + AnalysisTemplateRun, + AnalysisVariable, + NodeGraphType, + TagNode, + VariantTag, +) +from analysis.models.enums import ( + AnalysisTemplateType, + MinimisationResultType, + NodeStatus, + QuadSample, + SNPMatrix, + TrioSample, +) from analysis.models.mutational_signatures import MutationalSignature from analysis.models.nodes import node_utils -from analysis.models.nodes.analysis_node import NodeVCFFilter, AnalysisClassification, NodeTask, NodeCount -from analysis.models.nodes.node_counts import get_node_count_colors, get_node_counts_mine_and_available +from analysis.models.nodes.analysis_node import ( + AnalysisClassification, + NodeCount, + NodeTask, + NodeVCFFilter, +) +from analysis.models.nodes.node_counts import ( + get_node_count_colors, + get_node_counts_mine_and_available, +) from analysis.models.nodes.node_types import get_node_types_hash -from analysis.models.nodes.sources.cohort_node import CohortNodeZygosityFiltersCollection, CohortNodeZygosityFilter +from analysis.models.nodes.sources.cohort_node import ( + CohortNodeZygosityFilter, + CohortNodeZygosityFiltersCollection, +) from analysis.serializers import AnalysisNodeSerializer -from analysis.views.analysis_permissions import get_analysis_or_404, get_node_subclass_or_404, \ - get_node_subclass_or_non_fatal_exception +from analysis.views.analysis_permissions import ( + get_analysis_or_404, + get_node_subclass_or_404, + get_node_subclass_or_non_fatal_exception, +) from analysis.views.nodes.node_view import NodeView from annotation.models.models import MutationalSignatureInfo -from classification.views.views import create_classification_object, CreateClassificationForVariantView +from classification.views.views import ( + CreateClassificationForVariantView, + create_classification_object, +) from library import pandas_utils -from library.constants import WEEK_SECS, HOUR_SECS +from library.constants import HOUR_SECS, WEEK_SECS from library.django_utils import add_save_message, get_field_counts, set_form_read_only from library.guardian_utils import is_superuser -from library.utils import full_class_name, defaultdict_to_dict -from library.utils.database_utils import run_sql, queryset_to_sql +from library.utils import defaultdict_to_dict, full_class_name +from library.utils.database_utils import queryset_to_sql, run_sql from pedigree.models import Pedigree from seqauto.models import EnrichmentKit from snpdb.forms import SampleChoiceForm from snpdb.graphs import graphcache -from snpdb.models import UserSettings, Sample, \ - Cohort, CohortSample, ImportStatus, VCF, get_igv_data, Trio, Quad, Variant, GenomeBuild +from snpdb.models import ( + VCF, + Cohort, + CohortSample, + GenomeBuild, + ImportStatus, + Quad, + Sample, + Trio, + UserSettings, + Variant, + get_igv_data, +) from variantgrid.celery import app diff --git a/analysis/views/views_candidate_search.py b/analysis/views/views_candidate_search.py index 1464914a0..72977e8c3 100644 --- a/analysis/views/views_candidate_search.py +++ b/analysis/views/views_candidate_search.py @@ -4,14 +4,22 @@ from crispy_forms.helper import FormHelper from django.core.exceptions import PermissionDenied from django.http.response import HttpResponse -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.urls import reverse from django.views.decorators.http import require_POST from django.views.generic.base import TemplateView, View -from analysis.forms import AnalysisFilterForm, SampleCandidatesSearchForm, CandidateStatusForm, get_mult_choice_form -from analysis.models import CandidateSearchRun, Candidate, CandidateStatus, CandidateSearchType -from classification.views.views import CreateClassificationForVariantView, create_classification_object +from analysis.forms import ( + AnalysisFilterForm, + CandidateStatusForm, + SampleCandidatesSearchForm, + get_mult_choice_form, +) +from analysis.models import Candidate, CandidateSearchRun, CandidateSearchType, CandidateStatus +from classification.views.views import ( + CreateClassificationForVariantView, + create_classification_object, +) from snpdb.forms import SampleChoiceForm from snpdb.models import GenomeBuild diff --git a/analysis/views/views_grid.py b/analysis/views/views_grid.py index 7cd3d3695..c0f6b2c64 100644 --- a/analysis/views/views_grid.py +++ b/analysis/views/views_grid.py @@ -4,7 +4,7 @@ from django.contrib.postgres.aggregates.general import StringAgg from django.core.cache import cache -from django.http.response import StreamingHttpResponse, HttpResponseRedirect +from django.http.response import HttpResponseRedirect, StreamingHttpResponse from django.shortcuts import redirect from django.urls import reverse from django.utils.decorators import method_decorator @@ -14,14 +14,17 @@ from analysis import grids from analysis.grid_export import node_grid_get_export_iterator from analysis.models import AnalysisNode -from analysis.tasks.analysis_grid_export_tasks import export_cohort_to_downloadable_file, \ - export_sample_to_downloadable_file, get_grid_downloadable_file_params_hash +from analysis.tasks.analysis_grid_export_tasks import ( + export_cohort_to_downloadable_file, + export_sample_to_downloadable_file, + get_grid_downloadable_file_params_hash, +) from analysis.views.analysis_permissions import get_node_subclass_or_non_fatal_exception from analysis.views.node_json_view import NodeJSONGetView, NodeJSONViewMixin from library.constants import WEEK_SECS -from library.django_utils.major_operation import major_operation, TooManyMajorOperationsError +from library.django_utils.major_operation import TooManyMajorOperationsError, major_operation from library.utils.hash_utils import sha256sum_str -from snpdb.models import Sample, Cohort, CachedGeneratedFile +from snpdb.models import CachedGeneratedFile, Cohort, Sample from snpdb.models.models_variant import Variant _NODE_GRID_ALLOWED_PARAMS = { @@ -47,7 +50,7 @@ def _add_allowed_node_grid_params(url: str, params: dict) -> str: if key in _NODE_GRID_ALLOWED_PARAMS: cleaned_params[key] = value else: - logging.warning(f"Node redirect had disallowed GET param: %s", key) + logging.warning("Node redirect had disallowed GET param: %s", key) return f"{url}?" + urlencode(cleaned_params) diff --git a/analysis/views/views_json.py b/analysis/views/views_json.py index 0dff3a19d..a25b16fb2 100644 --- a/analysis/views/views_json.py +++ b/analysis/views/views_json.py @@ -1,7 +1,7 @@ import json import logging import random -from collections import defaultdict, Counter +from collections import Counter, defaultdict from celery.result import AsyncResult from django.conf import settings @@ -11,27 +11,41 @@ from django.views.decorators.cache import never_cache from django.views.decorators.http import require_POST -from analysis.models import AnalysisVariable, AnalysisTemplate, NodeCount, VariantTag, CandidateSearchRun, Candidate, \ - CandidateStatus +from analysis.models import ( + AnalysisTemplate, + AnalysisVariable, + Candidate, + CandidateSearchRun, + CandidateStatus, + NodeCount, + VariantTag, +) from analysis.models.enums import TagLocation from analysis.models.nodes import node_utils -from analysis.models.nodes.analysis_node import NodeStatus, AnalysisEdge, AnalysisNode, NodeTask +from analysis.models.nodes.analysis_node import AnalysisEdge, AnalysisNode, NodeStatus, NodeTask from analysis.models.nodes.filter_child import create_filter_child_node from analysis.models.nodes.filters.built_in_filter_node import BuiltInFilterNode from analysis.models.nodes.filters.selected_in_parent_node import NodeVariant, SelectedInParentNode from analysis.models.nodes.filters.venn_node import VennNode from analysis.models.nodes.node_types import get_node_types_hash_by_class_name -from analysis.models.nodes.node_utils import reload_analysis_nodes, update_analysis, \ - get_toposorted_nodes, get_rendering_dict -from analysis.serializers import VariantTagSerializer, CandidateSearchRunSerializer +from analysis.models.nodes.node_utils import ( + get_rendering_dict, + get_toposorted_nodes, + reload_analysis_nodes, + update_analysis, +) +from analysis.serializers import CandidateSearchRunSerializer, VariantTagSerializer from analysis.tasks.analysis_update_tasks import populate_clingen_alleles_from_analysis_node -from analysis.views.analysis_permissions import get_analysis_or_404, get_node_subclass_or_404, \ - get_node_subclass_or_non_fatal_exception +from analysis.views.analysis_permissions import ( + get_analysis_or_404, + get_node_subclass_or_404, + get_node_subclass_or_non_fatal_exception, +) from analysis.views.node_json_view import NodeJSONPostView from library.django_utils import require_superuser from ontology.models import OntologyTerm, OntologyVersion from ontology.serializers import OntologyTermSerializer -from snpdb.models import Tag, BuiltInFilters, GenomeBuild, Sample +from snpdb.models import BuiltInFilters, GenomeBuild, Sample, Tag from variantgrid.celery import app @@ -394,7 +408,7 @@ def nodes_tasks(request, analysis_id): summary[status] += 1 data = dict(summary) else: - data = {"error": f"No analysis workers found!"} + data = {"error": "No analysis workers found!"} return JsonResponse(data) diff --git a/analysis/views/views_karyomapping.py b/analysis/views/views_karyomapping.py index 9129f16fd..654ee6d1e 100644 --- a/analysis/views/views_karyomapping.py +++ b/analysis/views/views_karyomapping.py @@ -1,16 +1,19 @@ import csv -from collections import defaultdict, OrderedDict +from collections import OrderedDict, defaultdict from django.core.exceptions import PermissionDenied from django.http.response import StreamingHttpResponse -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls.base import reverse from django.views.decorators.cache import cache_page from django.views.decorators.vary import vary_on_cookie from analysis.forms import KaryomappingGeneForm, UserTrioForm -from analysis.models.models_karyomapping import KaryomappingAnalysis, KaryotypeBins, \ - KaryomappingGene +from analysis.models.models_karyomapping import ( + KaryomappingAnalysis, + KaryomappingGene, + KaryotypeBins, +) from library.constants import DAY_SECS from library.django_utils import add_save_message from library.utils import StashFile diff --git a/annotation/admin.py b/annotation/admin.py index 3afea1d38..bdfe7600f 100644 --- a/annotation/admin.py +++ b/annotation/admin.py @@ -1,4 +1,4 @@ -from datetime import timedelta, datetime +from datetime import datetime, timedelta from typing import Optional from django.contrib import admin, messages @@ -11,8 +11,16 @@ from annotation.clinvar_fetch_request import ClinVarFetchRequest from annotation.clinvar_xml_parser import CLINVAR_RECORD_CACHE_DAYS from annotation.clinvar_xml_parser_via_vcv import ClinVarXmlParserViaVCV -from annotation.models import Citation, CitationFetchRequest, ClinVarRecordCollection, ClinVarRecord, ClinVar, \ - AnnotationRun, VariantAnnotation, VariantAnnotationVersion +from annotation.models import ( + AnnotationRun, + Citation, + CitationFetchRequest, + ClinVar, + ClinVarRecord, + ClinVarRecordCollection, + VariantAnnotation, + VariantAnnotationVersion, +) from snpdb.admin_partition_archive_mixin import ArchivePartitionDataAdminMixin from snpdb.admin_utils import ModelAdminBasics, admin_action, admin_list_column, get_admin_url diff --git a/annotation/annotation_data/generate_annotation/dbnsfp_strip.py b/annotation/annotation_data/generate_annotation/dbnsfp_strip.py index 5fb4d1def..5249939e2 100755 --- a/annotation/annotation_data/generate_annotation/dbnsfp_strip.py +++ b/annotation/annotation_data/generate_annotation/dbnsfp_strip.py @@ -164,7 +164,7 @@ def build_pipeline(build: str, version: str, files: list[str], tmp_dir: str, # pipe early, zcat's SIGPIPE doesn't trip `set -o pipefail` (it would if # we used `zcat | head -n1 | ...`). f"read -r _hdr < <(zcat {shlex.quote(files[0])})", - f"printf '%s\\n' \"$_hdr\" \\", + "printf '%s\\n' \"$_hdr\" \\", f" | cut -f {cut_arg} \\", f" | awk 'BEGIN{{OFS=\"\\t\"}}{{ $1=\"#\"$1; print }}' > {header_file}", "", diff --git a/annotation/annotation_data/generate_annotation/denovo_db/denovo_db_tsv_to_vcf.py b/annotation/annotation_data/generate_annotation/denovo_db/denovo_db_tsv_to_vcf.py index 61de8cc2b..2fd433f20 100755 --- a/annotation/annotation_data/generate_annotation/denovo_db/denovo_db_tsv_to_vcf.py +++ b/annotation/annotation_data/generate_annotation/denovo_db/denovo_db_tsv_to_vcf.py @@ -60,7 +60,7 @@ def smart_open(path): p = Path(path) if p.suffix == ".gz": return gzip.open(p, "rt", encoding="utf-8", errors="replace") - return open(p, "rt", encoding="utf-8", errors="replace") + return open(p, encoding="utf-8", errors="replace") def normalize_chrom(s: str) -> str: diff --git a/annotation/annotation_data/generate_annotation/gnomad_data.py b/annotation/annotation_data/generate_annotation/gnomad_data.py index c4fe027ce..568d27c11 100755 --- a/annotation/annotation_data/generate_annotation/gnomad_data.py +++ b/annotation/annotation_data/generate_annotation/gnomad_data.py @@ -357,7 +357,7 @@ def calculate_allele_frequency(version, gnomad_input_vcf, af_output_vcf): infos[p] = str(af_popmax) elif p.startswith("AC_"): infos[p] = str(ac_popmax) - elif p.startswith(("AN_")): + elif p.startswith("AN_"): infos[p] = str(an_popmax) elif p in {"popmax", "grpmax"}: infos[p] = popmax diff --git a/annotation/annotation_version_querysets.py b/annotation/annotation_version_querysets.py index d8d54ca0f..48a0bf850 100644 --- a/annotation/annotation_version_querysets.py +++ b/annotation/annotation_version_querysets.py @@ -12,16 +12,16 @@ import operator from functools import reduce -from typing import TypeVar, Optional +from typing import Optional, TypeVar from django.conf import settings -from django.db.models import QuerySet, Model, F +from django.db.models import F, Model, QuerySet from django.db.models.query_utils import Q from annotation.models import AnnotationVersion, VariantAnnotation, VariantAnnotationPipelineType from library.django_utils.django_queryset_sql_transformer import get_queryset_with_transformer_hook from snpdb.archive import DataArchivedError -from snpdb.models import Variant, GenomeBuild +from snpdb.models import GenomeBuild, Variant def get_variant_queryset_for_latest_annotation_version(genome_build: GenomeBuild) -> QuerySet[Variant]: diff --git a/annotation/annotation_versions.py b/annotation/annotation_versions.py index a117959af..f5a2c14b0 100644 --- a/annotation/annotation_versions.py +++ b/annotation/annotation_versions.py @@ -2,12 +2,17 @@ import sys from django.db import transaction -from django.db.models.aggregates import Max, Min, Count +from django.db.models.aggregates import Count, Max, Min from django.utils import timezone from annotation.annotation_version_querysets import get_variants_qs_for_annotation -from annotation.models import AnnotationRun, AnnotationRangeLock, Variant, AnnotationStatus -from annotation.models import VariantAnnotationVersion +from annotation.models import ( + AnnotationRangeLock, + AnnotationRun, + AnnotationStatus, + Variant, + VariantAnnotationVersion, +) from annotation.vep_annotation import get_vep_variant_annotation_version_kwargs from library.django_utils import highest_pk from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/apps.py b/annotation/apps.py index 1f8f34242..9679f4dea 100644 --- a/annotation/apps.py +++ b/annotation/apps.py @@ -12,13 +12,18 @@ def ready(self): from Bio import Entrez from django.conf import settings + from annotation.models import CachedWebResource - from annotation.signals.manual_signals import clinvar_citations_post_save_handler - from annotation.signals import citation_preview, citation_search, clinvar_annotation_health_check - from annotation.signals.manual_signals import gene_counts_classification_withdraw_handler, \ - gene_counts_classification_publish_handler - from classification.models import Classification, classification_withdraw_signal, \ - classification_post_publish_signal + from annotation.signals.manual_signals import ( + clinvar_citations_post_save_handler, + gene_counts_classification_publish_handler, + gene_counts_classification_withdraw_handler, + ) + from classification.models import ( + Classification, + classification_post_publish_signal, + classification_withdraw_signal, + ) # pylint: enable=import-outside-toplevel,unused-import # Entrez wants both email and API key diff --git a/annotation/clinvar_fetch_request.py b/annotation/clinvar_fetch_request.py index e33c8e5ed..ac9e627ed 100644 --- a/annotation/clinvar_fetch_request.py +++ b/annotation/clinvar_fetch_request.py @@ -1,8 +1,9 @@ import logging import time +from collections.abc import Collection from dataclasses import dataclass, field from datetime import timedelta -from typing import Type, Optional, Collection +from typing import Optional from urllib.error import HTTPError from django.db import transaction @@ -10,7 +11,7 @@ from annotation.clinvar_xml_parser import CLINVAR_RECORD_CACHE_DAYS, ClinVarXmlParser from annotation.clinvar_xml_parser_via_vcv import ClinVarXmlParserViaVCV -from annotation.models import ClinVarRecordCollection, ClinVarVersion, ClinVar +from annotation.models import ClinVar, ClinVarRecordCollection, ClinVarVersion from snpdb.models import GenomeBuild @@ -31,7 +32,7 @@ class ClinVarFetchRequest: How old until the cache is considered stale, provide seconds=0 if you want to force a refresh """ - parser: Type[ClinVarXmlParser] = ClinVarXmlParserViaVCV + parser: type[ClinVarXmlParser] = ClinVarXmlParserViaVCV clinvar_versions: Optional[Collection[ClinVarVersion]] = None diff --git a/annotation/clinvar_xml_parser.py b/annotation/clinvar_xml_parser.py index 624b0d726..0d82c2d99 100644 --- a/annotation/clinvar_xml_parser.py +++ b/annotation/clinvar_xml_parser.py @@ -87,9 +87,9 @@ def assign_better_hgvs(self, text: str) -> str: def hgvs_score(some_hgvs): if some_hgvs is None: return 0 - if not "c." in some_hgvs: + if "c." not in some_hgvs: return 1 - if not "(" in some_hgvs: + if "(" not in some_hgvs: return 2 return 3 diff --git a/annotation/clinvar_xml_parser_via_vcv.py b/annotation/clinvar_xml_parser_via_vcv.py index 2dfa7fb97..05a17945c 100644 --- a/annotation/clinvar_xml_parser_via_vcv.py +++ b/annotation/clinvar_xml_parser_via_vcv.py @@ -3,11 +3,16 @@ from Bio import Entrez -from annotation.clinvar_xml_parser import ClinVarXmlParser, ClinVarXmlParserOutput, CLINVAR_REVIEW_STATUS_TO_STARS, \ - CLINVAR_TO_VG_CLIN_SIG, SOMATIC_CLIN_SIG_VALUE +from annotation.clinvar_xml_parser import ( + CLINVAR_REVIEW_STATUS_TO_STARS, + CLINVAR_TO_VG_CLIN_SIG, + SOMATIC_CLIN_SIG_VALUE, + ClinVarXmlParser, + ClinVarXmlParserOutput, +) from annotation.models import ClinVarRecord from classification.enums import AlleleOriginBucket -from library.utils.xml_utils import parser_path, PP +from library.utils.xml_utils import PP, parser_path class ConditionList: diff --git a/annotation/fake_annotation.py b/annotation/fake_annotation.py index 9d8bea311..2e06382ab 100644 --- a/annotation/fake_annotation.py +++ b/annotation/fake_annotation.py @@ -10,14 +10,27 @@ from django.utils import timezone from annotation.models import ClinVarReviewStatus, GeneAnnotationRelease -from annotation.models.models import VariantAnnotationVersion, ClinVarVersion, \ - HumanProteinAtlasAnnotationVersion, AnnotationVersion, ClinVar, ClinVarCitation, \ - ClinVarCitationsCollection, VariantAnnotation, AnnotationRun, AnnotationRangeLock, GeneAnnotationVersion +from annotation.models.models import ( + AnnotationRangeLock, + AnnotationRun, + AnnotationVersion, + ClinVar, + ClinVarCitation, + ClinVarCitationsCollection, + ClinVarVersion, + GeneAnnotationVersion, + HumanProteinAtlasAnnotationVersion, + VariantAnnotation, + VariantAnnotationVersion, +) from annotation.models.models_citations import CitationIdNormalized, CitationSource from genes.hgvs import HGVSMatcher from genes.models import GeneAnnotationImport from genes.models_enums import AnnotationConsortium -from ontology.tests.test_data_ontology import create_ontology_test_data, create_test_ontology_version +from ontology.tests.test_data_ontology import ( + create_ontology_test_data, + create_test_ontology_version, +) from snpdb.models import Variant from snpdb.models.models_genome import GenomeBuild from snpdb.tests.utils.vcf_testing_utils import slowly_create_loci_and_variants_for_vcf diff --git a/annotation/grids.py b/annotation/grids.py index 8328e1a3f..941d74554 100644 --- a/annotation/grids.py +++ b/annotation/grids.py @@ -2,12 +2,17 @@ from datetime import timedelta from typing import Any -from django.db.models import QuerySet, ExpressionWrapper, F, fields - -from annotation.models import VariantAnnotationVersion, AnnotationRun, AnnotationStatus, VariantAnnotationPipelineType +from django.db.models import ExpressionWrapper, F, QuerySet, fields + +from annotation.models import ( + AnnotationRun, + AnnotationStatus, + VariantAnnotationPipelineType, + VariantAnnotationVersion, +) from genes.models_enums import AnnotationConsortium from snpdb.models.models_genome import GenomeBuild -from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder, CellData +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder class AnnotationRunColumns(DatatableConfig): diff --git a/annotation/import_task_factories.py b/annotation/import_task_factories.py index 0cc2c4902..750e9f25f 100644 --- a/annotation/import_task_factories.py +++ b/annotation/import_task_factories.py @@ -1,9 +1,14 @@ import vcf -from annotation.tasks.import_clinvar_vcf_task import ImportCreateVersionForClinVarVCFTask, ProcessClinVarVCFDataTask, \ - ImportClinVarSuccessTask +from annotation.tasks.import_clinvar_vcf_task import ( + ImportClinVarSuccessTask, + ImportCreateVersionForClinVarVCFTask, + ProcessClinVarVCFDataTask, +) from annotation.vcf_files.import_clinvar_vcf import check_can_import_clinvar -from upload.import_task_factories.abstract_vcf_import_task_factory import AbstractVCFImportTaskFactory +from upload.import_task_factories.abstract_vcf_import_task_factory import ( + AbstractVCFImportTaskFactory, +) from upload.models import UploadedFileTypes diff --git a/annotation/management/commands/calculate_sample_stats.py b/annotation/management/commands/calculate_sample_stats.py index 88479780b..8a90e68c5 100644 --- a/annotation/management/commands/calculate_sample_stats.py +++ b/annotation/management/commands/calculate_sample_stats.py @@ -2,10 +2,15 @@ from django.core.management.base import BaseCommand -from annotation.models import SampleVariantAnnotationStats, \ - SampleVariantAnnotationStatsPassingFilter, SampleGeneAnnotationStats, \ - SampleGeneAnnotationStatsPassingFilter, SampleClinVarAnnotationStats, \ - SampleClinVarAnnotationStatsPassingFilter, VCFAnnotationStats +from annotation.models import ( + SampleClinVarAnnotationStats, + SampleClinVarAnnotationStatsPassingFilter, + SampleGeneAnnotationStats, + SampleGeneAnnotationStatsPassingFilter, + SampleVariantAnnotationStats, + SampleVariantAnnotationStatsPassingFilter, + VCFAnnotationStats, +) from annotation.tasks.calculate_sample_stats import calculate_needed_stats from snpdb.models import SampleStats, SampleStatsPassingFilter diff --git a/annotation/management/commands/fix_annotation_link_transcripts.py b/annotation/management/commands/fix_annotation_link_transcripts.py index 1c44c19e6..289458a16 100755 --- a/annotation/management/commands/fix_annotation_link_transcripts.py +++ b/annotation/management/commands/fix_annotation_link_transcripts.py @@ -3,10 +3,15 @@ import logging from django.core.management.base import BaseCommand -from django.db.models import Q, Func, Value, F - -from annotation.models import VariantAnnotation, VariantTranscriptAnnotation, VariantAnnotationVersion, \ - TranscriptVersion, defaultdict +from django.db.models import F, Func, Q, Value + +from annotation.models import ( + TranscriptVersion, + VariantAnnotation, + VariantAnnotationVersion, + VariantTranscriptAnnotation, + defaultdict, +) from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/management/commands/fix_annotation_sv_c_hgvs.py b/annotation/management/commands/fix_annotation_sv_c_hgvs.py index 52ea15b3d..9be52dc19 100755 --- a/annotation/management/commands/fix_annotation_sv_c_hgvs.py +++ b/annotation/management/commands/fix_annotation_sv_c_hgvs.py @@ -5,8 +5,14 @@ from django.core.management.base import BaseCommand -from annotation.models import VariantAnnotation, VariantTranscriptAnnotation, \ - TranscriptVersion, AnnotationRun, VariantAnnotationPipelineType, AnnotationStatus +from annotation.models import ( + AnnotationRun, + AnnotationStatus, + TranscriptVersion, + VariantAnnotation, + VariantAnnotationPipelineType, + VariantTranscriptAnnotation, +) from genes.hgvs import HGVSMatcher from snpdb.models import Variant from snpdb.models.models_genome import GenomeBuild @@ -55,7 +61,7 @@ def handle(self, *args, **options): hgvs_c = hgvs_matcher.variant_coordinate_to_hgvs_variant(variant_coordinate, transcript_accession) hgvs_c_results["ok"] += 1 - except Exception as e: + except Exception: hgvs_c = None # c.HGVS is ok to be blank hgvs_c_results["error"] += 1 diff --git a/annotation/management/commands/fix_annotation_sv_overlaps.py b/annotation/management/commands/fix_annotation_sv_overlaps.py index 3e64dcb19..65684d96e 100644 --- a/annotation/management/commands/fix_annotation_sv_overlaps.py +++ b/annotation/management/commands/fix_annotation_sv_overlaps.py @@ -6,7 +6,12 @@ from django.core.management.base import BaseCommand from django.db import transaction -from annotation.models import VariantAnnotation, VariantAnnotationVersion, VariantGeneOverlap, VEPSkippedReason +from annotation.models import ( + VariantAnnotation, + VariantAnnotationVersion, + VariantGeneOverlap, + VEPSkippedReason, +) from annotation.vcf_files.bulk_vep_vcf_annotation_inserter import SVGeneOverlapResolver diff --git a/annotation/management/commands/fix_columns_version2_damage_counts.py b/annotation/management/commands/fix_columns_version2_damage_counts.py index cb23004ca..358476e67 100644 --- a/annotation/management/commands/fix_columns_version2_damage_counts.py +++ b/annotation/management/commands/fix_columns_version2_damage_counts.py @@ -3,9 +3,9 @@ from django.conf import settings from django.core.management.base import BaseCommand -from django.db.models import Case, Value, IntegerField, When, Q, F +from django.db.models import Case, F, IntegerField, Q, Value, When -from annotation.models import VariantAnnotationVersion, VariantAnnotation +from annotation.models import VariantAnnotation, VariantAnnotationVersion class Command(BaseCommand): diff --git a/annotation/management/commands/fix_columns_version3_gnomad_hemi_count.py b/annotation/management/commands/fix_columns_version3_gnomad_hemi_count.py index a5a5804cc..55375c7a1 100644 --- a/annotation/management/commands/fix_columns_version3_gnomad_hemi_count.py +++ b/annotation/management/commands/fix_columns_version3_gnomad_hemi_count.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand from django.db.models import F -from annotation.models import VariantAnnotationVersion, VariantAnnotation +from annotation.models import VariantAnnotation, VariantAnnotationVersion class Command(BaseCommand): diff --git a/annotation/management/commands/fix_columns_version4_damage_counts.py b/annotation/management/commands/fix_columns_version4_damage_counts.py index 58a9017cc..d6bbe88e6 100644 --- a/annotation/management/commands/fix_columns_version4_damage_counts.py +++ b/annotation/management/commands/fix_columns_version4_damage_counts.py @@ -2,9 +2,9 @@ from functools import reduce from django.core.management.base import BaseCommand -from django.db.models import Case, Value, IntegerField, When, Q, F +from django.db.models import Case, F, IntegerField, Q, Value, When -from annotation.models import VariantAnnotationVersion, VariantAnnotation +from annotation.models import VariantAnnotation, VariantAnnotationVersion from annotation.pathogenicity_predictions import TOOLS diff --git a/annotation/management/commands/fix_impact_change_choice.py b/annotation/management/commands/fix_impact_change_choice.py index 5c27b8006..a0f366e99 100644 --- a/annotation/management/commands/fix_impact_change_choice.py +++ b/annotation/management/commands/fix_impact_change_choice.py @@ -1,9 +1,13 @@ from django.core.management.base import BaseCommand from django.db import connection -from django.db.models import Case, When, Value +from django.db.models import Case, Value, When from analysis.models import DamageNode -from annotation.models import VariantAnnotationVersion, VariantAnnotation, VariantTranscriptAnnotation +from annotation.models import ( + VariantAnnotation, + VariantAnnotationVersion, + VariantTranscriptAnnotation, +) class Command(BaseCommand): diff --git a/annotation/management/commands/fix_variant_annotation_add_hgvs_g.py b/annotation/management/commands/fix_variant_annotation_add_hgvs_g.py index 4f9b6bc5f..91bc85ff9 100644 --- a/annotation/management/commands/fix_variant_annotation_add_hgvs_g.py +++ b/annotation/management/commands/fix_variant_annotation_add_hgvs_g.py @@ -4,7 +4,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from annotation.models import VariantAnnotationVersion, VariantAnnotation +from annotation.models import VariantAnnotation, VariantAnnotationVersion from genes.hgvs import HGVSMatcher diff --git a/annotation/management/commands/gene_annotation.py b/annotation/management/commands/gene_annotation.py index e69577b9f..22a03ec0a 100644 --- a/annotation/management/commands/gene_annotation.py +++ b/annotation/management/commands/gene_annotation.py @@ -1,22 +1,35 @@ import argparse import os -from collections import defaultdict, Counter +from collections import Counter, defaultdict from datetime import timedelta from django.conf import settings from django.core.management import BaseCommand from django.utils import timezone -from annotation.models import GeneAnnotationVersion, OntologyTerm, GenomeBuild, AnnotationVersion, \ - InvalidAnnotationVersionError, GeneAnnotation, DBNSFPGeneAnnotationVersion, VariantAnnotationVersion +from annotation.models import ( + AnnotationVersion, + DBNSFPGeneAnnotationVersion, + GeneAnnotation, + GeneAnnotationVersion, + GenomeBuild, + InvalidAnnotationVersionError, + OntologyTerm, + VariantAnnotationVersion, +) from genes.gene_matching import ReleaseGeneMatcher -from genes.models import GeneAnnotationRelease, GnomADGeneConstraint, ReleaseGeneSymbolGene, Gene +from genes.models import Gene, GeneAnnotationRelease, GnomADGeneConstraint, ReleaseGeneSymbolGene from library.django_utils.django_file_utils import get_import_processing_filename -from ontology.models import OntologyService, GeneDiseaseClassification, OntologyTermRelation, \ - OntologyVersion, ONTOLOGY_RELATIONSHIP_MEDIUM_QUALITY_FILTER +from ontology.models import ( + ONTOLOGY_RELATIONSHIP_MEDIUM_QUALITY_FILTER, + GeneDiseaseClassification, + OntologyService, + OntologyTermRelation, + OntologyVersion, +) from ontology.ontology_traversal import get_ontology_traverser from ontology.panel_app_ontology import bulk_update_gene_relations, panel_app_bulk_data_age -from upload.vcf.sql_copy_files import write_sql_copy_csv, sql_copy_csv +from upload.vcf.sql_copy_files import sql_copy_csv, write_sql_copy_csv class Command(BaseCommand): @@ -285,7 +298,7 @@ def _add_new_columns_to_existing(self): snake = traverser.terms_for_gene_symbol(hgnc_ot.name, OntologyService.MONDO, max_depth=0) uc_symbol = gene_symbol.upper() - hgnc_data[uc_symbol]["mondo_terms"] = self.TERM_JOIN_STRING.join((str(lt) for lt in snake.leafs())) + hgnc_data[uc_symbol]["mondo_terms"] = self.TERM_JOIN_STRING.join(str(lt) for lt in snake.leafs()) hgnc_data[uc_symbol]["gene_disease"] = self._get_gene_disease(traverser, gene_symbol, Command.TERM_JOIN_STRING) @@ -351,7 +364,7 @@ def _add_missing_omim(self): if gene_symbol := gene_symbol_for_gene.get(ga.gene_id): try: snake = traverser.terms_for_gene_symbol(gene_symbol, OntologyService.OMIM, max_depth=1) - if omim_terms := self.TERM_JOIN_STRING.join((str(lt) for lt in snake.leafs())): + if omim_terms := self.TERM_JOIN_STRING.join(str(lt) for lt in snake.leafs()): ga.omim_terms = omim_terms update_records.append(ga) except ValueError: @@ -428,7 +441,7 @@ def _populate_gene_annotation_version(self, gav: GeneAnnotationVersion, gene_sym gene_symbol = hgnc_ot.name for ontology_service in [OntologyService.OMIM, OntologyService.HPO, OntologyService.MONDO]: snake = traverser.terms_for_gene_symbol(gene_symbol, ontology_service, max_depth=1) - service_terms[ontology_service] = self.TERM_JOIN_STRING.join((str(lt) for lt in snake.leafs())) + service_terms[ontology_service] = self.TERM_JOIN_STRING.join(str(lt) for lt in snake.leafs()) gene_disease_supportive_or_below, gene_disease_moderate_or_above = self._get_gene_disease(traverser, gene_symbol, @@ -475,7 +488,7 @@ def _populate_gene_annotation_version(self, gav: GeneAnnotationVersion, gene_sym for gene_id, ga_data in annotation_by_gene.items(): ga_data["gene_id"] = gene_id ga_data["version_id"] = gav.pk - gene_annotation_records.append(tuple((ga_data.get(k) for k in self.GENE_ANNOTATION_HEADER))) + gene_annotation_records.append(tuple(ga_data.get(k) for k in self.GENE_ANNOTATION_HEADER)) if gene_annotation_records: self._write_records(gav, gene_annotation_records) diff --git a/annotation/management/commands/human_protein_atlas_import.py b/annotation/management/commands/human_protein_atlas_import.py index 5d3425b87..101def87d 100644 --- a/annotation/management/commands/human_protein_atlas_import.py +++ b/annotation/management/commands/human_protein_atlas_import.py @@ -9,11 +9,11 @@ from django.core.management.base import BaseCommand, CommandError from annotation.models import HumanProteinAtlasAnnotationVersion, HumanProteinAtlasTissueSample -from genes.models import GeneSymbol, Gene +from genes.models import Gene, GeneSymbol from genes.models_enums import AnnotationConsortium from library.django_utils.django_file_utils import get_import_processing_filename from library.utils import file_sha256sum -from upload.vcf.sql_copy_files import write_sql_copy_csv, sql_copy_csv +from upload.vcf.sql_copy_files import sql_copy_csv, write_sql_copy_csv def get_or_create_hpa_samples_ids(df) -> dict: diff --git a/annotation/management/commands/import_dbnsfp_gene_annotation.py b/annotation/management/commands/import_dbnsfp_gene_annotation.py index c29674005..f164338c1 100644 --- a/annotation/management/commands/import_dbnsfp_gene_annotation.py +++ b/annotation/management/commands/import_dbnsfp_gene_annotation.py @@ -3,7 +3,7 @@ import pandas as pd from django.core.management import BaseCommand, CommandError -from annotation.models import DBNSFPGeneAnnotationVersion, DBNSFPGeneAnnotation +from annotation.models import DBNSFPGeneAnnotation, DBNSFPGeneAnnotationVersion from genes.models import GeneSymbol from library.pandas_utils import df_nan_to_none from library.utils import file_sha256sum diff --git a/annotation/management/commands/one_off_populate_missing_symbolic_hgvs.py b/annotation/management/commands/one_off_populate_missing_symbolic_hgvs.py index 6f2e4c4c5..2067d8f81 100644 --- a/annotation/management/commands/one_off_populate_missing_symbolic_hgvs.py +++ b/annotation/management/commands/one_off_populate_missing_symbolic_hgvs.py @@ -3,8 +3,12 @@ from django.core.management import BaseCommand -from annotation.models import VariantAnnotation, VariantTranscriptAnnotation, AbstractVariantAnnotation -from genes.hgvs import HGVSMatcher, HGVSException +from annotation.models import ( + AbstractVariantAnnotation, + VariantAnnotation, + VariantTranscriptAnnotation, +) +from genes.hgvs import HGVSException, HGVSMatcher from snpdb.models import GenomeBuild, Variant, VariantCoordinate @@ -33,7 +37,7 @@ def _update_annotation(self, v: Variant, variant_coordinate: VariantCoordinate, try: hgvs_c = matcher.variant_coordinate_to_hgvs_variant(variant_coordinate, transcript_accession) va_list.append(va) - except (ValueError, HGVSException) as e: + except (ValueError, HGVSException): # print(f"FAILED: {transcript_accession}: {e} - {quick_reject=}") pass diff --git a/annotation/management/commands/reset_annotation_states.py b/annotation/management/commands/reset_annotation_states.py index ee5da3ecb..51a269a42 100644 --- a/annotation/management/commands/reset_annotation_states.py +++ b/annotation/management/commands/reset_annotation_states.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand -from annotation.models import AnnotationRun, AnnotationRangeLock +from annotation.models import AnnotationRangeLock, AnnotationRun class Command(BaseCommand): diff --git a/annotation/management/commands/vcf_sample_gene_damage_counts.py b/annotation/management/commands/vcf_sample_gene_damage_counts.py index c5f792ff8..0d8528299 100755 --- a/annotation/management/commands/vcf_sample_gene_damage_counts.py +++ b/annotation/management/commands/vcf_sample_gene_damage_counts.py @@ -6,7 +6,9 @@ from annotation.models.models import VariantAnnotationVersion from annotation.models.models_gene_counts import CohortGeneCounts -from annotation.tasks.cohort_sample_gene_damage_counts import get_or_create_gene_count_type_and_values +from annotation.tasks.cohort_sample_gene_damage_counts import ( + get_or_create_gene_count_type_and_values, +) from snpdb.models import VCF, ImportStatus diff --git a/annotation/management/commands/vep_run.py b/annotation/management/commands/vep_run.py index a7f08fdca..98984554d 100644 --- a/annotation/management/commands/vep_run.py +++ b/annotation/management/commands/vep_run.py @@ -10,7 +10,7 @@ from django.core.management.base import BaseCommand from annotation.models import VariantAnnotationPipelineType -from annotation.vep_annotation import run_vep, VEPConfig +from annotation.vep_annotation import VEPConfig, run_vep from snpdb.models.models_genome import GenomeBuild DO_SMALL = False diff --git a/annotation/management/commands/vep_version.py b/annotation/management/commands/vep_version.py index c916d7765..e6df85f80 100644 --- a/annotation/management/commands/vep_version.py +++ b/annotation/management/commands/vep_version.py @@ -1,6 +1,10 @@ from django.core.management.base import BaseCommand -from annotation.vep_annotation import get_vep_version, VEPConfig, vep_dict_to_variant_annotation_version_kwargs +from annotation.vep_annotation import ( + VEPConfig, + get_vep_version, + vep_dict_to_variant_annotation_version_kwargs, +) from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/manual_variant_entry.py b/annotation/manual_variant_entry.py index 51aaf31ab..28f61b4e0 100644 --- a/annotation/manual_variant_entry.py +++ b/annotation/manual_variant_entry.py @@ -4,15 +4,28 @@ from django.core.exceptions import PermissionDenied from annotation.models import ManualVariantEntryType -from annotation.models.models import ManualVariantEntryCollection, ManualVariantEntry -from annotation.tasks.process_manual_variants_task import ManualVariantsPostInsertTask, get_manual_variant_coordinates +from annotation.models.models import ManualVariantEntry, ManualVariantEntryCollection +from annotation.tasks.process_manual_variants_task import ( + ManualVariantsPostInsertTask, + get_manual_variant_coordinates, +) from library.django_utils.django_file_utils import get_import_processing_dir from library.genomics.vcf_utils import write_vcf_from_variant_coordinates from library.utils import full_class_name from snpdb.models.models_enums import ImportSource from snpdb.models.models_genome import GenomeBuild -from upload.models import UploadPipeline, UploadedFile, UploadStep, UploadedManualVariantEntryCollection -from upload.models.models_enums import UploadedFileTypes, UploadStepTaskType, VCFPipelineStage, UploadStepOrigin +from upload.models import ( + UploadedFile, + UploadedManualVariantEntryCollection, + UploadPipeline, + UploadStep, +) +from upload.models.models_enums import ( + UploadedFileTypes, + UploadStepOrigin, + UploadStepTaskType, + VCFPipelineStage, +) from upload.upload_processing import process_upload_pipeline diff --git a/annotation/models.py b/annotation/models.py index 4b403173f..0a8c8a6b1 100644 --- a/annotation/models.py +++ b/annotation/models.py @@ -1,20 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from annotation.models.models import ClinVarVersion, ClinVar, ClinVarRecordCollection, ClinVarRecord, \ - ClinVarCitationsCollection, ClinVarCitation, DBNSFPGeneAnnotationVersion, DBNSFPGeneAnnotation, \ - GeneAnnotationVersion, GeneAnnotation, HumanProteinAtlasAnnotationVersion, HumanProteinAtlasTissueSample, \ - HumanProteinAtlasAnnotation, VariantAnnotationVersion, VCFAnnotationStats, AnnotationRangeLock, \ - AnnotationRun, VariantAnnotation, VariantTranscriptAnnotation, VariantGeneOverlap, ManualVariantEntryCollection, \ - ManualVariantEntry, CreatedManualVariant, AnnotationVersion, CachedWebResource, GeneSymbolCitation, \ - GenePubMedCount, MutationalSignatureInfo -from annotation.models.models_citations import Citation -from annotation.models.models_gene_counts import VariantSource, SampleAnnotationVersionVariantSource, GeneCountType, \ - GeneValue, GeneValueCountCollection, GeneValueCount, CohortGeneCounts -from annotation.models.models_phenotype_match import DescriptionProcessingStatus, PhenotypeDescription, TextPhenotype, \ - TextPhenotypeSentence, TextPhenotypeMatch, PatientTextPhenotype -from annotation.models.models_sample_stats import SampleVariantAnnotationStats, \ - SampleVariantAnnotationStatsPassingFilter, SampleGeneAnnotationStats, SampleGeneAnnotationStatsPassingFilter, \ - SampleClinVarAnnotationStats, SampleClinVarAnnotationStatsPassingFilter -from annotation.models.models_version_diff import VersionDiff, VersionDiffFromToResult, VersionDiffChangeCountResult, \ - VariantAnnotationVersionDiff # pylint: enable=unused-import diff --git a/annotation/models/models.py b/annotation/models/models.py index c8f2a1db0..7a5c5cac3 100644 --- a/annotation/models/models.py +++ b/annotation/models/models.py @@ -2,18 +2,19 @@ import os import re from collections import defaultdict +from collections.abc import Callable, Iterable from datetime import datetime from functools import cached_property -from typing import Optional, Callable, Iterable +from typing import Optional from Bio.Data.IUPACData import protein_letters_1to3 from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.fields import ArrayField from django.core.exceptions import PermissionDenied -from django.db import models, transaction, connection -from django.db.models import F, Q, QuerySet, Subquery, OuterRef, Min, Max -from django.db.models.deletion import PROTECT, CASCADE, SET_NULL +from django.db import connection, models, transaction +from django.db.models import F, Max, Min, OuterRef, Q, QuerySet, Subquery +from django.db.models.deletion import CASCADE, PROTECT, SET_NULL from django.db.models.functions import Coalesce, Greatest from django.db.models.signals import pre_delete from django.dispatch.dispatcher import receiver @@ -24,32 +25,61 @@ from psqlextra.models import PostgresPartitionedModel from psqlextra.types import PostgresPartitioningMethod -from annotation.external_search_terms import get_variant_search_terms, get_variant_pubmed_search_terms -from annotation.models.damage_enums import Polyphen2Prediction, FATHMMPrediction, MutationTasterPrediction, \ - SIFTPrediction, PathogenicityImpact, MutationAssessorPrediction, ALoFTPrediction, \ - AlphaMissensePrediction, ClinPredPrediction, MetaRNNPrediction, PrimateAIPrediction +from annotation.external_search_terms import ( + get_variant_pubmed_search_terms, + get_variant_search_terms, +) +from annotation.models.damage_enums import ( + ALoFTPrediction, + AlphaMissensePrediction, + ClinPredPrediction, + FATHMMPrediction, + MetaRNNPrediction, + MutationAssessorPrediction, + MutationTasterPrediction, + PathogenicityImpact, + Polyphen2Prediction, + PrimateAIPrediction, + SIFTPrediction, +) from annotation.models.models_citations import Citation, CitationFetchRequest, CitationFetchResponse +from annotation.models.models_enums import ( + AnnotationStatus, + ClinVarReviewStatus, + EssentialGeneCRISPR, + EssentialGeneCRISPR2, + EssentialGeneGeneTrap, + HumanProteinAtlasAbundance, + ManualVariantEntryType, + VariantAnnotationPipelineType, + VEPSkippedReason, +) from annotation.models.repeat_masker import RepeatMaskerSummary -from annotation.models.models_enums import AnnotationStatus, \ - ClinVarReviewStatus, VEPSkippedReason, \ - ManualVariantEntryType, HumanProteinAtlasAbundance, EssentialGeneCRISPR, EssentialGeneCRISPR2, \ - EssentialGeneGeneTrap, VariantAnnotationPipelineType from annotation.utils.clinvar_constants import CLINVAR_REVIEW_EXPERT_PANEL_STARS_VALUE from classification.enums import AlleleOriginBucket -from genes.models import GeneSymbol, Gene, TranscriptVersion, Transcript, GeneAnnotationRelease +from genes.models import Gene, GeneAnnotationRelease, GeneSymbol, Transcript, TranscriptVersion from genes.models_enums import AnnotationConsortium from library.django_utils import object_is_referenced from library.django_utils.data_archive_mixin import DataArchiveMixin from library.django_utils.django_partition import RelatedModelsPartitionModel -from library.log_utils import report_message -from snpdb.archive import DataArchivedError from library.genomics import parse_gnomad_coord from library.genomics.vcf_enums import VariantClass -from library.utils import invert_dict, name_from_filename, first, all_equal +from library.log_utils import report_message +from library.utils import all_equal, first, invert_dict, name_from_filename from ontology.models import OntologyVersion from patients.models_enums import GnomADPopulation -from snpdb.models import GenomeBuild, Variant, VariantGridColumn, Q, VCF, DBSNP_PATTERN, VARIANT_PATTERN, \ - HGVS_UNCLEANED_PATTERN, Allele, VARIANT_SYMBOLIC_PATTERN +from snpdb.archive import DataArchivedError +from snpdb.models import ( + DBSNP_PATTERN, + HGVS_UNCLEANED_PATTERN, + VARIANT_PATTERN, + VARIANT_SYMBOLIC_PATTERN, + VCF, + Allele, + GenomeBuild, + Q, + Variant, +) from snpdb.models.models_enums import ImportStatus @@ -193,7 +223,7 @@ def fix_name(name: str): name = name[25:] return name - db_names = list(sorted(fix_name(db_name) for db_name in re.split("[|,]", db_name_text))) + db_names = sorted(fix_name(db_name) for db_name in re.split("[|,]", db_name_text)) return db_names return [] @@ -353,10 +383,10 @@ def set_allele_for_variants(cls, allele): ).update(allele=allele) def records_with_min_stars(self, min_stars: int) -> list['ClinVarRecord']: - return list(sorted(self.clinvarrecord_set.filter(stars__gte=min_stars), reverse=True)) + return sorted(self.clinvarrecord_set.filter(stars__gte=min_stars), reverse=True) def update_with_records_and_save(self, records: list['ClinVarRecord']): - records = list(sorted(records, reverse=True)) + records = sorted(records, reverse=True) self.clinvarrecord_set.all().delete() for record in records: record.clinvar_record_collection = self @@ -415,11 +445,11 @@ def conditions(self) -> list[str]: return [] def mark_invalid(self): - setattr(self, '_invalid', True) + self._invalid = True def __bool__(self) -> bool: if hasattr(self, '_invalid'): - return not getattr(self, '_invalid') + return not self._invalid return True def __lt__(self, other): @@ -832,7 +862,10 @@ def get_raw_score_pathogenic_prediction_funcs(self) -> dict[str, Callable]: """Raw-score + pred contributions to predictions_num_pathogenic at v4. Empty pre-v4.""" if self.columns_version < 4: return {} - from annotation.pathogenicity_predictions import raw_score_pathogenic_funcs, pred_pathogenic_funcs + from annotation.pathogenicity_predictions import ( + pred_pathogenic_funcs, + raw_score_pathogenic_funcs, + ) return {**raw_score_pathogenic_funcs(), **pred_pathogenic_funcs()} @cached_property diff --git a/annotation/models/models_citations.py b/annotation/models/models_citations.py index df07c9e15..d18aa3b14 100644 --- a/annotation/models/models_citations.py +++ b/annotation/models/models_citations.py @@ -2,11 +2,12 @@ import itertools import re import typing +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field from datetime import timedelta from enum import Enum from functools import cached_property -from typing import Optional, Iterable, Union, Any, Iterator +from typing import Any, Optional, Union from Bio import Entrez, Medline from django.db import models @@ -693,7 +694,7 @@ def _load_from_entrez(self, entrez_db: EntrezDbType, ids: list[CitationIdNormali except Exception as ex: # if this fails it's probably because a single id in ids ruined it for everybody report_exc_info(extra_data={"target": f'Error when attempting to Entrez.efetch ids {ids}'}) - self._mark_error_if_not_fetched(ids, f'Error when attempting to Entrez.efetch ids {request_ids} : {str(ex)}') + self._mark_error_if_not_fetched(ids, f'Error when attempting to Entrez.efetch ids {request_ids} : {ex!s}') @staticmethod def _populate_from_entrez(citation: Citation, record: JsonObjType): @@ -735,7 +736,7 @@ def _load_from_nbk(self, ids: Iterable[CitationIdNormalized]): except RuntimeError as run_error: report_exc_info(extra_data={'bookshelf_rid': bookshelf_rid}) - self._mark_error_if_not_fetched([bookshelf_rid], f'Error when attempting to Entrez.efetch ids {bookshelf_rid} : {str(run_error)}') + self._mark_error_if_not_fetched([bookshelf_rid], f'Error when attempting to Entrez.efetch ids {bookshelf_rid} : {run_error!s}') @staticmethod def _populate_from_nbk(citation: Citation, record: JsonObjType): diff --git a/annotation/models/models_cohort_stats.py b/annotation/models/models_cohort_stats.py index e48beca54..be6158a9b 100644 --- a/annotation/models/models_cohort_stats.py +++ b/annotation/models/models_cohort_stats.py @@ -8,7 +8,7 @@ from django.db.models.query_utils import Q from django_extensions.db.models import TimeStampedModel -from annotation.models.models import VariantAnnotationVersion, ClinVarVersion, GeneAnnotationVersion +from annotation.models.models import ClinVarVersion, GeneAnnotationVersion, VariantAnnotationVersion from snpdb.models import SampleStatsCodeVersion diff --git a/annotation/models/models_gene_counts.py b/annotation/models/models_gene_counts.py index 4b9482cb7..668699ba1 100644 --- a/annotation/models/models_gene_counts.py +++ b/annotation/models/models_gene_counts.py @@ -2,16 +2,16 @@ from typing import Optional from django.db import models -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.db.models.deletion import CASCADE from model_utils.managers import InheritanceManager from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version -from annotation.models.models import VariantAnnotationVersion, VariantAnnotation +from annotation.models.models import VariantAnnotation, VariantAnnotationVersion from classification.models import Classification, ClassificationModification from genes.models import Gene from library.utils import rgb_invert -from snpdb.models import Sample, Cohort, ShareLevel, GenomeBuild, Variant +from snpdb.models import Cohort, GenomeBuild, Sample, ShareLevel, Variant from snpdb.models.models_enums import ProcessingStatus from variantgrid.celery import app diff --git a/annotation/models/models_phenotype_match.py b/annotation/models/models_phenotype_match.py index 7d6664f7b..34827a1ca 100644 --- a/annotation/models/models_phenotype_match.py +++ b/annotation/models/models_phenotype_match.py @@ -3,7 +3,7 @@ from cache_memoize import cache_memoize from django.contrib.auth.models import User from django.db import models -from django.db.models import QuerySet, OuterRef, Count, Subquery +from django.db.models import Count, OuterRef, QuerySet, Subquery from django.db.models.deletion import CASCADE, SET_NULL from annotation.phenotype_matcher import get_ambiguous_acronym_denylist diff --git a/annotation/models/models_sample_stats.py b/annotation/models/models_sample_stats.py index 7521d72b8..785359879 100644 --- a/annotation/models/models_sample_stats.py +++ b/annotation/models/models_sample_stats.py @@ -11,7 +11,7 @@ from django.db.models.deletion import CASCADE from django_extensions.db.models import TimeStampedModel -from annotation.models.models import VariantAnnotationVersion, ClinVarVersion, GeneAnnotationVersion +from annotation.models.models import ClinVarVersion, GeneAnnotationVersion, VariantAnnotationVersion from snpdb.models import Sample, SampleStatsCodeVersion from snpdb.models.models_enums import BuiltInFilters diff --git a/annotation/models/models_version_diff.py b/annotation/models/models_version_diff.py index a5a213bfe..fae698904 100644 --- a/annotation/models/models_version_diff.py +++ b/annotation/models/models_version_diff.py @@ -3,8 +3,7 @@ from functools import reduce import pandas as pd -from django.db import connection -from django.db import models +from django.db import connection, models from django.db.models.deletion import CASCADE from django.utils.timesince import timesince from model_utils.managers import InheritanceManager diff --git a/annotation/pathogenicity_predictions.py b/annotation/pathogenicity_predictions.py index 9cee48163..8c25e0551 100644 --- a/annotation/pathogenicity_predictions.py +++ b/annotation/pathogenicity_predictions.py @@ -1,5 +1,6 @@ +from collections.abc import Callable from dataclasses import dataclass -from typing import Callable, Optional +from typing import Optional @dataclass(frozen=True) diff --git a/annotation/phenotype_matcher.py b/annotation/phenotype_matcher.py index 29d986122..83b1b0fe7 100644 --- a/annotation/phenotype_matcher.py +++ b/annotation/phenotype_matcher.py @@ -2,7 +2,8 @@ import logging import re from collections import defaultdict -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any, Optional import Levenshtein from cache_memoize import cache_memoize @@ -10,7 +11,7 @@ from library.constants import DAY_SECS from library.log_utils import log_traceback from library.utils import is_not_none -from ontology.models import OntologyTerm, OntologyService, OntologyTermRelation, OntologyVersion +from ontology.models import OntologyService, OntologyTerm, OntologyTermRelation, OntologyVersion # There can be more than 1 term matching a string, eg OMIM has 1849 terms that match 2 or more IDs CodePK = Any diff --git a/annotation/phenotype_matching.py b/annotation/phenotype_matching.py index 80762e1ea..a1cd312d4 100644 --- a/annotation/phenotype_matching.py +++ b/annotation/phenotype_matching.py @@ -1,14 +1,20 @@ import logging import multiprocessing as mp import time -from typing import Optional, Iterable +from collections.abc import Iterable +from typing import Optional import nltk from django.conf import settings from django.db import connections -from annotation.models.models_phenotype_match import TextPhenotypeMatch, PhenotypeDescription, TextPhenotype, \ - TextPhenotypeSentence, filter_ambiguous_acronym_matches +from annotation.models.models_phenotype_match import ( + PhenotypeDescription, + TextPhenotype, + TextPhenotypeMatch, + TextPhenotypeSentence, + filter_ambiguous_acronym_matches, +) from annotation.phenotype_matcher import PhenotypeMatcher, SkipAllPhenotypeMatchException from annotation.phenotype_tokenizer import PhenotypeTokenizer from patients.models import Patient diff --git a/annotation/regexes.py b/annotation/regexes.py index bce35e7cf..b9a2b852c 100644 --- a/annotation/regexes.py +++ b/annotation/regexes.py @@ -1,7 +1,7 @@ import re from enum import Enum -from re import RegexFlag -from typing import Union, Match, Optional +from re import Match, RegexFlag +from typing import Optional, Union from annotation.models.models_citations import CitationIdNormalized from ontology.models import OntologyService @@ -212,7 +212,7 @@ def append_result_if_length(db_regex: DbRefRegex, match: Optional[Match], must_e return False result = DbRefRegexResult(cregx=db_regex, idx=id_group, match=match) - if not result in already_added: + if result not in already_added: results.append(result) already_added.add(result) return True diff --git a/annotation/serializers.py b/annotation/serializers.py index 04db426f6..ce979c367 100644 --- a/annotation/serializers.py +++ b/annotation/serializers.py @@ -3,8 +3,12 @@ from rest_framework import serializers from annotation.models import AnnotationStatus -from annotation.models.models import VariantAnnotationVersion, VariantAnnotation, ManualVariantEntryCollection -from snpdb.serializers import VariantSerializer, TimestampField +from annotation.models.models import ( + ManualVariantEntryCollection, + VariantAnnotation, + VariantAnnotationVersion, +) +from snpdb.serializers import TimestampField, VariantSerializer class VariantAnnotationVersionSerializer(serializers.ModelSerializer): diff --git a/annotation/signals/citation_preview.py b/annotation/signals/citation_preview.py index 685dc9274..e1feae9f5 100644 --- a/annotation/signals/citation_preview.py +++ b/annotation/signals/citation_preview.py @@ -2,7 +2,7 @@ from annotation.models import CitationFetchRequest from annotation.models.models_citations import CitationIdNormalized -from library.preview_request import preview_request_signal, PreviewRequest +from library.preview_request import PreviewRequest, preview_request_signal @receiver(signal=preview_request_signal) diff --git a/annotation/signals/citation_search.py b/annotation/signals/citation_search.py index d1590e21c..759c17a0b 100644 --- a/annotation/signals/citation_search.py +++ b/annotation/signals/citation_search.py @@ -1,8 +1,8 @@ import re -from annotation.models import CitationFetchRequest, Citation -from annotation.models.models_citations import CitationSource, CitationIdNormalized -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from annotation.models import Citation, CitationFetchRequest +from annotation.models.models_citations import CitationIdNormalized, CitationSource +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/annotation/signals/clinvar_annotation_health_check.py b/annotation/signals/clinvar_annotation_health_check.py index 8b9133784..322af06c9 100644 --- a/annotation/signals/clinvar_annotation_health_check.py +++ b/annotation/signals/clinvar_annotation_health_check.py @@ -3,8 +3,11 @@ from django.dispatch import receiver from annotation.models import ClinVarVersion -from library.health_check import HealthCheckRequest, HealthCheckAge, \ - health_check_overall_stats_signal +from library.health_check import ( + HealthCheckAge, + HealthCheckRequest, + health_check_overall_stats_signal, +) from snpdb.models import GenomeBuild diff --git a/annotation/tasks/annotate_variants.py b/annotation/tasks/annotate_variants.py index 72a895629..80a34108d 100644 --- a/annotation/tasks/annotate_variants.py +++ b/annotation/tasks/annotate_variants.py @@ -10,7 +10,7 @@ from django.utils import timezone from annotation.annotation_version_querysets import get_variants_qs_for_annotation -from annotation.annotsv_annotation import run_annotsv, annotsv_check_command_line_version_match +from annotation.annotsv_annotation import annotsv_check_command_line_version_match, run_annotsv from annotation.models import AnnotationStatus, GenomeBuild, VariantAnnotationPipelineType from annotation.models.models import AnnotationRun, InvalidAnnotationVersionError from annotation.signals.manual_signals import annotation_run_complete_signal @@ -18,11 +18,10 @@ from annotation.vep_annotation import get_vep_command, vep_check_command_line_version_match from eventlog.models import create_event from library.enums.log_level import LogLevel -from library.log_utils import get_traceback, report_message, log_traceback +from library.log_utils import get_traceback, log_traceback, report_message from library.utils import execute_cmd -from library.utils.file_utils import name_from_filename, mk_path_for_file -from snpdb.variants_to_vcf import write_contig_sorted_values_to_vcf_file, VARIANT_GRID_INFO_DICT - +from library.utils.file_utils import mk_path_for_file, name_from_filename +from snpdb.variants_to_vcf import VARIANT_GRID_INFO_DICT, write_contig_sorted_values_to_vcf_file # #2667: kick the single-authority dispatcher by name to avoid importing annotation_scheduler_task # (which imports this module). Mirror of analysis _trigger_rescheduling (#346). diff --git a/annotation/tasks/annotation_scheduler_task.py b/annotation/tasks/annotation_scheduler_task.py index c41dff3d9..4b0aa037b 100644 --- a/annotation/tasks/annotation_scheduler_task.py +++ b/annotation/tasks/annotation_scheduler_task.py @@ -9,15 +9,21 @@ from django.utils import timezone from annotation.annotation_version_querysets import get_variants_qs_for_annotation -from annotation.annotation_versions import get_annotation_range_lock_and_unannotated_count, \ - merge_pending_range_locks +from annotation.annotation_versions import ( + get_annotation_range_lock_and_unannotated_count, + merge_pending_range_locks, +) from annotation.celery_utils import annotation_worker_slots -from annotation.models import AnnotationRun, AnnotationStatus, VariantAnnotationPipelineType, \ - VariantAnnotationVersion -from annotation.models.models import AnnotationVersion, AnnotationRangeLock +from annotation.models import ( + AnnotationRun, + AnnotationStatus, + VariantAnnotationPipelineType, + VariantAnnotationVersion, +) +from annotation.models.models import AnnotationRangeLock, AnnotationVersion from annotation.tasks.annotate_variants import annotate_variants from library.log_utils import log_traceback -from snpdb.models import GenomeBuild, ImportStatus, Sample, VCF, Variant +from snpdb.models import VCF, GenomeBuild, ImportStatus, Sample, Variant @celery.shared_task(queue='scheduling_single_worker') diff --git a/annotation/tasks/calculate_sample_stats.py b/annotation/tasks/calculate_sample_stats.py index f988c86bc..96df2af23 100644 --- a/annotation/tasks/calculate_sample_stats.py +++ b/annotation/tasks/calculate_sample_stats.py @@ -10,8 +10,12 @@ from django.db.models.query_utils import Q from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version -from annotation.models import AnnotationVersion, CohortGenotypeVariantAnnotationStats, \ - CohortGenotypeGeneAnnotationStats, CohortGenotypeClinVarAnnotationStats +from annotation.models import ( + AnnotationVersion, + CohortGenotypeClinVarAnnotationStats, + CohortGenotypeGeneAnnotationStats, + CohortGenotypeVariantAnnotationStats, +) from annotation.models.damage_enums import PathogenicityImpact from annotation.models.models import InvalidAnnotationVersionError, VCFAnnotationStats from eventlog.models import create_event @@ -21,9 +25,19 @@ from library.git import Git from library.log_utils import get_traceback from library.utils.json_utils import canonical_filter_key -from snpdb.models import Cohort, CohortGenotypeCollection, CohortGenotypeStats, ImportStatus, VCF, Variant, \ - SampleStatsCodeVersion, Sequence, VCFLengthStatsCollection, VCFLengthStats -from snpdb.models import Zygosity +from snpdb.models import ( + VCF, + Cohort, + CohortGenotypeCollection, + CohortGenotypeStats, + ImportStatus, + SampleStatsCodeVersion, + Sequence, + Variant, + VCFLengthStats, + VCFLengthStatsCollection, + Zygosity, +) from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/tasks/cohort_sample_gene_damage_counts.py b/annotation/tasks/cohort_sample_gene_damage_counts.py index f598e9817..50be0123b 100644 --- a/annotation/tasks/cohort_sample_gene_damage_counts.py +++ b/annotation/tasks/cohort_sample_gene_damage_counts.py @@ -4,8 +4,14 @@ from celery.app.task import Task from annotation.models.models import VariantAnnotation, VariantAnnotationVersion -from annotation.models.models_gene_counts import GeneCountType, GeneValue, CohortGeneCounts, \ - SampleAnnotationVersionVariantSource, GeneValueCountCollection, GeneValueCount +from annotation.models.models_gene_counts import ( + CohortGeneCounts, + GeneCountType, + GeneValue, + GeneValueCount, + GeneValueCountCollection, + SampleAnnotationVersionVariantSource, +) from annotation.models.molecular_consequence_enums import MolecularConsequenceColors from classification.models import Classification from eventlog.models import create_event diff --git a/annotation/tasks/import_clinvar_vcf_task.py b/annotation/tasks/import_clinvar_vcf_task.py index 814b30a01..a1cca4448 100644 --- a/annotation/tasks/import_clinvar_vcf_task.py +++ b/annotation/tasks/import_clinvar_vcf_task.py @@ -1,8 +1,7 @@ import logging from annotation.models.models import ClinVarVersion -from annotation.vcf_files.import_clinvar_vcf import import_clinvar_vcf, \ - check_can_import_clinvar +from annotation.vcf_files.import_clinvar_vcf import check_can_import_clinvar, import_clinvar_vcf from upload.models import UploadedClinVarVersion from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask from upload.vcf.vcf_import import vcf_detect_genome_build diff --git a/annotation/tasks/process_manual_variants_task.py b/annotation/tasks/process_manual_variants_task.py index 0f0cb8e03..957a0a055 100644 --- a/annotation/tasks/process_manual_variants_task.py +++ b/annotation/tasks/process_manual_variants_task.py @@ -5,9 +5,12 @@ from annotation.models import ManualVariantEntryType from annotation.models.models import ManualVariantEntry from genes.hgvs import HGVSMatcher -from snpdb.clingen_allele import populate_clingen_alleles_for_variants, get_clingen_alleles_from_external_code +from snpdb.clingen_allele import ( + get_clingen_alleles_from_external_code, + populate_clingen_alleles_for_variants, +) from snpdb.models import Variant, VariantCoordinate -from snpdb.models.models_enums import ImportStatus, ClinGenAlleleExternalRecordType +from snpdb.models.models_enums import ClinGenAlleleExternalRecordType, ImportStatus from upload.models import SimpleVCFImportInfo from upload.models.models_enums import VCFImportInfoSeverity from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask diff --git a/annotation/templatetags/clinvar_tags.py b/annotation/templatetags/clinvar_tags.py index 3c38a0fa0..2df856086 100644 --- a/annotation/templatetags/clinvar_tags.py +++ b/annotation/templatetags/clinvar_tags.py @@ -5,11 +5,11 @@ from more_itertools import first from pydantic import ConfigDict -from annotation.models import ClinVar, AnnotationVersion, VariantAnnotation +from annotation.models import AnnotationVersion, ClinVar, VariantAnnotation from annotation.utils.clinvar_constants import CLINVAR_REVIEW_EXPERT_PANEL_STARS_VALUE from library.log_utils import report_exc_info from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import Allele, Variant, GenomeBuild +from snpdb.models import Allele, GenomeBuild, Variant register = Library() diff --git a/annotation/templatetags/gene_disease_tags.py b/annotation/templatetags/gene_disease_tags.py index 1c468159c..68c3d787e 100644 --- a/annotation/templatetags/gene_disease_tags.py +++ b/annotation/templatetags/gene_disease_tags.py @@ -2,7 +2,7 @@ from django.template import Library -from ontology.models import OntologyVersion, ONTOLOGY_RELATIONSHIP_NO_QUALITY_FILTER +from ontology.models import ONTOLOGY_RELATIONSHIP_NO_QUALITY_FILTER, OntologyVersion register = Library() diff --git a/annotation/tests/test_annotation_dispatch.py b/annotation/tests/test_annotation_dispatch.py index d4720182b..e1da828b5 100644 --- a/annotation/tests/test_annotation_dispatch.py +++ b/annotation/tests/test_annotation_dispatch.py @@ -17,17 +17,22 @@ from django.test.utils import override_settings from django.utils import timezone +from annotation.annotation_versions import merge_pending_range_locks from annotation.fake_annotation import get_fake_annotation_settings_dict, get_fake_vep_version -from annotation.models import AnnotationRangeLock, AnnotationRun, AnnotationVersion, VariantAnnotationVersion +from annotation.models import ( + AnnotationRangeLock, + AnnotationRun, + AnnotationVersion, + VariantAnnotationVersion, +) from annotation.models.models_enums import AnnotationStatus, VariantAnnotationPipelineType -from annotation.annotation_versions import merge_pending_range_locks from annotation.tasks import annotation_scheduler_task +from annotation.tasks.annotate_variants import annotate_variants from annotation.tasks.annotation_scheduler_task import ( _handle_range_lock, dispatch_annotation_runs, reclaim_stalled_annotation_runs, ) -from annotation.tasks.annotate_variants import annotate_variants from genes.models_enums import AnnotationConsortium from snpdb.models import GenomeBuild from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant diff --git a/annotation/tests/test_annotation_vcf.py b/annotation/tests/test_annotation_vcf.py index 879afece7..7d67e5fb5 100644 --- a/annotation/tests/test_annotation_vcf.py +++ b/annotation/tests/test_annotation_vcf.py @@ -4,16 +4,31 @@ from django.test import TestCase from django.test.utils import override_settings -from annotation.annotation_versions import get_or_create_variant_annotation_version_from_current_vep, \ - get_annotation_range_lock_and_unannotated_count +from annotation.annotation_versions import ( + get_annotation_range_lock_and_unannotated_count, + get_or_create_variant_annotation_version_from_current_vep, +) from annotation.fake_annotation import get_fake_annotation_settings_dict from annotation.models import VariantAnnotation -from annotation.models.damage_enums import PathogenicityImpact, ALoFTPrediction, AlphaMissensePrediction -from annotation.models.models import AnnotationRun, VariantAnnotationVersion, VariantTranscriptAnnotation +from annotation.models.damage_enums import ( + ALoFTPrediction, + AlphaMissensePrediction, + PathogenicityImpact, +) +from annotation.models.models import ( + AnnotationRun, + VariantAnnotationVersion, + VariantTranscriptAnnotation, +) from annotation.vcf_files.bulk_vep_vcf_annotation_inserter import BulkVEPVCFAnnotationInserter from annotation.vcf_files.import_vcf_annotations import import_vcf_annotations -from annotation.vep_annotation import vep_parse_version_line, get_vep_version_from_vcf, \ - vep_dict_to_variant_annotation_version_kwargs, VEPVersionMismatchError, VEPConfig +from annotation.vep_annotation import ( + VEPConfig, + VEPVersionMismatchError, + get_vep_version_from_vcf, + vep_dict_to_variant_annotation_version_kwargs, + vep_parse_version_line, +) from snpdb.models import Variant from snpdb.models.models_genome import GenomeBuild from snpdb.tests.utils.vcf_testing_utils import slowly_create_loci_and_variants_for_vcf diff --git a/annotation/tests/test_annotation_vcf_cnv.py b/annotation/tests/test_annotation_vcf_cnv.py index a8f71c0bf..c769bc594 100644 --- a/annotation/tests/test_annotation_vcf_cnv.py +++ b/annotation/tests/test_annotation_vcf_cnv.py @@ -10,8 +10,11 @@ from annotation.models.damage_enums import PathogenicityImpact from annotation.models.models import AnnotationRun, VariantAnnotationVersion from annotation.vcf_files.import_vcf_annotations import import_vcf_annotations -from annotation.vep_annotation import get_vep_version_from_vcf, vep_dict_to_variant_annotation_version_kwargs, \ - VEPConfig +from annotation.vep_annotation import ( + VEPConfig, + get_vep_version_from_vcf, + vep_dict_to_variant_annotation_version_kwargs, +) from library.genomics.vcf_enums import VariantClass from snpdb.models import Variant from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/tests/test_annotsv.py b/annotation/tests/test_annotsv.py index 2d7ba115f..4e6dd4c27 100644 --- a/annotation/tests/test_annotsv.py +++ b/annotation/tests/test_annotsv.py @@ -7,8 +7,8 @@ from annotation.annotation_versions import get_annotation_range_lock_and_unannotated_count from annotation.annotsv_annotation import ( - annotsv_check_command_line_version_match, AnnotSVVersionMismatchError, + annotsv_check_command_line_version_match, get_annotsv_command, run_annotsv, ) @@ -23,15 +23,14 @@ ) from annotation.vcf_files.import_vcf_annotations import import_vcf_annotations from annotation.vep_annotation import ( + VEPConfig, get_vep_version_from_vcf, vep_dict_to_variant_annotation_version_kwargs, - VEPConfig, ) from genes.models_enums import AnnotationConsortium from snpdb.models.models_genome import GenomeBuild from snpdb.tests.utils.vcf_testing_utils import slowly_create_loci_and_variants_for_vcf - TEST_DATA_DIR = os.path.join(settings.BASE_DIR, "annotation/tests/test_data") TEST_ANNOTSV_TSV = os.path.join(TEST_DATA_DIR, "annotsv", "test_grch37_sv.annotated.tsv") TEST_SV_VCF_GRCH37 = os.path.join(TEST_DATA_DIR, "test_columns_version4_grch37_sv.vep_annotated.vcf") diff --git a/annotation/tests/test_bugs.py b/annotation/tests/test_bugs.py index 99dfec0a2..e0befe1ab 100644 --- a/annotation/tests/test_bugs.py +++ b/annotation/tests/test_bugs.py @@ -2,15 +2,15 @@ Regression tests for confirmed bugs in the annotation app, plus sentinels for known fragile code paths that are likely to break when related code changes. """ +from types import SimpleNamespace + from django.test import TestCase from annotation.models.models import ClinVar, VariantAnnotation, VariantAnnotationVersion from annotation.models.models_enums import ClinVarReviewStatus from annotation.vcf_files.bulk_vep_vcf_annotation_inserter import BulkVEPVCFAnnotationInserter from classification.enums import AlleleOriginBucket -from library.genomics import overlap_fraction, Range -from types import SimpleNamespace - +from library.genomics import Range, overlap_fraction # --------------------------------------------------------------------------- # Bug: amino_acid_3_to_1 did not convert stop codon "Ter" → "*" diff --git a/annotation/tests/test_data_archive.py b/annotation/tests/test_data_archive.py index 0ee11c4eb..fc36ee3ad 100644 --- a/annotation/tests/test_data_archive.py +++ b/annotation/tests/test_data_archive.py @@ -15,7 +15,11 @@ get_variants_qs_for_annotation, ) from annotation.fake_annotation import get_fake_annotation_settings_dict, get_fake_vep_version -from annotation.models import AnnotationVersion, VariantAnnotationVersion, VariantTranscriptAnnotation +from annotation.models import ( + AnnotationVersion, + VariantAnnotationVersion, + VariantTranscriptAnnotation, +) from genes.models import Gene from genes.models_enums import AnnotationConsortium from snpdb.archive import DataArchivedError diff --git a/annotation/tests/test_data_fake_genes.py b/annotation/tests/test_data_fake_genes.py index 592cd31bf..316ffd7f5 100644 --- a/annotation/tests/test_data_fake_genes.py +++ b/annotation/tests/test_data_fake_genes.py @@ -1,5 +1,14 @@ -from genes.models import GeneSymbol, Transcript, Gene, GeneAnnotationImport, GeneVersion, TranscriptVersion, \ - GeneAnnotationRelease, ReleaseGeneSymbol, ReleaseGeneSymbolGene +from genes.models import ( + Gene, + GeneAnnotationImport, + GeneAnnotationRelease, + GeneSymbol, + GeneVersion, + ReleaseGeneSymbol, + ReleaseGeneSymbolGene, + Transcript, + TranscriptVersion, +) from genes.models_enums import AnnotationConsortium from snpdb.models import GenomeBuild diff --git a/annotation/tests/test_phenotype_matching.py b/annotation/tests/test_phenotype_matching.py index 513993f35..25901a896 100644 --- a/annotation/tests/test_phenotype_matching.py +++ b/annotation/tests/test_phenotype_matching.py @@ -5,7 +5,10 @@ from annotation.models.models_phenotype_match import PatientTextPhenotype from annotation.phenotype_matcher import PhenotypeMatcher from ontology.models import OntologyService -from ontology.tests.test_data_ontology import create_ontology_test_data, create_test_ontology_version +from ontology.tests.test_data_ontology import ( + create_ontology_test_data, + create_test_ontology_version, +) from patients.models import Patient diff --git a/annotation/tests/test_repeat_masker.py b/annotation/tests/test_repeat_masker.py index 49202bd35..13a42427a 100644 --- a/annotation/tests/test_repeat_masker.py +++ b/annotation/tests/test_repeat_masker.py @@ -1,8 +1,16 @@ from django.test import TestCase from annotation.models.repeat_masker import ( - RepeatMaskerSummary, classify_repeat, - SIMPLE_REPEAT, LOW_COMPLEXITY, SINE, LINE, LTR, DNA, RNA, OTHER, + DNA, + LINE, + LOW_COMPLEXITY, + LTR, + OTHER, + RNA, + SIMPLE_REPEAT, + SINE, + RepeatMaskerSummary, + classify_repeat, ) # Real value from issue #1580 (NM_182961.4(SYNE1):c.5270_23300del), truncated diff --git a/annotation/tests/test_urls.py b/annotation/tests/test_urls.py index 791e6ca00..1a6766161 100644 --- a/annotation/tests/test_urls.py +++ b/annotation/tests/test_urls.py @@ -2,9 +2,12 @@ from django.contrib.auth.models import User -from annotation.fake_annotation import get_fake_annotation_version, create_fake_clinvar_data, \ - create_fake_variant_annotation -from annotation.models import HumanProteinAtlasTissueSample, ClinVar, Citation, AnnotationRun +from annotation.fake_annotation import ( + create_fake_clinvar_data, + create_fake_variant_annotation, + get_fake_annotation_version, +) +from annotation.models import AnnotationRun, Citation, ClinVar, HumanProteinAtlasTissueSample from annotation.models.models_citations import CitationSource from library.django_utils.unittest_utils import URLTestCase from snpdb.models import Variant @@ -40,8 +43,8 @@ def setUpTestData(cls): cls.clinvar_id = clinvar.pk cls.variant_string = str(variant) - cls.pubmed_citations = "&".join((str(c) for c in Citation.objects.all().values_list("id", flat=True)[:2])) - cls.citations_ids_list = "/".join((str(c) for c in Citation.objects.all().values_list("id", flat=True)[:2])) + cls.pubmed_citations = "&".join(str(c) for c in Citation.objects.all().values_list("id", flat=True)[:2]) + cls.citations_ids_list = "/".join(str(c) for c in Citation.objects.all().values_list("id", flat=True)[:2]) cls.citations_ids_list_pubmed = pubmed_citation def testUrls(self): diff --git a/annotation/transcripts_annotation_selections.py b/annotation/transcripts_annotation_selections.py index 41f74150e..c6d7dbd47 100644 --- a/annotation/transcripts_annotation_selections.py +++ b/annotation/transcripts_annotation_selections.py @@ -6,11 +6,16 @@ from django.forms.models import model_to_dict from django.utils.timesince import timesince -from annotation.models import VEPSkippedReason, AnnotationStatus -from annotation.models.models import VariantAnnotation, AnnotationVersion, \ - InvalidAnnotationVersionError, VariantTranscriptAnnotation, AnnotationRun -from genes.hgvs import HGVSMatcher, HGVSException -from genes.models import TranscriptVersion, GnomADGeneConstraint, Transcript +from annotation.models import AnnotationStatus, VEPSkippedReason +from annotation.models.models import ( + AnnotationRun, + AnnotationVersion, + InvalidAnnotationVersionError, + VariantAnnotation, + VariantTranscriptAnnotation, +) +from genes.hgvs import HGVSException, HGVSMatcher +from genes.models import GnomADGeneConstraint, Transcript, TranscriptVersion from genes.models_enums import AnnotationConsortium from snpdb.models import Variant from snpdb.models.models_genome import GenomeBuild diff --git a/annotation/urls.py b/annotation/urls.py index 3e3265ebd..12cf3a0bf 100644 --- a/annotation/urls.py +++ b/annotation/urls.py @@ -1,6 +1,5 @@ from annotation import views, views_rest -from annotation.grids import AnnotationRunColumns, \ - VariantAnnotationVersionColumns +from annotation.grids import AnnotationRunColumns, VariantAnnotationVersionColumns from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/annotation/vcf_files/bulk_annotsv_tsv_inserter.py b/annotation/vcf_files/bulk_annotsv_tsv_inserter.py index 97f2d2eb8..89be1fc76 100644 --- a/annotation/vcf_files/bulk_annotsv_tsv_inserter.py +++ b/annotation/vcf_files/bulk_annotsv_tsv_inserter.py @@ -14,7 +14,6 @@ from annotation.models.models import AnnotationRun, VariantAnnotation - # AnnotSV preserves the input VCF INFO column when run with -SVinputInfo 1. # The dump VCF writes "variant_id=NNN" into INFO; we use that to join back. _VARIANT_ID_RE = re.compile(r"(?:^|;)variant_id=(\d+)") diff --git a/annotation/vcf_files/bulk_vep_vcf_annotation_inserter.py b/annotation/vcf_files/bulk_vep_vcf_annotation_inserter.py index 2b48f10dd..4cc8462bc 100644 --- a/annotation/vcf_files/bulk_vep_vcf_annotation_inserter.py +++ b/annotation/vcf_files/bulk_vep_vcf_annotation_inserter.py @@ -2,36 +2,53 @@ import operator import shutil import time -from collections import defaultdict, Counter +from collections import Counter, defaultdict +from collections.abc import Iterable from functools import cached_property -from typing import Optional, Iterable, TypeAlias +from typing import Optional, TypeAlias import intervaltree from django.conf import settings from annotation import vep_columns as vep_columns_registry -from annotation.models.damage_enums import SIFTPrediction, FATHMMPrediction, \ - MutationAssessorPrediction, MutationTasterPrediction, Polyphen2Prediction, \ - PathogenicityImpact, ALoFTPrediction, AlphaMissensePrediction, \ - MetaRNNPrediction, PrimateAIPrediction -from annotation.models.models import VariantAnnotation, \ - VariantTranscriptAnnotation, VariantAnnotationVersion, VariantGeneOverlap, AnnotationRun +from annotation.models.damage_enums import ( + ALoFTPrediction, + AlphaMissensePrediction, + FATHMMPrediction, + MetaRNNPrediction, + MutationAssessorPrediction, + MutationTasterPrediction, + PathogenicityImpact, + Polyphen2Prediction, + PrimateAIPrediction, + SIFTPrediction, +) +from annotation.models.models import ( + AnnotationRun, + VariantAnnotation, + VariantAnnotationVersion, + VariantGeneOverlap, + VariantTranscriptAnnotation, +) from annotation.models.models_enums import VariantAnnotationPipelineType, VEPCustom from annotation.refseq_ensembl_resolver import DBNSFPGeneResolver from annotation.vcf_files.vcf_types import VCFVariant from annotation.vep_annotation import VEPConfig from annotation.vep_columns import VEPColumnDef from genes.hgvs import HGVSMatcher -from genes.models import TranscriptVersion, GeneVersion +from genes.models import GeneVersion, TranscriptVersion from genes.models_enums import AnnotationConsortium from library.django_utils import get_model_fields -from library.django_utils.django_file_utils import get_import_processing_filename, get_import_processing_dir -from library.genomics import overlap_fraction, Range, parse_gnomad_coord +from library.django_utils.django_file_utils import ( + get_import_processing_dir, + get_import_processing_filename, +) +from library.genomics import Range, overlap_fraction, parse_gnomad_coord from library.genomics.vcf_enums import VariantClass from library.log_utils import log_traceback from library.utils import invert_dict, split_dict_multi_values from snpdb.models import GenomeBuild, VariantCoordinate -from upload.vcf.sql_copy_files import write_sql_copy_csv, sql_copy_csv +from upload.vcf.sql_copy_files import sql_copy_csv, write_sql_copy_csv VEP_SEPARATOR = '&' EMPTY_VALUES = {'', '.'} @@ -1069,7 +1086,7 @@ def process(self, raw_db_data: dict): update_fields[f] = VEP_SEPARATOR.join([r[f] for r in filtered_sv_records]) else: # Nothing left after filtering - need to blank out all our values - update_fields = {f: None for f in self.sv_fields} + update_fields = dict.fromkeys(self.sv_fields) raw_db_data.update(update_fields) diff --git a/annotation/vcf_files/import_clinvar_vcf.py b/annotation/vcf_files/import_clinvar_vcf.py index 5720c1b29..6295cb52c 100644 --- a/annotation/vcf_files/import_clinvar_vcf.py +++ b/annotation/vcf_files/import_clinvar_vcf.py @@ -12,7 +12,7 @@ from snpdb.models import VariantCoordinate from snpdb.variant_pk_lookup import VariantPKLookup from upload.models import UploadStep -from upload.vcf.sql_copy_files import write_sql_copy_csv, sql_copy_csv +from upload.vcf.sql_copy_files import sql_copy_csv, write_sql_copy_csv """ ##fileformat=VCFv4.1 diff --git a/annotation/vcf_files/import_vcf_annotations.py b/annotation/vcf_files/import_vcf_annotations.py index 165b63c93..72ab91e6f 100755 --- a/annotation/vcf_files/import_vcf_annotations.py +++ b/annotation/vcf_files/import_vcf_annotations.py @@ -6,7 +6,7 @@ from django.utils import timezone from annotation.annotation_version_querysets import get_variants_qs_for_annotation -from annotation.models import AnnotationRun, re, VEPSkippedReason +from annotation.models import AnnotationRun, VEPSkippedReason, re from annotation.models.models_enums import VariantAnnotationPipelineType from annotation.vcf_files.bulk_annotsv_tsv_inserter import import_annotsv_tsv from annotation.vcf_files.bulk_vep_vcf_annotation_inserter import BulkVEPVCFAnnotationInserter @@ -21,6 +21,7 @@ def import_vcf_annotations( vep_version_check: bool = True, delete_temp_files: bool = settings.IMPORT_PROCESSING_DELETE_TEMP_FILES_ON_SUCCESS): import cyvcf2 + from library.genomics.vcf_utils import cyvcf2_header_types annotation_run.upload_start = timezone.now() diff --git a/annotation/vep_annotation.py b/annotation/vep_annotation.py index 1ed772b81..96aa8b0d7 100644 --- a/annotation/vep_annotation.py +++ b/annotation/vep_annotation.py @@ -9,7 +9,7 @@ from annotation import vep_columns from annotation.fake_annotation import get_fake_vep_version -from annotation.models.models_enums import VEPPlugin, VEPCustom, VariantAnnotationPipelineType +from annotation.models.models_enums import VariantAnnotationPipelineType, VEPCustom, VEPPlugin from annotation.vep_columns import VEPColumnDef from annotation.vep_config import VEPConfig, parse_gnomad_version_from_filename from genes.models_enums import AnnotationConsortium diff --git a/annotation/views.py b/annotation/views.py index a25f04e89..cded7e4cc 100644 --- a/annotation/views.py +++ b/annotation/views.py @@ -1,43 +1,74 @@ import logging -from collections import defaultdict, Counter +from collections import Counter, defaultdict import cdot from django.conf import settings from django.contrib import messages -from django.http.response import HttpResponse, Http404, \ - JsonResponse -from django.shortcuts import get_object_or_404, render, redirect +from django.http.response import Http404, HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404, redirect, render from django.template.loader import render_to_string from django.views.decorators.cache import cache_page from django.views.decorators.http import require_POST from django.views.decorators.vary import vary_on_cookie from htmlmin.decorators import not_minified_response +from annotation import vep_columns from annotation.annotation_versions import vav_diff_vs_kwargs from annotation.clinvar_fetch_request import ClinVarFetchRequest from annotation.manual_variant_entry import create_manual_variants -from annotation.models import AnnotationVersion, AnnotationRun, VariantAnnotationVersion, \ - VariantAnnotationVersionDiff, Citation, InvalidAnnotationVersionError -from annotation import vep_columns -from annotation.pathogenicity_predictions import TOOLS -from annotation.models.models import CachedWebResource, HumanProteinAtlasAnnotationVersion, \ - HumanProteinAtlasAnnotation, DBNSFPGeneAnnotationVersion +from annotation.models import ( + AnnotationRun, + AnnotationVersion, + Citation, + InvalidAnnotationVersionError, + VariantAnnotationVersion, + VariantAnnotationVersionDiff, +) +from annotation.models.models import ( + CachedWebResource, + DBNSFPGeneAnnotationVersion, + HumanProteinAtlasAnnotation, + HumanProteinAtlasAnnotationVersion, +) from annotation.models.models_citations import CitationFetchRequest from annotation.models.models_enums import AnnotationStatus, VariantAnnotationPipelineType from annotation.models.models_version_diff import VersionDiff +from annotation.pathogenicity_predictions import TOOLS from annotation.tasks.annotate_variants import annotation_run_retry -from annotation.tasks.annotation_scheduler_task import annotation_scheduler, subdivide_annotation_range_lock -from annotation.vep_annotation import VEPConfig, get_vep_command, get_vep_variant_annotation_version_kwargs -from genes.models import GeneListCategory, GeneAnnotationImport, GeneVersion, TranscriptVersion, GeneSymbolAlias +from annotation.tasks.annotation_scheduler_task import ( + annotation_scheduler, + subdivide_annotation_range_lock, +) +from annotation.vep_annotation import ( + VEPConfig, + get_vep_command, + get_vep_variant_annotation_version_kwargs, +) +from genes.models import ( + GeneAnnotationImport, + GeneListCategory, + GeneSymbolAlias, + GeneVersion, + TranscriptVersion, +) from genes.models_enums import AnnotationConsortium, GeneSymbolAliasSource from library.constants import WEEK_SECS -from library.django_utils import require_superuser, get_field_counts +from library.django_utils import get_field_counts, require_superuser from library.utils import first -from ontology.models import OntologyTerm, OntologyService, OntologyImport, OntologyVersion -from snpdb.models import VariantGridColumn, SomalierConfig, GenomeBuild, VCF, UserSettings, ColumnAnnotationLevel +from ontology.models import OntologyImport, OntologyService, OntologyTerm, OntologyVersion +from snpdb.models import ( + VCF, + ColumnAnnotationLevel, + GenomeBuild, + SomalierConfig, + UserSettings, + VariantGridColumn, +) from variantgrid.celery import app -from variantgrid.deployment_validation.annotation_status_checks import \ - is_variant_annotation_version_populated, get_variant_annotation_progress +from variantgrid.deployment_validation.annotation_status_checks import ( + get_variant_annotation_progress, + is_variant_annotation_version_populated, +) from variantgrid.deployment_validation.somalier_check import verify_somalier_config diff --git a/annotation/views_rest.py b/annotation/views_rest.py index a1f984fee..d71043f8b 100644 --- a/annotation/views_rest.py +++ b/annotation/views_rest.py @@ -2,13 +2,16 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import never_cache from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.generics import RetrieveAPIView from rest_framework.response import Response from rest_framework.views import APIView -from annotation.models.models import VariantAnnotationVersion, ManualVariantEntryCollection -from annotation.serializers import VariantAnnotationSerializer, ManualVariantEntryCollectionSerializer +from annotation.models.models import ManualVariantEntryCollection, VariantAnnotationVersion +from annotation.serializers import ( + ManualVariantEntryCollectionSerializer, + VariantAnnotationSerializer, +) from snpdb.models import GenomeBuild, Variant diff --git a/classification/admin.py b/classification/admin.py index 2eb81abc9..75ab3fe64 100644 --- a/classification/admin.py +++ b/classification/admin.py @@ -1,5 +1,4 @@ # File for PyCharm DJango Structure plugin # noinspection PyUnresolvedReferences # pylint: disable=unused-import -from classification.admin import ClassificationImportRunAdmin, ClassificationAdmin, ClassificationImportAdmin, ClinicalContextAdmin, EvidenceKeyAdmin, ClassificationReportTemplateAdmin, DiscordanceReportAdmin, DiscordanceReportTriageAdmin, DiscordanceNotificationAdmin, UploadedClassificationsUnmappedAdmin, ResolvedVariantInfoAdmin, ImportedAlleleInfoAdmin, ImportedAlleleInfoValidationAdmin, ClinVarExportAdmin, ClinVarAlleleAdmin, ClinVarExportBatchAdmin, ConditionTextAdmin, ClassificationGroupingAdmin # pylint: enable=unused-import diff --git a/classification/admin/classification_admin.py b/classification/admin/classification_admin.py index f417d7ee6..3ffe6f409 100644 --- a/classification/admin/classification_admin.py +++ b/classification/admin/classification_admin.py @@ -1,41 +1,89 @@ import json from datetime import timedelta -from typing import Union, Optional +from typing import Optional, Union from django.contrib import admin, messages -from django.contrib.admin import RelatedFieldListFilter, BooleanFieldListFilter, DateFieldListFilter, TabularInline +from django.contrib.admin import ( + BooleanFieldListFilter, + DateFieldListFilter, + RelatedFieldListFilter, + TabularInline, +) from django.db.models import QuerySet from django.forms import Widget from django.utils import timezone from django.utils.safestring import SafeString from annotation.models.models import AnnotationVersion -from classification.autopopulate_evidence_keys.evidence_from_variant import get_evidence_fields_for_variant -from classification.classification_import import reattempt_variant_matching, variant_matching_dry_run +from classification.autopopulate_evidence_keys.evidence_from_variant import ( + get_evidence_fields_for_variant, +) +from classification.classification_import import ( + reattempt_variant_matching, + variant_matching_dry_run, +) from classification.enums import WithdrawReason -from classification.enums.classification_enums import EvidenceCategory, SpecialEKeys, SubmissionSource, ShareLevel -from classification.models import EvidenceKey, EvidenceKeyMap, DiscordanceReport, DiscordanceReportClassification, \ - ClinicalContext, ClassificationReportTemplate, ClassificationModification, \ - UploadedClassificationsUnmapped, ImportedAlleleInfo, ClassificationImport, ImportedAlleleInfoStatus, \ - classification_flag_types, DiscordanceReportTriage, ensure_discordance_report_triages_bulk, \ - DiscordanceReportTriageStatus, ClassificationGrouping, ClassificationGroupingEntry, \ - AlleleOriginGrouping, AlleleGrouping, ClassificationGroupingSearchTerm +from classification.enums.classification_enums import ( + EvidenceCategory, + ShareLevel, + SpecialEKeys, + SubmissionSource, +) +from classification.models import ( + AlleleGrouping, + AlleleOriginGrouping, + ClassificationGrouping, + ClassificationGroupingEntry, + ClassificationGroupingSearchTerm, + ClassificationImport, + ClassificationModification, + ClassificationReportTemplate, + ClinicalContext, + DiscordanceReport, + DiscordanceReportClassification, + DiscordanceReportTriage, + DiscordanceReportTriageStatus, + EvidenceKey, + EvidenceKeyMap, + ImportedAlleleInfo, + ImportedAlleleInfoStatus, + UploadedClassificationsUnmapped, + classification_flag_types, + ensure_discordance_report_triages_bulk, +) from classification.models.classification import Classification -from classification.models.classification_import_run import ClassificationImportRun, ClassificationImportRunStatus -from classification.models.classification_variant_info_models import ResolvedVariantInfo, ImportedAlleleInfoValidation -from classification.models.clinical_context_models import ClinicalContextRecalcTrigger, DiscordanceNotification +from classification.models.classification_import_run import ( + ClassificationImportRun, + ClassificationImportRunStatus, +) +from classification.models.classification_variant_info_models import ( + ImportedAlleleInfoValidation, + ResolvedVariantInfo, +) +from classification.models.clinical_context_models import ( + ClinicalContextRecalcTrigger, + DiscordanceNotification, +) from classification.models.clinical_context_utils import update_clinical_contexts from classification.models.discordance_lab_summaries import DiscordanceLabSummary from classification.models.discordance_models_utils import DiscordanceReportRowDataTriagesRowData from classification.signals import send_prepared_discordance_notifications -from classification.tasks.classification_import_map_and_insert_task import ClassificationImportMapInsertTask +from classification.tasks.classification_import_map_and_insert_task import ( + ClassificationImportMapInsertTask, +) from library.cache import timed_cache from library.django_utils import get_url_from_view_path from library.guardian_utils import admin_bot -from library.utils import ExportRow, export_column, ExportDataType, first -from ontology.models import OntologyTerm, AncestorCalculator -from snpdb.admin_utils import ModelAdminBasics, admin_action, admin_list_column, AllValuesChoicesFieldListFilter, \ - admin_model_action, get_admin_url +from library.utils import ExportDataType, ExportRow, export_column, first +from ontology.models import AncestorCalculator, OntologyTerm +from snpdb.admin_utils import ( + AllValuesChoicesFieldListFilter, + ModelAdminBasics, + admin_action, + admin_list_column, + admin_model_action, + get_admin_url, +) from snpdb.lab_picker import LabPickerData from snpdb.models import GenomeBuild, Lab @@ -740,7 +788,7 @@ def _labs_involved(self): @export_column("# classifications") def _classification_count(self): - return sum((summary.count for summary in self.summaries)) + return sum(summary.count for summary in self.summaries) @export_column("Status") def _status(self): @@ -881,11 +929,11 @@ def _cs_current(self): @export_column("Upgrade/Downgrade") def _upgrade_downgrade(self): - return "\n".join((DiscordanceReportAdminExport._up_down_for(summary) for summary in self.summaries)) + return "\n".join(DiscordanceReportAdminExport._up_down_for(summary) for summary in self.summaries) @export_column("Certainty") def _certainty(self): - return "\n".join((DiscordanceReportAdminExport._less_more_certain(summary) for summary in self.summaries)) + return "\n".join(DiscordanceReportAdminExport._less_more_certain(summary) for summary in self.summaries) @export_column("Triage", sub_data=DiscordanceReportRowDataTriagesRowData) def _triages(self): diff --git a/classification/admin/clinvar_export_admin.py b/classification/admin/clinvar_export_admin.py index 1a18b6b7c..3c1d48085 100644 --- a/classification/admin/clinvar_export_admin.py +++ b/classification/admin/clinvar_export_admin.py @@ -1,16 +1,27 @@ import re from datetime import timedelta -from django.contrib import messages, admin -from django.db.models import QuerySet, TextField, Q -from django.db.models.functions import Length, Cast +from django.contrib import admin, messages +from django.db.models import Q, QuerySet, TextField +from django.db.models.functions import Cast, Length from django.http import HttpRequest, JsonResponse from django.utils import timezone -from classification.models import ClinVarExport, ClinVarExportBatch, ClinVarAllele, ClinVarExportBatchStatus, \ - ClinVarExportRequest, ClinVarExportSubmission -from classification.models.clinvar_export_sync import clinvar_export_sync, ClinVarRequestException -from snpdb.admin_utils import AllValuesChoicesFieldListFilter, ModelAdminBasics, admin_action, admin_list_column +from classification.models import ( + ClinVarAllele, + ClinVarExport, + ClinVarExportBatch, + ClinVarExportBatchStatus, + ClinVarExportRequest, + ClinVarExportSubmission, +) +from classification.models.clinvar_export_sync import ClinVarRequestException, clinvar_export_sync +from snpdb.admin_utils import ( + AllValuesChoicesFieldListFilter, + ModelAdminBasics, + admin_action, + admin_list_column, +) class ClinVarExportSubmissionAdmin(admin.TabularInline): diff --git a/classification/apps.py b/classification/apps.py index 7d14402ce..798cbc4ec 100644 --- a/classification/apps.py +++ b/classification/apps.py @@ -9,5 +9,5 @@ class ClassificationConfig(AppConfig): def ready(self): # pylint: disable=import-outside-toplevel # imported to activate receivers - import classification.signals # pylint: disable=unused-import + pass # pylint: disable=unused-import # pylint: enable=import-outside-toplevel diff --git a/classification/autopopulate_evidence_keys/autopopulate_evidence_keys.py b/classification/autopopulate_evidence_keys/autopopulate_evidence_keys.py index e3719a729..84e9d69f9 100644 --- a/classification/autopopulate_evidence_keys/autopopulate_evidence_keys.py +++ b/classification/autopopulate_evidence_keys/autopopulate_evidence_keys.py @@ -10,15 +10,20 @@ from django.contrib.sites.models import Site from annotation.models.models import AnnotationVersion -from classification.autopopulate_evidence_keys.evidence_from_sample_and_patient import \ - get_evidence_fields_for_sample_and_patient -from classification.autopopulate_evidence_keys.evidence_from_variant import get_evidence_fields_for_variant, \ - AutopopulateData -from classification.enums import SubmissionSource, SpecialEKeys -from classification.models import EvidenceKey, Classification, ClassificationImport -from classification.tasks.classification_import_process_variants_task import liftover_classification_import +from classification.autopopulate_evidence_keys.evidence_from_sample_and_patient import ( + get_evidence_fields_for_sample_and_patient, +) +from classification.autopopulate_evidence_keys.evidence_from_variant import ( + AutopopulateData, + get_evidence_fields_for_variant, +) +from classification.enums import SpecialEKeys, SubmissionSource +from classification.models import Classification, ClassificationImport, EvidenceKey +from classification.tasks.classification_import_process_variants_task import ( + liftover_classification_import, +) from library.git import Git -from snpdb.models import GenomeBuild, ImportSource, Sample, Variant, Lab +from snpdb.models import GenomeBuild, ImportSource, Lab, Sample, Variant def create_classification_for_sample_and_variant_objects( diff --git a/classification/autopopulate_evidence_keys/evidence_from_sample_and_patient.py b/classification/autopopulate_evidence_keys/evidence_from_sample_and_patient.py index 2de97105e..fc7391658 100644 --- a/classification/autopopulate_evidence_keys/evidence_from_sample_and_patient.py +++ b/classification/autopopulate_evidence_keys/evidence_from_sample_and_patient.py @@ -3,8 +3,8 @@ from classification.autopopulate_evidence_keys.evidence_from_variant import AutopopulateData from classification.enums import SpecialEKeys -from patients.models_enums import Zygosity, Sex -from snpdb.models import Sample, Specimen, SampleGenotype +from patients.models_enums import Sex, Zygosity +from snpdb.models import Sample, SampleGenotype, Specimen from snpdb.models.models_variant import Variant diff --git a/classification/autopopulate_evidence_keys/evidence_from_variant.py b/classification/autopopulate_evidence_keys/evidence_from_variant.py index befbb5db9..d6dff54c1 100644 --- a/classification/autopopulate_evidence_keys/evidence_from_variant.py +++ b/classification/autopopulate_evidence_keys/evidence_from_variant.py @@ -1,5 +1,6 @@ import itertools -from typing import Iterable, Optional +from collections.abc import Iterable +from typing import Optional from django.conf import settings from django.contrib.sites.models import Site @@ -7,24 +8,29 @@ from analysis.models import VariantTag from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version from annotation.models import Citation -from annotation.models.damage_enums import FATHMMPrediction, \ - MutationTasterPrediction, Polyphen2Prediction, SIFTPrediction, \ - MutationAssessorPrediction, ALoFTPrediction, MetaRNNPrediction -from annotation.models.models import VariantAnnotation, AnnotationVersion, GenePubMedCount +from annotation.models.damage_enums import ( + ALoFTPrediction, + FATHMMPrediction, + MetaRNNPrediction, + MutationAssessorPrediction, + MutationTasterPrediction, + Polyphen2Prediction, + SIFTPrediction, +) +from annotation.models.models import AnnotationVersion, GenePubMedCount, VariantAnnotation from annotation.models.models_enums import ClinVarReviewStatus from annotation.transcripts_annotation_selections import VariantTranscriptSelections from annotation.vcf_files.bulk_vep_vcf_annotation_inserter import VEP_SEPARATOR -from classification.enums import SubmissionSource, \ - SpecialEKeys -from classification.models.evidence_key import EvidenceKeyMap, EvidenceKey +from classification.enums import SpecialEKeys, SubmissionSource +from classification.models.evidence_key import EvidenceKey, EvidenceKeyMap from genes.hgvs import HGVSMatcher -from genes.models import TranscriptVersion, GnomADGeneConstraint +from genes.models import GnomADGeneConstraint, TranscriptVersion from genes.models_enums import AnnotationConsortium from library.django_utils import get_choices_formatter from library.genomics.vcf_enums import VariantClass from library.log_utils import log_traceback from seqauto.models import get_20x_gene_coverage -from snpdb.clingen_allele import get_clingen_allele_for_variant, ClinGenAlleleAPIException +from snpdb.clingen_allele import ClinGenAlleleAPIException, get_clingen_allele_for_variant from snpdb.models import Variant, VariantZygosityCountCollection from snpdb.models.models_clingen_allele import ClinGenAllele from snpdb.models.models_enums import ColumnAnnotationLevel @@ -385,7 +391,7 @@ def get_evidence_fields_from_preferred_transcript( data[SpecialEKeys.C_HGVS] = c_hgvs.full_c_hgvs except Exception as e: value_obj = {} - data.message = f'Could not parse HGVS value {str(e)}' + data.message = f'Could not parse HGVS value {e!s}' data[SpecialEKeys.C_HGVS] = value_obj # If we classify against a transcript we don't have annotation for, try to grab p.HGVS from ClinGen diff --git a/classification/classification_changes.py b/classification/classification_changes.py index 1f3882b11..4dc4ac1bf 100644 --- a/classification/classification_changes.py +++ b/classification/classification_changes.py @@ -8,9 +8,13 @@ from django.utils.timezone import now from classification.enums import SubmissionSource -from classification.models import ClassificationModification, Classification, classification_flag_types -from flags.models import FlagComment, Flag, FlagResolution -from library.utils import IterableTransformer, IterableStitcher +from classification.models import ( + Classification, + ClassificationModification, + classification_flag_types, +) +from flags.models import Flag, FlagComment, FlagResolution +from library.utils import IterableStitcher, IterableTransformer from snpdb.models import Allele, Lab diff --git a/classification/classification_import.py b/classification/classification_import.py index 343df7bb4..932d5db85 100644 --- a/classification/classification_import.py +++ b/classification/classification_import.py @@ -8,17 +8,22 @@ from classification.models import ImportedAlleleInfo, ImportedAlleleInfoStatus from classification.models.classification import ClassificationImport from classification.models.classification_import_run import ClassificationImportRun -from classification.tasks.classification_import_process_variants_task import ClassificationImportProcessVariantsTask +from classification.tasks.classification_import_process_variants_task import ( + ClassificationImportProcessVariantsTask, +) from library.django_utils.django_file_utils import get_import_processing_dir -from library.genomics.vcf_utils import write_vcf_from_variant_coordinates, get_contigs_header_lines +from library.genomics.vcf_utils import get_contigs_header_lines, write_vcf_from_variant_coordinates from library.utils import full_class_name -from snpdb.models import Variant, ImportSource +from snpdb.models import ImportSource, Variant from snpdb.models.models_variant import VariantCoordinate -from snpdb.variant_pk_lookup import VariantPKLookup, VariantHash -from upload.models import UploadedFile, UploadPipeline, UploadStep, \ - UploadedClassificationImport -from upload.models.models_enums import UploadedFileTypes, UploadStepOrigin, \ - UploadStepTaskType, VCFPipelineStage +from snpdb.variant_pk_lookup import VariantHash, VariantPKLookup +from upload.models import UploadedClassificationImport, UploadedFile, UploadPipeline, UploadStep +from upload.models.models_enums import ( + UploadedFileTypes, + UploadStepOrigin, + UploadStepTaskType, + VCFPipelineStage, +) from upload.upload_processing import process_upload_pipeline # MAX_VCF_FIELD_LENGTH = 131072 @@ -93,7 +98,7 @@ def _classification_upload_pipeline( if unknown_variant_coordinates: working_dir = get_import_processing_dir(classification_import.pk, "classification_import") vcf_filename = os.path.join(working_dir, "classification_import.vcf") - used_chroms = set((vc.chrom for vc in unknown_variant_coordinates)) + used_chroms = set(vc.chrom for vc in unknown_variant_coordinates) header_lines = get_contigs_header_lines(classification_import.genome_build, use_accession=False, contig_allow_list=used_chroms) write_vcf_from_variant_coordinates(vcf_filename, unknown_variant_coordinates, header_lines=header_lines) diff --git a/classification/classification_stats.py b/classification/classification_stats.py index 76fc0632c..610898c85 100644 --- a/classification/classification_stats.py +++ b/classification/classification_stats.py @@ -1,13 +1,14 @@ import operator from collections import Counter, defaultdict -from typing import Optional, Any, Iterable +from collections.abc import Iterable +from typing import Any, Optional import numpy as np from django.conf import settings from django.contrib.auth.models import User from django.db.models import QuerySet -from classification.enums import ClinicalSignificance, AlleleOriginBucket +from classification.enums import AlleleOriginBucket, ClinicalSignificance from classification.enums.classification_enums import CriteriaEvaluation from classification.models import EvidenceKeyMap from classification.models.classification import Classification, ClassificationModification diff --git a/classification/criteria_strengths.py b/classification/criteria_strengths.py index c9ff9befe..b4687733c 100644 --- a/classification/criteria_strengths.py +++ b/classification/criteria_strengths.py @@ -1,7 +1,8 @@ from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property -from typing import Optional, Iterable, Union +from typing import Optional, Union from django.utils.safestring import SafeString diff --git a/classification/enums/classification_enums.py b/classification/enums/classification_enums.py index c25a9daa7..cb9669fb6 100644 --- a/classification/enums/classification_enums.py +++ b/classification/enums/classification_enums.py @@ -7,7 +7,7 @@ from django.contrib.auth.models import User from django.db.models import TextChoices -from library.guardian_utils import public_group, all_users_group +from library.guardian_utils import all_users_group, public_group from library.utils import ChoicesEnum CRITERIA_NOT_MET = 'NM' @@ -236,7 +236,9 @@ class EvidenceKeyValueType: ) -_ShareLevelData = typing.NamedTuple('ShareLevelData', [('index', int), ('label', str)]) +class _ShareLevelData(typing.NamedTuple): + index: int + label: str @total_ordering diff --git a/classification/evidence_key_rename.py b/classification/evidence_key_rename.py index 7cfca1224..de2c86f21 100644 --- a/classification/evidence_key_rename.py +++ b/classification/evidence_key_rename.py @@ -74,7 +74,7 @@ def sort_value(e_key_options: dict): else: return (e_key_options.get("label") or e_key_options.get("key")).lower() - self.options = list(sorted(self.options, key=sort_value)) + self.options = sorted(self.options, key=sort_value) self.e_key.options = self.options def save(self): diff --git a/classification/management/commands/blat_keys.py b/classification/management/commands/blat_keys.py index 048a0b281..47514134a 100644 --- a/classification/management/commands/blat_keys.py +++ b/classification/management/commands/blat_keys.py @@ -44,7 +44,7 @@ def get_dependency_lines(migration_script, replace): found_deps = False for line in lines: if found_deps: - if re.match("^\s+]$", line): + if re.match(r"^\s+]$", line): break dependency_lines.append(line) elif re.match(r"^\s+dependencies = \[", line): diff --git a/classification/management/commands/classification_cache_chgvs.py b/classification/management/commands/classification_cache_chgvs.py index 05f483668..691bccfc8 100644 --- a/classification/management/commands/classification_cache_chgvs.py +++ b/classification/management/commands/classification_cache_chgvs.py @@ -36,7 +36,7 @@ def handle(self, *args, **options): update_count += 1 if update_count % 100 == 0: print(f"Completed {update_count}") - print(f"Bulk Update of Cached c.hgvs - completed") - print(f"Biggest ref lengths are:") + print("Bulk Update of Cached c.hgvs - completed") + print("Biggest ref lengths are:") for conversion in conversions[::-1]: print(f"{conversion.ref_length} from vc.id {conversion.vc_id} {conversion.chgvs}") diff --git a/classification/management/commands/classification_db_refs.py b/classification/management/commands/classification_db_refs.py index a4abcffe7..217469694 100644 --- a/classification/management/commands/classification_db_refs.py +++ b/classification/management/commands/classification_db_refs.py @@ -2,7 +2,7 @@ from annotation.regexes import DbRegexes, db_ref_regexes from classification.enums import EvidenceKeyValueType -from classification.models import VCDataDict, EvidenceKeyMap, VCBlobKeys, Classification +from classification.models import Classification, EvidenceKeyMap, VCBlobKeys, VCDataDict from library.utils import JsonDiffs diff --git a/classification/management/commands/classification_ensure_alleles_and_liftover.py b/classification/management/commands/classification_ensure_alleles_and_liftover.py index 8d0f4a7a3..74704860b 100644 --- a/classification/management/commands/classification_ensure_alleles_and_liftover.py +++ b/classification/management/commands/classification_ensure_alleles_and_liftover.py @@ -1,15 +1,16 @@ from django.conf import settings from django.core.management.base import BaseCommand -from classification.autopopulate_evidence_keys.evidence_from_variant import \ - get_clingen_allele_and_evidence_value_for_variant +from classification.autopopulate_evidence_keys.evidence_from_variant import ( + get_clingen_allele_and_evidence_value_for_variant, +) from classification.enums import SpecialEKeys, SubmissionSource from classification.models.classification import Classification from library.git import Git from library.guardian_utils import admin_bot from snpdb.clingen_allele import populate_clingen_alleles_for_variants from snpdb.liftover import create_liftover_pipelines -from snpdb.models import Variant, Allele +from snpdb.models import Allele, Variant from snpdb.models.models_enums import ImportSource from snpdb.models.models_genome import GenomeBuild diff --git a/classification/management/commands/classification_export_w_annotations.py b/classification/management/commands/classification_export_w_annotations.py index f42328f16..ccc6a0c58 100644 --- a/classification/management/commands/classification_export_w_annotations.py +++ b/classification/management/commands/classification_export_w_annotations.py @@ -110,8 +110,8 @@ def handle(self, *args, **options): status = "transcript-annotated" except: all_transcript_versions = transcript_version.transcript.transcriptversion_set.filter(genome_build=genome_build) - lower_versions = list(sorted((v for v in all_transcript_versions if v.version < transcript_version.version), reverse=True)) - higher_versions = list(sorted((v for v in all_transcript_versions if v.version > transcript_version.version))) + lower_versions = sorted((v for v in all_transcript_versions if v.version < transcript_version.version), reverse=True) + higher_versions = sorted(v for v in all_transcript_versions if v.version > transcript_version.version) attempt_versions = higher_versions + lower_versions for attempt_version in attempt_versions: try: @@ -127,7 +127,7 @@ def handle(self, *args, **options): for key in transcript_annotation_keys: transcript_annotations_cells.append(annotation_data.get(key)) - except Exception as e: + except Exception: pass row = [ diff --git a/classification/management/commands/classification_groupings.py b/classification/management/commands/classification_groupings.py index f7c4068fa..2991c46e0 100644 --- a/classification/management/commands/classification_groupings.py +++ b/classification/management/commands/classification_groupings.py @@ -1,7 +1,14 @@ from django.core.management import BaseCommand -from classification.models import Classification, ClassificationModification, ClassificationSummaryCalculator -from classification.models.classification_grouping import ClassificationGrouping, AlleleOriginGrouping +from classification.models import ( + Classification, + ClassificationModification, + ClassificationSummaryCalculator, +) +from classification.models.classification_grouping import ( + AlleleOriginGrouping, + ClassificationGrouping, +) class Command(BaseCommand): diff --git a/classification/management/commands/classification_history_censor.py b/classification/management/commands/classification_history_censor.py index 6e47a86fe..5f5739f67 100644 --- a/classification/management/commands/classification_history_censor.py +++ b/classification/management/commands/classification_history_censor.py @@ -93,14 +93,14 @@ def handle(self, *args, **options): lab = Lab.objects.get(id=lab_id) else: lab = Lab.objects.get(group_name=lab_id) - print(f"Lab = {str(lab)}") + print(f"Lab = {lab!s}") if org_id: if org_id.isnumeric(): org = Organization.objects.get(id=org_id) else: org = Organization.objects.get(group_name=org_id) - print(f"Org = {str(org)}") + print(f"Org = {org!s}") print(f"Pattern = {pattern}, icase = {pattern_icase}") if not apply_remaining: diff --git a/classification/management/commands/classification_re_matching.py b/classification/management/commands/classification_re_matching.py index 00e888a1f..ed471daed 100644 --- a/classification/management/commands/classification_re_matching.py +++ b/classification/management/commands/classification_re_matching.py @@ -5,8 +5,13 @@ from django.core.management import BaseCommand from classification.classification_import import reattempt_variant_matching -from classification.models import Classification, ImportedAlleleInfo, DiscordanceReport, ClinVarExport, \ - ResolvedVariantInfo +from classification.models import ( + Classification, + ClinVarExport, + DiscordanceReport, + ImportedAlleleInfo, + ResolvedVariantInfo, +) from library.guardian_utils import admin_bot from snpdb.models import GenomeBuildPatchVersion @@ -118,7 +123,7 @@ def link_all_unlinked(self): rematch_count = 0 total_count = Classification.objects.count() - print(f"Will rematch classifications with no Allele Info, speed will vary based on how many classifications were imported with the same details") + print("Will rematch classifications with no Allele Info, speed will vary based on how many classifications were imported with the same details") print(f"Total classification count = {total_count}") bulk: list[Classification] = [] diff --git a/classification/management/commands/classification_set_allele_context.py b/classification/management/commands/classification_set_allele_context.py index 9a514ef3a..46e01c7d3 100644 --- a/classification/management/commands/classification_set_allele_context.py +++ b/classification/management/commands/classification_set_allele_context.py @@ -2,8 +2,8 @@ from django.core.management import BaseCommand from django.db.models import QuerySet -from classification.enums import SpecialEKeys, AlleleOriginBucket -from classification.models import Classification, EvidenceKeyMap, ClassificationModification +from classification.enums import AlleleOriginBucket, SpecialEKeys +from classification.models import Classification, ClassificationModification, EvidenceKeyMap from classification.models.clinical_context_utils import update_clinical_contexts from library.utils import iter_fixed_chunks diff --git a/classification/management/commands/classification_show_hgvs.py b/classification/management/commands/classification_show_hgvs.py index 19fed5ac3..55a6a2de6 100644 --- a/classification/management/commands/classification_show_hgvs.py +++ b/classification/management/commands/classification_show_hgvs.py @@ -24,6 +24,6 @@ def handle(self, *args, **options): converted_hgvs = matcher.variant_to_c_hgvs_parts(va.variant, transcript, throw_on_issue=True).full_c_hgvs if original_hgvs != converted_hgvs: cols = [vc.pk, vc.variant.pk, orig_allele.genome_build.name, original_hgvs, converted_hgvs] - print("\t".join((str(c) for c in cols))) + print("\t".join(str(c) for c in cols)) except Exception as e: print(e) diff --git a/classification/management/commands/clinvar_export_duplicate_check.py b/classification/management/commands/clinvar_export_duplicate_check.py index 2e960005c..ab1899f37 100644 --- a/classification/management/commands/clinvar_export_duplicate_check.py +++ b/classification/management/commands/clinvar_export_duplicate_check.py @@ -1,7 +1,7 @@ from django.core.management import BaseCommand from django.db.models import Count -from classification.models import ClinVarExport, ClinVarAllele +from classification.models import ClinVarAllele, ClinVarExport from ontology.models import AncestorCalculator, OntologySnake diff --git a/classification/management/commands/close_flags.py b/classification/management/commands/close_flags.py index f6be57ab1..69be32193 100644 --- a/classification/management/commands/close_flags.py +++ b/classification/management/commands/close_flags.py @@ -1,7 +1,7 @@ from django.core.management import BaseCommand from django.db.models import QuerySet -from flags.models import FlagComment, Flag, FlagResolution, FlagStatus, FlagType, FlagTypeResolution +from flags.models import Flag, FlagComment, FlagResolution, FlagStatus, FlagType, FlagTypeResolution class Command(BaseCommand): diff --git a/classification/management/commands/condition_matching_report.py b/classification/management/commands/condition_matching_report.py index 9c176ca9f..116aac4ad 100644 --- a/classification/management/commands/condition_matching_report.py +++ b/classification/management/commands/condition_matching_report.py @@ -6,8 +6,8 @@ from classification.models import ConditionTextMatch from genes.models import GeneSymbol -from library.utils import ExportRow, export_column, delimited_row -from ontology.models import OntologyTerm, OntologySnake, OntologyService +from library.utils import ExportRow, delimited_row, export_column +from ontology.models import OntologyService, OntologySnake, OntologyTerm @dataclass diff --git a/classification/management/commands/csv_classification_inserter.py b/classification/management/commands/csv_classification_inserter.py index 4b9b46d89..e03dc5524 100644 --- a/classification/management/commands/csv_classification_inserter.py +++ b/classification/management/commands/csv_classification_inserter.py @@ -70,7 +70,7 @@ def handle(self, *args, **options): count = count + 1 if count >= max_records: break - except Exception as e: + except Exception: log_traceback() raise finally: diff --git a/classification/management/commands/evidence_key_cleaner.py b/classification/management/commands/evidence_key_cleaner.py index 38653b08d..a6de389be 100644 --- a/classification/management/commands/evidence_key_cleaner.py +++ b/classification/management/commands/evidence_key_cleaner.py @@ -23,7 +23,7 @@ def handle(self, *args, **options): lab = Lab.objects.get(id=lab_id) else: lab = Lab.objects.get(group_name=lab_id) - print(f"Lab = {str(lab)}") + print(f"Lab = {lab!s}") qs = ClassificationModification.objects.filter(is_last_published=True, is_last_edited=True) if lab: @@ -39,7 +39,7 @@ def handle(self, *args, **options): print(f"{mod.classification.pk} has illegal evidence keys = {bonus_keys}") if apply: - patch = {k: None for k in bonus_keys} + patch = dict.fromkeys(bonus_keys) classification = mod.classification classification.patch_value(patch=patch, source=SubmissionSource.API, diff --git a/classification/management/commands/evidence_key_to_unit.py b/classification/management/commands/evidence_key_to_unit.py index ce11c2823..c91901bc0 100644 --- a/classification/management/commands/evidence_key_to_unit.py +++ b/classification/management/commands/evidence_key_to_unit.py @@ -1,7 +1,7 @@ from django.db.models import QuerySet from classification.enums import EvidenceKeyValueType -from classification.models import EvidenceKey, Classification, ClassificationModification +from classification.models import Classification, ClassificationModification, EvidenceKey class EvidenceKeyToUnit: diff --git a/classification/management/commands/fix_evidence_key_removal.py b/classification/management/commands/fix_evidence_key_removal.py index df8d36871..d57a49415 100644 --- a/classification/management/commands/fix_evidence_key_removal.py +++ b/classification/management/commands/fix_evidence_key_removal.py @@ -27,7 +27,7 @@ def remove_evidence_key(key: str): if fixed_records % 10 == 0: print(f"Fixed {fixed_records}") - print(f"Complete") + print("Complete") class Command(BaseCommand): diff --git a/classification/management/commands/fix_legacy_classification_alignment_gap_hgvs_matching.py b/classification/management/commands/fix_legacy_classification_alignment_gap_hgvs_matching.py index b71429cf5..d443647ee 100644 --- a/classification/management/commands/fix_legacy_classification_alignment_gap_hgvs_matching.py +++ b/classification/management/commands/fix_legacy_classification_alignment_gap_hgvs_matching.py @@ -82,10 +82,10 @@ def handle(self, *args, **options): # Revalidate user = admin_bot() - print(f"Revalidating...") + print("Revalidating...") for c in classification_qs: c.revalidate(user) - print(f"Rematching...") + print("Rematching...") matched_count = reattempt_variant_matching(user, classification_qs) print(f"{matched_count=}") diff --git a/classification/management/commands/fix_legacy_imported_allele_info_hgvs_error_provided_reference_length.py b/classification/management/commands/fix_legacy_imported_allele_info_hgvs_error_provided_reference_length.py index 0f68a79f3..98d6e9318 100644 --- a/classification/management/commands/fix_legacy_imported_allele_info_hgvs_error_provided_reference_length.py +++ b/classification/management/commands/fix_legacy_imported_allele_info_hgvs_error_provided_reference_length.py @@ -7,8 +7,10 @@ from pyhgvs import InvalidHGVSName from classification.classification_import import reattempt_variant_matching -from classification.models.classification_variant_info_models import ImportedAlleleInfo, \ - ImportedAlleleInfoStatus +from classification.models.classification_variant_info_models import ( + ImportedAlleleInfo, + ImportedAlleleInfoStatus, +) from genes.hgvs import HGVSMatcher from library.guardian_utils import admin_bot from snpdb.models import GenomeBuild diff --git a/classification/management/commands/fix_legacy_labs.py b/classification/management/commands/fix_legacy_labs.py index 44997db57..60e41acf1 100644 --- a/classification/management/commands/fix_legacy_labs.py +++ b/classification/management/commands/fix_legacy_labs.py @@ -1,4 +1,4 @@ -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.core.management import BaseCommand from classification.models import Classification diff --git a/classification/management/commands/fix_migrate_flags_to_imported_allele_info.py b/classification/management/commands/fix_migrate_flags_to_imported_allele_info.py index 712ab3067..03946bb44 100644 --- a/classification/management/commands/fix_migrate_flags_to_imported_allele_info.py +++ b/classification/management/commands/fix_migrate_flags_to_imported_allele_info.py @@ -5,7 +5,7 @@ from typing import Optional from classification.models import Classification, ImportedAlleleInfo -from flags.models import FlagComment, FlagType, FlagStatus +from flags.models import FlagComment, FlagStatus, FlagType from genes.hgvs import CHGVS from library.guardian_utils import admin_bot from snpdb.models import Allele, GenomeBuild @@ -168,7 +168,7 @@ def apply_to_imported_allele_infos(self, allele_id: Optional[int] = None): all_open_flags = set() all_open_flags.update(*(match.open_flags for match in matches)) - all_comments = list(sorted(all_comments, key=lambda c: c.created)) + all_comments = sorted(all_comments, key=lambda c: c.created) if not all_comments: continue diff --git a/classification/management/commands/fix_variant_coordinate_abbreviation.py b/classification/management/commands/fix_variant_coordinate_abbreviation.py index 43ea897c4..18ae65d6a 100644 --- a/classification/management/commands/fix_variant_coordinate_abbreviation.py +++ b/classification/management/commands/fix_variant_coordinate_abbreviation.py @@ -1,6 +1,6 @@ from django.core.management import BaseCommand -from classification.enums import SubmissionSource, SpecialEKeys +from classification.enums import SpecialEKeys, SubmissionSource from classification.models import Classification from library.guardian_utils import admin_bot diff --git a/classification/management/commands/fix_variant_matching.py b/classification/management/commands/fix_variant_matching.py index 2a253a366..f2f90e75b 100644 --- a/classification/management/commands/fix_variant_matching.py +++ b/classification/management/commands/fix_variant_matching.py @@ -3,7 +3,9 @@ from django.core.management import BaseCommand from django.db.models import Q -from classification.management.commands.fix_migrate_flags_to_imported_allele_info import FlagDatabase +from classification.management.commands.fix_migrate_flags_to_imported_allele_info import ( + FlagDatabase, +) from classification.models import Classification, ImportedAlleleInfo from classification.models.classification_variant_info_models import ResolvedVariantInfo from library.guardian_utils import admin_bot diff --git a/classification/management/commands/imported_allele_info_check.py b/classification/management/commands/imported_allele_info_check.py index df6946bae..cfb37fd07 100644 --- a/classification/management/commands/imported_allele_info_check.py +++ b/classification/management/commands/imported_allele_info_check.py @@ -1,7 +1,7 @@ from collections import Counter from dataclasses import dataclass from enum import Enum -from typing import Optional, Any +from typing import Any, Optional from django.core.management import BaseCommand from django.db.models import Max @@ -9,11 +9,11 @@ from django.utils.http import urlencode from classification.models import ImportedAlleleInfo -from genes.hgvs import HGVSMatcher, HGVSConverterType, VariantResolvingError -from genes.models import TranscriptVersion, TranscriptParts +from genes.hgvs import HGVSConverterType, HGVSMatcher, VariantResolvingError +from genes.models import TranscriptParts, TranscriptVersion from library.django_utils import get_url_from_view_path -from library.utils import ExportRow, export_column, delimited_row -from snpdb.models import Variant, GenomeBuild, VariantCoordinate +from library.utils import ExportRow, delimited_row, export_column +from snpdb.models import GenomeBuild, Variant, VariantCoordinate class Change(int, Enum): diff --git a/classification/management/commands/lab_record_id_changer.py b/classification/management/commands/lab_record_id_changer.py index 79aa646e7..90ab0999e 100644 --- a/classification/management/commands/lab_record_id_changer.py +++ b/classification/management/commands/lab_record_id_changer.py @@ -6,7 +6,7 @@ from classification.enums import SpecialEKeys from classification.models import Classification from genes.hgvs import CHGVS -from snpdb.models import Lab, GenomeBuild +from snpdb.models import GenomeBuild, Lab class Command(BaseCommand): diff --git a/classification/management/commands/labs_and_users.py b/classification/management/commands/labs_and_users.py index 45900c4e9..16d64b967 100644 --- a/classification/management/commands/labs_and_users.py +++ b/classification/management/commands/labs_and_users.py @@ -16,7 +16,7 @@ def csv_to_dict(file_path): header = None - with open(file_path, 'r') as csvfile: + with open(file_path) as csvfile: reader = csv.reader(csvfile, delimiter=',', quotechar='"') rows = [] for csv_row in reader: diff --git a/classification/management/commands/sync_condition_text_matches.py b/classification/management/commands/sync_condition_text_matches.py index 40babc558..f331655ef 100644 --- a/classification/management/commands/sync_condition_text_matches.py +++ b/classification/management/commands/sync_condition_text_matches.py @@ -2,8 +2,12 @@ from django.core.management.base import BaseCommand -from classification.models import ConditionText, sync_all_condition_resolutions_to_classifications, Classification, \ - ClinVarExport +from classification.models import ( + Classification, + ClinVarExport, + ConditionText, + sync_all_condition_resolutions_to_classifications, +) from classification.models.condition_text_matching import ConditionTextMatch diff --git a/classification/management/commands/unknown_evidence_key_cleaner.py b/classification/management/commands/unknown_evidence_key_cleaner.py index e63aacba7..81b67aa5a 100644 --- a/classification/management/commands/unknown_evidence_key_cleaner.py +++ b/classification/management/commands/unknown_evidence_key_cleaner.py @@ -22,7 +22,7 @@ def handle(self, *args, **options): lab = Lab.objects.get(id=lab_id) else: lab = Lab.objects.get(group_name=lab_id) - print(f"Lab = {str(lab)}") + print(f"Lab = {lab!s}") qs = ClassificationModification.objects.filter(is_last_published=True, is_last_edited=True) if lab: @@ -38,7 +38,7 @@ def handle(self, *args, **options): print(f"{mod.classification.pk} has illegal evidence keys = {bonus_keys}") if apply: - patch = {k: None for k in bonus_keys} + patch = dict.fromkeys(bonus_keys) classification = mod.classification classification.patch_value(patch=patch, source=SubmissionSource.API, diff --git a/classification/models.py b/classification/models.py index c0b4dc657..0a8c8a6b1 100644 --- a/classification/models.py +++ b/classification/models.py @@ -1,14 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from classification.models.uploaded_classifications_unmapped import UploadedClassificationsUnmapped -from classification.models.classification_import_run import ClassificationImportRun -from classification.models.classification_variant_info_models import HGVSConverterVersion, ResolvedVariantInfo, ImportedAlleleInfoValidation, ImportedAlleleInfo -from classification.models.evidence_key import EvidenceKey -from classification.models.classification import ClassificationImport, ClassificationImportAlleleSource, AllClassificationsAlleleSource, Classification, ClassificationModification -from classification.models.clinical_context_models import DiscordanceNotification, ClinicalContext -from classification.models.discordance_models import DiscordanceReport, DiscordanceReportClassification, DiscordanceReportTriage -from classification.models.variant_models import ClassificationAttachment -from classification.models.condition_text_matching import ConditionText, ConditionTextMatch -from classification.models.clinvar_export_models import ClinVarAllele, ClinVarExport, ClinVarExportBatch, ClinVarExportSubmission, ClinVarExportRequest -from classification.models.classification_report_models import ClassificationReportTemplate # pylint: enable=unused-import diff --git a/classification/models/abstract_utils.py b/classification/models/abstract_utils.py index 2b91980ad..8881f560b 100644 --- a/classification/models/abstract_utils.py +++ b/classification/models/abstract_utils.py @@ -1,7 +1,6 @@ from abc import abstractmethod from collections.abc import Hashable -from typing import TypeVar, Generic, Optional - +from typing import Generic, Optional, TypeVar CandidateType = TypeVar("CandidateType") EstablishedType = TypeVar("EstablishedType", bound=Hashable) diff --git a/classification/models/allele_overlaps.py b/classification/models/allele_overlaps.py index db9827cae..97d3118b3 100644 --- a/classification/models/allele_overlaps.py +++ b/classification/models/allele_overlaps.py @@ -1,20 +1,29 @@ import itertools from abc import ABC, abstractmethod from collections import defaultdict +from collections.abc import Iterator from dataclasses import dataclass from functools import cached_property, reduce -from typing import Optional, Iterator +from typing import Optional from django.db.models import Count, QuerySet, Subquery from classification.criteria_strengths import AcmgPointScore, CriteriaStrengths, CriteriaSummarizer -from classification.enums import SpecialEKeys, ShareLevel -from classification.models import ClassificationModification, ClinicalContext, ClassificationLabSummaryEntry, \ - ClassificationLabSummary, classification_flag_types, ClassificationFlagTypes, DiscordanceReport, Classification -from classification.models.clinical_context_models import DiscordanceStatus, DiscordanceLevel +from classification.enums import ShareLevel, SpecialEKeys +from classification.models import ( + Classification, + ClassificationFlagTypes, + ClassificationLabSummary, + ClassificationLabSummaryEntry, + ClassificationModification, + ClinicalContext, + DiscordanceReport, + classification_flag_types, +) +from classification.models.clinical_context_models import DiscordanceLevel, DiscordanceStatus from flags.models import Flag, FlagStatus from genes.hgvs import CHGVS -from library.utils import group_by_key, segment, first +from library.utils import first, group_by_key, segment from snpdb.lab_picker import LabPickerData from snpdb.models import Allele, Lab @@ -366,7 +375,7 @@ def overlaps(self) -> list[AlleleOverlap]: all_overlaps = [] cms_list: list[ClassificationModification] for allele, cms_list in group_by_key(cm_qs, lambda x: x.classification.allele_object): - if len(cms_list) >= 2 and any((cms.classification.lab_id in lab_ids for cms in cms_list)): + if len(cms_list) >= 2 and any(cms.classification.lab_id in lab_ids for cms in cms_list): overlap = AlleleOverlap(calculator_state=self.calculator_state, allele=allele) all_overlaps.append(overlap) cm: Classification diff --git a/classification/models/classification.py b/classification/models/classification.py index 16a5af3df..34629c9d1 100644 --- a/classification/models/classification.py +++ b/classification/models/classification.py @@ -3,11 +3,17 @@ import re import uuid from collections import Counter, namedtuple +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass -from datetime import datetime, timedelta, date +from datetime import date, datetime, timedelta from enum import Enum, StrEnum from functools import cached_property -from typing import Any, Dict, List, Union, Optional, Iterable, Callable, Mapping, TypedDict, Tuple, Set +from typing import ( + Any, + Optional, + TypedDict, + Union, +) import django.dispatch from datetimeutc.fields import DateTimeUTCField @@ -18,8 +24,8 @@ from django.db import models, transaction from django.db.models import TextField from django.db.models.deletion import CASCADE, PROTECT, SET_NULL -from django.db.models.expressions import RawSQL, OuterRef, Value, Subquery -from django.db.models.functions import LPad, Cast, Concat +from django.db.models.expressions import OuterRef, RawSQL, Subquery, Value +from django.db.models.functions import Cast, Concat, LPad from django.db.models.query import QuerySet from django.db.models.query_utils import Q from django.dispatch.dispatcher import receiver @@ -28,39 +34,76 @@ from django_extensions.db.models import TimeStampedModel from guardian.shortcuts import assign_perm, get_objects_for_user -from annotation.models.models import AnnotationVersion, VariantAnnotationVersion, VariantAnnotation -from annotation.regexes import db_ref_regexes, DbRegexes -from classification.enums import ClinicalSignificance, SubmissionSource, ShareLevel, SpecialEKeys, \ - CRITERIA_NOT_MET, ValidationCode, CriteriaEvaluation, WithdrawReason, AlleleOriginBucket +from annotation.models.models import AnnotationVersion, VariantAnnotation, VariantAnnotationVersion +from annotation.regexes import DbRegexes, db_ref_regexes +from classification.enums import ( + CRITERIA_NOT_MET, + AlleleOriginBucket, + ClinicalSignificance, + CriteriaEvaluation, + ShareLevel, + SpecialEKeys, + SubmissionSource, + ValidationCode, + WithdrawReason, +) from classification.models.classification_import_run import ClassificationImportRun from classification.models.classification_patcher import patch_fuzzy_age -from classification.models.classification_utils import \ - ValidationMerger, ClassificationJsonParams, PatchMeta, ClassificationPatchResponse -from classification.models.classification_variant_info_models import ImportedAlleleInfo, ImportedAlleleInfoStatus -from classification.models.evidence_key import EvidenceKeyValueType, \ - EvidenceKey, EvidenceKeyMap, VCDataDict, WipeMode, VCDataCell, EvidenceKeyOverrides +from classification.models.classification_utils import ( + ClassificationJsonParams, + ClassificationPatchResponse, + PatchMeta, + ValidationMerger, +) +from classification.models.classification_variant_info_models import ( + ImportedAlleleInfo, + ImportedAlleleInfoStatus, +) +from classification.models.evidence_key import ( + EvidenceKey, + EvidenceKeyMap, + EvidenceKeyOverrides, + EvidenceKeyValueType, + VCDataCell, + VCDataDict, + WipeMode, +) from classification.models.evidence_mixin import EvidenceMixin, VCPatch -from classification.models.evidence_mixin_summary_cache import ClassificationSummaryCalculator, \ - ClassificationSummaryCacheDict +from classification.models.evidence_mixin_summary_cache import ( + ClassificationSummaryCacheDict, + ClassificationSummaryCalculator, +) from classification.models.flag_types import classification_flag_types from flags.models import Flag, FlagPermissionLevel, FlagStatus -from flags.models.models import FlagsMixin, FlagCollection, FlagTypeContext, \ - flag_collection_extra_info_signal, FlagInfos -from genes.hgvs import HGVSMatcher, CHGVS +from flags.models.models import ( + FlagCollection, + FlagInfos, + FlagsMixin, + FlagTypeContext, + flag_collection_extra_info_signal, +) +from genes.hgvs import CHGVS, HGVSMatcher from genes.models import Gene, NoTranscript from library.cache import clear_cached_property from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin from library.guardian_utils import clear_permissions -from library.log_utils import report_exc_info, report_event -from library.preview_request import PreviewData, PreviewModelMixin, PreviewKeyValue -from library.utils import empty_to_none, nest_dict, cautious_attempt_html_to_text, \ - invalidate_cached_property, md5sum_str, get_timer, utc_from_timestamp -from ontology.models import OntologyTerm, OntologySnake, OntologyTermRelation +from library.log_utils import report_event, report_exc_info +from library.preview_request import PreviewData, PreviewKeyValue, PreviewModelMixin +from library.utils import ( + cautious_attempt_html_to_text, + empty_to_none, + get_timer, + invalidate_cached_property, + md5sum_str, + nest_dict, + utc_from_timestamp, +) +from ontology.models import OntologySnake, OntologyTerm, OntologyTermRelation from snpdb.clingen_allele import populate_clingen_alleles_for_variants from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import Variant, Lab, Sample +from snpdb.models import Lab, Sample, Variant from snpdb.models.models_genome import GenomeBuild -from snpdb.models.models_variant import AlleleSource, Allele, VariantAllele +from snpdb.models.models_variant import Allele, AlleleSource, VariantAllele from snpdb.user_settings_manager import UserSettingsManager ChgvsKey = namedtuple('CHGVS', ['short', 'column', 'build']) @@ -172,8 +215,8 @@ def get_extra_info(flag_infos: FlagInfos, user: User, **kwargs) -> None: # pyli :param user: The current user :param kwargs: Required by @receiver """ - from classification.models.discordance_models import DiscordanceReportClassification from classification.enums.discordance_enums import DiscordanceReportResolution + from classification.models.discordance_models import DiscordanceReportClassification vcs = Classification.objects.filter(flag_collection__in=flag_infos.ids).select_related('lab') drcs = DiscordanceReportClassification.objects.filter(classification_original__classification__in=vcs, @@ -232,8 +275,8 @@ class ConditionResolvedDict(TypedDict, total=False): @dataclass(frozen=True) class ConditionResolved: - terms: List[OntologyTerm] - plain_text_terms: List[str] = None + terms: list[OntologyTerm] + plain_text_terms: list[str] = None join: Optional['MultiCondition'] = None plain_text: Optional[str] = None # fallback, not populated in all contexts @@ -367,7 +410,7 @@ def format_term(term: OntologyTerm) -> str: join = self.join or MultiCondition.NOT_DECIDED text = f"{text}; {join.label}" - resolved_term_dicts: List[ConditionResolvedTermDict] = [ConditionResolved.term_to_dict(term) for term in + resolved_term_dicts: list[ConditionResolvedTermDict] = [ConditionResolved.term_to_dict(term) for term in self.terms] jsoned: ConditionResolvedDict = { "resolved_terms": resolved_term_dicts, @@ -609,7 +652,7 @@ def chgvs_grch38_compat(self) -> Optional[str]: return None @property - def metrics_logging_key(self) -> Tuple[str, Any]: + def metrics_logging_key(self) -> tuple[str, Any]: return "classification_id", self.pk @property @@ -674,7 +717,7 @@ def dashboard_total_unshared_classifications() -> int: return qs.exclude(share_level__in=ShareLevel.DISCORDANT_LEVEL_KEYS).count() @staticmethod - def dashboard_report_classifications_of_interest(since) -> List[ClassificationOutstandingIssues]: + def dashboard_report_classifications_of_interest(since) -> list[ClassificationOutstandingIssues]: min_age = django_timezone.now() - timedelta(minutes=2) # give records 2 minutes to matching properly before reporting time_range_q = Q(created__gte=since) & Q(created__lte=min_age) @@ -687,7 +730,7 @@ def dashboard_report_classifications_of_interest(since) -> List[ClassificationOu coi_qs = Classification.objects.filter(flag_q | (time_range_q & missing_chgvs_q)) coi_qs = coi_qs.order_by('-pk').select_related('lab', 'flag_collection') - summaries: List[ClassificationOutstandingIssues] = [] + summaries: list[ClassificationOutstandingIssues] = [] c: Classification for c in coi_qs: coi = ClassificationOutstandingIssues(c) @@ -840,7 +883,7 @@ def update_allele(self): def ensure_allele_info(self) -> Optional[ImportedAlleleInfo]: return self.ensure_allele_info_with_created()[0] - def ensure_allele_info_with_created(self, force_allele_info_update_check: bool = False) -> Tuple[ + def ensure_allele_info_with_created(self, force_allele_info_update_check: bool = False) -> tuple[ Optional[ImportedAlleleInfo], bool]: created = False if not self.allele_info or force_allele_info_update_check: @@ -945,7 +988,7 @@ def summarize_evidence_weights(evidence: dict, ekeys: Optional[EvidenceKeyMap] = weights = {} for ekey in ekeys.all_keys: if ekey.value_type == EvidenceKeyValueType.CRITERIA: - if ekey.key in evidence and evidence[ekey.key]: + if evidence.get(ekey.key): value = evidence[ekey.key] if isinstance(value, Mapping): value = value.get('value') @@ -971,12 +1014,12 @@ def summarize_evidence_weights(evidence: dict, ekeys: Optional[EvidenceKeyMap] = def create_with_response(user: User, lab: Lab, lab_record_id: Optional[str] = None, - data: Optional[Dict[str, Any]] = None, + data: Optional[dict[str, Any]] = None, save: bool = True, source: SubmissionSource = SubmissionSource.VARIANT_GRID, make_fields_immutable=False, populate_with_defaults=False, - **kwargs) -> Tuple['Classification', ClassificationPatchResponse]: + **kwargs) -> tuple['Classification', ClassificationPatchResponse]: """ :param user: The user creating this Classification :param lab: The lab the record will be created under @@ -1030,7 +1073,7 @@ def create_with_response(user: User, def create(user: User, lab: Lab, lab_record_id: Optional[str] = None, - data: Optional[Dict[str, Any]] = None, + data: Optional[dict[str, Any]] = None, save: bool = True, source: SubmissionSource = SubmissionSource.VARIANT_GRID, make_fields_immutable=False, @@ -1062,7 +1105,7 @@ def save(self, *args, **kwargs): if fix_permissions: self.fix_permissions() - def get_key_errors(self) -> Dict[str, Dict]: + def get_key_errors(self) -> dict[str, dict]: """ Returns a dict of key to validation error (first error in the case of multiple errors for one key). @@ -1093,7 +1136,7 @@ def _export_value(normal_value_obj: Any, key_data: EvidenceKey, export_key: str return None @staticmethod - def match_option(options, check_value) -> Optional[Dict[str, str]]: + def match_option(options, check_value) -> Optional[dict[str, str]]: for option in options: option_value = option.get('key') @@ -1128,12 +1171,12 @@ def match_option(options, check_value) -> Optional[Dict[str, str]]: return None @staticmethod - def process_option_values(cell: VCDataCell, values: List[Any]) -> Optional[List[str]]: + def process_option_values(cell: VCDataCell, values: list[Any]) -> Optional[list[str]]: e_key = cell.e_key options = e_key.virtual_options or [] # Do a case-insensitive check for each value against the key and any aliases # if there's a match to any of those, normalise back to the key (with the case of the key) - results: List[str] = [] + results: list[str] = [] # remove duplicates values = list(set(values)) for check_value in values: @@ -1276,7 +1319,7 @@ def process_entry(self, cell: VCDataCell, source: str): elif e_key.value_type in (EvidenceKeyValueType.MULTISELECT, EvidenceKeyValueType.SELECT, EvidenceKeyValueType.CRITERIA): - parts: List[Any] + parts: list[Any] if isinstance(value, str): if e_key.value_type == EvidenceKeyValueType.MULTISELECT and '|_' in str(value): parts = value.split('|_') @@ -1353,7 +1396,7 @@ def process_entry(self, cell: VCDataCell, source: str): elif e_key.value_type == EvidenceKeyValueType.DATE: try: Classification.to_date(value) - except ValueError as ve: + except ValueError: message = "Invalid date (expected yyyy-mm-dd)" cell.add_validation(code=ValidationCode.INVALID_DATE, severity='warning', message=message) @@ -1495,7 +1538,7 @@ def patch_history(self, patcher: Callable[[PatchMeta], None]): @transaction.atomic() def patch_value(self, - patch: Dict[str, Any], + patch: dict[str, Any], clear_all_fields: bool = False, user: Optional[User] = None, source: SubmissionSource = None, @@ -1505,7 +1548,7 @@ def patch_value(self, remove_api_immutable=False, initial_data=False, revalidate_all=False, - ignore_if_only_patching: Optional[Set[str]] = None, + ignore_if_only_patching: Optional[set[str]] = None, patch_known_keys_only: Optional[bool] = None) -> ClassificationPatchResponse: """ Creates a new ClassificationModification if the patch values are different to the current values @@ -1814,8 +1857,9 @@ def patch_value(self, if self.requires_auto_population: self.requires_auto_population = False - from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import \ - classification_auto_populate_fields + from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import ( + classification_auto_populate_fields, + ) genome_build = self.get_genome_build() patch_response += classification_auto_populate_fields(self, genome_build, save=save) @@ -1851,7 +1895,7 @@ def last_published_sync_records(self): return records @staticmethod - def validate_evidence(evidence: dict) -> List[Dict]: + def validate_evidence(evidence: dict) -> list[dict]: messages = [] for key, blob in evidence.items(): @@ -1891,7 +1935,7 @@ def has_errors(self) -> bool: return True return False - def validate(self) -> List[Dict]: + def validate(self) -> list[dict]: return Classification.validate_evidence(self.evidence) @staticmethod @@ -1983,7 +2027,7 @@ def lowest_share_level(self, user) -> ShareLevel: return ShareLevel.ALL_USERS return ShareLevel.PUBLIC - def get_visible_evidence(self, evidence, lowest_share_level: ShareLevel) -> Dict[str, Dict]: + def get_visible_evidence(self, evidence, lowest_share_level: ShareLevel) -> dict[str, dict]: """ Driven by EvidenceKey.max_share_level """ if lowest_share_level.index == 0: # No restrictions @@ -1999,7 +2043,7 @@ def get_visible_evidence(self, evidence, lowest_share_level: ShareLevel) -> Dict visible_evidence[k] = {'value': "(hidden)", 'hidden': True} return visible_evidence - def get_allele_info_dict(self) -> Optional[Dict[str, Any]]: + def get_allele_info_dict(self) -> Optional[dict[str, Any]]: allele_info_dict = {} if allele_info := self.allele_info: resolved_dict = { @@ -2135,8 +2179,8 @@ def get_variant_annotation(self, variant_annotation_version: VariantAnnotationVe if variant := self.get_variant_for_build(variant_annotation_version.genome_build): return variant.variantannotation_set.filter(version=variant_annotation_version).first() - def c_hgvs_all(self) -> List[CHGVS]: - all_chgvs: List[CHGVS] = [] + def c_hgvs_all(self) -> list[CHGVS]: + all_chgvs: list[CHGVS] = [] for genome_build in GenomeBuild.builds_with_annotation_cached(): if text := self.get_c_hgvs(genome_build): chgvs = CHGVS(full_c_hgvs=text) @@ -2202,7 +2246,7 @@ def get_c_hgvs(self, genome_build: GenomeBuild, use_compat: bool = False) -> Opt return self._generate_c_hgvs(genome_build) def __str__(self) -> str: - parts = [f"({str(self.id)})"] + parts = [f"({self.id!s})"] genome_build = GenomeBuildManager.get_current_genome_build() cached_c_hgvs = self.get_c_hgvs(genome_build=genome_build) if not cached_c_hgvs: @@ -2532,7 +2576,7 @@ def index(self): created__lt=self.created).count() @property - def evidence(self) -> Dict[str, Dict]: + def evidence(self) -> dict[str, dict]: if self.cached_evidence is None: if self.published_evidence is not None: self.cached_evidence = self.published_evidence @@ -2555,7 +2599,7 @@ def evidence(self) -> Dict[str, Dict]: self.cached_evidence = data return self.cached_evidence - def get_visible_evidence(self, user: User) -> Dict[str, Dict]: + def get_visible_evidence(self, user: User) -> dict[str, dict]: """ Driven by EvidenceKey.max_share_level """ lowest_share_level = self.classification.lowest_share_level(user) return self.classification.get_visible_evidence(self.evidence, lowest_share_level) @@ -2666,7 +2710,7 @@ def consensus_patch(self) -> VCPatch: keys = EvidenceKeyMap.cached() evidence = self.modification.published_evidence - consensus: Dict[str, Any] = {} + consensus: dict[str, Any] = {} # default allele origin - don't use copy consensus because that would copy "likely somatic" etc if allele_origin_bucket := self.modification.classification.allele_origin_bucket: @@ -2689,7 +2733,7 @@ def consensus_patch(self) -> VCPatch: continue consensus[key + '.' + part] = part_value - patch: Dict[str, Dict[str, Any]] = nest_dict(consensus) + patch: dict[str, dict[str, Any]] = nest_dict(consensus) return patch diff --git a/classification/models/classification_grouping.py b/classification/models/classification_grouping.py index 2a61e4aab..96264c7cf 100644 --- a/classification/models/classification_grouping.py +++ b/classification/models/classification_grouping.py @@ -1,24 +1,32 @@ import operator from dataclasses import dataclass, field from functools import cached_property, reduce -from typing import Optional, Set, Self, Tuple +from typing import Optional, Self import django from django.contrib.auth.models import User from django.contrib.postgres.fields import ArrayField from django.core.exceptions import PermissionDenied from django.db import models, transaction -from django.db.models import CASCADE, TextChoices, SET_NULL, IntegerChoices, Q, QuerySet +from django.db.models import CASCADE, SET_NULL, IntegerChoices, Q, QuerySet, TextChoices from django.urls import reverse from django_extensions.db.models import TimeStampedModel from frozendict import frozendict from more_itertools import last from classification.enums import AlleleOriginBucket, ShareLevel, SpecialEKeys -from classification.models import Classification, ImportedAlleleInfo, EvidenceKeyMap, ClassificationModification, \ - ConditionResolved -from classification.models.evidence_mixin_summary_cache import ClassificationSummaryCacheDict, \ - ClassificationSummaryCacheDictPathogenicity, ClassificationSummaryCacheDictSomatic +from classification.models import ( + Classification, + ClassificationModification, + ConditionResolved, + EvidenceKeyMap, + ImportedAlleleInfo, +) +from classification.models.evidence_mixin_summary_cache import ( + ClassificationSummaryCacheDict, + ClassificationSummaryCacheDictPathogenicity, + ClassificationSummaryCacheDictSomatic, +) from genes.models import GeneSymbol from library.utils import strip_json from ontology.models import OntologyTerm @@ -279,7 +287,7 @@ def get_absolute_url(self): return reverse('classification_grouping_detail', kwargs={"classification_grouping_id": self.pk}) @staticmethod - def _desired_grouping_for_classification(classification: Classification) -> Tuple[Optional['ClassificationGrouping'], bool]: + def _desired_grouping_for_classification(classification: Classification) -> tuple[Optional['ClassificationGrouping'], bool]: # withdrawn classifications are removed from groupings if classification.withdrawn: return None, False @@ -350,7 +358,7 @@ def classification_modifications(self) -> list[ClassificationModification]: all_classifications = self.classificationgroupingentry_set.values_list("classification", flat=True) all_modifications = ClassificationModification.objects.filter(classification_id__in=all_classifications, is_last_published=True) all_modifications = all_modifications.select_related("classification") - return list(sorted(all_modifications, key=lambda mod: mod.curated_date_check)) + return sorted(all_modifications, key=lambda mod: mod.curated_date_check) @cached_property def allele(self) -> Allele: @@ -379,8 +387,8 @@ def update(self): self.latest_allele_info = best_classification.classification.allele_info # TODO check for dirty values - all_terms: Set[OntologyTerm] = set() - all_free_text_conditions: Set[str] = set() + all_terms: set[OntologyTerm] = set() + all_free_text_conditions: set[str] = set() # UPDATE CLASSIFICATION / CLINICAL SIGNIFICANCE # TODO - CLINICAL SIGNIFICANCE @@ -424,12 +432,12 @@ def update(self): if None in all_zygosities: all_zygosities.remove(None) - all_zygosities = list(sorted(evidence_map[SpecialEKeys.ZYGOSITY].sort_values(all_zygosities))) + all_zygosities = sorted(evidence_map[SpecialEKeys.ZYGOSITY].sort_values(all_zygosities)) self.zygosity_values = all_zygosities self.conditions = strip_json(ConditionResolved( - terms=list(sorted(all_terms)), - plain_text_terms=list(sorted(all_free_text_conditions)) + terms=sorted(all_terms), + plain_text_terms=sorted(all_free_text_conditions) ).to_json(include_join=False)) self.classification_count = len(all_modifications) diff --git a/classification/models/classification_groups.py b/classification/models/classification_groups.py index b1db26c37..e37c6591c 100644 --- a/classification/models/classification_groups.py +++ b/classification/models/classification_groups.py @@ -1,16 +1,23 @@ +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property from itertools import groupby -from typing import Optional, Iterable, TypeVar, Generic +from typing import Generic, Optional, TypeVar import deprecation from django.contrib.auth.models import User from more_itertools import first from classification.criteria_strengths import CriteriaStrength -from classification.enums import SpecialEKeys, CriteriaEvaluation, ShareLevel -from classification.models import ClassificationModification, EvidenceKeyMap, CuratedDate, ConditionResolved, \ - classification_flag_types, ImportedAlleleInfo +from classification.enums import CriteriaEvaluation, ShareLevel, SpecialEKeys +from classification.models import ( + ClassificationModification, + ConditionResolved, + CuratedDate, + EvidenceKeyMap, + ImportedAlleleInfo, + classification_flag_types, +) from classification.models.flag_types import ClassificationFlagTypes from flags.models import Flag, FlagStatus from genes.hgvs import CHGVS, PHGVS @@ -231,7 +238,7 @@ def allele(self) -> Optional[Allele]: @property def allele_infos(self) -> list[ImportedAlleleInfo]: - return list(sorted({mod.classification.allele_info for mod in self.modifications if mod.classification.allele_info})) + return sorted({mod.classification.allele_info for mod in self.modifications if mod.classification.allele_info}) def diff_ids(self) -> str: return ",".join([str(cm.classification_id) for cm in self.modifications]) diff --git a/classification/models/classification_inserter.py b/classification/models/classification_inserter.py index a7dbecced..501a089ba 100644 --- a/classification/models/classification_inserter.py +++ b/classification/models/classification_inserter.py @@ -1,15 +1,23 @@ +from collections.abc import Mapping from functools import cached_property -from typing import Optional, Mapping +from typing import Optional from django.contrib.auth.models import User from django.db import transaction from django.utils.timezone import now # from classification.enums import ForceUpdate -from classification.enums import ShareLevel, SubmissionSource, SpecialEKeys -from classification.models import ClassificationProcessError, ClassificationRef, \ - EvidenceMixin, classification_flag_types, ClassificationJsonParams, ClassificationModification, \ - ClassificationPatchResponse, ClassificationImportRun +from classification.enums import ShareLevel, SpecialEKeys, SubmissionSource +from classification.models import ( + ClassificationImportRun, + ClassificationJsonParams, + ClassificationModification, + ClassificationPatchResponse, + ClassificationProcessError, + ClassificationRef, + EvidenceMixin, + classification_flag_types, +) from classification.models.classification_utils import ClassificationPatchStatus from classification.models.variant_resolver import VariantResolver from eventlog.models import create_event diff --git a/classification/models/classification_json.py b/classification/models/classification_json.py index 6357040fc..1d55e8974 100644 --- a/classification/models/classification_json.py +++ b/classification/models/classification_json.py @@ -2,11 +2,18 @@ from django.conf import settings -from classification.enums import ShareLevel, EvidenceKeyValueType, SpecialEKeys -from classification.models import Classification, ClassificationJsonParams, ClassificationModification, EvidenceKeyMap, \ - ImportedAlleleInfo -from classification.models.classification_json_definitions import ClassificationJsonAlleleDict, \ - ClassificationJsonAlleleRevolvedDict +from classification.enums import EvidenceKeyValueType, ShareLevel, SpecialEKeys +from classification.models import ( + Classification, + ClassificationJsonParams, + ClassificationModification, + EvidenceKeyMap, + ImportedAlleleInfo, +) +from classification.models.classification_json_definitions import ( + ClassificationJsonAlleleDict, + ClassificationJsonAlleleRevolvedDict, +) from genes.hgvs import CHGVS from library.django_utils import get_url_from_view_path diff --git a/classification/models/classification_json_definitions.py b/classification/models/classification_json_definitions.py index a3b47d535..297e67881 100644 --- a/classification/models/classification_json_definitions.py +++ b/classification/models/classification_json_definitions.py @@ -1,4 +1,4 @@ -from typing import Optional, TypedDict, Any +from typing import Any, Optional, TypedDict class ClassificationJsonLabDict(TypedDict): diff --git a/classification/models/classification_lab_summaries.py b/classification/models/classification_lab_summaries.py index ce56711b9..e8d9bc2ae 100644 --- a/classification/models/classification_lab_summaries.py +++ b/classification/models/classification_lab_summaries.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Optional -from classification.enums import SpecialEKeys, AlleleOriginBucket +from classification.enums import AlleleOriginBucket, SpecialEKeys from snpdb.models import Lab diff --git a/classification/models/classification_ref.py b/classification/models/classification_ref.py index 4ee9a176b..ef024be38 100644 --- a/classification/models/classification_ref.py +++ b/classification/models/classification_ref.py @@ -1,7 +1,7 @@ import re from datetime import datetime from functools import cached_property -from typing import Optional, Tuple +from typing import Optional from django.contrib.auth.models import User from django.core.exceptions import PermissionDenied @@ -10,8 +10,11 @@ from classification.enums import SubmissionSource from classification.models import ClassificationJsonParams, ClassificationPatchResponse -from classification.models.classification import Classification, \ - ClassificationProcessError, ClassificationModification +from classification.models.classification import ( + Classification, + ClassificationModification, + ClassificationProcessError, +) from library.utils import empty_to_none, utc_from_timestamp from snpdb.models import Lab @@ -120,7 +123,7 @@ def create_with_response(self, source: SubmissionSource, data: dict = None, save: bool = True, - make_fields_immutable=False) -> Tuple[Classification, ClassificationPatchResponse]: + make_fields_immutable=False) -> tuple[Classification, ClassificationPatchResponse]: if not self.lab: raise ValueError('Cannot create record without a lab') if self.exists(): diff --git a/classification/models/classification_utils.py b/classification/models/classification_utils.py index ce18f5d11..3f803647e 100644 --- a/classification/models/classification_utils.py +++ b/classification/models/classification_utils.py @@ -1,10 +1,11 @@ # used for validating multiple keys when one changes import operator from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from enum import Enum -from functools import reduce, cached_property -from typing import List, Set, Optional, Union, Any, Dict, Iterable +from functools import cached_property, reduce +from typing import Any, Optional, Union from django.contrib.auth.models import User from django.db.models import Q @@ -46,9 +47,9 @@ class ClassificationPatchStatus(str, Enum): class ClassificationPatchResponse(VarsDict): def __init__(self): - self.warnings: List[PatchMessage] = [] - self.modified_keys: Set[str] = set() - self.classification_json: Optional[Dict] = None + self.warnings: list[PatchMessage] = [] + self.modified_keys: set[str] = set() + self.classification_json: Optional[dict] = None self.internal_error: Optional[Any] = None self.withdrawn = None self.deleted = None @@ -202,7 +203,7 @@ def __init__(self, strip_complicated=False, strip_notes_and_explains=False, api_version=1, - hardcode_extra_data: Dict = None, + hardcode_extra_data: dict = None, fix_data_types=False, remove_acmg_namespace: Optional[bool] = None, inject_source_url: bool = False, @@ -298,7 +299,7 @@ def is_modified(self, key: str) -> bool: """ return self.revalidate_all or key in self.modified_keys - def intersection_modified(self, key_set: Set[str]) -> Set[str]: + def intersection_modified(self, key_set: set[str]) -> set[str]: """ Performs is_modified but over a set :param key_set: The set of str evidence keys we're asking about diff --git a/classification/models/classification_variant_fields_validation.py b/classification/models/classification_variant_fields_validation.py index 7a204888f..58da3c951 100644 --- a/classification/models/classification_variant_fields_validation.py +++ b/classification/models/classification_variant_fields_validation.py @@ -5,10 +5,9 @@ from django.conf import settings from django.dispatch.dispatcher import receiver -from classification.enums import ValidationCode, SpecialEKeys, AlleleOriginBucket +from classification.enums import AlleleOriginBucket, SpecialEKeys, ValidationCode from classification.models import EvidenceKeyMap, PatchMeta -from classification.models.classification import Classification, \ - classification_validation_signal +from classification.models.classification import Classification, classification_validation_signal from classification.models.classification_utils import ValidationMerger from genes.hgvs import HGVSMatcher from genes.models import NoTranscript diff --git a/classification/models/classification_variant_info_models.py b/classification/models/classification_variant_info_models.py index 748ccf44f..644fb790d 100644 --- a/classification/models/classification_variant_info_models.py +++ b/classification/models/classification_variant_info_models.py @@ -1,26 +1,36 @@ import logging from dataclasses import dataclass from datetime import timedelta -from typing import Optional, Any, TypedDict, Literal +from typing import Any, Literal, Optional, TypedDict import django.dispatch from django.conf import settings from django.contrib.auth.models import User from django.db import models -from django.db.models import TextField, ForeignKey, CASCADE, SET_NULL, OneToOneField, TextChoices, \ - CharField, JSONField, BooleanField, PROTECT +from django.db.models import ( + CASCADE, + PROTECT, + SET_NULL, + BooleanField, + CharField, + ForeignKey, + JSONField, + OneToOneField, + TextChoices, + TextField, +) from django.urls import reverse from django.utils.timezone import now from model_utils.models import TimeStampedModel -from genes.hgvs import HGVSMatcher, CHGVS, CHGVSDiff, HGVSConverterType, chgvs_diff_description -from genes.models import TranscriptVersion, GeneSymbol, Transcript, NoTranscript +from genes.hgvs import CHGVS, CHGVSDiff, HGVSConverterType, HGVSMatcher, chgvs_diff_description +from genes.models import GeneSymbol, NoTranscript, Transcript, TranscriptVersion from library.cache import timed_cache from library.django_utils.django_object_managers import ObjectManagerCachingRequest from library.log_utils import report_exc_info -from library.utils import pretty_label, IconWithTooltip, md5sum_str +from library.utils import IconWithTooltip, md5sum_str, pretty_label from library.utils.django_utils import get_cached_project_git_hash -from snpdb.models import GenomeBuild, Variant, Allele, GenomeBuildPatchVersion, VariantCoordinate +from snpdb.models import Allele, GenomeBuild, GenomeBuildPatchVersion, Variant, VariantCoordinate """ Now we have @@ -659,7 +669,7 @@ def all_chgvs(allele: Allele) -> list[CHGVS]: for rb in iai.resolved_builds: if c_hgvs := rb.c_hgvs_obj: all_chgvs.add(c_hgvs) - return list(sorted(all_chgvs, key=lambda x: (x.genome_build, x.sort_str))) + return sorted(all_chgvs, key=lambda x: (x.genome_build, x.sort_str)) @property def get_transcript(self) -> str: @@ -677,15 +687,15 @@ def gene_symbols(self) -> list[GeneSymbol]: if imported_gene_symbol_str := c_hgvs_obj.gene_symbol: if symbol := GeneSymbol.cast(imported_gene_symbol_str): gene_symbol_set.add(symbol) - return list(sorted(gene_symbol_set)) + return sorted(gene_symbol_set) @property def transcript_versions(self) -> list[TranscriptVersion]: - return list(sorted({build.transcript_version for build in self.resolved_builds if build.transcript_version})) + return sorted({build.transcript_version for build in self.resolved_builds if build.transcript_version}) @property def transcripts(self) -> list[Transcript]: - return list(sorted({build.transcript_version.transcript for build in self.resolved_builds if build.transcript_version})) + return sorted({build.transcript_version.transcript for build in self.resolved_builds if build.transcript_version}) @staticmethod def icon_for(status: str, include: bool) -> Optional[IconWithTooltip]: @@ -826,7 +836,7 @@ def dirty_check(self): if existing_vc and new_vc and existing_vc.ref != new_vc.ref: new_dirty_message = f"DIFF REF\n{cvc.message}\nRef {existing_vc.ref} -> {new_vc.ref}" else: - new_dirty_message = f"????\n{cvc.message}\n{repr(existing_vc)} -> {repr(new_vc)}" + new_dirty_message = f"????\n{cvc.message}\n{existing_vc!r} -> {new_vc!r}" def c_hgvs_diff_if_applicable(original_chgvs: str, new_chgvs: str): @@ -969,7 +979,7 @@ def set_variant_and_save(self, matched_variant: Variant, message: Optional[str] self.dirty_message = None if not force_update and self.matched_variant == matched_variant and self.status == ImportedAlleleInfoStatus.MATCHED_ALL_BUILDS: # nothing to do, and no force update, just update message if we need to - if message and message != self.message or self.dirty_message: + if (message and message != self.message) or self.dirty_message: self.message = message self.save() return diff --git a/classification/models/clinical_context_models.py b/classification/models/clinical_context_models.py index bd1947c85..504344a5a 100644 --- a/classification/models/clinical_context_models.py +++ b/classification/models/clinical_context_models.py @@ -1,7 +1,8 @@ +from collections.abc import Iterable from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import Optional, Iterable +from typing import Optional import django.dispatch from django.conf import settings @@ -11,10 +12,9 @@ from django.utils.timezone import now from django_extensions.db.models import TimeStampedModel -from classification.enums import ShareLevel, SpecialEKeys, AlleleOriginBucket +from classification.enums import AlleleOriginBucket, ShareLevel, SpecialEKeys from classification.enums.clinical_context_enums import ClinicalContextStatus -from classification.models.classification import Classification, \ - ClassificationModification +from classification.models.classification import Classification, ClassificationModification from classification.models.classification_import_run import ClassificationImportRun from flags.models import Flag, FlagStatus from flags.models.models import FlagsMixin, FlagTypeContext diff --git a/classification/models/clinical_context_utils.py b/classification/models/clinical_context_utils.py index 47c1496e9..2991d4bc2 100644 --- a/classification/models/clinical_context_utils.py +++ b/classification/models/clinical_context_utils.py @@ -1,8 +1,9 @@ from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass -from typing import List, Optional, Iterable +from typing import Optional -from classification.models import Classification, ClinicalContextRecalcTrigger, ClinicalContext +from classification.models import Classification, ClinicalContext, ClinicalContextRecalcTrigger from library.utils import first @@ -25,7 +26,7 @@ def merge(dirty_contexts: ['DirtyClinicalContext']) -> ['DirtyClinicalContext']: for dc in dirty_contexts: by_clinical_contexts[dc.clinical_context].append(dc) - flattened_dc: List[DirtyClinicalContext] = [] + flattened_dc: list[DirtyClinicalContext] = [] for dcc_list in by_clinical_contexts.values(): if len(dcc_list) == 1: flattened_dc.append(dcc_list[0]) @@ -55,7 +56,7 @@ def _assign_new_cc_reason(classification: Classification) -> Optional[ClinicalCo def _update_clinical_context( classification: Classification, - force_recalc_text: Optional[str] = None) -> List[DirtyClinicalContext]: + force_recalc_text: Optional[str] = None) -> list[DirtyClinicalContext]: """ :param classification: The classification to check the clinical context for :param force_recalc_text: If this has a value, and the classification is assigned to an allele, it's @@ -74,7 +75,7 @@ def _update_clinical_context( existing_clinical_context.recalc_and_save(cause=force_recalc_text, cause_code=ClinicalContextRecalcTrigger.SUBMISSION) return [] - contexts_to_recalculate: List[ClinicalContext] = [] + contexts_to_recalculate: list[ClinicalContext] = [] if existing_clinical_context: # since we're changing the clinical context, we're going to need to call recalc on the old one after @@ -126,7 +127,7 @@ def update_clinical_contexts( classifications: Iterable[Classification], force_recalc_text: Optional[str] = None): - all_dcs: List[DirtyClinicalContext] = [] + all_dcs: list[DirtyClinicalContext] = [] for classification in classifications: all_dcs += _update_clinical_context(classification, force_recalc_text) diff --git a/classification/models/clinvar_export_convertor.py b/classification/models/clinvar_export_convertor.py index 62ce21caf..cb1106f7f 100644 --- a/classification/models/clinvar_export_convertor.py +++ b/classification/models/clinvar_export_convertor.py @@ -1,20 +1,30 @@ import re +from collections.abc import Mapping from dataclasses import dataclass from functools import cached_property -from typing import Any, Mapping, TypedDict, Optional +from typing import Any, Optional, TypedDict from annotation.models import CitationFetchRequest from annotation.models.models_citations import CitationSource from annotation.regexes import db_citation_regexes -from classification.enums import SpecialEKeys, EvidenceKeyValueType, ShareLevel, AlleleOriginBucket -from classification.models import ClassificationModification, EvidenceKeyMap, EvidenceKey, \ - MultiCondition, ClinVarExport, classification_flag_types, Classification, ClinVarExportStatus, \ - ClinVarExportSubmission, CLINVAR_EXPORT_CONVERSION_VERSION +from classification.enums import AlleleOriginBucket, EvidenceKeyValueType, ShareLevel, SpecialEKeys +from classification.models import ( + CLINVAR_EXPORT_CONVERSION_VERSION, + Classification, + ClassificationModification, + ClinVarExport, + ClinVarExportStatus, + ClinVarExportSubmission, + EvidenceKey, + EvidenceKeyMap, + MultiCondition, + classification_flag_types, +) from genes.hgvs import CHGVS -from library.utils import html_to_text, JsonObjType, JsonDiffs, invalidate_cached_property -from ontology.models import OntologyTerm, OntologyService, OntologyTermStatus -from snpdb.models import ClinVarKey, ClinVarCitationsModes -from uicore.json.validated_json import JsonMessages, JSON_MESSAGES_EMPTY, ValidatedJson +from library.utils import JsonDiffs, JsonObjType, html_to_text, invalidate_cached_property +from ontology.models import OntologyService, OntologyTerm, OntologyTermStatus +from snpdb.models import ClinVarCitationsModes, ClinVarKey +from uicore.json.validated_json import JSON_MESSAGES_EMPTY, JsonMessages, ValidatedJson """ Code in this file is responsible for converting VariantGrid formatted classifications to ClinVar JSON @@ -390,7 +400,7 @@ def condition_to_json(condition: OntologyTerm) -> ValidatedJson: if condition.ontology_service == OntologyService.OMIM: id_part = str(condition.index) # OMIM is not 0 prefixed elif condition.ontology_service == OntologyService.ORPHANET: - id_part = f"ORPHA{str(condition.index)}" # ORPHA is not 0 prefixed + id_part = f"ORPHA{condition.index!s}" # ORPHA is not 0 prefixed db = "Orphanet" return ValidatedJson({ diff --git a/classification/models/clinvar_export_exclude_utils.py b/classification/models/clinvar_export_exclude_utils.py index 1e916cc37..388e12512 100644 --- a/classification/models/clinvar_export_exclude_utils.py +++ b/classification/models/clinvar_export_exclude_utils.py @@ -7,9 +7,14 @@ from django.db.models import QuerySet from django.dispatch import receiver -from classification.models import ClassificationModification, EvidenceMixin, classification_flag_types, \ - classification_post_publish_signal, Classification -from flags.models import FlagStatus, Flag +from classification.models import ( + Classification, + ClassificationModification, + EvidenceMixin, + classification_flag_types, + classification_post_publish_signal, +) +from flags.models import Flag, FlagStatus from library.guardian_utils import admin_bot from library.utils import get_timer from snpdb.models import ClinVarKey, ClinVarKeyExcludePattern diff --git a/classification/models/clinvar_export_models.py b/classification/models/clinvar_export_models.py index ea4186155..11aad1b62 100644 --- a/classification/models/clinvar_export_models.py +++ b/classification/models/clinvar_export_models.py @@ -1,6 +1,7 @@ +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property -from typing import Optional, Iterable +from typing import Optional from django.db import models, transaction from django.db.models import QuerySet, TextChoices @@ -11,10 +12,10 @@ from classification.enums import AlleleOriginBucket from classification.models import ClassificationModification, ConditionResolved -from library.preview_request import PreviewModelMixin, PreviewData, PreviewKeyValue -from library.utils import first, invalidate_cached_property, JsonObjType -from snpdb.models import ClinVarKey, Allele -from uicore.json.validated_json import ValidatedJson, JsonMessages +from library.preview_request import PreviewData, PreviewKeyValue, PreviewModelMixin +from library.utils import JsonObjType, first, invalidate_cached_property +from snpdb.models import Allele, ClinVarKey +from uicore.json.validated_json import JsonMessages, ValidatedJson CLINVAR_EXPORT_CONVERSION_VERSION = 4 diff --git a/classification/models/clinvar_export_prepare.py b/classification/models/clinvar_export_prepare.py index 0dc6f58ef..c4a818ea5 100644 --- a/classification/models/clinvar_export_prepare.py +++ b/classification/models/clinvar_export_prepare.py @@ -1,13 +1,19 @@ import itertools from collections import defaultdict -from typing import List, Optional, Set, Iterable +from collections.abc import Iterable +from typing import Optional from django.utils import timezone from django.utils.timezone import now -from classification.enums import ShareLevel, AlleleOriginBucket -from classification.models import ClinVarAllele, ClassificationModification, ClinVarExport, \ - ConditionResolved, ClinVarExportStatus +from classification.enums import AlleleOriginBucket, ShareLevel +from classification.models import ( + ClassificationModification, + ClinVarAllele, + ClinVarExport, + ClinVarExportStatus, + ConditionResolved, +) from classification.models.abstract_utils import ConsolidatingMerger from library.utils import pretty_collection from snpdb.lab_picker import LabPickerData @@ -26,11 +32,11 @@ def __init__(self, modification: ClassificationModification, allele_origin_bucket: Optional[AlleleOriginBucket] = None, condition_umbrella: Optional[ConditionResolved] = None, - failed_candidates: Optional[Set[ClassificationModification]] = None): + failed_candidates: Optional[set[ClassificationModification]] = None): self.modification = modification self.allele_origin_bucket = allele_origin_bucket or modification.classification.allele_origin_bucket self.condition_umbrella: ConditionResolved = condition_umbrella or modification.classification.condition_resolution_obj.as_mondo_if_possible() - self.failed_candidates: Set[ClassificationModification] = failed_candidates or set() + self.failed_candidates: set[ClassificationModification] = failed_candidates or set() if self.condition_umbrella is None or not bool(self.condition_umbrella.terms): raise ValueError("Candidate must have a resolved condition associated with it") @@ -47,11 +53,11 @@ def __init__(self, force_update: bool = True): self.clinvar_allele = clinvar_allele self.allele_origin_bucket = allele_origin_bucket - self.log: List[str] = [] + self.log: list[str] = [] self.force_update = force_update super().__init__() - def retrieve_established(self) -> Set[ClinVarExport]: + def retrieve_established(self) -> set[ClinVarExport]: return set(ClinVarExport.objects.filter(clinvar_allele=self.clinvar_allele)) def establish_new_candidate(self, new_candidate: ClassificationModificationCandidate) -> ClinVarExport: @@ -124,14 +130,14 @@ def merge_into_established_if_possible(self, established: ClinVarExport, new_can return False -ClinVarAlleleExportLog = List[str] +ClinVarAlleleExportLog = list[str] class ClinvarExportPrepare: @staticmethod def update_export_records(perspective: Optional[LabPickerData] = None): - clinvar_keys: Set[ClinVarKey] + clinvar_keys: set[ClinVarKey] if perspective: clinvar_keys = {lab.clinvar_key for lab in perspective.selected_labs if lab.clinvar_key} else: @@ -151,7 +157,7 @@ def process_allele( clinvar_key: ClinVarKey, allele: Allele, allele_origin_bucket: AlleleOriginBucket, - modifications: Iterable[ClassificationModification]) -> List[str]: + modifications: Iterable[ClassificationModification]) -> list[str]: clinvar_allele, _ = ClinVarAllele.objects.get_or_create( clinvar_key=clinvar_key, @@ -192,7 +198,7 @@ def process_allele( return log @staticmethod - def update_export_records_for_keys(clinvar_keys: Set[ClinVarKey]) -> ClinVarAlleleExportLog: + def update_export_records_for_keys(clinvar_keys: set[ClinVarKey]) -> ClinVarAlleleExportLog: # work on clinvar keys, not on labs, as a user could have access to one lab but the clinvar key might be for 2 # and a clinvar key has to get all labs updated or none, can't deal with partial clinvar_labs = Lab.objects.filter(clinvar_key__in=clinvar_keys) diff --git a/classification/models/clinvar_export_sync.py b/classification/models/clinvar_export_sync.py index 0a493c34c..867e1d67a 100644 --- a/classification/models/clinvar_export_sync.py +++ b/classification/models/clinvar_export_sync.py @@ -1,6 +1,6 @@ from enum import Enum, auto from functools import cached_property -from typing import TypedDict, Optional, Tuple +from typing import Optional, TypedDict import requests from django.conf import settings @@ -8,8 +8,14 @@ from requests import Response from classification.enums import AlleleOriginBucket -from classification.models import ClinVarExportBatch, ClinVarExportRequest, ClinVarExportRequestType, \ - ClinVarExportBatchStatus, ClinVarExportSubmission, ClinVarExportSubmissionStatus +from classification.models import ( + ClinVarExportBatch, + ClinVarExportBatchStatus, + ClinVarExportRequest, + ClinVarExportRequestType, + ClinVarExportSubmission, + ClinVarExportSubmissionStatus, +) from library.constants import MINUTE_SECS from library.log_utils import report_message from library.utils import JsonObjType @@ -158,7 +164,7 @@ def _send_data(self, response_status_code=response.status_code ) - def next_request(self, batch: ClinVarExportBatch) -> Tuple[ClinVarExportRequest, ClinVarResponseOutcome]: + def next_request(self, batch: ClinVarExportBatch) -> tuple[ClinVarExportRequest, ClinVarResponseOutcome]: if batch.allele_origin_bucket != AlleleOriginBucket.GERMLINE: raise ClinVarRequestException( exception_type=ClinVarRequestExceptionType.NOT_SUPPORTED_YET, diff --git a/classification/models/condition_text_matching.py b/classification/models/condition_text_matching.py index f5c896a68..1f3c7f2ed 100644 --- a/classification/models/condition_text_matching.py +++ b/classification/models/condition_text_matching.py @@ -1,16 +1,17 @@ import json import operator import re +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property, reduce from operator import attrgetter -from typing import Optional, Iterable +from typing import Optional import django from django.contrib.auth.models import User from django.contrib.postgres.fields import ArrayField from django.db import models, transaction -from django.db.models import CASCADE, QuerySet, SET_NULL, Q +from django.db.models import CASCADE, SET_NULL, Q, QuerySet from django.db.models.signals import post_save from django.dispatch import receiver from django.urls import reverse @@ -18,19 +19,40 @@ from model_utils.models import TimeStampedModel from annotation.regexes import db_ref_regexes -from classification.enums import SpecialEKeys, ShareLevel -from classification.models import Classification, ClassificationModification, classification_post_publish_signal, \ - flag_types, EvidenceKeyMap, ConditionResolvedDict, ConditionResolved, classification_flag_types +from classification.enums import ShareLevel, SpecialEKeys +from classification.models import ( + Classification, + ClassificationModification, + ConditionResolved, + ConditionResolvedDict, + EvidenceKeyMap, + classification_flag_types, + classification_post_publish_signal, + flag_types, +) from classification.models.condition_text_search import condition_text_search -from flags.models import flag_comment_action, Flag, FlagComment, FlagResolution +from flags.models import Flag, FlagComment, FlagResolution, flag_comment_action from genes.models import GeneSymbol, GeneSymbolAlias from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin from library.guardian_utils import admin_bot from library.log_utils import report_exc_info from library.utils import ArrayLength, get_timer -from ontology.models import OntologyTerm, OntologyService, OntologySnake, OntologyTermRelation, OntologyRelation -from ontology.ontology_matching import normalize_condition_text, \ - OPRPHAN_OMIM_TERMS, SearchText, pretty_set, PREFIX_SKIP_TERMS, IGNORE_TERMS, NON_PR_TERMS +from ontology.models import ( + OntologyRelation, + OntologyService, + OntologySnake, + OntologyTerm, + OntologyTermRelation, +) +from ontology.ontology_matching import ( + IGNORE_TERMS, + NON_PR_TERMS, + OPRPHAN_OMIM_TERMS, + PREFIX_SKIP_TERMS, + SearchText, + normalize_condition_text, + pretty_set, +) from snpdb.models import Lab condition_set_signal = django.dispatch.Signal() # args: "classification", "resolved_condition" diff --git a/classification/models/discordance_lab_summaries.py b/classification/models/discordance_lab_summaries.py index f12d5c545..1cdcc1e9c 100644 --- a/classification/models/discordance_lab_summaries.py +++ b/classification/models/discordance_lab_summaries.py @@ -4,8 +4,15 @@ from typing import Optional from classification.enums import SpecialEKeys -from classification.models import ClassificationLabSummary, DiscordanceReport, ClassificationLabSummaryEntry, \ - classification_flag_types, ClassificationFlagTypes, DiscordanceReportClassification, DiscordanceReportTriage +from classification.models import ( + ClassificationFlagTypes, + ClassificationLabSummary, + ClassificationLabSummaryEntry, + DiscordanceReport, + DiscordanceReportClassification, + DiscordanceReportTriage, + classification_flag_types, +) from snpdb.lab_picker import LabPickerData from snpdb.models import Lab @@ -18,7 +25,9 @@ class DiscordanceLabSummary(ClassificationLabSummary): @cached_property def embedded(self): if triage := self.triage: - from classification.views.discordance_report_triage_view import DiscordanceReportTriageView + from classification.views.discordance_report_triage_view import ( + DiscordanceReportTriageView, + ) return DiscordanceReportTriageView.lazy_render(triage) def _with_triage(self, triage: Optional[DiscordanceReportTriage]) -> 'DiscordanceLabSummary': @@ -67,12 +76,12 @@ def for_discordance_report(discordance_report: DiscordanceReport, perspective: L pending=pending )].append(drc) - dlses: list[DiscordanceLabSummary] = list(sorted([DiscordanceLabSummary( + dlses: list[DiscordanceLabSummary] = sorted([DiscordanceLabSummary( group=group, is_internal=group.lab in perspective.labs_if_not_admin, count=len(drcs), drcs=drcs - ) for group, drcs in group_counts.items()])) + ) for group, drcs in group_counts.items()]) triage_by_lab: dict[Lab, DiscordanceReportTriage] = {} for triage in discordance_report.discordancereporttriage_set.select_related('lab').all(): diff --git a/classification/models/discordance_models.py b/classification/models/discordance_models.py index 2f1e1b28f..a29320e31 100644 --- a/classification/models/discordance_models.py +++ b/classification/models/discordance_models.py @@ -1,33 +1,39 @@ import html from enum import Enum from functools import cached_property -from typing import Optional, Any, Union +from typing import Any, Optional, Union import django.dispatch from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.core.exceptions import PermissionDenied from django.db import models, transaction from django.db.models import TextChoices -from django.db.models.deletion import PROTECT, CASCADE +from django.db.models.deletion import CASCADE, PROTECT from django.urls.base import reverse from django.utils import timezone from django_extensions.db.models import TimeStampedModel from classification.enums.classification_enums import SpecialEKeys -from classification.enums.discordance_enums import DiscordanceReportResolution, ContinuedDiscordanceReason -from classification.models.classification import ClassificationModification, Classification -from classification.models.clinical_context_models import ClinicalContext, ClinicalContextRecalcTrigger -from classification.models.clinical_context_models import ClinicalContextChangeData +from classification.enums.discordance_enums import ( + ContinuedDiscordanceReason, + DiscordanceReportResolution, +) +from classification.models.classification import Classification, ClassificationModification +from classification.models.clinical_context_models import ( + ClinicalContext, + ClinicalContextChangeData, + ClinicalContextRecalcTrigger, +) from classification.models.flag_types import classification_flag_types from genes.hgvs import CHGVS from library.guardian_utils import admin_bot -from library.preview_request import PreviewModelMixin, PreviewKeyValue, PreviewData +from library.preview_request import PreviewData, PreviewKeyValue, PreviewModelMixin from library.utils import invalidate_cached_property from library.utils.django_utils import refresh_for_update -from review.models import ReviewableModelMixin, Review +from review.models import Review, ReviewableModelMixin from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import Lab, GenomeBuild +from snpdb.models import GenomeBuild, Lab discordance_change_signal = django.dispatch.Signal() # args: "discordance_report", "clinical_context_change_data:ClinicalContextChangeData" diff --git a/classification/models/discordance_models_utils.py b/classification/models/discordance_models_utils.py index 527d9d572..66ea2bed3 100644 --- a/classification/models/discordance_models_utils.py +++ b/classification/models/discordance_models_utils.py @@ -1,18 +1,26 @@ from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta from functools import cached_property -from typing import Iterable, Optional +from typing import Optional from django.utils import timezone from frozendict import frozendict from more_itertools import first -from classification.models import DiscordanceReport, DiscordanceReportTriage, DiscordanceReportTriageStatus, \ - ClassificationModification, ClassificationLabSummary, DiscordanceReportNextStep, DiscordanceReportClassification +from classification.models import ( + ClassificationLabSummary, + ClassificationModification, + DiscordanceReport, + DiscordanceReportClassification, + DiscordanceReportNextStep, + DiscordanceReportTriage, + DiscordanceReportTriageStatus, +) from genes.hgvs import CHGVS from library.django_utils import get_url_from_view_path -from library.utils import ExportRow, export_column, ExportDataType, pretty_label +from library.utils import ExportDataType, ExportRow, export_column, pretty_label from snpdb.lab_picker import LabPickerData from snpdb.models import Lab diff --git a/classification/models/evidence_key.py b/classification/models/evidence_key.py index 68b8665a5..66654df37 100644 --- a/classification/models/evidence_key.py +++ b/classification/models/evidence_key.py @@ -1,10 +1,11 @@ import math import re from collections import defaultdict +from collections.abc import Iterable, Mapping from copy import deepcopy from enum import Enum from functools import cached_property -from typing import Any, List, Optional, Dict, Iterable, Mapping, Union, Set, TypedDict, cast +from typing import Any, Optional, TypedDict, Union, cast import pydantic from django.conf import settings @@ -12,12 +13,21 @@ from django.db.models.deletion import SET_NULL from django_extensions.db.models import TimeStampedModel -from classification.enums import CriteriaEvaluation, SubmissionSource, SpecialEKeys -from classification.enums.classification_enums import EvidenceCategory, \ - EvidenceKeyValueType, ShareLevel -from classification.models.evidence_mixin import VCBlobDict, VCPatchValue, VCPatch, VCDbRefDict, EvidenceMixin +from classification.enums import CriteriaEvaluation, SpecialEKeys, SubmissionSource +from classification.enums.classification_enums import ( + EvidenceCategory, + EvidenceKeyValueType, + ShareLevel, +) +from classification.models.evidence_mixin import ( + EvidenceMixin, + VCBlobDict, + VCDbRefDict, + VCPatch, + VCPatchValue, +) from library.cache import timed_cache -from library.utils import empty_to_none, strip_json, first +from library.utils import empty_to_none, first, strip_json from snpdb.models import VariantGridColumn CLASSIFICATION_VALUE_TOLERANCE = 0.00000001 @@ -70,7 +80,7 @@ class EvidenceKeyOverrides(pydantic.BaseModel): def to_json(self): data = self.evidence_key_config.copy() - data["namespaces"] = list(sorted(self.namespaces)) + data["namespaces"] = sorted(self.namespaces) return data @staticmethod @@ -211,7 +221,7 @@ def redcap_key(self, suffix: int = None, is_note=False): return lower_key + suffix_str - def matched_options(self, normal_value_obj) -> List[Dict[str, str]]: + def matched_options(self, normal_value_obj) -> list[dict[str, str]]: """ Given a value (or possibly list of values) generate or find the options that match. e.g. for the value ["maternal","xsdfdwerew"] for variant inheritance we'd get @@ -244,7 +254,7 @@ def matched_options(self, normal_value_obj) -> List[Dict[str, str]]: return [{'key': value, 'label': value}] @staticmethod - def __special_up_options(options: List[Dict[str, Any]], default: str, allow_overrides=True) -> List[EvidenceKeyOption]: + def __special_up_options(options: list[dict[str, Any]], default: str, allow_overrides=True) -> list[EvidenceKeyOption]: """ Converts a CriteriaEvaluation options (e.g. CriteriaEvaluation.BENIGN_OPTIONS) and a default strength (e.g. BM) to a list of evidence keys with default and override populated appropriately @@ -303,17 +313,17 @@ def _virtual_options_for_criteria(self): return use_options @property - def virtual_options(self) -> Optional[List[EvidenceKeyOption]]: + def virtual_options(self) -> Optional[list[EvidenceKeyOption]]: if self.options: - return cast(List[EvidenceKeyOption], self.options) + return cast("list[EvidenceKeyOption]", self.options) if self.value_type == EvidenceKeyValueType.CRITERIA: return self._virtual_options_for_criteria() return None @cached_property - def option_indexes(self) -> Optional[Dict[str, int]]: - index_map: Optional[Dict[str, int]] = None + def option_indexes(self) -> Optional[dict[str, int]]: + index_map: Optional[dict[str, int]] = None if options := self.virtual_options: index_map = {} for index, option in enumerate(options): @@ -330,7 +340,7 @@ def sort_values(self, values: Iterable) -> list: sorted_list = sorted(values, key=lambda x: (sorter(x), x)) return sorted_list - def classification_sorter(self, evidence: Dict[str, Any]) -> Union[int, Any]: + def classification_sorter(self, evidence: dict[str, Any]) -> Union[int, Any]: """ Provide .classification_sorter as a Callable[dict aka ClassificationData] -> sortable """ @@ -346,13 +356,13 @@ def validate(self): raise ValueError(f'{self.key} option with space in it "{option_key}"') @property - def option_dictionary(self) -> Dict[str, str]: + def option_dictionary(self) -> dict[str, str]: if options := self.virtual_options: return {x.get('key'): x.get('label') for x in options} else: return {} - def option_dictionary_property(self, prop: str) -> Dict[str, Any]: + def option_dictionary_property(self, prop: str) -> dict[str, Any]: if options := self.virtual_options: return {x.get('key'): x.get(prop) for x in options if prop in x} else: @@ -419,7 +429,7 @@ def pretty_value(self, normal_value_obj: Any, dash_for_none: bool = False) -> Op value = [value] str_values = [] for val in value: - matched_option: Dict + matched_option: dict if val == '' or val is None: matched_option = next((option for option in options if option.get('key') is None or option.get('key') == ''), None) else: @@ -506,7 +516,7 @@ def pretty_value_for(item, key: str) -> str: return EvidenceKeyMap.cached_key(key).pretty_value(item.get(key)) @staticmethod - def __ordered_keys() -> List[EvidenceKey]: + def __ordered_keys() -> list[EvidenceKey]: # sort in code (rather than sql) as pretty_label isn't available normally key_entries = list(EvidenceKey.objects.all()) key_entries.sort(key=lambda k: (k.order, k.pretty_label.lower())) @@ -536,7 +546,7 @@ def instance() -> 'EvidenceKeyMap': def cached() -> 'EvidenceKeyMap': return EvidenceKeyMap.instance() - def __init__(self, ordered_keys: List[EvidenceKey], config: Optional[EvidenceKeyOverrides] = None): + def __init__(self, ordered_keys: list[EvidenceKey], config: Optional[EvidenceKeyOverrides] = None): if not config: config = EvidenceKeyOverrides() self._ordered_keys = ordered_keys @@ -594,34 +604,34 @@ def __getitem__(self, item) -> EvidenceKey: def __contains__(self, item): return item in self.key_dict - def immutable(self) -> List[EvidenceKey]: + def immutable(self) -> list[EvidenceKey]: return [eKey for eKey in self.all_keys if eKey.immutable] - def mandatory(self) -> List[EvidenceKey]: + def mandatory(self) -> list[EvidenceKey]: return [eKey for eKey in self.all_keys if eKey.mandatory] - def share_level(self, sl: ShareLevel) -> List[EvidenceKey]: + def share_level(self, sl: ShareLevel) -> list[EvidenceKey]: return [eKey for eKey in self.all_keys if eKey.max_share_level_enum == sl] - def share_level_and_higher(self, sl: ShareLevel) -> List[EvidenceKey]: + def share_level_and_higher(self, sl: ShareLevel) -> list[EvidenceKey]: return [eKey for eKey in self.all_keys if eKey.max_share_level_enum in ShareLevel.same_and_higher(sl)] - def criteria(self) -> List[EvidenceKey]: + def criteria(self) -> list[EvidenceKey]: """ :return: A list of ALL criteria EvidenceKeys, includes standard ACMG and custom ones with namespaces """ return [eKey for eKey in self.all_keys if eKey.value_type == EvidenceKeyValueType.CRITERIA] - def vital(self) -> List[EvidenceKey]: + def vital(self) -> list[EvidenceKey]: return [e_key for e_key in self.all_keys if e_key.is_vital_key] - def acmg_criteria(self) -> List[EvidenceKey]: + def acmg_criteria(self) -> list[EvidenceKey]: """ :return: A list of STANDARD ACMG criteria EvidenceKeys """ return self.criteria_for("acmg") - def criteria_for(self, namespace) -> List[EvidenceKey]: + def criteria_for(self, namespace) -> list[EvidenceKey]: crit = [eKey for eKey in self.criteria() if eKey.namespace == namespace] crit.sort(key=lambda k: k.pretty_label.lower()) return crit @@ -655,7 +665,7 @@ def __init__(self, data: VCPatch, e_key: EvidenceKey): self.validate = True def __str__(self): - return f'"{self.e_key.key}": {str(self.raw)}' + return f'"{self.e_key.key}": {self.raw!s}' @property def _my_data(self) -> VCBlobDict: @@ -760,11 +770,11 @@ def note(self, value: Optional[str]): self._ensure_my_data()['note'] = value @property - def db_refs(self) -> Optional[List[VCDbRefDict]]: + def db_refs(self) -> Optional[list[VCDbRefDict]]: return self._my_data.get('db_refs') @db_refs.setter - def db_refs(self, db_refs: Optional[List[VCDbRefDict]]): + def db_refs(self, db_refs: Optional[list[VCDbRefDict]]): self._ensure_my_data()['db_refs'] = db_refs def strip_non_client_submission(self): @@ -810,7 +820,7 @@ def __eq__(self, other): def __contains__(self, item): return item in self._my_data - def diff(self, dest: Optional['VCDataCell'], ignore_if_omitted: Optional[Set[str]] = None) -> Dict[str, Any]: + def diff(self, dest: Optional['VCDataCell'], ignore_if_omitted: Optional[set[str]] = None) -> dict[str, Any]: """ Given two dictionaries, returns only the entries that have changed :param dest: Another dict @@ -872,7 +882,7 @@ class VCDataDict: Represents an entire patch or base set of data for EvidenceKeys """ - def __init__(self, data: Dict[str, Any], evidence_keys: EvidenceKeyMap): + def __init__(self, data: dict[str, Any], evidence_keys: EvidenceKeyMap): if not isinstance(data, dict): raise ValueError('Data must be of type dict') self.data = data diff --git a/classification/models/evidence_mixin.py b/classification/models/evidence_mixin.py index 763f8496d..23d52152b 100644 --- a/classification/models/evidence_mixin.py +++ b/classification/models/evidence_mixin.py @@ -1,14 +1,15 @@ import re +from collections.abc import Mapping from dataclasses import dataclass from functools import cached_property -from typing import Dict, Any, Mapping, Optional, Union, List, TypedDict +from typing import Any, Optional, TypedDict, Union from django.conf import settings from annotation.models import CitationFetchRequest from annotation.models.models_citations import CitationFetchResponse from classification.criteria_strengths import CriteriaStrength, CriteriaStrengths -from classification.enums import SpecialEKeys, AlleleOriginBucket +from classification.enums import AlleleOriginBucket, SpecialEKeys from genes.hgvs import CHGVS, PHGVS from library.log_utils import report_message from library.utils import empty_to_none @@ -36,8 +37,8 @@ class VCBlobDict(TypedDict, total=False): note: str explain: str immutable: str - db_refs: List[VCDbRefDict] - validation: List[VCValidation] + db_refs: list[VCDbRefDict] + validation: list[VCValidation] class SomaticValueDict(TypedDict): @@ -107,8 +108,8 @@ def from_str(value: str): VCStoreValue = VCBlobDict VCPatchValue = Union[None, VCStoreValue] -VCStore = Dict[str, VCStoreValue] -VCPatch = Dict[str, VCPatchValue] +VCStore = dict[str, VCStoreValue] +VCPatch = dict[str, VCPatchValue] class EvidenceMixin: @@ -188,7 +189,7 @@ def get_genome_build_patch_version(self) -> GenomeBuildPatchVersion: raise ValueError("Classification does not have a value for genome build") @cached_property - def db_refs(self) -> List[VCDbRefDict]: + def db_refs(self) -> list[VCDbRefDict]: all_db_refs = [] for blob in self._evidence.values(): db_refs = blob.get('db_refs') @@ -213,7 +214,7 @@ def criteria_strengths(self, e_keys: Optional['EvidenceKeyMap'] = None) -> Crite if not e_keys: e_keys = EvidenceKeyMap.instance() - criteria: List[CriteriaStrength] = [] + criteria: list[CriteriaStrength] = [] for ek in e_keys.criteria(): if strength := self.get(ek.key): criteria.append(CriteriaStrength(ek, strength)) @@ -299,7 +300,7 @@ def _clean_key(key): return key @staticmethod - def to_patch(raw: Dict[str, Any]) -> VCPatch: + def to_patch(raw: dict[str, Any]) -> VCPatch: """ Cleans up a dictionary significantly ready for processing. Converts keys to numbers and letters, and all whitespace to underscores. diff --git a/classification/models/evidence_mixin_summary_cache.py b/classification/models/evidence_mixin_summary_cache.py index 2f356d96c..758272f5e 100644 --- a/classification/models/evidence_mixin_summary_cache.py +++ b/classification/models/evidence_mixin_summary_cache.py @@ -1,9 +1,9 @@ from dataclasses import dataclass from functools import cached_property -from typing import TypedDict, Optional, Self +from typing import Optional, Self, TypedDict from classification.criteria_strengths import CriteriaStrength -from classification.enums import AlleleOriginBucket, SpecialEKeys, CriteriaEvaluation +from classification.enums import AlleleOriginBucket, CriteriaEvaluation, SpecialEKeys from library.utils import strip_json """ @@ -116,7 +116,7 @@ def cache_dict(self) -> ClassificationSummaryCacheDict: @cached_property def pending_classification_value(self) -> Optional[str]: - from classification.models import classification_flag_types, ClassificationFlagTypes + from classification.models import ClassificationFlagTypes, classification_flag_types from flags.models import Flag, FlagStatus if flag := Flag.objects.filter( flag_type=classification_flag_types.classification_pending_changes, diff --git a/classification/models/uploaded_classifications_unmapped.py b/classification/models/uploaded_classifications_unmapped.py index e00f8ed6b..9ac593621 100644 --- a/classification/models/uploaded_classifications_unmapped.py +++ b/classification/models/uploaded_classifications_unmapped.py @@ -1,6 +1,6 @@ import html from functools import cached_property -from typing import Union, Optional, Any +from typing import Any, Optional, Union from django.contrib.auth.models import User from django.db import models diff --git a/classification/models/uploaded_file_types.py b/classification/models/uploaded_file_types.py index 433662a13..13f53bcf2 100644 --- a/classification/models/uploaded_file_types.py +++ b/classification/models/uploaded_file_types.py @@ -4,9 +4,10 @@ import re import tarfile from abc import ABC, abstractmethod +from collections.abc import Iterator from os import PathLike from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Optional, Union from zipfile import ZipFile diff --git a/classification/models/variant_resolver.py b/classification/models/variant_resolver.py index 5ed67f028..25de0ad5b 100644 --- a/classification/models/variant_resolver.py +++ b/classification/models/variant_resolver.py @@ -3,7 +3,7 @@ from classification.models import ClassificationImport, ImportedAlleleInfo, ImportedAlleleInfoStatus from classification.tasks.classification_import_task import process_classification_import_task -from snpdb.models import ImportSource, GenomeBuild +from snpdb.models import GenomeBuild, ImportSource class VariantResolver: diff --git a/classification/signals/classification_health_checks.py b/classification/signals/classification_health_checks.py index c6ce6d88f..e1489fbb4 100644 --- a/classification/signals/classification_health_checks.py +++ b/classification/signals/classification_health_checks.py @@ -1,13 +1,22 @@ from django.dispatch import receiver from classification.enums import ShareLevel -from classification.models import Classification, classification_flag_types, ImportedAlleleInfo, \ - ImportedAlleleInfoStatus +from classification.models import ( + Classification, + ImportedAlleleInfo, + ImportedAlleleInfoStatus, + classification_flag_types, +) from flags.models import FlagType from flags.models.flag_health_check import flag_chanced_since -from library.health_check import health_check_signal, \ - HealthCheckRequest, HealthCheckTotalAmount, HealthCheckRecentActivity, HealthCheckStat, \ - health_check_overall_stats_signal +from library.health_check import ( + HealthCheckRecentActivity, + HealthCheckRequest, + HealthCheckStat, + HealthCheckTotalAmount, + health_check_overall_stats_signal, + health_check_signal, +) from library.utils import limit_str """ diff --git a/classification/signals/classification_hooks_assign_owner.py b/classification/signals/classification_hooks_assign_owner.py index 75d70cc93..2d1633e39 100644 --- a/classification/signals/classification_hooks_assign_owner.py +++ b/classification/signals/classification_hooks_assign_owner.py @@ -1,11 +1,9 @@ from django.contrib.auth.models import User from django.dispatch.dispatcher import receiver -from classification.enums.classification_enums import SpecialEKeys, \ - ValidationCode +from classification.enums.classification_enums import SpecialEKeys, ValidationCode from classification.models import PatchMeta -from classification.models.classification import Classification, \ - classification_validation_signal +from classification.models.classification import Classification, classification_validation_signal from classification.models.classification_utils import ValidationMerger diff --git a/classification/signals/classification_hooks_discordance_notifications.py b/classification/signals/classification_hooks_discordance_notifications.py index f05e993cb..3deb27761 100644 --- a/classification/signals/classification_hooks_discordance_notifications.py +++ b/classification/signals/classification_hooks_discordance_notifications.py @@ -8,10 +8,17 @@ from django.utils import timezone from classification.enums import SpecialEKeys -from classification.models import DiscordanceReport, discordance_change_signal, EvidenceKeyMap, \ - ClassificationLabSummary -from classification.models.clinical_context_models import DiscordanceNotification, ClinicalContextChangeData, \ - ClinicalContextRecalcTrigger +from classification.models import ( + ClassificationLabSummary, + DiscordanceReport, + EvidenceKeyMap, + discordance_change_signal, +) +from classification.models.clinical_context_models import ( + ClinicalContextChangeData, + ClinicalContextRecalcTrigger, + DiscordanceNotification, +) from classification.models.discordance_models_utils import DiscordanceReportRowData from library.django_utils import get_url_from_view_path from library.log_utils import NotificationBuilder @@ -68,7 +75,7 @@ def send_prepared_discordance_notifications(outstanding_notifications: Optional[ dr_id = outstanding_notification.discordance_report_id unique_ids.add(dr_id) - dr_ids: list[int] = list(sorted(unique_ids)) + dr_ids: list[int] = sorted(unique_ids) drs: list[DiscordanceReport] = DiscordanceReport.objects.filter(pk__in=dr_ids).order_by('pk') dr_count = len(dr_ids) @@ -111,7 +118,7 @@ def report_url_for_id(the_id): lab_notification.add_field(label="Discordance Detected On", value=report_summary.date_detected_str) - c_hgvs_str = "\n".join((str(chgvs) for chgvs in report_summary.c_hgvses)) + c_hgvs_str = "\n".join(str(chgvs) for chgvs in report_summary.c_hgvses) lab_notification.add_field(label="c.HGVS", value=c_hgvs_str) sig_lab: ClassificationLabSummary diff --git a/classification/signals/classification_hooks_discordance_status.py b/classification/signals/classification_hooks_discordance_status.py index 6d1476aa1..2862fe53a 100644 --- a/classification/signals/classification_hooks_discordance_status.py +++ b/classification/signals/classification_hooks_discordance_status.py @@ -1,4 +1,3 @@ -from typing import Set from django.conf import settings from django.contrib.auth.models import User @@ -6,18 +5,27 @@ from django.db.models.signals import post_delete from django.dispatch.dispatcher import receiver -from classification.enums import SpecialEKeys, ShareLevel +from classification.enums import ShareLevel, SpecialEKeys from classification.models import ImportedAlleleInfo -from classification.models.classification import Classification, \ - classification_current_state_signal, \ - classification_post_publish_signal, \ - ClassificationModification, \ - classification_withdraw_signal +from classification.models.classification import ( + Classification, + ClassificationModification, + classification_current_state_signal, + classification_post_publish_signal, + classification_withdraw_signal, +) from classification.models.classification_utils import ValidationMerger from classification.models.classification_variant_info_models import allele_info_changed_signal -from classification.models.clinical_context_models import ClinicalContext, \ - clinical_context_signal, ClinicalContextRecalcTrigger, ClinicalContextChangeData -from classification.models.clinical_context_utils import update_clinical_context, update_clinical_contexts +from classification.models.clinical_context_models import ( + ClinicalContext, + ClinicalContextChangeData, + ClinicalContextRecalcTrigger, + clinical_context_signal, +) +from classification.models.clinical_context_utils import ( + update_clinical_context, + update_clinical_contexts, +) from classification.models.discordance_models import DiscordanceReport from classification.models.evidence_key import EvidenceKey, EvidenceKeyMap from classification.models.flag_types import classification_flag_types @@ -75,7 +83,7 @@ def published(sender, cs = EvidenceKey.objects.get(pk=SpecialEKeys.CLINICAL_SIGNIFICANCE).pretty_value( classification.evidence.get(SpecialEKeys.CLINICAL_SIGNIFICANCE)) or 'Unclassified' - diff_keys: Set[str] = set() + diff_keys: set[str] = set() if not first_publish: if previously_published: keys = set(newly_published.evidence) | set(previously_published.evidence) diff --git a/classification/signals/classification_hooks_grouping.py b/classification/signals/classification_hooks_grouping.py index 3b3e82fc3..4e49ffc35 100644 --- a/classification/signals/classification_hooks_grouping.py +++ b/classification/signals/classification_hooks_grouping.py @@ -3,14 +3,23 @@ from django.dispatch import receiver from classification.enums import ShareLevel -from classification.models import classification_post_publish_signal, Classification, ClassificationModification, \ - classification_withdraw_signal, allele_info_changed_signal, ImportedAlleleInfo, ClassificationImportRun, \ - condition_set_signal -from classification.models.classification_grouping import ClassificationGrouping, ClassificationGroupingEntry +from classification.models import ( + Classification, + ClassificationImportRun, + ClassificationModification, + ImportedAlleleInfo, + allele_info_changed_signal, + classification_post_publish_signal, + classification_withdraw_signal, + condition_set_signal, +) +from classification.models.classification_grouping import ( + ClassificationGrouping, + ClassificationGroupingEntry, +) from classification.models.classification_import_run import classification_imports_complete_signal from library.utils import get_timer - ### # Capture events that could require a ClassificationGrouping to update. # If the ClassificationGrouping for a classificaiton might change (or be created for the first time) diff --git a/classification/signals/classification_hooks_grouping_search_terms.py b/classification/signals/classification_hooks_grouping_search_terms.py index 0997f1023..c45a28848 100644 --- a/classification/signals/classification_hooks_grouping_search_terms.py +++ b/classification/signals/classification_hooks_grouping_search_terms.py @@ -1,12 +1,19 @@ from collections import defaultdict -from typing import Iterable, Optional, Set +from collections.abc import Iterable +from typing import Optional from django.dispatch import receiver from annotation.models import AnnotationVersion, VariantAnnotation -from classification.models import ClassificationGroupingSearchTermStub, ClassificationGroupingSearchTermType, \ - ClassificationGrouping, ImportedAlleleInfo, classification_grouping_search_term_signal, \ - ClassificationGroupingSearchTermBuilder, ClinVarExport +from classification.models import ( + ClassificationGrouping, + ClassificationGroupingSearchTermBuilder, + ClassificationGroupingSearchTermStub, + ClassificationGroupingSearchTermType, + ClinVarExport, + ImportedAlleleInfo, + classification_grouping_search_term_signal, +) from genes.models import GeneSymbol, GeneSymbolAlias, GeneVersion, TranscriptVersion from ontology.models import OntologyTerm from snpdb.models import GenomeBuild, Variant @@ -70,7 +77,7 @@ def stub_for(gene_symbol: GeneSymbol | str): @receiver(classification_grouping_search_term_signal) def _condition_terms(grouping: ClassificationGrouping, **kwargs) -> Optional[Iterable[ClassificationGroupingSearchTermStub]]: - all_terms: Set[OntologyTerm] = set() + all_terms: set[OntologyTerm] = set() all_stubs: list[ClassificationGroupingSearchTermStub] = [] # redundantly duplicate a lot for the condition free text diff --git a/classification/signals/classification_hooks_import_notifications.py b/classification/signals/classification_hooks_import_notifications.py index fdde66fd0..9a7eaea9c 100644 --- a/classification/signals/classification_hooks_import_notifications.py +++ b/classification/signals/classification_hooks_import_notifications.py @@ -2,12 +2,13 @@ from django.dispatch.dispatcher import receiver from classification.models import ClinicalContext -from classification.models.classification_import_run import ClassificationImportRun, \ - classification_imports_complete_signal +from classification.models.classification_import_run import ( + ClassificationImportRun, + classification_imports_complete_signal, +) from classification.models.clinical_context_models import ClinicalContextRecalcTrigger from classification.signals import send_prepared_discordance_notifications -from flags.models.models import FlagCollection, \ - flag_collection_extra_info_signal, FlagInfos +from flags.models.models import FlagCollection, FlagInfos, flag_collection_extra_info_signal @receiver(flag_collection_extra_info_signal, sender=FlagCollection) diff --git a/classification/signals/classification_hooks_pending_flags.py b/classification/signals/classification_hooks_pending_flags.py index 71faa8527..6eace1167 100644 --- a/classification/signals/classification_hooks_pending_flags.py +++ b/classification/signals/classification_hooks_pending_flags.py @@ -1,7 +1,11 @@ from django.dispatch import receiver -from classification.models import classification_flag_types, Classification, ClassificationSummaryCalculator -from flags.models import flag_comment_action, Flag, FlagComment, FlagResolution +from classification.models import ( + Classification, + ClassificationSummaryCalculator, + classification_flag_types, +) +from flags.models import Flag, FlagComment, FlagResolution, flag_comment_action @receiver(flag_comment_action, sender=Flag) diff --git a/classification/signals/classification_hooks_share_flags.py b/classification/signals/classification_hooks_share_flags.py index 76e8c0e0e..588fa96ba 100644 --- a/classification/signals/classification_hooks_share_flags.py +++ b/classification/signals/classification_hooks_share_flags.py @@ -3,8 +3,7 @@ from django.dispatch.dispatcher import receiver from classification.enums.classification_enums import ShareLevel -from classification.models.classification import \ - Classification, classification_revalidate_signal +from classification.models.classification import Classification, classification_revalidate_signal from classification.models.flag_types import classification_flag_types """ diff --git a/classification/signals/classification_hooks_significant_change.py b/classification/signals/classification_hooks_significant_change.py index 1b8a9c57f..7506d2a45 100644 --- a/classification/signals/classification_hooks_significant_change.py +++ b/classification/signals/classification_hooks_significant_change.py @@ -6,9 +6,16 @@ from django.urls import reverse from django_messages.admin import User -from classification.enums import ShareLevel, SpecialEKeys, ClinicalSignificance -from classification.models import classification_post_publish_signal, Classification, classification_flag_types, \ - EvidenceKey, ClassificationFlagTypes, ClassificationModification, EvidenceKeyMap +from classification.enums import ClinicalSignificance, ShareLevel, SpecialEKeys +from classification.models import ( + Classification, + ClassificationFlagTypes, + ClassificationModification, + EvidenceKey, + EvidenceKeyMap, + classification_flag_types, + classification_post_publish_signal, +) from library.django_utils import get_url_from_view_path from library.log_utils import NotificationBuilder from library.utils import get_timer diff --git a/classification/signals/classification_hooks_variants_classification_changes.py b/classification/signals/classification_hooks_variants_classification_changes.py index 6ce189c4f..2d16584d8 100644 --- a/classification/signals/classification_hooks_variants_classification_changes.py +++ b/classification/signals/classification_hooks_variants_classification_changes.py @@ -1,8 +1,13 @@ from django.contrib.auth.models import User from django.dispatch import receiver -from classification.models import classification_post_publish_signal, Classification, ClassificationModification, \ - variants_classification_changed_signal, ImportedAlleleInfo +from classification.models import ( + Classification, + ClassificationModification, + ImportedAlleleInfo, + classification_post_publish_signal, + variants_classification_changed_signal, +) from classification.models.classification_variant_info_models import allele_info_changed_signal from snpdb.models import Allele diff --git a/classification/signals/classification_liftover.py b/classification/signals/classification_liftover.py index ce5c191b5..8cecc62b5 100644 --- a/classification/signals/classification_liftover.py +++ b/classification/signals/classification_liftover.py @@ -5,7 +5,7 @@ from classification.models import ImportedAlleleInfo from library.log_utils import report_event from snpdb.clingen_allele import populate_clingen_alleles_for_variants -from snpdb.models import liftover_run_complete_signal, LiftoverRun, AlleleConversionTool, Variant +from snpdb.models import AlleleConversionTool, LiftoverRun, Variant, liftover_run_complete_signal @receiver(liftover_run_complete_signal, sender=LiftoverRun) diff --git a/classification/signals/classification_search.py b/classification/signals/classification_search.py index 1498c46c2..d294bc6f4 100644 --- a/classification/signals/classification_search.py +++ b/classification/signals/classification_search.py @@ -10,11 +10,11 @@ from annotation.models import VariantAnnotationVersion from classification.enums import AlleleOriginBucket from classification.models import Classification, ClassificationModification -from library.preview_request import preview_extra_signal, PreviewKeyValue +from library.preview_request import PreviewKeyValue, preview_extra_signal from ontology.models import OntologyTerm from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import Lab, Organization, Allele, Variant, GenomeBuild -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.models import Allele, GenomeBuild, Lab, Organization, Variant +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/classification/signals/discordance_report_review_detail.py b/classification/signals/discordance_report_review_detail.py index 6ad330f1e..c1faa1512 100644 --- a/classification/signals/discordance_report_review_detail.py +++ b/classification/signals/discordance_report_review_detail.py @@ -8,7 +8,7 @@ from classification.enums import SpecialEKeys from classification.models import DiscordanceReport, EvidenceKeyMap -from review.models import review_detail_signal, Review +from review.models import Review, review_detail_signal from snpdb.models import Lab @@ -45,7 +45,7 @@ def discordance_report_changes_summary(sender, instance: Review, **kwargs): if changes := data.get("changes"): rows = [] t = loader.get_template("classification/snippets/pending_change.html") - changes_d = list(sorted(PendingChange.from_dict(change) for change in changes)) + changes_d = sorted(PendingChange.from_dict(change) for change in changes) for change in changes_d: row = t.render(context={"change": change}) rows.append(row) diff --git a/classification/signals/discordance_report_search.py b/classification/signals/discordance_report_search.py index 0db1e96e5..3549119ff 100644 --- a/classification/signals/discordance_report_search.py +++ b/classification/signals/discordance_report_search.py @@ -1,7 +1,7 @@ import re from classification.models import DiscordanceReport -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/classification/signals/discordance_report_triage_changes.py b/classification/signals/discordance_report_triage_changes.py index 83fc8212b..c1ccb2f34 100644 --- a/classification/signals/discordance_report_triage_changes.py +++ b/classification/signals/discordance_report_triage_changes.py @@ -1,7 +1,10 @@ from django.dispatch import receiver -from classification.models import DiscordanceReport, discordance_change_signal, \ - ensure_discordance_report_triages_for +from classification.models import ( + DiscordanceReport, + discordance_change_signal, + ensure_discordance_report_triages_for, +) """ Creates, closes or re-opens triages base don activity of the discordance diff --git a/classification/tasks/classification_candidate_search_tasks.py b/classification/tasks/classification_candidate_search_tasks.py index 51f22c091..13d152486 100644 --- a/classification/tasks/classification_candidate_search_tasks.py +++ b/classification/tasks/classification_candidate_search_tasks.py @@ -1,19 +1,19 @@ import logging import operator from collections import defaultdict +from collections.abc import Iterable from functools import reduce -from typing import Iterable from django.db.models import Q, QuerySet from analysis.models import Candidate from analysis.tasks.abstract_candidate_search_task import AbstractCandidateSearchTask -from annotation.models import AnnotationVersion, VariantAnnotation, ClinVar, ClinVarReviewStatus +from annotation.models import AnnotationVersion, ClinVar, ClinVarReviewStatus, VariantAnnotation from classification.enums import AlleleOriginBucket, ClinicalSignificance from classification.models import Classification, ClassificationModification, EvidenceKey from classification.models.classification_utils import classification_gene_symbol_filter from classification.views.classification_datatables import ClassificationColumns -from snpdb.models import Sample, GenomeBuild +from snpdb.models import GenomeBuild, Sample from snpdb.sample_filters import get_sample_ontology_q, get_sample_qc_gene_list_gene_symbol_q from variantgrid.celery import app diff --git a/classification/tasks/classification_import_map_and_insert_task.py b/classification/tasks/classification_import_map_and_insert_task.py index 1e9b1fbb0..d1c0bb96a 100644 --- a/classification/tasks/classification_import_map_and_insert_task.py +++ b/classification/tasks/classification_import_map_and_insert_task.py @@ -10,9 +10,12 @@ from django.conf import settings from classification.enums import SubmissionSource -from classification.models import UploadedClassificationsUnmapped, UploadedClassificationsUnmappedStatus +from classification.models import ( + UploadedClassificationsUnmapped, + UploadedClassificationsUnmappedStatus, +) from classification.models.classification_import_run import ClassificationImportRun -from library.log_utils import report_message, NotificationBuilder +from library.log_utils import NotificationBuilder, report_message from library.utils import batch_iterator, pretty_label from variantgrid.celery import app @@ -116,7 +119,7 @@ def run(self, upload_classifications_unmapped_id: int, import_records: bool = Tr "--publish", shlex.quote(publish), "--org", shlex.quote(upload_file.lab.organization.group_name), "--lab", shlex.quote(upload_file.lab.group_name.split("/")[1]), - "--env", f"file" + "--env", "file" ] if include_source: args.append("--include_source") @@ -145,7 +148,7 @@ def run(self, upload_classifications_unmapped_id: int, import_records: bool = Tr validation_list_file = output_dir / "validation_list.json" fatal_error = None - with open(validation_summary_file, 'r') as validation_handle: + with open(validation_summary_file) as validation_handle: validation_json = json.load(validation_handle) fatal_error = validation_json.get('fatal_error') upload_file.validation_summary = validation_json @@ -171,7 +174,7 @@ def run(self, upload_classifications_unmapped_id: int, import_records: bool = Tr nb.add_field(formatted_key, value) nb.send() - with open(validation_list_file, 'r') as validation_handle: + with open(validation_list_file) as validation_handle: validation_list = json.load(validation_handle) upload_file.validation_list = validation_list @@ -183,7 +186,7 @@ def run(self, upload_classifications_unmapped_id: int, import_records: bool = Tr if import_records: import_id = upload_file.file_data.filename + "_" + upload_file.lab.group_name - with open(classifications_file, 'r') as file_handle: + with open(classifications_file) as file_handle: ClassificationImportMapInsertTask.update_status(upload_file, UploadedClassificationsUnmappedStatus.Importing) def row_generator() -> dict: diff --git a/classification/tasks/classification_import_process_variants_task.py b/classification/tasks/classification_import_process_variants_task.py index 715ee4993..99725e973 100644 --- a/classification/tasks/classification_import_process_variants_task.py +++ b/classification/tasks/classification_import_process_variants_task.py @@ -1,4 +1,4 @@ -from typing import Optional, Any +from typing import Any, Optional from django.conf import settings @@ -6,7 +6,7 @@ from classification.models.classification import ClassificationImport from snpdb.clingen_allele import populate_clingen_alleles_for_variants from snpdb.liftover import create_liftover_pipelines -from snpdb.models import GenomeBuild, ImportSource, Variant, VariantCoordinate, Allele +from snpdb.models import Allele, GenomeBuild, ImportSource, Variant, VariantCoordinate from snpdb.variant_pk_lookup import VariantPKLookup from upload.models import ModifiedImportedVariant, UploadStep from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask diff --git a/classification/templatetags/classification_tags.py b/classification/templatetags/classification_tags.py index cbe08fe7c..48d51f6c0 100644 --- a/classification/templatetags/classification_tags.py +++ b/classification/templatetags/classification_tags.py @@ -1,9 +1,10 @@ import json import logging import uuid +from collections.abc import Collection, Iterable from datetime import timedelta from html import escape -from typing import Union, Optional, Iterable, Any, Collection +from typing import Any, Optional, Union from django.conf import settings from django.contrib.auth.models import User @@ -13,18 +14,30 @@ from django.utils.safestring import mark_safe from django.utils.timezone import localtime -from classification.criteria_strengths import CriteriaStrength, AcmgPointScore +from classification.criteria_strengths import AcmgPointScore, CriteriaStrength from classification.enums import SpecialEKeys from classification.enums.classification_enums import ShareLevel -from classification.models import ConditionTextMatch, ConditionResolved, ClassificationLabSummary, ImportedAlleleInfo, \ - EvidenceMixin, ClassificationSummaryCacheDictPathogenicity -from classification.models.classification import ClassificationModification, Classification -from classification.models.classification_groups import ClassificationGroup, ClassificationGroups, \ - ClassificationGroupUtils +from classification.models import ( + ClassificationLabSummary, + ClassificationSummaryCacheDictPathogenicity, + ConditionResolved, + ConditionTextMatch, + EvidenceMixin, + ImportedAlleleInfo, +) +from classification.models.classification import Classification, ClassificationModification +from classification.models.classification_groups import ( + ClassificationGroup, + ClassificationGroups, + ClassificationGroupUtils, +) from classification.models.classification_ref import ClassificationRef from classification.models.clinical_context_models import ClinicalContext from classification.models.discordance_models import DiscordanceReport -from classification.models.discordance_models_utils import DiscordanceReportRowData, DiscordanceReportTableData +from classification.models.discordance_models_utils import ( + DiscordanceReportRowData, + DiscordanceReportTableData, +) from classification.models.evidence_key import EvidenceKey, EvidenceKeyMap from classification.models.evidence_mixin import VCDbRefDict from eventlog.models import ViewEvent @@ -183,7 +196,7 @@ def render_ekey(val, key: Optional[str] = None, value_if_none: Optional[str] = N elif (isinstance(val, list) or isinstance(val, set)) and len(val) == 0: if value_if_none is not None: return value_if_none - return mark_safe(f'-') + return mark_safe('-') return pretty_val @@ -228,7 +241,7 @@ def clinical_significance_values(vcm: ClassificationModification): "css_class": "cs cs-" + (value.lower() if value else "none") }] - if always_show_somatic or somatic_dict and somatic_dict.get("classification"): + if always_show_somatic or (somatic_dict and somatic_dict.get("classification")): somatic_key = EvidenceKeyMap.cached_key(SpecialEKeys.SOMATIC_CLINICAL_SIGNIFICANCE) value = somatic_dict.get("clinical_significance") label = somatic_key.pretty_value(value, value) or "No Data" diff --git a/classification/tests/models/test_classification_modifications.py b/classification/tests/models/test_classification_modifications.py index 6f9b27027..c52774e60 100644 --- a/classification/tests/models/test_classification_modifications.py +++ b/classification/tests/models/test_classification_modifications.py @@ -1,7 +1,11 @@ from django.test import TestCase, override_settings -from classification.enums import SubmissionSource, SpecialEKeys -from classification.models.classification import Classification, ClassificationModification, VCBlobKeys +from classification.enums import SpecialEKeys, SubmissionSource +from classification.models.classification import ( + Classification, + ClassificationModification, + VCBlobKeys, +) from classification.models.classification_ref import ClassificationRef from classification.models.classification_utils import ClassificationJsonParams from classification.tests.models.test_utils import ClassificationTestUtils diff --git a/classification/tests/models/test_classification_quirks.py b/classification/tests/models/test_classification_quirks.py index 3798ac9fd..fd87b537b 100644 --- a/classification/tests/models/test_classification_quirks.py +++ b/classification/tests/models/test_classification_quirks.py @@ -1,7 +1,7 @@ from django.test import TestCase -from classification.enums import SubmissionSource, CriteriaEvaluation, ValidationCode, SpecialEKeys -from classification.models import VCDataDict, EvidenceKeyMap +from classification.enums import CriteriaEvaluation, SpecialEKeys, SubmissionSource, ValidationCode +from classification.models import EvidenceKeyMap, VCDataDict from classification.models.classification import Classification from classification.models.classification_inserter import BulkClassificationInserter from classification.tests.models.test_utils import ClassificationTestUtils diff --git a/classification/tests/models/test_classification_utils.py b/classification/tests/models/test_classification_utils.py index ac21b7bcb..b4928c66a 100644 --- a/classification/tests/models/test_classification_utils.py +++ b/classification/tests/models/test_classification_utils.py @@ -1,7 +1,7 @@ from django.test import TestCase from classification.enums import ShareLevel -from classification.models import ValidationMerger, EvidenceMixin, Classification +from classification.models import Classification, EvidenceMixin, ValidationMerger class ClassificationTestCaseUtils(TestCase): diff --git a/classification/tests/models/test_utils.py b/classification/tests/models/test_utils.py index 2e1d6d181..e00376fd6 100644 --- a/classification/tests/models/test_utils.py +++ b/classification/tests/models/test_utils.py @@ -1,6 +1,6 @@ from django.contrib.auth.models import User -from snpdb.models import Lab, Organization, Country +from snpdb.models import Country, Lab, Organization class ClassificationTestUtils: diff --git a/classification/tests/utils/test_clinvar_export.py b/classification/tests/utils/test_clinvar_export.py index c2d61fe6a..20067efc7 100644 --- a/classification/tests/utils/test_clinvar_export.py +++ b/classification/tests/utils/test_clinvar_export.py @@ -3,17 +3,28 @@ from django.test import TestCase, override_settings -from classification.enums import SpecialEKeys, SubmissionSource, ShareLevel -from classification.models import Classification, ClinVarExport, ClinVarExportBatch, ClinVarExportStatus, \ - ClinVarExportRequestType, ClinVarExportRequest, ClinVarExportBatchStatus, ImportedAlleleInfo -from classification.models.classification_variant_info_models import ResolvedVariantInfo, ImportedAlleleInfoValidation +from classification.enums import ShareLevel, SpecialEKeys, SubmissionSource +from classification.models import ( + Classification, + ClinVarExport, + ClinVarExportBatch, + ClinVarExportBatchStatus, + ClinVarExportRequest, + ClinVarExportRequestType, + ClinVarExportStatus, + ImportedAlleleInfo, +) +from classification.models.classification_variant_info_models import ( + ImportedAlleleInfoValidation, + ResolvedVariantInfo, +) from classification.models.clinvar_export_prepare import ClinvarExportPrepare -from classification.models.clinvar_export_sync import clinvar_export_sync, ClinVarResponseOutcome +from classification.models.clinvar_export_sync import ClinVarResponseOutcome, clinvar_export_sync from classification.tests.models.test_utils import ClassificationTestUtils from library.guardian_utils import admin_bot from library.utils import JsonObjType -from snpdb.models import GenomeBuild, ClinVarKey, GenomeBuildPatchVersion -from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant, create_mock_allele +from snpdb.models import ClinVarKey, GenomeBuild, GenomeBuildPatchVersion +from snpdb.tests.utils.vcf_testing_utils import create_mock_allele, slowly_create_test_variant def mock_send_data( diff --git a/classification/tests/utils/test_regexes.py b/classification/tests/utils/test_regexes.py index ee256a958..522e792e6 100644 --- a/classification/tests/utils/test_regexes.py +++ b/classification/tests/utils/test_regexes.py @@ -1,6 +1,6 @@ from django.test.testcases import TestCase -from annotation.regexes import db_ref_regexes, DbRegexes, db_citation_regexes +from annotation.regexes import DbRegexes, db_citation_regexes, db_ref_regexes class RegexTests(TestCase): diff --git a/classification/tests/utils/test_resolved_conditions.py b/classification/tests/utils/test_resolved_conditions.py index 2d0914974..10c79f823 100644 --- a/classification/tests/utils/test_resolved_conditions.py +++ b/classification/tests/utils/test_resolved_conditions.py @@ -4,7 +4,7 @@ from classification.models import ConditionResolved, MultiCondition from classification.tests.utils.data_utils import ConditionMock -from ontology.models import OntologyTerm, OntologySnake, OntologyTermRelation +from ontology.models import OntologySnake, OntologyTerm, OntologyTermRelation class ResolvedConditionTest(TestCase): diff --git a/classification/tests/utils/test_urls.py b/classification/tests/utils/test_urls.py index 9e1474fe8..7da80e33f 100644 --- a/classification/tests/utils/test_urls.py +++ b/classification/tests/utils/test_urls.py @@ -2,13 +2,24 @@ from django.contrib.auth.models import User -from annotation.fake_annotation import get_fake_annotation_version, create_fake_variants -from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import \ - create_classification_for_sample_and_variant_objects +from annotation.fake_annotation import create_fake_variants, get_fake_annotation_version +from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import ( + create_classification_for_sample_and_variant_objects, +) from classification.models import EvidenceKey -from library.django_utils.unittest_utils import prevent_request_warnings, URLTestCase -from snpdb.models import GenomeBuild, Variant, ClinGenAllele, Allele, VariantAllele, AlleleOrigin, Lab, Organization, \ - Country, UserSettings +from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings +from snpdb.models import ( + Allele, + AlleleOrigin, + ClinGenAllele, + Country, + GenomeBuild, + Lab, + Organization, + UserSettings, + Variant, + VariantAllele, +) class Test(URLTestCase): diff --git a/classification/tests/views/test_classification_view.py b/classification/tests/views/test_classification_view.py index 244f86d91..ed5f41ce5 100644 --- a/classification/tests/views/test_classification_view.py +++ b/classification/tests/views/test_classification_view.py @@ -2,7 +2,7 @@ from unittest import skip from deepdiff import DeepDiff -from django.test import TestCase, RequestFactory, override_settings +from django.test import RequestFactory, TestCase, override_settings from classification.enums import EvidenceKeyValueType, SubmissionSource from classification.models import Classification, EvidenceKey diff --git a/classification/urls.py b/classification/urls.py index d6a2ec1d8..4ad34efb4 100644 --- a/classification/urls.py +++ b/classification/urls.py @@ -1,41 +1,86 @@ from rest_framework import routers -from classification.views import clinvar_export_view, search_view_metrics, classification_candidate_search_view -from classification.views import views, classification_dashboard_view, \ - classification_export_view, views_autocomplete, \ - classification_accumulation_graph +from classification.views import ( + classification_accumulation_graph, + classification_candidate_search_view, + classification_dashboard_view, + classification_export_view, + clinvar_export_view, + search_view_metrics, + views, + views_autocomplete, +) from classification.views.allele_grouping_datatables import AlleleGroupingColumns from classification.views.classification_dashboard_view import issues_download from classification.views.classification_datatables import ClassificationColumns -from classification.views.classification_email_view import summary_email_preview_html, \ - summary_email_preview_text +from classification.views.classification_email_view import ( + summary_email_preview_html, + summary_email_preview_text, +) from classification.views.classification_export_view import ClassificationApiExportView from classification.views.classification_grouping_datatables import ClassificationGroupingColumns -from classification.views.classification_overlaps_view import view_overlaps, post_clinical_context, \ - view_clinical_context, view_overlaps_detail -from classification.views.classification_overlaps_vus_view import view_overlaps_vus, view_overlaps_vus_detail -from classification.views.classification_view import ClassificationView, LabGeneClassificationCountsView -from classification.views.classification_view_metrics import view_classification_metrics, \ - view_page_metrics_detail +from classification.views.classification_overlaps_view import ( + post_clinical_context, + view_clinical_context, + view_overlaps, + view_overlaps_detail, +) +from classification.views.classification_overlaps_vus_view import ( + view_overlaps_vus, + view_overlaps_vus_detail, +) +from classification.views.classification_view import ( + ClassificationView, + LabGeneClassificationCountsView, +) +from classification.views.classification_view_metrics import ( + view_classification_metrics, + view_page_metrics_detail, +) from classification.views.clinvar_export_view import ClinVarMatchView, clinvar_match_detail -from classification.views.condition_match_test_view import condition_match_test_view, \ - condition_match_test_download_view, condition_obsoletes_view -from classification.views.condition_matching_view import condition_matching_view, condition_matchings_view, \ - ConditionTextColumns, ConditionTextMatchingAPI +from classification.views.condition_match_test_view import ( + condition_match_test_download_view, + condition_match_test_view, + condition_obsoletes_view, +) +from classification.views.condition_matching_view import ( + ConditionTextColumns, + ConditionTextMatchingAPI, + condition_matching_view, + condition_matchings_view, +) from classification.views.discordance_report_triage_view import DiscordanceReportTriageView -from classification.views.discordance_report_views import discordance_report_view, export_discordance_report, \ - discordance_reports_view, discordance_reports_history_detail, discordance_reports_active_detail, \ - discordance_report_review, action_discordance_report_review, discordance_reports_download +from classification.views.discordance_report_views import ( + action_discordance_report_review, + discordance_report_review, + discordance_report_view, + discordance_reports_active_detail, + discordance_reports_download, + discordance_reports_history_detail, + discordance_reports_view, + export_discordance_report, +) from classification.views.evidence_keys_view import EvidenceKeysView -from classification.views.exports.classification_export_formatter_redcap import redcap_data_dictionary -from classification.views.imported_allele_info_view import view_imported_allele_info, ImportedAlleleInfoColumns, \ - view_imported_allele_info_detail, download_allele_info -from classification.views.views import classification_import_tool, AutopopulateView +from classification.views.exports.classification_export_formatter_redcap import ( + redcap_data_dictionary, +) +from classification.views.imported_allele_info_view import ( + ImportedAlleleInfoColumns, + download_allele_info, + view_imported_allele_info, + view_imported_allele_info_detail, +) +from classification.views.views import AutopopulateView, classification_import_tool from classification.views.views_hgvs_resolution_tool import hgvs_resolution_tool -from classification.views.views_uploaded_classifications_unmapped import UploadedClassificationsUnmappedView, \ - UploadedClassificationsUnmappedColumns, download_classification_unmapped_file, \ - view_uploaded_classification_unmapped, view_uploaded_classification_unmapped_detail, \ - view_uploaded_classification_unmapped_validation_detail, upload_classification_unmapped_download_validation +from classification.views.views_uploaded_classifications_unmapped import ( + UploadedClassificationsUnmappedColumns, + UploadedClassificationsUnmappedView, + download_classification_unmapped_file, + upload_classification_unmapped_download_validation, + view_uploaded_classification_unmapped, + view_uploaded_classification_unmapped_detail, + view_uploaded_classification_unmapped_validation_detail, +) from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/classification/utils/clinvar_matcher.py b/classification/utils/clinvar_matcher.py index a052775ed..1e6cdb927 100644 --- a/classification/utils/clinvar_matcher.py +++ b/classification/utils/clinvar_matcher.py @@ -13,20 +13,26 @@ import json import re from collections import defaultdict +from collections.abc import Iterator from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import Optional, Iterator +from typing import Optional -from annotation.models import ClinVar, AnnotationVersion +from annotation.models import AnnotationVersion, ClinVar from classification.enums import SpecialEKeys -from classification.models import Classification, ClinVarExport, ClinVarAllele, EvidenceKeyMap, \ - ConditionResolved +from classification.models import ( + Classification, + ClinVarAllele, + ClinVarExport, + ConditionResolved, + EvidenceKeyMap, +) from genes.hgvs import CHGVS, HGVSException from library.guardian_utils import admin_bot from library.log_utils import report_exc_info -from ontology.models import OntologyTerm, OntologySnake, OntologyTermRelation -from snpdb.models import GenomeBuild, Variant, Allele, ClinVarKey +from ontology.models import OntologySnake, OntologyTerm, OntologyTermRelation +from snpdb.models import Allele, ClinVarKey, GenomeBuild, Variant from snpdb.search import SearchInput from snpdb.signals.variant_search import search_hgvs diff --git a/classification/variant_card.py b/classification/variant_card.py index 6c279e213..3b1780f86 100644 --- a/classification/variant_card.py +++ b/classification/variant_card.py @@ -4,13 +4,21 @@ from django.contrib.auth.models import User from django.db.models import Q, QuerySet -from annotation.manual_variant_entry import check_can_create_variants, CreateManualVariantForbidden +from annotation.manual_variant_entry import CreateManualVariantForbidden, check_can_create_variants from annotation.models import VariantAnnotation from annotation.templatetags.clinvar_tags import ClinVarDetails from classification.models import Classification, ImportedAlleleInfo from snpdb.liftover import allele_can_attempt_liftover -from snpdb.models import Allele, GenomeBuild, VariantAllele, \ - Variant, AlleleOrigin, AlleleMergeLog, LiftoverRun, AlleleLiftover +from snpdb.models import ( + Allele, + AlleleLiftover, + AlleleMergeLog, + AlleleOrigin, + GenomeBuild, + LiftoverRun, + Variant, + VariantAllele, +) from snpdb.variant_links import variant_link_info @@ -118,11 +126,9 @@ def clinvar_data(self) -> ClinVarDetails: @cached_property def imported_allele_infos(self): - return list( - sorted(ImportedAlleleInfo.objects.filter( + return sorted(ImportedAlleleInfo.objects.filter( allele=self.allele ).all()) - ) @property def imported_allele_info_label(self): diff --git a/classification/views/allele_grouping_datatables.py b/classification/views/allele_grouping_datatables.py index 61bef3d50..2343a6375 100644 --- a/classification/views/allele_grouping_datatables.py +++ b/classification/views/allele_grouping_datatables.py @@ -1,17 +1,22 @@ from functools import cached_property from typing import Optional -from django.db.models import QuerySet, OuterRef, Subquery +from django.db.models import OuterRef, QuerySet, Subquery from django.http import HttpRequest from classification.enums import AlleleOriginBucket -from classification.models import AlleleOriginGrouping, ClassificationGrouping, OverlapStatus, AlleleGrouping +from classification.models import ( + AlleleGrouping, + AlleleOriginGrouping, + ClassificationGrouping, + OverlapStatus, +) from genes.hgvs import CHGVS from library.cache import timed_cache from library.utils import JsonDataType from snpdb.lab_picker import LabPickerData -from snpdb.models import GenomeBuild, UserSettings, Lab -from snpdb.views.datatable_view import DatatableConfig, RichColumn, CellData, SortOrder +from snpdb.models import GenomeBuild, Lab, UserSettings +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder @timed_cache(size_limit=1, ttl=60) @@ -61,7 +66,7 @@ def render_labs(self, row: CellData) -> set[Lab]: for ag in ag.allele_origin_dict.values(): for cg in ag.classificationgrouping_set.all(): labs.add(cg.lab) - labs_list = list(sorted(labs)) + labs_list = sorted(labs) return "".join("
" + str(lab) +"
" for lab in labs_list) def c_hgvs_for(self, cg: ClassificationGrouping) -> CHGVS: @@ -82,7 +87,7 @@ def render_allele(self, row: CellData) -> JsonDataType: allele_group = _allele_group(row.get("allele")) # FIXME cache this cgs = ClassificationGrouping.objects.filter(allele_origin_grouping__allele_grouping=allele_group.pk, dirty=False) - all_chgvs = list(sorted({self.c_hgvs_for(cg) for cg in cgs})) + all_chgvs = sorted({self.c_hgvs_for(cg) for cg in cgs}) c_hgvs_json: JsonDataType if all_chgvs: c_hgvs_json = all_chgvs[0].to_json() diff --git a/classification/views/classification_accumulation_graph.py b/classification/views/classification_accumulation_graph.py index 3095a66da..0490c01a2 100644 --- a/classification/views/classification_accumulation_graph.py +++ b/classification/views/classification_accumulation_graph.py @@ -1,17 +1,23 @@ from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta -from enum import IntEnum, Enum +from enum import Enum, IntEnum from functools import total_ordering -from typing import Any, Optional, Union, Iterable +from typing import Any, Optional, Union import pandas as pd from django.http import StreamingHttpResponse -from classification.enums import ShareLevel, AlleleOriginBucket -from classification.models import classification_flag_types, Classification, ClassificationModification, EvidenceKeyMap +from classification.enums import AlleleOriginBucket, ShareLevel +from classification.models import ( + Classification, + ClassificationModification, + EvidenceKeyMap, + classification_flag_types, +) from flags.models import FlagComment -from library.utils import delimited_row, IterableStitcher, IterableTransformer +from library.utils import IterableStitcher, IterableTransformer, delimited_row from snpdb.models import Lab diff --git a/classification/views/classification_candidate_search_view.py b/classification/views/classification_candidate_search_view.py index 49f1248e8..b881f4fd8 100644 --- a/classification/views/classification_candidate_search_view.py +++ b/classification/views/classification_candidate_search_view.py @@ -1,19 +1,31 @@ from crispy_forms.helper import FormHelper -from crispy_forms.layout import Layout, Field +from crispy_forms.layout import Field, Layout from django.conf import settings from django.http.response import HttpResponse -from django.shortcuts import render, get_object_or_404 +from django.shortcuts import get_object_or_404, render from analysis.forms import SampleCandidatesSearchForm from analysis.models import CandidateSearchRun, CandidateSearchType -from analysis.views.views_candidate_search import AbstractCandidateSearchView, AbstractNewCandidateSearchView -from classification.forms import ClassificationAlleleOriginForm, ClinicalSignificanceForm, \ - ClassificationEvidenceUpdateForm +from analysis.views.views_candidate_search import ( + AbstractCandidateSearchView, + AbstractNewCandidateSearchView, +) +from classification.forms import ( + ClassificationAlleleOriginForm, + ClassificationEvidenceUpdateForm, + ClinicalSignificanceForm, +) from genes.forms import GeneSymbolForm from genes.models import SampleGeneList from ontology.forms import PhenotypeMultipleSelectForm -from snpdb.forms import UserSelectForm, LabSelectForm, LabMultiSelectForm, ProjectChoiceForm, VCFChoiceForm, \ - SampleMultiForm +from snpdb.forms import ( + LabMultiSelectForm, + LabSelectForm, + ProjectChoiceForm, + SampleMultiForm, + UserSelectForm, + VCFChoiceForm, +) from snpdb.models import Lab, Sample from snpdb.user_settings_manager import UserSettingsManager diff --git a/classification/views/classification_dashboard_view.py b/classification/views/classification_dashboard_view.py index 31bb8b5ab..bdf3d7dc6 100644 --- a/classification/views/classification_dashboard_view.py +++ b/classification/views/classification_dashboard_view.py @@ -11,14 +11,25 @@ from termsandconditions.decorators import terms_required from classification.enums import ShareLevel -from classification.models import classification_flag_types, ClinVarExport, DiscordanceReportClassification, \ - DiscordanceReport, ConditionText, ConditionTextMatch +from classification.models import ( + ClinVarExport, + ConditionText, + ConditionTextMatch, + DiscordanceReport, + DiscordanceReportClassification, + classification_flag_types, +) from classification.models.classification import Classification from classification.models.clinvar_export_sync import clinvar_export_sync -from classification.models.discordance_models_utils import DiscordanceReportRowData, DiscordanceReportTableData, \ - DiscordanceReportCategories -from classification.views.classification_accumulation_graph import \ - AccumulationReportMode, get_accumulation_graph_data +from classification.models.discordance_models_utils import ( + DiscordanceReportCategories, + DiscordanceReportRowData, + DiscordanceReportTableData, +) +from classification.views.classification_accumulation_graph import ( + AccumulationReportMode, + get_accumulation_graph_data, +) from classification.views.exports import ClassificationExportFormatterFlags from classification.views.exports.classification_export_filter import ClassificationFilter from flags.models import FlagCollection diff --git a/classification/views/classification_datatables.py b/classification/views/classification_datatables.py index bcdc8e802..8b51daa1d 100644 --- a/classification/views/classification_datatables.py +++ b/classification/views/classification_datatables.py @@ -1,17 +1,27 @@ import logging import operator from functools import cached_property, reduce -from typing import Dict, Any, List, Optional +from typing import Any, Optional from django.conf import settings -from django.db.models import Q, When, Case, TextField, Value, IntegerField, QuerySet +from django.db.models import Case, IntegerField, Q, QuerySet, TextField, Value, When from django.db.models.fields.json import KeyTextTransform, KeyTransform -from django.db.models.functions import Lower, Cast +from django.db.models.functions import Cast, Lower from django.http import HttpRequest -from classification.enums import SpecialEKeys, EvidenceCategory, ShareLevel, AlleleOriginBucket, ClinicalSignificance -from classification.models import ClassificationModification, EvidenceKeyMap, \ - ImportedAlleleInfo, DiscordanceReport +from classification.enums import ( + AlleleOriginBucket, + ClinicalSignificance, + EvidenceCategory, + ShareLevel, + SpecialEKeys, +) +from classification.models import ( + ClassificationModification, + DiscordanceReport, + EvidenceKeyMap, + ImportedAlleleInfo, +) from classification.models.classification_utils import classification_gene_symbol_filter from flags.models import FlagCollection, FlagStatus from genes.hgvs import CHGVS @@ -19,7 +29,7 @@ from library.utils import JsonDataType from ontology.models import OntologyTerm from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import UserSettings, GenomeBuild, Variant, Lab +from snpdb.models import GenomeBuild, Lab, UserSettings, Variant from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder ALLELE_GERMLINE_VALUES = ['germline', 'likely_germline'] @@ -37,17 +47,17 @@ class ClassificationColumns(DatatableConfig[ClassificationModification]): "pathogenic": ClinicalSignificance.PATHOGENIC, } - def render_somatic(self, row: Dict[str, Any]) -> JsonDataType: + def render_somatic(self, row: dict[str, Any]) -> JsonDataType: if row["classification__allele_origin_bucket"] != "G": return row["classification__summary__somatic"] - def render_classification(self, row: Dict[str, Any]) -> JsonDataType: + def render_classification(self, row: dict[str, Any]) -> JsonDataType: return { SpecialEKeys.CLINICAL_SIGNIFICANCE: row[f"published_evidence__{SpecialEKeys.CLINICAL_SIGNIFICANCE}__value"] } - def render_c_hgvs(self, row: Dict[str, Any]) -> JsonDataType: - def get_preferred_chgvs_json() -> Dict: + def render_c_hgvs(self, row: dict[str, Any]) -> JsonDataType: + def get_preferred_chgvs_json() -> dict: nonlocal row for index, genome_build in enumerate(self.genome_build_prefs): try: @@ -84,14 +94,14 @@ def get_preferred_chgvs_json() -> Dict: return response - def render_condition(self, row: Dict[str, Any]) -> JsonDataType: + def render_condition(self, row: dict[str, Any]) -> JsonDataType: if cr := row['classification__condition_resolution']: return cr else: return {"display_text": row['published_evidence__condition__value']} - def classification_id(self, row: Dict[str, Any]) -> JsonDataType: - matches: Optional[Dict[str, str]] = None + def classification_id(self, row: dict[str, Any]) -> JsonDataType: + matches: Optional[dict[str, str]] = None if id_filter := self.get_query_param("id_filter"): matches = {} id_keys = self.id_columns @@ -119,7 +129,7 @@ def classification_id(self, row: Dict[str, Any]) -> JsonDataType: } @cached_property - def genome_build_prefs(self) -> List[GenomeBuild]: + def genome_build_prefs(self) -> list[GenomeBuild]: return GenomeBuild.builds_with_annotation_priority(GenomeBuildManager.get_current_genome_build()) @cached_property @@ -127,7 +137,7 @@ def genome_build_preferred(self) -> GenomeBuild: return self.genome_build_prefs[0] def __init__(self, request: HttpRequest): - self.term_cache: Dict[str, OntologyTerm] = {} + self.term_cache: dict[str, OntologyTerm] = {} super().__init__(request) user_settings = UserSettings.get_for_user(self.user) @@ -141,7 +151,7 @@ def __init__(self, request: HttpRequest): except ValueError: pass if not (c_hgvs and genomic_sort): - possible_builds = ', '.join((str(gb) for gb in self.genome_build_prefs)) + possible_builds = ', '.join(str(gb) for gb in self.genome_build_prefs) raise ValueError(f"Couldn't find c_hgvs and genomic_sort in builds: {possible_builds}") self.rich_columns = [ @@ -232,7 +242,7 @@ def __init__(self, request: HttpRequest): key='classification__sample__name', name='sample_name', label='Sample', - client_renderer=f'VCTable.sample', + client_renderer='VCTable.sample', enabled=settings.CLASSIFICATION_GRID_SHOW_SAMPLE, orderable=True ), @@ -328,7 +338,7 @@ def get_initial_queryset(self) -> QuerySet[ClassificationModification]: return initial_qs @cached_property - def id_columns(self) -> List[str]: + def id_columns(self) -> list[str]: keys = EvidenceKeyMap.instance() return [e_key.key for e_key in keys.all_keys if '_id' in e_key.key and e_key.evidence_category in (EvidenceCategory.HEADER_PATIENT, EvidenceCategory.HEADER_TEST, EvidenceCategory.SIGN_OFF)] @@ -344,7 +354,7 @@ def get_clinical_significance_q(clinical_significances: dict) -> Q | None: q = Q(clinical_significance__in=clinical_significance_list) return q - def value_columns(self) -> List[str]: + def value_columns(self) -> list[str]: all_columns = super().value_columns() if self.get_query_param("id_filter"): id_keys = self.id_columns @@ -353,7 +363,7 @@ def value_columns(self) -> List[str]: return all_columns def filter_queryset(self, qs: QuerySet[ClassificationModification]) -> QuerySet[ClassificationModification]: - filters: List[Q] = [] + filters: list[Q] = [] if settings.CLASSIFICATION_GRID_SHOW_USERNAME: if user_id := self.get_query_param('user'): filters.append(Q(classification__user__pk=user_id)) @@ -373,7 +383,7 @@ def filter_queryset(self, qs: QuerySet[ClassificationModification]) -> QuerySet[ if id_filter := self.get_query_param("id_filter"): id_keys = self.id_columns - ids_contain_q_list: List[Q] = [] + ids_contain_q_list: list[Q] = [] for id_key in id_keys: ids_contain_q_list.append(Q(**{f'published_evidence__{id_key}__value__icontains': id_filter})) id_filter_q = reduce(operator.or_, ids_contain_q_list) diff --git a/classification/views/classification_email_view.py b/classification/views/classification_email_view.py index 561a80526..9cc591f48 100644 --- a/classification/views/classification_email_view.py +++ b/classification/views/classification_email_view.py @@ -13,14 +13,18 @@ from django.utils.timesince import timesince from classification.enums.discordance_enums import DiscordanceReportResolution -from classification.models import Classification, classification_flag_types, \ - DiscordanceReportClassification, DiscordanceReport +from classification.models import ( + Classification, + DiscordanceReport, + DiscordanceReportClassification, + classification_flag_types, +) from classification.models.discordance_models_utils import DiscordanceReportCategories from email_manager.models import EmailLog -from flags.models import FlagCollection, Flag +from flags.models import Flag, FlagCollection from library.log_utils import report_exc_info, report_message from snpdb.lab_picker import LabPickerData -from snpdb.models import Lab, UserSettings, GenomeBuild +from snpdb.models import GenomeBuild, Lab, UserSettings EmailOutput = collections.namedtuple('EmailOutput', 'subject html text') diff --git a/classification/views/classification_export_report.py b/classification/views/classification_export_report.py index 911d5aade..c35194d37 100644 --- a/classification/views/classification_export_report.py +++ b/classification/views/classification_export_report.py @@ -6,8 +6,7 @@ from django.template import engines from classification.models import ClassificationJsonParams, ClassificationReportTemplate -from classification.models.classification import ClassificationModification, \ - Classification +from classification.models.classification import Classification, ClassificationModification from classification.models.evidence_key import EvidenceKeyMap from snpdb.models import GenomeBuild @@ -87,4 +86,4 @@ def _get_unknown_evidence(content, row_data) -> list[str]: continue if evidence not in row_data: unknown_evidence.add(evidence) - return list(sorted(unknown_evidence)) + return sorted(unknown_evidence) diff --git a/classification/views/classification_export_utils.py b/classification/views/classification_export_utils.py index 09d652373..0efcef18e 100644 --- a/classification/views/classification_export_utils.py +++ b/classification/views/classification_export_utils.py @@ -1,7 +1,8 @@ +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import Iterable, Optional, Any, Mapping, Callable +from typing import Any, Optional from django.contrib.auth.models import User from django.db.models import Count @@ -9,7 +10,7 @@ from classification.enums import CriteriaEvaluation from classification.models.classification import ClassificationModification -from classification.models.evidence_key import EvidenceKeyMap, EvidenceKey +from classification.models.evidence_key import EvidenceKey, EvidenceKeyMap from genes.hgvs import CHGVS from library.cache import clear_cached_property @@ -33,7 +34,7 @@ def value_for(self, ekey: EvidenceKey, value, pretty: bool = False, cell_formatt value = ekey.pretty_value(value) else: if isinstance(value, list): - value = ', '.join((str(item) for item in value)) + value = ', '.join(str(item) for item in value) elif value is True: return 'TRUE' elif value is False: diff --git a/classification/views/classification_export_view.py b/classification/views/classification_export_view.py index 332012ac6..ddbccdb7a 100644 --- a/classification/views/classification_export_view.py +++ b/classification/views/classification_export_view.py @@ -8,7 +8,7 @@ from django.db.models import Q from django.http.request import HttpRequest from django.http.response import HttpResponse, HttpResponseBase -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.urls.base import reverse from django.utils import timezone from drf_spectacular.types import OpenApiTypes @@ -23,10 +23,14 @@ from classification.models.classification_ref import ClassificationRef from classification.views.classification_export_report import ClassificationReport from classification.views.exports import ClassificationExportFormatterCSV -from classification.views.exports.classification_export_filter import ClassificationFilter, \ - classification_export_user_strings_to_q +from classification.views.exports.classification_export_filter import ( + ClassificationFilter, + classification_export_user_strings_to_q, +) from classification.views.exports.classification_export_formatter_csv import FormatDetailsCSV -from classification.views.exports.classification_export_formatter_redcap import export_redcap_definition +from classification.views.exports.classification_export_formatter_redcap import ( + export_redcap_definition, +) from classification.views.exports.classification_export_view import serve_export from library.django_utils import get_url_from_view_path from library.log_utils import report_exc_info @@ -119,7 +123,7 @@ def _export_view_context(request: HttpRequest) -> dict: format_vcf ] - labs_for_user = list(sorted(Lab.valid_labs_qs(request.user, admin_check=True))) + labs_for_user = sorted(Lab.valid_labs_qs(request.user, admin_check=True)) downloadable_field_counts = len(EvidenceKeyMap.cached().vital()) all_field_counts = len(EvidenceKeyMap.cached().all_keys) @@ -310,7 +314,7 @@ def internal_lab_download(request): genome_build=genome_build, allele_origin_filter=allele_origin, min_share_level=share_level, - file_prefix=f"Internal_lab_report", + file_prefix="Internal_lab_report", include_sources=user_labs, extra_filter=extra_filter_qs ) diff --git a/classification/views/classification_grouping_datatables.py b/classification/views/classification_grouping_datatables.py index 1076bacc8..8ff5d6ad3 100644 --- a/classification/views/classification_grouping_datatables.py +++ b/classification/views/classification_grouping_datatables.py @@ -1,23 +1,32 @@ import operator from functools import cached_property, reduce -from typing import Dict, List, Optional +from typing import Optional from django.conf import settings -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.http import HttpRequest from more_itertools import first -from classification.enums import AlleleOriginBucket, EvidenceCategory, SpecialEKeys, ShareLevel -from classification.models import ClassificationGrouping, ImportedAlleleInfo, ClassificationGroupingSearchTerm, \ - ClassificationGroupingSearchTermType, EvidenceKeyMap, ClassificationModification, ClassificationGroupingEntry, \ - Classification, DiscordanceReport, DiscordanceReportClassification +from classification.enums import AlleleOriginBucket, EvidenceCategory, ShareLevel, SpecialEKeys +from classification.models import ( + Classification, + ClassificationGrouping, + ClassificationGroupingEntry, + ClassificationGroupingSearchTerm, + ClassificationGroupingSearchTermType, + ClassificationModification, + DiscordanceReport, + DiscordanceReportClassification, + EvidenceKeyMap, + ImportedAlleleInfo, +) from genes.hgvs import CHGVS from genes.models import GeneSymbol, TranscriptVersion from library.utils import JsonDataType -from ontology.models import OntologyTerm, OntologyTermRelation, OntologySnake +from ontology.models import OntologySnake, OntologyTerm, OntologyTermRelation from snpdb.genome_build_manager import GenomeBuildManager from snpdb.models import GenomeBuild, Variant -from snpdb.views.datatable_view import DatatableConfig, RichColumn, DC, SortOrder, CellData +from snpdb.views.datatable_view import DC, CellData, DatatableConfig, RichColumn, SortOrder class ClassificationGroupingColumns(DatatableConfig[ClassificationGrouping]): @@ -35,7 +44,7 @@ class ClassificationGroupingColumns(DatatableConfig[ClassificationGrouping]): def render_row_header(self, row: CellData) -> JsonDataType: - matches: Optional[Dict[str, str]] = None + matches: Optional[dict[str, str]] = None search: Optional[str] = None if settings.CLASSIFICATION_ID_FILTER: @@ -106,12 +115,12 @@ def _render_date(self, row: CellData) -> JsonDataType: } @cached_property - def genome_build_prefs(self) -> List[GenomeBuild]: + def genome_build_prefs(self) -> list[GenomeBuild]: builds = GenomeBuild.builds_with_annotation_priority(GenomeBuildManager.get_current_genome_build()) return [gb for gb in builds if gb in ImportedAlleleInfo.supported_genome_builds()] def render_c_hgvs(self, row: CellData) -> JsonDataType: - def get_preferred_chgvs_json() -> Dict: + def get_preferred_chgvs_json() -> dict: nonlocal row for index, genome_build in enumerate(self.genome_build_prefs): if c_hgvs_string := row.get(ImportedAlleleInfo.column_name_for_build(genome_build, "latest_allele_info")): @@ -154,7 +163,7 @@ def get_initial_queryset(self) -> QuerySet[DC]: page = self.get_query_param('page_id') - filters: List[Q] = [] + filters: list[Q] = [] # run the filters that are perma-applied on certain pages @@ -186,7 +195,7 @@ def filter_queryset(self, qs: QuerySet[ClassificationGrouping]) -> QuerySet[Clas # run the filters that are optionally applied - filters: List[Q] = [] + filters: list[Q] = [] if lab_id := self.get_query_param('lab'): lab_list = lab_id.split(",") filters.append(Q(lab_id__in=lab_list)) @@ -226,7 +235,7 @@ def discordance_report(self) -> Optional[DiscordanceReport]: return dr @cached_property - def id_columns(self) -> List[str]: + def id_columns(self) -> list[str]: keys = EvidenceKeyMap.instance() return [e_key.key for e_key in keys.all_keys if '_id' in e_key.key and e_key.evidence_category in ( @@ -248,7 +257,7 @@ def classification_filter_to_grouping(self, cm_q: Q) -> Q: ) def id_filter(self, text: str): - ids_contain_q_list: List[Q] = [] + ids_contain_q_list: list[Q] = [] for id_key in self.id_columns: ids_contain_q_list.append(Q(**{f'published_evidence__{id_key}__value__icontains': text})) id_filter_q = reduce(operator.or_, ids_contain_q_list) diff --git a/classification/views/classification_overlaps_view.py b/classification/views/classification_overlaps_view.py index 9171f51b6..c64854754 100644 --- a/classification/views/classification_overlaps_view.py +++ b/classification/views/classification_overlaps_view.py @@ -7,12 +7,15 @@ from django.db.models import Model from django.http.request import HttpRequest from django.http.response import HttpResponseBase -from django.shortcuts import render, redirect, get_object_or_404 +from django.shortcuts import get_object_or_404, redirect, render from classification.models import ClassificationRef from classification.models.allele_overlaps import OverlapsCalculator from classification.models.classification import Classification -from classification.models.clinical_context_models import ClinicalContext, ClinicalContextRecalcTrigger +from classification.models.clinical_context_models import ( + ClinicalContext, + ClinicalContextRecalcTrigger, +) from classification.models.flag_types import classification_flag_types from library.django_utils import require_superuser from snpdb.lab_picker import LabPickerData diff --git a/classification/views/classification_view.py b/classification/views/classification_view.py index f15bd394c..3a7e88624 100644 --- a/classification/views/classification_view.py +++ b/classification/views/classification_view.py @@ -6,15 +6,15 @@ from django.core.exceptions import PermissionDenied from django.utils.decorators import method_decorator from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.generics import get_object_or_404 from rest_framework.response import Response -from rest_framework.status import HTTP_200_OK, HTTP_500_INTERNAL_SERVER_ERROR, HTTP_400_BAD_REQUEST +from rest_framework.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_500_INTERNAL_SERVER_ERROR from rest_framework.views import APIView from classification.classification_stats import get_lab_gene_counts from classification.enums import ClinicalSignificance -from classification.models import ClassificationRef, ClassificationJsonParams +from classification.models import ClassificationJsonParams, ClassificationRef from classification.models.classification_import_run import ClassificationImportRun from classification.models.classification_inserter import BulkClassificationInserter from library.utils import empty_to_none, force_json diff --git a/classification/views/classification_view_metrics.py b/classification/views/classification_view_metrics.py index 25db3f49f..8ad76bc9f 100644 --- a/classification/views/classification_view_metrics.py +++ b/classification/views/classification_view_metrics.py @@ -1,12 +1,13 @@ from collections import defaultdict +from collections.abc import Callable from dataclasses import dataclass -from datetime import timedelta, datetime +from datetime import datetime, timedelta from functools import cached_property -from typing import Any, Callable, TypeVar, Generic, Optional, Type +from typing import Any, Generic, Optional, TypeVar from django.contrib.auth.models import User from django.core.exceptions import PermissionDenied -from django.db.models import Model, Count, Q, QuerySet +from django.db.models import Count, Model, Q, QuerySet from django.http import HttpRequest from django.http.response import HttpResponseBase from django.shortcuts import render @@ -98,7 +99,7 @@ def count_field(self, field_name: str, resolver: Optional[Callable]) -> list[Cou return sorted((Counted(pk, count, resolver) for pk, count in id_to_count.items()), reverse=True) @staticmethod - def resolver_for_model(model: Type[Model]): + def resolver_for_model(model: type[Model]): def resolver(pk: Any): if pk and pk != "undefined": if first := model.objects.filter(pk=pk).first(): diff --git a/classification/views/clinvar_export_view.py b/classification/views/clinvar_export_view.py index 3ac57b328..65d9685f6 100644 --- a/classification/views/clinvar_export_view.py +++ b/classification/views/clinvar_export_view.py @@ -5,33 +5,55 @@ from collections import defaultdict from dataclasses import dataclass from functools import cached_property -from typing import Dict, Any, Optional, List, Set +from typing import Any, Optional from django.contrib import messages -from django.db.models import QuerySet, When, Value, Case, IntegerField, Count, Q, TextField +from django.db.models import Case, Count, IntegerField, Q, QuerySet, TextField, Value, When from django.db.models.functions import Cast -from django.http import HttpResponse, StreamingHttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse, StreamingHttpResponse from django.http.response import HttpResponseBase -from django.shortcuts import render, redirect, get_object_or_404 +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.views import View from django.views.decorators.http import require_POST from classification.enums import SpecialEKeys -from classification.models import ClinVarExport, ClinVarExportBatch, ClinVarExportBatchStatus, \ - EvidenceKeyMap, ClinVarExportStatus, ClinVarExportSubmission +from classification.models import ( + ClinVarExport, + ClinVarExportBatch, + ClinVarExportBatchStatus, + ClinVarExportStatus, + ClinVarExportSubmission, + EvidenceKeyMap, +) from classification.models.clinvar_export_prepare import ClinvarExportPrepare -from classification.utils.clinvar_matcher import ClinVarLegacyRow, ClinVarLegacyMatches, ClinVarLegacyExportMatchType +from classification.utils.clinvar_matcher import ( + ClinVarLegacyExportMatchType, + ClinVarLegacyMatches, + ClinVarLegacyRow, +) from classification.views.classification_dashboard_view import ClassificationDashboard from genes.hgvs import CHGVS from library.cache import timed_cache -from library.django_utils import add_save_message, get_url_from_view_path, require_superuser, RequireSuperUserView -from library.utils import html_to_text, export_column, ExportRow, local_date_string, ExportDataType, JsonDataType +from library.django_utils import ( + RequireSuperUserView, + add_save_message, + get_url_from_view_path, + require_superuser, +) +from library.utils import ( + ExportDataType, + ExportRow, + JsonDataType, + export_column, + html_to_text, + local_date_string, +) from library.utils.django_utils import render_ajax_view -from ontology.models import OntologyTerm, AncestorCalculator, OntologyTermRelation +from ontology.models import AncestorCalculator, OntologyTerm, OntologyTermRelation from snpdb.lab_picker import LabPickerData -from snpdb.models import ClinVarKey, Lab, Allele, GenomeBuild -from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder, CellData +from snpdb.models import Allele, ClinVarKey, GenomeBuild, Lab +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder @timed_cache(size_limit=30, ttl=60) @@ -86,7 +108,7 @@ def clinvar_export_batch_detail(request, clinvar_export_batch_id: int): }) -def _export_id_to_batch_ids(qs: QuerySet[ClinVarExport]) -> Dict[int, List[int]]: +def _export_id_to_batch_ids(qs: QuerySet[ClinVarExport]) -> dict[int, list[int]]: export_to_batches = defaultdict(list) for export_id, batch_id in ClinVarExportSubmission.objects.filter( clinvar_export__in=qs.values_list("id", flat=True)).order_by( @@ -97,7 +119,7 @@ def _export_id_to_batch_ids(qs: QuerySet[ClinVarExport]) -> Dict[int, List[int]] class ClinVarExportColumns(DatatableConfig[ClinVarExport]): - def render_allele(self, row: Dict[str, Any]) -> str: + def render_allele(self, row: dict[str, Any]) -> str: allele = allele_for(row["clinvar_allele__allele"]) return f"{allele:CA}" @@ -148,7 +170,7 @@ def render_c_hgvs(self, row: CellData) -> JsonDataType: else: c_hgvs_str = row["classification_based_on__classification__allele_info__grch38__c_hgvs"] - data: Dict[str, Any] + data: dict[str, Any] c_hgvs = CHGVS(c_hgvs_str) if c_hgvs.raw_c != c_hgvs.full_c_hgvs: data = { @@ -267,7 +289,7 @@ def clinvar_export_review(request: HttpRequest, clinvar_export_id: int) -> HttpR clinvar_allele_id=clinvar_export.clinvar_allele_id ).exclude(pk=clinvar_export_id).all()) if clinvar_exports_for_allele: - all_condition_terms: Set[OntologyTerm] = set() + all_condition_terms: set[OntologyTerm] = set() clinvar_exports_for_allele.append(clinvar_export) for clinvar_allele in clinvar_exports_for_allele: for term in clinvar_allele.condition_resolved.terms: @@ -300,14 +322,14 @@ def clinvar_export_history(request: HttpRequest, clinvar_export_id: int) -> Http @dataclass(frozen=True) class ClinVarExportSummaryData: clinvar_export: ClinVarExport - batch_ids: Optional[List[int]] + batch_ids: Optional[list[int]] @dataclass(frozen=True) class ClinVarExportSummary(ExportRow): clinvar_export: ClinVarExport - batch_ids: Optional[List[int]] + batch_ids: Optional[list[int]] has_duplicates: bool = False @property @@ -451,7 +473,7 @@ def _common_condition(self): all_conditions = list() for other in all_for_allele: all_conditions += other.condition_resolved.terms - all_conditions = list(sorted(set(all_conditions))) + all_conditions = sorted(set(all_conditions)) all_conditions_str = ", ".join(f"{term.pk} {term.name}" for term in all_conditions) try: if common_condition := AncestorCalculator.common_ancestor(all_conditions): @@ -462,7 +484,7 @@ def _common_condition(self): @export_column("Messages") def _messages(self): - all_messages: List[str] = [] + all_messages: list[str] = [] if clinvar_error := self.clinvar_export.last_submission_error: all_messages.append(f"(CLINVAR ERROR) {clinvar_error}") @@ -589,7 +611,7 @@ def clinvar_export_create_batch(request: HttpRequest, clinvar_key_id: str) -> Ht missing_str = ", ".join(str(m) for m in missing) messages.add_message(request, level=messages.ERROR, message=f"Could not find some of the IDs for this ClinVarKey - not creating a batch. Missing IDs : {missing_str}") elif id_count == 0: - messages.add_message(request, level=messages.ERROR, message=f"No IDs provided") + messages.add_message(request, level=messages.ERROR, message="No IDs provided") else: batches = ClinVarExportBatch.create_batches(clinvar_export_qs, force_update=True) for batch in batches: @@ -647,7 +669,7 @@ def post(self, request, **kwargs): clinvar_key: ClinVarKey = get_object_or_404(ClinVarKey, pk=clinvar_key_id) clinvar_key.check_user_can_access(request.user) - clinvar_legacy_rows: List[ClinVarLegacyRow] = [] + clinvar_legacy_rows: list[ClinVarLegacyRow] = [] try: file_obj = io.StringIO(request.FILES.get('file').read().decode("utf-8")) clinvar_legacy_rows = list(ClinVarLegacyRow.load_file(file_obj, clinvar_key)) @@ -669,7 +691,7 @@ def clinvar_match_detail(request, clinvar_key_id: str): data_str = request.GET.get('data_str') legacy_row = ClinVarLegacyRow.from_data_str(clinvar_key, data_str) - matches: List[ClinVarLegacyMatches] = legacy_row.find_variant_grid_allele() if data_str else [] + matches: list[ClinVarLegacyMatches] = legacy_row.find_variant_grid_allele() if data_str else [] # see if we match to a ClinVarExport, but not using SCV # implying that the SCV might need to be copied over diff --git a/classification/views/condition_match_test_view.py b/classification/views/condition_match_test_view.py index 6f12ac5f4..0e43ea702 100644 --- a/classification/views/condition_match_test_view.py +++ b/classification/views/condition_match_test_view.py @@ -6,19 +6,31 @@ from django.contrib import messages from django.db.models import Q from django.db.models.functions import Length -from django.http import StreamingHttpResponse, HttpRequest +from django.http import HttpRequest, StreamingHttpResponse from django.http.response import HttpResponseBase from django.shortcuts import render -from classification.models import ConditionText, top_level_suggestion, condition_matching_suggestions, \ - ConditionMatchingSuggestion, ConditionTextMatch +from classification.models import ( + ConditionMatchingSuggestion, + ConditionText, + ConditionTextMatch, + condition_matching_suggestions, + top_level_suggestion, +) from genes.models import GeneSymbol from library.django_utils import require_superuser from library.log_utils import report_exc_info from library.utils import delimited_row -from ontology.models import OntologySnake, OntologyVersion, OntologyTermStatus, OntologyImportSource, \ - OntologyTermRelation, OntologyTerm, OntologyTermDescendant, \ - ONTOLOGY_RELATIONSHIP_STANDARD_QUALITY_FILTER +from ontology.models import ( + ONTOLOGY_RELATIONSHIP_STANDARD_QUALITY_FILTER, + OntologyImportSource, + OntologySnake, + OntologyTerm, + OntologyTermDescendant, + OntologyTermRelation, + OntologyTermStatus, + OntologyVersion, +) from ontology.ontology_matching import OntologyMatching, SearchText, normalize_condition_text diff --git a/classification/views/condition_matching_view.py b/classification/views/condition_matching_view.py index 8e25bf779..6d322510a 100644 --- a/classification/views/condition_matching_view.py +++ b/classification/views/condition_matching_view.py @@ -3,15 +3,21 @@ from django.contrib.auth.models import User from django.db.models import QuerySet -from django.shortcuts import render, get_object_or_404 +from django.shortcuts import get_object_or_404, render from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema from guardian.shortcuts import get_objects_for_user from rest_framework.response import Response from rest_framework.views import APIView -from classification.models import ConditionTextMatch, ConditionText, update_condition_text_match_counts, MultiCondition, \ - ConditionMatchingSuggestion, condition_matching_suggestions +from classification.models import ( + ConditionMatchingSuggestion, + ConditionText, + ConditionTextMatch, + MultiCondition, + condition_matching_suggestions, + update_condition_text_match_counts, +) from classification.views.classification_dashboard_view import ClassificationDashboard from library.utils import empty_to_none from ontology.models import OntologyTerm diff --git a/classification/views/discordance_report_views.py b/classification/views/discordance_report_views.py index fb7c18882..64760430c 100644 --- a/classification/views/discordance_report_views.py +++ b/classification/views/discordance_report_views.py @@ -10,15 +10,29 @@ from django.dispatch import receiver from django.http import HttpRequest from django.http.response import HttpResponse, HttpResponseBase -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.urls import reverse from classification.enums import SpecialEKeys -from classification.enums.discordance_enums import ContinuedDiscordanceReason, DiscordanceReportResolution -from classification.models import ClassificationModification, DiscordanceReportClassification, ClinicalContext, \ - EvidenceKeyMap, classification_flag_types, discordance_change_signal, \ - ClassificationFlagTypes, ClinicalContextChangeData, ClinicalContextRecalcTrigger -from classification.models.classification_groups import ClassificationGroupUtils, ClassificationGroups +from classification.enums.discordance_enums import ( + ContinuedDiscordanceReason, + DiscordanceReportResolution, +) +from classification.models import ( + ClassificationFlagTypes, + ClassificationModification, + ClinicalContext, + ClinicalContextChangeData, + ClinicalContextRecalcTrigger, + DiscordanceReportClassification, + EvidenceKeyMap, + classification_flag_types, + discordance_change_signal, +) +from classification.models.classification_groups import ( + ClassificationGroups, + ClassificationGroupUtils, +) from classification.models.discordance_models import DiscordanceReport from classification.models.discordance_models_utils import DiscordanceReportRowData from classification.models.evidence_key import EvidenceKeyOption @@ -29,11 +43,11 @@ from classification.views.exports.classification_export_formatter_csv import FormatDetailsCSV from genes.hgvs import CHGVS from library.log_utils import log_admin_change -from library.preview_request import preview_extra_signal, PreviewKeyValue +from library.preview_request import PreviewKeyValue, preview_extra_signal from review.models import Review from snpdb.genome_build_manager import GenomeBuildManager from snpdb.lab_picker import LabPickerData -from snpdb.models import Lab, GenomeBuild, Allele +from snpdb.models import Allele, GenomeBuild, Lab from uicore.views.ajax_form_view import LazyRender diff --git a/classification/views/exports/classification_export_decorator.py b/classification/views/exports/classification_export_decorator.py index f71ff0dea..95b945964 100644 --- a/classification/views/exports/classification_export_decorator.py +++ b/classification/views/exports/classification_export_decorator.py @@ -1,8 +1,10 @@ -from typing import Callable +from collections.abc import Callable from django.http import HttpRequest -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) ExportFormatterFactory = Callable[[HttpRequest], ClassificationExportFormatter] diff --git a/classification/views/exports/classification_export_filter.py b/classification/views/exports/classification_export_filter.py index 6e50f9aa0..0d1e44077 100644 --- a/classification/views/exports/classification_export_filter.py +++ b/classification/views/exports/classification_export_filter.py @@ -1,29 +1,49 @@ import operator +from collections.abc import Callable, Iterator from dataclasses import dataclass, field from datetime import datetime from enum import Enum from functools import cached_property, reduce -from typing import List, Type, Union, Set, Optional, Dict, Iterator, Any, Callable +from typing import Any, Optional, Union from django.conf import settings from django.contrib.auth.models import User -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.http import HttpRequest from guardian.shortcuts import get_objects_for_user from threadlocals.threadlocals import get_current_request -from annotation.annotation_version_querysets import get_variant_queryset_for_latest_annotation_version -from classification.enums import ShareLevel, ClinicalContextStatus, AlleleOriginBucket +from annotation.annotation_version_querysets import ( + get_variant_queryset_for_latest_annotation_version, +) +from classification.enums import AlleleOriginBucket, ClinicalContextStatus, ShareLevel from classification.enums.discordance_enums import DiscordanceReportResolution -from classification.models import ClassificationModification, Classification, classification_flag_types, \ - DiscordanceReport, ClinicalContext, ImportedAlleleInfo, ClinVarExport -from flags.models import FlagsMixin, Flag, FlagComment -from genes.models import GeneSymbolAlias, GeneSymbol +from classification.models import ( + Classification, + ClassificationModification, + ClinicalContext, + ClinVarExport, + DiscordanceReport, + ImportedAlleleInfo, + classification_flag_types, +) +from flags.models import Flag, FlagComment, FlagsMixin +from genes.models import GeneSymbol, GeneSymbolAlias from genes.signals.gene_symbol_search import GENE_SYMBOL_PATTERN -from library.utils import batch_iterator, local_date_string, http_header_date_now +from library.utils import batch_iterator, http_header_date_now, local_date_string from snpdb.clingen_allele import get_clingen_allele -from snpdb.models import GenomeBuild, Lab, Organization, Allele, Variant, AlleleOriginFilterDefault, ClinGenAllele, \ - VariantCoordinate, VARIANT_PATTERN, VARIANT_SYMBOLIC_PATTERN +from snpdb.models import ( + VARIANT_PATTERN, + VARIANT_SYMBOLIC_PATTERN, + Allele, + AlleleOriginFilterDefault, + ClinGenAllele, + GenomeBuild, + Lab, + Organization, + Variant, + VariantCoordinate, +) from snpdb.signals.variant_search import get_results_from_variant_coordinate @@ -36,7 +56,7 @@ class ClassificationIssue: @property def message(self) -> Optional[str]: - messages: List[str] = [] + messages: list[str] = [] if self.withdrawn: messages.append("Classification has been withdrawn") # if self.transcript_version or self.matching_warning: @@ -78,7 +98,7 @@ class AlleleData: TODO, make this a method and make calling it an exception if we didn't split by allele origin """ - all_cms: List[ClassificationIssue] = field(default_factory=list) # misleading name, should be all_ci or something + all_cms: list[ClassificationIssue] = field(default_factory=list) # misleading name, should be all_ci or something def sort(self): def record_order(ci: ClassificationIssue): @@ -88,7 +108,7 @@ def record_order(ci: ClassificationIssue): cached_allele: Optional[Allele] = None cached_variant: Optional[Variant] = None - cached_data: Dict[str, Any] = None + cached_data: dict[str, Any] = None @staticmethod def from_allele_info(source: 'ClassificationFilter', allele_info: ImportedAlleleInfo, allele_origin_bucket: AlleleOriginBucket): @@ -146,7 +166,7 @@ def genome_build(self) -> GenomeBuild: return self.source.genome_build @property - def cms(self) -> List[ClassificationModification]: + def cms(self) -> list[ClassificationModification]: # The classifications that should be exported (passed validation, not withdrawn) return [ci.classification for ci in self.all_cms if not ci.has_issue] @@ -158,11 +178,11 @@ def cms_allele_origins(self) -> set[AlleleOriginBucket]: return all_allele_origins @property - def cms_regardless_of_issues(self) -> List[ClassificationModification]: + def cms_regardless_of_issues(self) -> list[ClassificationModification]: return [ci.classification for ci in self.all_cms] @property - def issues(self) -> List[ClassificationIssue]: + def issues(self) -> list[ClassificationIssue]: # All Classification Issues that actually have issues return [ci for ci in self.all_cms if ci.has_issue] @@ -170,7 +190,7 @@ def __bool__(self): return bool(self.all_cms) -def flag_ids_to(model: Type[FlagsMixin], qs: Union[QuerySet[Flag], QuerySet[FlagComment]]) -> Set[int]: +def flag_ids_to(model: type[FlagsMixin], qs: Union[QuerySet[Flag], QuerySet[FlagComment]]) -> set[int]: """ Convert a qs of Flags or FlagComments to ids of Classification/Allele/etc. :param model: The Model to get the IDs of e.g. Classification, Allele, assumes the flag qs was for Flags that @@ -350,8 +370,8 @@ class ClassificationFilter: allele_origin_filter: AlleleOriginFilterDefault = AlleleOriginFilterDefault.SHOW_ALL extra_filter: Optional[Q] = None allele_origin_split: bool = False # if true, subdivide allele data by allele origin bucket - exclude_sources: Optional[Set[Union[Lab, Organization]]] = None - include_sources: Optional[Set[Lab]] = None + exclude_sources: Optional[set[Union[Lab, Organization]]] = None + include_sources: Optional[set[Lab]] = None since: Optional[datetime] = None min_share_level: ShareLevel = ShareLevel.LAB transcript_strategy: TranscriptStrategy = TranscriptStrategy.ALL @@ -379,9 +399,9 @@ def description(self): if since := self.since: parts.append(f"Since {since.strftime('%Y%m%d')}") if self.allele: - parts.append(f"Limited to Single Allele") + parts.append("Limited to Single Allele") if self.min_share_level == ShareLevel.ALL_USERS: - parts.append(f"Shared Data Only") + parts.append("Shared Data Only") if not parts: return "No Filters" return ", ".join(parts) @@ -419,7 +439,7 @@ def date_str(self) -> str: return local_date_string() @staticmethod - def _string_to_group_name(model: Type[Union[Lab, Organization]], group_names: str) -> Union[Set[Lab], Set[Organization]]: + def _string_to_group_name(model: type[Union[Lab, Organization]], group_names: str) -> Union[set[Lab], set[Organization]]: """ Converts comma separated group names to the org or lab objects. Invalid org/lab group names will be ignored @@ -527,13 +547,13 @@ def c_hgvs_col(self): return 'classification__allele_info__grch38__c_hgvs' @cached_property - def _discordant_classification_ids(self) -> Dict[int, DiscordanceReportStatus]: + def _discordant_classification_ids(self) -> dict[int, DiscordanceReportStatus]: """ Returns a set of classification IDs where the classification id discordant Ids are not necessarily part of this import :return: A set of classification IDs """ - discordance_status: Dict[int, DiscordanceReportStatus] = {} + discordance_status: dict[int, DiscordanceReportStatus] = {} for cc in ClinicalContext.objects.filter(status=ClinicalContextStatus.DISCORDANT): if dr := DiscordanceReport.latest_report(cc): status: Optional[DiscordanceReportStatus] @@ -554,7 +574,7 @@ def is_discordant(self, cm: ClassificationModification) -> DiscordanceReportStat return self._discordant_classification_ids.get(cm.classification_id) @cached_property - def _since_flagged_classification_ids(self) -> Set[int]: + def _since_flagged_classification_ids(self) -> set[int]: """ TODO rename to indicate this is a flag check only Returns classification ids that have had flags change since the since date @@ -587,18 +607,18 @@ def _passes_since(self, allele_data: AlleleData) -> bool: return False @property - def _share_levels(self) -> Set[ShareLevel]: + def _share_levels(self) -> set[ShareLevel]: """ :return: A set of all ShareLevels that should be considered """ - share_levels: Set[ShareLevel] = set() + share_levels: set[ShareLevel] = set() for sl in ShareLevel.ALL_LEVELS: if sl >= self.min_share_level: share_levels.add(sl) return share_levels @cached_property - def excluded_record_filters(self) -> List[str]: + def excluded_record_filters(self) -> list[str]: """ Returns a list of record filters that are not valid """ @@ -675,7 +695,7 @@ def cms_qs(self) -> QuerySet[ClassificationModification]: if self.transcript_strategy == TranscriptStrategy.REFSEQ: from classification.views.classification_export_view import ALISSA_ACCEPTED_TRANSCRIPTS - acceptable_transcripts: List[Q] = [ + acceptable_transcripts: list[Q] = [ Q(**{f'{self.c_hgvs_col}__startswith': tran}) for tran in ALISSA_ACCEPTED_TRANSCRIPTS ] cms = cms.filter(reduce(operator.or_, acceptable_transcripts)) @@ -733,7 +753,7 @@ def _allele_data_filtered(self) -> Iterator[AlleleData]: if self._passes_since(allele_data): yield allele_data - def allele_data_filtered_pre_processed(self, batch_processor: Optional[Callable[[List[AlleleData]], None]] = None) -> Iterator[AlleleData]: + def allele_data_filtered_pre_processed(self, batch_processor: Optional[Callable[[list[AlleleData]], None]] = None) -> Iterator[AlleleData]: for batch in batch_iterator(self._allele_data_filtered(), batch_size=100): # AlleleData.pre_process(batch) if batch_processor: diff --git a/classification/views/exports/classification_export_formatter.py b/classification/views/exports/classification_export_formatter.py index a4258e3d4..790d4a027 100644 --- a/classification/views/exports/classification_export_formatter.py +++ b/classification/views/exports/classification_export_formatter.py @@ -1,21 +1,25 @@ import zipfile from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from datetime import datetime from io import StringIO -from typing import Optional, Iterator, Any, Callable, Iterable +from typing import Any, Optional from django.http import HttpResponse, StreamingHttpResponse from django.http.response import HttpResponseBase from django.shortcuts import render from more_itertools import peekable -from stream_zip import stream_zip, ZIP_64 +from stream_zip import ZIP_64, stream_zip from threadlocals.threadlocals import get_current_request -from classification.views.exports.classification_export_filter import AlleleData, ClassificationFilter +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) from library.guardian_utils import bot_group from library.log_utils import NotificationBuilder, report_exc_info -from snpdb.models import GenomeBuild, AlleleOriginFilterDefault +from snpdb.models import AlleleOriginFilterDefault, GenomeBuild @dataclass(frozen=True) @@ -194,7 +198,7 @@ def _yield_streaming_zip_entries(self) -> Iterator[tuple[str, datetime, int, Any except: report_exc_info() def yield_error_bytes(): - yield "An error occurred generating the file".encode() + yield b"An error occurred generating the file" yield "error.txt", modified_at, perms, ZIP_64, yield_error_bytes() raise diff --git a/classification/views/exports/classification_export_formatter_clinvar.py b/classification/views/exports/classification_export_formatter_clinvar.py index e5465ad34..253b5d3c9 100644 --- a/classification/views/exports/classification_export_formatter_clinvar.py +++ b/classification/views/exports/classification_export_formatter_clinvar.py @@ -1,24 +1,39 @@ import logging +from collections.abc import Callable from datetime import datetime from enum import Enum from functools import cached_property from itertools import groupby -from typing import Optional, Callable +from typing import Optional -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.http import HttpRequest from django.urls import reverse from annotation.clinvar_fetch_request import ClinVarFetchRequest -from annotation.models import ClinVar, ClinVarVersion, ClinVarRecord, ClinVarReviewStatus +from annotation.models import ClinVar, ClinVarRecord, ClinVarReviewStatus, ClinVarVersion from annotation.utils.clinvar_constants import CLINVAR_REVIEW_EXPERT_PANEL_STARS_VALUE -from classification.enums import SpecialEKeys, AlleleOriginBucket -from classification.models import EvidenceKeyMap, ClassificationModification -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.enums import AlleleOriginBucket, SpecialEKeys +from classification.models import ClassificationModification, EvidenceKeyMap +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from library.django_utils import get_url_from_view_path -from library.utils import ExportRow, export_column, delimited_row, first, ExportDataType, ExportTweak +from library.utils import ( + ExportDataType, + ExportRow, + ExportTweak, + delimited_row, + export_column, + first, +) from snpdb.models import GenomeBuild, VariantAllele diff --git a/classification/views/exports/classification_export_formatter_condition_resolution.py b/classification/views/exports/classification_export_formatter_condition_resolution.py index 6ad368048..ef5ad127d 100644 --- a/classification/views/exports/classification_export_formatter_condition_resolution.py +++ b/classification/views/exports/classification_export_formatter_condition_resolution.py @@ -1,18 +1,32 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import Optional from django.http import HttpRequest from classification.enums import SpecialEKeys from classification.models import ClassificationModification, ConditionTextMatch -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from genes.models import GeneSymbol -from library.utils import ExportRow, export_column, delimited_row -from ontology.models import OntologyTerm, OntologyRelation, OntologyImportSource, \ - PanelAppClassification, OntologySnake, OntologyService, GeneDiseaseClassification, \ - ONTOLOGY_RELATIONSHIP_NO_QUALITY_FILTER +from library.utils import ExportRow, delimited_row, export_column +from ontology.models import ( + ONTOLOGY_RELATIONSHIP_NO_QUALITY_FILTER, + GeneDiseaseClassification, + OntologyImportSource, + OntologyRelation, + OntologyService, + OntologySnake, + OntologyTerm, + PanelAppClassification, +) @dataclass(frozen=True) @@ -145,10 +159,10 @@ def content_type(self) -> str: def extension(self) -> str: return "csv" - def header(self) -> List[str]: + def header(self) -> list[str]: return [delimited_row(ClassificationConditionResolutionRow.csv_header())] - def footer(self) -> List[str]: + def footer(self) -> list[str]: return [] def row(self, allele_data: AlleleData) -> list[str]: diff --git a/classification/views/exports/classification_export_formatter_csv.py b/classification/views/exports/classification_export_formatter_csv.py index 013455bcf..d7334734a 100644 --- a/classification/views/exports/classification_export_formatter_csv.py +++ b/classification/views/exports/classification_export_formatter_csv.py @@ -1,22 +1,36 @@ from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import Optional, Any +from typing import Any, Optional from django.conf import settings from django.http import HttpRequest from classification.models import Classification, ClassificationModification, EvidenceKeyMap from classification.models.classification_groups import ClassificationGroupUtils -from classification.views.classification_export_utils import UsedKeyTracker, KeyValueFormatter -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import AlleleData, ClassificationFilter, \ - DiscordanceReportStatus -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter, \ - ClassificationExportExtraData +from classification.views.classification_export_utils import KeyValueFormatter, UsedKeyTracker +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, + DiscordanceReportStatus, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportExtraData, + ClassificationExportFormatter, +) from classification.views.exports.classification_export_utils import CitationCounter from library.django_utils import get_url_from_view_path -from library.utils import delimited_row, export_column, ExportRow, ExportDataType, html_to_text, ExportTweak +from library.utils import ( + ExportDataType, + ExportRow, + ExportTweak, + delimited_row, + export_column, + html_to_text, +) from snpdb.models import GenomeBuild diff --git a/classification/views/exports/classification_export_formatter_flags.py b/classification/views/exports/classification_export_formatter_flags.py index e8a3c263e..09a6aad94 100644 --- a/classification/views/exports/classification_export_formatter_flags.py +++ b/classification/views/exports/classification_export_formatter_flags.py @@ -1,17 +1,25 @@ +from collections.abc import Iterator from dataclasses import dataclass -from typing import Optional, Iterator +from typing import Optional from django.http import HttpRequest from classification.enums import SpecialEKeys from classification.models import ClassificationModification, EvidenceKeyMap -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData, \ - ClassificationIssue -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter -from flags.models import FlagComment, Flag +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, + ClassificationIssue, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) +from flags.models import Flag, FlagComment from library.django_utils import get_url_from_view_path -from library.utils import export_column, ExportDataType, ExportRow, delimited_row +from library.utils import ExportDataType, ExportRow, delimited_row, export_column @dataclass(frozen=True) diff --git a/classification/views/exports/classification_export_formatter_json.py b/classification/views/exports/classification_export_formatter_json.py index 5c89e3e0d..412c111cb 100644 --- a/classification/views/exports/classification_export_formatter_json.py +++ b/classification/views/exports/classification_export_formatter_json.py @@ -6,11 +6,22 @@ from django.conf import settings from django.http import HttpRequest -from classification.models import ClassificationJsonParams, ClassificationModification, EvidenceKeyMap -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData, \ - DiscordanceReportStatus -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.models import ( + ClassificationJsonParams, + ClassificationModification, + EvidenceKeyMap, +) +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, + DiscordanceReportStatus, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) @dataclass(frozen=True) diff --git a/classification/views/exports/classification_export_formatter_keys.py b/classification/views/exports/classification_export_formatter_keys.py index 3a2127775..9c1aa69da 100644 --- a/classification/views/exports/classification_export_formatter_keys.py +++ b/classification/views/exports/classification_export_formatter_keys.py @@ -1,13 +1,20 @@ import json -from typing import Optional, Any +from typing import Any, Optional from django.http import HttpRequest from classification.models import EvidenceKey, EvidenceKeyMap from classification.models.evidence_mixin import VCBlobDict -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) """ For generating a report about the usage of evidence keys. diff --git a/classification/views/exports/classification_export_formatter_lab_compare.py b/classification/views/exports/classification_export_formatter_lab_compare.py index 865180e4a..8de639f45 100644 --- a/classification/views/exports/classification_export_formatter_lab_compare.py +++ b/classification/views/exports/classification_export_formatter_lab_compare.py @@ -8,12 +8,19 @@ from classification.enums import SpecialEKeys from classification.models import ClassificationModification, EvidenceKeyMap from classification.views.classification_export_view import InvalidExportParameter -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from library.django_utils import get_url_from_view_path -from library.utils import ExportRow, export_column, delimited_row, ExportTweak -from snpdb.models import Lab, Allele +from library.utils import ExportRow, ExportTweak, delimited_row, export_column +from snpdb.models import Allele, Lab class ClassificationLab(ExportRow): @@ -146,7 +153,7 @@ def __init__(self, classification_filter: ClassificationFilter): super().__init__(classification_filter) if not self.classification_filter.include_sources or len(self.classification_filter.include_sources) != 2: raise InvalidExportParameter("Lab Compare requires the inclusion of 2 labs.") - two_labs = list(sorted(self.classification_filter.include_sources)) + two_labs = sorted(self.classification_filter.include_sources) self.lab_a = two_labs[0] self.lab_b = two_labs[1] self.e_keys = EvidenceKeyMap.cached() diff --git a/classification/views/exports/classification_export_formatter_mvl.py b/classification/views/exports/classification_export_formatter_mvl.py index 0f5d6878d..10ee5ef3c 100644 --- a/classification/views/exports/classification_export_formatter_mvl.py +++ b/classification/views/exports/classification_export_formatter_mvl.py @@ -11,16 +11,26 @@ from annotation.views import simple_citation_html from classification.enums import SpecialEKeys from classification.models import ClassificationModification, EvidenceKeyMap -from classification.models.classification_groups import ClassificationGroups, ClassificationGroupUtils +from classification.models.classification_groups import ( + ClassificationGroups, + ClassificationGroupUtils, +) from classification.views.classification_export_utils import ConflictStrategy from classification.views.classification_export_view import InvalidExportParameter -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import AlleleData, ClassificationFilter, \ - DiscordanceReportStatus -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, + DiscordanceReportStatus, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from classification.views.exports.classification_export_utils import CHGVSData, CitationCounter from library.django_utils import get_url_from_view_path -from library.utils import delimited_row, export_column, ExportRow, ExportTweak +from library.utils import ExportRow, ExportTweak, delimited_row, export_column from snpdb.models import Allele, AlleleOriginFilterDefault diff --git a/classification/views/exports/classification_export_formatter_redcap.py b/classification/views/exports/classification_export_formatter_redcap.py index 8774eec44..424d0fef6 100644 --- a/classification/views/exports/classification_export_formatter_redcap.py +++ b/classification/views/exports/classification_export_formatter_redcap.py @@ -1,7 +1,7 @@ import csv import io import re -from typing import Iterable +from collections.abc import Iterable from django.db.models import OrderBy from django.db.models.expressions import RawSQL @@ -9,15 +9,26 @@ from django.http.response import HttpResponseBase from classification.enums import EvidenceKeyValueType -from classification.models import EvidenceKey, Classification, EvidenceKeyMap, ClassificationModification +from classification.models import ( + Classification, + ClassificationModification, + EvidenceKey, + EvidenceKeyMap, +) from classification.views.classification_export_utils import KeyValueFormatter, UsedKeyTracker -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from classification.views.exports.classification_export_utils import CitationCounter from library.utils import delimited_row - # WARNING: REDCap export ignores the since parameter # WARNING: REDCap export ignores variant warnings diff --git a/classification/views/exports/classification_export_formatter_spelling.py b/classification/views/exports/classification_export_formatter_spelling.py index 959469ca8..bf87a5c44 100644 --- a/classification/views/exports/classification_export_formatter_spelling.py +++ b/classification/views/exports/classification_export_formatter_spelling.py @@ -8,11 +8,18 @@ from classification.enums import SpecialEKeys from classification.models import ClassificationModification -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) from library.django_utils import get_url_from_view_path -from library.utils import ExportRow, export_column, delimited_row +from library.utils import ExportRow, delimited_row, export_column from library.utils.nltk_utils import ensure_nltk_data RE_HAS_BAD_CHAR = re.compile(r"[\d._]") diff --git a/classification/views/exports/classification_export_formatter_vcf.py b/classification/views/exports/classification_export_formatter_vcf.py index 550f19a9e..d859407a9 100644 --- a/classification/views/exports/classification_export_formatter_vcf.py +++ b/classification/views/exports/classification_export_formatter_vcf.py @@ -7,17 +7,30 @@ from django.http import HttpRequest from django.urls import reverse -from classification.enums import SpecialEKeys, AlleleOriginBucket -from classification.models import EvidenceKeyMap, ClassificationModification -from classification.views.exports.classification_export_decorator import register_classification_exporter -from classification.views.exports.classification_export_filter import ClassificationFilter, AlleleData -from classification.views.exports.classification_export_formatter import ClassificationExportFormatter -from classification.views.exports.vcf_export_utils import ExportVCF, export_vcf_info_cell, VCFHeaderType, \ - VCFHeaderNumberSpecial, VCFHeader, VCFExportTweak +from classification.enums import AlleleOriginBucket, SpecialEKeys +from classification.models import ClassificationModification, EvidenceKeyMap +from classification.views.exports.classification_export_decorator import ( + register_classification_exporter, +) +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) +from classification.views.exports.classification_export_formatter import ( + ClassificationExportFormatter, +) +from classification.views.exports.vcf_export_utils import ( + ExportVCF, + VCFExportTweak, + VCFHeader, + VCFHeaderNumberSpecial, + VCFHeaderType, + export_vcf_info_cell, +) from library.django_utils import get_url_from_view_path from library.utils import local_date_str_no_dash from ontology.models import OntologyTerm -from snpdb.models import Variant, Allele +from snpdb.models import Allele, Variant class VCFTargetSystem(str, Enum): @@ -200,7 +213,7 @@ def condition_terms(self): all_plain_text.add(plain) elif condition := cm.get(SpecialEKeys.CONDITION): all_plain_text.add(condition) - return [f"{t.id} {t.name}" for t in sorted(all_terms)] + list(sorted(all_plain_text)) + return [f"{t.id} {t.name}" for t in sorted(all_terms)] + sorted(all_plain_text) @export_vcf_info_cell( header_id="classification", diff --git a/classification/views/exports/classification_export_utils.py b/classification/views/exports/classification_export_utils.py index 0074831aa..208df589f 100644 --- a/classification/views/exports/classification_export_utils.py +++ b/classification/views/exports/classification_export_utils.py @@ -1,13 +1,17 @@ from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Iterable, Any +from typing import Any from annotation.models import Citation, CitationFetchRequest from annotation.models.models_citations import CitationIdNormalized, CitationSource from classification.enums import SpecialEKeys from classification.models import ClassificationModification from classification.views.classification_export_utils import TranscriptGroup, VariantWithChgvs -from classification.views.exports.classification_export_filter import AlleleData, ClassificationFilter +from classification.views.exports.classification_export_filter import ( + AlleleData, + ClassificationFilter, +) from genes.hgvs import CHGVS from library.log_utils import report_message diff --git a/classification/views/exports/classification_export_view.py b/classification/views/exports/classification_export_view.py index 54957021a..bfeadee26 100644 --- a/classification/views/exports/classification_export_view.py +++ b/classification/views/exports/classification_export_view.py @@ -1,7 +1,9 @@ from django.http import HttpRequest from django.http.response import HttpResponseBase -from classification.views.exports.classification_export_decorator import classification_exporter_for_request +from classification.views.exports.classification_export_decorator import ( + classification_exporter_for_request, +) def serve_export(request: HttpRequest) -> HttpResponseBase: diff --git a/classification/views/exports/vcf_export_utils.py b/classification/views/exports/vcf_export_utils.py index 3fe33bbe5..ee7ce136b 100644 --- a/classification/views/exports/vcf_export_utils.py +++ b/classification/views/exports/vcf_export_utils.py @@ -1,13 +1,14 @@ import inspect import urllib +from collections.abc import Iterable from dataclasses import dataclass from enum import Enum -from typing import Any, Optional, Iterable, TypeAlias, Protocol +from typing import Any, Optional, Protocol, TypeAlias from django.conf import settings -from library.utils import ExportTweak, local_date_str_no_dash, get_decorated_methods -from snpdb.models import Variant, GenomeBuild, GenomeBuildContig, Contig, Allele +from library.utils import ExportTweak, get_decorated_methods, local_date_str_no_dash +from snpdb.models import Allele, Contig, GenomeBuild, GenomeBuildContig, Variant """ This works much like ExportRow @export_column but for VCFs diff --git a/classification/views/imported_allele_info_view.py b/classification/views/imported_allele_info_view.py index 1e7c3de92..127ba2b22 100644 --- a/classification/views/imported_allele_info_view.py +++ b/classification/views/imported_allele_info_view.py @@ -3,22 +3,25 @@ from functools import reduce from typing import Optional -from django.db.models import QuerySet, Q, Count +from django.db.models import Count, Q, QuerySet from django.http import HttpRequest -from django.shortcuts import render, get_object_or_404 +from django.shortcuts import get_object_or_404, render from requests import Response -from classification.models import ImportedAlleleInfo, ImportedAlleleInfoStatus, Classification, \ - ClassificationModification +from classification.models import ( + Classification, + ClassificationModification, + ImportedAlleleInfo, + ImportedAlleleInfoStatus, +) from classification.models.classification_variant_info_models import ImportedAlleleInfoValidation from genes.hgvs import CHGVS, CHGVSDiff, chgvs_diff_description -from library.django_utils import get_url_from_view_path -from library.django_utils import require_superuser -from library.utils import MultiDiff, MultiDiffInput, ExportRow, export_column +from library.django_utils import get_url_from_view_path, require_superuser +from library.utils import ExportRow, MultiDiff, MultiDiffInput, export_column from library.utils.django_utils import render_ajax_view from snpdb.admin_utils import get_admin_url -from snpdb.models import GenomeBuild, Lab, Allele -from snpdb.views.datatable_view import DatatableConfig, RichColumn, CellData, SortOrder +from snpdb.models import Allele, GenomeBuild, Lab +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder class ImportedAlleleInfoColumns(DatatableConfig[ImportedAlleleInfo]): diff --git a/classification/views/search_view_metrics.py b/classification/views/search_view_metrics.py index 3561187bf..950e89b4b 100644 --- a/classification/views/search_view_metrics.py +++ b/classification/views/search_view_metrics.py @@ -1,7 +1,7 @@ from collections import defaultdict +from collections.abc import Iterator from dataclasses import dataclass, field -from datetime import timedelta, date -from typing import Iterator +from datetime import date, timedelta from django.contrib.auth.models import User from django.db.models import Q diff --git a/classification/views/views.py b/classification/views/views.py index 463c9379e..04b7ccd40 100644 --- a/classification/views/views.py +++ b/classification/views/views.py @@ -3,11 +3,11 @@ import re from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Optional, Any, Union +from typing import Any, Optional, Union import rest_framework from crispy_forms.bootstrap import FieldWithButtons -from crispy_forms.layout import Layout, Field, Submit +from crispy_forms.layout import Field, Layout, Submit from django.conf import settings from django.contrib import messages from django.contrib.auth.models import User @@ -16,10 +16,10 @@ from django.http import StreamingHttpResponse from django.http.request import HttpRequest from django.http.response import HttpResponse -from django.shortcuts import render, get_object_or_404, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.timezone import now -from django.views.decorators.http import require_POST, require_http_methods +from django.views.decorators.http import require_http_methods, require_POST from django.views.generic import TemplateView from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema @@ -30,17 +30,41 @@ from rest_framework.views import APIView from annotation.transcripts_annotation_selections import VariantTranscriptSelections -from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import \ - create_classification_for_sample_and_variant_objects, generate_auto_populate_data +from classification.autopopulate_evidence_keys.autopopulate_evidence_keys import ( + create_classification_for_sample_and_variant_objects, + generate_auto_populate_data, +) from classification.classification_changes import ClassificationChanges -from classification.classification_stats import get_grouped_classification_counts, \ - get_classification_counts, get_criteria_counts -from classification.enums import SubmissionSource, SpecialEKeys, ShareLevel, WithdrawReason, AlleleOriginBucket +from classification.classification_stats import ( + get_classification_counts, + get_criteria_counts, + get_grouped_classification_counts, +) +from classification.enums import ( + AlleleOriginBucket, + ShareLevel, + SpecialEKeys, + SubmissionSource, + WithdrawReason, +) from classification.forms import ClassificationAlleleOriginForm -from classification.models import ClassificationAttachment, Classification, \ - ClassificationRef, ClassificationJsonParams, ClassificationConsensus, ClassificationReportTemplate, ReportNames, \ - ConditionResolvedDict, DiscordanceReport, ClassificationGrouping, AlleleGrouping, AlleleOriginGrouping, \ - ImportedAlleleInfo, ImportedAlleleInfoStatus, ClassificationImportRun +from classification.models import ( + AlleleGrouping, + AlleleOriginGrouping, + Classification, + ClassificationAttachment, + ClassificationConsensus, + ClassificationGrouping, + ClassificationImportRun, + ClassificationJsonParams, + ClassificationRef, + ClassificationReportTemplate, + ConditionResolvedDict, + DiscordanceReport, + ImportedAlleleInfo, + ImportedAlleleInfoStatus, + ReportNames, +) from classification.models.classification import ClassificationModification from classification.models.classification_import_run import ClassificationImportRunStatus from classification.models.clinical_context_models import ClinicalContext @@ -48,23 +72,32 @@ from classification.models.flag_types import classification_flag_types from classification.views.classification_dashboard_view import ClassificationDashboard from classification.views.classification_datatables import ClassificationColumns -from classification.views.exports import ClassificationExportFormatterCSV, ClassificationExportFormatterRedCap +from classification.views.exports import ( + ClassificationExportFormatterCSV, + ClassificationExportFormatterRedCap, +) from classification.views.exports.classification_export_filter import ClassificationFilter from classification.views.exports.classification_export_formatter_csv import FormatDetailsCSV from flags.models import Flag, FlagComment from flags.models.models import FlagType from genes.forms import GeneSymbolForm from genes.hgvs import HGVSMatcher -from library.django_utils import require_superuser, get_url_from_view_path -from library.django_utils.file_uploads import filepond_upload_receive, filepond_process_response +from library.django_utils import get_url_from_view_path, require_superuser +from library.django_utils.file_uploads import filepond_process_response, filepond_upload_receive from library.log_utils import log_traceback from library.utils import delimited_row from library.utils.django_utils import render_ajax_view from library.utils.file_utils import rm_if_exists -from snpdb.forms import SampleChoiceForm, UserSelectForm, LabSelectForm, LabMultiSelectForm, UserLabChoiceForm +from snpdb.forms import ( + LabMultiSelectForm, + LabSelectForm, + SampleChoiceForm, + UserLabChoiceForm, + UserSelectForm, +) from snpdb.genome_build_manager import GenomeBuildManager from snpdb.lab_picker import LabPickerData -from snpdb.models import Variant, UserSettings, Sample, Lab, Allele +from snpdb.models import Allele, Lab, Sample, UserSettings, Variant from snpdb.models.models_genome import GenomeBuild from snpdb.user_settings_manager import UserSettingsManager from uicore.utils.form_helpers import form_helper_horizontal @@ -857,7 +890,7 @@ def lab_gene_classification_counts(request): if settings.CLASSIFICATION_STATS_USE_SHARED: visibility = "Shared" else: - visibility = f"Visible to user" + visibility = "Visible to user" data_columns_whitelist = { @@ -956,7 +989,7 @@ def de_number(clin_sig: str) -> str: response = StreamingHttpResponse(yield_data(), content_type='text/tsv') # modified_str = now().strftime("%a, %d %b %Y %H:%M:%S GMT") # e.g. 'Wed, 21 Oct 2015 07:28:00 GMT' # response['Last-Modified'] = modified_str - response['Content-Disposition'] = f'attachment; filename="clin_sig_changes.tsv"' + response['Content-Disposition'] = 'attachment; filename="clin_sig_changes.tsv"' return response @@ -1005,7 +1038,7 @@ def from_grouping(allele_grouping: AlleleGrouping, user: User) -> list['AlleleOr visible_groups.append( AlleleOriginGroupingVisible( allele_origin_grouping=allele_origin_grouping, - classification_groupings=list(sorted(allele_origin_grouping.classificationgrouping_set.all())), + classification_groupings=sorted(allele_origin_grouping.classificationgrouping_set.all()), discordance_reports=discordance_reports ) ) diff --git a/classification/views/views_hgvs_resolution_tool.py b/classification/views/views_hgvs_resolution_tool.py index 04917a893..bd306c006 100644 --- a/classification/views/views_hgvs_resolution_tool.py +++ b/classification/views/views_hgvs_resolution_tool.py @@ -7,7 +7,7 @@ from classification.models import ImportedAlleleInfo from genes.hgvs import HGVSConverterType, HGVSMatcher, VariantResolvingError -from genes.models import TranscriptVersion, TranscriptParts +from genes.models import TranscriptParts, TranscriptVersion from library.django_utils import require_superuser from library.utils import all_equal from snpdb.models import GenomeBuild, VariantCoordinate diff --git a/classification/views/views_uploaded_classifications_unmapped.py b/classification/views/views_uploaded_classifications_unmapped.py index fd056fb54..c3cd6f742 100644 --- a/classification/views/views_uploaded_classifications_unmapped.py +++ b/classification/views/views_uploaded_classifications_unmapped.py @@ -1,23 +1,32 @@ import datetime +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable, Any, Optional +from typing import Any, Optional import django from django.contrib import messages from django.contrib.auth.models import User from django.core.exceptions import PermissionDenied -from django.db.models import QuerySet, Q -from django.http import HttpResponseRedirect, HttpRequest, StreamingHttpResponse +from django.db.models import Q, QuerySet +from django.http import HttpRequest, HttpResponseRedirect, StreamingHttpResponse from django.shortcuts import render from django.urls import reverse from django.utils.timezone import now from django.views import View -from classification.models import ClassificationImportRun, resolve_uploaded_url_to_handle, FileHandle, \ - UploadedClassificationsUnmappedValidationRow -from classification.models.uploaded_classifications_unmapped import UploadedClassificationsUnmapped, \ - UploadedClassificationsUnmappedStatus -from classification.tasks.classification_import_map_and_insert_task import ClassificationImportMapInsertTask +from classification.models import ( + ClassificationImportRun, + FileHandle, + UploadedClassificationsUnmappedValidationRow, + resolve_uploaded_url_to_handle, +) +from classification.models.uploaded_classifications_unmapped import ( + UploadedClassificationsUnmapped, + UploadedClassificationsUnmappedStatus, +) +from classification.tasks.classification_import_map_and_insert_task import ( + ClassificationImportMapInsertTask, +) from library.django_utils import get_url_from_view_path from library.log_utils import NotificationBuilder, report_exc_info from library.utils import empty_to_none diff --git a/config/pylint3.rc b/config/pylint3.rc index 5da007f46..4a3e3dbce 100644 --- a/config/pylint3.rc +++ b/config/pylint3.rc @@ -437,7 +437,20 @@ disable=raw-checker-failed, too-many-positional-arguments, import-outside-toplevel, cyclic-import, - pointless-string-statement + pointless-string-statement, + # --- Slimmed: the following are now owned by ruff (see ruff.toml). --- + # pylint is kept only for inference-heavy E-level checks (wrong arg + # counts, no-value-for-parameter, used-before-assignment, ...) and + # pylint_django. Style / imports / refactors are ruff's job. + C, # convention: naming, docstrings, formatting, import style + R, # refactoring: too-many-*, no-else-*, comprehensions (ruff B/C4/SIM) + unused-import, # ruff F401 + unused-variable, # ruff F841 + unused-wildcard-import, # ruff F403/F405 + wildcard-import, # ruff F403 + reimported, # ruff F811 + wrong-import-position, # ruff E402 / I001 + dangerous-default-value # ruff B006 # Enable the message, report, category or checker with the given id(s). You can diff --git a/eventlog/apps.py b/eventlog/apps.py index fa2a1a645..6c4521693 100644 --- a/eventlog/apps.py +++ b/eventlog/apps.py @@ -8,5 +8,5 @@ class OntologyConfig(AppConfig): def ready(self): # pylint: disable=import-outside-toplevel # imported to activate receivers - from eventlog.signals import active_users_health_check # pylint: disable=unused-import + pass # pylint: disable=unused-import # pylint: enable=import-outside-toplevel diff --git a/eventlog/middleware.py b/eventlog/middleware.py index c1d73be4d..7611436d5 100644 --- a/eventlog/middleware.py +++ b/eventlog/middleware.py @@ -1,4 +1,4 @@ -from typing import Optional, Any +from typing import Any, Optional from django.conf import settings from django.contrib.auth.models import User diff --git a/eventlog/models.py b/eventlog/models.py index d116975cc..4d1cf7a01 100644 --- a/eventlog/models.py +++ b/eventlog/models.py @@ -2,7 +2,7 @@ import logging from typing import Optional, Union -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.contrib.auth.signals import user_logged_in from django.db import models from django.db.models.deletion import SET_NULL diff --git a/eventlog/signals/active_users_health_check.py b/eventlog/signals/active_users_health_check.py index d8a9c7720..1a076306f 100644 --- a/eventlog/signals/active_users_health_check.py +++ b/eventlog/signals/active_users_health_check.py @@ -4,11 +4,20 @@ from django.db.models import Q from django.dispatch import receiver -from classification.models import DiscordanceReportTriage, ClinVarExportBatch, ClinVarExportBatchStatus +from classification.models import ( + ClinVarExportBatch, + ClinVarExportBatchStatus, + DiscordanceReportTriage, +) from eventlog.models import Event, ViewEvent from library.guardian_utils import bot_group -from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckRecentActivity, \ - health_check_overall_stats_signal, HealthCheckTotalAmount +from library.health_check import ( + HealthCheckRecentActivity, + HealthCheckRequest, + HealthCheckTotalAmount, + health_check_overall_stats_signal, + health_check_signal, +) from snpdb.models import UserPreview diff --git a/eventlog/urls.py b/eventlog/urls.py index 89c01bbbf..1189576c4 100644 --- a/eventlog/urls.py +++ b/eventlog/urls.py @@ -3,7 +3,6 @@ from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path - urlpatterns = [ path('', views.eventlog, name='eventlog'), path('detail/', views.eventlog_detail, name='eventlog_detail'), diff --git a/flags/admin.py b/flags/admin.py index cbfd15b0e..9ba4118af 100644 --- a/flags/admin.py +++ b/flags/admin.py @@ -1,9 +1,9 @@ from django.contrib import admin from django.contrib.admin import RelatedFieldListFilter, TabularInline -from flags.models import Flag, FlagComment, FlagCollection +from flags.models import Flag, FlagCollection, FlagComment from flags.models.models import FlagType -from snpdb.admin_utils import ModelAdminBasics, AllValuesChoicesFieldListFilter +from snpdb.admin_utils import AllValuesChoicesFieldListFilter, ModelAdminBasics class FlagCommentAdminTabular(TabularInline): diff --git a/flags/models.py b/flags/models.py index 5b39ce543..0a8c8a6b1 100644 --- a/flags/models.py +++ b/flags/models.py @@ -1,4 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from flags.models.models import FlagResolution, FlagTypeContext, FlagType, FlagTypeResolution, Flag, FlagCollection, FlagComment # pylint: enable=unused-import diff --git a/flags/models/flag_health_check.py b/flags/models/flag_health_check.py index efeb3a600..3ac243d9c 100644 --- a/flags/models/flag_health_check.py +++ b/flags/models/flag_health_check.py @@ -1,10 +1,11 @@ # NOTE it would be ideal for this to react to notification instead of having to be called directly +from collections.abc import Iterable from datetime import datetime -from typing import Optional, Iterable +from typing import Optional from django.db.models import Q -from flags.models import FlagType, FlagComment, Flag, FlagStatus, FlagResolution +from flags.models import Flag, FlagComment, FlagResolution, FlagStatus, FlagType class FlagDelta: diff --git a/flags/models/models.py b/flags/models/models.py index 1ca2354f8..0a2187734 100644 --- a/flags/models/models.py +++ b/flags/models/models.py @@ -1,9 +1,10 @@ import datetime import logging from collections import defaultdict -from functools import cached_property, total_ordering, reduce +from collections.abc import Iterable +from functools import cached_property, reduce, total_ordering from operator import __and__ -from typing import Optional, Union, Iterable, Any, TypeVar +from typing import Any, Optional, TypeVar, Union import django.dispatch from django.contrib.auth.models import User @@ -18,7 +19,7 @@ from library.django_utils.django_object_managers import ObjectManagerCachingImmutable from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin from library.guardian_utils import admin_bot -from library.utils import ModelUtilsMixin, ChoicesEnum +from library.utils import ChoicesEnum, ModelUtilsMixin flag_collection_extra_info_signal = django.dispatch.Signal() # args: "flag_infos", "user" diff --git a/flags/views/views.py b/flags/views/views.py index 097dd4e66..9af4e678f 100644 --- a/flags/views/views.py +++ b/flags/views/views.py @@ -1,21 +1,22 @@ import datetime from collections import defaultdict -from typing import Iterable, Any, Union, Optional +from collections.abc import Iterable +from typing import Any, Optional, Union from django.conf import settings -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter from django.contrib.auth.models import User from django.utils import timezone +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.response import Response from rest_framework.views import APIView -from flags.models import Flag, FlagComment, FlagType, FlagCollection, FlagPermissionLevel +from flags.models import Flag, FlagCollection, FlagComment, FlagPermissionLevel, FlagType from flags.models.enums import FlagStatus from flags.models.models import FlagResolution, FlagTypeResolution, fetch_flag_infos from library.django_utils import ensure_timezone_aware from library.utils import empty_to_none -from snpdb.models import Lab, AvatarDetails +from snpdb.models import AvatarDetails, Lab class CommentDetails: diff --git a/genes/admin.py b/genes/admin.py index 682f896e8..e04859498 100644 --- a/genes/admin.py +++ b/genes/admin.py @@ -5,8 +5,8 @@ from guardian.admin import GuardedModelAdmin from genes import models -from genes.models import GeneSymbol, GeneCoverageCollection -from snpdb.admin_utils import ModelAdminBasics, admin_list_column, admin_action +from genes.models import GeneCoverageCollection, GeneSymbol +from snpdb.admin_utils import ModelAdminBasics, admin_action, admin_list_column from snpdb.archive import ArchivePreconditionError diff --git a/genes/apps.py b/genes/apps.py index 9ed9a695a..f1ff3a927 100644 --- a/genes/apps.py +++ b/genes/apps.py @@ -8,18 +8,24 @@ class GenesConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from genes.signals import gene_search, gene_symbol_search, transcript_search from annotation.models.models import CachedWebResource from genes.models import CachedThirdPartyGeneList - from genes.signals.manual_signals import hgnc_post_save_handler, lrg_ref_seq_gene_post_save_handler, \ - mane_post_save_handler, \ - panel_app_england_panels_post_save_handler, panel_app_australia_panels_post_save_handler, \ - pfam_post_save_handler, \ - gnomad_gene_constraint_post_save_handler, cached_third_part_gene_list_pre_delete_handler, \ - refseq_gene_summary_post_save_handler, refseq_gene_info_post_save_handler, \ - refseq_sequence_info_post_save_handler, refseq_gene_pub_med_count_post_save_handler, \ - uniprot_post_save_handler + from genes.signals.manual_signals import ( + cached_third_part_gene_list_pre_delete_handler, + gnomad_gene_constraint_post_save_handler, + hgnc_post_save_handler, + lrg_ref_seq_gene_post_save_handler, + mane_post_save_handler, + panel_app_australia_panels_post_save_handler, + panel_app_england_panels_post_save_handler, + pfam_post_save_handler, + refseq_gene_info_post_save_handler, + refseq_gene_pub_med_count_post_save_handler, + refseq_gene_summary_post_save_handler, + refseq_sequence_info_post_save_handler, + uniprot_post_save_handler, + ) # pylint: enable=import-outside-toplevel,unused-import post_save.connect(gnomad_gene_constraint_post_save_handler, sender=CachedWebResource) diff --git a/genes/cached_web_resource/gnomad_gene_constraint.py b/genes/cached_web_resource/gnomad_gene_constraint.py index 767b5a682..8f8a6f3b1 100644 --- a/genes/cached_web_resource/gnomad_gene_constraint.py +++ b/genes/cached_web_resource/gnomad_gene_constraint.py @@ -13,7 +13,7 @@ import pandas as pd -from genes.models import GnomADGeneConstraint, GeneSymbol, TranscriptVersion +from genes.models import GeneSymbol, GnomADGeneConstraint, TranscriptVersion from library.pandas_utils import df_nan_to_none from snpdb.models import GenomeBuild diff --git a/genes/cached_web_resource/hgnc.py b/genes/cached_web_resource/hgnc.py index a702f2aa9..44701f8b6 100644 --- a/genes/cached_web_resource/hgnc.py +++ b/genes/cached_web_resource/hgnc.py @@ -9,10 +9,17 @@ from annotation.models import CachedWebResource from genes.gene_matching import ReleaseGeneMatcher -from genes.models import HGNCImport, GeneAnnotationRelease, GeneSymbol, HGNC, GeneSymbolAlias, UniProt -from genes.models_enums import HGNCStatus, GeneSymbolAliasSource +from genes.models import ( + HGNC, + GeneAnnotationRelease, + GeneSymbol, + GeneSymbolAlias, + HGNCImport, + UniProt, +) +from genes.models_enums import GeneSymbolAliasSource, HGNCStatus from library.constants import MINUTE_SECS -from library.django_utils import get_model_fields, get_field_counts +from library.django_utils import get_field_counts, get_model_fields HGNC_BASE_URL = "https://rest.genenames.org/fetch/status/" diff --git a/genes/cached_web_resource/mane.py b/genes/cached_web_resource/mane.py index 7464468ab..84a7e5f05 100644 --- a/genes/cached_web_resource/mane.py +++ b/genes/cached_web_resource/mane.py @@ -6,7 +6,7 @@ import requests from annotation.models import CachedWebResource -from genes.models import TranscriptVersion, GeneSymbol, GeneVersion, MANE, HGNC +from genes.models import HGNC, MANE, GeneSymbol, GeneVersion, TranscriptVersion from genes.models_enums import MANEStatus from library.constants import MINUTE_SECS from snpdb.models import GenomeBuild diff --git a/genes/cached_web_resource/pfam.py b/genes/cached_web_resource/pfam.py index cf388b9b0..edb7eba1a 100644 --- a/genes/cached_web_resource/pfam.py +++ b/genes/cached_web_resource/pfam.py @@ -12,7 +12,14 @@ import pandas as pd -from genes.models import TranscriptVersion, Pfam, Transcript, PfamSequence, PfamSequenceIdentifier, PfamDomains +from genes.models import ( + Pfam, + PfamDomains, + PfamSequence, + PfamSequenceIdentifier, + Transcript, + TranscriptVersion, +) from genes.models_enums import AnnotationConsortium BULK_INSERT_SIZE = 2000 diff --git a/genes/cached_web_resource/refseq.py b/genes/cached_web_resource/refseq.py index 51c26d8d8..cfc62df35 100644 --- a/genes/cached_web_resource/refseq.py +++ b/genes/cached_web_resource/refseq.py @@ -8,12 +8,19 @@ from Bio import Entrez, SeqIO from annotation.models import CachedWebResource, GenePubMedCount -from genes.models import Gene, GeneSymbol, GeneSymbolAlias, TranscriptVersionSequenceInfoFastaFileImport, Transcript, \ - TranscriptVersionSequenceInfo, TranscriptVersion +from genes.models import ( + Gene, + GeneSymbol, + GeneSymbolAlias, + Transcript, + TranscriptVersion, + TranscriptVersionSequenceInfo, + TranscriptVersionSequenceInfoFastaFileImport, +) from genes.models_enums import AnnotationConsortium, GeneSymbolAliasSource from library.constants import MINUTE_SECS from library.django_utils import chunked_queryset -from library.utils import sha256sum_str, iter_http_lines +from library.utils import iter_http_lines, sha256sum_str def store_refseq_gene_summary_from_web(cached_web_resource: CachedWebResource): diff --git a/genes/canonical_transcripts/create_canonical_transcripts.py b/genes/canonical_transcripts/create_canonical_transcripts.py index 0b17c56ef..c99436276 100644 --- a/genes/canonical_transcripts/create_canonical_transcripts.py +++ b/genes/canonical_transcripts/create_canonical_transcripts.py @@ -3,7 +3,7 @@ import pandas as pd from genes.gene_matching import GeneSymbolMatcher -from genes.models import CanonicalTranscriptCollection, CanonicalTranscript, TranscriptVersion +from genes.models import CanonicalTranscript, CanonicalTranscriptCollection, TranscriptVersion from library.utils import file_sha256sum from library.utils.database_utils import sql_delete_qs from snpdb.models import GenomeBuild diff --git a/genes/custom_text_gene_list.py b/genes/custom_text_gene_list.py index 9c8b28293..b4a23ca65 100644 --- a/genes/custom_text_gene_list.py +++ b/genes/custom_text_gene_list.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import User from genes.gene_matching import GeneSymbolMatcher -from genes.models import GeneListCategory, GeneList, CustomTextGeneList +from genes.models import CustomTextGeneList, GeneList, GeneListCategory from library.utils import sha256sum_str from snpdb.models import ImportStatus diff --git a/genes/forms.py b/genes/forms.py index b66e2aaee..d45a8da9b 100644 --- a/genes/forms.py +++ b/genes/forms.py @@ -4,8 +4,16 @@ from django.forms.widgets import HiddenInput, TextInput from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.models import PanelAppPanel, GeneListCategory, GeneList, CustomTextGeneList, \ - GeneSymbol, Gene, Transcript, GeneAnnotationRelease +from genes.models import ( + CustomTextGeneList, + Gene, + GeneAnnotationRelease, + GeneList, + GeneListCategory, + GeneSymbol, + PanelAppPanel, + Transcript, +) from library.django_utils.autocomplete_utils import ModelSelect2 from library.forms import ROFormMixin from snpdb.forms import BaseDeclareForm, GenomeBuildAutocompleteForwardMixin diff --git a/genes/gene_matching.py b/genes/gene_matching.py index ac82f8e9d..9a28e0307 100644 --- a/genes/gene_matching.py +++ b/genes/gene_matching.py @@ -1,15 +1,23 @@ import logging import re from collections import defaultdict, namedtuple +from collections.abc import Iterable from functools import cached_property -from typing import Iterable from django.db.models import Q from django.db.models.functions import Upper -from genes.models import GeneSymbol, GeneSymbolAlias, GeneListGeneSymbol, GeneAnnotationRelease, GeneVersion, \ - ReleaseGeneSymbol, ReleaseGeneSymbolGene, HGNC -from genes.models_enums import HGNCStatus, GeneSymbolAliasSource +from genes.models import ( + HGNC, + GeneAnnotationRelease, + GeneListGeneSymbol, + GeneSymbol, + GeneSymbolAlias, + GeneVersion, + ReleaseGeneSymbol, + ReleaseGeneSymbolGene, +) +from genes.models_enums import GeneSymbolAliasSource, HGNCStatus from library.cache import timed_cache from library.utils import clean_string diff --git a/genes/graphs/gene_list_chromosome_graph.py b/genes/graphs/gene_list_chromosome_graph.py index 339d22823..d7e096f0f 100644 --- a/genes/graphs/gene_list_chromosome_graph.py +++ b/genes/graphs/gene_list_chromosome_graph.py @@ -4,7 +4,7 @@ from matplotlib.patches import Rectangle from genes.models import GeneList -from library.graphs.chromosomes_graph import read_cytoband, centromere_mid +from library.graphs.chromosomes_graph import centromere_mid, read_cytoband from library.utils import sha256sum_str from snpdb.graphs.graphcache import CacheableGraph from snpdb.models.models_enums import AssemblyMoleculeType diff --git a/genes/grids.py b/genes/grids.py index c3d2c57eb..66b6007c4 100644 --- a/genes/grids.py +++ b/genes/grids.py @@ -6,25 +6,42 @@ from django.conf import settings from django.contrib.postgres.aggregates.general import StringAgg from django.core.exceptions import PermissionDenied -from django.db.models import Count, TextField, QuerySet, OuterRef, Subquery, IntegerField +from django.db.models import Count, IntegerField, OuterRef, QuerySet, Subquery, TextField from django.http import HttpRequest from django.shortcuts import get_object_or_404 from django.urls.base import reverse from analysis.models import VariantTag -from annotation.models.models import AnnotationVersion, GeneAnnotationVersion, InvalidAnnotationVersionError -from genes.models import CanonicalTranscript, GeneList, GeneSymbol, \ - GeneCoverageCanonicalTranscript, CanonicalTranscriptCollection, GeneCoverageCollection, TranscriptVersion, \ - GeneListGeneSymbol, GeneAnnotationRelease, ReleaseGeneVersion, GeneSymbolWiki +from annotation.models.models import ( + AnnotationVersion, + GeneAnnotationVersion, + InvalidAnnotationVersionError, +) +from genes.models import ( + CanonicalTranscript, + CanonicalTranscriptCollection, + GeneAnnotationRelease, + GeneCoverageCanonicalTranscript, + GeneCoverageCollection, + GeneList, + GeneListGeneSymbol, + GeneSymbol, + GeneSymbolWiki, + ReleaseGeneVersion, + TranscriptVersion, +) from genes.models_enums import AnnotationConsortium, GeneSymbolAliasSource from library.django_utils.jqgrid_view import JQGridViewOp from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig -from library.utils import update_dict_of_dict_values, JsonDataType +from library.utils import JsonDataType, update_dict_of_dict_values from snpdb.grid_columns.custom_columns import get_custom_column_fields_override_and_sample_position from snpdb.grids import AbstractVariantGrid -from snpdb.models import UserSettings, Q, VariantGridColumn, Tag, ImportStatus -from snpdb.models.models_genome import GenomeBuild, Contig -from snpdb.variant_queries import get_variant_queryset_for_gene_symbol, variant_qs_filter_has_internal_data +from snpdb.models import ImportStatus, Q, Tag, UserSettings, VariantGridColumn +from snpdb.models.models_genome import Contig, GenomeBuild +from snpdb.variant_queries import ( + get_variant_queryset_for_gene_symbol, + variant_qs_filter_has_internal_data, +) from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder diff --git a/genes/hgvs/biocommons_hgvs/data_provider.py b/genes/hgvs/biocommons_hgvs/data_provider.py index ee8d17c57..203bf38d0 100644 --- a/genes/hgvs/biocommons_hgvs/data_provider.py +++ b/genes/hgvs/biocommons_hgvs/data_provider.py @@ -1,10 +1,10 @@ from typing import Optional -from cdot.hgvs.dataproviders import LocalDataProvider, FastaSeqFetcher, ChainedSeqFetcher +from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, LocalDataProvider from django.conf import settings from hgvs.exceptions import HGVSDataNotAvailableError -from genes.models import TranscriptVersion, TranscriptVersionSequenceInfo, NoTranscript +from genes.models import NoTranscript, TranscriptVersion, TranscriptVersionSequenceInfo from genes.transcripts_utils import get_refseq_type from snpdb.models import Contig diff --git a/genes/hgvs/biocommons_hgvs/hgvs_converter_biocommons.py b/genes/hgvs/biocommons_hgvs/hgvs_converter_biocommons.py index fa8eea023..57419f420 100644 --- a/genes/hgvs/biocommons_hgvs/hgvs_converter_biocommons.py +++ b/genes/hgvs/biocommons_hgvs/hgvs_converter_biocommons.py @@ -2,17 +2,25 @@ from typing import Optional from bioutils.sequences import reverse_complement -from hgvs.exceptions import HGVSDataNotAvailableError, HGVSError, HGVSUnsupportedOperationError, HGVSInvalidVariantError +from hgvs.exceptions import ( + HGVSDataNotAvailableError, + HGVSError, + HGVSInvalidVariantError, + HGVSUnsupportedOperationError, +) from hgvs.sequencevariant import SequenceVariant from hgvs_shim import BioCommonsHGVSConverter as _BioCommonsHGVSConverterBase from hgvs_shim.hgvs_converter_biocommons import BioCommonsHGVSVariant -from genes.hgvs import HGVSNomenclatureException, HGVSImplementationException from genes.hgvs.biocommons_hgvs.data_provider import DjangoTranscriptDataProvider -from genes.hgvs.hgvs_converter import HgvsMatchRefAllele, HgvsOriginallyNormalized, HGVSConverterType +from genes.hgvs.hgvs_converter import ( + HGVSConverterType, + HgvsMatchRefAllele, + HgvsOriginallyNormalized, +) from genes.models import TranscriptVersion from genes.transcripts_utils import get_refseq_type -from snpdb.models import GenomeBuild, VariantCoordinate, Contig +from snpdb.models import Contig, GenomeBuild, VariantCoordinate class HgvsMatchTranscriptAndGenomeRefAllele(HgvsMatchRefAllele): diff --git a/genes/hgvs/hgvs.py b/genes/hgvs/hgvs.py index 39e7fc5fa..592382155 100644 --- a/genes/hgvs/hgvs.py +++ b/genes/hgvs/hgvs.py @@ -6,7 +6,7 @@ from Bio.Data.IUPACData import protein_letters_1to3_extended -from genes.models import TranscriptVersion, TranscriptParts, Transcript, LRGRefSeqGene +from genes.models import LRGRefSeqGene, Transcript, TranscriptParts, TranscriptVersion from snpdb.models.models_genome import GenomeBuild diff --git a/genes/hgvs/hgvs_converter.py b/genes/hgvs/hgvs_converter.py index ddf66cbb0..eab72f172 100644 --- a/genes/hgvs/hgvs_converter.py +++ b/genes/hgvs/hgvs_converter.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Optional -from genes.hgvs import HGVSVariant, HGVSNomenclatureException +from genes.hgvs import HGVSNomenclatureException, HGVSVariant from snpdb.models import GenomeBuild, VariantCoordinate diff --git a/genes/hgvs/hgvs_converter_combo.py b/genes/hgvs/hgvs_converter_combo.py index cd43fd60e..5825c6b03 100644 --- a/genes/hgvs/hgvs_converter_combo.py +++ b/genes/hgvs/hgvs_converter_combo.py @@ -1,9 +1,9 @@ import logging -from genes.hgvs import HGVSVariant, HGVSException +from genes.hgvs import HGVSException, HGVSVariant from genes.hgvs.hgvs_converter import HGVSConverter, HgvsMatchRefAllele, HgvsOriginallyNormalized from library.utils import all_equal -from snpdb.models import VariantCoordinate, GenomeBuild +from snpdb.models import GenomeBuild, VariantCoordinate class ComboCheckerHGVSConverter(HGVSConverter): diff --git a/genes/hgvs/hgvs_matcher.py b/genes/hgvs/hgvs_matcher.py index 52dd3858c..82bcd4960 100644 --- a/genes/hgvs/hgvs_matcher.py +++ b/genes/hgvs/hgvs_matcher.py @@ -9,22 +9,37 @@ from django.db.models import Max, Min from django.utils.timezone import now -from genes.hgvs import HGVSVariant, CHGVS, HGVSImplementationException, HGVSNomenclatureException +from genes.hgvs import CHGVS, HGVSImplementationException, HGVSNomenclatureException, HGVSVariant from genes.hgvs.biocommons_hgvs.hgvs_converter_biocommons import BioCommonsHGVSConverter -from genes.hgvs.hgvs_converter import HGVSConverterType, HgvsMatchRefAllele, HGVSConverter, HgvsOriginallyNormalized +from genes.hgvs.hgvs_converter import ( + HGVSConverter, + HGVSConverterType, + HgvsMatchRefAllele, + HgvsOriginallyNormalized, +) from genes.hgvs.hgvs_converter_combo import ComboCheckerHGVSConverter from genes.hgvs.pyhgvs.hgvs_converter_pyhgvs import PyHGVSConverter -from genes.models import TranscriptVersion, Transcript, LRGRefSeqGene, BadTranscript, \ - NoTranscript, TranscriptParts +from genes.models import ( + BadTranscript, + LRGRefSeqGene, + NoTranscript, + Transcript, + TranscriptParts, + TranscriptVersion, +) from genes.models_enums import HGVSKind -from genes.transcripts_utils import transcript_is_lrg, looks_like_transcript, looks_like_hgvs_prefix +from genes.transcripts_utils import looks_like_hgvs_prefix, looks_like_transcript, transcript_is_lrg from library.constants import WEEK_SECS from library.log_utils import report_exc_info from library.utils import clean_string -from snpdb.clingen_allele import get_clingen_allele_from_hgvs, \ - ClinGenAlleleServerException, ClinGenAlleleAPIException, get_clingen_allele_for_variant_coordinate, \ - clingen_check_variant_length -from snpdb.models import Variant, ClinGenAllele +from snpdb.clingen_allele import ( + ClinGenAlleleAPIException, + ClinGenAlleleServerException, + clingen_check_variant_length, + get_clingen_allele_for_variant_coordinate, + get_clingen_allele_from_hgvs, +) +from snpdb.models import ClinGenAllele, Variant from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_variant import VariantCoordinate diff --git a/genes/hgvs/pyhgvs/hgvs_converter_pyhgvs.py b/genes/hgvs/pyhgvs/hgvs_converter_pyhgvs.py index d971482a1..2df0cfdfa 100644 --- a/genes/hgvs/pyhgvs/hgvs_converter_pyhgvs.py +++ b/genes/hgvs/pyhgvs/hgvs_converter_pyhgvs.py @@ -3,11 +3,11 @@ import pyhgvs from hgvs_shim import PyHGVSConverter as _PyHGVSConverter from hgvs_shim.hgvs_converter_pyhgvs import PyHGVSVariant -from pyhgvs import get_genomic_sequence, HGVSName +from pyhgvs import HGVSName, get_genomic_sequence from pyhgvs.utils import make_transcript from genes.hgvs import HGVSException -from genes.hgvs.hgvs_converter import HGVSConverter, HgvsMatchRefAllele, HGVSConverterType +from genes.hgvs.hgvs_converter import HGVSConverter, HGVSConverterType, HgvsMatchRefAllele from genes.transcripts_utils import transcript_is_lrg from snpdb.models import VariantCoordinate diff --git a/genes/management/commands/fix_hgnc.py b/genes/management/commands/fix_hgnc.py index 1eb64eaa3..5138169a5 100644 --- a/genes/management/commands/fix_hgnc.py +++ b/genes/management/commands/fix_hgnc.py @@ -1,6 +1,6 @@ from django.core.management import BaseCommand -from genes.gene_matching import ReleaseGeneMatcher, HGNCMatcher +from genes.gene_matching import HGNCMatcher, ReleaseGeneMatcher from genes.models import GeneAnnotationRelease, GeneVersion from genes.models_enums import AnnotationConsortium diff --git a/genes/management/commands/fix_rematch_release_symbols_to_genes.py b/genes/management/commands/fix_rematch_release_symbols_to_genes.py index 1d96722e6..105d2bb96 100644 --- a/genes/management/commands/fix_rematch_release_symbols_to_genes.py +++ b/genes/management/commands/fix_rematch_release_symbols_to_genes.py @@ -1,7 +1,7 @@ from django.core.management import BaseCommand from genes.gene_matching import ReleaseGeneMatcher -from genes.models import GeneAnnotationRelease, ReleaseGeneSymbolGene, ReleaseGeneSymbol +from genes.models import GeneAnnotationRelease, ReleaseGeneSymbol, ReleaseGeneSymbolGene class Command(BaseCommand): diff --git a/genes/management/commands/import_canonical_transcript.py b/genes/management/commands/import_canonical_transcript.py index f0316e85d..d91e3ac9f 100644 --- a/genes/management/commands/import_canonical_transcript.py +++ b/genes/management/commands/import_canonical_transcript.py @@ -4,9 +4,11 @@ from django.core.management.base import BaseCommand -from genes.canonical_transcripts.create_canonical_transcripts import create_canonical_transcript_collection +from genes.canonical_transcripts.create_canonical_transcripts import ( + create_canonical_transcript_collection, +) from library.utils import invert_dict -from snpdb.models import GenomeBuild, AnnotationConsortium +from snpdb.models import AnnotationConsortium, GenomeBuild class Command(BaseCommand): diff --git a/genes/management/commands/import_cdot_latest.py b/genes/management/commands/import_cdot_latest.py index ad3625e4d..f84e2a219 100644 --- a/genes/management/commands/import_cdot_latest.py +++ b/genes/management/commands/import_cdot_latest.py @@ -3,7 +3,11 @@ import logging import requests -from cdot.data_release import get_latest_combo_file_urls, get_latest_data_release_tag_name, _get_version_from_tag_name +from cdot.data_release import ( + _get_version_from_tag_name, + get_latest_combo_file_urls, + get_latest_data_release_tag_name, +) from django.core.management import BaseCommand from genes.management.commands.import_gene_annotation import Command as GeneAnnotationCommand @@ -79,7 +83,7 @@ def handle(self, *args, **options): annotation_consortium = AnnotationConsortium(ac_lookup[annotation_consortium_label]) combo_file_url = get_single_element(combo_files) - logging.info(f"%s/%s - downloading: %s", + logging.info("%s/%s - downloading: %s", genome_build, annotation_consortium.label, combo_file_url) response = requests.get(combo_file_url, stream=True, timeout=2 * MINUTE_SECS) with gzip.GzipFile(fileobj=response.raw) as fz: diff --git a/genes/management/commands/import_gene_annotation.py b/genes/management/commands/import_gene_annotation.py index be44fddf8..668002210 100644 --- a/genes/management/commands/import_gene_annotation.py +++ b/genes/management/commands/import_gene_annotation.py @@ -9,8 +9,18 @@ from annotation.models import VariantAnnotationVersion from genes.cached_web_resource.refseq import retrieve_refseq_gene_summaries from genes.gene_matching import ReleaseGeneMatcher -from genes.models import GeneSymbol, GeneAnnotationImport, Gene, GeneVersion, TranscriptVersion, Transcript, HGNC, \ - GeneAnnotationRelease, ReleaseGeneVersion, ReleaseTranscriptVersion +from genes.models import ( + HGNC, + Gene, + GeneAnnotationImport, + GeneAnnotationRelease, + GeneSymbol, + GeneVersion, + ReleaseGeneVersion, + ReleaseTranscriptVersion, + Transcript, + TranscriptVersion, +) from genes.models_enums import AnnotationConsortium from library.utils import invert_dict from library.utils.file_utils import open_handle_gzip diff --git a/genes/models.py b/genes/models.py index 4577fa764..5f871ea0f 100644 --- a/genes/models.py +++ b/genes/models.py @@ -4,27 +4,28 @@ import re import shutil import types -from collections import defaultdict, Counter +from collections import Counter, defaultdict +from collections.abc import Iterable from dataclasses import dataclass from datetime import timedelta from functools import cached_property, total_ordering from io import StringIO -from typing import Optional, Union, Iterable, Any -from urllib.error import URLError, HTTPError +from typing import Any, Optional, Union +from urllib.error import HTTPError, URLError import requests from Bio import Entrez, SeqIO from cache_memoize import cache_memoize from cdot.pyhgvs.pyhgvs_transcript import PyHGVSTranscriptFactory from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.contrib.postgres.aggregates import StringAgg from django.core.cache import cache -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist, MultipleObjectsReturned -from django.db import models, IntegrityError, transaction +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist, PermissionDenied +from django.db import IntegrityError, models, transaction from django.db.models import QuerySet, TextField -from django.db.models.deletion import CASCADE, SET_NULL, PROTECT -from django.db.models.functions import Upper, Collate +from django.db.models.deletion import CASCADE, PROTECT, SET_NULL +from django.db.models.functions import Collate, Upper from django.db.models.query_utils import Q from django.db.models.signals import post_save, pre_delete from django.dispatch import receiver @@ -37,26 +38,34 @@ from requests import RequestException from genes.gene_coverage import load_gene_coverage_df -from genes.models_enums import AnnotationConsortium, HGNCStatus, GeneSymbolAliasSource, MANEStatus +from genes.models_enums import AnnotationConsortium, GeneSymbolAliasSource, HGNCStatus, MANEStatus from library.cache import timed_cache -from library.constants import HOUR_SECS, WEEK_SECS, MINUTE_SECS, DAY_SECS +from library.constants import DAY_SECS, HOUR_SECS, MINUTE_SECS, WEEK_SECS from library.django_utils import SortByPKMixin from library.django_utils.data_archive_mixin import DataArchiveMixin from library.django_utils.django_object_managers import ObjectManagerCachingRequest from library.django_utils.django_partition import RelatedModelsPartitionModel from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin -from snpdb.archive import DataArchivedError -from library.guardian_utils import assign_permission_to_user_and_groups, DjangoPermission, admin_bot, \ - add_public_group_read_permission +from library.guardian_utils import ( + DjangoPermission, + add_public_group_read_permission, + admin_bot, + assign_permission_to_user_and_groups, +) from library.log_utils import log_traceback from library.preview_request import PreviewData, PreviewModelMixin -from library.utils import get_single_element, iter_fixed_chunks, FormerTuple +from library.utils import FormerTuple, get_single_element, iter_fixed_chunks from library.utils.file_utils import mk_path -from snpdb.models import Wiki, Company, Sample, DataState +from snpdb.archive import DataArchivedError +from snpdb.models import Company, DataState, Sample, Wiki from snpdb.models.models_enums import ImportStatus -from snpdb.models.models_genome import GenomeBuild, Contig -from upload.vcf.sql_copy_files import write_sql_copy_csv, gene_coverage_canonical_transcript_sql_copy_csv, \ - gene_coverage_sql_copy_csv, GENE_COVERAGE_HEADER +from snpdb.models.models_genome import Contig, GenomeBuild +from upload.vcf.sql_copy_files import ( + GENE_COVERAGE_HEADER, + gene_coverage_canonical_transcript_sql_copy_csv, + gene_coverage_sql_copy_csv, + write_sql_copy_csv, +) class HGNCImport(TimeStampedModel): @@ -389,19 +398,19 @@ def alias_symbol_strs(self) -> list[str]: for alias_summary in self.alias_list: if not alias_summary.different_genes: gene_symbol_strs.add(alias_summary.other_symbol) - return list(sorted(gene_symbol_strs)) + return sorted(gene_symbol_strs) @cached_property def alias_symbols_in_db(self) -> list[GeneSymbolAlias]: - return list(sorted([alias for alias in self.alias_list if not alias.different_genes and alias.other_symbol_in_database])) + return sorted([alias for alias in self.alias_list if not alias.different_genes and alias.other_symbol_in_database]) @cached_property def aliases_out(self) -> list[GeneSymbolAliasSummary]: - return list(sorted([alias for alias in self.alias_list if not alias.my_symbol_is_main])) + return sorted([alias for alias in self.alias_list if not alias.my_symbol_is_main]) @cached_property def aliases_in(self) -> list[GeneSymbolAliasSummary]: - return list(sorted([alias for alias in self.alias_list if alias.my_symbol_is_main])) + return sorted([alias for alias in self.alias_list if alias.my_symbol_is_main]) class GeneAnnotationImport(TimeStampedModel): @@ -1017,7 +1026,7 @@ def get_transcript_version(genome_build: GenomeBuild, transcript_name, best_atte break transcript_version = transcript_versions_qs.filter(version=use_version).last() else: - version_list = ', '.join((str(v) for v in possible_versions)) + version_list = ', '.join(str(v) for v in possible_versions) raise MissingTranscript(f"No Transcript for '{transcript_name}' (build: {genome_build}) - but there are entries for versions {version_list}") from exc else: transcript_version = transcript_versions_qs.last() diff --git a/genes/panel_app.py b/genes/panel_app.py index 4ec20449e..4bb612e92 100644 --- a/genes/panel_app.py +++ b/genes/panel_app.py @@ -7,8 +7,15 @@ from annotation.models import CachedWebResource from genes.gene_matching import GeneSymbolMatcher -from genes.models import PanelAppPanelRelevantDisorders, PanelAppPanel, PanelAppServer, PanelAppPanelLocalCache, \ - PanelAppPanelLocalCacheGeneSymbol, GeneSymbol, create_fake_gene_list +from genes.models import ( + GeneSymbol, + PanelAppPanel, + PanelAppPanelLocalCache, + PanelAppPanelLocalCacheGeneSymbol, + PanelAppPanelRelevantDisorders, + PanelAppServer, + create_fake_gene_list, +) from genes.serializers import GeneListGeneSymbolSerializer from library.constants import MINUTE_SECS diff --git a/genes/serializers.py b/genes/serializers.py index eb8efc905..ab218a55a 100644 --- a/genes/serializers.py +++ b/genes/serializers.py @@ -1,8 +1,22 @@ from rest_framework import serializers -from genes.models import GeneInfo, GeneListCategory, GeneList, Gene, Transcript, GeneListGeneSymbol, \ - GeneAnnotationRelease, SampleGeneList, ActiveSampleGeneList, GeneSymbol, TranscriptVersion, GeneVersion, HGNC, \ - GeneCoverageCollection, GeneCoverageCanonicalTranscript +from genes.models import ( + HGNC, + ActiveSampleGeneList, + Gene, + GeneAnnotationRelease, + GeneCoverageCanonicalTranscript, + GeneCoverageCollection, + GeneInfo, + GeneList, + GeneListCategory, + GeneListGeneSymbol, + GeneSymbol, + GeneVersion, + SampleGeneList, + Transcript, + TranscriptVersion, +) from snpdb.models import Company from snpdb.serializers import GenomeBuildSerializer diff --git a/genes/signals/gene_search.py b/genes/signals/gene_search.py index 81b964c64..bda4917bb 100644 --- a/genes/signals/gene_search.py +++ b/genes/signals/gene_search.py @@ -2,7 +2,7 @@ from genes.models import Gene from genes.models_enums import AnnotationConsortium -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver GENE_PATTERN = re.compile(r"^(?PENSG|GENE\s*(?:ID)?\s*:\s*)(?P\d+)$", re.IGNORECASE) GENE_VERSION_PATTERN = re.compile(r"^ENSG(?P\d+)\.(?P\d+)$", re.IGNORECASE) diff --git a/genes/signals/gene_symbol_search.py b/genes/signals/gene_symbol_search.py index 35a58f623..46572b6c0 100644 --- a/genes/signals/gene_symbol_search.py +++ b/genes/signals/gene_symbol_search.py @@ -7,8 +7,13 @@ from classification.models.classification_utils import classification_gene_symbol_filter from classification.signals.classification_search import classification_qs_to_extras from genes.models import GeneSymbol, GeneSymbolAlias -from library.preview_request import preview_extra_signal, PreviewKeyValue -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, SearchResultMatchStrength +from library.preview_request import PreviewKeyValue, preview_extra_signal +from snpdb.search import ( + SearchExample, + SearchInputInstance, + SearchResultMatchStrength, + search_receiver, +) GENE_SYMBOL_PATTERN = re.compile(r"^[a-zA-Z][\da-zA-Z\-\.]+$") diff --git a/genes/signals/manual_signals.py b/genes/signals/manual_signals.py index 720b30ae2..7df7673de 100644 --- a/genes/signals/manual_signals.py +++ b/genes/signals/manual_signals.py @@ -4,11 +4,20 @@ from annotation.models.models import CachedWebResource from genes.models import GeneListCategory -from genes.tasks.cached_web_resource_tasks import PanelAppEnglandPanelsWebResourceTask, \ - PanelAppAustraliaPanelsWebResourceTask, GnomADGeneConstraintWebResourceTask, PfamWebResourceTask, \ - UniProtWebResourceTask, RefSeqGeneSummaryWebResourceTask, HGNCWebResourceTask, LRGRefSeqGeneWebResourceTask, \ - RefSeqGeneInfoWebResourceTask, RefSeqSequenceInfoWebResourceTask, MANEWebResourceTask, \ - RefSeqGenePubMedCountWebResourceTask +from genes.tasks.cached_web_resource_tasks import ( + GnomADGeneConstraintWebResourceTask, + HGNCWebResourceTask, + LRGRefSeqGeneWebResourceTask, + MANEWebResourceTask, + PanelAppAustraliaPanelsWebResourceTask, + PanelAppEnglandPanelsWebResourceTask, + PfamWebResourceTask, + RefSeqGeneInfoWebResourceTask, + RefSeqGenePubMedCountWebResourceTask, + RefSeqGeneSummaryWebResourceTask, + RefSeqSequenceInfoWebResourceTask, + UniProtWebResourceTask, +) # Remember to connect these handlers up in apps.GenesConfig # For some reason this doesn't work as a variable, has to be stored here... diff --git a/genes/signals/transcript_search.py b/genes/signals/transcript_search.py index 24f24756f..e8f310208 100644 --- a/genes/signals/transcript_search.py +++ b/genes/signals/transcript_search.py @@ -2,7 +2,7 @@ from genes.models import Transcript, TranscriptVersion from library.utils import first -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver TRANSCRIPT_PATTERN = re.compile(r"^(ENST|NM_|NR_|XR_)\d+\.?\d*$", re.IGNORECASE) diff --git a/genes/tasks/cached_web_resource_tasks.py b/genes/tasks/cached_web_resource_tasks.py index b0f392223..4827b30f8 100644 --- a/genes/tasks/cached_web_resource_tasks.py +++ b/genes/tasks/cached_web_resource_tasks.py @@ -4,8 +4,12 @@ from genes.cached_web_resource.lrg_ref_seq_gene import store_lrg_ref_seq_gene_from_web from genes.cached_web_resource.mane import store_mane_from_web from genes.cached_web_resource.pfam import store_pfam_from_web -from genes.cached_web_resource.refseq import store_refseq_gene_summary_from_web, store_refseq_gene_info_from_web, \ - store_refseq_sequence_info_from_web, store_gene2pubmed_from_web +from genes.cached_web_resource.refseq import ( + store_gene2pubmed_from_web, + store_refseq_gene_info_from_web, + store_refseq_gene_summary_from_web, + store_refseq_sequence_info_from_web, +) from genes.cached_web_resource.uniprot import store_uniprot_from_web from genes.models import PanelAppServer from genes.panel_app import store_panel_app_panels_from_web diff --git a/genes/tasks/gene_coverage_tasks.py b/genes/tasks/gene_coverage_tasks.py index d36a976f2..08c944890 100644 --- a/genes/tasks/gene_coverage_tasks.py +++ b/genes/tasks/gene_coverage_tasks.py @@ -7,7 +7,7 @@ from genes.canonical_transcripts.canonical_transcript_manager import CanonicalTranscriptManager from genes.gene_matching import GeneSymbolMatcher -from genes.models import GeneCoverageCollection, GeneCoverageCanonicalTranscript, TranscriptVersion +from genes.models import GeneCoverageCanonicalTranscript, GeneCoverageCollection, TranscriptVersion from seqauto.models import EnrichmentKit from snpdb.models import DataState diff --git a/genes/templatetags/gene_grid_tags.py b/genes/templatetags/gene_grid_tags.py index e48069b87..ec068a2ca 100644 --- a/genes/templatetags/gene_grid_tags.py +++ b/genes/templatetags/gene_grid_tags.py @@ -2,9 +2,14 @@ from django.db.models.query_utils import Q from django.template import Library -from genes.forms import GeneListCategoryAutocompleteForm, NamedCustomGeneListForm, GeneSymbolForm, \ - GeneAnnotationReleaseForm, panel_app_server_autocomplete_form_factory -from genes.models import GeneInfo, GeneListCategory, PanelAppServer, GeneAnnotationRelease +from genes.forms import ( + GeneAnnotationReleaseForm, + GeneListCategoryAutocompleteForm, + GeneSymbolForm, + NamedCustomGeneListForm, + panel_app_server_autocomplete_form_factory, +) +from genes.models import GeneAnnotationRelease, GeneInfo, GeneListCategory, PanelAppServer from library.log_utils import log_traceback from ontology.forms import HPOForm, OMIMForm from pathtests.forms import ActivePathologyTestForm, SelectPathologyTestVersionForm diff --git a/genes/templatetags/panel_app_tags.py b/genes/templatetags/panel_app_tags.py index dd2765b75..c1f2b4647 100644 --- a/genes/templatetags/panel_app_tags.py +++ b/genes/templatetags/panel_app_tags.py @@ -1,6 +1,6 @@ from django.template import Library -from genes.models import PanelAppServer, GeneSymbol +from genes.models import GeneSymbol, PanelAppServer register = Library() diff --git a/genes/tests/test_clean_hgvs.py b/genes/tests/test_clean_hgvs.py index dd156f751..282f5e4ca 100644 --- a/genes/tests/test_clean_hgvs.py +++ b/genes/tests/test_clean_hgvs.py @@ -1,6 +1,6 @@ from django.test import TestCase, override_settings -from genes.hgvs import HGVSMatcher, HGVSConverterType +from genes.hgvs import HGVSConverterType, HGVSMatcher from snpdb.models import GenomeBuild diff --git a/genes/tests/test_gene_matching.py b/genes/tests/test_gene_matching.py index 20601a6f4..15512ff34 100644 --- a/genes/tests/test_gene_matching.py +++ b/genes/tests/test_gene_matching.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import User from django.test.testcases import TestCase -from genes.gene_matching import GeneSymbolMatcher, MAX_GENE_SYMBOL_LENGTH, tokenize_gene_symbols +from genes.gene_matching import MAX_GENE_SYMBOL_LENGTH, GeneSymbolMatcher, tokenize_gene_symbols from genes.models import GeneList, GeneSymbol diff --git a/genes/tests/test_genes_adversarial.py b/genes/tests/test_genes_adversarial.py index e90d10293..607a4f20a 100644 --- a/genes/tests/test_genes_adversarial.py +++ b/genes/tests/test_genes_adversarial.py @@ -6,23 +6,23 @@ No DB needed for most tests (pure Python / mock-based). """ import types + from django.test import TestCase +from genes.gene_matching import tokenize_gene_symbols from genes.hgvs.hgvs import ( CHGVS, PHGVS, CHGVSDiff, chgvs_diff_description, ) -from genes.gene_matching import tokenize_gene_symbols +from genes.models import TranscriptVersion from genes.transcripts_utils import ( get_refseq_type, - looks_like_transcript, looks_like_hgvs_prefix, + looks_like_transcript, transcript_is_lrg, ) -from genes.models import TranscriptVersion - # --------------------------------------------------------------------------- # Helpers diff --git a/genes/tests/test_hgvs.py b/genes/tests/test_hgvs.py index 65776cbcf..803f76136 100644 --- a/genes/tests/test_hgvs.py +++ b/genes/tests/test_hgvs.py @@ -7,8 +7,11 @@ from pyhgvs import HGVSName # This is used for pyhgvs specific test from annotation.fake_annotation import get_fake_annotation_version -from annotation.tests.test_data_fake_genes import create_fake_transcript_version, create_gata2_transcript_version -from genes.hgvs import HGVSMatcher, HGVSException, HGVSConverterType +from annotation.tests.test_data_fake_genes import ( + create_fake_transcript_version, + create_gata2_transcript_version, +) +from genes.hgvs import HGVSConverterType, HGVSException, HGVSMatcher from genes.hgvs.hgvs_matcher import FakeTranscriptVersion from genes.hgvs.pyhgvs.hgvs_converter_pyhgvs import PyHGVSVariant from snpdb.models import GenomeBuild, VariantCoordinate @@ -155,11 +158,11 @@ def test_sort_transcript_versions(self): version = 4 key_up_then_down = HGVSMatcher._get_sort_key_transcript_version_and_methods(version) - sorted_up_then_down = list(sorted(transcript_version_and_methods, key=key_up_then_down)) + sorted_up_then_down = sorted(transcript_version_and_methods, key=key_up_then_down) self.assertEqual(sorted_up_then_down, expected_up_then_down, "Sorted up then down") key_closest = HGVSMatcher._get_sort_key_transcript_version_and_methods(version, closest=True) - sorted_closest = list(sorted(transcript_version_and_methods, key=key_closest)) + sorted_closest = sorted(transcript_version_and_methods, key=key_closest) self.assertEqual(sorted_closest, expected_closest, "Sorted closest") def test_hgvs_pyhgvs(self): diff --git a/genes/tests/test_panel_app.py b/genes/tests/test_panel_app.py index 530f90e31..5af5c6e0d 100644 --- a/genes/tests/test_panel_app.py +++ b/genes/tests/test_panel_app.py @@ -2,7 +2,12 @@ from django.test import TestCase -from genes.models import GeneSymbol, PanelAppServer, PanelAppPanel, PanelAppPanelLocalCacheGeneSymbol +from genes.models import ( + GeneSymbol, + PanelAppPanel, + PanelAppPanelLocalCacheGeneSymbol, + PanelAppServer, +) from genes.panel_app import get_panel_app_local_cache diff --git a/genes/tests/test_urls.py b/genes/tests/test_urls.py index a2e8ff3ef..278d47c15 100644 --- a/genes/tests/test_urls.py +++ b/genes/tests/test_urls.py @@ -8,12 +8,22 @@ from annotation.fake_annotation import get_fake_annotation_version from annotation.models import CachedWebResource from annotation.tests.test_data_fake_genes import create_fake_transcript_version -from genes.models import CanonicalTranscriptCollection, GeneCoverageCollection, GeneCoverage, GeneList, PanelAppPanel, \ - GeneListCategory, Gene, CanonicalTranscript, GeneListGeneSymbol, PanelAppServer +from genes.models import ( + CanonicalTranscript, + CanonicalTranscriptCollection, + Gene, + GeneCoverage, + GeneCoverageCollection, + GeneList, + GeneListCategory, + GeneListGeneSymbol, + PanelAppPanel, + PanelAppServer, +) from genes.models_enums import AnnotationConsortium from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings -from ontology.models import OntologyService, OntologyImport, OntologyTerm -from snpdb.models import ImportStatus, DataState +from ontology.models import OntologyImport, OntologyService, OntologyTerm +from snpdb.models import DataState, ImportStatus from snpdb.models.models_genome import GenomeBuild from snpdb.tests.utils.fake_cohort_data import create_fake_trio diff --git a/genes/urls.py b/genes/urls.py index 743cd2b33..fe9e10d52 100644 --- a/genes/urls.py +++ b/genes/urls.py @@ -1,9 +1,21 @@ -from genes.grids import GeneListGenesColumns, GenesGrid, QCGeneCoverageGrid, \ - UncoveredGenesGrid, GeneSymbolVariantsGrid, GeneSymbolWikiColumns, \ - GeneListColumns, CanonicalTranscriptCollectionColumns, CanonicalTranscriptColumns +from genes.grids import ( + CanonicalTranscriptCollectionColumns, + CanonicalTranscriptColumns, + GeneListColumns, + GeneListGenesColumns, + GenesGrid, + GeneSymbolVariantsGrid, + GeneSymbolWikiColumns, + QCGeneCoverageGrid, + UncoveredGenesGrid, +) from genes.views import views, views_autocomplete, views_rest -from genes.views.views_hotspot_graphs import HotspotGraphView, ClassificationsHotspotGraphView, CohortHotspotGraphView, \ - PublicRUNX1HotspotGraphView +from genes.views.views_hotspot_graphs import ( + ClassificationsHotspotGraphView, + CohortHotspotGraphView, + HotspotGraphView, + PublicRUNX1HotspotGraphView, +) from library.django_utils.jqgrid_view import JQGridView from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/genes/views/views.py b/genes/views/views.py index 07827707b..9579bc6db 100644 --- a/genes/views/views.py +++ b/genes/views/views.py @@ -1,10 +1,11 @@ import operator from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timedelta from functools import cached_property, reduce from itertools import combinations -from typing import Optional, Iterable, Union, Any +from typing import Any, Optional, Union from django.conf import settings from django.contrib import messages @@ -12,7 +13,7 @@ from django.db.models.aggregates import Count from django.db.models.query_utils import Q from django.http.response import JsonResponse -from django.shortcuts import render, get_object_or_404, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.datastructures import OrderedSet from django.views.decorators.cache import cache_page @@ -20,31 +21,58 @@ from analysis.models import VariantTag from annotation.models import Citation -from annotation.models.models import AnnotationVersion, DBNSFPGeneAnnotationVersion, DBNSFPGeneAnnotation +from annotation.models.models import ( + AnnotationVersion, + DBNSFPGeneAnnotation, + DBNSFPGeneAnnotationVersion, +) from classification.models import ClassificationModification from classification.models.classification_utils import classification_gene_symbol_filter from classification.views.exports import ClassificationExportFormatterCSV from classification.views.exports.classification_export_filter import ClassificationFilter from classification.views.exports.classification_export_formatter_csv import FormatDetailsCSV from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.forms import GeneListForm, NamedCustomGeneListForm, UserGeneListForm, CustomGeneListForm, \ - GeneSymbolForm, GeneAnnotationReleaseGenomeBuildForm +from genes.forms import ( + CustomGeneListForm, + GeneAnnotationReleaseGenomeBuildForm, + GeneListForm, + GeneSymbolForm, + NamedCustomGeneListForm, + UserGeneListForm, +) from genes.graphs.gene_list_chromosome_graph import GeneListChromosomeGraph from genes.hgvs import HGVSMatcher -from genes.models import GeneInfo, CanonicalTranscriptCollection, GeneListCategory, \ - GeneList, GeneCoverageCollection, GeneCoverageCanonicalTranscript, \ - CustomTextGeneList, Transcript, Gene, TranscriptVersion, GeneSymbol, GeneCoverage, \ - PanelAppServer, SampleGeneList, HGNC, GeneVersion, TranscriptVersionSequenceInfo, NoTranscript, GnomADGeneConstraint +from genes.models import ( + HGNC, + CanonicalTranscriptCollection, + CustomTextGeneList, + Gene, + GeneCoverage, + GeneCoverageCanonicalTranscript, + GeneCoverageCollection, + GeneInfo, + GeneList, + GeneListCategory, + GeneSymbol, + GeneVersion, + GnomADGeneConstraint, + NoTranscript, + PanelAppServer, + SampleGeneList, + Transcript, + TranscriptVersion, + TranscriptVersionSequenceInfo, +) from genes.models_enums import AnnotationConsortium from genes.serializers import SampleGeneListSerializer from library.constants import WEEK_SECS -from library.django_utils import get_field_counts, add_save_message -from library.utils import defaultdict_to_dict, LazyAttribute, full_class_name -from ontology.models import OntologySnake, OntologyService, OntologyTerm +from library.django_utils import add_save_message, get_field_counts +from library.utils import LazyAttribute, defaultdict_to_dict, full_class_name +from ontology.models import OntologyService, OntologySnake, OntologyTerm from seqauto.models import EnrichmentKit from snpdb.genome_build_manager import GenomeBuildManager from snpdb.graphs import graphcache -from snpdb.models import VariantZygosityCountCollection, Sample, VariantGridColumn +from snpdb.models import Sample, VariantGridColumn, VariantZygosityCountCollection from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_user_settings import UserSettings from snpdb.variant_queries import get_has_classifications_q, get_variant_queryset_for_gene_symbol @@ -337,7 +365,7 @@ def unmatched_classifications(self) -> QuerySet[ClassificationModification]: ).filter(reduce(operator.or_, evidence_q_list)) classifications_qs = ClassificationModification.filter_for_user(user=self.user, queryset=classifications_qs) classifications_qs = classifications_qs.select_related('classification', 'classification__lab') - return list(sorted(classifications_qs[0:100], key=lambda c: c.curated_date_check, reverse=True)) + return sorted(classifications_qs[0:100], key=lambda c: c.curated_date_check, reverse=True) @cached_property def unmatched_classifications_title(self): diff --git a/genes/views/views_autocomplete.py b/genes/views/views_autocomplete.py index f25f40000..c57c63ecc 100644 --- a/genes/views/views_autocomplete.py +++ b/genes/views/views_autocomplete.py @@ -6,8 +6,16 @@ from django.views.decorators.cache import cache_page from django.views.decorators.vary import vary_on_cookie -from genes.models import PanelAppPanel, GeneList, GeneSymbol, Gene, Transcript, GeneAnnotationRelease, PanelAppServer -from library.constants import HOUR_SECS, WEEK_SECS, MINUTE_SECS +from genes.models import ( + Gene, + GeneAnnotationRelease, + GeneList, + GeneSymbol, + PanelAppPanel, + PanelAppServer, + Transcript, +) +from library.constants import HOUR_SECS, MINUTE_SECS, WEEK_SECS from library.django_utils.autocomplete_utils import AutocompleteView diff --git a/genes/views/views_hotspot_graphs.py b/genes/views/views_hotspot_graphs.py index 475940b37..b2fd45fff 100644 --- a/genes/views/views_hotspot_graphs.py +++ b/genes/views/views_hotspot_graphs.py @@ -15,11 +15,11 @@ from annotation.models.models import AnnotationVersion, VariantAnnotation, VariantAnnotationVersion from annotation.models.molecular_consequence_enums import MolecularConsequenceColors from classification.enums import ShareLevel -from classification.models import ClassificationModification, Classification -from genes.models import Transcript, Gene, TranscriptVersion +from classification.models import Classification, ClassificationModification +from genes.models import Gene, Transcript, TranscriptVersion from library.constants import MINUTE_SECS from library.utils import segment -from snpdb.models import CohortGenotypeCollection, Cohort, VariantZygosityCountCollection +from snpdb.models import Cohort, CohortGenotypeCollection, VariantZygosityCountCollection from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_variant import Variant @@ -105,7 +105,7 @@ def _pick_transcripts(self) -> tuple[Optional[TranscriptVersion], str, list[tupl raise ValueError("At least one of 'gene_symbol', 'gene_id' or 'transcript_id' must be in url kwargs") # Sort by canonical then choose higher version if any ties - tv_list = list(sorted(tv_qs, key=lambda tv: (tv.canonical_score, tv.version), reverse=True)) + tv_list = sorted(tv_qs, key=lambda tv: (tv.canonical_score, tv.version), reverse=True) if tv_list: # First, we look for canonical in our gene annotation release canonical, non_canonical = segment(tv_list, filter_func=lambda tv: tv.canonical_score) diff --git a/genes/views/views_rest.py b/genes/views/views_rest.py index ff71dce76..71c32bf5e 100644 --- a/genes/views/views_rest.py +++ b/genes/views/views_rest.py @@ -8,21 +8,35 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework import permissions -from rest_framework.generics import ListAPIView -from rest_framework.generics import RetrieveAPIView +from rest_framework.generics import ListAPIView, RetrieveAPIView from rest_framework.response import Response from rest_framework.status import HTTP_200_OK from rest_framework.views import APIView from genes.gene_matching import GeneSymbolMatcher -from genes.models import GeneInfo, GeneList, GeneAnnotationRelease, \ - ReleaseGeneSymbolGene, PanelAppServer, SampleGeneList, ActiveSampleGeneList, create_fake_gene_list -from genes.panel_app import get_panel_app_panel_as_gene_list_json -from genes.panel_app import get_panel_app_results_by_gene_symbol_json -from genes.serializers import GeneInfoSerializer, GeneListGeneSymbolSerializer, GeneListSerializer, \ - GeneAnnotationReleaseSerializer, SampleGeneListSerializer +from genes.models import ( + ActiveSampleGeneList, + GeneAnnotationRelease, + GeneInfo, + GeneList, + PanelAppServer, + ReleaseGeneSymbolGene, + SampleGeneList, + create_fake_gene_list, +) +from genes.panel_app import ( + get_panel_app_panel_as_gene_list_json, + get_panel_app_results_by_gene_symbol_json, +) +from genes.serializers import ( + GeneAnnotationReleaseSerializer, + GeneInfoSerializer, + GeneListGeneSymbolSerializer, + GeneListSerializer, + SampleGeneListSerializer, +) from library.constants import HOUR_SECS, WEEK_SECS from library.django_utils.django_rest_utils import MultipleFieldLookupMixin from library.guardian_utils import DjangoPermission @@ -114,7 +128,7 @@ def post(self, request, **kwargs): gene_additions, gene_deletions = self.get_gene_list_modifications(request.data) modification_info = f"Added manually by {request.user} on {timezone.now()}" - gene_additions_modification_info = {gene: modification_info for gene in gene_additions} + gene_additions_modification_info = dict.fromkeys(gene_additions, modification_info) num_added, num_deleted = gene_list.add_and_remove_gene_symbols(gene_additions, gene_deletions, gene_additions_modification_info) return Response(status=HTTP_200_OK, data={"num_added": num_added, "num_deleted": num_deleted}) diff --git a/library/django_utils/django_object_managers.py b/library/django_utils/django_object_managers.py index 7a04c35d2..9db3c634f 100644 --- a/library/django_utils/django_object_managers.py +++ b/library/django_utils/django_object_managers.py @@ -1,11 +1,11 @@ -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, Optional from django.conf import settings from django.db.models import Manager, QuerySet from frozendict import frozendict -from threadlocals.threadlocals import set_request_variable, get_request_variable +from threadlocals.threadlocals import get_request_variable, set_request_variable @dataclass(frozen=True) diff --git a/library/django_utils/django_partition.py b/library/django_utils/django_partition.py index 213c00688..31cea398a 100644 --- a/library/django_utils/django_partition.py +++ b/library/django_utils/django_partition.py @@ -6,7 +6,7 @@ from django.utils.text import slugify from library.log_utils import log_traceback -from library.utils import single_quote, double_quote +from library.utils import double_quote, single_quote from library.utils.database_utils import run_sql diff --git a/library/django_utils/django_postgres.py b/library/django_utils/django_postgres.py index 1fed27c0c..38d7ab14d 100644 --- a/library/django_utils/django_postgres.py +++ b/library/django_utils/django_postgres.py @@ -40,7 +40,7 @@ def model_to_insert_sql(model_list: list[Model], db_table: str = None, ignore_fi compiler = q.get_compiler('default') # Normally, execute sets this, but we don't want to call execute - setattr(compiler, 'return_id', False) + compiler.return_id = False raw_statements = compiler.as_sql() model._meta.db_table = old_table # Put table name back diff --git a/library/django_utils/django_queryset_sql_transformer.py b/library/django_utils/django_queryset_sql_transformer.py index 077305e38..12962647e 100644 --- a/library/django_utils/django_queryset_sql_transformer.py +++ b/library/django_utils/django_queryset_sql_transformer.py @@ -1,4 +1,4 @@ -from typing import Type, Any +from typing import Any from django.db import connections from django.db.models.query import QuerySet @@ -27,7 +27,7 @@ class TransformerSQLUpdateCompiler(TransformerSQLMixin, SQLUpdateCompiler): pass class QueryTransformerCompilerMixin: - compiler_klass: Type[Any] + compiler_klass: type[Any] def get_compiler(self, using=None, connection=None, elide_empty=True): if using is None and connection is None: diff --git a/library/django_utils/django_rest_utils.py b/library/django_utils/django_rest_utils.py index d00c09d30..8a9eb6b2c 100644 --- a/library/django_utils/django_rest_utils.py +++ b/library/django_utils/django_rest_utils.py @@ -1,6 +1,7 @@ from rest_framework import serializers from rest_framework.generics import get_object_or_404 + class MultipleFieldLookupMixin: """ Apply this mixin to any view or viewset to get multiple field filtering diff --git a/library/django_utils/guardian_permissions_mixin.py b/library/django_utils/guardian_permissions_mixin.py index e6a76fc5d..e4bd49d90 100644 --- a/library/django_utils/guardian_permissions_mixin.py +++ b/library/django_utils/guardian_permissions_mixin.py @@ -4,9 +4,9 @@ from django.contrib.auth.models import Group, User from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404 -from guardian.shortcuts import get_objects_for_user, get_objects_for_group, get_group_perms +from guardian.shortcuts import get_group_perms, get_objects_for_group, get_objects_for_user -from library.guardian_utils import assign_permission_to_user_and_groups, DjangoPermission +from library.guardian_utils import DjangoPermission, assign_permission_to_user_and_groups class GuardianPermissionsMixin: diff --git a/library/django_utils/jqgrid_view.py b/library/django_utils/jqgrid_view.py index ba810cf6f..296f4e31b 100644 --- a/library/django_utils/jqgrid_view.py +++ b/library/django_utils/jqgrid_view.py @@ -1,14 +1,15 @@ import csv import json -from typing import Type, Any, Optional, Iterator +from collections.abc import Iterator +from typing import Any, Optional from django.core.exceptions import PermissionDenied -from django.http.response import JsonResponse, HttpResponse, StreamingHttpResponse +from django.http.response import HttpResponse, JsonResponse, StreamingHttpResponse from django.urls.base import resolve, reverse from django.utils.text import slugify from django.views.generic.base import View -from library.utils import nice_class_name, StashFile +from library.utils import StashFile, nice_class_name from library.utils.date_utils import local_date_string @@ -26,7 +27,7 @@ class JQGridView(View): perm_path('analyses/grid//', JQGridView.as_view(grid=AnalysesGrid, delete=True), name='analyses_grid'), """ - grid: Optional[Type[Any]] = None # JqGridUserRowConfig (or grid initialised w/user, has delete_row method) + grid: Optional[type[Any]] = None # JqGridUserRowConfig (or grid initialised w/user, has delete_row method) delete_row = False csv_download = False # via request - can also do via JSON @@ -53,7 +54,7 @@ def _load_grid(self, request, *args, **kwargs): msg = f"{nice_class_name(self)}.grid not set" raise ValueError(msg) else: - grid_klass: Type[Any] = self.grid + grid_klass: type[Any] = self.grid return self.create_grid_from_request(request, grid_klass, **kwargs) diff --git a/library/django_utils/tests/test_major_operation.py b/library/django_utils/tests/test_major_operation.py index 7a5f32f92..daa6c7cb6 100644 --- a/library/django_utils/tests/test_major_operation.py +++ b/library/django_utils/tests/test_major_operation.py @@ -9,7 +9,7 @@ from django.core.cache import cache from django.test import TestCase, override_settings -from library.django_utils.major_operation import major_operation, TooManyMajorOperationsError +from library.django_utils.major_operation import TooManyMajorOperationsError, major_operation LOCMEM_CACHE = { "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, diff --git a/library/enums/time_enums.py b/library/enums/time_enums.py index 9acfbc9a8..0ea6a762b 100644 --- a/library/enums/time_enums.py +++ b/library/enums/time_enums.py @@ -1,4 +1,4 @@ -from django.db.models.functions import TruncHour, TruncDay, TruncWeek, TruncMonth, TruncYear +from django.db.models.functions import TruncDay, TruncHour, TruncMonth, TruncWeek, TruncYear class TimePeriod: diff --git a/library/forms.py b/library/forms.py index d169add09..0f5ddbfc8 100644 --- a/library/forms.py +++ b/library/forms.py @@ -5,6 +5,7 @@ from django import forms from django.forms.widgets import TextInput + class NumberInput(TextInput): input_type = 'number' diff --git a/library/genomics/calculate_cancer_mutation_signatures.py b/library/genomics/calculate_cancer_mutation_signatures.py index 6706e3413..cd1d491d5 100644 --- a/library/genomics/calculate_cancer_mutation_signatures.py +++ b/library/genomics/calculate_cancer_mutation_signatures.py @@ -234,7 +234,7 @@ def calculate_signatures(self): # --------------------------------------------------------------------------- # parse the signature data def ParseSignatureData(self, sig_data_file): - with open(sig_data_file, "r", encoding="utf-8") as fin: + with open(sig_data_file, encoding="utf-8") as fin: # how many columns are there? how many signatures? header = fin.readline() for idx, col_name in enumerate(header.strip().split("\t")[3:]): diff --git a/library/genomics/vcf_utils.py b/library/genomics/vcf_utils.py index 52f02fe2e..9a4573369 100644 --- a/library/genomics/vcf_utils.py +++ b/library/genomics/vcf_utils.py @@ -3,15 +3,16 @@ import os import re from collections import defaultdict -from typing import Optional, Iterable, IO, Union +from collections.abc import Iterable +from typing import IO, Optional, Union import cyvcf2 import vcf from django.conf import settings from library.genomics.vcf_enums import VCFSymbolicAllele -from library.utils import open_handle_gzip, open_file_or_filename -from snpdb.models import Variant, Sequence, GenomeFasta, SequenceRole, VariantCoordinate +from library.utils import open_file_or_filename, open_handle_gzip +from snpdb.models import GenomeFasta, Sequence, SequenceRole, Variant, VariantCoordinate def cyvcf2_header_types(cyvcf2_reader) -> defaultdict: diff --git a/library/guardian_utils.py b/library/guardian_utils.py index 4754cc9ff..9ec2b6661 100644 --- a/library/guardian_utils.py +++ b/library/guardian_utils.py @@ -5,7 +5,7 @@ from django.contrib.auth.models import Group, User from django.core.exceptions import PermissionDenied from django.db.models import Model, QuerySet -from guardian.shortcuts import get_groups_with_perms, get_users_with_perms, remove_perm, assign_perm +from guardian.shortcuts import assign_perm, get_groups_with_perms, get_users_with_perms, remove_perm def is_superuser(user): @@ -25,7 +25,7 @@ def public_group(): return g -@lru_cache() +@lru_cache def _cached_admin_bot() -> User: return User.objects.get(username='admin_bot') diff --git a/library/health_check.py b/library/health_check.py index 6f69a8b1d..88f603a95 100644 --- a/library/health_check.py +++ b/library/health_check.py @@ -1,8 +1,8 @@ import itertools from abc import ABC from dataclasses import dataclass -from datetime import timedelta, datetime -from typing import Union, Optional, Type +from datetime import datetime, timedelta +from typing import Optional, Union import django.dispatch from django.db.models import Model, Q @@ -11,7 +11,7 @@ from library.log_utils import NotificationBuilder from library.preview_request import PreviewData -from library.utils import flatten_nested_lists, model_has_field, limit_str +from library.utils import flatten_nested_lists, limit_str, model_has_field """ HealthChecks are generated nightly and posted in Slack. @@ -103,7 +103,7 @@ def sort_order(cls): @staticmethod def simple_report( health_request: HealthCheckRequest, - model: Type[Model], + model: type[Model], emoji: str, created: bool = False, modified: bool = False) -> list['HealthCheckRecentActivity']: diff --git a/library/jqgrid/jqgrid.py b/library/jqgrid/jqgrid.py index 7652e335d..d2653c394 100644 --- a/library/jqgrid/jqgrid.py +++ b/library/jqgrid/jqgrid.py @@ -34,9 +34,9 @@ from functools import reduce from django.core.exceptions import FieldError, ImproperlyConfigured -from django.core.paginator import Paginator, InvalidPage +from django.core.paginator import InvalidPage, Paginator from django.core.serializers.json import DjangoJSONEncoder -from django.db.models import fields, JSONField, F +from django.db.models import F, JSONField, fields from django.db.models.fields.json import KeyTransform from django.db.models.query_utils import Q from django.utils.encoding import smart_str diff --git a/library/log_utils.py b/library/log_utils.py index dd75f44b7..4e80c7f4d 100644 --- a/library/log_utils.py +++ b/library/log_utils.py @@ -7,13 +7,12 @@ from abc import ABC, abstractmethod from logging import StreamHandler from re import Match -from typing import Optional, Any, Union +from typing import Any, Optional, Union import markdown import requests import rollbar from django.conf import settings -from django.contrib.admin.options import get_content_type_for_model from django.contrib.auth.models import User from django.db.models import Model from django.forms import ModelForm diff --git a/library/preview_request.py b/library/preview_request.py index d3233e139..6534fdcca 100644 --- a/library/preview_request.py +++ b/library/preview_request.py @@ -1,7 +1,8 @@ +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from functools import cached_property -from typing import Optional, Union, Any, Callable, Type +from typing import Any, Optional, Union from django.db.models import Model from django.dispatch import Signal @@ -169,7 +170,7 @@ def preview(self) -> 'PreviewData': return self.preview_with() -PreviewCoordinator = Union[Type[PreviewModelMixin], PreviewProxyModel] +PreviewCoordinator = Union[type[PreviewModelMixin], PreviewProxyModel] @dataclass(eq=True) diff --git a/library/tests/test_utils.py b/library/tests/test_utils.py index a94a0f213..9c19113e9 100644 --- a/library/tests/test_utils.py +++ b/library/tests/test_utils.py @@ -17,13 +17,13 @@ server_side_format_percent, ) from library.utils.collection_utils import ( - LimitedCollection, IterableStitcher, + LimitedCollection, batch_iterator, flatten_nested_lists, group_by_key, ) -from library.utils.date_utils import calculate_age, parse_yymm, month_range +from library.utils.date_utils import calculate_age, month_range, parse_yymm from library.utils.file_utils import IteratorFile, file_to_array from library.utils.hash_utils import sha256sum_str, string_deterministic_hash from library.utils.json_utils import JsonDiffs, make_json_safe_in_place, strip_json @@ -35,7 +35,6 @@ split_dict_multi_values, ) - # --------------------------------------------------------------------------- # unit_percent.py # --------------------------------------------------------------------------- diff --git a/library/uptime_check.py b/library/uptime_check.py index 76834389d..a899ffbda 100644 --- a/library/uptime_check.py +++ b/library/uptime_check.py @@ -8,7 +8,6 @@ from library.utils import text_utils - ### # Uptime Check is similar to health check but # The results is to be displayed to a public web page, so it needs to not show sensitive information diff --git a/library/utils/collection_utils.py b/library/utils/collection_utils.py index 4518f36bd..c754eb926 100644 --- a/library/utils/collection_utils.py +++ b/library/utils/collection_utils.py @@ -3,9 +3,10 @@ import re from abc import ABC from collections import defaultdict +from collections.abc import Callable, Iterable, Iterator, Sequence from dataclasses import dataclass from itertools import islice -from typing import Iterable, Iterator, TypeVar, Any, Generic, Callable, Optional, Sequence, Union +from typing import Any, Generic, Optional, TypeVar, Union from django.utils.functional import SimpleLazyObject @@ -409,6 +410,6 @@ def __getitem__(self, item): def __eq__(self, other): # default implementation assumes all parts implement __eq__ - if not type(self) is type(other): + if type(self) is not type(other): return False return self.as_tuple == other.as_tuple diff --git a/library/utils/database_utils.py b/library/utils/database_utils.py index 9544d82c6..3ee5a0498 100644 --- a/library/utils/database_utils.py +++ b/library/utils/database_utils.py @@ -1,7 +1,9 @@ -from typing import Iterable, Optional, Any +from collections.abc import Iterable +from typing import Any, Optional import sqlparse from django.db import connection, transaction + # 970: Added transaction wrapper due to Postgres hanging query from django.db.models import QuerySet diff --git a/library/utils/date_utils.py b/library/utils/date_utils.py index fc09e8c5a..66f35da4a 100644 --- a/library/utils/date_utils.py +++ b/library/utils/date_utils.py @@ -1,4 +1,4 @@ -from datetime import date, datetime, timedelta, timezone +from datetime import UTC, date, datetime, timedelta from typing import Optional from dateutil import tz @@ -76,4 +76,4 @@ def parse_http_header_date(date_str: str): def utc_from_timestamp(ts) -> datetime: - return datetime.fromtimestamp(ts, tz=timezone.utc) + return datetime.fromtimestamp(ts, tz=UTC) diff --git a/library/utils/diff_utils.py b/library/utils/diff_utils.py index 9f305af27..c7be9de5a 100644 --- a/library/utils/diff_utils.py +++ b/library/utils/diff_utils.py @@ -3,7 +3,8 @@ import re from dataclasses import dataclass, field from html import escape -from typing import Optional, Any, Pattern +from re import Pattern +from typing import Any, Optional from django.utils.safestring import SafeString diff --git a/library/utils/export_utils.py b/library/utils/export_utils.py index 25f6211d3..8446e115f 100644 --- a/library/utils/export_utils.py +++ b/library/utils/export_utils.py @@ -1,11 +1,12 @@ import inspect import json import re +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum from itertools import chain -from typing import Iterable, Optional, Any, Type, Iterator, Callable, Protocol +from typing import Any, Optional, Protocol from dateutil.tz import gettz from django.conf import settings @@ -26,7 +27,7 @@ class ExportDataType(str, Enum): class ExportCellMethod(Protocol): line_number: int label: str - sub_data: Optional[Type['ExportRow']] + sub_data: Optional[type['ExportRow']] categories: Optional[dict[Any, Any]] data_type: ExportDataType def __call__(self, *args) -> Any: ... @@ -34,7 +35,7 @@ def __call__(self, *args) -> Any: ... def export_column( label: Optional[str] = None, - sub_data: Optional[Type] = None, + sub_data: Optional[type] = None, categories: dict[str, Any] = None, data_type: ExportDataType = ExportDataType.any): """ @@ -227,7 +228,7 @@ def get_export_methods(cls, export_tweak: ExportTweak = ExportTweak.DEFAULT) -> return get_decorated_methods(cls, categories=export_tweak.categories, attribute="is_export") @classmethod - def _data_generator(cls: Type, data: Iterable[Any]) -> Iterator[Any]: + def _data_generator(cls: type, data: Iterable[Any]) -> Iterator[Any]: for row_data in data: if row_data is None: continue diff --git a/library/utils/file_utils.py b/library/utils/file_utils.py index 9c12d469d..b68e9444f 100644 --- a/library/utils/file_utils.py +++ b/library/utils/file_utils.py @@ -4,7 +4,7 @@ import subprocess from hashlib import md5 from pathlib import Path -from typing import Optional, Union, IO +from typing import IO, Optional, Union def open_file_or_filename(f, mode='r', **kwargs): diff --git a/library/utils/json_utils.py b/library/utils/json_utils.py index 2366f1f2f..afca8d1d6 100644 --- a/library/utils/json_utils.py +++ b/library/utils/json_utils.py @@ -1,9 +1,10 @@ import json -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod +from collections.abc import Mapping from dataclasses import dataclass from decimal import Decimal from functools import cached_property -from typing import Union, Any, Mapping, Optional +from typing import Any, Optional, Union def canonical_filter_key(filter_dict: Optional[dict]) -> Optional[str]: @@ -176,7 +177,7 @@ def to_json(self, before_label: str = "before", after_label: str = "after") -> J @staticmethod def differences(obj1: JsonDataType, obj2: JsonDataType) -> 'JsonDiffs': - diffs: list['JsonDiff'] = [] + diffs: list[JsonDiff] = [] JsonDiffs._differences(obj1, obj2, [], diffs) diffs.sort() return JsonDiffs(diffs) diff --git a/library/utils/misc_utils.py b/library/utils/misc_utils.py index 3de60b71d..27c6a28b2 100644 --- a/library/utils/misc_utils.py +++ b/library/utils/misc_utils.py @@ -6,7 +6,7 @@ import time from enum import Enum from json.encoder import JSONEncoder -from typing import TypeVar, Optional, Any +from typing import Any, Optional, TypeVar from urllib.parse import urlparse from django.core.serializers import serialize diff --git a/library/utils/model_utils.py b/library/utils/model_utils.py index 0d6ab154a..3bedad0e7 100644 --- a/library/utils/model_utils.py +++ b/library/utils/model_utils.py @@ -1,4 +1,3 @@ -from typing import Type from django.core.exceptions import FieldDoesNotExist from django.db import models @@ -11,7 +10,7 @@ class ModelUtilsMixin: """ @classmethod - def get(cls: Type[Model], value): + def get(cls: type[Model], value): if value is None: return None if isinstance(value, cls): @@ -33,7 +32,7 @@ class AsciiValue(models.Func): function = 'ASCII' -def model_has_field(model: Type[Model], field_name: str) -> bool: +def model_has_field(model: type[Model], field_name: str) -> bool: try: if field_name.endswith('_id'): field = model._meta.get_field(field_name.strip('_id')) diff --git a/library/utils/nltk_utils.py b/library/utils/nltk_utils.py index 2a1eb4551..730d7985a 100644 --- a/library/utils/nltk_utils.py +++ b/library/utils/nltk_utils.py @@ -1,5 +1,6 @@ import nltk + def ensure_nltk_data(resource): try: nltk.data.find(resource) diff --git a/library/utils/text_utils.py b/library/utils/text_utils.py index 61a254728..bcc17ef59 100644 --- a/library/utils/text_utils.py +++ b/library/utils/text_utils.py @@ -3,7 +3,8 @@ import math import re import string -from typing import Collection, Any, Optional, Callable +from collections.abc import Callable, Collection +from typing import Any, Optional from rich.text import Text diff --git a/library/utils/timer_utils.py b/library/utils/timer_utils.py index 2c656ae91..dad88e7d2 100644 --- a/library/utils/timer_utils.py +++ b/library/utils/timer_utils.py @@ -1,4 +1,4 @@ -from dataclasses import field, dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta from functools import reduce @@ -47,7 +47,7 @@ def tick(self, description: str): self.start = now def __str__(self): - return "\n".join((str(debug_time) for debug_time in self.times.values())) + return "\n".join(str(debug_time) for debug_time in self.times.values()) class NullTimer(DebugTimer): diff --git a/library/utils/xml_utils.py b/library/utils/xml_utils.py index d15d157f6..460d00b59 100644 --- a/library/utils/xml_utils.py +++ b/library/utils/xml_utils.py @@ -1,5 +1,6 @@ import inspect -from typing import Callable, Optional, Any, Union +from collections.abc import Callable +from typing import Any, Optional, Union from lxml import etree from lxml.etree import Element diff --git a/manual/management/commands/manual_complete.py b/manual/management/commands/manual_complete.py index 1769462e6..ce3aee8af 100644 --- a/manual/management/commands/manual_complete.py +++ b/manual/management/commands/manual_complete.py @@ -1,7 +1,7 @@ from django.core.management import CommandParser from django.core.management.base import BaseCommand -from manual.models import ManualMigrationTask, ManualMigrationAttempt +from manual.models import ManualMigrationAttempt, ManualMigrationTask class Command(BaseCommand): diff --git a/manual/management/commands/manual_request.py b/manual/management/commands/manual_request.py index 20b38c0ad..67fa1acf3 100644 --- a/manual/management/commands/manual_request.py +++ b/manual/management/commands/manual_request.py @@ -1,7 +1,7 @@ from django.core.management import CommandParser from django.core.management.base import BaseCommand -from manual.models import ManualMigrationTask, ManualMigrationRequired +from manual.models import ManualMigrationRequired, ManualMigrationTask class Command(BaseCommand): diff --git a/manual/models.py b/manual/models.py index 35f959c8c..0a8c8a6b1 100644 --- a/manual/models.py +++ b/manual/models.py @@ -1,5 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from manual.models.deployment_models import Deployment -from manual.models.manual_migration_models import ManualMigrationTask, ManualMigrationRequired, ManualMigrationAttempt # pylint: enable=unused-import diff --git a/manual/models/manual_migration_models.py b/manual/models/manual_migration_models.py index e62646e7f..d65da3ee1 100644 --- a/manual/models/manual_migration_models.py +++ b/manual/models/manual_migration_models.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from datetime import datetime -from typing import Optional, Any +from typing import Any, Optional from dateutil import tz from django.db import models diff --git a/manual/operations/manual_operations.py b/manual/operations/manual_operations.py index f7e928d0a..cfbf26c65 100644 --- a/manual/operations/manual_operations.py +++ b/manual/operations/manual_operations.py @@ -1,4 +1,5 @@ -from typing import Optional, Union, Callable +from collections.abc import Callable +from typing import Optional, Union from django.db.migrations.operations.base import Operation diff --git a/manual/tests/test_urls.py b/manual/tests/test_urls.py index 690d5f673..fcfc9cfb9 100644 --- a/manual/tests/test_urls.py +++ b/manual/tests/test_urls.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from library.django_utils.unittest_utils import URLTestCase -from manual.models.manual_migration_models import ManualMigrationTask, ManualMigrationAttempt +from manual.models.manual_migration_models import ManualMigrationAttempt, ManualMigrationTask class Test(URLTestCase): diff --git a/manual/views/views.py b/manual/views/views.py index 754c877c5..8f2fb088c 100644 --- a/manual/views/views.py +++ b/manual/views/views.py @@ -10,7 +10,7 @@ from library.django_utils import require_superuser from library.git import Git -from manual.models import ManualMigrationAttempt, ManualMigrationTask, ManualMigrationOutstanding +from manual.models import ManualMigrationAttempt, ManualMigrationOutstanding, ManualMigrationTask from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder """ diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..7f3f3d58f --- /dev/null +++ b/mypy.ini @@ -0,0 +1,42 @@ +[mypy] +# Gradual typing setup. Starts permissive so the existing ~127k LOC don't +# drown new work in errors. Tighten per-app via the [mypy-.*] sections +# below as each app gets cleaned up. +python_version = 3.12 +plugins = + mypy_django_plugin.main, + mypy_drf_plugin.main + +# --- Global leniency (gradual adoption) --- +# Don't force annotations everywhere yet - we add them app-by-app. +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False +# Lots of 3rd-party deps ship no type info (cyvcf2, pysam, hgvs, ...). +ignore_missing_imports = True +# Keep output focused on the files we ask for, not their entire import graph. +follow_imports = silent + +# Useful low-noise checks that catch real bugs even in untyped code. +warn_redundant_casts = True +warn_unused_ignores = False +no_implicit_optional = True +strict_equality = True + +# Skip generated / vendored code. +exclude = (?x)( + /migrations/ + | ^variantgrid/static_files/ + | ^variantgrid/sitestatic/ + ) + +[mypy.plugins.django-stubs] +django_settings_module = variantgrid.settings + +# --- Per-app strictness ratchet --- +# As each app is annotated and made clean, turn these on. snpdb is the +# foundation so it's the first target. +# +# [mypy-snpdb.*] +# disallow_untyped_defs = True +# check_untyped_defs = True diff --git a/ontology/admin.py b/ontology/admin.py index dc857b2c7..e11d6aef4 100644 --- a/ontology/admin.py +++ b/ontology/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from ontology.models import OntologyTerm, OntologyImport, OntologyTermRelation +from ontology.models import OntologyImport, OntologyTerm, OntologyTermRelation from snpdb.admin_utils import ModelAdminBasics diff --git a/ontology/apps.py b/ontology/apps.py index 941bbb64f..7d415d7b3 100644 --- a/ontology/apps.py +++ b/ontology/apps.py @@ -9,8 +9,6 @@ class OntologyConfig(AppConfig): def ready(self): # pylint: disable=import-outside-toplevel,unused-import # imported to activate receivers - from ontology.signals import ontology_health_check - from ontology.signals import ontology_preview from annotation.models import CachedWebResource from ontology.signals.signals import gencc_post_save_handler diff --git a/ontology/gencc.py b/ontology/gencc.py index eaa0ad266..a0cc49e6d 100644 --- a/ontology/gencc.py +++ b/ontology/gencc.py @@ -11,7 +11,7 @@ from library.constants import MINUTE_SECS from library.pandas_utils import df_nan_to_none from library.utils import md5sum_str -from ontology.models import OntologyRelation, GeneDiseaseClassification +from ontology.models import GeneDiseaseClassification, OntologyRelation from ontology.ontology_builder import OntologyBuilder diff --git a/ontology/management/commands/ontology_import.py b/ontology/management/commands/ontology_import.py index 297ab3e68..02b98ff19 100644 --- a/ontology/management/commands/ontology_import.py +++ b/ontology/management/commands/ontology_import.py @@ -16,8 +16,16 @@ from genes.models import HGNC, HGNCImport from library.utils import file_md5sum from ontology.gencc import load_gencc -from ontology.models import OntologyService, OntologyRelation, OntologyTerm, OntologyImportSource, OntologyImport, \ - OntologyTermRelation, OntologyVersion, OntologyTermStatus +from ontology.models import ( + OntologyImport, + OntologyImportSource, + OntologyRelation, + OntologyService, + OntologyTerm, + OntologyTermRelation, + OntologyTermStatus, + OntologyVersion, +) from ontology.ontology_builder import OntologyBuilder, OntologyBuilderDataUpToDateException """ @@ -93,7 +101,7 @@ def load_mondo(filename: str, force: bool): ontology_builder.cache_everything() data_file: dict - with open(filename, 'r') as json_file: + with open(filename) as json_file: data_file = json.load(json_file) node_to_hgnc_id: [str, str] = {} @@ -329,7 +337,7 @@ def load_hpo(filename: str, force: bool): definition=term.definition, primary_source=True, status=status, - aliases=list(sorted(aliases)) + aliases=sorted(aliases) ) children = itertools.islice(term.subclasses(), 1, None) @@ -460,7 +468,7 @@ def load_omim(filename: str, force: bool): ontology_builder.ensure_hash_changed(data_hash=file_hash) # don't re-import if hash hasn't changed ontology_builder.cache_everything() - with open(filename, "r") as csv_file: + with open(filename) as csv_file: csv_reader = csv.reader(csv_file, delimiter='\t') next(csv_reader) # title row next(csv_reader) # date row (worth reading e.g. "Generated: 20201-02-04") diff --git a/ontology/models.py b/ontology/models.py index 396b1557a..0a8c8a6b1 100644 --- a/ontology/models.py +++ b/ontology/models.py @@ -1,4 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from ontology.models.models_ontology import OntologyImport, OntologyTerm, OntologyTermRelation, OntologyVersion # pylint: enable=unused-import diff --git a/ontology/models/models_ontology.py b/ontology/models/models_ontology.py index ecdde8a44..b5c0892fa 100644 --- a/ontology/models/models_ontology.py +++ b/ontology/models/models_ontology.py @@ -6,15 +6,16 @@ import logging import re from collections import defaultdict +from collections.abc import Iterable, Iterator from dataclasses import dataclass from functools import cached_property -from typing import Optional, Union, Iterable, Any, Iterator +from typing import Any, Optional, Union from cache_memoize import cache_memoize from django.conf import settings from django.contrib.postgres.fields import ArrayField -from django.db import models, connection -from django.db.models import PROTECT, CASCADE, QuerySet, Q, Max, TextChoices +from django.db import connection, models +from django.db.models import CASCADE, PROTECT, Max, Q, QuerySet, TextChoices from django.http import Http404 from django.urls import reverse from model_utils.models import TimeStampedModel, now @@ -848,7 +849,7 @@ def moi_and_submitters(self) -> tuple[list[str], list[str]]: for source in extra["sources"]: moi.add(source["mode_of_inheritance"]) submitters.add(source["submitter"]) - return list(sorted(moi)), list(sorted(submitters)) + return sorted(moi), sorted(submitters) @cache_memoize(DAY_SECS) def cached_gene_symbols_for_terms_tuple(self, terms_tuple: tuple[int]) -> QuerySet: @@ -1085,7 +1086,7 @@ def snake_from(term: OntologyTerm, to_ontology: OntologyService, if otr_qs is None: otr_qs = OntologyVersion.get_latest_and_live_ontology_qs() - from ontology.ontology_traversal import bfs_to_ontology, _make_db_step_fn + from ontology.ontology_traversal import _make_db_step_fn, bfs_to_ontology step_fn = _make_db_step_fn(otr_qs, quality_filter) return bfs_to_ontology(term, to_ontology, max_depth, step_fn) @@ -1253,13 +1254,13 @@ def __getitem__(self, item): return self.snakes[item] def leafs(self) -> list[OntologyTerm]: - return list(sorted({snake.leaf_term for snake in self})) + return sorted({snake.leaf_term for snake in self}) def leaf_relations(self, ontology_relation: str = None) -> list[OntologyTermRelation]: relations = {snake.leaf_relationship for snake in self} if ontology_relation: relations = {otr for otr in relations if otr.relation == ontology_relation} - return list(sorted(relations)) + return sorted(relations) class SingleTermH: diff --git a/ontology/models/ontology_search.py b/ontology/models/ontology_search.py index 56cdbefae..8f4892ab4 100644 --- a/ontology/models/ontology_search.py +++ b/ontology/models/ontology_search.py @@ -5,9 +5,16 @@ from django.db.models.functions import Lower from library.preview_request import PreviewProxyModel -from ontology.models import OntologyTerm, OntologyService, OntologyTermStatus -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ALPHA_MIN, SearchResult, \ - SearchResultMatchStrength, SearchMessage +from ontology.models import OntologyService, OntologyTerm, OntologyTermStatus +from snpdb.search import ( + HAS_3_ALPHA_MIN, + SearchExample, + SearchInputInstance, + SearchMessage, + SearchResult, + SearchResultMatchStrength, + search_receiver, +) def validate_ontology(term: OntologyTerm, preview_proxy: Optional[PreviewProxyModel] = None) -> SearchResult: diff --git a/ontology/ontology_builder.py b/ontology/ontology_builder.py index 9bfe5a4a2..81e1b8615 100644 --- a/ontology/ontology_builder.py +++ b/ontology/ontology_builder.py @@ -1,15 +1,16 @@ import logging +from collections.abc import Iterable from dataclasses import dataclass -from datetime import timedelta, datetime +from datetime import datetime, timedelta from enum import Enum from functools import cached_property -from typing import Optional, TypeVar, Generic, Iterable, Type +from typing import Generic, Optional, TypeVar from django.db.models import Model from django.utils import timezone from model_utils.models import now -from ontology.models import OntologyTermRelation, OntologyTerm, OntologyImport, OntologyTermStatus +from ontology.models import OntologyImport, OntologyTerm, OntologyTermRelation, OntologyTermStatus class OntologyBuilderDataUpToDateException(Exception): @@ -66,7 +67,7 @@ def modify(self, ontology_import: OntologyImport): self.obj.modified = timezone.now() @staticmethod - def bulk_apply(model: Type[Model], cache: Iterable['CachedObj'], fields: list[str], verbose=False): + def bulk_apply(model: type[Model], cache: Iterable['CachedObj'], fields: list[str], verbose=False): created = [c.obj for c in cache if c.status == ModifiedStatus.CREATED] modified = [c.obj for c in cache if c.status == ModifiedStatus.MODIFIED] if verbose: diff --git a/ontology/ontology_matching.py b/ontology/ontology_matching.py index a8fb2f68b..8e89a9627 100644 --- a/ontology/ontology_matching.py +++ b/ontology/ontology_matching.py @@ -1,15 +1,16 @@ import re +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property -from typing import Optional, Any, Iterable, TypedDict +from typing import Any, Optional, TypedDict from django.urls import reverse from annotation.regexes import db_ref_regexes from classification.models.condition_text_search import condition_text_search -from library.log_utils import report_message, report_exc_info +from library.log_utils import report_exc_info, report_message from library.utils import empty_to_none -from ontology.models import OntologyTerm, OntologyService, OntologySnake +from ontology.models import OntologyService, OntologySnake, OntologyTerm class OntologySnakeJson(TypedDict): @@ -210,7 +211,7 @@ def matches(self, term: OntologyTerm) -> Optional[MatchInfo]: def effective_equals(self, other: 'SearchText') -> bool: # TODO handle a little bit off by 1 letter matching - return self.prefix_terms == other.prefix_terms and self.suffix_terms == other.suffix_terms or self.all_terms == other.all_terms + return (self.prefix_terms == other.prefix_terms and self.suffix_terms == other.suffix_terms) or self.all_terms == other.all_terms def pretty_set(s: Iterable[str]) -> str: diff --git a/ontology/ontology_traversal.py b/ontology/ontology_traversal.py index c2ad87948..c694bc7c4 100644 --- a/ontology/ontology_traversal.py +++ b/ontology/ontology_traversal.py @@ -6,12 +6,14 @@ once. """ from collections import defaultdict -from typing import Callable, Optional, Protocol, Union +from collections.abc import Callable +from typing import Protocol, Union from django.db.models import Q from genes.models import GeneSymbol from ontology.models import ( + ONTOLOGY_RELATIONSHIP_STANDARD_QUALITY_FILTER, GeneDiseaseClassification, OntologyImportSource, OntologyRelation, @@ -21,10 +23,8 @@ OntologyTermRelation, OntologyVersion, PanelAppClassification, - ONTOLOGY_RELATIONSHIP_STANDARD_QUALITY_FILTER, ) - _EXCLUDED_RELATIONS = frozenset({ OntologyRelation.IS_A, OntologyRelation.EXACT_SYNONYM, diff --git a/ontology/panel_app_ontology.py b/ontology/panel_app_ontology.py index 8ab6ac348..f1e269fab 100644 --- a/ontology/panel_app_ontology.py +++ b/ontology/panel_app_ontology.py @@ -1,21 +1,34 @@ import re import urllib.parse from collections import defaultdict +from collections.abc import Callable from dataclasses import dataclass from datetime import timedelta -from typing import Callable, Union, Optional, Any +from typing import Any, Optional, Union from django.conf import settings from django.db import transaction from django.utils import timezone from genes.models import GeneSymbol, PanelAppServer -from genes.panel_app import get_panel_app_results_by_gene_symbol_json, get_request, PANEL_APP_SEARCH_BY_GENES_BASE_PATH +from genes.panel_app import ( + PANEL_APP_SEARCH_BY_GENES_BASE_PATH, + get_panel_app_results_by_gene_symbol_json, + get_request, +) from library.cache import timed_cache from library.log_utils import report_exc_info, report_message from library.utils import md5sum_str -from ontology.models import OntologyTerm, OntologyRelation, OntologyImportSource, OntologyImport, OntologyTermRelation, \ - OntologyTermStatus, OntologyIdNormalized, PanelAppClassification +from ontology.models import ( + OntologyIdNormalized, + OntologyImport, + OntologyImportSource, + OntologyRelation, + OntologyTerm, + OntologyTermRelation, + OntologyTermStatus, + PanelAppClassification, +) from ontology.ontology_builder import OntologyBuilder, OntologyBuilderDataUpToDateException # increment if you change the logic of parsing ontology terms from PanelApp @@ -53,7 +66,7 @@ def parse_data(raw_data: dict) -> 'PanelAppResult': for phenotype_row in phenotypes: hash_str += phenotype_row + ";" found_term = False - from annotation.regexes import db_ref_regexes, DbRegexes + from annotation.regexes import DbRegexes, db_ref_regexes for result in db_ref_regexes.search(phenotype_row): if result.cregx in (DbRegexes.OMIM, DbRegexes.MONDO): all_terms.add(result.id_fixed) diff --git a/ontology/signals/ontology_health_check.py b/ontology/signals/ontology_health_check.py index 9960e698b..a9bd26f7b 100644 --- a/ontology/signals/ontology_health_check.py +++ b/ontology/signals/ontology_health_check.py @@ -3,8 +3,11 @@ from django.dispatch import receiver -from library.health_check import HealthCheckRequest, HealthCheckAge, \ - health_check_overall_stats_signal +from library.health_check import ( + HealthCheckAge, + HealthCheckRequest, + health_check_overall_stats_signal, +) from ontology.models import OntologyImport diff --git a/ontology/signals/ontology_preview.py b/ontology/signals/ontology_preview.py index 855dc4200..5a35f0f28 100644 --- a/ontology/signals/ontology_preview.py +++ b/ontology/signals/ontology_preview.py @@ -1,6 +1,6 @@ from django.dispatch import receiver -from library.preview_request import preview_request_signal, PreviewRequest +from library.preview_request import PreviewRequest, preview_request_signal from ontology.models import OntologyTerm diff --git a/ontology/templatetags/ontology_tags.py b/ontology/templatetags/ontology_tags.py index 9a0f85e10..5f104134b 100644 --- a/ontology/templatetags/ontology_tags.py +++ b/ontology/templatetags/ontology_tags.py @@ -1,12 +1,19 @@ import itertools import uuid +from collections.abc import Iterable, Iterator from dataclasses import dataclass -from typing import Optional, Union, Iterable, Iterator +from typing import Optional, Union from django.template import Library -from ontology.models import OntologyTerm, OntologyTermRelation, GeneDiseaseClassification, OntologyService, \ - OntologySnake, PanelAppClassification +from ontology.models import ( + GeneDiseaseClassification, + OntologyService, + OntologySnake, + OntologyTerm, + OntologyTermRelation, + PanelAppClassification, +) from ontology.ontology_matching import OntologyMatch register = Library() diff --git a/ontology/tests/test_data_ontology.py b/ontology/tests/test_data_ontology.py index d8398319a..d9e3ce114 100644 --- a/ontology/tests/test_data_ontology.py +++ b/ontology/tests/test_data_ontology.py @@ -5,7 +5,7 @@ from django.utils import timezone from ontology.management.commands import ontology_import -from ontology.models import OntologyVersion, OntologyImport +from ontology.models import OntologyImport, OntologyVersion def create_ontology_test_data(): diff --git a/ontology/tests/test_ontology_traversal.py b/ontology/tests/test_ontology_traversal.py index 607bf9bbf..a9ba1a647 100644 --- a/ontology/tests/test_ontology_traversal.py +++ b/ontology/tests/test_ontology_traversal.py @@ -2,7 +2,10 @@ from ontology.models import OntologyService, OntologyTerm from ontology.ontology_traversal import DbOntologyTraverser, MemoryOntologyTraverser -from ontology.tests.test_data_ontology import create_ontology_test_data, create_test_ontology_version +from ontology.tests.test_data_ontology import ( + create_ontology_test_data, + create_test_ontology_version, +) class OntologyTraversalParityTest(TestCase): diff --git a/ontology/urls.py b/ontology/urls.py index a980e61e2..3d6e2b84c 100644 --- a/ontology/urls.py +++ b/ontology/urls.py @@ -1,6 +1,10 @@ from ontology import views_autocomplete from ontology.views import OntologyTermView, ontology_term_text -from ontology.views_rest import SearchMondoText, OntologyTermGeneListView, GeneDiseaseRelationshipView +from ontology.views_rest import ( + GeneDiseaseRelationshipView, + OntologyTermGeneListView, + SearchMondoText, +) from variantgrid.perm_path import path urlpatterns = [ diff --git a/ontology/views.py b/ontology/views.py index ed4dcb1f7..0184b261a 100644 --- a/ontology/views.py +++ b/ontology/views.py @@ -5,8 +5,14 @@ from annotation.models import patients_qs_for_ontology_term from library.utils import LimitedCollection -from ontology.models import OntologyTerm, OntologyTermRelation, OntologyService, OntologySnake, OntologyRelation, \ - ONTOLOGY_RELATIONSHIP_MINIMUM_QUALITY_FILTER +from ontology.models import ( + ONTOLOGY_RELATIONSHIP_MINIMUM_QUALITY_FILTER, + OntologyRelation, + OntologyService, + OntologySnake, + OntologyTerm, + OntologyTermRelation, +) from ontology.panel_app_ontology import update_gene_relations @@ -36,7 +42,7 @@ def get_context_data(self, **kwargs): # the reverse of gene_relationships term_relationships = OntologySnake.snake_from(term=term, to_ontology=OntologyService.MONDO, quality_filter=ONTOLOGY_RELATIONSHIP_MINIMUM_QUALITY_FILTER).snakes + \ OntologySnake.snake_from(term=term, to_ontology=OntologyService.OMIM, quality_filter=ONTOLOGY_RELATIONSHIP_MINIMUM_QUALITY_FILTER).snakes - term_relationships = list(sorted((snake.reverse() for snake in term_relationships), key=lambda snake: snake.leaf_term)) + term_relationships = sorted((snake.reverse() for snake in term_relationships), key=lambda snake: snake.leaf_term) term_relationships = LimitedCollection(term_relationships, 250) else: raw_gene_relationships = sorted(OntologySnake.snake_from(term=term, to_ontology=OntologyService.HGNC, quality_filter=ONTOLOGY_RELATIONSHIP_MINIMUM_QUALITY_FILTER), key=lambda snake: snake.leaf_relationship.dest_term.short) diff --git a/ontology/views_autocomplete.py b/ontology/views_autocomplete.py index 2b9295fe0..d6eaa35a1 100644 --- a/ontology/views_autocomplete.py +++ b/ontology/views_autocomplete.py @@ -9,7 +9,7 @@ from library.constants import HOUR_SECS from library.django_utils.autocomplete_utils import AutocompleteView -from ontology.models import OntologyTerm, OntologyService, OntologyTermStatus +from ontology.models import OntologyService, OntologyTerm, OntologyTermStatus class AbstractOntologyTermAutocompleteView(abc.ABC, AutocompleteView): diff --git a/ontology/views_rest.py b/ontology/views_rest.py index b0d01fd49..1f5fef5c4 100644 --- a/ontology/views_rest.py +++ b/ontology/views_rest.py @@ -4,7 +4,7 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.response import Response from rest_framework.status import HTTP_200_OK from rest_framework.views import APIView @@ -12,8 +12,11 @@ from genes.models import GeneListGeneSymbol, create_fake_gene_list from genes.serializers import GeneListGeneSymbolSerializer from library.constants import WEEK_SECS -from ontology.models import OntologyTerm, OntologyVersion, \ - ONTOLOGY_RELATIONSHIP_MEDIUM_QUALITY_FILTER +from ontology.models import ( + ONTOLOGY_RELATIONSHIP_MEDIUM_QUALITY_FILTER, + OntologyTerm, + OntologyVersion, +) from ontology.ontology_matching import OntologyMatching from ontology.serializers import OntologyTermRelationSerializer diff --git a/pathtests/forms.py b/pathtests/forms.py index 6dc2192ea..9a7323bc7 100644 --- a/pathtests/forms.py +++ b/pathtests/forms.py @@ -6,8 +6,7 @@ from genes.models import GeneList from library.django_utils.autocomplete_utils import ModelSelect2 from library.utils import is_not_none -from pathtests.models import PathologyTest, PathologyTestVersion, Case, \ - PathologyTestOrder +from pathtests.models import Case, PathologyTest, PathologyTestOrder, PathologyTestVersion from seqauto.models import EnrichmentKit diff --git a/pathtests/grids.py b/pathtests/grids.py index ea0bf8ca7..1e6468c59 100644 --- a/pathtests/grids.py +++ b/pathtests/grids.py @@ -1,5 +1,5 @@ from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig -from pathtests.models import PathologyTestOrder, Case, PathologyTest +from pathtests.models import Case, PathologyTest, PathologyTestOrder class PathologyTestOrdersGrid(JqGridUserRowConfig): diff --git a/pathtests/models.py b/pathtests/models.py index d2c547fe9..12ad276d3 100644 --- a/pathtests/models.py +++ b/pathtests/models.py @@ -1,20 +1,24 @@ from typing import Union -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.db import models -from django.db.models.deletion import CASCADE, SET_NULL, PROTECT +from django.db.models.deletion import CASCADE, PROTECT, SET_NULL from django.urls.base import reverse from django_extensions.db.models import TimeStampedModel from genes.models import GeneList, GeneSymbol from library.enums import ModificationOperation from library.preview_request import PreviewModelMixin -from pathtests.models_enums import PathologyTestGeneModificationOutcome, \ - CaseState, InvestigationType, CaseWorkflowStatus -from patients.models import Patient, Clinician, ExternallyManagedModel, TEST_PATIENT_KWARGS +from pathtests.models_enums import ( + CaseState, + CaseWorkflowStatus, + InvestigationType, + PathologyTestGeneModificationOutcome, +) +from patients.models import TEST_PATIENT_KWARGS, Clinician, ExternallyManagedModel, Patient from patients.models_enums import PopulationGroup -from seqauto.models import Experiment, SequencingRun, EnrichmentKit -from snpdb.models import Wiki, Sample +from seqauto.models import EnrichmentKit, Experiment, SequencingRun +from snpdb.models import Sample, Wiki from snpdb.models.models_enums import ImportStatus diff --git a/pathtests/serializers.py b/pathtests/serializers.py index bb58ecb98..fa7201f3c 100644 --- a/pathtests/serializers.py +++ b/pathtests/serializers.py @@ -1,7 +1,11 @@ from rest_framework import serializers from genes.serializers import GeneListSerializer, GeneSymbolSerializer -from pathtests.models import PathologyTestVersion, PathologyTest, PathologyTestGeneModificationRequest +from pathtests.models import ( + PathologyTest, + PathologyTestGeneModificationRequest, + PathologyTestVersion, +) from snpdb.serializers import UserSerializer diff --git a/pathtests/urls.py b/pathtests/urls.py index bb3274e6c..cb6a6f930 100644 --- a/pathtests/urls.py +++ b/pathtests/urls.py @@ -1,6 +1,6 @@ from library.django_utils.jqgrid_view import JQGridView from pathtests import views, views_autocomplete, views_rest -from pathtests.grids import PathologyTestOrdersGrid, CasesGrid, PathologyTestsGrid +from pathtests.grids import CasesGrid, PathologyTestOrdersGrid, PathologyTestsGrid from variantgrid.perm_path import path urlpatterns = [ diff --git a/pathtests/views.py b/pathtests/views.py index 686d95a35..d9e1d499d 100644 --- a/pathtests/views.py +++ b/pathtests/views.py @@ -7,21 +7,35 @@ from django.contrib.auth.models import User from django.core.exceptions import PermissionDenied from django.db.models.query_utils import Q -from django.http.response import JsonResponse, HttpResponse -from django.shortcuts import render, get_object_or_404, redirect +from django.http.response import HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404, redirect, render from django.utils import timezone from django.views.decorators.http import require_POST -from genes.models import GeneListCategory, GeneList, GeneSymbol +from genes.models import GeneList, GeneListCategory, GeneSymbol from genes.views.views import add_gene_list_unmatched_genes_message from library.django_utils import add_save_message -from pathtests.forms import SelectPathologyTestForm, SelectPathologyTestVersionForm, \ - PathologyTestOrderForm, CaseForm, CreatePathologyTestForm, PathologyTestVersionForm -from pathtests.models import PathologyTest, PathologyTestVersion, \ - PathologyTestGeneModificationRequest, PathologyTestGeneModificationOutcome, get_cases_qs, PathologyTestOrder, \ - Case, get_external_order_system_last_checked, ActivePathologyTestVersion +from pathtests.forms import ( + CaseForm, + CreatePathologyTestForm, + PathologyTestOrderForm, + PathologyTestVersionForm, + SelectPathologyTestForm, + SelectPathologyTestVersionForm, +) +from pathtests.models import ( + ActivePathologyTestVersion, + Case, + PathologyTest, + PathologyTestGeneModificationOutcome, + PathologyTestGeneModificationRequest, + PathologyTestOrder, + PathologyTestVersion, + get_cases_qs, + get_external_order_system_last_checked, +) from patients.forms import external_pk_autocomplete_form_factory -from patients.models import Clinician, get_lead_scientist_users_for_user, FollowLeadScientist +from patients.models import Clinician, FollowLeadScientist, get_lead_scientist_users_for_user from snpdb.models.models_enums import ImportStatus diff --git a/pathtests/views_rest.py b/pathtests/views_rest.py index e7c81590e..0ad4123cf 100644 --- a/pathtests/views_rest.py +++ b/pathtests/views_rest.py @@ -1,11 +1,11 @@ from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework.exceptions import NotFound from rest_framework.generics import RetrieveAPIView, get_object_or_404 from rest_framework.response import Response from rest_framework.views import APIView -from pathtests.models import PathologyTestVersion, PathologyTest +from pathtests.models import PathologyTest, PathologyTestVersion from pathtests.serializers import PathologyTestVersionSerializer diff --git a/patients/apps.py b/patients/apps.py index 3da611cae..ea43c767f 100644 --- a/patients/apps.py +++ b/patients/apps.py @@ -7,5 +7,5 @@ class PatientsConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from patients.signals import external_pk_search, patient_search + pass # pylint: enable=import-outside-toplevel,unused-import diff --git a/patients/forms.py b/patients/forms.py index e69f237ea..92fb110af 100644 --- a/patients/forms.py +++ b/patients/forms.py @@ -1,12 +1,17 @@ from dal import forward from django import forms -from django.forms.models import inlineformset_factory, ALL_FIELDS +from django.forms.models import ALL_FIELDS, inlineformset_factory from django.forms.widgets import TextInput from library.django_utils.autocomplete_utils import ModelSelect2 from library.guardian_utils import assign_permission_to_user_and_groups -from patients.models import Patient, Specimen, ExternalPK, PatientModification, \ - PatientRecordOriginType +from patients.models import ( + ExternalPK, + Patient, + PatientModification, + PatientRecordOriginType, + Specimen, +) from patients.models_enums import PopulationGroup diff --git a/patients/grids.py b/patients/grids.py index ac6a6b9bb..e52b5e6cc 100644 --- a/patients/grids.py +++ b/patients/grids.py @@ -1,7 +1,7 @@ from functools import partial from django.contrib.postgres.aggregates.general import StringAgg -from django.db.models import TextField, QuerySet +from django.db.models import QuerySet, TextField from django.db.models.aggregates import Count from django.db.models.query_utils import Q from django.http import HttpRequest @@ -10,10 +10,10 @@ from annotation.models.models_phenotype_match import PATIENT_ONTOLOGY_TERM_PATH from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from ontology.grids import AbstractOntologyGenesGrid -from ontology.models import OntologyTerm, OntologyService -from patients.models import PatientRecords, Patient, PatientRecord +from ontology.models import OntologyService, OntologyTerm +from patients.models import Patient, PatientRecord, PatientRecords from patients.models_enums import PatientRecordMatchType -from snpdb.views.datatable_view import DatatableConfig, RichColumn, CellData +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn class PatientListGrid(JqGridUserRowConfig): diff --git a/patients/import_records.py b/patients/import_records.py index 7a5b06dbc..005fc7666 100644 --- a/patients/import_records.py +++ b/patients/import_records.py @@ -21,9 +21,15 @@ from annotation.phenotype_matching import bulk_patient_phenotype_matching from library.guardian_utils import assign_permission_to_user_and_groups from library.pandas_utils import df_nan_to_none -from patients.models import PatientColumns, PatientRecord, Specimen, Patient, \ - PatientModification, PatientRecordOriginType -from patients.models_enums import Sex, NucleicAcid, Mutation, PatientRecordMatchType +from patients.models import ( + Patient, + PatientColumns, + PatientModification, + PatientRecord, + PatientRecordOriginType, + Specimen, +) +from patients.models_enums import Mutation, NucleicAcid, PatientRecordMatchType, Sex from snpdb.models import Sample UNKNOWN_STRING = 'UNKNOWN' # Upper diff --git a/patients/management/commands/match_patient_phenotypes.py b/patients/management/commands/match_patient_phenotypes.py index ca9c1983c..6eca15b23 100644 --- a/patients/management/commands/match_patient_phenotypes.py +++ b/patients/management/commands/match_patient_phenotypes.py @@ -4,7 +4,11 @@ from django.core.management.base import BaseCommand from django.db.models import Count -from annotation.models.models_phenotype_match import TextPhenotype, PatientTextPhenotype, TextPhenotypeMatch +from annotation.models.models_phenotype_match import ( + PatientTextPhenotype, + TextPhenotype, + TextPhenotypeMatch, +) from annotation.phenotype_matching import bulk_patient_phenotype_matching diff --git a/patients/models.py b/patients/models.py index 37281b6d9..6c8df9e2c 100644 --- a/patients/models.py +++ b/patients/models.py @@ -12,15 +12,23 @@ from django_extensions.db.models import TimeStampedModel from annotation.models.has_phenotype_description_mixin import HasPhenotypeDescriptionMixin -from library.django_utils import single_string_to_first_last_name_q, \ - ensure_mutally_exclusive_fields_not_set +from library.django_utils import ( + ensure_mutally_exclusive_fields_not_set, + single_string_to_first_last_name_q, +) from library.django_utils.django_file_system_storage import PrivateUploadStorage from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin from library.enums.file_attachments import AttachmentFileType from library.enums.titles import Title -from library.preview_request import PreviewData, PreviewModelMixin, PreviewKeyValue +from library.preview_request import PreviewData, PreviewKeyValue, PreviewModelMixin from library.utils import calculate_age -from patients.models_enums import NucleicAcid, Mutation, Sex, PopulationGroup, PatientRecordMatchType +from patients.models_enums import ( + Mutation, + NucleicAcid, + PatientRecordMatchType, + PopulationGroup, + Sex, +) TEST_PATIENT_KWARGS = {"first_name": "PATIENT", "last_name": "TESTPATIENT"} diff --git a/patients/models_enums.py b/patients/models_enums.py index 50ce9d855..96797a9bf 100644 --- a/patients/models_enums.py +++ b/patients/models_enums.py @@ -1,7 +1,7 @@ from django.db import models from django.db.models import Q -from library.utils import invert_dict, Constant, get_single_element +from library.utils import Constant, get_single_element, invert_dict class NucleicAcid(models.TextChoices): diff --git a/patients/signals/external_pk_search.py b/patients/signals/external_pk_search.py index 5712f48b5..3310b1b29 100644 --- a/patients/signals/external_pk_search.py +++ b/patients/signals/external_pk_search.py @@ -2,8 +2,7 @@ from rest_framework.exceptions import PermissionDenied from patients.models import ExternalPK -from snpdb.search import search_receiver, HAS_ALPHA_PATTERN, \ - SearchInputInstance, SearchExample +from snpdb.search import HAS_ALPHA_PATTERN, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/patients/signals/patient_search.py b/patients/signals/patient_search.py index ed6b193a2..4481ca293 100644 --- a/patients/signals/patient_search.py +++ b/patients/signals/patient_search.py @@ -1,14 +1,14 @@ from django.contrib.auth.models import User -from django.db.models import Value, CharField, Count +from django.db.models import CharField, Count, Value from django.db.models.functions import Concat, Lower from django.dispatch import receiver from annotation.models import patients_qs_for_ontology_term -from library.preview_request import preview_extra_signal, PreviewKeyValue +from library.preview_request import PreviewKeyValue, preview_extra_signal from ontology.models import OntologyTerm from patients.models import Patient from snpdb.models import Sample -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ANY +from snpdb.search import HAS_3_ANY, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/patients/templatetags/patient_graph_tags.py b/patients/templatetags/patient_graph_tags.py index 1407db9d6..d257d6992 100644 --- a/patients/templatetags/patient_graph_tags.py +++ b/patients/templatetags/patient_graph_tags.py @@ -3,7 +3,7 @@ from django.db.models.aggregates import Count from django.template import Library -from ontology.models import OntologyTerm, OntologyService +from ontology.models import OntologyService, OntologyTerm from patients.models import Patient GENES_COLOR = "#99CD83" diff --git a/patients/templatetags/phenotype_tags.py b/patients/templatetags/phenotype_tags.py index 71ceb6109..cd53e9088 100644 --- a/patients/templatetags/phenotype_tags.py +++ b/patients/templatetags/phenotype_tags.py @@ -4,7 +4,7 @@ from django.template import Library from genes.forms import GeneSymbolForm -from ontology.forms import HPOForm, OMIMForm, MONDOForm +from ontology.forms import HPOForm, MONDOForm, OMIMForm register = Library() diff --git a/patients/tests/test_patients_bugs.py b/patients/tests/test_patients_bugs.py index 86f66408f..1852bc195 100644 --- a/patients/tests/test_patients_bugs.py +++ b/patients/tests/test_patients_bugs.py @@ -4,7 +4,8 @@ Tests that expose confirmed bugs are expected to FAIL until the bug is fixed. """ import os -from datetime import date, datetime, timezone as dt_timezone +from datetime import UTC, date, datetime +from datetime import timezone as dt_timezone from django.contrib.auth.models import User from django.test import TestCase @@ -12,8 +13,15 @@ from library.guardian_utils import assign_permission_to_user_and_groups from patients.import_records import parse_boolean, parse_choice, process_record from patients.models import ( - Clinician, ExternalModelManager, ExternalPK, Patient, PatientColumns, - PatientImport, PatientModification, PatientRecords, Specimen, + Clinician, + ExternalModelManager, + ExternalPK, + Patient, + PatientColumns, + PatientImport, + PatientModification, + PatientRecords, + Specimen, ) from patients.models_enums import Sex from snpdb.models import ImportSource @@ -24,7 +32,7 @@ def _make_row(**overrides): - row = {col: None for col in PatientColumns.COLUMNS} + row = dict.fromkeys(PatientColumns.COLUMNS) row[PatientColumns.PATIENT_LAST_NAME] = "IMPORTTESTLAST" row[PatientColumns.PATIENT_FIRST_NAME] = "IMPORTTESTFIRST" row.update(overrides) @@ -116,7 +124,7 @@ def test_age_calculated_from_dob_and_collection_date(self): specimen = Specimen.objects.create( reference_id="AGECALC001", patient=self.patient, - collection_date=datetime(2020, 6, 15, tzinfo=dt_timezone.utc), + collection_date=datetime(2020, 6, 15, tzinfo=UTC), ) self.assertEqual(specimen.age_at_collection_date, 40) diff --git a/patients/tests/test_urls.py b/patients/tests/test_urls.py index 5d6736d26..9b6790785 100644 --- a/patients/tests/test_urls.py +++ b/patients/tests/test_urls.py @@ -7,11 +7,18 @@ from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings from library.enums.titles import Title from library.guardian_utils import assign_permission_to_user_and_groups -from patients.models import Clinician, ExternalPK, ExternalModelManager, Patient, PatientRecords, PatientImport, \ - Specimen -from snpdb.models import Sex, ImportSource +from patients.models import ( + Clinician, + ExternalModelManager, + ExternalPK, + Patient, + PatientImport, + PatientRecords, + Specimen, +) +from snpdb.models import ImportSource, Sex from snpdb.models.models_genome import GenomeBuild -from upload.models import UploadedFile, UploadedPatientRecords, UploadedFileTypes +from upload.models import UploadedFile, UploadedFileTypes, UploadedPatientRecords class Test(URLTestCase): diff --git a/patients/urls.py b/patients/urls.py index 08ed22dda..1fe964ae3 100644 --- a/patients/urls.py +++ b/patients/urls.py @@ -1,6 +1,11 @@ from library.django_utils.jqgrid_view import JQGridView from patients import views, views_autocomplete -from patients.grids import PatientListGrid, PatientOntologyGenesGrid, PatientRecordColumns, PatientRecordsColumns +from patients.grids import ( + PatientListGrid, + PatientOntologyGenesGrid, + PatientRecordColumns, + PatientRecordsColumns, +) from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/patients/views.py b/patients/views.py index 932aca6b6..d204c3b09 100644 --- a/patients/views.py +++ b/patients/views.py @@ -3,21 +3,28 @@ import pandas as pd from django.conf import settings from django.http.response import HttpResponse, JsonResponse -from django.shortcuts import render, get_object_or_404 -from django.views.decorators.http import require_POST, require_http_methods +from django.shortcuts import get_object_or_404, render +from django.views.decorators.http import require_http_methods, require_POST from annotation.models.models_phenotype_match import TextPhenotypeMatch from annotation.phenotype_matching import create_phenotype_description from library.django_utils import add_save_message, set_form_read_only -from library.django_utils.file_uploads import filepond_upload_receive, filepond_process_response +from library.django_utils.file_uploads import filepond_process_response, filepond_upload_receive from library.log_utils import log_traceback from library.utils import invert_dict from library.utils.file_utils import rm_if_exists -from ontology.forms import OMIMForm, HPOForm, HGNCForm, MONDOForm +from ontology.forms import HGNCForm, HPOForm, MONDOForm, OMIMForm from patients import forms -from patients.forms import PatientSearchForm, PatientContactForm -from patients.models import PatientColumns, PatientRecords, Patient, PatientModification, PatientRecordOriginType, \ - PatientAttachment, PatientRecord +from patients.forms import PatientContactForm, PatientSearchForm +from patients.models import ( + Patient, + PatientAttachment, + PatientColumns, + PatientModification, + PatientRecord, + PatientRecordOriginType, + PatientRecords, +) from snpdb.models import Sample from uicore.utils.form_helpers import form_helper_horizontal @@ -216,7 +223,7 @@ def get_patient_upload_csv(filename, sample_qs, columns_lookup=None): } sample_values_qs = sample_qs.values(*columns_lookup) - empty_row = {c: '' for c in PatientColumns.COLUMNS} + empty_row = dict.fromkeys(PatientColumns.COLUMNS, '') rows = [] for values in sample_values_qs: data = empty_row.copy() @@ -257,7 +264,7 @@ def patients(request): if valid: patient = form.save() form = forms.PatientForm(user=request.user) # clear form for next patient - msg = f"Patient #{patient.pk}: {str(patient)}" + msg = f"Patient #{patient.pk}: {patient!s}" else: msg = "Patient" add_save_message(request, valid, msg, created=True) diff --git a/patients/views_autocomplete.py b/patients/views_autocomplete.py index 7fc9c1fb1..1e25efe6e 100644 --- a/patients/views_autocomplete.py +++ b/patients/views_autocomplete.py @@ -11,7 +11,7 @@ from library.constants import MINUTE_SECS from library.django_utils.autocomplete_utils import AutocompleteView -from patients.models import Patient, Clinician, Specimen, ExternalPK +from patients.models import Clinician, ExternalPK, Patient, Specimen @method_decorator([cache_page(MINUTE_SECS), vary_on_cookie], name='dispatch') diff --git a/pedigree/apps.py b/pedigree/apps.py index b6376d467..8808063f0 100644 --- a/pedigree/apps.py +++ b/pedigree/apps.py @@ -7,5 +7,5 @@ class PedigreeConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from pedigree.signals import pedigree_search + pass # pylint: enable=import-outside-toplevel,unused-import diff --git a/pedigree/grids.py b/pedigree/grids.py index cfc5d0de6..20bfab0d7 100644 --- a/pedigree/grids.py +++ b/pedigree/grids.py @@ -2,7 +2,7 @@ from django.http import HttpRequest from pedigree.models import PedFile, Pedigree -from snpdb.models import UserGridConfig, ImportStatus +from snpdb.models import ImportStatus, UserGridConfig from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder diff --git a/pedigree/models.py b/pedigree/models.py index 637a34855..05f73ea88 100644 --- a/pedigree/models.py +++ b/pedigree/models.py @@ -1,4 +1,4 @@ -from typing import Iterable +from collections.abc import Iterable from django.contrib.auth.models import User from django.db import models @@ -7,11 +7,13 @@ from model_utils.models import TimeStampedModel from library.django_utils import SortByPKMixin -from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin, \ - GuardianPermissionsMixin +from library.django_utils.guardian_permissions_mixin import ( + GuardianPermissionsAutoInitialSaveMixin, + GuardianPermissionsMixin, +) from library.preview_request import PreviewModelMixin from patients.models_enums import Sex -from snpdb.models import ImportStatus, Cohort, CohortSample, Sample, SomalierRelate +from snpdb.models import Cohort, CohortSample, ImportStatus, Sample, SomalierRelate class PedFile(GuardianPermissionsMixin, models.Model): diff --git a/pedigree/ped/import_ped.py b/pedigree/ped/import_ped.py index 44be0aa10..12211ea10 100644 --- a/pedigree/ped/import_ped.py +++ b/pedigree/ped/import_ped.py @@ -5,9 +5,9 @@ from guardian.shortcuts import assign_perm from toposort import toposort -from pedigree.models import PedFile, PedFileRecord, PedFileFamily, create_automatch_pedigree -from pedigree.ped.ped_file_utils import get_sex, get_affection, PED_COLUMNS, get_parent_id -from snpdb.models import ImportStatus, Cohort +from pedigree.models import PedFile, PedFileFamily, PedFileRecord, create_automatch_pedigree +from pedigree.ped.ped_file_utils import PED_COLUMNS, get_affection, get_parent_id, get_sex +from snpdb.models import Cohort, ImportStatus def save_ped_records(ped_file_family, family_df, dependency_graph): diff --git a/pedigree/signals/pedigree_search.py b/pedigree/signals/pedigree_search.py index 0c3a40c0b..08305da5a 100644 --- a/pedigree/signals/pedigree_search.py +++ b/pedigree/signals/pedigree_search.py @@ -1,5 +1,5 @@ from pedigree.models import Pedigree -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/pedigree/urls.py b/pedigree/urls.py index dad0e90e3..cf28c1970 100644 --- a/pedigree/urls.py +++ b/pedigree/urls.py @@ -3,7 +3,6 @@ from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path - urlpatterns = [ path('pedigrees', views.pedigrees, name='pedigrees'), path('view_pedigree/', views.view_pedigree, name='view_pedigree'), diff --git a/pedigree/views.py b/pedigree/views.py index a07258211..5526c9ada 100644 --- a/pedigree/views.py +++ b/pedigree/views.py @@ -4,15 +4,21 @@ from django.forms.formsets import formset_factory from django.forms.models import ModelChoiceField from django.http.response import HttpResponseRedirect -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls.base import reverse from library.utils import full_class_name from pedigree import forms from pedigree.forms import BaseCohortSamplesForPedFileRecordsFormSet from pedigree.graphs.pedigree_chart import PedigreeChart -from pedigree.models import PedFile, Pedigree, PedFileFamily, PedFileRecord, \ - CohortSamplePedFileRecord, create_automatch_pedigree +from pedigree.models import ( + CohortSamplePedFileRecord, + PedFile, + PedFileFamily, + PedFileRecord, + Pedigree, + create_automatch_pedigree, +) from snpdb.forms import UserCohortForm from snpdb.graphs import graphcache from snpdb.models import Cohort, UserGridConfig diff --git a/requirements.in b/requirements.in index fccc6d62c..2e3da033b 100644 --- a/requirements.in +++ b/requirements.in @@ -90,3 +90,13 @@ django_threadlocals frozendict pydantic>=2.4.0 deepdiff + +# --- linting & type checking (dev only) --- +# Run: mypy --config-file mypy.ini snpdb (or any app) +# Lint: ./scripts/linting/run_pylint.sh | ruff check +ruff +mypy +django-stubs +djangorestframework-stubs +pylint +pylint-django diff --git a/requirements.txt b/requirements.txt index 365779443..eec5152da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.in +# uv pip compile requirements.in -o requirements.txt amqp==5.3.1 # via kombu annotated-types==0.7.0 @@ -8,12 +8,18 @@ ansimarkup==2.0.0 # via django-postgres-extra asgiref==3.11.1 # via django +ast-serialize==0.5.0 + # via mypy +astroid==4.0.4 + # via pylint asttokens==3.0.1 # via stack-data attrs==26.1.0 # via # bioutils # hgvs + # jsonschema + # referencing beautifulsoup4==4.14.3 # via # -r requirements.in @@ -96,6 +102,8 @@ deepdiff==9.1.0 # via -r requirements.in deprecation==2.1.0 # via -r requirements.in +dill==0.4.1 + # via pylint django==6.0.5 # via # -r requirements.in @@ -120,8 +128,12 @@ django==6.0.5 # django-recaptcha # django-starfield # django-storages + # django-stubs + # django-stubs-ext # django-termsandconditions # djangorestframework + # drf-spectacular + # drf-spectacular-sidecar # easy-thumbnails # martor # mozilla-django-oidc @@ -183,6 +195,12 @@ django-starfield @ git+https://github.com/davmlaw/django-starfield@a7bb433bb645a # via -r requirements.in django-storages==1.14.6 # via -r requirements.in +django-stubs==6.0.5 + # via + # -r requirements.in + # djangorestframework-stubs +django-stubs-ext==6.0.5 + # via django-stubs django-termsandconditions==2.1.1 # via -r requirements.in django-threadlocals==0.10 @@ -191,6 +209,8 @@ djangorestframework==3.17.1 # via # -r requirements.in # drf-spectacular +djangorestframework-stubs==3.17.0 + # via -r requirements.in dnspython==2.8.0 # via django-avatar drf-spectacular==0.29.0 @@ -244,18 +264,20 @@ ipython==8.39.0 # -r requirements.in # biocommons-seqrepo # hgvs +isort==8.0.1 + # via pylint jedi==0.20.0 # via ipython -jsonschema==4.26.0 - # via drf-spectacular -jsonschema-specifications==2025.9.1 - # via jsonschema jmespath==1.1.0 # via # boto3 # botocore joblib==1.5.3 # via nltk +jsonschema==4.26.0 + # via drf-spectacular +jsonschema-specifications==2025.9.1 + # via jsonschema kiwisolver==1.5.0 # via matplotlib kombu==5.6.2 @@ -266,6 +288,8 @@ lazy==1.6 # via cdot levenshtein==0.27.3 # via -r requirements.in +librt==0.11.0 + # via mypy lxml==6.1.1 # via -r requirements.in markdown==3.10.2 @@ -280,12 +304,18 @@ matplotlib==3.10.9 # via -r requirements.in matplotlib-inline==0.2.2 # via ipython +mccabe==0.7.0 + # via pylint mdurl==0.1.2 # via markdown-it-py more-itertools==11.1.0 # via -r requirements.in mozilla-django-oidc==5.0.2 # via -r requirements.in +mypy==2.1.0 + # via -r requirements.in +mypy-extensions==1.1.0 + # via mypy nameparser==1.1.3 # via -r requirements.in networkx==3.6.1 @@ -323,6 +353,8 @@ parsley==1.3 # via hgvs parso==0.8.7 # via jedi +pathspec==1.1.1 + # via mypy ped-parser==1.6.6 # via -r requirements.in pexpect==4.9.0 @@ -335,6 +367,8 @@ pillow==12.2.0 # reportlab pip==26.1.1 # via pyhgvs +platformdirs==4.10.0 + # via pylint pluggy==1.6.0 # via pytest prompt-toolkit==3.0.52 @@ -368,6 +402,15 @@ pyhgvs @ git+https://github.com/SACGF/hgvs@69ff4bdd09088be798708702779a5ba59f1c5 # via -r requirements.in pyjwt==2.13.0 # via mozilla-django-oidc +pylint==4.0.5 + # via + # -r requirements.in + # pylint-django + # pylint-plugin-utils +pylint-django==2.7.0 + # via -r requirements.in +pylint-plugin-utils==0.9.0 + # via pylint-django pyparsing==3.3.2 # via matplotlib pypdf==6.12.2 @@ -432,6 +475,8 @@ rpds-py==2026.5.1 # via # jsonschema # referencing +ruff==0.15.16 + # via -r requirements.in s3transfer==0.17.1 # via boto3 scipy==1.17.1 @@ -460,6 +505,8 @@ stream-zip==0.0.84 # via -r requirements.in tabulate==0.10.0 # via yoyo-migrations +tomlkit==0.15.0 + # via pylint toposort==1.10 # via -r requirements.in tqdm==4.67.3 @@ -470,13 +517,22 @@ traitlets==5.15.0 # via # ipython # matplotlib-inline +types-pyyaml==6.0.12.20260518 + # via + # django-stubs + # djangorestframework-stubs typing-extensions==4.15.0 # via # beautifulsoup4 # biocommons-seqrepo # django-guardian + # django-stubs + # django-stubs-ext + # djangorestframework-stubs + # mypy # pydantic # pydantic-core + # referencing # typing-inspection typing-inspection==0.4.2 # via pydantic diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..7d4dbf735 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,56 @@ +# Ruff: fast linter + formatter. Owns style / imports / modernization / common +# refactors. Coexists with a slimmed pylint (config/pylint3.rc), which keeps only +# the inference-heavy checks + pylint_django that ruff can't replicate, and with +# mypy (mypy.ini) for type checking. Three complementary layers. + +target-version = "py312" +line-length = 100 + +extend-exclude = [ + "**/migrations/**", + "variantgrid/static_files", + "variantgrid/sitestatic", + "**/env/**", +] + +[lint] +select = [ + "F", # pyflakes - undefined names, unused imports/vars + "E", # pycodestyle errors + "W", # pycodestyle warnings (whitespace - replaces the autopep8 pass) + "I", # isort - import ordering + "UP", # pyupgrade - modern py312 syntax + "B", # flake8-bugbear - likely bugs + "C4", # flake8-comprehensions + "DJ", # flake8-django + "TC", # flake8-type-checking - move typing-only imports into TYPE_CHECKING + "RUF", # ruff-specific +] +ignore = [ + "E501", # line-too-long - matches pylint (too many currently) + "E741", # ambiguous name l/I/O - pylint good-names allows i,j,k + "B008", # function call in arg default - common Django/DRF idiom (Depends-style) + "B905", # zip() without strict= - too noisy for now + "RUF012", # mutable class attrs need ClassVar - clashes with Django model idiom + "DJ001", # null=True on string field - existing schema, not actionable in lint + # PEP 604 union rewrites are NOT safe to auto-apply here: this codebase has + # annotations where a name is shadowed (e.g. a dataclass field named `datetime`) + # or is a non-type that Optional[...]/Union[...] silently tolerated. Converting to + # `X | None` makes the annotation eager-evaluate and raise TypeError at import. + # Re-enable per-file only after adding `from __future__ import annotations`. + "UP007", # Union[X, Y] -> X | Y + "UP045", # Optional[X] -> X | None +] + +[lint.per-file-ignores] +# __init__.py: re-exports are intentional (F401), and import order is frequently +# deliberate to manage circular dependencies, so do not sort imports here (I001). +"__init__.py" = ["F401", "I001"] +"**/tests/**" = ["F401", "F841"] # unused fixtures/locals common in tests + +[lint.isort] +known-first-party = [ + "analysis", "annotation", "classification", "eventlog", "flags", "genes", + "library", "manual", "ontology", "pathtests", "patients", "pedigree", + "seqauto", "snpdb", "sync", "uicore", "upload", "variantgrid", "variantopedia", +] diff --git a/scripts/linting/format_code.sh b/scripts/linting/format_code.sh index f60cd889c..acd454faa 100755 --- a/scripts/linting/format_code.sh +++ b/scripts/linting/format_code.sh @@ -2,4 +2,9 @@ VG_DIR=$(dirname $0)/../.. -autopep8 ${VG_DIR} --recursive --select=W293,W391,E203,E242,E251,E252,E261,E27,E303,W291,W292,W293,W391 --in-place --exclude "*env/*" +# Ruff replaces the old autopep8 pass. Safe auto-fixes only: whitespace, +# unused imports, import ordering, modern-syntax. Config: ruff.toml +ruff check "${VG_DIR}" --fix + +# Full black-style reformatting is opt-in (large diff) - uncomment to adopt: +# ruff format "${VG_DIR}" diff --git a/seqauto/apps.py b/seqauto/apps.py index 9f18be492..8adda4b93 100644 --- a/seqauto/apps.py +++ b/seqauto/apps.py @@ -9,5 +9,5 @@ class SeqautoConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from seqauto.signals import experiment_search, sequencing_run_search, enrichment_kit_search + pass # pylint: enable=import-outside-toplevel,unused-import diff --git a/seqauto/forms.py b/seqauto/forms.py index 9ae97f060..2c26b76dc 100644 --- a/seqauto/forms.py +++ b/seqauto/forms.py @@ -4,7 +4,7 @@ from library.django_utils.autocomplete_utils import ModelSelect2 from library.forms import ROFormMixin from seqauto import models -from seqauto.models import QCType, QCColumn, EnrichmentKit, SequencingRun +from seqauto.models import EnrichmentKit, QCColumn, QCType, SequencingRun from seqauto.models.models_enums import QCCompareType, QCGraphTypes2 from snpdb.forms import BaseDeclareForm diff --git a/seqauto/graphs/sequencing_run_qc_graph.py b/seqauto/graphs/sequencing_run_qc_graph.py index 7401ae930..79410de72 100644 --- a/seqauto/graphs/sequencing_run_qc_graph.py +++ b/seqauto/graphs/sequencing_run_qc_graph.py @@ -8,8 +8,13 @@ from library.graphs.graph_utils import ForceMandKIntFormatter from seqauto.models import SequencingRun from seqauto.models.models_enums import QCCompareType -from seqauto.qc.sequencing_run_utils import ILLUMINA_FLOWCELL_QC_COLUMNS, SEQUENCING_RUN_QC_COLUMNS, PAIRED_END_READS, \ - get_q30_col_name, get_sequencing_run_data +from seqauto.qc.sequencing_run_utils import ( + ILLUMINA_FLOWCELL_QC_COLUMNS, + PAIRED_END_READS, + SEQUENCING_RUN_QC_COLUMNS, + get_q30_col_name, + get_sequencing_run_data, +) from snpdb.graphs.graphcache import CacheableGraph diff --git a/seqauto/grids/qc_data_grids.py b/seqauto/grids/qc_data_grids.py index 5f5624f45..fd71c9bd9 100644 --- a/seqauto/grids/qc_data_grids.py +++ b/seqauto/grids/qc_data_grids.py @@ -2,7 +2,7 @@ from library.django_utils.jqgrid_view import JQGridViewOp from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig -from seqauto.models import IlluminaFlowcellQC, FastQC, Flagstats, QCExecSummary +from seqauto.models import FastQC, Flagstats, IlluminaFlowcellQC, QCExecSummary class IlluminaFlowcellQCGrid(JqGridUserRowConfig): diff --git a/seqauto/grids/seqauto_grids.py b/seqauto/grids/seqauto_grids.py index f8ee6785c..9ade85a39 100644 --- a/seqauto/grids/seqauto_grids.py +++ b/seqauto/grids/seqauto_grids.py @@ -7,8 +7,13 @@ from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from library.pandas_jqgrid import DataFrameJqGrid from library.pandas_utils import nan_to_none -from seqauto.models import SeqAutoRun, SequencingSample, GoldCoverageSummary, \ - GoldReference, EnrichmentKit +from seqauto.models import ( + EnrichmentKit, + GoldCoverageSummary, + GoldReference, + SeqAutoRun, + SequencingSample, +) from seqauto.seqauto_stats import get_sample_enrichment_kits_df, group_enrichment_kits_df diff --git a/seqauto/grids/sequencing_data_grids.py b/seqauto/grids/sequencing_data_grids.py index 14b6ecbc7..2bea89d3b 100644 --- a/seqauto/grids/sequencing_data_grids.py +++ b/seqauto/grids/sequencing_data_grids.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.postgres.aggregates.general import StringAgg -from django.db.models import TextField, QuerySet +from django.db.models import QuerySet, TextField from django.db.models.aggregates import Count from django.db.models.functions import Cast from django.db.models.query_utils import Q @@ -10,9 +10,17 @@ from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from library.utils import JsonDataType -from seqauto.models import SequencingRun, BamFile, UnalignedReads, SingleSampleVCF, QC, Experiment, EnrichmentKit, \ - EnrichmentKitType -from snpdb.models import UserGridConfig, DataState +from seqauto.models import ( + QC, + BamFile, + EnrichmentKit, + EnrichmentKitType, + Experiment, + SequencingRun, + SingleSampleVCF, + UnalignedReads, +) +from snpdb.models import DataState, UserGridConfig from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder diff --git a/seqauto/grids/sequencing_software_versions_grids.py b/seqauto/grids/sequencing_software_versions_grids.py index 2ad572351..e758efb5f 100644 --- a/seqauto/grids/sequencing_software_versions_grids.py +++ b/seqauto/grids/sequencing_software_versions_grids.py @@ -1,7 +1,7 @@ from django.db.models import QuerySet from django.http import HttpRequest -from seqauto.models import Library, Sequencer, Aligner, Assay, VariantCaller, VariantCallingPipeline +from seqauto.models import Aligner, Assay, Library, Sequencer, VariantCaller, VariantCallingPipeline from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder diff --git a/seqauto/illumina/illuminate_report.py b/seqauto/illumina/illuminate_report.py index 5d967dde1..7057b117d 100644 --- a/seqauto/illumina/illuminate_report.py +++ b/seqauto/illumina/illuminate_report.py @@ -2,7 +2,7 @@ from collections import defaultdict from library.utils import FLOAT_REGEX -from seqauto.models.models_enums import SequencerRead, PairedEnd +from seqauto.models.models_enums import PairedEnd, SequencerRead from snpdb.models import DataState diff --git a/seqauto/illumina/run_parameters.py b/seqauto/illumina/run_parameters.py index b98c69815..76c28ee05 100644 --- a/seqauto/illumina/run_parameters.py +++ b/seqauto/illumina/run_parameters.py @@ -1,5 +1,4 @@ import os - import xml.etree.ElementTree as ET diff --git a/seqauto/management/commands/import_sequencing_info.py b/seqauto/management/commands/import_sequencing_info.py index 22fd914d2..d547930ca 100644 --- a/seqauto/management/commands/import_sequencing_info.py +++ b/seqauto/management/commands/import_sequencing_info.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand from library.log_utils import console_logger -from seqauto.models import Sequencer, SequencingInfo, EnrichmentKit +from seqauto.models import EnrichmentKit, Sequencer, SequencingInfo from snpdb.models import Lab, LabProject LAB_NAME = "lab name" diff --git a/seqauto/management/commands/import_softwarepipeline.py b/seqauto/management/commands/import_softwarepipeline.py index 9a7af644f..ff6af47ca 100644 --- a/seqauto/management/commands/import_softwarepipeline.py +++ b/seqauto/management/commands/import_softwarepipeline.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand from library.log_utils import console_logger -from seqauto.models import SoftwarePipeline, SoftwarePipelineNode, SoftwarePipelineEdge +from seqauto.models import SoftwarePipeline, SoftwarePipelineEdge, SoftwarePipelineNode PIPELINE_NAME = "Pipeline Name" PIPELINE_VERSION = "Pipeline Version" diff --git a/seqauto/management/commands/reload_qc_gene_coverage.py b/seqauto/management/commands/reload_qc_gene_coverage.py index 1d5d01cdb..66f5cd64c 100644 --- a/seqauto/management/commands/reload_qc_gene_coverage.py +++ b/seqauto/management/commands/reload_qc_gene_coverage.py @@ -12,7 +12,7 @@ from genes.models import GeneCoverageCollection from genes.tasks.gene_coverage_tasks import reload_gene_coverage_collection from library.log_utils import log_traceback -from seqauto.models import SequencingRun, QCGeneCoverage +from seqauto.models import QCGeneCoverage, SequencingRun from snpdb.models import DataState diff --git a/seqauto/management/commands/seqauto_api_client.py b/seqauto/management/commands/seqauto_api_client.py index ea136757b..ac4450c7b 100755 --- a/seqauto/management/commands/seqauto_api_client.py +++ b/seqauto/management/commands/seqauto_api_client.py @@ -80,7 +80,7 @@ def handle(self, *args, **options): kwargs["files"] = {"file": open(filename, 'rb')} kwargs["params"] = {"path": filename} else: - with open(filename, "r") as f: + with open(filename) as f: json_data = json.load(f) kwargs["json"] = json_data diff --git a/seqauto/management/commands/seqauto_api_client2.py b/seqauto/management/commands/seqauto_api_client2.py index 9b7c9bc38..4fb096718 100755 --- a/seqauto/management/commands/seqauto_api_client2.py +++ b/seqauto/management/commands/seqauto_api_client2.py @@ -4,10 +4,10 @@ import os from dataclasses import dataclass, field from datetime import date, datetime -from typing import Optional, List, Dict +from typing import Optional import requests -from dataclasses_json import dataclass_json, config +from dataclasses_json import config, dataclass_json from django.conf import settings from django.core.management.base import BaseCommand @@ -43,7 +43,7 @@ class SequencingSample: sample_project: Optional[str] = None is_control: bool = False failed: bool = False - data: List[dict] = field(default_factory=lambda: [], metadata=config(field_name="sequencingsampledata_set")) + data: list[dict] = field(default_factory=lambda: [], metadata=config(field_name="sequencingsampledata_set")) @dataclass_json @@ -53,7 +53,7 @@ class SampleSheet: sequencing_run: SequencingRun file_last_modified: datetime hash: str - sequencing_samples: List[SequencingSample] = field(metadata=config(field_name="sequencingsample_set")) + sequencing_samples: list[SequencingSample] = field(metadata=config(field_name="sequencingsample_set")) @dataclass_json @@ -140,7 +140,7 @@ class QC: class QCGeneList: path: str qc: QC - gene_list: List[str] + gene_list: list[str] @dataclass_json @@ -243,7 +243,7 @@ def create_joint_called_vcf(self, joint_called_vcf: JointCalledVCF): # Backwards-compat alias (predates the JointCalledVCF rename) create_sample_sheet_combined_vcf_file = create_joint_called_vcf - def create_sequencing_data(self, sample_sheet_lookup: SampleSheetLookup, sequencing_files: List[SequencingFile]): + def create_sequencing_data(self, sample_sheet_lookup: SampleSheetLookup, sequencing_files: list[SequencingFile]): records = [] for sf in sequencing_files: data = sf.to_dict() @@ -268,7 +268,7 @@ def create_qc_gene_list(self, qc_gene_list: QCGeneList): return self._post("seqauto/api/v1/qc_gene_list/", json_data) - def create_multiple_qc_gene_lists(self, qc_gene_lists: List[QCGeneList]): + def create_multiple_qc_gene_lists(self, qc_gene_lists: list[QCGeneList]): json_data = { "records": [ qcgl.to_dict() for qcgl in qc_gene_lists @@ -282,7 +282,7 @@ def create_qc_exec_stats(self, qc_exec_stats: QCExecStats): return self._post("seqauto/api/v1/qc_exec_summary/", json_data) - def create_multiple_qc_exec_stats(self, qc_exec_stats: List[QCExecStats]): + def create_multiple_qc_exec_stats(self, qc_exec_stats: list[QCExecStats]): json_data = { "records": [ qces.to_dict() for qces in qc_exec_stats @@ -291,7 +291,7 @@ def create_multiple_qc_exec_stats(self, qc_exec_stats: List[QCExecStats]): return self._post("seqauto/api/v1/qc_exec_summary/bulk_create", json_data) - def create_multiple_qc_gene_coverage(self, qc_gene_coverage_list: List[QCGeneCoverage]): + def create_multiple_qc_gene_coverage(self, qc_gene_coverage_list: list[QCGeneCoverage]): json_data = { "records": [ qcgc.to_dict() for qcgc in qc_gene_coverage_list @@ -501,7 +501,7 @@ def handle(self, *args, **options): print("-" * 50) @staticmethod - def _get_qc_by_sample_name(sample_sheet_lookup: SampleSheetLookup, sequencing_files: List[SequencingFile]) -> Dict[ + def _get_qc_by_sample_name(sample_sheet_lookup: SampleSheetLookup, sequencing_files: list[SequencingFile]) -> dict[ str, QC]: bam_and_vcf_by_name = {} for sf in sequencing_files: diff --git a/seqauto/management/commands/set_gold_standard_runs.py b/seqauto/management/commands/set_gold_standard_runs.py index 5dd4fbcdf..769a111de 100644 --- a/seqauto/management/commands/set_gold_standard_runs.py +++ b/seqauto/management/commands/set_gold_standard_runs.py @@ -6,7 +6,7 @@ from django.core.management.base import BaseCommand from library.utils.file_utils import file_to_array -from seqauto.models import EnrichmentKit, SequencingRun, GoldReference, ImportStatus +from seqauto.models import EnrichmentKit, GoldReference, ImportStatus, SequencingRun from seqauto.tasks.gold_summary_tasks import calculate_gold_summary diff --git a/seqauto/models.py b/seqauto/models.py index 75e23a3f5..0a8c8a6b1 100644 --- a/seqauto/models.py +++ b/seqauto/models.py @@ -1,11 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from seqauto.models.models_seqauto import SeqAutoRun, SeqAutoRecord, SeqAutoMessage, SequencingRun, SequencingRunWiki, \ - SampleSheet, SequencingRunCurrentSampleSheet, SequencingSample, SequencingSampleData, SampleFromSequencingSample, \ - VCFFromSequencingRun, IlluminaFlowcellQC, ReadQ30, IlluminaIndexQC, Fastq, FastQC, UnalignedReads, BamFile, \ - Flagstats, SingleSampleVCF, JointCalledVCF, QC, QCGeneList, QCExecSummary, ExecSummaryReferenceRange, \ - QCGeneCoverage, GoldReference, GoldGeneCoverageCollection, GoldCoverageSummary, QCType, QCColumn, JobScript -from seqauto.models.models_sequencing import SequencerModel, Sequencer, EnrichmentKit, Library, Assay, Experiment, \ - SequencingInfo -from seqauto.models.models_software import VariantCaller, Aligner, VariantCallingPipeline, SoftwarePipeline # pylint: enable=unused-import diff --git a/seqauto/models/models_seqauto.py b/seqauto/models/models_seqauto.py index 8ab9c4ee5..242704d95 100644 --- a/seqauto/models/models_seqauto.py +++ b/seqauto/models/models_seqauto.py @@ -10,20 +10,30 @@ from cache_memoize import cache_memoize from django.conf import settings from django.contrib import messages -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.contrib.postgres.fields import DecimalRangeField from django.core.cache import cache from django.db import models -from django.db.models import Value, When, Case, IntegerField, Max -from django.db.models.deletion import SET_NULL, CASCADE, PROTECT +from django.db.models import Case, IntegerField, Max, Value, When +from django.db.models.deletion import CASCADE, PROTECT, SET_NULL from django.db.models.signals import post_delete, pre_delete from django.dispatch.dispatcher import receiver from django.urls.base import reverse from django.utils.timezone import make_aware from django_extensions.db.models import TimeStampedModel -from genes.models import GeneListCategory, CustomTextGeneList, GeneList, GeneCoverageCollection, \ - Transcript, GeneSymbol, SampleGeneList, TranscriptVersion, GeneCoverageCanonicalTranscript, ActiveSampleGeneList +from genes.models import ( + ActiveSampleGeneList, + CustomTextGeneList, + GeneCoverageCanonicalTranscript, + GeneCoverageCollection, + GeneList, + GeneListCategory, + GeneSymbol, + SampleGeneList, + Transcript, + TranscriptVersion, +) from library.constants import DAY_SECS from library.enums.log_level import LogLevel from library.genomics.vcf_utils import get_variant_caller_and_version_from_vcf @@ -34,17 +44,23 @@ from patients.models import FakeData, Patient from seqauto.illumina import illuminate_report from seqauto.illumina.illumina_sequencers import SEQUENCING_RUN_REGEX -from seqauto.models.models_enums import DataGeneration, SequencerRead, PairedEnd, \ - SequencingFileType, JobScriptStatus, SeqAutoRunStatus -from seqauto.models.models_sequencing import Sequencer, EnrichmentKit, Experiment +from seqauto.models.models_enums import ( + DataGeneration, + JobScriptStatus, + PairedEnd, + SeqAutoRunStatus, + SequencerRead, + SequencingFileType, +) +from seqauto.models.models_sequencing import EnrichmentKit, Experiment, Sequencer from seqauto.models.models_software import Aligner, VariantCaller from seqauto.qc.exec_summary import load_exec_summary from seqauto.qc.fastqc_parser import read_fastqc_data from seqauto.qc.flag_stats import load_flagstats from seqauto.qc.qc_utils import meta_data_file from seqauto.signals.signals_list import sequencing_run_sample_sheet_created_signal -from snpdb.models import VCF, Sample, GenomeBuild, DataState, InheritanceManager, Wiki -from snpdb.models.models_enums import ImportStatus, ImportSource +from snpdb.models import VCF, DataState, GenomeBuild, InheritanceManager, Sample, Wiki +from snpdb.models.models_enums import ImportSource, ImportStatus from variantgrid.celery import app @@ -445,7 +461,9 @@ def set_as_current_sample_sheet(self, sequencing_run, created=False, seqauto_run current_ss = sequencing_run.sequencingruncurrentsamplesheet on_disk_not_current = current_ss.sample_sheet != self if on_disk_not_current: - from seqauto.sequencing_files.create_resource_models import current_sample_sheet_changed + from seqauto.sequencing_files.create_resource_models import ( + current_sample_sheet_changed, + ) current_sample_sheet_changed(seqauto_run, current_ss, self) except SequencingRunCurrentSampleSheet.DoesNotExist: diff --git a/seqauto/models/models_sequencing.py b/seqauto/models/models_sequencing.py index 9e04bcc52..d5530467d 100644 --- a/seqauto/models/models_sequencing.py +++ b/seqauto/models/models_sequencing.py @@ -2,11 +2,17 @@ from django.db.models import CASCADE, PROTECT from django.urls import reverse -from genes.models import GeneList, CanonicalTranscriptCollection +from genes.models import CanonicalTranscriptCollection, GeneList from library.preview_request import PreviewModelMixin from seqauto.illumina import illumina_sequencers from seqauto.models.models_enums import DataGeneration, EnrichmentKitType -from snpdb.models import Manufacturer, GenomicIntervalsCollection, SET_NULL, LabProject, VariantsType +from snpdb.models import ( + SET_NULL, + GenomicIntervalsCollection, + LabProject, + Manufacturer, + VariantsType, +) class SequencerModel(models.Model): diff --git a/seqauto/models/models_software.py b/seqauto/models/models_software.py index 9a7564cc6..0ab7d601a 100644 --- a/seqauto/models/models_software.py +++ b/seqauto/models/models_software.py @@ -1,7 +1,7 @@ from django.db import models from django.db.models import CASCADE from django.urls import reverse -from django_dag.models import node_factory, edge_factory +from django_dag.models import edge_factory, node_factory from snpdb.models import SoftwareVersion diff --git a/seqauto/pbs/create_jobs.py b/seqauto/pbs/create_jobs.py index d3ed2650b..dab2e2982 100644 --- a/seqauto/pbs/create_jobs.py +++ b/seqauto/pbs/create_jobs.py @@ -6,11 +6,20 @@ from django.conf import settings from library.utils.file_utils import add_permissions_to_file, mk_path_for_file -from seqauto.job_scripts import get_job_data, create_bash_script -from seqauto.models import SingleSampleVCF, SampleSheet, BamFile, \ - SequencingFileType, QC, JointCalledVCF, IlluminaFlowcellQC, \ - FastQC, Flagstats, JobScript -from seqauto.pbs.pbs_scripts import get_dependency_flags, create_pbs_script +from seqauto.job_scripts import create_bash_script, get_job_data +from seqauto.models import ( + QC, + BamFile, + FastQC, + Flagstats, + IlluminaFlowcellQC, + JobScript, + JointCalledVCF, + SampleSheet, + SequencingFileType, + SingleSampleVCF, +) +from seqauto.pbs.pbs_scripts import create_pbs_script, get_dependency_flags from snpdb.models import DataState diff --git a/seqauto/qc/sequencing_run_utils.py b/seqauto/qc/sequencing_run_utils.py index 3ec3803d8..badca61e1 100644 --- a/seqauto/qc/sequencing_run_utils.py +++ b/seqauto/qc/sequencing_run_utils.py @@ -8,7 +8,7 @@ from collections import defaultdict -from seqauto.models import SequencingRun, IlluminaFlowcellQC, ReadQ30, QCExecSummary, QCType +from seqauto.models import IlluminaFlowcellQC, QCExecSummary, QCType, ReadQ30, SequencingRun from seqauto.models.models_enums import QCCompareType ILLUMINA_FLOWCELL_QC_COLUMNS = ["mean_cluster_density", "mean_pf_cluster_density", "total_clusters", diff --git a/seqauto/seqauto_stats.py b/seqauto/seqauto_stats.py index 8febe4321..93273d76e 100644 --- a/seqauto/seqauto_stats.py +++ b/seqauto/seqauto_stats.py @@ -6,8 +6,8 @@ import pandas as pd from django.conf import settings -from library.utils.date_utils import parse_yymm, get_months_since, month_range -from seqauto.models import SequencingSample, SequencingRun +from library.utils.date_utils import get_months_since, month_range, parse_yymm +from seqauto.models import SequencingRun, SequencingSample def get_sample_enrichment_kits_df(): @@ -93,7 +93,7 @@ def group_enrichment_kits_df(df, by_column, max_groups=None, max_years=None): if max_groups is not None and len(enrichment_kit_data) > max_groups: named_groups = max_groups - 1 enrichment_kit_data_sum = [(name, array, sum(array)) for name, array in enrichment_kit_data] - enrichment_kit_data_sum = list(sorted(enrichment_kit_data_sum, key=operator.itemgetter(2), reverse=True)) + enrichment_kit_data_sum = sorted(enrichment_kit_data_sum, key=operator.itemgetter(2), reverse=True) enrichment_kit_data = [] for name, array, _ in enrichment_kit_data_sum[:named_groups]: enrichment_kit_data.append((name, array)) diff --git a/seqauto/sequencing_files/create_resource_models.py b/seqauto/sequencing_files/create_resource_models.py index 0dd50e783..80b7db0d0 100644 --- a/seqauto/sequencing_files/create_resource_models.py +++ b/seqauto/sequencing_files/create_resource_models.py @@ -4,7 +4,8 @@ import sys import traceback from collections import Counter, defaultdict -from typing import Iterable, Optional +from collections.abc import Iterable +from typing import Optional import pandas as pd from django.conf import settings @@ -16,16 +17,39 @@ from library.enums.log_level import LogLevel from library.log_utils import get_traceback, log_traceback from library.utils import file_md5sum -from library.utils.file_utils import name_from_filename, file_to_array +from library.utils.file_utils import file_to_array, name_from_filename from seqauto.illumina.run_parameters import get_run_parameters from seqauto.illumina.samplesheet import convert_sheet_to_df, samplesheet_is_valid -from seqauto.models import Sequencer, SequencingRun, SequencingSample, SequencingSampleData, Fastq, SampleSheet, \ - UnalignedReads, BamFile, SingleSampleVCF, QC, JointCalledVCF, IlluminaFlowcellQC, FastQC, Flagstats, \ - DontAutoLoadException, Experiment, SampleFromSequencingSample, QCGeneList, \ - get_samples_by_sequencing_sample, QCGeneCoverage, SeqAutoMessage, SeqAutoRecord, get_variant_caller_from_vcf_file +from seqauto.models import ( + QC, + BamFile, + DontAutoLoadException, + Experiment, + Fastq, + FastQC, + Flagstats, + IlluminaFlowcellQC, + JointCalledVCF, + QCGeneCoverage, + QCGeneList, + SampleFromSequencingSample, + SampleSheet, + SeqAutoMessage, + SeqAutoRecord, + Sequencer, + SequencingRun, + SequencingSample, + SequencingSampleData, + SingleSampleVCF, + UnalignedReads, + get_samples_by_sequencing_sample, + get_variant_caller_from_vcf_file, +) from seqauto.models.models_enums import SequencingFileType -from seqauto.signals.signals_list import sequencing_run_current_sample_sheet_changed_signal, \ - sequencing_run_created_signal +from seqauto.signals.signals_list import ( + sequencing_run_created_signal, + sequencing_run_current_sample_sheet_changed_signal, +) from snpdb.models import DataState from upload.models import BackendVCF from upload.vcf.vcf_import import link_samples_and_vcfs_to_sequencing @@ -143,11 +167,11 @@ class FlowcellChecker: def __init__(self): self.skip_patterns = [] - skip_patterns = getattr(settings, "SEQAUTO_SKIP_FLOWCELLS_PATTERNS") + skip_patterns = settings.SEQAUTO_SKIP_FLOWCELLS_PATTERNS if skip_patterns: self.skip_patterns.extend(skip_patterns) - skip_file = getattr(settings, "SEQAUTO_SKIP_FLOWCELLS_FILE") + skip_file = settings.SEQAUTO_SKIP_FLOWCELLS_FILE if skip_file: with open(skip_file) as f: for line in f: @@ -159,7 +183,7 @@ def __init__(self): logging.info("Flowcell Skip patterns:") logging.info(self.skip_patterns) - self.skip_flowcell_filename = getattr(settings, "SEQAUTO_SKIP_INDIVIDUAL_FLOWCELL_FILE") + self.skip_flowcell_filename = settings.SEQAUTO_SKIP_INDIVIDUAL_FLOWCELL_FILE def skip(self, sequencing_run_dir): for p in self.skip_patterns: diff --git a/seqauto/serializers/seqauto_qc_serializers.py b/seqauto/serializers/seqauto_qc_serializers.py index 3bd144844..11e0f7bf1 100644 --- a/seqauto/serializers/seqauto_qc_serializers.py +++ b/seqauto/serializers/seqauto_qc_serializers.py @@ -1,10 +1,25 @@ from rest_framework import serializers -from genes.serializers import SampleGeneListSerializer, GeneCoverageCollectionSerializer -from seqauto.models import IlluminaFlowcellQC, QCGeneList, QC, QCGeneCoverage, QCExecSummary, FastQC, SequencingSample, \ - SampleSheet, SequencingRun, SingleSampleVCF -from seqauto.serializers.sequencing_serializers import SampleSheetLookupSerializer, FastqSerializer, \ - BamFilePathSerializer, SingleSampleVCFPathSerializer, SequencingSampleLookupSerializer +from genes.serializers import GeneCoverageCollectionSerializer, SampleGeneListSerializer +from seqauto.models import ( + QC, + FastQC, + IlluminaFlowcellQC, + QCExecSummary, + QCGeneCoverage, + QCGeneList, + SampleSheet, + SequencingRun, + SequencingSample, + SingleSampleVCF, +) +from seqauto.serializers.sequencing_serializers import ( + BamFilePathSerializer, + FastqSerializer, + SampleSheetLookupSerializer, + SequencingSampleLookupSerializer, + SingleSampleVCFPathSerializer, +) from snpdb.models import DataState diff --git a/seqauto/serializers/seqauto_serializers.py b/seqauto/serializers/seqauto_serializers.py index 288ae13b8..fd488d671 100644 --- a/seqauto/serializers/seqauto_serializers.py +++ b/seqauto/serializers/seqauto_serializers.py @@ -1,8 +1,11 @@ import numpy as np from rest_framework import serializers -from genes.serializers import TranscriptSerializer, GeneSymbolSerializer, \ - TranscriptVersionSerializer +from genes.serializers import ( + GeneSymbolSerializer, + TranscriptSerializer, + TranscriptVersionSerializer, +) from seqauto.models import GoldCoverageSummary, GoldReference from seqauto.serializers.enrichment_kit_serializers import EnrichmentKitSummarySerializer diff --git a/seqauto/serializers/sequencing_serializers.py b/seqauto/serializers/sequencing_serializers.py index 0b5da1aaf..c7eb16cf4 100644 --- a/seqauto/serializers/sequencing_serializers.py +++ b/seqauto/serializers/sequencing_serializers.py @@ -2,11 +2,26 @@ from rest_framework import serializers -from seqauto.models import Sequencer, Experiment, VariantCaller, SequencingRun, SequencerModel, SampleSheet, \ - SequencingSampleData, SequencingSample, UnalignedReads, Flagstats, JointCalledVCF, SingleSampleVCF, \ - BamFile, Fastq, Aligner, PairedEnd +from seqauto.models import ( + Aligner, + BamFile, + Experiment, + Fastq, + Flagstats, + JointCalledVCF, + PairedEnd, + SampleSheet, + Sequencer, + SequencerModel, + SequencingRun, + SequencingSample, + SequencingSampleData, + SingleSampleVCF, + UnalignedReads, + VariantCaller, +) from seqauto.serializers import EnrichmentKitSerializer, EnrichmentKitSummarySerializer -from snpdb.models import Manufacturer, DataState +from snpdb.models import DataState, Manufacturer class ManufacturerSerializer(serializers.ModelSerializer): diff --git a/seqauto/signals/enrichment_kit_search.py b/seqauto/signals/enrichment_kit_search.py index 5ea830b9f..6a45d0fed 100644 --- a/seqauto/signals/enrichment_kit_search.py +++ b/seqauto/signals/enrichment_kit_search.py @@ -1,5 +1,5 @@ from seqauto.models import EnrichmentKit -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ANY +from snpdb.search import HAS_3_ANY, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/seqauto/signals/experiment_search.py b/seqauto/signals/experiment_search.py index f73483442..04e132a72 100644 --- a/seqauto/signals/experiment_search.py +++ b/seqauto/signals/experiment_search.py @@ -1,5 +1,5 @@ from seqauto.models import Experiment -from snpdb.search import search_receiver, HAS_ALPHA_PATTERN, SearchInputInstance +from snpdb.search import HAS_ALPHA_PATTERN, SearchInputInstance, search_receiver @search_receiver(search_type=Experiment, pattern=HAS_ALPHA_PATTERN) diff --git a/seqauto/signals/sequencing_run_search.py b/seqauto/signals/sequencing_run_search.py index e82c8c515..4146a05a5 100644 --- a/seqauto/signals/sequencing_run_search.py +++ b/seqauto/signals/sequencing_run_search.py @@ -1,7 +1,7 @@ import re from seqauto.models import SequencingRun -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver SEQUENCING_RUN_REGEX = re.compile(r"\d{6}[_-](NS|NB|M|D|SN|K|ST|A)(.{3,7})_\d{4}_(0{9}-.{5}|.{10})") diff --git a/seqauto/tasks/gold_summary_tasks.py b/seqauto/tasks/gold_summary_tasks.py index f387b70c5..ba44f9073 100644 --- a/seqauto/tasks/gold_summary_tasks.py +++ b/seqauto/tasks/gold_summary_tasks.py @@ -18,11 +18,17 @@ from scipy import stats from eventlog.models import create_event -from genes.models import GeneCoverageCollection, GeneCoverageCanonicalTranscript +from genes.models import GeneCoverageCanonicalTranscript, GeneCoverageCollection from library.enums.log_level import LogLevel from library.log_utils import get_traceback, log_traceback from library.utils import get_single_element -from seqauto.models import SequencingRun, GoldReference, GoldCoverageSummary, GoldGeneCoverageCollection, EnrichmentKit +from seqauto.models import ( + EnrichmentKit, + GoldCoverageSummary, + GoldGeneCoverageCollection, + GoldReference, + SequencingRun, +) from snpdb.models import DataState from snpdb.models.models_enums import ImportStatus diff --git a/seqauto/tasks/scan_run_jobs.py b/seqauto/tasks/scan_run_jobs.py index ea511b7c8..dc6368d56 100644 --- a/seqauto/tasks/scan_run_jobs.py +++ b/seqauto/tasks/scan_run_jobs.py @@ -9,7 +9,7 @@ from django.utils import timezone from library.log_utils import get_traceback -from library.utils.file_utils import name_from_filename, mk_path +from library.utils.file_utils import mk_path, name_from_filename from seqauto.models import SeqAutoRun from seqauto.models.models_enums import SequencingFileType from seqauto.pbs.create_jobs import create_jobs_and_launch_script diff --git a/seqauto/tests/test_joint_called_vcf.py b/seqauto/tests/test_joint_called_vcf.py index cee254892..af90d7ade 100644 --- a/seqauto/tests/test_joint_called_vcf.py +++ b/seqauto/tests/test_joint_called_vcf.py @@ -16,14 +16,14 @@ SequencingRun, SequencingRunCurrentSampleSheet, SequencingSample, - UnalignedReads, SingleSampleVCF, + UnalignedReads, VariantCaller, VCFFromSequencingRun, ) from seqauto.models.models_enums import DataGeneration, PairedEnd from seqauto.serializers.sequencing_serializers import JointCalledVCFSerializer -from snpdb.models import VCF, Sample, DataState +from snpdb.models import VCF, DataState, Sample from upload.models import BackendVCF, UploadedFile, UploadedVCF from upload.vcf.vcf_import import link_samples_and_vcfs_to_sequencing diff --git a/seqauto/tests/test_models.py b/seqauto/tests/test_models.py index c2a0713eb..d21168854 100644 --- a/seqauto/tests/test_models.py +++ b/seqauto/tests/test_models.py @@ -5,15 +5,31 @@ from django.test import TestCase from genes.canonical_transcripts.canonical_transcript_manager import CanonicalTranscriptManager -from genes.canonical_transcripts.create_canonical_transcripts import create_canonical_transcript_collection +from genes.canonical_transcripts.create_canonical_transcripts import ( + create_canonical_transcript_collection, +) from genes.gene_matching import GeneSymbolMatcher from genes.models import GeneCoverageCollection, TranscriptVersion -from seqauto.models import SequencerModel, Sequencer, SequencingRun, SampleSheet, \ - SequencingRunCurrentSampleSheet, SequencingSample, Fastq, UnalignedReads, Aligner, \ - BamFile, SingleSampleVCF, QC, VariantCaller, EnrichmentKit, QCGeneCoverage +from seqauto.models import ( + QC, + Aligner, + BamFile, + EnrichmentKit, + Fastq, + QCGeneCoverage, + SampleSheet, + Sequencer, + SequencerModel, + SequencingRun, + SequencingRunCurrentSampleSheet, + SequencingSample, + SingleSampleVCF, + UnalignedReads, + VariantCaller, +) from seqauto.models.models_enums import DataGeneration from seqauto.tasks.gold_summary_tasks import calculate_gold_summary -from snpdb.models import Manufacturer, GenomeBuild, DataState +from snpdb.models import DataState, GenomeBuild, Manufacturer class TestSeqAutoModels(TestCase): diff --git a/seqauto/tests/test_urls.py b/seqauto/tests/test_urls.py index 8b7659d38..ecaa831f1 100644 --- a/seqauto/tests/test_urls.py +++ b/seqauto/tests/test_urls.py @@ -3,8 +3,15 @@ from django.contrib.auth.models import User from library.django_utils.unittest_utils import URLTestCase -from seqauto.models import QCColumn, EnrichmentKit, SequencingRun, SequencerModel, DataGeneration, Sequencer -from snpdb.models import Manufacturer, DataState +from seqauto.models import ( + DataGeneration, + EnrichmentKit, + QCColumn, + Sequencer, + SequencerModel, + SequencingRun, +) +from snpdb.models import DataState, Manufacturer class Test(URLTestCase): diff --git a/seqauto/urls.py b/seqauto/urls.py index 299373549..a275af05e 100644 --- a/seqauto/urls.py +++ b/seqauto/urls.py @@ -3,22 +3,64 @@ from library.django_utils.jqgrid_view import JQGridView from seqauto import views, views_autocomplete, views_rest -from seqauto.grids.qc_data_grids import IlluminaFlowcellQCGrid, FastQCGrid, FlagstatsGrid, \ - QCExecSummaryGrid -from seqauto.grids.seqauto_grids import SeqAutoRunsGrid, EnrichmentKitGeneCoverageGrid, \ - GoldCoverageSummaryGrid, SequencingSamplesGrid, SequencingSamplesHistoricalGrid -from seqauto.grids.sequencing_data_grids import SequencingRunListGrid, \ - UnalignedReadsListGrid, BamFileListGrid, SingleSampleVCFListGrid, QCFileListGrid, \ - EnrichmentKitColumns, ExperimentColumns -from seqauto.grids.sequencing_software_versions_grids import LibraryColumns, SequencerColumns, \ - AssayColumns, AlignerColumns, VariantCallerColumns, VariantCallingPipelineColumns -from seqauto.views import SequencerUpdate, LibraryUpdate, AssayUpdate, VariantCallerUpdate, \ - AlignerUpdate, VariantCallingPipelineUpdate -from seqauto.views_rest import SequencingRunViewSet, EnrichmentKitViewSet, SequencerModelViewSet, SequencerViewSet, \ - ExperimentViewSet, VariantCallerViewSet, SingleSampleVCFViewSet, JointCalledVCFViewSet, FastQCViewSet, \ - SampleSheetViewSet, IlluminaFlowcellQCViewSet, QCGeneListViewSet, QCGeneCoverageViewSet, \ - QCExecSummaryViewSet, QCGeneListBulkCreateView, SequencingFilesBulkCreateView, QCExecSummaryBulkCreateView, \ - QCGeneCoverageBulkCreateView +from seqauto.grids.qc_data_grids import ( + FastQCGrid, + FlagstatsGrid, + IlluminaFlowcellQCGrid, + QCExecSummaryGrid, +) +from seqauto.grids.seqauto_grids import ( + EnrichmentKitGeneCoverageGrid, + GoldCoverageSummaryGrid, + SeqAutoRunsGrid, + SequencingSamplesGrid, + SequencingSamplesHistoricalGrid, +) +from seqauto.grids.sequencing_data_grids import ( + BamFileListGrid, + EnrichmentKitColumns, + ExperimentColumns, + QCFileListGrid, + SequencingRunListGrid, + SingleSampleVCFListGrid, + UnalignedReadsListGrid, +) +from seqauto.grids.sequencing_software_versions_grids import ( + AlignerColumns, + AssayColumns, + LibraryColumns, + SequencerColumns, + VariantCallerColumns, + VariantCallingPipelineColumns, +) +from seqauto.views import ( + AlignerUpdate, + AssayUpdate, + LibraryUpdate, + SequencerUpdate, + VariantCallerUpdate, + VariantCallingPipelineUpdate, +) +from seqauto.views_rest import ( + EnrichmentKitViewSet, + ExperimentViewSet, + FastQCViewSet, + IlluminaFlowcellQCViewSet, + JointCalledVCFViewSet, + QCExecSummaryBulkCreateView, + QCExecSummaryViewSet, + QCGeneCoverageBulkCreateView, + QCGeneCoverageViewSet, + QCGeneListBulkCreateView, + QCGeneListViewSet, + SampleSheetViewSet, + SequencerModelViewSet, + SequencerViewSet, + SequencingFilesBulkCreateView, + SequencingRunViewSet, + SingleSampleVCFViewSet, + VariantCallerViewSet, +) from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/seqauto/views.py b/seqauto/views.py index d0d57f230..6f87b3990 100644 --- a/seqauto/views.py +++ b/seqauto/views.py @@ -8,8 +8,8 @@ from django.core.exceptions import PermissionDenied from django.db.models.aggregates import Count from django.db.models.query_utils import Q -from django.http.response import JsonResponse, HttpResponse -from django.shortcuts import render, get_object_or_404, redirect +from django.http.response import HttpResponse, JsonResponse +from django.shortcuts import get_object_or_404, redirect, render from django.utils.decorators import method_decorator from django.utils.safestring import mark_safe from django.views.decorators.http import require_POST @@ -22,23 +22,53 @@ from library.log_utils import log_traceback from library.utils import full_class_name from seqauto import forms -from seqauto.forms import SequencingRunForm, AllEnrichmentKitForm, AutocompleteSequencingRunForm +from seqauto.forms import AllEnrichmentKitForm, AutocompleteSequencingRunForm, SequencingRunForm from seqauto.graphs.index_metrics_qc_graph import IndexMetricsQCGraph from seqauto.graphs.qc_exec_summary_graph import QCExecSummaryGraph from seqauto.graphs.sequencing_run_qc_graph import SequencingRunQCGraph from seqauto.illumina.run_parameters import get_run_parameters -from seqauto.models import BamFile, SequencingRun, FastQC, Flagstats, UnalignedReads, QCType, SingleSampleVCF, QC, \ - Experiment, SequencingSample, JointCalledVCF, QCExecSummary, IlluminaFlowcellQC, SeqAutoRun, \ - Library, Sequencer, Assay, Aligner, VariantCaller, VariantCallingPipeline, SoftwarePipelineNode, \ - GoldReference, GoldGeneCoverageCollection, EnrichmentKit, QCGeneCoverage, QCColumn +from seqauto.models import ( + QC, + Aligner, + Assay, + BamFile, + EnrichmentKit, + Experiment, + FastQC, + Flagstats, + GoldGeneCoverageCollection, + GoldReference, + IlluminaFlowcellQC, + JointCalledVCF, + Library, + QCColumn, + QCExecSummary, + QCGeneCoverage, + QCType, + SeqAutoRun, + Sequencer, + SequencingRun, + SequencingSample, + SingleSampleVCF, + SoftwarePipelineNode, + UnalignedReads, + VariantCaller, + VariantCallingPipeline, +) from seqauto.models.models_enums import QCCompareType, SequencingFileType -from seqauto.qc.sequencing_run_utils import get_sequencing_run_data, get_qc_exec_summary_data, \ - get_sequencing_run_columns, SEQUENCING_RUN_QC_COLUMNS +from seqauto.qc.sequencing_run_utils import ( + SEQUENCING_RUN_QC_COLUMNS, + get_qc_exec_summary_data, + get_sequencing_run_columns, + get_sequencing_run_data, +) from seqauto.seqauto_stats import get_sample_enrichment_kits_df -from seqauto.sequencing_files.create_resource_models import assign_old_sample_sheet_data_to_current_sample_sheet +from seqauto.sequencing_files.create_resource_models import ( + assign_old_sample_sheet_data_to_current_sample_sheet, +) from seqauto.tasks.scan_run_jobs import process_seq_auto_run from snpdb.graphs import graphcache -from snpdb.models import Sample, UserSettings, DataState +from snpdb.models import DataState, Sample, UserSettings def sequencing_data(request): diff --git a/seqauto/views_autocomplete.py b/seqauto/views_autocomplete.py index 3f333ba71..1acbb93b5 100644 --- a/seqauto/views_autocomplete.py +++ b/seqauto/views_autocomplete.py @@ -1,9 +1,9 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page -from library.constants import WEEK_SECS, MINUTE_SECS +from library.constants import MINUTE_SECS, WEEK_SECS from library.django_utils.autocomplete_utils import AutocompleteView -from seqauto.models import QCColumn, EnrichmentKit, SequencingRun +from seqauto.models import EnrichmentKit, QCColumn, SequencingRun @method_decorator(cache_page(WEEK_SECS), name='dispatch') diff --git a/seqauto/views_rest.py b/seqauto/views_rest.py index a7fd8e504..7a43020e3 100644 --- a/seqauto/views_rest.py +++ b/seqauto/views_rest.py @@ -10,9 +10,9 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view from rest_framework import status -from rest_framework.generics import get_object_or_404, RetrieveAPIView +from rest_framework.generics import RetrieveAPIView, get_object_or_404 from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.viewsets import ModelViewSet @@ -21,18 +21,52 @@ from genes.views.views import get_coverage_stats from library.constants import WEEK_SECS from library.utils import defaultdict_to_dict -from seqauto.models import GoldCoverageSummary, EnrichmentKit, SequencerModel, Sequencer, Experiment, VariantCaller, \ - SequencingRun, SampleSheet, SingleSampleVCF, JointCalledVCF, FastQC, QCExecSummary, QCGeneCoverage, QCGeneList, \ - QC, IlluminaFlowcellQC -from seqauto.serializers import EnrichmentKitSerializer, \ - GoldCoverageSummarySerializer, EnrichmentKitSummarySerializer -from seqauto.serializers.seqauto_qc_serializers import FastQCSerializer, QCExecSummarySerializer, \ - QCGeneCoverageSerializer, QCGeneListSerializer, QCSerializer, IlluminaFlowcellQCSerializer, \ - QCGeneListCreateSerializer, QCGeneListBulkCreateSerializer, QCExecSummaryBulkCreateSerializer, \ - QCGeneCoverageBulkCreateSerializer -from seqauto.serializers.sequencing_serializers import SequencerModelSerializer, SequencerSerializer, \ - ExperimentSerializer, VariantCallerSerializer, SequencingRunSerializer, SampleSheetSerializer, \ - SingleSampleVCFSerializer, JointCalledVCFSerializer, SequencingFilesBulkCreateSerializer +from seqauto.models import ( + QC, + EnrichmentKit, + Experiment, + FastQC, + GoldCoverageSummary, + IlluminaFlowcellQC, + JointCalledVCF, + QCExecSummary, + QCGeneCoverage, + QCGeneList, + SampleSheet, + Sequencer, + SequencerModel, + SequencingRun, + SingleSampleVCF, + VariantCaller, +) +from seqauto.serializers import ( + EnrichmentKitSerializer, + EnrichmentKitSummarySerializer, + GoldCoverageSummarySerializer, +) +from seqauto.serializers.seqauto_qc_serializers import ( + FastQCSerializer, + IlluminaFlowcellQCSerializer, + QCExecSummaryBulkCreateSerializer, + QCExecSummarySerializer, + QCGeneCoverageBulkCreateSerializer, + QCGeneCoverageSerializer, + QCGeneListBulkCreateSerializer, + QCGeneListCreateSerializer, + QCGeneListSerializer, + QCSerializer, +) +from seqauto.serializers.sequencing_serializers import ( + ExperimentSerializer, + JointCalledVCFSerializer, + SampleSheetSerializer, + SequencerModelSerializer, + SequencerSerializer, + SequencingFilesBulkCreateSerializer, + SequencingRunSerializer, + SingleSampleVCFSerializer, + VariantCallerSerializer, +) class EnrichmentKitSummaryView(RetrieveAPIView): diff --git a/snpdb/admin.py b/snpdb/admin.py index c4b1f0f8b..3bfd344e6 100644 --- a/snpdb/admin.py +++ b/snpdb/admin.py @@ -9,12 +9,29 @@ from snpdb import models from snpdb.admin_partition_archive_mixin import ArchivePartitionDataAdminMixin -from snpdb.admin_utils import ModelAdminBasics, GuardedModelAdminBasics, admin_list_column, \ - admin_action +from snpdb.admin_utils import ( + GuardedModelAdminBasics, + ModelAdminBasics, + admin_action, + admin_list_column, +) from snpdb.liftover import liftover_alleles -from snpdb.models import Allele, VariantAllele, ClinVarKey, ClinVarKeyExcludePattern, UserSettingsOverride, \ - LabUserSettingsOverride, OrganizationUserSettingsOverride, UserPageAck, Organization, Lab, GlobalSettings, Variant, \ - AlleleLiftover, SiteMessage +from snpdb.models import ( + Allele, + AlleleLiftover, + ClinVarKey, + ClinVarKeyExcludePattern, + GlobalSettings, + Lab, + LabUserSettingsOverride, + Organization, + OrganizationUserSettingsOverride, + SiteMessage, + UserPageAck, + UserSettingsOverride, + Variant, + VariantAllele, +) from snpdb.models.models_genome import GenomeBuild diff --git a/snpdb/admin_utils.py b/snpdb/admin_utils.py index a348c4f6d..7e5421ab4 100644 --- a/snpdb/admin_utils.py +++ b/snpdb/admin_utils.py @@ -1,23 +1,24 @@ import inspect +from collections.abc import Iterator from functools import cached_property -from typing import Optional, Iterator, Type +from typing import Optional from dateutil.tz import gettz from django.conf import settings from django.contrib import admin, messages from django.db import models -from django.db.models import AutoField, ForeignKey, DateTimeField, Model +from django.db.models import AutoField, DateTimeField, ForeignKey, Model from django.forms import Widget -from django.http import StreamingHttpResponse, HttpResponseRedirect +from django.http import HttpResponseRedirect, StreamingHttpResponse from django.http.response import HttpResponseBase -from django.urls import path, NoReverseMatch, reverse +from django.urls import NoReverseMatch, path, reverse from django.utils.encoding import smart_str from django.utils.safestring import SafeString from django_json_widget.widgets import JSONEditorWidget from guardian.admin import GuardedModelAdminMixin from library.log_utils import log_admin_change -from library.utils import delimited_row, WrappablePartial, limit_str +from library.utils import WrappablePartial, delimited_row, limit_str class AllValuesChoicesFieldListFilter(admin.AllValuesFieldListFilter): @@ -354,7 +355,7 @@ def get_admin_url(obj: Model): except NoReverseMatch: return None -def get_admin_model_url(model_type: Type[Model]): +def get_admin_model_url(model_type: type[Model]): try: meta = model_type._meta path = f"admin:{meta.app_label}_{meta.model_name}_changelist" diff --git a/snpdb/apps.py b/snpdb/apps.py index 5123e1998..60aa77ea8 100644 --- a/snpdb/apps.py +++ b/snpdb/apps.py @@ -13,24 +13,16 @@ class SnpdbConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from snpdb.models import Trio - from django.contrib.auth.models import User, Group + from django.contrib.auth.models import Group, User + from seqauto.signals.signals_list import backend_vcf_import_success_signal - from snpdb.signals.signal_handlers import backend_vcf_import_success_handler, trio_post_save_handler, \ - user_post_save_handler, group_post_save_handler - from snpdb.signals import vcf_health_check - from snpdb.signals import disk_usage_health_check - from snpdb.signals import lab_search - from snpdb.signals import organization_search - from snpdb.signals import user_search - from snpdb.signals import cohort_search - from snpdb.signals import sample_search - from snpdb.signals import vcf_search - from snpdb.signals import variant_search - from snpdb.signals import variant_zygosity_preview_extra - from snpdb.signals import clinvar_export_search - from snpdb.signals import scv_search - from snpdb.signals import genomics_search + from snpdb.models import Trio + from snpdb.signals.signal_handlers import ( + backend_vcf_import_success_handler, + group_post_save_handler, + trio_post_save_handler, + user_post_save_handler, + ) # pylint: enable=import-outside-toplevel,unused-import backend_vcf_import_success_signal.connect(backend_vcf_import_success_handler) diff --git a/snpdb/bcftools_liftover.py b/snpdb/bcftools_liftover.py index 3b86ea12a..3cd17a25b 100644 --- a/snpdb/bcftools_liftover.py +++ b/snpdb/bcftools_liftover.py @@ -113,7 +113,7 @@ def get_reject_vcf_filename(out_vcf_filename: str) -> str: def count_non_header_lines(filename) -> int: count = 0 - with open(filename, 'r') as file: + with open(filename) as file: for line in file: if not line.startswith('#'): count += 1 diff --git a/snpdb/clingen_allele.py b/snpdb/clingen_allele.py index ef058c8d1..a7bbc749d 100644 --- a/snpdb/clingen_allele.py +++ b/snpdb/clingen_allele.py @@ -31,10 +31,22 @@ from library.django_utils import thread_safe_unique_together_get_or_create from library.django_utils.django_file_utils import get_import_processing_filename from library.genomics.vcf_enums import VCFSymbolicAllele -from library.utils import iter_fixed_chunks, get_single_element -from snpdb.models import Allele, ClinGenAllele, GenomeBuild, Variant, VariantAllele, Contig, GenomeFasta, \ - VariantCoordinate -from snpdb.models.models_enums import AlleleOrigin, AlleleConversionTool, ClinGenAlleleExternalRecordType +from library.utils import get_single_element, iter_fixed_chunks +from snpdb.models import ( + Allele, + ClinGenAllele, + Contig, + GenomeBuild, + GenomeFasta, + Variant, + VariantAllele, + VariantCoordinate, +) +from snpdb.models.models_enums import ( + AlleleConversionTool, + AlleleOrigin, + ClinGenAlleleExternalRecordType, +) class ClinGenAlleleServerException(ClinGenAllele.ClinGenAlleleRegistryException): diff --git a/snpdb/common_variants.py b/snpdb/common_variants.py index a8baa0a76..da534c993 100644 --- a/snpdb/common_variants.py +++ b/snpdb/common_variants.py @@ -3,12 +3,18 @@ from celery.canvas import Signature from django.conf import settings -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.dispatch import receiver from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version from classification.models import Classification, variants_classification_changed_signal -from snpdb.models import CohortGenotypeCommonFilterVersion, Variant, Allele, VariantAllele, CommonVariantClassified +from snpdb.models import ( + Allele, + CohortGenotypeCommonFilterVersion, + CommonVariantClassified, + Variant, + VariantAllele, +) def get_common_filter(genome_build) -> Optional[CohortGenotypeCommonFilterVersion]: diff --git a/snpdb/forms.py b/snpdb/forms.py index d5390d1f1..c42b40341 100644 --- a/snpdb/forms.py +++ b/snpdb/forms.py @@ -2,15 +2,15 @@ from functools import cached_property from crispy_forms.bootstrap import FieldWithButtons -from crispy_forms.layout import Layout, Submit, Field +from crispy_forms.layout import Field, Layout, Submit from dal import forward from django import forms from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError -from django.forms import EmailInput, URLInput, inlineformset_factory, ALL_FIELDS +from django.forms import ALL_FIELDS, EmailInput, URLInput, inlineformset_factory from django.forms.forms import DeclarativeFieldsMetaclass -from django.forms.widgets import TextInput, HiddenInput, NullBooleanSelect +from django.forms.widgets import HiddenInput, NullBooleanSelect, TextInput from guardian import shortcuts from guardian.shortcuts import assign_perm, remove_perm @@ -20,14 +20,29 @@ from library.django_utils.autocomplete_utils import ModelSelect2, ModelSelect2Multiple from library.forms import ROFormMixin from library.guardian_utils import DjangoPermission -from snpdb import models -from snpdb.models import VCF, Sample, Cohort, UserContact, Tag, UserSettings, GenomicIntervalsCollection, \ - ImportStatus, SettingsInitialGroupPermission, LabUserSettingsOverride, UserSettingsOverride, \ - OrganizationUserSettingsOverride, CustomColumnsCollection, Project, VariantsType, SampleFilePath from patients.models import Patient, Specimen +from snpdb import models +from snpdb.models import ( + VCF, + Cohort, + CustomColumnsCollection, + GenomicIntervalsCollection, + ImportStatus, + LabUserSettingsOverride, + OrganizationUserSettingsOverride, + Project, + Sample, + SampleFilePath, + SettingsInitialGroupPermission, + Tag, + UserContact, + UserSettings, + UserSettingsOverride, + VariantsType, +) from snpdb.models.models import Lab, Organization from snpdb.models.models_genome import GenomeBuild -from uicore.utils.form_helpers import form_helper_horizontal, FormHelperHelper +from uicore.utils.form_helpers import FormHelperHelper, form_helper_horizontal from variantgrid.perm_path import get_visible_url_names diff --git a/snpdb/genome/reference_contigs.py b/snpdb/genome/reference_contigs.py index 6283622d8..75ca19ad7 100644 --- a/snpdb/genome/reference_contigs.py +++ b/snpdb/genome/reference_contigs.py @@ -19,7 +19,7 @@ from django.utils.text import slugify from library.pandas_utils import read_csv_skip_header -from snpdb.models import SequenceRole, AssemblyMoleculeType +from snpdb.models import AssemblyMoleculeType, SequenceRole GRCH37 = "GCF_000001405.25_GRCh37.p13_assembly_report.txt" ASSEMBLY_REPORTS = { @@ -107,8 +107,8 @@ def create_build_and_contigs(get_model: callable, build_name, alias=None, igv_ge genome_build = GenomeBuild.objects.create(**kwargs) logging.info("Created build %s", genome_build) - role_lookup = dict(((k.label, k.value) for k in SequenceRole)) - molecule_type_lookup = dict(((k.label, k.value) for k in AssemblyMoleculeType)) + role_lookup = dict((k.label, k.value) for k in SequenceRole) + molecule_type_lookup = dict((k.label, k.value) for k in AssemblyMoleculeType) molecule_type_lookup["na"] = None i = 0 diff --git a/snpdb/genome_build_manager.py b/snpdb/genome_build_manager.py index f1b6a9eaa..a9a86c0a6 100644 --- a/snpdb/genome_build_manager.py +++ b/snpdb/genome_build_manager.py @@ -2,9 +2,14 @@ from typing import Optional from django.core.handlers.wsgi import WSGIRequest -from threadlocals.threadlocals import get_current_user, get_request_variable, set_request_variable, get_current_request - -from snpdb.models import UserSettings, GenomeBuild +from threadlocals.threadlocals import ( + get_current_request, + get_current_user, + get_request_variable, + set_request_variable, +) + +from snpdb.models import GenomeBuild, UserSettings # hardcoded to look for 37 or 38, needs support for other genome builds GENOME_BUILD_RE = re.compile(r"(?:^|/)(GRCh3[78])(?:\?|$|/)") diff --git a/snpdb/graphs/allele_frequency_graph.py b/snpdb/graphs/allele_frequency_graph.py index e6d497733..6445be588 100644 --- a/snpdb/graphs/allele_frequency_graph.py +++ b/snpdb/graphs/allele_frequency_graph.py @@ -6,7 +6,7 @@ from library.utils import sha256sum_str from snpdb.graphs.graphcache import CacheableGraph -from snpdb.models import Sample, Variant, CohortGenotype +from snpdb.models import CohortGenotype, Sample, Variant class AlleleFrequencyHistogramGraph(CacheableGraph): diff --git a/snpdb/grid_columns/custom_columns.py b/snpdb/grid_columns/custom_columns.py index 3b204bc8c..8ec4a084d 100644 --- a/snpdb/grid_columns/custom_columns.py +++ b/snpdb/grid_columns/custom_columns.py @@ -1,6 +1,6 @@ from django.contrib.auth.models import User from django.contrib.postgres.aggregates import StringAgg -from django.db.models import Q, OuterRef, Subquery, Max, Value +from django.db.models import Max, OuterRef, Q, Subquery, Value from django.db.models.functions import Coalesce from analysis.models import VariantTag @@ -58,7 +58,7 @@ def get_custom_column_fields_override_and_sample_position(custom_columns_collect "clinvar__highest_pathogenicity", "clinvar__clinical_significance"] for field in ID_FORMATTER_REQUIRED_FIELDS: - if not field in fields: + if field not in fields: fields.append(field) ov = override.get(field, {}) ov["hidden"] = True diff --git a/snpdb/grid_columns/grid_sample_columns.py b/snpdb/grid_columns/grid_sample_columns.py index a6652ac74..3c5132cc0 100644 --- a/snpdb/grid_columns/grid_sample_columns.py +++ b/snpdb/grid_columns/grid_sample_columns.py @@ -4,12 +4,12 @@ This is in snpdb not analysis as it needs to be called by snpdb.tasks.cohort_genotype_tasks and I don't want cross-dependencies between those apps """ -from typing import Iterable +from collections.abc import Iterable from django.db.models import Value from django.db.models.functions import Coalesce -from snpdb.models import CohortGenotype, VCF, Cohort +from snpdb.models import VCF, Cohort, CohortGenotype def get_available_format_columns(cohorts): diff --git a/snpdb/grids.py b/snpdb/grids.py index 916e0d0d3..9192a2509 100644 --- a/snpdb/grids.py +++ b/snpdb/grids.py @@ -1,10 +1,10 @@ import operator from functools import reduce -from typing import Optional, Any +from typing import Any, Optional from django.conf import settings from django.contrib.postgres.aggregates.general import StringAgg -from django.db.models import F, IntegerField, OuterRef, QuerySet, Subquery, Value, Func +from django.db.models import F, Func, IntegerField, OuterRef, QuerySet, Subquery, Value from django.db.models.aggregates import Count, Max from django.db.models.fields import CharField, TextField from django.db.models.query_utils import Q @@ -13,20 +13,38 @@ from django.urls import reverse from guardian.shortcuts import get_objects_for_user -from annotation.models import ManualVariantEntryCollection, PATIENT_ONTOLOGY_TERM_PATH +from annotation.models import PATIENT_ONTOLOGY_TERM_PATH, ManualVariantEntryCollection from library.django_utils import get_url_from_view_path from library.genomics.vcf_enums import INFO_LIFTOVER_SWAPPED_REF_ALT from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from library.unit_percent import get_allele_frequency_formatter -from library.utils import calculate_age, JsonDataType +from library.utils import JsonDataType, calculate_age from ontology.models import OntologyService from snpdb.grid_columns.custom_columns import get_variantgrid_extra_annotate -from snpdb.models import VCF, Cohort, CohortGenotypeStats, Sample, ImportStatus, \ - GenomicIntervalsCollection, CustomColumnsCollection, Variant, Trio, Quad, UserGridConfig, GenomeBuild, ClinGenAllele, \ - VariantZygosityCountCollection, TagColorsCollection, LiftoverRun, AlleleConversionTool, AlleleLiftover, \ - ProcessingStatus, Allele +from snpdb.models import ( + VCF, + Allele, + AlleleConversionTool, + AlleleLiftover, + ClinGenAllele, + Cohort, + CohortGenotypeStats, + CustomColumnsCollection, + GenomeBuild, + GenomicIntervalsCollection, + ImportStatus, + LiftoverRun, + ProcessingStatus, + Quad, + Sample, + TagColorsCollection, + Trio, + UserGridConfig, + Variant, + VariantZygosityCountCollection, +) from snpdb.sample_filters import get_sample_ontology_q, get_sample_qc_gene_list_gene_symbol_q -from snpdb.tasks.soft_delete_tasks import soft_delete_vcfs, remove_soft_deleted_vcfs_task +from snpdb.tasks.soft_delete_tasks import remove_soft_deleted_vcfs_task, soft_delete_vcfs from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder from uicore.templatetags.js_tags import jsonify_for_js diff --git a/snpdb/lab_picker.py b/snpdb/lab_picker.py index dffa07cdb..6dddef1f6 100644 --- a/snpdb/lab_picker.py +++ b/snpdb/lab_picker.py @@ -1,7 +1,8 @@ import itertools +from collections.abc import Iterable, Iterator from dataclasses import dataclass from functools import cached_property -from typing import Optional, Union, Iterator, Iterable +from typing import Optional, Union from django.contrib.auth.models import User from django.db.models import QuerySet @@ -11,7 +12,7 @@ from library.guardian_utils import admin_bot from library.utils import first -from snpdb.models import Lab, Organization, GenomeBuild, UserSettings +from snpdb.models import GenomeBuild, Lab, Organization, UserSettings @dataclass diff --git a/snpdb/liftover.py b/snpdb/liftover.py index bcdc0b807..421144959 100644 --- a/snpdb/liftover.py +++ b/snpdb/liftover.py @@ -6,8 +6,9 @@ import operator import os from collections import defaultdict +from collections.abc import Iterable from functools import reduce -from typing import Iterable, Optional +from typing import Optional from django.conf import settings from django.contrib.auth.models import User @@ -15,14 +16,19 @@ from genes.hgvs import HGVSMatcher from library.django_utils.django_file_utils import get_import_processing_dir -from library.genomics.vcf_utils import write_vcf_from_variant_coordinates, get_contigs_header_lines +from library.genomics.vcf_utils import get_contigs_header_lines, write_vcf_from_variant_coordinates from library.guardian_utils import admin_bot from snpdb.bcftools_liftover import bcftools_pre_liftover_error_check from snpdb.clingen_allele import populate_clingen_alleles_for_variants -from snpdb.models.models_enums import ImportSource, AlleleConversionTool, AlleleOrigin, ProcessingStatus +from snpdb.models.models_enums import ( + AlleleConversionTool, + AlleleOrigin, + ImportSource, + ProcessingStatus, +) from snpdb.models.models_genome import GenomeBuild -from snpdb.models.models_variant import LiftoverRun, Allele, Variant, VariantAllele, AlleleLiftover -from upload.models import UploadedFile, UploadedLiftover, UploadPipeline, UploadedFileTypes +from snpdb.models.models_variant import Allele, AlleleLiftover, LiftoverRun, Variant, VariantAllele +from upload.models import UploadedFile, UploadedFileTypes, UploadedLiftover, UploadPipeline from upload.upload_processing import process_upload_pipeline # VariantCoordinate can be None, with an error string at the end @@ -88,7 +94,7 @@ def create_liftover_pipelines(user: User, alleles: Iterable[Allele], if vcf_ids: # Need to write VCF and run # BCFTools uses chromosomes not contigs - used_chroms = set((vc.chrom for vc in variant_coordinates)) + used_chroms = set(vc.chrom for vc in variant_coordinates) header_lines = get_contigs_header_lines(vcf_genome_build, use_accession=False, contig_allow_list=used_chroms) write_vcf_from_variant_coordinates(vcf_filename, variant_coordinates=variant_coordinates, @@ -234,8 +240,8 @@ def _liftover_using_dest_variant_coordinate(allele, dest_genome_build: GenomeBui Optionally pass in hgvs_matcher to save re-instantiating it all the time """ from annotation.models import VariantAnnotationVersion - from snpdb.models.models_dbsnp import DbSNP from genes.hgvs import get_hgvs_variant_coordinate + from snpdb.models.models_dbsnp import DbSNP conversion_tool = None g_hgvs = None diff --git a/snpdb/management/commands/clean_orphan_obj_perms_safe.py b/snpdb/management/commands/clean_orphan_obj_perms_safe.py index 30187603b..7ad3928d5 100644 --- a/snpdb/management/commands/clean_orphan_obj_perms_safe.py +++ b/snpdb/management/commands/clean_orphan_obj_perms_safe.py @@ -1,7 +1,7 @@ from itertools import chain from django.core.management.base import BaseCommand -from guardian.utils import get_user_obj_perms_model, get_group_obj_perms_model +from guardian.utils import get_group_obj_perms_model, get_user_obj_perms_model class Command(BaseCommand): diff --git a/snpdb/management/commands/clingen_allele_linked_variants_reference_base_check.py b/snpdb/management/commands/clingen_allele_linked_variants_reference_base_check.py index 5cc9931dd..5b05e2bde 100644 --- a/snpdb/management/commands/clingen_allele_linked_variants_reference_base_check.py +++ b/snpdb/management/commands/clingen_allele_linked_variants_reference_base_check.py @@ -2,7 +2,7 @@ from django.core.management import BaseCommand -from snpdb.models import Variant, Allele, ClinGenAllele, Contig, AlleleLiftover +from snpdb.models import Allele, AlleleLiftover, ClinGenAllele, Contig, Variant class Command(BaseCommand): @@ -34,13 +34,13 @@ def handle(self, *args, **options): try: clingen_vc = allele.clingen_allele.get_variant_coordinate(va.genome_build) if existing_vc != clingen_vc: - logging.info(f"{allele} has variant {repr(existing_vc)} not matching expected for build {va.genome_build}: {repr(clingen_vc)}") + logging.info(f"{allele} has variant {existing_vc!r} not matching expected for build {va.genome_build}: {clingen_vc!r}") if not dry_run: liftover_res = AlleleLiftover.objects.filter(allele=allele, liftover__genome_build=va.genome_build).delete() - logging.info(f"Removing liftover record: %s", liftover_res) + logging.info("Removing liftover record: %s", liftover_res) va_res = va.delete() - logging.info(f"Unlinking variant: %s", va_res) + logging.info("Unlinking variant: %s", va_res) except (ClinGenAllele.ClinGenBuildNotInResponseError, Contig.ContigNotInBuildError): pass diff --git a/snpdb/management/commands/clingen_allele_replace.py b/snpdb/management/commands/clingen_allele_replace.py index 34dccb801..c02fcb57b 100644 --- a/snpdb/management/commands/clingen_allele_replace.py +++ b/snpdb/management/commands/clingen_allele_replace.py @@ -23,7 +23,7 @@ def handle(self, *args, **options): cga_qs = cga_qs.filter(modified__lte=before_date) if indels: - filter_message.append(f"(Indels only)") + filter_message.append("(Indels only)") variants_with_clingen = Variant.objects.filter(variantallele__allele__clingen_allele__in=cga_qs) indel_qs = variants_with_clingen.exclude(Variant.get_snp_q()) values_qs = indel_qs.values_list("variantallele__allele__clingen_allele__id", flat=True) diff --git a/snpdb/management/commands/create_zygosity_counts_for_existing_vcfs.py b/snpdb/management/commands/create_zygosity_counts_for_existing_vcfs.py index 82a28203d..966e84de8 100644 --- a/snpdb/management/commands/create_zygosity_counts_for_existing_vcfs.py +++ b/snpdb/management/commands/create_zygosity_counts_for_existing_vcfs.py @@ -2,9 +2,17 @@ from django.core.management.base import BaseCommand -from snpdb.models import VCF, VariantZygosityCountForVCF, VariantZygosityCountForSample, VariantZygosityCountCollection +from snpdb.models import ( + VCF, + VariantZygosityCountCollection, + VariantZygosityCountForSample, + VariantZygosityCountForVCF, +) from snpdb.models.models_enums import ImportStatus -from snpdb.variant_zygosity_count import create_variant_zygosity_counts, update_variant_zygosity_count_for_vcf +from snpdb.variant_zygosity_count import ( + create_variant_zygosity_counts, + update_variant_zygosity_count_for_vcf, +) class Command(BaseCommand): diff --git a/snpdb/management/commands/delete_unused_variants.py b/snpdb/management/commands/delete_unused_variants.py index 0e7cec790..4e57afca9 100644 --- a/snpdb/management/commands/delete_unused_variants.py +++ b/snpdb/management/commands/delete_unused_variants.py @@ -3,14 +3,26 @@ from django.db.models import Max from analysis.models import AllVariantsNode, Candidate, IntersectionNode, NodeVariant, VariantTag -from annotation.models import VariantAnnotation, VariantTranscriptAnnotation, VariantGeneOverlap, \ - ClinVar, CreatedManualVariant, AnnotationRangeLock +from annotation.models import ( + AnnotationRangeLock, + ClinVar, + CreatedManualVariant, + VariantAnnotation, + VariantGeneOverlap, + VariantTranscriptAnnotation, +) from classification.models import Classification, ImportedAlleleInfo, ResolvedVariantInfo -from snpdb.models import Variant, VariantZygosityCount, VariantCollectionRecord, \ - CohortGenotype, CommonVariantClassified, VariantAllele, VariantWiki +from snpdb.models import ( + CohortGenotype, + CommonVariantClassified, + Variant, + VariantAllele, + VariantCollectionRecord, + VariantWiki, + VariantZygosityCount, +) from upload.models import ModifiedImportedVariant, UploadedVCF - # Every model with a FK to Variant whose presence means the variant is still referenced and must be # kept, as (model, fk_column). A variant is "unused" only if no row in any of these (or in # PRELOAD_VARIANT_RELATIONS below) points at it. Keep this in sync with FKs to Variant - a missing diff --git a/snpdb/management/commands/fix_locus_ref_n.py b/snpdb/management/commands/fix_locus_ref_n.py index d0e28de27..d84a2f9e6 100644 --- a/snpdb/management/commands/fix_locus_ref_n.py +++ b/snpdb/management/commands/fix_locus_ref_n.py @@ -4,7 +4,7 @@ import logging import os import subprocess -from collections import defaultdict, Counter +from collections import Counter, defaultdict from datetime import datetime import cyvcf2 @@ -15,7 +15,7 @@ from library.genomics.vcf_utils import get_contigs_header_lines, write_vcf_from_variant_coordinates from library.utils import mk_path -from snpdb.models import GenomeBuild, Sequence, Locus, Variant, VariantCoordinate +from snpdb.models import GenomeBuild, Locus, Sequence, Variant, VariantCoordinate from upload.models import ModifiedImportedVariant @@ -64,7 +64,7 @@ def handle(self, *args, **options): if vcf_ids: mk_path(processing_dir) vcf_input_filename = os.path.join(processing_dir, f"old_variants_ref_n_{genome_build.name}.vcf") - used_chroms = set((vc.chrom for vc in variant_coordinates)) + used_chroms = set(vc.chrom for vc in variant_coordinates) header_lines = get_contigs_header_lines(genome_build, use_accession=True, contig_allow_list=used_chroms) logging.info("Writing old variants ref n VCF: %s", vcf_input_filename) diff --git a/snpdb/management/commands/fix_tag_colors_collection_permissions.py b/snpdb/management/commands/fix_tag_colors_collection_permissions.py index e036db376..6695d4963 100644 --- a/snpdb/management/commands/fix_tag_colors_collection_permissions.py +++ b/snpdb/management/commands/fix_tag_colors_collection_permissions.py @@ -1,7 +1,10 @@ from django.core.management.base import BaseCommand # Can be removed once environments migrated through snpdb/0089 -from library.guardian_utils import assign_permission_to_user_and_groups, add_public_group_read_permission +from library.guardian_utils import ( + add_public_group_read_permission, + assign_permission_to_user_and_groups, +) from snpdb.models import TagColorsCollection diff --git a/snpdb/management/commands/one_off_fix_csv_export_quoting.py b/snpdb/management/commands/one_off_fix_csv_export_quoting.py index 6ba7766cb..4cc914a48 100644 --- a/snpdb/management/commands/one_off_fix_csv_export_quoting.py +++ b/snpdb/management/commands/one_off_fix_csv_export_quoting.py @@ -18,7 +18,7 @@ def fix_csv(csv_file, fixed_csv_filename): quoting=csv.QUOTE_NONE) logging.info("Fixing in file: %s", fixed_csv_filename) - with open(fixed_csv_filename, "wt") as out_csv_file: + with open(fixed_csv_filename, 'w') as out_csv_file: writer = csv.writer(out_csv_file, dialect='excel', quoting=csv.QUOTE_MINIMAL) for r in reader: writer.writerow(r) diff --git a/snpdb/management/commands/one_off_fix_symbolic_variants.py b/snpdb/management/commands/one_off_fix_symbolic_variants.py index a51fcdb0e..61d4e104f 100644 --- a/snpdb/management/commands/one_off_fix_symbolic_variants.py +++ b/snpdb/management/commands/one_off_fix_symbolic_variants.py @@ -4,7 +4,7 @@ from annotation.models import AnnotationRangeLock, ClinVar from genes.hgvs import HGVSMatcher -from snpdb.models import Variant, Sequence, GenomeBuild, Locus +from snpdb.models import GenomeBuild, Locus, Sequence, Variant class Command(BaseCommand): diff --git a/snpdb/management/commands/one_off_fix_variant_end.py b/snpdb/management/commands/one_off_fix_variant_end.py index 8d1c03024..d3a56de2f 100644 --- a/snpdb/management/commands/one_off_fix_variant_end.py +++ b/snpdb/management/commands/one_off_fix_variant_end.py @@ -3,8 +3,8 @@ import numpy as np from django.conf import settings from django.core.management.base import BaseCommand -from django.db.models import OuterRef, Subquery, F -from django.db.models.functions import Length, Abs +from django.db.models import F, OuterRef, Subquery +from django.db.models.functions import Abs, Length from annotation.models import AnnotationRangeLock from library.utils import mk_path diff --git a/snpdb/management/commands/one_off_legacy_populate_allele_liftover.py b/snpdb/management/commands/one_off_legacy_populate_allele_liftover.py index 94af236a0..bc069bf30 100644 --- a/snpdb/management/commands/one_off_legacy_populate_allele_liftover.py +++ b/snpdb/management/commands/one_off_legacy_populate_allele_liftover.py @@ -2,11 +2,20 @@ from collections import defaultdict from django.core.management.base import BaseCommand -from django.db.models import Min, F, Count +from django.db.models import Count, F, Min from library.guardian_utils import admin_bot -from snpdb.models import GenomeBuild, Allele, ClinGenAllele, Contig, VariantAllele, \ - AlleleLiftover, LiftoverRun, AlleleConversionTool, ProcessingStatus +from snpdb.models import ( + Allele, + AlleleConversionTool, + AlleleLiftover, + ClinGenAllele, + Contig, + GenomeBuild, + LiftoverRun, + ProcessingStatus, + VariantAllele, +) class Command(BaseCommand): @@ -99,7 +108,7 @@ def handle(self, *args, **options): logging.info("Assigning alleles to Liftover runs - %d/%d", i, num_liftover_runs) try: status = lr.uploadedliftover.uploaded_file.uploadpipeline.status - except Exception as e: + except Exception: if lr not in known_missing_upload_pipelines: logging.error("Couldn't get upload pipeline status for run: %s", lr) continue diff --git a/snpdb/management/commands/possible_variantgrid_columns.py b/snpdb/management/commands/possible_variantgrid_columns.py index 714f22cf4..053b6bad0 100644 --- a/snpdb/management/commands/possible_variantgrid_columns.py +++ b/snpdb/management/commands/possible_variantgrid_columns.py @@ -1,9 +1,17 @@ from django.core.management.base import BaseCommand -from annotation.models import ClinVar, VariantAnnotation, GeneAnnotation, DBNSFPGeneAnnotation -from genes.models import Gene, TranscriptVersion, GeneVersion, GeneSymbolWiki, HGNC, UniProt +from annotation.models import ClinVar, DBNSFPGeneAnnotation, GeneAnnotation, VariantAnnotation +from genes.models import HGNC, Gene, GeneSymbolWiki, GeneVersion, TranscriptVersion, UniProt from library.django_utils import get_model_fields -from snpdb.models import Variant, Locus, VariantZygosityCount, Contig, ClinGenAllele, VariantWiki, VariantGridColumn +from snpdb.models import ( + ClinGenAllele, + Contig, + Locus, + Variant, + VariantGridColumn, + VariantWiki, + VariantZygosityCount, +) class Command(BaseCommand): diff --git a/snpdb/management/commands/set_custom_columns_permissions.py b/snpdb/management/commands/set_custom_columns_permissions.py index 3bb695e20..102ebc2ae 100644 --- a/snpdb/management/commands/set_custom_columns_permissions.py +++ b/snpdb/management/commands/set_custom_columns_permissions.py @@ -1,6 +1,9 @@ from django.core.management.base import BaseCommand -from library.guardian_utils import assign_permission_to_user_and_groups, add_public_group_read_permission +from library.guardian_utils import ( + add_public_group_read_permission, + assign_permission_to_user_and_groups, +) from snpdb.models import CustomColumnsCollection diff --git a/snpdb/management/commands/somalier_existing_vcfs.py b/snpdb/management/commands/somalier_existing_vcfs.py index fbcc9e783..bbe0810fb 100644 --- a/snpdb/management/commands/somalier_existing_vcfs.py +++ b/snpdb/management/commands/somalier_existing_vcfs.py @@ -1,8 +1,8 @@ from django.conf import settings from django.core.management.base import BaseCommand -from snpdb.models import VCF, log_traceback, SomalierVCFExtract, GenomeBuild, SomalierRelatePairs -from snpdb.tasks.somalier_tasks import somalier_vcf_id, somalier_all_samples +from snpdb.models import VCF, GenomeBuild, SomalierRelatePairs, SomalierVCFExtract, log_traceback +from snpdb.tasks.somalier_tasks import somalier_all_samples, somalier_vcf_id class Command(BaseCommand): diff --git a/snpdb/models.py b/snpdb/models.py index 6c5322dd4..0a8c8a6b1 100644 --- a/snpdb/models.py +++ b/snpdb/models.py @@ -1,29 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from snpdb.models.models import Tag, CachedGeneratedFile, Company, Manufacturer, Wiki, ImportedWikiCollection, \ - ImportedWiki, Organization, ClinVarKey, ClinVarKeyExcludePattern, Country, State, Lab, LabHead, UserAward, \ - LabProject, SiteMessage -from snpdb.models.models_clingen_allele import ClinGenAllele -from snpdb.models.models_cohort import Cohort, CohortSample, CohortGenotypeTaskVersion, \ - CohortGenotypeCommonFilterVersion, CommonVariantClassified, CohortGenotypeCollection, CohortGenotype, Trio, \ - CohortVersion, SubCohortVariantCollection -from snpdb.models.models_columns import VariantGridColumn, ColumnVCFInfo, CustomColumnsCollection, CustomColumn -from snpdb.models.models_dbsnp import DbSNP -from snpdb.models.models_genome import GenomeBuild, GenomeBuildPatchVersion, Contig, GenomeBuildContig, GenomeFasta, \ - GenomeFastaContig -from snpdb.models.models_genomic_interval import GenomicIntervalsCategory, GenomicIntervalsCollection, GenomicInterval -from snpdb.models.models_somalier import SomalierVCFExtract, SomalierSampleExtract, SomalierAncestryRun, \ - SomalierAncestry, SomalierCohortRelate, SomalierTrioRelate, SomalierAllSamplesRelate, SomalierRelatePairs -from snpdb.models.models_user_settings import UserDataPrefix, TagColorsCollection, TagColor, UserPageAck, \ - UserGridConfig, SettingsOverride, GlobalSettings, OrganizationUserSettingsOverride, LabUserSettingsOverride, \ - UserSettingsOverride, SettingsInitialGroupPermission, NodeCountSettingsCollection, NodeCountSettings, UserContact -from snpdb.models.models_variant import Allele, AlleleMergeLog, Sequence, Locus, Variant, VariantWiki, VariantAllele, \ - VariantCollection, VariantCollectionRecord, AlleleSource, VariantAlleleSource, VariantAlleleCollectionSource, \ - VariantAlleleCollectionRecord, LiftoverRun, AlleleLiftover -from snpdb.models.models_cohort_stats import CohortGenotypeStats -from snpdb.models.models_vcf import Project, VCF, VCFFilter, VCFTag, Sample, SampleFilePath, SampleTag, VCFAlleleSource, \ - SampleStatsCodeVersion, SampleStats, SampleStatsPassingFilter, SampleLocusCount, SampleLabProject, \ - VCFSourceSettings, VCFBedIntersection -from snpdb.models.models_zygosity_counts import VariantZygosityCountCollection, VariantZygosityCount, \ - VariantZygosityCountForVCF, VariantZygosityCountForSample # pylint: enable=unused-import diff --git a/snpdb/models/models.py b/snpdb/models/models.py index 9675a3562..8161f10bf 100644 --- a/snpdb/models/models.py +++ b/snpdb/models/models.py @@ -15,17 +15,17 @@ from functools import cached_property, total_ordering from html import escape from re import RegexFlag -from typing import TypedDict, Optional +from typing import Optional, TypedDict from celery import signature from celery.result import AsyncResult from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.core.cache import cache from django.core.exceptions import FieldDoesNotExist, PermissionDenied, ValidationError from django.db import models from django.db.models import QuerySet, TextChoices -from django.db.models.deletion import SET_NULL, CASCADE, PROTECT +from django.db.models.deletion import CASCADE, PROTECT, SET_NULL from django.db.models.signals import pre_delete from django.dispatch import receiver from django.urls import reverse @@ -41,7 +41,7 @@ from library.django_utils.django_object_managers import ObjectManagerCachingRequest from library.enums.log_level import LogLevel from library.preview_request import PreviewModelMixin -from library.utils import import_class, JsonObjType +from library.utils import JsonObjType, import_class from snpdb.models.models_enums import UserAwardLevel @@ -714,7 +714,7 @@ class UserAwards: def __init__(self, user: User): award_qs = UserAward.objects.filter(user=user).all() - award_list: list[UserAward] = list(sorted(award_qs, key=lambda x: (not x.active, 100 - UserAwardLevel(x.award_level).int_value, x.award_text))) + award_list: list[UserAward] = sorted(award_qs, key=lambda x: (not x.active, 100 - UserAwardLevel(x.award_level).int_value, x.award_text)) self.all_awards = award_list self.awards = [award for award in award_list if award.active] diff --git a/snpdb/models/models_clingen_allele.py b/snpdb/models/models_clingen_allele.py index 7e12d2319..2c9b70434 100644 --- a/snpdb/models/models_clingen_allele.py +++ b/snpdb/models/models_clingen_allele.py @@ -11,7 +11,7 @@ from django_extensions.db.models import TimeStampedModel from snpdb.models.models_enums import SequenceRole -from snpdb.models.models_genome import GenomeBuild, Contig +from snpdb.models.models_genome import Contig, GenomeBuild class ClinGenAllele(TimeStampedModel): diff --git a/snpdb/models/models_cohort.py b/snpdb/models/models_cohort.py index 9c047429a..130a353f5 100644 --- a/snpdb/models/models_cohort.py +++ b/snpdb/models/models_cohort.py @@ -1,9 +1,9 @@ import logging from functools import cached_property -from typing import Union, Optional +from typing import Optional, Union import celery -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.contrib.postgres.fields.array import ArrayField from django.core.exceptions import PermissionDenied from django.db import models @@ -11,8 +11,8 @@ from django.db.models.aggregates import Max from django.db.models.deletion import CASCADE, DO_NOTHING, PROTECT from django.db.models.expressions import F, Value -from django.db.models.query_utils import Q, FilteredRelation -from django.db.models.signals import pre_delete, post_delete +from django.db.models.query_utils import FilteredRelation, Q +from django.db.models.signals import post_delete, pre_delete from django.dispatch.dispatcher import receiver from django.shortcuts import get_object_or_404 from django.urls.base import reverse @@ -25,10 +25,10 @@ from library.django_utils.django_postgres import PostgresRealField from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin from library.guardian_utils import DjangoPermission -from library.preview_request import PreviewModelMixin, PreviewKeyValue +from library.preview_request import PreviewKeyValue, PreviewModelMixin from library.utils import invert_dict from patients.models_enums import Zygosity -from snpdb.models.models_enums import ImportStatus, CohortGenotypeCollectionType, ProcessingStatus +from snpdb.models.models_enums import CohortGenotypeCollectionType, ImportStatus, ProcessingStatus from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_variant import Variant, VariantCollection from snpdb.models.models_vcf import VCF, Sample diff --git a/snpdb/models/models_columns.py b/snpdb/models/models_columns.py index d5040aa8e..fbbd10a8e 100644 --- a/snpdb/models/models_columns.py +++ b/snpdb/models/models_columns.py @@ -4,7 +4,6 @@ from django.conf import settings from django.contrib.auth.models import User from django.db import models -from django.db.models import Q from django.db.models.deletion import CASCADE from django.urls import reverse from guardian.shortcuts import get_objects_for_group diff --git a/snpdb/models/models_enums.py b/snpdb/models/models_enums.py index 357c5b01e..7f236db32 100644 --- a/snpdb/models/models_enums.py +++ b/snpdb/models/models_enums.py @@ -1,5 +1,4 @@ from enum import Enum -from typing import Set from django.core.exceptions import ObjectDoesNotExist from django.db import models @@ -182,7 +181,7 @@ class AlleleOriginFilterDefault(models.TextChoices): SOMATIC = "S", "Somatic" @property - def buckets(self) -> Set[AlleleOriginBucket]: + def buckets(self) -> set[AlleleOriginBucket]: if self == AlleleOriginFilterDefault.SHOW_ALL: return {AlleleOriginBucket.values} elif self == AlleleOriginFilterDefault.GERMLINE: diff --git a/snpdb/models/models_genome.py b/snpdb/models/models_genome.py index 2c5407706..c84b691d0 100644 --- a/snpdb/models/models_genome.py +++ b/snpdb/models/models_genome.py @@ -2,8 +2,9 @@ import operator import os import re +from collections.abc import Iterable from functools import cached_property, reduce -from typing import Optional, Iterable +from typing import Optional from django.conf import settings from django.db import models @@ -21,7 +22,7 @@ from library.preview_request import PreviewModelMixin from library.utils import invert_dict from snpdb.genome.fasta_index import load_genome_fasta_index -from snpdb.models.models_enums import SequenceRole, AssemblyMoleculeType +from snpdb.models.models_enums import AssemblyMoleculeType, SequenceRole class GenomeBuild(models.Model, SortMetaOrderingMixin, PreviewModelMixin): diff --git a/snpdb/models/models_somalier.py b/snpdb/models/models_somalier.py index 1bdc50d26..f959638e1 100644 --- a/snpdb/models/models_somalier.py +++ b/snpdb/models/models_somalier.py @@ -2,8 +2,8 @@ import os import shutil import uuid +from collections.abc import Iterable from subprocess import CalledProcessError -from typing import Iterable from django.conf import settings from django.db import models @@ -17,8 +17,8 @@ from library.django_utils import get_url_from_media_root_filename from library.utils import execute_cmd from patients.models_enums import Sex -from pedigree.ped.export_ped import write_unrelated_ped, write_trio_ped -from snpdb.models import Sample, VCF, Cohort, Trio, SuperPopulationCode, ImportStatus +from pedigree.ped.export_ped import write_trio_ped, write_unrelated_ped +from snpdb.models import VCF, Cohort, ImportStatus, Sample, SuperPopulationCode, Trio from snpdb.models.models_enums import ProcessingStatus diff --git a/snpdb/models/models_user_settings.py b/snpdb/models/models_user_settings.py index 3b17e7659..7a5bac5bb 100644 --- a/snpdb/models/models_user_settings.py +++ b/snpdb/models/models_user_settings.py @@ -1,16 +1,17 @@ import dataclasses from collections import defaultdict +from collections.abc import Iterable from dataclasses import dataclass from functools import cached_property -from typing import Optional, List, Tuple, Dict, Set, Iterable, Any +from typing import Any, Optional from avatar.templatetags.avatar_tags import avatar_url from dateutil.tz import gettz from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.core.exceptions import ValidationError from django.db import models -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models.deletion import CASCADE, SET_NULL from django.urls import reverse from django_extensions.db.models import TimeStampedModel from model_utils.managers import InheritanceManager @@ -18,11 +19,11 @@ from library.django_utils import thread_safe_unique_together_get_or_create from library.django_utils.avatar import SpaceThemedAvatarProvider from library.django_utils.guardian_permissions_mixin import GuardianPermissionsAutoInitialSaveMixin -from library.preview_request import PreviewData, PreviewModelMixin, PreviewKeyValue -from library.utils import string_deterministic_hash, rgb_invert +from library.preview_request import PreviewData, PreviewKeyValue, PreviewModelMixin +from library.utils import rgb_invert, string_deterministic_hash from snpdb.models import AlleleOriginFilterDefault, UserAwards -from snpdb.models.models import Tag, Lab, Organization -from snpdb.models.models_columns import CustomColumnsCollection, CustomColumn +from snpdb.models.models import Lab, Organization, Tag +from snpdb.models.models_columns import CustomColumn, CustomColumnsCollection from snpdb.models.models_enums import BuiltInFilters from snpdb.models.models_genome import GenomeBuild @@ -60,7 +61,7 @@ def increment_version(self): self.version_id += 1 self.save() - def get_user_colors_by_tag(self) -> Dict[str, Dict]: + def get_user_colors_by_tag(self) -> dict[str, dict]: user_colors_by_tag = {} for tag_id, rgb in self.tagcolor_set.all().values_list('tag', 'rgb'): user_colors_by_tag[tag_id] = { @@ -286,7 +287,7 @@ def preview(self): title = "" if self.user.is_superuser: title = "Admin" - elif labs := list(sorted(Lab.valid_labs_qs(self.user))): + elif labs := sorted(Lab.valid_labs_qs(self.user)): if len(labs) > 1: title = f"{len(labs)} lab affiliations" else: @@ -394,7 +395,7 @@ def default_allele_origin(self) -> AlleleOriginFilterDefault: default_lab: Optional[Lab] oauth_sub: str timezone: str - _settings_overrides: List[SettingsOverride] + _settings_overrides: list[SettingsOverride] @property def tz(self): @@ -414,7 +415,7 @@ def default_lab_safe(self) -> Lab: raise ValueError("User doesn't have access to any Labs") @staticmethod - def get_settings_overrides(user=None, lab=None, organization=None) -> List[SettingsOverride]: + def get_settings_overrides(user=None, lab=None, organization=None) -> list[SettingsOverride]: user_settings_override = None lab_settings_override = None @@ -445,7 +446,7 @@ def get_settings_overrides(user=None, lab=None, organization=None) -> List[Setti @staticmethod def get_for(user: Optional[User] = None, lab: Optional[Lab] = None, organization: Optional[Organization] = None): override_fields = [s.name for s in dataclasses.fields(UserSettings)] - kwargs = {f: None for f in override_fields} # Need to pass all params + kwargs = dict.fromkeys(override_fields) # Need to pass all params settings_overrides = UserSettings.get_settings_overrides(user=user, lab=lab, organization=organization) kwargs["_settings_overrides"] = settings_overrides for so in settings_overrides: @@ -467,13 +468,13 @@ def get_for_user(user: User) -> 'UserSettings': return UserSettingsManager.get_user_settings(user) @cached_property - def initial_perm_read_and_write_groups(self) -> Tuple[Set[Group], Set[Group]]: + def initial_perm_read_and_write_groups(self) -> tuple[set[Group], set[Group]]: groups = self.user.groups.all() settings_overrides = self._settings_overrides return self.get_initial_perm_read_and_write_groups(groups, settings_overrides) @staticmethod - def get_initial_perm_read_and_write_groups(groups, settings_overrides) -> Tuple[Set[Group], Set[Group]]: + def get_initial_perm_read_and_write_groups(groups, settings_overrides) -> tuple[set[Group], set[Group]]: group_read = defaultdict(lambda x: False) group_write = defaultdict(lambda x: False) qs = SettingsInitialGroupPermission.objects.filter(group__in=groups) @@ -488,13 +489,13 @@ def get_initial_perm_read_and_write_groups(groups, settings_overrides) -> Tuple[ write_groups = {g for g, write_perm in group_write.items() if write_perm} return read_groups, write_groups - def get_override_source_and_values_before_user(self) -> Tuple[Dict[str, str], Dict[str, str]]: + def get_override_source_and_values_before_user(self) -> tuple[dict[str, str], dict[str, str]]: override_fields = [s.name for s in dataclasses.fields(UserSettings)] parent_overrides = self._settings_overrides[:-1] # Skip last, which is User override return self.get_override_source_and_values(override_fields, parent_overrides) @staticmethod - def get_override_source_and_values(override_fields: Iterable[str], parent_overrides: Iterable[SettingsOverride]) -> Tuple[Dict[str, str], Dict[str, str]]: + def get_override_source_and_values(override_fields: Iterable[str], parent_overrides: Iterable[SettingsOverride]) -> tuple[dict[str, str], dict[str, str]]: override_source = {} override_values = {} for so in parent_overrides: @@ -541,7 +542,7 @@ def get_genome_build_or_default(user: User, genome_build_name: Optional[str] = N return genome_build @staticmethod - def get_lab_and_error(user: User) -> Tuple[Optional[Lab], Optional[str]]: + def get_lab_and_error(user: User) -> tuple[Optional[Lab], Optional[str]]: lab_error = None lab = None user_settings = UserSettings.get_for_user(user) diff --git a/snpdb/models/models_variant.py b/snpdb/models/models_variant.py index 4e6c5f2d7..fdcf00a47 100644 --- a/snpdb/models/models_variant.py +++ b/snpdb/models/models_variant.py @@ -1,19 +1,20 @@ import logging import re +from collections.abc import Iterable from functools import cached_property -from typing import Optional, Iterable, Union, Any +from typing import Any, Optional, Union import django import pydantic from bioutils.sequences import reverse_complement from django.conf import settings from django.contrib.auth.models import User -from django.db import models, IntegrityError -from django.db.models import Value, QuerySet +from django.db import IntegrityError, models +from django.db.models import QuerySet, Value from django.db.models.deletion import CASCADE, DO_NOTHING from django.db.models.fields import TextField from django.db.models.functions.text import Concat -from django.db.models.query_utils import Q, FilteredRelation +from django.db.models.query_utils import FilteredRelation, Q from django.dispatch import receiver from django.urls.base import reverse from django.utils.text import slugify @@ -21,15 +22,15 @@ from model_utils.managers import InheritanceManager from pydantic import field_validator -from flags.models import FlagCollection, flag_collection_extra_info_signal, FlagInfos +from flags.models import FlagCollection, FlagInfos, flag_collection_extra_info_signal from flags.models.models import FlagsMixin, FlagTypeContext from library.django_utils.data_archive_mixin import DataArchiveMixin from library.django_utils.django_object_managers import ObjectManagerCachingRequest from library.django_utils.django_partition import RelatedModelsPartitionModel from library.genomics import format_chrom -from library.genomics.vcf_enums import VCFSymbolicAllele, INFO_LIFTOVER_SWAPPED_REF_ALT +from library.genomics.vcf_enums import INFO_LIFTOVER_SWAPPED_REF_ALT, VCFSymbolicAllele from library.guardian_utils import admin_bot -from library.preview_request import PreviewModelMixin, PreviewKeyValue +from library.preview_request import PreviewKeyValue, PreviewModelMixin from library.utils import FormerTuple, sha256sum_str from snpdb.models import Wiki from snpdb.models.models_clingen_allele import ClinGenAllele @@ -325,7 +326,7 @@ def from_string(variant_string: str, genome_build): return VariantCoordinate.from_variant_match(full_match, genome_build) elif full_match := VARIANT_SYMBOLIC_PATTERN.fullmatch(variant_string): return VariantCoordinate.from_symbolic_match(full_match, genome_build) - regex_patterns = ", ".join((str(s) for s in (VARIANT_PATTERN, VARIANT_SYMBOLIC_PATTERN))) + regex_patterns = ", ".join(str(s) for s in (VARIANT_PATTERN, VARIANT_SYMBOLIC_PATTERN)) raise ValueError(f"{variant_string=} did not match against {regex_patterns=}") @staticmethod diff --git a/snpdb/models/models_vcf.py b/snpdb/models/models_vcf.py index d9d5a8813..d1f930ae5 100644 --- a/snpdb/models/models_vcf.py +++ b/snpdb/models/models_vcf.py @@ -5,12 +5,12 @@ from typing import Optional, Union from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.contrib.postgres.fields.array import ArrayField -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import models -from django.db.models import Lookup, Field -from django.db.models.deletion import SET_NULL, CASCADE +from django.db.models import Field, Lookup +from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.functions import Substr from django.db.models.query_utils import Q from django.db.models.signals import pre_delete @@ -26,14 +26,20 @@ from library.genomics.vcf_enums import VariantClass from library.guardian_utils import DjangoPermission from library.log_utils import log_traceback, report_event -from library.preview_request import PreviewModelMixin, PreviewKeyValue +from library.preview_request import PreviewKeyValue, PreviewModelMixin from patients.models import FakeData, Patient, Specimen from patients.models_enums import Sex -from snpdb.models.models import Tag, LabProject -from snpdb.models.models_enums import ImportStatus, VariantsType, ProcessingStatus, SampleFileType, VCFInfoTypes +from snpdb.models.models import LabProject, Tag +from snpdb.models.models_enums import ( + ImportStatus, + ProcessingStatus, + SampleFileType, + VariantsType, + VCFInfoTypes, +) from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_genomic_interval import GenomicIntervalsCollection -from snpdb.models.models_variant import Variant, VariantCollection, AlleleSource +from snpdb.models.models_variant import AlleleSource, Variant, VariantCollection @Field.register_lookup @@ -430,7 +436,8 @@ def delete_internal_data(self): # a snpdb-internal load-order cycle (CohortGenotypeStats → SampleStatsCodeVersion # in this module) and a snpdb→annotation cycle. from annotation.models import ( - CohortGenotypeClinVarAnnotationStats, CohortGenotypeGeneAnnotationStats, + CohortGenotypeClinVarAnnotationStats, + CohortGenotypeGeneAnnotationStats, CohortGenotypeVariantAnnotationStats, ) from snpdb.models.models_cohort_stats import CohortGenotypeStats as _CGS @@ -579,7 +586,7 @@ def _sample_formatter_func(sample): for t in sample_label_template.split("||"): try: return t % params - except (ValueError, KeyError) as e: + except (ValueError, KeyError): pass # In theory this should be valid due to form validator, but just in case return sample.name diff --git a/snpdb/sample_file_path.py b/snpdb/sample_file_path.py index bde56ff60..fec93f5a7 100644 --- a/snpdb/sample_file_path.py +++ b/snpdb/sample_file_path.py @@ -1,7 +1,7 @@ from collections import OrderedDict from library.common_dir import get_common_prefix_dirs -from snpdb.models import Sample, UserDataPrefix, SampleFilePath +from snpdb.models import Sample, SampleFilePath, UserDataPrefix def get_bam_paths_and_user_data_paths(user, bam_file_paths): diff --git a/snpdb/search.py b/snpdb/search.py index 0968da379..821cdb606 100644 --- a/snpdb/search.py +++ b/snpdb/search.py @@ -3,23 +3,29 @@ import operator import re from collections import defaultdict +from collections.abc import Callable, Iterable from dataclasses import dataclass, field from enum import Enum from functools import cached_property, reduce -from re import IGNORECASE -from typing import Optional, Type, Pattern, Callable, Any, Match, Union, Iterable +from re import IGNORECASE, Match, Pattern +from typing import Any, Optional, Union from django.conf import settings from django.contrib.auth.models import User -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.dispatch import Signal from more_itertools import take from library.enums.log_level import LogLevel -from library.log_utils import report_exc_info, report_message, log_level_to_int, log_level_to_bootstrap +from library.log_utils import ( + log_level_to_bootstrap, + log_level_to_int, + report_exc_info, + report_message, +) from library.preview_request import PreviewCoordinator, PreviewData from library.utils import clean_string, first, remove_duplicates_from_list -from snpdb.models import UserSettings, GenomeBuild, Variant, Allele +from snpdb.models import Allele, GenomeBuild, UserSettings, Variant search_signal = Signal() HAS_ALPHA_PATTERN = re.compile(r"[a-zA-Z]") @@ -119,7 +125,9 @@ def search(self) -> list['SearchResponse']: def get_visible_variants(self, genome_build: GenomeBuild) -> QuerySet[Variant]: """ Shariant wants to restrict search to only classified variants """ - from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version + from annotation.annotation_version_querysets import ( + get_variant_queryset_for_annotation_version, + ) from annotation.models import AnnotationVersion from classification.models import Classification @@ -150,7 +158,7 @@ class SearchInputInstance: Useful for @search_receivers that have a capture group, not real bonus otherwise """ - expected_type: Type + expected_type: type search_input: SearchInput match: Match @@ -447,7 +455,7 @@ def genome_builds(self) -> Optional[set[GenomeBuild]]: @property def annotation_consortia(self) -> Optional[list['AnnotationConsortia']]: if consortia := self.preview.annotation_consortia: - return list(sorted(consortia)) + return sorted(consortia) def _default_make_search_result(obj) -> Optional['SearchResult']: @@ -644,8 +652,8 @@ def __post_init__(self): for i, result in enumerate(self.results): result.original_order = i result.parent = self - result.messages = list(sorted(result.messages)) - self.results = list(sorted(self.results)) + result.messages = sorted(result.messages) + self.results = sorted(self.results) self.messages_overall = [om.with_response(self) for om in self.messages_overall] @property @@ -695,7 +703,7 @@ def _convert_variant_search_response_to_allele_search_response(variant_response: for allele, variant_results in allele_to_variants.items(): genome_builds = set().union(*[vr.preview.genome_builds for vr in variant_results if vr.preview.genome_builds]) - all_messages = list(sorted(set().union(*(v.messages for v in variant_results)))) + all_messages = sorted(set().union(*(v.messages for v in variant_results))) strongest_match = max(variant.match_strength for variant in variant_results) @@ -756,7 +764,7 @@ class SearchResponsesCombined: def __init__(self, search_input: SearchInput, responses: list[SearchResponse]): self.search_input = search_input - self.responses = list(sorted(responses)) + self.responses = sorted(responses) @cached_property def search_counts(self) -> list[SearchCount]: @@ -772,7 +780,7 @@ def search_counts(self) -> list[SearchCount]: sc.resolved += len(response.results) sc.total += response.total_count - return list(sorted((sc for sc in counts.values() if sc.resolved), key=lambda x: x.category)) + return sorted((sc for sc in counts.values() if sc.resolved), key=lambda x: x.category) @property def has_excluded_records(self): @@ -955,7 +963,7 @@ def search_func(sender: Any, search_input: SearchInput, **kwargs): example=example, matched_pattern=matched_pattern, results=results, - messages_overall=list(sorted(overall_messages)), + messages_overall=sorted(overall_messages), total_count=total_count ) diff --git a/snpdb/serializers.py b/snpdb/serializers.py index 789caa0d0..0b515947c 100644 --- a/snpdb/serializers.py +++ b/snpdb/serializers.py @@ -3,9 +3,9 @@ from django.contrib.auth.models import User from rest_framework import serializers -from snpdb.models import Variant, Locus, Trio, Quad +from snpdb.models import Locus, Quad, Trio, Variant from snpdb.models.models_clingen_allele import ClinGenAllele -from snpdb.models.models_genome import GenomeBuild, Contig +from snpdb.models.models_genome import Contig, GenomeBuild from snpdb.models.models_variant import Allele, VariantAllele from snpdb.models.models_vcf import Project from snpdb.variant_links import variant_link_info diff --git a/snpdb/signals/clinvar_export_search.py b/snpdb/signals/clinvar_export_search.py index 1ea76fb3a..3ef6fe684 100644 --- a/snpdb/signals/clinvar_export_search.py +++ b/snpdb/signals/clinvar_export_search.py @@ -3,7 +3,14 @@ from classification.models import ClinVarExport, ClinVarExportBatch from library.enums.log_level import LogLevel -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, SearchMessageOverall, CE_SEARCH, CB_SEARCH +from snpdb.search import ( + CB_SEARCH, + CE_SEARCH, + SearchExample, + SearchInputInstance, + SearchMessageOverall, + search_receiver, +) @search_receiver( diff --git a/snpdb/signals/cohort_search.py b/snpdb/signals/cohort_search.py index d343b0070..3982ea0e5 100644 --- a/snpdb/signals/cohort_search.py +++ b/snpdb/signals/cohort_search.py @@ -1,5 +1,5 @@ from snpdb.models import Cohort -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/signals/disk_usage_health_check.py b/snpdb/signals/disk_usage_health_check.py index b3fbaf682..8a9a044d7 100644 --- a/snpdb/signals/disk_usage_health_check.py +++ b/snpdb/signals/disk_usage_health_check.py @@ -1,7 +1,10 @@ from django.dispatch import receiver -from library.health_check import HealthCheckRequest, HealthCheckCapacity, \ - health_check_overall_stats_signal +from library.health_check import ( + HealthCheckCapacity, + HealthCheckRequest, + health_check_overall_stats_signal, +) from variantgrid.deployment_validation.disk_usage import get_disk_usage_objects diff --git a/snpdb/signals/genomics_search.py b/snpdb/signals/genomics_search.py index f644cf278..3511eb0d9 100644 --- a/snpdb/signals/genomics_search.py +++ b/snpdb/signals/genomics_search.py @@ -4,8 +4,8 @@ from django.db.models import Q from library.enums.log_level import LogLevel -from snpdb.models import GenomeBuild, Contig -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, SearchMessageOverall +from snpdb.models import Contig, GenomeBuild +from snpdb.search import SearchExample, SearchInputInstance, SearchMessageOverall, search_receiver @search_receiver( diff --git a/snpdb/signals/lab_search.py b/snpdb/signals/lab_search.py index ad039e059..c01e688f9 100644 --- a/snpdb/signals/lab_search.py +++ b/snpdb/signals/lab_search.py @@ -1,6 +1,6 @@ from snpdb.models import Lab -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ALPHA_MIN +from snpdb.search import HAS_3_ALPHA_MIN, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/signals/organization_search.py b/snpdb/signals/organization_search.py index 2e19ba9b7..59a914272 100644 --- a/snpdb/signals/organization_search.py +++ b/snpdb/signals/organization_search.py @@ -1,5 +1,5 @@ from snpdb.models import Organization -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ALPHA_MIN +from snpdb.search import HAS_3_ALPHA_MIN, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/signals/sample_search.py b/snpdb/signals/sample_search.py index bf592bfcb..a48d00720 100644 --- a/snpdb/signals/sample_search.py +++ b/snpdb/signals/sample_search.py @@ -1,5 +1,5 @@ from snpdb.models import Sample -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_3_ANY +from snpdb.search import HAS_3_ANY, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/signals/scv_search.py b/snpdb/signals/scv_search.py index 09f093edc..38a9f3793 100644 --- a/snpdb/signals/scv_search.py +++ b/snpdb/signals/scv_search.py @@ -3,7 +3,13 @@ from classification.models import ClinVarExport from library.enums.log_level import LogLevel -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, HAS_SCV, SearchMessageOverall +from snpdb.search import ( + HAS_SCV, + SearchExample, + SearchInputInstance, + SearchMessageOverall, + search_receiver, +) @search_receiver( diff --git a/snpdb/signals/signal_handlers.py b/snpdb/signals/signal_handlers.py index 1010a07c6..15885e5f9 100644 --- a/snpdb/signals/signal_handlers.py +++ b/snpdb/signals/signal_handlers.py @@ -6,7 +6,7 @@ from analysis.tasks.karyomapping_tasks import create_genome_karyomapping_for_trio from library.guardian_utils import admin_bot from library.log_utils import AdminNotificationBuilder -from snpdb.models import UserDataPrefix, SettingsInitialGroupPermission, Organization, Lab +from snpdb.models import Lab, Organization, SettingsInitialGroupPermission, UserDataPrefix from snpdb.tasks.vcf_bed_file_task import create_backend_vcf_bed_intersections diff --git a/snpdb/signals/user_search.py b/snpdb/signals/user_search.py index 287789e12..4d0753fb5 100644 --- a/snpdb/signals/user_search.py +++ b/snpdb/signals/user_search.py @@ -1,11 +1,10 @@ from django.conf import settings from django.contrib.auth.models import User -from django.db.models import Value, CharField -from django.db.models.functions import Lower, Concat +from django.db.models import CharField, Value +from django.db.models.functions import Concat, Lower from snpdb.models import UserPreview -from snpdb.search import search_receiver, SearchInputInstance, \ - HAS_ALPHA_PATTERN, SearchExample +from snpdb.search import HAS_ALPHA_PATTERN, SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/signals/variant_search.py b/snpdb/signals/variant_search.py index b5b890b4d..a62e9610a 100644 --- a/snpdb/signals/variant_search.py +++ b/snpdb/signals/variant_search.py @@ -2,8 +2,9 @@ import logging import re from collections import defaultdict +from collections.abc import Callable, Iterable from itertools import zip_longest -from typing import Optional, Iterable, Union, Callable +from typing import Optional, Union from django.conf import settings from django.contrib.auth.models import User @@ -12,22 +13,42 @@ from hgvs_shim import HGVSException, HGVSImplementationException, HGVSNomenclatureException from annotation.cosmic import CosmicAPI -from annotation.manual_variant_entry import check_can_create_variants, CreateManualVariantForbidden +from annotation.manual_variant_entry import CreateManualVariantForbidden, check_can_create_variants from classification.models import Classification, CreateNoClassificationForbidden -from genes.hgvs import HGVSMatcher, VariantResolvingError, HgvsOriginallyNormalized +from genes.hgvs import HGVSMatcher, HgvsOriginallyNormalized, VariantResolvingError from genes.hgvs.hgvs_converter import HgvsMatchRefAllele -from genes.models import MissingTranscript, MANE, TranscriptVersion, BadTranscript +from genes.models import MANE, BadTranscript, MissingTranscript, TranscriptVersion from genes.models_enums import AnnotationConsortium from library.enums.log_level import LogLevel from library.genomics import format_chrom from library.log_utils import report_exc_info from library.preview_request import PreviewData from snpdb.clingen_allele import get_clingen_allele -from snpdb.models import Variant, LOCUS_PATTERN, LOCUS_NO_REF_PATTERN, DbSNP, DBSNP_PATTERN, VariantCoordinate, \ - ClinGenAllele, GenomeBuild, Contig, HGVS_UNCLEANED_PATTERN, VARIANT_PATTERN, VARIANT_SYMBOLIC_PATTERN, Allele, \ - Sequence -from snpdb.search import search_receiver, SearchInputInstance, SearchExample, SearchResult, SearchMessageOverall, \ - SearchMessage, INVALID_INPUT +from snpdb.models import ( + DBSNP_PATTERN, + HGVS_UNCLEANED_PATTERN, + LOCUS_NO_REF_PATTERN, + LOCUS_PATTERN, + VARIANT_PATTERN, + VARIANT_SYMBOLIC_PATTERN, + Allele, + ClinGenAllele, + Contig, + DbSNP, + GenomeBuild, + Sequence, + Variant, + VariantCoordinate, +) +from snpdb.search import ( + INVALID_INPUT, + SearchExample, + SearchInputInstance, + SearchMessage, + SearchMessageOverall, + SearchResult, + search_receiver, +) from upload.models import ModifiedImportedVariant COSMIC_PATTERN = re.compile(r"^(COS[VM])[0-9]{3,}$", re.IGNORECASE) @@ -584,7 +605,7 @@ def _search_hgvs(hgvs_string: str, user: User, genome_build: GenomeBuild, visibl except MissingTranscript: pass - except Contig.ContigNotInBuildError as e: + except Contig.ContigNotInBuildError: # HGVS is valid but g.HGVS contig from a different genome build than this one # we don't want to throw an error for 37 contig not in 38 (if both enabled) # but we do want to throw one if the contig is completely unrecognised diff --git a/snpdb/signals/variant_zygosity_preview_extra.py b/snpdb/signals/variant_zygosity_preview_extra.py index 71476c04a..ff7e48a54 100644 --- a/snpdb/signals/variant_zygosity_preview_extra.py +++ b/snpdb/signals/variant_zygosity_preview_extra.py @@ -4,12 +4,14 @@ from django.utils.safestring import SafeString from registration.forms import User -from annotation.annotation_version_querysets import get_variant_queryset_for_latest_annotation_version +from annotation.annotation_version_querysets import ( + get_variant_queryset_for_latest_annotation_version, +) from library.log_utils import get_current_logged_in_user -from library.preview_request import preview_extra_signal, PreviewKeyValue +from library.preview_request import PreviewKeyValue, preview_extra_signal from library.utils import first from snpdb.genome_build_manager import GenomeBuildManager -from snpdb.models import Allele, Variant, VariantZygosityCountCollection, GenomeBuild +from snpdb.models import Allele, GenomeBuild, Variant, VariantZygosityCountCollection from variantopedia.interesting_nearby import interesting_summary diff --git a/snpdb/signals/vcf_health_check.py b/snpdb/signals/vcf_health_check.py index e818480e5..83bb9c69d 100644 --- a/snpdb/signals/vcf_health_check.py +++ b/snpdb/signals/vcf_health_check.py @@ -1,9 +1,14 @@ from django.db.models import Max from django.dispatch import receiver -from library.health_check import health_check_signal, HealthCheckRequest, HealthCheckRecentActivity, \ - health_check_overall_stats_signal, HealthCheckTotalAmount -from snpdb.models import VCF, Variant, Sample +from library.health_check import ( + HealthCheckRecentActivity, + HealthCheckRequest, + HealthCheckTotalAmount, + health_check_overall_stats_signal, + health_check_signal, +) +from snpdb.models import VCF, Sample, Variant from variantgrid.perm_path import get_visible_url_names diff --git a/snpdb/signals/vcf_search.py b/snpdb/signals/vcf_search.py index 1f441dc1b..3074fb69c 100644 --- a/snpdb/signals/vcf_search.py +++ b/snpdb/signals/vcf_search.py @@ -1,5 +1,5 @@ from snpdb.models import VCF -from snpdb.search import search_receiver, SearchInputInstance, SearchExample +from snpdb.search import SearchExample, SearchInputInstance, search_receiver @search_receiver( diff --git a/snpdb/tasks/cohort_genotype_tasks.py b/snpdb/tasks/cohort_genotype_tasks.py index c89801645..5828c6205 100644 --- a/snpdb/tasks/cohort_genotype_tasks.py +++ b/snpdb/tasks/cohort_genotype_tasks.py @@ -7,14 +7,23 @@ from celery.result import AsyncResult from django.db.models.query_utils import Q -from library.django_utils.django_postgres import pg_sql_array, model_to_insert_sql +from library.django_utils.django_postgres import model_to_insert_sql, pg_sql_array from library.log_utils import log_traceback from library.utils import single_quote from library.utils.database_utils import run_sql from patients.models_enums import Zygosity from snpdb.common_variants import get_common_filter -from snpdb.models import Cohort, ImportStatus, CohortGenotypeCommonFilterVersion, Variant, CommonVariantClassified, \ - CohortGenotypeCollection, CohortGenotype, CohortGenotypeTaskVersion, CohortGenotypeCollectionType +from snpdb.models import ( + Cohort, + CohortGenotype, + CohortGenotypeCollection, + CohortGenotypeCollectionType, + CohortGenotypeCommonFilterVersion, + CohortGenotypeTaskVersion, + CommonVariantClassified, + ImportStatus, + Variant, +) from snpdb.tasks.sub_cohort_tasks import enqueue_sub_cohort_any_sample_called_vc diff --git a/snpdb/tasks/liftover_tasks.py b/snpdb/tasks/liftover_tasks.py index 70c36906f..8c3e82e1e 100644 --- a/snpdb/tasks/liftover_tasks.py +++ b/snpdb/tasks/liftover_tasks.py @@ -5,7 +5,7 @@ from library.log_utils import log_traceback from snpdb.liftover import create_liftover_pipelines -from snpdb.models import GenomeBuild, ImportSource, Allele +from snpdb.models import Allele, GenomeBuild, ImportSource @celery.shared_task @@ -20,5 +20,5 @@ def liftover_alleles(username, genome_build_name): logging.info("creating liftover pipelines from %s to %s", other_build, genome_build) create_liftover_pipelines(user, alleles, ImportSource.WEB, other_build, [genome_build]) logging.info("/ finished creating pipelines") - except Exception as e: + except Exception: log_traceback() diff --git a/snpdb/tasks/soft_delete_tasks.py b/snpdb/tasks/soft_delete_tasks.py index b7c918abc..0c9e66664 100644 --- a/snpdb/tasks/soft_delete_tasks.py +++ b/snpdb/tasks/soft_delete_tasks.py @@ -6,8 +6,10 @@ from library.guardian_utils import check_can_write from library.log_utils import log_traceback from snpdb.models import VCF, ImportStatus, Sample -from snpdb.variant_zygosity_count import update_all_variant_zygosity_counts_for_vcf, \ - update_all_variant_zygosity_counts_for_sample +from snpdb.variant_zygosity_count import ( + update_all_variant_zygosity_counts_for_sample, + update_all_variant_zygosity_counts_for_vcf, +) def soft_delete_vcfs(user, *vcf_ids): diff --git a/snpdb/tasks/somalier_tasks.py b/snpdb/tasks/somalier_tasks.py index ccaaf2b91..174fea15a 100644 --- a/snpdb/tasks/somalier_tasks.py +++ b/snpdb/tasks/somalier_tasks.py @@ -10,12 +10,27 @@ from django.db import IntegrityError from library.django_utils.django_file_utils import get_import_processing_dir -from library.log_utils import log_traceback, get_traceback +from library.log_utils import get_traceback, log_traceback from library.utils import execute_cmd from patients.models_enums import Zygosity -from snpdb.models import VCF, Cohort, Trio, SomalierVCFExtract, SomalierConfig, SomalierAncestryRun, \ - SomalierCohortRelate, SomalierRelate, SomalierTrioRelate, SomalierAncestry, SuperPopulationCode, \ - SomalierSampleExtract, ProcessingStatus, SomalierAllSamplesRelate, AbstractSomalierModel, SomalierRelatePairs +from snpdb.models import ( + VCF, + AbstractSomalierModel, + Cohort, + ProcessingStatus, + SomalierAllSamplesRelate, + SomalierAncestry, + SomalierAncestryRun, + SomalierCohortRelate, + SomalierConfig, + SomalierRelate, + SomalierRelatePairs, + SomalierSampleExtract, + SomalierTrioRelate, + SomalierVCFExtract, + SuperPopulationCode, + Trio, +) from snpdb.variants_to_vcf import vcf_export_to_file @@ -179,7 +194,7 @@ def somalier_all_samples(): pass # Sample was deleted - just don't store all_samples.status = ProcessingStatus.SUCCESS - except Exception as e: + except Exception: tb = get_traceback() logging.error(tb) all_samples.error_exception = tb diff --git a/snpdb/tasks/sub_cohort_tasks.py b/snpdb/tasks/sub_cohort_tasks.py index 2dd920cda..2abf56bf1 100644 --- a/snpdb/tasks/sub_cohort_tasks.py +++ b/snpdb/tasks/sub_cohort_tasks.py @@ -8,8 +8,14 @@ from library.utils.database_utils import run_sql from patients.models_enums import Zygosity from snpdb.archive import DataArchivedError -from snpdb.models import Cohort, CohortGenotypeCollection, CohortVersion, SubCohortVariantCollection, \ - VariantCollection, ProcessingStatus +from snpdb.models import ( + Cohort, + CohortGenotypeCollection, + CohortVersion, + ProcessingStatus, + SubCohortVariantCollection, + VariantCollection, +) def enqueue_sub_cohort_any_sample_called_vc(cohort: Cohort, run_async: bool = True) -> Optional[str]: @@ -56,7 +62,7 @@ def build_sub_cohort_any_sample_called_vc_task(cohort_id): # Variants kept by the EXCLUDE filter = those where NOT every sub-cohort sample is missing. This is # the same regex CohortGenotypeCollection.get_zygosity_q(exclude=True) uses at filter time. missing = [Zygosity.UNKNOWN_ZYGOSITY, Zygosity.MISSING] - sample_zygosities_dict = {s: missing for s in cohort.get_samples()} + sample_zygosities_dict = dict.fromkeys(cohort.get_samples(), missing) regex_string = cgc.get_sample_zygosity_regex(sample_zygosities_dict, {}) regex_excl = f"^((?!{regex_string}))" diff --git a/snpdb/tasks/vcf_bed_file_task.py b/snpdb/tasks/vcf_bed_file_task.py index 9eb4f3791..1de05f9e9 100644 --- a/snpdb/tasks/vcf_bed_file_task.py +++ b/snpdb/tasks/vcf_bed_file_task.py @@ -8,8 +8,8 @@ import celery from django.conf import settings -from library.log_utils import log_traceback, get_traceback -from snpdb.models import VariantCollection, Variant, VCFBedIntersection +from library.log_utils import get_traceback, log_traceback +from snpdb.models import Variant, VariantCollection, VCFBedIntersection from snpdb.models.models_enums import ProcessingStatus from snpdb.variants_to_vcf import write_qs_to_vcf_file_sort_alphabetically diff --git a/snpdb/tasks/vcf_zygosity_count_tasks.py b/snpdb/tasks/vcf_zygosity_count_tasks.py index b187381d3..7d31d70bc 100644 --- a/snpdb/tasks/vcf_zygosity_count_tasks.py +++ b/snpdb/tasks/vcf_zygosity_count_tasks.py @@ -1,6 +1,6 @@ import celery -from snpdb.models import VariantZygosityCountCollection, VCF +from snpdb.models import VCF, VariantZygosityCountCollection from snpdb.variant_zygosity_count import update_variant_zygosity_count_for_vcf diff --git a/snpdb/templatetags/jqgrid_tags.py b/snpdb/templatetags/jqgrid_tags.py index d9f4499a7..4a0374969 100644 --- a/snpdb/templatetags/jqgrid_tags.py +++ b/snpdb/templatetags/jqgrid_tags.py @@ -4,7 +4,7 @@ from django.urls.base import reverse from django.urls.exceptions import NoReverseMatch -from snpdb.models import UserSettings, UserGridConfig +from snpdb.models import UserGridConfig, UserSettings register = Library() diff --git a/snpdb/templatetags/lab_location_tags.py b/snpdb/templatetags/lab_location_tags.py index 90f527342..6b4ac303b 100644 --- a/snpdb/templatetags/lab_location_tags.py +++ b/snpdb/templatetags/lab_location_tags.py @@ -7,8 +7,8 @@ from django import template -from django.template.loader import render_to_string from django.conf import settings +from django.template.loader import render_to_string register = template.Library() diff --git a/snpdb/templatetags/model_tags.py b/snpdb/templatetags/model_tags.py index 58348f2e5..735085d53 100644 --- a/snpdb/templatetags/model_tags.py +++ b/snpdb/templatetags/model_tags.py @@ -2,7 +2,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe -from snpdb.models import Trio, Quad +from snpdb.models import Quad, Trio register = Library() diff --git a/snpdb/templatetags/sample_graph_tags.py b/snpdb/templatetags/sample_graph_tags.py index 5e2005db0..e1def8926 100644 --- a/snpdb/templatetags/sample_graph_tags.py +++ b/snpdb/templatetags/sample_graph_tags.py @@ -1,8 +1,13 @@ import numpy as np -from django.db.models.aggregates import Min, Max, Count +from django.db.models.aggregates import Count, Max, Min from django.template.library import Library -from library.utils.date_utils import diff_month, date_to_month_year_string, month_year_string, month_range +from library.utils.date_utils import ( + date_to_month_year_string, + diff_month, + month_range, + month_year_string, +) from snpdb.models.models_vcf import VCF register = Library() @@ -23,7 +28,7 @@ def samples_by_month_graph(context): num_months = diff_month(ld, ed) labels = month_range(ed.month, ed.year, 0, num_months, fmt=month_year_string) - samples_by_month = {m: 0 for m in labels} + samples_by_month = dict.fromkeys(labels, 0) for d, count in vcf_qs.annotate(count=Count("sample")).values_list("date", "count"): l = date_to_month_year_string(d) diff --git a/snpdb/templatetags/user_tags.py b/snpdb/templatetags/user_tags.py index d7711edc2..4e4805cc7 100644 --- a/snpdb/templatetags/user_tags.py +++ b/snpdb/templatetags/user_tags.py @@ -1,7 +1,7 @@ import datetime from dataclasses import dataclass from functools import cached_property -from typing import Union, Any +from typing import Any, Union from django import template from django.contrib.auth.models import User diff --git a/snpdb/templatetags/wiki_tags.py b/snpdb/templatetags/wiki_tags.py index 7f14d0b09..3b90d60fe 100644 --- a/snpdb/templatetags/wiki_tags.py +++ b/snpdb/templatetags/wiki_tags.py @@ -23,5 +23,5 @@ def wiki_editor(context, wiki, class_name, unique_keyword, unique_value): 'unique_keyword': unique_keyword, 'unique_value': unique_value, 'has_write_permission': wiki.can_write(request.user)} - t = loader.get_template(f"snpdb/tags/wiki_tag.html") + t = loader.get_template("snpdb/tags/wiki_tag.html") return t.render(context, request=request) diff --git a/snpdb/tests/test_clingen_allele.py b/snpdb/tests/test_clingen_allele.py index 94dc19fa2..31366fa88 100644 --- a/snpdb/tests/test_clingen_allele.py +++ b/snpdb/tests/test_clingen_allele.py @@ -1,10 +1,18 @@ from django.test import TestCase from annotation.fake_annotation import get_fake_annotation_version -from snpdb.clingen_allele import ClinGenAlleleServerException, get_clingen_allele, \ - ClinGenAlleleAPIException, get_clingen_allele_for_variant, variant_allele_clingen +from snpdb.clingen_allele import ( + ClinGenAlleleAPIException, + ClinGenAlleleServerException, + get_clingen_allele, + get_clingen_allele_for_variant, + variant_allele_clingen, +) from snpdb.models import GenomeBuild -from snpdb.tests.utils.mock_clingen_api import MockClinGenAlleleRegistryAPI, MockServerErrorClinGenAlleleRegistryAPI +from snpdb.tests.utils.mock_clingen_api import ( + MockClinGenAlleleRegistryAPI, + MockServerErrorClinGenAlleleRegistryAPI, +) from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant diff --git a/snpdb/tests/test_cohort.py b/snpdb/tests/test_cohort.py index e772d189f..f2495a904 100644 --- a/snpdb/tests/test_cohort.py +++ b/snpdb/tests/test_cohort.py @@ -2,7 +2,7 @@ from django.test import TestCase from annotation.fake_annotation import get_fake_annotation_version -from snpdb.models import GenomeBuild, Cohort, CohortSample +from snpdb.models import Cohort, CohortSample, GenomeBuild from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_and_launch_task from snpdb.tests.utils.fake_cohort_data import create_fake_trio diff --git a/snpdb/tests/test_cohort_genotype.py b/snpdb/tests/test_cohort_genotype.py index 27e54723e..27a8182e4 100644 --- a/snpdb/tests/test_cohort_genotype.py +++ b/snpdb/tests/test_cohort_genotype.py @@ -2,8 +2,11 @@ from django.test import TestCase from annotation.fake_annotation import get_fake_annotation_version -from snpdb.models import GenomeBuild, CohortGenotypeCollection -from snpdb.tasks.cohort_genotype_tasks import cohort_genotype_task, create_cohort_genotype_collection +from snpdb.models import CohortGenotypeCollection, GenomeBuild +from snpdb.tasks.cohort_genotype_tasks import ( + cohort_genotype_task, + create_cohort_genotype_collection, +) from snpdb.tests.utils.fake_cohort_data import create_fake_trio diff --git a/snpdb/tests/test_data_archive_mixin.py b/snpdb/tests/test_data_archive_mixin.py index de7b4c0ef..9081e6079 100644 --- a/snpdb/tests/test_data_archive_mixin.py +++ b/snpdb/tests/test_data_archive_mixin.py @@ -12,9 +12,17 @@ from annotation.models import VariantAnnotationVersion from genes.models import GeneCoverageCollection -from snpdb.archive import archive_vcf, ArchivePreconditionError, DataArchivedError, mark_vcf_archive_started -from snpdb.models import GenomeBuild, VCF -from snpdb.models.models_zygosity_counts import VariantZygosityCountCollection, VariantZygosityCountForVCF +from snpdb.archive import ( + ArchivePreconditionError, + DataArchivedError, + archive_vcf, + mark_vcf_archive_started, +) +from snpdb.models import VCF, GenomeBuild +from snpdb.models.models_zygosity_counts import ( + VariantZygosityCountCollection, + VariantZygosityCountForVCF, +) from snpdb.tasks.vcf_archive_tasks import archive_vcf_task from snpdb.tests.utils.fake_cohort_data import create_fake_cohort diff --git a/snpdb/tests/test_library_utils.py b/snpdb/tests/test_library_utils.py index 689eff9f4..2b5322a36 100644 --- a/snpdb/tests/test_library_utils.py +++ b/snpdb/tests/test_library_utils.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import UTC, datetime, timezone from django.test import TestCase @@ -28,5 +28,5 @@ def test_markdown(self): def test_utc_from_timestamp(self): ts = 0 - expected = datetime(1970, 1, 1, tzinfo=timezone.utc) + expected = datetime(1970, 1, 1, tzinfo=UTC) self.assertEqual(utc_from_timestamp(ts), expected) diff --git a/snpdb/tests/test_library_vcf_utils.py b/snpdb/tests/test_library_vcf_utils.py index 1d82d3f81..9fee56217 100644 --- a/snpdb/tests/test_library_vcf_utils.py +++ b/snpdb/tests/test_library_vcf_utils.py @@ -4,9 +4,13 @@ from django.conf import settings from django.test import TestCase -from library.genomics.vcf_utils import write_vcf_from_variant_coordinates, vcf_to_variant_coordinates, \ - vcf_to_variant_coordinates_and_records, vcf_get_ref_alt_svlen_and_modification -from snpdb.models import VariantCoordinate, Sequence +from library.genomics.vcf_utils import ( + vcf_get_ref_alt_svlen_and_modification, + vcf_to_variant_coordinates, + vcf_to_variant_coordinates_and_records, + write_vcf_from_variant_coordinates, +) +from snpdb.models import Sequence, VariantCoordinate from upload.models import ModifiedImportedVariant diff --git a/snpdb/tests/test_liftover.py b/snpdb/tests/test_liftover.py index ef70bbe93..7d7268496 100644 --- a/snpdb/tests/test_liftover.py +++ b/snpdb/tests/test_liftover.py @@ -2,9 +2,12 @@ from annotation.fake_annotation import get_fake_annotation_version from snpdb.clingen_allele import get_clingen_allele -from snpdb.liftover import _liftover_using_existing_contig, _liftover_using_dest_variant_coordinate, \ - _liftover_using_source_variant_coordinate -from snpdb.models import GenomeBuild, AlleleConversionTool, VariantCoordinate +from snpdb.liftover import ( + _liftover_using_dest_variant_coordinate, + _liftover_using_existing_contig, + _liftover_using_source_variant_coordinate, +) +from snpdb.models import AlleleConversionTool, GenomeBuild, VariantCoordinate from snpdb.tests.utils.mock_clingen_api import MockClinGenAlleleRegistryAPI from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant diff --git a/snpdb/tests/test_urls.py b/snpdb/tests/test_urls.py index 8c7b3549c..6ca82bdab 100644 --- a/snpdb/tests/test_urls.py +++ b/snpdb/tests/test_urls.py @@ -4,15 +4,18 @@ from annotation.fake_annotation import get_fake_annotation_version from annotation.tests.test_data_fake_genes import create_fake_transcript_version -from library.django_utils.unittest_utils import prevent_request_warnings, URLTestCase +from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings from library.guardian_utils import assign_permission_to_user_and_groups from snpdb.models import TagColorsCollection from snpdb.models.models_cohort import Cohort from snpdb.models.models_columns import CustomColumnsCollection from snpdb.models.models_enums import ImportStatus from snpdb.models.models_genome import GenomeBuild -from snpdb.models.models_genomic_interval import GenomicIntervalsCollection, GenomicIntervalsCategory -from snpdb.tests.utils.fake_cohort_data import create_fake_trio, create_fake_quad +from snpdb.models.models_genomic_interval import ( + GenomicIntervalsCategory, + GenomicIntervalsCollection, +) +from snpdb.tests.utils.fake_cohort_data import create_fake_quad, create_fake_trio class Test(URLTestCase): diff --git a/snpdb/tests/utils/fake_cohort_data.py b/snpdb/tests/utils/fake_cohort_data.py index 3a449d0f2..141fa92b5 100644 --- a/snpdb/tests/utils/fake_cohort_data.py +++ b/snpdb/tests/utils/fake_cohort_data.py @@ -2,9 +2,19 @@ from django.utils import timezone from library.guardian_utils import assign_permission_to_user_and_groups -from pedigree.models import Pedigree, PedFileFamily, PedFile -from snpdb.models import CohortGenotypeCollection, Trio, Quad, CohortSample, ImportStatus, Sample, VCF, GenomeBuild, \ - Cohort, VCFFilter +from pedigree.models import PedFile, PedFileFamily, Pedigree +from snpdb.models import ( + VCF, + Cohort, + CohortGenotypeCollection, + CohortSample, + GenomeBuild, + ImportStatus, + Quad, + Sample, + Trio, + VCFFilter, +) def create_fake_cohort(user: User, genome_build: GenomeBuild) -> Cohort: diff --git a/snpdb/tests/utils/vcf_testing_utils.py b/snpdb/tests/utils/vcf_testing_utils.py index fa3645d7e..e1aed3cf0 100644 --- a/snpdb/tests/utils/vcf_testing_utils.py +++ b/snpdb/tests/utils/vcf_testing_utils.py @@ -1,8 +1,17 @@ import vcf from library.utils import sha256sum_str -from snpdb.models import Locus, Variant, Sequence, GenomeBuild, Allele, VariantAllele, AlleleOrigin, \ - AlleleConversionTool, VariantCoordinate +from snpdb.models import ( + Allele, + AlleleConversionTool, + AlleleOrigin, + GenomeBuild, + Locus, + Sequence, + Variant, + VariantAllele, + VariantCoordinate, +) def slowly_create_test_variant(chrom: str, position: int, ref: str, alt: str, genome_build: GenomeBuild) -> Variant: diff --git a/snpdb/urls.py b/snpdb/urls.py index 5e14782ce..38c45f234 100644 --- a/snpdb/urls.py +++ b/snpdb/urls.py @@ -1,15 +1,27 @@ -from django.contrib.auth.views import PasswordChangeView, PasswordChangeDoneView +from django.contrib.auth.views import PasswordChangeDoneView, PasswordChangeView from django.urls import include from django.urls.conf import path as path_standard from django.views.generic import RedirectView from library.django_utils.jqgrid_view import JQGridView from library.preview_request import preview_view -from snpdb.grids import CohortListColumns, CohortSampleListGrid, SamplesListGrid, GenomicIntervalsListColumns, \ - CustomColumnsCollectionColumns, QuadsListColumns, TriosListColumns, VCFListGrid, TagColorsCollectionColumns, \ - LiftoverRunColumns, LiftoverRunAlleleLiftoverColumns, AlleleLiftoverFailureColumns, \ - ManualVariantEntryCollectionColumns, SampleColumns -from snpdb.views import views, views_json, views_rest, views_autocomplete +from snpdb.grids import ( + AlleleLiftoverFailureColumns, + CohortListColumns, + CohortSampleListGrid, + CustomColumnsCollectionColumns, + GenomicIntervalsListColumns, + LiftoverRunAlleleLiftoverColumns, + LiftoverRunColumns, + ManualVariantEntryCollectionColumns, + QuadsListColumns, + SampleColumns, + SamplesListGrid, + TagColorsCollectionColumns, + TriosListColumns, + VCFListGrid, +) +from snpdb.views import views, views_autocomplete, views_json, views_rest from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path diff --git a/snpdb/user_settings_manager.py b/snpdb/user_settings_manager.py index 3ffc8b398..d0f1aacbe 100644 --- a/snpdb/user_settings_manager.py +++ b/snpdb/user_settings_manager.py @@ -2,9 +2,9 @@ from dateutil.tz import gettz from django.contrib.auth.models import User -from threadlocals.threadlocals import get_current_user, set_request_variable, get_request_variable +from threadlocals.threadlocals import get_current_user, get_request_variable, set_request_variable -from snpdb.models import UserSettings, AvatarDetails +from snpdb.models import AvatarDetails, UserSettings class UserSettingsManager: diff --git a/snpdb/utils.py b/snpdb/utils.py index 76a9c6b54..9db1f350e 100644 --- a/snpdb/utils.py +++ b/snpdb/utils.py @@ -7,7 +7,7 @@ from email_manager.models import EmailLog from library.log_utils import NotificationBuilder, send_notification from library.utils import empty_to_none -from snpdb.models import Lab, UserSettings, Tag, TagColorsCollection +from snpdb.models import Lab, Tag, TagColorsCollection, UserSettings class LabNotificationBuilder(NotificationBuilder): diff --git a/snpdb/variant_links.py b/snpdb/variant_links.py index 6d2fab118..eda6b3299 100644 --- a/snpdb/variant_links.py +++ b/snpdb/variant_links.py @@ -1,7 +1,7 @@ from typing import Any from genes.hgvs import HGVSException -from snpdb.models import Variant, GenomeBuild +from snpdb.models import GenomeBuild, Variant def variant_link_info(variant: Variant, genome_build: GenomeBuild) -> dict[str, Any]: diff --git a/snpdb/variant_pk_lookup.py b/snpdb/variant_pk_lookup.py index 1c1f691f3..90e9bf485 100644 --- a/snpdb/variant_pk_lookup.py +++ b/snpdb/variant_pk_lookup.py @@ -6,18 +6,19 @@ import logging import os from collections import defaultdict -from typing import Iterable, TypeAlias, Tuple, Callable, Collection, Any +from collections.abc import Callable, Collection, Iterable +from typing import Any, TypeAlias -from django.db.models import Q, Value, TextField, QuerySet +from django.db.models import Q, QuerySet, TextField, Value from django.db.models.aggregates import Max from django.db.models.functions import Concat from library.utils import sha256sum_str -from snpdb.models import Variant, Locus, Sequence, GenomeBuild, VariantCoordinate +from snpdb.models import GenomeBuild, Locus, Sequence, Variant, VariantCoordinate from upload.vcf import sql_copy_files -VariantHash: TypeAlias = Tuple[int, int, int, int, int | str] -LocusHash: TypeAlias = Tuple[int, int, int] +VariantHash: TypeAlias = tuple[int, int, int, int, int | str] +LocusHash: TypeAlias = tuple[int, int, int] LociHash: TypeAlias = Any AnyHash: TypeAlias = VariantHash | LocusHash | LociHash diff --git a/snpdb/variant_queries.py b/snpdb/variant_queries.py index 484c0b5a9..7fb95b8c1 100644 --- a/snpdb/variant_queries.py +++ b/snpdb/variant_queries.py @@ -7,7 +7,7 @@ from analysis.models import VariantTag from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version from annotation.models import AnnotationVersion -from genes.models import GeneSymbol, Gene +from genes.models import Gene, GeneSymbol from snpdb.models import VariantZygosityCountCollection from snpdb.models.models_variant import VariantAllele diff --git a/snpdb/variant_sample_information.py b/snpdb/variant_sample_information.py index c64b62c32..af53e8077 100644 --- a/snpdb/variant_sample_information.py +++ b/snpdb/variant_sample_information.py @@ -7,12 +7,19 @@ from django.contrib.postgres.aggregates.general import StringAgg from django.db.models import Q, TextField -from annotation.models.models_phenotype_match import PATIENT_TPM_PATH, PATIENT_ONTOLOGY_TERM_PATH +from annotation.models.models_phenotype_match import PATIENT_ONTOLOGY_TERM_PATH, PATIENT_TPM_PATH from library.unit_percent import format_af from ontology.models import OntologyService from patients.models import Patient from patients.models_enums import Zygosity -from snpdb.models import Variant, Sample, Locus, CohortGenotypeCollection, GenomeBuild, CohortGenotype +from snpdb.models import ( + CohortGenotype, + CohortGenotypeCollection, + GenomeBuild, + Locus, + Sample, + Variant, +) class VariantSampleInformation: diff --git a/snpdb/variant_zygosity_count.py b/snpdb/variant_zygosity_count.py index e622be3de..209a6b13e 100644 --- a/snpdb/variant_zygosity_count.py +++ b/snpdb/variant_zygosity_count.py @@ -11,8 +11,15 @@ from library.enums.log_level import LogLevel from library.log_utils import get_traceback from library.utils.database_utils import run_sql -from snpdb.models import VCF, VariantZygosityCountForVCF, VariantZygosityCountForSample, Sample, Variant, \ - VariantZygosityCount, VariantZygosityCountCollection +from snpdb.models import ( + VCF, + Sample, + Variant, + VariantZygosityCount, + VariantZygosityCountCollection, + VariantZygosityCountForSample, + VariantZygosityCountForVCF, +) def _sample_excluded_from_variant_zygosity_count(sample: Sample) -> bool: diff --git a/snpdb/variants_to_vcf.py b/snpdb/variants_to_vcf.py index efea921fc..a5195635d 100644 --- a/snpdb/variants_to_vcf.py +++ b/snpdb/variants_to_vcf.py @@ -3,7 +3,7 @@ from bgzip import BGZipWriter from library.genomics.vcf_utils import vcf_allele_is_symbolic -from snpdb.models import VCF, Zygosity, Sample +from snpdb.models import VCF, Sample, Zygosity from snpdb.vcf_export_utils import get_vcf_header_from_contigs, get_vcf_header_lines @@ -167,7 +167,7 @@ def sample_name_func(s): ad = '.' if dp is None: dp = '.' - sample = ":".join((str(s) for s in (gt, ad, dp))) + sample = ":".join(str(s) for s in (gt, ad, dp)) samples_list.append(sample) row = [chrom, str(position), str(pk), ref, alt or ref, '.', '.', '.', vcf_format] + samples_list diff --git a/snpdb/views/datatable_mixins.py b/snpdb/views/datatable_mixins.py index 4a86fb582..0f455e494 100644 --- a/snpdb/views/datatable_mixins.py +++ b/snpdb/views/datatable_mixins.py @@ -21,7 +21,7 @@ def default(self, obj): return super().default(obj) -class JSONResponseMixin(object): +class JSONResponseMixin: is_clean = False def render_to_response(self, context): diff --git a/snpdb/views/datatable_view.py b/snpdb/views/datatable_view.py index 5c1745f2a..9a5cac08a 100644 --- a/snpdb/views/datatable_view.py +++ b/snpdb/views/datatable_view.py @@ -1,22 +1,22 @@ -# -*- coding: utf-8 -*- import enum import itertools import logging import operator +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from functools import cached_property, reduce -from typing import Optional, Any, Callable, Union, TypeVar, Generic, Type, List +from typing import Any, Generic, Optional, TypeVar, Union from django.contrib.auth.models import User from django.db import models -from django.db.models import QuerySet, Q, F, OrderBy +from django.db.models import F, OrderBy, Q, QuerySet from django.http import HttpRequest, QueryDict from django.urls import reverse from kombu.utils import json from library.log_utils import report_exc_info -from library.utils import pretty_label, nice_class_name, JsonDataType, JsonObjType, full_class_name +from library.utils import JsonDataType, JsonObjType, full_class_name, nice_class_name, pretty_label from snpdb.views.datatable_mixins import JSONResponseView logger = logging.getLogger(__name__) @@ -85,7 +85,7 @@ def __init__(self, enabled: bool = True, renderer: Optional[Callable[[CellData], JsonDataType]] = None, default_sort: Optional[SortOrder] = None, - order_sequence: Optional[List[SortOrder]] = None, + order_sequence: Optional[list[SortOrder]] = None, client_renderer: Optional[str] = None, client_renderer_td: Optional[str] = None, visible: bool = True, @@ -361,7 +361,7 @@ def pre_render(self, qs: QuerySet[DC]): pass @cached_property - def _model(self) -> Type[DC]: + def _model(self) -> type[DC]: return self.get_initial_queryset().model def view_primary_key(self, row: CellData) -> JsonDataType: @@ -395,7 +395,7 @@ class DatabaseTableView(Generic[DC], JSONResponseView): config: DatatableConfig max_display_length = 100 - column_class: Type[DC] = None + column_class: type[DC] = None def config_for_request(self, request: HttpRequest) -> DatatableConfig[DC]: return self.column_class(request) @@ -507,7 +507,7 @@ def prepare_results(self, qs: QuerySet[DC]): value = self.render_cell(row=row, column=rc) row_json[rc.name] = value if row_css := self.config.row_css(row): - row_json["row_css"] = row_css; + row_json["row_css"] = row_css data.append(row_json) return data diff --git a/snpdb/views/views.py b/snpdb/views/views.py index e5d3329b5..5b147b83b 100644 --- a/snpdb/views/views.py +++ b/snpdb/views/views.py @@ -3,23 +3,28 @@ import logging import os from collections import OrderedDict, defaultdict -from typing import Iterable +from collections.abc import Iterable import numpy as np import pandas as pd from celery.result import AsyncResult from django.conf import settings from django.contrib import messages -from django.contrib.auth.models import User, Group -from django.core.exceptions import PermissionDenied, ImproperlyConfigured, ObjectDoesNotExist +from django.contrib.auth.models import Group, User +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist, PermissionDenied from django.db.utils import IntegrityError -from django.forms.models import inlineformset_factory, ALL_FIELDS -from django.utils.html import escape +from django.forms.models import ALL_FIELDS, inlineformset_factory from django.forms.widgets import TextInput from django.http import HttpRequest -from django.http.response import HttpResponse, HttpResponseRedirect, HttpResponseServerError, JsonResponse -from django.shortcuts import get_object_or_404, render, redirect +from django.http.response import ( + HttpResponse, + HttpResponseRedirect, + HttpResponseServerError, + JsonResponse, +) +from django.shortcuts import get_object_or_404, redirect, render from django.urls.base import reverse +from django.utils.html import escape from django.views.decorators.cache import cache_page from django.views.decorators.http import require_POST from django.views.decorators.vary import vary_on_cookie @@ -33,55 +38,126 @@ from analysis.models import AnalysisTemplate from analysis.tasks.analysis_grid_export_tasks import get_annotated_download_files_cgf from annotation.forms import GeneCountTypeChoiceForm -from annotation.manual_variant_entry import create_manual_variants, can_create_variants -from annotation.models import AnnotationVersion, CohortGenotypeVariantAnnotationStats, \ - CohortGenotypeGeneAnnotationStats, CohortGenotypeClinVarAnnotationStats +from annotation.manual_variant_entry import can_create_variants, create_manual_variants +from annotation.models import ( + AnnotationVersion, + CohortGenotypeClinVarAnnotationStats, + CohortGenotypeGeneAnnotationStats, + CohortGenotypeVariantAnnotationStats, +) from annotation.models.models import ManualVariantEntryCollection, VariantAnnotationVersion -from annotation.models.models_gene_counts import GeneValueCountCollection, \ - GeneCountType, SampleAnnotationVersionVariantSource, CohortGeneCounts +from annotation.models.models_gene_counts import ( + CohortGeneCounts, + GeneCountType, + GeneValueCountCollection, + SampleAnnotationVersionVariantSource, +) from annotation.serializers import ManualVariantEntryCollectionSerializer from classification.classification_stats import get_grouped_classification_counts from classification.enums import AlleleOriginBucket from classification.models.clinvar_export_sync import clinvar_export_sync -from classification.views.classification_accumulation_graph import get_accumulation_graph_data, \ - AccumulationReportMode +from classification.views.classification_accumulation_graph import ( + AccumulationReportMode, + get_accumulation_graph_data, +) from classification.views.classification_datatables import ClassificationColumns from genes.custom_text_gene_list import create_custom_text_gene_list -from genes.forms import CustomGeneListForm, UserGeneListForm, GeneAndTranscriptForm -from genes.models import GeneListCategory, CustomTextGeneList, GeneList +from genes.forms import CustomGeneListForm, GeneAndTranscriptForm, UserGeneListForm +from genes.models import CustomTextGeneList, GeneList, GeneListCategory from library import uptime_check -from library.constants import WEEK_SECS, HOUR_SECS -from library.django_utils import add_save_message, get_model_fields, set_form_read_only, require_superuser, \ - get_field_counts +from library.constants import HOUR_SECS, WEEK_SECS +from library.django_utils import ( + add_save_message, + get_field_counts, + get_model_fields, + require_superuser, + set_form_read_only, +) from library.django_utils.guardian_permissions_mixin import GuardianPermissionsMixin from library.guardian_utils import DjangoPermission from library.keycloak import Keycloak from library.utils import full_class_name, import_class, rgb_invert from ontology.models import OntologyTerm from patients.forms import PatientForm -from patients.models import Patient, Clinician +from patients.models import Clinician, Patient from patients.views import get_patient_upload_csv from snpdb import forms -from snpdb.archive import DataArchivedError, ArchivePreconditionError, check_vcf_archive_precondition, \ - mark_vcf_archive_started -from snpdb.forms import SampleChoiceForm, VCFChoiceForm, \ - UserSettingsOverrideForm, UserForm, UserContactForm, SampleForm, TagForm, SettingsInitialGroupPermissionForm, \ - OrganizationForm, LabForm, LabUserSettingsOverrideForm, OrganizationUserSettingsOverrideForm +from snpdb.archive import ( + ArchivePreconditionError, + DataArchivedError, + check_vcf_archive_precondition, + mark_vcf_archive_started, +) +from snpdb.forms import ( + LabForm, + LabUserSettingsOverrideForm, + OrganizationForm, + OrganizationUserSettingsOverrideForm, + SampleChoiceForm, + SampleForm, + SettingsInitialGroupPermissionForm, + TagForm, + UserContactForm, + UserForm, + UserSettingsOverrideForm, + VCFChoiceForm, +) from snpdb.graphs import graphcache from snpdb.graphs.allele_frequency_graph import AlleleFrequencyHistogramGraph from snpdb.graphs.chromosome_density_graph import SampleChromosomeDensityGraph from snpdb.graphs.chromosome_intervals_graph import ChromosomeIntervalsGraph from snpdb.graphs.homozygosity_percent_graph import HomozygosityPercentGraph from snpdb.import_status import set_vcf_and_samples_import_status -from snpdb.models import CachedGeneratedFile, VariantGridColumn, UserSettings, \ - VCF, CustomColumnsCollection, CustomColumn, Cohort, \ - CohortSample, GenomicIntervalsCollection, Sample, UserDataPrefix, UserGridConfig, \ - get_igv_data, SampleLocusCount, UserContact, Tag, Wiki, Organization, GenomeBuild, \ - Trio, Quad, AbstractNodeCountSettings, CohortGenotypeCollection, UserSettingsOverride, NodeCountSettingsCollection, \ - Lab, LabUserSettingsOverride, OrganizationUserSettingsOverride, LabHead, SomalierRelatePairs, \ - VariantZygosityCountCollection, VariantZygosityCountForVCF, ClinVarKey, AvatarDetails, State, \ - CohortGenotypeStats, TagColorsCollection, Contig, LiftoverRun, Allele, AlleleLiftover, VCFLengthStatsCollection -from snpdb.models.models_enums import ProcessingStatus, ImportStatus, BuiltInFilters, AlleleConversionTool +from snpdb.models import ( + VCF, + AbstractNodeCountSettings, + Allele, + AlleleLiftover, + AvatarDetails, + CachedGeneratedFile, + ClinVarKey, + Cohort, + CohortGenotypeCollection, + CohortGenotypeStats, + CohortSample, + Contig, + CustomColumn, + CustomColumnsCollection, + GenomeBuild, + GenomicIntervalsCollection, + Lab, + LabHead, + LabUserSettingsOverride, + LiftoverRun, + NodeCountSettingsCollection, + Organization, + OrganizationUserSettingsOverride, + Quad, + Sample, + SampleLocusCount, + SomalierRelatePairs, + State, + Tag, + TagColorsCollection, + Trio, + UserContact, + UserDataPrefix, + UserGridConfig, + UserSettings, + UserSettingsOverride, + VariantGridColumn, + VariantZygosityCountCollection, + VariantZygosityCountForVCF, + VCFLengthStatsCollection, + Wiki, + get_igv_data, +) +from snpdb.models.models_enums import ( + AlleleConversionTool, + BuiltInFilters, + ImportStatus, + ProcessingStatus, +) from snpdb.sample_file_path import get_example_replacements from snpdb.tasks.liftover_tasks import liftover_alleles from snpdb.tasks.soft_delete_tasks import soft_delete_vcfs @@ -1026,7 +1102,7 @@ def view_lab(request, lab_id: int): if settings.CLASSIFICATION_STATS_USE_SHARED: visibility = "Shared" else: - visibility = f"Created" + visibility = "Created" context = { "lab": lab, @@ -1822,7 +1898,7 @@ def global_sample_gene_matrix(request): try: genome_build = GenomeBuild.builds_with_annotation().get() except GenomeBuild.MultipleObjectsReturned: - msg = f"settings.PUBLIC_SAMPLE_GENE_MATRIX_GENOME_BUILD must be set when there are multiple genome builds" + msg = "settings.PUBLIC_SAMPLE_GENE_MATRIX_GENOME_BUILD must be set when there are multiple genome builds" raise ImproperlyConfigured(msg) else: genome_build = GenomeBuild.get_name_or_alias(genome_build_name) diff --git a/snpdb/views/views_autocomplete.py b/snpdb/views/views_autocomplete.py index f5bca7da1..2e6355da5 100644 --- a/snpdb/views/views_autocomplete.py +++ b/snpdb/views/views_autocomplete.py @@ -9,8 +9,21 @@ from library.constants import MINUTE_SECS from library.django_utils.autocomplete_utils import AutocompleteView -from snpdb.models import VCF, Sample, Cohort, CustomColumnsCollection, CustomColumn, Tag, Trio, Quad, \ - Lab, GenomicIntervalsCollection, GenomeBuild, ImportStatus, Project +from snpdb.models import ( + VCF, + Cohort, + CustomColumn, + CustomColumnsCollection, + GenomeBuild, + GenomicIntervalsCollection, + ImportStatus, + Lab, + Project, + Quad, + Sample, + Tag, + Trio, +) from snpdb.models.models_genome import Contig diff --git a/snpdb/views/views_json.py b/snpdb/views/views_json.py index f8acb1363..6dfe91911 100755 --- a/snpdb/views/views_json.py +++ b/snpdb/views/views_json.py @@ -8,7 +8,14 @@ from django.views.decorators.http import require_POST from library.django_utils import require_superuser -from snpdb.models import CachedGeneratedFile, Cohort, Sample, VCF, CustomColumnsCollection, TagColorsCollection +from snpdb.models import ( + VCF, + CachedGeneratedFile, + Cohort, + CustomColumnsCollection, + Sample, + TagColorsCollection, +) from snpdb.tasks.clingen_tasks import populate_clingen_alleles_from_vcf from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_and_launch_task from snpdb.tasks.vcf_zygosity_count_tasks import update_variant_zygosity_count_for_vcf_task diff --git a/snpdb/views/views_rest.py b/snpdb/views/views_rest.py index 31ec391eb..ab4172662 100644 --- a/snpdb/views/views_rest.py +++ b/snpdb/views/views_rest.py @@ -2,7 +2,7 @@ from django.views.decorators.cache import cache_page from django.views.decorators.vary import vary_on_cookie from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework import permissions, viewsets from rest_framework.generics import RetrieveAPIView, get_object_or_404 from rest_framework.response import Response @@ -11,9 +11,14 @@ from library.constants import MINUTE_SECS from patients.models_enums import Zygosity from snpdb.clingen_allele import get_variant_allele_for_variant -from snpdb.models import Sample, Variant, Trio, Quad, GenomeBuild +from snpdb.models import GenomeBuild, Quad, Sample, Trio, Variant from snpdb.models.models_vcf import Project -from snpdb.serializers import QuadSerializer, TrioSerializer, VariantAlleleSerializer, ProjectSerializer +from snpdb.serializers import ( + ProjectSerializer, + QuadSerializer, + TrioSerializer, + VariantAlleleSerializer, +) class VariantZygosityForSampleView(APIView): diff --git a/sync/admin.py b/sync/admin.py index ae61d29bb..ade97147c 100644 --- a/sync/admin.py +++ b/sync/admin.py @@ -6,7 +6,7 @@ from django.db.models import QuerySet from snpdb.admin_utils import ModelAdminBasics, admin_action, admin_list_column -from sync.models import SyncRun, ClassificationModificationSyncRecord +from sync.models import ClassificationModificationSyncRecord, SyncRun from sync.models.models import SyncDestination from sync.sync_runner import sync_runner_for_destination @@ -23,7 +23,7 @@ def get_form(self, request, obj=None, **kwargs): def _run_sync(self, request, queryset: QuerySet[SyncDestination], max_rows: Optional[int] = None): for sync_destination in queryset: sync_destination.run(full_sync=False, max_rows=max_rows) - self.message_user(request, message=f"Completed {str(sync_destination)} row limit = {max_rows}") + self.message_user(request, message=f"Completed {sync_destination!s} row limit = {max_rows}") @admin_action("Validate configuration") def validate_configuration(self, request, queryset: QuerySet[SyncDestination]): @@ -61,7 +61,7 @@ def run_sync_full(self, request, queryset): sync_destination: SyncDestination for sync_destination in queryset: sync_destination.run(full_sync=True) - self.message_user(request, message=f"Completed {str(sync_destination)}") + self.message_user(request, message=f"Completed {sync_destination!s}") @admin.register(SyncRun) diff --git a/sync/alissa/alissa_download.py b/sync/alissa/alissa_download.py index 775fc9b63..baf405953 100644 --- a/sync/alissa/alissa_download.py +++ b/sync/alissa/alissa_download.py @@ -1,12 +1,17 @@ from django.core.files.uploadedfile import SimpleUploadedFile -from classification.models import resolve_uploaded_url_to_handle, UploadedClassificationsUnmappedStatus, \ - UploadedClassificationsUnmapped -from classification.tasks.classification_import_map_and_insert_task import ClassificationImportMapInsertTask +from classification.models import ( + UploadedClassificationsUnmapped, + UploadedClassificationsUnmappedStatus, + resolve_uploaded_url_to_handle, +) +from classification.tasks.classification_import_map_and_insert_task import ( + ClassificationImportMapInsertTask, +) from library.constants import MINUTE_SECS from library.guardian_utils import admin_bot from snpdb.models import Lab -from sync.sync_runner import SyncRunner, register_sync_runner, SyncRunInstance +from sync.sync_runner import SyncRunInstance, SyncRunner, register_sync_runner _ALISSA_DOWNLOAD_TIMEOUT = MINUTE_SECS * 10 # set the timeout to 10 minutes diff --git a/sync/alissa/alissa_upload.py b/sync/alissa/alissa_upload.py index 34131c210..442e3f8f6 100644 --- a/sync/alissa/alissa_upload.py +++ b/sync/alissa/alissa_upload.py @@ -7,15 +7,17 @@ from classification.enums import ShareLevel from classification.views.exports import ClassificationExportFormatterMVL from classification.views.exports.classification_export_filter import ClassificationFilter -from classification.views.exports.classification_export_formatter_mvl import FormatDetailsMVL, \ - FormatDetailsMVLFileFormat +from classification.views.exports.classification_export_formatter_mvl import ( + FormatDetailsMVL, + FormatDetailsMVLFileFormat, +) from library.constants import MINUTE_SECS from library.guardian_utils import admin_bot from library.log_utils import AdminNotificationBuilder, report_exc_info from library.utils import ExportRow, export_column -from snpdb.models import Lab, GenomeBuild, Organization +from snpdb.models import GenomeBuild, Lab, Organization from sync.models import SyncRun -from sync.sync_runner import SyncRunner, register_sync_runner, SyncRunInstance +from sync.sync_runner import SyncRunInstance, SyncRunner, register_sync_runner class AlissaImportOption(str, Enum): @@ -217,5 +219,5 @@ def _message(self): ) ) - sorted_data = list(sorted(all_issues, key=lambda x: (x.severity, x.c_hgvs))) + sorted_data = sorted(all_issues, key=lambda x: (x.severity, x.c_hgvs)) return AlissaRowInfoExport.streaming_csv(data=sorted_data, filename=f"sync_run_{sync_run.pk}") diff --git a/sync/apps.py b/sync/apps.py index 5700c6af9..8dab18a10 100644 --- a/sync/apps.py +++ b/sync/apps.py @@ -7,5 +7,5 @@ class SyncConfig(AppConfig): # noinspection PyUnresolvedReferences def ready(self): # pylint: disable=import-outside-toplevel,unused-import - from sync.signals import sync_health_check + pass # pylint: enable=import-outside-toplevel,unused-import diff --git a/sync/models.py b/sync/models.py index 08ad0e578..0a8c8a6b1 100644 --- a/sync/models.py +++ b/sync/models.py @@ -1,5 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from sync.models.models import SyncDestination, SyncRun -from sync.models.models_classification_sync import ClassificationModificationSyncRecord # pylint: enable=unused-import diff --git a/sync/models/models.py b/sync/models/models.py index ce5d11d04..5642ab73f 100644 --- a/sync/models/models.py +++ b/sync/models/models.py @@ -82,4 +82,4 @@ class SyncRun(TimeStampedModel): meta = models.JSONField(null=True, blank=True, default=None) def __str__(self): - return f'{str(self.destination)} {str(self.created)}' + return f'{self.destination!s} {self.created!s}' diff --git a/sync/models/models_classification_sync.py b/sync/models/models_classification_sync.py index 8b5dd6025..87f5a5b03 100644 --- a/sync/models/models_classification_sync.py +++ b/sync/models/models_classification_sync.py @@ -7,7 +7,7 @@ from django_extensions.db.models import TimeStampedModel from classification.models.classification import ClassificationModification -from sync.models.models import SyncRun, SyncDestination +from sync.models.models import SyncDestination, SyncRun class ClassificationModificationSyncRecord(TimeStampedModel): diff --git a/sync/shariant/historical_ekey_converter.py b/sync/shariant/historical_ekey_converter.py index f5b18dfcc..430e24eb0 100644 --- a/sync/shariant/historical_ekey_converter.py +++ b/sync/shariant/historical_ekey_converter.py @@ -1,5 +1,6 @@ import copy -from typing import Mapping, Any +from collections.abc import Mapping +from typing import Any from django.contrib.auth.models import User diff --git a/sync/shariant/variant_grid_download.py b/sync/shariant/variant_grid_download.py index a8375b9ce..f24ef8a47 100644 --- a/sync/shariant/variant_grid_download.py +++ b/sync/shariant/variant_grid_download.py @@ -8,10 +8,10 @@ from library.constants import MINUTE_SECS from library.guardian_utils import admin_bot from library.oauth import ServerAuth -from library.utils import make_json_safe_in_place, batch_iterator -from snpdb.models.models import Lab, Organization, Country +from library.utils import batch_iterator, make_json_safe_in_place +from snpdb.models.models import Country, Lab, Organization from sync.models.models import SyncRun -from sync.sync_runner import SyncRunner, register_sync_runner, SyncRunInstance +from sync.sync_runner import SyncRunInstance, SyncRunner, register_sync_runner @register_sync_runner(config={"type": {"shariant", "variantgrid"}, "direction": "download"}) diff --git a/sync/shariant/variant_grid_upload.py b/sync/shariant/variant_grid_upload.py index acf76b92d..9aa3eab29 100644 --- a/sync/shariant/variant_grid_upload.py +++ b/sync/shariant/variant_grid_upload.py @@ -1,5 +1,6 @@ import socket -from typing import Iterable, TypeVar, Union +from collections.abc import Iterable +from typing import TypeVar, Union from django.db.models import QuerySet @@ -14,7 +15,7 @@ from sync.models.models_classification_sync import ClassificationModificationSyncRecord from sync.shariant.historical_ekey_converter import HistoricalEKeyConverter from sync.shariant.query_json_filter import QueryJsonFilter -from sync.sync_runner import register_sync_runner, SyncRunner, SyncRunInstance +from sync.sync_runner import SyncRunInstance, SyncRunner, register_sync_runner # add variant_type to private fields as the key has been deprecated SHARIANT_PRIVATE_FIELDS = [ diff --git a/sync/signals/sync_health_check.py b/sync/signals/sync_health_check.py index 8dba533c3..84d61debb 100644 --- a/sync/signals/sync_health_check.py +++ b/sync/signals/sync_health_check.py @@ -2,9 +2,12 @@ from django.dispatch import receiver -from library.health_check import HealthCheckAge, HealthCheckRequest, \ - health_check_overall_stats_signal -from sync.models import SyncRun, SyncDestination, SyncStatus +from library.health_check import ( + HealthCheckAge, + HealthCheckRequest, + health_check_overall_stats_signal, +) +from sync.models import SyncDestination, SyncRun, SyncStatus @receiver(signal=health_check_overall_stats_signal) diff --git a/sync/sync_runner.py b/sync/sync_runner.py index c20051399..75a294e09 100644 --- a/sync/sync_runner.py +++ b/sync/sync_runner.py @@ -1,9 +1,10 @@ import json from abc import ABC, abstractmethod +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from functools import cached_property -from typing import Optional, Callable +from typing import Optional from library.oauth import ServerAuth from library.utils import parse_http_header_date diff --git a/sync/tests/test_historical_converter.py b/sync/tests/test_historical_converter.py index 6363d419e..a2b69e30c 100644 --- a/sync/tests/test_historical_converter.py +++ b/sync/tests/test_historical_converter.py @@ -12,7 +12,7 @@ from django.contrib.auth.models import User from django.test import TestCase -from classification.models.evidence_key import EvidenceKey, EvidenceCategory, EvidenceKeyValueType +from classification.models.evidence_key import EvidenceCategory, EvidenceKey, EvidenceKeyValueType from library.genomics.vcf_enums import VariantClass from sync.shariant.historical_ekey_converter import HistoricalEKeyConverter diff --git a/uicore/json/validated_json.py b/uicore/json/validated_json.py index 90c296498..5e0bf4e3d 100644 --- a/uicore/json/validated_json.py +++ b/uicore/json/validated_json.py @@ -1,10 +1,11 @@ import copy import json +from collections.abc import Iterator from dataclasses import dataclass, field from functools import cached_property -from typing import Iterator, Union, Optional, Any +from typing import Any, Optional, Union -from library.utils import invalidate_cached_property, JsonDataType +from library.utils import JsonDataType, invalidate_cached_property """ ValidatedJson (with JSonMessages) is used for serializing to JSon where there's also the need to providing infos or warnings diff --git a/uicore/templatetags/js_tags.py b/uicore/templatetags/js_tags.py index 3af69ad50..5a74955ad 100644 --- a/uicore/templatetags/js_tags.py +++ b/uicore/templatetags/js_tags.py @@ -7,13 +7,13 @@ from datetime import date, timedelta from decimal import Decimal from html import escape -from typing import Union, Any, Optional +from typing import Any, Optional, Union from django import template from django.db.models import TextChoices -from django.utils.safestring import mark_safe, SafeString +from django.utils.safestring import SafeString, mark_safe -from library.utils import format_significant_digits, JsonDataType, format_diff_text +from library.utils import JsonDataType, format_diff_text, format_significant_digits from snpdb.user_settings_manager import UserSettingsManager from uicore.json.validated_json import ValidatedJson @@ -161,7 +161,7 @@ def code_json(data: JsonDataType, css_class: Optional[str] = "", dash_if_empty: data = data.serialize() # note that we still want to print data if it's "false" or "0" but not if it's None or an empty dict or list - if dash_if_empty and data is None or (isinstance(data, collections.abc.Sized) and len(data) == 0): + if (dash_if_empty and data is None) or (isinstance(data, collections.abc.Sized) and len(data) == 0): return {"blank": True} if not css_class: diff --git a/uicore/templatetags/ui_help.py b/uicore/templatetags/ui_help.py index 28e869b84..a048f64df 100644 --- a/uicore/templatetags/ui_help.py +++ b/uicore/templatetags/ui_help.py @@ -10,7 +10,7 @@ from library.log_utils import report_message from library.utils import html_id_safe -from uicore.templatetags.ui_utils import parse_tag, TagUtils +from uicore.templatetags.ui_utils import TagUtils, parse_tag register = Library() diff --git a/uicore/templatetags/ui_tabs_builder.py b/uicore/templatetags/ui_tabs_builder.py index 92d8611ca..9d8e29ce9 100644 --- a/uicore/templatetags/ui_tabs_builder.py +++ b/uicore/templatetags/ui_tabs_builder.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass from typing import Any, Optional -from urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse from django import template from django.http import HttpRequest @@ -10,7 +10,7 @@ from django.utils.text import slugify from library.utils.django_utils import is_ajax -from uicore.templatetags.ui_utils import parse_tag, TagUtils +from uicore.templatetags.ui_utils import TagUtils, parse_tag register = template.Library() diff --git a/uicore/templatetags/ui_utils.py b/uicore/templatetags/ui_utils.py index db6dd2c86..a2d337ba2 100644 --- a/uicore/templatetags/ui_utils.py +++ b/uicore/templatetags/ui_utils.py @@ -1,8 +1,9 @@ import json import re import uuid +from collections.abc import Iterable from html import escape -from typing import Optional, Any, Iterable +from typing import Any, Optional from django import template from django.contrib.auth.models import User @@ -15,7 +16,7 @@ from library.enums.log_level import LogLevel from library.log_utils import log_level_to_bootstrap from library.preview_request import PreviewModelMixin -from library.utils import diff_text, html_id_safe, emoji_to_unicode, format_diff_text, pretty_label +from library.utils import diff_text, emoji_to_unicode, format_diff_text, html_id_safe, pretty_label from snpdb.admin_utils import get_admin_url from uicore.views.ajax_form_view import LazyRender from variantgrid.perm_path import get_visible_url_names @@ -256,7 +257,7 @@ def render(self, context): if output: output = output.strip() give_div_id = complete_id - if not complete_id and not ' tuple[str, AjaxFormM ) elif mode == AjaxFormMode.EMBEDDED_CARD: return ( - f'
', + '
', AjaxFormMode.CARD, '
' ) diff --git a/uicore/widgets/radio_other_widget.py b/uicore/widgets/radio_other_widget.py index 7f1c81ffe..f40164802 100644 --- a/uicore/widgets/radio_other_widget.py +++ b/uicore/widgets/radio_other_widget.py @@ -112,7 +112,7 @@ def value_from_datadict(self, data, files, name): else: missing_other = True - values=list(sorted(values)) + values=sorted(values) if missing_other: values = ValuesMissingOther(values) return values diff --git a/upload/admin.py b/upload/admin.py index 51b7731d7..99e44ec8b 100644 --- a/upload/admin.py +++ b/upload/admin.py @@ -5,8 +5,15 @@ from snpdb.admin_utils import ModelAdminBasics, admin_list_column from snpdb.models import ProcessingStatus from snpdb.user_settings_manager import UserSettingsManager -from .models import UploadStep, UploadPipeline, UploadedFile, UploadedVCF, UploadedClassificationImport, \ - ModifiedImportedVariant + +from .models import ( + ModifiedImportedVariant, + UploadedClassificationImport, + UploadedFile, + UploadedVCF, + UploadPipeline, + UploadStep, +) class UploadStepStatusFilter(admin.SimpleListFilter): diff --git a/upload/apps.py b/upload/apps.py index 52d9781d5..f6feada03 100644 --- a/upload/apps.py +++ b/upload/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class UploadConfig(AppConfig): name = 'upload' diff --git a/upload/bed_file_processing.py b/upload/bed_file_processing.py index fa27a527b..f887917fa 100644 --- a/upload/bed_file_processing.py +++ b/upload/bed_file_processing.py @@ -19,7 +19,7 @@ def process_bed_file(bed_file, processed_file, has_chr): bed_file = os.path.realpath(bed_file) if not os.path.exists(bed_file): msg = f"'{bed_file}' does not exist" - raise IOError(msg) + raise OSError(msg) mk_path(settings.PROCESSED_BED_FILES_DIR) diff --git a/upload/forms.py b/upload/forms.py index 93aef5899..d7d1ab2f9 100644 --- a/upload/forms.py +++ b/upload/forms.py @@ -1,6 +1,6 @@ from django import forms -from upload.models import UploadSettings, UploadedFileTypes +from upload.models import UploadedFileTypes, UploadSettings class UploadSettingsForm(forms.ModelForm): diff --git a/upload/grids.py b/upload/grids.py index 02a68f665..732d9d78b 100644 --- a/upload/grids.py +++ b/upload/grids.py @@ -8,8 +8,8 @@ from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig from snpdb.models import ProcessingStatus from snpdb.models.models_variant import Variant -from snpdb.views.datatable_view import DatatableConfig, RichColumn, CellData -from upload.models import UploadStep, ModifiedImportedVariant, UploadPipeline, VCFPipelineStage +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn +from upload.models import ModifiedImportedVariant, UploadPipeline, UploadStep, VCFPipelineStage class UploadStepColumns(DatatableConfig[UploadStep]): diff --git a/upload/import_task_factories/abstract_vcf_import_task_factory.py b/upload/import_task_factories/abstract_vcf_import_task_factory.py index a0bebbd41..317185134 100644 --- a/upload/import_task_factories/abstract_vcf_import_task_factory.py +++ b/upload/import_task_factories/abstract_vcf_import_task_factory.py @@ -2,7 +2,6 @@ import operator from abc import abstractmethod from functools import reduce -from typing import Type from django.conf import settings @@ -10,8 +9,13 @@ from upload.import_task_factories.import_task_factory import ImportTaskFactory from upload.models import UploadStep, UploadStepTaskType, VCFPipelineStage from upload.tasks.vcf.import_vcf_step_task import pipeline_start_task -from upload.tasks.vcf.import_vcf_tasks import CheckStartAnnotationTask, ScheduleMultiFileOutputTasksTask, \ - PreprocessVCFTask, UploadPipelineFinishedTask, DoNothingVCFTask +from upload.tasks.vcf.import_vcf_tasks import ( + CheckStartAnnotationTask, + DoNothingVCFTask, + PreprocessVCFTask, + ScheduleMultiFileOutputTasksTask, + UploadPipelineFinishedTask, +) class AbstractVCFImportTaskFactory(ImportTaskFactory): @@ -66,7 +70,7 @@ def get_create_data_from_vcf_header_task(self, upload_pipeline, input_filename): **kwargs) return create_data_from_vcf_header_task_clazz.si(unknown_variants_step.pk, 0) - def _get_preprocess_class(self) -> Type: + def _get_preprocess_class(self) -> type: return PreprocessVCFTask def get_preprocess_vcf_step_and_task(self, upload_pipeline, input_filename): diff --git a/upload/import_task_factories/import_task_factories.py b/upload/import_task_factories/import_task_factories.py index ccff2c780..6f43d539c 100644 --- a/upload/import_task_factories/import_task_factories.py +++ b/upload/import_task_factories/import_task_factories.py @@ -1,30 +1,58 @@ -from typing import Type import pandas as pd from django.conf import settings from library.django_utils.django_file_utils import get_import_processing_filename -from library.utils import import_class, full_class_name +from library.utils import full_class_name, import_class from patients.models import PatientColumns -from upload.import_task_factories.abstract_vcf_import_task_factory import AbstractVCFImportTaskFactory +from upload.import_task_factories.abstract_vcf_import_task_factory import ( + AbstractVCFImportTaskFactory, +) from upload.import_task_factories.import_task_factory import ImportTaskFactory -from upload.models import UploadedFileTypes, UploadedBed, \ - UploadedGeneList, UploadedPatientRecords, UploadedPedFile, UploadedVCF, \ - UploadedGeneCoverage, UploadStep, UploadStepTaskType, UploadedWikiCollection, VCFPipelineStage +from upload.models import ( + UploadedBed, + UploadedFileTypes, + UploadedGeneCoverage, + UploadedGeneList, + UploadedPatientRecords, + UploadedPedFile, + UploadedVCF, + UploadedWikiCollection, + UploadStep, + UploadStepTaskType, + VCFPipelineStage, +) from upload.tasks.import_bedfile_task import ImportBedFileTask from upload.tasks.import_gene_coverage_task import ImportGeneCoverageTask from upload.tasks.import_gene_list_task import ImportGeneListTask from upload.tasks.import_patient_records_task import ImportPatientRecords from upload.tasks.import_ped_task import ImportPedTask from upload.tasks.import_variant_tags_task import VariantTagsCreateVCFTask, VariantTagsInsertTask -from upload.tasks.import_wiki_task import ImportGeneWikiCollection, VariantWikiCreateVCFTask, VariantWikiInsertTask -from upload.tasks.vcf.genotype_vcf_tasks import VCFCheckAnnotationTask, ProcessGenotypeVCFDataTask, \ - ImportGenotypeVCFSuccessTask, UpdateVariantZygosityCountsTask, SampleLocusCountsTask, \ - ImportCreateVCFModelForGenotypeVCFTask, SomalierVCFTask -from upload.tasks.vcf.import_vcf_tasks import ProcessVCFSetMaxVariantTask, \ - ImportCreateUploadedVCFTask, ProcessVCFLinkAllelesSetMaxVariantTask, LiftoverCompleteTask, LiftoverCreateVCFTask, \ - PreprocessAndAnnotateVCFTask, ProcessVCFClinGenAlleleTask, ProcessVCFLinkManualVariantEntrySetMaxVariantTask, \ - LiftoverPreprocessVCFTask +from upload.tasks.import_wiki_task import ( + ImportGeneWikiCollection, + VariantWikiCreateVCFTask, + VariantWikiInsertTask, +) +from upload.tasks.vcf.genotype_vcf_tasks import ( + ImportCreateVCFModelForGenotypeVCFTask, + ImportGenotypeVCFSuccessTask, + ProcessGenotypeVCFDataTask, + SampleLocusCountsTask, + SomalierVCFTask, + UpdateVariantZygosityCountsTask, + VCFCheckAnnotationTask, +) +from upload.tasks.vcf.import_vcf_tasks import ( + ImportCreateUploadedVCFTask, + LiftoverCompleteTask, + LiftoverCreateVCFTask, + LiftoverPreprocessVCFTask, + PreprocessAndAnnotateVCFTask, + ProcessVCFClinGenAlleleTask, + ProcessVCFLinkAllelesSetMaxVariantTask, + ProcessVCFLinkManualVariantEntrySetMaxVariantTask, + ProcessVCFSetMaxVariantTask, +) class BedImportTaskFactory(ImportTaskFactory): @@ -129,7 +157,7 @@ def get_data_classes(self): def get_create_data_from_vcf_header_task_class(self): return ImportCreateVCFModelForGenotypeVCFTask - def _get_preprocess_class(self) -> Type: + def _get_preprocess_class(self) -> type: return PreprocessAndAnnotateVCFTask def get_known_variants_parallel_vcf_processing_task_class(self): @@ -231,7 +259,7 @@ def get_pre_vcf_task(self, upload_pipeline): pre_vcf_task = LiftoverCreateVCFTask.si(unknown_variants_step.pk, 0) return pre_vcf_task - def _get_preprocess_class(self) -> Type: + def _get_preprocess_class(self) -> type: return LiftoverPreprocessVCFTask def get_create_data_from_vcf_header_task_class(self): diff --git a/upload/import_task_factories/import_task_factory.py b/upload/import_task_factories/import_task_factory.py index 84a41fd0a..de96ac0a3 100644 --- a/upload/import_task_factories/import_task_factory.py +++ b/upload/import_task_factories/import_task_factory.py @@ -1,7 +1,7 @@ import importlib import inspect from abc import ABC, abstractmethod -from typing import Iterable, Type +from collections.abc import Iterable from django.conf import settings from django.contrib.auth.models import User @@ -19,7 +19,7 @@ def get_uploaded_file_type(self) -> str: pass @abstractmethod - def get_data_classes(self) -> Iterable[Type[models.Model]]: + def get_data_classes(self) -> Iterable[type[models.Model]]: """ e.g. return UploadedVCF, UploadedGeneList """ pass diff --git a/upload/management/commands/fix_vcf_info_and_format_models.py b/upload/management/commands/fix_vcf_info_and_format_models.py index 30b1d9de0..a0dd42ef3 100644 --- a/upload/management/commands/fix_vcf_info_and_format_models.py +++ b/upload/management/commands/fix_vcf_info_and_format_models.py @@ -11,7 +11,7 @@ from library.genomics.vcf_utils import cyvcf2_header_types from library.utils.file_utils import mk_path from snpdb.models import VCF -from upload.vcf.vcf_import import create_vcf_info, create_vcf_format +from upload.vcf.vcf_import import create_vcf_format, create_vcf_info class Command(BaseCommand): diff --git a/upload/management/commands/vcf_clean_alts.py b/upload/management/commands/vcf_clean_alts.py index 67923bee9..5a93178e8 100644 --- a/upload/management/commands/vcf_clean_alts.py +++ b/upload/management/commands/vcf_clean_alts.py @@ -9,7 +9,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from library.genomics.vcf_enums import VCFColumns, INFO_LIFTOVER_SWAPPED_REF_ALT +from library.genomics.vcf_enums import INFO_LIFTOVER_SWAPPED_REF_ALT, VCFColumns from library.genomics.vcf_utils import parse_vcf_info_column diff --git a/upload/management/commands/vcf_clean_and_filter.py b/upload/management/commands/vcf_clean_and_filter.py index c1715d777..1cd621de8 100644 --- a/upload/management/commands/vcf_clean_and_filter.py +++ b/upload/management/commands/vcf_clean_and_filter.py @@ -68,7 +68,7 @@ def handle(self, *args, **options): vcf_header_lines = [] if replace_header: - with open(replace_header, "r") as rh_f: + with open(replace_header) as rh_f: vcf_header_lines = rh_f.readlines() first_non_header_line = True diff --git a/upload/models.py b/upload/models.py index 0c7dbec74..0a8c8a6b1 100644 --- a/upload/models.py +++ b/upload/models.py @@ -1,5 +1,3 @@ # This file exists for PyCharm's Django Structure plugin # pylint: disable=unused-import -from upload.models.models import UploadedFile, UploadPipeline, ToolVersion, UploadStep, UploadStepMultiFileOutput, VCFImporter, UploadedVCF, UploadedVCFPendingAnnotation, BackendVCF, VCFImportInfo, SimpleVCFImportInfo, ModifiedImportedVariants, ModifiedImportedVariant, VCFSkippedContigs, VCFSkippedContig, UploadSettings, UploadSettingsFileType -from upload.models.models_uploaded_files import UploadedGeneList, UploadedBed, UploadedPedFile, UploadedPatientRecords, UploadedGeneCoverage, UploadedManualVariantEntryCollection, UploadedClassificationImport, UploadedLiftover, UploadedWikiCollection, UploadedClinVarVersion, UploadedVariantTags # pylint: enable=unused-import diff --git a/upload/models/models.py b/upload/models/models.py index bba93f6bf..0df2e3c75 100644 --- a/upload/models/models.py +++ b/upload/models/models.py @@ -8,14 +8,14 @@ from typing import Optional, Union from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import models, transaction -from django.db.models import Func, F, Value, CharField +from django.db.models import CharField, F, Func, Value from django.db.models.aggregates import Max from django.db.models.deletion import CASCADE, SET_NULL from django.db.models.query import QuerySet -from django.db.models.signals import pre_delete, post_save +from django.db.models.signals import post_save, pre_delete from django.dispatch.dispatcher import receiver from django.urls import reverse from django.utils import timezone @@ -26,17 +26,29 @@ from library.django_utils.django_file_system_storage import PrivateUploadStorage from library.django_utils.django_file_utils import get_import_processing_dir from library.enums.log_level import LogLevel -from library.log_utils import report_message, report_exc_info +from library.log_utils import report_exc_info, report_message from library.utils import file_sha256sum from library.utils.file_utils import mk_path -from seqauto.models import SingleSampleVCF, JointCalledVCF, get_samples_by_sequencing_sample, VariantCaller +from seqauto.models import ( + JointCalledVCF, + SingleSampleVCF, + VariantCaller, + get_samples_by_sequencing_sample, +) from snpdb.import_status import set_vcf_and_samples_import_status -from snpdb.models import VCF, Variant, SoftwareVersion, GenomeBuild, VariantCoordinate -from snpdb.models.models_enums import ImportSource, ProcessingStatus, ImportStatus +from snpdb.models import VCF, GenomeBuild, SoftwareVersion, Variant, VariantCoordinate +from snpdb.models.models_enums import ImportSource, ImportStatus, ProcessingStatus from snpdb.tasks.soft_delete_tasks import soft_delete_vcfs from snpdb.user_settings_manager import UserSettingsManager -from upload.models.models_enums import UploadedFileTypes, VCFPipelineStage, \ - UploadStepTaskType, TimeFilterMethod, VCFImportInfoSeverity, UploadStepOrigin, ModifiedImportedVariantOperation +from upload.models.models_enums import ( + ModifiedImportedVariantOperation, + TimeFilterMethod, + UploadedFileTypes, + UploadStepOrigin, + UploadStepTaskType, + VCFImportInfoSeverity, + VCFPipelineStage, +) from variantgrid.celery import app diff --git a/upload/models/models_uploaded_files.py b/upload/models/models_uploaded_files.py index 4c5778df1..e60ce47ed 100644 --- a/upload/models/models_uploaded_files.py +++ b/upload/models/models_uploaded_files.py @@ -8,13 +8,13 @@ from django.dispatch import receiver from analysis.models.models_variant_tag import VariantTagsImport -from annotation.models.models import ManualVariantEntryCollection, ClinVarVersion +from annotation.models.models import ClinVarVersion, ManualVariantEntryCollection from classification.models import ClassificationImport -from genes.models import GeneList, GeneCoverageCollection +from genes.models import GeneCoverageCollection, GeneList from library.utils.file_utils import name_from_filename from patients.models import PatientRecords from pedigree.models import PedFile -from snpdb.models import GenomicIntervalsCollection, ImportStatus, Sample, ImportedWikiCollection +from snpdb.models import GenomicIntervalsCollection, ImportedWikiCollection, ImportStatus, Sample from snpdb.models.models_variant import LiftoverRun from upload.bed_file_processing import process_bed_file from upload.models import UploadedFile diff --git a/upload/tasks/import_bedfile_task.py b/upload/tasks/import_bedfile_task.py index dffc93757..7fb2f1b3e 100644 --- a/upload/tasks/import_bedfile_task.py +++ b/upload/tasks/import_bedfile_task.py @@ -5,7 +5,12 @@ from guardian.shortcuts import assign_perm from library.genomics.bed_file import BedFileReader -from snpdb.models import GenomicIntervalsCollection, ImportStatus, GenomicIntervalsCategory, GenomeBuild +from snpdb.models import ( + GenomeBuild, + GenomicIntervalsCategory, + GenomicIntervalsCollection, + ImportStatus, +) from upload.models import UploadedBed from upload.tasks.import_task import ImportTask from variantgrid.celery import app diff --git a/upload/tasks/import_gene_coverage_task.py b/upload/tasks/import_gene_coverage_task.py index d78ce8b4c..e44d56bdf 100644 --- a/upload/tasks/import_gene_coverage_task.py +++ b/upload/tasks/import_gene_coverage_task.py @@ -6,10 +6,10 @@ from genes.canonical_transcripts.canonical_transcript_manager import CanonicalTranscriptManager from genes.gene_matching import GeneSymbolMatcher -from genes.models import GeneCoverageCollection, AnnotationConsortium, TranscriptVersion +from genes.models import AnnotationConsortium, GeneCoverageCollection, TranscriptVersion from library.log_utils import log_traceback from library.utils.file_utils import name_from_filename -from snpdb.models import Sample, GenomeBuild, DataState +from snpdb.models import DataState, GenomeBuild, Sample from upload.models import UploadedGeneCoverage from upload.tasks.import_task import ImportTask from variantgrid.celery import app diff --git a/upload/tasks/import_gene_list_task.py b/upload/tasks/import_gene_list_task.py index b3ae8b1c9..6227dbb95 100644 --- a/upload/tasks/import_gene_list_task.py +++ b/upload/tasks/import_gene_list_task.py @@ -1,4 +1,4 @@ -from genes.gene_matching import tokenize_gene_symbols, GeneSymbolMatcher +from genes.gene_matching import GeneSymbolMatcher, tokenize_gene_symbols from genes.models import GeneList from snpdb.models import ImportStatus from upload.models import UploadedGeneList diff --git a/upload/tasks/import_patient_records_task.py b/upload/tasks/import_patient_records_task.py index 679dd5efe..b5962f133 100644 --- a/upload/tasks/import_patient_records_task.py +++ b/upload/tasks/import_patient_records_task.py @@ -1,5 +1,5 @@ from patients.import_records import import_patient_records -from patients.models import PatientRecords, PatientImport +from patients.models import PatientImport, PatientRecords from upload.models import UploadedPatientRecords from upload.tasks.import_task import ImportTask from variantgrid.celery import app diff --git a/upload/tasks/import_ped_task.py b/upload/tasks/import_ped_task.py index f5561c473..af4572809 100644 --- a/upload/tasks/import_ped_task.py +++ b/upload/tasks/import_ped_task.py @@ -2,7 +2,7 @@ from django.conf import settings -from pedigree.ped.import_ped import import_ped, automatch_pedigree_samples +from pedigree.ped.import_ped import automatch_pedigree_samples, import_ped from upload.models import UploadedPedFile from upload.tasks.import_task import ImportTask from variantgrid.celery import app diff --git a/upload/tasks/import_variant_tags_task.py b/upload/tasks/import_variant_tags_task.py index e210fc25d..fda088393 100644 --- a/upload/tasks/import_variant_tags_task.py +++ b/upload/tasks/import_variant_tags_task.py @@ -5,16 +5,21 @@ from dateutil import parser from django.utils.timezone import make_aware -from analysis.models import VariantTagsImport, ImportedVariantTag, VariantTag, TagLocation +from analysis.models import ImportedVariantTag, TagLocation, VariantTag, VariantTagsImport from library.django_utils import UserMatcher from library.genomics.vcf_utils import write_vcf_from_variant_coordinates from library.guardian_utils import assign_permission_to_user_and_groups from library.pandas_utils import df_nan_to_none from library.utils import invert_dict from snpdb.liftover import create_liftover_pipelines -from snpdb.models import GenomeBuild, ImportSource, Tag, VariantAllele, VariantCoordinate, Allele +from snpdb.models import Allele, GenomeBuild, ImportSource, Tag, VariantAllele, VariantCoordinate from snpdb.variant_pk_lookup import VariantPKLookup -from upload.models import UploadedVariantTags, UploadStep, ModifiedImportedVariant, SimpleVCFImportInfo +from upload.models import ( + ModifiedImportedVariant, + SimpleVCFImportInfo, + UploadedVariantTags, + UploadStep, +) from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask from variantgrid.celery import app diff --git a/upload/tasks/import_wiki_task.py b/upload/tasks/import_wiki_task.py index dd40dd2cf..4edc9e339 100644 --- a/upload/tasks/import_wiki_task.py +++ b/upload/tasks/import_wiki_task.py @@ -10,7 +10,7 @@ from library.django_utils import UserMatcher from library.genomics.vcf_utils import write_vcf_from_variant_coordinates from library.pandas_utils import df_nan_to_none -from snpdb.models import ImportedWikiCollection, GenomeBuild, ImportedWiki, VariantWiki, Variant +from snpdb.models import GenomeBuild, ImportedWiki, ImportedWikiCollection, Variant, VariantWiki from upload.models import UploadedWikiCollection, UploadStep from upload.tasks.import_task import ImportTask from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask diff --git a/upload/tasks/vcf/genotype_vcf_tasks.py b/upload/tasks/vcf/genotype_vcf_tasks.py index 17093e515..c709bc3ab 100644 --- a/upload/tasks/vcf/genotype_vcf_tasks.py +++ b/upload/tasks/vcf/genotype_vcf_tasks.py @@ -12,13 +12,23 @@ from seqauto.signals.signals_list import backend_vcf_import_success_signal from snpdb.import_status import set_vcf_and_samples_import_status from snpdb.models import VCF -from snpdb.models.models_enums import ImportStatus, VariantsType, ProcessingStatus +from snpdb.models.models_enums import ImportStatus, ProcessingStatus, VariantsType from snpdb.tasks.sample_locus_count_task import do_sample_locus_count_for_vcf_id from snpdb.tasks.somalier_tasks import somalier_vcf_id -from snpdb.variant_zygosity_count import update_all_variant_zygosity_counts_for_vcf, \ - create_variant_zygosity_counts -from upload.models import VCFPipelineStage, UploadStep, UploadStepTaskType, UploadedVCFPendingAnnotation, \ - UploadPipeline, SimpleVCFImportInfo, SkipUploadStepException, ModifiedImportedVariants +from snpdb.variant_zygosity_count import ( + create_variant_zygosity_counts, + update_all_variant_zygosity_counts_for_vcf, +) +from upload.models import ( + ModifiedImportedVariants, + SimpleVCFImportInfo, + SkipUploadStepException, + UploadedVCFPendingAnnotation, + UploadPipeline, + UploadStep, + UploadStepTaskType, + VCFPipelineStage, +) from upload.signals.signals import vcf_import_success_signal from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask from upload.upload_processing import process_upload_pipeline @@ -29,8 +39,11 @@ class ImportCreateVCFModelForGenotypeVCFTask(ImportVCFStepTask): """ Create VCF model from header """ def process_items(self, upload_step): - from upload.vcf.vcf_import import create_vcf_from_vcf, create_cohort_genotype_collection_from_vcf, \ - configure_vcf_from_header + from upload.vcf.vcf_import import ( + configure_vcf_from_header, + create_cohort_genotype_collection_from_vcf, + create_vcf_from_vcf, + ) vcf_filename = upload_step.input_filename upload_pipeline = upload_step.upload_pipeline @@ -104,7 +117,7 @@ class ProcessGenotypeVCFDataTask(ImportVCFStepTask): (ie via ImportGenotypeVCFTask) - this can run in parallel """ def process_items(self, upload_step): - from upload.vcf.vcf_import import import_vcf_file, genotype_vcf_processor_factory + from upload.vcf.vcf_import import genotype_vcf_processor_factory, import_vcf_file upload_pipeline = upload_step.upload_pipeline uploaded_vcf = upload_pipeline.uploadedvcf diff --git a/upload/tasks/vcf/import_vcf_step_task.py b/upload/tasks/vcf/import_vcf_step_task.py index 48acf4a94..329182401 100644 --- a/upload/tasks/vcf/import_vcf_step_task.py +++ b/upload/tasks/vcf/import_vcf_step_task.py @@ -5,15 +5,22 @@ import celery from celery.app.task import Task from celery.canvas import chain -from django.db.models.aggregates import Min, Max +from django.db.models.aggregates import Max, Min from django.db.models.expressions import F from django.utils import timezone from library.log_utils import get_traceback -from library.utils import import_class, full_class_name -from upload.models import UploadPipeline, ProcessingStatus, \ - UploadStep, PipelineFailedJobTerminateEarlyException, VCFPipelineStage, SkipUploadStepException, UploadStepTaskType, \ - SimpleVCFImportInfo +from library.utils import full_class_name, import_class +from upload.models import ( + PipelineFailedJobTerminateEarlyException, + ProcessingStatus, + SimpleVCFImportInfo, + SkipUploadStepException, + UploadPipeline, + UploadStep, + UploadStepTaskType, + VCFPipelineStage, +) class ImportVCFStepTask(Task): diff --git a/upload/tasks/vcf/import_vcf_tasks.py b/upload/tasks/vcf/import_vcf_tasks.py index b8fea6939..7129593e6 100644 --- a/upload/tasks/vcf/import_vcf_tasks.py +++ b/upload/tasks/vcf/import_vcf_tasks.py @@ -12,13 +12,18 @@ from library.utils import import_class from snpdb.bcftools_liftover import bcftools_liftover from snpdb.models import AlleleLiftover -from snpdb.models.models_enums import ProcessingStatus, AlleleConversionTool -from upload.models import UploadStep, UploadedVCF, ModifiedImportedVariants +from snpdb.models.models_enums import AlleleConversionTool, ProcessingStatus +from upload.models import ModifiedImportedVariants, UploadedVCF, UploadStep from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask from upload.upload_processing import process_vcf_file -from upload.vcf.bulk_allele_linking_vcf_processor import BulkAlleleLinkingVCFProcessor, FailedLiftoverVCFProcessor +from upload.vcf.bulk_allele_linking_vcf_processor import ( + BulkAlleleLinkingVCFProcessor, + FailedLiftoverVCFProcessor, +) from upload.vcf.bulk_clingen_allele_vcf_processor import BulkClinGenAlleleVCFProcessor -from upload.vcf.bulk_manual_variant_entry_linking_vcf_processor import BulkManualVariantEntryLinkingVCFProcessor +from upload.vcf.bulk_manual_variant_entry_linking_vcf_processor import ( + BulkManualVariantEntryLinkingVCFProcessor, +) from upload.vcf.bulk_minimal_vcf_processor import BulkMinimalVCFProcessor from upload.vcf.vcf_import import import_vcf_file from upload.vcf.vcf_preprocess import preprocess_vcf diff --git a/upload/tasks/vcf/unknown_variants_task.py b/upload/tasks/vcf/unknown_variants_task.py index 08ff91734..a06bbd29a 100644 --- a/upload/tasks/vcf/unknown_variants_task.py +++ b/upload/tasks/vcf/unknown_variants_task.py @@ -8,11 +8,11 @@ from django.core.cache import cache from library.genomics.vcf_utils import vcf_get_ref_alt_svlen_and_modification -from library.utils.file_utils import name_from_filename, mk_path +from library.utils.file_utils import mk_path, name_from_filename from snpdb import variant_collection from snpdb.models import VariantCoordinate from snpdb.variant_pk_lookup import VariantPKLookup -from upload.models import UploadStep, UploadStepTaskType, VCFPipelineStage, ModifiedImportedVariant +from upload.models import ModifiedImportedVariant, UploadStep, UploadStepTaskType, VCFPipelineStage from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask from upload.vcf import sql_copy_files from variantgrid.celery import app @@ -141,7 +141,7 @@ def process_items(self, upload_step): LOCK_ID = "insert-unknown-variants-lock" if not settings.UPLOAD_ENABLED: - raise ValueError(f"Uploads disabled, this should not have been called!") + raise ValueError("Uploads disabled, this should not have been called!") if cache.add(LOCK_ID, "true", LOCK_EXPIRE): # fails if already exists try: diff --git a/upload/tests/test_urls.py b/upload/tests/test_urls.py index 06451fca9..a7e0b3bbc 100644 --- a/upload/tests/test_urls.py +++ b/upload/tests/test_urls.py @@ -5,9 +5,9 @@ from annotation.fake_annotation import get_fake_annotation_version from library.django_utils.unittest_utils import URLTestCase, prevent_request_warnings -from snpdb.models import ImportSource, ProcessingStatus, VCF +from snpdb.models import VCF, ImportSource, ProcessingStatus from snpdb.models.models_genome import GenomeBuild -from upload.models import UploadedFile, UploadPipeline, UploadedVCF, UploadedFileTypes +from upload.models import UploadedFile, UploadedFileTypes, UploadedVCF, UploadPipeline class Test(URLTestCase): diff --git a/upload/tests/test_vcf_import_info.py b/upload/tests/test_vcf_import_info.py index b17eb5139..0a19a13f9 100644 --- a/upload/tests/test_vcf_import_info.py +++ b/upload/tests/test_vcf_import_info.py @@ -3,13 +3,18 @@ from annotation.fake_annotation import get_fake_annotation_version from library.utils import sha256sum_str -from snpdb.models import Sequence, GenomeBuild +from snpdb.models import GenomeBuild, Sequence from snpdb.models.models_enums import ImportSource from snpdb.tests.utils.vcf_testing_utils import slowly_create_test_variant from upload.models import ( - UploadedFile, UploadPipeline, UploadStep, UploadedFileTypes, - SimpleVCFImportInfo, ModifiedImportedVariants, ModifiedImportedVariant, + ModifiedImportedVariant, ModifiedImportedVariantOperation, + ModifiedImportedVariants, + SimpleVCFImportInfo, + UploadedFile, + UploadedFileTypes, + UploadPipeline, + UploadStep, ) diff --git a/upload/tests/vcf/test_vcf_detect_build.py b/upload/tests/vcf/test_vcf_detect_build.py index 3c42ffa20..c7e2c996f 100644 --- a/upload/tests/vcf/test_vcf_detect_build.py +++ b/upload/tests/vcf/test_vcf_detect_build.py @@ -4,7 +4,7 @@ from django.conf import settings from django.test import TestCase -from upload.vcf.vcf_import import vcf_detect_genome_build, GenomeBuildDetectionException +from upload.vcf.vcf_import import GenomeBuildDetectionException, vcf_detect_genome_build class TestVCFDetectBuild(TestCase): diff --git a/upload/tests/vcf/test_vcf_processors.py b/upload/tests/vcf/test_vcf_processors.py index 3dfee2462..29a4e5114 100644 --- a/upload/tests/vcf/test_vcf_processors.py +++ b/upload/tests/vcf/test_vcf_processors.py @@ -8,11 +8,11 @@ from annotation.fake_annotation import get_fake_annotation_version from library.utils import sha256sum_str from snpdb.models import ImportSource, Sequence -from upload.models import UploadedFile, UploadPipeline, UploadedVCF, UploadStep, UploadedFileTypes +from upload.models import UploadedFile, UploadedFileTypes, UploadedVCF, UploadPipeline, UploadStep from upload.vcf.bulk_genotype_vcf_processor import BulkGenotypeVCFProcessor from upload.vcf.bulk_no_genotype_vcf_processor import BulkNoGenotypeVCFProcessor from upload.vcf.sql_copy_files import COHORT_GENOTYPE_HEADER -from upload.vcf.vcf_import import create_vcf_from_vcf, create_cohort_genotype_collection_from_vcf +from upload.vcf.vcf_import import create_cohort_genotype_collection_from_vcf, create_vcf_from_vcf class TestVCFProcessors(TestCase): diff --git a/upload/upload_processing.py b/upload/upload_processing.py index c9eb68487..305050377 100644 --- a/upload/upload_processing.py +++ b/upload/upload_processing.py @@ -4,7 +4,13 @@ from eventlog.models import create_event from upload.import_task_factories.import_task_factory import get_import_task_factories -from upload.models import UploadPipeline, ProcessingStatus, UploadedFileTypes, UploadedFile, UploadStepOrigin +from upload.models import ( + ProcessingStatus, + UploadedFile, + UploadedFileTypes, + UploadPipeline, + UploadStepOrigin, +) def get_upload_processing_task(file_type, upload_pipeline): diff --git a/upload/upload_stats.py b/upload/upload_stats.py index 8888ca983..8891cfe77 100644 --- a/upload/upload_stats.py +++ b/upload/upload_stats.py @@ -1,15 +1,15 @@ from collections import Counter, defaultdict -from typing import TypeAlias, Optional +from typing import Optional, TypeAlias import numpy as np import pandas as pd from django.db.models import ExpressionWrapper, F, fields -from django.db.models.aggregates import Sum, Count +from django.db.models.aggregates import Count, Sum from django.utils import timezone from django.utils.datastructures import OrderedSet from snpdb.models import VCF -from upload.models import UploadStep, UploadPipeline +from upload.models import UploadPipeline, UploadStep def get_upload_stats(uploadstep_qs, max_step_names: int = None): diff --git a/upload/urls.py b/upload/urls.py index bd0182688..0ec955f1a 100644 --- a/upload/urls.py +++ b/upload/urls.py @@ -1,7 +1,10 @@ from library.django_utils.jqgrid_view import JQGridView from snpdb.views.datatable_view import DatabaseTableView -from upload.grids import UploadPipelineModifiedVariantsGrid, UploadPipelineSkippedAnnotationGrid, \ - UploadStepColumns +from upload.grids import ( + UploadPipelineModifiedVariantsGrid, + UploadPipelineSkippedAnnotationGrid, + UploadStepColumns, +) from upload.views import views from upload.views.views import view_upload_step_detail from upload.views.views_rest import APIFileUploadView diff --git a/upload/vcf/abstract_bulk_vcf_processor.py b/upload/vcf/abstract_bulk_vcf_processor.py index ef391a0df..90f3f537e 100644 --- a/upload/vcf/abstract_bulk_vcf_processor.py +++ b/upload/vcf/abstract_bulk_vcf_processor.py @@ -10,8 +10,14 @@ from library.genomics.vcf_utils import vcf_get_ref_alt_svlen_and_modification from snpdb.models import VariantCoordinate from snpdb.variant_pk_lookup import VariantPKLookup -from upload.models import UploadStep, ModifiedImportedVariant, UploadStepTaskType, VCFPipelineStage, \ - SimpleVCFImportInfo, ModifiedImportedVariantOperation +from upload.models import ( + ModifiedImportedVariant, + ModifiedImportedVariantOperation, + SimpleVCFImportInfo, + UploadStep, + UploadStepTaskType, + VCFPipelineStage, +) from upload.tasks.vcf.import_sql_copy_task import ImportModifiedImportedVariantSQLCopyTask from upload.vcf.sql_copy_files import write_sql_copy_csv diff --git a/upload/vcf/bulk_allele_linking_vcf_processor.py b/upload/vcf/bulk_allele_linking_vcf_processor.py index c774e57c2..7072f9a01 100644 --- a/upload/vcf/bulk_allele_linking_vcf_processor.py +++ b/upload/vcf/bulk_allele_linking_vcf_processor.py @@ -4,8 +4,16 @@ from django.conf import settings from library.utils import invert_dict -from snpdb.models import AlleleOrigin, VariantAllele, Allele, SequenceRole, Variant, AlleleLiftover, ProcessingStatus, \ - VariantCoordinate +from snpdb.models import ( + Allele, + AlleleLiftover, + AlleleOrigin, + ProcessingStatus, + SequenceRole, + Variant, + VariantAllele, + VariantCoordinate, +) from upload.models import ModifiedImportedVariant from upload.vcf.bulk_minimal_vcf_processor import BulkMinimalVCFProcessor diff --git a/upload/vcf/bulk_genotype_vcf_processor.py b/upload/vcf/bulk_genotype_vcf_processor.py index a600b607c..0e8b1f9c6 100644 --- a/upload/vcf/bulk_genotype_vcf_processor.py +++ b/upload/vcf/bulk_genotype_vcf_processor.py @@ -15,17 +15,24 @@ from library.django_utils.django_file_utils import get_import_processing_filename from library.genomics.vcf_enums import VCFConstant from library.git import Git -from library.utils import double_quote, json_default_converter, AsciiValue +from library.utils import AsciiValue, double_quote, json_default_converter from library.utils.database_utils import postgres_arrays from patients.models_enums import Zygosity from snpdb.common_variants import get_classified_high_frequency_variants_qs from snpdb.models import CohortGenotype, VariantCoordinate, VCFFilter from snpdb.models.models_enums import ProcessingStatus -from upload.models import UploadPipeline, PipelineFailedJobTerminateEarlyException, \ - VCFImporter, UploadStep, UploadStepTaskType, VCFPipelineStage, ModifiedImportedVariantOperation +from upload.models import ( + ModifiedImportedVariantOperation, + PipelineFailedJobTerminateEarlyException, + UploadPipeline, + UploadStep, + UploadStepTaskType, + VCFImporter, + VCFPipelineStage, +) from upload.tasks.vcf.import_sql_copy_task import ImportCohortGenotypeSQLCopyTask from upload.vcf.abstract_bulk_vcf_processor import AbstractBulkVCFProcessor -from upload.vcf.sql_copy_files import write_sql_copy_csv, COHORT_GENOTYPE_HEADER +from upload.vcf.sql_copy_files import COHORT_GENOTYPE_HEADER, write_sql_copy_csv class BulkGenotypeVCFProcessor(AbstractBulkVCFProcessor): @@ -317,7 +324,7 @@ def finished_locus(self): else: # TODO: In Python3.10 we can replace this check with zip(strict=True) if len(self.locus_cohort_genotypes) != len(self.locus_allele_depths): - raise ValueError(f"Locus cohort genotype and locus Allele depths not equal!") + raise ValueError("Locus cohort genotype and locus Allele depths not equal!") # Calculate ourselves across locus for cgt, ad in zip(self.locus_cohort_genotypes, self.locus_allele_depths): diff --git a/upload/vcf/vcf_import.py b/upload/vcf/vcf_import.py index 770628818..6810523f3 100644 --- a/upload/vcf/vcf_import.py +++ b/upload/vcf/vcf_import.py @@ -13,19 +13,46 @@ from django_messages.models import Message from library.genomics.vcf_enums import VCFConstant -from library.genomics.vcf_utils import cyvcf2_header_types, cyvcf2_header_get, cyvcf2_get_contig_lengths_dict +from library.genomics.vcf_utils import ( + cyvcf2_get_contig_lengths_dict, + cyvcf2_header_get, + cyvcf2_header_types, +) from library.guardian_utils import assign_permission_to_user_and_groups -from library.utils import invert_dict, get_single_element -from seqauto.models import JointCalledVCF, SingleSampleVCF, VCFFromSequencingRun, \ - SampleFromSequencingSample, QCGeneList +from library.utils import get_single_element, invert_dict +from seqauto.models import ( + JointCalledVCF, + QCGeneList, + SampleFromSequencingSample, + SingleSampleVCF, + VCFFromSequencingRun, +) from seqauto.signals.signals_list import backend_vcf_import_start_signal -from snpdb.models import VCF, ImportStatus, Sample, VCFFilter, \ - Cohort, CohortSample, UserSettings, VCFSourceSettings, SampleFilePath, VCFInfo, AbstractVCFField, VCFFormat -from snpdb.models.models_enums import ImportSource, VariantsType, SampleFileType, VCFInfoTypes +from snpdb.models import ( + VCF, + AbstractVCFField, + Cohort, + CohortSample, + ImportStatus, + Sample, + SampleFilePath, + UserSettings, + VCFFilter, + VCFFormat, + VCFInfo, + VCFSourceSettings, +) +from snpdb.models.models_enums import ImportSource, SampleFileType, VariantsType, VCFInfoTypes from snpdb.models.models_genome import GenomeBuild from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_collection -from upload.models import UploadedVCF, PipelineFailedJobTerminateEarlyException, \ - BackendVCF, UploadStep, UploadStepTaskType, VCFPipelineStage +from upload.models import ( + BackendVCF, + PipelineFailedJobTerminateEarlyException, + UploadedVCF, + UploadStep, + UploadStepTaskType, + VCFPipelineStage, +) from upload.tasks.vcf.import_sql_copy_task import ImportModifiedImportedVariantSQLCopyTask from upload.vcf.bulk_genotype_vcf_processor import BulkGenotypeVCFProcessor from upload.vcf.bulk_no_genotype_vcf_processor import BulkNoGenotypeVCFProcessor diff --git a/upload/vcf/vcf_preprocess.py b/upload/vcf/vcf_preprocess.py index 9dd1740d6..7ad9506d8 100644 --- a/upload/vcf/vcf_preprocess.py +++ b/upload/vcf/vcf_preprocess.py @@ -1,7 +1,7 @@ import glob import logging import os -from subprocess import Popen, PIPE, CalledProcessError +from subprocess import PIPE, CalledProcessError, Popen import pandas as pd from django.conf import settings @@ -11,10 +11,22 @@ from library.genomics.vcf_enums import INFO_LIFTOVER_SWAPPED_REF_ALT from library.genomics.vcf_utils import write_cleaned_vcf_header from library.utils.file_utils import name_from_filename -from upload.models import ModifiedImportedVariants, ToolVersion, UploadStep, \ - UploadStepTaskType, VCFSkippedContigs, VCFSkippedContig, UploadStepMultiFileOutput, VCFPipelineStage, \ - SimpleVCFImportInfo, ModifiedImportedVariant -from upload.tasks.vcf.unknown_variants_task import SeparateUnknownVariantsTask, AnnotateImportedVCFTask +from upload.models import ( + ModifiedImportedVariant, + ModifiedImportedVariants, + SimpleVCFImportInfo, + ToolVersion, + UploadStep, + UploadStepMultiFileOutput, + UploadStepTaskType, + VCFPipelineStage, + VCFSkippedContig, + VCFSkippedContigs, +) +from upload.tasks.vcf.unknown_variants_task import ( + AnnotateImportedVCFTask, + SeparateUnknownVariantsTask, +) def get_bcftools_tool_version(bcftools_command): @@ -30,7 +42,7 @@ def get_bcftools_tool_version(bcftools_command): if not bcftools_version.startswith("bcftools"): raise CalledProcessError(f"Expected to find bcftools on 1st line of output of '{bcftools_command} --version' output: {bcftools_version}") htslib_version = output_list[1] - if not "htslib" in htslib_version: + if "htslib" not in htslib_version: raise CalledProcessError(f"Expected to find htslib version on 2nd line of '{bcftools_command} --version' output: {htslib_version}") version = f"{bcftools_version}, {htslib_version}" @@ -212,7 +224,7 @@ def preprocess_vcf(upload_step, annotate_gnomad_af=False, disable_swap=False): stderr_output = None stderr_filename = stderr_filenames.get(sub_step_name) if stderr_filename: - with open(stderr_filename, "r") as sf: + with open(stderr_filename) as sf: stderr_output = sf.read() if stderr_output and len(stderr_output) > MAX_STDERR_OUTPUT: half = MAX_STDERR_OUTPUT // 2 diff --git a/upload/views/views.py b/upload/views/views.py index 728b8c87a..687c208ea 100644 --- a/upload/views/views.py +++ b/upload/views/views.py @@ -10,29 +10,44 @@ from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db.models import Q from django.http.response import HttpResponse, HttpResponseRedirect, JsonResponse -from django.shortcuts import render, get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.urls.base import reverse from django.utils import timezone from django.utils.timesince import timesince from django.views.decorators.cache import never_cache -from django.views.decorators.http import require_POST, require_http_methods +from django.views.decorators.http import require_http_methods, require_POST from django_downloadview import PathDownloadView from annotation.models import AnnotationRun from annotation.views import get_build_contigs from eventlog.models import create_event -from library.django_utils.file_uploads import filepond_upload_receive, filepond_process_response +from library.django_utils.file_uploads import filepond_process_response, filepond_upload_receive from library.enums.log_level import LogLevel from library.log_utils import log_traceback from library.utils.django_utils import render_ajax_view from snpdb.models import VCF from upload import forms, upload_processing, upload_stats -from upload.models import UploadPipeline, UploadedFile, ProcessingStatus, UploadedFileTypes, \ - UploadSettings, ImportSource, UploadStep, VCFSkippedContigs, \ - VCFImportInfo, SimpleVCFImportInfo, ModifiedImportedVariant, TimeFilterMethod -from upload.uploaded_file_type import get_upload_data_for_uploaded_file, \ - get_uploaded_file_type, get_url_and_data_for_uploaded_file_data, \ - retry_upload_pipeline, get_import_tasks_by_extension +from upload.models import ( + ImportSource, + ModifiedImportedVariant, + ProcessingStatus, + SimpleVCFImportInfo, + TimeFilterMethod, + UploadedFile, + UploadedFileTypes, + UploadPipeline, + UploadSettings, + UploadStep, + VCFImportInfo, + VCFSkippedContigs, +) +from upload.uploaded_file_type import ( + get_import_tasks_by_extension, + get_upload_data_for_uploaded_file, + get_uploaded_file_type, + get_url_and_data_for_uploaded_file_data, + retry_upload_pipeline, +) UPLOADED_FILE_CONTEXT = {UploadedFileTypes.VCF: "uploaded_vcf", UploadedFileTypes.GENE_LIST: "uploaded_gene_list"} diff --git a/variantopedia/grids.py b/variantopedia/grids.py index dfe84bde7..650b0799f 100644 --- a/variantopedia/grids.py +++ b/variantopedia/grids.py @@ -1,26 +1,28 @@ import operator import re from functools import reduce -from typing import Optional, Any +from typing import Any, Optional from django.conf import settings -from django.core.paginator import Paginator, InvalidPage +from django.core.paginator import InvalidPage, Paginator from django.db import connection -from django.db.models import QuerySet, Q +from django.db.models import Q, QuerySet from django.http import HttpRequest from django.shortcuts import get_object_or_404 -from analysis.models import VariantTag, Analysis -from annotation.annotation_version_querysets import get_variant_queryset_for_latest_annotation_version, \ - get_variant_queryset_for_annotation_version +from analysis.models import Analysis, VariantTag +from annotation.annotation_version_querysets import ( + get_variant_queryset_for_annotation_version, + get_variant_queryset_for_latest_annotation_version, +) from annotation.models import AnnotationVersion, VariantAnnotation from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig -from library.utils import update_dict_of_dict_values, JsonDataType +from library.utils import JsonDataType, update_dict_of_dict_values from snpdb.grid_columns.custom_columns import get_custom_column_fields_override_and_sample_position from snpdb.grids import AbstractVariantGrid -from snpdb.models import Variant, VariantZygosityCountCollection, GenomeBuild, Tag, VariantWiki -from snpdb.models.models_user_settings import UserSettings, UserGridConfig -from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder, CellData +from snpdb.models import GenomeBuild, Tag, Variant, VariantWiki, VariantZygosityCountCollection +from snpdb.models.models_user_settings import UserGridConfig, UserSettings +from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder from variantopedia.interesting_nearby import get_nearby_qs diff --git a/variantopedia/interesting_nearby.py b/variantopedia/interesting_nearby.py index d4ea1fbbe..980cd48ba 100644 --- a/variantopedia/interesting_nearby.py +++ b/variantopedia/interesting_nearby.py @@ -1,14 +1,14 @@ import operator import re -from collections import defaultdict, Counter +from collections import Counter, defaultdict from functools import reduce from django.conf import settings from django.contrib.postgres.aggregates import StringAgg -from django.db.models import Count, Sum, Q +from django.db.models import Count, Q, Sum from annotation.annotation_version_querysets import get_variant_queryset_for_annotation_version -from annotation.models import VariantTranscriptAnnotation, AnnotationVersion +from annotation.models import AnnotationVersion, VariantTranscriptAnnotation from classification.enums import ClinicalSignificance from classification.models import Classification from genes.models import GeneSymbol diff --git a/variantopedia/management/commands/deployment_check.py b/variantopedia/management/commands/deployment_check.py index 48305d1af..49903c4e6 100644 --- a/variantopedia/management/commands/deployment_check.py +++ b/variantopedia/management/commands/deployment_check.py @@ -3,9 +3,14 @@ from django.conf import settings from django.core.management.base import BaseCommand -from variantgrid.deployment_validation.annotation_files_check import annotation_data_exists, check_cdot_data -from variantgrid.deployment_validation.annotation_status_checks import check_annotation_versions, \ - check_variant_annotation_runs_status +from variantgrid.deployment_validation.annotation_files_check import ( + annotation_data_exists, + check_cdot_data, +) +from variantgrid.deployment_validation.annotation_status_checks import ( + check_annotation_versions, + check_variant_annotation_runs_status, +) from variantgrid.deployment_validation.celery_checks import check_celery_tasks from variantgrid.deployment_validation.classification_checks import check_classification_reports from variantgrid.deployment_validation.column_check import check_variantgrid_columns diff --git a/variantopedia/server_status.py b/variantopedia/server_status.py index f0ac638ec..49b4406de 100644 --- a/variantopedia/server_status.py +++ b/variantopedia/server_status.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.utils.timezone import localtime -from library.health_check import HealthCheckRequest, HealthCheckRecentActivity, health_check_signal +from library.health_check import HealthCheckRecentActivity, HealthCheckRequest, health_check_signal from library.utils import flatten_nested_lists diff --git a/variantopedia/tests/test_search.py b/variantopedia/tests/test_search.py index f314abd47..6e3d7dc4d 100644 --- a/variantopedia/tests/test_search.py +++ b/variantopedia/tests/test_search.py @@ -3,7 +3,7 @@ from annotation.fake_annotation import get_fake_annotation_version from annotation.tests.test_data_fake_genes import create_fake_transcript_version -from snpdb.models import GenomeBuild, ClinGenAllele +from snpdb.models import ClinGenAllele, GenomeBuild from snpdb.search import search_data from variantopedia.models import SearchTypes diff --git a/variantopedia/tests/test_urls.py b/variantopedia/tests/test_urls.py index 771cd248e..16df95229 100644 --- a/variantopedia/tests/test_urls.py +++ b/variantopedia/tests/test_urls.py @@ -4,11 +4,24 @@ from django.contrib.auth.models import User from analysis.models import VariantTag -from annotation.fake_annotation import get_fake_annotation_version, create_fake_variants, create_fake_variant_annotation +from annotation.fake_annotation import ( + create_fake_variant_annotation, + create_fake_variants, + get_fake_annotation_version, +) from annotation.tests.test_data_fake_genes import create_fake_transcript_version from library.django_utils.unittest_utils import URLTestCase -from snpdb.models import Variant, ClinGenAllele, Allele, VariantAllele, AlleleOrigin, Tag, \ - VariantZygosityCountCollection, VariantZygosityCount, VariantWiki +from snpdb.models import ( + Allele, + AlleleOrigin, + ClinGenAllele, + Tag, + Variant, + VariantAllele, + VariantWiki, + VariantZygosityCount, + VariantZygosityCountCollection, +) from snpdb.models.models_genome import GenomeBuild from snpdb.tests.utils.mock_clingen_api import MockClinGenAlleleRegistryAPI diff --git a/variantopedia/urls.py b/variantopedia/urls.py index fd61b0721..da5b4d948 100644 --- a/variantopedia/urls.py +++ b/variantopedia/urls.py @@ -2,8 +2,15 @@ from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import path from variantopedia import views -from variantopedia.grids import AllVariantsGrid, NearbyVariantsGrid, TaggedVariantGrid, \ - VariantTagsGrid, VariantWikiColumns, VariantTagCountsColumns, VariantTagDetailColumns +from variantopedia.grids import ( + AllVariantsGrid, + NearbyVariantsGrid, + TaggedVariantGrid, + VariantTagCountsColumns, + VariantTagDetailColumns, + VariantTagsGrid, + VariantWikiColumns, +) urlpatterns = [ path('variants', views.variants, name='variants'), diff --git a/variantopedia/views.py b/variantopedia/views.py index de2d9457e..c42423f7b 100644 --- a/variantopedia/views.py +++ b/variantopedia/views.py @@ -4,7 +4,7 @@ import re from collections import defaultdict from dataclasses import dataclass -from datetime import timedelta, datetime +from datetime import datetime, timedelta from functools import reduce from typing import Any, Optional @@ -14,20 +14,34 @@ from django.core.exceptions import PermissionDenied from django.db import connection from django.forms import model_to_dict -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.timesince import timesince from django.utils.timezone import localtime from django.views.decorators.http import require_POST from analysis.models import VariantTag -from annotation.models import AnnotationRun, AnnotationVersion, ClassificationModification, Classification, \ - VariantAnnotationVersion, VariantAnnotation, AnnotationStatus, ClinVarRecordCollection from annotation.manual_variant_entry import check_can_create_variants +from annotation.models import ( + AnnotationRun, + AnnotationStatus, + AnnotationVersion, + Classification, + ClassificationModification, + ClinVarRecordCollection, + VariantAnnotation, + VariantAnnotationVersion, +) from annotation.transcripts_annotation_selections import VariantTranscriptSelections from classification.enums import AlleleOriginBucket, ShareLevel, SpecialEKeys -from classification.models import ClassificationGrouping, AlleleOriginGrouping, DiscordanceReport, OverlapStatus, \ - EvidenceKeyMap, ClassificationGroupingEntry +from classification.models import ( + AlleleOriginGrouping, + ClassificationGrouping, + ClassificationGroupingEntry, + DiscordanceReport, + EvidenceKeyMap, + OverlapStatus, +) from classification.models.classification_import_run import ClassificationImportRun from classification.variant_card import AlleleCard from classification.views.exports import ClassificationExportFormatterCSV @@ -36,12 +50,17 @@ from eventlog.models import create_event from genes.hgvs import HGVSMatcher from genes.models import CanonicalTranscriptCollection, GeneSymbol -from library.django_utils import require_superuser, highest_pk, get_field_counts +from library.django_utils import get_field_counts, highest_pk, require_superuser from library.django_utils.jqgrid_view import JQGridView from library.git import Git from library.guardian_utils import admin_bot from library.health_check import HealthCheckRequest, health_check_overall_stats_signal -from library.log_utils import log_traceback, report_message, slack_bot_username, AdminNotificationBuilder +from library.log_utils import ( + AdminNotificationBuilder, + log_traceback, + report_message, + slack_bot_username, +) from library.utils import flatten_nested_lists from pathtests.models import cases_for_user from patients.models import Clinician @@ -51,8 +70,18 @@ from snpdb.forms import TagForm, get_settings_form_features from snpdb.genome_build_manager import GenomeBuildManager from snpdb.liftover import create_liftover_pipelines -from snpdb.models import Variant, Sample, VCF, get_igv_data, Allele, AlleleConversionTool, ImportSource, AlleleOrigin, \ - VariantGridColumn, Tag +from snpdb.models import ( + VCF, + Allele, + AlleleConversionTool, + AlleleOrigin, + ImportSource, + Sample, + Tag, + Variant, + VariantGridColumn, + get_igv_data, +) from snpdb.models.models_genome import GenomeBuild from snpdb.models.models_user_settings import UserSettings from snpdb.search import search_data @@ -63,8 +92,12 @@ from variantgrid.celery import app from variantgrid.tasks.server_monitoring_tasks import get_disk_messages from variantopedia import forms -from variantopedia.grids import VariantTagsGrid, TaggedVariantGrid -from variantopedia.interesting_nearby import get_nearby_qs, get_method_summaries, get_nearby_summaries +from variantopedia.grids import TaggedVariantGrid, VariantTagsGrid +from variantopedia.interesting_nearby import ( + get_method_summaries, + get_nearby_qs, + get_nearby_summaries, +) from variantopedia.server_status import get_dashboard_notices from variantopedia.tasks.server_status_tasks import notify_server_status_now