Add Python linting & type-checking tooling (ruff + mypy/django-stubs), slim pylint, apply safe auto-fixes

analysis/analysis_import_export.py

@@ -5,7 +5,13 @@
 from django.contrib.auth.models import User
 from django.core.serializers import serialize
 
-from analysis.models import Analysis, GenomeBuild, AnnotationVersion, AnalysisEdge, AnalysisTemplateType
+from analysis.models import (
+    Analysis,
+    AnalysisEdge,
+    AnalysisTemplateType,
+    AnnotationVersion,
+    GenomeBuild,
+)
 from analysis.models.nodes.node_utils import reload_analysis_nodes
 from analysis.serializers import AnalysisNodeSerializer, AnalysisSerializer
 

analysis/analysis_templates.py

@@ -1,18 +1,26 @@
 import logging
 import re
 from dataclasses import dataclass
-from typing import Optional, List
+from typing import Optional
 
 from auditlog.context import disable_auditlog
 from django.contrib.auth.models import User
 
-from analysis.models import Analysis, AnalysisNode, AnalysisTemplate, AnalysisTemplateRun, \
-    AnalysisTemplateRunArgument, SampleAnalysisTemplateRun, CohortAnalysisTemplateRun, AutoLaunchAnalysisTemplate
+from analysis.models import (
+    Analysis,
+    AnalysisNode,
+    AnalysisTemplate,
+    AnalysisTemplateRun,
+    AnalysisTemplateRunArgument,
+    AutoLaunchAnalysisTemplate,
+    CohortAnalysisTemplateRun,
+    SampleAnalysisTemplateRun,
+)
 from analysis.models.nodes.node_utils import get_toposorted_nodes, reload_analysis_nodes
 from analysis.related_analyses import get_related_analysis_details_for_samples
 from genes.models import ActiveSampleGeneList
 from library.guardian_utils import add_public_group_read_permission
-from snpdb.models import Sample, GenomeBuild, Cohort
+from snpdb.models import Cohort, GenomeBuild, Sample
 
 
 def run_analysis_template(analysis_template: AnalysisTemplate,
@@ -123,7 +131,7 @@ def match(self) -> bool:
         return self.enrichment_kit_match and self.sample_regex_match
 
 
-def get_auto_launch_analysis_template_matches(user, sample_enrichment_kit_name, sample_name) -> List[AutoLaunchAnalysisTemplateMatch]:
+def get_auto_launch_analysis_template_matches(user, sample_enrichment_kit_name, sample_name) -> list[AutoLaunchAnalysisTemplateMatch]:
     matches = []
     templates_qs = AnalysisTemplate.filter_for_user(user)
     for auto_launch in AutoLaunchAnalysisTemplate.objects.filter(template__in=templates_qs):

analysis/apps.py

@@ -9,14 +9,21 @@ class AnalysisConfig(AppConfig):
     def ready(self):
         # pylint: disable=import-outside-toplevel,unused-import
         # imported to activate receivers
-        from analysis.signals import analysis_search, analysis_health_check
 
         from analysis.models import VariantTag
-        from analysis.signals.signal_handlers import variant_tag_create, variant_tag_delete, \
-            handle_vcf_import_success, handle_active_sample_gene_list_created
-        from analysis.signals.source_data_invalidation import handle_sample_pre_delete, \
-            handle_cohort_pre_delete, handle_trio_pre_delete, handle_pedigree_pre_delete, \
-            handle_quad_pre_delete
+        from analysis.signals.signal_handlers import (
+            handle_active_sample_gene_list_created,
+            handle_vcf_import_success,
+            variant_tag_create,
+            variant_tag_delete,
+        )
+        from analysis.signals.source_data_invalidation import (
+            handle_cohort_pre_delete,
+            handle_pedigree_pre_delete,
+            handle_quad_pre_delete,
+            handle_sample_pre_delete,
+            handle_trio_pre_delete,
+        )
         from genes.models import ActiveSampleGeneList
         from pedigree.models import Pedigree
         from snpdb.models import Cohort, Quad, Sample, Trio

analysis/forms/forms.py

@@ -4,16 +4,30 @@
 from collections import defaultdict
 
 from crispy_forms.bootstrap import FieldWithButtons, StrictButton
-from crispy_forms.layout import Layout, Field
+from crispy_forms.layout import Field, Layout
 from dal import forward
 from django import forms
 from django.core.exceptions import ValidationError
 from django.forms import inlineformset_factory
 from django.forms.widgets import TextInput
 
-from analysis.models import Analysis, NodeGraphType, FilterNodeItem, AnalysisTemplate, AnalysisTemplateVersion, \
-    AnalysisNode, CandidateStatus, AutoLaunchAnalysisTemplate
-from analysis.models.enums import SNPMatrix, AnalysisTemplateType, TrioSample, QuadSample, AnalysisType
+from analysis.models import (
+    Analysis,
+    AnalysisNode,
+    AnalysisTemplate,
+    AnalysisTemplateVersion,
+    AutoLaunchAnalysisTemplate,
+    CandidateStatus,
+    FilterNodeItem,
+    NodeGraphType,
+)
+from analysis.models.enums import (
+    AnalysisTemplateType,
+    AnalysisType,
+    QuadSample,
+    SNPMatrix,
+    TrioSample,
+)
 from analysis.models.models_karyomapping import KaryomappingGene
 from analysis.models.nodes.node_types import get_nodes_by_classification
 from annotation.models.models import AnnotationVersion, VariantAnnotationVersion
@@ -23,7 +37,7 @@
 from library.guardian_utils import assign_permission_to_user_and_groups
 from seqauto.models import EnrichmentKit
 from snpdb.forms import GenomeBuildAutocompleteForwardMixin, UserSettingsGenomeBuildMixin
-from snpdb.models import CustomColumnsCollection, VariantGridColumn, Trio, UserSettings
+from snpdb.models import CustomColumnsCollection, Trio, UserSettings, VariantGridColumn
 from uicore.utils.form_helpers import form_helper_horizontal
 
 

analysis/forms/forms_nodes.py

@@ -3,13 +3,13 @@
 from dal import forward
 from django import forms
 from django.forms.models import fields_for_model
-from django.forms.widgets import TextInput, HiddenInput
+from django.forms.widgets import HiddenInput, TextInput
 from django.utils.text import slugify
 from django_starfield import Stars
 
 from analysis import models
-from analysis.models import AnalysisNode, AnalysisTemplateType, Analysis, MOINode
-from analysis.models.nodes.analysis_node import NodeVCFFilter, NodeAlleleFrequencyFilter
+from analysis.models import Analysis, AnalysisNode, AnalysisTemplateType, MOINode
+from analysis.models.nodes.analysis_node import NodeAlleleFrequencyFilter, NodeVCFFilter
 from analysis.models.nodes.filters.conservation_node import ConservationNode
 from analysis.models.nodes.filters.damage_node import DamageNode
 from analysis.models.nodes.filters.gene_list_node import GeneListNode
@@ -23,24 +23,27 @@
 from analysis.models.nodes.filters.venn_node import VennNode
 from analysis.models.nodes.sources.all_variants_node import AllVariantsNode
 from analysis.models.nodes.sources.classifications_node import ClassificationsNode
-from analysis.models.nodes.sources.cohort_node import CohortNode, CohortNodeZygosityFiltersCollection, \
-    CohortNodeZygosityFilter
+from analysis.models.nodes.sources.cohort_node import (
+    CohortNode,
+    CohortNodeZygosityFilter,
+    CohortNodeZygosityFiltersCollection,
+)
 from analysis.models.nodes.sources.pedigree_node import PedigreeNode
-from analysis.models.nodes.sources.sample_node import SampleNode
 from analysis.models.nodes.sources.quad_node import QuadNode
+from analysis.models.nodes.sources.sample_node import SampleNode
 from analysis.models.nodes.sources.trio_node import TrioNode
 from annotation.models import VariantAnnotation
 from annotation.pathogenicity_predictions import TOOLS
 from genes.custom_text_gene_list import create_custom_text_gene_list
-from genes.hgvs import get_hgvs_variant_coordinate, get_hgvs_variant, HGVSException
-from genes.models import GeneListCategory, CustomTextGeneList, GeneList, PanelAppPanel
+from genes.hgvs import HGVSException, get_hgvs_variant, get_hgvs_variant_coordinate
+from genes.models import CustomTextGeneList, GeneList, GeneListCategory, PanelAppPanel
 from library.django_utils.autocomplete_utils import ModelSelect2, ModelSelect2Multiple
 from library.forms import NumberInput
 from library.utils import sha256sum_str
 from ontology.models import OntologyTerm
 from patients.models_enums import GnomADPopulation
 from snpdb.forms import GenomeBuildAutocompleteForwardMixin
-from snpdb.models import GenomicInterval, Sample, VCFFilter, Tag, Lab
+from snpdb.models import GenomicInterval, Lab, Sample, Tag, VCFFilter
 from snpdb.models.models_genome import Contig
 
 # Can use this for ModelForm.exclude to only use node specific fields

analysis/grid_export.py

@@ -2,11 +2,12 @@
 import operator
 import re
 from collections import Counter
+from collections.abc import Iterator
 from io import StringIO
-from typing import Iterator, Optional
+from typing import Optional
 
-from vcf import Writer, Reader
-from vcf.model import _Substitution, _Record, make_calldata_tuple, _Call
+from vcf import Reader, Writer
+from vcf.model import _Call, _Record, _Substitution, make_calldata_tuple
 
 from analysis.grids import ExportVariantGrid
 from analysis.models import AnalysisNode
@@ -16,7 +17,7 @@
 from library.django_utils.jqgrid_view import grid_export_csv
 from library.utils import StashFile
 from patients.models_enums import Zygosity
-from snpdb.models import Sample, ColumnVCFInfo, VCFInfoTypes
+from snpdb.models import ColumnVCFInfo, Sample, VCFInfoTypes
 from snpdb.vcf_export_utils import get_vcf_header_from_contigs
 
 

analysis/grids.py

@@ -2,24 +2,39 @@
 import operator
 import time
 from collections import defaultdict
+from collections.abc import Callable
 from functools import reduce
-from typing import Optional, Any, Callable
+from typing import Any, Optional
 
 import pandas as pd
 from auditlog.models import LogEntry
 from django.conf import settings
 from django.contrib.postgres.aggregates import StringAgg
 from django.core.exceptions import PermissionDenied
-from django.db.models import Max, F, Q, QuerySet
+from django.db.models import F, Max, Q, QuerySet
 from django.db.models.functions import Substr
 from django.shortcuts import get_object_or_404
 from django.urls.base import reverse
 from django.utils.functional import SimpleLazyObject
 
-from analysis.models import Analysis, AnalysisNode, NodeCount, NodeStatus, AnalysisTemplate, GroupOperation, \
-    CandidateSearchRun, CandidateSearchType, Candidate, CandidateStatus, AnalysisType
+from analysis.models import (
+    Analysis,
+    AnalysisNode,
+    AnalysisTemplate,
+    AnalysisType,
+    Candidate,
+    CandidateSearchRun,
+    CandidateSearchType,
+    CandidateStatus,
+    GroupOperation,
+    NodeCount,
+    NodeStatus,
+)
 from analysis.models.models_karyomapping import KaryomappingAnalysis
-from analysis.models.nodes.analysis_node import get_extra_filters_q, NodeColumnSummaryCacheCollection
+from analysis.models.nodes.analysis_node import (
+    NodeColumnSummaryCacheCollection,
+    get_extra_filters_q,
+)
 from analysis.views.analysis_permissions import get_node_subclass_or_404
 from annotation.models import HumanProteinAtlasAnnotation
 from genes.grids import GeneListGenesColumns
@@ -28,18 +43,29 @@
 from library.jqgrid.jqgrid_user_row_config import JqGridUserRowConfig
 from library.pandas_jqgrid import DataFrameJqGrid
 from library.unit_percent import get_allele_frequency_formatter
-from library.utils import update_dict_of_dict_values, JsonDataType, sha256sum_str
+from library.utils import JsonDataType, sha256sum_str, update_dict_of_dict_values
 from ontology.grids import AbstractOntologyGenesGrid
-from ontology.models import OntologyTermRelation, GeneDiseaseClassification, OntologyVersion
+from ontology.models import GeneDiseaseClassification, OntologyTermRelation, OntologyVersion
 from patients.models_enums import Zygosity
-from snpdb.grid_columns.custom_columns import get_custom_column_fields_override_and_sample_position, \
-    get_variantgrid_extra_annotate
-from snpdb.grid_columns.grid_sample_columns import get_available_format_columns, \
-    get_variantgrid_zygosity_annotation_kwargs
+from snpdb.grid_columns.custom_columns import (
+    get_custom_column_fields_override_and_sample_position,
+    get_variantgrid_extra_annotate,
+)
+from snpdb.grid_columns.grid_sample_columns import (
+    get_available_format_columns,
+    get_variantgrid_zygosity_annotation_kwargs,
+)
 from snpdb.grids import AbstractVariantGrid
-from snpdb.models import VariantGridColumn, UserGridConfig, VCFFilter, Sample, CohortGenotype, ProcessingStatus
+from snpdb.models import (
+    CohortGenotype,
+    ProcessingStatus,
+    Sample,
+    UserGridConfig,
+    VariantGridColumn,
+    VCFFilter,
+)
 from snpdb.models.models_genome import GenomeBuild
-from snpdb.views.datatable_view import DatatableConfig, RichColumn, SortOrder, CellData
+from snpdb.views.datatable_view import CellData, DatatableConfig, RichColumn, SortOrder
 
 
 class VariantGrid(AbstractVariantGrid):
@@ -230,7 +256,7 @@ def get_grid_genotype_columns_and_overrides(cohorts, visibility,
         # Some legacy data (Missing data in FreeBayes before PythonKnownVariantsImporter v12) has -2147483647 for
         # empty values (what CyVCF2 returns using format()) @see https://github.com/SACGF/variantgrid/issues/59
         MISSING_VALUES = [CohortGenotype.MISSING_NUMBER_VALUE, CohortGenotype.MISSING_FT_VALUE, -2147483648]
-        packed_data_replace.update({mv: VariantGrid.GENOTYPE_COLUMNS_MISSING_VALUE for mv in MISSING_VALUES})
+        packed_data_replace.update(dict.fromkeys(MISSING_VALUES, VariantGrid.GENOTYPE_COLUMNS_MISSING_VALUE))
 
         # We now have separate aliases for packed data, so each cohort handled separately
         sample_cohort_index = {}

analysis/management/commands/analyses_process_incomplete_nodes.py

@@ -2,7 +2,7 @@
 
 from django.core.management.base import BaseCommand
 
-from analysis.models import NodeStatus, AnalysisNode
+from analysis.models import AnalysisNode, NodeStatus
 from analysis.models.models_analysis import Analysis
 from analysis.models.nodes.node_utils import update_analysis
 

analysis/management/commands/analysis_create_default_templates.py

@@ -5,8 +5,8 @@
 from django.core.management.base import BaseCommand
 
 from analysis.analysis_import_export import analysis_import
-from analysis.models import AnalysisTemplateType, AnalysisTemplate, AnalysisTemplateVersion
-from library.guardian_utils import admin_bot, add_public_group_read_permission
+from analysis.models import AnalysisTemplate, AnalysisTemplateType, AnalysisTemplateVersion
+from library.guardian_utils import add_public_group_read_permission, admin_bot
 from snpdb.models import GenomeBuild
 
 

analysis/management/commands/fix_analysis_permissions.py

@@ -1,6 +1,7 @@
 from django.core.management.base import BaseCommand
 
 from analysis.models.models_analysis import Analysis
+
 # Analysis templates didn't set permissions correctly until fix on October 19th 2020
 # Need to fix legacy data created in systems before this fix: https://github.com/SACGF/variantgrid/issues/84
 # Can be removed once environments fixed

analysis/management/commands/fix_legacy_phenotype_nodes.py

@@ -9,7 +9,7 @@
 from django.db import IntegrityError
 
 from analysis.models import PhenotypeNodeOntologyTerm
-from ontology.models import OntologyTerm, OntologyService
+from ontology.models import OntologyService, OntologyTerm
 
 
 class Command(BaseCommand):

analysis/management/commands/fix_liftover_existing_variant_tags.py

@@ -5,7 +5,7 @@
 from library.utils import iter_fixed_chunks
 from snpdb.clingen_allele import populate_clingen_alleles_for_variants
 from snpdb.liftover import create_liftover_pipelines
-from snpdb.models import GenomeBuild, ImportSource, Variant, Allele
+from snpdb.models import Allele, GenomeBuild, ImportSource, Variant
 
 
 class Command(BaseCommand):

analysis/management/commands/fix_variant_tag_permissions.py

@@ -1,5 +1,5 @@
 from django.core.management.base import BaseCommand
-from guardian.shortcuts import get_groups_with_perms, get_group_perms, assign_perm
+from guardian.shortcuts import assign_perm, get_group_perms, get_groups_with_perms
 
 from analysis.models.models_analysis import Analysis
 from analysis.models.models_variant_tag import VariantTag

analysis/management/commands/profile_analysis_nodes.py

@@ -29,13 +29,12 @@
 import socket
 import sys
 import time
-import traceback
 from datetime import datetime
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connection
-from django.db.models import F, Q
+from django.db.models import Q
 from django.db.models.functions import Substr as DjSubstr
 
 from analysis.models import Analysis
@@ -45,7 +44,6 @@
 from snpdb.models import Cohort, Sample, Trio, Variant, VariantCollection
 from snpdb.models.models_enums import ProcessingStatus
 
-
 CSV_FIELDS = [
     "source",
     "analysis_id",

analysis/models.py

@@ -1,28 +1,3 @@
 # This file exists for PyCharm's Django Structure plugin
 # pylint: disable=unused-import
-from analysis.models.gene_counts import NodeGenesCountCollection, NodeGenesCount
-from analysis.models.models_analysis import Analysis, AnalysisLock, AnalysisNodeCountConfiguration, \
-    AnalysisNodeCountConfigRecord, AnalysisVariable, AnalysisTemplate, AnalysisTemplateVersion, AnalysisTemplateRun, \
-    AnalysisTemplateRunArgument, CohortAnalysisTemplateRun, SampleAnalysisTemplateRun
-from analysis.models.models_karyomapping import KaryomappingAnalysis, KaryomappingGene, GenomeKaryomappingCounts, \
-    ContigKaryomappingCounts
-from analysis.models.models_variant_tag import VariantTagsImport, ImportedVariantTag, VariantTag
-from analysis.models.mutational_signatures import MutationalSignatureCalculator, MutationalSignature, \
-    MutationalSignatureMinimisationResult, MutationalSignatureMutationCount
-from analysis.models.nodes.analysis_node import NodeTask, NodeWiki, AnalysisNodeAlleleSource, NodeVersion, NodeCache, \
-    NodeCount, NodeColumnSummaryCacheCollection, NodeColumnSummaryData, NodeVCFFilter, NodeAlleleFrequencyFilter, \
-    NodeAlleleFrequencyRange, AnalysisClassification
-from analysis.models.nodes.filters.filter_node import FilterNodeItem
-from analysis.models.nodes.filters.gene_list_node import GeneListNodeGeneList, GeneListNodePanelAppPanel
-from analysis.models.nodes.filters.moi_node import MOINodeOntologyTerm, MOINodeModeOfInheritance, MOINodeSubmitter
-from analysis.models.nodes.filters.phenotype_node import PhenotypeNodeOntologyTerm
-from analysis.models.nodes.filters.population_node import PopulationNodeGnomADPopulation
-from analysis.models.nodes.filters.selected_in_parent_node import NodeVariant
-from analysis.models.nodes.filters.tag_node import TagNodeTag
-from analysis.models.nodes.filters.venn_node import VennNodeCache
-from analysis.models.nodes.node_types import NodeGraphType
-from analysis.models.nodes.sources.all_variants_node import AllVariantsNode
-from analysis.models.nodes.sources.classifications_node import ClassificationsNodeLab
-from analysis.models.nodes.sources.cohort_node import CohortNode, CohortNodeZygosityFiltersCollection, \
-    CohortNodeZygosityFilter
 # pylint: enable=unused-import