diff --git a/.github/workflows/kg-build-part2.yml b/.github/workflows/kg-build-part2.yml index 4800f752..1dfab333 100644 --- a/.github/workflows/kg-build-part2.yml +++ b/.github/workflows/kg-build-part2.yml @@ -1,7 +1,7 @@ name: KG Build - Part 2 (Construct Knowledge Graphs) on: schedule: - - cron: '0 0 25 * *' # runs at 00:00:00 UTC on the second day of each month + - cron: '0 0 29 * *' # runs at 00:00:00 UTC on the second day of each month env: PROJECT_ID: ${{ secrets.GCE_PROJECT }} GCS_SERVICE_ACCOUNT: ${{ secrets.GCE_SA_KEY }} diff --git a/.gitignore b/.gitignore index 57de4583..3f6b8f76 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ builds/temp/* #### External Libraries pkt_kg/libs/deepwalk_c_master/* pkt_kg/libs/walking-rdf-and-owl-master/* +pkt_kg/libs/pylucene* #### Scripts pkt_kg/kg_embedding_visualizer.py @@ -48,7 +49,8 @@ scratch*.py /resources/embeddings/* /resources/knowledge_graphs/ /resources/kr_model/ -/resources/node_data/* +/resources/metadata/* +!/resources/metadata/pheknowlator_source_metadata.xlsx /resources/ontologies/* /resources/owl_decoding/* /resources/processed_data/* @@ -60,7 +62,7 @@ scratch*.py !/resources/edge_data/README.md !/resources/embeddings/README.md !/resources/knowledge_graphs/README.md -!/resources/node_data/README.md +!/resources/metadata/README.md !/resources/ontologies/ontology_source_metadata.txt !/resources/ontologies/README.md !/resources/owl_decoding/README.md diff --git a/Main.py b/Main.py index b8d366be..69e95ec8 100644 --- a/Main.py +++ b/Main.py @@ -26,7 +26,7 @@ def main(): parser.add_argument('-b', '--kg', help='build type: "partial", "full", or "post-closure"', required=True) parser.add_argument('-r', '--rel', help='yes/no - adding inverse relations to knowledge graph', required=True) parser.add_argument('-s', '--owl', help='yes/no - removing OWL Semantics from knowledge graph', required=True) - parser.add_argument('-m', '--nde', help='yes/no - adding node metadata to knowledge graph', required=True) + parser.add_argument('-m', '--mta', help='yes/no - adding entity metadata to knowledge graph', required=True) parser.add_argument('-o', '--out', help='name/path to directory where to write knowledge graph', required=True) args = parser.parse_args() @@ -85,21 +85,21 @@ def main(): if args.kg == 'partial': kg = PartialBuild(construction=args.app, - node_data=args.nde, + node_data=args.mta, inverse_relations=args.rel, decode_owl=args.owl, cpus=cpus, write_location=args.out) elif args.kg == 'post-closure': kg = PostClosureBuild(construction=args.app, - node_data=args.nde, + node_data=args.mta, inverse_relations=args.rel, decode_owl=args.owl, cpus=cpus, write_location=args.out) else: kg = FullBuild(construction=args.app, - node_data=args.nde, + node_data=args.mta, inverse_relations=args.rel, decode_owl=args.owl, cpus=cpus, diff --git a/README.rst b/README.rst index e7f09f14..599e2b98 100644 --- a/README.rst +++ b/README.rst @@ -129,7 +129,7 @@ The ``pkt_kg`` library requires a specific project directory structure. | | | knowledge_graphs/ | | - | node_data/ + | metadata/ | | | ontologies/ | | @@ -165,7 +165,9 @@ The `KG Construction`_ Wiki page provides a detailed description of the knowledg * `resources/construction_approach/subclass_construction_map.pkl`_ * `resources/Master_Edge_List_Dict.json`_ ➞ *automatically created after edge list construction* -* `resources/node_data/node_metadata_dict.pkl `__ ➞ *if adding metadata for new edges to the knowledge graph* +* `resources/metadata/entity_metadata_dict.pkl `__ ➞ *if adding metadata for new edges to the +knowledge graph* * `resources/knowledge_graphs/PheKnowLator_MergedOntologies*.owl`_ ➞ *see* `ontology README`_ *for information* * `resources/relations_data/RELATIONS_LABELS.txt`_ * `resources/relations_data/INVERSE_RELATIONS.txt`_ ➞ *if including inverse relations* @@ -221,7 +223,7 @@ The program can be run locally using the `main.py`_ script or using the `main.ip kg = PartialBuild(kg_version='v2.0.0', write_location='./resources/knowledge_graphs', construction='subclass, - node_data='yes, + metadata='yes, inverse_relations='yes', cpus=available_cpus, decode_owl='yes') @@ -437,7 +439,8 @@ Callahan TJ, Tripodi IJ, Hunter LE, Baumgartner WA. `A Framework for Automated C .. _`resources/Master_Edge_List_Dict.json`: https://www.dropbox.com/s/t8sgzd847t1rof4/Master_Edge_List_Dict.json?dl=1 -.. _`resources/node_data/node_metadata_dict.pkl`: https://github.com/callahantiff/PheKnowLator/blob/master/resources/node_data/README.md +.. _`resources/metadata/entity_metadata_dict.pkl`: https://github +.com/callahantiff/PheKnowLator/blob/master/resources/metadata/README.md .. _`resources/knowledge_graphs/PheKnowLator_MergedOntologies*.owl`: https://www.dropbox.com/s/75lkod7vzpgjdaq/PheKnowLator_MergedOntologiesGeneID_Normalized_Cleaned.owl?dl=1 diff --git a/builds/build_requirements.txt b/builds/build_requirements.txt index 79229c25..22cb615d 100644 --- a/builds/build_requirements.txt +++ b/builds/build_requirements.txt @@ -4,14 +4,15 @@ google==1.9.3 google-api-core==1.24.1 google-api-python-client~=1.7.9 google-cloud-storage==1.28.0 -lxml==4.6.5 +lxml>=4.6.5 networkx==2.4 -numpy==1.21.0 +numpy==1.19.5 openpyxl==3.0.3 oauth2client~=4.1.3 Owlready2==0.25 pandas==1.0.5 python-json-logger==2.0.1 +pyyaml ray~=1.1.0 rdflib==4.2.2 reactome2py==0.0.8 diff --git a/builds/data_preprocessing.py b/builds/data_preprocessing.py index 0d19e221..0a73084c 100755 --- a/builds/data_preprocessing.py +++ b/builds/data_preprocessing.py @@ -5,6 +5,7 @@ # import fnmatch import glob # import itertools +import json import logging.config import networkx # type: ignore import numpy # type: ignore @@ -13,6 +14,7 @@ import pickle import re import requests +import shutil import sys from google.cloud import storage # type: ignore @@ -138,7 +140,7 @@ def _preprocess_hgnc_data(self) -> pandas.DataFrame: 'name', 'location', 'alias_name']] hgnc.rename(columns={'uniprot_ids': 'uniprot_id', 'location': 'map_location', 'locus_type': 'hgnc_gene_type'}, inplace=True) - hgnc['hgnc_id'].replace('.*\:', '', inplace=True, regex=True) # strip 'HGNC' off of the identifiers + hgnc['hgnc_id'] = hgnc['hgnc_id'].str.replace('.*\:', '', regex=True) # strip 'HGNC' off of the identifiers hgnc.fillna('None', inplace=True) # replace NaN with 'None' hgnc['entrez_id'] = hgnc['entrez_id'].apply(lambda x: str(int(x)) if x != 'None' else 'None') # make col str # combine certain columns into single column @@ -150,12 +152,13 @@ def _preprocess_hgnc_data(self) -> pandas.DataFrame: 'name', 'synonyms'], '|') # reformat hgnc gene type for v in self.genomic_type_mapper['hgnc_gene_type'].keys(): - explode_df_hgnc['hgnc_gene_type'].replace(v, self.genomic_type_mapper['hgnc_gene_type'][v], inplace=True) + explode_df_hgnc['hgnc_gene_type'] = explode_df_hgnc['hgnc_gene_type'].str.replace( + v, self.genomic_type_mapper['hgnc_gene_type'][v]) # reformat master hgnc gene type explode_df_hgnc['master_gene_type'] = explode_df_hgnc['hgnc_gene_type'] master_dict = self.genomic_type_mapper['hgnc_master_gene_type'] for val in master_dict.keys(): - explode_df_hgnc['master_gene_type'].replace(val, master_dict[val], inplace=True) + explode_df_hgnc['master_gene_type'] = explode_df_hgnc['master_gene_type'].str.replace(val, master_dict[val]) # post-process reformatted data explode_df_hgnc.drop(['alias_symbol', 'alias_name'], axis=1, inplace=True) # remove original gene type column explode_df_hgnc.drop_duplicates(inplace=True) @@ -189,16 +192,21 @@ def _preprocess_ensembl_data(self) -> pandas.DataFrame: 'ensembl_gene_type', 'transcript_name', 'ensembl_transcript_type']) # reformat ensembl gene type gene_dict = self.genomic_type_mapper['ensembl_gene_type'] - for val in gene_dict.keys(): ensembl_geneset['ensembl_gene_type'].replace(val, gene_dict[val], inplace=True) + for val in gene_dict.keys(): + ensembl_geneset['ensembl_gene_type'] = ensembl_geneset['ensembl_gene_type'].str.replace(val, gene_dict[val]) # reformat master gene type ensembl_geneset['master_gene_type'] = ensembl_geneset['ensembl_gene_type'] gene_dict = self.genomic_type_mapper['ensembl_master_gene_type'] - for val in gene_dict.keys(): ensembl_geneset['master_gene_type'].replace(val, gene_dict[val], inplace=True) + for val in gene_dict.keys(): + ensembl_geneset['master_gene_type'] = ensembl_geneset['master_gene_type'].str.replace(val, gene_dict[val]) # reformat master transcript type - ensembl_geneset['ensembl_transcript_type'].replace('vault_RNA', 'vaultRNA', inplace=True, regex=False) + ensembl_geneset['ensembl_transcript_type'] = ensembl_geneset['ensembl_transcript_type'].str.replace( + 'vault_RNA', 'vaultRNA', regex=False) ensembl_geneset['master_transcript_type'] = ensembl_geneset['ensembl_transcript_type'] trans_d = self.genomic_type_mapper['ensembl_master_transcript_type'] - for val in trans_d.keys(): ensembl_geneset['master_transcript_type'].replace(val, trans_d[val], inplace=True) + for val in trans_d.keys(): + ensembl_geneset['master_transcript_type'] = ensembl_geneset['master_transcript_type'].str.replace( + val, trans_d[val]) # post-process reformatted data ensembl_geneset.drop_duplicates(inplace=True) @@ -226,8 +234,6 @@ def merges_ensembl_mapping_data(self) -> pandas.DataFrame: ensembl_uniprot = ensembl_uniprot.loc[ensembl_uniprot['uniprot_id'].apply(lambda x: '-' not in x)] ensembl_uniprot = ensembl_uniprot.loc[ensembl_uniprot['info_type'].apply(lambda x: x == 'DIRECT')] ensembl_uniprot = ensembl_uniprot.loc[ensembl_uniprot['xref_identity'].apply(lambda x: x != 'None')] - # ensembl_uniprot['master_gene_type'] = ['protein-coding'] * len(ensembl_uniprot) - # ensembl_uniprot['master_transcript_type'] = ['protein-coding'] * len(ensembl_uniprot) ensembl_uniprot.drop(drop_cols, axis=1, inplace=True) ensembl_uniprot.drop_duplicates(subset=None, keep='first', inplace=True) # entrez data @@ -279,7 +285,8 @@ def _preprocess_uniprot_data(self) -> pandas.DataFrame: # explode nested data and perform light value reformatting explode_df_uniprot = explodes_data(uniprot.copy(), ['transcript_stable_id', 'entrez_id', 'hgnc_id'], ';') explode_df_uniprot = explodes_data(explode_df_uniprot.copy(), ['symbol', 'synonyms'], '|') - explode_df_uniprot['transcript_stable_id'].replace('\s.*', '', inplace=True, regex=True) # strip uniprot names + explode_df_uniprot['transcript_stable_id'] = explode_df_uniprot['transcript_stable_id'].str.replace( + '\s.*', '', regex=True) # strip uniprot explode_df_uniprot.drop(['Status'], axis=1, inplace=True) explode_df_uniprot.drop_duplicates(inplace=True) @@ -324,16 +331,18 @@ def _preprocess_ncbi_data(self) -> pandas.DataFrame: explode_df_ncbi_gene['entrez_gene_type'] = explode_df_ncbi_gene['type_of_gene'] gene_dict = self.genomic_type_mapper['entrez_gene_type'] for val in gene_dict.keys(): - explode_df_ncbi_gene['entrez_gene_type'].replace(val, gene_dict[val], inplace=True) + explode_df_ncbi_gene['entrez_gene_type'] = explode_df_ncbi_gene['entrez_gene_type'].str.replace( + val, gene_dict[val]) # reformat master gene type explode_df_ncbi_gene['master_gene_type'] = explode_df_ncbi_gene['entrez_gene_type'] gene_dict = self.genomic_type_mapper['master_gene_type'] for val in gene_dict.keys(): - explode_df_ncbi_gene['master_gene_type'].replace(val, gene_dict[val], inplace=True) + explode_df_ncbi_gene['master_gene_type'] = explode_df_ncbi_gene['master_gene_type'].str.replace( + val, gene_dict[val]) # post-process reformatted data - explode_df_ncbi_gene['hgnc_id'] = explode_df_ncbi_gene['hgnc_id'].replace('HGNC:', '', regex=True) - explode_df_ncbi_gene['ensembl_gene_id'] = explode_df_ncbi_gene['ensembl_gene_id'].replace('Ensembl:', '', - regex=True) + explode_df_ncbi_gene['hgnc_id'] = explode_df_ncbi_gene['hgnc_id'].str.replace('HGNC:', '', regex=True) + explode_df_ncbi_gene['ensembl_gene_id'] = explode_df_ncbi_gene['ensembl_gene_id'].str.replace('Ensembl:', '', + regex=True) explode_df_ncbi_gene.drop(['type_of_gene', 'dbXrefs', 'description', 'Nomenclature_status', 'Modification_date', 'LocusTag', '#tax_id', 'Full_name_from_nomenclature_authority', 'Feature_type', 'Symbol_from_nomenclature_authority'], axis=1, inplace=True) @@ -355,8 +364,8 @@ def _preprocess_protein_ontology_mapping_data(self) -> pandas.DataFrame: pro = self.reads_gcs_bucket_data_to_df(f_name='promapping.txt', delm='\t', head=col_names) pro = pro.loc[pro['Entry'].apply(lambda x: x.startswith('UniProtKB:') and '_VAR' not in x and ', ' not in x)] pro = pro.loc[pro['pro_mapping'].apply(lambda x: x.startswith('exact'))] - pro['pro_id'].replace('PR:', 'PR_', inplace=True, regex=True) # replace PR: with PR_ - pro['Entry'].replace('(^\w*\:)', '', inplace=True, regex=True) # remove ids which appear before ':' + pro['pro_id'] = pro['pro_id'].str.replace('PR:', 'PR_', regex=True) # replace PR: with PR_ + pro['Entry'] = pro['Entry'].str.replace('(^\w*\:)', '', regex=True) # remove ids which appear before ':' pro = pro.loc[pro['pro_id'].apply(lambda x: '-' not in x)] # remove isoforms pro.rename(columns={'Entry': 'uniprot_id'}, inplace=True) pro.drop(['pro_mapping'], axis=1, inplace=True); pro.drop_duplicates(subset=None, keep='first', inplace=True) @@ -412,12 +421,14 @@ def _fixes_genomic_symbols(self) -> pandas.DataFrame: else: clean_dates.append(x) merged_data['symbol'] = clean_dates; merged_data.fillna('None', inplace=True) # make sure that all gene and transcript type columns have none recoded to unknown or not protein-coding - merged_data['hgnc_gene_type'].replace('None', 'unknown', inplace=True, regex=False) - merged_data['ensembl_gene_type'].replace('None', 'unknown', inplace=True, regex=False) - merged_data['entrez_gene_type'].replace('None', 'unknown', inplace=True, regex=False) - merged_data['master_gene_type'].replace('None', 'unknown', inplace=True, regex=False) - merged_data['master_transcript_type'].replace('None', 'not protein-coding', inplace=True, regex=False) - merged_data['ensembl_transcript_type'].replace('None', 'unknown', inplace=True, regex=False) + merged_data['hgnc_gene_type'] = merged_data['hgnc_gene_type'].str.replace('None', 'unknown', regex=False) + merged_data['ensembl_gene_type'] = merged_data['ensembl_gene_type'].str.replace('None', 'unknown', regex=False) + merged_data['entrez_gene_type'] = merged_data['entrez_gene_type'].str.replace('None', 'unknown', regex=False) + merged_data['master_gene_type'] = merged_data['master_gene_type'].str.replace('None', 'unknown', regex=False) + merged_data['master_transcript_type'] = merged_data['master_transcript_type'].str.replace( + 'None', 'not protein-coding', regex=False) + merged_data['ensembl_transcript_type'] = merged_data['ensembl_transcript_type'].str.replace( + 'None', 'unknown', regex=False) merged_data_clean = merged_data.drop_duplicates() return merged_data_clean @@ -496,6 +507,50 @@ def creates_master_genomic_identifier_map(self) -> Dict: return reformatted_mapped_identifiers + def _write_genomic_entity_metadata(self): + """Process the dictionary created in the prior steps in order to assist with creating a master metadata file + for all nodes that are a genomic entity (i.e., genes, transcripts, or proteins). + """ + + reformatted_mapped_identifiers = self.creates_master_genomic_identifier_map() + out_location = self.temp_dir + '/GENOMIC_ENTITY_METADATA.jsonl' + + for key, value in tqdm(reformatted_mapped_identifiers.items()): + old_prefix = '_'.join(key.split('_')[0:-1]); idx = key.split('_')[-1]; pass_var = True; new_prefix = None + if old_prefix == 'entrez_id': new_prefix = 'NCBIGene' + elif old_prefix in ['ensembl_gene_id', 'protein_stable_id', 'transcript_stable_id']: new_prefix = 'ensembl' + elif old_prefix == 'pro_id': new_prefix = 'PR' + else: pass_var = False + if pass_var and new_prefix is not None: + updated_key = new_prefix + ':' + idx; master_metadata_dict = {updated_key: {}} + for x in value: + i, j = '_'.join(x.split('_')[0:-1]), x.split('_')[-1] + if 'type' in i: continue + elif i == 'entrez_id': new_i = 'NCBIGene'; j = new_i + ':' + j + elif i == 'ensembl_gene_id': new_i = 'ensembl gene'; j = 'ensembl:' + j + elif i == 'protein_stable_id': new_i = 'ensembl protein'; j = 'ensembl:' + j + elif i == 'transcript_stable_id': new_i = 'ensembl transcript'; j = 'ensembl:' + j + elif i == 'pro_id_PR': new_i = 'PR'; j = new_i + ':' + j + elif i == 'hgnc_id': new_i = 'HGNC_ID'; j = new_i + ':' + j + elif i == 'uniprot_id': new_i = 'uniprot'; j = new_i + ':' + j + elif i == 'symbol': new_i = 'GeneSymbol'; j = new_i + ':' + j + else: + if i == 'synonyms': new_i = 'Synonyms' + elif i == 'name': new_i = 'Label' + elif i == 'Other_designations': new_i = 'Synonyms'; j = j.split('|') + else: new_i = i + if new_i in master_metadata_dict[updated_key].keys(): + if isinstance(j, list): master_metadata_dict[updated_key][new_i] += j + else: master_metadata_dict[updated_key][new_i] += [j] + else: master_metadata_dict[updated_key][new_i] = [j] + # write entry + dump_jsonl([master_metadata_dict], out_location) + + # load data to cloud + uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, '/GENOMIC_ENTITY_METADATA.jsonl') + + return None + def generates_specific_genomic_identifier_maps(self) -> None: """Method takes a list of information needed to create mappings between specific sets of genomic identifiers. @@ -509,23 +564,35 @@ def generates_specific_genomic_identifier_maps(self) -> None: reformatted_mapped_identifiers = self.creates_master_genomic_identifier_map() gene_sets = [ ['ENSEMBL_GENE_ENTREZ_GENE_MAP.txt', 'ensembl_gene_id', 'entrez_id', 'ensembl_gene_type', - 'entrez_gene_type', 'gene_type_update', 'gene_type_update', False, False], + 'entrez_gene_type', 'gene_type_update', 'gene_type_update', [1, 1, 'NCBIGene_']], ['ENSEMBL_TRANSCRIPT_PROTEIN_ONTOLOGY_MAP.txt', 'transcript_stable_id', 'pro_id', 'ensembl_transcript_type', - None, 'transcript_type_update', None, False, True], + None, 'transcript_type_update', None, [0, 5, 'ensembl_']], ['ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt', 'entrez_id', 'transcript_stable_id', 'entrez_gene_type', - 'ensembl_transcript_type', 'gene_type_update', 'transcript_type_update', False, False], + 'ensembl_transcript_type', 'gene_type_update', 'transcript_type_update', [0, 7, 'NCBIGene_'], + [1, 1, 'ensembl_']], ['ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt', 'entrez_id', 'pro_id', 'entrez_gene_type', None, 'gene_type_update', - None, False, True], + None, [0, 5, 'NCBIGene_']], ['GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt', 'symbol', 'transcript_stable_id', 'master_gene_type', - 'ensembl_transcript_type', 'gene_type_update', 'transcript_type_update', False, False], - ['STRING_PRO_ONTOLOGY_MAP.txt', 'protein_stable_id', 'pro_id', None, None, None, None, False, True], - ['UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt', 'uniprot_id', 'pro_id', None, None, None, None, False, True] + 'ensembl_transcript_type', 'gene_type_update', 'transcript_type_update', [1, 1, 'ensembl_']], + ['STRING_PRO_ONTOLOGY_MAP.txt', 'protein_stable_id', 'pro_id', None, None, None, None, [0, 0, '9606.']], + ['UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt', 'uniprot_id', 'pro_id', None, None, None, None, None], + ['UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt', 'uniprot_id', 'entrez_id', None, 'master_gene_type', None, + 'gene_type_update', [1, 1, 'NCBIGene_']] ] for x in gene_sets: genomic_id_mapper(reformatted_mapped_identifiers, self.temp_dir + '/' + x[0], # type: ignore x[1], x[2], x[3], x[4], x[5], x[6]) # type: ignore + + if x[-1] is not None: + df = pandas.read_csv(self.temp_dir + '/' + x[0], header=None, delimiter='\t', low_memory=False) + for i in x[7:]: + df[i[1]] = i[2] + df[i[0]].astype(str) + df = df.replace('None', numpy.nan).dropna(axis=1, how="all") + df.to_csv(self.temp_dir + '/' + x[0], header=None, sep='\t', index=False) + uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, x[0]) # type: ignore + self._write_genomic_entity_metadata() # write genomic metadata return None @@ -619,12 +686,12 @@ def creates_chebi_to_mesh_identifier_mappings(self) -> None: # write results and push data to gcs bucket filename = 'MESH_CHEBI_MAP.txt' with open(self.temp_dir + '/' + filename, 'w') as out: - for pair in mesh_edges: out.write(pair[0] + '\t' + pair[1] + '\n') + for pair in mesh_edges: out.write(pair[0].replace('_', ':') + '\t' + pair[1] + '\n') uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename) return None - def _preprocess_mondo_mapping_data(self) -> Dict: + def _preprocess_mondo_mapping_data(self) -> pandas.DataFrame: """Method processes MonDO Disease Ontology (MONDO) ontology data in order to create a dictionary that aligns MONDO concepts with other types of disease terminology identifiers. This is done by obtaining database cross-references (dbxrefs) for each ontology and then combining the results into a single large dictionary @@ -642,9 +709,23 @@ def _preprocess_mondo_mapping_data(self) -> Dict: mondo_dict = {str(k).lower().split('/')[-1]: {str(i).split('/')[-1].replace('_', ':') for i in v} for k, v in dbxref_res.items() if 'MONDO' in str(v)} - return mondo_dict - - def _preprocess_hpo_mapping_data(self) -> Dict: + # convert to pandas DataFrame + temp_list = [] + for k, v in mondo_dict.items(): + if k.startswith('umls:'): new_k = k.split(':')[-1].upper() + elif k.startswith('hp:'): new_k = k.upper() + elif k.startswith('mesh:'): new_k = 'MESH:' + k.split(':')[-1].upper() + elif k.startswith('orphanet:'): new_k = 'ORPHA:' + k.split(':')[-1].upper() + elif k.startswith('omimps:'): new_k = 'OMIM:' + k.split(':')[-1].upper() + else: new_k = k + for i in v: + temp_list += [[new_k, i.replace(':', '_')]]; temp_list += [[i, i.replace(':', '_')]] + # convert to + mondo_df = pandas.DataFrame({'other_id': [x[0] for x in temp_list], 'ontology_id': [x[1] for x in temp_list]}) + + return mondo_df + + def _preprocess_hpo_mapping_data(self) -> pandas.DataFrame: """Method processes Human Phenotype Ontology (HPO) ontology data in order to create a dictionary that aligns HPO concepts with other types of disease terminology identifiers. This is done by obtaining database cross-references (dbxrefs) for each ontology and then combining the results into a single large dictionary @@ -662,12 +743,89 @@ def _preprocess_hpo_mapping_data(self) -> Dict: hp_dict = {str(k).lower().split('/')[-1]: {str(i).split('/')[-1].replace('_', ':') for i in v} for k, v in dbxref_res.items() if 'HP' in str(v)} - return hp_dict + # convert to pandas DataFrame + temp_list = [] + for k, v in hp_dict.items(): + if k.startswith('umls:'): new_k = k.split(':')[-1].upper() + elif k.startswith('mondo:'): new_k = k.upper() + elif k.startswith('msh:'): new_k = 'MESH:' + k.split(':')[-1].upper() + elif k.startswith('orpha:'): new_k = 'ORPHA:' + k.split(':')[-1].upper() + else: new_k = k + for i in v: + temp_list += [[new_k, i.replace(':', '_')]]; temp_list += [[i, i.replace(':', '_')]] + # convert to + hp_df = pandas.DataFrame({'other_id': [x[0] for x in temp_list], 'ontology_id': [x[1] for x in temp_list]}) + + return hp_df + + def reads_disgenet_data(self) -> pandas.DataFrame: + """Reads in disease mapping data from DisGeNET. + + Returns: + data: A pandas DataFrame object. + """ + + data = self.reads_gcs_bucket_data_to_df(f_name='disease_mappings.tsv', delm='\t', head=0) + + # reformat data + data['vocabulary'] = data['vocabulary'].str.lower() + data['diseaseId'] = data['diseaseId'].str.lower() + data['vocabulary'] = data['vocabulary'].str.replace('hpo', 'HP') + data['vocabulary'] = data['vocabulary'].str.replace('mondo', 'MONDO') + data['vocabulary'] = data['vocabulary'].str.replace('msh', 'MESH') + data['vocabulary'] = data['vocabulary'].str.replace('omim', 'OMIM') + data['vocabulary'] = data['vocabulary'].str.replace('do', 'doid') + data['vocabulary'] = data['vocabulary'].str.replace('ordo', 'ORPHA') + data['vocabulary'] = data['vocabulary'].str.replace('ORPHAid', 'ORPHA') + # capitalize UMLS id + data['diseaseId'] = data['diseaseId'].str.upper() + # create a disease code column + data['code'] = data['vocabulary'] + ':' + data['code'] + data['code'] = data['code'].str.replace('HP:HP:', 'HP:') + # rename columns + data.rename(columns={'diseaseId': 'cui', 'vocabularyName': 'code_name'}, inplace=True) + # remove unneeded columns + data = data[['cui', 'code', 'code_name', 'vocabulary']].drop_duplicates() + + return data + + def reads_medgen_data(self) -> pandas.DataFrame: + """Reads in disease mapping data from MedGen. + + Returns: + data: A pandas DataFrame object. + """ + + data = self.reads_gcs_bucket_data_to_df(f_name='MGCONSO.RRF', delm='|', head=0) + + # reformat data + data = data[data['SUPPRESS'] == 'N'].drop_duplicates() + data = data[data['SAB'].isin(['HPO', 'MONDO', 'MSH', 'ORDO', 'OMIM'])].drop_duplicates() + # reformat codes + data['temp_code'] = data.apply(lambda x: 'MESH:' + x['CODE'] if x['SAB'] == 'MSH' + else 'OMIM:' + x['CODE'] if x['SAB'] == 'OMIM' + else 'ORPHA:' + x['SDUI'].split('_')[-1] if x['SAB'] == 'ORDO' + else x['SDUI'] if x['SAB'] == 'HPO' + else x['SDUI'] if x['SAB'] == 'MONDO' + else 'None', axis=1) + # add rows for MedGen identifiers + temp = data[['#CUI']]; temp['temp_code'] = 'MedGen:' + data['#CUI'] + data = pandas.concat([data, temp]) + # remove unneeded columns + data = data[['#CUI', 'temp_code', 'STR', 'SAB']].drop_duplicates() + # rename columns + data.rename(columns={'#CUI': 'cui', 'STR': 'code_name', + 'temp_code': 'code', 'SAB': 'vocabulary'}, inplace=True) + # reformat vocabulary ids + data['vocabulary'] = data['vocabulary'].str.replace('HPO', 'HP') + data['vocabulary'] = data['vocabulary'].str.replace('MSH', 'MESH') + + return data def creates_disease_identifier_mappings(self) -> None: """Creates Human Phenotype Ontology (HPO) and MonDO Disease Ontology (MONDO) dbxRef maps and then uses them - with the DisGEeNET UMLS disease mappings to create a master mapping between all disease identifiers to HPO - and MONDO. + with the DisGEeNET UMLS and MedGen disease mappings to create a master mapping between all disease identifiers + to HPO and MONDO. Returns: None. @@ -675,44 +833,42 @@ def creates_disease_identifier_mappings(self) -> None: log_str = 'Creating Phenotype and Disease ID Cross-Map Data'; print(log_str); logger.info(log_str) - mondo_dict, hp_dict = self._preprocess_mondo_mapping_data(), self._preprocess_hpo_mapping_data() - data = self.reads_gcs_bucket_data_to_df(f_name='disease_mappings.tsv', delm='\t', head=0) - data['vocabulary'], data['diseaseId'] = data['vocabulary'].str.lower(), data['diseaseId'].str.lower() - data['vocabulary'] = ['doid' if x == 'do' else 'ordoid' if x == 'ordo' else x for x in data['vocabulary']] - # get all CUIs mapped to HPO and MONDO - ont_dict: Dict = {}; disease_data_keep = data.query('vocabulary == "hpo" | vocabulary == "mondo"') - for idx, row in tqdm(disease_data_keep.iterrows(), total=disease_data_keep.shape[0]): - if row['vocabulary'] == 'mondo': key, value = 'umls:' + row['diseaseId'], 'MONDO:' + row['code'] - else: key, value = 'umls:' + row['diseaseId'], row['code'] - if key in ont_dict.keys(): ont_dict[key] |= {value} - else: ont_dict[key] = {value} - for key in tqdm(ont_dict.keys()): # add ontology mappings from MONDO and HPO - if key in mondo_dict.keys(): ont_dict[key] = set(list(ont_dict[key]) + list(mondo_dict[key])) - if key in hp_dict.keys(): ont_dict[key] = set(list(ont_dict[key]) + list(hp_dict[key])) - # get all rows for HPO/MONDO CUIs to obtain mappings to other disease identifiers - disease_dict: Dict = {}; disease_data_other = data[data.diseaseId.isin(disease_data_keep['diseaseId'])] - for idx, row in tqdm(disease_data_other.iterrows(), total=disease_data_other.shape[0]): - vocab, ids, code = row['vocabulary'], row['diseaseId'], row['code'] - if vocab == 'mondo' or vocab == 'hpo': - key, value = 'umls:' + ids.lower(), code - if key in disease_dict.keys(): disease_dict[key] |= {value} - else: disease_dict[key] = {value} - else: - if 'mondo' not in code or 'hp' not in code: - if ':' not in code: key, value = vocab + ':' + code, ont_dict['umls:' + ids] - else: key, value = code, ont_dict['umls:' + ids] - if key in disease_dict.keys(): disease_dict[key] |= value - else: disease_dict[key] = value - # save data and push to GCS bucket - file1, file2 = 'DISEASE_MONDO_MAP.txt', 'PHENOTYPE_HPO_MAP.txt' - with open(self.temp_dir + '/' + file1, 'w') as out1, open(self.temp_dir + '/' + file2, 'w') as out2: - for k, v in tqdm({**disease_dict, **mondo_dict, **hp_dict}.items()): - if any(x for x in v if x.startswith('MONDO')): - for idx in [x.replace(':', '_') for x in v if 'MONDO' in x]: - out1.write(k.upper().split(':')[-1] + '\t' + idx + '\n') - if any(x for x in v if x.startswith('HP')): - for idx in [x.replace(':', '_') for x in v if 'HP' in x]: - out2.write(k.upper().split(':')[-1] + '\t' + idx + '\n') + disease_map_df = pandas.concat([self._preprocess_mondo_mapping_data(), self._preprocess_hpo_mapping_data()]) + disease_data = pandas.concat([self.reads_disgenet_data(), self.reads_medgen_data()]).drop_duplicates() + + # find cuis that map to HP or MONDO + disease_data_keep = disease_data.copy() + disease_data_keep = disease_data_keep.query('vocabulary == "HP" | vocabulary == "MONDO"') + disease_data_keep = disease_data_keep[['cui', 'code']] + cui_list = set(disease_data_keep['cui']) + # obtain a list of other ids that map to the cuis + temp_df = disease_data[disease_data['cui'].isin(cui_list)] + # merge back with original data and rename the columns + merged_temp = temp_df.merge(disease_data_keep, on='cui') + merged_temp = merged_temp[['code_x', 'code_y', 'code_name', 'vocabulary']].drop_duplicates() + merged_temp.rename(columns={'code_x': 'cui', 'code_y': 'code'}, inplace=True) + # combine the columns back to main data + disease_mapping_data = pandas.concat([disease_data, merged_temp]).drop_duplicates() + disease_mapping_data = disease_mapping_data[['cui', 'code']].drop_duplicates() + # merge ontology and other mappings together and clean up file + cleaned_disease_map = disease_mapping_data.merge(disease_map_df, left_on='cui', right_on='other_id') + cleaned_disease_map = cleaned_disease_map[['cui', 'ontology_id']] + cleaned_disease_map.rename(columns={'cui': 'disease_id'}, inplace=True) + # format ontology identifiers + cleaned_disease_map['ontology_id'] = cleaned_disease_map['ontology_id'].str.replace(':', '_') + cleaned_disease_map['vocabulary'] = cleaned_disease_map['ontology_id'].str.replace('\_.*', '', regex=True) + cleaned_disease_map.drop_duplicates(inplace=True) + + # write data + # split data by ontology and write to file + mondo_map = cleaned_disease_map[cleaned_disease_map['vocabulary'] == 'MONDO'].drop_duplicates() + hp_map = cleaned_disease_map[cleaned_disease_map['vocabulary'] == 'HP'].drop_duplicates() + mondo_map = mondo_map[['disease_id', 'ontology_id']]; hp_map = hp_map[['disease_id', 'ontology_id']] + + # write data + file1 = 'DISEASE_MONDO_MAP.txt'; file2 = 'PHENOTYPE_HPO_MAP.txt' + mondo_map.to_csv(self.temp_dir + '/' + file1, header=None, index=False, sep='\t') + hp_map.to_csv(self.temp_dir + '/' + file2, header=None, index=False, sep='\t') uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, file1) uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, file2) @@ -759,63 +915,123 @@ def _extracts_hpa_tissue_information(self) -> pandas.DataFrame: filename = 'HPA_tissues.txt' with open(self.temp_dir + '/' + filename, 'w') as outfile: for x in tqdm(list(hpa.columns)): - if x.endswith('[NX]'): outfile.write(x.split('RNA - ')[-1].split(' [NX]')[:-1][0] + '\n') + if x.endswith('[nTPM]'): outfile.write(x.split('RNA - ')[-1].split(' [nTPM]')[:-1][0] + '\n') uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename) return hpa - def processes_hpa_gtex_data(self) -> None: - """Method processes and combines gene expression experiment results from the Human protein Atlas (HPA) and the - Genotype-Tissue Expression Project (GTEx). Additional details provided below on how each source are processed. - - HPA: The HPA data is reformatted so all tissue, cell, cell lines, and fluid types are stored as a nested - list. The anatomy type is specified as an item in the list according to its type. - - GTEx: All protein-coding genes that appear in the HPA data set are removed. Then, only those non-coding - genes with a median expression >= 1.0 are maintained. GTEx data are formatted such the anatomical - entities are stored as columns and genes stored as rows, thus the expression filtering step is - performed while also reformatting the file, resulting in a nested list. + def _processes_hpa_data(self) -> Union[List, pandas.DataFrame]: + """The HPA data is reformatted so all tissue, cell, cell lines, and fluid types are stored as a nested list. + The anatomy type is specified as an item in the list according to its type. Returns: - None. + hpa_results: A nested list of processed HPA data. """ - log_str = 'Creating Human Protein Atlas and GTEx Cross-Map Data'; print(log_str); logger.info(log_str) + hpa = self._extracts_hpa_tissue_information() - hpa = self._extracts_hpa_tissue_information(); f_name = 'GTEx_Analysis_*_RNASeQC*_gene_median_tpm.gct' - gtex = self.reads_gcs_bucket_data_to_df(f_name=f_name, delm='\t', skip=2, head=0) - gtex.fillna('None', inplace=True); gtex['Name'].replace('(\..*)', '', inplace=True, regex=True) # process human protein atlas data hpa_results = [] for idx, row in tqdm(hpa.iterrows(), total=hpa.shape[0]): - ens, gene, uniprot, evid = str(row['Ensembl']), str(row['Gene']), str(row['Uniprot']), str(row['Evidence']) - if row['RNA tissue specific NX'] != 'None': - for x in row['RNA tissue specific NX'].split(';'): - hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]] - if row['RNA cell line specific NX'] != 'None': - for x in row['RNA cell line specific NX'].split(';'): - hpa_results += [[ens, gene, uniprot, evid, 'cell line', str(x.split(':')[0])]] - if row['RNA brain regional specific NX'] != 'None': - for x in row['RNA brain regional specific NX'].split(';'): - hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]] - if row['RNA blood cell specific NX'] != 'None': - for x in row['RNA blood cell specific NX'].split(';'): - hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]] - if row['RNA blood lineage specific NX'] != 'None': - for x in row['RNA blood lineage specific NX'].split(';'): - hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]] + ens = str(row['Ensembl']); gene = str(row['Gene']); uni = str(row['Uniprot']) + evid = str(row['Evidence']); sub = str(row['Subcellular location']); source = 'The Human Protein Atlas' + if row['RNA tissue specific nTPM'] != 'None': + row_val = row['RNA tissue specific nTPM'] + if ';' in row_val: + for x in row_val.split(';'): + x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]] + else: + x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]] + if row['RNA cell line specific nTPM'] != 'None': + row_val = row['RNA cell line specific nTPM'] + if ';' in row_val: + for x in row_val.split(';'): + x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + else: + x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + if row['RNA brain regional specific nTPM'] != 'None': + row_val = row['RNA brain regional specific nTPM'] + if ';' in row_val: + for x in row_val.split(';'): + x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]] + else: + x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]] + if row['RNA blood cell specific nTPM'] != 'None': + row_val = row['RNA blood cell specific nTPM'] + if ';' in row_val: + for x in row_val.split(';'): + x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + else: + x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + if row['RNA blood lineage specific nTPM'] != 'None': + row_val = row['RNA blood lineage specific nTPM'] + if ';' in row_val: + for x in row_val.split(';'): + x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + else: + x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1]) + hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]] + + return hpa_results, hpa + + def _processes_gtex_data(self, hpa_df: pandas.DataFrame) -> List: + """All protein-coding genes that appear in the HPA data set are removed. Then, only those non-coding genes + with a median expression >= 1.0 are maintained. GTEx data are formatted such the anatomical entities are + stored as columns and genes stored as rows, thus the expression filtering step is performed while also + reformatting the file, resulting in a nested list. + + Args: + hpa_df: A Pandas DataFrame containining HPA data. + + Returns: + gtex_results: A nested list of processed HPA data. + """ + + f_name = 'GTEx_Analysis_*_RNASeQC*_gene_median_tpm.gct' + gtex = self.reads_gcs_bucket_data_to_df(f_name=f_name, delm='\t', skip=2, head=0) + gtex.fillna('None', inplace=True); gtex['Name'] = gtex['Name'].str.replace('(\..*)', '', regex=True) + # process gtex data -- using only those protein-coding genes not already in hpa - gtex_results, hpa_genes = [], list(hpa['Ensembl'].drop_duplicates(keep='first', inplace=False)) + gtex_results, hpa_genes = [], list(hpa_df['Ensembl'].drop_duplicates(keep='first', inplace=False)) gtex = gtex.loc[gtex['Name'].apply(lambda i: i not in hpa_genes)] + # loop over data and re-organize + source = 'Genotype-Tissue Expression (GTEx) Project' for idx, row in tqdm(gtex.iterrows(), total=gtex.shape[0]): for col in list(gtex.columns)[2:]: - typ = 'cell line' if 'Cells' in col else 'anatomy' - if row[col] >= 1.0: - evidence = 'Evidence at transcript level' - gtex_results += [[str(row['Name']), str(row['Description']), 'None', evidence, typ, str(col)]] + typ = 'cell line' if 'Cells' in col else 'anatomy'; evid = 'Evidence at transcript level' + gtex_results += [[str(row['Name']), str(row['Description']), + 'None', evid, typ, 'None', col, float(row[col]), source]] + + return gtex_results + + def processes_hpa_gtex_data(self) -> None: + """Method processes and combines gene expression experiment results from the Human protein Atlas (HPA) and the + Genotype-Tissue Expression Project (GTEx). Additional details provided below on how each source are processed. + + Returns: + None. + """ + + log_str = 'Creating Human Protein Atlas and GTEx Cross-Map Data'; print(log_str); logger.info(log_str) + + hpa_results, hpa = self._processes_hpa_data() + gtex_results = self._processes_gtex_data(hpa) + # write results filename = 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt' with open(self.temp_dir + '/' + filename, 'w') as out: - for x in hpa_results + gtex_results: - out.write(x[0] + '\t' + x[1] + '\t' + x[2] + '\t' + x[3] + '\t' + x[4] + '\t' + x[5] + '\n') + for x in tqdm(hpa_results + gtex_results): + out.write(x[0] + '\t' + x[1] + '\t' + x[2] + '\t' + x[3] + '\t' + x[4] + '\t' + x[5] + '\t' + + x[6] + '\t' + str(x[7]) + '\t' + x[8] + '\n') uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename) return None @@ -1259,6 +1475,119 @@ def processes_relation_ontology_data(self) -> None: return None + def _processes_variant_summary_data(self) -> pandas.DataFrame: + """Data from ClinVar (variant_summary) is downloaded and the file is cleaned to handle missing data, unneeded + variables are removed, identifiers and date fields are cleaned and reformatted, and rows without valid + disease/phenotype identifiers are removed. + + Returns: + var_summary_update: A Pandas DataFrame containing processed clinvar data. + """ + + var_summary = self.reads_gcs_bucket_data_to_df(f_name='variant_summary.txt', delm='\t', head=0) + + # replace "na" and "-" with NaN + var_summary = var_summary.replace('na', numpy.nan); var_summary = var_summary.replace('-', numpy.nan) + # handle ids that are coded as missing (i.e., -1) + var_summary['GeneID'] = var_summary['GeneID'].replace(-1, numpy.nan) + var_summary['RS# (dbSNP)'] = var_summary['RS# (dbSNP)'].replace(-1, numpy.nan) + # convert date format + var_summary['LastEvaluated'] = var_summary['LastEvaluated'].str.replace('None', '') + var_summary['LastEvaluated'] = pandas.to_datetime(var_summary['LastEvaluated']) + var_summary['LastEvaluated'] = var_summary['LastEvaluated'].dt.strftime('%B %d, %Y') + var_summary['LastEvaluated'] = var_summary['LastEvaluated'].replace('', numpy.nan) + # rename variables + var_summary.rename(columns={'#AlleleID': 'AlleleID', 'nsv/esv (dbVar)': 'nsv', 'Name': 'VariantName'}, + inplace=True) + # update variable types + var_summary['GeneID'] = var_summary['GeneID'].astype('Int64') + var_summary['RS# (dbSNP)'] = var_summary['RS# (dbSNP)'].astype('Int64') + + # subset df to process multiple assembly entries + var_summary_update_assemb = var_summary.copy() + var_summary_update_assemb = var_summary_update_assemb[['VariationID', 'Assembly', 'ChromosomeAccession', + 'Chromosome', 'Start', 'Stop', 'ReferenceAllele', + 'AlternateAllele', 'Cytogenetic', + 'PositionVCF']].drop_duplicates() + # identify columns to process + assemb_cols = ['ChromosomeAccession', 'Chromosome', 'Start', 'Stop', 'ReferenceAllele', + 'AlternateAllele', 'Cytogenetic', 'PositionVCF', 'ReferenceAlleleVCF', 'AlternateAlleleVCF'] + # group data by variant + df = var_summary_update_assemb.fillna('None') + df = df.groupby('VariationID').apply( + lambda g: str(g.drop(['VariationID'], axis=1).to_dict('records'))).to_dict() + # convert to Pandas DataFrame + df_items = df.items() + temp_df = pandas.DataFrame({'VariationID': [x[0] for x in df_items], 'Assembly': [x[1] for x in df_items]}) + # join temp df with original data + var_summary_assemb = var_summary.copy().drop(assemb_cols + ['Assembly'], axis=1) + var_summary_update = var_summary_assemb.merge(temp_df, on='VariationID', how='left') + var_summary_update.drop_duplicates(inplace=True) # drop duplicates + + # process and clean up phenotype identifiers + var_summary_update['Phenotype'] = var_summary_update['PhenotypeIDS'].str.replace('|', ';').str.replace(',', ';') + var_summary_update['OtherIDs'] = var_summary_update['OtherIDs'].str.replace(';', '|').str.replace(',', '|') + # remove unneeded variables + drop_list = ['PhenotypeList', 'PhenotypeIDS'] + var_summary_update = var_summary_update.drop(drop_list, axis=1).drop_duplicates() + # replace NaN with 'None' + var_summary_update['Phenotype'] = var_summary_update['Phenotype'].fillna('None') + # reformat phenotypeIDS and trim leading whitespace from unnested columns + var_summary_update['Phenotype'] = var_summary_update['Phenotype'].apply( + lambda x: ';'.join(set(x for x in ['MONDO:' + i.split(':')[-1] if i.startswith('MONDO') + else 'HP:' + i.split(':')[-1] if i.startswith('Human Phenotype') + else 'ORPHA:' + i.split(':')[-1] if i.startswith('Orphanet') + else 'None' if i.endswith(' conditions') + else i for i in x.split(';')] if x != 'None'))) + var_summary_update.drop_duplicates(inplace=True) # drop duplicates + + return var_summary_update + + def _processes_var_citation_data(self) -> pandas.DataFrame: + """Data from ClinVar (var_citations) is downloaded and the file is cleaned to handle missing data, unneeded + variables are removed, and a new citation field is created. + + Returns: + var_citations: A Pandas DataFrame containing processed clinvar data. + """ + + var_citations = self.reads_gcs_bucket_data_to_df(f_name='var_citations.txt', delm='\t', head=0) + + # replace "na" and "-" with NaN + var_citations = var_citations.replace('na', numpy.nan); var_citations = var_citations.replace('-', numpy.nan) + # combine citation information + var_citations['Citation'] = var_citations['citation_source'] + ':' + var_citations['citation_id'] + # remove unneeded variables + drop_list = ['citation_source', 'citation_id'] + var_citations = var_citations.drop(drop_list, axis=1).drop_duplicates() + # group data by citations + var_citations = var_citations.groupby('VariationID').Citation.agg([('Citation', '|'.join)]).reset_index() + var_citations = var_citations.drop_duplicates().sort_values(by=['VariationID']) + + return var_citations + + def _processes_allele_gene_data(self) -> pandas.DataFrame: + """Data from ClinVar (allele_gene) is downloaded and the file is cleaned to handle missing data, unneeded + variables are removed, and the file is reduced to only contain a subset of relevant variables. + + Returns: + allele_gene: A Pandas DataFrame containing processed clinvar data. + """ + + allele_gene = self.reads_gcs_bucket_data_to_df(f_name='allele_gene.txt', delm='\t', head=0) + + # replace "na" and "-" with NaN + allele_gene = allele_gene.replace('na', numpy.nan) + allele_gene = allele_gene.replace('-', numpy.nan) + # handle gene ids that may be coded as -1 + allele_gene['GeneID'] = allele_gene['GeneID'].replace(-1, numpy.nan) + # rename variables + allele_gene.rename(columns={'#AlleleID': 'AlleleID', 'Symbol': 'GeneSymbol', 'Name': 'GeneName'}, inplace=True) + # update variable types + allele_gene['GeneID'] = allele_gene['GeneID'].astype('Int64') + + return allele_gene + def processes_clinvar_data(self) -> None: """Processes ClinVar data by performing light tidying and filtering and then outputs data needed to create mappings between genes, variants, and phenotypes. @@ -1269,16 +1598,48 @@ def processes_clinvar_data(self) -> None: log_str = 'Generating ClinVar Cross-Mapping Data'; print(log_str); logger.info(log_str) - f_name = 'variant_summary.txt' - clinvar_data = self.reads_gcs_bucket_data_to_df(f_name=f_name, delm='\t', head=0) - clinvar_data.fillna('None', inplace=True) - # explode nested data - explode_df_clinvar = explodes_data(clinvar_data.copy(), ['PhenotypeIDS'], ';') - explode_df_clinvar = explodes_data(explode_df_clinvar.copy(), ['PhenotypeIDS'], ',') - explode_df_clinvar['PhenotypeIDS'].replace('Orphanet:ORPHA', 'ORPHA:', inplace=True, regex=True) - explode_df_clinvar['PhenotypeIDS'].replace('Human Phenotype Ontology:HP:', 'HP_', inplace=True, regex=True) - filename = 'CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt' - explode_df_clinvar.to_csv(self.temp_dir + '/' + filename, sep='\t', encoding='utf-8', index=False) + # obtain processed data sets + var_summary_update = self._processes_variant_summary_data() + var_citations = self._processes_var_citation_data() + allele_gene = self._processes_allele_gene_data() + + # merge var_summary and var_citation data + merge_cols = list(set(var_summary_update.columns).intersection(set(var_citations.columns))) + var_summary_merged = var_summary_update.merge(var_citations, on=merge_cols, how='left') + # added allele_gene data + merge_cols = list(set(var_summary_merged.columns).intersection(set(allele_gene.columns))) + var_summary_merged = var_summary_merged.merge(allele_gene, on=merge_cols, how='left') + var_summary_merged['GenesPerAlleleID'] = var_summary_merged['GenesPerAlleleID'].astype('Int64') + # reduce data set to extract variant gene edges + var_summary_merged_gene = var_summary_merged.copy() + var_summary_merged_gene = var_summary_merged_gene[[ + 'VariationID', 'AlleleID', 'RS# (dbSNP)', 'Type', 'VariantName', 'OtherIDs', 'GeneID', 'GeneSymbol', + 'GeneName', 'GenesPerAlleleID', 'Assembly', 'Category', 'Guidelines', 'TestedInGTR', 'RCVaccession', + 'LastEvaluated', 'ReviewStatus', 'ClinicalSignificance', 'ClinSigSimple', 'Origin', 'OriginSimple', + 'Source', 'SubmitterCategories', 'NumberSubmitters', 'Citation']] + var_summary_merged_gene.drop_duplicates(inplace=True) + var_summary_merged_gene = var_summary_merged_gene.dropna(subset=['GeneID']) + var_summary_merged_gene['GeneID'] = 'NCBIGene_' + var_summary_merged_gene['GeneID'].astype(str) + var_summary_merged_gene['VariationID'] = 'clinvar_' + var_summary_merged_gene['VariationID'].astype(str) + filename = 'CLINVAR_VARIANT_GENE_EDGES.txt' + var_summary_merged_gene.to_csv(self.temp_dir + '/' + filename, sep='\t', encoding='utf-8', index=False) + uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename) + # reduce data set to extract variant-disease/phenotype edges + var_summary_merged_disease = var_summary_merged.copy() + var_summary_merged_disease = var_summary_merged_disease[[ + 'VariationID', 'RS# (dbSNP)', 'Type', 'VariantName', 'RCVaccession', 'LastEvaluated', 'ReviewStatus', + 'ClinicalSignificance', 'ClinSigSimple', 'NumberSubmitters', 'SubmitterCategories', 'Guidelines', + 'GeneID', 'TestedInGTR', 'Origin', 'OriginSimple', 'Assembly', 'Phenotype', 'Citation', 'OtherIDs']] + var_summary_merged_disease.drop_duplicates(inplace=True) + # expand results by disease identifier, remove phenotype rows with None, and drop duplicates + cols = ['Phenotype'] + for col in tqdm(cols): var_summary_merged_disease = var_summary_merged_disease.assign( + **{col: var_summary_merged_disease[col].str.split(';')}).explode(col) + var_summary_merged_disease = var_summary_merged_disease[var_summary_merged_disease['Phenotype'] != 'None'] + var_summary_merged_disease.drop_duplicates(inplace=True) + var_summary_merged_disease['VariationID'] = 'clinvar_' + var_summary_merged_disease['VariationID'].astype(str) + filename = 'CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt' + var_summary_merged_gene.to_csv(self.temp_dir + '/' + filename, sep='\t', encoding='utf-8', index=False) uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename) return None @@ -1300,157 +1661,161 @@ def processes_cofactor_catalyst_data(self) -> None: filename1, filename2 = 'UNIPROT_PROTEIN_COFACTOR.txt', 'UNIPROT_PROTEIN_CATALYST.txt' with open(self.temp_dir + '/' + filename1, 'w') as out1, open(self.temp_dir + '/' + filename2, 'w') as out2: for line in tqdm(data): - if 'CHEBI' in line.split('\t')[4]: # cofactors + status = line.split('\t')[1]; upt_id = line.split('\t')[0]; upt_entry = line.split('\t')[2] + pr_id = 'PR_' + line.split('\t')[3].strip(';') + # get cofactors + if 'CHEBI' in line.split('\t')[4]: for i in line.split('\t')[4].split(';'): chebi = i.split('[')[-1].replace(']', '').replace(':', '_') - out1.write('PR_' + line.split('\t')[3].strip(';') + '\t' + chebi + '\n') - if 'CHEBI' in line.split('\t')[5]: # catalysts - for j in line.split('\t')[5].split(';'): - chebi = j.split('[')[-1].replace(']', '').replace(':', '_') - out2.write('PR_' + line.split('\t')[3].strip(';') + '\t' + chebi + '\n') + out1.write(pr_id + '\t' + chebi + '\t' + status + '\t' + upt_id + '\t' + upt_entry + '\n') + # get catalysts + if 'CHEBI' in line.split('\t')[5]: + for i in line.strip('\n').split('\t')[5].split(';'): + chebi = i.split('[')[-1].replace(']', '').replace(':', '_') + out2.write(pr_id + '\t' + chebi + '\t' + status + '\t' + upt_id + '\t' + upt_entry + '\n') # push data to gsc bucket uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename1) uploads_data_to_gcs_bucket(self.bucket, self.processed_data, self.temp_dir, filename2) return None - def _creates_gene_metadata_dict(self) -> Dict: - """Creates a dictionary to store labels, synonyms, and a description for each Entrez gene identifier present in - the input data file. - - Returns: - gene_metadata_dict: A dict containing metadata that's keyed by Entrez gene identifier and whose values are - dicts containing label, description, and synonym information. For example: - {{'http://www.ncbi.nlm.nih.gov/gene/1': { - 'Label': 'A1BG', - 'Description': "A1BG is 'protein-coding' and is located on chromosome 19 (19q13.43).", - 'Synonym': 'HEL-S-163pA|A1B|ABG|HYST2477alpha-1B-glycoprotein|GAB'}, ...} - """ - - log_str = 'Generating Metadata for Gene Identifiers'; print('\t- ' + log_str); logger.info(log_str) - - f_name = 'Homo_sapiens.gene_info' - x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) - data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) - data = data.loc[data['#tax_id'].apply(lambda i: i == 9606)] - data.fillna('None', inplace=True); data.replace('-', 'None', inplace=True, regex=False) - # create metadata - genes, lab, desc, syn = [], [], [], [] - for idx, row in tqdm(data.iterrows(), total=data.shape[0]): - gene_id, sym, defn, gene_type = row['GeneID'], row['Symbol'], row['description'], row['type_of_gene'] - chrom, map_loc, s1, s2 = row['chromosome'], row['map_location'], row['Synonyms'], row['Other_designations'] - if gene_id != 'None': - genes.append('http://www.ncbi.nlm.nih.gov/gene/' + str(gene_id)) - if sym != 'None' or sym != '': lab.append(sym) - else: lab.append('Entrez_ID:' + gene_id) - if 'None' not in [defn, gene_type, chrom, map_loc]: - desc_str = "{} has locus group '{}' and is located on chromosome {} ({})." - desc.append(desc_str.format(sym, gene_type, chrom, map_loc)) - else: desc.append("{} locus group '{}'.".format(sym, gene_type)) - if s1 != 'None' and s2 != 'None': - syn.append('|'.join(set([x for x in (s1 + s2).split('|') if x != 'None' or x != '']))) - elif s1 != 'None': syn.append('|'.join(set([x for x in s1.split('|') if x != 'None' or x != '']))) - elif s2 != 'None': syn.append('|'.join(set([x for x in s2.split('|') if x != 'None' or x != '']))) - else: syn.append('None') - # combine into new data frame then convert it to dictionary - metadata = pandas.DataFrame(list(zip(genes, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) - metadata = metadata.astype(str); metadata.drop_duplicates(subset='ID', inplace=True) - metadata.set_index('ID', inplace=True); gene_metadata_dict = metadata.to_dict('index') - - return gene_metadata_dict - - def _creates_transcript_metadata_dict(self) -> Dict: - """Creates a dictionary to store labels, synonyms, and a description for each Entrez gene identifier present - in the input data file. - - Returns: - rna_metadata_dict: A dict containing metadata that's keyed by Ensembl transcript identifier and whose values - are a dict containing label, description, and synonym information. For example: - {'https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=ENST00000456328': { - 'Label': 'DDX11L1-202', - 'Description': "Transcript DDX11L1-202 is classified as type 'processed_transcript'.", - 'Synonym': 'None'}, ...} - """ - - log_str = 'Generating Metadata for Transcript Identifiers'; print('\t- ' + log_str); logger.info(log_str) - - f_name = 'ensembl_identifier_data_cleaned.txt' - x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) - dup_cols = ['transcript_stable_id', 'transcript_name', 'ensembl_transcript_type'] - data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) - data = data.loc[data['transcript_stable_id'].apply(lambda i: i != 'None')] - data.drop(['ensembl_gene_id', 'symbol', 'protein_stable_id', 'uniprot_id', 'master_transcript_type', - 'entrez_id', 'ensembl_gene_type', 'master_gene_type', 'symbol'], axis=1, inplace=True) - data.drop_duplicates(subset=dup_cols, keep='first', inplace=True); data.fillna('None', inplace=True) - # create metadata - rna, lab, desc, syn = [], [], [], [] - for idx, row in tqdm(data.iterrows(), total=data.shape[0]): - rna_id, ent_type, nme = row[dup_cols[0]], row[dup_cols[2]], row[dup_cols[1]] - rna.append('https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=' + rna_id) - if nme != 'None': lab.append(nme) - else: lab.append('Ensembl_Transcript_ID:' + rna_id); nme = 'Ensembl_Transcript_ID:' + rna_id - if ent_type != 'None': desc.append("Transcript {} is classified as type '{}'.".format(nme, ent_type)) - else: desc.append('None') - syn.append('None') - # combine into new data frame then convert it to dictionary - metadata = pandas.DataFrame(list(zip(rna, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) - metadata = metadata.astype(str); metadata.drop_duplicates(subset='ID', inplace=True) - metadata.set_index('ID', inplace=True); rna_metadata_dict = metadata.to_dict('index') - - return rna_metadata_dict - - def _creates_variant_metadata_dict(self) -> Dict: - """Creates a dictionary to store labels, synonyms, and a description for each ClinVar variant identifier present - in the input data file. - - Returns: - variant_metadata_dict: A dict containing metadata that's keyed by ClinVar variant identifier and whose - values are a dict containing label, description, and synonym information. For example: - {{'https://www.ncbi.nlm.nih.gov/snp/rs141138948': { - 'Label': 'NM_016042.4(EXOSC3):c.395A>C (p.Asp132Ala)', - 'Description': "This variant is a germline single nucleotide variant on chromosome 9 - (NC_000009.12, start:37783993/stop:37783993 positions,cytogenetic location:9p13.2) and - has clinical significance 'Pathogenic/Likely pathogenic'. This entry is for the GRCh38 and was - last reviewed on Sep 30, 2020 with review status 'criteria provided, multiple submitters, - no conflict'.", 'Synonym': 'None'}, ...} - """ - - log_str = 'Generating Metadata for Variant IDs'; print('\t- ' + log_str); logger.info(log_str) - - f_name = 'variant_summary.txt' - x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) - data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) - data = data.loc[data['Assembly'].apply(lambda i: i == 'GRCh38')] - data = data.loc[data['RS# (dbSNP)'].apply(lambda i: i != -1)] - data = data[['#AlleleID', 'Type', 'Name', 'ClinicalSignificance', 'RS# (dbSNP)', 'Origin', 'Start', 'Stop', - 'ChromosomeAccession', 'Chromosome', 'ReferenceAllele', 'Assembly', 'AlternateAllele', - 'Cytogenetic', 'ReviewStatus', 'LastEvaluated']] - data.replace('na', 'None', inplace=True); data.fillna('None', inplace=True) - data.sort_values('LastEvaluated', ascending=False, inplace=True) - data.drop_duplicates(subset='RS# (dbSNP)', keep='first', inplace=True) - # create metadata - var, label, desc, syn = [], [], [], [] - for idx, row in tqdm(data.iterrows(), total=data.shape[0]): - var_id, lab = row['RS# (dbSNP)'], row['Name'] - if var_id != 'None': - var.append('https://www.ncbi.nlm.nih.gov/snp/rs' + str(var_id)) - if lab != 'None': label.append(lab) - else: label.append('dbSNP_ID:rs' + str(var_id)) - sent = "This variant is a {} {} located on chromosome {} ({}, start:{}/stop:{} positions, " + \ - "cytogenetic location:{}) and has clinical significance '{}'. " + \ - "This entry is for the {} and was last reviewed on {} with review status '{}'." - desc.append( - sent.format(row['Origin'].replace(';', '/'), row['Type'].replace(';', '/'), row['Chromosome'], - row['ChromosomeAccession'], row['Start'], row['Stop'], row['Cytogenetic'], - row['ClinicalSignificance'], row['Assembly'], row['LastEvaluated'], - row['ReviewStatus']).replace('None', 'UNKNOWN')) - syn.append('None') - # combine into new data frame then convert it to dictionary - metadata = pandas.DataFrame(list(zip(var, label, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) - metadata.drop_duplicates(inplace=True); metadata = metadata.astype(str) - metadata.set_index('ID', inplace=True); variant_metadata_dict = metadata.to_dict('index') - - return variant_metadata_dict + # def _creates_gene_metadata_dict(self) -> Dict: + # """Creates a dictionary to store labels, synonyms, and a description for each Entrez gene identifier present in + # the input data file. + # + # Returns: + # gene_metadata_dict: A dict containing metadata that's keyed by Entrez gene identifier and whose values are + # dicts containing label, description, and synonym information. For example: + # {{'http://www.ncbi.nlm.nih.gov/gene/1': { + # 'Label': 'A1BG', + # 'Description': "A1BG is 'protein-coding' and is located on chromosome 19 (19q13.43).", + # 'Synonym': 'HEL-S-163pA|A1B|ABG|HYST2477alpha-1B-glycoprotein|GAB'}, ...} + # """ + # + # log_str = 'Generating Metadata for Gene Identifiers'; print('\t- ' + log_str); logger.info(log_str) + # + # f_name = 'Homo_sapiens.gene_info' + # x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) + # data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) + # data = data.loc[data['#tax_id'].apply(lambda i: i == 9606)] + # data.fillna('None', inplace=True); data.replace('-', 'None', inplace=True, regex=False) + # # create metadata + # genes, lab, desc, syn = [], [], [], [] + # for idx, row in tqdm(data.iterrows(), total=data.shape[0]): + # gene_id, sym, defn, gene_type = row['GeneID'], row['Symbol'], row['description'], row['type_of_gene'] + # chrom, map_loc, s1, s2 = row['chromosome'], row['map_location'], row['Synonyms'], row['Other_designations'] + # if gene_id != 'None': + # genes.append('http://www.ncbi.nlm.nih.gov/gene/' + str(gene_id)) + # if sym != 'None' or sym != '': lab.append(sym) + # else: lab.append('Entrez_ID:' + gene_id) + # if 'None' not in [defn, gene_type, chrom, map_loc]: + # desc_str = "{} has locus group '{}' and is located on chromosome {} ({})." + # desc.append(desc_str.format(sym, gene_type, chrom, map_loc)) + # else: desc.append("{} locus group '{}'.".format(sym, gene_type)) + # if s1 != 'None' and s2 != 'None': + # syn.append('|'.join(set([x for x in (s1 + s2).split('|') if x != 'None' or x != '']))) + # elif s1 != 'None': syn.append('|'.join(set([x for x in s1.split('|') if x != 'None' or x != '']))) + # elif s2 != 'None': syn.append('|'.join(set([x for x in s2.split('|') if x != 'None' or x != '']))) + # else: syn.append('None') + # # combine into new data frame then convert it to dictionary + # metadata = pandas.DataFrame(list(zip(genes, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) + # metadata = metadata.astype(str); metadata.drop_duplicates(subset='ID', inplace=True) + # metadata.set_index('ID', inplace=True); gene_metadata_dict = metadata.to_dict('index') + # + # return gene_metadata_dict + # + # def _creates_transcript_metadata_dict(self) -> Dict: + # """Creates a dictionary to store labels, synonyms, and a description for each Entrez gene identifier present + # in the input data file. + # + # Returns: + # rna_metadata_dict: A dict containing metadata that's keyed by Ensembl transcript identifier and whose values + # are a dict containing label, description, and synonym information. For example: + # {'https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=ENST00000456328': { + # 'Label': 'DDX11L1-202', + # 'Description': "Transcript DDX11L1-202 is classified as type 'processed_transcript'.", + # 'Synonym': 'None'}, ...} + # """ + # + # log_str = 'Generating Metadata for Transcript Identifiers'; print('\t- ' + log_str); logger.info(log_str) + # + # f_name = 'ensembl_identifier_data_cleaned.txt' + # x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) + # dup_cols = ['transcript_stable_id', 'transcript_name', 'ensembl_transcript_type'] + # data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) + # data = data.loc[data['transcript_stable_id'].apply(lambda i: i != 'None')] + # data.drop(['ensembl_gene_id', 'symbol', 'protein_stable_id', 'uniprot_id', 'master_transcript_type', + # 'entrez_id', 'ensembl_gene_type', 'master_gene_type', 'symbol'], axis=1, inplace=True) + # data.drop_duplicates(subset=dup_cols, keep='first', inplace=True); data.fillna('None', inplace=True) + # # create metadata + # rna, lab, desc, syn = [], [], [], [] + # for idx, row in tqdm(data.iterrows(), total=data.shape[0]): + # rna_id, ent_type, nme = row[dup_cols[0]], row[dup_cols[2]], row[dup_cols[1]] + # rna.append('https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=' + rna_id) + # if nme != 'None': lab.append(nme) + # else: lab.append('Ensembl_Transcript_ID:' + rna_id); nme = 'Ensembl_Transcript_ID:' + rna_id + # if ent_type != 'None': desc.append("Transcript {} is classified as type '{}'.".format(nme, ent_type)) + # else: desc.append('None') + # syn.append('None') + # # combine into new data frame then convert it to dictionary + # metadata = pandas.DataFrame(list(zip(rna, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) + # metadata = metadata.astype(str); metadata.drop_duplicates(subset='ID', inplace=True) + # metadata.set_index('ID', inplace=True); rna_metadata_dict = metadata.to_dict('index') + # + # return rna_metadata_dict + # + # def _creates_variant_metadata_dict(self) -> Dict: + # """Creates a dictionary to store labels, synonyms, and a description for each ClinVar variant identifier present + # in the input data file. + # + # Returns: + # variant_metadata_dict: A dict containing metadata that's keyed by ClinVar variant identifier and whose + # values are a dict containing label, description, and synonym information. For example: + # {{'https://www.ncbi.nlm.nih.gov/snp/rs141138948': { + # 'Label': 'NM_016042.4(EXOSC3):c.395A>C (p.Asp132Ala)', + # 'Description': "This variant is a germline single nucleotide variant on chromosome 9 + # (NC_000009.12, start:37783993/stop:37783993 positions,cytogenetic location:9p13.2) and + # has clinical significance 'Pathogenic/Likely pathogenic'. This entry is for the GRCh38 and was + # last reviewed on Sep 30, 2020 with review status 'criteria provided, multiple submitters, + # no conflict'.", 'Synonym': 'None'}, ...} + # """ + # + # log_str = 'Generating Metadata for Variant IDs'; print('\t- ' + log_str); logger.info(log_str) + # + # f_name = 'variant_summary.txt' + # x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) + # data = pandas.read_csv(x, header=0, delimiter='\t', low_memory=False) + # data = data.loc[data['Assembly'].apply(lambda i: i == 'GRCh38')] + # data = data.loc[data['RS# (dbSNP)'].apply(lambda i: i != -1)] + # data = data[['#AlleleID', 'Type', 'Name', 'ClinicalSignificance', 'RS# (dbSNP)', 'Origin', 'Start', 'Stop', + # 'ChromosomeAccession', 'Chromosome', 'ReferenceAllele', 'Assembly', 'AlternateAllele', + # 'Cytogenetic', 'ReviewStatus', 'LastEvaluated']] + # data.replace('na', 'None', inplace=True); data.fillna('None', inplace=True) + # data.sort_values('LastEvaluated', ascending=False, inplace=True) + # data.drop_duplicates(subset='RS# (dbSNP)', keep='first', inplace=True) + # # create metadata + # var, label, desc, syn = [], [], [], [] + # for idx, row in tqdm(data.iterrows(), total=data.shape[0]): + # var_id, lab = row['RS# (dbSNP)'], row['Name'] + # if var_id != 'None': + # var.append('https://www.ncbi.nlm.nih.gov/snp/rs' + str(var_id)) + # if lab != 'None': label.append(lab) + # else: label.append('dbSNP_ID:rs' + str(var_id)) + # sent = "This variant is a {} {} located on chromosome {} ({}, start:{}/stop:{} positions, " + \ + # "cytogenetic location:{}) and has clinical significance '{}'. " + \ + # "This entry is for the {} and was last reviewed on {} with review status '{}'." + # desc.append( + # sent.format(row['Origin'].str.replace(';', '/'), row['Type'].replace(';', '/'), row['Chromosome'], + # row['ChromosomeAccession'], row['Start'], row['Stop'], row['Cytogenetic'], + # row['ClinicalSignificance'], row['Assembly'], row['LastEvaluated'], + # row['ReviewStatus']).replace('None', 'UNKNOWN')) + # syn.append('None') + # # combine into new data frame then convert it to dictionary + # metadata = pandas.DataFrame(list(zip(var, label, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym']) + # metadata.drop_duplicates(inplace=True); metadata = metadata.astype(str) + # metadata.set_index('ID', inplace=True); variant_metadata_dict = metadata.to_dict('index') + # + # return variant_metadata_dict @staticmethod def _metadata_api_mapper(nodes: List[str]) -> pandas.DataFrame: @@ -1509,7 +1874,7 @@ def _creates_pathway_metadata_dict(self) -> Dict: g = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name1, self.temp_dir) data1 = pandas.read_csv(g, header=None, delimiter='\t', skiprows=4, low_memory=False) data1 = data1.loc[data1[12].apply(lambda x: x == 'taxon:9606')] - data1[5].replace('REACTOME:', '', inplace=True, regex=True) + data1[5] = data1[5].str.replace('REACTOME:', '', regex=True) # reactome CHEBI data f_name2 = 'ChEBI2Reactome_All_Levels.txt' h = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name2, self.temp_dir) @@ -1544,7 +1909,7 @@ def _creates_relations_metadata_dict(self) -> Dict: f_name = 'ro_with_imports.owl' x = downloads_data_from_gcs_bucket(self.bucket, self.original_data, self.processed_data, f_name, self.temp_dir) ro_graph = Graph().parse(x) - relation_metadata_dict, obo = {}, Namespace('http://purl.obolibrary.org/obo/') + relation_metadata_dict = {} cls = [x for x in gets_ontology_classes(ro_graph) if '/RO_' in str(x)] + \ [x for x in gets_object_properties(ro_graph) if '/RO_' in str(x)] master_synonyms = [x for x in ro_graph if 'synonym' in str(x[1]).lower() and isinstance(x[0], URIRef)] @@ -1559,28 +1924,203 @@ def _creates_relations_metadata_dict(self) -> Dict: return relation_metadata_dict - def creates_non_ontology_class_metadata_dict(self) -> None: - """Combines the gene metadata, transcript metadata, variant metadata, pathway metadata, and relations - metadata dictionaries into a single large metadata dictionary. See example output below: - { - 'nodes': { - 'http://www.ncbi.nlm.nih.gov/gene/1': { - 'Label': 'A1BG', - 'Description': "A1BG has locus group protein-coding' and is located on chromosome 19 (19q13.43).", - 'Synonym': 'HYST2477alpha-1B-glycoprotein|HEL-S-163pA|ABG|A1B|GAB'} ... }, - 'relations': { - 'http://purl.obolibrary.org/obo/RO_0002533': { - 'Label': 'sequence atomic unit', - 'Description': 'Any individual unit of a collection of like units arranged in a linear order', - 'Synonym': 'None'} ... } + def _loads_mapping_data(self) -> Dict: + """ + + Returns: + id_map_dict: A dictionary of Pandas DataFrame objects keyed by variable name. + """ + + id_map_dict: Dict = { + 'rna_map': pandas.read_csv(self.temp_dir + '/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt', + header=None, delimiter='\t', low_memory=False, usecols=[0, 1, 2, 4], + names=['Entrez_Gene_IDs', 'Ensembl_Transcript_IDs', 'Entrez_Gene_Type', + 'Ensembl_Transcript_Type', 'Master_Gene_Type', 'Master_Transcript_Type', + 'Entrez_Gene_prefix']), + 'entrez_pro_map' : pandas.read_csv(self.temp_dir + '/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt', + header=None, delimiter='\t', low_memory=False, usecols=[0, 1, 2, 4], + names=['Gene_IDs', 'Protein_Ontology_IDs', 'Entrez_Gene_Type', + 'Master_Gene_Type', 'Entrez_Gene_Prefix']), + 'string_pro_map': pandas.read_csv(self.temp_dir + '/STRING_PRO_ONTOLOGY_MAP.txt', + header=None, delimiter='\t', low_memory=False, usecols=[0, 1], + names=['STRING_IDs', 'Protein_Ontology_IDs']), + 'uniprot_pro_map': pandas.read_csv(self.temp_dir + '/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt', + header=None, delimiter='\t', low_memory=False, usecols=[0, 1], + names=['Uniprot_Accession_IDs', 'Protein_Ontology_IDs']), + 'uniprot_entrez_data': pandas.read_csv(self.temp_dir + '/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt', + header=None, delimiter='\t', low_memory=False, usecols=[0, 1, 2, 3], + names=['Uniprot_Accession_IDs', 'Entrez_Gene_IDs', + 'master_gene_type', 'gene_type_update']), + 'mesh_chebi_map': pandas.read_csv(self.temp_dir + '/MESH_CHEBI_MAP.txt', header=None, + names=['MESH_ID', 'CHEBI_ID'], delimiter='\t'), + 'disease_maps': pandas.read_csv(self.temp_dir + '/DISEASE_MONDO_MAP.txt', header=None, + names=['Disease_IDs', 'MONDO_IDs'], delimiter='\t'), + 'phenotype_maps': pandas.read_csv(self.temp_dir + '/PHENOTYPE_HPO_MAP.txt', header=None, + names=['Disease_IDs', 'HP_IDs'], delimiter='\t') } + return id_map_dict + + def _creates_genomic_metadata_dict(self) -> Dict: + """Process a genomic metadata dictionary created in the prior steps in order to assist with creating a master + metadata file for all nodes that are a genomic entity (i.e., genes, transcripts, or proteins). + + Returns: + genomic_metadata: A nested dictionary of genomic metadata keyed by NCBIGene, ensembl, and Protein Ontology + identifiers. + """ + + filepath = self.temp_dir + '/Merged_gene_rna_protein_identifiers.pkl' + max_bytes = 2**31 - 1; input_size = os.path.getsize(filepath); bytes_in = bytearray(0) + with open(filepath, 'rb') as f_in: + for _ in range(0, input_size, max_bytes): + bytes_in += f_in.read(max_bytes) + reformatted_mapped_identifiers = pickle.loads(bytes_in) + + # clean up data for use with master metadata + genomic_metadata = dict() + for key, value in tqdm(reformatted_mapped_identifiers.items()): + old_prefix = '_'.join(key.split('_')[0:-1]); idx = key.split('_')[-1]; pass_var = True; new_prefix = None + if old_prefix == 'entrez_id': new_prefix = 'NCBIGene' + elif old_prefix in ['ensembl_gene_id', 'protein_stable_id', 'transcript_stable_id']: new_prefix = 'ensembl' + elif old_prefix == 'pro_id_PR': new_prefix = 'PR' + else: pass_var = False + if pass_var and new_prefix is not None: + updated_key = new_prefix + '_' + idx; master_metadata_dict = {updated_key: {}} + for x in value: + i, j = '_'.join(x.split('_')[0:-1]), x.split('_')[-1] + if 'type' in i: continue + elif i == 'entrez_id': new_i = 'NCBIGene'; j = new_i + '_' + j + elif i == 'ensembl_gene_id': new_i = 'ensembl gene'; j = 'ensembl_' + j + elif i == 'protein_stable_id': new_i = 'ensembl protein'; j = 'ensembl_' + j + elif i == 'transcript_stable_id': new_i = 'ensembl transcript'; j = 'ensembl_' + j + elif i == 'pro_id_PR': new_i = 'PR'; j = new_i + '_' + j + elif i == 'hgnc_id': new_i = 'HGNC_ID'; j = new_i + '_' + j + elif i == 'uniprot_id': new_i = 'uniprot'; j = new_i + '_' + j + elif i == 'symbol': new_i = 'GeneSymbol'; j = new_i + '_' + j + else: + if i == 'synonyms': new_i = 'Synonyms' + elif i == 'name': new_i = 'Label' + elif i == 'Other_designations': new_i = 'Synonyms'; j = j.split('|') + else: new_i = i + if new_i in master_metadata_dict[updated_key].keys(): + if isinstance(j, list): master_metadata_dict[updated_key][new_i] += j + else: master_metadata_dict[updated_key][new_i] += [j] + else: master_metadata_dict[updated_key][new_i] = [j] + genomic_metadata[updated_key] = master_metadata_dict + + return genomic_metadata + + def _processes_ctd_gene_inx_data(self, master, g_dict, mesh_chebi_map, rna_map, entrez_pro_map) -> Dict: + """This function processes the CTD_chem_gene_ixns.tsv file and obtains the following node and edge metadata: + Nodes: + - ChemicalID: A string containing the concept's MESH identifier. + - CasRN: A string containing a CAS Registry Number. + - ChemicalName: A string containing the concept's synonym. + - Organism: A string containing the name of an organism. + - GeneSymbol: A string containing the concept's gene symbol. + Relations: + - Interaction: A string describing a chemical-gene/protein/rna interaction. + - InteractionActions: A "|"-delimited list of the actions that underlie an interaction. + - PubMedIDs: |'-delimited list of PubMed identifiers that do not include a prefix. + + Args: + master: The master metadata dictionary keyed by nodes and relations. + g_dict: A nested dictionary of genomic metadata keyed by NCBIGene, ensembl, and Protein Ontology IDs. + mesh_chebi_map: A Pandas DataFrame that contains MeSH-CHEBI identifier mappings. + rna_map: A Pandas DataFrame that contains Entrez Gene-Ensembl Transcript identifier mappings. + entrez_pro_map: A Pandas DataFrame that contains Entrez Gene-Protein Ontology identifier mappings. + + Returns: + master_dict: The master metadata dictionary keyed by nodes and relations. + """ + + # download and process data + url = 'http://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz'; f_name = self.temp_dir + '/CTD_chem_gene_ixns.tsv' + if not os.path.exists(f_name): data_downloader(url, f_name) + df = pandas.read_csv(f_name, header=0, delimiter='\t', skiprows=27) + df = df[df['# ChemicalName'] != '#']; df = df[df['OrganismID'] == 9606]; df = df[df['PubMedIDs'] != numpy.nan] + df['ChemicalID'] = 'MESH:' + df['ChemicalID'] + df['GeneID'] = df['GeneID'].astype('Int64'); df['OrganismID'] = df['OrganismID'].astype('Int64') + # merge identifier maps + df = df.merge(mesh_chebi_map, left_on='ChemicalID', right_on='MESH_ID') + df = df.merge(rna_map, left_on='GeneID', right_on='Entrez_Gene_IDs') + df = df.merge(entrez_pro_map, left_on='GeneID', right_on='Gene_IDs') + + for idx, row in tqdm(df.iterrows(), total=df.shape[0]): + chebi = row['CHEBI_ID'].rstrip(); n_key = None; genomic_info = None; r_key = None; form = None + chemical_name = row['# ChemicalName']; chemical_id = row['ChemicalID'].rstrip(); casrn = row['CasRN'] + evidence = [{'CTD_Interaction': row['Interaction'], 'CTD_InteractionActions': row['InteractionActions'], + 'CTD_PubMedIDs': row['PubMedIDs']}] + g_info = None; relation_key = '{}-{}'.format(chebi, n_key); edge_type = None + if row['GeneForms'] == 'gene': + n_key = row['Entrez_Gene_prefix'].rstrip(); edge_type = 'chemical-gene'; form = row['GeneForms'] + if n_key in g_dict.keys(): g_info = g_dict[n_key] + if row['GeneForms'] == 'protein': + n_key = row['Protein_Ontology_IDs'].rstrip(); edge_type = 'chemical-protein'; form = row['GeneForms'] + if n_key in g_dict.keys(): g_info = g_dict[n_key] + if row['GeneForms'] == 'rna': + n_key = row['Ensembl_Transcript_IDs'].rstrip(); edge_type = 'chemical-rna'; form = row['GeneForms'] + if n_key in g_dict.keys(): g_info = g_dict[n_key] + if form is not None: + # add node data to dictionary + if chebi in master['nodes'].keys(): + if 'CTD_ChemicalName' in master['nodes'][chebi].keys(): + master['nodes'][chebi]['CTD_ChemicalName'] |= {chemical_name} + else: master['nodes'][chebi]['CTD_ChemicalName'] = {chemical_name} + if 'CTD_ChemicalID' in master['nodes'][chebi].keys(): + master['nodes'][chebi]['ChemicalID'] |= {chemical_id} + else: master['nodes'][chebi]['ChemicalID'] = {chemical_id} + if 'CTD_CasRN' in master['nodes'][chebi].keys(): master['nodes'][chebi]['CTD_CasRN'] |= {casrn} + else: master['nodes'][chebi]['CTD_CasRN'] = {casrn} + else: + master['nodes'][n_key] = {'CTD_GeneForms': form} + if g_info is not None: master['nodes'][n_key]['genomic_data'] = {n_key: g_info} + master['nodes'][chebi] = {} + master['nodes'][chebi]['ChemicalID'] = {chemical_id} + master['nodes'][chebi]['CTD_CasRN'] = {casrn} + master['nodes'][chebi]['CTD_ChemicalName'] = {chemical_name} + # add relation data to dictionary + if r_key in master['relations'][edge_type].keys(): + if 'CTD_Evidence' in master['relations'][edge_type][r_key].keys(): + master['relations'][edge_type][r_key]['CTD_Evidence'] += [evidence] + else: master['relations'][edge_type][r_key]['CTD_Evidence'] = [evidence] + else: + master['relations'][edge_type][r_key] = {} + master['relations'][edge_type][r_key]['CTD_Evidence'] = [evidence] + + return master + + + + def creates_metadata_dict(self) -> None: + """Creates a single large metadata dictionary that is keyed by nodes and relations and contains a variety of + metadata. See the following file for additional details: + https://github.com/callahantiff/PheKnowLator/tree/master/resources/pheknowlator_source_metadata.xlsx. + Returns: None. """ log_str = 'Creating Master Metadata Dictionary for Non-Ontology Entities'; print(log_str); logger.info(log_str) + # load identifier mapping data + id_map = self._loads_mapping_data() + rna_map = id_map['rna_map']; entrez_pro_map = id_map['entrez_pro_map']; disease_maps = id_map['disease_maps'] + string_pro_map = id_map['string_pro_map']; uniprot_pro_map = id_map['uniprot_pro_map'] + uniprot_entrez_data = id_map['uniprot_entrez_data']; mesh_chebi_map = id_map['mesh_chebi_map'] + phenotype_maps = id_map['phenotype_maps'] + + # create dictionary + master_dict = {'nodes': {}, 'relations': {}} + + # obtain metadata dictionaries + genomic = self._creates_genomic_metadata_dict() + master_dict = self._processes_ctd_gene_inx_data(master_dict, genomic, mesh_chebi_map, rna_map, entrez_pro_map) + + + + # create single dictionary of master_metadata_dictionary = {'nodes': {**self._creates_gene_metadata_dict(), **self._creates_transcript_metadata_dict(), @@ -1663,7 +2203,7 @@ def preprocesses_build_data(self) -> None: # STEP 10: Non-Ontology Metadata Dictionary log_str = 'STEP 10: CREATING OBO-ONTOLOGY METADATA DICTIONARY'; print('\n' + log_str); logger.info(log_str) - self.creates_non_ontology_class_metadata_dict() + self.creates_metadata_dict() uploads_data_to_gcs_bucket(self.bucket, self.log_location, log_dir, log) return None diff --git a/builds/data_to_download.txt b/builds/data_to_download.txt index 2b9728bb..93ed62ff 100755 --- a/builds/data_to_download.txt +++ b/builds/data_to_download.txt @@ -24,6 +24,7 @@ mesh2021.nt, ftp://nlmpubs.nlm.nih.gov/online/mesh/rdf/2021/mesh2021.nt names.tsv, ftp://ftp.ebi.ac.uk/pub/databases/chebi/Flat_file_tab_delimited/names.tsv.gz # disease and phenotype identifiers disease_mappings.tsv, https://www.disgenet.org/static/disgenet_ap1/files/downloads/disease_mappings.tsv.gz +MGCONSO.RRF, https://ftp.ncbi.nlm.nih.gov/pub/medgen/MGCONSO.RRF.gz # human protein atlas/gtex tissue/cells - uberon + cell ontology + cell line ontology proteinatlas_search.tsv.gz, https://www.proteinatlas.org/api/search_download.php?search=&columns=g,eg,up,pe,rnatsm,rnaclsm,rnacasm,rnabrsm,rnabcsm,rnablsm,scl,t_RNA_adipose_tissue,t_RNA_adrenal_gland,t_RNA_amygdala,t_RNA_appendix,t_RNA_basal_ganglia,t_RNA_bone_marrow,t_RNA_breast,t_RNA_cerebellum,t_RNA_cerebral_cortex,t_RNA_cervix,_uterine,t_RNA_colon,t_RNA_corpus_callosum,t_RNA_ductus_deferens,t_RNA_duodenum,t_RNA_endometrium_1,t_RNA_epididymis,t_RNA_esophagus,t_RNA_fallopian_tube,t_RNA_gallbladder,t_RNA_heart_muscle,t_RNA_hippocampal_formation,t_RNA_hypothalamus,t_RNA_kidney,t_RNA_liver,t_RNA_lung,t_RNA_lymph_node,t_RNA_midbrain,t_RNA_olfactory_region,t_RNA_ovary,t_RNA_pancreas,t_RNA_parathyroid_gland,t_RNA_pituitary_gland,t_RNA_placenta,t_RNA_pons_and_medulla,t_RNA_prostate,t_RNA_rectum,t_RNA_retina,t_RNA_salivary_gland,t_RNA_seminal_vesicle,t_RNA_skeletal_muscle,t_RNA_skin_1,t_RNA_small_intestine,t_RNA_smooth_muscle,t_RNA_spinal_cord,t_RNA_spleen,t_RNA_stomach_1,t_RNA_testis,t_RNA_thalamus,t_RNA_thymus,t_RNA_thyroid_gland,t_RNA_tongue,t_RNA_tonsil,t_RNA_urinary_bladder,t_RNA_vagina,t_RNA_B-cells,t_RNA_dendritic_cells,t_RNA_granulocytes,t_RNA_monocytes,t_RNA_NK-cells,t_RNA_T-cells,t_RNA_total_PBMC,cell_RNA_A-431,cell_RNA_A549,cell_RNA_AF22,cell_RNA_AN3-CA,cell_RNA_ASC_diff,cell_RNA_ASC_TERT1,cell_RNA_BEWO,cell_RNA_BJ,cell_RNA_BJ_hTERT+,cell_RNA_BJ_hTERT+_SV40_Large_T+,cell_RNA_BJ_hTERT+_SV40_Large_T+_RasG12V,cell_RNA_CACO-2,cell_RNA_CAPAN-2,cell_RNA_Daudi,cell_RNA_EFO-21,cell_RNA_fHDF/TERT166,cell_RNA_HaCaT,cell_RNA_HAP1,cell_RNA_HBEC3-KT,cell_RNA_HBF_TERT88,cell_RNA_HDLM-2,cell_RNA_HEK_293,cell_RNA_HEL,cell_RNA_HeLa,cell_RNA_Hep_G2,cell_RNA_HHSteC,cell_RNA_HL-60,cell_RNA_HMC-1,cell_RNA_HSkMC,cell_RNA_hTCEpi,cell_RNA_hTEC/SVTERT24-B,cell_RNA_hTERT-HME1,cell_RNA_HUVEC_TERT2,cell_RNA_K-562,cell_RNA_Karpas-707,cell_RNA_LHCN-M2,cell_RNA_MCF7,cell_RNA_MOLT-4,cell_RNA_NB-4,cell_RNA_NTERA-2,cell_RNA_PC-3,cell_RNA_REH,cell_RNA_RH-30,cell_RNA_RPMI-8226,cell_RNA_RPTEC_TERT1,cell_RNA_RT4,cell_RNA_SCLC-21H,cell_RNA_SH-SY5Y,cell_RNA_SiHa,cell_RNA_SK-BR-3,cell_RNA_SK-MEL-30,cell_RNA_T-47d,cell_RNA_THP-1,cell_RNA_TIME,cell_RNA_U-138_MG,cell_RNA_U-2_OS,cell_RNA_U-2197,cell_RNA_U-251_MG,cell_RNA_U-266/70,cell_RNA_U-266/84,cell_RNA_U-698,cell_RNA_U-87_MG,cell_RNA_U-937,cell_RNA_WM-115,blood_RNA_basophil,blood_RNA_classical_monocyte,blood_RNA_eosinophil,blood_RNA_gdT-cell,blood_RNA_intermediate_monocyte,blood_RNA_MAIT_T-cell,blood_RNA_memory_B-cell,blood_RNA_memory_CD4_T-cell,blood_RNA_memory_CD8_T-cell,blood_RNA_myeloid_DC,blood_RNA_naive_B-cell,blood_RNA_naive_CD4_T-cell,blood_RNA_naive_CD8_T-cell,blood_RNA_neutrophil,blood_RNA_NK-cell,blood_RNA_non-classical_monocyte,blood_RNA_plasmacytoid_DC,blood_RNA_T-reg,blood_RNA_total_PBMC,brain_RNA_amygdala,brain_RNA_basal_ganglia,brain_RNA_cerebellum,brain_RNA_cerebral_cortex,brain_RNA_hippocampal_formation,brain_RNA_hypothalamus,brain_RNA_midbrain,brain_RNA_olfactory_region,brain_RNA_pons_and_medulla,brain_RNA_thalamus&format=tsv GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct, https://storage.googleapis.com/gtex_analysis_v8/rna_seq_data/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz @@ -38,7 +39,9 @@ genomic_sequence_ontology_mappings.xlsx, https://storage.googleapis.com/pheknowl # protein ontology consortium sparql query results human_pro_classes.html, https://sparql.proconsortium.org/virtuoso/sparql?query=PREFIX+obo%3A+%3Chttp%3A%2F%2Fpurl.obolibrary.org%2Fobo%2F%3E%0D%0A%0D%0ASELECT+%3FPRO_term%0D%0AFROM+%3Chttp%3A%2F%2Fpurl.obolibrary.org%2Fobo%2Fpr%3E%0D%0AWHERE+%7B%0D%0A+++++++%3FPRO_term+rdf%3Atype+owl%3AClass+.%0D%0A+++++++%3FPRO_term+rdfs%3AsubClassOf+%3Frestriction+.%0D%0A+++++++%3Frestriction+owl%3AonProperty+obo%3ARO_0002160+.%0D%0A+++++++%3Frestriction+owl%3AsomeValuesFrom+obo%3ANCBITaxon_9606+.%0D%0A%0D%0A+++++++%23+use+this+to+filter-out+things+like+hgnc+ids%0D%0A+++++++FILTER+%28regex%28%3FPRO_term%2C%22http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2F*%22%29%29+.%0D%0A%7D&format=text%2Fhtml&debug= # clinvar variant-diseases and phenotypes -variant_summary.txt, ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz +variant_summary.txt, https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz +var_citations.txt, https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/var_citations.txt +allele_gene.txt, https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/allele_gene.txt.gz # uniprot protein-cofactor and protein-catalyst uniprot-cofactor-catalyst.tab, https://www.uniprot.org/uniprot/?query=&fil=organism%3A%22Homo%20sapiens%20(Human)%20%5B9606%5D%22&columns=id%2Creviewed%2Centry%20name%2Cdatabase(PRO)%2Cchebi(Cofactor)%2Cchebi(Catalytic%20activity)&format=tab diff --git a/generates_dependency_documents.py b/generates_dependency_documents.py index 26b09a02..be362fd6 100644 --- a/generates_dependency_documents.py +++ b/generates_dependency_documents.py @@ -32,8 +32,10 @@ class DocumentationMaker(object): def __init__(self, edge_count: int, write_location: str = './resources') -> None: # check edge count - if not isinstance(edge_count, int): raise ValueError('edge_count must be an integer (i.e. "1" not "one").') - else: self.edge_count = edge_count + if not isinstance(edge_count, int): + raise ValueError('edge_count must be an integer (i.e. "1" not "one").') + else: + self.edge_count = edge_count # make sure that the specified location to write data exists if os.path.exists(write_location): @@ -63,7 +65,7 @@ def information_getter(self) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, print('GATHERING INFORMATION FOR EDGE: {count}/{total}'.format(count=edge, total=self.edge_count)) print('#' * 40) - edge_name = input('Please enter the edge type (e.g. "gene-protein", "disease-chemical"): ') + edge_type = input('Please enter the edge type (e.g. "gene-protein", "disease-chemical"): ') print('\n') ont = input('Is one or both of the nodes in edge an ontology? Please enter "one" or "both": ') @@ -76,9 +78,6 @@ def information_getter(self) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, ont_data[ont_edge] = input('Provide an owl or obo URL for this ontology: ') print('\n') - data_type = input('Provide the data types for each node in the edge (e.g. "class" or "entity" (for ' - 'non-class data) each node in the edge separated by "-" --> "class-entity"): ') - print('\n') delimiter = input('Provide the character used to split each row into columns (e.g. "t" or ","): ') print('\n') @@ -111,29 +110,22 @@ def information_getter(self) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, '"RO_0000056"): ') print('\n') - subj_uri = input('Provide the Universal Resource Identifier that will be connected to the subject node (' - '(e.g. "http://purl.obolibrary.org/obo/"): ') - print('\n') - - obj_uri = input('Provide the Universal Resource Identifier that will be connected to the object node: ') - print('\n') - - source_label = input('Source Identifier Formatting (i.e. GO:12838340, when we need ' - 'GO_12838340).\n\nProvide the following 3 items:\n(1) Character to split existing ' - 'source labels (e.g. ":" in GO:1283834);\n(2) New label to replace existing label) ' - 'for subject node (e.g. "GO_");\n(3) New label to replace existing label) for ' - 'object node (e.g. GO_).\n\nEnter each item separated by ";". If the existing label ' - 'is correct, press "enter": ') or ';;' + identifier_prefix_information = input('Source Identifier Formatting (i.e., GO:12838340, when we need ' + 'GO_12838340).\n\nProvide the following 3 items:\n(1) Character to ' + 'split existing CURIE (e.g., ":" in GO:1283834);\n(2) New subject ' + 'prefix to replace existing one (e.g. "GO_");\n(3) New object ' + 'prefix to replace existing one (e.g., GO_).\n\nEnter each item ' + 'separated by ";". If the existing prefix is correct, press "enter": ' + ') or ";;"') print('\n') # add edge data to dictionary - resource_data[edge_name] = '{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}|{9}'.format(source_label, data_type, - edge_relation, subj_uri, - obj_uri, delimiter, col_idx, - id_maps, evi_crit, filt_crit) + resource_data[edge_type] = '{0}|{1}|{2}|{3}|{4}|{5}|{6}'.format(identifier_prefix_information, + edge_relation, delimiter, col_idx, + id_maps, evi_crit, filt_crit) # get edge data sources - edge_data[edge_name] = input('Provide a URL or file path to data used to create this edge: ') + edge_data[edge_type] = input('Provide a URL or file path to data used to create this edge: ') return resource_data, ont_data, edge_data @@ -158,47 +150,48 @@ def writes_out_document(self, data: Dict[str, str], delimiter: str, filename: st def main(): - # print initial message for user print('\n\n' + '***' * 50) print('INPUT DOCUMENT BUILDER\n\nThis program will help you generate the input documentation needed to run ' 'PheKnowLator by asking specific information about each edge type in the knowledge graph.\nIt will help ' - 'you create three documents: (1) resource_info.txt; (2) ontology_source_info.txt; and (3) ' - 'edge_source_info.txt.\nAn example of the data this program expects to find within each of these ' + 'you create three documents:\n\t\t(1) resource_info.txt\n\t\t(2) ontology_source_info.txt\n\t\t(3) ' + 'edge_source_info.txt\nAn example of the data this program expects to find within each of these ' 'documents is shown below:\n\n(1) resource_info.txt: This document represents each edge type as a single ' - '"|" delimited string and contains a total of 11 items:\n\t(1) Edge Type: A string containing a "-" ' - 'delimited edge label (node1-node2)\n\t(2) Source Labels: 3 ";"-delimited strings (e.g. ":;GO_;GO_)":\n\t\t' - '-the character to split existing labels (e.g. ":" in GO:1283834)\n\t\t-a new label for the subject ' - 'node\n\t\t-a new label for the object node. If the existing label is correct, use ";;";\n\t(3) Data ' - 'Type: A label of "class", "entity" ( fornon-ontology data) provided for each node and separated by "-" (' - 'e.g. "class-class", "class-entity", "entity-class");\n\t(4) Edge Relation: A Relation Ontology identifier to' - 'be used as an edge between the nodes (e.g. "RO_0000056")\n\t(5) Subject URI: A Universal Resource Identifier' - ' that will be connected to the subject node in the Edge Type (e.g. "http://purl.uniprot.org/geneid/");\n\t' - '(6) Object URI that will be connected to the object node in the Edge Type (e.g. ' - '"http://purl.obolibrary.org/obo/");\n\t(7) Delimiter: A character used to split input text rows into ' - 'columns (e.g. "t" or ",");\n\t(8) Column Indices: two column indices separated by ";" (e.g. 0;4 for the ' - 'first and third columns);\n\t(9) Identifier Maps: A string indicating the column index in the input data' - ' source needing identifier mapping and a file pointing to mapping data, for example:\n' - '\t\t"2:./resources/processed_data/mapping_file_1.txt;4:./resources/processed_data/mapping_file_2.txt" ' - 'means:\n\t\t\t-mapping data from the first node in the edge to the 0th column in ' - '"mapping_file_1.txt"\n\t\t\t-mapping data from the second node in the edge to the 4th column in ' - '"mapping_file_2.txt");\n\t(10) Evidence Criteria: Sets of 3 "::"-separated items, where each set is ' - 'composed of three pieces of ";"-separated information (e.g. "4;!=;IEA::8;<;0.0001" - means:\n\t\t-filter ' - 'the 4th column to keep rows that do not contain "IEA"\n\t\t-filter the 8th column to keep rows with a ' - 'value less than "0.0001");\n\t(11) Filter Criteria: Sets of 3 "::"-separated items, where each set is ' - 'composed of three pieces of ";"-separated information (e.g. "5;==;P::7;==;9606" - means:\n\t\t-filter the ' - '5th column to only include rows with "P"\n\t\tfilter the 7th column to only include rows containing ' - '"99606").\n\n\tAn example line from the resource_info.txt file is shown ' - 'below:\n\t\tchemical-gene|;MESH_;|class-class|;MESH_;|class-class|RO_0002434|http://purl.obolibrary.org' - '/obo/\n\t\t|http://purl.uniprot.org/geneid/|#|t|1;4|0:./resources/data_maps/MESH_CHEBI_MAP.txt|None|7' - ';==;9606\n\n(2) ontology_source_info.txt: This document contains a ","-delimited line for each ontology ' - 'source used, for example:\n\t"chemical, http://purl.obolibrary.org/obo/chebi.owl"\n\t"gene, ' - 'http://purl.obolibrary.org/obo/so.owl"\n\n(3) edge_source_info.txt: This document contains a ",' - '"-delimited line for each edge data source, for example:\n\t"chemical-gene, ' + '"|" delimited string and contains a total of 9 items:\n\t(1) edge_type: A string label for an edge ' + '(node1-node2). The label matches what is used in the edge_source_list.txt and ontology_source_list.txt files' + '\n\t(2) prefixes: A ";"-separated string where the first item is the final prefix for the subject node and ' + 'the second is the final prefix for the object node. . All prefixes should be the ' + 'preferred prefix from the BioRegistry\n\t\t(https://bioregistry.io/registry/);\n\t(3) relation: An OBO ' + 'Foundry ontology CURIE (e.g., RO_0000056)\n\t(4) delimiter: A character used to split rows from an input ' + 'data source into columns (e.g., "t" for tab-delimited data or "," for comma-delimited data);\n\t(5) ' + 'column_indexes: Two-column indexes separated by ";" (e.g., "0;4" for the first and third columns in the ' + 'input data source);\n\t(5) IdentifierMaps: A string of mapping information for each node in an edge. For ' + 'example, the string "2:mapping_file_1.txt;4:mapping_file_2.txt" means that the first node require\n\t\tdata ' + 'contained in the 2nd column of the "mapping_file_1.txt" and the second node requires data from the 4th ' + 'column in the "mapping_file_2.txt" file;\n\t(6) evidence_criteria: Evidence criteria that can be used to ' + 'filter an input data source (e.g., scores above a certain cut-off). An evidence set is composed of 3 pieces ' + 'of ";"\n\t\t-separated information. Multiple filtering sets can be passed, where each set is separated by ' + '"::". Consider the following example: "4;!=;IEA::8;<;0.0001"):\n\t\t\t1. The index of the column to apply ' + 'the evidence criteria to (e.g., "4" and "8" in the example above)\n\t\t\t2. The operator (i.e., "==", ' + '"!=", "<", ">", "<=", ">=", "in", ".startswith()", ".endswith()") to use when filtering (e.g., "!=" and ' + '"<" in the example above).\n\t\t\t3.The value (i.e., "int", "float", "str", "list") to filter on (e.g., ' + '"IEA" and "0.0001" in the example above);\n\t(7) filtering_criteria: Criteria that can be used to filter ' + 'an input data source (e.g., human proteins). An evidence set is composed of 3 pieces of ";"-separated ' + 'information. \n\t\tMultiple filtering sets can be passed as demonstrated by the example above, where each ' + 'set is separated by "::". Consider the following example: "5;==;P::7;==;9606"):\n\t\t\t1. The index of the ' + 'column to apply the evidence criteria to (e.g., "5" and "7" in the example above)\n\t\t\t2. The operator ' + '(i.e., "==", "!=", "<", ">", "<=", ">=", "in", ".startswith()", ".endswith()") to use when filtering ' + '(e.g., "==" and "==" in the example above)\n\t\t\t3. The value (i.e., "int", "float", "str", "list") to ' + 'filter on (e.g., "P" and "9606" in the example above).\n\n\tAn example line from the resource_info.txt file ' + 'is shown below:\n\t\tchemical-gene|;MESH;|RO_0002434|#|t|1;4|0:./resources/data_maps/' + 'MESH_CHEBI_MAP.txt|None|7;==;9606\n\n(2) ontology_source_info.txt: This document contains a "|"-delimited ' + 'line for each ontology source used, for example:\n\t"chemical|http://purl.obolibrary.org/obo/chebi.owl"' + '\n\t"gene|http://purl.obolibrary.org/obo/so.owl"\n\n(3) edge_source_info.txt: This document contains a ' + '"|"-delimited line for each edge data source, for example:\n\t"chemical-gene|' 'http://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz"\n\nIf you would like more information on the ' - 'dependency documents need to run PheKnowLator, please visit the following Wiki page: ' + 'dependency documents need to run PheKnowLator, please visit the following Wiki page:\n' 'https://github.com/callahantiff/PheKnowLator/wiki/Dependencies.') - print('***' * 50 + '\n') + print('***' * 60 + '\n') # initialize class edge_count = int(input('EDGE COUNT: Enter the number of edge types to create: ')) @@ -212,10 +205,10 @@ def main(): edge_maker.writes_out_document(edge_data[0], '|', 'resource_info.txt') # write out ontology data - edge_maker.writes_out_document(edge_data[1], ', ', 'ontology_source_list.txt') + edge_maker.writes_out_document(edge_data[1], '|', 'ontology_source_list.txt') # write out edge data - edge_maker.writes_out_document(edge_data[2], ', ', 'edge_source_list.txt') + edge_maker.writes_out_document(edge_data[2], '|', 'edge_source_list.txt') if __name__ == '__main__': diff --git a/main.ipynb b/main.ipynb index 3f944dc9..b35aa737 100644 --- a/main.ipynb +++ b/main.ipynb @@ -14,7 +14,7 @@ "\n", "**Author:** [TJCallahan](https://mail.google.com/mail/u/0/?view=cm&fs=1&tf=1&to=callahantiff@gmail.com) \n", "**GitHub Repository:** [PheKnowLator](https://github.com/callahantiff/PheKnowLator/wiki) \n", - "**Current Release:** **[`v2.0.0`](https://github.com/callahantiff/PheKnowLator/wiki/v2.0.0)**\n", + "**Current Release:** **`v4.0.0`**\n", "\n", "
\n", "\n", @@ -35,7 +35,7 @@ "metadata": {}, "source": [ "## Notebook Purpose\n", - "**Wiki Page:** **[`Release v2.0.0`](https://github.com/callahantiff/PheKnowLator/wiki/v2.0.0)**\n", + "**Wiki Page:** **`Release v4.0.0`**\n", "\n", "
\n", "\n", @@ -50,7 +50,7 @@ " - [`ontology_source_list.txt`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/ontology_source_list.txt)\n", " - [`edge_source_list.txt`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/edge_source_list.txt) \n", "\n", - "3. Prepare [relations](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#relations-data) and [node metadata](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#node-metadata) files prior to running the scripts. \n", + "3. Prepare [relations](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#relations-data) and [metadata](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#metadata) files prior to running the scripts. \n", "\n", "4. Select a knowledge graph build type (i.e. `full`, `partial`, or `post-closure`) and construction method (i.e. `instance-based` or `subclass-based`). \n", "\n", @@ -295,7 +295,7 @@ "**Wiki Pages:** \n", "- **[`KG-Construction`](https://github.com/callahantiff/PheKnowLator/wiki/KG-Construction)** \n", "- **[`relations-data`](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#relations-data)** \n", - "- **[`node-metadata`](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#node-metadata)** \n", + "- **[`metadata`](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#metadata)** \n", "\n", "**Jupyter Notebooks:** \n", "- [`Data_Preparation.ipynb`](https://github.com/callahantiff/PheKnowLator/blob/master/notebooks/Data_Preparation.ipynb) \n", @@ -307,7 +307,7 @@ "**Assumptions:** \n", "- Construction Approach. If using the `subclass-based` construction approach, please make sure that a `pickled` dictionary mapping each non-ontology data node to an existing ontology class is created and added to the `./resources/knowledge_graph` directory (please see [here](https://github.com/callahantiff/PheKnowLator/tree/master/resources/knowledge_graphs#construction-method) for additional information). \n", "- Relations Data. If inverse relation data is going to be used to build the knowledge graph, that it has been generated and added to the `./resources/relations_data` directory (please see [here](https://github.com/callahantiff/PheKnowLator/blob/master/resources/relations_data/README.md) for additional information). \n", - "- Node Metadata. If node metadata is going to be used to build the knowledge graph, that it has been generated and added to the `./resources/node_metadata` directory (please see [here](https://github.com/callahantiff/PheKnowLator/blob/master/resources/node_data/README.md) for additional information). \n", + "- Entity Metadata. If entity metadata is going to be used to build the knowledge graph, it has been generated and added to the `./resources/metadata` directory (please see [here](https://github.com/callahantiff/PheKnowLator/blob/master/resources/metadata/README.md) for additional information). \n", "- Decoding OWL Semantics. If decoding OWL-Semantics, please make sure to provide a list of owl:Property types to keep is created and added to the `./resources/knowledge_graph` directory (please see [here](https://github.com/callahantiff/PheKnowLator/wiki/OWL-NETS-2.0) for additional information). \n", "\n", "
\n", @@ -339,7 +339,7 @@ "\n", "4. Filter OWL Semantics. Filter the knowledge graph with the goal of removing all edges that contain entities that are needed to support owl semantics, but are not biologically meaningful (please see [here](https://github.com/callahantiff/PheKnowLator/wiki/OWL-NETS-2.0) for additional information).\n", "\n", - "5. Save Edge Lists and Node Metadata. Several versions of the knowledge graph are saved, including: the full knowledge graph (`owl` or Networkx MultiDiGraph `pickle`), triple lists (i.e. integer index and identifier labeled edge lists with a dictionary that maps between the integer indices and node identifiers), and a file of metadata (i.e. identifiers, labels, synonyms, and descriptions) for all nodes in the knowledge graph. \n", + "5. Save Edge Lists and Entity Metadata. Several versions of the knowledge graph are saved, including: the full knowledge graph (`owl` or Networkx MultiDiGraph `pickle`), triple lists (i.e. integer index and identifier labeled edge lists with a dictionary that maps between the integer indexes and node identifiers), and a file of metadata (i.e. identifiers, labels, synonyms, and descriptions) for all nodes in the knowledge graph. \n", "\n", "
\n", "\n", @@ -359,7 +359,7 @@ "# specify input arguments\n", "build = 'full'\n", "construction_approach = 'subclass'\n", - "add_node_data_to_kg = 'yes'\n", + "add_metadata_to_kg = 'yes'\n", "add_inverse_relations_to_kg = 'yes'\n", "decode_owl_semantics = 'yes'\n", "kg_directory_location = './resources/knowledge_graphs'\n" @@ -374,21 +374,21 @@ "# construct knowledge graphs\n", "if build == 'partial':\n", " kg = PartialBuild(construction=construction_approach,\n", - " node_data=add_node_data_to_kg,\n", + " metadata=add_metadata_to_kg,\n", " inverse_relations=add_inverse_relations_to_kg,\n", " decode_owl=decode_owl_semantics,\n", " cpus=cpus,\n", " write_location=kg_directory_location)\n", "elif build == 'post-closure':\n", " kg = PostClosureBuild(construction=construction_approach,\n", - " node_data=add_node_data_to_kg,\n", + " metadata=add_metadata_to_kg,\n", " inverse_relations=add_inverse_relations_to_kg,\n", " decode_owl=decode_owl_semantics,\n", " cpus=cpus,\n", " write_location=kg_directory_location)\n", "else:\n", " kg = FullBuild(construction=construction_approach,\n", - " node_data=add_node_data_to_kg,\n", + " metadata=add_metadata_to_kg,\n", " inverse_relations=add_inverse_relations_to_kg,\n", " decode_owl=decode_owl_semantics,\n", " cpus=cpus,\n", diff --git a/notebooks/Data_Preparation.ipynb b/notebooks/Data_Preparation.ipynb index af1fd624..aefef06b 100644 --- a/notebooks/Data_Preparation.ipynb +++ b/notebooks/Data_Preparation.ipynb @@ -6,21 +6,23 @@ "collapsed": true }, "source": [ + "

\n", + " \n", + "

\n", + "\n", "***\n", "***\n", "\n", - "\n", - "\n", "## Pre-Knowledge Graph Build Data Preparation\n", "***\n", "\n", "**Author:** [TJCallahan](https://mail.google.com/mail/u/0/?view=cm&fs=1&tf=1&to=callahantiff@gmail.com) \n", "**GitHub Repository:** [PheKnowLator](https://github.com/callahantiff/PheKnowLator/wiki) \n", - "**Release:** **[v2.0.0](https://github.com/callahantiff/PheKnowLator/wiki/v2.0.0)**\n", + "**Release:** **[`v4.0.0`](https://github.com/callahantiff/PheKnowLator/wiki/v4.0.0)**\n", " \n", "
\n", " \n", - "**Purpose:** This notebook serves as a script to download and process data in order to generate mapping and filtering data needed to build edges for the PheKnowLator knowledge graph. For more information on the data sources utilize within this script, please see the [Data Sources](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources) Wiki page.\n", + "**Purpose:** This notebook serves as a script to download and process data in order to generate mapping and filtering data needed to build edges for the PheKnowLator knowledge graph. For more information on the data sources utilize within this script, please see the [Data Sources](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources) Wiki page.\n", "\n", "
\n", "\n", @@ -32,7 +34,7 @@ "\n", "**Dependencies:** \n", "- **Scripts**: This notebook utilizes several helper functions, which are stored in the [`data_utils.py`](https://github.com/callahantiff/PheKnowLator/blob/master/pkt_kg/utils/data_utils.py) and [`kg_utils.py`](https://github.com/callahantiff/PheKnowLator/blob/master/pkt_kg/utils/kg_utils.py) scripts. \n", - "- **Data**: Hyperlinks to all downloaded and generated data sources are provided through [this](https://console.cloud.google.com/storage/browser/pheknowlator/release_v2.0.0?project=pheknowlator) dedicated Google Cloud Storage Bucket. This notebook will download everything that is needed for you. \n", + "- **Data**: Hyperlinks to all downloaded and generated data sources are provided through [this](https://console.cloud.google.com/storage/browser/pheknowlator/release_v4.0.0?project=pheknowlator) dedicated Google Cloud Storage Bucket. This notebook will download everything that is needed for you. \n", "_____\n", "***" ] @@ -44,14 +46,15 @@ "## Table of Contents\n", "***\n", "\n", - "### [Create Identifier Maps ](#create-identifier-maps) \n", + "### [Identifier Maps ](#create-identifier-maps) \n", "- [HUMAN TRANSCRIPT, GENE, AND PROTEIN IDENTIFIER MAPPING](#human-transcript,-gene,-and-protein-identifier-mapping)\n", " - [Entrez Gene-Ensembl Transcript](#entrezgene-ensembltranscript) \n", " - [Entrez Gene-Protein Ontology](#entrezgene-proteinontology) \n", " - [Ensembl Gene-Entrez Gene](#ensemblgene-entrezgene)\n", " - [Gene Symbol-Ensembl Transcript](#genesymbol-ensembltranscript) \n", " - [STRING-Protein Ontology](#string-proteinontology) \n", - " - [Uniprot Accession-Protein Ontology](#uniprotaccession-proteinontology)\n", + " - [Uniprot Accession-Protein Ontology](#uniprotaccession-proteinontology) \n", + " - [Uniprot Accession-Entrez Gene](#uniprotaccession-entrezgene)\n", " \n", "\n", "- [OTHER IDENTIFIER MAPPING](#other-identifier-mapping) \n", @@ -62,7 +65,7 @@ " - [Genomic Identifiers - Sequence Ontology](#genomic-soo) \n", "\n", "\n", - "### [Create Edge Datasets](#create-edge-datasets)\n", + "### [Edge Datasets](#create-edge-datasets)\n", "- [ONTOLOGIES](#ontologies) \n", " - [Protein Ontology](#protein-ontology) \n", " - [Relations Ontology](#relations-ontology) \n", @@ -73,12 +76,24 @@ " - [Uniprot Protein-Cofactor and Protein-Catalyst](#uniprot-protein-cofactorcatalyst) \n", "\n", "\n", - "### [Create Instance Data and/or Subclass Metadata](#create-instance-metadata) \n", - "- [Genes/RNA](#gene-and-rna-metadata)\n", - "- [Pathways](#pathway-metadata)\n", - "- [Variants](#variant-metadata) \n", - "- [Relations](#relations-metadata) \n", - "\n", + "### [Node and Relation Metadata](#node-relation-metadata) \n", + "- [CTD_chem_gene_ixns.tsv](#chemical-gene) \n", + "- [CTD_chem_go_enriched.tsv](#chemical-go) \n", + "- [CTD_chemicals_diseases.tsv](#chemical-disease) \n", + "- [CTD_genes_pathways.tsv](#gene-pathway) \n", + "- [goa_human.gaf](#goa) \n", + "- [COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt](#gene-gene) \n", + "- [phenotype.hpoa](#phenotype-disease) \n", + "- [ChEBI2Reactome_All_Levels.txt](#chemical-pathway) \n", + "- [gene_association.reactome](#reactome-goa) \n", + "- [UniProt2Reactome_All_Levels.txt](#uniprot-react) \n", + "- [CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt](#variant-disease) \n", + "- [CLINVAR_VARIANT_GENE_EDGES.txt](#variant-gene) \n", + "- [HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt](#hpa) \n", + "- [UNIPROT_PROTEIN_CATALYST.txt](#uniprot-catalyst) \n", + "- [UNIPROT_PROTEIN_COFACTOR.txt](#uniprot-cofactor) \n", + "- [9606.protein.links.v11.0.txt.gz](#protein-protein) \n", + "- [curated_gene_disease_associations.tsv](#gene-phen) \n", "____" ] }, @@ -114,7 +129,7 @@ "metadata": {}, "outputs": [], "source": [ - "# if running a local version of pkt_kg, uncomment the code below\n", + "# # if running a local version of pkt_kg, uncomment the code below\n", "# import sys\n", "# sys.path.append('../')" ] @@ -129,6 +144,7 @@ "import datetime\n", "import glob\n", "import itertools\n", + "import json\n", "import networkx\n", "import numpy\n", "import os\n", @@ -137,6 +153,7 @@ "import pickle\n", "import re\n", "import requests\n", + "import shutil\n", "import sys\n", "\n", "from collections import Counter\n", @@ -144,7 +161,7 @@ "from rdflib import Graph, Namespace, URIRef, BNode, Literal\n", "from rdflib.namespace import OWL, RDF, RDFS\n", "from reactome2py import content\n", - "from tqdm import tqdm\n", + "from tqdm.notebook import tqdm\n", "from typing import Dict\n", "\n", "from pkt_kg.utils import * # import pkt_kg utility script containing helper functions" @@ -171,7 +188,7 @@ "relations_data_location = '../resources/relations_data/'\n", "\n", "# directory to write node metadata to\n", - "node_data_location = '../resources/node_data/'\n", + "metadata_location = '../resources/metadata/'\n", "\n", "# directory to write kg construction approach dictionary to\n", "construction_approach_location = '../resources/construction_approach/'\n", @@ -213,10 +230,10 @@ "\n", "**Data Source Wiki Pages:** \n", "- [Ensembl](https://uswest.ensembl.org/) \n", - "- [Uniprot Knowledgebase](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#uniprot-knowledgebase) \n", + "- [Uniprot Knowledgebase](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#universal-protein-resource-knowledgebase) \n", "- [HGNC](ftp://ftp.ebi.ac.uk/pub/databases/genenames/new/tsv/hgnc_complete_set.txt) \n", - "- [NCBI Gene](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#ncbi-gene) \n", - "- [Protein Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#protein-ontology)\n", + "- [NCBI Gene](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#national-center-for-biotechnology-information-gene) \n", + "- [Protein Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources/#protein-ontology)\n", "\n", "
\n", "\n", @@ -227,6 +244,7 @@ "- [Gene Symbol-Ensembl Transcript](#genesymbol-ensembltranscript) \n", "- [STRING-Protein Ontology](#string-proteinontology) \n", "- [Uniprot Accession-Protein Ontology](#uniprotaccession-proteinontology)\n", + "- [Uniprot Accession-Entrez Gene](#uniprotaccession-entrezgene)\n", "\n", "
\n", "\n", @@ -1016,7 +1034,7 @@ "hgnc = hgnc.loc[hgnc['status'].apply(lambda x: x == 'Approved')]\n", "hgnc = hgnc[['hgnc_id', 'entrez_id', 'ensembl_gene_id', 'uniprot_ids', 'symbol', 'locus_type', 'alias_symbol', 'name', 'location', 'alias_name']]\n", "hgnc.rename(columns={'uniprot_ids': 'uniprot_id', 'location': 'map_location', 'locus_type': 'hgnc_gene_type'}, inplace=True)\n", - "hgnc['hgnc_id'].replace('.*\\:', '', inplace=True, regex=True) # strip 'HGNC' off of the identifiers\n", + "hgnc['hgnc_id'] = hgnc['hgnc_id'].str.replace('.*\\:', '', regex=True) # strip 'HGNC' off of the identifiers\n", "hgnc.fillna('None', inplace=True) # replace NaN with 'None'\n", "hgnc['entrez_id'] = hgnc['entrez_id'].apply(lambda x: str(int(x)) if x != 'None' else 'None') # make col str\n", "\n", @@ -1030,13 +1048,13 @@ "\n", "# reformat hgnc gene type\n", "for val in genomic_type_mapper['hgnc_gene_type'].keys():\n", - " explode_df_hgnc['hgnc_gene_type'].replace(val, genomic_type_mapper['hgnc_gene_type'][val], inplace=True)\n", + " explode_df_hgnc['hgnc_gene_type'] = explode_df_hgnc['hgnc_gene_type'].str.replace(val, genomic_type_mapper['hgnc_gene_type'][val])\n", "\n", "# reformat master hgnc gene type\n", "explode_df_hgnc['master_gene_type'] = explode_df_hgnc['hgnc_gene_type']\n", "master_dict = genomic_type_mapper['hgnc_master_gene_type']\n", "for val in master_dict.keys():\n", - " explode_df_hgnc['master_gene_type'].replace(val, master_dict[val], inplace=True)\n", + " explode_df_hgnc['master_gene_type'] = explode_df_hgnc['master_gene_type'].str.replace(val, master_dict[val])\n", "\n", "# post-process reformatted data\n", "explode_df_hgnc.drop(['alias_symbol', 'alias_name'], axis=1, inplace=True) # remove original gene type column\n", @@ -1104,16 +1122,19 @@ "\n", "# reformat ensembl gene type\n", "gene_dict = genomic_type_mapper['ensembl_gene_type']\n", - "for val in gene_dict.keys(): ensembl_geneset['ensembl_gene_type'].replace(val, gene_dict[val], inplace=True)\n", + "for val in gene_dict.keys():\n", + " ensembl_geneset['ensembl_gene_type'] = ensembl_geneset['ensembl_gene_type'].str.replace(val, gene_dict[val])\n", "# reformat master gene type\n", "ensembl_geneset['master_gene_type'] = ensembl_geneset['ensembl_gene_type']\n", "gene_dict = genomic_type_mapper['ensembl_master_gene_type']\n", - "for val in gene_dict.keys(): ensembl_geneset['master_gene_type'].replace(val, gene_dict[val], inplace=True)\n", + "for val in gene_dict.keys():\n", + " ensembl_geneset['master_gene_type'] = ensembl_geneset['master_gene_type'].str.replace(val, gene_dict[val])\n", "# reformat master transcript type\n", - "ensembl_geneset['ensembl_transcript_type'].replace('vault_RNA', 'vaultRNA', inplace=True, regex=False)\n", + "ensembl_geneset['ensembl_transcript_type'] = ensembl_geneset['ensembl_transcript_type'].str.replace('vault_RNA', 'vaultRNA', regex=False)\n", "ensembl_geneset['master_transcript_type'] = ensembl_geneset['ensembl_transcript_type']\n", "trans_dict = genomic_type_mapper['ensembl_master_transcript_type']\n", - "for val in trans_dict.keys(): ensembl_geneset['master_transcript_type'].replace(val, trans_dict[val], inplace=True)\n", + "for val in trans_dict.keys():\n", + " ensembl_geneset['master_transcript_type'] = ensembl_geneset['master_transcript_type'].str.replace(val, trans_dict[val])\n", "\n", "# post-process reformatted data\n", "ensembl_geneset.drop_duplicates(subset=None, keep='first', inplace=True)\n", @@ -1275,7 +1296,7 @@ "\n", "The URL to access the results of this query is obtained by clicking on the share symbol and copying the free-text from the box. To obtain the data in a tab-delimited format the following string is appended to the end of the URL: \"&format=tab\".\n", "\n", - "**NOTE.** Be sure to obtain a new URL from the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) when rebuilding to ensure you are getting the most up-to-date data. This query was last generated on `12/02/2020`." + "**NOTE.** Be sure to obtain a new URL from the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) when rebuilding to ensure you are getting the most up-to-date data. This query was last generated on `01/30/2022`." ] }, { @@ -1326,7 +1347,7 @@ "explode_df_uniprot = explodes_data(explode_df_uniprot.copy(), ['symbol', 'synonyms'], '|')\n", "\n", "# strip out uniprot names\n", - "explode_df_uniprot['transcript_stable_id'].replace('\\s.*','', inplace=True, regex=True)\n", + "explode_df_uniprot['transcript_stable_id'] = explode_df_uniprot['transcript_stable_id'].str.replace('\\s.*','', regex=True)\n", "\n", "# remove duplicates\n", "explode_df_uniprot.drop(['Status'], axis=1, inplace=True)\n", @@ -1395,18 +1416,20 @@ "# reformat entrez gene type\n", "explode_df_ncbi_gene['entrez_gene_type'] = explode_df_ncbi_gene['type_of_gene']\n", "gene_dict = genomic_type_mapper['entrez_gene_type']\n", - "for val in gene_dict.keys(): explode_df_ncbi_gene['entrez_gene_type'].replace(val, gene_dict[val], inplace=True)\n", + "for val in gene_dict.keys():\n", + " explode_df_ncbi_gene['entrez_gene_type'] = explode_df_ncbi_gene['entrez_gene_type'].str.replace(val, gene_dict[val])\n", "# reformat master gene type\n", "explode_df_ncbi_gene['master_gene_type'] = explode_df_ncbi_gene['entrez_gene_type']\n", "gene_dict = genomic_type_mapper['master_gene_type']\n", - "for val in gene_dict.keys(): explode_df_ncbi_gene['master_gene_type'].replace(val, gene_dict[val], inplace=True)\n", + "for val in gene_dict.keys():\n", + " explode_df_ncbi_gene['master_gene_type'] = explode_df_ncbi_gene['master_gene_type'].str.replace(val, gene_dict[val])\n", "\n", "# post-process reformatted data\n", "explode_df_ncbi_gene.drop(['type_of_gene', 'dbXrefs', 'description', 'Nomenclature_status', 'Modification_date',\n", " 'LocusTag', '#tax_id', 'Full_name_from_nomenclature_authority', 'Feature_type',\n", " 'Symbol_from_nomenclature_authority'], axis=1, inplace=True)\n", - "explode_df_ncbi_gene['hgnc_id'] = explode_df_ncbi_gene['hgnc_id'].replace('HGNC:', '', regex=True)\n", - "explode_df_ncbi_gene['ensembl_gene_id'] = explode_df_ncbi_gene['ensembl_gene_id'].replace('Ensembl:', '', regex=True)\n", + "explode_df_ncbi_gene['hgnc_id'] = explode_df_ncbi_gene['hgnc_id'].str.replace('HGNC:', '', regex=True)\n", + "explode_df_ncbi_gene['ensembl_gene_id'] = explode_df_ncbi_gene['ensembl_gene_id'].str.replace('Ensembl:', '', regex=True)\n", "explode_df_ncbi_gene.drop_duplicates(subset=None, keep='first', inplace=True)\n", "\n", "# preview data\n", @@ -1453,8 +1476,8 @@ "source": [ "pro_map = pro_map.loc[pro_map['entry'].apply(lambda x: x.startswith('Uni') and '_VAR' not in x and ', ' not in x)] # keep 'UniProtKB' rows\n", "pro_map = pro_map.loc[pro_map['pro_mapping'].apply(lambda x: x.startswith('exact'))] # keep exact mappings\n", - "pro_map['pro_id'].replace('PR:','PR_', inplace=True, regex=True) # replace PR: with PR_\n", - "pro_map['entry'].replace('(^\\w*\\:)','', inplace=True, regex=True) # remove id prefixes\n", + "pro_map['pro_id'] = pro_map['pro_id'].str.replace('PR:','PR_', regex=True) # replace PR: with PR_\n", + "pro_map['entry'] = pro_map['entry'].str.replace('(^\\w*\\:)','', regex=True) # remove id prefixes\n", "pro_map = pro_map.loc[pro_map['pro_id'].apply(lambda x: '-' not in x)] # remove isoforms\n", "pro_map.rename(columns={'entry': 'uniprot_id'}, inplace=True) # rename columns before merging\n", "pro_map.drop(['pro_mapping'], axis=1, inplace=True) # remove uneeded columns\n", @@ -1604,12 +1627,12 @@ "merged_data.fillna('None', inplace=True)\n", "\n", "# make sure that all gene and transcript type colunmns have none recoded to unknown or not protein-coding\n", - "merged_data['hgnc_gene_type'].replace('None', 'unknown', inplace=True, regex=False)\n", - "merged_data['ensembl_gene_type'].replace('None', 'unknown', inplace=True, regex=False)\n", - "merged_data['entrez_gene_type'].replace('None', 'unknown', inplace=True, regex=False)\n", - "merged_data['master_gene_type'].replace('None', 'unknown', inplace=True, regex=False)\n", - "merged_data['master_transcript_type'].replace('None', 'not protein-coding', inplace=True, regex=False)\n", - "merged_data['ensembl_transcript_type'].replace('None', 'unknown', inplace=True, regex=False)\n", + "merged_data['hgnc_gene_type'] = merged_data['hgnc_gene_type'].str.replace('None', 'unknown', regex=False)\n", + "merged_data['ensembl_gene_type'] = merged_data['ensembl_gene_type'].str.replace('None', 'unknown', regex=False)\n", + "merged_data['entrez_gene_type'] = merged_data['entrez_gene_type'].str.replace('None', 'unknown', regex=False)\n", + "merged_data['master_gene_type'] = merged_data['master_gene_type'].str.replace('None', 'unknown', regex=False)\n", + "merged_data['master_transcript_type'] = merged_data['master_transcript_type'].str.replace('None', 'not protein-coding', regex=False)\n", + "merged_data['ensembl_transcript_type'] = merged_data['ensembl_transcript_type'].str.replace('None', 'unknown', regex=False)\n", "\n", "# remove duplicates\n", "merged_data_clean = merged_data.drop_duplicates(subset=None, keep='first')\n", @@ -1739,7 +1762,7 @@ "# for _ in range(0, input_size, max_bytes):\n", "# bytes_in += f_in.read(max_bytes)\n", "\n", - "# # load ickled data\n", + "# # load pickled data\n", "# reformatted_mapped_identifiers = pickle.loads(bytes_in)" ] }, @@ -1781,6 +1804,12 @@ " 'Ensembl_Gene_Type', 'Entrez_Gene_Type',\n", " 'Master_Gene_Type1', 'Master_Gene_Type2'])\n", "\n", + "# add prefix to output edge\n", + "egeg_data['Entrez_Gene_IDs'] = 'NCBIGene_' + egeg_data['Entrez_Gene_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "egeg_data.to_csv(processed_data_location + 'ENSEMBL_GENE_ENTREZ_GENE_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} ensembl gene-entrez gene edges'.format(edge_count=len(egeg_data)))\n", "egeg_data.head(n=5)" ] @@ -1821,6 +1850,12 @@ " names=['Ensembl_Transcript_IDs', 'Protein_Ontology_IDs',\n", " 'Ensembl_Transcript_Type', 'Master_Transcript_Type'])\n", "\n", + "# add prefix to output edge\n", + "etpr_data['Ensembl_Transcript_ID_Edge'] = 'ensembl_' + etpr_data['Ensembl_Transcript_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "etpr_data.to_csv(processed_data_location + 'ENSEMBL_TRANSCRIPT_PROTEIN_ONTOLOGY_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} ensembl transcript-protein ontology edges'.format(edge_count=len(etpr_data)))\n", "etpr_data.head(n=5)" ] @@ -1862,6 +1897,13 @@ " 'Entrez_Gene_Type', 'Ensembl_Transcript_Type',\n", " 'Master_Gene_Type', 'Master_Transcript_Type'])\n", "\n", + "# add prefix to output edge\n", + "eet_data['Ensembl_Transcript_IDs'] = 'ensembl_' + eet_data['Ensembl_Transcript_IDs'].astype(str)\n", + "eet_data['Entrez_Gene_Edge'] = 'NCBIGene_' + eet_data['Entrez_Gene_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "eet_data.to_csv(processed_data_location + 'ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} entrez gene identifiers-ensembl transcript edges'.format(edge_count=len(eet_data)))\n", "eet_data.head(n=5)" ] @@ -1904,6 +1946,12 @@ " names=['Gene_IDs', 'Protein_Ontology_IDs',\n", " 'Entrez_Gene_Type', 'Master_Gene_Type'])\n", "\n", + "# add prefix to output edge\n", + "egpr_data['Entrez_Gene_Edge'] = 'NCBIGene_' + egpr_data['Gene_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "egpr_data.to_csv(processed_data_location + 'ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} entrez gene-protein ontology edges'.format(edge_count=len(egpr_data)))\n", "egpr_data.head(n=5)" ] @@ -1948,6 +1996,12 @@ " 'Gene_Type', 'Ensembl_Transcript_Type',\n", " 'Master_Gene_Type', 'Master_Transcript_Type'])\n", "\n", + "# add prefix to output edge\n", + "set_data['Ensembl_Transcript_IDs'] = 'ensembl_' + set_data['Ensembl_Transcript_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "set_data.to_csv(processed_data_location + 'GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} gene symbol-ensembl transcript edges'.format(edge_count=len(set_data.drop_duplicates())))\n", "set_data.head(n=5)" ] @@ -1987,6 +2041,12 @@ " header=None, delimiter='\\t', low_memory=False, usecols=[0, 1],\n", " names=['STRING_IDs', 'Protein_Ontology_IDs'])\n", "\n", + "# add prefix to output edge\n", + "stpr_data['STRING_IDs'] = '9606.' + stpr_data['STRING_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "stpr_data.to_csv(processed_data_location + 'STRING_PRO_ONTOLOGY_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", "print('There are {edge_count} string-protein ontology edges'.format(edge_count=len(stpr_data.drop_duplicates())))\n", "stpr_data.head(n=5)" ] @@ -2036,6 +2096,53 @@ "uapr_data.head(n=5)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "### Uniprot Accession-Entrez Gene \n", + "\n", + "**Purpose:** To map Uniprot accession identifiers to Entrez Gene identifiers when creating the following edges: \n", + "- gene-gene \n", + "\n", + "**Output:** `UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "genomic_id_mapper(reformatted_mapped_identifiers,\n", + " processed_data_location + 'UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt',\n", + " 'uniprot_id', 'entrez_id', None, 'master_gene_type', None, 'gene_type_update')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load data, print the number of rows, and preview it\n", + "uaeg_data = pandas.read_csv(processed_data_location + 'UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False, usecols=[0, 1, 2, 3],\n", + " names=['Uniprot_Accession_IDs', 'Entrez_Gene_IDs',\n", + " 'master_gene_type', 'gene_type_update'])\n", + "\n", + "# add prefix to output edge\n", + "uaeg_data['Entrez_Gene_IDs'] = 'NCBIGene_' + uaeg_data['Entrez_Gene_IDs'].astype(str)\n", + "\n", + "# write data back to file\n", + "uaeg_data.to_csv(processed_data_location + 'UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt', header=None, sep='\\t', index=False)\n", + "\n", + "print('There are {edge_count} uniprot accession-entrez gene edges'.format(edge_count=len(uaeg_data.drop_duplicates())))\n", + "uaeg_data.head(n=5)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2062,7 +2169,7 @@ "source": [ "### ChEBI-MeSH Identifiers \n", "\n", - "**Data Source Wiki Page:** [mapping-mesh-to-chebi](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#mapping-mesh-identifiers-to-chebi-identifiers) \n", + "**Data Source Wiki Page:** [mapping-mesh-to-chebi](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#mapping-mesh-identifiers-to-chebi-identifiers) \n", "\n", "**Purpose:** Map MeSH identifiers to ChEBI identifiers when creating the following edges: \n", "- chemical-gene \n", @@ -2226,7 +2333,7 @@ "# write resulting mappings\n", "with open(processed_data_location + 'MESH_CHEBI_MAP.txt', 'w') as out:\n", " for pair in mesh_edges:\n", - " out.write(pair[0] + '\\t' + pair[1] + '\\n')" + " out.write(pair[0].replace('_', ':') + '\\t' + pair[1] + '\\n')" ] }, { @@ -2251,9 +2358,11 @@ "\n", "### Disease and Phenotype Identifiers \n", "\n", - "**Data Source Wiki Page:** [DisGeNET](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#disgenet) \n", + "**Data Source Wiki Page:** \n", + "- [DisGeNET](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#disgenet) \n", + "- [MedGen](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#national-center-for-biotechnology-information-medgen) \n", "\n", - "**Purpose:** This script downloads the Human Phenotype Ontology (HPO), the MonDO Disease Ontology (MONDO), and [disease_mappings.tsv](https://www.disgenet.org/static/disgenet_ap1/files/downloads/disease_mappings.tsv.gz) in order to map UMLS identifiers to HPO and MONDO identifiers when creating the following edges: \n", + "**Purpose:** This script downloads the Human Phenotype Ontology (HPO), the MonDO Disease Ontology (MONDO), [disease_mappings.tsv](https://www.disgenet.org/static/disgenet_ap1/files/downloads/disease_mappings.tsv.gz), and [MGCONSO.RRF](https://ftp.ncbi.nlm.nih.gov/pub/medgen/MGCONSO.RRF.gz) in order to map UMLS identifiers to HPO and MONDO identifiers when creating the following edges: \n", "- chemical-disease \n", "- disease-phenotype \n", "- chemical-phenotype \n", @@ -2295,7 +2404,24 @@ "mondo_dict = {str(k).lower().split('/')[-1]: {str(i).split('/')[-1].replace('_', ':') for i in v} for k, v in dbxref_res.items() if 'MONDO' in str(v)}\n", "\n", "# pickle dictionary\n", - "pickle.dump(mondo_dict, open(processed_data_location + 'Mondo_Identifier_Map.pkl', 'wb'), protocol=4)" + "pickle.dump(mondo_dict, open(processed_data_location + 'Mondo_Identifier_Map.pkl', 'wb'), protocol=4)\n", + "\n", + "# convert to pandas DataFrame\n", + "temp_list = []\n", + "for k, v in mondo_dict.items():\n", + " if k.startswith('umls:'): new_k = k.split(':')[-1].upper()\n", + " elif k.startswith('hp:'): new_k = k.upper()\n", + " elif k.startswith('mesh:'): new_k = 'MESH:' + k.split(':')[-1].upper()\n", + " elif k.startswith('orphanet:'): new_k = 'ORPHA:' + k.split(':')[-1].upper()\n", + " elif k.startswith('omimps:'): new_k = 'OMIM:' + k.split(':')[-1].upper()\n", + " else: new_k = k\n", + " for i in v:\n", + " temp_list += [[new_k, i.replace(':', '_')]]\n", + " temp_list += [[i, i.replace(':', '_')]]\n", + "\n", + " # convert to \n", + "mondo_df = pandas.DataFrame({'other_id': [x[0] for x in temp_list],\n", + " 'ontology_id': [x[1] for x in temp_list]})" ] }, { @@ -2328,7 +2454,43 @@ "hp_dict = {str(k).lower().split('/')[-1]: {str(i).split('/')[-1].replace('_', ':') for i in v} for k, v in dbxref_res.items() if 'HP' in str(v)}\n", "\n", "# pickle dictionary\n", - "pickle.dump(hp_dict, open(processed_data_location + 'HPO_Identifier_Map.pkl', 'wb'), protocol=4)" + "pickle.dump(hp_dict, open(processed_data_location + 'HPO_Identifier_Map.pkl', 'wb'), protocol=4)\n", + "\n", + "# convert to pandas DataFrame\n", + "temp_list = []\n", + "for k, v in hp_dict.items():\n", + " if k.startswith('umls:'): new_k = k.split(':')[-1].upper()\n", + " elif k.startswith('mondo:'): new_k = k.upper()\n", + " elif k.startswith('msh:'): new_k = 'MESH:' + k.split(':')[-1].upper()\n", + " elif k.startswith('orpha:'): new_k = 'ORPHA:' + k.split(':')[-1].upper()\n", + " else: new_k = k\n", + " for i in v:\n", + " temp_list += [[new_k, i.replace(':', '_')]]\n", + " temp_list += [[i, i.replace(':', '_')]]\n", + "\n", + "# convert to \n", + "hp_df = pandas.DataFrame({'other_id': [x[0] for x in temp_list],\n", + " 'ontology_id': [x[1] for x in temp_list]})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Combine MONDO and HP Disease Mapping DataFrames into a Single DataFrame*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# combine data frames\n", + "disease_map_df = pandas.concat([mondo_df, hp_df])\n", + "\n", + "# preview data\n", + "disease_map_df.head(n=5)" ] }, { @@ -2356,7 +2518,26 @@ "# reformat data\n", "disease_data['vocabulary'] = disease_data['vocabulary'].str.lower()\n", "disease_data['diseaseId'] = disease_data['diseaseId'].str.lower()\n", - "disease_data['vocabulary'] = ['doid' if x == 'do' else 'ordoid' if x == 'ordo' else x for x in disease_data['vocabulary']]\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('hpo', 'HP')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('mondo', 'MONDO')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('msh', 'MESH')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('omim', 'OMIM')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('do', 'doid')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('ordo', 'ORPHA')\n", + "disease_data['vocabulary'] = disease_data['vocabulary'].str.replace('ORPHAid', 'ORPHA')\n", + "\n", + "# capitalize UMLS id\n", + "disease_data['diseaseId'] = disease_data['diseaseId'].str.upper()\n", + "\n", + "# create a disease code column\n", + "disease_data['code'] = disease_data['vocabulary'] + ':' + disease_data['code']\n", + "disease_data['code'] = disease_data['code'].str.replace('HP:HP:', 'HP:')\n", + "\n", + "# rename columns\n", + "disease_data.rename(columns={'diseaseId': 'cui', 'vocabularyName': 'code_name'}, inplace=True)\n", + "\n", + "# remove unneeded columns\n", + "disease_data = disease_data[['cui', 'code', 'code_name', 'vocabulary']].drop_duplicates()\n", "\n", "# preview data\n", "disease_data.head(n=3)" @@ -2366,8 +2547,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "_Build Disease Identifier Dictionary_ \n", - "In order to improve efficiency when mapping different disease terminology identifiers to the [MonDO Disease Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#mondo-disease-ontology) and [Human Phenotype Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#human-phenotype-ontology), we create a dictionary of disease identifiers." + "***\n", + "**MedGen Disease Mappings**" ] }, { @@ -2376,22 +2557,51 @@ "metadata": {}, "outputs": [], "source": [ - "# get all CUIs found with HPO and MONDO\n", - "disease_data_keep = disease_data.query('vocabulary == \"hpo\" | vocabulary == \"mondo\"')\n", + "# download data\n", + "url = 'https://ftp.ncbi.nlm.nih.gov/pub/medgen/MGCONSO.RRF.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'MGCONSO.RRF'):\n", + " data_downloader(url, unprocessed_data_location)\n", + " \n", + "# load data and clean data\n", + "medgen_data = pandas.read_csv(unprocessed_data_location + 'MGCONSO.RRF', header=0, delimiter='|')\n", + "medgen_data = medgen_data[medgen_data['SUPPRESS'] == 'N'].drop_duplicates()\n", + "medgen_data = medgen_data[medgen_data['SAB'].isin(['HPO', 'MONDO', 'MSH', 'ORDO', 'OMIM'])].drop_duplicates()\n", + "\n", + "# reformat codes\n", + "medgen_data['temp_code'] = medgen_data.apply(lambda x: 'MESH:' + x['CODE'] if x['SAB'] == 'MSH'\n", + " else 'OMIM:' + x['CODE'] if x['SAB'] == 'OMIM'\n", + " else 'ORPHA:' + x['SDUI'].split('_')[-1] if x['SAB'] == 'ORDO'\n", + " else x['SDUI'] if x['SAB'] == 'HPO'\n", + " else x['SDUI'] if x['SAB'] == 'MONDO'\n", + " else 'None', axis=1)\n", + "\n", + "# add rows for MedGen identifiers\n", + "temp = medgen_data[['#CUI']]\n", + "temp['temp_code'] = 'MedGen:' + medgen_data['#CUI']\n", + "medgen_data = pandas.concat([medgen_data, temp])\n", + "\n", + "# remove unneeded columns\n", + "medgen_data = medgen_data[['#CUI', 'temp_code', 'STR', 'SAB']].drop_duplicates()\n", + "\n", + "# rename columns\n", + "medgen_data.rename(columns={'#CUI': 'cui',\n", + " 'STR': 'code_name',\n", + " 'temp_code': 'code',\n", + " 'SAB': 'vocabulary'}, inplace=True)\n", "\n", - "# create mondo and hpo dictionary\n", - "hp_mondo_dict = {}\n", - "for idx, row in tqdm(disease_data_keep.iterrows(), total=disease_data_keep.shape[0]):\n", - " if row['vocabulary'] == 'mondo': key, value = 'umls:' + row['diseaseId'], 'MONDO:' + row['code']\n", - " else: key, value = 'umls:' + row['diseaseId'], row['code']\n", - " if key in hp_mondo_dict.keys(): hp_mondo_dict[key] |= {value}\n", - " else: hp_mondo_dict[key] = {value}\n", - "# add ontology mappings from MONDO and HPO\n", - "for key in tqdm(hp_mondo_dict.keys()):\n", - " if key in mondo_dict.keys():\n", - " hp_mondo_dict[key] = set(list(hp_mondo_dict[key]) + list(mondo_dict[key]))\n", - " if key in hp_dict.keys():\n", - " hp_mondo_dict[key] = set(list(hp_mondo_dict[key]) + list(hp_dict[key]))" + "# reformat vocabulary ids\n", + "medgen_data['vocabulary'] = medgen_data['vocabulary'].str.replace('HPO', 'HP')\n", + "medgen_data['vocabulary'] = medgen_data['vocabulary'].str.replace('MSH', 'MESH')\n", + "\n", + "# preview data\n", + "medgen_data.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Combine DisGeNET and MedGen Mappings*" ] }, { @@ -2400,25 +2610,61 @@ "metadata": {}, "outputs": [], "source": [ - "# get all rows for HPO/MONDO CUIs to obtain mappings to other disease identifiers\n", - "disease_data_other = disease_data[disease_data.diseaseId.isin(disease_data_keep['diseaseId'])]\n", + "# combine data\n", + "disease_mapping_data = pandas.concat([disease_data, medgen_data]).drop_duplicates()\n", "\n", - "# get all other codes that map to MONDO or HPO by hopping through MONDO/HPO relevant CUIs\n", - "disease_dict = {}\n", - "for idx, row in tqdm(disease_data_other.iterrows(), total=disease_data_other.shape[0]):\n", - " if row['vocabulary'] == 'mondo' or row['vocabulary'] == 'hpo':\n", - " key, value = 'umls:' + row['diseaseId'].lower(), row['code']\n", - " if key in disease_dict.keys(): disease_dict[key] |= {value}\n", - " else: disease_dict[key] = {value}\n", - " else:\n", - " if 'mondo' not in row['code'] or 'hp' not in row['code']:\n", - " if ':' not in row['code']: key, value = row['vocabulary'] + ':' + row['code'], hp_mondo_dict['umls:' + row['diseaseId']]\n", - " else: key, value = row['code'], hp_mondo_dict['umls:' + row['diseaseId']]\n", - " if key in disease_dict.keys(): disease_dict[key] |= value\n", - " else: disease_dict[key] = value\n", + "# preview data\n", + "disease_mapping_data.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_Build Disease Identifier Dictionary_ \n", + "In order to improve efficiency when mapping different disease terminology identifiers to the [MonDO Disease Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#mondo-disease-ontology) and [Human Phenotype Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-phenotype-ontology), we create a dictionary of disease identifiers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# find cuis that map to HP or MONDO\n", + "disease_data_keep = disease_mapping_data.copy()\n", + "disease_data_keep = disease_data_keep.query('vocabulary == \"HP\" | vocabulary == \"MONDO\"')\n", + "disease_data_keep = disease_data_keep[['cui', 'code']]\n", + "cui_list = set(disease_data_keep['cui'])\n", + "\n", + "# obtain a list of other ids that map to the cuis\n", + "temp_df = disease_mapping_data[disease_mapping_data['cui'].isin(cui_list)]\n", + "\n", + "# merge back with original data\n", + "merged_temp = temp_df.merge(disease_data_keep, on='cui')\n", + "merged_temp = merged_temp[['code_x', 'code_y', 'code_name', 'vocabulary']].drop_duplicates()\n", + "\n", + "# rename the columns\n", + "merged_temp.rename(columns={'code_x': 'cui', 'code_y': 'code'}, inplace=True)\n", + "\n", + "# combine the columns back to main data\n", + "disease_mapping_data = pandas.concat([disease_mapping_data, merged_temp]).drop_duplicates()\n", + "disease_mapping_data = disease_mapping_data[['cui', 'code']].drop_duplicates()\n", + "\n", + "# merge ontology and other mappings together\n", + "cleaned_disease_map = disease_mapping_data.merge(disease_map_df, left_on='cui', right_on='other_id')\n", + "\n", + "# clean up file\n", + "cleaned_disease_map = cleaned_disease_map[['cui', 'ontology_id']]\n", + "cleaned_disease_map.rename(columns={'cui': 'disease_id'}, inplace=True)\n", + "\n", + "# format ontology identifiers\n", + "cleaned_disease_map['ontology_id'] = cleaned_disease_map['ontology_id'].str.replace(':', '_')\n", + "cleaned_disease_map['vocabulary'] = cleaned_disease_map['ontology_id'].str.replace('\\_.*', '', regex=True)\n", + "cleaned_disease_map.drop_duplicates(inplace=True)\n", "\n", - "# add ontology dictionaries\n", - "disease_dict = {**disease_dict, **mondo_dict, **hp_dict}" + "# preview data\n", + "cleaned_disease_map.head(n=3)" ] }, { @@ -2431,17 +2677,21 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "code_folding": [] + }, "outputs": [], "source": [ - "with open(processed_data_location + 'DISEASE_MONDO_MAP.txt', 'w') as outfile1, open(processed_data_location + 'PHENOTYPE_HPO_MAP.txt', 'w') as outfile2:\n", - " for k, v in tqdm(disease_dict.items()):\n", - " if any(x for x in v if x.startswith('MONDO')):\n", - " for idx in [x.replace(':', '_') for x in v if 'MONDO' in x]:\n", - " outfile1.write(k.upper().split(':')[-1] + '\\t' + idx + '\\n')\n", - " if any(x for x in v if x.startswith('HP')):\n", - " for idx in [x.replace(':', '_') for x in v if 'HP' in x]:\n", - " outfile2.write(k.upper().split(':')[-1] + '\\t' + idx + '\\n')" + "# split data by ontology and write to file\n", + "mondo_map = cleaned_disease_map[cleaned_disease_map['vocabulary'] == 'MONDO'].drop_duplicates()\n", + "hp_map = cleaned_disease_map[cleaned_disease_map['vocabulary'] == 'HP'].drop_duplicates()\n", + "mondo_map = mondo_map[['disease_id', 'ontology_id']]\n", + "hp_map = hp_map[['disease_id', 'ontology_id']]\n", + "\n", + "\n", + "# write data\n", + "mondo_map.to_csv(processed_data_location + 'DISEASE_MONDO_MAP.txt', header=None, index=False, sep='\\t')\n", + "hp_map.to_csv(processed_data_location + 'PHENOTYPE_HPO_MAP.txt', header=None, index=False, sep='\\t')" ] }, { @@ -2493,12 +2743,12 @@ "### Human Protein Atlas/GTEx Tissue/Cells - UBERON + Cell Ontology + Cell Line Ontology \n", "\n", "**Data Source Wiki Page:** \n", - "- [human-protein-atlas](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#human-protein-atlas) \n", - "- [genotype-tissue-expression-project](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#the-genotype-tissue-expression-gtex-project) \n", + "- [human-protein-atlas](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-protein-atlas) \n", + "- [genotype-tissue-expression-project](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#genotype-tissue-expression-project) \n", "\n", "
\n", "\n", - "**Purpose:** Downloads a query for cell, tissue, and blood types with overexpressed protein-coding genes in the human proteome ([`proteinatlas_search.tsv`](https://www.proteinatlas.org/api/search_download.php?search=&columns=g,eg,up,pe,rnatsm,rnaclsm,rnacasm,rnabrsm,rnabcsm,rnablsm,scl,t_RNA_adipose_tissue,t_RNA_adrenal_gland,t_RNA_amygdala,t_RNA_appendix,t_RNA_basal_ganglia,t_RNA_bone_marrow,t_RNA_breast,t_RNA_cerebellum,t_RNA_cerebral_cortex,t_RNA_cervix,_uterine,t_RNA_colon,t_RNA_corpus_callosum,t_RNA_ductus_deferens,t_RNA_duodenum,t_RNA_endometrium_1,t_RNA_epididymis,t_RNA_esophagus,t_RNA_fallopian_tube,t_RNA_gallbladder,t_RNA_heart_muscle,t_RNA_hippocampal_formation,t_RNA_hypothalamus,t_RNA_kidney,t_RNA_liver,t_RNA_lung,t_RNA_lymph_node,t_RNA_midbrain,t_RNA_olfactory_region,t_RNA_ovary,t_RNA_pancreas,t_RNA_parathyroid_gland,t_RNA_pituitary_gland,t_RNA_placenta,t_RNA_pons_and_medulla,t_RNA_prostate,t_RNA_rectum,t_RNA_retina,t_RNA_salivary_gland,t_RNA_seminal_vesicle,t_RNA_skeletal_muscle,t_RNA_skin_1,t_RNA_small_intestine,t_RNA_smooth_muscle,t_RNA_spinal_cord,t_RNA_spleen,t_RNA_stomach_1,t_RNA_testis,t_RNA_thalamus,t_RNA_thymus,t_RNA_thyroid_gland,t_RNA_tongue,t_RNA_tonsil,t_RNA_urinary_bladder,t_RNA_vagina,t_RNA_B-cells,t_RNA_dendritic_cells,t_RNA_granulocytes,t_RNA_monocytes,t_RNA_NK-cells,t_RNA_T-cells,t_RNA_total_PBMC,cell_RNA_A-431,cell_RNA_A549,cell_RNA_AF22,cell_RNA_AN3-CA,cell_RNA_ASC_diff,cell_RNA_ASC_TERT1,cell_RNA_BEWO,cell_RNA_BJ,cell_RNA_BJ_hTERT+,cell_RNA_BJ_hTERT+_SV40_Large_T+,cell_RNA_BJ_hTERT+_SV40_Large_T+_RasG12V,cell_RNA_CACO-2,cell_RNA_CAPAN-2,cell_RNA_Daudi,cell_RNA_EFO-21,cell_RNA_fHDF/TERT166,cell_RNA_HaCaT,cell_RNA_HAP1,cell_RNA_HBEC3-KT,cell_RNA_HBF_TERT88,cell_RNA_HDLM-2,cell_RNA_HEK_293,cell_RNA_HEL,cell_RNA_HeLa,cell_RNA_Hep_G2,cell_RNA_HHSteC,cell_RNA_HL-60,cell_RNA_HMC-1,cell_RNA_HSkMC,cell_RNA_hTCEpi,cell_RNA_hTEC/SVTERT24-B,cell_RNA_hTERT-HME1,cell_RNA_HUVEC_TERT2,cell_RNA_K-562,cell_RNA_Karpas-707,cell_RNA_LHCN-M2,cell_RNA_MCF7,cell_RNA_MOLT-4,cell_RNA_NB-4,cell_RNA_NTERA-2,cell_RNA_PC-3,cell_RNA_REH,cell_RNA_RH-30,cell_RNA_RPMI-8226,cell_RNA_RPTEC_TERT1,cell_RNA_RT4,cell_RNA_SCLC-21H,cell_RNA_SH-SY5Y,cell_RNA_SiHa,cell_RNA_SK-BR-3,cell_RNA_SK-MEL-30,cell_RNA_T-47d,cell_RNA_THP-1,cell_RNA_TIME,cell_RNA_U-138_MG,cell_RNA_U-2_OS,cell_RNA_U-2197,cell_RNA_U-251_MG,cell_RNA_U-266/70,cell_RNA_U-266/84,cell_RNA_U-698,cell_RNA_U-87_MG,cell_RNA_U-937,cell_RNA_WM-115,blood_RNA_basophil,blood_RNA_classical_monocyte,blood_RNA_eosinophil,blood_RNA_gdT-cell,blood_RNA_intermediate_monocyte,blood_RNA_MAIT_T-cell,blood_RNA_memory_B-cell,blood_RNA_memory_CD4_T-cell,blood_RNA_memory_CD8_T-cell,blood_RNA_myeloid_DC,blood_RNA_naive_B-cell,blood_RNA_naive_CD4_T-cell,blood_RNA_naive_CD8_T-cell,blood_RNA_neutrophil,blood_RNA_NK-cell,blood_RNA_non-classical_monocyte,blood_RNA_plasmacytoid_DC,blood_RNA_T-reg,blood_RNA_total_PBMC,brain_RNA_amygdala,brain_RNA_basal_ganglia,brain_RNA_cerebellum,brain_RNA_cerebral_cortex,brain_RNA_hippocampal_formation,brain_RNA_hypothalamus,brain_RNA_midbrain,brain_RNA_olfactory_region,brain_RNA_pons_and_medulla,brain_RNA_thalamus&format=tsv)) via [API](https://www.proteinatlas.org/about/help/dataaccess) and median gene-level TPM by tissue for all genes that are not protein-coding ([`GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct`](https://storage.googleapis.com/gtex_analysis_v8/rna_seq_data/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz)) in order to create mappings between cell and tissue type strings to the Uber-Anatomy, Cell Ontology, and Cell Line Ontology concepts (see [human-protein-atlas](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#human-protein-atlas) for details on the mapping process). The mappings are then used to create the following edge types: \n", + "**Purpose:** Downloads a query for cell, tissue, and blood types with overexpressed protein-coding genes in the human proteome ([`proteinatlas_search.tsv`](https://www.proteinatlas.org/api/search_download.php?search=&columns=g,eg,up,pe,rnatsm,rnaclsm,rnacasm,rnabrsm,rnabcsm,rnablsm,scl,t_RNA_adipose_tissue,t_RNA_adrenal_gland,t_RNA_amygdala,t_RNA_appendix,t_RNA_basal_ganglia,t_RNA_bone_marrow,t_RNA_breast,t_RNA_cerebellum,t_RNA_cerebral_cortex,t_RNA_cervix,_uterine,t_RNA_colon,t_RNA_corpus_callosum,t_RNA_ductus_deferens,t_RNA_duodenum,t_RNA_endometrium_1,t_RNA_epididymis,t_RNA_esophagus,t_RNA_fallopian_tube,t_RNA_gallbladder,t_RNA_heart_muscle,t_RNA_hippocampal_formation,t_RNA_hypothalamus,t_RNA_kidney,t_RNA_liver,t_RNA_lung,t_RNA_lymph_node,t_RNA_midbrain,t_RNA_olfactory_region,t_RNA_ovary,t_RNA_pancreas,t_RNA_parathyroid_gland,t_RNA_pituitary_gland,t_RNA_placenta,t_RNA_pons_and_medulla,t_RNA_prostate,t_RNA_rectum,t_RNA_retina,t_RNA_salivary_gland,t_RNA_seminal_vesicle,t_RNA_skeletal_muscle,t_RNA_skin_1,t_RNA_small_intestine,t_RNA_smooth_muscle,t_RNA_spinal_cord,t_RNA_spleen,t_RNA_stomach_1,t_RNA_testis,t_RNA_thalamus,t_RNA_thymus,t_RNA_thyroid_gland,t_RNA_tongue,t_RNA_tonsil,t_RNA_urinary_bladder,t_RNA_vagina,t_RNA_B-cells,t_RNA_dendritic_cells,t_RNA_granulocytes,t_RNA_monocytes,t_RNA_NK-cells,t_RNA_T-cells,t_RNA_total_PBMC,cell_RNA_A-431,cell_RNA_A549,cell_RNA_AF22,cell_RNA_AN3-CA,cell_RNA_ASC_diff,cell_RNA_ASC_TERT1,cell_RNA_BEWO,cell_RNA_BJ,cell_RNA_BJ_hTERT+,cell_RNA_BJ_hTERT+_SV40_Large_T+,cell_RNA_BJ_hTERT+_SV40_Large_T+_RasG12V,cell_RNA_CACO-2,cell_RNA_CAPAN-2,cell_RNA_Daudi,cell_RNA_EFO-21,cell_RNA_fHDF/TERT166,cell_RNA_HaCaT,cell_RNA_HAP1,cell_RNA_HBEC3-KT,cell_RNA_HBF_TERT88,cell_RNA_HDLM-2,cell_RNA_HEK_293,cell_RNA_HEL,cell_RNA_HeLa,cell_RNA_Hep_G2,cell_RNA_HHSteC,cell_RNA_HL-60,cell_RNA_HMC-1,cell_RNA_HSkMC,cell_RNA_hTCEpi,cell_RNA_hTEC/SVTERT24-B,cell_RNA_hTERT-HME1,cell_RNA_HUVEC_TERT2,cell_RNA_K-562,cell_RNA_Karpas-707,cell_RNA_LHCN-M2,cell_RNA_MCF7,cell_RNA_MOLT-4,cell_RNA_NB-4,cell_RNA_NTERA-2,cell_RNA_PC-3,cell_RNA_REH,cell_RNA_RH-30,cell_RNA_RPMI-8226,cell_RNA_RPTEC_TERT1,cell_RNA_RT4,cell_RNA_SCLC-21H,cell_RNA_SH-SY5Y,cell_RNA_SiHa,cell_RNA_SK-BR-3,cell_RNA_SK-MEL-30,cell_RNA_T-47d,cell_RNA_THP-1,cell_RNA_TIME,cell_RNA_U-138_MG,cell_RNA_U-2_OS,cell_RNA_U-2197,cell_RNA_U-251_MG,cell_RNA_U-266/70,cell_RNA_U-266/84,cell_RNA_U-698,cell_RNA_U-87_MG,cell_RNA_U-937,cell_RNA_WM-115,blood_RNA_basophil,blood_RNA_classical_monocyte,blood_RNA_eosinophil,blood_RNA_gdT-cell,blood_RNA_intermediate_monocyte,blood_RNA_MAIT_T-cell,blood_RNA_memory_B-cell,blood_RNA_memory_CD4_T-cell,blood_RNA_memory_CD8_T-cell,blood_RNA_myeloid_DC,blood_RNA_naive_B-cell,blood_RNA_naive_CD4_T-cell,blood_RNA_naive_CD8_T-cell,blood_RNA_neutrophil,blood_RNA_NK-cell,blood_RNA_non-classical_monocyte,blood_RNA_plasmacytoid_DC,blood_RNA_T-reg,blood_RNA_total_PBMC,brain_RNA_amygdala,brain_RNA_basal_ganglia,brain_RNA_cerebellum,brain_RNA_cerebral_cortex,brain_RNA_hippocampal_formation,brain_RNA_hypothalamus,brain_RNA_midbrain,brain_RNA_olfactory_region,brain_RNA_pons_and_medulla,brain_RNA_thalamus&format=tsv)) via [API](https://www.proteinatlas.org/about/help/dataaccess) and median gene-level TPM by tissue for all genes that are not protein-coding ([`GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct`](https://storage.googleapis.com/gtex_analysis_v8/rna_seq_data/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz)) in order to create mappings between cell and tissue type strings to the Uber-Anatomy, Cell Ontology, and Cell Line Ontology concepts (see [human-protein-atlas](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-protein-atlas) for details on the mapping process). The mappings are then used to create the following edge types: \n", "- rna-cell line \n", "- rna-tissue type \n", "- protein-cell line \n", @@ -2546,8 +2796,8 @@ "# retrieve terms to map and write results\n", "with open(unprocessed_data_location + 'HPA_tissues.txt', 'w') as outfile:\n", " for x in tqdm(list(hpa.columns)):\n", - " if x.endswith('[NX]'):\n", - " outfile.write(x.split('RNA - ')[-1].split(' [NX]')[:-1][0] + '\\n')" + " if x.endswith('[nTPM]'):\n", + " outfile.write(x.split('RNA - ')[-1].split(' [nTPM]')[:-1][0] + '\\n')" ] }, { @@ -2556,7 +2806,7 @@ "source": [ "***\n", "**Genotype-Tissue Expression Project** \n", - "Import the tissues, cells, cell lines, and fluids that we externally mapped from HPA and GTEx data to [UBERON](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#uber-anatomy-ontology), the [Cell Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#cell-ontology), and the [Cell Line Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#cell-line-ontology)." + "Import the tissues, cells, cell lines, and fluids that we externally mapped from HPA and GTEx data to [UBERON](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#uber-anatomy-ontology), the [Cell Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#cell-ontology), and the [Cell Line Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#cell-line-ontology)." ] }, { @@ -2573,7 +2823,7 @@ "# load data\n", "gtex = pandas.read_csv(unprocessed_data_location + 'GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct', header=0, skiprows=2, delimiter='\\t')\n", "gtex.fillna('None', inplace=True) # replace NaN with 'None'\n", - "gtex['Name'].replace('(\\..*)','', inplace=True, regex=True) # remove identifier type, which appears after '.'\n" + "gtex['Name'] = gtex['Name'].str.replace('(\\..*)','', regex=True) # remove identifier type, which appears after '.'\n" ] }, { @@ -2639,7 +2889,7 @@ "**Create Edge Data Set**\n", "\n", "_Human Protein Atlas_ \n", - "The `HPA` data is looped over and reformatted such that all tissue, cell, cell lines, and fluid types are stored as a nested list. The anatomy type is specified as an item in the list according to its type in order to make mapping more efficient while building the knowledge graph edge list." + "hpaThe `HPA` data is looped over and reformatted such that all tissue, cell, cell lines, and fluid types are stored as a nested list. The anatomy type is specified as an item in the list according to its type in order to make mapping more efficient while building the knowledge graph edge list." ] }, { @@ -2650,22 +2900,53 @@ "source": [ "hpa_results = []\n", "for idx, row in tqdm(hpa.iterrows(), total=hpa.shape[0]):\n", - " ens, gene, uniprot, evid = str(row['Ensembl']), str(row['Gene']), str(row['Uniprot']), str(row['Evidence'])\n", - " if row['RNA tissue specific NX'] != 'None':\n", - " for x in row['RNA tissue specific NX'].split(';'):\n", - " hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]]\n", - " if row['RNA cell line specific NX'] != 'None':\n", - " for x in row['RNA cell line specific NX'].split(';'):\n", - " hpa_results += [[ens, gene, uniprot, evid, 'cell line', str(x.split(':')[0])]]\n", - " if row['RNA brain regional specific NX'] != 'None':\n", - " for x in row['RNA brain regional specific NX'].split(';'):\n", - " hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]]\n", - " if row['RNA blood cell specific NX'] != 'None':\n", - " for x in row['RNA blood cell specific NX'].split(';'):\n", - " hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]]\n", - " if row['RNA blood lineage specific NX'] != 'None':\n", - " for x in row['RNA blood lineage specific NX'].split(';'):\n", - " hpa_results += [[ens, gene, uniprot, evid, 'anatomy', str(x.split(':')[0])]]" + " ens = str(row['Ensembl']); gene = str(row['Gene']); uni = str(row['Uniprot'])\n", + " evid = str(row['Evidence']); sub = str(row['Subcellular location']); source = 'The Human Protein Atlas'\n", + " if row['RNA tissue specific nTPM'] != 'None':\n", + " row_val = row['RNA tissue specific nTPM']\n", + " if ';' in row_val:\n", + " for x in row_val.split(';'):\n", + " x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1])\n", + " hpa_results += [ [ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]]\n", + " else:\n", + " x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]]\n", + " if row['RNA cell line specific nTPM'] != 'None':\n", + " row_val = row['RNA cell line specific nTPM']\n", + " if ';' in row_val:\n", + " for x in row_val.split(';'):\n", + " x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]\n", + " else:\n", + " x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]\n", + " if row['RNA brain regional specific nTPM'] != 'None':\n", + " row_val = row['RNA brain regional specific nTPM']\n", + " if ';' in row_val:\n", + " for x in row_val.split(';'):\n", + " x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]]\n", + " else:\n", + " x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'anatomy', 'None', x1, x2, source]]\n", + " if row['RNA blood cell specific nTPM'] != 'None':\n", + " row_val = row['RNA blood cell specific nTPM']\n", + " if ';' in row_val:\n", + " for x in row_val.split(';'):\n", + " x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]\n", + " else:\n", + " x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]\n", + " if row['RNA blood lineage specific nTPM'] != 'None':\n", + " row_val = row['RNA blood lineage specific nTPM']\n", + " if ';' in row_val:\n", + " for x in row_val.split(';'):\n", + " x1 = str(x.split(':')[0]); x2 = float(x.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]\n", + " else:\n", + " x1 = str(row_val.split(':')[0]); x2 = float(row_val.split(': ')[1])\n", + " hpa_results += [[ens, gene, uni, evid, 'cell line', sub, x1, x2, source]]" ] }, { @@ -2684,16 +2965,25 @@ "source": [ "# remove rows that contain protein coding genes already in the hpa data\n", "hpa_genes = list(hpa['Ensembl'].drop_duplicates(keep='first', inplace=False))\n", - "gtex = gtex.loc[gtex['Name'].apply(lambda x: x not in hpa_genes)]\n", - "\n", + "gtex = gtex.loc[gtex['Name'].apply(lambda x: x not in hpa_genes)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# loop over data and re-organize - only keep results with tpm >= 1 and if gene symbol is not a protein-coding gene\n", "gtex_results = []\n", + "source = 'Genotype-Tissue Expression (GTEx) Project'\n", "for idx, row in tqdm(gtex.iterrows(), total=gtex.shape[0]):\n", " for col in list(gtex.columns)[2:]:\n", " typ = 'cell line' if 'Cells' in col else 'anatomy'\n", - " if row[col] >= 1.0:\n", - " evidence = 'Evidence at transcript level'\n", - " gtex_results += [[str(row['Name']), str(row['Description']), 'None', evidence, typ, str(col)]]" + " evidence = 'Evidence at transcript level'\n", + " gtex_results += [[str(row['Name']), str(row['Description']), 'None', evidence, typ, 'None', col, float(row[col]), source]]\n", + " \n", + " " ] }, { @@ -2711,7 +3001,7 @@ "source": [ "with open(processed_data_location + 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt', 'w') as out:\n", " for x in tqdm(hpa_results + gtex_results):\n", - " out.write(x[0] + '\\t' + x[1] + '\\t' + x[2] + '\\t' + x[3] + '\\t' + x[4] + '\\t' + x[5] + '\\n')" + " out.write(x[0] + '\\t' + x[1] + '\\t' + x[2] + '\\t' + x[3] + '\\t' + x[4] + '\\t' + x[5] + '\\t' + x[6] + '\\t' + str(x[7]) + '\\t' + x[8] + '\\n')" ] }, { @@ -2723,7 +3013,9 @@ "# load data, return edge count, and preview it\n", "hpa_edges = pandas.read_csv(processed_data_location + 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt',\n", " header=None, low_memory=False, sep='\\t',\n", - " names=['Ensembl_IDs', 'Gene_Symbols', 'Uniprot_IDs', 'Evidence', 'Anatomy_Type', 'Anatomy'])\n", + " names=['Ensembl_IDs', 'Gene_Symbols', 'Uniprot_IDs', 'Evidence',\n", + " 'Anatomy_Type', 'Subcellular_Location', 'Anatomy', 'Expresison_Value',\n", + " 'Source'])\n", "\n", "print('There are {edge_count} edges'.format(edge_count=len(hpa_edges)))\n", "hpa_edges.head(n=5)" @@ -2744,7 +3036,7 @@ "\n", "### Mapping Reactome Pathways to the Pathway Ontology \n", "\n", - "**Data Source Wiki Page:** [Pathway Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#pathway-ontology) \n", + "**Data Source Wiki Page:** [Pathway Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#pathway-ontology) \n", "\n", "**Purpose:** This script downloads the [canonical pathways](http://compath.scai.fraunhofer.de/export_mappings) and [kegg-reactome pathway mappings](https://github.com/ComPath/resources/blob/master/mappings/kegg_reactome.csv) files from the [ComPath Ecosystem](https://github.com/ComPath) in order to create the following identifier mappings: \n", "- `Reactome Pathway Identifiers` ➞ `KEGG Pathway Identifiers` ➞ `Pathway Ontology Identifiers` \n", @@ -2870,7 +3162,7 @@ " data_downloader(url, unprocessed_data_location)\n", "\n", "# load data\n", - "reactome_pathways2 = pandas.read_csv(unprocessed_data_location + 'gene_association.reactome', header=None, delimiter='\\t', skiprows=3, low_memory=False)" + "reactome_pathways2 = pandas.read_csv(unprocessed_data_location + 'gene_association.reactome', header=None, delimiter='\\t', skiprows=4, low_memory=False)" ] }, { @@ -3084,7 +3376,7 @@ "\n", "### Mapping Genomic Identifiers to the Sequence Ontology \n", "\n", - "**Data Source Wiki Page:** [Sequence Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/_edit#sequence-ontology) \n", + "**Data Source Wiki Page:** [Sequence Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources/_edit#sequence-ontology) \n", "\n", "**Purpose:** This script downloads the `genomic_sequence_ontology_mappings.xlsx` file in order to create the following identifier mappings: \n", "- `Gene BioTypes` ➞ `Sequence Ontology Identifiers` \n", @@ -3312,7 +3604,7 @@ "***\n", "### Protein Ontology \n", "\n", - "**Data Source Wiki Page:** [protein-ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#human-phenotype-ontology) \n", + "**Data Source Wiki Page:** [protein-ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-phenotype-ontology) \n", "\n", "**Purpose:** This script uses [OWLTools](https://github.com/owlcollab/owltools) to download the [pr.owl](http://purl.obolibrary.org/obo/pr.owl) (with imports) file from [ProConsortium.org](https://proconsortium.org/) in order to create a version of the ontology that contains only human proteins. This is achieved by performing forward and reverse breadth first search over all proteins which are `owl:subClassOf` [Homo sapiens protein](https://proconsortium.org/app/entry/PR%3A000029067/).\n", "\n", @@ -3404,7 +3696,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "code_folding": [] + "code_folding": [ + 0 + ] }, "outputs": [], "source": [ @@ -3533,7 +3827,7 @@ "metadata": {}, "outputs": [], "source": [ - "gets_ontology_statistics(ontology_data_location + 'pr_with_imports.owl')" + "gets_ontology_statistics(ontology_data_location + 'pr_with_imports.owl', '../pkt_kg/libs/owltools')" ] }, { @@ -3546,7 +3840,7 @@ "\n", "### Relations Ontology \n", "\n", - "**Data Source Wiki Page:** [RO](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#relation-ontology) \n", + "**Data Source Wiki Page:** [Relations Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#relations-ontology) \n", "\n", "**Purpose:** This script downloads the [ro.owl](http://purl.obolibrary.org/obo/ro.owl) file from [obofoundry.org](http://www.obofoundry.org/) in order to obtain all `ObjectProperties` and their inverse relations. \n", "\n", @@ -3687,14 +3981,95 @@ "***\n", "### Clinvar Variant-Diseases and Phenotypes \n", "\n", - "**Data Source Wiki Page:** [Clinvar](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#clinvar) \n", + "**Data Source Wiki Page:** [Clinvar](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#clinvar) \n", "\n", - "**Purpose:** This script downloads the [variant_summary.txt](ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz) file from [ClinVar](https://www.ncbi.nlm.nih.gov/clinvar/) in order to create the following edges: \n", + "**Purpose:** This script downloads the data files list below in order to create the following edges: \n", "- gene-variant \n", "- variant-disease \n", "- variant-phenotype \n", "\n", - "**Output:** `CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt`\n" + "**Data Files:** \n", + "Details on each file have been taken from this [README](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/README.txt) and are provided in relevant code chunks below. \n", + "##### *Core Data Files* \n", + "- [`variant_summary.txt.gz`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz) \n", + "\n", + "##### *Metadata Files* \n", + "- [`var_citations.txt`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/var_citations.txt) \n", + "- [`allele_gene.txt.gz`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/allele_gene.txt.gz) \n", + "\n", + "**Output:** \n", + "- `CLINVAR_VARIANT_GENE_EDGES.txt` \n", + "- `CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt`\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "#### Download and Process Core Data Files \n", + "***\n", + "\n", + "*Data Files:* \n", + "- [`variant_summary.txt.gz`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz) \n", + "\n", + "*Processing Details* \n", + "The first step is down the `variant_summary.txt.gz` file. After downloading, the file is cleaned to handle missing data, unneeded variables are removed, identifiers and date fields are cleaned and reformatted, and rows without valid disease/phenotype identifiers are removed. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "[**`variant_summary.txt.gz`**](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz)\n", + "\n", + "> A tab-delimited report based on each variant at a location on the genome for which data have been submitted to ClinVar. \n", + "The data for the variant are reported for each assembly, so most variants have a line for GRCh37 (hg19) and another line for GRCh38 (hg38).\n", + ">\n", + "> - AlleleID: integer value as stored in the AlleleID field in ClinVar \n", + "> - Type: character, the type of variant represented by the AlleleID \n", + "> - Name: character, ClinVar's preferred name for the record with this AlleleID \n", + "> - GeneID: integer, GeneID in NCBI's Gene database, reported if there is a single gene, otherwise reported as -1. \n", + "> - GeneSymbol: character, comma-separated list of GeneIDs overlapping the variant \n", + "> - HGNC_ID: string, of format HGNC:integer, reported if there is a single GeneID. \n", + "> - ClinicalSignificance: character, comma-separated list of aggregate values of clinical significance calculated for this variant. \n", + "> - ClinSigSimple: integer, \n", + " 0 = no current value of Likely pathogenic or Pathogenic;\n", + " 1 = at least one current record submitted with an interpretation of Likely pathogenic or \n", + " Pathogenic (independent of whether that record includes assertion criteria and \n", + " evidence) \n", + " -1 = no values for clinical significance at all for this variant or set of variants; \n", + " used for the \"included\" variants that are only in ClinVar because they are included\n", + " in a haplotype or genotype with an interpretation \n", + "> - LastEvaluated: date, the latest date any submitter reported clinical significance \n", + "> - RS# (dbSNP): integer, rs# in dbSNP, reported as -1 if missing \n", + "> - nsv/esv (dbVar): character, the NSV identifier for the region in dbVar \n", + "> - RCVaccession: character, list of RCV accessions that report this variant \n", + "> - PhenotypeIDs: character, list of identifiers for phenotype(s) interpreted for this variant. If more than 5 conditions are reported, the number of conditions is reported instead. \n", + "> - PhenotypeList: character, list of names corresponding to PhenotypeIDs. If more than 5 conditions are reported, the number of conditions is reported instead. \n", + "> - Origin: character, list of all allelic origins for this variant \n", + "> - OriginSimple: character, processed from Origin to make it easier to distinguish between germline and somatic \n", + "> - Assembly: character, name of the assembly on which locations are based \n", + "> - ChromosomeAccession: Accession and version of the RefSeq sequence defining the position reported in the start and stop columns. \n", + "> - Chromosome: character, chromosomal location \n", + "> - Start: integer, starting location, right-shifted, in pter->qter orientation \n", + "> - Stop: integer, end location, right-shifted, in pter->qter orientation \n", + "> - ReferenceAllele: The reference allele using the right-shifted location in Start and Stop. \n", + "> - AlternateAllele: The alternate allele using the right-shifted location in Start and Stop. \n", + "> - Cytogenetic: character, ISCN band\n", + "> - ReviewStatus: character, highest review status for reporting this measure. \n", + "> - NumberSubmitters: integer, number of submitters describing this variant \n", + "> - Guidelines: character, ACMG only right now \n", + "> - TestedInGTR: character, Y/N for Yes/No if there is a test registered as specific to this variant in the NIH Genetic Testing Registry (GTR) \n", + "> - OtherIDs: character, list of other identifiers or sources of information about this variant \n", + "> - SubmitterCategories: coded value to indicate whether data were submitted by another resource (1), any other type of source (2), both (3), or none (4) \n", + "> - VariationID: The identifier ClinVar uses specific to the AlleleID. Not all VariationIDS that may be related to the AlleleID are reported in this file. \n", + "> - PositionVCF: integer, starting location, left-shifted, in pter->qter orientation \n", + "> - ReferenceAlleleVCF: The reference allele using the left-shifted location in vcf_pos. \n", + "> - AlternateAlleleVCF: The alternate allele using the left-shifted location in vcf_pos. " ] }, { @@ -3704,19 +4079,54 @@ "outputs": [], "source": [ "# download data\n", - "url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz'\n", + "url = 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz'\n", "if not os.path.exists(unprocessed_data_location + 'variant_summary.txt'):\n", " data_downloader(url, unprocessed_data_location)\n", "\n", "# load data\n", - "clinvar_data = pandas.read_csv(unprocessed_data_location + 'variant_summary.txt', header=0, delimiter='\\t', low_memory=False)" + "var_summary = pandas.read_csv(unprocessed_data_location + 'variant_summary.txt',\n", + " header=0, delimiter='\\t', low_memory=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# replace \"na\" and \"-\" with NaN\n", + "var_summary = var_summary.replace('na', numpy.nan)\n", + "var_summary = var_summary.replace('-', numpy.nan)\n", + "\n", + "# handle ids that are coded as missing (i.e., -1)\n", + "var_summary['GeneID'] = var_summary['GeneID'].replace(-1, numpy.nan)\n", + "var_summary['RS# (dbSNP)'] = var_summary['RS# (dbSNP)'].replace(-1, numpy.nan)\n", + "\n", + "# convert date format\n", + "var_summary['LastEvaluated'] = var_summary['LastEvaluated'].str.replace('None', '')\n", + "var_summary['LastEvaluated'] = pandas.to_datetime(var_summary['LastEvaluated'])\n", + "var_summary['LastEvaluated'] = var_summary['LastEvaluated'].dt.strftime('%B %d, %Y')\n", + "var_summary['LastEvaluated'] = var_summary['LastEvaluated'].replace('', numpy.nan)\n", + "\n", + "# rename variables\n", + "var_summary.rename(columns={'#AlleleID': 'AlleleID',\n", + " 'nsv/esv (dbVar)': 'nsv',\n", + " 'Name': 'VariantName'}, inplace=True)\n", + "\n", + "# update variable types\n", + "var_summary['GeneID'] = var_summary['GeneID'].astype('Int64')\n", + "var_summary['RS# (dbSNP)'] = var_summary['RS# (dbSNP)'].astype('Int64')\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} variant edges'.format(edge_count=len(var_summary)))\n", + "var_summary.head(n=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "_Preprocess Data_" + "*Address Duplicate Rows for GRCh37 and GRCh38 Assemblies*" ] }, { @@ -3725,96 +4135,130 @@ "metadata": {}, "outputs": [], "source": [ - "# replace NaN with 'None'\n", - "clinvar_data.fillna('None', inplace=True)\n", + "# subset df\n", + "var_summary_update_assemb = var_summary.copy()\n", + "var_summary_update_assemb = var_summary_update_assemb[['VariationID', 'Assembly', 'ChromosomeAccession',\n", + " 'Chromosome', 'Start', 'Stop', 'ReferenceAllele',\n", + " 'AlternateAllele', 'Cytogenetic', 'PositionVCF']].drop_duplicates()\n", "\n", - "# explode nested data\n", - "explode_df_clinvar = explodes_data(clinvar_data.copy(), ['PhenotypeIDS'], ';')\n", - "explode_df_clinvar = explodes_data(explode_df_clinvar.copy(), ['PhenotypeIDS'], ',')\n", + "# identify columns to process\n", + "assemb_cols = ['ChromosomeAccession', 'Chromosome', 'Start', 'Stop', 'ReferenceAllele',\n", + " 'AlternateAllele','Cytogenetic', 'PositionVCF', 'ReferenceAlleleVCF', 'AlternateAlleleVCF']\n", "\n", - "# edit column formatting\n", - "explode_df_clinvar['PhenotypeIDS'].replace('Orphanet:ORPHA','ORPHA:', inplace=True, regex=True)\n", - "explode_df_clinvar['PhenotypeIDS'].replace('Human Phenotype Ontology:HP:','HP_', inplace=True, regex=True)\n", + "# group data by variant\n", + "df = var_summary_update_assemb.fillna('None')\n", + "df = df.groupby('VariationID').apply(lambda g: str(g.drop(['VariationID'], axis=1).to_dict('records'))).to_dict()\n", "\n", - "# write data\n", - "explode_df_clinvar.to_csv(processed_data_location + 'CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt', header=True, sep='\\t', encoding='utf-8', index=False)\n", + "# convert to Pandas DataFrame\n", + "df_items = df.items()\n", + "temp_df = pandas.DataFrame({'VariationID': [x[0] for x in df_items], 'Assembly': [x[1] for x in df_items]})\n", + "\n", + "# join temp df with original data\n", + "var_summary_assemb = var_summary.copy().drop(assemb_cols + ['Assembly'], axis = 1)\n", + "var_summary_update = var_summary_assemb.merge(temp_df, on='VariationID', how='left')\n", + "\n", + "# drop duplicates\n", + "var_summary_update.drop_duplicates(inplace=True)\n", "\n", "# print row count and preview data\n", - "print('There are {edge_count} variant edges'.format(edge_count=len(explode_df_clinvar)))\n", - "explode_df_clinvar.head(n=5)" + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_update)))\n", + "var_summary_update.head(n=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "*Process `PhenotypeIDS` and `PhenotypeList` Columns*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [] + }, + "outputs": [], + "source": [ + "# clean-up identifiers\n", + "var_summary_update['Phenotype'] = var_summary_update['PhenotypeIDS'].str.replace('|', ';').str.replace(',', ';')\n", + "var_summary_update['OtherIDs'] = var_summary_update['OtherIDs'].str.replace(';', '|').str.replace(',', '|')\n", "\n", - "
\n", - "\n", - "***\n", + "# remove unneeded variables\n", + "drop_list = ['PhenotypeList', 'PhenotypeIDS']\n", + "var_summary_update = var_summary_update.drop(drop_list, axis = 1).drop_duplicates()\n", "\n", - "### Uniprot Protein-Cofactor and Protein-Catalyst \n", + "# replace NaN with 'None'\n", + "var_summary_update['Phenotype'] = var_summary_update['Phenotype'].fillna('None')\n", "\n", - "**Data Source Wiki Page:** [Uniprot](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#uniprot-knowledgebase) \n", + "# reformat phenotypeIDS and trim leading whitespace from unnested columns\n", + "var_summary_update['Phenotype'] = var_summary_update['Phenotype'].apply(\n", + " lambda x: ';'.join(set(x for x in ['MONDO:' + i.split(':')[-1] if i.startswith('MONDO')\n", + " else 'HP:' + i.split(':')[-1] if i.startswith('Human Phenotype')\n", + " else 'ORPHA:' + i.split(':')[-1] if i.startswith('Orphanet')\n", + " else 'None' if i.endswith(' conditions')\n", + " else i for i in x.split(';')] if x != 'None')))\n", "\n", - "**Purpose:** This script downloads the [uniprot-cofactor-catalyst.tab](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources/#uniprot-knowledgebase) file from the [Uniprot Knowledge Base](https://www.uniprot.org) in order to create the following edges: \n", - "- protein-cofactor \n", - "- protein-catalyst \n", + "# drop duplicates\n", + "var_summary_update.drop_duplicates(inplace=True)\n", "\n", - "**Data:** This data was obtained by querying the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) using the *reviewed:yes AND organism:\"Homo sapiens (Human) [9606]\"\"* keyword and including the following columns:\n", - "- Entry (Standard) \n", - "- Status (Standard) \n", - "- PRO (*Miscellaneous*) \n", - "- ChEBI (Cofactor) (*Chemical entities*) \n", - "- ChEBI (Catalytic activity) (*Chemical entities*) \n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_update)))\n", + "var_summary_update.head(n=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", "\n", - "The URL to access the results of this query is obtained by clicking on the share symbol and copying the free-text from the box. To obtain the data in a tab-delimited format the following string is appended to the end of the URL: \"&format=tab\".\n", + "#### Metadata Files \n", + "***\n", "\n", - "**NOTE.** Be sure to obtain a new URL from the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) when rebuilding to ensure you are getting the most up-to-date data. This query was last generated on `12/02/2020`.\n", + "*Data Files:* \n", + "- [`var_citations.txt`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/var_citations.txt) \n", + "- [`allele_gene.txt.gz`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/allele_gene.txt.gz) \n", + "- [`gene_specific_summary.txt`](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/gene_specific_summary.txt) \n", "\n", - "
\n", + "*Processing Details* \n", + "Step 1: The first step is down the files. After downloading, the files are cleaned to handle missing data, unneeded variables are removed, and identifiers and date fields are cleaned and reformatted. \n", "\n", - "**Output:** \n", - "- protein-cofactor ➞ `UNIPROT_PROTEIN_COFACTOR.txt`\n", - "- protein-catalyst ➞ `UNIPROT_PROTEIN_CATALYST.txt`\n" + "Step 2: Merge each cleaned file with the processed variant summary data from the prior steps." ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# download data\n", - "url = 'https://www.uniprot.org/uniprot/?query=&fil=organism%3A%22Homo%20sapiens%20(Human)%20%5B9606%5D%22&columns=id%2Creviewed%2Centry%20name%2Cdatabase(PRO)%2Cchebi(Cofactor)%2Cchebi(Catalytic%20activity)&format=tab'\n", - "if not os.path.exists(unprocessed_data_location + 'uniprot-cofactor-catalyst.tab'):\n", - " data_downloader(url, unprocessed_data_location, 'uniprot-cofactor-catalyst.tab')\n", + "
\n", "\n", - "# upload datta\n", - "data = open(unprocessed_data_location + 'uniprot-cofactor-catalyst.tab').readlines()\n", + "[**`var_citations.txt`**](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/var_citations.txt)\n", "\n", - "# reformat data and write it out\n", - "with open(processed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt', 'w') as outfile1, open(processed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt', 'w') as outfile2:\n", - " for line in tqdm(data):\n", - " # get cofactors\n", - " if 'CHEBI' in line.split('\\t')[4]: \n", - " for i in line.split('\\t')[4].split(';'):\n", - " chebi = i.split('[')[-1].replace(']', '').replace(':', '_')\n", - " outfile1.write('PR_' + line.split('\\t')[3].strip(';') + '\\t' + chebi + '\\n')\n", - " # get catalysts\n", - " if 'CHEBI' in line.split('\\t')[5]: \n", - " for i in line.split('\\t')[5].split(';'):\n", - " chebi = i.split('[')[-1].replace(']', '').replace(':', '_')\n", - " outfile2.write('PR_' + line.split('\\t')[3].strip(';') + '\\t' + chebi + '\\n')" + "> A tab-delimited report of citations associated with data in ClinVar, connected to the AlleleID, the VariationID, and either rs# from dbSNP or nsv in dbVar.\n", + ">\n", + "> - AlleleID: integer value as stored in the AlleleID field in ClinVar \n", + "> - VariationID: The identifier ClinVar uses to anchor its default display \n", + "> - rs: rs identifier from dbSNP, null if missing \n", + "> - nsv: nsv identifier from dbVar, null if missing \n", + "> - citation_source: The source of the citation, either PubMed, PubMedCentral, or the NCBI Bookshelf \n", + "> - citation_id: The identifier used by that source " ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "***\n", + "# download data\n", + "url = 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/var_citations.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'var_citations.txt'):\n", + " data_downloader(url, unprocessed_data_location)\n", "\n", - "**Cofactor Data** " + "# load data\n", + "var_citations = pandas.read_csv(unprocessed_data_location + 'var_citations.txt',\n", + " header=0, delimiter='\\t', low_memory=False)" ] }, { @@ -3823,21 +4267,353 @@ "metadata": {}, "outputs": [], "source": [ - "# load data, print row count, and preview it\n", - "pcp1_data = pandas.read_csv(processed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt', header=None, names=['Protein_Ontology_IDs', 'CHEBI_IDs'], delimiter='\\t')\n", + "# replace \"na\" and \"-\" with NaN\n", + "var_citations = var_citations.replace('na', numpy.nan)\n", + "var_citations = var_citations.replace('-', numpy.nan)\n", "\n", - "print('There are {edge_count} protein-cofactor edges'.format(edge_count=len(pcp1_data)))\n", - "pcp1_data.head(n=5)" + "# combine citation information\n", + "var_citations['Citation'] = var_citations['citation_source'] + ':' + var_citations['citation_id']\n", + "# remove unneeded variables\n", + "drop_list = ['citation_source', 'citation_id']\n", + "var_citations = var_citations.drop(drop_list, axis = 1).drop_duplicates()\n", + "\n", + "# group data by citations\n", + "var_citations = var_citations.groupby('VariationID').Citation.agg([('Citation', '|'.join)]).reset_index()\n", + "var_citations = var_citations.drop_duplicates().sort_values(by=['VariationID'])\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_citations)))\n", + "var_citations.head(n=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", + "
\n", "\n", + "[**`allele_gene.txt.gz`**](https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/allele_gene.txt.gz)\n", "\n", - "**Catalyst Data** " + "> Reports per ClinVar's AlleleID, the genes that are related to that gene and how they are related.\n", + ">\n", + "> - AlleleID: the integer identifier assigned by ClinVar to each simple allele\n", + "> - GeneID: integer, GeneID in NCBI's Gene database \n", + "> - Symbol: character, Symbol preferred in NCBI's Gene database. Is the symbol from HGNC when available \n", + "> - Name: character, full name of the gene \n", + "> - GenesPerAlleleID: integer, number of genes related to the allele \n", + "> - Category: character, type of allele-gene relationship. The values for category are:\n", + "> - asserted, but not computed: Submitted as related to a gene, but not within the location of that gene on the genome \n", + "> - genes overlapped by variant: The gene and variant overlap \n", + "> - near gene, downstream: Outside the location of the gene on the genome, within 5 kb \n", + "> - near gene, upstream: Outside the location of the gene on the genome, within 5 kb \n", + "> - within multiple genes by overlap: The variant is within genes that overlap on the genome. Includes introns \n", + "> - within single gene: The variant is in only one gene. Includes introns \n", + "> - Source: character, was the relationship submitted or calculated? " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/allele_gene.txt.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'allele_gene.txt'):\n", + " data_downloader(url, unprocessed_data_location)\n", + "\n", + "# load data\n", + "allele_gene = pandas.read_csv(unprocessed_data_location + 'allele_gene.txt',\n", + " header=0, delimiter='\\t', low_memory=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# replace \"na\" and \"-\" with NaN\n", + "allele_gene = allele_gene.replace('na', numpy.nan)\n", + "allele_gene = allele_gene.replace('-', numpy.nan)\n", + "\n", + "# handle gene ids that may be coded as -1\n", + "allele_gene['GeneID'] = allele_gene['GeneID'].replace(-1, numpy.nan)\n", + "\n", + "# rename variables\n", + "allele_gene.rename(columns={'#AlleleID': 'AlleleID',\n", + " 'Symbol': 'GeneSymbol',\n", + " 'Name': 'GeneName'}, inplace=True)\n", + "\n", + "# update variable types\n", + "allele_gene['GeneID'] = allele_gene['GeneID'].astype('Int64')\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(allele_gene)))\n", + "allele_gene.head(n=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "_Merge and Process Data Sources_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Merge `var_summary_update` with `var_citations` data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge data\n", + "merge_cols = list(set(var_summary_update.columns).intersection(set(var_citations.columns)))\n", + "var_summary_merged = var_summary_update.merge(var_citations, on=merge_cols, how='left')\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_merged)))\n", + "var_summary_merged.head(n=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Merge merged `var_summary_update` with `allele_gene` data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge data\n", + "merge_cols = list(set(var_summary_merged.columns).intersection(set(allele_gene.columns)))\n", + "var_summary_merged = var_summary_merged.merge(allele_gene, on=merge_cols, how='left')\n", + "\n", + "# update variable types\n", + "var_summary_merged['GenesPerAlleleID'] = var_summary_merged['GenesPerAlleleID'].astype('Int64')\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_merged)))\n", + "var_summary_merged.head(n=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Write Edge Lists**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*`variant`-`gene` Edges*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# reduce data set\n", + "var_summary_merged_gene = var_summary_merged.copy()\n", + "var_summary_merged_gene = var_summary_merged_gene[[\n", + " 'VariationID', 'AlleleID', 'RS# (dbSNP)', 'Type', 'VariantName',\n", + " 'OtherIDs', 'GeneID', 'GeneSymbol', 'GeneName', 'GenesPerAlleleID',\n", + " 'Assembly', 'Category', 'Guidelines', 'TestedInGTR', 'RCVaccession', 'LastEvaluated',\n", + " 'ReviewStatus', 'ClinicalSignificance', 'ClinSigSimple', 'Origin', 'OriginSimple', 'Source',\n", + " 'SubmitterCategories', 'NumberSubmitters', 'Citation']]\n", + "var_summary_merged_gene.drop_duplicates(inplace=True)\n", + "\n", + "# remove any rows missing a gene id\n", + "var_summary_merged_gene = var_summary_merged_gene.dropna(subset=['GeneID'])\n", + "\n", + "# head prefix to output\n", + "var_summary_merged_gene['GeneID'] = 'NCBIGene_' + var_summary_merged_gene['GeneID'].astype(str)\n", + "var_summary_merged_gene['VariationID'] = 'clinvar_' + var_summary_merged_gene['VariationID'].astype(str)\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_merged_gene)))\n", + "var_summary_merged_gene.head(n=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# write out data\n", + "var_summary_merged_gene.to_csv(open(processed_data_location + 'CLINVAR_VARIANT_GENE_EDGES.txt', 'w'),\n", + " index=False, header=True, sep='\\t')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*`variant`-`disease` / `variant`-`phenotype` Edges*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# reduce data set\n", + "var_summary_merged_disease = var_summary_merged.copy()\n", + "var_summary_merged_disease = var_summary_merged_disease[[\n", + " 'VariationID', 'AlleleID', 'RS# (dbSNP)', 'Type', 'VariantName', 'RCVaccession',\n", + " 'LastEvaluated', 'ReviewStatus', 'ClinicalSignificance', 'ClinSigSimple', 'GeneID',\n", + " 'NumberSubmitters', 'SubmitterCategories', 'Guidelines', 'TestedInGTR',\n", + " 'Origin', 'OriginSimple', 'Assembly', 'Phenotype', 'Citation', 'OtherIDs']]\n", + "var_summary_merged_disease.drop_duplicates(inplace=True)\n", + "\n", + "# expand results by disease identifier\n", + "cols = ['Phenotype']\n", + "for col in tqdm(cols): var_summary_merged_disease = var_summary_merged_disease.assign(**{col: var_summary_merged_disease[col].str.split(';')}).explode(col)\n", + " \n", + "# remove phenotype rows with None and drop duplicates\n", + "var_summary_merged_disease = var_summary_merged_disease[var_summary_merged_disease['Phenotype'] != 'None']\n", + "var_summary_merged_disease.drop_duplicates(inplace=True)\n", + "\n", + "# head prefix to output\n", + "var_summary_merged_disease['VariationID'] = 'clinvar_' + var_summary_merged_disease['VariationID'].astype(str)\n", + "\n", + "# print row count and preview data\n", + "print('There are {edge_count} edges'.format(edge_count=len(var_summary_merged_disease)))\n", + "var_summary_merged_disease.head(n=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# write data to file\n", + "var_summary_merged_disease.to_csv(open(processed_data_location + 'CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt', 'w'),\n", + " index=False, header=True, sep='\\t')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "***\n", + "\n", + "### Uniprot Protein-Cofactor and Protein-Catalyst \n", + "\n", + "**Data Source Wiki Page:** [UniProt](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#universal-protein-resource-knowledgebase) \n", + "\n", + "**Purpose:** This script downloads the [uniprot-cofactor-catalyst.tab](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#universal-protein-resource-knowledgebase) file from the [Uniprot Knowledge Base](https://www.uniprot.org) in order to create the following edges: \n", + "- protein-cofactor \n", + "- protein-catalyst \n", + "\n", + "**Data:** This data was obtained by querying the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) using the *reviewed:yes AND organism:\"Homo sapiens (Human) [9606]\"\"* keyword and including the following columns:\n", + "- Entry (Standard) \n", + "- Status (Standard) \n", + "- PRO (*Miscellaneous*) \n", + "- ChEBI (Cofactor) (*Chemical entities*) \n", + "- ChEBI (Catalytic activity) (*Chemical entities*) \n", + "\n", + "The URL to access the results of this query is obtained by clicking on the share symbol and copying the free-text from the box. To obtain the data in a tab-delimited format the following string is appended to the end of the URL: \"&format=tab\".\n", + "\n", + "**NOTE.** Be sure to obtain a new URL from the [UniProt Knowledgebase](https://www.uniprot.org/uniprot/) when rebuilding to ensure you are getting the most up-to-date data. This query was last generated on `12/02/2020`.\n", + "\n", + "
\n", + "\n", + "**Output:** \n", + "- protein-cofactor ➞ `UNIPROT_PROTEIN_COFACTOR.txt`\n", + "- protein-catalyst ➞ `UNIPROT_PROTEIN_CATALYST.txt`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://www.uniprot.org/uniprot/?query=&fil=organism%3A%22Homo%20sapiens%20(Human)%20%5B9606%5D%22&columns=id%2Creviewed%2Centry%20name%2Cdatabase(PRO)%2Cchebi(Cofactor)%2Cchebi(Catalytic%20activity)&format=tab'\n", + "if not os.path.exists(unprocessed_data_location + 'uniprot-cofactor-catalyst.tab'):\n", + " data_downloader(url, unprocessed_data_location, 'uniprot-cofactor-catalyst.tab')\n", + "\n", + "# upload data\n", + "data = open(unprocessed_data_location + 'uniprot-cofactor-catalyst.tab').readlines()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# reformat data and write it out\n", + "with open(processed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt', 'w') as outfile1, open(processed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt', 'w') as outfile2:\n", + " for line in tqdm(data):\n", + " status = line.split('\\t')[1]; upt_id = line.split('\\t')[0]; upt_entry = line.split('\\t')[2]\n", + " pr_id = 'PR_' + line.split('\\t')[3].strip(';')\n", + " # get cofactors\n", + " if 'CHEBI' in line.split('\\t')[4]: \n", + " for i in line.split('\\t')[4].split(';'):\n", + " chebi = i.split('[')[-1].replace(']', '').replace(':', '_')\n", + " outfile1.write(pr_id + '\\t' + chebi + '\\t' + status + '\\t' + upt_id + '\\t' + upt_entry + '\\n')\n", + " # get catalysts\n", + " if 'CHEBI' in line.split('\\t')[5]: \n", + " for i in line.strip('\\n').split('\\t')[5].split(';'):\n", + " chebi = i.split('[')[-1].replace(']', '').replace(':', '_')\n", + " outfile2.write(pr_id + '\\t' + chebi + '\\t' + status + '\\t' + upt_id + '\\t' + upt_entry + '\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "**Cofactor Data** " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load data, print row count, and preview it\n", + "pcp1_data = pandas.read_csv(processed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt', header=None,\n", + " names=['Protein_Ontology_IDs', 'CHEBI_IDs', 'Status', 'Uniprot_ID', 'Uniprot_Entry_name'],\n", + " delimiter='\\t')\n", + "\n", + "print('There are {edge_count} protein-cofactor edges'.format(edge_count=len(pcp1_data)))\n", + "pcp1_data.head(n=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "\n", + "**Catalyst Data** " ] }, { @@ -3847,109 +4623,2737 @@ "outputs": [], "source": [ "# load data, print row count, and preview it\n", - "pcp2_data = pandas.read_csv(processed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt', header=None, names=['Protein_Ontology_IDs', 'CHEBI_IDs'], delimiter='\\t')\n", + "pcp2_data = pandas.read_csv(processed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt', header=None,\n", + " names=['Protein_Ontology_IDs', 'CHEBI_IDs', 'Status', 'Uniprot_ID', 'Uniprot_Entry_name'],\n", + " delimiter='\\t')\n", "\n", "print('There are {edge_count} protein-catalyst edges'.format(edge_count=len(pcp2_data)))\n", "pcp2_data.head(n=5)" ] }, { - "cell_type": "markdown", + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "***\n", + "***\n", + "### NODE AND RELATION METADATA\n", + "***\n", + "\n", + "**Data Source Wiki Page:** [Dependencies](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies/#metadata) \n", + "\n", + "**Purpose:** The goal of this section is to obtain metadata for each entity that is not from an ontology and all relations used in the knowledge graph. \n", + "\n", + "
\n", + "\n", + "**Metadata:** \n", + "A variety of metadata are pulled from the data sources that are used to support external edges added to enhance the core set of ontologies. For the monthly PheknowLator builds, please see [`pheknowlator_source_metadata.xlsx`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/pheknowlator_source_metadata.xlsx) spreadsheet. This spreadsheet has two tabs, one for nodes and one for edges. Each each entity (i.e., node or relation) there are several columns, including descriptions of the metadata, the variable type, and even examples of values for each type of metadata. \n", + "\n", + "*Example Metadata Dictionary Output*. The code snippet below is meant to provide a snapshot of how data are organized in the metadata dictionary. As demonstrated by this example, there are three high-level keys: \n", + " - `nodes`: Nodes are keyed by CURIE. Every node has a `Label`, `Description`, `Synonym`, and `Dbxref` (whenever possible). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. \n", + " - `edges`: Edges are keyed by a label which represents the edge type (the same label that is used in `resource_info.txt` and `edge_source_list.txt` files). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. \n", + " - `relations`: Relations or `owl:ObjectProperty` objects are keyed by CURIE. Similar to nodes, every relation has a `Label`, `Description`, and `Synonym` (whenever possible). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. \n", + "\n", + "```python\n", + "{\n", + " 'nodes': {\n", + " 'NCBIGene_2052': {\n", + " 'Label': 'EPHX1',\n", + " 'Description': \"EPHX1 has locus group 'protein-coding' and is located on chromosome 1 (1q42.12).\",\n", + " 'Synonym': 'epoxide hydrolase 1, microsomal (xenobiotic)|epoxide hydratase|EPHX|HYL1|MEHepoxide hydrolase 1|epoxide hydrolase 1 microsomal|EPOX',\n", + " 'Dbxref': 'MIM:132810|HGNC:HGNC:3401|Ensembl:ENSG00000143819', ... },\n", + " 'CHEBI_4592': {\n", + " 'Label': 'Dihydroxycarbazepine',\n", + " 'Description': \"None\",\n", + " 'Synonym': '10,11-Dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide|10,11-Dihydroxycarbamazepine',\n", + " 'Dbxref': 'CAS:35079-97-1|KEGG:C07495',\n", + " 'CTD_chem_gene_ixns.tsv.gz': { \n", + " 'CTD_ChemicalID': {'MESH:C004822'},\n", + " 'CTD_CasRN': {'35079-97-1'},\n", + " 'CTD_ChemicalName': {'10,11-dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide'}}, ... }, ... },\n", + " 'edges': {\n", + " 'chemical-gene': {\n", + " 'CHEBI_4592-NCBIGene_2052': {\n", + " {'CTD_chem_gene_ixns.tsv': {\n", + " 'CTD_Evidence': [{'CTD_Interaction': '[EPHX1 gene SNP affects the metabolism of carbamazepine epoxide] which affects the chemical synthesis of 10,11-dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide',\n", + " 'CTD_InteractionActions': 'affects^chemical synthesis|affects^metabolic processing',\n", + " 'CTD_PubMedIDs': '15692831'}]}}, ...}, ...}, ...}, \n", + " 'relations': {\n", + " 'RO_0002434': {\n", + " 'Label': 'interacts with',\n", + " 'Description': 'A relationship that holds between two entities in which the processes executed by the two entities are causally connected.',\n", + " 'Synonym': 'in pairwise interaction with'}, ... }\n", + "}\n", + "```\n", + "\n", + "
\n", + "\n", + "\n", + "NOTE. All entity metadata are written to the `metadata` directory as a `pickled` dictionary called `entity_metadata_dict.pkl`. The algorithm will look for this dictionary in the `metadata` directory and if it is not there, then no entity metadata will be created.\n", + "\n", + "
\n", + "\n", + "### Prepare Metadata Dictionaries\n", + "***\n", + "\n", + "**Purpose:** To create the resources needed in order to create metadata dictionaries. This process has the following steps:\n", + "\n", + "**1. [Generate Metadata Dictionaries](#generate-metadata-dictionaries):** In order to obtain metadata, we first read in the data source for each type and convert it into a dictionary. Then, each metadata dictionary is merged together and saved to a `master_metadata_dictionary`, keyed by identifier.\n", + " - Input Datasets: \n", + " - [CTD_chem_gene_ixns.tsv](http://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz) \n", + " - Edges: `chemical-gene`, `chemical-protein`, `chemical-rna` \n", + " - Identifier Maps: \n", + " - Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + " - Proteins: [ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt) \n", + " - RNA: [ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt) \n", + " - [CTD_chem_go_enriched.tsv](http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz) \n", + " - Edges: `chemical-gobp`, `chemical-gocc`, `chemical-gomf` \n", + " - Identifier Maps: \n", + " - Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + " - [CTD_chemicals_diseases.tsv](http://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz) \n", + " - Edges: `chemical-disease`, `chemical-phenotype` \n", + " - Identifier Maps: \n", + " - Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + " - Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt) \n", + " - Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + " - [ChEBI2Reactome_All_Levels.txt](https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt) \n", + " - Edge: `chemical-pathway` \n", + " - [goa_human.gaf](http://current.geneontology.org/annotations/goa_human.gaf.gz) \n", + " - Edges: `protein-gobp`, `protein-gocc`, `protein-gomf` \n", + " - Identifier Maps: \n", + " - Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + " - [COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt](http://genemania.org/data/current/Homo_sapiens.COMBINED/COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt) \n", + " - Edge: `gene-gene` \n", + " - Identifier Maps: \n", + " - Genes: [UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt) \n", + " - [phenotype.hpoa](http://purl.obolibrary.org/obo/hp/hpoa/phenotype.hpoa) \n", + " - Edge: `disease-phenotype` \n", + " - Identifier Maps: \n", + " - Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt) \n", + " - [ChEBI2Reactome_All_Levels.txt](https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt) \n", + " - Edge: `chemical-pathway` \n", + " - [gene_association.reactome](https://reactome.org/download/current/gene_association.reactome.gz) \n", + " - Edge: `gobp-pathway`, `pathway-gocc`, `pathway-gomf` \n", + " - [UniProt2Reactome_All_Levels.txt](https://reactome.org/download/current/UniProt2Reactome_All_Levels.txt) \n", + " - Edge: `protein-pathway` \n", + " - Identifier Maps: \n", + " - Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + " - [CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt) \n", + " - Edge: `variant-disease`, `variant-disease` \n", + " - Identifier Maps: \n", + " - Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt)\n", + " - Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + " - [CLINVAR_VARIANT_GENE_EDGES.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_EDGES.txt) \n", + " - Edge: `variant-gene` \n", + " - [HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt) \n", + " - Edge: `protein-anatomy`, `protein-cell`, `rna-anatomy`, `rna-cell` \n", + " - Identifier Maps: \n", + " - Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + "\t\t- Anatomy: [HPA_GTEx_TISSUE_CELL_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt)\n", + " - Cells: [HPA_GTEx_TISSUE_CELL_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt) \n", + " - RNA: [GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt) \n", + " - [UNIPROT_PROTEIN_CATALYST.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_CATALYST.txt) \n", + " - Edge: `protein-catalyst`\n", + " - [UNIPROT_PROTEIN_COFACTOR.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_COFACTOR.txt) \n", + " - Edge: `protein-cofactor`\n", + " - [9606.protein.links.v11.0.txt.gz](https://stringdb-static.org/download/protein.links.v11.0/9606.protein.links.v11.0.txt.gz) \n", + " - Edge: `protein-protein` \n", + " - Identifier Maps: \n", + " - Proteins: [STRING_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/STRING_PRO_ONTOLOGY_MAP.txt)\n", + " - [curated_gene_disease_associations.tsv](https://www.disgenet.org/static/disgenet_ap1/files/downloads/curated_gene_disease_associations.tsv.gz) \n", + " - Edge: `gene-disease`, `gene-phenotype` \n", + " - Identifier Maps: \n", + " - Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt)\n", + " - Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + " \n", + "
\n", + "\n", + "**2. [Write Metadata Files](#write-metadata-files):** The `master_metadata_dictionary` dictionary from _Step 1_ is `pickled` and saved to the `resources/metadata/entity_metadata_dict.pkl` directory.\n", + "\n", + "
\n", + "\n", + "***" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create the shell for the node and relation dictionary\n", + "master_metadata_dictionary = {'nodes': {}, 'relations': {}, 'edges': {}}\n", + "\n", + "# # create temp metadata directory\n", + "# temp_location = metadata_location + 'temp'\n", + "# if os.path.exists(temp_location): shutil.rmtree(temp_location)\n", + "# os.mkdir(temp_location)\n", + "# os.mkdir(temp_location + '/nodes')\n", + "# os.mkdir(temp_location + '/relations')\n", + "# os.mkdir(temp_location + '/edges')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "### Generate Metadata Dictionaries \n", + "\n", + "There are two types of data that are processed when building the metadata dictionary. The first type of data is *Primary*, meaning it consists of a small set of variables that are collected for all entities that are included in the knowledge graph (i.e., `Label`, `Description`, `DbXref`, `Synonym`). These data are collected for entities of type: genes, RNA, variants, and pathways. *Secondary* data are then collected for all edges in the knowledge graph that include entities that are not obtained from an ontology. For these sources, metadata may differ by source. \n", + "- [Primary Metadata Elements](#primary) \n", + "- [Secondary Metadata Elements](#secondary)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Primary Metadata Elements \n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Genes Metadata Dictionary \n", + "\n", + "**Data Source Wiki Page:** [National Center for Biotechnology Information Gene](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#national-center-for-biotechnology-information-gene)\n", + "\n", + "The nested dictionary of gene metadata is created by looping over the cleaned human [National Center for Biotechnology Information Gene](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#national-center-for-biotechnology-information-gene) identifier data set ([`ensembl_identifier_data_cleaned.txt`](ftp://ftp.ncbi.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz)). The `keys` of the dictionary are `Entrez gene identifiers` and the `values` are dictionaries for each metadata type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# entrez gene data\n", + "entrez_gene_data = pandas.read_csv(unprocessed_data_location + 'Homo_sapiens.gene_info', header=0, delimiter='\\t', low_memory=False)\n", + "\n", + "# remove all rows that are not human\n", + "entrez_gene_data = entrez_gene_data.loc[entrez_gene_data['#tax_id'].apply(lambda x: x == 9606)]\n", + "\n", + "# replace NaN and '-' with 'None'\n", + "entrez_gene_data.fillna('None', inplace=True)\n", + "entrez_gene_data.replace('-','None', inplace=True, regex=False)\n", + "\n", + "# update prefixes\n", + "entrez_gene_data['GeneID'] = 'NCBIGene_' + entrez_gene_data['GeneID'].astype('str')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create metadata\n", + "for idx, row in tqdm(entrez_gene_data.iterrows(), total=entrez_gene_data.shape[0]):\n", + " if row['GeneID'] != 'None':\n", + " genes, lab, desc, syn = [], [], [], []\n", + " gene_id, sym, defn = row['GeneID'], row['Symbol'], row['description']\n", + " gene_type, dbxref = row['type_of_gene'], row['dbXrefs']\n", + " chrom, map_loc, s1, s2 = row['chromosome'], row['map_location'], row['Synonyms'], row['Other_designations']\n", + " genes.append('http://www.ncbi.nlm.nih.gov/gene/' + str(gene_id))\n", + " if sym != 'None' or sym != '': lab.append(sym)\n", + " else: lab.append('Entrez_ID:' + gene_id)\n", + " if 'None' not in [defn, gene_type, chrom, map_loc]:\n", + " desc_str = \"{} has locus group '{}' and is located on chromosome {} ({}).\"\n", + " desc.append(desc_str.format(sym, gene_type, chrom, map_loc))\n", + " else: desc.append(\"{} locus group '{}'.\".format(sym, gene_type))\n", + " if s1 != 'None' and s2 != 'None': syn.append('|'.join(set([x for x in (s1 + s2).split('|') if x != 'None' or x != ''])))\n", + " elif s1 != 'None': syn.append('|'.join(set([x for x in s1.split('|') if x != 'None' or x != ''])))\n", + " elif s2 != 'None': syn.append('|'.join(set([x for x in s2.split('|') if x != 'None' or x != ''])))\n", + " else: syn.append('None')\n", + " # update master dictionary\n", + " master_metadata_dictionary['nodes'][gene_id] = {\n", + " 'Label': ''.join(lab),\n", + " 'Description': ''.join(desc),\n", + " 'Synonym': '|'.join(syn),\n", + " 'Dbxref': dbxref}\n", + "\n", + "# delete unneeded data\n", + "del entrez_gene_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### RNA Metadata Dictionary \n", + "\n", + "**Data Source Wiki Page:** [Ensembl](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#ensembl)\n", + "\n", + "The nested dictionary of rna metadata is created by looping over the cleaned human [Ensembl](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#ensembl) gene, RNA, and protein identifier data set (`ensembl_identifier_data_cleaned.txt`). The `keys` of the dictionary are `Ensembl transcript identifiers` and the `values` are dictionaries for each metadata type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load data\n", + "rna_gene_data = pandas.read_csv(processed_data_location + 'ensembl_identifier_data_cleaned.txt', header=0, delimiter='\\t', low_memory=False)\n", + "\n", + "# remove rows without identifiers\n", + "rna_gene_data = rna_gene_data.loc[rna_gene_data['transcript_stable_id'].apply(lambda x: x != 'None')]\n", + "\n", + "# remove unneeded columns\n", + "rna_gene_data.drop(['ensembl_gene_id', 'symbol', 'protein_stable_id', 'uniprot_id', 'master_transcript_type',\n", + " 'entrez_id', 'ensembl_gene_type', 'master_gene_type', 'symbol'], axis=1, inplace=True)\n", + "\n", + "# remove duplicates\n", + "rna_gene_data.drop_duplicates(subset=['transcript_stable_id', 'transcript_name', 'ensembl_transcript_type'], keep='first', inplace=True)\n", + "\n", + "# update prefixes\n", + "rna_gene_data['transcript_stable_id'] = 'ensembl_' + rna_gene_data['transcript_stable_id'].astype('str')\n", + "\n", + "# replace NaN with 'None'\n", + "rna_gene_data.fillna('None', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create metadata\n", + "for idx, row in tqdm(rna_gene_data.iterrows(), total=rna_gene_data.shape[0]):\n", + " rna, lab, desc, syn = [], [], [], []\n", + " rna_id = row['transcript_stable_id']\n", + " ent_type, nme = row['ensembl_transcript_type'], row['transcript_name']\n", + " rna.append('https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=' + rna_id)\n", + " if nme != 'None': lab.append(nme)\n", + " else:\n", + " lab.append('Ensembl_Transcript_ID:' + rna_id)\n", + " nme = 'Ensembl_Transcript_ID:' + rna_id\n", + " if ent_type != 'None': desc.append(\"Transcript {} is classified as type '{}'.\".format(nme, ent_type))\n", + " else: desc.append('None')\n", + " syn.append('None')\n", + " \n", + " # update master dictionary\n", + " master_metadata_dictionary['nodes'][rna_id] = {\n", + " 'Label': ''.join(lab),\n", + " 'Description': ''.join(desc),\n", + " 'Synonym': '|'.join(syn)}\n", + "\n", + "# delete unneeded data\n", + "del rna_gene_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Variant Metadata Dictionary \n", + "\n", + "**Data Source Wiki Page:** [ClinVar Variant](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#clinvar)\n", + "\n", + "The nested dictionary of rna metadata is created by looping over the human [ClinVar Variant](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#clinvar) identifier data set ([`variant_summary.txt`](ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz)). The `keys` of the dictionary are `dbSNP identifiers` and the `values` are dictionaries for each metadata type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'variant_summary.txt'):\n", + " data_downloader(url, unprocessed_data_location)\n", + "\n", + "# load data\n", + "var_data = pandas.read_csv(unprocessed_data_location + 'variant_summary.txt', header=0, delimiter='\\t', low_memory=False)\n", + "\n", + "# remove rows without identifiers\n", + "var_data = var_data.loc[var_data['Assembly'].apply(lambda x: x == 'GRCh38')]\n", + "var_data = var_data.loc[var_data['RS# (dbSNP)'].apply(lambda x: x != -1)]\n", + "\n", + "# de-dup data\n", + "var_metadata = var_data[['VariationID', '#AlleleID', 'Type', 'Name', 'ClinicalSignificance', 'RS# (dbSNP)', 'Origin',\n", + " 'ChromosomeAccession', 'Chromosome', 'Start', 'Stop', 'ReferenceAllele', 'OtherIDs',\n", + " 'Assembly', 'AlternateAllele','Cytogenetic', 'ReviewStatus', 'LastEvaluated']] \n", + "# update prefixes\n", + "var_metadata['VariationID'] = 'clinvar_' + var_metadata['VariationID'].astype('str')\n", + "\n", + "\n", + "# replace NaN with 'None'\n", + "var_metadata.replace('na', 'None', inplace=True)\n", + "var_metadata.fillna('None', inplace=True)\n", + "\n", + "# remove duplicate dbSNP ids by choosing the most recent reviewed variant\n", + "var_metadata.sort_values('LastEvaluated', ascending=False, inplace=True)\n", + "var_metadata.drop_duplicates(subset='RS# (dbSNP)', keep='first', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create metadata\n", + "for idx, row in tqdm(var_metadata.iterrows(), total=var_metadata.shape[0]):\n", + " if row['VariationID'] != 'None':\n", + " variant, label, desc, syn = [], [], [], []\n", + " var_id, lab, dbxref = row['VariationID'], row['Name'], row['OtherIDs']\n", + " variant.append('https://www.ncbi.nlm.nih.gov/snp/rs' + str(var_id))\n", + " if lab != 'None': label.append(lab)\n", + " else: label.append('dbSNP_ID:rs' + str(var_id))\n", + " sent = \"This variant is a {} {} located on chromosome {} ({}, start:{}/stop:{} positions, \" +\\\n", + " \"cytogenetic location:{}) and has clinical significance '{}'. \" +\\\n", + " \"This entry is for the {} and was last reviewed on {} with review status '{}'.\"\n", + " desc.append(sent.format(row['Origin'].replace(';', '/'), row['Type'].replace(';', '/'), row['Chromosome'], row['ChromosomeAccession'],\n", + " row['Start'], row['Stop'], row['Cytogenetic'], row['ClinicalSignificance'],\n", + " row['Assembly'], row['LastEvaluated'], row['ReviewStatus']).replace('None', 'UNKNOWN'))\n", + " syn.append('None')\n", + " \n", + " # update master dictionary\n", + " master_metadata_dictionary['nodes'][var_id] = {\n", + " 'Label': ''.join(lab),\n", + " 'Description': ''.join(desc),\n", + " 'Synonym': '|'.join(syn),\n", + " 'Dbxref': dbxref}\n", + "\n", + "# delete unneeded data\n", + "del var_metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Pathway Metadata Dictionary \n", + "\n", + "**Data Source Wiki Page:** [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#reactome-pathway-database)\n", + "\n", + "The nested dictionary of pathway metadata is created by looping over the human [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#reactome-pathway-database) identifier data set ([`ReactomePathways.txt`](https://reactome.org/download/current/ReactomePathways.txt)); Reactome-Gene Association data ([`gene_association.reactome.gz`](https://reactome.org/download/current/gene_association.reactome.gz)), and Reactome-ChEBI data ([`ChEBI2Reactome_All_Levels.txt`](https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt)). The `keys` of the dictionary are `Reactome identifiers` and the `values` are dictionaries for each metadata type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download reactome pathways data\n", + "url = 'https://reactome.org/download/current/ReactomePathways.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'ReactomePathways.txt'):\n", + " data_downloader(url, unprocessed_data_location)\n", + "# load data\n", + "reactome_pathways = pandas.read_csv(unprocessed_data_location + 'ReactomePathways.txt', header=None, delimiter='\\t', low_memory=False)\n", + "reactome_pathways = reactome_pathways.loc[reactome_pathways[2].apply(lambda x: x == 'Homo sapiens')] \n", + "\n", + "# download reactome gene association data\n", + "url = 'https://reactome.org/download/current/gene_association.reactome.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'gene_association.reactome'):\n", + " data_downloader(url, unprocessed_data_location)\n", + "# load data\n", + "reactome_pathways2 = pandas.read_csv(unprocessed_data_location + 'gene_association.reactome', header=None, delimiter='\\t', skiprows=4, low_memory=False)\n", + "reactome_pathways2 = reactome_pathways2.loc[reactome_pathways2[12].apply(lambda x: x == 'taxon:9606')]\n", + "reactome_pathways2[5] = reactome_pathways2[5].str.replace('REACTOME:','', regex=True) \n", + "\n", + "# download reactome CHEBI data\n", + "url = 'https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt'):\n", + " data_downloader(url, unprocessed_data_location)\n", + "# load data\n", + "reactome_pathways3 = pandas.read_csv(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt', header=None, delimiter='\\t', low_memory=False)\n", + "# remove all non-human pathways and save as list\n", + "reactome_pathways3 = reactome_pathways3.loc[reactome_pathways3[5].apply(lambda x: x == 'Homo sapiens')] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get metadata\n", + "nodes = list(set(reactome_pathways[0]) | set(reactome_pathways2[5]) | set(reactome_pathways3[1]))\n", + "pathway_metadata_final = metadata_api_mapper(nodes)\n", + "\n", + "# update dictionary\n", + "pathway_metadata_final['ID'] = pathway_metadata_final['ID'].map('reactome_{}'.format)\n", + "pathway_metadata_final.set_index('ID', inplace=True)\n", + "\n", + "# add entries to existing dictionary\n", + "master_metadata_dictionary['nodes'].update(pathway_metadata_final.to_dict('index'))\n", + "\n", + "# delete unneeded data\n", + "del reactome_pathways, reactome_pathways2, reactome_pathways3" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Relations Metadata Dictionary \n", + "\n", + "**Data Source Wiki Page:** [Relations Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#relations-ontology)\n", + "\n", + "The nested dictionary of relation metadata is created by looping over the human [Relations Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#relations-ontology) identifier data set (`ro_with_imports.owl`). The `keys` of the dictionary are `Relations Ontology identifiers` and the `values` are dictionaries for each metadata type." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download ontology\n", + "if not os.path.exists(unprocessed_data_location + 'ro_with_imports.owl'):\n", + " command = '{} {} --merge-import-closure -o {}'\n", + " os.system(command.format(owltools_location, 'http://purl.obolibrary.org/obo/ro.owl',\n", + " unprocessed_data_location + 'ro_with_imports.owl'))\n", + "# load graph\n", + "ro_graph = Graph().parse(unprocessed_data_location + 'ro_with_imports.owl')\n", + "print('There are {} edges in the ontology (date:{})'.format(len(ro_graph), datetime.datetime.now().strftime('%m/%d/%Y')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get metadata\n", + "relation_metadata_dict, obo = {}, Namespace('http://purl.obolibrary.org/obo/')\n", + "\n", + "# get ontology information\n", + "cls = [x for x in gets_ontology_classes(ro_graph) if '/RO_' in str(x)] +\\\n", + " [x for x in gets_object_properties(ro_graph) if '/RO_' in str(x)]\n", + "master_synonyms = [x for x in ro_graph if 'synonym' in str(x[1]).lower() and isinstance(x[0], URIRef)]\n", + "\n", + "for x in tqdm(cls):\n", + " # labels\n", + " cls_label = [x for x in ro_graph.objects(x, RDFS.label) if '@' not in n3(x) or '@en' in n3(x)]\n", + " labels = str(cls_label[0]) if len(cls_label) > 0 else 'None'\n", + " # synonyms\n", + " cls_syn = [str(i[2]) for i in master_synonyms if x == i[0]]\n", + " synonym = str(cls_syn[0]) if len(cls_syn) > 0 else 'None'\n", + " # description\n", + " cls_desc = [x for x in ro_graph.objects(x, obo.IAO_0000115) if '@' not in n3(x) or '@en' in n3(x)]\n", + " desc = '|'.join([str(cls_desc[0])]) if len(cls_desc) > 0 else 'None'\n", + " \n", + " relation_metadata_dict[str(x).split('/')[-1]] = {\n", + " 'Label': labels, 'Description': desc, 'Synonym': synonym\n", + " }\n", + "\n", + "# add entries to existing dictionary\n", + "master_metadata_dictionary['relations'].update(relation_metadata_dict)\n", + "\n", + "# delete unneeded data\n", + "del ro_graph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "#### Secondary Metadata Elements \n", + "\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Download Identifier Maps\n", + "\n", + "This code chunk downloads identifier mapping files that were creating in the prior steps.\n", + "\n", + "- Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + "- Genes: [UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt) \n", + "- Proteins: \n", + " - [ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt) \n", + " - [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + " - [STRING_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/STRING_PRO_ONTOLOGY_MAP.txt)\n", + "- RNA: \n", + " - [ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt) \n", + " - [GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt)\n", + "- Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt) \n", + "- Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# entrez-ensembl map\n", + "rna_map = pandas.read_csv(processed_data_location + 'ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False,\n", + " names=['Entrez_Gene_IDs', 'Ensembl_Transcript_IDs', 'Entrez_Gene_Type',\n", + " 'Ensembl_Transcript_Type', 'Master_Gene_Type', 'Master_Transcript_Type',\n", + " 'Entrez_Gene_prefix'])\n", + "# entrez-pro map\n", + "entrez_pro_map = pandas.read_csv(processed_data_location + 'ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False, usecols = [0, 1, 2, 4],\n", + " names=['Gene_IDs', 'Protein_Ontology_IDs', 'Entrez_Gene_Type',\n", + " 'Master_Gene_Type', 'Entrez_Gene_Prefix'])\n", + "# symbol-ensembl map\n", + "symbol_transcript_map = pandas.read_csv(processed_data_location + 'GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False,\n", + " names=['Gene_Symbols', 'Ensembl_Transcript_IDs',\n", + " 'Gene_Type', 'Ensembl_Transcript_Type',\n", + " 'Master_Gene_Type', 'Master_Transcript_Type'])\n", + "\n", + "# string-pro map\n", + "string_pro_map = pandas.read_csv(processed_data_location + 'STRING_PRO_ONTOLOGY_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False, usecols=[0, 1],\n", + " names=['STRING_IDs', 'Protein_Ontology_IDs'])\n", + "# uniprot-pro map\n", + "uniprot_pro_map = pandas.read_csv(processed_data_location + 'UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False, usecols=[0, 1],\n", + " names=['Uniprot_Accession_IDs', 'Protein_Ontology_IDs'])\n", + "# uniprot-entrez gene map\n", + "uniprot_entrez_data = pandas.read_csv(processed_data_location + 'UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt',\n", + " header=None, delimiter='\\t', low_memory=False, usecols=[0, 1, 2, 3],\n", + " names=['Uniprot_Accession_IDs', 'Entrez_Gene_IDs',\n", + " 'master_gene_type', 'gene_type_update'])\n", + "# mesh-chebi map\n", + "mesh_chebi_map = pandas.read_csv(processed_data_location + 'MESH_CHEBI_MAP.txt', header=None, \n", + " names=['MESH_ID', 'CHEBI_ID'], delimiter='\\t')\n", + "# disease maps\n", + "disease_maps = pandas.read_csv(processed_data_location + 'DISEASE_MONDO_MAP.txt', header=None,\n", + " names=['Disease_IDs', 'MONDO_IDs'], delimiter='\\t')\n", + "# phenotype maps\n", + "phenotype_maps = pandas.read_csv(processed_data_location + 'PHENOTYPE_HPO_MAP.txt', header=None,\n", + " names=['Disease_IDs', 'HP_IDs'], delimiter='\\t')\n", + "\n", + "# cells and anatomical entities\n", + "anatomy_maps = pandas.read_csv(processed_data_location + 'HPA_GTEx_TISSUE_CELL_MAP.txt', header=None,\n", + " names=['anatomy_ids', 'ontolgoy_ids'], delimiter='\\t')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "##### Genomic Entity Metadata \n", + "\n", + "Process the dictionary created in the prior steps in order to assist with creating a master metadata file for all nodes that are a genomic entity (i.e., genes, transcripts, or proteins). Some example output is shown below:\n", + "\n", + "``` python\n", + "{'NCBIGene_51471': {\n", + " 'Synonyms': ['acetyltransferase 1',\n", + " 'Hcml2',\n", + " 'probable N-acetyltransferase 8B',\n", + " 'CML2',\n", + " 'putative N-acetyltransferase 8B',\n", + " 'ATase1',\n", + " 'N-acetyltransferase 8B (putative, gene/pseudogene)',\n", + " 'N-acetyltransferase Camello 2',\n", + " 'NAT8BP',\n", + " 'camello-like protein 2',\n", + " 'N-acetyltransferase 8B (GCN5-related, putative, gene/pseudogene)',\n", + " 'putative N-acetyltransferase 8B',\n", + " 'ATase1',\n", + " 'N-acetyltransferase 8B (GCN5-related, putative, gene/pseudogene)',\n", + " 'N-acetyltransferase Camello 2',\n", + " 'acetyltransferase 1',\n", + " 'camello-like protein 2',\n", + " 'probable N-acetyltransferase 8B'],\n", + " 'PR': ['PR_Q9UHF3'],\n", + " 'GeneSymbol': ['GeneSymbol_NAT8BP',\n", + " 'GeneSymbol_Hcml2',\n", + " 'GeneSymbol_CML2',\n", + " 'GeneSymbol_NAT8B'],\n", + " 'ensembl gene': ['ensembl_ENSG00000204872'],\n", + " 'ensembl protein': ['ensembl_ENSP00000485054'],\n", + " 'map_location': ['2p13.1'],\n", + " 'Label': ['N-acetyltransferase 8B (putative, gene/pseudogene)'],\n", + " 'ensembl transcript': ['ensembl_ENST00000377712'],\n", + " 'transcript_name': ['NAT8B-201'],\n", + " 'HGNC_ID': ['HGNC_ID_30235'],\n", + " 'chromosome': ['2'],\n", + " 'uniprot': ['uniprot_Q9UHF3']}}\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load data -- only reload if the dictionary has not been populated\n", + "if not 'reformatted_mapped_identifiers' in locals():\n", + " filepath = processed_data_location + 'Merged_gene_rna_protein_identifiers.pkl'\n", + " max_bytes = 2**31 - 1; input_size = os.path.getsize(filepath); bytes_in = bytearray(0)\n", + " with open(filepath, 'rb') as f_in:\n", + " for _ in range(0, input_size, max_bytes):\n", + " bytes_in += f_in.read(max_bytes)\n", + " reformatted_mapped_identifiers = pickle.loads(bytes_in)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# clean up data for use with master metadata\n", + "genomic_metadata = dict()\n", + "for key, value in tqdm(reformatted_mapped_identifiers.items()):\n", + " old_prefix = '_'.join(key.split('_')[0:-1]); idx = key.split('_')[-1]; pass_var = True; new_prefix = None\n", + " if old_prefix == 'entrez_id': new_prefix = 'NCBIGene'\n", + " elif old_prefix in ['ensembl_gene_id', 'protein_stable_id', 'transcript_stable_id']: new_prefix = 'ensembl'\n", + " elif old_prefix == 'pro_id_PR': new_prefix = 'PR'\n", + " else: pass_var = False\n", + " if pass_var and new_prefix is not None:\n", + " updated_key = new_prefix + '_' + idx; master_metadata_dict = {updated_key: {}}\n", + " for x in value:\n", + " i, j = '_'.join(x.split('_')[0:-1]), x.split('_')[-1]\n", + " if 'type' in i: continue\n", + " elif i == 'entrez_id': new_i = 'NCBIGene'; j = new_i + '_' + j\n", + " elif i == 'ensembl_gene_id': new_i = 'ensembl gene'; j = 'ensembl_' + j\n", + " elif i == 'protein_stable_id': new_i = 'ensembl protein'; j = 'ensembl_' + j\n", + " elif i == 'transcript_stable_id': new_i = 'ensembl transcript'; j = 'ensembl_' + j\n", + " elif i == 'pro_id_PR': new_i = 'PR'; j = new_i + '_' + j\n", + " elif i == 'hgnc_id': new_i = 'HGNC_ID'; j = new_i + '_' + j\n", + " elif i == 'uniprot_id': new_i = 'uniprot'; j = new_i + '_' + j\n", + " elif i == 'symbol': new_i = 'GeneSymbol'; j = new_i + '_' + j\n", + " else:\n", + " if i == 'synonyms': new_i = 'Synonyms'\n", + " elif i == 'name': new_i = 'Label'\n", + " elif i == 'Other_designations': new_i = 'Synonyms'; j = j.split('|')\n", + " else: new_i = i\n", + " if new_i in master_metadata_dict[updated_key].keys():\n", + " if isinstance(j , list): master_metadata_dict[updated_key][new_i] += j\n", + " else: master_metadata_dict[updated_key][new_i] += [j]\n", + " else: master_metadata_dict[updated_key][new_i] = [j]\n", + " genomic_metadata[updated_key] = master_metadata_dict\n", + " \n", + "# delete unneeded data\n", + "del reformatted_mapped_identifiers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `CTD_chem_gene_ixns.tsv` \n", + "\n", + "**Data Source Wiki Page:** [Comparative Toxicogenomics Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#comparative-toxicogenomics-database)\n", + "\n", + "\n", + "**Edges:** \n", + "- `chemical-gene` \n", + "- `chemical-protein` \n", + "- `chemical-rna` \n", + "\n", + "**Identifier Maps:** \n", + "- Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + "- Proteins: [ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt) \n", + "- RNA: [ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt)\n", + "\n", + "This chunk process the [`CTD_chem_gene_ixns.tsv`](http://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + "_chemical_ \n", + " - `ChemicalID`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, MeSH Identifier. Variable is provided as a string without a prefix. \n", + " - `CasRN`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, a string containing a CAS Registry Number, if available. \n", + " - `ChemicalName`: A string containing the concept's synonym. If derived from an ontology, the string will be prefixed by the synonym type. If not, a string containing the name of the chemical. \n", + " \n", + " _Gene, RNA, and Protein_ \n", + " - `GenomicInformation`: A dictionary of gene, RNA, and protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Interaction`: A string describing a chemical-gene/protein/rna interaction. \n", + " - `InteractionActions`: A \"|\"-delimited list of the actions that underlie an interaction. \n", + " - `PubMedIDs`: |'-delimited list of PubMed identifiers that do not include a prefix. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://ctdbase.org/reports/CTD_chem_gene_ixns.tsv.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'CTD_chem_gene_ixns.tsv'):\n", + " data_downloader(url, unprocessed_data_location, 'CTD_chem_gene_ixns.tsv')\n", + "\n", + "# load data\n", + "ctd_gene_inx = pandas.read_csv(unprocessed_data_location + 'CTD_chem_gene_ixns.tsv', header=0, delimiter='\\t', skiprows=27)\n", + "ctd_gene_inx = ctd_gene_inx[ctd_gene_inx['# ChemicalName'] != '#']\n", + "ctd_gene_inx = ctd_gene_inx[ctd_gene_inx['OrganismID'] == 9606]\n", + "ctd_gene_inx = ctd_gene_inx[ctd_gene_inx['PubMedIDs'] != numpy.nan]\n", + "ctd_gene_inx.fillna('None', inplace=True)\n", + "# fix variable typing\n", + "ctd_gene_inx['GeneID'] = ctd_gene_inx['GeneID'].astype('Int64')\n", + "ctd_gene_inx['OrganismID'] = ctd_gene_inx['OrganismID'].astype('Int64')\n", + "# update prefix\n", + "ctd_gene_inx['ChemicalID'] = 'MESH:' + ctd_gene_inx['ChemicalID']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge identifier maps\n", + "ctd_gene_inx = ctd_gene_inx.merge(mesh_chebi_map, left_on='ChemicalID', right_on='MESH_ID')\n", + "ctd_gene_inx = ctd_gene_inx.merge(rna_map, left_on='GeneID', right_on='Entrez_Gene_IDs')\n", + "ctd_gene_inx = ctd_gene_inx.merge(entrez_pro_map, left_on='GeneID', right_on='Gene_IDs')\n", + "\n", + "# visualize data\n", + "ctd_gene_inx.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'] = {'chemical-gene': {}, 'chemical-rna': {}, 'chemical-protein': {}}\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(ctd_gene_inx.iterrows(), total=ctd_gene_inx.shape[0]):\n", + " chebi = row['CHEBI_ID'].rstrip(); gene_form = None\n", + " chemical_name = row['# ChemicalName']; chemical_id = row['ChemicalID'].rstrip(); casrn = row['CasRN']\n", + " evidence = [{'CTD_Interaction': row['Interaction'],\n", + " 'CTD_InteractionActions': row['InteractionActions'],\n", + " 'CTD_PubMedIDs': row['PubMedIDs']}]\n", + " if row['GeneForms'] == 'gene':\n", + " node_key = row['Entrez_Gene_prefix'].rstrip(); gene_form = row['GeneForms']\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " else: genomic_info_dict = None\n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-gene'\n", + " if row['GeneForms'] == 'protein':\n", + " node_key = row['Protein_Ontology_IDs'].rstrip(); gene_form = row['GeneForms']\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " else: genomic_info_dict = None\n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-protein'\n", + " if row['GeneForms'] == 'mRNA':\n", + " node_key = row['Ensembl_Transcript_IDs'].rstrip(); gene_form = row['GeneForms']\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " else: genomic_info_dict = None\n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-rna'\n", + " if gene_form is not None:\n", + " # add chebi metadata\n", + " if chebi in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][chebi].keys():\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalName'] |= {chemical_name}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalID'] |= {chemical_id}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_CasRN'] |= {casrn}\n", + " else:\n", + " master_metadata_dictionary['nodes'][chebi].update({\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({chebi: {\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}}})\n", + " \n", + " # add genomic information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'CTD_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['CTD_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['CTD_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'CTD_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'CTD_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'CTD_Evidence': evidence, 'Type': edge_type}}})\n", + " \n", + "# delete unneeded data\n", + "del ctd_gene_inx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `CTD_chem_go_enriched.tsv` \n", + "\n", + "**Data Source Wiki Page:** [Comparative Toxicogenomics Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#comparative-toxicogenomics-database)\n", + "\n", + "**Edges:** \n", + "- `chemical-gobp` \n", + "- `chemical-gocc` \n", + "- `chemical-gomf` \n", + "\n", + "**Identifier Maps:** \n", + "- Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + "\n", + "This chunk process the [`CTD_chem_go_enriched.tsv`](http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Chemical_ \n", + " - `ChemicalID`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, MeSH Identifier. Variable is provided as a string without a prefix. \n", + " - `CasRN`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, a string containing a CAS Registry Number, if available. \n", + " - `ChemicalName`: A string containing the concept's synonym. If derived from an ontology, the string will be prefixed by the synonym type. If not, a string containing the name of the chemical. \n", + " \n", + " _GO Biological Process, Cellular Component, Molecular Function_ \n", + " - `GOTermName`: A string containing the concept's synonym. \n", + " - `Ontology`: A string naming the GO Ontology subset. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `HighestGOLevel`: The highest level to which the GO term is assigned within the GO hierarchical ontology. Many GO terms are located at multiple levels within the ontology; only the highest level is displayed. Level 1 constitutes “children” of the most general Biological Process, Cellular Component, and Molecular Function terms. Source: http://ctdbase.org/help/chemGODetailHelp.jsp. \n", + " - `Pvalue`: Raw P-value. Source: http://ctdbase.org/help/chemGODetailHelp.jsp. \n", + " - `CorrectedPValue`:The corrected p-value calculated using the Bonferroni multiple testing adjustment. Source: http://ctdbase.org/help/chemGODetailHelp.jsp. \n", + " - `TargetMatchQty`: The count of matches to the target. Source: http://ctdbase.org/help/chemGODetailHelp.jsp. \n", + " - `TargetTotalQty`: The total matches to the target. Source: http://ctdbase.org/help/chemGODetailHelp.jsp.\n", + " - `BackgroundMatchQty`: The count of matches to the genome. Source: http://ctdbase.org/help/chemGODetailHelp.jsp.\n", + " - `BackgroundTotalQty`: The total matches to the genome. Source: http://ctdbase.org/help/chemGODetailHelp.jsp." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'CTD_chem_go_enriched.tsv'):\n", + " data_downloader(url, unprocessed_data_location, 'CTD_chem_go_enriched.tsv')\n", + "\n", + "# load data\n", + "ctd_chem_go = pandas.read_csv(unprocessed_data_location + 'CTD_chem_go_enriched.tsv', header=0, delimiter='\\t', skiprows=27)\n", + "ctd_chem_go = ctd_chem_go[ctd_chem_go['# ChemicalName'] != '#']\n", + "ctd_chem_go.fillna('None', inplace=True)\n", + "# update prefix\n", + "ctd_chem_go['ChemicalID'] = 'MESH:' + ctd_chem_go['ChemicalID']\n", + "ctd_chem_go['GOTermID'] = ctd_chem_go['GOTermID'].str.replace(':', '_')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge identifier maps\n", + "ctd_chem_go = ctd_chem_go.merge(mesh_chebi_map, left_on='ChemicalID', right_on='MESH_ID')\n", + "\n", + "# visualize data\n", + "ctd_chem_go.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'chemical-gobp': {}, 'chemical-gocc': {}, 'chemical-gomf': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(ctd_chem_go.iterrows(), total=ctd_chem_go.shape[0]):\n", + " chebi = row['CHEBI_ID'].rstrip(); node_key = row['GOTermID']\n", + " chemical_name = row['# ChemicalName']; chemical_id = row['ChemicalID'].rstrip(); casrn = row['CasRN']\n", + " ontology = row['Ontology']; go_name = row['GOTermName']\n", + " evidence = [{'CTD_Pvalue': row['PValue'],\n", + " 'CTD_CorrectedPValue': row['CorrectedPValue'],\n", + " 'CTD_TargetMatchQty': row['TargetMatchQty'],\n", + " 'CTD_TargetTotalQty': row['TargetTotalQty'],\n", + " 'CTD_BackgroundMatchQty': row['BackgroundMatchQty'],\n", + " 'CTD_BackgroundTotalQty': row['BackgroundTotalQty'],\n", + " 'CTD_HighestGOLevel': row['HighestGOLevel']}]\n", + " # specify edge type, which is related to the ontology aspect\n", + " if ontology == 'Biological Process': edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-gobp'\n", + " if ontology == 'Cellular Component': edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-gocc'\n", + " if ontology == 'Molecular Function': edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-gomf' \n", + " \n", + " # add chebi metadata\n", + " if chebi in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][chebi].keys():\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalName'] |= {chemical_name}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalID'] |= {chemical_id}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_CasRN'] |= {casrn}\n", + " else:\n", + " master_metadata_dictionary['nodes'][chebi].update({\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({chebi: {\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}}})\n", + " \n", + " # add go information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['CTD_Ontology'] |= {ontology}\n", + " master_metadata_dictionary['nodes'][node_key][url]['CTD_GOTermName'] |= {go_name}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({\n", + " url: {'CTD_Ontology': {ontology},\n", + " 'CTD_GOTermName': {go_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {\n", + " url: {'CTD_Ontology': {ontology},\n", + " 'CTD_GOTermName': {go_name}}}})\n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'CTD_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['CTD_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['CTD_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'CTD_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'CTD_Evidence': evidence, 'Type': edge_type}}) \n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'CTD_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del ctd_chem_go" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `CTD_chemicals_diseases.tsv` \n", + "\n", + "**Data Source Wiki Page:** [Comparative Toxicogenomics Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#comparative-toxicogenomics-database)\n", + "\n", + "**Edges:** \n", + "- `chemical-disease` \n", + "- `chemical-phenotype` \n", + "\n", + "**Identifier Maps:** \n", + "- Chemicals: [MESH_CHEBI_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/MESH_CHEBI_MAP.txt) \n", + "- Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt) \n", + "- Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + "\n", + "This chunk process the [`CTD_chemicals_diseases.tsv`](http://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Chemical_ \n", + " - `ChemicalID`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, MeSH Identifier. Variable is provided as a string without a prefix. \n", + " - `CasRN`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. If not, a string containing a CAS Registry Number, if available. \n", + " - `ChemicalName`: A string containing the concept's synonym. If derived from an ontology, the string will be prefixed by the synonym type. If not, a string containing the name of the chemical. \n", + " \n", + " _Disease, Phenotype_ \n", + " - `DiseaseName`: A string containing the concept's synonym. \n", + " - `DiseaseID`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. \n", + " - `OmimIDs`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `DirectEvidence`: '|'-delimited list of strings that include keywords. \n", + " - `InferenceScore`: The inference score (float) reflects the degree of similarity between CTD chemical–gene–disease networks and a similar scale-free random network. The higher the score, the more likely the inference network has atypical connectivity. \n", + " - `PubMedIDs`: |'-delimited list of PubMed identifiers that do not include a prefix. \n", + " - `InferenceGeneSymbol`: A string containing the gene symbol. The genes on which the inferred association is based (i.e., genes that have curated interactions with the chemical and curated associations with the disease). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'CTD_chemicals_diseases.tsv'):\n", + " data_downloader(url, unprocessed_data_location, 'CTD_chemicals_diseases.tsv')\n", + "\n", + "# load data\n", + "ctd_chem_dis = pandas.read_csv(unprocessed_data_location + 'CTD_chemicals_diseases.tsv', header=0, delimiter='\\t', skiprows=27)\n", + "ctd_chem_dis = ctd_chem_dis[ctd_chem_dis['# ChemicalName'] != '#']\n", + "ctd_chem_dis = ctd_chem_dis[ctd_chem_dis['PubMedIDs'] != numpy.nan]\n", + "ctd_chem_dis = ctd_chem_dis[ctd_chem_dis['DiseaseID'] != numpy.nan]\n", + "# update prefix\n", + "ctd_chem_dis['ChemicalID'] = 'MESH:' + ctd_chem_dis ['ChemicalID']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ctd_chem_dis = ctd_chem_dis.merge(mesh_chebi_map, left_on='ChemicalID', right_on='MESH_ID')\n", + "ctd_chem_dis = ctd_chem_dis.merge(disease_maps, left_on='DiseaseID', right_on='Disease_IDs')\n", + "ctd_chem_dis = ctd_chem_dis.merge(phenotype_maps, left_on='DiseaseID', right_on='Disease_IDs')\n", + "\n", + "# visualize data\n", + "ctd_chem_dis.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'chemical-disease': {}, 'chemical-phenotype': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(ctd_chem_dis.iterrows(), total=ctd_chem_dis.shape[0]):\n", + " chebi = row['CHEBI_ID'].rstrip()\n", + " chemical_name = row['# ChemicalName']; chemical_id = row['ChemicalID'].rstrip(); casrn = row['CasRN']\n", + " dis_name = row['DiseaseName']; dis_id = row['DiseaseID']\n", + " omim = row['OmimIDs'] if not pandas.isna(row['OmimIDs']) else 'None'\n", + " evidence = [{'CTD_DirectEvidence': row['DirectEvidence'] if not pandas.isna(row['DirectEvidence']) else 'None',\n", + " 'CTD_InferenceScore': row['InferenceScore'] if not pandas.isna(row['InferenceScore']) else 'None',\n", + " 'CTD_PubMedIDs': row['PubMedIDs'],\n", + " 'CTD_InferenceGeneSymbol': row['InferenceGeneSymbol'] if not pandas.isna(row['InferenceGeneSymbol']) else 'None'}]\n", + " for node_key in [row['MONDO_IDs'], row['HP_IDs']]:\n", + " if not pandas.isna(node_key) and node_key.startswith('MONDO'):\n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-disease'\n", + " if not pandas.isna(node_key) and node_key.startswith('HP'):\n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-phenotype'\n", + " \n", + " # add chebi metadata\n", + " if chebi in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][chebi].keys():\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalName'] |= {chemical_name}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_ChemicalID'] |= {chemical_id}\n", + " master_metadata_dictionary['nodes'][chebi][url]['CTD_CasRN'] |= {casrn}\n", + " else:\n", + " master_metadata_dictionary['nodes'][chebi].update({\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({chebi: {\n", + " url: {'CTD_ChemicalID': {chemical_id},\n", + " 'CTD_CasRN': {casrn},\n", + " 'CTD_ChemicalName': {chemical_name}}}})\n", + " \n", + " # add disease information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key]:\n", + " master_metadata_dictionary['nodes'][node_key][url]['CTD_DiseaseName'] |= {dis_name}\n", + " master_metadata_dictionary['nodes'][node_key][url]['CTD_DiseaseID'] |= {dis_id}\n", + " master_metadata_dictionary['nodes'][node_key][url]['CTD_OmimIDs'] |= {omim}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({\n", + " url: {'CTD_DiseaseName': {dis_name},\n", + " 'CTD_DiseaseID': {dis_id},\n", + " 'CTD_OmimIDs': {omim}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {\n", + " url: {'CTD_DiseaseName': {dis_name},\n", + " 'CTD_DiseaseID': {dis_id},\n", + " 'CTD_OmimIDs': {omim}}}})\n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'CTD_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['CTD_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['CTD_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'CTD_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'CTD_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'CTD_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del ctd_chem_dis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `ChEBI2Reactome_All_Levels.txt` \n", + "\n", + "**Data Source Wiki Page:** [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#reactome-pathway-database) \n", + "\n", + "\n", + "**Edges:** \n", + "- `chemical-pathway` \n", + "\n", + "This chunk process the [`ChEBI2Reactome_All_Levels.txt`](https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt) file and obtains the following node metadata: \n", + "- **Nodes:** \n", + " _Pathway_ \n", + " - `DBReference`: A string containing the concept's database cross-reference, which is formatted as prefix:ID. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `EvidenceID`: A string containing an evidence code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'ChEBI2Reactome_All_Levels.txt')\n", + "\n", + "# load data\n", + "rtm_chem_path = pandas.read_csv(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt', header=None, delimiter='\\t', skiprows=0)\n", + "rtm_chem_path = rtm_chem_path[rtm_chem_path[5] == 'Homo sapiens']\n", + "rtm_chem_path.fillna('None', inplace=True)\n", + "# update prefix\n", + "rtm_chem_path[0] = 'CHEBI_' + rtm_chem_path[0].astype('str')\n", + "rtm_chem_path[1] = 'reactome_' + rtm_chem_path[1]\n", + "\n", + "# visualize data\n", + "rtm_chem_path.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'chemical-pathway': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(rtm_chem_path.iterrows(), total=rtm_chem_path.shape[0]):\n", + " chebi = row[0].rstrip(); node_key = row[1]; path_name = row[3]\n", + " evidence = [{'CTD_EvidenceID': row[4]}] \n", + " edge_key = '{}-{}'.format(chebi, node_key); edge_type = 'chemical-pathway' \n", + " \n", + " # add reactome information \n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['Reactome_PathwayName'] |= {path_name}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({\n", + " url: {'Reactome_PathwayName': {path_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {\n", + " url: {'Reactome_PathwayName': {path_name}}}})\n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'Reactome_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['Reactome_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['Reactome_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'Reactome_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'Reactome_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'Reactome_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del rtm_chem_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `goa_human.gaf` \n", + "\n", + "**Data Source Wiki Page:** [Gene Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#gene-ontology) \n", + "\n", + "\n", + "**Edges:** \n", + "- `protein-gobp` \n", + "- `protein-gocc` \n", + "- `protein-gomf` \n", + "\n", + "**Identifier Maps:** \n", + "- Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + "This chunk process the [`goa_human.gaf`](http://current.geneontology.org/annotations/goa_human.gaf.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + "_GO Biological Process, Cellular Component, and Molecular Function_ \n", + " - `DB_Object_Name`: A string containing the concept's synonym. \n", + " - `DB_Object_Synonym`: A string containing the concept's synonym. \n", + " - `DB_Object_Symbol`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. \n", + " - `With_Or_From`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. \n", + " - `DB_Object_Type`: A string indicating the type of object that has been annotated. \n", + " \n", + " _Protein_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Qualifier`: Some annotations are modified by qualifiers, which have specific usage rules and meanings within GO. \n", + " - `DB_Reference`: One or more unique identifiers for a single source cited as an authority for the attribution of the GO ID to the DB Object ID. This may be a literature reference or a database record. The syntax is DB:accession_number. \n", + " - `EvidenceCode`: Each annotation includes an evidence code to indicate how the annotation to a particular term is supported\n", + " - Inferred from Experiment (EXP)\n", + " - Inferred from Direct Assay (IDA)\n", + " - Inferred from Physical Interaction (IPI)\n", + " - Inferred from Mutant Phenotype (IMP)\n", + " - Inferred from Genetic Interaction (IGI)\n", + " - Inferred from Expression Pattern (IEP)\n", + " - Inferred from High Throughput Experiment (HTP)\n", + " - Inferred from High Throughput Direct Assay (HDA)\n", + " - Inferred from High Throughput Mutant Phenotype (HMP)\n", + " - Inferred from High Throughput Genetic Interaction (HGI)\n", + " - Inferred from High Throughput Expression Pattern (HEP)\n", + " - Inferred from Biological aspect of Ancestor (IBA)\n", + " - Inferred from Biological aspect of Descendant (IBD)\n", + " - Inferred from Key Residues (IKR)\n", + " - Inferred from Rapid Divergence (IRD)\n", + " - Inferred from Sequence or structural Similarity (ISS)\n", + " - Inferred from Sequence Orthology (ISO)\n", + " - Inferred from Sequence Alignment (ISA)\n", + " - Inferred from Sequence Model (ISM)\n", + " - Inferred from Genomic Context (IGC)\n", + " - Inferred from Reviewed Computational Analysis (RCA)\n", + " - Traceable Author Statement (TAS)\n", + " - Non-traceable Author Statement (NAS)\n", + " - Inferred by Curator (IC)\n", + " - No biological Data available (ND)\n", + " - Inferred from Electronic Annotation (IEA) \n", + " - `AssignedBy`: A string indicating who assigned the association. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://current.geneontology.org/annotations/goa_human.gaf.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'goa_human.gaf'):\n", + " data_downloader(url, unprocessed_data_location, 'goa_human.gaf')\n", + "\n", + "# load data\n", + "goa_gene = pandas.read_csv(unprocessed_data_location + 'goa_human.gaf', header=None, delimiter='\\t', skiprows=41, low_memory=False)\n", + "goa_gene = goa_gene[goa_gene[12] == 'taxon:9606']\n", + "goa_gene = goa_gene[goa_gene[3] != 'NOT']\n", + "goa_gene = goa_gene[goa_gene[11] == 'protein']\n", + "# fix prefix\n", + "goa_gene[4] = goa_gene[4].str.replace(':', '_')\n", + "goa_gene.fillna('None', inplace=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "goa_gene = goa_gene.merge(uniprot_pro_map, left_on=1, right_on='Uniprot_Accession_IDs')\n", + "\n", + "# visualize data\n", + "goa_gene.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'protein-gobp': {}, 'protein-gocc': {}, 'protein-gomf': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(goa_gene.iterrows(), total=goa_gene.shape[0]):\n", + " pr = row['Protein_Ontology_IDs'].rstrip(); node_key = row[4]\n", + " pr_db = row[9]; pr_syn = row[10]; pr_symb = row[2]; aspect = row[8]; db_with = row[7]\n", + " evidence = [{'GOA_Qualifier': row[3], 'GOA_DB_Reference': row[5],\n", + " 'GOA_EvidenceCode': row[6], 'GOA_AssignedBy': row[14]}]\n", + " if aspect == 'P': edge_key = '{}-{}'.format(pr, node_key); edge_type = 'protein-gobp'\n", + " if aspect == 'C': edge_key = '{}-{}'.format(pr, node_key); edge_type = 'protein-gocc'\n", + " if aspect == 'F': edge_key = '{}-{}'.format(pr, node_key); edge_type = 'protein-gomf' \n", + " if pr in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[pr]\n", + " else: genomic_info_dict = None\n", + " \n", + " # add pr information\n", + " if pr in master_metadata_dictionary['nodes'].keys(): \n", + " if url in master_metadata_dictionary['nodes'][pr].keys():\n", + " master_metadata_dictionary['nodes'][pr][url]['GOA_DB_Object_Name'] |= {pr_db}\n", + " master_metadata_dictionary['nodes'][pr][url]['GOA_DB_Object_Synonym'] |= {pr_syn}\n", + " master_metadata_dictionary['nodes'][pr][url]['GOA_DB_Object_Symbol'] |= {pr_symb}\n", + " master_metadata_dictionary['nodes'][pr][url]['GOA_With_Or_From'] |= {db_with}\n", + " else:\n", + " master_metadata_dictionary['nodes'][pr].update({\n", + " url: {'GOA_DB_Object_Name': {pr_db},\n", + " 'GOA_DB_Object_Synonym': {pr_syn},\n", + " 'GOA_DB_Object_Symbol': {pr_symb},\n", + " 'GOA_With_Or_From': {db_with}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({pr: {\n", + " url: {'GOA_DB_Object_Name': {pr_db},\n", + " 'GOA_DB_Object_Synonym': {pr_syn},\n", + " 'GOA_DB_Object_Symbol': {pr_symb},\n", + " 'GOA_With_Or_From': {db_with}}}})\n", + " \n", + " # add genomic information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'GOA_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['GOA_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['GOA_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'GOA_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'GOA_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'GOA_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del goa_gene" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt` \n", + "\n", + "**Data Source Wiki Page:** [GeneMANIA](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#genemania) \n", + "\n", + "**Edges:** \n", + "- `gene-gene` \n", + "\n", + "**Identifier Maps:** \n", + "- Genes: [UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt) \n", + "\n", + "This chunk process the [`COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt`](http://genemania.org/data/current/Homo_sapiens.COMBINED/COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt) file and obtains the following edge metadata: \n", + "- **Nodes:** \n", + " _Genes_ \n", + " - `GenomicInformation`: A dictionary of gene identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "- **Edges:** \n", + " - `Weight`: Assumes the input gene list is related through GO biological processes. The score will vary depending on the type of network, but in general is a number ranging from zero (no interaction) to 1 (strong interaction). See `PMID:25254104` for more detail. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://genemania.org/data/current/Homo_sapiens.COMBINED/COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt')\n", + "\n", + "# load data\n", + "gm_gene_gene = pandas.read_csv(unprocessed_data_location + 'COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt', header=0, delimiter='\\t', skiprows=0)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gm_gene_gene = gm_gene_gene.merge(uniprot_entrez_data, left_on='Gene_A', right_on='Uniprot_Accession_IDs')\n", + "gm_gene_gene.rename(columns={'Entrez_Gene_IDs': 'Entrez_Gene_A'}, inplace=True)\n", + "gm_gene_gene = gm_gene_gene.merge(uniprot_entrez_data, left_on='Gene_B', right_on='Uniprot_Accession_IDs')\n", + "gm_gene_gene.rename(columns={'Entrez_Gene_IDs': 'Entrez_Gene_B'}, inplace=True)\n", + "\n", + "# visualize data\n", + "gm_gene_gene.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'gene-gene': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(gm_gene_gene.iterrows(), total=gm_gene_gene.shape[0]):\n", + " genes = [row['Entrez_Gene_A'], row['Entrez_Gene_B']]; weight = row['Weight']; gene_info = []\n", + " edge_key = '{}-{}'.format(row['Entrez_Gene_A'], row['Entrez_Gene_B']); edge_type = 'gene-gene' \n", + " \n", + " for node_key in genes:\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " else: genomic_info_dict = None\n", + " \n", + " # add genomic information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}}) \n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'GeneMania_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " master_metadata_dictionary['edges'][edge_key][url]['GeneMania_Evidence'] = weight\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'GeneMania_Evidence': weight})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'GeneMania_Evidence': weight, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'GeneMania_Evidence': weight, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del gm_gene_gene" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `phenotype.hpoa` \n", + "\n", + "**Data Source Wiki Page:** [Human Phenotype Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-phenotype-ontology) \n", + "\n", + "**Edges:** \n", + "- `disease-phenotype` \n", + "\n", + "**Identifier Maps:** \n", + "- Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt) \n", + "\n", + "This chunk process the [`phenotype.hpoa`](http://purl.obolibrary.org/obo/hp/hpoa/phenotype.hpoa) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Disease, Phenotype_ \n", + " - `DiseaseName`: A string containing the concept's synonym. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Reference`: This required field indicates the source of the information used for the annotation. This may be the clinical experience of the annotator or may be taken from an article as indicated by a PubMed id. Each collaborating center of the Human Phenotype Ontology consortium is assigned a HPO:Ref id. In addition, if appropriate, a PubMed id for an article describing the clinical abnormality may be used. \n", + " - `Evidence`: This required field indicates the level of evidence supporting the annotation. Annotations that have been extracted by parsing the Clinical Features sections of the omim.txt file are assigned the evidence code IEA. Other codes include PCS for published clinical study. This should be used for information extracted from articles in the medical literature. ICE can be used for annotations based on individual clinical experience. This may be appropriate for disorders with a limited amount of published data. This must be accompanied by an entry in the DB:Reference field denoting the individual or center performing the annotation together with an identifier. For instance, GH:007 might be used to refer to the seventh such annotation made by a specialist from Gotham Hospital (assuming the prefix GH has been registered with the HPO). Finally we have TAS, which stands for “traceable author statement”, usually reviews or disease entries (e.g. OMIM) that only refers to the original publication.. \n", + " - `Frequency`: A term-id from the HPO-sub-ontology below the term Frequency.\n", + " There are three allowed options for this field.\n", + " 1. A term-id from the HPO-sub-ontology below the term Frequency.\n", + " 2. A count of patients affected within a cohort. For instance, 7/13 would indicate that 7 of the 13 patients with the specified disease were found to have the phenotypic abnormality referred to by the HPO term in question in the study referred to by the DB_Reference\n", + " 3. A percentage value such as 17%, again referring to the percentage of patients found to have the phenotypic abnormality referred to by the HPO term in question in the study referred to by the DB_Reference. If possible, the 7/13 format is preferred over the percentage format if the exact data is available.. \n", + " - `Sex`: This field contains the strings MALE or FEMALE if the annotation in question is limited to males or females. This field refers to the phenotypic (and not the chromosomal) sex, and does not intend to capture the further complexities of sex determination. If a phenotype is limited to one or the other sex, then the corresponding term from the Clinical modifier subontology should also be used in the Modifier field.\n", + " - `Modifier`: A term from the Clinical modifier subontology. \n", + " - `Aspect`: One of P (Phenotypic abnormality), I (inheritance), C (onset and clinical course). This field is mandatory; cardinality 1. \n", + " - `Biocuration`: This refers to the center or user making the annotation and the date on which the annotation was made; format is YYYY-MM-DD this field is mandatory. Multiple entries can be separated by a semicolon if an annotation was revised, e.g., HPO:skoehler[2010-04-21];HPO:lcarmody[2019-06-02]. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'http://purl.obolibrary.org/obo/hp/hpoa/phenotype.hpoa'\n", + "if not os.path.exists(unprocessed_data_location + 'phenotype.hpoa'):\n", + " data_downloader(url, unprocessed_data_location, 'phenotype.hpoa')\n", + "\n", + "# load data\n", + "hpo_dis_phe = pandas.read_csv(unprocessed_data_location + 'phenotype.hpoa', header=0, delimiter='\\t', skiprows=4, low_memory=False)\n", + "hpo_dis_phe = hpo_dis_phe[hpo_dis_phe['Qualifier'] != 'NOT']\n", + "hpo_dis_phe.fillna('None', inplace=True)\n", + "\n", + "# fix prefix\n", + "hpo_dis_phe['HPO_ID'] = hpo_dis_phe['HPO_ID'].str.replace(':', '_')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hpo_dis_phe = hpo_dis_phe.merge(disease_maps, left_on='#DatabaseID', right_on='Disease_IDs')\n", + "\n", + "# visualize data\n", + "hpo_dis_phe.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'disease-phenotype': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(hpo_dis_phe.iterrows(), total=hpo_dis_phe.shape[0]):\n", + " concepts = [row['MONDO_IDs'], row['HPO_ID']]\n", + " disease_names = {row['MONDO_IDs']: {row['DiseaseName']}, row['HPO_ID']: {'None'}}\n", + " edge_key = '{}-{}'.format(row['MONDO_IDs'], row['HPO_ID']); edge_type = 'disease-phenotype'\n", + " evidence = [{'HPO_Reference': row['Reference'],\n", + " 'HPO_EvidenceCode': row['Evidence'],\n", + " 'HPO_Frequency': row['Frequency'],\n", + " 'HPO_Sex': row['Sex'],\n", + " 'HPO_Modifier': row['Modifier'],\n", + " 'HPO_Aspect': row['Aspect'],\n", + " 'HPO_Biocuration': row['Biocuration']}]\n", + " for node_key in concepts:\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['HPO_DiseaseName'] |= disease_names[node_key]\n", + " else: master_metadata_dictionary['nodes'][node_key].update({url: {'HPO_DiseaseName': disease_names[node_key]}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {url: {'HPO_DiseaseName': disease_names[node_key]}}})\n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'HPO_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['HPO_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['HPO_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'HPO_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'HPO_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'HPO_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del hpo_dis_phe" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `gene_association.reactome` \n", + "\n", + "**Data Source Wiki Page:** [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#reactome-pathway-database) \n", + "\n", + "**Edges:** \n", + "- `gobp-pathway` \n", + "- `pathway-gocc` \n", + "- `pathway-gomf` \n", + "\n", + "This chunk process the [`gene_association.reactome.tsv`](https://reactome.org/download/current/gene_association.reactome.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Pathway_ \n", + " - `DBReference`: A string containing the concept's database cross-reference, which is formatted as Prefix:ID. \n", + " \n", + " _Gene Ontology Biological Proces, Cellular Component, Molecular Function_\n", + " - `Aspect`: A variable indicating what GO subontology is being used. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `EvidenceCode`: Each annotation includes an evidence code to indicate how the annotation to a particular term is supported\n", + " - Inferred from Experiment (EXP)\n", + " - Inferred from Direct Assay (IDA)\n", + " - Inferred from Physical Interaction (IPI)\n", + " - Inferred from Mutant Phenotype (IMP)\n", + " - Inferred from Genetic Interaction (IGI)\n", + " - Inferred from Expression Pattern (IEP)\n", + " - Inferred from High Throughput Experiment (HTP)\n", + " - Inferred from High Throughput Direct Assay (HDA)\n", + " - Inferred from High Throughput Mutant Phenotype (HMP)\n", + " - Inferred from High Throughput Genetic Interaction (HGI)\n", + " - Inferred from High Throughput Expression Pattern (HEP)\n", + " - Inferred from Biological aspect of Ancestor (IBA)\n", + " - Inferred from Biological aspect of Descendant (IBD)\n", + " - Inferred from Key Residues (IKR)\n", + " - Inferred from Rapid Divergence (IRD)\n", + " - Inferred from Sequence or structural Similarity (ISS)\n", + " - Inferred from Sequence Orthology (ISO)\n", + " - Inferred from Sequence Alignment (ISA)\n", + " - Inferred from Sequence Model (ISM)\n", + " - Inferred from Genomic Context (IGC)\n", + " - Inferred from Reviewed Computational Analysis (RCA)\n", + " - Traceable Author Statement (TAS)\n", + " - Non-traceable Author Statement (NAS)\n", + " - Inferred by Curator (IC)\n", + " - No biological Data available (ND)\n", + " - Inferred from Electronic Annotation (IEA) \n", + " - `AssignedBy`: A string indicating who assigned the association. \n", + " - `Qualifier`: Some annotations are modified by qualifiers, which have specific usage rules and meanings within GO. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://reactome.org/download/current/gene_association.reactome.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'gene_association.reactome'):\n", + " data_downloader(url, unprocessed_data_location, 'gene_association.reactome')\n", + "\n", + "# load data\n", + "rce_go_ptw = pandas.read_csv(unprocessed_data_location + 'gene_association.reactome', header=None, delimiter='\\t', skiprows=4)\n", + "rce_go_ptw.fillna('None', inplace=True)\n", + "rce_go_ptw = rce_go_ptw[rce_go_ptw[12] == 'taxon:9606']\n", + "rce_go_ptw = rce_go_ptw[[x.startswith('REACTOME') for x in rce_go_ptw[5]]]\n", + "\n", + "# fix variable prefixing\n", + "rce_go_ptw[4] = rce_go_ptw[4].str.replace(':', '_')\n", + "rce_go_ptw[5] = rce_go_ptw[5].str.replace('REACTOME:', 'reactome_')\n", + "\n", + "# visualize data\n", + "rce_go_ptw.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'gobp-pathway': {}, 'pathway-gocc': {}, 'pathway-gomf': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(rce_go_ptw.iterrows(), total=rce_go_ptw.shape[0]):\n", + " react = row[5].rstrip(); node_key = row[4]; pathway_db = row[0]; aspect = row[8]\n", + " evidence = [{'Reactome_EvidenceCode': row[6],\n", + " 'Reactome_AssignedBy': row[14],\n", + " 'Reactome_Qualifier': row[3]}]\n", + " # specify edge type, which is related to the ontology aspect\n", + " if aspect == 'P': edge_key = '{}-{}'.format(node_key, react); edge_type = 'gobp-pathway'\n", + " if aspect == 'C': edge_key = '{}-{}'.format(react, node_key); edge_type = 'pathway-gocc'\n", + " if aspect == 'F': edge_key = '{}-{}'.format(react, node_key); edge_type = 'pathway-gomf' \n", + " \n", + " # add reactome metadata\n", + " if react in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][react].keys():\n", + " master_metadata_dictionary['nodes'][react][url]['Reactome_DBReference'] |= {pathway_db}\n", + " else: master_metadata_dictionary['nodes'][react].update({url: {'Reactome_DBReference': {pathway_db}}})\n", + " else: master_metadata_dictionary['nodes'].update({react: {url: {'Reactome_DBReference': {pathway_db}}}})\n", + " \n", + " # add go information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['Reactome_Aspect'] |= {aspect}\n", + " else: master_metadata_dictionary['nodes'][node_key].update({url: {'Reactome_Aspect': {aspect}}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {url: {'Reactome_Aspect': {aspect}}}})\n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'Reactome_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['Reactome_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['Reactome_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'Reactome_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'Reactome_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'Reactome_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del rce_go_ptw" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `UniProt2Reactome_All_Levels.txt` \n", + "\n", + "**Data Source Wiki Page:** [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#reactome-pathway-database) \n", + "\n", + "**Edges:** \n", + "- `protein-pathway` \n", + "\n", + "**Identifier Maps:** \n", + "- Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + "This chunk process the [`UniProt2Reactome_All_Levels.txt`](https://reactome.org/download/current/UniProt2Reactome_All_Levels.txt) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Pathway_ \n", + " - `PathwayName`: A string containing the concept's label. \n", + " \n", + " _Protein_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `EvidenceID`: Each annotation includes an evidence code to indicate how the annotation to a particular term is supported\n", + " - Inferred from Experiment (EXP)\n", + " - Inferred from Direct Assay (IDA)\n", + " - Inferred from Physical Interaction (IPI)\n", + " - Inferred from Mutant Phenotype (IMP)\n", + " - Inferred from Genetic Interaction (IGI)\n", + " - Inferred from Expression Pattern (IEP)\n", + " - Inferred from High Throughput Experiment (HTP)\n", + " - Inferred from High Throughput Direct Assay (HDA)\n", + " - Inferred from High Throughput Mutant Phenotype (HMP)\n", + " - Inferred from High Throughput Genetic Interaction (HGI)\n", + " - Inferred from High Throughput Expression Pattern (HEP)\n", + " - Inferred from Biological aspect of Ancestor (IBA)\n", + " - Inferred from Biological aspect of Descendant (IBD)\n", + " - Inferred from Key Residues (IKR)\n", + " - Inferred from Rapid Divergence (IRD)\n", + " - Inferred from Sequence or structural Similarity (ISS)\n", + " - Inferred from Sequence Orthology (ISO)\n", + " - Inferred from Sequence Alignment (ISA)\n", + " - Inferred from Sequence Model (ISM)\n", + " - Inferred from Genomic Context (IGC)\n", + " - Inferred from Reviewed Computational Analysis (RCA)\n", + " - Traceable Author Statement (TAS)\n", + " - Non-traceable Author Statement (NAS)\n", + " - Inferred by Curator (IC)\n", + " - No biological Data available (ND)\n", + " - Inferred from Electronic Annotation (IEA) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://reactome.org/download/current/UniProt2Reactome_All_Levels.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'UniProt2Reactome_All_Levels.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'UniProt2Reactome_All_Levels.txt')\n", + "\n", + "# load data\n", + "rce_prot_pth = pandas.read_csv(unprocessed_data_location + 'UniProt2Reactome_All_Levels.txt', header=None, delimiter='\\t', skiprows=0)\n", + "rce_prot_pth = rce_prot_pth[rce_prot_pth[5] == 'Homo sapiens']\n", + "# fix prefixes\n", + "rce_prot_pth[1] = 'reactome_' + rce_prot_pth[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rce_prot_pth = rce_prot_pth.merge(uniprot_pro_map, left_on=0, right_on='Uniprot_Accession_IDs')\n", + "\n", + "# visualize data\n", + "rce_prot_pth.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'protein-pathway': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(rce_prot_pth.iterrows(), total=rce_prot_pth.shape[0]):\n", + " pr = row['Protein_Ontology_IDs'].rstrip(); node_key = row[1]; react_name = row[3]\n", + " evidence = [{'Reactome_EvidenceID': row[4]}]\n", + " edge_key = '{}-{}'.format(pr, node_key); edge_type = 'protein-pathway'\n", + " if pr in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[pr]\n", + " else: genomic_info_dict = None\n", + " \n", + " # add reactome information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['Reactome_PathwayName'] |= {react_name}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({url: {'Reactome_PathwayName': {react_name}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {url: {'Reactome_PathwayName': {react_name}}}})\n", + " \n", + " # add genomic information\n", + " if pr in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][pr].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][pr].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({pr: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({pr: {'genomic_data': 'None'}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'Reactome_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['Reactome_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['Reactome_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'Reactome_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'Reactome_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'Reactome_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del rce_prot_pth" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt` \n", + "\n", + "**Data Source Wiki Page:** [ClinVar](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#clinvar) \n", + "\n", + "**Edges:** \n", + "- `variant-disease` \n", + "- `variant-phenotype` \n", + "\n", + "**Identifier Maps:** \n", + "- Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt)\n", + "- Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + "\n", + "This chunk process the [`CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Disease, Phenotype_ \n", + " - `Phenotype`: A string containing a disease identifier and prefix. Sources are OMIM, MedGen (UMLS), and Orphanet. \n", + " \n", + " _Variant_ \n", + " - `VariantName`: A string containing the name of the variant. \n", + " - `rs_id`: An integer that represents a dbSNP identifier. \n", + " - `AlleleID`: An integer that represents an Allele identifier. \n", + " - `RCVaccession`: An integer that represents an RCV accession identifier. \n", + " - `Type`: Character, the type of variant represented by the AlleleID. \n", + " - `Assembly`: A list of dictionaries, stored as a string, that contains information related to the assembly (i.e., Assembly, ChromosomeAccession, Chromosome, Start, Stop, ReferenceAlel, AlernateAllel, Cytogenetic, and PositionVCF). \n", + "\n", + "\n", + "- **Edges:** \n", + " - `OtherIDs`: A \"|\"-delimited list of other identifiers associated with the variant edge. Note that each identifier included also includes a prefix. \n", + " - `GeneID`: An identifier for the gene associated with each variant (wherever possible). \n", + " - `Guidelines`: Character, ACMG only right now. \n", + " - `TestedInGTR`: Character, Y/N for Yes/No if there is a test registered as specific to this variant in the NIH Genetic Testing Registry (GTR). \n", + " - `LastEvaluated`: Date, the latest date any submitter reported clinical significance. \n", + " - `ReviewStatus`: Character, highest review status for reporting this measure. \n", + " - `ClinicalSignificance`: Character, comma-separated list of aggregate values of clinical significance calculated for this variant. \n", + " - `ClinSigSimple`: Integer, \n", + " 0 = no current value of Likely pathogenic or Pathogenic; \n", + " 1 = at least one current record submitted with an interpretation of Likely pathogenic or Pathogenic (independent of whether that record includes assertion criteria and evidence). \n", + " -1 = no values for clinical significance at all for this variant or set of variants; used for the \"included\" variants that are only in ClinVar because they are included in a haplotype or genotype with an interpretation. \n", + " - `Origin`: Character, list of all allelic origins for this variant. \n", + " - `OriginSimple`: Character, processed from Origin to make it easier to distinguish between germline and somatic. \n", + " - `SubmitterCategories`: Coded value to indicate whether data were submitted by another resource (1), any other type of source (2), both (3), or none (4). \n", + " - `NumberSubmitters`: Integer, number of submitters describing this variant \n", + " - `Citation`: A \"|\"-delimited list of evidence supporting the variant association. Sources are either PubMed, PubMedCentral, or the NCBI Bookshelf. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt')\n", + "\n", + "# load data\n", + "clv_var_dis = pandas.read_csv(unprocessed_data_location + 'CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt', header=0, delimiter='\\t', low_memory=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "clv_var_dis = clv_var_dis.merge(disease_maps, left_on='Phenotype', right_on='Disease_IDs')\n", + "clv_var_dis = clv_var_dis.merge(phenotype_maps, left_on='Phenotype', right_on='Disease_IDs')\n", + "\n", + "# visualize data\n", + "clv_var_dis.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'variant-disease': {}, 'variant-phenotype': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(clv_var_dis.iterrows(), total=clv_var_dis.shape[0]):\n", + " node_key = row['VariationID']; rcv = row['RCVaccession']; var_type = row['Type']\n", + " pheno = row['Phenotype']; rs_id = row['RS# (dbSNP)']; allele_id = row['AlleleID']\n", + " assembly = row['Assembly']; var_name = row['VariantName']\n", + " evidence = [{'ClinVar_GeneID': row['GeneID'],\n", + " 'ClinVar_OtherIDs': row['OtherIDs'],\n", + " 'ClinVar_Guidelines': row['Guidelines'],\n", + " 'ClinVar_TestedInGTR': row['TestedInGTR'],\n", + " 'ClinVar_LastEvaluated': row['LastEvaluated'],\n", + " 'ClinVar_ReviewStatus': row['ReviewStatus'],\n", + " 'ClinVar_ClinicalSignificance': row['ClinicalSignificance'],\n", + " 'ClinVar_ClinSigSimple': row['ClinSigSimple'],\n", + " 'ClinVar_Origin': row['Origin'],\n", + " 'ClinVar_OriginSimple': row['OriginSimple'],\n", + " 'ClinVar_SubmitterCategories': row['SubmitterCategories'],\n", + " 'ClinVar_NumberSubmitters': row['NumberSubmitters'],\n", + " 'ClinVar_Citation': row['Citation']}] \n", + " for idx in [row['MONDO_IDs'].rstrip(), row['HP_IDs'].rstrip()]:\n", + " if idx.startswith('MONDO'): edge_key = '{}-{}'.format(node_key, idx); edge_type = 'variant-disease'\n", + " else: edge_key = '{}-{}'.format(node_key, idx); edge_type = 'variant-phenotype'\n", + " \n", + " # add disease/phenotype metadata\n", + " if idx in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][idx].keys():\n", + " master_metadata_dictionary['nodes'][idx][url]['ClinVar_Phenotype'] |= {pheno}\n", + " else: master_metadata_dictionary['nodes'][idx].update({url: {'ClinVar_Phenotype': {pheno}}})\n", + " else: master_metadata_dictionary['nodes'].update({idx: {url: {'ClinVar_Phenotype': {pheno}}}})\n", + "\n", + " # add variant information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_VariantName'] |= {var_name}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_rs_id'] |= {rs_id}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_AlleleID'] |= {allele_id}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_RCVaccession'] |= {rcv}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_Type'] |= {var_type}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_Assembly'] |= {assembly}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({\n", + " url: {'ClinVar_rs_id': {rs_id},\n", + " 'ClinVar_VariantName': {var_name},\n", + " 'ClinVar_AlleleID': {allele_id},\n", + " 'ClinVar_RCVaccession': {rcv},\n", + " 'ClinVar_Type': {var_type},\n", + " 'ClinVar_Assembly': {assembly}\n", + " }})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {\n", + " url: {'ClinVar_rs_id': {rs_id},\n", + " 'ClinVar_VariantName': {var_name},\n", + " 'ClinVar_AlleleID': {allele_id},\n", + " 'ClinVar_RCVaccession': {rcv},\n", + " 'ClinVar_Type': {var_type},\n", + " 'ClinVar_Assembly': {assembly}}}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'ClinVar_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['ClinVar_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['ClinVar_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'ClinVar_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'ClinVar_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'ClinVar_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del clv_var_dis" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `CLINVAR_VARIANT_GENE_EDGES.txt` \n", + "\n", + "**Data Source Wiki Page:** [ClinVar](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#clinvar) \n", + "\n", + "**Edges:** \n", + "- `variant-gene` \n", + "\n", + "This chunk process the [`CLINVAR_VARIANT_GENE_EDGES.txt`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_EDGES.txt) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Variant_ \n", + " - `VariantName`: A string containing the name of the variant. \n", + " - `rs_id`: An integer that represents a dbSNP identifier. \n", + " - `AlleleID`: An integer that represents an Allele identifier. \n", + " - `RCVaccession`: An integer that represents an RCV accession identifier. \n", + " - `Type`: Character, the type of variant represented by the AlleleID. \n", + " - `Assembly`: A list of dictionaries, stored as a string, that contains information related to the assembly (i.e., Assembly, ChromosomeAccession, Chromosome, Start, Stop, ReferenceAlel, AlernateAllel, Cytogenetic, and PositionVCF). \n", + " - `GenesPerAlleleID`: An integer that represents the count of genes that are found in the allele which corresponds to the variant. \n", + " - `Category`: The type of allele-gene relationship. The values for category are:\n", + " - Asserted, but not computed: Submitted as related to a gene, but not within the location of that gene on the genome\n", + " - Genes overlapped by variant: The gene and variant overlap\n", + " - Near gene, downstream: Outside the location of the gene on the genome, within 5 kb\n", + " -Near gene, upstream: Outside the location of the gene on the genome, within 5 kb\n", + " - Within multiple genes by overlap: The variant is within genes that overlap on the genome. Includes introns\n", + " - Within single gene: The variant is in only one gene. Includes introns\n", + " \n", + " _Gene_ \n", + " - `GenomicInformation`: A dictionary of gene identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `OtherIDs`: A \"|\"-delimited list of other identifiers associated with the variant edge. Note that each identifier included also includes a prefix. \n", + " - `Guidelines`: Character, ACMG only right now. \n", + " - `TestedInGTR`: Character, Y/N for Yes/No if there is a test registered as specific to this variant in the NIH Genetic Testing Registry (GTR). \n", + " - `LastEvaluated`: Date, the latest date any submitter reported clinical significance. \n", + " - `ReviewStatus`: Character, highest review status for reporting this measure. \n", + " - `ClinicalSignificance`: Character, comma-separated list of aggregate values of clinical significance calculated for this variant. \n", + " - `ClinSigSimple`: Integer, \n", + " 0 = no current value of Likely pathogenic or Pathogenic; \n", + " 1 = at least one current record submitted with an interpretation of Likely pathogenic or Pathogenic (independent of whether that record includes assertion criteria and evidence). \n", + " -1 = no values for clinical significance at all for this variant or set of variants; used for the \"included\" variants that are only in ClinVar because they are included in a haplotype or genotype with an interpretation. \n", + " - `Origin`: Character, list of all allelic origins for this variant. \n", + " - `OriginSimple`: Character, processed from Origin to make it easier to distinguish between germline and somatic. \n", + " - `SubmitterCategories`: Coded value to indicate whether data were submitted by another resource (1), any other type of source (2), both (3), or none (4). \n", + " - `NumberSubmitters`: Integer, number of submitters describing this variant \n", + " - `Citation`: A \"|\"-delimited list of evidence supporting the variant association. Sources are either PubMed, PubMedCentral, or the NCBI Bookshelf. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_EDGES.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'CLINVAR_VARIANT_GENE_EDGES.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'CLINVAR_VARIANT_GENE_EDGES.txt')\n", + "\n", + "# load data\n", + "clv_var_gene = pandas.read_csv(unprocessed_data_location + 'CLINVAR_VARIANT_GENE_EDGES.txt', header=0, delimiter='\\t', low_memory=False)\n", + "\n", + "# visualize data\n", + "clv_var_gene.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'variant-gene': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(clv_var_gene.iterrows(), total=clv_var_gene.shape[0]):\n", + " node_key = row['VariationID']; rcv = row['RCVaccession']; var_type = row['Type']\n", + " gene = row['GeneID']; var_name = row['VariantName']\n", + " rs_id = row['RS# (dbSNP)']; allele_id = row['AlleleID']\n", + " assembly = row['Assembly']; gpa = row['GenesPerAlleleID']; category = row['Category']\n", + " evidence = [{'ClinVar_OtherIDs': row['OtherIDs'],\n", + " 'ClinVar_Guidelines': row['Guidelines'],\n", + " 'ClinVar_TestedInGTR': row['TestedInGTR'],\n", + " 'ClinVar_LastEvaluated': row['LastEvaluated'],\n", + " 'ClinVar_ReviewStatus': row['ReviewStatus'],\n", + " 'ClinVar_ClinicalSignificance': row['ClinicalSignificance'],\n", + " 'ClinVar_ClinSigSimple': row['ClinSigSimple'],\n", + " 'ClinVar_Origin': row['Origin'],\n", + " 'ClinVar_OriginSimple': row['OriginSimple'],\n", + " 'ClinVar_SubmitterCategories': row['SubmitterCategories'],\n", + " 'ClinVar_NumberSubmitters': row['NumberSubmitters'],\n", + " 'ClinVar_Citation': row['Citation']}] \n", + " edge_key = '{}-{}'.format(node_key, gene); edge_type = 'variant-gene'\n", + "\n", + " # add genomic information\n", + " if gene in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[gene]\n", + " else: genomic_info_dict = None\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][gene].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][gene].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({gene: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({gene: {'genomic_data': 'None'}})\n", + " \n", + " # add variant information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_VariantName'] |= {var_name}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_rs_id'] |= {rs_id}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_AlleleID'] |= {allele_id}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_RCVaccession'] |= {rcv}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_Type'] |= {var_type}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_Assembly'] |= {assembly}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_GenesPerAlleleID'] |= {gpa}\n", + " master_metadata_dictionary['nodes'][node_key][url]['ClinVar_Category'] |= {category}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({\n", + " url: {'ClinVar_rs_id': {rs_id},\n", + " 'ClinVar_VariantName': {var_name},\n", + " 'ClinVar_AlleleID': {allele_id},\n", + " 'ClinVar_RCVaccession': {rcv},\n", + " 'ClinVar_Type': {var_type},\n", + " 'ClinVar_Assembly': {assembly},\n", + " 'ClinVar_GenesPerAlleleID': {gpa},\n", + " 'ClinVar_Category': {category}\n", + " }})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {\n", + " url: {'ClinVar_rs_id': {rs_id},\n", + " 'ClinVar_VariantName': {var_name},\n", + " 'ClinVar_AlleleID': {allele_id},\n", + " 'ClinVar_RCVaccession': {rcv},\n", + " 'ClinVar_Type': {var_type},\n", + " 'ClinVar_Assembly': {assembly},\n", + " 'ClinVar_GenesPerAlleleID': {gpa},\n", + " 'ClinVar_Category': {category}}}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'ClinVar_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['ClinVar_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['ClinVar_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'ClinVar_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'ClinVar_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'ClinVar_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del clv_var_gene " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt` \n", + "\n", + "**Data Source Wiki Page:** \n", + "- [Genotype-Tissue Expression Project](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#genotype-tissue-expression-project) \n", + "- [Human Protein Atlas](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#human-protein-atlas) \n", + "\n", + "**Edges:** \n", + "- `protein-anatomy` \n", + "- `protein-cell` \n", + "- `rna-anatomy` \n", + "- `rna-cell` \n", + "\n", + "**Identifier Maps:** \n", + "- Proteins: [UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt) \n", + "- Anatomy: [HPA_GTEx_TISSUE_CELL_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt)\n", + "- Cells: [HPA_GTEx_TISSUE_CELL_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt) \n", + "- RNA: [GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt) \n", + "\n", + "This chunk process the [`HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Anatomy, Cell_ \n", + " - `Anatomy`: A string containing the concept's synonym. If derived from an ontology, the string will be prefixed by the synonym type. \n", + " - `Anatomy_Type`: A string indicating the type of annotation. \n", + " - `Subcellular_Location`: A string containing a subcellular compartment. \n", + " \n", + " _Protein, RNA_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Expression_Value`: The expression value derived from the experiments. \n", + " - `Source`: A string indicating the source of the data (i.e., Human Protein Atlas or the Genotype-Tissue Expression project). \n", + " - `Evidence`: A string indicating if the evidence is at the transcript or protein level. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt')\n", + "\n", + "# load data\n", + "hpa_gen_ant = pandas.read_csv(unprocessed_data_location + 'HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt', header=None, delimiter='\\t', skiprows=0)\n", + "\n", + "# filter data\n", + "hpa_gen_ant = hpa_gen_ant[hpa_gen_ant[3] != 'No human protein/transcript evidence']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " *Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hpa_gen_ant = hpa_gen_ant.merge(uniprot_pro_map, left_on=2, right_on='Uniprot_Accession_IDs')\n", + "hpa_gen_ant = hpa_gen_ant.merge(anatomy_maps, left_on=6, right_on='anatomy_ids')\n", + "hpa_gen_ant = hpa_gen_ant.merge(symbol_transcript_map, left_on=1, right_on='Gene_Symbols')\n", + "\n", + "# visualize data\n", + "hpa_gen_ant.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'protein-anatomy': {}, 'protein-cell': {}, 'rna-anatomy': {}, 'rna-cell': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(hpa_gen_ant.iterrows(), total=hpa_gen_ant.shape[0]):\n", + " node_key = row['ontolgoy_ids']; anatomy = row[6]; anatomy_type = row[4]; subcell = row[5]\n", + " evidence = [{'HPA_GTEx_Expression_Value': row[7], 'HPA_GTEx_Source': row[8], 'HPA_GTEx_Evidence': row[3]}] \n", + " protein = row['Protein_Ontology_IDs'].rstrip(); rna = row['Ensembl_Transcript_IDs']\n", + " if row[3] == 'Evidence at protein level' and row[4] == 'anatomy':\n", + " node_key2 = protein; edge_key = '{}-{}'.format(node_key2, node_key); edge_type = 'protein-anatomy'\n", + " elif row[3] == 'Evidence at protein level' and row[4] != 'anatomy':\n", + " node_key2 = protein; edge_key = '{}-{}'.format(node_key2, node_key); edge_type = 'protein-cell'\n", + " elif row[3] == 'Evidence at transcript level' and row[4] == 'anatomy':\n", + " node_key2 = rna; edge_key = '{}-{}'.format(node_key2, node_key); edge_type = 'rna-anatomy'\n", + " elif row[3] == 'Evidence at transcript level' and row[4] != 'anatomy':\n", + " node_key2 = rna; edge_key = '{}-{}'.format(node_key2, node_key); edge_type = 'protein-cell'\n", + " else: pass\n", + " \n", + " # add anatomical information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key].keys():\n", + " master_metadata_dictionary['nodes'][node_key][url]['HPA_GTEx_Anatomy'] |= {anatomy}\n", + " master_metadata_dictionary['nodes'][node_key][url]['HPA_GTEx_Anatomy_Type'] |= {anatomy_type}\n", + " master_metadata_dictionary['nodes'][node_key][url]['HPA_GTEx_Subcellular_Location'] |= {subcell}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key].update({url: {\n", + " 'HPA_GTEx_Anatomy': {anatomy},\n", + " 'HPA_GTEx_Anatomy_Type': {anatomy_type},\n", + " 'HPA_GTEx_Subcellular_Location': {subcell}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key: {url: {\n", + " 'HPA_GTEx_Anatomy': {anatomy},\n", + " 'HPA_GTEx_Anatomy_Type': {anatomy_type},\n", + " 'HPA_GTEx_Subcellular_Location': {subcell}}}})\n", + "\n", + " # add genomic information\n", + " if node_key2 in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key2]\n", + " if node_key2 in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key2].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key2].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key2: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key2: {'genomic_data': 'None'}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'HPA_GTEx_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['HPA_GTEx_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['HPA_GTEx_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'HPA_GTEx_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'HPA_GTEx_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'HPA_GTEx__Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del hpa_gen_ant " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `UNIPROT_PROTEIN_CATALYST.txt` \n", + "\n", + "**Data Source Wiki Page:** [Universal Protein Resource Knowledgebase](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#universal-protein-resource-knowledgebase) \n", + "\n", + "**Edges:** \n", + "- `protein-catalyst` \n", + "\n", + "This chunk process the [`UNIPROT_PROTEIN_CATALYST.txt`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_CATALYST.txt) file and obtains the following node and edge metadata: \n", + " \n", + "- **Nodes:** \n", + " _Protein_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Status`: A string to indicate the status of the entry in Uniprot. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_CATALYST.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'UNIPROT_PROTEIN_CATALYST.txt')\n", + "\n", + "# load data\n", + "upt_prot_cat = pandas.read_csv(unprocessed_data_location + 'UNIPROT_PROTEIN_CATALYST.txt', header=None, delimiter='\\t', skiprows=0)\n", + "\n", + "# visualize data\n", + "upt_prot_cat.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# master_metadata_dictionary['edges'].update({'protein-catalyst': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(upt_prot_cat.iterrows(), total=upt_prot_cat.shape[0]):\n", + " node_key = row[0]; chebi = row[1]; evidence = [{'Uniprot_Status': row[2]}] \n", + " edge_key = '{}-{}'.format(node_key, chebi); edge_type = 'protein-catalyst'\n", + " \n", + " # add catalyst information\n", + " if chebi in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][chebi].keys():\n", + " master_metadata_dictionary['nodes'][chebi][url]['Uniprot_CHEBI'] |= {chebi}\n", + " else: master_metadata_dictionary['nodes'][chebi].update({url: {'Uniprot_CHEBI': {chebi}}})\n", + " else: master_metadata_dictionary['nodes'].update({chebi: {url: {'Uniprot_CHEBI': {chebi}}}})\n", + " \n", + " # add genomic information\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}})\n", + "\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'Uniprot_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['Uniprot_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['Uniprot_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'Uniprot_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'Uniprot_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'Uniprot_Evidence': evidence, 'Type': edge_type}}})\n", + "\n", + "# delete unneeded data\n", + "del upt_prot_cat" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***\n", + "\n", + "#### `UNIPROT_PROTEIN_COFACTOR.txt` \n", + "\n", + "**Data Source Wiki Page:** [Universal Protein Resource Knowledgebase](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#universal-protein-resource-knowledgebase) \n", + "\n", + "**Edges:** \n", + "- `protein-cofactor` \n", + "\n", + "This chunk process the [`UNIPROT_PROTEIN_COFACTOR.txt`](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_COFACTOR.txt) file and obtains the following node and edge metadata: \n", + " \n", + "- **Nodes:** \n", + " _Protein_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", + "\n", + "- **Edges:** \n", + " - `Status`: A string to indicate the status of the entry in Uniprot. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data\n", + "url = 'https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_COFACTOR.txt'\n", + "if not os.path.exists(unprocessed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt'):\n", + " data_downloader(url, unprocessed_data_location, 'UNIPROT_PROTEIN_COFACTOR.txt')\n", + "\n", + "# load data\n", + "upt_prot_cof = pandas.read_csv(unprocessed_data_location + 'UNIPROT_PROTEIN_COFACTOR.txt', header=None, delimiter='\\t', skiprows=0)\n", + "\n", + "# visualize data\n", + "upt_prot_cof.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "
\n", - "\n", - "***\n", - "***\n", - "### INSTANCE AND/OR SUBCLASS (NON-ONTOLOGY CLASS) METADATA \n", - "***\n", - "\n", - "**Data Source Wiki Page:** [Dependencies](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies/#node-metadata) \n", - "\n", - "**Purpose:** The goal of this section is to obtain metadata for each non-ontology instance and/or subclass data source and all relations used in the knowledge graph. For **[`Release V2.0.0`](https://github.com/callahantiff/PheKnowLator/wiki/v2.0.0)**, the following are non-ontology instance and/or subclass data and require the compiling of metadata:\n", - "- [Genes](#gene-metadata)\n", - "- [RNA](#rna-metadata)\n", - "- [Variants](#variant-metadata) \n", - "- [Pathways](#pathway-metadata)\n", - "- [Relations](#relations-metadata)\n", - "\n", - "
\n", - "\n", - "**Metadata:** The metadata we will gather includes: \n", + "# master_metadata_dictionary['edges'].update({'protein-cofactor': {}})\n", "\n", - "| **Metadata Type** | **Definition** | **Example Node** | **Example Node Metadata** | \n", - "| :---: | :---: | :---: | :---: | \n", - "| Label | The primary label or name for the node | `R-HSA-1006173` | \"CFH:Host cell surface\" | \n", - "| Description | A definition or other useful details about the node | `rs794727058` | This `germline` `single nucleotide variant` located on chromosome `5 (GRCh38: NC_000005.10, start/stop positions (126555930/126555930))` with `pathogenic` clinical significance and a last review date of `2/23/2015` (review status: `criteria provided, single submitter`). | \n", - "| Synonym | Alternative terms used for a node | `81399` | \"OR1-1, OR7-21\" | \n", - "\n", - "The metadata information will be used to create the following edges in the knowledge graph: \n", - "- **Label** ➞ node `rdfs:label` \n", - "- **Description** ➞ node `obo:IAO_0000115` description \n", - "- **Synonyms** ➞ node `oboInOwl:hasExactSynonym` synonym \n", - "\n", - "
\n", - "\n", - "*NOTE. All node metadata are written to the `node_data` directory as a `pickled` dictionary called `node_metadata_dict.pkl`. The algorithm will look for this dictionary in the `node_data` directory and if it is not there, then no node metadata will be created.*\n", + "# create dictionary\n", + "for idx, row in tqdm(upt_prot_cof.iterrows(), total=upt_prot_cof.shape[0]):\n", + " node_key = row[0]; chebi = row[1]; evidence = [{'Uniprot_Status': row[2]}] \n", + " edge_key = '{}-{}'.format(node_key, chebi); edge_type = 'protein-cofactor'\n", + " \n", + " # add catalyst information\n", + " if chebi in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][chebi].keys():\n", + " master_metadata_dictionary['nodes'][chebi][url]['Uniprot_CHEBI'] |= {chebi}\n", + " else: master_metadata_dictionary['nodes'][chebi].update({url: {'Uniprot_CHEBI': {chebi}}})\n", + " else: master_metadata_dictionary['nodes'].update({chebi: {url: {'Uniprot_CHEBI': {chebi}}}})\n", + " \n", + " # add genomic information\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}})\n", "\n", - "
\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'Uniprot_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['Uniprot_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['Uniprot_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'Uniprot_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'Uniprot_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'Uniprot_Evidence': evidence, 'Type': edge_type}}})\n", "\n", - "### Prepare Metadata Dictionaries\n", + "# delete unneeded data\n", + "del upt_prot_cof" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "***\n", "\n", - "**Purpose:** To create the resources needed in order to create metadata dictionaries, which are in turn used to obtain metadata for instance and/or subclass data nodes. This process has the following steps:\n", + "#### `9606.protein.links.v11.0.txt.gz` \n", "\n", - "**1. [Generate Metadata Dictionaries](#generate-metadata-dictionaries):** In order to efficiently obtain metadata for all non-ontology instance and/or subclass data nodes and all relations, we first read in the data for each type (i.e. genes, rna, pathways, variants, and relations) and convert them into a dictionary. Then, each metadata dictionary is merged together and saved to a `master_metadata_dictionary`, keyed by identifier.\n", - " - Input Datasets: \n", - " - Genes ➞ `Homo_sapiens.gene_info` \n", - " - RNA ➞ `ensembl_identifier_data_cleaned.txt` \n", - " - Pathways ➞ [`reactome2py API`](https://github.com/reactome/reactome2py) ; `ReactomePathways.txt`; `gene_association.reactome.gz`; `ChEBI2Reactome_All_Levels.txt`; `kegg_reactome.csv` \n", - " - Variants ➞ `variant_summary.txt` \n", - " - Relations ➞ `ro_with_imports.owl` \n", - " \n", - "Example Metadata Dictionary Output:\n", + "**Data Source Wiki Page:** [Search Tool for Recurring Instances of Neighbouring Genes Database](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#search-tool-for-recurring-instances-of-neighbouring-genes-database) \n", "\n", - "```python\n", - "{\n", - " 'nodes': {\n", - " 'http://www.ncbi.nlm.nih.gov/gene/1': {\n", - " 'Label': 'A1BG',\n", - " 'Description': \"A1BG has locus group protein-coding' and is located on chromosome 19 (19q13.43).\",\n", - " 'Synonym': 'HYST2477alpha-1B-glycoprotein|HEL-S-163pA|ABG|A1B|GAB'} ... },\n", - " 'relations': {\n", - " 'http://purl.obolibrary.org/obo/RO_0002533': {\n", - " 'Label': 'sequence atomic unit',\n", - " 'Description': 'Any individual unit of a collection of like units arranged in a linear order',\n", - " 'Synonym': 'None'} ... }\n", - "}\n", - "```\n", + "**Edges:** \n", + "- `protein-protein` \n", "\n", - "
\n", + "**Identifier Maps:** \n", + "- Proteins: [STRING_PRO_ONTOLOGY_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/STRING_PRO_ONTOLOGY_MAP.txt) \n", "\n", - "**2. [Write Metadata Files](#write-metadata-files):** The `master_metadata_dictionary` dictionary from _Step 1_ is `pickled` and saved to the `resources/node_data/` directory.\n", + "This chunk process the [`9606.protein.links.v11.0.txt.gz`](https://stringdb-static.org/download/protein.links.v11.0/9606.protein.links.v11.0.txt.gz) file and obtains the following node and edge metadata: \n", + " \n", + "- **Nodes:** \n", + " _Protein_ \n", + " - `GenomicInformation`: A dictionary of protein identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", "\n", - "
\n", "\n", - "***" + "- **Edges:** \n", + " - `combined_score`: The combined score is computed by combining the probabilities from the different evidence channels and corrected for the probability of randomly observing an interaction. Scores range from 0-1000. For a more detailed description please see PMID:15608232. " ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Generate Metadata Dictionaries \n", - "In this step, the goal is to create a metadata dictionary for each node type that does not rely on API data. In this case, only the **Gene**, **RNA**, and **Variant** nodes require data that is not from an API.\n" + "# download data\n", + "url = 'https://stringdb-static.org/download/protein.links.v11.0/9606.protein.links.v11.0.txt.gz'\n", + "if not os.path.exists(unprocessed_data_location + '9606.protein.links.v11.0.txt'):\n", + " data_downloader(url, unprocessed_data_location, '9606.protein.links.v11.0.txt')\n", + "\n", + "# load data\n", + "stg_prot_prot = pandas.read_csv(unprocessed_data_location + '9606.protein.links.v11.0.txt', header=0, delimiter=' ', skiprows=0)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", - "\n", - "#### Genes Metadata Dictionary \n", - "\n", - "The nested dictionary of gene metadata is created by looping over the merged data described in the prior column. The `keys` of the dictionary are `Entrez gene identifiers` and the `values` are dictionaries for each metadata type." + " *Merge Identifier Maps*" ] }, { @@ -3958,15 +7362,18 @@ "metadata": {}, "outputs": [], "source": [ - "# entrez gene data\n", - "entrez_gene_data = pandas.read_csv(unprocessed_data_location + 'Homo_sapiens.gene_info', header=0, delimiter='\\t', low_memory=False)\n", + "stg_prot_prot = stg_prot_prot.merge(string_pro_map, left_on='protein1', right_on='STRING_IDs')\n", + "stg_prot_prot = stg_prot_prot.merge(string_pro_map, left_on='protein2', right_on='STRING_IDs')\n", "\n", - "# remove all rows that are not human\n", - "entrez_gene_data = entrez_gene_data.loc[entrez_gene_data['#tax_id'].apply(lambda x: x == 9606)]\n", - "\n", - "# replace NaN and '-' with 'None'\n", - "entrez_gene_data.fillna('None', inplace=True)\n", - "entrez_gene_data.replace('-','None', inplace=True, regex=False)" + "# visualize data\n", + "stg_prot_prot.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" ] }, { @@ -3975,32 +7382,38 @@ "metadata": {}, "outputs": [], "source": [ - "# create metadata\n", - "genes, lab, desc, syn = [], [], [], []\n", - "for idx, row in tqdm(entrez_gene_data.iterrows(), total=entrez_gene_data.shape[0]):\n", - " gene_id, sym, defn, gene_type = row['GeneID'], row['Symbol'], row['description'], row['type_of_gene']\n", - " chrom, map_loc, s1, s2 = row['chromosome'], row['map_location'], row['Synonyms'], row['Other_designations']\n", - " if gene_id != 'None':\n", - " genes.append('http://www.ncbi.nlm.nih.gov/gene/' + str(gene_id))\n", - " if sym != 'None' or sym != '': lab.append(sym)\n", - " else: lab.append('Entrez_ID:' + gene_id)\n", - " if 'None' not in [defn, gene_type, chrom, map_loc]:\n", - " desc_str = \"{} has locus group '{}' and is located on chromosome {} ({}).\"\n", - " desc.append(desc_str.format(sym, gene_type, chrom, map_loc))\n", - " else: desc.append(\"{} locus group '{}'.\".format(sym, gene_type))\n", - " if s1 != 'None' and s2 != 'None': syn.append('|'.join(set([x for x in (s1 + s2).split('|') if x != 'None' or x != ''])))\n", - " elif s1 != 'None': syn.append('|'.join(set([x for x in s1.split('|') if x != 'None' or x != ''])))\n", - " elif s2 != 'None': syn.append('|'.join(set([x for x in s2.split('|') if x != 'None' or x != ''])))\n", - " else: syn.append('None')\n", + "# master_metadata_dictionary['edges'].update({'protein-protein': {}})\n", "\n", - "# combine into new data frame\n", - "metadata = pandas.DataFrame(list(zip(genes, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym'])\n", - "metadata = metadata.astype(str)\n", - "metadata.drop_duplicates(subset='ID', keep='first', inplace=True)\n", + "# create dictionary\n", + "for idx, row in tqdm(stg_prot_prot.iterrows(), total=stg_prot_prot.shape[0]):\n", + " proteins = [row['Protein_Ontology_IDs_x'], row['Protein_Ontology_IDs_y']]; score = row['combined_score']; gene_info = []\n", + " edge_key = '{}-{}'.format(row['Protein_Ontology_IDs_x'], row['Protein_Ontology_IDs_y']); edge_type = 'protein-protein' \n", + " \n", + " for node_key in proteins:\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " else: genomic_info_dict = None\n", + " \n", + " # add genomic information\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}}) \n", + " \n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'String_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " master_metadata_dictionary['edges'][edge_key][url]['String_Evidence'] = score\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'String_Evidence': score})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'String_Evidence': score, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'String_Evidence': score, 'Type': edge_type}}})\n", "\n", - "# convert df to dictionary\n", - "metadata.set_index('ID', inplace=True)\n", - "gene_metadata_dict = metadata.to_dict('index')" + "# delete unneeded data\n", + "del stg_prot_prot" ] }, { @@ -4009,9 +7422,40 @@ "source": [ "***\n", "\n", - "#### RNA Metadata Dictionary \n", + "#### `curated_gene_disease_associations.tsv` \n", + "\n", + "**Data Source Wiki Page:** [DisGeNET](https://github.com/callahantiff/PheKnowLator/wiki/v4-Data-Sources#disgenet)\n", + "\n", + "**Edges:** \n", + "- `gene-disease` \n", + "- `gene-phenotype` \n", + "\n", + "**Identifier Maps:** \n", + "- Diseases: [DISEASE_MONDO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/DISEASE_MONDO_MAP.txt)\n", + "- Phenotypes: [PHENOTYPE_HPO_MAP.txt](https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/PHENOTYPE_HPO_MAP.txt) \n", + "\n", + "This chunk process the [curated_gene_disease_associations.tsv](https://www.disgenet.org/static/disgenet_ap1/files/downloads/curated_gene_disease_associations.tsv.gz) file and obtains the following node and edge metadata: \n", + "- **Nodes:** \n", + " _Disease, Phenotype_ \n", + " - `diseaseId`: A string containing the concept's database cross-reference, which is formatted as DB:ID. If not, a MeSH or OMIM identifier. Variable is provided as a string with the \"MESH\" or \"OMIM\" prefix in all caps. \n", + " - `diseaseName`: A string containing the concept's synonym. If derived from an ontology, the string will be prefixed by the synonym type. \n", + " - `diseaseSematicType`: A string containing a high-level grouper or typing variable for the disease. \n", + " - `diseaseClass`: A \";\"-delimnited list of ICD codes that can be used to classify the disease. \n", + " \n", + " _Gene_ \n", + " - `GenomicInformation`: A dictionary of gene identifier information. See the [Genomic Entity Metadata](#genomicinfo) code chunk for more details. \n", + "\n", "\n", - "The nested dictionary of rna metadata is created by looping over the cleaned human [Ensembl](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#ensembl) gene, RNA, and protein identifier data set (`ensembl_identifier_data_cleaned.txt`). The `keys` of the dictionary are `Ensembl transcript identifiers` and the `values` are dictionaries for each metadata type." + "- **Edges:** \n", + " - `DSI`: The Disease Similarity Index ranges from from 0.25 to 1. It is calculated as: DSI = log2(# diseases assoc with gene/total # of diseases in DisGeNET) / log2(1/total # of diseases in DisGeNET) \n", + " - `DPI`: The Disease Pleiotropy Index ranges from 0 to 1. it is calculated as: DPI = (# of MeSH disease classes of disease assoc with gene/total # of MeSH disease classes)*100. \n", + " - `score`: The score range from 0 to 1, and take into account the number and type of sources (level of curation, model organisms), and the number of publications supporting the association. \n", + " - `EI`: The Evidence Index (EL) is a metric developed by ClinGen that measures the strength of evidence of a gene-disease relationship that correlates to a qualitative classification: \"Definitive\", \"Strong\", \"Moderate\", \"Limited\", \"Disputed\" ([PMID:28552198](https://www.ncbi.nlm.nih.gov/pubmed/28552198)). EI=1 indicates that all the publications support the GDA or the VDA, while EI<1 indicates that there are publications that assert that there is no association between the gene/variants and the disease. If the gene/variant has no EI value, it indicates that the index has not been computed for this association. It is calculated as: EI = (# positive pubs/total # of pubs). \n", + " - `YearInitial`: First time that the association was reported. \n", + " - `YearFinal`: Last time that the association was reported. \n", + " - `NofPmids`: Count of associated Pubmed IDs. \n", + " - `NofSnps`: Count of associated SNPs. \n", + " - `Source`: The original source reporting the Gene-Disease Association. " ] }, { @@ -4020,21 +7464,48 @@ "metadata": {}, "outputs": [], "source": [ - "# load data\n", - "rna_gene_data = pandas.read_csv(processed_data_location + 'ensembl_identifier_data_cleaned.txt', header=0, delimiter='\\t', low_memory=False)\n", + "# download data\n", + "url = 'https://www.disgenet.org/static/disgenet_ap1/files/downloads/curated_gene_disease_associations.tsv.gz'\n", + "if not os.path.exists(unprocessed_data_location + 'curated_gene_disease_associations.tsv'):\n", + " data_downloader(url, unprocessed_data_location, 'curated_gene_disease_associations.tsv')\n", "\n", - "# remove rows without identifiers\n", - "rna_gene_data = rna_gene_data.loc[rna_gene_data['transcript_stable_id'].apply(lambda x: x != 'None')]\n", + "# load data\n", + "dgt_dis_gene = pandas.read_csv(unprocessed_data_location + 'curated_gene_disease_associations.tsv', header=0, delimiter='\\t', skiprows=0)\n", + "dgt_dis_gene = dgt_dis_gene[dgt_dis_gene['diseaseType'] != 'group']\n", "\n", - "# remove unneeded columns\n", - "rna_gene_data.drop(['ensembl_gene_id', 'symbol', 'protein_stable_id', 'uniprot_id', 'master_transcript_type',\n", - " 'entrez_id', 'ensembl_gene_type', 'master_gene_type', 'symbol'], axis=1, inplace=True)\n", + "# fix variable typing\n", + "dgt_dis_gene['YearInitial'] = dgt_dis_gene['YearInitial'].astype('float').astype('Int64')\n", + "dgt_dis_gene['YearFinal'] = dgt_dis_gene['YearFinal'].astype('float').astype('Int64')\n", "\n", - "# remove duplicates\n", - "rna_gene_data.drop_duplicates(subset=['transcript_stable_id', 'transcript_name', 'ensembl_transcript_type'], keep='first', inplace=True)\n", + "# fix prefix\n", + "dgt_dis_gene['geneId'] = 'NCBIGene_' + dgt_dis_gene['geneId'].astype('str')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Merge Identifier Maps*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dgt_dis_gene = dgt_dis_gene.merge(disease_maps, left_on='diseaseId', right_on='Disease_IDs')\n", + "dgt_dis_gene = dgt_dis_gene.merge(phenotype_maps, left_on='diseaseId', right_on='Disease_IDs')\n", "\n", - "# replace NaN with 'None'\n", - "rna_gene_data.fillna('None', inplace=True)" + "# visualize data\n", + "dgt_dis_gene.head(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Create Metadata Dictionary*" ] }, { @@ -4043,39 +7514,89 @@ "metadata": {}, "outputs": [], "source": [ - "# create metadata\n", - "rna, lab, desc, syn = [], [], [], []\n", - "for idx, row in tqdm(rna_gene_data.iterrows(), total=rna_gene_data.shape[0]):\n", - " rna_id, ent_type, nme = row['transcript_stable_id'], row['ensembl_transcript_type'], row['transcript_name']\n", - " rna.append('https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=' + rna_id)\n", - " if nme != 'None':\n", - " lab.append(nme)\n", + "# master_metadata_dictionary['edges'].update({'gene-disease': {}, 'gene-phenotype': {}})\n", + "\n", + "# create dictionary\n", + "for idx, row in tqdm(dgt_dis_gene.iterrows(), total=dgt_dis_gene.shape[0]):\n", + " node_key = row['geneId']; dis_id = row['diseaseId']; dis_name = row['diseaseName']\n", + " sem_type = row['diseaseSemanticType']; dis_cls = row['diseaseClass']\n", + " evidence = [{'DisGeNET_DSI': row['DSI'] if not pandas.isna(row['DSI']) else 'None',\n", + " 'DisGeNET_DPI': row['DPI'] if not pandas.isna(row['DPI']) else 'None',\n", + " 'DisGeNET_score': row['score'] if not pandas.isna(row['score']) else 'None',\n", + " 'DisGeNET_EI': row['EI'] if not pandas.isna(row['EI']) else 'None',\n", + " 'DisGeNET_YearInitial': row['YearInitial'] if not pandas.isna(row['YearInitial']) else 'None',\n", + " 'DisGeNET_YearFinal': row['YearFinal'] if not pandas.isna(row['YearFinal']) else 'None',\n", + " 'DisGeNET_NofPmids': row['NofPmids'],\n", + " 'DisGeNET_NofSnps': row['NofSnps']}] \n", + " if row['diseaseType'] == 'disease':\n", + " node_key2 = row['MONDO_IDs']; edge_key = '{}-{}'.format(node_key, node_key2); edge_type = 'gene-disease'\n", " else:\n", - " lab.append('Ensembl_Transcript_ID:' + rna_id)\n", - " nme = 'Ensembl_Transcript_ID:' + rna_id\n", - " if ent_type != 'None': desc.append(\"Transcript {} is classified as type '{}'.\".format(nme, ent_type))\n", - " else: desc.append('None')\n", - " syn.append('None')\n", + " node_key2 = row['HP_IDs']; edge_key = '{}-{}'.format(node_key, node_key2); edge_type = 'gene-phenotype'\n", + " \n", + " # add disease/phenotype information\n", + " if node_key2 in master_metadata_dictionary['nodes'].keys():\n", + " if url in master_metadata_dictionary['nodes'][node_key2].keys():\n", + " master_metadata_dictionary['nodes'][node_key2][url]['DisGeNET_diseaseId'] |= {dis_id}\n", + " master_metadata_dictionary['nodes'][node_key2][url]['DisGeNET_diseaseName'] |= {dis_name}\n", + " master_metadata_dictionary['nodes'][node_key2][url]['DisGeNET_diseaseSemanticType'] |= {sem_type}\n", + " master_metadata_dictionary['nodes'][node_key2][url]['DisGeNET_diseaseClass'] |= {dis_cls}\n", + " else:\n", + " master_metadata_dictionary['nodes'][node_key2].update({url: {\n", + " 'DisGeNET_diseaseId': {dis_id},\n", + " 'DisGeNET_diseaseName': {dis_name},\n", + " 'DisGeNET_diseaseSemanticType': {sem_type},\n", + " 'DisGeNET_diseaseClass': {dis_cls}}})\n", + " else:\n", + " master_metadata_dictionary['nodes'].update({node_key2: {url: {\n", + " 'DisGeNET_diseaseId': {dis_id},\n", + " 'DisGeNET_diseaseName': {dis_name},\n", + " 'DisGeNET_diseaseSemanticType': {sem_type},\n", + " 'DisGeNET_diseaseClass': {dis_cls}}}})\n", + "\n", + " # add genomic information\n", + " if node_key in genomic_metadata.keys(): genomic_info_dict = genomic_metadata[node_key]\n", + " if node_key in master_metadata_dictionary['nodes'].keys():\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'][node_key].update({'genomic_data': genomic_info_dict})\n", + " else: master_metadata_dictionary['nodes'][node_key].update({'genomic_data': 'None'})\n", + " else:\n", + " if genomic_info_dict is not None:\n", + " master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': genomic_info_dict}})\n", + " else: master_metadata_dictionary['nodes'].update({node_key: {'genomic_data': 'None'}})\n", "\n", - "# combine into new data frame\n", - "metadata = pandas.DataFrame(list(zip(rna, lab, desc, syn)), columns=['ID', 'Label', 'Description', 'Synonym'])\n", - "metadata = metadata.astype(str)\n", - "metadata.drop_duplicates(subset='ID', keep='first', inplace=True)\n", + " # add relation data to dictionary\n", + " if edge_key in master_metadata_dictionary['edges'].keys():\n", + " if url in master_metadata_dictionary['edges'][edge_key].keys():\n", + " if 'DisGeNET_Evidence' in master_metadata_dictionary['edges'][edge_key][url].keys():\n", + " inital_ev = master_metadata_dictionary['edges'][edge_key][url]\n", + " inital_ev = inital_ev['DisGeNET_Evidence'] + evidence\n", + " ev = [json.loads(i) for i in set(json.dumps(item, sort_keys=True) for item in inital_ev)]\n", + " master_metadata_dictionary['edges'][edge_key][url]['DisGeNET_Evidence'] = ev\n", + " else: master_metadata_dictionary['edges'][edge_key][url].update({'DisGeNET_Evidence': evidence})\n", + " else: master_metadata_dictionary['edges'][edge_key].update({url: {'DisGeNET_Evidence': evidence, 'Type': edge_type}})\n", + " else: master_metadata_dictionary['edges'].update({edge_key: {url: {'DisGeNET_Evidence': evidence, 'Type': edge_type}}})\n", "\n", - "# convert df to dictionary\n", - "metadata.set_index('ID', inplace=True)\n", - "rna_metadata_dict = metadata.to_dict('index')" + "# delete unneeded data\n", + "del dgt_dis_gene" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", + "
\n", "\n", - "#### Variant Metadata Dictionary \n", + "***\n", "\n", - "The nested dictionary of rna metadata is created by looping over the human [ClinVar Variant](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#clinvar) identifier data set (`variant_summary.txt`). The `keys` of the dictionary are `dbSNP identifiers` and the `values` are dictionaries for each metadata type." + "#### Save Metadata Dictionary\n", + "Write the metadata dictionary to a file named `entity_metadata_dict.pkl` and located in the `resources/metadata/` directory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Node Data*" ] }, { @@ -4084,30 +7605,10 @@ "metadata": {}, "outputs": [], "source": [ - "# download data\n", - "url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz'\n", - "if not os.path.exists(unprocessed_data_location + 'variant_summary.txt'):\n", - " data_downloader(url, unprocessed_data_location)\n", - "\n", - "# load data\n", - "var_data = pandas.read_csv(unprocessed_data_location + 'variant_summary.txt', header=0, delimiter='\\t', low_memory=False)\n", - "\n", - "# remove rows without identifiers\n", - "var_data = var_data.loc[var_data['Assembly'].apply(lambda x: x == 'GRCh38')]\n", - "var_data = var_data.loc[var_data['RS# (dbSNP)'].apply(lambda x: x != -1)]\n", - "\n", - "# de-dup data\n", - "var_metadata = var_data[['#AlleleID', 'Type', 'Name', 'ClinicalSignificance', 'RS# (dbSNP)', 'Origin',\n", - " 'ChromosomeAccession', 'Chromosome', 'Start', 'Stop', 'ReferenceAllele',\n", - " 'Assembly', 'AlternateAllele','Cytogenetic', 'ReviewStatus', 'LastEvaluated']] \n", - "\n", - "# replace NaN with 'None'\n", - "var_metadata.replace('na', 'None', inplace=True)\n", - "var_metadata.fillna('None', inplace=True)\n", - "\n", - "# remove duplicate dbSNP ids by choosing the most recent reviewed variant\n", - "var_metadata.sort_values('LastEvaluated', ascending=False, inplace=True)\n", - "var_metadata.drop_duplicates(subset='RS# (dbSNP)', keep='first', inplace=True)" + "# # create list of dictionaries\n", + "# print('Creating Node List ...')\n", + "# node_list = [{k: v} for k, v in tqdm(master_metadata_dictionary['nodes'].items())]\n", + "# master_metadata_dictionary['nodes'] = {} " ] }, { @@ -4116,41 +7617,28 @@ "metadata": {}, "outputs": [], "source": [ - "# create metadata\n", - "variant, label, desc, syn = [], [], [], []\n", - "for idx, row in tqdm(var_metadata.iterrows(), total=var_metadata.shape[0]):\n", - " var_id, lab = row['RS# (dbSNP)'], row['Name']\n", - " if var_id != 'None':\n", - " variant.append('https://www.ncbi.nlm.nih.gov/snp/rs' + str(var_id))\n", - " if lab != 'None': label.append(lab)\n", - " else: label.append('dbSNP_ID:rs' + str(var_id))\n", - " sent = \"This variant is a {} {} located on chromosome {} ({}, start:{}/stop:{} positions, \" +\\\n", - " \"cytogenetic location:{}) and has clinical significance '{}'. \" +\\\n", - " \"This entry is for the {} and was last reviewed on {} with review status '{}'.\"\n", - " desc.append(sent.format(row['Origin'].replace(';', '/'), row['Type'].replace(';', '/'), row['Chromosome'], row['ChromosomeAccession'],\n", - " row['Start'], row['Stop'], row['Cytogenetic'], row['ClinicalSignificance'],\n", - " row['Assembly'], row['LastEvaluated'], row['ReviewStatus']).replace('None', 'UNKNOWN'))\n", - " syn.append('None')\n", + "node_temp = {}\n", + "for k, v in tqdm(master_metadata_dictionary['nodes'].items()):\n", + " file_loc = metadata_location + 'temp/nodes/' + k + '.json'\n", + " # write data to temp directory\n", + " dump_jsonl([v[k]], file_loc)\n", + " # add dictionary entry with file path\n", + " node_temp[k] = file_loc\n", + " # delete entry\n", + " del master_metadata_dictionary['nodes'][k]\n", + "\n", + "# update nodes entry\n", + "master_metadata_dictionary['nodes'] = node_temp\n", " \n", - "# combine into new data frame\n", - "var_metadata_final = pandas.DataFrame(list(zip(variant, label, desc, syn)), columns =['ID', 'Label', 'Description', 'Synonym'])\n", - "var_metadata_final.drop_duplicates(subset=None, keep='first', inplace=True)\n", - "var_metadata_final = var_metadata_final.astype(str)\n", - "\n", - "# convert df to dictionary\n", - "var_metadata_final.set_index('ID', inplace=True)\n", - "var_metadata_dict = var_metadata_final.to_dict('index') " + "# remove unneeded data\n", + "del node_list" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", - "\n", - "#### Pathway Metadata Dictionary \n", - "\n", - "The nested dictionary of pathway metadata is created by looping over the human [Reactome Pathway Database](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#reactome-pathway-database) identifier data set (`ReactomePathways.txt`); Reactome-Gene Association data (`gene_association.reactome.gz`), and Reactome-ChEBI data (`ChEBI2Reactome_All_Levels.txt`). The `keys` of the dictionary are `Reactome identifiers` and the `values` are dictionaries for each metadata type." + "*Relation Data*" ] }, { @@ -4159,31 +7647,9 @@ "metadata": {}, "outputs": [], "source": [ - "# download reactome pathways data\n", - "url = 'https://reactome.org/download/current/ReactomePathways.txt'\n", - "if not os.path.exists(unprocessed_data_location + 'ReactomePathways.txt'):\n", - " data_downloader(url, unprocessed_data_location)\n", - "# load data\n", - "reactome_pathways = pandas.read_csv(unprocessed_data_location + 'ReactomePathways.txt', header=None, delimiter='\\t', low_memory=False)\n", - "reactome_pathways = reactome_pathways.loc[reactome_pathways[2].apply(lambda x: x == 'Homo sapiens')] \n", - "\n", - "# reactome gene association data\n", - "url = 'https://reactome.org/download/current/gene_association.reactome.gz'\n", - "if not os.path.exists(unprocessed_data_location + 'gene_association.reactome'):\n", - " data_downloader(url, unprocessed_data_location)\n", - "# load data\n", - "reactome_pathways2 = pandas.read_csv(unprocessed_data_location + 'gene_association.reactome', header=None, delimiter='\\t', skiprows=3, low_memory=False)\n", - "reactome_pathways2 = reactome_pathways2.loc[reactome_pathways2[12].apply(lambda x: x == 'taxon:9606')]\n", - "reactome_pathways2[5].replace('REACTOME:','', inplace=True, regex=True) \n", - "\n", - "# reactome CHEBI data\n", - "url = 'https://reactome.org/download/current/ChEBI2Reactome_All_Levels.txt'\n", - "if not os.path.exists(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt'):\n", - " data_downloader(url, unprocessed_data_location)\n", - "# load data\n", - "reactome_pathways3 = pandas.read_csv(unprocessed_data_location + 'ChEBI2Reactome_All_Levels.txt', header=None, delimiter='\\t', low_memory=False)\n", - "# remove all non-human pathways and save as list\n", - "reactome_pathways3 = reactome_pathways3.loc[reactome_pathways3[5].apply(lambda x: x == 'Homo sapiens')] " + "# print('\\nCreating Relations List ...')\n", + "# relation_list = [{k: v} for k, v in tqdm(master_metadata_dictionary['relations'].items())]\n", + "# master_metadata_dictionary['relations'] = {} " ] }, { @@ -4192,27 +7658,28 @@ "metadata": {}, "outputs": [], "source": [ - "# get metadata\n", - "nodes = list(set(reactome_pathways[0]) | set(reactome_pathways2[5]) | set(reactome_pathways3[1]))\n", - "pathway_metadata_final = metadata_api_mapper(nodes)\n", - "\n", - "# update dictionary\n", - "pathway_metadata_final['ID'] = pathway_metadata_final['ID'].map('https://reactome.org/content/detail/{}'.format)\n", - "pathway_metadata_final.set_index('ID', inplace=True)\n", - "\n", - "# convert df to dictionary\n", - "pathway_metadata_dict = pathway_metadata_final.to_dict('index') " + "relations_temp = {}\n", + "for k, v in tqdm(master_metadata_dictionary['relations'].items()):\n", + " file_loc = metadata_location + 'temp/relations/' + k + '.json'\n", + " # write data to temp directory\n", + " dump_jsonl([v[k]], file_loc)\n", + " # add dictionary entry with file path\n", + " relations_temp[k] = file_loc\n", + " # delete entry\n", + " del master_metadata_dictionary['relations'][k]\n", + "\n", + "# update nodes entry\n", + "master_metadata_dictionary['relations'] = relations_temp\n", + " \n", + "# remove unneeded data\n", + "del relation_list" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", - "\n", - "#### Relations Metadata Dictionary \n", - "\n", - "The nested dictionary of relation metadata is created by looping over the human [Relations Ontology](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#relations-ontology) identifier data set (`ro_with_imports.owl`). The `keys` of the dictionary are `Relations Ontology identifiers` and the `values` are dictionaries for each metadata type." + "*Edge Data*" ] }, { @@ -4221,14 +7688,9 @@ "metadata": {}, "outputs": [], "source": [ - "# download ontology\n", - "if not os.path.exists(unprocessed_data_location + 'ro_with_imports.owl'):\n", - " command = '{} {} --merge-import-closure -o {}'\n", - " os.system(command.format(owltools_location, 'http://purl.obolibrary.org/obo/ro.owl',\n", - " unprocessed_data_location + 'ro_with_imports.owl'))\n", - "# load graph\n", - "ro_graph = Graph().parse(unprocessed_data_location + 'ro_with_imports.owl')\n", - "print('There are {} edges in the ontology (date:{})'.format(len(ro_graph), datetime.datetime.now().strftime('%m/%d/%Y')))" + "# # print('\\nCreating Edge List ...')\n", + "# # edge_list = [{k: v} for k, v in tqdm(master_metadata_dictionary['edges'].items())]\n", + "# master_metadata_dictionary['edges'] = {} " ] }, { @@ -4237,39 +7699,29 @@ "metadata": {}, "outputs": [], "source": [ - "# get metadata\n", - "relation_metadata_dict, obo = {}, Namespace('http://purl.obolibrary.org/obo/')\n", - "\n", - "# get ontology information\n", - "cls = [x for x in gets_ontology_classes(ro_graph) if '/RO_' in str(x)] +\\\n", - " [x for x in gets_object_properties(ro_graph) if '/RO_' in str(x)]\n", - "master_synonyms = [x for x in ro_graph if 'synonym' in str(x[1]).lower() and isinstance(x[0], URIRef)]\n", + "edges_temp = {}\n", + "for k, v in tqdm(master_metadata_dictionary['edges'].items()):\n", + " file_loc = metadata_location + 'temp/edges/' + k + '.json'\n", + " # write data to temp directory\n", + " dump_jsonl([v[k]], file_loc)\n", + " # add dictionary entry with file path\n", + " edges_temp[k] = file_loc\n", + " # delete entry\n", + " del master_metadata_dictionary['edges'][k]\n", + " \n", "\n", - "for x in tqdm(cls):\n", - " # labels\n", - " cls_label = [x for x in ro_graph.objects(x, RDFS.label) if '@' not in n3(x) or '@en' in n3(x)]\n", - " labels = str(cls_label[0]) if len(cls_label) > 0 else 'None'\n", - " # synonyms\n", - " cls_syn = [str(i[2]) for i in master_synonyms if x == i[0]]\n", - " synonym = str(cls_syn[0]) if len(cls_syn) > 0 else 'None'\n", - " # description\n", - " cls_desc = [x for x in ro_graph.objects(x, obo.IAO_0000115) if '@' not in n3(x) or '@en' in n3(x)]\n", - " desc = '|'.join([str(cls_desc[0])]) if len(cls_desc) > 0 else 'None'\n", + "# update nodes entry\n", + "master_metadata_dictionary['edges'] = edges_temp\n", " \n", - " relation_metadata_dict[str(x)] = {\n", - " 'Label': labels, 'Description': desc, 'Synonym': synonym\n", - " }" + "# remove unneeded data\n", + "del edge_list" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "***\n", - "\n", - "**Create Master Metadata Dictionary** \n", - "\n", - "To make it easier to navigate the mapping of each instance node in an edge, a master dictionary is created and keyed by node type. This is most useful when both nodes in an edge are instances, but of different data types (e.g. `gene-rna`).\n" + "*Save Updated Metadata Dictionary to Metadata Location*" ] }, { @@ -4278,25 +7730,17 @@ "metadata": {}, "outputs": [], "source": [ - "# combine all metadata dictionaries\n", - "master_metadata_dictionary = {'nodes': {**gene_metadata_dict,\n", - " **rna_metadata_dict,\n", - " **var_metadata_dict,\n", - " **pathway_metadata_dict},\n", - " 'relations': relation_metadata_dict}\n", + "# save a copy of the dictionary\n", + "# output > 4GB requires special approach: https://stackoverflow.com/questions/42653386/does-pickle-randomly-fail-with-oserror-on-large-files\n", + "filepath = metadata_location + 'entity_metadata_dict.pkl'\n", "\n", - "# verify metadata strings are properly formatted\n", - "temp_copy = master_metadata_dictionary.copy(); master_metadata_dictionary = dict()\n", - "for key, value in tqdm(temp_copy.items()):\n", - " master_metadata_dictionary[key] = {}\n", - " for ent_key, ent_value in value.items():\n", - " updated_inner_dict = {k: re.sub('\\s\\s+', ' ', v.replace('\\n', ' '))\n", - " if v is not None else v for k, v in ent_value.items()}\n", - " master_metadata_dictionary[key][ent_key] = updated_inner_dict\n", - "del temp_copy\n", + "# defensive way to write pickle.write, allowing for very large files on all platforms\n", + "max_bytes, bytes_out = 2**31 - 1, pickle.dumps(master_metadata_dictionary)\n", + "n_bytes = sys.getsizeof(bytes_out)\n", "\n", - "# save dictionary locally\n", - "pickle.dump(master_metadata_dictionary, open(node_data_location + 'node_metadata_dict.pkl', 'wb'), protocol=4)" + "with open(filepath, 'wb') as f_out:\n", + " for idx in range(0, n_bytes, max_bytes):\n", + " f_out.write(bytes_out[idx:idx+max_bytes])" ] }, { @@ -4309,6 +7753,8 @@ "***\n", "***\n", "\n", + "This Notebook is part of the [**PheKnowLator Ecosystem**](https://zenodo.org/communities/pheknowlator-ecosystem/edit/)\n", + "\n", "```\n", "@misc{callahan_tj_2019_3401437,\n", " author = {Callahan, TJ},\n", @@ -4318,7 +7764,9 @@ " doi = {10.5281/zenodo.3401437},\n", " url = {https://doi.org/10.5281/zenodo.3401437}\n", "}\n", - "```" + "```\n", + "\n", + "***" ] } ], diff --git a/notebooks/Entity_Search_Examples.ipynb b/notebooks/Entity_Search_Examples.ipynb new file mode 100644 index 00000000..1141a61b --- /dev/null +++ b/notebooks/Entity_Search_Examples.ipynb @@ -0,0 +1,1213 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "

\n", + " \n", + "

\n", + "\n", + "***\n", + "***\n", + "\n", + "# Knowledge Graph Entity Search Examples\n", + "\n", + "***\n", + "\n", + "**Author:** [TJCallahan](http://tiffanycallahan.com/) \n", + "**GitHub Repository:** [PheKnowLator](https://github.com/callahantiff/PheKnowLator/wiki) \n", + "**Wiki Page:** [OWL-NETS-2.0](https://github.com/callahantiff/PheKnowLator/wiki/OWL-NETS-2.0) \n", + "**Release:** **[v3.0.0](https://github.com/callahantiff/PheKnowLator/wiki/v3.0.0)** \n", + " \n", + "
\n", + "\n", + "## Purpose \n", + "The goal of this notebook is to explore different ways to examine relationships between different types of entities in a PheKnowLator knowledge graph.\n", + "\n", + "### Notebook Organization \n", + "- [Set-Up Environment](#set-environment) \n", + "- [Knowledge Graph Data](#kg-data) \n", + "- [Knowledge-based Characterization](#kg-characterization) \n", + " - [Node-Level Characterization](#node-level) \n", + " - [Path-Level Characterization](#path-level) \n", + "\n", + "***\n", + "***\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "*** \n", + "## Set-Up Environment \n", + "*** \n", + "___\n", + "\n", + "### Dependencies: [pkt_kg](https://pypi.org/project/pkt-kg/), [networkx](https://pypi.org/project/networkx/), [rdflib](https://pypi.org/project/rdflib/)\n", + "\n", + "To prepare for the tutorial we need to make sure that the all needed libraries are downloaded and imported. If you don't already have `pkt_kg`, `rdflib`, and `networkx` installed, you can extend the code chunk below to include any libraries that you need to download. In addition to downloading needed libraries, you will also need to download the specific version of each knowledge graph that you want to analyze. Each data source is briefly described in the next section. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# # uncomment and run to install any required modules from notebooks/requirements.txt\n", + "# import sys\n", + "# !{sys.executable} -m pip install -r ../../notebooks/requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# # if running a local version (i.e., forked from GitHub) of pkt_kg, uncomment the code below\n", + "# import sys\n", + "# sys.path.append('../')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# import needed libraries\n", + "import json\n", + "import networkx as nx\n", + "import os\n", + "import pandas as pd\n", + "import pickle\n", + "import random\n", + "import re\n", + "\n", + "from pkt_kg.utils import *\n", + "from rdflib import Graph, Namespace, URIRef, BNode, Literal\n", + "from rdflib.namespace import RDFS\n", + "from tqdm.notebook import tqdm\n", + "from typing import Callable, Dict, List, Optional, Union\n", + "\n", + "# create namespace for OBO ontologies\n", + "obo = Namespace('http://purl.obolibrary.org/obo/')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper Functions \n", + "Helper functions used only by this notebook that are needed to process and label knowledge graph node and edge entities." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "code_folding": [ + 0, + 18, + 44, + 69 + ] + }, + "outputs": [], + "source": [ + "def format_path_ancestors(anc_dict: Dict, node_metadata: Dict) -> List:\n", + " \"\"\"Processes a dictionary of node ancestors into a list.\n", + "\n", + " Args:\n", + " anc_dict: A dictionary where keys are ints formatted as strings and values are sets of URL strings for each\n", + " concept that was found at that level. The level is the distance in the hierarchy from the searched node.\n", + " node_metadata: A nested dictionary containing node attributes.\n", + "\n", + " Returns:\n", + " ancestors: A nested list where each inner list contains ontology identifier strings.\n", + " \"\"\"\n", + "\n", + " ancestors = [['{} ({})'.format(node_metadata[str(x)]['label'], x) for x in anc_dict[str(k)]]\n", + " for k in sorted([int(x) for x in anc_dict.keys()])]\n", + "\n", + " return ancestors\n", + "\n", + "\n", + "def formats_node_information(node: URIRef, neighborhood: List, metadata_dict: Dict, verbose: bool=False) -> None:\n", + " \"\"\"Processes neighborhood results.\n", + " \n", + " Args:\n", + " node: A string containing a node URL.\n", + " neighborhood: A nested list of strings, where each string contains a node identifier.\n", + " metadata_dict: A nested dictionary containing node attributes.\n", + " verbose: A bool indicating whether or not node and edge metadata should be printed.\n", + " \n", + " Returns:\n", + " None\n", + " \"\"\"\n", + " \n", + " for e, o in neighborhood:\n", + " spe = '\\n' if neighborhood.index([e, o]) == 0 else '\\n\\n'\n", + " s, s_label = str(node[0]).split('/')[-1], metadata_dict[str(node[0])]['label']\n", + " e_label = metadata_dict[str(e)]['label']\n", + " o, o_label, o_def = str(o).split('/')[-1], metadata_dict[str(o)]['label'], metadata_dict[str(o)]['description']\n", + " if verbose:\n", + " if o_def != 'None': print(spe + '>>> {} ({}) - {} - {} ({})\\n{} Definition: {}'.format(s_label, s, e_label, o_label, o, o, o_def))\n", + " else: print(spe + '>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " else: print('>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " \n", + " return None\n", + " \n", + "\n", + "def metadata_formatter(s: str, o: str, metadata_dict: Dict) -> None:\n", + " \"\"\"Function looks up edge-level metadata and prints it.\n", + " \n", + " Args:\n", + " s: A string containing the identifier for the subject node of a predicate or triple.\n", + " o: A string containing the identifier for the object node of a predicate or triple.\n", + " metadata_dict: A nested dictionary containing node and edge-level metadata.\n", + " \n", + " Returns:\n", + " None.\n", + " \"\"\"\n", + " \n", + " s = s + '-reactome_' if 'R-HSA' in s else s\n", + " o = o + '-reactome_' if 'R-HSA' in o else o\n", + " \n", + " if s + '-' + o in metadata_dict['edges'].keys():\n", + " print('\\nEdge Evidence'); print(json.dumps(metadata_dict['edges'][s + '-' + o], indent=4))\n", + " elif o + '-' + s in metadata_dict['edges'].keys():\n", + " print('\\nEdge Evidence'); print(json.dumps(metadata_dict['edges'][o + '-' + s], indent=4))\n", + " else:\n", + " pass\n", + " \n", + " return None\n", + "\n", + "\n", + "def formats_path_information(kg: nx.multidigraph.MultiDiGraph, paths: List, path_type: str, metadata_func: Callable, metadata_dict: Dict, node_metadata: Dict, verbose: bool=False, rand: bool=False, sample_size: int=10) -> None:\n", + " \"\"\"Processes shortest and simple path results.\n", + " \n", + " Args:\n", + " kg: A networkx MultiDiGraph object.\n", + " paths: A nested list of strings, where each string contains an an entity identifier.\n", + " path_type: A string, either 'simple' or 'shortest' that indicates the types of paths to process.\n", + " metadata_func: A function that processes edge metadata.\n", + " metadata_dict: A nested dictionary containing node and edge-level metadata. \n", + " node_metadata: A nested dictionary containing node attributes.\n", + " verbose: A bool indicating whether or not node and edge metadata should be printed.\n", + " rand: A bool indicating whether or not to draw random samples from the path.\n", + " sample_size: An integer used when rand is True to specify the size of the random sample to draw.\n", + " \n", + " Returns:\n", + " None\n", + " \"\"\"\n", + " \n", + " if path_type == 'shortest': \n", + " for i in range(0, len(paths[0]) - 1):\n", + " s = paths[0][i]; o = paths[0][i + 1]\n", + " edges = kg.get_edge_data(*(s, o)).keys()\n", + " for e in edges:\n", + " spe = '\\n' if list(edges).index(e) == 0 else '\\n\\n\\n'\n", + " s, s_label = str(s).split('/')[-1], node_metadata[str(s)]['label']\n", + " e_label = node_metadata[str(e)]['label']\n", + " o, o_label, o_def = str(o).split('/')[-1], node_metadata[str(o)]['label'], node_metadata[str(o)]['description']\n", + " if verbose:\n", + " if o_def != 'None': print(spe + '>>> {} ({}) - {} - {} ({})\\n\\n{} Definition: {}'.format(s_label, s, e_label, o_label, o, o, o_def))\n", + " else: print(spe + '>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " metadata_func(s, o, metadata_dict)\n", + " else: print('>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " else:\n", + " if rand: paths = random.sample(paths, sample_size)\n", + " for path in paths:\n", + " print('*' * 100)\n", + " for i in range(0, len(path) - 1):\n", + " spe = '\\n' if i == 0 else '\\n\\n\\n'\n", + " s = path[i]; o = path[i + 1]; edges = kg.get_edge_data(*(s, o))\n", + " try: edges.keys()\n", + " except AttributeError: edges = kg.get_edge_data(*(o, s))\n", + " for e in edges.keys():\n", + " s, s_label = str(s).split('/')[-1], node_metadata[str(s)]['label']\n", + " e_label = node_metadata[str(e)]['label']\n", + " o, o_label, o_def = str(o).split('/')[-1], node_metadata[str(o)]['label'], node_metadata[str(o)]['description']\n", + " if verbose:\n", + " if o_def != 'None': print(spe + '>>> {} ({}) - {} - {} ({})\\n\\n{} Definition: {}'.format(s_label, s, e_label, o_label, o, o, o_def))\n", + " else: print(spe + '>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " metadata_func(s, o, metadata_dict)\n", + " else: print('>>> {} ({}) - {} - {} ({})'.format(s_label, s, e_label, o_label, o))\n", + " print('*' * 100); print('\\n')\n", + " \n", + " return None\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "***\n", + "\n", + "## Knowledge Graph Data \n", + "***\n", + "___\n", + "\n", + "This notebook was built using a `v3.0.2` OWL-NETS-abstracted subclass-based build with inverse relations, which is publicly available and can be downloaded using the following links: \n", + "- [PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NetworkxMultiDiGraph.gpickle](https://storage.googleapis.com/pheknowlator/current_build/knowledge_graphs/subclass_builds/inverse_relations/owlnets/PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NetworkxMultiDiGraph.gpickle) \n", + "- [PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NodeLabels.txt](https://storage.googleapis.com/pheknowlator/current_build/knowledge_graphs/subclass_builds/inverse_relations/owlnets/PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NodeLabels.txt) \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Download Data \n", + "***\n", + "\n", + "The knowledge graph data is publicly available and downloaded from the PheKnowLator project's Google Cloud Storage Bucket: https://console.cloud.google.com/storage/browser/pheknowlator/. Data will be downloaded to a temporary directory created in the PheKnowLator root directory (`PheKnowLator/temp_directory`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# notebook will create a temporary directory and will download data to it\n", + "write_location = '../temp_directory/'\n", + "if not os.path.exists(write_location): os.mkdir(write_location)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# download data to the data directory\n", + "data_urls = [\n", + " 'https://storage.googleapis.com/pheknowlator/current_build/knowledge_graphs/subclass_builds/inverse_relations/owlnets/PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NetworkxMultiDiGraph.gpickle',\n", + " 'https://storage.googleapis.com/pheknowlator/current_build/knowledge_graphs/subclass_builds/inverse_relations/owlnets/PheKnowLator_v3.0.2_full_subclass_inverseRelations_OWLNETS_NodeLabels.txt',\n", + " 'https://www.dropbox.com/s/ev0ea6v6fu70fbl/entity_metadata_dict.pkl?dl=1'\n", + "]\n", + "\n", + "for url in data_urls:\n", + " file_name = url.split('/')[-1] if 'entity_metadata_dict.pkl' not in url else re.sub(r'\\?.*', '', url.split('/')[-1])\n", + " if not os.path.exists(write_location + file_name): data_downloader(url, write_location, file_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Loading Data\n", + "***\n", + "\n", + "The knowledge graph will be loaded as a `networkx` MultiDiGraph object and the node labels will be read in and converted to a dictionary to enable easy access to node labels and other relevant metadata." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Knowledge Graph" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load the knowledge graph\n", + "kg = nx.read_gpickle(write_location + data_urls[0].split('/')[-1])\n", + "print('The knowledge graph contains {} nodes and {} edges'.format(len(kg.nodes()), len(kg.edges())))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# convert multidigraph to undirected graph -- needed to run some of the algorithms\n", + "undirected_kg = kg.to_undirected()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load Node Metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# read in node metadata\n", + "node_data = pd.read_csv(write_location + data_urls[1].split('/')[-1], header=0, sep=r\"\\t\", encoding=\"utf8\", engine='python', quoting=3)\n", + "node_data['entity_uri'] = node_data['entity_uri'].str.strip('<>') # remove angle brackets\n", + "node_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# convert node data to dictionary\n", + "node_data_dict = dict()\n", + "for idx, row in tqdm(node_data.iterrows(), total=node_data.shape[0]):\n", + " node_data_dict[row['entity_uri']] = {\n", + " 'label': row['label'],\n", + " 'description': row['description/definition'],\n", + " 'synonym': row['synonym']\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load Node and Edge Evidence\n", + "This file is temporary while the next release is being formatted." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filepath = write_location + re.sub(r'\\?.*', '', data_urls[2].split('/')[-1])\n", + "max_bytes = 2**31 - 1; input_size = os.path.getsize(filepath); bytes_in = bytearray(0)\n", + "with open(filepath, 'rb') as f_in:\n", + " for _ in tqdm(range(0, input_size, max_bytes)):\n", + " bytes_in += f_in.read(max_bytes)\n", + "metadata_dict = pickle.loads(bytes_in)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "***\n", + "\n", + "## Knowledge-based Characterization \n", + "***\n", + "____\n", + "\n", + "The goal is to use the knowledge graph to explore what we know about specific concepts as well as what we can say about pairs of concepts. Additional details are presented by comparison below:\n", + "\n", + "#### [Node-Level](#node-level)\n", + " - Node Ancestry: Identify all ancestors for each node up to the root.\n", + " - Node Neighborhood: Returns all nodes reachable from a node of interest via a single directed edge. \n", + "\n", + "\n", + "#### [Path-Level](#path-level)\n", + " - All Shortest Paths: Returns the shortest simple path, if there are multiple paths of the same length then they are all returned.\n", + " - All Simple Paths: A simple path is a path with no repeated nodes. These nodes are identified using a modified depth-first search. Given that there are a lot of these, the initial output is limited to a random draw of 10 paths of length 10 from the first 100 derived paths.\n", + " \n", + "
\n", + "\n", + "**Important.** Output for the node neighborhood and simple and shortest paths are printed twice. The first time (`verbose=False`), there is minimal node and edge evidence printed. The second time (`verbose=True`), node definitions and any available evidence from the source resources used to build the edge are printed. Note that the metadata for the edges in the neighborhood will only include a definition for the nodes that are connected to each primary node of interest.\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "### Node-Level Characterization \n", + "***\n", + "\n", + "This section characterizes the following concepts:\n", + "- [benazepril (`CHEBI_3011`)](#chebi1) \n", + "- [hydrochlorothiazide (`CHEBI_5778`)](#chebi2) \n", + "- [Acute Myocardial Infarction (`MONDO_0004781`)](#mondo1) \n", + "- [Myocardial infarction (`HP_0001658`)](#hpo1)\n", + "\n", + "*Note*. All output is presented twice for each analysis, the first without any metadata/evidence and the second time, with metadata. This is done to facilitate readability." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### benazepril (`CHEBI_3011`) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Ancestors*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# examine the node's ancestors\n", + "prefix = 'CHEBI'; node = [obo.CHEBI_3011]\n", + "chebi3011_anc_dict = processes_ancestor_path_list(nx_ancestor_search(kg, node.copy(), prefix))\n", + "chebi3011_ancestors = format_path_ancestors(chebi3011_anc_dict, node_data_dict)\n", + "\n", + "# print results -- nodes are ordered by seniority (higher numbers indicate closer to root)\n", + "print('Ancestors of {}\\n'.format(node[0]))\n", + "for level in range(len(chebi3011_ancestors)):\n", + " print('Level: {}'.format(str(level + 1)))\n", + " for v in chebi3011_ancestors[level]:\n", + " print('\\t- {}'.format(re.sub('http://purl.obolibrary.org/obo/', '', v)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Neighborhood*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's neigborhood\n", + "nodes = list(kg.neighbors(node[0]))\n", + "neighbors = [a for b in [[[i, n] for j in [kg.get_edge_data(*(node[0], n)).keys()]\n", + " for i in j] for n in nodes] for a in b]\n", + "chebi3011_sorted_neigbors = sorted(neighbors, key=lambda x: (str(x[1]).split('/')[-1], x[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes without definitions\n", + "formats_node_information(node, chebi3011_sorted_neigbors, node_data_dict, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes with definitions\n", + "formats_node_information(node, chebi3011_sorted_neigbors, node_data_dict, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**hydrochlorothiazide (`CHEBI_5778`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Ancestors*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# examine the node's ancestors\n", + "prefix = 'CHEBI'; node = [obo.CHEBI_5778]\n", + "path_list = nx_ancestor_search(kg, node.copy(), prefix)\n", + "chebi5778_anc_dict = processes_ancestor_path_list(nx_ancestor_search(kg, node.copy(), prefix))\n", + "chebi5778_ancestors = format_path_ancestors(chebi5778_anc_dict, node_data_dict)\n", + "\n", + "# print results -- nodes are ordered by seniority (higher numbers indicate closer to root)\n", + "print('Ancestors of {}\\n'.format(node[0]))\n", + "for level in range(len(chebi5778_ancestors)):\n", + " print('Level: {}'.format(str(level + 1)))\n", + " for v in chebi5778_ancestors[level]:\n", + " print('\\t- {}'.format(re.sub('http://purl.obolibrary.org/obo/', '', v)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Neighborhood*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's neigborhood\n", + "nodes = list(kg.neighbors(node[0]))\n", + "neighbors = [a for b in [[[i, n] for j in [kg.get_edge_data(*(node[0], n)).keys()]\n", + " for i in j] for n in nodes] for a in b]\n", + "chebi5778_sorted_neigbors = sorted(neighbors, key=lambda x: (str(x[1]).split('/')[-1], x[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes without definitions\n", + "formats_node_information(node, chebi5778_sorted_neigbors, node_data_dict, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes with definitions\n", + "formats_node_information(node, chebi5778_sorted_neigbors, node_data_dict, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**Myocardial Infarction (`MONDO_0005068`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Ancestors*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's ancestors\n", + "prefix = 'MONDO'; node = [obo.MONDO_0005068]\n", + "path_list = nx_ancestor_search(kg, node.copy(), prefix)\n", + "mondo0005068_anc_dict = processes_ancestor_path_list(nx_ancestor_search(kg, node.copy(), prefix))\n", + "mondo0005068_ancestors = format_path_ancestors(mondo0005068_anc_dict, node_data_dict)\n", + "\n", + "# print results -- nodes are ordered by seniority (higher numbers indicate closer to root)\n", + "print('Ancestors of {}\\n'.format(node[0]))\n", + "for level in range(len(mondo0005068_ancestors)):\n", + " print('Level: {}'.format(str(level + 1)))\n", + " for v in mondo0005068_ancestors[level]:\n", + " print('\\t- {}'.format(re.sub('http://purl.obolibrary.org/obo/', '', v)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Neighborhood*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's neigborhood\n", + "nodes = list(kg.neighbors(node[0]))\n", + "neighbors = [a for b in [[[i, n] for j in [kg.get_edge_data(*(node[0], n)).keys()]\n", + " for i in j] for n in nodes] for a in b]\n", + "mondo0005068_sorted_neigbors = sorted(neighbors, key=lambda x: (str(x[1]).split('/')[-1], x[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes without definitions\n", + "formats_node_information(node, mondo0005068_sorted_neigbors, node_data_dict, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes with definitions\n", + "formats_node_information(node, mondo0005068_sorted_neigbors, node_data_dict, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**Myocardial infarction (`HP_0001658`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Ancestors*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's ancestors\n", + "prefix = 'HP'; node = [obo.HP_0001658]\n", + "path_list = nx_ancestor_search(kg, node.copy(), prefix)\n", + "hp0001658_anc_dict = processes_ancestor_path_list(nx_ancestor_search(kg, node.copy(), prefix))\n", + "hp0001658_ancestors = format_path_ancestors(hp0001658_anc_dict, node_data_dict)\n", + "\n", + "# print results -- nodes are ordered by seniority (higher numbers indicate closer to root)\n", + "print('Ancestors of {}\\n'.format(node[0]))\n", + "for level in range(len(hp0001658_ancestors)):\n", + " print('Level: {}'.format(str(level + 1)))\n", + " for v in hp0001658_ancestors[level]:\n", + " print('\\t- {}'.format(re.sub('http://purl.obolibrary.org/obo/', '', v)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Neighborhood*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# examine the node's neigborhood\n", + "nodes = list(kg.neighbors(node[0]))\n", + "neighbors = [a for b in [[[i, n] for j in [kg.get_edge_data(*(node[0], n)).keys()]\n", + " for i in j] for n in nodes] for a in b]\n", + "hp0001658_sorted_neigbors = sorted(neighbors, key=lambda x: (str(x[1]).split('/')[-1], x[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes without definitions\n", + "formats_node_information(node, hp0001658_sorted_neigbors, node_data_dict, verbose=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print nodes with definitions\n", + "formats_node_information(node, hp0001658_sorted_neigbors, node_data_dict, verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "### Path-Level Characterization \n", + "\n", + "***\n", + "\n", + "This section characterizes the following concept pairs:\n", + "- [benazepril (`CHEBI_3011`) - Myocardial Infarction (`MONDO_0005068`)](#pair1) \n", + "- [hydrochlorothiazide (`CHEBI_5778`) - Myocardial Infarction (`MONDO_0005068`)](#pair2) \n", + "- [benazepril (`CHEBI_3011`) - Myocardial infarction (`HP_0001658`)](#pair3) \n", + "- [hydrochlorothiazide (`CHEBI_5778`) - Myocardial infarction (`HP_0001658`)](#pair4) \n", + "\n", + "*Note*. All output is presented twice for each analysis, the first without any metadata/evidence and the second time, with metadata. This is done to facilitate readability." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**benazepril (`CHEBI_3011`) - Myocardial Infarction (`MONDO_0005068`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Shortest Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# look at all shortest paths between the nodes in pair\n", + "shortest_paths = list(nx.all_shortest_paths(kg, obo.CHEBI_3011, obo.MONDO_0005068))\n", + "formats_path_information(kg=kg,\n", + " paths=shortest_paths,\n", + " path_type='shortest',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Simple Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# look at all simple paths between the nodes in pair\n", + "simple_paths = []; counter = 0\n", + "for path in tqdm(nx.all_simple_paths(undirected_kg, source=obo.CHEBI_3011, target=obo.MONDO_0005068, cutoff=10)):\n", + " simple_paths += [path]\n", + " if counter == 100: break\n", + " else: counter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# print path information -- without definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=False,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print path information -- with definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**hydrochlorothiazide (`CHEBI_5778`) - Myocardial Infarction (`MONDO_0005068`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Shortest Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# look at all shortest paths between the nodes in pair\n", + "shortest_paths = list(nx.all_shortest_paths(kg, obo.CHEBI_5778, obo.MONDO_0005068))\n", + "formats_path_information(kg=kg,\n", + " paths=shortest_paths,\n", + " path_type='shortest',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Simple Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# look at all simple paths between the nodes in pair\n", + "simple_paths = []; counter = 0\n", + "for path in tqdm(nx.all_simple_paths(undirected_kg, source=obo.CHEBI_5778, target=obo.MONDO_0005068, cutoff=10)):\n", + " simple_paths += [path]\n", + " if counter == 100: break\n", + " else: counter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# print path information -- without definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=False,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# print path information -- with definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**benazepril (`CHEBI_3011`) - Myocardial infarction (`HP_0001658`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Shortest Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# look at all shortest paths between the nodes in pair\n", + "shortest_paths = list(nx.all_shortest_paths(kg, obo.CHEBI_3011, obo.HP_0001658))\n", + "formats_path_information(kg=kg,\n", + " paths=shortest_paths,\n", + " path_type='shortest',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Simple Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# look at all simple paths between the nodes in pair\n", + "simple_paths = []; counter = 0\n", + "for path in tqdm(nx.all_simple_paths(undirected_kg, source=obo.CHEBI_3011, target=obo.HP_0001658, cutoff=10)):\n", + " simple_paths += [path]\n", + " if counter == 100: break\n", + " else: counter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print path information -- without definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=False,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print path information -- with definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "**hydrochlorothiazide (`CHEBI_5778`) - Myocardial infarction (`HP_0001658`)** " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Shortest Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# look at all shortest paths between the nodes in pair\n", + "shortest_paths = list(nx.all_shortest_paths(kg, obo.CHEBI_5778, obo.HP_0001658))\n", + "formats_path_information(kg=kg,\n", + " paths=shortest_paths,\n", + " path_type='shortest',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict, \n", + " verbose=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Simple Paths*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# look at all simple paths between the nodes in pair\n", + "simple_paths = []; counter = 0\n", + "for path in tqdm(nx.all_simple_paths(undirected_kg, source=obo.CHEBI_5778, target=obo.HP_0001658, cutoff=10)):\n", + " simple_paths += [path]\n", + " if counter == 100: break\n", + " else: counter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print path information -- without definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=False,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# print path information -- with definitions and metadata\n", + "formats_path_information(kg=kg,\n", + " paths=simple_paths,\n", + " path_type='simple',\n", + " metadata_func=metadata_formatter,\n", + " metadata_dict=metadata_dict,\n", + " node_metadata=node_data_dict,\n", + " verbose=True,\n", + " rand=True,\n", + " sample_size=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "***\n", + "***\n", + "\n", + "This Notebook is part of the [**PheKnowLator Ecosystem**](https://zenodo.org/communities/pheknowlator-ecosystem/edit/)\n", + "\n", + "```\n", + "@misc{callahan_tj_2019_3401437,\n", + " author = {Callahan, TJ},\n", + " title = {PheKnowLator},\n", + " month = mar,\n", + " year = 2019,\n", + " doi = {10.5281/zenodo.3401437},\n", + " url = {https://doi.org/10.5281/zenodo.3401437}\n", + "}\n", + "```\n", + "\n", + "***" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/notebooks/OWLNETS_Example_Application.ipynb b/notebooks/OWLNETS_Example_Application.ipynb index 9b40251f..e410e463 100644 --- a/notebooks/OWLNETS_Example_Application.ipynb +++ b/notebooks/OWLNETS_Example_Application.ipynb @@ -6,11 +6,13 @@ "collapsed": true }, "source": [ + "

\n", + " \n", + "

\n", + "\n", "***\n", "***\n", "\n", - "\n", - "\n", "## OWL-NETS Application - Example\n", "\n", "***\n", @@ -198,7 +200,7 @@ "from functools import reduce\n", "from rdflib import Graph, Namespace, URIRef, BNode, Literal\n", "from rdflib.namespace import OWL, RDF, RDFS\n", - "from tqdm import tqdm" + "from tqdm.notebook import tqdm" ] }, { @@ -818,6 +820,32 @@ " definitions = entity_metadata['relations'][x]['definitions']\n", " out.write(x + '\\t' + namespace + '\\t' + labels + '\\t' + definitions + '\\n')" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "***\n", + "***\n", + "\n", + "This Notebook is part of the [**PheKnowLator Ecosystem**](https://zenodo.org/communities/pheknowlator-ecosystem/edit/)\n", + "\n", + "```\n", + "@misc{callahan_tj_2019_3401437,\n", + " author = {Callahan, TJ},\n", + " title = {PheKnowLator},\n", + " month = mar,\n", + " year = 2019,\n", + " doi = {10.5281/zenodo.3401437},\n", + " url = {https://doi.org/10.5281/zenodo.3401437}\n", + "}\n", + "```\n", + "\n", + "***" + ] } ], "metadata": { diff --git a/notebooks/Ontology_Cleaning.ipynb b/notebooks/Ontology_Cleaning.ipynb index 79001acd..c916c3cc 100644 --- a/notebooks/Ontology_Cleaning.ipynb +++ b/notebooks/Ontology_Cleaning.ipynb @@ -4,11 +4,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "

\n", + " \n", + "

\n", + "\n", "***\n", "***\n", "\n", - "\n", - "\n", "## Pre-Knowledge Graph Build Ontology Cleaning\n", "***\n", "***\n", @@ -214,7 +216,7 @@ "import shutil\n", "\n", "from rdflib import Graph\n", - "from tqdm import tqdm\n", + "from tqdm.notebook import tqdm\n", "\n", "# import script containing helper functions\n", "from pkt_kg.utils import * \n", @@ -575,6 +577,8 @@ "***\n", "***\n", "\n", + "This Notebook is part of the [**PheKnowLator Ecosystem**](https://zenodo.org/communities/pheknowlator-ecosystem/edit/)\n", + "\n", "```\n", "@misc{callahan_tj_2019_3401437,\n", " author = {Callahan, TJ},\n", @@ -584,7 +588,9 @@ " doi = {10.5281/zenodo.3401437},\n", " url = {https://doi.org/10.5281/zenodo.3401437}\n", "}\n", - "```" + "```\n", + "\n", + "***" ] } ], diff --git a/notebooks/RDF_Graph_Processing_Example.ipynb b/notebooks/RDF_Graph_Processing_Example.ipynb index aa699373..a5a22905 100644 --- a/notebooks/RDF_Graph_Processing_Example.ipynb +++ b/notebooks/RDF_Graph_Processing_Example.ipynb @@ -4,11 +4,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "

\n", + " \n", + "

\n", + "\n", "***\n", "***\n", "\n", - "\n", - "\n", "## Working with RDF Graphs\n", "\n", "***\n", @@ -88,7 +90,7 @@ "import os\n", "\n", "from rdflib import Graph, Namespace, URIRef, BNode, Literal\n", - "from tqdm import tqdm\n", + "from tqdm.notebook import tqdm\n", "\n", "from pkt_kg.utils import * # provides access to helper functions" ] @@ -842,6 +844,39 @@ "gene_drug_disease_graph.serialize(write_location + 'pkt_DrugGeneDisease_subgraph.nt', format='nt')\n", "nx.write_gpickle(nx_graph_dgd, write_location + 'pkt_DrugGeneDisease_NetworkxMultiDiGraph.gpickle')" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "***\n", + "***\n", + "\n", + "This Notebook is part of the [**PheKnowLator Ecosystem**](https://zenodo.org/communities/pheknowlator-ecosystem/edit/)\n", + "\n", + "```\n", + "@misc{callahan_tj_2019_3401437,\n", + " author = {Callahan, TJ},\n", + " title = {PheKnowLator},\n", + " month = mar,\n", + " year = 2019,\n", + " doi = {10.5281/zenodo.3401437},\n", + " url = {https://doi.org/10.5281/zenodo.3401437}\n", + "}\n", + "```\n", + "\n", + "***" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/requirements.txt b/notebooks/requirements.txt index 29285e99..7cd27151 100644 --- a/notebooks/requirements.txt +++ b/notebooks/requirements.txt @@ -2,11 +2,12 @@ Cython>=0.29.14 ipywidgets>=7.7.0 more-itertools>=8.6.0 networkx>=2.4 -numpy>=1.18.1 +numpy>=1.19.5 openpyxl>=3.0.3 pandas>=1.0.5 psutil>=5.6.3 python-json-logger>=2.0.1 +pyyaml ray>=1.1.0 rdflib>=4.2.2 reactome2py>=0.0.8 diff --git a/pkt_kg/downloads.py b/pkt_kg/downloads.py index 7bc5d27d..ca2a23d2 100644 --- a/pkt_kg/downloads.py +++ b/pkt_kg/downloads.py @@ -40,11 +40,9 @@ class DataSource(object): important information on each of the files that is downloaded. - The class has two subclasses which inherit its methods. Each subclass contains an altered version of the primary classes methods that are specialized for that specific data type. - Attributes: data_path: A string file path/name to a text file storing URLs of different sources to download. resource_data: A string pointing to a data file that contains the contents of resource_info. - Raises: TypeError: If the file pointed to by data_path is not type str. IOError: If the file pointed to by data_path does not exist. @@ -90,12 +88,10 @@ def __init__(self, data_path: str, resource_data: Optional[str] = None) -> None: def parses_resource_file(self) -> None: """Verifies that an input file contains data and then outputs a dictionary where each item is a line from the input file. - Returns: source_list: A dictionary, where the key is the type of data and the value is the file path or url. For example: {'chemical-gomf', 'http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz', 'phenotype': 'http://purl.obolibrary.org/obo/hp.owl'} - Raises: ValueError: If the file does not contain data. ValueError: If there some of the input URLs were improperly formatted. @@ -105,12 +101,10 @@ def parses_resource_file(self) -> None: def downloads_data_from_url(self) -> None: """Downloads each data source from a list and writes the downloaded file to a directory. - Returns: data_files: A dictionary mapping each source identifier to the local location where it was downloaded. For example: {'chemical-gomf', 'resources/edge_data/chemical-gomf_CTD_chem_go_enriched.tsv', 'phenotype': 'resources/ontologies/hp_with_imports.owl'} - Raises: ValueError: If not all of the URLs returned valid data. """ @@ -121,10 +115,8 @@ def downloads_data_from_url(self) -> None: def extracts_edge_metadata(edge) -> Tuple[str, str, str]: """Processes edge data metadata and returns a dictionary where the keys are the edge type and the values are a list containing mapping and filtering information. - Args: edge: A pipe-delimited string containing information about the edge. For example, - Returns: mapping: Identifier mapping information stored as a node and a filepath to perform identifier mapping on (e.g. node1 - './filepath/mapping_data.txt). @@ -153,7 +145,6 @@ def extracts_edge_metadata(edge) -> Tuple[str, str, str]: def _writes_source_metadata_locally(self) -> None: """Writes metadata for each imported data source to a text file. - Returns: None """ @@ -178,7 +169,6 @@ def generates_source_metadata(self) -> None: sources that will be used to map identifiers or filter the data. 3 - Data Information: information on the data including: downloaded url, download date, file size in bytes, and the local file location it was downloaded to - Example: EDGE: chemical-gobp DATA PROCESSING INFO @@ -190,7 +180,6 @@ def generates_source_metadata(self) -> None: - DOWNLOAD_DATE = 01/14/2020 - FILE_SIZE_IN_BYTES = 760612373 - DOWNLOADED_FILE_LOCATION = ./resources/edge_data/chemical-gobp_CTD_chem_go_enriched.tsv - Returns: None. """ @@ -234,12 +223,10 @@ def gets_data_type(self) -> str: def parses_resource_file(self) -> None: """Parses data from a file and outputs a list where each item is a line from the input text file. - Returns: source_list: A dictionary, where the key is the type of data and the value is the file path or url. See example below: {'chemical-gomf', 'http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz', 'phenotype': 'http://purl.obolibrary.org/obo/hp.owl'} - Raises: TypeError: If the file does not contain data. ValueError: If there some of the input URLs were improperly formatted. @@ -258,15 +245,12 @@ def parses_resource_file(self) -> None: def downloads_data_from_url(self, owltools_location: str = os.path.abspath('./pkt_kg/libs/owltools')) -> None: """Takes a string representing a file path/name to a text file as an argument. The function assumes that each item in the input file list is an URL to an OWL/OBO ontology. - For each URL, the referenced ontology is downloaded, and used as input to an OWLTools command line argument ( https://github.com/owlcollab/owltools/wiki/Extract-Properties-Command), which facilitates the downloading of ontologies that are imported by the primary ontology. The function will save the downloaded ontology + imported ontologies. - Args: owltools_location: A string pointing to the location of the owl tools library. - Returns: data_files: A dictionary mapping each source identifier to the local location where it was downloaded. For example: {'chemical-gomf', 'resources/edge_data/chemical-gomf_CTD_chem_go_enriched.tsv', @@ -312,12 +296,10 @@ def gets_data_type(self) -> str: def parses_resource_file(self) -> None: """Verifies a file contains data and then outputs a list where each item is a line from the input text file. - Returns: source_list: A dictionary, where the key is the type of data and the value is the file path or url. See example below: {'chemical-gomf', 'http://ctdbase.org/reports/CTD_chem_go_enriched.tsv.gz', 'phenotype': 'http://purl.obolibrary.org/obo/hp.owl'} - Raises: TypeError: If the file does not contain data. """ @@ -335,7 +317,6 @@ def parses_resource_file(self) -> None: def downloads_data_from_url(self) -> None: """Takes a string representing a file path/name to a text file as an argument. The function assumes that each item in the input file list is a valid URL. - Returns: data_files: A dictionary mapping each source identifier to the local location where it was downloaded. For example: {'chemical-gomf', 'resources/edge_data/chemical-gomf_CTD_chem_go_enriched.tsv', diff --git a/pkt_kg/knowledge_graph.py b/pkt_kg/knowledge_graph.py index cedf8461..82a6cb22 100644 --- a/pkt_kg/knowledge_graph.py +++ b/pkt_kg/knowledge_graph.py @@ -48,7 +48,6 @@ class KGBuilder(object): build types. The current construction approaches are Instance-based and Subclass-based. The three build types are (1) Full (i.e. runs all build steps in the algorithm); (2) Partial (i.e. runs all of the build steps through adding new edges); and (3) Post-Closure: Runs the remaining build steps over a closed knowledge graph. - Attributes: construction: A string indicating the construction approach (i.e. instance or subclass). node_data: A string ("yes" or "no") indicating whether or not to add node data to the knowledge graph. @@ -56,7 +55,6 @@ class KGBuilder(object): decode_owl: A string containing "yes" or "no" indicating whether owl semantics should be removed. cpus: An integer indicating the number of workers to use. write_location: An optional string passed to specify the primary directory to write to. - Raises: ValueError: If the formatting of kg_version is incorrect (i.e. not "v.#.#.#"). ValueError: If write_location, edge_data does not contain a valid filepath. @@ -143,7 +141,6 @@ def reverse_relation_processor(self) -> None: relation data or relation data identifiers and labels. Examples of each dictionary are provided below: relations_dict: {'RO_0002551': 'has skeleton', 'RO_0002442': 'mutualistically interacts with} inverse_relations_dict: {'RO_0000056': 'RO_0000057', 'RO_0000079': 'RO_0000085'} - Returns: None. """ @@ -165,7 +162,6 @@ def construct_knowledge_graph(self) -> None: Create graph subsets; (4) Process node metadata; (5) Merge ontologies; (6) Add master edge list to merged ontologies; (7) Extract and write node metadata; (8) Decode OWL-encoded classes; and (8) Output knowledge graph files and create edge lists. - Returns: None. """ @@ -180,7 +176,6 @@ def gets_build_type(self) -> str: class EdgeConstructor(object): """Inner class object used to facilitate ray parallelization. - Attributes: construction: A string indicating the construction approach (i.e. instance or subclass). edge_data: A nested dictionary keyed by edge type that contains all information needed to construct an edge. @@ -224,13 +219,10 @@ def error_dict_getter(self) -> Dict: def verifies_object_property(self, object_property: URIRef) -> None: """Adds an object property to a knowledge graph. - Args: object_property: A string containing an obo ontology object property. - Returns: None. - Raises: TypeError: If the object_property is not type rdflib.term.URIRef """ @@ -247,13 +239,11 @@ def verifies_object_property(self, object_property: URIRef) -> None: def checks_classes(self, edge_info) -> bool: """Determines whether or not an edge is safe to add to the knowledge graph by making sure that any ontology class nodes are also present in the current list of classes from the merged ontologies graph. - Args: edge_info: A dict of information needed to add edge to graph, for example: {'n1': 'class', 'n2': 'class','rel': 'RO_0002606', 'inv_rel': 'RO_0002615', 'uri': ['https://www.ncbi.nlm.nih.gov/gene/', 'http://purl.obolibrary.org/obo/'], 'edges': ['CHEBI_81395', 'DOID_12858']} - Returns: True - if the class node is already in the graph or nodes are both non-class entities. False - if the edge contains at least 1 ontology class that is not present in the graph. @@ -269,11 +259,9 @@ def checks_relations(self, relation: str, edge_list: Union[List, Set]) -> Option """Determines whether or not an inverse relation should be created and added to the graph and verifies that a relation and its inverse (if it exists) are both an existing owl:ObjectProperty in the graph. - Args: relation: A string that contains the relation assigned to edge in resource_info.txt (e.g. 'RO_0000056'). edge_list: A list or set of knowledge graph edges. For example: {["8837", "4283"], ["8837", "839"]} - Returns: A string containing an ontology identifier (e.g. "RO_0000056) or None. Value depends on: - inverse relation, if the stored relation has an inverse relation @@ -294,12 +282,10 @@ def checks_relations(self, relation: str, edge_list: Union[List, Set]) -> Option @staticmethod def gets_edge_statistics(edge_type: str, results: Set, entity_info: List) -> str: """Calculates the number of nodes and edges involved in constructing an edge type. - Args: edge_type: A string point to a specific edge type (e.g. 'chemical-disease). results: A set of tuples representing the complete set of triples from the construction process. entity_info: 3 items: 1-2 are sets of node tuples and 3 is the total count of non-OWL edges. - Returns: formatted_str: A string containing edge statistics. """ @@ -314,10 +300,8 @@ def gets_edge_statistics(edge_type: str, results: Set, entity_info: List) -> str def creates_new_edges(self, edge_type: str) -> Graph: """Takes a dictionary of information needed to construct and edge creates the associated triples. - Args: edge_type: A list of strings representing the types of edges to build. - Returns: graph: An RDFLib Graph object. """ @@ -362,10 +346,8 @@ def construct_knowledge_graph(self) -> None: knowledge graph and intends to run a reasoner over it. The partial build includes the following steps: (1) Process relation/inverse relations; (2) Merge ontologies; (3) Process node metadata; (4) Create graph subsets; and (5) Add master edge list to merged ontologies. - Returns: None. - Raises: TypeError: If the ontologies directory is empty. """ @@ -445,14 +427,11 @@ def construct_knowledge_graph(self) -> None: """Builds a post-closure knowledge graph. This build is recommended when one has previously performed a "partial" knowledge graph build and then ran a reasoner over it. This build type inputs the closed partially built knowledge graph and completes the build process. - The post-closure build utilizes the following steps: (1) Process relation and inverse relation data; (2) Load closed knowledge graph; (3) Process node metadata; (4) Create graph subsets; (5) Decode OWL-encoded classes; (6) Output knowledge graph files and create edge lists; and (7) Extract and write node metadata. - Returns: None. - Raises: OSError: If closed knowledge graph file does not exist. TypeError: If the closed knowledge graph file is empty. @@ -535,7 +514,6 @@ def construct_knowledge_graph(self) -> None: relations; (2) Merge ontologies; (3) Process node metadata; (4) Create graph subsets; (5) Add master edge list to merged ontologies; (6) Decode OWL-encoded classes; (7) Output knowledge graphs and create edge lists and (8) Extract and write node metadata. - Returns: None. """ @@ -584,8 +562,8 @@ def construct_knowledge_graph(self) -> None: actors = [ray.remote(self.EdgeConstructor).remote(args) for _ in range(self.cpus)] # type: ignore for i in range(0, len(edges)): [actors[i].creates_new_edges.remote(j) for j in edges[i]] # type: ignore _ = ray.wait([x.graph_getter.remote() for x in actors], num_returns=len(actors)) # type: ignore - res = ray.get([x.graph_getter.remote() for x in actors]); g1 = [x[0] for x in res] # type: ignore - g2 = [x[1] for x in res] + res = ray.get([x.graph_getter.remote() for x in actors]) # type: ignore + g1 = [x[0] for x in res]; g2 = [x[1] for x in res] error_dicts = dict(ChainMap(*ray.get([x.error_dict_getter.remote() for x in actors]))) # type: ignore del actors if len(error_dicts.keys()) > 0: # output error logs diff --git a/pkt_kg/metadata.py b/pkt_kg/metadata.py index 0ec8dc98..1522ca05 100644 --- a/pkt_kg/metadata.py +++ b/pkt_kg/metadata.py @@ -36,7 +36,6 @@ class Metadata(object): """Class helps manage knowledge graph metadata. - Attributes: kg_version: A string that contains the version of the knowledge graph build. write_location: A filepath to the knowledge graph directory (e.g. './resources/knowledge_graphs). @@ -71,7 +70,6 @@ def metadata_processor(self) -> None: """Loads a directory of node and relations data. The dictionary is nested with the outer keys corresponding to the metadata type (i.e. "nodes" or "relations") and the values containing dictionaries keyed by URI and values containing a dictionary of metadata. - Returns: None. """ @@ -105,10 +103,8 @@ def extract_metadata(self, graph: Graph) -> None: owl:ObjectProperty). Each metadata type is saved as a dictionary key with the actual string stored as the value. The metadata types are packaged as a dictionary which is stored as the value to the node identifier as the key. - Args: graph: An rdflib graph object. - Returns: None. """ @@ -156,12 +152,10 @@ def creates_node_metadata(self, ent: List, e_type: Optional[List] = None, key_ty """Given a node in the knowledge graph, if the node is not an ontology class and if it has metadata information, then new edges are created to add the metadata to the knowledge graph. Metadata that is added includes: labels, descriptions, and synonyms. - Args: ent: A list of two node identifiers (e.g. ['http://example/3075', 'http://example/1080']). e_type: A list of types for each node in nodes (e.g. ['entity', 'entity']). key_type: A string indicating if the key should be 'nodes' or 'relations (default='nodes'). - Returns: edges: A list of tuples containing RDFLib objects used to add metadata to a knowledge graph. """ @@ -195,11 +189,9 @@ def creates_node_metadata(self, ent: List, e_type: Optional[List] = None, key_ty def adds_ontology_annotations(self, filename: str, graph: Graph) -> Graph: """Updates the ontology annotation information for an input knowledge graph or ontology. - Args: filename: A string containing the name of a knowledge graph. graph: An rdflib graph object. - Returns: graph: An rdflib graph object with edited ontology annotations. """ @@ -234,16 +226,13 @@ def output_metadata(self, node_integer_map: Dict, graph: Union[Set, Graph]) -> N """Loops over the self.node_dict dictionary and writes out the data to a file locally. The data is stored as a tab-delimited '.txt' file with four columns: (1) node identifier; (2) node label; (3) node description or definition; and (4) node synonym. - NOTE. Not every node in the knowledge class will have metadata. There are some non-ontology nodes that are added (e.g. Ensembl transcript identifiers) that at the time of adding did not include labels, synonyms, or definitions. While these nodes have valid metadata through their original provider, this data may not have been available for download and thus would not have been added to the node_dict. - Args: node_integer_map: A dictionary where keys are integers and values are node and relation identifiers. graph: A set of RDFLib Graph object triples or an RDFLib Graph. - Returns: None. """ diff --git a/pkt_kg/owlnets.py b/pkt_kg/owlnets.py index 499c74c5..ea6c2274 100644 --- a/pkt_kg/owlnets.py +++ b/pkt_kg/owlnets.py @@ -425,14 +425,14 @@ def detects_complement_of_constructed_classes(self, node_info: Dict, node: URIRe else: return False @staticmethod - def returns_object_property(sub: URIRef, obj: URIRef, prop: URIRef = None) -> URIRef: + def returns_object_property(sub: URIRef, obj: URIRef, prop: Optional[URIRef] = None) -> URIRef: """Checks the subject and object node types in order to determine the correct type of owl:ObjectProperty. The following ObjectProperties are returned for each of the following subject-object types: - - subject + object are not PATO terms + prop is None --> rdfs:subClassOf - - sub + obj are PATO terms + prop is None --> rdfs:subClassOf - - sub is not a PATO term, but obj is a PATO term --> owl:RO_000086 - - sub is a PATO term + obj is a PATO term + prop is not None --> prop + - if sub + obj are PATO terms + prop is None --> rdfs:subClassOf + - elif sub is not a PATO term, but obj is a PATO term --> obo:RO_000086 + - elif prop is not None --> prop + - else --> rdfs:subClassOf Args: sub: An rdflib.term object. @@ -443,9 +443,9 @@ def returns_object_property(sub: URIRef, obj: URIRef, prop: URIRef = None) -> UR An rdflib.term object that represents an owl:ObjectProperty. """ - if ('PATO' in sub and 'PATO' in obj) and not prop: return RDFS.subClassOf - elif ('PATO' not in sub and 'PATO' not in obj) and not prop: return RDFS.subClassOf - elif 'PATO' not in sub and 'PATO' in obj: return URIRef(obo + 'RO_0000086') + if ('PATO' in sub and 'PATO' in obj) and prop is None: return RDFS.subClassOf + elif 'PATO' not in sub and 'PATO' in obj: return obo.RO_0000086 + elif prop is None: return RDFS.subClassOf else: return prop @staticmethod @@ -499,20 +499,7 @@ def parses_constructors(self, node: URIRef, edges: Dict, class_dict: Dict, relat -> Tuple[Set, Optional[Dict]]: """Traverses a dictionary of rdflib objects used in the owl:unionOf or owl:intersectionOf constructors, from which the original set of edges used to the construct the class_node are edited, such that all owl-encoded - information is removed. For example: - INPUT: - - - - - - - - - - - - OUTPUT: [(CL_0000995, rdfs:subClassOf, CL_0001021), (CL_0000995, rdfs:subClassOf, CL_0001026)] + information is removed. See examples here: https://github.com/callahantiff/PheKnowLator/wiki/OWL-NETS-2.0. Args: node: An rdflib term of type URIRef or BNode that references an OWL-encoded class. @@ -526,21 +513,24 @@ def parses_constructors(self, node: URIRef, edges: Dict, class_dict: Dict, relat """ cleaned: Set = set() - if 'unionOf' in edges.keys() or 'intersectionOf' in edges.keys(): - batch = class_dict[edges['unionOf' if 'unionOf' in edges.keys() else 'intersectionOf']] - else: batch = edges + if 'unionOf' in edges.keys(): batch = class_dict[edges['unionOf']]; keyword = 'union' + elif 'intersectionOf' in edges.keys(): batch = class_dict[edges['intersectionOf']]; keyword = 'intersection' + else: batch = edges; keyword = 'other' while batch: if ('first' in batch.keys() and 'rest' in batch.keys()) and 'type' not in batch.keys(): if isinstance(batch['first'], URIRef) and isinstance(batch['rest'], BNode): obj_property = self.returns_object_property(node, batch['first'], relation) if node != batch['first']: - cleaned |= {(node, obj_property, batch['first'])} + if keyword == 'union': cleaned |= {(batch['first'], obj_property, node)} + else: cleaned |= {(node, obj_property, batch['first'])} batch = class_dict[batch['rest']] if 'rest' in batch.keys() else None else: batch = class_dict[batch['rest']] elif isinstance(batch['first'], URIRef) and isinstance(batch['rest'], URIRef): obj_property = self.returns_object_property(node, batch['first'], relation) - cleaned |= {(node, obj_property, batch['first'])}; batch = None + if keyword == 'union': cleaned |= {(batch['first'], obj_property, node)} + else: cleaned |= {(node, obj_property, batch['first'])} + batch = None else: batch = self.parses_anonymous_axioms(batch, class_dict) else: break @@ -595,6 +585,36 @@ class (referenced by node) in order to remove owl-encoded information. An exampl return cleaned, results[1] else: return cleaned, axioms + @staticmethod + def verifies_cleaned_classes(cleaned_classes: Set) -> Set: + """Verifies a set of cleaned tuples to ensure that there are not duplicate triples (i.e., subject-object + pairs with different properties). The function assumes that a duplicate tuple will include RDFS.subClassOf, + which should be removed. + + Args: + cleaned_classes: A set of tuples, where each tuple contains three URIRef objects. + + Returns: + A set of tuples, where each tuple contains a cleaned triple comprised of three URIRef objects. + """ + + org = len([x[0::2] for x in list(cleaned_classes)]) + unq = len(set([x[0::2] for x in list(cleaned_classes)])) + + if org == unq: return cleaned_classes + else: + cleaned_dict: Dict = dict(); verified_classes: Set = set() + for s, p, o in cleaned_classes: + key = '{}--{}'.format(str(s), str(o)) + if key in cleaned_dict.keys(): cleaned_dict[key] += [str(p)] + else: cleaned_dict[key] = [str(p)] + for k, v in cleaned_dict.items(): + s = URIRef(k.split('--')[0]); o = URIRef(k.split('--')[1]) + if len(v) > 1 and str(RDFS.subClassOf) in v: p = URIRef([x for x in v if x != str(RDFS.subClassOf)][0]) + else: p = URIRef(v[0]) + verified_classes |= {(s, p, o)} + return verified_classes + def cleans_owl_encoded_entities(self, node_list: List, verbose: bool = True) -> None: """Loops over a all owl:Class and owl: Axiom objects and decodes the OWL semantics returning the corresponding triples for each type without OWL semantics. @@ -620,8 +640,9 @@ def cleans_owl_encoded_entities(self, node_list: List, verbose: bool = True) -> if not neg and not comp: node, org = (node_info[0], node) if isinstance(node, BNode) else (node, node) cleaned_entities |= {org}; cleaned_classes: Set = set() - bnodes = set(x for x in self.graph.objects(org) if isinstance(x, BNode)) - for element in (bnodes if len(bnodes) > 0 else node_info[1].keys()): + # bnodes = set(x for x in self.graph.objects(org) if isinstance(x, BNode)) + # for element in (bnodes if len(bnodes) > 1 else node_info[1].keys()): + for element in node_info[1].keys(): edges = node_info[1][element] while edges: if 'subClassOf' in edges.keys(): @@ -636,11 +657,12 @@ def cleans_owl_encoded_entities(self, node_list: List, verbose: bool = True) -> results = self.parses_restrictions(node, edges, node_info[1]) if results is not None: cleaned_classes |= results[0]; edges = results[1] else: edges = None - else: # catch all other axioms -- only catching owl:onProperty + else: # catch all other axioms -- currently only catching owl:onProperty misc = [x for x in edges.keys() if x not in ['type', 'first', 'rest', 'onProperty']] edges = None; self.owl_nets_dict['misc'][n3(node)] = {tuple(misc)} - decoded_graph = adds_edges_to_graph(decoded_graph, list(cleaned_classes), False) - self.owl_nets_dict['decoded_entities'][n3(node)] = cleaned_classes + verified_classes = self.verifies_cleaned_classes(cleaned_classes) + decoded_graph = adds_edges_to_graph(decoded_graph, list(verified_classes), False) + self.owl_nets_dict['decoded_entities'][n3(node)] = verified_classes self.graph = decoded_graph; self.graph = self.cleans_decoded_graph(verbose) # ; pbar.close() return None @@ -667,7 +689,6 @@ def makes_graph_connected(self, graph: Graph, common_ancestor: Union[URIRef, str log_str = 'Obtaining node list'; print(log_str); logger.info(log_str) anc_node, roots = common_ancestor if isinstance(common_ancestor, URIRef) else URIRef(common_ancestor), set() nodes = set([x for x in tqdm(list(graph.subjects()) + list(graph.objects())) if isinstance(x, URIRef)]) - print('Identifying root nodes') for x in tqdm(nodes): ancs = gets_entity_ancestors(graph, [x], RDFS.subClassOf) @@ -679,7 +700,6 @@ def makes_graph_connected(self, graph: Graph, common_ancestor: Union[URIRef, str try: ancs = [mode(ancs)] except StatisticsError: ancs = sample(ancs, 1) if not any(x for x in ancs if x in roots) else [] roots |= {ancs[0]} if len(ancs) > 0 else {x} - log_str = 'Updating graph connectivity'; print(log_str); logger.info(log_str) rel = RDF.type if self.kg_construct_approach == 'instance' else RDFS.subClassOf needed_triples = set((URIRef(x), rel, anc_node) for x in roots if x != anc_node) @@ -701,13 +721,10 @@ def purifies_graph_build(self, graph: Graph) -> Graph: """ log_str = 'Purifying Graph Based on Construction Approach'; logger.info(log_str); print(log_str) - org_rel = RDF.type if self.kg_construct_approach == 'subclass' else RDFS.subClassOf pure_rel = RDFS.subClassOf if org_rel == RDF.type else RDF.type - log_str = 'Determining what triples need purification'; print(log_str); logger.info(log_str) triples = list(graph.triples((None, org_rel, None))) - log_str = 'Processing {} {} triples'.format(len(triples), org_rel); print(log_str); logger.info(log_str) for edge in tqdm(triples): graph.add((edge[0], pure_rel, edge[2])); graph.remove(edge) diff --git a/pkt_kg/utils/__init__.py b/pkt_kg/utils/__init__.py index 75ddb510..36348066 100644 --- a/pkt_kg/utils/__init__.py +++ b/pkt_kg/utils/__init__.py @@ -6,13 +6,17 @@ from .kg_utils import * -__all__ = ['url_download', 'ftp_url_download', 'gzipped_ftp_url_download', 'zipped_url_download', - 'gzipped_url_download', 'data_downloader', 'explodes_data', 'chunks', 'metadata_dictionary_mapper', - 'metadata_api_mapper', 'genomic_id_mapper', 'outputs_dictionary_data', 'gets_ontology_statistics', - 'gets_ontology_classes', 'gets_deprecated_ontology_classes', 'gets_object_properties', - 'gets_ontology_class_dbxrefs', 'gets_ontology_class_synonyms', 'merges_ontologies', - 'ontology_file_formatter', 'adds_edges_to_graph', 'remove_edges_from_graph', 'gets_entity_ancestors', - 'connected_components', 'removes_self_loops', 'derives_graph_statistics', 'splits_knowledge_graph', - 'adds_namespace_to_bnodes', 'removes_namespace_from_bnodes', 'updates_pkt_namespace_identifiers', - 'finds_node_type', 'updates_graph_namespace', 'maps_ids_to_integers', 'n3', 'appends_to_existing_file', - 'deduplicates_file', 'merges_files', 'convert_to_networkx', 'sublist_creator', 'gets_ontology_definitions'] +__all__ = ['adds_edges_to_graph', 'adds_namespace_to_bnodes', 'appends_to_existing_file', 'chunks', + 'connected_components', 'convert_to_networkx', 'data_downloader', 'deduplicates_file', + 'derives_graph_statistics', 'dump_jsonl', 'explodes_data', 'finds_node_type', 'ftp_url_download', + 'genomic_id_mapper', + # 'gets_biolink_information', + 'gets_deprecated_ontology_classes', + 'gets_entity_ancestors', 'gets_object_properties', 'gets_ontology_class_dbxrefs', + 'gets_ontology_class_synonyms', 'gets_ontology_classes', 'gets_ontology_definitions', + 'gets_ontology_statistics', 'gzipped_ftp_url_download', 'gzipped_url_download', 'load_jsonl', + 'maps_ids_to_integers', 'merges_files', 'merges_ontologies', 'metadata_api_mapper', + 'metadata_dictionary_mapper', 'n3', 'nx_ancestor_search', 'obtains_entity_url', 'ontology_file_formatter', + 'outputs_dictionary_data', 'processes_ancestor_path_list', 'remove_edges_from_graph', + 'removes_namespace_from_bnodes', 'removes_self_loops', 'splits_knowledge_graph', 'sublist_creator', + 'updates_graph_namespace', 'updates_pkt_namespace_identifiers', 'url_download', 'zipped_url_download'] diff --git a/pkt_kg/utils/data_utils.py b/pkt_kg/utils/data_utils.py index 82ebcd92..8afcec3d 100644 --- a/pkt_kg/utils/data_utils.py +++ b/pkt_kg/utils/data_utils.py @@ -23,9 +23,15 @@ * deduplicates_file * merges_files * sublist_creator +* obtains_entity_url +* gets_biolink_information + +Inputs data +* load_jsonl Outputs data * outputs_dictionary_data +* dump_jsonl """ # import needed libraries @@ -40,9 +46,12 @@ import requests import shutil import urllib3 # type: ignore +import yaml # type: ignore from contextlib import closing from io import BytesIO +from json.decoder import JSONDecodeError +from rdflib import Graph # type: ignore from reactome2py import content # type: ignore from tqdm import tqdm # type: ignore from typing import Dict, Generator, List, Optional, Union @@ -279,9 +288,16 @@ def metadata_api_mapper(nodes: List[str]) -> pd.DataFrame: results = content.query_ids(ids=','.join(request_ids)) if results is not None and (isinstance(results, List) or results['code'] != 404): for row in results: - ids.append(row['stId']); labels.append(row['displayName']); desc.append('None') + ids.append(row['stId']); labels.append(row['displayName']) if row['displayName'] != row['name']: synonyms.append('|'.join(row['name'])) else: synonyms.append('None') + if 'summation' in row.keys(): + definition = '|'.join([x['text'] for x in row['summation']]) + if 'literatureReference' in row.keys(): + lit_ev = '|'.join([x['url'] for x in row['literatureReference'] if 'url' in x.keys()]) + else: lit_ev = '' + desc.append('{} Literature References: {}.'.format(definition, lit_ev)) + else: desc.append('None') # combine into new data frame metadata = pd.DataFrame(list(zip(ids, labels, desc, synonyms)), columns=['ID', 'Label', 'Description', 'Synonym']) @@ -478,3 +494,148 @@ def sublist_creator(actors: Union[Dict, List], chunk_size: int) -> List: else: updated_lists = lists return updated_lists + + +def obtains_entity_url(prefix: str, identifier: Union[int, str], url: Optional[str] = None) -> str: + """Function takes a prefix and identifier for an entity, looks it up in the BioRegistry API and returns a + resolvable URL. Information on the BioRegistry can be found here: https://bioregistry.io/. + + Args: + prefix: A string containing the prefix or name of a resources (e.g., "chebi"). + identifier: A string or integer containing an entity identifier (e.g., "138488"). + url: A string containing a url. + + Returns: + entity_url: A string containing a valid BioRegistry URL. + + Raises: + ValueError: If a JSONDecodeError is thrown, a ValueError is raised to alert the user that a bad identifier or + prefix was provided. + """ + + prefix = prefix.lower(); identifier = str(identifier) + entity_url: str = ''; res: Optional[Dict] = None + obo_url = 'http://purl.obolibrary.org/obo/' + obo_ont_prefixes = ['BFO', 'CHEBI', 'DOID', 'GO', 'OBI', 'PATO', 'PO', 'PR', 'XAO', 'ZFA', 'AEO', 'AGRO', 'AISM', + 'AMPHX', 'APO', 'APOLLO_SV', 'ARO', 'BCO', 'BSPO', 'BTO', 'CARO', 'CDAO', 'CDNO', 'CHEMINF', + 'CHIRO', 'CHMO', 'CIDO', 'CIO', 'CL', 'CLAO', 'CLO', 'CLYH', 'CMO', 'COB', 'COLAO', 'CRO', + 'CTENO', 'CTO', 'CVDO', 'DDANAT', 'DDPHENO', 'DIDEO', 'DISDRIV', 'DPO', 'DRON', 'DUO', + 'ECAO', 'ECO', 'ECOCORE', 'ECTO', 'EMAPA', 'ENVO', 'EUPATH', 'EXO', 'FAO', 'FBBI', 'FBBT', + 'FBCV', 'FBDV', 'FIDEO', 'FLOPO', 'FMA', 'FOBI', 'FOODON', 'FOVT', 'FYPO', 'GECKO', + 'GENEPIO', 'GENO', 'GEO', 'GNO', 'HANCESTRO', 'HAO', 'HOM', 'HSAPDV', 'HSO', 'HTN', 'IAO', + 'ICEO', 'ICO', 'IDO', 'INO', 'LABO', 'LEPAO', 'MA', 'MAXO', 'MCO', 'MF', 'MFMO', 'MFOEM', + 'MFOMD', 'MI', 'MIAPA', 'MICRO', 'MMO', 'MMUSDV', 'MOD', 'MONDO', 'MOP', 'MP', 'MPATH', + 'MPIO', 'MRO', 'MS', 'NBO', 'NCBITAXON', 'NCIT', 'NCRO', 'NOMEN', 'OAE', 'OARCS', 'OBA', + 'OBCS', 'OBIB', 'OGG', 'OGMS', 'OGSF', 'OHD', 'OHMI', 'OHPI', 'OLATDV', 'OMIT', 'OMO', 'OMP', + 'OMRSE', 'ONE', 'ONS', 'ONTOAVIDA', 'ONTONEO', 'OOSTT', 'OPL', 'OPMI', 'ORNASEQ', 'OVAE', + 'PCO', 'PDRO', 'PDUMDV', 'PECO', 'PHIPO', 'PLANA', 'PLANP', 'PORO', 'PPO', 'PSDO', 'PSO', + 'PW', 'RBO', 'RO', 'RS', 'RXNO', 'SEPIO', 'SO', 'SPD', 'STATO', 'SWO', 'SYMP', 'TAXRANK', + 'TO', 'TRANS', 'TTO', 'TXPO', 'UBERON', 'UO', 'UPHENO', 'VO', 'VT', 'VTO', 'WBBT', 'WBLS', + 'WBPHENOTYPE', 'XCO', 'XLMOD', 'XPO', 'ZECO', 'ZFS', 'ZP', 'EPIO', 'GSSO', 'HP', 'KISAO', + 'MAMO', 'SBO', 'SCDO', 'SIBO', 'FIX', 'VARIO', 'OGI', 'REX', 'CEPH', 'EHDAA2', 'GAZ', 'RNAO', + 'UPA', 'ERO', 'IDOMAL', 'MIRO', 'TADS', 'TGMA', ] + + try: + res = requests.get('https://bioregistry.io/api/reference/' + prefix + ':' + identifier).json() + except JSONDecodeError: + if prefix.upper() in obo_ont_prefixes: entity_url = obo_url + prefix.upper() + '_' + identifier + elif url is not None: entity_url = url + else: raise ValueError('Error: Invalid prefix or identifier provided. Please check your input and try again.') + if entity_url == '' and res is not None: entity_url = res['providers']['bioregistry'] + + return entity_url + + +# def gets_biolink_information(entity: str, entity_label: Optional[str] = None, biolink_loc='./resources/') -> str: +# """Function takes an entity CURIE and label and returns its BioLink Model type. First, the function uses the +# TranslatorSRI API. If that does not return a match, the function then downloads (if not already downloaded) a +# yaml file of the current BioLink model and searches it. If that also does not return a match, then the function +# formats the entity's label and returns it as the biolink type. Examples are shown below. For entities, +# this function relies on the TranslatorSRI application (https://github.com/TranslatorSRI/NodeNormalization). +# +# Assumptions: If more than 1 BioLink type is provided, the function is designed to take the first one. +# +# EXAMPLE OUTPUT: +# - ""CHEBI:16753" --> biolink:SmallMolecule +# - "RO:0002512" --> biolink:translation_of +# +# Args: +# entity: A string containing an entity CURIE (e.g., CHEBI:16753) or None. +# entity_label: A string representing an entity label. +# biolink_loc: A string containing a location to a biolink yaml file. +# +# Returns: +# biolink_type: A string containing a BioLink model type for a node or an predication. +# """ +# +# # check for biolink data being downloaded +# biolink_file = 'https://raw.githubusercontent.com/biolink/biolink-model/master/biolink-model.yaml' +# if not os.path.exists(biolink_loc + 'biolink-model.yaml'): data_downloader(biolink_file, biolink_loc) +# biolink_data = yaml.load(open(biolink_loc + 'biolink-model.yaml'), Loader=yaml.FullLoader) +# +# # find entities bioLink type +# entity = entity.replace('_', ':') +# result = requests.get('https://nodenormalization-sri.renci.org/get_normalized_nodes', params={'curie': entity}) +# res = result.json() +# +# if res[entity] is not None: biolink_type = res[entity]['type'][0] +# else: # checks the biolink yaml for the entity CURIE +# temp_idx = [k for k in biolink_data['slots'].keys() +# if ('exact_mappings' in biolink_data['slots'][k].keys() +# and entity in biolink_data['slots'][k]['exact_mappings']) +# or ('narrow_mappings' in biolink_data['slots'][k].keys() +# and entity in biolink_data['slots'][k]['narrow_mappings'])] +# if len(temp_idx) > 0: biolink_type = 'biolink:{}'.format(temp_idx[0].replace(' ', '_')) +# else: # checks the biolink yaml for the entity label +# if entity_label is not None: +# entity_label = entity_label.lower() +# temp_str = [k for k in biolink_data['slots'].keys() if k == entity_label] +# if len(temp_str) > 0: biolink_type = 'biolink:{}'.format(temp_str[0].replace(' ', '_')) +# else: biolink_type = 'biolink:{}'.format(entity_label.replace(' ', '_')) +# else: biolink_type = 'biolink:Other' +# +# return biolink_type + + +def dump_jsonl(data: List, output_path: str) -> None: + """Write list of objects to a JSON lines file. This function was modified from: + https://galea.medium.com/how-to-love-jsonl-using-json-line-format-in-your-workflow-b6884f65175b + + Args: + data: A list of Dict objects. + output_path: A string containing a location to write data to. + + Returns: + None. + """ + + with open(output_path, 'a+', encoding='utf-8') as f: + for line in data: + temp_dict = dict() + # check for nested dictionaries + for k, v in line.items(): + if isinstance(v, dict): temp_dict[k] = str(v) + else: temp_dict[k] = v + json_record = json.dumps(temp_dict, ensure_ascii=False) + f.write(json_record + '\n') + + return None + + +def load_jsonl(input_path: str) -> Dict: + """Read list of objects from a JSON lines file. This function was modified from: + https://galea.medium.com/how-to-love-jsonl-using-json-line-format-in-your-workflow-b6884f65175b + + Args: + input_path: A string containing a location to a jsonl file. + + Returns: + data_dict: A Dict object of the data contained in the object pointed to by input_path. + """ + + data_dict: Dict = dict() + with open(input_path, 'r', encoding='utf-8') as f: + for line in f: + data_dict.update(**json.loads(line.rstrip('\n|\r'))) + + return data_dict diff --git a/pkt_kg/utils/kg_utils.py b/pkt_kg/utils/kg_utils.py index 430a3751..08ff4156 100644 --- a/pkt_kg/utils/kg_utils.py +++ b/pkt_kg/utils/kg_utils.py @@ -26,6 +26,8 @@ * removes_namespace_from_bnodes * updates_pkt_namespace_identifiers * splits_knowledge_graph +* nx_ancestor_search +* processes_ancestor_path_list Writes Triple Lists * maps_ids_to_integers @@ -48,11 +50,12 @@ from more_itertools import unique_everseen # type: ignore from rdflib import BNode, Graph, Literal, Namespace, URIRef # type: ignore from rdflib.namespace import OWL, RDF, RDFS # type: ignore +# noinspection PyProtectedMember from rdflib.plugins.serializers.nt import _quoteLiteral # type: ignore import subprocess from tqdm import tqdm # type: ignore -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Union from pkt_kg.utils import * # set-up environment variables @@ -688,7 +691,9 @@ def maps_ids_to_integers(graph: Union[Graph, Set], write_location: str, output_i s, p, o = s.encode('utf-8').decode(), p.encode('utf-8').decode(), o.encode('utf-8').decode() ids.write(s + '\t' + p + '\t' + o + '\n') output_triples += 1 -# ints.close(), ids.close() + + # TODO: add an edge identifier and make sure that the output is zipped. + # CHECK - verify we get the number of edges that we would expect to get if graph_len != output_triples: raise ValueError('ERROR: The number of triples is incorrect!') else: @@ -775,3 +780,52 @@ def appends_to_existing_file(edges: Union[List, Set, Graph], filepath: str, sep: out.close() return None + + +def nx_ancestor_search(kg: nx.multidigraph.MultiDiGraph, nodes: List, prefix: str, anc_list: Optional[List] = None) ->\ + Union[Callable, List]: + """Returns all ancestors nodes reachable through a direct edge. The returned list is ordered by seniority. + + Args: + kg: A networkx MultiDiGraph object. + nodes: A list of RDFLib URIRef objects or None. + prefix: A string containing an ontology prefix (e.g., MONDO). + anc_list: A list that is empty or that contains RDFLib URIRef objects. + + Returns: + anc_list: A list of period-delimited strings, where each string represents a path + """ + + ancestor_list = [] if anc_list is None else anc_list + + if len(nodes) == 0: return ancestor_list + else: + node = nodes.pop(); node_list = list(kg.neighbors(node)) + neighborhood = [a for b in [[[i, n] for j in [kg.get_edge_data(*(node, n)).keys()] + for i in j] for n in node_list] for a in b] + ancestors = [x[1] for x in neighborhood if (prefix in str(x[1]) and x[0] == RDFS.subClassOf)] + if len(ancestors) > 0: + ancestor_list += [[str(x) for x in ancestors]] + nodes += ancestors + return nx_ancestor_search(kg, nodes, prefix, ancestor_list) + + +def processes_ancestor_path_list(path_list: List) -> Dict: + """Processes a nested list of ancestor paths into a dictionary. + + Args: + path_list: A nested list of ontology URLs, where each list represents a set of ancestors. + + Returns: + ancestors: A dictionary where keys are ints formatted as strings and values are sets of URL strings for each + concept that was found at that level. The level is the distance in the hierarchy from the searched node. + """ + + anc_dict: Dict = dict() + for path in path_list: + for x in path: + idx = max([i for i, j in enumerate(path_list) if x in j]) + if str(idx) in anc_dict.keys(): anc_dict[str(idx)] |= {x} + else: anc_dict[str(idx)] = {x} + + return anc_dict diff --git a/resources/edge_source_list.txt b/resources/edge_source_list.txt index 3f9c0d16..ab977b3b 100644 --- a/resources/edge_source_list.txt +++ b/resources/edge_source_list.txt @@ -1,33 +1,40 @@ -chemical-disease, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chemicals_diseases.tsv -chemical-gene, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_gene_ixns.tsv -chemical-gobp, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv -chemical-gocc, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv -chemical-gomf, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv -chemical-pathway, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/ChEBI2Reactome_All_Levels.txt -chemical-phenotype, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chemicals_diseases.tsv -chemical-protein, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_gene_ixns.tsv -disease-phenotype, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/phenotype.hpoa -gene-disease, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/curated_gene_disease_associations.tsv -gene-gene, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt -gene-pathway, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_genes_pathways.tsv -gene-phenotype, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/curated_gene_disease_associations.tsv -gene-protein, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt -gene-rna, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt -gobp-pathway, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome -pathway-gocc, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome -pathway-gomf, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome -protein-anatomy, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt -protein-catalyst, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_CATALYST.txt -protein-cell, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt -protein-cofactor, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_COFACTOR.txt -protein-gobp, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf -protein-gocc, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf -protein-gomf, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf -protein-pathway, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/UniProt2Reactome_All_Levels.txt -protein-protein, https://storage.googleapis.com/pheknowlator/current_build/data/original_data/9606.protein.links.v11.0.txt -rna-anatomy, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt -rna-cell, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt -rna-protein, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENSEMBL_TRANSCRIPT_PROTEIN_ONTOLOGY_MAP.txt -variant-disease, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt -variant-gene, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt -variant-phenotype, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_DISEASE_PHENOTYPE_EDGES.txt \ No newline at end of file +##################################################################################################################################################### +#### edge_source_info.txt (last updated: December 27, 2021) +### Each column is separated by a pipe (i.e., "|") and includes the following: +# edge_type: A string label for an edge (node1-node2). The label matches what is used in the resource_info.txt and ontology_source_list.txt files. +# url: A string containing a URL to the primary data source for the edge. +##################################################################################################################################################### +chemical-disease|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chemicals_diseases.tsv +chemical-gene|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_gene_ixns.tsv +chemical-gobp|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv +chemical-gocc|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv +chemical-gomf|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_go_enriched.tsv +chemical-pathway|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/ChEBI2Reactome_All_Levels.txt +chemical-phenotype|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chemicals_diseases.tsv +chemical-protein|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_gene_ixns.tsv +chemical-rna|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_chem_gene_ixns.tsv +disease-phenotype|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/phenotype.hpoa +gene-disease|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/curated_gene_disease_associations.tsv +gene-gene|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/COMBINED.DEFAULT_NETWORKS.BP_COMBINING.txt +gene-pathway|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/CTD_genes_pathways.tsv +gene-phenotype|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/curated_gene_disease_associations.tsv +gene-protein|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt +gene-rna|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt +gobp-pathway|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome +pathway-gocc|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome +pathway-gomf|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/gene_association.reactome +protein-anatomy|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt +protein-catalyst|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_CATALYST.txt +protein-cell|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt +protein-cofactor|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/UNIPROT_PROTEIN_COFACTOR.txt +protein-gobp|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf +protein-gocc|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf +protein-gomf|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/goa_human.gaf +protein-pathway|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/UniProt2Reactome_All_Levels.txt +protein-protein|https://storage.googleapis.com/pheknowlator/current_build/data/original_data/9606.protein.links.v11.0.txt +rna-anatomy|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt +rna-cell|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/HPA_GTEX_RNA_GENE_PROTEIN_EDGES.txt +rna-protein|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ENSEMBL_TRANSCRIPT_PROTEIN_ONTOLOGY_MAP.txt +variant-disease|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt +variant-gene|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_GENE_EDGES.txt +variant-phenotype|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/CLINVAR_VARIANT_DISEASE_PHENOTYPE_EDGES.txt \ No newline at end of file diff --git a/resources/metadata/README.md b/resources/metadata/README.md new file mode 100644 index 00000000..c18688c8 --- /dev/null +++ b/resources/metadata/README.md @@ -0,0 +1,76 @@ +*** +## Preparing Node and Entity Metadata +*** +*** + +**Wiki Page:** **[`Dependencies`](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#node-metadata)** +**Jupyter Notebook:** **[`Data_Preparation.ipynb`](https://github.com/callahantiff/PheKnowLator/blob/master/notebooks/Data_Preparation.ipynb)** + +**Generated Output:** `./resources/metadata/entity_metadata_dict.pkl` + +___ + +A variety of metadata are pulled from the data sources that are used to support external edges added to +enhance the core set of ontologies. For the monthly PheKnowLator builds, please see [`pheknowlator_source_metadata. +xlsx`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/metadata/pheknowlator_source_metadata.xlsx) +spreadsheet. This spreadsheet has two tabs, one for nodes and one for edges. Each entity (i.e., node or relation) there are several columns, including descriptions of the metadata, the variable type, and even examples of values for each type of metadata. + +*Example Metadata Dictionary Output*. The code snippet below is meant to provide a snapshot of how data are organized in the metadata dictionary. As demonstrated by this example, there are three high-level keys: + - `nodes`: Nodes are keyed by CURIE. Every node has a `Label`, `Description`, `Synonym`, and `Dbxref` (whenever possible). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. + - `edges`: Edges are keyed by a label which represents the edge type (the same label that is used in `resource_info.txt` and `edge_source_list.txt` files). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. + - `relations`: Relations or `owl:ObjectProperty` objects are keyed by CURIE. Similar to nodes, every relation has a `Label`, `Description`, and `Synonym` (whenever possible). Metadata that are obtained from specific sources that are not ontologies are added as a nested dictionary keyed by the filename. + +```python +{ + 'nodes': { + 'NCBIGene_2052': { + 'Label': 'EPHX1', + 'Description': "EPHX1 has locus group 'protein-coding' and is located on chromosome 1 (1q42.12).", + 'Synonym': 'epoxide hydrolase 1, microsomal (xenobiotic)|epoxide hydratase|EPHX|HYL1|MEHepoxide hydrolase 1|epoxide hydrolase 1 microsomal|EPOX', + 'Dbxref': 'MIM:132810|HGNC:HGNC:3401|Ensembl:ENSG00000143819', ... }, + 'CHEBI_4592': { + 'Label': 'Dihydroxycarbazepine', + 'Description': "None", + 'Synonym': '10,11-Dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide|10,11-Dihydroxycarbamazepine', + 'Dbxref': 'CAS:35079-97-1|KEGG:C07495', + 'CTD_chem_gene_ixns.tsv.gz': { + 'CTD_ChemicalID': {'MESH:C004822'}, + 'CTD_CasRN': {'35079-97-1'}, + 'CTD_ChemicalName': {'10,11-dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide'}}, ... }, ... }, + 'edges': { + 'chemical-gene': { + 'CHEBI_4592-NCBIGene_2052': { + {'CTD_chem_gene_ixns.tsv': { + 'CTD_Evidence': [{'CTD_Interaction': '[EPHX1 gene SNP affects the metabolism of carbamazepine epoxide] which affects the chemical synthesis of 10,11-dihydro-10,11-dihydroxy-5H-dibenzazepine-5-carboxamide', + 'CTD_InteractionActions': 'affects^chemical synthesis|affects^metabolic processing', + 'CTD_PubMedIDs': '15692831'}]}}, ... }, ... }, ... }, + 'relations': { + 'RO_0002434': { + 'Label': 'interacts with', + 'Description': 'A relationship that holds between two entities in which the processes executed by the two entities are causally connected.', + 'Synonym': 'in pairwise interaction with'}, ... } +} +``` + +
+ +**Purpose:** +The knowledge graph can be built with or without the inclusion of node and relation metadata (i.e. +labels, descriptions or definitions, and synonyms). If you'd like to create and use node metadata, please run the +[`Data_Preparation.ipynb`](https://github.com/callahantiff/PheKnowLator/blob/master/notebooks/Data_Preparation.ipynb) +Jupyter Notebook and run the code chunks listed under the **NODE AND RELATION METADATA** section. These code chunks +should be run before the knowledge graph is constructed. For more details on what these data sources are and how +they are created, please see the `metatadata` [`README.md`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/metadata/README.md). + +
+ +🛑 *CONSTRAINTS* 🛑 +The algorithm makes the following assumptions: +- If metadata is provided, only those edges with nodes that have metadata will be created; valid edges without metadata will be discarded. +- Metadata will be divided into `nodes`, `relations`, and `edges`. For `nodes` and `relations`, entities will be + keyed by CURIE. For `edges`, entities will be keyed by their edge type (i.e., the same label that is used in + `resource_info.txt` and `edge_source_list.txt` files). +- For each `node` and `node` entity identifier we try to obtain at least the following metadata: `Label`, + `Description`, and `Synonym`. +- Metadata that are obtained from specific sources that are not ontologies will be added as a nested dictionary that is + keyed by the filename. diff --git a/resources/metadata/pheknowlator_source_metadata.xlsx b/resources/metadata/pheknowlator_source_metadata.xlsx new file mode 100644 index 00000000..3ce8ab1e Binary files /dev/null and b/resources/metadata/pheknowlator_source_metadata.xlsx differ diff --git a/resources/node_data/README.md b/resources/node_data/README.md deleted file mode 100644 index 84661419..00000000 --- a/resources/node_data/README.md +++ /dev/null @@ -1,54 +0,0 @@ -*** -## Creating Instance Data Node Metadata -*** -*** - -**Wiki Page:** **[`Dependencies`](https://github.com/callahantiff/PheKnowLator/wiki/Dependencies#node-metadata)** -**Jupyter Notebook:** **[`Data_Preparation.ipynb`](https://github.com/callahantiff/PheKnowLator/blob/master/notebooks/Data_Preparation.ipynb)** - -___ - -**Purpose:** The knowledge graph can be built with or without the inclusion of node and relation metadata (i.e. -labels, descriptions or definitions, and synonyms). If you'd like to create and use node metadata, please run the -[`Data_Preparation.ipynb`](https://github.com/callahantiff/PheKnowLator/blob/master/notebooks/Data_Preparation.ipynb) Jupyter Notebook and run the code chunks listed under the **INSTANCE AND/OR SUBCLASS (NON-ONTOLOGY CLASS) METADATA** section. These code chunks should be run before the knowledge graph is constructed. For more details on what these data sources are and how they are created, please see the `node_data` [`README.md`](https://github.com/callahantiff/PheKnowLator/blob/master/resources/node_data/README.md). - -Example structure of the metadata dictionary is shown below: - -```python -{ - 'nodes': { - 'http://www.ncbi.nlm.nih.gov/gene/1': { - 'Label': 'A1BG', - 'Description': "A1BG has locus group protein-coding' and is located on chromosome 19 (19q13.43).", - 'Synonym': 'HYST2477alpha-1B-glycoprotein|HEL-S-163pA|ABG|A1B|GAB'} ... }, - 'relations': { - 'http://purl.obolibrary.org/obo/RO_0002533': { - 'Label': 'sequence atomic unit', - 'Description': 'Any individual unit of a collection of like units arranged in a linear order', - 'Synonym': 'None'} ... } -} -``` - -
- -🛑 *CONSTRAINTS* 🛑 -The algorithm makes the following assumptions: -- If metadata is provided, only those edges with nodes that have metadata will be created; valid edges without metadata will be discarded. -- Metadata for all non-ontology nodes and all relations for edges added to the core set of ontologies will be saved as a dictionary in the `./resources/node_data/node_metadata_dict.pkl` repository. -- For each identifier we try to obtain the following metadata: `Label`, `Description`, and `Synonym`. An example of these data types is shown below for a [`gene`](https://github.com/callahantiff/PheKnowLator/wiki/v2-Data-Sources#ncbi-gene) identifier `5620`: - -| **Metadata Type** | **Definition** | **Metadata** | -| :---: | :--- | :--- | -| ID | Node identifiers for instance data sources | `5620` | -| Label | The primary label or name for the node | `LANCL2` | -| Description | A definition or other useful details about the node | `Lanc Like 2` is a `protein-coding` gene that is located on chromosome `7` (map_location: `7p11.2`) | -| Synonym | Alternative terms used for a node | `GPR69B`, `TASP`, `lanC-like protein 2`, `G protein-coupled receptor 69B`, `LanC (bacterial lantibiotic synthetase component C)-like 2`, `LanC lantibiotic synthetase component C-like 2`, `testis-specific adriamycin sensitivity protein` | - -
- -#### Metadata + PheKnowLator -*** -The metadata will be used to create the following edges in the knowledge graph: -- **Label** ➞ node `rdfs:label` -- **Description** ➞ node `obo:IAO_0000115` description -- **Synonyms** ➞ node `oboInOwl:hasExactSynonym` synonym diff --git a/resources/ontology_source_list.txt b/resources/ontology_source_list.txt index d901f1eb..6dc71b1d 100644 --- a/resources/ontology_source_list.txt +++ b/resources/ontology_source_list.txt @@ -1,11 +1,18 @@ -phenotype, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/hp_with_imports.owl -go, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/go_with_imports.owl -disease, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/mondo_with_imports.owl -vaccine, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/vo_with_imports.owl -chemical, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/chebi_with_imports.owl -anatomy, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ext_with_imports.owl -cell, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/clo_with_imports.owl -protein, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/pr_with_imports.owl -genomic, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/so_with_imports.owl -pathway, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/pw_with_imports.owl -relation, https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ro_with_imports.owl \ No newline at end of file +################################################################################################################################################### +#### ontology_source_info.txt (last updated: December 27, 2021) +### Each column is separated by a pipe (i.e., "|") and includes the following: +# ontology: A string label for an edge (node1-node2). The label matches what is used in the resource_info.txt and +edge_source_list.txt files. +# url: A string containing a URL to the ontology file. +#################################################################################################################################################### +phenotype|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/hp_with_imports.owl +go|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/go_with_imports.owl +disease|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/mondo_with_imports.owl +vaccine|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/vo_with_imports.owl +chemical|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/chebi_with_imports.owl +anatomy|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ext_with_imports.owl +cell|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/clo_with_imports.owl +protein|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/pr_with_imports.owl +genomic|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/so_with_imports.owl +pathway|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/pw_with_imports.owl +relation|https://storage.googleapis.com/pheknowlator/current_build/data/processed_data/ro_with_imports.owl \ No newline at end of file diff --git a/resources/resource_info.txt b/resources/resource_info.txt index 37b8c0f5..cd07bd89 100644 --- a/resources/resource_info.txt +++ b/resources/resource_info.txt @@ -1,33 +1,61 @@ -chemical-disease|:;MESH_;|class-class|RO_0002606|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/DISEASE_MONDO_MAP.txt|5;!=;''|None -chemical-gene|;MESH_;|class-entity|RO_0002434|http://purl.obolibrary.org/obo/|http://www.ncbi.nlm.nih.gov/gene/|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt|9;affects;not in x|6;==;Homo sapiens::5;.startswith('gene'); -chemical-gobp|:;MESH_;GO_|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|8;<=;1.04e-47|3;==;Biological Process -chemical-gocc|:;MESH_;GO_|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|8;<=;1.04e-47|3;==;Cellular Component -chemical-gomf|:;MESH_;GO_|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|8;<=;1.04e-47|3;==;Molecular Function -chemical-pathway|;CHEBI_;|class-entity|RO_0000056|http://purl.obolibrary.org/obo/|https://reactome.org/content/detail/|t|0;1|None|None|5;==;Homo sapiens -chemical-phenotype|:;MESH_;|class-class|RO_0002606|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|5;!=;''|None -chemical-protein|;MESH_;|class-class|RO_0002434|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt|9;affects;not in x|6;==;Homo sapiens::5;.startswith('protein'); -disease-phenotype|:;;HP_|class-class|RO_0002200|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|0;3|0:./resources/processed_data/DISEASE_MONDO_MAP.txt|None|None|None -gene-disease|;;|entity-class|RO_0003302|http://www.ncbi.nlm.nih.gov/gene/|http://purl.obolibrary.org/obo/|t|0;4|1:./resources/processed_data/DISEASE_MONDO_MAP.txt|10;>=;1.0|None -gene-gene|;;|entity-entity|RO_0002435|http://www.ncbi.nlm.nih.gov/gene/|http://www.ncbi.nlm.nih.gov/gene/|t|0;1|0:./resources/processed_data/ENSEMBL_GENE_ENTREZ_GENE_MAP.txt;1:./resources/processed_data/ENSEMBL_GENE_ENTREZ_GENE_MAP.txt|None|None -gene-pathway|:;;|entity-entity|RO_0000056|http://www.ncbi.nlm.nih.gov/gene/|https://reactome.org/content/detail/|t|1;3|None|None|3;.startswith('REACT:R-HSA-'); -gene-phenotype|;;|entity-class|RO_0003302|http://www.ncbi.nlm.nih.gov/gene/|http://purl.obolibrary.org/obo/|t|0;4|1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|10;>=;1.0|None -gene-protein|;;|entity-class|RO_0002205|http://www.ncbi.nlm.nih.gov/gene/|http://purl.obolibrary.org/obo/|t|0;1|None|None|4;==;protein-coding -gene-rna|;;|entity-entity|RO_0002511|http://www.ncbi.nlm.nih.gov/gene/|https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=|t|0;1|None|None|None -gobp-pathway|:;GO_;|class-entity|RO_0009501|http://purl.obolibrary.org/obo/|https://reactome.org/content/detail/|t|4;5|None|None|8;==;P::12;==;taxon:9606::5;.startswith('REACTOME'); -pathway-gocc|:;;GO_|entity-class|RO_0002180|https://reactome.org/content/detail/|http://purl.obolibrary.org/obo/|t|5;4|None|None|8;==;C::12;==;taxon:9606::5;.startswith('REACTOME'); -pathway-gomf|:;;GO_|entity-class|RO_0000085|https://reactome.org/content/detail/|http://purl.obolibrary.org/obo/|t|5;4|None|None|8;==;F::12;==;taxon:9606::5;.startswith('REACTOME'); -protein-anatomy|;;|class-class|RO_0001025|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|2;5|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at protein level::4;==;anatomy -protein-catalyst|;;|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|0;1|None|None|None|None -protein-cell|;;|class-class|RO_0001025|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|2;5|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at protein level::4;==;cell line -protein-cofactor|;;|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|0;1|None|None|None|None -protein-gobp|:;;GO_|class-class|RO_0000056|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;P::12;==;taxon:9606 -protein-gocc|:;;GO_|class-class|RO_0001025|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;C::12;==;taxon:9606 -protein-gomf|:;;GO_|class-class|RO_0000085|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;F::12;==;taxon:9606 -protein-pathway|;;|class-entity|RO_0000056|http://purl.obolibrary.org/obo/|https://reactome.org/content/detail/|t|0;1|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|5;==;Homo sapiens -protein-protein|9606.;;|class-class|RO_0002436|http://purl.obolibrary.org/obo/|http://purl.obolibrary.org/obo/|''|0;1|0:./resources/processed_data/STRING_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/STRING_PRO_ONTOLOGY_MAP.txt|2;>=;700|None -rna-anatomy|;;|entity-class|RO_0001025|https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=|http://purl.obolibrary.org/obo/|t|1;5|0:./resources/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at transcript level::4;==;anatomy -rna-cell|;;|entity-class|RO_0001025|https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=|http://purl.obolibrary.org/obo/|t|1;5|0:./resources/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at transcript level::4;==;cell line -rna-protein|;;|entity-class|RO_0002513|https://uswest.ensembl.org/Homo_sapiens/Transcript/Summary?t=|http://purl.obolibrary.org/obo/|t|0;1|None|None|4;==;protein-coding -variant-disease|:;rs;|entity-class|RO_0003302|https://www.ncbi.nlm.nih.gov/snp/|http://purl.obolibrary.org/obo/|t|9;12|1:./resources/processed_data/DISEASE_MONDO_MAP.txt|24;in;["criteria provided, multiple submitters, no conflicts", "reviewed by expert panel", "practice guideline"]::7;==;1|9;!=;-1::16;==;GRCh38::8-9;dedup;desc -variant-gene|;rs;|entity-entity|RO_0002566|https://www.ncbi.nlm.nih.gov/snp/|http://www.ncbi.nlm.nih.gov/gene/|t|9;3|None|24;in;["criteria provided, multiple submitters, no conflicts", "reviewed by expert panel", "practice guideline"]|9;!=;-1::3;!=;-1::16;==;GRCh38::8-9;dedup;desc -variant-phenotype|:;rs;|entity-class|RO_0003302|https://www.ncbi.nlm.nih.gov/snp/|http://purl.obolibrary.org/obo/|t|9;12|1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|24;in;["criteria provided, multiple submitters, no conflicts", "reviewed by expert panel", "practice guideline"]::7;==;1|9;!=;-1::16;==;GRCh38::8-9;dedup;desc \ No newline at end of file +###################################################################################################################################################################################### +#### resource_info.txt (last updated: December 27, 2021) +### Each column is separated by a pipe (i.e., "|") and includes the following: +# edge_type: A string label for an edge (node1-node2). The label matches what is used in the edge_source_list.txt and ontology_source_list.txt files. +# prefixes: A ";"-separated string where the first item is the final prefix for the subject node and the second is the final prefix for the object node. +# All prefixes should be the preferred prefix from the BioRegistry (https://bioregistry.io/registry/). +# relation: An OBO Foundry ontology CURIE (e.g., RO_0000056). +# delimiter: A character used to split rows from an input data source into columns (e.g., "t" for tab-delimited data or "," for comma-delimited data). +# column_indexes: Two-column indexes separated by ";" (e.g., "0;4" for the first and third columns in the input data source). +# identifier_maps: A string of mapping information for each node in an edge. For example, the string "2:mapping_file_1.txt;4:mapping_file_2.txt" means that +# the first node requires data contained in the 2nd column of the "mapping_file_1.txt" and the second node requires data from the 4th column +# in the "mapping_file_2.txt" file. +# evidence_criteria: Evidence criteria that can be used to filter an input data source (e.g., scores above a certain cut-off). An evidence set is composed of 3 +# pieces of ";"-separated information. Multiple evidence sets can be passed, where each set is separated by "::". Consider the following +# example: "4;!=;IEA::8;<;0.0001": +# 1. The index of the column to apply the evidence criteria to (e.g., "4" and "8" in the example above). +# 2. The operator (i.e., "==", "!=", "<", ">", "<=", ">=", "in", ".startswith()", ".endswith()") to use when filtering (e.g., "!=" and "<" +# in the example above). +# 3. The value (i.e., "int", "float", "str", "list") to filter on (e.g., "IEA" and "0.0001" in the example above). +# filter_criteria: Filtering criteria that can be used to filter an input data source (e.g., human proteins). An evidence set is composed of 3 pieces of ";"- +# separated information. Multiple filtering sets can be passed, where each set is separated by "::". Consider the following example: +# "5;==;P::7;==;9606"): +# 1. The index of the column to apply the evidence criteria to (e.g., "5" and "7" in the example above). +# 2. The operator (i.e., "==", "!=", "<", ">", "<=", ">=", "in", ".startswith()", ".endswith()") to use when filtering (e.g., "==" and "==" +# in the example above). +# 3. The value (i.e., "int", "float", "str", "list") to filter on (e.g., "P" and "9606" in the example above). +###################################################################################################################################################################################### +chemical-disease|CHEBI;|RO_0002606|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/DISEASE_MONDO_MAP.txt|9;!=;''|None +chemical-gene|CHEBI;NCBIGene|RO_0002434|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt|10;!=;''|6;==;Homo sapiens::5;.startswith('gene'); +chemical-gobp|CHEBI;GO|RO_0002436|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|None|3;==;Biological Process +chemical-gocc|CHEBI;GO|RO_0002436|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|None|3;==;Cellular Component +chemical-gomf|CHEBI;GO|RO_0002436|t|1;5|0:./resources/processed_data/MESH_CHEBI_MAP.txt|None|3;==;Molecular Function +chemical-pathway|CHEBI;reactome|RO_0000056|t|0;1|None|None|5;==;Homo sapiens +chemical-phenotype|CHEBI;HP|RO_0002606|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|9;!=;''|None +chemical-protein|CHEBI;PR|RO_0002434|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/ENTREZ_GENE_PRO_ONTOLOGY_MAP.txt|10;!=;''|6;==;Homo sapiens::5;.startswith('protein'); +chemical-rna|CHEBI;ensembl|RO_0002434|t|1;4|0:./resources/processed_data/MESH_CHEBI_MAP.txt;1:./resources/processed_data/ENTREZ_GENE_ENSEMBL_TRANSCRIPT_MAP.txt|10;!=;''|6;==;Homo sapiens::5;.startswith('mRNA'); +disease-phenotype|MONDO;HP|RO_0002200|t|0;3|0:./resources/processed_data/DISEASE_MONDO_MAP.txt|None|None|2;!=;NOT +gene-disease|NCBIGene;MONDO|RO_0003302|t|0;4|1:./resources/processed_data/DISEASE_MONDO_MAP.txt|None|6;==;disease +gene-gene|NCBIGene;NCBIGene|RO_0002435|t|0;1|0:./resources/processed_data/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt;1:./resources/processed_data/UNIPROT_ACCESSION_ENTREZ_GENE_MAP.txt|None|None +gene-pathway|NCBIGene;reactome|RO_0000056|t|1;3|None|None|3;.startswith('REACT:R-HSA-'); +gene-phenotype|NCBIGene;HP|RO_0003302|t|0;4|1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|None|6;==;phenotype +gene-protein|NCBIGene;PR|RO_0002205|t|4;1|None|None|3;==;protein-coding +gene-rna|NCBIGene;ensembl|RO_0002511|t|6;1|None|None|None +gobp-pathway|GO;reactome|RO_0009501|t|4;5|None|None|8;==;P::12;==;taxon:9606::5;.startswith('REACTOME'); +pathway-gocc|reactome;GO|RO_0002180|t|5;4|None|None|8;==;C::12;==;taxon:9606::5;.startswith('REACTOME'); +pathway-gomf|reactome;GO|RO_0000085|t|5;4|None|None|8;==;F::12;==;taxon:9606::5;.startswith('REACTOME'); +protein-anatomy|PR;UBERON|RO_0001025|t|2;6|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at protein level::4;==;anatomy +protein-catalyst|PR;CHEBI|RO_0002436|t|0;1|None|None|None|None +protein-cell|PR;CL|RO_0001025|t|2;6|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at protein level::4;==;cell line +protein-cofactor|PR;CHEBI|RO_0002436|t|0;1|None|None|None|None +protein-gobp|PR;GO|RO_0000056|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;P::12;==;taxon:9606::3;not in;["NOT"]::11;==;protein +protein-gocc|PR;GO|RO_0001025|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;C::12;==;taxon:9606::3;not in;["NOT"]::11;==;protein +protein-gomf|PR;GO|RO_0000085|t|1;4|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|8;==;F::12;==;taxon:9606::3;not in;["NOT"]::11;==;protein +protein-pathway|PR;reactome|RO_0000056|t|0;1|0:./resources/processed_data/UNIPROT_ACCESSION_PRO_ONTOLOGY_MAP.txt|None|5;==;Homo sapiens +protein-protein|PR;PR|RO_0002436|''|0;1|0:./resources/processed_data/STRING_PRO_ONTOLOGY_MAP.txt;1:./resources/processed_data/STRING_PRO_ONTOLOGY_MAP.txt|None|None +rna-anatomy|ensembl;UBERON|RO_0001025|t|1;6|0:./resources/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at transcript level::4;==;anatomy +rna-cell|ensembl;CL|RO_0001025|t|1;6|0:./resources/processed_data/GENE_SYMBOL_ENSEMBL_TRANSCRIPT_MAP.txt;1:./resources/processed_data/HPA_GTEx_TISSUE_CELL_MAP.txt|None|3;==;Evidence at transcript level::4;==;cell line. +rna-protein|ensembl;PR|RO_0002513|t|4;1|None|None|3;==;protein-coding +variant-disease|clinvar;MONDO|RO_0003302|t|0:18|1:./resources/processed_data/DISEASE_MONDO_MAP.txt|None|None +variant-gene|clinvar;NCBIGene|RO_0002566|t|0;6|None|None|None +variant-phenotype|clinvar;HP|RO_0003302|t|0;18|1:./resources/processed_data/PHENOTYPE_HPO_MAP.txt|None|None \ No newline at end of file diff --git a/resources/~$pheknowlator_source_metadata.xlsx b/resources/~$pheknowlator_source_metadata.xlsx new file mode 100644 index 00000000..e8ac2539 Binary files /dev/null and b/resources/~$pheknowlator_source_metadata.xlsx differ diff --git a/setup.py b/setup.py index 3a5cfce1..54663215 100644 --- a/setup.py +++ b/setup.py @@ -74,11 +74,12 @@ def find_version(*file_paths): 'Cython>=0.29.14', 'more-itertools', 'networkx', - 'numpy>=1.18.1', + 'numpy>=1.19.5', 'openpyxl>=3.0.3', 'pandas>=1.0.5', 'psutil', 'python-json-logger', + 'pyyaml', 'ray', 'rdflib', 'reactome2py', diff --git a/tests/test_data_utils_miscellaneous.py b/tests/test_data_utils_miscellaneous.py index 32182f31..ab11e52b 100644 --- a/tests/test_data_utils_miscellaneous.py +++ b/tests/test_data_utils_miscellaneous.py @@ -4,7 +4,6 @@ import shutil import unittest -from tqdm import tqdm from typing import List from pkt_kg.utils import * @@ -154,6 +153,137 @@ def tests_sublist_creator_list(self): return None + def tests_obtains_entity_url_good(self): + """Tests the obtains_entity_url method when a valid prefix and identifier are passed.""" + + # set-up input + prefix = 'chebi'; identifier = '138488' + + # test function + entity_uri = obtains_entity_url(prefix, identifier) + self.assertEqual(entity_uri, 'https://bioregistry.io/chebi:138488') + + return None + + def tests_obtains_entity_url_obo(self): + """Tests the obtains_entity_url method when an obo identifier is passed.""" + + # set-up input + prefix = 'pr'; identifier = 'A5D8V7' + + # test function + entity_uri = obtains_entity_url(prefix, identifier) + self.assertEqual(entity_uri, 'http://purl.obolibrary.org/obo/PR_A5D8V7') + + return None + + def tests_obtains_entity_url_bad1(self): + """Tests the obtains_entity_url method when an invalid identifier is passed.""" + + # set-up input + prefix = 'hpo'; identifier = 't' + + # test function + self.assertRaises(ValueError, obtains_entity_url, prefix, identifier) + + return None + + def tests_obtains_entity_url_bad2(self): + """Tests the obtains_entity_url method when an invalid identifier is passed, but a valid url is passed.""" + + # set-up input + prefix = 'swrl'; identifier = 'Variable'; url = 'http://www.w3.org/2003/11/swrl#Variable' + + # test function + entity_uri = obtains_entity_url(prefix, identifier, url) + self.assertEqual(entity_uri, 'http://www.w3.org/2003/11/swrl#Variable') + + return None + + # def tests_gets_biolink_information_entity(self): + # """Tests the gets_biolink_information function when provided a valid entity CURIE.""" + # + # # set-up input + # entity = 'CHEBI:16753'; entity_label = None + # + # # test function + # res = gets_biolink_information(entity, entity_label, self.dir_loc + '/') + # self.assertEqual(res, 'biolink:SmallMolecule') + # + # return None + # + # def tests_gets_biolink_information_entitylabel(self): + # """Tests the gets_biolink_information function when provided a valid entity CURIE and label are provided.""" + # + # # set-up input + # entity = 'RO:0002436'; entity_label = 'molecularly interacts with' + # + # # test function + # res = gets_biolink_information(entity, entity_label, self.dir_loc + '/') + # self.assertEqual(res, 'biolink:molecularly_interacts_with') + # + # return None + # + # def tests_gets_biolink_information_entitylabel2(self): + # """Tests the gets_biolink_information function when provided a valid entity CURIE and label are provided.""" + # + # # set-up input + # entity = 'rdfs:subClassOf'; entity_label = 'subclass of' + # + # # test function + # res = gets_biolink_information(entity, entity_label, self.dir_loc + '/') + # self.assertEqual(res, 'biolink:subclass_of') + # + # return None + # + # def tests_gets_biolink_information_entitylabel3(self): + # """Tests the gets_biolink_information function when provided a valid entity CURIE and label that cannot be + # found in the model or API.""" + # + # # set-up input + # entity = 'RO:0000000'; entity_label = None + # + # # test function + # res = gets_biolink_information(entity, entity_label, self.dir_loc + '/') + # self.assertEqual(res, 'biolink:Other') + # + # return None + + def tests_dump_jsonl(self): + """Tests the dump_jsonl function.""" + + # set-up input + out_location = self.dir_loc + '/out.jsonl' + + # test function + for url in ['https://chordanalytics.ca/', 'https://github.com/agalea91']: + webpage_data = {'page_url': url, 'status_code': 200} + dump_jsonl([webpage_data], out_location) + + self.assertTrue(os.path.exists(out_location)) + self.assertTrue(os.stat(out_location).st_size > 0) + + return None + + def tests_load_jsonl(self): + """Tests the load_jsonl function.""" + + # set-up input + out_location = self.dir_loc + '/out.jsonl' + for url in ['https://chordanalytics.ca/', 'https://github.com/agalea91']: + webpage_data = {url: {'status_code': 200}} + dump_jsonl([webpage_data], out_location) + + # test function + data_dict = load_jsonl(out_location) + test_dict = {'https://chordanalytics.ca/': "{'status_code': 200}", + 'https://github.com/agalea91': "{'status_code': 200}"} + + self.assertIsInstance(data_dict, dict) + self.assertEqual(data_dict, test_dict) + + return None + def tearDown(self): # remove temp directory diff --git a/tests/test_kg_utils.py b/tests/test_kg_utils.py index 168ad3a6..d17e952f 100644 --- a/tests/test_kg_utils.py +++ b/tests/test_kg_utils.py @@ -775,3 +775,47 @@ def test_updates_pkt_namespace_identifiers_edges2(self): RDFS.subClassOf, URIRef('http://www.ncbi.nlm.nih.gov/gene/4841'))) in result_graph) return None + + def tests_nx_ancestor_search(self): + """Tests the nx_ancestor_search method.""" + + # create test data + graph = Graph().parse(self.dir_loc + '/so_with_imports.owl') + kg = nx.MultiDiGraph() + for s, p, o in graph: + kg.add_node(s, key=n3(s)); kg.add_node(o, key=n3(o)) + kg.add_edge(s, o, **{'key': p, 'weight': 0.0}) + nodes = [obo.SO_0001544] + prefix = 'SO' + + # test method + result_list = nx_ancestor_search(kg, nodes, prefix) + self.assertIsInstance(result_list, List) + self.assertEqual(len(result_list), 5) + self.assertEqual(result_list, [['http://purl.obolibrary.org/obo/SO_0001543'], + ['http://purl.obolibrary.org/obo/SO_0001538'], + ['http://purl.obolibrary.org/obo/SO_0002218'], + ['http://purl.obolibrary.org/obo/SO_0001536'], + ['http://purl.obolibrary.org/obo/SO_0001060']]) + + return None + + def test_processes_ancestor_path_list(self): + """Tests the processes_ancestor_path_list method.""" + + # create test data + path = [['http://purl.obolibrary.org/obo/SO_0001543'], ['http://purl.obolibrary.org/obo/SO_0001538'], + ['http://purl.obolibrary.org/obo/SO_0002218'], ['http://purl.obolibrary.org/obo/SO_0001536'], + ['http://purl.obolibrary.org/obo/SO_0001060']] + + # test method + result = processes_ancestor_path_list(path) + self.assertIsInstance(result, Dict) + self.assertEqual(len(result.keys()), 5) + self.assertEqual(result['0'], {'http://purl.obolibrary.org/obo/SO_0001543'}) + self.assertEqual(result['1'], {'http://purl.obolibrary.org/obo/SO_0001538'}) + self.assertEqual(result['2'], {'http://purl.obolibrary.org/obo/SO_0002218'}) + self.assertEqual(result['3'], {'http://purl.obolibrary.org/obo/SO_0001536'}) + self.assertEqual(result['4'], {'http://purl.obolibrary.org/obo/SO_0001060'}) + + return None diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 8db89c5b..ec64866f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -78,8 +78,8 @@ def test_metadata_processor(self): return None - def test_creates_node_metadata_nodes(self): - """Tests the creates_node_metadata method.""" + def test_creates_entity_metadata_nodes(self): + """Tests the creates_entity_metadata method.""" self.metadata.node_data = [self.metadata.node_data[0].replace('.pkl', '_test.pkl')] self.metadata.extract_metadata(self.graph) @@ -116,8 +116,8 @@ def test_creates_node_metadata_nodes(self): return None - def test_creates_node_metadata_relations(self): - """Tests the creates_node_metadata method.""" + def test_creates_entity_metadata_relations(self): + """Tests the creates_entity_metadata method.""" self.metadata.node_data = [self.metadata.node_data[0].replace('.pkl', '_test.pkl')] self.metadata.extract_metadata(self.graph) @@ -139,8 +139,8 @@ def test_creates_node_metadata_relations(self): return None - def test_creates_node_metadata_none(self): - """Tests the creates_node_metadata method when node_dict is None.""" + def test_creates_entity_metadata_none(self): + """Tests the creates_entity_metadata method when node_dict is None.""" self.metadata.node_data = [self.metadata.node_data[0].replace('.pkl', '_test.pkl')] self.metadata.extract_metadata(self.graph) diff --git a/tests/test_owlnets.py b/tests/test_owlnets.py index 2025348c..a884a096 100644 --- a/tests/test_owlnets.py +++ b/tests/test_owlnets.py @@ -433,17 +433,17 @@ def test_returns_object_property(self): """Tests the returns_object_property method.""" # when sub and obj are PATO terms and property is none - res1 = self.owl_nets.returns_object_property(obo.PATO_0001199, obo.PATO_0000402, None) + res1 = self.owl_nets.returns_object_property(obo.PATO_0001199, obo.PATO_0000402) self.assertIsInstance(res1, URIRef) self.assertEqual(res1, RDFS.subClassOf) # when sub and obj are NOT PATO terms and property is none - res2 = self.owl_nets.returns_object_property(obo.SO_0000784, obo.GO_2000380, None) + res2 = self.owl_nets.returns_object_property(obo.SO_0000784, obo.GO_2000380) self.assertIsInstance(res2, URIRef) self.assertEqual(res2, RDFS.subClassOf) # when the obj is a PATO term and property is none - res3 = self.owl_nets.returns_object_property(obo.SO_0000784, obo.PATO_0001199, None) + res3 = self.owl_nets.returns_object_property(obo.SO_0000784, obo.PATO_0001199) self.assertIsInstance(res3, URIRef) self.assertEqual(res3, obo.RO_0000086) @@ -458,8 +458,8 @@ def test_returns_object_property(self): self.assertEqual(res5, obo.RO_0002202) # when sub is a PATO term and property is none - res6 = self.owl_nets.returns_object_property(obo.PATO_0001199, obo.SO_0000784, None) - self.assertEqual(res6, None) + res6 = self.owl_nets.returns_object_property(obo.PATO_0001199, obo.SO_0000784) + self.assertEqual(res6, RDFS.subClassOf) return None @@ -526,7 +526,7 @@ def test_parses_constructors_intersection(self): # set-up inputs node = obo.SO_0000034 node_info = self.owl_nets.creates_edge_dictionary(node) - bnodes = set(x for x in self.owl_nets.graph.objects(node, None) if isinstance(x, BNode)) + bnodes = set(x for x in self.owl_nets.graph.objects(node) if isinstance(x, BNode)) edges = {k: v for k, v in node_info[1].items() if 'intersectionOf' in v.keys() and k in bnodes} edges = node_info[1][list(x for x in bnodes if x in edges.keys())[0]] @@ -539,12 +539,12 @@ def test_parses_constructors_intersection(self): return None def test_parses_constructors_intersection2(self): - """Tests the parses_constructors method for the UnionOf class constructor""" + """Tests the parses_constructors method for the intersectionOf class constructor""" # set-up inputs node = obo.SO_0000078 node_info = self.owl_nets.creates_edge_dictionary(node) - bnodes = set(x for x in self.owl_nets.graph.objects(node, None) if isinstance(x, BNode)) + bnodes = set(x for x in self.owl_nets.graph.objects(node) if isinstance(x, BNode)) edges = {k: v for k, v in node_info[1].items() if 'intersectionOf' in v.keys() and k in bnodes} edges = node_info[1][list(x for x in bnodes if x in edges.keys())[0]] @@ -556,13 +556,41 @@ def test_parses_constructors_intersection2(self): return None + def test_parses_constructors_union(self): + """Tests the parses_constructors method for the unionOf class constructor""" + + # instantiate class + self.kg_filename2 = '/clo_with_imports.owl' + self.graph2 = Graph().parse('http://purl.obolibrary.org/obo/clo.owl', format='xml') + self.owl_nets3 = OwlNets(kg_construct_approach='subclass', graph=self.graph2, + write_location=self.write_location, filename=self.kg_filename2) + dir_loc_owltools = os.path.join(os.path.dirname(__file__), 'utils/owltools') + self.owl_nets3.owl_tools = os.path.abspath(dir_loc_owltools) + + # set-up inputs + node = obo.CL_0000995 + node_info = self.owl_nets3.creates_edge_dictionary(node) + bnodes = set(x for x in self.owl_nets3.graph.objects(node) if isinstance(x, BNode)) + edges = {k: v for k, v in node_info[1].items() if 'unionOf' in v.keys() and k in bnodes} + edges = node_info[1][list(x for x in bnodes if x in edges.keys())[0]] + + # test method + res = self.owl_nets3.parses_constructors(node, edges, node_info[1]) + self.assertIsInstance(res, Tuple) + self.assertEqual(sorted(list(res[0])), + [(obo.CL_0001021, RDFS.subClassOf, obo.CL_0000995), + (obo.CL_0001026, RDFS.subClassOf, obo.CL_0000995)]) + self.assertEqual(res[1], None) + + return None + def test_parses_restrictions(self): """Tests the parses_restrictions method.""" # set-up inputs node = obo.SO_0000078 node_info = self.owl_nets.creates_edge_dictionary(node) - bnodes = set(x for x in self.owl_nets.graph.objects(node, None) if isinstance(x, BNode)) + bnodes = set(x for x in self.owl_nets.graph.objects(node) if isinstance(x, BNode)) edges = {k: v for k, v in node_info[1].items() if ('type' in v.keys() and v['type'] == OWL.Restriction) and k in bnodes} edges = node_info[1][list(x for x in bnodes if x in edges.keys())[0]] @@ -576,6 +604,24 @@ def test_parses_restrictions(self): return None + def test_verifies_cleaned_classes(self): + """Tests the verifies_cleaned_classes method""" + + # create input data + cleaned_classes = {(obo.HP_0000602, obo.BFO_0000051, obo.HP_0000597), + (obo.HP_0000602, RDFS.subClassOf, obo.HP_0000597), + (obo.HP_0007715, RDFS.subClassOf, obo.HP_0000597), + (obo.HP_0007715, obo.BFO_0000051, obo.HP_0000597)} + cleaned_result = sorted(list({(obo.HP_0000602, obo.BFO_0000051, obo.HP_0000597), + (obo.HP_0007715, obo.BFO_0000051, obo.HP_0000597)})) + + # test method + verified_classes = self.owl_nets.verifies_cleaned_classes(cleaned_classes) + self.assertIsInstance(verified_classes, Set) + self.assertEqual(sorted(list(verified_classes)), cleaned_result) + + return None + def test_cleans_owl_encoded_entities(self): """Tests the cleans_owl_encoded_entities method"""