From 6681c7c3f0eff4ced7b589216fbf927f01acdd97 Mon Sep 17 00:00:00 2001 From: Arnav G <73305591+arnavg115@users.noreply.github.com> Date: Sun, 8 Mar 2026 12:13:18 -0700 Subject: [PATCH 1/4] Various Optimizations to LL_net and LINGER_tr --- code/lingergrn-1.106/LingerGRN/LINGER_tr.py | 849 ++++++++++-------- code/lingergrn-1.106/LingerGRN/LL_net.py | 307 ++++--- code/lingergrn-1.106/LingerGRN/TF_activity.py | 132 ++- 3 files changed, 740 insertions(+), 548 deletions(-) diff --git a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py index d693ae3..62fcb1e 100644 --- a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py +++ b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py @@ -1,499 +1,608 @@ +import os +import random + +# load data +import numpy as np +import pandas as pd +import shap import torch import torch.nn as nn import torch.optim as optim -from torch.nn import functional as F -from scipy.stats import pearsonr -from scipy.stats import spearmanr -#load data -import numpy as np -import pandas as pd -import random -from torch.optim import Adam -import os -from sklearn.linear_model import ElasticNet +from scipy.stats import pearsonr, spearmanr from sklearn.datasets import make_regression +from sklearn.linear_model import ElasticNet from sklearn.model_selection import KFold -import shap -hidden_size = 64 +from torch.nn import functional as F +from torch.optim import Adam + +hidden_size = 64 hidden_size2 = 16 output_size = 1 -from joblib import Parallel,delayed +from joblib import Parallel, delayed + seed_value = 42 +# cpu is more efficient in this case. +# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +device = torch.device("cpu") +print(f"Using device: {device}") + + class Net(nn.Module): - def __init__(self,input_size,activef): + def __init__(self, input_size, activef): super(Net, self).__init__() self.fc1 = nn.Linear(input_size, 64) self.fc2 = nn.Linear(64, 16) self.fc3 = nn.Linear(16, output_size) - self.activef=activef + self.activef = activef + def forward(self, x): - #x = torch.sigmoid(self.fc1(x)) - if self.activef=='ReLU': + # x = torch.sigmoid(self.fc1(x)) + if self.activef == "ReLU": x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) - if self.activef=='sigmoid': + if self.activef == "sigmoid": x = F.sigmoid(self.fc1(x)) x = F.sigmoid(self.fc2(x)) - if self.activef=='tanh': + if self.activef == "tanh": x = F.tanh(self.fc1(x)) x = F.tanh(self.fc2(x)) x = self.fc3(x) return x -#EWC -def EWC(fisher,params,net): + +# EWC +def EWC(fisher, params, net): params_n = list(net.parameters()) - EWC=0 - i=0 - p=params_n[0] - cost=(p-params[i])*fisher*(p-params[i]) - EWC=EWC+cost.sum() + EWC = 0 + i = 0 + p = params_n[0] + cost = (p - params[i]) * fisher * (p - params[i]) + EWC = EWC + cost.sum() return EWC -def sc_nn(ii,gene_chr,TFindex,TFindex_bulk,REindex,REindex_bulk,REindex_bulk_match,Target,netall,adj_matrix_all,Exp,TF_match,input_size_all,fisherall,Opn,l1_lambda,fisher_w,activef): +def sc_nn( + ii, + gene_chr, + TFindex, + TFindex_bulk, + REindex, + REindex_bulk, + REindex_bulk_match, + Target, + netall, + adj_matrix_all, + Exp, + TF_match, + input_size_all, + fisherall, + Opn, + l1_lambda, + fisher_w, + activef, +): warnings.filterwarnings("ignore") alpha = 1 - eps=1e-12 - alpha = torch.tensor(alpha,dtype=torch.float32) - gene_idx=gene_chr['id_s'].values[ii]-1 - gene_idx_b=int(gene_chr['id_b'].values[ii])-1 - TFidxtemp=TFindex[gene_idx] - TFidxtemp=TFidxtemp.split('_') - TFidxtemp=[int(TFidxtemp[k])+1 for k in range(len(TFidxtemp))] - TFidxtemp_b=TFindex_bulk[gene_idx_b] - TFidxtemp_b=TFidxtemp_b.split('_') - TFidxtemp_b=[int(TFidxtemp_b[k]) for k in range(len(TFidxtemp_b))] - TFtemp=Exp[np.array(TFidxtemp)-1,:] - REidxtemp=REindex[gene_idx] - REidxtemp_b_m=REindex_bulk_match[gene_idx] - REidxtemp_b=REindex_bulk[gene_idx_b] - REidxtemp=str(REidxtemp).split('_') - REidxtemp_b_m=str(REidxtemp_b_m).split('_') - REidxtemp_b=str(REidxtemp_b).split('_') - if (len(REidxtemp)==1)&(REidxtemp[0]=='nan'): - REidxtemp=[] - REidxtemp_b_m=[] - inputs=TFtemp+1-1 - L=np.zeros([len(TFidxtemp)+len(REidxtemp),len(TFidxtemp)+len(REidxtemp)]) - L=torch.tensor(L, dtype=torch.float32) + eps = 1e-12 + alpha = torch.tensor(alpha, dtype=torch.float32).to(device) + gene_idx = gene_chr["id_s"].values[ii] - 1 + gene_idx_b = int(gene_chr["id_b"].values[ii]) - 1 + TFidxtemp = TFindex[gene_idx] + TFidxtemp = TFidxtemp.split("_") + TFidxtemp = [int(TFidxtemp[k]) + 1 for k in range(len(TFidxtemp))] + TFidxtemp_b = TFindex_bulk[gene_idx_b] + TFidxtemp_b = TFidxtemp_b.split("_") + TFidxtemp_b = [int(TFidxtemp_b[k]) for k in range(len(TFidxtemp_b))] + TFtemp = Exp[np.array(TFidxtemp) - 1, :] + REidxtemp = REindex[gene_idx] + REidxtemp_b_m = REindex_bulk_match[gene_idx] + REidxtemp_b = REindex_bulk[gene_idx_b] + REidxtemp = str(REidxtemp).split("_") + REidxtemp_b_m = str(REidxtemp_b_m).split("_") + REidxtemp_b = str(REidxtemp_b).split("_") + if (len(REidxtemp) == 1) & (REidxtemp[0] == "nan"): + REidxtemp = [] + REidxtemp_b_m = [] + inputs = TFtemp + 1 - 1 + L = np.zeros([len(TFidxtemp) + len(REidxtemp), len(TFidxtemp) + len(REidxtemp)]) + L = torch.tensor(L, dtype=torch.float32).to(device) else: - REidxtemp=[int(REidxtemp[k])+1 for k in range(len(REidxtemp))] - REidxtemp_b_m=[int(REidxtemp_b_m[k])+1 for k in range(len(REidxtemp_b_m))] - REtemp=Opn[np.array(REidxtemp)-1,:] - inputs=np.vstack((TFtemp, REtemp)) - adj_matrix=np.zeros([len(TFidxtemp)+len(REidxtemp),len(TFidxtemp)+len(REidxtemp)]) - AA=adj_matrix_all[np.array(REidxtemp)-1,:] - AA=AA[:,np.array(TFidxtemp)-1] - adj_matrix[:len(TFidxtemp),-len(REidxtemp):]=AA.T - adj_matrix[-len(REidxtemp):,:len(TFidxtemp)]=AA + REidxtemp = [int(REidxtemp[k]) + 1 for k in range(len(REidxtemp))] + REidxtemp_b_m = [int(REidxtemp_b_m[k]) + 1 for k in range(len(REidxtemp_b_m))] + REtemp = Opn[np.array(REidxtemp) - 1, :] + inputs = np.vstack((TFtemp, REtemp)) + adj_matrix = np.zeros( + [len(TFidxtemp) + len(REidxtemp), len(TFidxtemp) + len(REidxtemp)] + ) + AA = adj_matrix_all[np.array(REidxtemp) - 1, :] + AA = AA[:, np.array(TFidxtemp) - 1] + adj_matrix[: len(TFidxtemp), -len(REidxtemp) :] = AA.T + adj_matrix[-len(REidxtemp) :, : len(TFidxtemp)] = AA A = torch.tensor(adj_matrix, dtype=torch.float32) D = torch.diag(A.sum(1)) degree = A.sum(dim=1) degree += eps D_sqrt_inv = 1 / degree.sqrt() D_sqrt_inv = torch.diag(D_sqrt_inv) - L = D_sqrt_inv@(D - A)@D_sqrt_inv - if (len(REidxtemp_b)==1)&(REidxtemp_b[0]=='nan'): - REidxtemp_b=[] + L = D_sqrt_inv @ (D - A) @ D_sqrt_inv + L = L.to(device) + if (len(REidxtemp_b) == 1) & (REidxtemp_b[0] == "nan"): + REidxtemp_b = [] else: - REidxtemp_b=[int(REidxtemp_b[k]) for k in range(len(REidxtemp_b))] - targets = torch.tensor(Target[gene_idx,:]) - inputs = torch.tensor(inputs,dtype=torch.float32) + REidxtemp_b = [int(REidxtemp_b[k]) for k in range(len(REidxtemp_b))] + targets = torch.tensor(Target[gene_idx, :]).to(device) + inputs = torch.tensor(inputs, dtype=torch.float32).to(device) targets = targets.type(torch.float32) mean = inputs.mean(dim=1) std = inputs.std(dim=1) - inputs = (inputs.T - mean) / (std+eps) - inputs=inputs.T - num_nodes=inputs.shape[0] - y=targets.reshape(len(targets),1) - #trainData testData - input_size=int(input_size_all[gene_idx_b]) - loaded_net = Net(input_size,activef) + inputs = (inputs.T - mean) / (std + eps) + inputs = inputs.T + num_nodes = inputs.shape[0] + y = targets.reshape(len(targets), 1) + # trainData testData + input_size = int(input_size_all[gene_idx_b]) + loaded_net = Net(input_size, activef).to(device) loaded_net.load_state_dict(netall[gene_idx_b]) params = list(loaded_net.parameters()) - fisher0=fisherall[gene_idx_b][0].data.clone() - data0=pd.DataFrame(TFidxtemp) - data1=pd.DataFrame(TFidxtemp_b) - data0.columns=['TF'] - data1.columns=['TF'] - A=TF_match.loc[data0['TF'].values-1]['id_b'] - data0=pd.DataFrame(A) - data0.columns=['TF'] - data1['id_b']=data1.index - data0['id_s']=range(0,len(A)) - merge_TF=pd.merge(data0,data1,how='left',on='TF') - if (len(REidxtemp)>0)&(len(REidxtemp_b)>0): - data0=pd.DataFrame(REidxtemp_b_m) - data1=pd.DataFrame(REidxtemp_b) - data0.columns=['RE'] - data1.columns=['RE'] - data0['id_s']=data0.index - data1['id_b']=data1.index - merge_RE=pd.merge(data0,data1,how='left',on='RE') - if merge_RE['id_b'].isna().sum()==0: - good=1 - indexall=merge_TF['id_b'].values.tolist()+(merge_RE['id_b'].values+merge_TF.shape[0]).tolist() - else: - good=0 + fisher0 = fisherall[gene_idx_b][0].data.clone().to(device) + data0 = pd.DataFrame(TFidxtemp) + data1 = pd.DataFrame(TFidxtemp_b) + data0.columns = ["TF"] + data1.columns = ["TF"] + A = TF_match.loc[data0["TF"].values - 1]["id_b"] + data0 = pd.DataFrame(A) + data0.columns = ["TF"] + data1["id_b"] = data1.index + data0["id_s"] = range(0, len(A)) + merge_TF = pd.merge(data0, data1, how="left", on="TF") + if (len(REidxtemp) > 0) & (len(REidxtemp_b) > 0): + data0 = pd.DataFrame(REidxtemp_b_m) + data1 = pd.DataFrame(REidxtemp_b) + data0.columns = ["RE"] + data1.columns = ["RE"] + data0["id_s"] = data0.index + data1["id_b"] = data1.index + merge_RE = pd.merge(data0, data1, how="left", on="RE") + if merge_RE["id_b"].isna().sum() == 0: + good = 1 + indexall = ( + merge_TF["id_b"].values.tolist() + + (merge_RE["id_b"].values + merge_TF.shape[0]).tolist() + ) + else: + good = 0 else: - indexall=merge_TF['id_b'].values.tolist() - good=1 - if good==1: - fisher=fisher0[:,np.array(indexall,dtype=int)] - params_bulk = params[0][:,np.array(indexall,dtype=int)] + indexall = merge_TF["id_b"].values.tolist() + good = 1 + if good == 1: + fisher = fisher0[:, np.array(indexall, dtype=int)] + params_bulk = params[0][:, np.array(indexall, dtype=int)] with torch.no_grad(): - params_bulk = params_bulk.detach() - num_nodes=inputs.shape[0] + params_bulk = params_bulk.detach() + num_nodes = inputs.shape[0] n_folds = 5 - kf = KFold(n_splits=n_folds,shuffle=True,random_state=0) + kf = KFold(n_splits=n_folds, shuffle=True, random_state=0) fold_size = len(inputs.T) // n_folds input_size = num_nodes mse_loss = nn.MSELoss() - y_pred_all=0*(y+1-1) - y_pred_all1=0*(y+1-1) - y_pred_all1=y_pred_all1.numpy().reshape(-1) + y_pred_all = 0 * (y + 1 - 1) + y_pred_all1 = 0 * (y + 1 - 1) + y_pred_all1 = y_pred_all1.cpu().numpy().reshape(-1) X_tr = inputs.T y_tr = y torch.manual_seed(seed_value) - net = Net(input_size,activef) - optimizer = Adam(net.parameters(),lr=0.01,weight_decay=l1_lambda) - #optimizer = Adam(net.parameters(),weight_decay=1) - # Perform backpropagation - Loss0=np.zeros([100,1]) + net = Net(input_size, activef).to(device) + optimizer = Adam(net.parameters(), lr=0.01, weight_decay=l1_lambda) + # optimizer = Adam(net.parameters(),weight_decay=1) + # Perform backpropagation + Loss0 = np.zeros([100, 1]) for i in range(100): # Perform forward pass y_pred = net(X_tr) # Calculate loss l1_norm = sum(torch.linalg.norm(p, 1) for p in net.parameters()) - #loss_EWC=EWC(fisher,params_bulk,net); - l2_bulk = -1* fisher_w* sum(sum(torch.mul(params_bulk,net.fc1.weight))) - lap_reg = alpha * torch.trace(torch.mm(torch.mm(net.fc1.weight, L), net.fc1.weight.t())) - loss = mse_loss(y_pred, y_tr) +l1_norm*l1_lambda+l2_bulk+lap_reg - Loss0[i,0]=loss.detach().numpy() + # loss_EWC=EWC(fisher,params_bulk,net); + l2_bulk = -1 * fisher_w * sum(sum(torch.mul(params_bulk, net.fc1.weight))) + lap_reg = alpha * torch.trace( + torch.mm(torch.mm(net.fc1.weight, L), net.fc1.weight.t()) + ) + loss = mse_loss(y_pred, y_tr) + l1_norm * l1_lambda + l2_bulk + lap_reg + Loss0[i, 0] = loss.detach().cpu().numpy() # Perform backpropagation optimizer.zero_grad() loss.backward() optimizer.step() np.random.seed(42) background = X_tr[np.random.choice(X_tr.shape[0], 50, replace=False)] - explainer = shap.DeepExplainer(net,background) + explainer = shap.DeepExplainer(net, background) shap_values = explainer.shap_values(X_tr) warnings.resetwarnings() - return net,shap_values,0.5,0.5,1,Loss0 + net = net.to("cpu") + return net, shap_values, 0.5, 0.5, 1, Loss0 else: warnings.resetwarnings() - return 0,0,0,0,0,0 - -def get_TSS(GRNdir,genome,TSS_dis): - #import pyensembl -# Initialize Ensembl database for the desired genome assembly - #ensembl = pyensembl.EnsemblRelease(release=release, species=species) # For hg19 -# ensembl = pyensembl.EnsemblRelease(release=104, species='mouse') # For mm10 -# Get all genes in the genome - #genes = ensembl.genes() -# Retrieve TSS positions for each gene and store them in a list - #tss_positions = [] - #strand=[] - #chrom=[] - #genesymbol=[] - #for gene in genes: - #tss_positions.append(gene.transcripts[0].start) - #strand.append(gene.strand) - #chrom.append('chr'+gene.contig) - #genesymbol.append(gene.name) + return 0, 0, 0, 0, 0, 0 + + +def get_TSS(GRNdir, genome, TSS_dis): + # import pyensembl + # Initialize Ensembl database for the desired genome assembly + # ensembl = pyensembl.EnsemblRelease(release=release, species=species) # For hg19 + # ensembl = pyensembl.EnsemblRelease(release=104, species='mouse') # For mm10 + # Get all genes in the genome + # genes = ensembl.genes() + # Retrieve TSS positions for each gene and store them in a list + # tss_positions = [] + # strand=[] + # chrom=[] + # genesymbol=[] + # for gene in genes: + # tss_positions.append(gene.transcripts[0].start) + # strand.append(gene.strand) + # chrom.append('chr'+gene.contig) + # genesymbol.append(gene.name) import pandas as pd - Tssdf = pd.read_csv(GRNdir+'TSS_'+genome+'.txt',sep='\t',header=None) - Tssdf.columns=['chr','TSS','symbol','strand'] - Tssdf['1M-']=Tssdf['TSS']-TSS_dis - Tssdf['1M+']=Tssdf['TSS']+TSS_dis - temp=Tssdf['1M-'].values - temp[temp<1]=1 - Tssdf['1M-']=temp - Tssdf=Tssdf[Tssdf['symbol']!=''] - Tssdf[['chr','1M-','1M+','symbol','TSS', 'strand']].to_csv('data/TSS_extend_1M.txt',sep='\t',index=None) - -def load_data(GRNdir,outdir): - gene_all=pd.DataFrame([]) + + Tssdf = pd.read_csv(GRNdir + "TSS_" + genome + ".txt", sep="\t", header=None) + Tssdf.columns = ["chr", "TSS", "symbol", "strand"] + Tssdf["1M-"] = Tssdf["TSS"] - TSS_dis + Tssdf["1M+"] = Tssdf["TSS"] + TSS_dis + temp = Tssdf["1M-"].values + temp[temp < 1] = 1 + Tssdf["1M-"] = temp + Tssdf = Tssdf[Tssdf["symbol"] != ""] + Tssdf[["chr", "1M-", "1M+", "symbol", "TSS", "strand"]].to_csv( + "data/TSS_extend_1M.txt", sep="\t", index=None + ) + + +def load_data(GRNdir, outdir): + gene_all = pd.DataFrame([]) for i in range(22): - chr='chr'+str(i+1) - gene_file=GRNdir+chr+'_gene.txt' - data0=pd.read_csv(gene_file,sep='\t',header=None) - data0['chr']=chr - data0['id_b']=data0.index+1 - gene_all=pd.concat([gene_all,data0]) - chr='chrX' - gene_file=GRNdir+chr+'_gene.txt' - data0=pd.read_csv(gene_file,sep='\t',header=None) - data0['chr']=chr - data0['id_b']=data0.index+1 - gene_all=pd.concat([gene_all,data0]) - gene_file=outdir+'Symbol.txt' - data0=pd.read_csv(gene_file,sep='\t',header=None) - data0.columns=['Symbol'] - data0['id_s']=data0.index+1 - gene_all.columns=['Symbol','chr','id_b'] - data_merge=pd.merge(data0,gene_all,how='left',on='Symbol') - TFName_b=pd.read_csv(GRNdir+'TFName.txt',header=None,sep='\t') - TFName_s=pd.read_csv(outdir+'TFName.txt',header=None,sep='\t') - TFName_b.columns=['TF'] - TFName_s.columns=['TF'] - TFName_b['id_b']=TFName_b.index+1# index from 1 - TFName_s['id_s']=TFName_s.index+1# index from 1 - TF_match=pd.merge(TFName_s,TFName_b,how='left',on='TF') - Opn_file=outdir+'Openness.txt' - idx_file=outdir+'index.txt' - geneexp_file=outdir+'Exp.txt' - Target=pd.read_csv(geneexp_file,header=None,sep='\t') - Target=Target.values - #def sc_NN(gene_file,Opn_file,idx_file,geneexp_file,out_PCC,out_net): - #alpha = torch.tensor(alpha,dtype=torch.float32) - bind_file=outdir+'TF_binding.txt' - adj_matrix_all=pd.read_csv(bind_file,header=None,sep='\t') - adj_matrix_all=adj_matrix_all.values - TFExp_file=outdir+'TFexp.txt' - Opn=pd.read_csv(Opn_file,header=None,sep='\t') - Opn=Opn.values - idx=pd.read_csv(idx_file,header=None,sep='\t') - Exp=pd.read_csv(TFExp_file,header=None,sep='\t') - Exp=Exp.values - return Exp,idx,Opn,adj_matrix_all,Target,data_merge,TF_match -def sc_nn_NN(ii,RE_TGlink_temp,Target,Exp,Opn,l1_lambda,activef): + chr = "chr" + str(i + 1) + gene_file = GRNdir + chr + "_gene.txt" + data0 = pd.read_csv(gene_file, sep="\t", header=None) + data0["chr"] = chr + data0["id_b"] = data0.index + 1 + gene_all = pd.concat([gene_all, data0]) + chr = "chrX" + gene_file = GRNdir + chr + "_gene.txt" + data0 = pd.read_csv(gene_file, sep="\t", header=None) + data0["chr"] = chr + data0["id_b"] = data0.index + 1 + gene_all = pd.concat([gene_all, data0]) + gene_file = outdir + "Symbol.txt" + data0 = pd.read_csv(gene_file, sep="\t", header=None) + data0.columns = ["Symbol"] + data0["id_s"] = data0.index + 1 + gene_all.columns = ["Symbol", "chr", "id_b"] + data_merge = pd.merge(data0, gene_all, how="left", on="Symbol") + TFName_b = pd.read_csv(GRNdir + "TFName.txt", header=None, sep="\t") + TFName_s = pd.read_csv(outdir + "TFName.txt", header=None, sep="\t") + TFName_b.columns = ["TF"] + TFName_s.columns = ["TF"] + TFName_b["id_b"] = TFName_b.index + 1 # index from 1 + TFName_s["id_s"] = TFName_s.index + 1 # index from 1 + TF_match = pd.merge(TFName_s, TFName_b, how="left", on="TF") + Opn_file = outdir + "Openness.txt" + idx_file = outdir + "index.txt" + geneexp_file = outdir + "Exp.txt" + Target = pd.read_csv(geneexp_file, header=None, sep="\t") + Target = Target.values + # def sc_NN(gene_file,Opn_file,idx_file,geneexp_file,out_PCC,out_net): + # alpha = torch.tensor(alpha,dtype=torch.float32) + bind_file = outdir + "TF_binding.txt" + adj_matrix_all = pd.read_csv(bind_file, header=None, sep="\t") + adj_matrix_all = adj_matrix_all.values + TFExp_file = outdir + "TFexp.txt" + Opn = pd.read_csv(Opn_file, header=None, sep="\t") + Opn = Opn.values + idx = pd.read_csv(idx_file, header=None, sep="\t") + Exp = pd.read_csv(TFExp_file, header=None, sep="\t") + Exp = Exp.values + return Exp, idx, Opn, adj_matrix_all, Target, data_merge, TF_match + + +def sc_nn_NN(ii, RE_TGlink_temp, Target, Exp, Opn, l1_lambda, activef): warnings.filterwarnings("ignore") alpha = 1 - eps=1e-12 - alpha = torch.tensor(alpha,dtype=torch.float32) + eps = 1e-12 + alpha = torch.tensor(alpha, dtype=torch.float32).to(device) if RE_TGlink_temp[0] in Exp.index: TFtemp = Exp.drop([RE_TGlink_temp[0]]).values else: - TFtemp=Exp.values - REtemp=Opn.loc[RE_TGlink_temp[1]].values - inputs=np.vstack((TFtemp, REtemp)) - targets = torch.tensor(Target.loc[RE_TGlink_temp[0],:]) - inputs = torch.tensor(inputs,dtype=torch.float32) - targets = targets.type(torch.float32) + TFtemp = Exp.values + REtemp = Opn.loc[RE_TGlink_temp[1]].values + inputs = np.vstack((TFtemp, REtemp)) + targets = torch.tensor(Target.loc[RE_TGlink_temp[0], :]).to(device) + inputs = torch.tensor(inputs, dtype=torch.float32).to(device) + targets = targets.type(torch.float32).to(device) mean = inputs.mean(dim=1) std = inputs.std(dim=1) - inputs = (inputs.T - mean) / (std+eps) - inputs=inputs.T - num_nodes=inputs.shape[0] - y=targets.reshape(len(targets),1) - #trainData testData - input_size=int(num_nodes) + inputs = (inputs.T - mean) / (std + eps) + inputs = inputs.T + num_nodes = inputs.shape[0] + y = targets.reshape(len(targets), 1) + # trainData testData + input_size = int(num_nodes) mse_loss = nn.MSELoss() - y_pred_all=0*(y+1-1) - y_pred_all1=0*(y+1-1) - y_pred_all1=y_pred_all1.numpy().reshape(-1) + y_pred_all = 0 * (y + 1 - 1) + y_pred_all1 = 0 * (y + 1 - 1) + y_pred_all1 = y_pred_all1.cpu().numpy().reshape(-1) X_tr = inputs.T y_tr = y torch.manual_seed(seed_value) - net = Net(input_size,activef) - optimizer = Adam(net.parameters(),lr=0.01,weight_decay=l1_lambda) - #optimizer = Adam(net.parameters(),weight_decay=1) - # Perform backpropagation - Loss0=np.zeros([100,1]) + net = Net(input_size, activef).to(device) + optimizer = Adam(net.parameters(), lr=0.01, weight_decay=l1_lambda) + # optimizer = Adam(net.parameters(),weight_decay=1) + # Perform backpropagation + Loss0 = np.zeros([100, 1]) for i in range(100): - # Perform forward pass + # Perform forward pass y_pred = net(X_tr) - # Calculate loss + # Calculate loss l1_norm = sum(torch.linalg.norm(p, 1) for p in net.parameters()) - #loss_EWC=EWC(fisher,params_bulk,net); - #l2_bulk = -1* fisher_w* sum(sum(torch.mul(params_bulk,net.fc1.weight))) - #lap_reg = alpha * torch.trace(torch.mm(torch.mm(net.fc1.weight, L), net.fc1.weight.t())) - loss = mse_loss(y_pred, y_tr) +l1_norm*l1_lambda#+l2_bulk+lap_reg - Loss0[i,0]=loss.detach().numpy() - # Perform backpropagation + # loss_EWC=EWC(fisher,params_bulk,net); + # l2_bulk = -1* fisher_w* sum(sum(torch.mul(params_bulk,net.fc1.weight))) + # lap_reg = alpha * torch.trace(torch.mm(torch.mm(net.fc1.weight, L), net.fc1.weight.t())) + loss = mse_loss(y_pred, y_tr) + l1_norm * l1_lambda # +l2_bulk+lap_reg + Loss0[i, 0] = loss.detach().cpu().numpy() + # Perform backpropagation optimizer.zero_grad() loss.backward() optimizer.step() np.random.seed(42) background = X_tr[np.random.choice(X_tr.shape[0], 50, replace=False)] - explainer = shap.DeepExplainer(net,background) + explainer = shap.DeepExplainer(net, background) shap_values = explainer.shap_values(X_tr) warnings.resetwarnings() - return net,shap_values,Loss0 - + net = net.to("cpu") + return net, shap_values, Loss0 -def load_data_scNN(GRNdir,species): + +def load_data_scNN(GRNdir, species): import pandas as pd - if species=='New': - Match2=pd.read_csv(GRNdir+'MotifMatch.txt',header=0,sep='\t') + + if species == "New": + Match2 = pd.read_csv(GRNdir + "MotifMatch.txt", header=0, sep="\t") else: - Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+species+'.txt',header=None,sep='\t') - Match2.columns = ['Motif','TF'] - TFName = pd.DataFrame(Match2['TF'].unique()) - Target=pd.read_csv('data/TG_pseudobulk.tsv',sep=',',header=0,index_col=0) - TFlist=list(set(Target.index)&set(TFName[0].values)) - Exp=Target.loc[TFlist] - Opn=pd.read_csv('data/RE_pseudobulk.tsv',sep=',',header=0,index_col=0) - RE_TGlink=pd.read_csv('data/RE_gene_distance.txt',sep='\t',header=0) - RE_TGlink = RE_TGlink.groupby('gene').apply(lambda x: x['RE'].values.tolist()).reset_index() - geneoverlap=list(set(Target.index)&set(RE_TGlink['gene'])) - RE_TGlink.index=RE_TGlink['gene'] - RE_TGlink=RE_TGlink.loc[geneoverlap] - RE_TGlink=RE_TGlink.reset_index(drop=True) - return Exp,Opn,Target,RE_TGlink + Match2 = pd.read_csv( + GRNdir + "Match_TF_motif_" + species + ".txt", header=None, sep="\t" + ) + Match2.columns = ["Motif", "TF"] + TFName = pd.DataFrame(Match2["TF"].unique()) + Target = pd.read_csv("data/TG_pseudobulk.tsv", sep=",", header=0, index_col=0) + TFlist = list(set(Target.index) & set(TFName[0].values)) + Exp = Target.loc[TFlist] + Opn = pd.read_csv("data/RE_pseudobulk.tsv", sep=",", header=0, index_col=0) + RE_TGlink = pd.read_csv("data/RE_gene_distance.txt", sep="\t", header=0) + RE_TGlink = ( + RE_TGlink.groupby("gene").apply(lambda x: x["RE"].values.tolist()).reset_index() + ) + geneoverlap = list(set(Target.index) & set(RE_TGlink["gene"])) + RE_TGlink.index = RE_TGlink["gene"] + RE_TGlink = RE_TGlink.loc[geneoverlap] + RE_TGlink = RE_TGlink.reset_index(drop=True) + return Exp, Opn, Target, RE_TGlink + def RE_TG_dis(outdir): + import numpy as np import pandas as pd import pybedtools - import numpy as np - print('Overlap the regions with gene loc ...') - import os# Create the directory + + print("Overlap the regions with gene loc ...") + import os # Create the directory + current_directory = os.getcwd() os.makedirs(outdir, exist_ok=True) import pandas as pd - peakList=pd.read_csv(current_directory+'/data/Peaks.txt',index_col=None,header=None) - peakList1=[temp.split(':')[0] for temp in peakList[0].values.tolist()] - peakList2=[temp.split(':')[1].split('-')[0] for temp in peakList[0].values.tolist()] - peakList3=[temp.split(':')[1].split('-')[1] for temp in peakList[0].values.tolist()] - peakList['chr']=peakList1 - peakList['start']=peakList2 - peakList['end']=peakList3 - peakList[['chr','start','end']].to_csv(current_directory+'/data/Peaks.bed',sep='\t',header=None,index=None) - TSS_1M=pd.read_csv(current_directory+'/data/TSS_extend_1M.txt',sep='\t',header=0) - TSS_1M.to_csv(current_directory+'/data/TSS_extend_1M.bed',sep='\t',header=None,index=None) - a = pybedtools.example_bedtool(current_directory+'/data/Peaks.bed') - b = pybedtools.example_bedtool(current_directory+'/data/TSS_extend_1M.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'temp.bed') - a_with_b=pd.read_csv(outdir+'temp.bed',sep='\t',header=None) - a_with_b['RE']=a_with_b[0].astype(str) + ':' + a_with_b[1].astype(str) + '-' + a_with_b[2].astype(str) - temp=a_with_b[['RE',6]] - temp.columns=[['RE','gene']] - temp['distance']=np.abs(a_with_b[7]-a_with_b[1]) - temp.to_csv(current_directory+'/data/RE_gene_distance.txt',sep='\t',index=None) - -from tqdm import tqdm -import warnings + + peakList = pd.read_csv( + current_directory + "/data/Peaks.txt", index_col=None, header=None + ) + peakList1 = [temp.split(":")[0] for temp in peakList[0].values.tolist()] + peakList2 = [ + temp.split(":")[1].split("-")[0] for temp in peakList[0].values.tolist() + ] + peakList3 = [ + temp.split(":")[1].split("-")[1] for temp in peakList[0].values.tolist() + ] + peakList["chr"] = peakList1 + peakList["start"] = peakList2 + peakList["end"] = peakList3 + peakList[["chr", "start", "end"]].to_csv( + current_directory + "/data/Peaks.bed", sep="\t", header=None, index=None + ) + TSS_1M = pd.read_csv( + current_directory + "/data/TSS_extend_1M.txt", sep="\t", header=0 + ) + TSS_1M.to_csv( + current_directory + "/data/TSS_extend_1M.bed", sep="\t", header=None, index=None + ) + a = pybedtools.example_bedtool(current_directory + "/data/Peaks.bed") + b = pybedtools.example_bedtool(current_directory + "/data/TSS_extend_1M.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "temp.bed") + a_with_b = pd.read_csv(outdir + "temp.bed", sep="\t", header=None) + a_with_b["RE"] = ( + a_with_b[0].astype(str) + + ":" + + a_with_b[1].astype(str) + + "-" + + a_with_b[2].astype(str) + ) + temp = a_with_b[["RE", 6]] + temp.columns = [["RE", "gene"]] + temp["distance"] = np.abs(a_with_b[7] - a_with_b[1]) + temp.to_csv(current_directory + "/data/RE_gene_distance.txt", sep="\t", index=None) + + import time -import pandas as pd +import warnings + import numpy as np -def training(GRNdir,method,outdir,activef,species): - if method=='LINGER': - hidden_size = 64 +import pandas as pd +from tqdm import tqdm + + +def training(GRNdir, method, outdir, activef, species): + if method == "LINGER": + hidden_size = 64 hidden_size2 = 16 output_size = 1 - l1_lambda = 0.01 - alpha_l = 0.01#elastic net parameter - lambda0 = 0.00 #bulk - fisher_w=0.1 - n_jobs=16 - - Exp,idx,Opn,adj_matrix_all,Target,data_merge,TF_match=load_data(GRNdir,outdir) - data_merge.to_csv(outdir+'data_merge.txt',sep='\t') - chrall=['chr'+str(i+1) for i in range(22)] - chrall.append('chrX') - import warnings + l1_lambda = 0.01 + alpha_l = 0.01 # elastic net parameter + lambda0 = 0.00 # bulk + fisher_w = 0.1 + n_jobs = 16 + + Exp, idx, Opn, adj_matrix_all, Target, data_merge, TF_match = load_data( + GRNdir, outdir + ) + data_merge.to_csv(outdir + "data_merge.txt", sep="\t") + chrall = ["chr" + str(i + 1) for i in range(22)] + chrall.append("chrX") import time + import warnings + from tqdm import tqdm + for i in range(23): - netall_s={} - shapall_s={} - result=np.zeros([data_merge.shape[0],2]) - Lossall=np.zeros([data_merge.shape[0],100]) - chr=chrall[i] - print(chr) - idx_file1=GRNdir+chr+'_index.txt' - idx_file_all=GRNdir+chr+'_index_all.txt' - idx_bulk=pd.read_csv(idx_file1,header=None,sep='\t') - idxRE_all=pd.read_csv(idx_file_all,header=None,sep='\t') - gene_chr=data_merge[data_merge['chr']==chr] - N=len(gene_chr) - TFindex=idx.values[:,2] - REindex=idx.values[:,1] - REindex_bulk_match=idx.values[:,3] - REindex_bulk=idxRE_all.values[:,0] - TFindex_bulk=idx_bulk.values[:,2] - input_size_all=idx_bulk.values[:,3] - fisherall = torch.load(GRNdir+'fisher_'+chr+'.pt') - netall=torch.load(GRNdir+'all_models_'+chr+'.pt') - + netall_s = {} + shapall_s = {} + result = np.zeros([data_merge.shape[0], 2]) + Lossall = np.zeros([data_merge.shape[0], 100]) + chr = chrall[i] + idx_file1 = GRNdir + chr + "_index.txt" + idx_file_all = GRNdir + chr + "_index_all.txt" + idx_bulk = pd.read_csv(idx_file1, header=None, sep="\t") + idxRE_all = pd.read_csv(idx_file_all, header=None, sep="\t") + gene_chr = data_merge[data_merge["chr"] == chr] + N = len(gene_chr) + TFindex = idx.values[:, 2] + REindex = idx.values[:, 1] + REindex_bulk_match = idx.values[:, 3] + REindex_bulk = idxRE_all.values[:, 0] + TFindex_bulk = idx_bulk.values[:, 2] + input_size_all = idx_bulk.values[:, 3] + fisherall = torch.load(GRNdir + "fisher_" + chr + ".pt") + netall = torch.load(GRNdir + "all_models_" + chr + ".pt") + for ii in tqdm(range(N)): warnings.filterwarnings("ignore") - res=sc_nn(ii,gene_chr,TFindex,TFindex_bulk,REindex,REindex_bulk,REindex_bulk_match,Target,netall,adj_matrix_all,Exp,TF_match,input_size_all,fisherall,Opn,l1_lambda,fisher_w,activef) + res = sc_nn( + ii, + gene_chr, + TFindex, + TFindex_bulk, + REindex, + REindex_bulk, + REindex_bulk_match, + Target, + netall, + adj_matrix_all, + Exp, + TF_match, + input_size_all, + fisherall, + Opn, + l1_lambda, + fisher_w, + activef, + ) warnings.resetwarnings() - index_all=gene_chr.index[ii] - if res[4]==1: - result[index_all,0]=res[2] - result[index_all,1]=res[3] - netall_s[index_all]=res[0] - shapall_s[index_all]=res[1] - Lossall[index_all,:]=res[5].T + index_all = gene_chr.index[ii] + if res[4] == 1: + result[index_all, 0] = res[2] + result[index_all, 1] = res[3] + netall_s[index_all] = res[0] + shapall_s[index_all] = res[1] + Lossall[index_all, :] = res[5].T else: - result[index_all,0]=-100 - result=pd.DataFrame(result) - result.index=data_merge['Symbol'].values - genetemp=data_merge[data_merge['chr']==chr]['Symbol'].values - result=result.loc[genetemp] - result.to_csv(outdir+'result_'+chr+'.txt',sep='\t') - torch.save(netall_s,outdir+'net_'+chr+'.pt') - torch.save(shapall_s,outdir+'shap_'+chr+'.pt') - Lossall=pd.DataFrame(Lossall) - Lossall.index=data_merge['Symbol'].values - Lossall=Lossall.loc[genetemp] - Lossall.to_csv(outdir+'Loss_'+chr+'.txt',sep='\t') - if method=='scNN': - hidden_size = 64 + result[index_all, 0] = -100 + result = pd.DataFrame(result) + result.index = data_merge["Symbol"].values + genetemp = data_merge[data_merge["chr"] == chr]["Symbol"].values + result = result.loc[genetemp] + result.to_csv(outdir + "result_" + chr + ".txt", sep="\t") + torch.save(netall_s, outdir + "net_" + chr + ".pt") + torch.save(shapall_s, outdir + "shap_" + chr + ".pt") + Lossall = pd.DataFrame(Lossall) + Lossall.index = data_merge["Symbol"].values + Lossall = Lossall.loc[genetemp] + Lossall.to_csv(outdir + "Loss_" + chr + ".txt", sep="\t") + if method == "scNN": + hidden_size = 64 hidden_size2 = 16 output_size = 1 - l1_lambda = 0.01 - alpha_l = 0.01#elastic net parameter - lambda0 = 0.00 #bulk - fisher_w=0.1 - n_jobs=16 - Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,species) - import warnings + l1_lambda = 0.01 + alpha_l = 0.01 # elastic net parameter + lambda0 = 0.00 # bulk + fisher_w = 0.1 + n_jobs = 16 + Exp, Opn, Target, RE_TGlink = load_data_scNN(GRNdir, species) import time + import warnings + from tqdm import tqdm - netall_s={} - shapall_s={} - #result=np.zeros([data_merge.shape[0],2]) - chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] - RE_TGlink['chr']=chrall - chrlist=RE_TGlink['chr'].unique() + + netall_s = {} + shapall_s = {} + # result=np.zeros([data_merge.shape[0],2]) + chrall = [RE_TGlink[0][i][0].split(":")[0] for i in range(RE_TGlink.shape[0])] + RE_TGlink["chr"] = chrall + chrlist = RE_TGlink["chr"].unique() for jj in tqdm(range(len(chrlist))): - chrtemp=chrlist[jj] - RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrtemp] - Lossall=np.zeros([RE_TGlink1.shape[0],100]) - for ii in range(RE_TGlink1.shape[0]): + chrtemp = chrlist[jj] + RE_TGlink1 = RE_TGlink[RE_TGlink["chr"] == chrtemp] + Lossall = np.zeros([RE_TGlink1.shape[0], 100]) + for ii in range(RE_TGlink1.shape[0]): warnings.filterwarnings("ignore") - #res = Parallel(n_jobs=n_jobs)(delayed(sc_nn_NN)(ii,RE_TGlink_temp,Target,netall,Exp,Opn,l1_lambda,activef) for ii in tqdm(range(RE_TGlink.shape[0])) - RE_TGlink_temp=RE_TGlink1.values[ii,:] - res=sc_nn_NN(ii,RE_TGlink_temp,Target,Exp,Opn,l1_lambda,activef) + # res = Parallel(n_jobs=n_jobs)(delayed(sc_nn_NN)(ii,RE_TGlink_temp,Target,netall,Exp,Opn,l1_lambda,activef) for ii in tqdm(range(RE_TGlink.shape[0])) + RE_TGlink_temp = RE_TGlink1.values[ii, :] + res = sc_nn_NN(ii, RE_TGlink_temp, Target, Exp, Opn, l1_lambda, activef) warnings.resetwarnings() - netall_s[ii]=res[0] - shapall_s[ii]=res[1] - Lossall[ii,:]=res[2].T - torch.save(netall_s,outdir+chrtemp+'_net.pt') - torch.save(shapall_s,outdir+chrtemp+'_shap.pt') - Lossall=pd.DataFrame(Lossall) - Lossall.index=RE_TGlink1['gene'].values - Lossall.to_csv(outdir+chrtemp+'_Loss.txt',sep='\t') - RE_TGlink.to_csv(outdir+'RE_TGlink.txt',sep='\t',index=None) - - -def get_TSS_ensembl(genome_short,gtf_file,GRNdir): - import pyensembl + netall_s[ii] = res[0] + shapall_s[ii] = res[1] + Lossall[ii, :] = res[2].T + torch.save(netall_s, outdir + chrtemp + "_net.pt") + torch.save(shapall_s, outdir + chrtemp + "_shap.pt") + Lossall = pd.DataFrame(Lossall) + Lossall.index = RE_TGlink1["gene"].values + Lossall.to_csv(outdir + chrtemp + "_Loss.txt", sep="\t") + RE_TGlink.to_csv(outdir + "RE_TGlink.txt", sep="\t", index=None) + + +def get_TSS_ensembl(genome_short, gtf_file, GRNdir): import subprocess + + import pyensembl from pyensembl import Genome + ensembl = Genome( - reference_name=genome_short, - annotation_name="My_annotation", - gtf_path_or_url=gtf_file) + reference_name=genome_short, + annotation_name="My_annotation", + gtf_path_or_url=gtf_file, + ) ensembl.index() genes = ensembl.genes() -# Retrieve TSS positions for each gene and store them in a list + # Retrieve TSS positions for each gene and store them in a list tss_positions = [] - strand=[] - chrom=[] - genesymbol=[] + strand = [] + chrom = [] + genesymbol = [] for gene in genes: tss_positions.append(gene.transcripts[0].start) strand.append(gene.strand) - chrom.append('chr'+gene.contig) + chrom.append("chr" + gene.contig) genesymbol.append(gene.name) import pandas as pd - Tssdf = pd.DataFrame({'chr': chrom, 'TSS': tss_positions, 'symbol': genesymbol,'strand': strand}) - Tssdf.to_csv(GRNdir+'TSS_'+genome_short+'.txt',sep='\t',index=None,header=0) \ No newline at end of file + + Tssdf = pd.DataFrame( + {"chr": chrom, "TSS": tss_positions, "symbol": genesymbol, "strand": strand} + ) + Tssdf.to_csv( + GRNdir + "TSS_" + genome_short + ".txt", sep="\t", index=None, header=0 + ) diff --git a/code/lingergrn-1.106/LingerGRN/LL_net.py b/code/lingergrn-1.106/LingerGRN/LL_net.py index fdfb092..ffe3e33 100644 --- a/code/lingergrn-1.106/LingerGRN/LL_net.py +++ b/code/lingergrn-1.106/LingerGRN/LL_net.py @@ -1,22 +1,23 @@ +import csv +import os + +#load data +import random + import numpy as np import pandas as pd -from scipy.sparse import coo_matrix -from scipy.sparse import csc_matrix -from tqdm import tqdm import torch -import csv import torch.nn as nn import torch.optim as optim -from torch.nn import functional as F -from scipy.stats import pearsonr -from scipy.stats import spearmanr -#load data -import random -from torch.optim import Adam -import os -from sklearn.linear_model import ElasticNet +from scipy.sparse import coo_matrix, csc_matrix +from scipy.stats import pearsonr, spearmanr from sklearn.datasets import make_regression +from sklearn.linear_model import ElasticNet from sklearn.model_selection import KFold +from torch.nn import functional as F +from torch.optim import Adam +from tqdm import tqdm + hidden_size = 64 hidden_size2 = 16 output_size = 1 @@ -28,7 +29,7 @@ def __init__(self,input_size,activef): self.fc1 = nn.Linear(input_size, 64) self.fc2 = nn.Linear(64, 16) self.fc3 = nn.Linear(16, output_size) - self.activef=activef + self.activef = activef def forward(self, x): #x = torch.sigmoid(self.fc1(x)) if self.activef=='ReLU': @@ -44,14 +45,24 @@ def forward(self, x): return x def cosine_similarity_0(X): - A=X.T/((X**2).sum(axis=1)**(1/2)+((X**2).sum(axis=1)**(1/2)).mean()/1000000) - return np.dot(A.T,A) + norm_term = np.linalg.norm(X, axis = 1) # calculate the norm term once rather than recalculate + eps = norm_term.mean() / 1e6 + A=X/(norm_term[:,np.newaxis] + eps) + return np.dot(A, A.T) + +# torch norm, should be slightly faster than np +def cosine_similarity_0_torch(X): + with torch.no_grad(): + norm_term = torch.norm(X, p=2, dim=1) + eps = norm_term.mean() / 1e6 + A = X / (norm_term.unsqueeze(-1) + eps) + return torch.mm(A, A.t()) + - def list2mat(df,i_n,j_n,x_n): TFs = df[j_n].unique() REs = df[i_n].unique() -#Initialize matrix as numpy array +#Initialize matrix as numpy array #Map row and col indices for lookup row_map = {r:i for i,r in enumerate(REs)} col_map = {c:i for i,c in enumerate(TFs)} @@ -65,14 +76,14 @@ def list2mat(df,i_n,j_n,x_n): def list2mat_s(df,REs,TFs,i_n,j_n,x_n): -#Initialize matrix as numpy array +#Initialize matrix as numpy array #Map row and col indices for lookup row_map = {r:i for i,r in enumerate(REs)} col_map = {c:i for i,c in enumerate(TFs)} row_indices = np.array([row_map[row] for row in df[i_n]]) col_indices = np.array([col_map[col] for col in df[j_n]]) - from scipy.sparse import coo_matrix import scipy.sparse as sp + from scipy.sparse import coo_matrix matrix = sp.csr_matrix((df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs))) return matrix,REs,TFs @@ -100,9 +111,9 @@ def merge_columns_in_bed_file2(file_path,startcol): return merged_values def format_RE_tran12(region): chr, range_ = region.split(":") - start, end = range_.split("-") + start, end = range_.split("-") return "_".join([chr, start, end]) -def get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName): +def get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName, torch_cosine = False): index_all=data_merge_temp[j] result={'TF':[],'RE':[],'score':[]} result = pd.DataFrame(result) @@ -111,23 +122,29 @@ def get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName): TFidxtemp=TFindex[index_all] TFidxtemp=TFidxtemp.split('_') TFidxtemp=[int(TFidxtemp[i]) for i in range(len(TFidxtemp))] - TFName_temp=TFName[np.array(TFidxtemp)] + TFName_temp=TFName[np.array(TFidxtemp)] REidxtemp=REindex[index_all] if REidxtemp=='': REidxtemp=[] else: REidxtemp=REidxtemp.split('_') - REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] #146 RE idx + REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] #146 RE idx if len(REidxtemp)>0: - corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + if not torch_cosine: + corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + else: + corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() REName_temp=REName[np.array(REidxtemp)] corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] + results = [] for k in range(len(REidxtemp)): datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) datatemp['TF']=TFName_temp.tolist() datatemp['RE']=REName_temp[k] - result=pd.concat([result,datatemp]) + results.append(datatemp) + result = pd.concat(results) return result + def load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN): TFbinding=pd.read_csv(GRNdir+'TF_binding_'+chrN+'.txt',sep='\t',index_col=0) TFbinding1=np.zeros([len(O_overlap_u),TFbinding.shape[1]]) @@ -150,7 +167,7 @@ def load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN): TFbinding=pd.DataFrame(TFbinding2,index=O_overlap,columns=TFbinding.columns) return TFbinding -def load_region(GRNdir,genome,chrN,outdir): +def load_region(GRNdir,genome,chrN,outdir): O_overlap=merge_columns_in_bed_file(outdir+'Region_overlap_'+chrN+'.bed',1) N_overlap=merge_columns_in_bed_file(outdir+'Region_overlap_'+chrN+'.bed',4) O_overlap_u=list(set(O_overlap)) @@ -169,7 +186,7 @@ def load_region(GRNdir,genome,chrN,outdir): O_overlap_hg19_u=hg19_region.index[idx].tolist() return O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u -def load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u): +def load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u): #print('load prior TF-RE for '+chrN+'...') mat=pd.read_csv(GRNdir+'Primary_TF_RE_'+chrN+'.txt',sep='\t',index_col=0) mat1=np.zeros([len(O_overlap_u),mat.shape[1]]) @@ -221,18 +238,22 @@ def TF_RE_LINGER_chr(chr,outdir): resultlist=[0 for i in range(times+1)] for ii in tqdm(range(times)): result_all=pd.DataFrame([]) + results = [] for j in range(ii*batchsize,(ii+1)*batchsize): if (AAA[j]>0)&(AAA[j]<10): result=get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName) - result_all=pd.concat([result_all,result],axis=0) + results.append(result) + result_all = pd.concat(results) result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() resultlist[ii]=result_all result_all=pd.DataFrame([]) ii=ii+1 + results = [] for j in range(ii*batchsize,N): if (AAA[j]>0)&(AAA[j]<10): result=get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName) - result_all=pd.concat([result_all,result],axis=0) + results.append(result) + result_all = pd.concat(results) if result_all.shape[0]>0: result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() resultlist[ii]=result_all @@ -275,14 +296,17 @@ def TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,outdir): mean_S = S.groupby(S.index).max() return mean_S import ast -def TF_RE_scNN(TFName,geneName,net_all,RE_TGlink,REName): + + +def TF_RE_scNN(TFName,geneName,net_all,RE_TGlink,REName, torch_cosine = False): batchsize=50 REName=pd.DataFrame(range(len(REName)),index=REName) N=RE_TGlink.shape[0] times=int(np.floor(N/batchsize)) resultlist=[0 for i in range(times+1)] for ii in range(times): - result_all=pd.DataFrame([]) + # result_all=pd.DataFrame([]) + results = [] for j in range(ii*batchsize,(ii+1)*batchsize): RE_TGlink_temp=RE_TGlink.values[j,:] temps=list(net_all[j].parameters())[0] @@ -291,22 +315,27 @@ def TF_RE_scNN(TFName,geneName,net_all,RE_TGlink,REName): TFidxtemp=np.array(range(len(TFName))) TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] if len(REidxtemp)>0: - corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + if not torch_cosine: + corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + else: + corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] - result={'TF':[],'RE':[],'score':[]} - result = pd.DataFrame(result) + # result={'TF':[],'RE':[],'score':[]} + # result = pd.DataFrame(result) + for k in range(len(REidxtemp)): datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) datatemp['TF']=TFName[TFidxtemp].tolist() datatemp['RE']=REidxtemp[k] - result=pd.concat([result,datatemp]) - result_all=pd.concat([result_all,result],axis=0) + results.append(datatemp) + result_all=pd.concat(results,axis=0) result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() #print(result_all) resultlist[ii]=result_all result_all=pd.DataFrame([]) ii=times if N>ii*batchsize: + results = [] for j in range(ii*batchsize,N): RE_TGlink_temp=RE_TGlink.values[j,:] temps=list(net_all[j].parameters())[0] @@ -315,29 +344,30 @@ def TF_RE_scNN(TFName,geneName,net_all,RE_TGlink,REName): TFidxtemp=np.array(range(len(TFName))) TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] if len(REidxtemp)>0: - corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + if not torch_cosine: + corr_matrix = cosine_similarity_0(temps.detach().numpy().T) + else: + corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] - result={'TF':[],'RE':[],'score':[]} - result = pd.DataFrame(result) for k in range(len(REidxtemp)): datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) datatemp['TF']=TFName[TFidxtemp].tolist() datatemp['RE']=REidxtemp[k] - result=pd.concat([result,datatemp]) - result_all=pd.concat([result_all,result],axis=0) + results.append(datatemp) + result_all = pd.concat(results) result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() #print(result_all) resultlist[ii]=result_all result_all=pd.concat(resultlist,axis=0) result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() return result_all - + def load_data_scNN(GRNdir,genome): import pandas as pd genome_map=pd.read_csv(GRNdir+'genome_map_homer.txt',sep='\t',header=0) - genome_map.index=genome_map['genome_short'].values + genome_map.index=genome_map['genome_short'].values if genome in genome_map.index: - Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+genome_map.loc[genome]['species_ensembl']+'.txt',sep='\t',header=0) + Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+genome_map.loc[genome]['species_ensembl']+'.txt',sep='\t',header=0) else: Match2=pd.read_csv(GRNdir+'MotifMatch.txt',sep='\t',header=0) TFName = pd.DataFrame(Match2['TF'].unique()) @@ -351,24 +381,25 @@ def load_data_scNN(GRNdir,genome): RE_TGlink.index=RE_TGlink['gene'] RE_TGlink=RE_TGlink.loc[geneoverlap] RE_TGlink=RE_TGlink.reset_index(drop=True) - return Exp,Opn,Target,RE_TGlink - - -def TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): - from tqdm import tqdm + return Exp,Opn,Target,RE_TGlink + + +def TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir, torch_cosine = True): import numpy as np import pandas as pd + from tqdm import tqdm print('Generating cellular population TF binding strength ...') chrom = ['chr'+str(i+1) for i in range(22)] chrom.append('chrX') - if method=='baseline': + results = [] + + if method =='baseline': result=pd.DataFrame() for i in tqdm(range(23)): chrN=chrom[i] out=TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,outdir) - out.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') - #result=pd.concat([result,out],axis=1).fillna(0) - result = pd.concat([result, out], join='outer', axis=0) + out.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') + results.append(out) if method=='LINGER': result=pd.DataFrame() for i in tqdm(range(23)): @@ -380,8 +411,9 @@ def TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): TG=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) TFoverlap = list(set(TFs) & set(TG.index)) mat = mat[TFoverlap] - mat.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') - result = pd.concat([result, mat], join='outer', axis=0) + mat.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') + results.append(mat) + if method=='scNN': Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) @@ -391,25 +423,26 @@ def TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): REName=Opn.index geneName=Target.index TFName=Exp.index - result_all=pd.DataFrame([]) for jj in tqdm(range(0,len(chrlist))): chrtemp=chrlist[jj] RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrtemp] net_all=torch.load(outdir+chrtemp+'_net.pt') - result=TF_RE_scNN(TFName,geneName,net_all,RE_TGlink1,REName) - result.to_csv(outdir+chrtemp+'_cell_population_TF_RE_binding.txt',sep='\t') - result_all=pd.concat([result_all,result],axis=0) - result=result_all.copy() - result.to_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t') - + result_scnn=TF_RE_scNN(TFName,geneName,net_all,RE_TGlink1,REName, torch_cosine=torch_cosine) + result_scnn.to_csv(outdir+chrtemp+'_cell_population_TF_RE_binding.txt',sep='\t') + results.append(result_scnn) + + # result=result_all.copy() + result = pd.concat(results, join="outer", axis=0) + result.to_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t') + def load_TFbinding_scNN(GRNdir,outdir,genome): - import pandas as pd import numpy as np + import pandas as pd genome_map=pd.read_csv(GRNdir+'genome_map_homer.txt',sep='\t',header=0) - genome_map.index=genome_map['genome_short'].values - A=pd.read_csv(outdir+'MotifTarget.bed',sep='\t',header=0,index_col=None) + genome_map.index=genome_map['genome_short'].values + A=pd.read_csv(outdir+'MotifTarget.bed',sep='\t',header=0,index_col=None) #Motif_binding,REs1,motifs=list2mat(A,'PositionID','Motif Name','MotifScore') - A['MotifScore']=np.log(1+A['MotifScore']); + A['MotifScore']=np.log(1+A['MotifScore']); if genome in genome_map.index: Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+genome_map.loc[genome]['species_ensembl']+'.txt',sep='\t',header=0) else: @@ -446,7 +479,7 @@ def cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome if len(other_RE)>0: B_arr = pd.DataFrame(np.zeros((len(other_RE), mat.shape[1])), columns=mat.columns, index=other_RE) mat = pd.concat([mat, B_arr]) - mat = mat.loc[N_overlap] + mat = mat.loc[N_overlap] if method=='baseline': mat=load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u) mat.index=N_overlap @@ -495,32 +528,35 @@ def cell_type_specific_TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,celltype label=adata_RNA.obs['label'].values.tolist() labelset=list(set(label)) if (celltype == 'all')&(method!='scNN'): + results = [] for label0 in labelset: - print('Generate cell type specitic TF binding potential for cell type '+ str(label0)+'...') + print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') result=pd.DataFrame() from tqdm import tqdm for i in tqdm(range(22)): chrN='chr'+str(i+1) mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,label0,outdir,method,mat) - #result=pd.concat([result,out],axis=1).fillna(0) - result = pd.concat([result, out], join='outer', axis=0) + + results.append(out) chrN='chrX' mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,label0,outdir,method,mat) - result = pd.concat([result, out], join='outer', axis=0).fillna(0) + results.append(out) + result = pd.concat(results, join='outer', axis=0).fillna(0) result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt', sep='\t') elif method!='scNN': result=pd.DataFrame() from tqdm import tqdm chrom=['chr'+str(i+1) for i in range(22)] chrom.append('chrX') + results = [] for i in tqdm(range(23)): chrN=chrom[i] mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,celltype,outdir,method,mat) - #result=pd.concat([result,out],axis=1).fillna(0) - result = pd.concat([result, out], join='outer', axis=0) + result = results.append(out) + result = pd.concat(results, axis = 1).fillna(0) result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(celltype)+'.txt', sep='\t') elif (celltype == 'all')&(method=='scNN'): A=pd.read_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t',header=0,index_col=0) @@ -541,7 +577,7 @@ def cell_type_specific_TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,celltype TFbinding1 = pd.DataFrame(TFbinding1,index=mat.index,columns=TFoverlap) TFbinding=TFbinding1.copy() for label0 in labelset: - print('Generate cell type specitic TF binding potential for cell type '+ str(label0)+'...') + print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') from tqdm import tqdm temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) @@ -568,7 +604,7 @@ def cell_type_specific_TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,celltype REidx=pd.DataFrame(range(mat.shape[0]),index=mat.index) TFbinding1[REidx.loc[TFbinding.index][0].values,:]=TFbinding.values TFbinding1 = pd.DataFrame(TFbinding1,index=mat.index,columns=TFoverlap) - print('Generate cell type specitic TF binding potential for cell type '+ str(label0)+'...') + print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T @@ -576,13 +612,14 @@ def cell_type_specific_TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,celltype RE=RE.loc[REs] result=cell_type_specific_TF_RE_binding_score_scNN(mat,TFbinding,RE,TG,TFoverlap) result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt', sep='\t') - + def load_shap(chr,outdir): - import torch - import pandas as pd - import numpy as np import csv + + import numpy as np + import pandas as pd + import torch #print('loading shapley value '+chr+' ...') shap_all=torch.load(outdir+"shap_"+chr+".pt") import pandas as pd @@ -631,7 +668,7 @@ def cis_shap(chr,outdir): if (REidxtemp[0]=='') : REidxtemp=[] else: - REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] + REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] if len(REidxtemp)>0: REName_temp=REName[np.array(REidxtemp)] for k in range(len(REidxtemp)): @@ -674,8 +711,8 @@ def trans_shap(chr,outdir): mat=pd.DataFrame(mat,index=TGs,columns=TFs) mat.fillna(0, inplace=True) return mat - -def load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap): + +def load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap): #print('load prior RE-TG ...') from scipy.sparse import coo_matrix primary_s=pd.read_csv(GRNdir+'Primary_RE_TG_'+chrN+'.txt',sep='\t') @@ -740,11 +777,11 @@ def load_RE_TG_distance(GRNdir,chrN,O_overlap_hg19_u,O_overlap_u,O_overlap,TGove return array -def load_RE_TG_scNN(outdir): +def load_RE_TG_scNN(outdir): #print('load prior RE-TG ...') - from scipy.sparse import coo_matrix - import pandas as pd import numpy as np + import pandas as pd + from scipy.sparse import coo_matrix dis=pd.read_csv('data/RE_gene_distance.txt',sep='\t',header=0) dis['distance']=np.exp(-(0.5+dis['distance']/25000)) REs=dis['RE'].unique() @@ -761,11 +798,10 @@ def load_RE_TG_scNN(outdir): distance,REs,TGs=list2mat_s(dis,REoverlap,TGoverlap,'RE','gene','distance') return distance,cisGRN,REoverlap,TGoverlap -def cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir): +def cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir): import numpy as np import pandas as pd - from scipy.sparse import csc_matrix - from scipy.sparse import coo_matrix + from scipy.sparse import coo_matrix, csc_matrix O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) sparse_S,TGset=load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap) RE=pd.DataFrame(adata_ATAC.X.toarray().T,index=adata_ATAC.var['gene_ids'].values,columns=adata_ATAC.obs['barcode'].values) @@ -788,10 +824,10 @@ def cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir): Score=np.multiply(sparse_S.values,sparse_dis.values) Score=pd.DataFrame(Score,index=N_overlap,columns=TGoverlap) Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] - rows, cols = np.nonzero(Score.values) + data = Score.values[Score.values!=0] + rows, cols = np.nonzero(Score.values) coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) + combined = np.zeros([len(data),3], dtype=object) combined[:,0]=Score.index[coo.row] combined[:,1]=np.array(TGoverlap)[coo.col] combined[:,2]=coo.data @@ -828,23 +864,25 @@ def cis_shap_scNN(chrtemp,outdir,RE_TGlink1,REName,TFName): return RE_TG -def cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): +def cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): from tqdm import tqdm chrom=['chr'+str(i+1) for i in range(22)] chrom.append('chrX') + results = [] + # every if statement is independent since method can only be one value if method=='baseline': - result=pd.DataFrame([]) + # result=pd.DataFrame([]) for i in tqdm(range(23)): chrN=chrom[i] temp=cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir) temp.columns=['RE','TG','Score'] - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) if method=='LINGER': - result=pd.DataFrame([]) + # result=pd.DataFrame([]) for i in tqdm(range(23)): chrN=chrom[i] temp=cis_shap(chrN,outdir) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) if method=='scNN': Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) @@ -859,15 +897,15 @@ def cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): chrN=chrlist[i] RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrN] temp=cis_shap_scNN(chrN,outdir,RE_TGlink1,REName,TFName) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) + result = pd.concat(results, axis=0, join="outer") result.to_csv(outdir+'cell_population_cis_regulatory.txt',sep='\t',header=None,index=None) -def cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir): +def cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir): import numpy as np import pandas as pd - from scipy.sparse import csc_matrix - from scipy.sparse import coo_matrix + from scipy.sparse import coo_matrix, csc_matrix O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) sparse_S,TGset=load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap) label=adata_RNA.obs['label'].values.tolist() @@ -895,20 +933,19 @@ def cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,cellt Score=csc_matrix(RE_temp).multiply(sparse_S.values).multiply(sparse_dis.values).multiply(csc_matrix(TG_temp.T)).toarray() Score=pd.DataFrame(Score,index=N_overlap,columns=TGoverlap) Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] - rows, cols = np.nonzero(Score.values) + data = Score.values[Score.values!=0] + rows, cols = np.nonzero(Score.values) coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) + combined = np.zeros([len(data),3], dtype=object) combined[:,0]=Score.index[coo.row] combined[:,1]=np.array(TGoverlap)[coo.col] combined[:,2]=coo.data resultall=pd.DataFrame(combined) - return resultall + return resultall def cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs): import numpy as np import pandas as pd - from scipy.sparse import csr_matrix - from scipy.sparse import coo_matrix + from scipy.sparse import coo_matrix, csr_matrix RE=RE.loc[REs] ## select the genes #target_col_indices = [col_dict[col] for col in TGoverlap] @@ -923,16 +960,16 @@ def cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs): row_indices=np.array(REs)[row_indices] col_indices = np.array(TGs)[col_indices] values = Score.data - combined = np.zeros([len(row_indices),3], dtype=object) + combined = np.zeros([len(row_indices),3], dtype=object) combined[:,0]=row_indices combined[:,1]=col_indices combined[:,2]=values resultall=pd.DataFrame(combined) - return resultall + return resultall -def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdir,method): - import pandas as pd +def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdir,method): import numpy as np + import pandas as pd label=adata_RNA.obs['label'].values.tolist() labelset=list(set(label)) chrom=['chr'+str(i+1) for i in range(22)] @@ -942,23 +979,27 @@ def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdi for label0 in labelset: label0=str(label0) result=pd.DataFrame([]) + results = [] for i in tqdm(range(23)): chrN=chrom[i] temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,label0,outdir) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) chrN='chrX' temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,label0,outdir) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) + result=pd.concat(results,axis=0,join='outer') result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+str(label0)+'.txt',sep='\t',header=None,index=None) elif (method!='scNN'): result=pd.DataFrame([]) + results = [] for i in tqdm(range(23)): chrN=chrom[i] temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) chrN='chrX' temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir) - result=pd.concat([result,temp],axis=0,join='outer') + results.append(temp) + result=pd.concat(results,axis=0,join='outer') result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+celltype+'.txt',sep='\t',header=None,index=None) elif (celltype=='all')&(method=='scNN'): distance,cisGRN,REs,TGs=load_RE_TG_scNN(outdir) @@ -969,10 +1010,10 @@ def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdi temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) del temp - + result=cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs) result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+label0+'.txt',sep='\t',header=None,index=None) - else: + else: label0=celltype label0=str(label0) temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T @@ -982,7 +1023,7 @@ def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdi del temp result=cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs) result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+label0+'.txt',sep='\t',header=None,index=None) - + def trans_shap_scNN(chrtemp,outdir,RE_TGlink1,REName,TFName): import ast TG_1=[] @@ -1017,9 +1058,9 @@ def trans_shap_scNN(chrtemp,outdir,RE_TGlink1,REName,TFName): def load_cis(Binding,celltype,outdir): - from scipy.sparse import coo_matrix - import pandas as pd import numpy as np + import pandas as pd + from scipy.sparse import coo_matrix if celltype=='': cis=pd.read_csv(outdir+'cell_population_cis_regulatory.txt',sep='\t',header=None) else: @@ -1051,7 +1092,7 @@ def load_TF_TG( GRNdir, TFset,TGset): a=list(range(1,23)) a.append('X') for i in a: - chrN='chr'+str(i) + chrN='chr'+str(i) TF_TG = pd.read_csv(GRNdir+'Primary_TF_TG_'+chrN+'.txt',sep='\t') TF_TG = TF_TG[TF_TG['TF'].isin(TFset)] TF_TG = TF_TG[TF_TG['TG'].isin(TGset)] @@ -1074,13 +1115,11 @@ def load_TF_TG( GRNdir, TFset,TGset): def trans_reg(GRNdir,method,outdir,genome): import ast - import pandas as pd - from scipy.sparse import coo_matrix + import numpy as np import pandas as pd - from scipy.sparse import csc_matrix - from scipy.sparse import coo_matrix - print('Generate trans-regulatory netowrk ...') + from scipy.sparse import coo_matrix, csc_matrix + print('Generating trans-regulatory netowrk ...') if method=='baseline': Binding=pd.read_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t',index_col=0) cis=load_cis(Binding,'',outdir) @@ -1093,10 +1132,13 @@ def trans_reg(GRNdir,method,outdir,genome): chrom=['chr'+str(i+1) for i in range(22)] chrom.append('chrX') S=pd.DataFrame([]) + results = [] for i in tqdm(range(23)): chrN=chrom[i] temp=trans_shap(chrN,outdir) - S=pd.concat([S,temp],axis=0,join='outer') + results.append(temp) + + S = pd.concat(results, axis = 0, join="outer") elif method=='scNN': Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) @@ -1106,21 +1148,22 @@ def trans_reg(GRNdir,method,outdir,genome): REName=Opn.index geneName=Target.index TFName=Exp.index - result=pd.DataFrame([]) S=pd.DataFrame([]) + results = [] for i in tqdm(range(len(chrlist))): chrN=chrlist[i] RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrN] temp=trans_shap_scNN(chrN,outdir,RE_TGlink1,REName,TFName) - S=pd.concat([S,temp],axis=0,join='outer') - print('Save trans-regulatory netowrk ...') + results.append(temp) + + S=pd.concat(results,axis=0,join='outer') + print('Saving trans-regulatory netowrk ...') S.to_csv(outdir+'cell_population_trans_regulatory.txt',sep='\t') def cell_type_specific_trans_reg(GRNdir,adata_RNA,celltype,outdir): - import pandas as pd import numpy as np - from scipy.sparse import csc_matrix - from scipy.sparse import coo_matrix + import pandas as pd + from scipy.sparse import coo_matrix, csc_matrix label=adata_RNA.obs['label'].values.tolist() labelset=list(set(label)) if celltype=='all': diff --git a/code/lingergrn-1.106/LingerGRN/TF_activity.py b/code/lingergrn-1.106/LingerGRN/TF_activity.py index 60f9d1f..4f9e0d3 100644 --- a/code/lingergrn-1.106/LingerGRN/TF_activity.py +++ b/code/lingergrn-1.106/LingerGRN/TF_activity.py @@ -1,12 +1,36 @@ -def quantile_normalize(df): - rank_mean = df.stack().groupby(df.rank(method='first').stack().astype(int)).mean() - return df.rank(method='min').stack().astype(int).map(rank_mean).unstack() -def bulk_reg(outdir,GRNdir,genome,chrN): - from LingerGRN.LL_net import load_region - from LingerGRN.LL_net import load_TFbinding - from LingerGRN.LL_net import load_TF_RE - from LingerGRN.LL_net import load_RE_TG - from LingerGRN.LL_net import load_RE_TG_distance +# def quantile_normalize(df): +# rank_mean = df.stack().groupby(df.rank(method='first').stack().astype(int)).mean() +# return df.rank(method='min').stack().astype(int).map(rank_mean).unstack() +# +from scipy.stats import rankdata + + +# approx 4x faster to use np and scipy over pandas methods +# tie_break: how ties betw values are broken for finding the means, if true then ties are assigned unique vals based on appearance +# order. This is the same as the original function. False: ties are arbitrarily assigned values (faster). +def quantile_normalize(df, tie_break: bool = True): + arr = df.values + if tie_break: + sort_order = rankdata(arr, method='ordinal', axis=0).astype(int) - 1 + sorted_arr = arr[sort_order, np.arange(arr.shape[1])] + else: + sorted_arr = np.sort(arr, axis = 0) + means = np.mean(sorted_arr, axis = -1) + # mask = np.argsort(np.argsort(arr, axis = 0), axis = 0) + mask = (rankdata(arr, method="min", axis = 0).astype(int)-1) + + return pd.DataFrame(means[mask], columns = df.columns, index = df.index) + + + +def bulk_reg(outdir,GRNdir,genome,chrN): + from LingerGRN.LL_net import ( + load_RE_TG, + load_RE_TG_distance, + load_region, + load_TF_RE, + load_TFbinding, + ) from scipy.sparse import coo_matrix O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(outdir,GRNdir,genome,chrN) TFbinding=load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN) @@ -27,10 +51,10 @@ def bulk_reg(outdir,GRNdir,genome,chrN): Score=sparse_S.multiply(sparse_dis.values) Score=pd.DataFrame(Score.values,index=N_overlap,columns=TGset) Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] - rows, cols = np.nonzero(Score.values) + data = Score.values[Score.values!=0] + rows, cols = np.nonzero(Score.values) coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) + combined = np.zeros([len(data),3], dtype=object) combined[:,0]=Score.index[coo.row] combined[:,1]=np.array(TGset)[coo.col] combined[:,2]=coo.data @@ -56,26 +80,33 @@ def TF_RE2m(result_RE_TG,REset): cis=sparse_S.toarray() cis=pd.DataFrame(cis,index=REset,columns=TGset) return cis -import scipy.io as sio import numpy as np import pandas as pd +import scipy.io as sio + + def regulon(outdir,adata_RNA,GRNdir,network,genome): # Load data from MATLAB .mat files if network=='cell population': trans_reg = pd.read_csv(outdir+'cell_population_trans_regulatory.txt',sep='\t',index_col=0) # Apply quantile normalization to 'trans_reg_n' - elif network=='general': + elif network=='general': from tqdm import tqdm chrom=['chr'+str(i+1) for i in range(22)] chrom.append('chrX') result_TF_RE=pd.DataFrame([]) result_RE_TG=pd.DataFrame([]) + tf_res = [] + re_tgs = [] for i in tqdm(range(23)): chrN=chrom[i] TF_RE,RE_TG=bulk_reg(outdir,GRNdir,genome,chrN) RE_TG.columns=['RE','TG','Score'] - result_RE_TG=pd.concat([result_RE_TG,RE_TG],axis=0,join='outer') - result_TF_RE=pd.concat([result_TF_RE,TF_RE],axis=0,join='outer') + tf_res.append(TF_RE) + re_tgs.append(RE_TG) + result_RE_TG=pd.concat(re_tgs,axis=0,join='outer') + result_TF_RE=pd.concat(tf_res,axis=0,join='outer') + TFset=result_TF_RE.columns REset=result_TF_RE.index cis=TF_RE2m(result_RE_TG,REset) @@ -84,24 +115,25 @@ def regulon(outdir,adata_RNA,GRNdir,network,genome): TF_TG=load_TF_TG(GRNdir, TFset,TGset) trans_reg=np.matmul(result_TF_RE.values.T, cis.values).T*(TF_TG.values) trans_reg=pd.DataFrame(trans_reg, index=TGset,columns=TFset) - else: + else: trans_reg = pd.read_csv(outdir+'cell_type_specific_trans_regulatory_'+network+'.txt',sep='\t',index_col=0) #RNA=pd.read_csv(outdir+RNA_file,sep='\t',index_col=0) RNA=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) - gene_overlap=list(set(trans_reg.index)&set(RNA.index)) + gene_overlap=list(set(trans_reg.index)&set(RNA.index)) RNA=RNA.loc[gene_overlap] trans_reg=trans_reg.loc[gene_overlap] row=trans_reg.sum(axis=1).values[:,np.newaxis]+trans_reg.sum(axis=1).mean()*0.000001 colsum=trans_reg.sum(axis=0).values[:,np.newaxis]+trans_reg.sum(axis=0).mean()*0.000001 - data_norm=trans_reg.values/(row*colsum.T)*row.sum() + # data_norm=trans_reg.values/(row*colsum.T)*row.sum() E=(row*colsum.T)/row.sum() - - #trans_reg=trans_reg/trans_reg.sum(axis=0) + + #trans_reg=trans_reg/trans_reg.sum(axis=0) #trans_reg_norm = quantile_normalize(trans_reg.T) RNA=RNA/RNA.sum(axis=0) #neccessary RNA_norm=quantile_normalize(RNA) #RNA_norm=RNA - regulon=(np.dot(trans_reg.values.T,RNA_norm.values)-np.dot(E.T,RNA_norm.values))/np.dot(E.T,RNA_norm.values) + e_rna = np.dot(E.T,RNA_norm.values) + regulon=(np.dot(trans_reg.values.T,RNA_norm.values)-e_rna)/ e_rna regulon=pd.DataFrame(regulon,index=trans_reg.columns,columns=RNA_norm.columns) return regulon def master_regulator(regulon_score,adata_RNA,celltype): @@ -122,19 +154,17 @@ def master_regulator(regulon_score,adata_RNA,celltype): # Initialize an empty DataFrame to store the t-test results t_test_results = np.zeros((regulon_score.shape[0],2)) # Iterate over the columns/variables in X and Y - for i in range(X.shape[0]): - row= regulon_score.index[i] - # Perform the t-test between X and Y for the current variable - t_stat, p_value = stats.ttest_ind(X.loc[row], Y.loc[row],alternative='greater') - # Append the results to the DataFrame - t_test_results[i,0] = t_stat - t_test_results[i,1] = p_value - t_test_results[np.isnan(t_test_results)]=1 - t_test_results=pd.DataFrame(t_test_results,index=regulon_score.index,columns=['t_stat','p_value']) - t_test_results['adj_p']= smm.multipletests(t_test_results['p_value'], method='fdr_bh')[1] + t_stat, p_value = stats.ttest_ind(X, Y, axis=1, alternative="greater") + t_test_results = pd.DataFrame( + {"t_stat": t_stat, "p_value": p_value}, + index=regulon_score.index + ) + t_test_results.fillna({"t_stat": 0, "p_value": 1}, inplace=True) + t_test_results["adj_p"] = smm.multipletests(t_test_results["p_value"], method="fdr_bh")[1] elif celltype=='all': label_set=np.array(list(set(label['celltype'].values))) t_test_results = np.zeros((regulon_score.shape[0],3*len(label_set))) + res = [] for j in range(len(label_set)): idx=regulon_score.columns[np.isin(label['celltype'],label_set[j])] X=regulon_score[idx] @@ -142,16 +172,26 @@ def master_regulator(regulon_score,adata_RNA,celltype): Y=regulon_score[idx] # Initialize an empty DataFrame to store the t-test results # Iterate over the columns/variables in X and Y - for i in range(X.shape[0]): - row= regulon_score.index[i] - # Perform the t-test between X and Y for the current variable - t_stat, p_value = stats.ttest_ind(X.loc[row], Y.loc[row],alternative='greater') - # Append the results to the DataFrame - p_value = np.nan_to_num(p_value, nan=1) - t_test_results[i,3*j+1] = p_value - t_test_results[i,3*j] = t_stat - t_test_results[:,3*j+2]=smm.multipletests(t_test_results[:,3*j+1], method='fdr_bh')[1] - t_test_results=pd.DataFrame(t_test_results,index=regulon_score.index) + t_stat, p_value = stats.ttest_ind(X, Y,alternative='greater', axis=1) + t_test_results = pd.DataFrame( + {"t_stat": t_stat, "p_value": p_value}, + index=regulon_score.index + ) + t_test_results.fillna({"t_stat": 0, "p_value": 1}, inplace=True) + t_test_results["adj_p"] = smm.multipletests(t_test_results["p_value"], method="fdr_bh")[1] + res.append(t_test_results) + # for i in range(X.shape[0]): + # row= regulon_score.index[i] + # # Perform the t-test between X and Y for the current variable + # t_stat, p_value = stats.ttest_ind(X.loc[row], Y.loc[row],alternative='greater') + # # Append the results to the DataFrame + # p_value = np.nan_to_num(p_value, nan=1) + # t_test_results[i,3*j+1] = p_value + # t_test_results[i,3*j] = t_stat + # t_test_results[:,3*j+2]=smm.multipletests(t_test_results[:,3*j+1], method='fdr_bh')[1] + t_test_results = pd.concat(res, axis = 1) + # t_test_results=pd.DataFrame(t_test_results,index=regulon_score.index) + col=[0 for kk in range(len(label_set)*3)] for j in range(len(label_set)): col[3*j]=label_set[j]+'_t_stat' @@ -160,9 +200,9 @@ def master_regulator(regulon_score,adata_RNA,celltype): t_test_results.columns=col return t_test_results def heatmap_cluster(regulon_score,adata_RNA,save,outdir): - import seaborn as sns import matplotlib.pyplot as plt import numpy as np + import seaborn as sns from scipy.stats import zscore # Generate random data for the heatmap Vars=regulon_score.var(axis=1) @@ -187,7 +227,7 @@ def heatmap_cluster(regulon_score,adata_RNA,save,outdir): plt.savefig(outdir+"heatmap_activity.png", format='png', bbox_inches='tight') # Finally, display the plot plt.show() - + def box_comp(TFName,adata_RNA,celltype1,celltype2,datatype,regulon_score,save,outdir): import numpy as np data=np.zeros(regulon_score.shape[1]) @@ -208,9 +248,9 @@ def box_comp(TFName,adata_RNA,celltype1,celltype2,datatype,regulon_score,save,ou else: G1=TFexp[np.array(label)==celltype1] G2=TFexp[np.array(label)==celltype2] - import seaborn as sns import matplotlib.pyplot as plt import numpy as np + import seaborn as sns # Combine the vectors into a single list data = [G1, G2] # Set up the figure size and style @@ -230,4 +270,4 @@ def box_comp(TFName,adata_RNA,celltype1,celltype2,datatype,regulon_score,save,ou if save==True: plt.savefig(outdir+"box_plot_"+TFName+'_'+datatype+'_'+celltype1+'_'+celltype2+".png", format='png', bbox_inches='tight') # Finally, display the plot - plt.show() \ No newline at end of file + plt.show() From dcd11bbbc43b3b12a789fa4d7da9e5dd7839f46e Mon Sep 17 00:00:00 2001 From: Arnav G <73305591+arnavg115@users.noreply.github.com> Date: Sun, 8 Mar 2026 12:27:03 -0700 Subject: [PATCH 2/4] formatting and isort --- code/lingergrn-1.106/LingerGRN/Benchmk.py | 134 +- code/lingergrn-1.106/LingerGRN/Compare.py | 481 ++-- code/lingergrn-1.106/LingerGRN/LINGER_tr.py | 25 +- code/lingergrn-1.106/LingerGRN/LL_net.py | 1952 ++++++++++------- code/lingergrn-1.106/LingerGRN/LingerGRN.py | 2 +- code/lingergrn-1.106/LingerGRN/TF_activity.py | 396 ++-- code/lingergrn-1.106/LingerGRN/perturb.py | 405 ++-- code/lingergrn-1.106/LingerGRN/preprocess.py | 647 +++--- code/lingergrn-1.106/LingerGRN/pseudo_bulk.py | 119 +- code/lingergrn-1.106/setup.py | 33 +- 10 files changed, 2428 insertions(+), 1766 deletions(-) diff --git a/code/lingergrn-1.106/LingerGRN/Benchmk.py b/code/lingergrn-1.106/LingerGRN/Benchmk.py index d6ad849..3fcded6 100644 --- a/code/lingergrn-1.106/LingerGRN/Benchmk.py +++ b/code/lingergrn-1.106/LingerGRN/Benchmk.py @@ -1,12 +1,14 @@ -import seaborn as sns import matplotlib.colors as mcolors +import seaborn as sns + + def generate_colors(N): """ Generate N visually appealing colors using seaborn color palette. - + Args: N (int): The number of colors to generate. - + Returns: list: A list of N RGB tuples representing the generated colors. """ @@ -14,85 +16,99 @@ def generate_colors(N): colors = [mcolors.rgb2hex(color_palette[i]) for i in range(N)] return colors -def bm_trans(TFName,Method_name,Groundtruth,Infer_trans,outdir,filetype): - import pandas as pd - import numpy as np + +def bm_trans(TFName, Method_name, Groundtruth, Infer_trans, outdir, filetype): import matplotlib.pyplot as plt - from sklearn.metrics import roc_curve, roc_auc_score + import numpy as np + import pandas as pd import seaborn as sns -# Set the working directory -# Use the appropriate path for your system + from sklearn.metrics import roc_auc_score, roc_curve + + # Set the working directory + # Use the appropriate path for your system import os - data0=pd.read_csv(Groundtruth,sep='\t', skiprows=5,header=0) - data1=data0.groupby(['symbol'])['score'].max() - label=data1.sort_values(axis=0, ascending=False) - label=label.reset_index() + + data0 = pd.read_csv(Groundtruth, sep="\t", skiprows=5, header=0) + data1 = data0.groupby(["symbol"])["score"].max() + label = data1.sort_values(axis=0, ascending=False) + label = label.reset_index() N = 1000 # top 500 TG as the ground truth - label = label['symbol'].iloc[:N] + label = label["symbol"].iloc[:N] # Load data - colors=generate_colors(len(Infer_trans)) + colors = generate_colors(len(Infer_trans)) for i in range(len(Infer_trans)): - if filetype=='list': - data2 = pd.read_csv(Infer_trans[i], sep='\t') - #label = pd.read_csv('~/SC_NET/all_data/result/LL_sc/max/' + info0['id'][i] + '_gene_score_5fold.txt', header=True) - loc = data2[data2['TF'] == TFName].index - TGName = data2['TG'].values[loc].tolist() + if filetype == "list": + data2 = pd.read_csv(Infer_trans[i], sep="\t") + # label = pd.read_csv('~/SC_NET/all_data/result/LL_sc/max/' + info0['id'][i] + '_gene_score_5fold.txt', header=True) + loc = data2[data2["TF"] == TFName].index + TGName = data2["TG"].values[loc].tolist() TGset = TGName.copy() - Score = data2['score'].values[loc] - if filetype=='matrix': - data2=pd.read_csv(Infer_trans[i], sep='\t',header=0,index_col=0) - TGset=data2.index - Score=data2[TFName].values + Score = data2["score"].values[loc] + if filetype == "matrix": + data2 = pd.read_csv(Infer_trans[i], sep="\t", header=0, index_col=0) + TGset = data2.index + Score = data2[TFName].values d1 = np.zeros(len(TGset)) loc = np.where(np.isin(TGset, label))[0] d1[loc] = 1 fpr, tpr, thresholds = roc_curve(d1, Score) -# Compute AUC + # Compute AUC auc = roc_auc_score(d1, Score) - # Plot the ROC curve - plt.plot(fpr, tpr, color=colors[i], label=Method_name[i]+': (AUC = %0.2f)' % auc) - plt.plot([0, 1], [0, 1], color='black', linestyle='--') + # Plot the ROC curve + plt.plot( + fpr, tpr, color=colors[i], label=Method_name[i] + ": (AUC = %0.2f)" % auc + ) + plt.plot([0, 1], [0, 1], color="black", linestyle="--") plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) - plt.xlabel('False Positive Rate') - plt.ylabel('True Positive Rate') - plt.title('Receiver Operating Characteristic') + plt.xlabel("False Positive Rate") + plt.ylabel("True Positive Rate") + plt.title("Receiver Operating Characteristic") plt.legend(loc="lower right") -# Display the plot in the notebook - plt.savefig(outdir+"trans_roc_curve"+TFName+".png", format='png', bbox_inches='tight') + # Display the plot in the notebook + plt.savefig( + outdir + "trans_roc_curve" + TFName + ".png", format="png", bbox_inches="tight" + ) plt.show() plt.close() - from sklearn.metrics import precision_recall_curve - from sklearn.metrics import average_precision_score + from sklearn.metrics import average_precision_score, precision_recall_curve + for i in range(len(Infer_trans)): - if filetype=='list': - data2 = pd.read_csv(Infer_trans[i], sep='\t') - #label = pd.read_csv('~/SC_NET/all_data/result/LL_sc/max/' + info0['id'][i] + '_gene_score_5fold.txt', header=True) - loc = data2[data2['TF'] == TFName].index - TGName = data2['TG'].values[loc].tolist() + if filetype == "list": + data2 = pd.read_csv(Infer_trans[i], sep="\t") + # label = pd.read_csv('~/SC_NET/all_data/result/LL_sc/max/' + info0['id'][i] + '_gene_score_5fold.txt', header=True) + loc = data2[data2["TF"] == TFName].index + TGName = data2["TG"].values[loc].tolist() TGset = TGName.copy() - Score = data2['score'].values[loc] - if filetype=='matrix': - data2=pd.read_csv(Infer_trans[i], sep='\t',header=0,index_col=0) - TGset=data2.index - Score=data2[TFName].values + Score = data2["score"].values[loc] + if filetype == "matrix": + data2 = pd.read_csv(Infer_trans[i], sep="\t", header=0, index_col=0) + TGset = data2.index + Score = data2[TFName].values d1 = np.zeros(len(TGset)) loc = np.where(np.isin(TGset, label))[0] d1[loc] = 1 -# Assuming you have the true labels (y_true) and predicted probabilities (y_scores) for your classifier -# Calculate the average precision score (AUPR + # Assuming you have the true labels (y_true) and predicted probabilities (y_scores) for your classifier + # Calculate the average precision score (AUPR aupr = average_precision_score(d1, Score) - auprr=aupr*len(d1)/sum(d1) -# Assuming you have the true labels (y_true) and predicted probabilities (y_scores) for your classifier -# Calculate precision and recall values + auprr = aupr * len(d1) / sum(d1) + # Assuming you have the true labels (y_true) and predicted probabilities (y_scores) for your classifier + # Calculate precision and recall values precision, recall, _ = precision_recall_curve(d1, Score) - # Plot precision-recall curve - plt.plot(recall, precision, color=colors[i], label=Method_name[i]+': (AUPR ratio = %0.2f)' % auprr) + # Plot precision-recall curve + plt.plot( + recall, + precision, + color=colors[i], + label=Method_name[i] + ": (AUPR ratio = %0.2f)" % auprr, + ) plt.ylim([0.0, 0.6]) - plt.xlabel('Recall') - plt.ylabel('Precision') - plt.title('Precision-Recall Curve') - plt.legend(loc='upper right') - plt.savefig(outdir+"trans_pr_curve"+TFName+".png", format='png', bbox_inches='tight') + plt.xlabel("Recall") + plt.ylabel("Precision") + plt.title("Precision-Recall Curve") + plt.legend(loc="upper right") + plt.savefig( + outdir + "trans_pr_curve" + TFName + ".png", format="png", bbox_inches="tight" + ) plt.show() - plt.close() \ No newline at end of file + plt.close() diff --git a/code/lingergrn-1.106/LingerGRN/Compare.py b/code/lingergrn-1.106/LingerGRN/Compare.py index 7c514cb..f0706fb 100644 --- a/code/lingergrn-1.106/LingerGRN/Compare.py +++ b/code/lingergrn-1.106/LingerGRN/Compare.py @@ -1,285 +1,364 @@ -from scipy import stats -import pandas as pd import numpy as np -def assignLabel(W,p): - W=W/(W.sum(axis=0)+1**(-6)) - W2=W.T/(W.T.sum(axis=0)+ 10**(-4)); - max_values = np.max(W2, axis=0)#col max +import pandas as pd +from scipy import stats + + +def assignLabel(W, p): + W = W / (W.sum(axis=0) + 1 ** (-6)) + W2 = W.T / (W.T.sum(axis=0) + 10 ** (-4)) + max_values = np.max(W2, axis=0) # col max max_indices = np.argmax(W2, axis=0) - quantile = np.percentile(max_values, p*100) - S_gene=W[:,0]; - S_gene[:]=0 - K=W.shape[1] + quantile = np.percentile(max_values, p * 100) + S_gene = W[:, 0] + S_gene[:] = 0 + K = W.shape[1] for i in range(K): - S_gene[(max_values>quantile)&(max_indices==i)]=i+1 - return S_gene,W2 -import numpy as np + S_gene[(max_values > quantile) & (max_indices == i)] = i + 1 + return S_gene, W2 + + import matplotlib.pyplot as plt +import numpy as np + -def qq_pval(p1,names,celltype): +def qq_pval(p1, names, celltype): p1[np.isnan(p1)] = 1 d = np.sort(p1) f = np.argsort(p1) p11 = np.arange(1, len(p1) + 1) / len(p1) plt.scatter(-np.log10(p11), -np.log10(d)) for i in range(K): - plt.text(-np.log10(p11[i])+np.log10(K)/20, -np.log10(d[i]), np.array(names)[f[i]], fontsize=8) + plt.text( + -np.log10(p11[i]) + np.log10(K) / 20, + -np.log10(d[i]), + np.array(names)[f[i]], + fontsize=8, + ) m = round(-np.log10(d[0]) * 100) / 100 - plt.plot(np.arange(0, m + 0.01, 0.01), np.arange(0, m + 0.01, 0.01), '-r', linewidth=1) - plt.xlabel('Theoretical Quantiles') - plt.ylabel('Sample Quantiles') + plt.plot( + np.arange(0, m + 0.01, 0.01), np.arange(0, m + 0.01, 0.01), "-r", linewidth=1 + ) + plt.xlabel("Theoretical Quantiles") + plt.ylabel("Sample Quantiles") plt.title(celltype) plt.show() # Display the plot plt.close() + class Module_obj: def __init__(self): - import pandas as pd import numpy as np + import pandas as pd + self.S_TG = pd.DataFrame() # Initialize A.x as an empty DataFrame self.pvalue_all = pd.DataFrame() # Initialize A.y as an empty list - self.tvalue_all = pd.DataFrame() - self.p_fisher = pd.DataFrame() - self.odds_ratio_fisher=pd.DataFrame() - - -def diff_Module(Exp_TG,metadata,S_TG,K): - celltype=metadata['celltype'].unique().tolist() - pvalue_all= np.zeros((K, len(celltype))) - tvalue_all= np.zeros((K, len(celltype))) + self.tvalue_all = pd.DataFrame() + self.p_fisher = pd.DataFrame() + self.odds_ratio_fisher = pd.DataFrame() + + +def diff_Module(Exp_TG, metadata, S_TG, K): + celltype = metadata["celltype"].unique().tolist() + pvalue_all = np.zeros((K, len(celltype))) + tvalue_all = np.zeros((K, len(celltype))) from scipy import stats from statsmodels.stats.multitest import multipletests + for k in range(len(celltype)): - temp=Exp_TG.iloc[:,metadata['celltype'].values==celltype[k]] - aud_idxtemp=metadata[(metadata['celltype'].values==celltype[k])]['group'].values - Exp_mean=stats.zscore(temp.T).T.groupby(S_TG['Module'].values).mean() - Exp_mean=Exp_mean.loc[range(1,K+1)] - X=Exp_mean.values[:,(aud_idxtemp==1)] - Y=Exp_mean.values[:,(aud_idxtemp==0)] - p_values = np.zeros((K, )) - t_values = np.zeros((K, )) + temp = Exp_TG.iloc[:, metadata["celltype"].values == celltype[k]] + aud_idxtemp = metadata[(metadata["celltype"].values == celltype[k])][ + "group" + ].values + Exp_mean = stats.zscore(temp.T).T.groupby(S_TG["Module"].values).mean() + Exp_mean = Exp_mean.loc[range(1, K + 1)] + X = Exp_mean.values[:, (aud_idxtemp == 1)] + Y = Exp_mean.values[:, (aud_idxtemp == 0)] + p_values = np.zeros((K,)) + t_values = np.zeros((K,)) from scipy.stats import ttest_ind + for i in range(K): t_values[i], p_values[i] = ttest_ind(X[i], Y[i]) - #p_values = np.nan_to_num(p_values, nan=1) - pvalue_all[:,k]=p_values - tvalue_all[:,k]=t_values - - adjusted_p_values = multipletests(p_values, method='fdr_bh')[1] - pvalue_all=pd.DataFrame(pvalue_all,index=['M'+str(i+1) for i in range(K)],columns=celltype) - tvalue_all=pd.DataFrame(tvalue_all,index=['M'+str(i+1) for i in range(K)],columns=celltype) - return pvalue_all,tvalue_all + # p_values = np.nan_to_num(p_values, nan=1) + pvalue_all[:, k] = p_values + tvalue_all[:, k] = t_values + adjusted_p_values = multipletests(p_values, method="fdr_bh")[1] + pvalue_all = pd.DataFrame( + pvalue_all, index=["M" + str(i + 1) for i in range(K)], columns=celltype + ) + tvalue_all = pd.DataFrame( + tvalue_all, index=["M" + str(i + 1) for i in range(K)], columns=celltype + ) + return pvalue_all, tvalue_all -def GWAS_Module_enrich(S_TG,TGset,GWASgene,K): + +def GWAS_Module_enrich(S_TG, TGset, GWASgene, K): from collections import Counter - counts = Counter(S_TG['Module'].values) + + counts = Counter(S_TG["Module"].values) from scipy.stats import fisher_exact - p_fisher=np.zeros((K,GWASgene.shape[1])); - odds_ratio_fisher=np.zeros((K,GWASgene.shape[1])); + + p_fisher = np.zeros((K, GWASgene.shape[1])) + odds_ratio_fisher = np.zeros((K, GWASgene.shape[1])) for i in range(GWASgene.shape[1]): - AUD_genes=GWASgene[GWASgene['GWAS_'+(str(i+1))]==1].index - N=len(set(TGset)&set(AUD_genes)) + AUD_genes = GWASgene[GWASgene["GWAS_" + (str(i + 1))] == 1].index + N = len(set(TGset) & set(AUD_genes)) for k in range(K): - Noverlap=len(set(TGset[S_TG['Module'].values==k+1])&set(AUD_genes)) - contingency_table = np.array([[Noverlap, counts[k+1]-Noverlap], [N-Noverlap, len(TGset)- N-counts[k+1]+Noverlap]]) - odds_ratio, p_value = fisher_exact(contingency_table,alternative='greater') - p_fisher[k,i]=odds_ratio - odds_ratio_fisher[k,i]=p_value - odds_ratio_fisher=pd.DataFrame(odds_ratio_fisher,index=['M'+str(j+1) for j in range(K)],columns=['GWAS_'+(str(j+1)) for j in range(GWASgene.shape[1])]) - p_fisher=pd.DataFrame(p_fisher,index=['M'+str(j+1) for j in range(K)],columns=['GWAS_'+(str(j+1)) for j in range(GWASgene.shape[1])]) - return p_fisher,odds_ratio_fisher - -def Module_trans(outdir,metadata,TG_pseudobulk,K,GWASfile=None): + Noverlap = len(set(TGset[S_TG["Module"].values == k + 1]) & set(AUD_genes)) + contingency_table = np.array( + [ + [Noverlap, counts[k + 1] - Noverlap], + [N - Noverlap, len(TGset) - N - counts[k + 1] + Noverlap], + ] + ) + odds_ratio, p_value = fisher_exact(contingency_table, alternative="greater") + p_fisher[k, i] = odds_ratio + odds_ratio_fisher[k, i] = p_value + odds_ratio_fisher = pd.DataFrame( + odds_ratio_fisher, + index=["M" + str(j + 1) for j in range(K)], + columns=["GWAS_" + (str(j + 1)) for j in range(GWASgene.shape[1])], + ) + p_fisher = pd.DataFrame( + p_fisher, + index=["M" + str(j + 1) for j in range(K)], + columns=["GWAS_" + (str(j + 1)) for j in range(GWASgene.shape[1])], + ) + return p_fisher, odds_ratio_fisher + + +def Module_trans(outdir, metadata, TG_pseudobulk, K, GWASfile=None): import numpy as np from scipy import stats - print('loading GRN......') - trans_reg=pd.read_csv(outdir+'cell_population_trans_regulatory.txt',sep='\t',index_col=0) - #print(trans_reg) - #TG_pseudobulk=TG_pseudobulk_all - TFset=trans_reg.columns - TGset=trans_reg.index - #TG_pseudobulk=TG_pseudobulk/TG_pseudobulk.mean() + + print("loading GRN......") + trans_reg = pd.read_csv( + outdir + "cell_population_trans_regulatory.txt", sep="\t", index_col=0 + ) + # print(trans_reg) + # TG_pseudobulk=TG_pseudobulk_all + TFset = trans_reg.columns + TGset = trans_reg.index + # TG_pseudobulk=TG_pseudobulk/TG_pseudobulk.mean() idx = [s[:3] != "MIR" for s in TG_pseudobulk.index] - TG_pseudobulk=TG_pseudobulk.loc[TG_pseudobulk.index[idx]] - R1 = stats.zscore(trans_reg,1);R1[np.isnan(R1)] = 0.0 - R2 = stats.zscore(trans_reg,0);R2[np.isnan(R2)] = 0.0 - #R1[R1<0]=0;R2[R2<0]=0 - #X=TG_pseudobulk.loc[TGset] + TG_pseudobulk = TG_pseudobulk.loc[TG_pseudobulk.index[idx]] + R1 = stats.zscore(trans_reg, 1) + R1[np.isnan(R1)] = 0.0 + R2 = stats.zscore(trans_reg, 0) + R2[np.isnan(R2)] = 0.0 + # R1[R1<0]=0;R2[R2<0]=0 + # X=TG_pseudobulk.loc[TGset] from sklearn.preprocessing import quantile_transform - #Exp=quantile_transform(np.log2(X + 1), n_quantiles=10, random_state=0, copy=True) - #Exp=quantile_transform(np.log2(TG_pseudobulk + 1), n_quantiles=10, random_state=0, copy=True) - Exp=pd.DataFrame(TG_pseudobulk,index=TG_pseudobulk.index,columns=TG_pseudobulk.columns) - Z=R1+R2 - Z[Z<0]=0 - print('identify modules......') + + # Exp=quantile_transform(np.log2(X + 1), n_quantiles=10, random_state=0, copy=True) + # Exp=quantile_transform(np.log2(TG_pseudobulk + 1), n_quantiles=10, random_state=0, copy=True) + Exp = pd.DataFrame( + TG_pseudobulk, index=TG_pseudobulk.index, columns=TG_pseudobulk.columns + ) + Z = R1 + R2 + Z[Z < 0] = 0 + print("identify modules......") from sklearn.decomposition import NMF - nmf = NMF(n_components=K, init='random', random_state=0) + + nmf = NMF(n_components=K, init="random", random_state=0) W = nmf.fit_transform(Z) H = nmf.components_ - [S_TG,W2]=assignLabel(W,0.9); - [S_TF,H2]=assignLabel(H.T,0.8); - #Exp_mean=stats.zscore(Exp_TG.T).T.groupby(S_TG).mean() - S_TG=pd.DataFrame(S_TG,index=TGset,columns=['Module']) - Exp_TF=Exp.loc[TFset] - Exp_TG=Exp.loc[TGset] - print('differential modules......') - pvalue_all,tvalue_all=diff_Module(Exp_TG,metadata,S_TG,K) - if GWASfile is not None: - print('GWAS enrich......') - GWASgene=pd.DataFrame(index=TG_pseudobulk.index) + [S_TG, W2] = assignLabel(W, 0.9) + [S_TF, H2] = assignLabel(H.T, 0.8) + # Exp_mean=stats.zscore(Exp_TG.T).T.groupby(S_TG).mean() + S_TG = pd.DataFrame(S_TG, index=TGset, columns=["Module"]) + Exp_TF = Exp.loc[TFset] + Exp_TG = Exp.loc[TGset] + print("differential modules......") + pvalue_all, tvalue_all = diff_Module(Exp_TG, metadata, S_TG, K) + if GWASfile is not None: + print("GWAS enrich......") + GWASgene = pd.DataFrame(index=TG_pseudobulk.index) for i in range(len(GWASfile)): - temp=pd.read_csv(GWASfile[i],sep='\t',header=None) - idx=np.zeros((GWASgene.shape[0],1)) - idx[GWASgene.index.isin(temp[0].values),:]=1 - GWASgene['GWAS_'+(str(i+1))]=idx - p_fisher,odds_ratio_fisher=GWAS_Module_enrich(S_TG,TGset,GWASgene,K) - Module_result=Module_obj() - Module_result.S_TG=S_TG - Module_result.pvalue_all=pvalue_all - Module_result.tvalue_all=tvalue_all - Module_result.p_fisher=p_fisher - Module_result.odds_ratio_fisher=odds_ratio_fisher + temp = pd.read_csv(GWASfile[i], sep="\t", header=None) + idx = np.zeros((GWASgene.shape[0], 1)) + idx[GWASgene.index.isin(temp[0].values), :] = 1 + GWASgene["GWAS_" + (str(i + 1))] = idx + p_fisher, odds_ratio_fisher = GWAS_Module_enrich(S_TG, TGset, GWASgene, K) + Module_result = Module_obj() + Module_result.S_TG = S_TG + Module_result.pvalue_all = pvalue_all + Module_result.tvalue_all = tvalue_all + Module_result.p_fisher = p_fisher + Module_result.odds_ratio_fisher = odds_ratio_fisher return Module_result else: - Module_result=Module_obj() - Module_result.S_TG=S_TG - Module_result.pvalue_all=pvalue_all - Module_result.tvalue_all=tvalue_all + Module_result = Module_obj() + Module_result.S_TG = S_TG + Module_result.pvalue_all = pvalue_all + Module_result.tvalue_all = tvalue_all return Module_result -def remove_covariate(TG_pseudobulk,aud_idx,celltype): - celltypeset=list(set(celltype)) - Exp_norm=TG_pseudobulk.values.copy() + +def remove_covariate(TG_pseudobulk, aud_idx, celltype): + celltypeset = list(set(celltype)) + Exp_norm = TG_pseudobulk.values.copy() for i in range(len(celltypeset)): - Nseq=np.array(range(len(celltype))) - Cov=aud_idx.iloc[Nseq[celltype==celltypeset[i]],:] + Nseq = np.array(range(len(celltype))) + Cov = aud_idx.iloc[Nseq[celltype == celltypeset[i]], :] Cov_product = np.dot(Cov.T, Cov) # Calculate the pseudo-inverse of Cov_product Cov_product_pinv = np.linalg.pinv(Cov_product) # Calculate beta - Exp=TG_pseudobulk.iloc[:,Nseq[celltype==celltypeset[i]]] + Exp = TG_pseudobulk.iloc[:, Nseq[celltype == celltypeset[i]]] beta = np.dot(np.dot(Exp, Cov), Cov_product_pinv) - Exp_norm1=Exp-np.dot(beta, Cov.T)+np.dot(np.reshape(beta[:, 0], (-1, 1)),np.ones((1,Exp.shape[1]))) - Exp_norm1[Exp_norm1<0]=0 - Exp_norm[:,celltype==celltypeset[i]]=Exp_norm1 - Exp_norm=pd.DataFrame(Exp_norm,columns=TG_pseudobulk.columns,index=TG_pseudobulk.index) + Exp_norm1 = ( + Exp + - np.dot(beta, Cov.T) + + np.dot(np.reshape(beta[:, 0], (-1, 1)), np.ones((1, Exp.shape[1]))) + ) + Exp_norm1[Exp_norm1 < 0] = 0 + Exp_norm[:, celltype == celltypeset[i]] = Exp_norm1 + Exp_norm = pd.DataFrame( + Exp_norm, columns=TG_pseudobulk.columns, index=TG_pseudobulk.index + ) return Exp_norm + +import numpy as np import pandas as pd -import numpy as np -def runWGCNA(celltypetemp,TG_pseudobulk_all,metadata): - metadata_temp=metadata[metadata['celltype'].isin([celltypetemp])] - idx=np.array([i for i in range(metadata.shape[0])]) - idx=idx[metadata['celltype'].isin([celltypetemp])] - TG_pseudobulk=TG_pseudobulk_all.iloc[:,idx] + + +def runWGCNA(celltypetemp, TG_pseudobulk_all, metadata): + metadata_temp = metadata[metadata["celltype"].isin([celltypetemp])] + idx = np.array([i for i in range(metadata.shape[0])]) + idx = idx[metadata["celltype"].isin([celltypetemp])] + TG_pseudobulk = TG_pseudobulk_all.iloc[:, idx] print(TG_pseudobulk.shape) - TG_pseudobulk=TG_pseudobulk.groupby(TG_pseudobulk.index).mean() + TG_pseudobulk = TG_pseudobulk.groupby(TG_pseudobulk.index).mean() print(TG_pseudobulk.shape) - expression=TG_pseudobulk.T + expression = TG_pseudobulk.T print(expression.shape) - geneList=pd.DataFrame(TG_pseudobulk.index,index=TG_pseudobulk.index) - geneList.columns=['gene_name'] - geneList['gene_type']='protein_coding' + geneList = pd.DataFrame(TG_pseudobulk.index, index=TG_pseudobulk.index) + geneList.columns = ["gene_name"] + geneList["gene_type"] = "protein_coding" import PyWGCNA - expression.to_csv(celltypetemp+'_expression_WGCNA.csv') - pyWGCNA_5xFAD = PyWGCNA.WGCNA(name=celltypetemp, - geneExpPath=celltypetemp+'_expression_WGCNA.csv', - outputPath='', - save=True) + + expression.to_csv(celltypetemp + "_expression_WGCNA.csv") + pyWGCNA_5xFAD = PyWGCNA.WGCNA( + name=celltypetemp, + geneExpPath=celltypetemp + "_expression_WGCNA.csv", + outputPath="", + save=True, + ) pyWGCNA_5xFAD.preprocess() pyWGCNA_5xFAD.findModules() - metadata_temp.to_csv(celltypetemp+'_sampleInfo.csv',sep=',') - pyWGCNA_5xFAD.updateSampleInfo(path=celltypetemp+'_sampleInfo.csv', sep=',') - pyWGCNA_5xFAD.setMetadataColor('group', {0: 'green', - 1: 'yellow'}) + metadata_temp.to_csv(celltypetemp + "_sampleInfo.csv", sep=",") + pyWGCNA_5xFAD.updateSampleInfo(path=celltypetemp + "_sampleInfo.csv", sep=",") + pyWGCNA_5xFAD.setMetadataColor("group", {0: "green", 1: "yellow"}) pyWGCNA_5xFAD.updateGeneInfo(geneList) pyWGCNA_5xFAD.analyseWGCNA() pyWGCNA_5xFAD.saveWGCNA() -def correlation_FC(x,y,method): + +def correlation_FC(x, y, method): import numpy as np from scipy import stats + # Loop through each column of y and calculate correlation with x correlations = [] - correlationsp=[] + correlationsp = [] for i in range(y.shape[1]): - if method=='pearsonr': - r, p = stats.pearsonr(x.ravel(), y.values[:,i]) - if method=='spearmanr': - r, p = stats.spearmanr(x.ravel(), y.values[:,i]) + if method == "pearsonr": + r, p = stats.pearsonr(x.ravel(), y.values[:, i]) + if method == "spearmanr": + r, p = stats.spearmanr(x.ravel(), y.values[:, i]) correlations.append(r) correlationsp.append(p) - correlations=pd.DataFrame(correlations,index=y.columns) - correlationsp=pd.DataFrame(correlationsp,index=y.columns) - return correlations,correlationsp + correlations = pd.DataFrame(correlations, index=y.columns) + correlationsp = pd.DataFrame(correlationsp, index=y.columns) + return correlations, correlationsp -def driver_score(expression,aud_idx,GRN,outdir,adjust_method,corr_method): - print('loading GRN......') - import numpy as np +def driver_score(expression, aud_idx, GRN, outdir, adjust_method, corr_method): + print("loading GRN......") + import numpy as np from statsmodels.stats.multitest import multipletests - allcelltype=aud_idx['celltype'].unique() - C_result=pd.DataFrame([]) - P_result=pd.DataFrame([]) - Q_result=pd.DataFrame([]) - reg=pd.read_csv(outdir+'cell_population_'+GRN+'.txt',sep='\t',index_col=0) - #print(reg.shape) - if reg.shape[1]<4: - reg = reg.pivot(index='RE', columns='TF', values='score').fillna(0) + + allcelltype = aud_idx["celltype"].unique() + C_result = pd.DataFrame([]) + P_result = pd.DataFrame([]) + Q_result = pd.DataFrame([]) + reg = pd.read_csv(outdir + "cell_population_" + GRN + ".txt", sep="\t", index_col=0) + # print(reg.shape) + if reg.shape[1] < 4: + reg = reg.pivot(index="RE", columns="TF", values="score").fillna(0) reg = reg.fillna(0) - cols=reg.sum(axis=0).values - rows=reg.sum(axis=1).values - E=np.reshape(rows,(rows.shape[0],1))*np.reshape(cols,(1,cols.shape[0]))/rows.sum() - #print(E.mean().mean()*10**(-4)) - E=E+E.mean().mean()*10**(-4) - reg=(reg-E)/E - reg[reg<0]=0 - overlap=list(set(expression.index)&set(reg.index)) - reg=reg.loc[overlap] - expression=expression.loc[overlap] - cols=expression.sum(axis=0).values - rows=expression.sum(axis=1).values - E=np.reshape(rows,(rows.shape[0],1))*np.reshape(cols,(1,cols.shape[0]))/rows.sum() - E=E+E.mean().mean()*10**(-1) - expression=expression/E + cols = reg.sum(axis=0).values + rows = reg.sum(axis=1).values + E = ( + np.reshape(rows, (rows.shape[0], 1)) + * np.reshape(cols, (1, cols.shape[0])) + / rows.sum() + ) + # print(E.mean().mean()*10**(-4)) + E = E + E.mean().mean() * 10 ** (-4) + reg = (reg - E) / E + reg[reg < 0] = 0 + overlap = list(set(expression.index) & set(reg.index)) + reg = reg.loc[overlap] + expression = expression.loc[overlap] + cols = expression.sum(axis=0).values + rows = expression.sum(axis=1).values + E = ( + np.reshape(rows, (rows.shape[0], 1)) + * np.reshape(cols, (1, cols.shape[0])) + / rows.sum() + ) + E = E + E.mean().mean() * 10 ** (-1) + expression = expression / E for i in range(len(allcelltype)): - print('cell type '+ allcelltype[i]) - aud_idx1=aud_idx.reset_index() - exp_temp=expression.iloc[:,aud_idx1[aud_idx1['celltype']==allcelltype[i]].index] - aud_idx1=aud_idx1[aud_idx1['celltype']==allcelltype[i]] - Mean1=exp_temp[aud_idx1[aud_idx1['group']==1]['index']].mean(axis=1)+10**(-6) - Mean0=exp_temp[aud_idx1[aud_idx1['group']==0]['index']].mean(axis=1)+10**(-6) - FC=pd.DataFrame(Mean1/Mean0) - FC=FC.loc[reg.index] - #print(FC) + print("cell type " + allcelltype[i]) + aud_idx1 = aud_idx.reset_index() + exp_temp = expression.iloc[ + :, aud_idx1[aud_idx1["celltype"] == allcelltype[i]].index + ] + aud_idx1 = aud_idx1[aud_idx1["celltype"] == allcelltype[i]] + Mean1 = exp_temp[aud_idx1[aud_idx1["group"] == 1]["index"]].mean( + axis=1 + ) + 10 ** (-6) + Mean0 = exp_temp[aud_idx1[aud_idx1["group"] == 0]["index"]].mean( + axis=1 + ) + 10 ** (-6) + FC = pd.DataFrame(Mean1 / Mean0) + FC = FC.loc[reg.index] + # print(FC) print(np.isnan(FC).sum()) - c,cp=correlation_FC(np.log(FC[0]).values,reg,corr_method) - #idx=pd.DataFrame(range(expression.shape[0]),index=expression.index) - C_result=pd.concat([C_result,c],axis=1) - P_result=pd.concat([P_result,cp],axis=1) - cp=cp.fillna(1) - adjusted_p_values = pd.DataFrame(multipletests(cp[0].values, method=adjust_method)[1],index=c.index) - #print(adjusted_p_values) - Q_result=pd.concat([Q_result,adjusted_p_values],axis=1) - C_result.columns=allcelltype - P_result.columns=allcelltype - Q_result.columns=allcelltype - return C_result,P_result,Q_result - -def driver_result(C_result,Q_result,K): + c, cp = correlation_FC(np.log(FC[0]).values, reg, corr_method) + # idx=pd.DataFrame(range(expression.shape[0]),index=expression.index) + C_result = pd.concat([C_result, c], axis=1) + P_result = pd.concat([P_result, cp], axis=1) + cp = cp.fillna(1) + adjusted_p_values = pd.DataFrame( + multipletests(cp[0].values, method=adjust_method)[1], index=c.index + ) + # print(adjusted_p_values) + Q_result = pd.concat([Q_result, adjusted_p_values], axis=1) + C_result.columns = allcelltype + P_result.columns = allcelltype + Q_result.columns = allcelltype + return C_result, P_result, Q_result + + +def driver_result(C_result, Q_result, K): import pandas as pd - # Create a sample DataFrame of size 100x7 + + # Create a sample DataFrame of size 100x7 # Rank all values in the DataFrame - top_5_rows=[] + top_5_rows = [] for column in C_result.columns: sorted_column = C_result[column].sort_values(ascending=False) - top_5_rows =top_5_rows+sorted_column.index[:K].tolist() - top_5_rows=list(set(top_5_rows)) + top_5_rows = top_5_rows + sorted_column.index[:K].tolist() + top_5_rows = list(set(top_5_rows)) for column in C_result.columns: sorted_column = C_result[column].sort_values(ascending=True) - top_5_rows =top_5_rows+sorted_column.index[:K].tolist() - top_5_rows=list(set(top_5_rows)) - return C_result.loc[top_5_rows],Q_result.loc[top_5_rows] - + top_5_rows = top_5_rows + sorted_column.index[:K].tolist() + top_5_rows = list(set(top_5_rows)) + return C_result.loc[top_5_rows], Q_result.loc[top_5_rows] diff --git a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py index 62fcb1e..00e0df9 100644 --- a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py +++ b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py @@ -1,5 +1,6 @@ import os import random +import warnings # load data import numpy as np @@ -14,6 +15,7 @@ from sklearn.model_selection import KFold from torch.nn import functional as F from torch.optim import Adam +from tqdm import tqdm hidden_size = 64 hidden_size2 = 16 @@ -242,7 +244,6 @@ def get_TSS(GRNdir, genome, TSS_dis): # strand.append(gene.strand) # chrom.append('chr'+gene.contig) # genesymbol.append(gene.name) - import pandas as pd Tssdf = pd.read_csv(GRNdir + "TSS_" + genome + ".txt", sep="\t", header=None) Tssdf.columns = ["chr", "TSS", "symbol", "strand"] @@ -362,7 +363,6 @@ def sc_nn_NN(ii, RE_TGlink_temp, Target, Exp, Opn, l1_lambda, activef): def load_data_scNN(GRNdir, species): - import pandas as pd if species == "New": Match2 = pd.read_csv(GRNdir + "MotifMatch.txt", header=0, sep="\t") @@ -388,16 +388,12 @@ def load_data_scNN(GRNdir, species): def RE_TG_dis(outdir): - import numpy as np - import pandas as pd import pybedtools print("Overlap the regions with gene loc ...") - import os # Create the directory current_directory = os.getcwd() os.makedirs(outdir, exist_ok=True) - import pandas as pd peakList = pd.read_csv( current_directory + "/data/Peaks.txt", index_col=None, header=None @@ -439,14 +435,6 @@ def RE_TG_dis(outdir): temp.to_csv(current_directory + "/data/RE_gene_distance.txt", sep="\t", index=None) -import time -import warnings - -import numpy as np -import pandas as pd -from tqdm import tqdm - - def training(GRNdir, method, outdir, activef, species): if method == "LINGER": hidden_size = 64 @@ -464,10 +452,6 @@ def training(GRNdir, method, outdir, activef, species): data_merge.to_csv(outdir + "data_merge.txt", sep="\t") chrall = ["chr" + str(i + 1) for i in range(22)] chrall.append("chrX") - import time - import warnings - - from tqdm import tqdm for i in range(23): netall_s = {} @@ -543,10 +527,6 @@ def training(GRNdir, method, outdir, activef, species): fisher_w = 0.1 n_jobs = 16 Exp, Opn, Target, RE_TGlink = load_data_scNN(GRNdir, species) - import time - import warnings - - from tqdm import tqdm netall_s = {} shapall_s = {} @@ -598,7 +578,6 @@ def get_TSS_ensembl(genome_short, gtf_file, GRNdir): strand.append(gene.strand) chrom.append("chr" + gene.contig) genesymbol.append(gene.name) - import pandas as pd Tssdf = pd.DataFrame( {"chr": chrom, "TSS": tss_positions, "symbol": genesymbol, "strand": strand} diff --git a/code/lingergrn-1.106/LingerGRN/LL_net.py b/code/lingergrn-1.106/LingerGRN/LL_net.py index ffe3e33..5b77335 100644 --- a/code/lingergrn-1.106/LingerGRN/LL_net.py +++ b/code/lingergrn-1.106/LingerGRN/LL_net.py @@ -1,11 +1,11 @@ import csv import os - -#load data import random import numpy as np import pandas as pd + +# load data import torch import torch.nn as nn import torch.optim as optim @@ -18,38 +18,45 @@ from torch.optim import Adam from tqdm import tqdm -hidden_size = 64 +hidden_size = 64 hidden_size2 = 16 output_size = 1 seed_value = 42 torch.manual_seed(seed_value) + + class Net(nn.Module): - def __init__(self,input_size,activef): + def __init__(self, input_size, activef): super(Net, self).__init__() self.fc1 = nn.Linear(input_size, 64) self.fc2 = nn.Linear(64, 16) self.fc3 = nn.Linear(16, output_size) self.activef = activef + def forward(self, x): - #x = torch.sigmoid(self.fc1(x)) - if self.activef=='ReLU': + # x = torch.sigmoid(self.fc1(x)) + if self.activef == "ReLU": x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) - if self.activef=='sigmoid': + if self.activef == "sigmoid": x = F.sigmoid(self.fc1(x)) x = F.sigmoid(self.fc2(x)) - if self.activef=='tanh': + if self.activef == "tanh": x = F.tanh(self.fc1(x)) x = F.tanh(self.fc2(x)) x = self.fc3(x) return x + def cosine_similarity_0(X): - norm_term = np.linalg.norm(X, axis = 1) # calculate the norm term once rather than recalculate + norm_term = np.linalg.norm( + X, axis=1 + ) # calculate the norm term once rather than recalculate eps = norm_term.mean() / 1e6 - A=X/(norm_term[:,np.newaxis] + eps) + A = X / (norm_term[:, np.newaxis] + eps) return np.dot(A, A.T) + # torch norm, should be slightly faster than np def cosine_similarity_0_torch(X): with torch.no_grad(): @@ -59,673 +66,848 @@ def cosine_similarity_0_torch(X): return torch.mm(A, A.t()) -def list2mat(df,i_n,j_n,x_n): +def list2mat(df, i_n, j_n, x_n): TFs = df[j_n].unique() REs = df[i_n].unique() -#Initialize matrix as numpy array -#Map row and col indices for lookup - row_map = {r:i for i,r in enumerate(REs)} - col_map = {c:i for i,c in enumerate(TFs)} + # Initialize matrix as numpy array + # Map row and col indices for lookup + row_map = {r: i for i, r in enumerate(REs)} + col_map = {c: i for i, c in enumerate(TFs)} row_indices = np.array([row_map[row] for row in df[i_n]]) col_indices = np.array([col_map[col] for col in df[j_n]]) from scipy.sparse import coo_matrix - matrix = coo_matrix((df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs))) - mat=coo_matrix.toarray(matrix) - return mat,REs,TFs + matrix = coo_matrix( + (df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs)) + ) + mat = coo_matrix.toarray(matrix) + return mat, REs, TFs -def list2mat_s(df,REs,TFs,i_n,j_n,x_n): -#Initialize matrix as numpy array -#Map row and col indices for lookup - row_map = {r:i for i,r in enumerate(REs)} - col_map = {c:i for i,c in enumerate(TFs)} +def list2mat_s(df, REs, TFs, i_n, j_n, x_n): + # Initialize matrix as numpy array + # Map row and col indices for lookup + row_map = {r: i for i, r in enumerate(REs)} + col_map = {c: i for i, c in enumerate(TFs)} row_indices = np.array([row_map[row] for row in df[i_n]]) col_indices = np.array([col_map[col] for col in df[j_n]]) import scipy.sparse as sp from scipy.sparse import coo_matrix - matrix = sp.csr_matrix((df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs))) - return matrix,REs,TFs -def merge_columns_in_bed_file(file_path,startcol): + matrix = sp.csr_matrix( + (df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs)) + ) + return matrix, REs, TFs + + +def merge_columns_in_bed_file(file_path, startcol): merged_values = [] - with open(file_path, 'r') as file: + with open(file_path, "r") as file: for line in file: - columns = line.strip().split('\t') - col1 = columns[-1+startcol] + columns = line.strip().split("\t") + col1 = columns[-1 + startcol] col2 = columns[startcol] - col3 = columns[1+startcol] + col3 = columns[1 + startcol] merged_value = f"{col1}:{col2}-{col3}" merged_values.append(merged_value) return merged_values -def merge_columns_in_bed_file2(file_path,startcol): + + +def merge_columns_in_bed_file2(file_path, startcol): merged_values = [] - with open(file_path, 'r') as file: + with open(file_path, "r") as file: for line in file: - columns = line.strip().split('\t') - col1 = columns[-1+startcol] + columns = line.strip().split("\t") + col1 = columns[-1 + startcol] col2 = columns[startcol] - col3 = columns[1+startcol] + col3 = columns[1 + startcol] merged_value = f"{col1}_{col2}_{col3}" merged_values.append(merged_value) return merged_values + + def format_RE_tran12(region): chr, range_ = region.split(":") start, end = range_.split("-") return "_".join([chr, start, end]) -def get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName, torch_cosine = False): - index_all=data_merge_temp[j] - result={'TF':[],'RE':[],'score':[]} + + +def get_TF_RE( + data_merge_temp, j, net_all, TFindex, TFName, REindex, REName, torch_cosine=False +): + index_all = data_merge_temp[j] + result = {"TF": [], "RE": [], "score": []} result = pd.DataFrame(result) - #for ii in range(1): - temps=list(net_all[index_all].parameters())[0] - TFidxtemp=TFindex[index_all] - TFidxtemp=TFidxtemp.split('_') - TFidxtemp=[int(TFidxtemp[i]) for i in range(len(TFidxtemp))] - TFName_temp=TFName[np.array(TFidxtemp)] - REidxtemp=REindex[index_all] - if REidxtemp=='': - REidxtemp=[] + # for ii in range(1): + temps = list(net_all[index_all].parameters())[0] + TFidxtemp = TFindex[index_all] + TFidxtemp = TFidxtemp.split("_") + TFidxtemp = [int(TFidxtemp[i]) for i in range(len(TFidxtemp))] + TFName_temp = TFName[np.array(TFidxtemp)] + REidxtemp = REindex[index_all] + if REidxtemp == "": + REidxtemp = [] else: - REidxtemp=REidxtemp.split('_') - REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] #146 RE idx - if len(REidxtemp)>0: + REidxtemp = REidxtemp.split("_") + REidxtemp = [int(REidxtemp[i]) for i in range(len(REidxtemp))] # 146 RE idx + if len(REidxtemp) > 0: if not torch_cosine: corr_matrix = cosine_similarity_0(temps.detach().numpy().T) else: corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() - REName_temp=REName[np.array(REidxtemp)] - corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] + REName_temp = REName[np.array(REidxtemp)] + corr_matrix = corr_matrix[: len(TFidxtemp), len(TFidxtemp) :] results = [] for k in range(len(REidxtemp)): - datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) - datatemp['TF']=TFName_temp.tolist() - datatemp['RE']=REName_temp[k] + datatemp = pd.DataFrame({"score": corr_matrix[:, k].tolist()}) + datatemp["TF"] = TFName_temp.tolist() + datatemp["RE"] = REName_temp[k] results.append(datatemp) result = pd.concat(results) return result -def load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN): - TFbinding=pd.read_csv(GRNdir+'TF_binding_'+chrN+'.txt',sep='\t',index_col=0) - TFbinding1=np.zeros([len(O_overlap_u),TFbinding.shape[1]]) - TFbinding1=np.zeros([len(O_overlap_u),TFbinding.shape[1]]) - O_overlap1=list(set(O_overlap_hg19_u)&set(TFbinding.index)) - List=pd.DataFrame(range(len(TFbinding.index)), index=TFbinding.index) - index0=List.loc[O_overlap1][0].values - #O_overlap_df=pd.DataFrame(range(len(O_overlap)), index=O_overlap) - O_overlap_hg19_u_df=pd.DataFrame(range(len(O_overlap_hg19_u)), index=O_overlap_hg19_u) - #hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) - index1=O_overlap_hg19_u_df.loc[O_overlap1][0].values - #index1=O_overlap_df.loc[index1][0].values - TFbinding1[index1,:]=TFbinding.iloc[index0,:].values - #TFbinding=pd.DataFrame(TFbinding1,index=O_overlap_u,columns=TFbinding.columns) - O_overlap_u_df=pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) - hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) - TFbinding2=np.zeros([len(O_overlap),TFbinding.shape[1]]) - index=O_overlap_u_df.loc[O_overlap][0].values - TFbinding2=TFbinding1[index,:] - TFbinding=pd.DataFrame(TFbinding2,index=O_overlap,columns=TFbinding.columns) + +def load_TFbinding(GRNdir, O_overlap, O_overlap_u, O_overlap_hg19_u, chrN): + TFbinding = pd.read_csv( + GRNdir + "TF_binding_" + chrN + ".txt", sep="\t", index_col=0 + ) + TFbinding1 = np.zeros([len(O_overlap_u), TFbinding.shape[1]]) + TFbinding1 = np.zeros([len(O_overlap_u), TFbinding.shape[1]]) + O_overlap1 = list(set(O_overlap_hg19_u) & set(TFbinding.index)) + List = pd.DataFrame(range(len(TFbinding.index)), index=TFbinding.index) + index0 = List.loc[O_overlap1][0].values + # O_overlap_df=pd.DataFrame(range(len(O_overlap)), index=O_overlap) + O_overlap_hg19_u_df = pd.DataFrame( + range(len(O_overlap_hg19_u)), index=O_overlap_hg19_u + ) + # hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) + index1 = O_overlap_hg19_u_df.loc[O_overlap1][0].values + # index1=O_overlap_df.loc[index1][0].values + TFbinding1[index1, :] = TFbinding.iloc[index0, :].values + # TFbinding=pd.DataFrame(TFbinding1,index=O_overlap_u,columns=TFbinding.columns) + O_overlap_u_df = pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) + hg19_38 = pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) + TFbinding2 = np.zeros([len(O_overlap), TFbinding.shape[1]]) + index = O_overlap_u_df.loc[O_overlap][0].values + TFbinding2 = TFbinding1[index, :] + TFbinding = pd.DataFrame(TFbinding2, index=O_overlap, columns=TFbinding.columns) return TFbinding -def load_region(GRNdir,genome,chrN,outdir): - O_overlap=merge_columns_in_bed_file(outdir+'Region_overlap_'+chrN+'.bed',1) - N_overlap=merge_columns_in_bed_file(outdir+'Region_overlap_'+chrN+'.bed',4) - O_overlap_u=list(set(O_overlap)) - N_overlap_u=list(set(N_overlap)) - #O_all=merge_columns_in_bed_file(GRNdir+'Peaks_'+chrN+'.bed',1) - hg19_region=merge_columns_in_bed_file(GRNdir+'hg19_Peaks_'+chrN+'.bed',1) - hg19_region=pd.DataFrame(range(len(hg19_region)),index=hg19_region) - hg38_region=merge_columns_in_bed_file(GRNdir+'hg38_Peaks_'+chrN+'.bed',1) - hg38_region=pd.DataFrame(range(len(hg38_region)),index=hg38_region) - if genome=='hg19': - idx=hg19_region.loc[O_overlap_u][0].values - O_overlap_u=hg38_region.index[idx].tolist() - O_overlap_hg19_u=hg19_region.index[idx].tolist() - if genome=='hg38': - idx=hg38_region.loc[O_overlap_u][0].values - O_overlap_hg19_u=hg19_region.index[idx].tolist() - return O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u - -def load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u): - #print('load prior TF-RE for '+chrN+'...') - mat=pd.read_csv(GRNdir+'Primary_TF_RE_'+chrN+'.txt',sep='\t',index_col=0) - mat1=np.zeros([len(O_overlap_u),mat.shape[1]]) - O_overlap1=list(set(O_overlap_u)&set(mat.index)) - List=pd.DataFrame(range(len(mat.index)), index=mat.index) + +def load_region(GRNdir, genome, chrN, outdir): + O_overlap = merge_columns_in_bed_file(outdir + "Region_overlap_" + chrN + ".bed", 1) + N_overlap = merge_columns_in_bed_file(outdir + "Region_overlap_" + chrN + ".bed", 4) + O_overlap_u = list(set(O_overlap)) + N_overlap_u = list(set(N_overlap)) + # O_all=merge_columns_in_bed_file(GRNdir+'Peaks_'+chrN+'.bed',1) + hg19_region = merge_columns_in_bed_file(GRNdir + "hg19_Peaks_" + chrN + ".bed", 1) + hg19_region = pd.DataFrame(range(len(hg19_region)), index=hg19_region) + hg38_region = merge_columns_in_bed_file(GRNdir + "hg38_Peaks_" + chrN + ".bed", 1) + hg38_region = pd.DataFrame(range(len(hg38_region)), index=hg38_region) + if genome == "hg19": + idx = hg19_region.loc[O_overlap_u][0].values + O_overlap_u = hg38_region.index[idx].tolist() + O_overlap_hg19_u = hg19_region.index[idx].tolist() + if genome == "hg38": + idx = hg38_region.loc[O_overlap_u][0].values + O_overlap_hg19_u = hg19_region.index[idx].tolist() + return O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u + + +def load_TF_RE(GRNdir, chrN, O_overlap, O_overlap_u, O_overlap_hg19_u): + # print('load prior TF-RE for '+chrN+'...') + mat = pd.read_csv(GRNdir + "Primary_TF_RE_" + chrN + ".txt", sep="\t", index_col=0) + mat1 = np.zeros([len(O_overlap_u), mat.shape[1]]) + O_overlap1 = list(set(O_overlap_u) & set(mat.index)) + List = pd.DataFrame(range(len(mat.index)), index=mat.index) index0 = List.loc[O_overlap1][0].values - O_overlap_u_df=pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) + O_overlap_u_df = pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) index1 = O_overlap_u_df.loc[O_overlap1][0].values - mat1[index1,:] = mat.iloc[index0,:].values - #mat = pd.DataFrame(mat1,index=O_overlap_u,columns=mat.columns) - O_overlap_u_df=pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) - hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) - mat2=np.zeros([len(O_overlap),mat.shape[1]]) - index=O_overlap_u_df.loc[O_overlap][0].values - mat2=mat1[index,:] - mat=pd.DataFrame(mat2,index=O_overlap,columns=mat.columns) + mat1[index1, :] = mat.iloc[index0, :].values + # mat = pd.DataFrame(mat1,index=O_overlap_u,columns=mat.columns) + O_overlap_u_df = pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) + hg19_38 = pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) + mat2 = np.zeros([len(O_overlap), mat.shape[1]]) + index = O_overlap_u_df.loc[O_overlap][0].values + mat2 = mat1[index, :] + mat = pd.DataFrame(mat2, index=O_overlap, columns=mat.columns) return mat -def TF_RE_LINGER_chr(chr,outdir): - REName = 'data/Peaks.txt' -# Open the file in read mode + + +def TF_RE_LINGER_chr(chr, outdir): + REName = "data/Peaks.txt" + # Open the file in read mode with open(REName, "r") as file: - # Create a CSV reader - reader = csv.reader(file, delimiter='\t') - # Read the first column and store it in a list + # Create a CSV reader + reader = csv.reader(file, delimiter="\t") + # Read the first column and store it in a list first_column = [row[0] for row in reader] - REName=np.array(first_column) - idx_file=outdir+'index.txt' + REName = np.array(first_column) + idx_file = outdir + "index.txt" from scipy.stats import zscore - data0=pd.read_csv(outdir+'result_'+chr+'.txt',sep='\t') - data0.columns=['gene','x','y'] - idx_file=outdir+'index.txt' - idx=pd.read_csv(idx_file,sep='\t',header=None) - idx.columns=['gene','REid','TF_id','REid_b'] - idx.fillna('', inplace=True) - TFName=outdir+'TFName.txt' - TFName=pd.read_csv(TFName,sep='\t',header=None) - TFName.columns=['Name'] - TFName=TFName['Name'].values - TFindex=idx['TF_id'].values - REindex=idx['REid'].values - geneName=idx['gene'].values - net_all=torch.load(outdir+"net_"+chr+".pt") - data_merge=pd.read_csv(outdir+'data_merge.txt',sep='\t',header=0,index_col=0) - data_merge_temp=data_merge[data_merge['chr']==chr].index - batchsize=50 - AAA=np.abs(data0[['x']].values) - N=data_merge_temp.shape[0] - times=int(np.floor(N/batchsize)) - resultlist=[0 for i in range(times+1)] + + data0 = pd.read_csv(outdir + "result_" + chr + ".txt", sep="\t") + data0.columns = ["gene", "x", "y"] + idx_file = outdir + "index.txt" + idx = pd.read_csv(idx_file, sep="\t", header=None) + idx.columns = ["gene", "REid", "TF_id", "REid_b"] + idx.fillna("", inplace=True) + TFName = outdir + "TFName.txt" + TFName = pd.read_csv(TFName, sep="\t", header=None) + TFName.columns = ["Name"] + TFName = TFName["Name"].values + TFindex = idx["TF_id"].values + REindex = idx["REid"].values + geneName = idx["gene"].values + net_all = torch.load(outdir + "net_" + chr + ".pt") + data_merge = pd.read_csv(outdir + "data_merge.txt", sep="\t", header=0, index_col=0) + data_merge_temp = data_merge[data_merge["chr"] == chr].index + batchsize = 50 + AAA = np.abs(data0[["x"]].values) + N = data_merge_temp.shape[0] + times = int(np.floor(N / batchsize)) + resultlist = [0 for i in range(times + 1)] for ii in tqdm(range(times)): - result_all=pd.DataFrame([]) + result_all = pd.DataFrame([]) results = [] - for j in range(ii*batchsize,(ii+1)*batchsize): - if (AAA[j]>0)&(AAA[j]<10): - result=get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName) + for j in range(ii * batchsize, (ii + 1) * batchsize): + if (AAA[j] > 0) & (AAA[j] < 10): + result = get_TF_RE( + data_merge_temp, j, net_all, TFindex, TFName, REindex, REName + ) results.append(result) result_all = pd.concat(results) - result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() - resultlist[ii]=result_all - result_all=pd.DataFrame([]) - ii=ii+1 + result_all = result_all.groupby(["TF", "RE"])["score"].max().reset_index() + resultlist[ii] = result_all + result_all = pd.DataFrame([]) + ii = ii + 1 results = [] - for j in range(ii*batchsize,N): - if (AAA[j]>0)&(AAA[j]<10): - result=get_TF_RE(data_merge_temp,j,net_all,TFindex,TFName,REindex,REName) + for j in range(ii * batchsize, N): + if (AAA[j] > 0) & (AAA[j] < 10): + result = get_TF_RE( + data_merge_temp, j, net_all, TFindex, TFName, REindex, REName + ) results.append(result) result_all = pd.concat(results) - if result_all.shape[0]>0: - result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() - resultlist[ii]=result_all - result_all1=pd.concat(resultlist,axis=0) - A=result_all1.groupby(['TF', 'RE'])['score'].max().reset_index() - mat,REs,TFs=list2mat(A,'RE','TF','score') - mat=pd.DataFrame(mat,index=REs,columns=TFs) + if result_all.shape[0] > 0: + result_all = result_all.groupby(["TF", "RE"])["score"].max().reset_index() + resultlist[ii] = result_all + result_all1 = pd.concat(resultlist, axis=0) + A = result_all1.groupby(["TF", "RE"])["score"].max().reset_index() + mat, REs, TFs = list2mat(A, "RE", "TF", "score") + mat = pd.DataFrame(mat, index=REs, columns=TFs) return mat -def TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,outdir): + +def TF_RE_binding_chr(adata_RNA, adata_ATAC, GRNdir, chrN, genome, outdir): ## the regions - O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) - import numpy as np - import pandas as pd -## read the count file. - #RE=pd.DataFrame(adata_ATAC.raw.X.toarray().T,index=adata_ATAC.raw.var['gene_ids'].values,columns=adata_ATAC.obs['barcode'].values) - TG=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) -## cell annotation -## extact the overlapped peaks. - #RE=RE.loc[N_overlap] - #TFbinding=load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN) - mat=load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u) + O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u = load_region( + GRNdir, genome, chrN, outdir + ) + + ## read the count file. + # RE=pd.DataFrame(adata_ATAC.raw.X.toarray().T,index=adata_ATAC.raw.var['gene_ids'].values,columns=adata_ATAC.obs['barcode'].values) + TG = pd.DataFrame( + adata_RNA.X.toarray().T, + index=adata_RNA.var["gene_ids"].values, + columns=adata_RNA.obs["barcode"].values, + ) + ## cell annotation + ## extact the overlapped peaks. + # RE=RE.loc[N_overlap] + # TFbinding=load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN) + mat = load_TF_RE(GRNdir, chrN, O_overlap, O_overlap_u, O_overlap_hg19_u) TFs = mat.columns TFoverlap = list(set(TFs) & set(TG.index)) mat = mat[TFoverlap] - #TFbinding = TFbinding[TFoverlap] - #TF = TG.loc[TFoverlap] - #mat_m=np.mean(mat.values[mat>0]) - #mat = mat / mat_m - mat.values[mat.values<0]=0 - #TFbinding = TFbinding / TFbinding.mean(axis=1).mean() - #TF_cluster = TF.values.mean(axis=1) - #TF_cluster = TF_cluster[None,:] - #RE_cluster = RE.values.mean(axis=1) - #RE_cluster = RE_cluster[:,None] - #S = np.log(RE_cluster+0.1) + np.log(mat+TFbinding+0.1) + np.log(TF_cluster+0.1) - S = mat#+TFbinding - #S = np.exp(S) - S.index=N_overlap + # TFbinding = TFbinding[TFoverlap] + # TF = TG.loc[TFoverlap] + # mat_m=np.mean(mat.values[mat>0]) + # mat = mat / mat_m + mat.values[mat.values < 0] = 0 + # TFbinding = TFbinding / TFbinding.mean(axis=1).mean() + # TF_cluster = TF.values.mean(axis=1) + # TF_cluster = TF_cluster[None,:] + # RE_cluster = RE.values.mean(axis=1) + # RE_cluster = RE_cluster[:,None] + # S = np.log(RE_cluster+0.1) + np.log(mat+TFbinding+0.1) + np.log(TF_cluster+0.1) + S = mat # +TFbinding + # S = np.exp(S) + S.index = N_overlap mean_S = S.groupby(S.index).max() return mean_S + + import ast -def TF_RE_scNN(TFName,geneName,net_all,RE_TGlink,REName, torch_cosine = False): - batchsize=50 - REName=pd.DataFrame(range(len(REName)),index=REName) - N=RE_TGlink.shape[0] - times=int(np.floor(N/batchsize)) - resultlist=[0 for i in range(times+1)] +def TF_RE_scNN(TFName, geneName, net_all, RE_TGlink, REName, torch_cosine=False): + batchsize = 50 + REName = pd.DataFrame(range(len(REName)), index=REName) + N = RE_TGlink.shape[0] + times = int(np.floor(N / batchsize)) + resultlist = [0 for i in range(times + 1)] for ii in range(times): # result_all=pd.DataFrame([]) results = [] - for j in range(ii*batchsize,(ii+1)*batchsize): - RE_TGlink_temp=RE_TGlink.values[j,:] - temps=list(net_all[j].parameters())[0] + for j in range(ii * batchsize, (ii + 1) * batchsize): + RE_TGlink_temp = RE_TGlink.values[j, :] + temps = list(net_all[j].parameters())[0] actual_list = ast.literal_eval(RE_TGlink_temp[1]) - REidxtemp=REName.loc[actual_list].index - TFidxtemp=np.array(range(len(TFName))) - TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] - if len(REidxtemp)>0: + REidxtemp = REName.loc[actual_list].index + TFidxtemp = np.array(range(len(TFName))) + TFidxtemp = TFidxtemp[TFName != RE_TGlink_temp[0]] + if len(REidxtemp) > 0: if not torch_cosine: corr_matrix = cosine_similarity_0(temps.detach().numpy().T) else: - corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() - corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] + corr_matrix = ( + cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() + ) + corr_matrix = corr_matrix[: len(TFidxtemp), len(TFidxtemp) :] # result={'TF':[],'RE':[],'score':[]} # result = pd.DataFrame(result) for k in range(len(REidxtemp)): - datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) - datatemp['TF']=TFName[TFidxtemp].tolist() - datatemp['RE']=REidxtemp[k] + datatemp = pd.DataFrame({"score": corr_matrix[:, k].tolist()}) + datatemp["TF"] = TFName[TFidxtemp].tolist() + datatemp["RE"] = REidxtemp[k] results.append(datatemp) - result_all=pd.concat(results,axis=0) - result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() - #print(result_all) - resultlist[ii]=result_all - result_all=pd.DataFrame([]) - ii=times - if N>ii*batchsize: + result_all = pd.concat(results, axis=0) + result_all = result_all.groupby(["TF", "RE"])["score"].max().reset_index() + # print(result_all) + resultlist[ii] = result_all + result_all = pd.DataFrame([]) + ii = times + if N > ii * batchsize: results = [] - for j in range(ii*batchsize,N): - RE_TGlink_temp=RE_TGlink.values[j,:] - temps=list(net_all[j].parameters())[0] + for j in range(ii * batchsize, N): + RE_TGlink_temp = RE_TGlink.values[j, :] + temps = list(net_all[j].parameters())[0] actual_list = ast.literal_eval(RE_TGlink_temp[1]) - REidxtemp=REName.loc[actual_list].index - TFidxtemp=np.array(range(len(TFName))) - TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] - if len(REidxtemp)>0: + REidxtemp = REName.loc[actual_list].index + TFidxtemp = np.array(range(len(TFName))) + TFidxtemp = TFidxtemp[TFName != RE_TGlink_temp[0]] + if len(REidxtemp) > 0: if not torch_cosine: corr_matrix = cosine_similarity_0(temps.detach().numpy().T) else: - corr_matrix = cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() - corr_matrix=corr_matrix[:len(TFidxtemp),len(TFidxtemp):] + corr_matrix = ( + cosine_similarity_0_torch(temps.t()).detach().cpu().numpy() + ) + corr_matrix = corr_matrix[: len(TFidxtemp), len(TFidxtemp) :] for k in range(len(REidxtemp)): - datatemp=pd.DataFrame({'score':corr_matrix[:,k].tolist()}) - datatemp['TF']=TFName[TFidxtemp].tolist() - datatemp['RE']=REidxtemp[k] + datatemp = pd.DataFrame({"score": corr_matrix[:, k].tolist()}) + datatemp["TF"] = TFName[TFidxtemp].tolist() + datatemp["RE"] = REidxtemp[k] results.append(datatemp) result_all = pd.concat(results) - result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() - #print(result_all) - resultlist[ii]=result_all - result_all=pd.concat(resultlist,axis=0) - result_all=result_all.groupby(['TF', 'RE'])['score'].max().reset_index() + result_all = result_all.groupby(["TF", "RE"])["score"].max().reset_index() + # print(result_all) + resultlist[ii] = result_all + result_all = pd.concat(resultlist, axis=0) + result_all = result_all.groupby(["TF", "RE"])["score"].max().reset_index() return result_all -def load_data_scNN(GRNdir,genome): - import pandas as pd - genome_map=pd.read_csv(GRNdir+'genome_map_homer.txt',sep='\t',header=0) - genome_map.index=genome_map['genome_short'].values + +def load_data_scNN(GRNdir, genome): + genome_map = pd.read_csv(GRNdir + "genome_map_homer.txt", sep="\t", header=0) + genome_map.index = genome_map["genome_short"].values if genome in genome_map.index: - Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+genome_map.loc[genome]['species_ensembl']+'.txt',sep='\t',header=0) + Match2 = pd.read_csv( + GRNdir + + "Match_TF_motif_" + + genome_map.loc[genome]["species_ensembl"] + + ".txt", + sep="\t", + header=0, + ) else: - Match2=pd.read_csv(GRNdir+'MotifMatch.txt',sep='\t',header=0) - TFName = pd.DataFrame(Match2['TF'].unique()) - Target=pd.read_csv('data/TG_pseudobulk.tsv',sep=',',header=0,index_col=0) - TFlist=list(set(Target.index)&set(TFName[0].values)) - Exp=Target.loc[TFlist] - Opn=pd.read_csv('data/RE_pseudobulk.tsv',sep=',',header=0,index_col=0) - RE_TGlink=pd.read_csv('data/RE_gene_distance.txt',sep='\t',header=0) - RE_TGlink = RE_TGlink.groupby('gene').apply(lambda x: x['RE'].values.tolist()).reset_index() - geneoverlap=list(set(Target.index)&set(RE_TGlink['gene'])) - RE_TGlink.index=RE_TGlink['gene'] - RE_TGlink=RE_TGlink.loc[geneoverlap] - RE_TGlink=RE_TGlink.reset_index(drop=True) - return Exp,Opn,Target,RE_TGlink - - -def TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir, torch_cosine = True): - import numpy as np - import pandas as pd - from tqdm import tqdm - print('Generating cellular population TF binding strength ...') - chrom = ['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') + Match2 = pd.read_csv(GRNdir + "MotifMatch.txt", sep="\t", header=0) + TFName = pd.DataFrame(Match2["TF"].unique()) + Target = pd.read_csv("data/TG_pseudobulk.tsv", sep=",", header=0, index_col=0) + TFlist = list(set(Target.index) & set(TFName[0].values)) + Exp = Target.loc[TFlist] + Opn = pd.read_csv("data/RE_pseudobulk.tsv", sep=",", header=0, index_col=0) + RE_TGlink = pd.read_csv("data/RE_gene_distance.txt", sep="\t", header=0) + RE_TGlink = ( + RE_TGlink.groupby("gene").apply(lambda x: x["RE"].values.tolist()).reset_index() + ) + geneoverlap = list(set(Target.index) & set(RE_TGlink["gene"])) + RE_TGlink.index = RE_TGlink["gene"] + RE_TGlink = RE_TGlink.loc[geneoverlap] + RE_TGlink = RE_TGlink.reset_index(drop=True) + return Exp, Opn, Target, RE_TGlink + + +def TF_RE_binding( + GRNdir, adata_RNA, adata_ATAC, genome, method, outdir, torch_cosine=True +): + + print("Generating cellular population TF binding strength ...") + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") results = [] - if method =='baseline': - result=pd.DataFrame() + if method == "baseline": + result = pd.DataFrame() for i in tqdm(range(23)): - chrN=chrom[i] - out=TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,outdir) - out.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') + chrN = chrom[i] + out = TF_RE_binding_chr(adata_RNA, adata_ATAC, GRNdir, chrN, genome, outdir) + out.to_csv(outdir + chrN + "_cell_population_TF_RE_binding.txt", sep="\t") results.append(out) - if method=='LINGER': - result=pd.DataFrame() + if method == "LINGER": + result = pd.DataFrame() for i in tqdm(range(23)): - chrN=chrom[i] - print('Generating cellular population TF binding strength for '+chrN) - mat=TF_RE_LINGER_chr(chrN,outdir) + chrN = chrom[i] + print("Generating cellular population TF binding strength for " + chrN) + mat = TF_RE_LINGER_chr(chrN, outdir) TFs = mat.columns -## read the count file. - TG=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) + ## read the count file. + TG = pd.DataFrame( + adata_RNA.X.toarray().T, + index=adata_RNA.var["gene_ids"].values, + columns=adata_RNA.obs["barcode"].values, + ) TFoverlap = list(set(TFs) & set(TG.index)) mat = mat[TFoverlap] - mat.to_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t') + mat.to_csv(outdir + chrN + "_cell_population_TF_RE_binding.txt", sep="\t") results.append(mat) - if method=='scNN': - Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) - RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) - RE_TGlink.columns=[0,1,'chr'] - #chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] - chrlist=RE_TGlink['chr'].unique() - REName=Opn.index - geneName=Target.index - TFName=Exp.index - for jj in tqdm(range(0,len(chrlist))): - chrtemp=chrlist[jj] - RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrtemp] - net_all=torch.load(outdir+chrtemp+'_net.pt') - result_scnn=TF_RE_scNN(TFName,geneName,net_all,RE_TGlink1,REName, torch_cosine=torch_cosine) - result_scnn.to_csv(outdir+chrtemp+'_cell_population_TF_RE_binding.txt',sep='\t') + if method == "scNN": + Exp, Opn, Target, RE_TGlink = load_data_scNN(GRNdir, genome) + RE_TGlink = pd.read_csv(outdir + "RE_TGlink.txt", sep="\t", header=0) + RE_TGlink.columns = [0, 1, "chr"] + # chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] + chrlist = RE_TGlink["chr"].unique() + REName = Opn.index + geneName = Target.index + TFName = Exp.index + for jj in tqdm(range(0, len(chrlist))): + chrtemp = chrlist[jj] + RE_TGlink1 = RE_TGlink[RE_TGlink["chr"] == chrtemp] + net_all = torch.load(outdir + chrtemp + "_net.pt") + result_scnn = TF_RE_scNN( + TFName, geneName, net_all, RE_TGlink1, REName, torch_cosine=torch_cosine + ) + result_scnn.to_csv( + outdir + chrtemp + "_cell_population_TF_RE_binding.txt", sep="\t" + ) results.append(result_scnn) # result=result_all.copy() result = pd.concat(results, join="outer", axis=0) - result.to_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t') - -def load_TFbinding_scNN(GRNdir,outdir,genome): - import numpy as np - import pandas as pd - genome_map=pd.read_csv(GRNdir+'genome_map_homer.txt',sep='\t',header=0) - genome_map.index=genome_map['genome_short'].values - A=pd.read_csv(outdir+'MotifTarget.bed',sep='\t',header=0,index_col=None) - #Motif_binding,REs1,motifs=list2mat(A,'PositionID','Motif Name','MotifScore') - A['MotifScore']=np.log(1+A['MotifScore']); + result.to_csv(outdir + "cell_population_TF_RE_binding.txt", sep="\t") + + +def load_TFbinding_scNN(GRNdir, outdir, genome): + + genome_map = pd.read_csv(GRNdir + "genome_map_homer.txt", sep="\t", header=0) + genome_map.index = genome_map["genome_short"].values + A = pd.read_csv(outdir + "MotifTarget.bed", sep="\t", header=0, index_col=None) + # Motif_binding,REs1,motifs=list2mat(A,'PositionID','Motif Name','MotifScore') + A["MotifScore"] = np.log(1 + A["MotifScore"]) if genome in genome_map.index: - Match2=pd.read_csv(GRNdir+'Match_TF_motif_'+genome_map.loc[genome]['species_ensembl']+'.txt',sep='\t',header=0) + Match2 = pd.read_csv( + GRNdir + + "Match_TF_motif_" + + genome_map.loc[genome]["species_ensembl"] + + ".txt", + sep="\t", + header=0, + ) else: - Match2=pd.read_csv(GRNdir+'MotifMatch.txt',sep='\t',header=0) - TF_binding,REs1,motifs=list2mat(A,'PositionID','Motif Name','MotifScore') - TF_binding1=pd.DataFrame(TF_binding.T,index=motifs,columns=REs1) - TF_binding1['motif']=motifs - TF_binding1=TF_binding1.merge(Match2,how='inner',left_on='motif',right_on='Motif') - TF_binding=TF_binding1.groupby(['TF'])[REs1].max() - TF_binding=TF_binding.reset_index() - TF_binding.index=TF_binding['TF'] - TF_binding=TF_binding[REs1] + Match2 = pd.read_csv(GRNdir + "MotifMatch.txt", sep="\t", header=0) + TF_binding, REs1, motifs = list2mat(A, "PositionID", "Motif Name", "MotifScore") + TF_binding1 = pd.DataFrame(TF_binding.T, index=motifs, columns=REs1) + TF_binding1["motif"] = motifs + TF_binding1 = TF_binding1.merge( + Match2, how="inner", left_on="motif", right_on="Motif" + ) + TF_binding = TF_binding1.groupby(["TF"])[REs1].max() + TF_binding = TF_binding.reset_index() + TF_binding.index = TF_binding["TF"] + TF_binding = TF_binding[REs1] return TF_binding.T -def cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,celltype,outdir,method,mat): + +def cell_type_specific_TF_RE_binding_chr( + adata_RNA, adata_ATAC, GRNdir, chrN, genome, celltype, outdir, method, mat +): + ## the regions ## the regions - ## the regions - O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) - import numpy as np - import pandas as pd - label=adata_RNA.obs['label'].values.tolist() - labelset=list(set(label)) - temp=adata_ATAC.X[np.array(label)==celltype,:].mean(axis=0) - RE=pd.DataFrame(temp.T,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==celltype,:].mean(axis=0) - TG=pd.DataFrame(temp.T,index=adata_RNA.var['gene_ids'].values,columns=['values']) + O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u = load_region( + GRNdir, genome, chrN, outdir + ) + + label = adata_RNA.obs["label"].values.tolist() + labelset = list(set(label)) + temp = adata_ATAC.X[np.array(label) == celltype, :].mean(axis=0) + RE = pd.DataFrame( + temp.T, index=adata_ATAC.var["gene_ids"].values, columns=["values"] + ) + temp = adata_RNA.X[np.array(label) == celltype, :].mean(axis=0) + TG = pd.DataFrame( + temp.T, index=adata_RNA.var["gene_ids"].values, columns=["values"] + ) del temp -## cell annotation -## extact the overlapped peaks. - RE=RE.loc[N_overlap] - TFbinding=load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN) - if method=='LINGER': - other_RE=list(set(N_overlap)-set(mat.index)) - if len(other_RE)>0: - B_arr = pd.DataFrame(np.zeros((len(other_RE), mat.shape[1])), columns=mat.columns, index=other_RE) + ## cell annotation + ## extact the overlapped peaks. + RE = RE.loc[N_overlap] + TFbinding = load_TFbinding(GRNdir, O_overlap, O_overlap_u, O_overlap_hg19_u, chrN) + if method == "LINGER": + other_RE = list(set(N_overlap) - set(mat.index)) + if len(other_RE) > 0: + B_arr = pd.DataFrame( + np.zeros((len(other_RE), mat.shape[1])), + columns=mat.columns, + index=other_RE, + ) mat = pd.concat([mat, B_arr]) mat = mat.loc[N_overlap] - if method=='baseline': - mat=load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u) - mat.index=N_overlap + if method == "baseline": + mat = load_TF_RE(GRNdir, chrN, O_overlap, O_overlap_u, O_overlap_hg19_u) + mat.index = N_overlap TFs = mat.columns TFoverlap = list(set(TFs) & set(TG.index)) mat = mat[TFoverlap] TFbinding = TFbinding[TFoverlap] - TFbinding.index=N_overlap + TFbinding.index = N_overlap TF = TG.loc[TFoverlap] - mat_m=np.mean(mat.values[mat>0]) + mat_m = np.mean(mat.values[mat > 0]) mat = mat / mat_m - mat.values[mat.values<0]=0 + mat.values[mat.values < 0] = 0 TFbinding = TFbinding / TFbinding.mean(axis=1).mean() - TF_cluster = TF.values#[:,np.array(label)==celltype].mean(axis=1) - TF_cluster=TF_cluster/TF_cluster.mean() - #TF_cluster = TF_cluster[None,:] - RE_cluster = RE.values#[:,np.array(label)==celltype].mean(axis=1) - RE_cluster=RE_cluster/RE_cluster.mean() - #RE_cluster = RE_cluster[:,None] - S = (np.log(RE_cluster+0.1) + np.log(mat+TFbinding+0.1)).T + np.log(TF_cluster+0.1) + TF_cluster = TF.values # [:,np.array(label)==celltype].mean(axis=1) + TF_cluster = TF_cluster / TF_cluster.mean() + # TF_cluster = TF_cluster[None,:] + RE_cluster = RE.values # [:,np.array(label)==celltype].mean(axis=1) + RE_cluster = RE_cluster / RE_cluster.mean() + # RE_cluster = RE_cluster[:,None] + S = (np.log(RE_cluster + 0.1) + np.log(mat + TFbinding + 0.1)).T + np.log( + TF_cluster + 0.1 + ) S = np.exp(S.T) - S.index=N_overlap + S.index = N_overlap S_all = S.groupby(S.index).max() return S_all - -def cell_type_specific_TF_RE_binding_score_scNN(mat,TFbinding,RE,TG,TFoverlap): +def cell_type_specific_TF_RE_binding_score_scNN(mat, TFbinding, RE, TG, TFoverlap): TF = TG.loc[TFoverlap] - mat_m=np.mean(mat.values[mat>0]) + mat_m = np.mean(mat.values[mat > 0]) mat = mat / mat_m - mat.values[mat.values<0]=0 + mat.values[mat.values < 0] = 0 TFbinding = TFbinding / TFbinding.mean(axis=1).mean() - TF_cluster = TF.values#[:,np.array(label)==celltype].mean(axis=1) - TF_cluster=TF_cluster/TF_cluster.mean() - #TF_cluster = TF_cluster[None,:] - RE_cluster = RE.values#[:,np.array(label)==celltype].mean(axis=1) - RE_cluster=RE_cluster/RE_cluster.mean() - #RE_cluster = RE_cluster[:,None] - S = (np.log(RE_cluster+0.1) + np.log(mat+TFbinding+0.1)).T + np.log(TF_cluster+0.1) + TF_cluster = TF.values # [:,np.array(label)==celltype].mean(axis=1) + TF_cluster = TF_cluster / TF_cluster.mean() + # TF_cluster = TF_cluster[None,:] + RE_cluster = RE.values # [:,np.array(label)==celltype].mean(axis=1) + RE_cluster = RE_cluster / RE_cluster.mean() + # RE_cluster = RE_cluster[:,None] + S = (np.log(RE_cluster + 0.1) + np.log(mat + TFbinding + 0.1)).T + np.log( + TF_cluster + 0.1 + ) S = np.exp(S.T) - S.index=mat.index + S.index = mat.index return S -def cell_type_specific_TF_RE_binding(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdir,method): - label=adata_RNA.obs['label'].values.tolist() - labelset=list(set(label)) - if (celltype == 'all')&(method!='scNN'): + +def cell_type_specific_TF_RE_binding( + GRNdir, adata_RNA, adata_ATAC, genome, celltype, outdir, method +): + label = adata_RNA.obs["label"].values.tolist() + labelset = list(set(label)) + if (celltype == "all") & (method != "scNN"): results = [] for label0 in labelset: - print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') - result=pd.DataFrame() - from tqdm import tqdm + print( + "Generating cell type specitic TF binding potential for cell type " + + str(label0) + + "..." + ) + result = pd.DataFrame() for i in tqdm(range(22)): - chrN='chr'+str(i+1) - mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) - out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,label0,outdir,method,mat) + chrN = "chr" + str(i + 1) + mat = pd.read_csv( + outdir + chrN + "_cell_population_TF_RE_binding.txt", + sep="\t", + index_col=0, + header=0, + ) + out = cell_type_specific_TF_RE_binding_chr( + adata_RNA, + adata_ATAC, + GRNdir, + chrN, + genome, + label0, + outdir, + method, + mat, + ) results.append(out) - chrN='chrX' - mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) - out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,label0,outdir,method,mat) + chrN = "chrX" + mat = pd.read_csv( + outdir + chrN + "_cell_population_TF_RE_binding.txt", + sep="\t", + index_col=0, + header=0, + ) + out = cell_type_specific_TF_RE_binding_chr( + adata_RNA, adata_ATAC, GRNdir, chrN, genome, label0, outdir, method, mat + ) results.append(out) - result = pd.concat(results, join='outer', axis=0).fillna(0) - result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt', sep='\t') - elif method!='scNN': - result=pd.DataFrame() - from tqdm import tqdm - chrom=['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') + result = pd.concat(results, join="outer", axis=0).fillna(0) + result.to_csv( + outdir + "cell_type_specific_TF_RE_binding_" + str(label0) + ".txt", + sep="\t", + ) + elif method != "scNN": + result = pd.DataFrame() + + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") results = [] for i in tqdm(range(23)): - chrN=chrom[i] - mat=pd.read_csv(outdir+chrN+'_cell_population_TF_RE_binding.txt',sep='\t',index_col=0,header=0) - out=cell_type_specific_TF_RE_binding_chr(adata_RNA,adata_ATAC,GRNdir,chrN,genome,celltype,outdir,method,mat) + chrN = chrom[i] + mat = pd.read_csv( + outdir + chrN + "_cell_population_TF_RE_binding.txt", + sep="\t", + index_col=0, + header=0, + ) + out = cell_type_specific_TF_RE_binding_chr( + adata_RNA, + adata_ATAC, + GRNdir, + chrN, + genome, + celltype, + outdir, + method, + mat, + ) result = results.append(out) - result = pd.concat(results, axis = 1).fillna(0) - result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(celltype)+'.txt', sep='\t') - elif (celltype == 'all')&(method=='scNN'): - A=pd.read_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t',header=0,index_col=0) - mat,REs,TFs=list2mat(A,'RE','TF','score') - mat=pd.DataFrame(mat,index=REs,columns=TFs) + result = pd.concat(results, axis=1).fillna(0) + result.to_csv( + outdir + "cell_type_specific_TF_RE_binding_" + str(celltype) + ".txt", + sep="\t", + ) + elif (celltype == "all") & (method == "scNN"): + A = pd.read_csv( + outdir + "cell_population_TF_RE_binding.txt", + sep="\t", + header=0, + index_col=0, + ) + mat, REs, TFs = list2mat(A, "RE", "TF", "score") + mat = pd.DataFrame(mat, index=REs, columns=TFs) TFs = mat.columns - TFbinding=load_TFbinding_scNN(GRNdir,outdir,genome) - TG=pd.DataFrame([],index=adata_RNA.var['gene_ids'].values) + TFbinding = load_TFbinding_scNN(GRNdir, outdir, genome) + TG = pd.DataFrame([], index=adata_RNA.var["gene_ids"].values) TFoverlap = list(set(TFs) & set(TG.index)) - TFoverlap=list(set(TFoverlap) & set(TFbinding.columns)) + TFoverlap = list(set(TFoverlap) & set(TFbinding.columns)) mat = mat[TFoverlap] TFbinding = TFbinding[TFoverlap] - REoverlap=list(set(TFbinding.index)&set(mat.index)) - TFbinding=TFbinding.loc[REoverlap] - TFbinding1=np.zeros((mat.shape[0],len(TFoverlap))) - REidx=pd.DataFrame(range(mat.shape[0]),index=mat.index) - TFbinding1[REidx.loc[TFbinding.index][0].values,:]=TFbinding.values - TFbinding1 = pd.DataFrame(TFbinding1,index=mat.index,columns=TFoverlap) - TFbinding=TFbinding1.copy() + REoverlap = list(set(TFbinding.index) & set(mat.index)) + TFbinding = TFbinding.loc[REoverlap] + TFbinding1 = np.zeros((mat.shape[0], len(TFoverlap))) + REidx = pd.DataFrame(range(mat.shape[0]), index=mat.index) + TFbinding1[REidx.loc[TFbinding.index][0].values, :] = TFbinding.values + TFbinding1 = pd.DataFrame(TFbinding1, index=mat.index, columns=TFoverlap) + TFbinding = TFbinding1.copy() for label0 in labelset: - print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') - from tqdm import tqdm - temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T - RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T - TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) - RE=RE.loc[REs] - result=cell_type_specific_TF_RE_binding_score_scNN(mat,TFbinding,RE,TG,TFoverlap) - result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt', sep='\t') + print( + "Generating cell type specitic TF binding potential for cell type " + + str(label0) + + "..." + ) + + temp = adata_ATAC.X[np.array(label) == label0, :].mean(axis=0).T + RE = pd.DataFrame( + temp, index=adata_ATAC.var["gene_ids"].values, columns=["values"] + ) + temp = adata_RNA.X[np.array(label) == label0, :].mean(axis=0).T + TG = pd.DataFrame( + temp, index=adata_RNA.var["gene_ids"].values, columns=["values"] + ) + RE = RE.loc[REs] + result = cell_type_specific_TF_RE_binding_score_scNN( + mat, TFbinding, RE, TG, TFoverlap + ) + result.to_csv( + outdir + "cell_type_specific_TF_RE_binding_" + str(label0) + ".txt", + sep="\t", + ) else: - label0=celltype - A=pd.read_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t',header=0,index_col=0) - mat,REs,TFs=list2mat(A,'RE','TF','score') - mat=pd.DataFrame(mat,index=REs,columns=TFs) + label0 = celltype + A = pd.read_csv( + outdir + "cell_population_TF_RE_binding.txt", + sep="\t", + header=0, + index_col=0, + ) + mat, REs, TFs = list2mat(A, "RE", "TF", "score") + mat = pd.DataFrame(mat, index=REs, columns=TFs) TFs = mat.columns - TFbinding=load_TFbinding_scNN(GRNdir,outdir,genome) - TG=pd.DataFrame([],index=adata_RNA.var['gene_ids'].values) + TFbinding = load_TFbinding_scNN(GRNdir, outdir, genome) + TG = pd.DataFrame([], index=adata_RNA.var["gene_ids"].values) TFoverlap = list(set(TFs) & set(TG.index)) - TFoverlap=list(set(TFoverlap) & set(TFbinding.columns)) + TFoverlap = list(set(TFoverlap) & set(TFbinding.columns)) mat = mat[TFoverlap] TFbinding = TFbinding[TFoverlap] - REoverlap=list(set(TFbinding.index)&set(RE.index)) - TFbinding=TFbinding.loc[REoverlap] - TFbinding1=np.zeros((mat.shape[0],len(TFoverlap))) - REidx=pd.DataFrame(range(mat.shape[0]),index=mat.index) - TFbinding1[REidx.loc[TFbinding.index][0].values,:]=TFbinding.values - TFbinding1 = pd.DataFrame(TFbinding1,index=mat.index,columns=TFoverlap) - print('Generating cell type specitic TF binding potential for cell type '+ str(label0)+'...') - temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T - RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T - TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) - RE=RE.loc[REs] - result=cell_type_specific_TF_RE_binding_score_scNN(mat,TFbinding,RE,TG,TFoverlap) - result.to_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt', sep='\t') - - -def load_shap(chr,outdir): - import csv - - import numpy as np - import pandas as pd - import torch - #print('loading shapley value '+chr+' ...') - shap_all=torch.load(outdir+"shap_"+chr+".pt") - import pandas as pd - idx_file=outdir+'index.txt' - TFName=outdir+'TFName.txt' - #TFE=Input_dir+'TFexp.txt' - REName = 'data/Peaks.txt' -# Open the file in read mode + REoverlap = list(set(TFbinding.index) & set(RE.index)) + TFbinding = TFbinding.loc[REoverlap] + TFbinding1 = np.zeros((mat.shape[0], len(TFoverlap))) + REidx = pd.DataFrame(range(mat.shape[0]), index=mat.index) + TFbinding1[REidx.loc[TFbinding.index][0].values, :] = TFbinding.values + TFbinding1 = pd.DataFrame(TFbinding1, index=mat.index, columns=TFoverlap) + print( + "Generating cell type specitic TF binding potential for cell type " + + str(label0) + + "..." + ) + temp = adata_ATAC.X[np.array(label) == label0, :].mean(axis=0).T + RE = pd.DataFrame( + temp, index=adata_ATAC.var["gene_ids"].values, columns=["values"] + ) + temp = adata_RNA.X[np.array(label) == label0, :].mean(axis=0).T + TG = pd.DataFrame( + temp, index=adata_RNA.var["gene_ids"].values, columns=["values"] + ) + RE = RE.loc[REs] + result = cell_type_specific_TF_RE_binding_score_scNN( + mat, TFbinding, RE, TG, TFoverlap + ) + result.to_csv( + outdir + "cell_type_specific_TF_RE_binding_" + str(label0) + ".txt", + sep="\t", + ) + + +def load_shap(chr, outdir): + + # print('loading shapley value '+chr+' ...') + shap_all = torch.load(outdir + "shap_" + chr + ".pt") + + idx_file = outdir + "index.txt" + TFName = outdir + "TFName.txt" + # TFE=Input_dir+'TFexp.txt' + REName = "data/Peaks.txt" + # Open the file in read mode with open(REName, "r") as file: - # Create a CSV reader - reader = csv.reader(file, delimiter='\t') - # Read the first column and store it in a list + # Create a CSV reader + reader = csv.reader(file, delimiter="\t") + # Read the first column and store it in a list first_column = [row[0] for row in reader] - REName=np.array(first_column) - idx=pd.read_csv(idx_file,sep='\t',header=None) - idx.fillna('', inplace=True) - TFName=pd.read_csv(TFName,sep='\t',header=None) - import numpy as np - idx.columns=['gene','REid','TF_id','REid_b'] - TFName.columns=['Name'] - TFName=TFName['Name'].values - #TFE=pd.read_csv(TFE,header=None,sep='\t') + REName = np.array(first_column) + idx = pd.read_csv(idx_file, sep="\t", header=None) + idx.fillna("", inplace=True) + TFName = pd.read_csv(TFName, sep="\t", header=None) + + idx.columns = ["gene", "REid", "TF_id", "REid_b"] + TFName.columns = ["Name"] + TFName = TFName["Name"].values + # TFE=pd.read_csv(TFE,header=None,sep='\t') from scipy.stats import zscore - TFindex=idx['TF_id'].values - REindex=idx['REid'].values - geneName=idx['gene'].values - data_merge=pd.read_csv(outdir+'data_merge.txt',sep='\t',header=0,index_col=0) - data_merge_temp=data_merge[data_merge['chr']==chr] - return data_merge_temp,geneName,REindex,TFindex,shap_all,TFName,REName -def cis_shap(chr,outdir): - RE_2=[] - TG_2=[] - score_2=[] - data_merge_temp,geneName,REindex,TFindex,shap_all,TFName,REName=load_shap(chr,outdir) - from tqdm import tqdm + + TFindex = idx["TF_id"].values + REindex = idx["REid"].values + geneName = idx["gene"].values + data_merge = pd.read_csv(outdir + "data_merge.txt", sep="\t", header=0, index_col=0) + data_merge_temp = data_merge[data_merge["chr"] == chr] + return data_merge_temp, geneName, REindex, TFindex, shap_all, TFName, REName + + +def cis_shap(chr, outdir): + RE_2 = [] + TG_2 = [] + score_2 = [] + data_merge_temp, geneName, REindex, TFindex, shap_all, TFName, REName = load_shap( + chr, outdir + ) + for j in tqdm(range(data_merge_temp.shape[0])): - ii=data_merge_temp.index[j] + ii = data_merge_temp.index[j] if ii in shap_all.keys(): - AA0=shap_all[ii] - REidxtemp=REindex[ii] - REidxtemp=str(REidxtemp).split('_') - #AA0[:,0:len(TFidxtemp)]=np.multiply(AA0[:,0:len(TFidxtemp)],TFE.values[np.array(TFidxtemp),:].T) - temps=np.abs(AA0).mean(axis=0) - #zscored_arr = zscore(temps) + AA0 = shap_all[ii] + REidxtemp = REindex[ii] + REidxtemp = str(REidxtemp).split("_") + # AA0[:,0:len(TFidxtemp)]=np.multiply(AA0[:,0:len(TFidxtemp)],TFE.values[np.array(TFidxtemp),:].T) + temps = np.abs(AA0).mean(axis=0) + # zscored_arr = zscore(temps) zscored_arr = np.nan_to_num(temps, nan=0.0) - if (REidxtemp[0]=='') : - REidxtemp=[] + if REidxtemp[0] == "": + REidxtemp = [] else: - REidxtemp=[int(REidxtemp[i]) for i in range(len(REidxtemp))] - if len(REidxtemp)>0: - REName_temp=REName[np.array(REidxtemp)] + REidxtemp = [int(REidxtemp[i]) for i in range(len(REidxtemp))] + if len(REidxtemp) > 0: + REName_temp = REName[np.array(REidxtemp)] for k in range(len(REidxtemp)): TG_2.append(geneName[ii]) RE_2.append(REName_temp[k]) - score_2.append(zscored_arr[k+len(zscored_arr)-len(REidxtemp)]) - RE_TG=pd.DataFrame(TG_2) - RE_TG.columns=['TG'] - RE_TG['RE']=RE_2 - RE_TG['score']=score_2 - RE_TG=RE_TG.groupby(['RE', 'TG'])['score'].max().reset_index() + score_2.append(zscored_arr[k + len(zscored_arr) - len(REidxtemp)]) + RE_TG = pd.DataFrame(TG_2) + RE_TG.columns = ["TG"] + RE_TG["RE"] = RE_2 + RE_TG["score"] = score_2 + RE_TG = RE_TG.groupby(["RE", "TG"])["score"].max().reset_index() return RE_TG -def trans_shap(chr,outdir): - TG_1=[] - TF_1=[] - score_1=[] - data_merge_temp,geneName,REindex,TFindex,shap_all,TFName,REName=load_shap(chr,outdir) - from tqdm import tqdm + + +def trans_shap(chr, outdir): + TG_1 = [] + TF_1 = [] + score_1 = [] + data_merge_temp, geneName, REindex, TFindex, shap_all, TFName, REName = load_shap( + chr, outdir + ) + for j in tqdm(range(data_merge_temp.shape[0])): - ii=data_merge_temp.index[j] + ii = data_merge_temp.index[j] if ii in shap_all.keys(): - AA0=shap_all[ii] - TFidxtemp=TFindex[ii] - TFidxtemp=TFidxtemp.split('_') - TFidxtemp=[int(TFidxtemp[i]) for i in range(len(TFidxtemp))] - TFName_temp=TFName[np.array(TFidxtemp)] - #AA0[:,0:len(TFidxtemp)]=np.multiply(AA0[:,0:len(TFidxtemp)],TFE.values[np.array(TFidxtemp),:].T) - temps=np.abs(AA0).mean(axis=0) - #zscored_arr = zscore(temps) + AA0 = shap_all[ii] + TFidxtemp = TFindex[ii] + TFidxtemp = TFidxtemp.split("_") + TFidxtemp = [int(TFidxtemp[i]) for i in range(len(TFidxtemp))] + TFName_temp = TFName[np.array(TFidxtemp)] + # AA0[:,0:len(TFidxtemp)]=np.multiply(AA0[:,0:len(TFidxtemp)],TFE.values[np.array(TFidxtemp),:].T) + temps = np.abs(AA0).mean(axis=0) + # zscored_arr = zscore(temps) zscored_arr = np.nan_to_num(temps, nan=0.0) for k in range(len(TFidxtemp)): TG_1.append(geneName[ii]) TF_1.append(TFName_temp[k]) score_1.append(zscored_arr[k]) - TF_TG=pd.DataFrame(TG_1) - TF_TG.columns=['TG'] - TF_TG['TF']=TF_1 - TF_TG['score']=score_1 - mat,TGs,TFs=list2mat(TF_TG,'TG','TF','score') - mat=pd.DataFrame(mat,index=TGs,columns=TFs) + TF_TG = pd.DataFrame(TG_1) + TF_TG.columns = ["TG"] + TF_TG["TF"] = TF_1 + TF_TG["score"] = score_1 + mat, TGs, TFs = list2mat(TF_TG, "TG", "TF", "score") + mat = pd.DataFrame(mat, index=TGs, columns=TFs) mat.fillna(0, inplace=True) return mat -def load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap): - #print('load prior RE-TG ...') + +def load_RE_TG(GRNdir, chrN, O_overlap_u, O_overlap_hg19_u, O_overlap): + # print('load prior RE-TG ...') from scipy.sparse import coo_matrix - primary_s=pd.read_csv(GRNdir+'Primary_RE_TG_'+chrN+'.txt',sep='\t') - primary_s["RE"] = primary_s["RE"].apply(lambda x: x.split('_')[0]+':'+x.split('_')[1]+'-'+x.split('_')[2]) + + primary_s = pd.read_csv(GRNdir + "Primary_RE_TG_" + chrN + ".txt", sep="\t") + primary_s["RE"] = primary_s["RE"].apply( + lambda x: x.split("_")[0] + ":" + x.split("_")[1] + "-" + x.split("_")[2] + ) primary_s = primary_s[primary_s["RE"].isin(O_overlap_u)] - TGset=primary_s["TG"].unique() - REset=O_overlap_u + TGset = primary_s["TG"].unique() + REset = O_overlap_u # Create a dictionary mapping column names and row names to integer indices col_dict = {col: i for i, col in enumerate(TGset)} row_dict = {row: i for i, row in enumerate(REset)} -# Map the column names and row names to integer indices in the DataFrame - primary_s.loc[:,"col_index"] = primary_s["TG"].map(col_dict) - primary_s.loc[:,"row_index"] = primary_s["RE"].map(row_dict) + # Map the column names and row names to integer indices in the DataFrame + primary_s.loc[:, "col_index"] = primary_s["TG"].map(col_dict) + primary_s.loc[:, "row_index"] = primary_s["RE"].map(row_dict) # Extract the column indices, row indices, and values from the DataFrame col_indices = primary_s["col_index"].tolist() row_indices = primary_s["row_index"].tolist() @@ -735,344 +917,436 @@ def load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap): sparse_S.colnames = TGset sparse_S.rownames = REset array = sparse_S.toarray() - O_overlap_u_df=pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) - hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) - array2=np.zeros([len(O_overlap),array.shape[1]]) - index=O_overlap_u_df.loc[O_overlap][0].values - array2=array[index,:] - array=pd.DataFrame(array2,index=O_overlap,columns=TGset) - return array,TGset -def load_RE_TG_distance(GRNdir,chrN,O_overlap_hg19_u,O_overlap_u,O_overlap,TGoverlap): - #print('load RE-TG distance for '+chrN+'...') + O_overlap_u_df = pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) + hg19_38 = pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) + array2 = np.zeros([len(O_overlap), array.shape[1]]) + index = O_overlap_u_df.loc[O_overlap][0].values + array2 = array[index, :] + array = pd.DataFrame(array2, index=O_overlap, columns=TGset) + return array, TGset + + +def load_RE_TG_distance( + GRNdir, chrN, O_overlap_hg19_u, O_overlap_u, O_overlap, TGoverlap +): + # print('load RE-TG distance for '+chrN+'...') from scipy.sparse import coo_matrix - Dis=pd.read_csv(GRNdir+'RE_TG_distance_'+chrN+'.txt',sep='\t',header=None) - Dis.columns=['RE','TG','dis'] - Dis["RE"] = Dis["RE"].apply(lambda x: x.split('_')[0]+':'+x.split('_')[1]+'-'+x.split('_')[2]) + + Dis = pd.read_csv(GRNdir + "RE_TG_distance_" + chrN + ".txt", sep="\t", header=None) + Dis.columns = ["RE", "TG", "dis"] + Dis["RE"] = Dis["RE"].apply( + lambda x: x.split("_")[0] + ":" + x.split("_")[1] + "-" + x.split("_")[2] + ) Dis = Dis[Dis["RE"].isin(O_overlap_hg19_u)] - Dis = Dis[Dis['TG'].isin(TGoverlap)] + Dis = Dis[Dis["TG"].isin(TGoverlap)] col_dict = {col: i for i, col in enumerate(TGoverlap)} row_dict = {row: i for i, row in enumerate(O_overlap_hg19_u)} -# Map the column names and row names to integer indices in the DataFrame - Dis.loc[:,"col_index"] = Dis["TG"].map(col_dict) - Dis.loc[:,"row_index"] = Dis["RE"].map(row_dict) + # Map the column names and row names to integer indices in the DataFrame + Dis.loc[:, "col_index"] = Dis["TG"].map(col_dict) + Dis.loc[:, "row_index"] = Dis["RE"].map(row_dict) col_indices = Dis["col_index"].tolist() row_indices = Dis["row_index"].tolist() values = Dis["dis"].tolist() -# Create the sparse matrix using coo_matrix - sparse_dis = coo_matrix((values, (row_indices, col_indices)),shape=(len(O_overlap_u), len(TGoverlap))) + # Create the sparse matrix using coo_matrix + sparse_dis = coo_matrix( + (values, (row_indices, col_indices)), shape=(len(O_overlap_u), len(TGoverlap)) + ) sparse_dis.colnames = TGoverlap sparse_dis.rownames = O_overlap_u sparse_dis = sparse_dis.tocsc() - A=sparse_dis.multiply(1 / 25000) - A.data +=0.5 + A = sparse_dis.multiply(1 / 25000) + A.data += 0.5 A.data = np.exp(-A.data) - sparse_dis=A + sparse_dis = A array = sparse_dis.toarray() - O_overlap_u_df=pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) - hg19_38=pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) - array2=np.zeros([len(O_overlap),array.shape[1]]) - index=O_overlap_u_df.loc[O_overlap][0].values - array2=array[index,:] - array=pd.DataFrame(array2,index=O_overlap,columns=TGoverlap) + O_overlap_u_df = pd.DataFrame(range(len(O_overlap_u)), index=O_overlap_u) + hg19_38 = pd.DataFrame(O_overlap_u, index=O_overlap_hg19_u) + array2 = np.zeros([len(O_overlap), array.shape[1]]) + index = O_overlap_u_df.loc[O_overlap][0].values + array2 = array[index, :] + array = pd.DataFrame(array2, index=O_overlap, columns=TGoverlap) return array def load_RE_TG_scNN(outdir): - #print('load prior RE-TG ...') - import numpy as np - import pandas as pd + # print('load prior RE-TG ...') + from scipy.sparse import coo_matrix - dis=pd.read_csv('data/RE_gene_distance.txt',sep='\t',header=0) - dis['distance']=np.exp(-(0.5+dis['distance']/25000)) - REs=dis['RE'].unique() - TGs=dis['gene'].unique() - cis=pd.read_csv(outdir+'cell_population_cis_regulatory.txt',sep='\t',header=None) - cis.columns=['RE','TG','score'] - REs2=cis['RE'].unique() - TGs2=cis['TG'].unique() - REoverlap=list(set(REs2)&set(REs)) - TGoverlap=list(set(TGs)&set(TGs2)) - cisGRN,REs2,TGs2=list2mat_s(cis,REoverlap,TGoverlap,'RE','TG','score') - dis=dis[dis['RE'].isin(REoverlap)] - dis=dis[dis['gene'].isin(TGoverlap)] - distance,REs,TGs=list2mat_s(dis,REoverlap,TGoverlap,'RE','gene','distance') - return distance,cisGRN,REoverlap,TGoverlap - -def cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir): - import numpy as np - import pandas as pd + + dis = pd.read_csv("data/RE_gene_distance.txt", sep="\t", header=0) + dis["distance"] = np.exp(-(0.5 + dis["distance"] / 25000)) + REs = dis["RE"].unique() + TGs = dis["gene"].unique() + cis = pd.read_csv( + outdir + "cell_population_cis_regulatory.txt", sep="\t", header=None + ) + cis.columns = ["RE", "TG", "score"] + REs2 = cis["RE"].unique() + TGs2 = cis["TG"].unique() + REoverlap = list(set(REs2) & set(REs)) + TGoverlap = list(set(TGs) & set(TGs2)) + cisGRN, REs2, TGs2 = list2mat_s(cis, REoverlap, TGoverlap, "RE", "TG", "score") + dis = dis[dis["RE"].isin(REoverlap)] + dis = dis[dis["gene"].isin(TGoverlap)] + distance, REs, TGs = list2mat_s(dis, REoverlap, TGoverlap, "RE", "gene", "distance") + return distance, cisGRN, REoverlap, TGoverlap + + +def cis_reg_chr(GRNdir, adata_RNA, adata_ATAC, genome, chrN, outdir): + from scipy.sparse import coo_matrix, csc_matrix - O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) - sparse_S,TGset=load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap) - RE=pd.DataFrame(adata_ATAC.X.toarray().T,index=adata_ATAC.var['gene_ids'].values,columns=adata_ATAC.obs['barcode'].values) -## cell annotation -## extact the overlapped peaks. - RE=RE.loc[N_overlap] - RE=RE.mean(axis=1) - RE=RE/RE.mean()+0.1 + + O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u = load_region( + GRNdir, genome, chrN, outdir + ) + sparse_S, TGset = load_RE_TG(GRNdir, chrN, O_overlap_u, O_overlap_hg19_u, O_overlap) + RE = pd.DataFrame( + adata_ATAC.X.toarray().T, + index=adata_ATAC.var["gene_ids"].values, + columns=adata_ATAC.obs["barcode"].values, + ) + ## cell annotation + ## extact the overlapped peaks. + RE = RE.loc[N_overlap] + RE = RE.mean(axis=1) + RE = RE / RE.mean() + 0.1 ## select the genes - TG=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) - TGoverlap=list(set(TGset)&set(TG.index)) - #target_col_indices = [col_dict[col] for col in TGoverlap] + TG = pd.DataFrame( + adata_RNA.X.toarray().T, + index=adata_RNA.var["gene_ids"].values, + columns=adata_RNA.obs["barcode"].values, + ) + TGoverlap = list(set(TGset) & set(TG.index)) + # target_col_indices = [col_dict[col] for col in TGoverlap] sparse_S = sparse_S[TGoverlap] - TG=TG.loc[TGoverlap] - TG=TG.mean(axis=1) - TG=TG/TG.mean()+0.1 - sparse_dis=load_RE_TG_distance(GRNdir,chrN,O_overlap_hg19_u,O_overlap_u,O_overlap,TGoverlap) - sparse_S+=0.1 - #Score=csc_matrix(RE).T.multiply(sparse_S.values).multiply(sparse_dis.values).multiply(csc_matrix(TG)).toarray() - Score=np.multiply(sparse_S.values,sparse_dis.values) - Score=pd.DataFrame(Score,index=N_overlap,columns=TGoverlap) - Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] + TG = TG.loc[TGoverlap] + TG = TG.mean(axis=1) + TG = TG / TG.mean() + 0.1 + sparse_dis = load_RE_TG_distance( + GRNdir, chrN, O_overlap_hg19_u, O_overlap_u, O_overlap, TGoverlap + ) + sparse_S += 0.1 + # Score=csc_matrix(RE).T.multiply(sparse_S.values).multiply(sparse_dis.values).multiply(csc_matrix(TG)).toarray() + Score = np.multiply(sparse_S.values, sparse_dis.values) + Score = pd.DataFrame(Score, index=N_overlap, columns=TGoverlap) + Score = Score.groupby(Score.index).max() + data = Score.values[Score.values != 0] rows, cols = np.nonzero(Score.values) - coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) - combined[:,0]=Score.index[coo.row] - combined[:,1]=np.array(TGoverlap)[coo.col] - combined[:,2]=coo.data - combined=pd.DataFrame(combined) + coo = coo_matrix((data, (rows, cols)), shape=Score.shape) + combined = np.zeros([len(data), 3], dtype=object) + combined[:, 0] = Score.index[coo.row] + combined[:, 1] = np.array(TGoverlap)[coo.col] + combined[:, 2] = coo.data + combined = pd.DataFrame(combined) return combined -def cis_shap_scNN(chrtemp,outdir,RE_TGlink1,REName,TFName): + +def cis_shap_scNN(chrtemp, outdir, RE_TGlink1, REName, TFName): import ast - REName=pd.DataFrame(range(len(REName)),index=REName) - RE_2=[] - TG_2=[] - score_2=[] - shap_all=torch.load(outdir+chrtemp+"_shap"+".pt") - N=RE_TGlink1.shape[0] + + REName = pd.DataFrame(range(len(REName)), index=REName) + RE_2 = [] + TG_2 = [] + score_2 = [] + shap_all = torch.load(outdir + chrtemp + "_shap" + ".pt") + N = RE_TGlink1.shape[0] for ii in tqdm(range(N)): - AA0=shap_all[ii] - RE_TGlink_temp=RE_TGlink1.values[ii,:] + AA0 = shap_all[ii] + RE_TGlink_temp = RE_TGlink1.values[ii, :] actual_list = ast.literal_eval(RE_TGlink_temp[1]) - REidxtemp=REName.loc[actual_list].index - TFidxtemp=np.array(range(len(TFName))) - TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] - if len(REidxtemp)>0: - temps=np.abs(AA0).mean(axis=0) + REidxtemp = REName.loc[actual_list].index + TFidxtemp = np.array(range(len(TFName))) + TFidxtemp = TFidxtemp[TFName != RE_TGlink_temp[0]] + if len(REidxtemp) > 0: + temps = np.abs(AA0).mean(axis=0) zscored_arr = np.nan_to_num(temps, nan=0.0) for k in range(len(REidxtemp)): TG_2.append(RE_TGlink_temp[0]) RE_2.append(REidxtemp[k]) - score_2.append(zscored_arr[k+len(zscored_arr)-len(REidxtemp)]) - RE_TG=pd.DataFrame(TG_2) - RE_TG.columns=['TG'] - RE_TG['RE']=RE_2 - RE_TG['score']=score_2 - RE_TG=RE_TG.groupby(['RE', 'TG'])['score'].max().reset_index() + score_2.append(zscored_arr[k + len(zscored_arr) - len(REidxtemp)]) + RE_TG = pd.DataFrame(TG_2) + RE_TG.columns = ["TG"] + RE_TG["RE"] = RE_2 + RE_TG["score"] = score_2 + RE_TG = RE_TG.groupby(["RE", "TG"])["score"].max().reset_index() return RE_TG -def cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,method,outdir): - from tqdm import tqdm - chrom=['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') +def cis_reg(GRNdir, adata_RNA, adata_ATAC, genome, method, outdir): + + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") results = [] # every if statement is independent since method can only be one value - if method=='baseline': + if method == "baseline": # result=pd.DataFrame([]) for i in tqdm(range(23)): - chrN=chrom[i] - temp=cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,outdir) - temp.columns=['RE','TG','Score'] + chrN = chrom[i] + temp = cis_reg_chr(GRNdir, adata_RNA, adata_ATAC, genome, chrN, outdir) + temp.columns = ["RE", "TG", "Score"] results.append(temp) - if method=='LINGER': + if method == "LINGER": # result=pd.DataFrame([]) for i in tqdm(range(23)): - chrN=chrom[i] - temp=cis_shap(chrN,outdir) + chrN = chrom[i] + temp = cis_shap(chrN, outdir) results.append(temp) - if method=='scNN': - Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) - RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) - RE_TGlink.columns=[0,1,'chr'] - #chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] - chrlist=RE_TGlink['chr'].unique() - REName=Opn.index - geneName=Target.index - TFName=Exp.index - result=pd.DataFrame([]) + if method == "scNN": + Exp, Opn, Target, RE_TGlink = load_data_scNN(GRNdir, genome) + RE_TGlink = pd.read_csv(outdir + "RE_TGlink.txt", sep="\t", header=0) + RE_TGlink.columns = [0, 1, "chr"] + # chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] + chrlist = RE_TGlink["chr"].unique() + REName = Opn.index + geneName = Target.index + TFName = Exp.index + result = pd.DataFrame([]) for i in tqdm(range(len(chrlist))): - chrN=chrlist[i] - RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrN] - temp=cis_shap_scNN(chrN,outdir,RE_TGlink1,REName,TFName) + chrN = chrlist[i] + RE_TGlink1 = RE_TGlink[RE_TGlink["chr"] == chrN] + temp = cis_shap_scNN(chrN, outdir, RE_TGlink1, REName, TFName) results.append(temp) result = pd.concat(results, axis=0, join="outer") - result.to_csv(outdir+'cell_population_cis_regulatory.txt',sep='\t',header=None,index=None) + result.to_csv( + outdir + "cell_population_cis_regulatory.txt", sep="\t", header=None, index=None + ) -def cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir): - import numpy as np - import pandas as pd +def cell_type_specific_cis_reg_chr( + GRNdir, adata_RNA, adata_ATAC, genome, chrN, celltype, outdir +): + from scipy.sparse import coo_matrix, csc_matrix - O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(GRNdir,genome,chrN,outdir) - sparse_S,TGset=load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap) - label=adata_RNA.obs['label'].values.tolist() - labelset=list(set(label)) - temp=adata_ATAC.X[np.array(label)==celltype,:].mean(axis=0).T - RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==celltype,:].mean(axis=0).T - TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) + + O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u = load_region( + GRNdir, genome, chrN, outdir + ) + sparse_S, TGset = load_RE_TG(GRNdir, chrN, O_overlap_u, O_overlap_hg19_u, O_overlap) + label = adata_RNA.obs["label"].values.tolist() + labelset = list(set(label)) + temp = adata_ATAC.X[np.array(label) == celltype, :].mean(axis=0).T + RE = pd.DataFrame(temp, index=adata_ATAC.var["gene_ids"].values, columns=["values"]) + temp = adata_RNA.X[np.array(label) == celltype, :].mean(axis=0).T + TG = pd.DataFrame(temp, index=adata_RNA.var["gene_ids"].values, columns=["values"]) del temp -## cell annotation -## extact the overlapped peaks. - RE=RE.loc[N_overlap] + ## cell annotation + ## extact the overlapped peaks. + RE = RE.loc[N_overlap] ## select the genes - TGoverlap=list(set(TGset)&set(TG.index)) - #target_col_indices = [col_dict[col] for col in TGoverlap] + TGoverlap = list(set(TGset) & set(TG.index)) + # target_col_indices = [col_dict[col] for col in TGoverlap] sparse_S = sparse_S[TGoverlap] - TG=TG.loc[TGoverlap] - sparse_dis=load_RE_TG_distance(GRNdir,chrN,O_overlap_hg19_u,O_overlap_u,O_overlap,TGoverlap) - sparse_S+=0.1 -## cell annotation - TG_temp=TG.values#[:,np.array(label)==celltype].mean(axis=1) - TG_temp=TG_temp/TG_temp.mean()+0.1 - RE_temp=RE.values#[:,np.array(label)==celltype].mean(axis=1) - RE_temp=RE_temp/RE_temp.mean()+0.1 - Score=csc_matrix(RE_temp).multiply(sparse_S.values).multiply(sparse_dis.values).multiply(csc_matrix(TG_temp.T)).toarray() - Score=pd.DataFrame(Score,index=N_overlap,columns=TGoverlap) - Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] + TG = TG.loc[TGoverlap] + sparse_dis = load_RE_TG_distance( + GRNdir, chrN, O_overlap_hg19_u, O_overlap_u, O_overlap, TGoverlap + ) + sparse_S += 0.1 + ## cell annotation + TG_temp = TG.values # [:,np.array(label)==celltype].mean(axis=1) + TG_temp = TG_temp / TG_temp.mean() + 0.1 + RE_temp = RE.values # [:,np.array(label)==celltype].mean(axis=1) + RE_temp = RE_temp / RE_temp.mean() + 0.1 + Score = ( + csc_matrix(RE_temp) + .multiply(sparse_S.values) + .multiply(sparse_dis.values) + .multiply(csc_matrix(TG_temp.T)) + .toarray() + ) + Score = pd.DataFrame(Score, index=N_overlap, columns=TGoverlap) + Score = Score.groupby(Score.index).max() + data = Score.values[Score.values != 0] rows, cols = np.nonzero(Score.values) - coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) - combined[:,0]=Score.index[coo.row] - combined[:,1]=np.array(TGoverlap)[coo.col] - combined[:,2]=coo.data - resultall=pd.DataFrame(combined) + coo = coo_matrix((data, (rows, cols)), shape=Score.shape) + combined = np.zeros([len(data), 3], dtype=object) + combined[:, 0] = Score.index[coo.row] + combined[:, 1] = np.array(TGoverlap)[coo.col] + combined[:, 2] = coo.data + resultall = pd.DataFrame(combined) return resultall -def cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs): - import numpy as np - import pandas as pd + + +def cell_type_specific_cis_reg_scNN(distance, cisGRN, RE, TG, REs, TGs): + from scipy.sparse import coo_matrix, csr_matrix - RE=RE.loc[REs] + + RE = RE.loc[REs] ## select the genes - #target_col_indices = [col_dict[col] for col in TGoverlap] - TG=TG.loc[TGs] + # target_col_indices = [col_dict[col] for col in TGoverlap] + TG = TG.loc[TGs] ## cell annotation - TG_temp=TG.values#[:,np.array(label)==celltype].mean(axis=1) - TG_temp=TG_temp/TG_temp.mean()+0.1 - RE_temp=RE.values#[:,np.array(label)==celltype].mean(axis=1) - RE_temp=RE_temp/RE_temp.mean()+0.1 - Score=(cisGRN.multiply(csr_matrix(RE_temp))).multiply(distance).multiply(csr_matrix(TG_temp.T)) + TG_temp = TG.values # [:,np.array(label)==celltype].mean(axis=1) + TG_temp = TG_temp / TG_temp.mean() + 0.1 + RE_temp = RE.values # [:,np.array(label)==celltype].mean(axis=1) + RE_temp = RE_temp / RE_temp.mean() + 0.1 + Score = ( + (cisGRN.multiply(csr_matrix(RE_temp))) + .multiply(distance) + .multiply(csr_matrix(TG_temp.T)) + ) row_indices, col_indices = Score.nonzero() - row_indices=np.array(REs)[row_indices] + row_indices = np.array(REs)[row_indices] col_indices = np.array(TGs)[col_indices] values = Score.data - combined = np.zeros([len(row_indices),3], dtype=object) - combined[:,0]=row_indices - combined[:,1]=col_indices - combined[:,2]=values - resultall=pd.DataFrame(combined) + combined = np.zeros([len(row_indices), 3], dtype=object) + combined[:, 0] = row_indices + combined[:, 1] = col_indices + combined[:, 2] = values + resultall = pd.DataFrame(combined) return resultall -def cell_type_specific_cis_reg(GRNdir,adata_RNA,adata_ATAC,genome,celltype,outdir,method): - import numpy as np - import pandas as pd - label=adata_RNA.obs['label'].values.tolist() - labelset=list(set(label)) - chrom=['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') - from tqdm import tqdm - if (celltype=='all')&(method!='scNN'): + +def cell_type_specific_cis_reg( + GRNdir, adata_RNA, adata_ATAC, genome, celltype, outdir, method +): + + label = adata_RNA.obs["label"].values.tolist() + labelset = list(set(label)) + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") + + if (celltype == "all") & (method != "scNN"): for label0 in labelset: - label0=str(label0) - result=pd.DataFrame([]) + label0 = str(label0) + result = pd.DataFrame([]) results = [] for i in tqdm(range(23)): - chrN=chrom[i] - temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,label0,outdir) + chrN = chrom[i] + temp = cell_type_specific_cis_reg_chr( + GRNdir, adata_RNA, adata_ATAC, genome, chrN, label0, outdir + ) results.append(temp) - chrN='chrX' - temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,label0,outdir) + chrN = "chrX" + temp = cell_type_specific_cis_reg_chr( + GRNdir, adata_RNA, adata_ATAC, genome, chrN, label0, outdir + ) results.append(temp) - result=pd.concat(results,axis=0,join='outer') - result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+str(label0)+'.txt',sep='\t',header=None,index=None) - elif (method!='scNN'): - result=pd.DataFrame([]) - results = [] - for i in tqdm(range(23)): - chrN=chrom[i] - temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir) - results.append(temp) - chrN='chrX' - temp=cell_type_specific_cis_reg_chr(GRNdir,adata_RNA,adata_ATAC,genome,chrN,celltype,outdir) + result = pd.concat(results, axis=0, join="outer") + result.to_csv( + outdir + "cell_type_specific_cis_regulatory_" + str(label0) + ".txt", + sep="\t", + header=None, + index=None, + ) + elif method != "scNN": + result = pd.DataFrame([]) + results = [] + for i in tqdm(range(23)): + chrN = chrom[i] + temp = cell_type_specific_cis_reg_chr( + GRNdir, adata_RNA, adata_ATAC, genome, chrN, celltype, outdir + ) results.append(temp) - result=pd.concat(results,axis=0,join='outer') - result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+celltype+'.txt',sep='\t',header=None,index=None) - elif (celltype=='all')&(method=='scNN'): - distance,cisGRN,REs,TGs=load_RE_TG_scNN(outdir) + chrN = "chrX" + temp = cell_type_specific_cis_reg_chr( + GRNdir, adata_RNA, adata_ATAC, genome, chrN, celltype, outdir + ) + results.append(temp) + result = pd.concat(results, axis=0, join="outer") + result.to_csv( + outdir + "cell_type_specific_cis_regulatory_" + celltype + ".txt", + sep="\t", + header=None, + index=None, + ) + elif (celltype == "all") & (method == "scNN"): + distance, cisGRN, REs, TGs = load_RE_TG_scNN(outdir) for label0 in labelset: - label0=str(label0) - temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T - RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T - TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) + label0 = str(label0) + temp = adata_ATAC.X[np.array(label) == label0, :].mean(axis=0).T + RE = pd.DataFrame( + temp, index=adata_ATAC.var["gene_ids"].values, columns=["values"] + ) + temp = adata_RNA.X[np.array(label) == label0, :].mean(axis=0).T + TG = pd.DataFrame( + temp, index=adata_RNA.var["gene_ids"].values, columns=["values"] + ) del temp - result=cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs) - result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+label0+'.txt',sep='\t',header=None,index=None) + result = cell_type_specific_cis_reg_scNN(distance, cisGRN, RE, TG, REs, TGs) + result.to_csv( + outdir + "cell_type_specific_cis_regulatory_" + label0 + ".txt", + sep="\t", + header=None, + index=None, + ) else: - label0=celltype - label0=str(label0) - temp=adata_ATAC.X[np.array(label)==label0,:].mean(axis=0).T - RE=pd.DataFrame(temp,index=adata_ATAC.var['gene_ids'].values,columns=['values']) - temp=adata_RNA.X[np.array(label)==label0,:].mean(axis=0).T - TG=pd.DataFrame(temp,index=adata_RNA.var['gene_ids'].values,columns=['values']) + label0 = celltype + label0 = str(label0) + temp = adata_ATAC.X[np.array(label) == label0, :].mean(axis=0).T + RE = pd.DataFrame( + temp, index=adata_ATAC.var["gene_ids"].values, columns=["values"] + ) + temp = adata_RNA.X[np.array(label) == label0, :].mean(axis=0).T + TG = pd.DataFrame( + temp, index=adata_RNA.var["gene_ids"].values, columns=["values"] + ) del temp - result=cell_type_specific_cis_reg_scNN(distance,cisGRN,RE,TG,REs,TGs) - result.to_csv(outdir+'cell_type_specific_cis_regulatory_'+label0+'.txt',sep='\t',header=None,index=None) + result = cell_type_specific_cis_reg_scNN(distance, cisGRN, RE, TG, REs, TGs) + result.to_csv( + outdir + "cell_type_specific_cis_regulatory_" + label0 + ".txt", + sep="\t", + header=None, + index=None, + ) + -def trans_shap_scNN(chrtemp,outdir,RE_TGlink1,REName,TFName): +def trans_shap_scNN(chrtemp, outdir, RE_TGlink1, REName, TFName): import ast - TG_1=[] - TF_1=[] - score_1=[] - REName=pd.DataFrame(range(len(REName)),index=REName) - shap_all=torch.load(outdir+chrtemp+"_shap"+".pt") - N=RE_TGlink1.shape[0] - from tqdm import tqdm + + TG_1 = [] + TF_1 = [] + score_1 = [] + REName = pd.DataFrame(range(len(REName)), index=REName) + shap_all = torch.load(outdir + chrtemp + "_shap" + ".pt") + N = RE_TGlink1.shape[0] + for ii in range(N): - AA0=shap_all[ii] - RE_TGlink_temp=RE_TGlink1.values[ii,:] + AA0 = shap_all[ii] + RE_TGlink_temp = RE_TGlink1.values[ii, :] actual_list = ast.literal_eval(RE_TGlink_temp[1]) - REidxtemp=REName.loc[actual_list].index - TFidxtemp=np.array(range(len(TFName))) - TFidxtemp=TFidxtemp[TFName!=RE_TGlink_temp[0]] - temps=np.abs(AA0).mean(axis=0) - #zscored_arr = zscore(temps) + REidxtemp = REName.loc[actual_list].index + TFidxtemp = np.array(range(len(TFName))) + TFidxtemp = TFidxtemp[TFName != RE_TGlink_temp[0]] + temps = np.abs(AA0).mean(axis=0) + # zscored_arr = zscore(temps) zscored_arr = np.nan_to_num(temps, nan=0.0) for k in range(len(TFidxtemp)): TG_1.append(RE_TGlink_temp[0]) TF_1.append(TFName[TFidxtemp[k]]) score_1.append(zscored_arr[k]) - TF_TG=pd.DataFrame(TG_1) - TF_TG.columns=['TG'] - TF_TG['TF']=TF_1 - TF_TG['score']=score_1 - mat,TGs,TFs=list2mat(TF_TG,'TG','TF','score') - mat=pd.DataFrame(mat,index=TGs,columns=TFs) + TF_TG = pd.DataFrame(TG_1) + TF_TG.columns = ["TG"] + TF_TG["TF"] = TF_1 + TF_TG["score"] = score_1 + mat, TGs, TFs = list2mat(TF_TG, "TG", "TF", "score") + mat = pd.DataFrame(mat, index=TGs, columns=TFs) mat.fillna(0, inplace=True) return mat -def load_cis(Binding,celltype,outdir): - import numpy as np - import pandas as pd +def load_cis(Binding, celltype, outdir): + from scipy.sparse import coo_matrix - if celltype=='': - cis=pd.read_csv(outdir+'cell_population_cis_regulatory.txt',sep='\t',header=None) + + if celltype == "": + cis = pd.read_csv( + outdir + "cell_population_cis_regulatory.txt", sep="\t", header=None + ) else: - cis=pd.read_csv(outdir+'cell_type_specific_cis_regulatory_'+celltype+'.txt',sep='\t',header=None) - cis.columns=['RE','TG','Score'] - TGset=cis['TG'].unique() - REset=Binding.index - TFset=Binding.columns + cis = pd.read_csv( + outdir + "cell_type_specific_cis_regulatory_" + celltype + ".txt", + sep="\t", + header=None, + ) + cis.columns = ["RE", "TG", "Score"] + TGset = cis["TG"].unique() + REset = Binding.index + TFset = Binding.columns col_dict = {col: i for i, col in enumerate(TGset)} row_dict = {row: i for i, row in enumerate(REset)} - cis=cis[cis["RE"].isin(REset)] -# Map the column names and row names to integer indices in the DataFrame + cis = cis[cis["RE"].isin(REset)] + # Map the column names and row names to integer indices in the DataFrame cis["col_index"] = cis["TG"].map(col_dict) cis["row_index"] = cis["RE"].map(row_dict) # Extract the column indices, row indices, and values from the DataFrame @@ -1080,22 +1354,25 @@ def load_cis(Binding,celltype,outdir): row_indices = cis["row_index"].tolist() values = cis["Score"].tolist() # Create the sparse matrix using coo_matrix - sparse_S = coo_matrix((values, (row_indices, col_indices)),shape=(len(REset), len(TGset))) + sparse_S = coo_matrix( + (values, (row_indices, col_indices)), shape=(len(REset), len(TGset)) + ) sparse_S.colnames = TGset sparse_S.rownames = REset - cis=sparse_S.toarray() - cis=pd.DataFrame(cis,index=REset,columns=TGset) + cis = sparse_S.toarray() + cis = pd.DataFrame(cis, index=REset, columns=TGset) return cis -def load_TF_TG( GRNdir, TFset,TGset): - TF_TG_all=np.zeros([len(TGset),len(TFset)]) - a=list(range(1,23)) - a.append('X') + +def load_TF_TG(GRNdir, TFset, TGset): + TF_TG_all = np.zeros([len(TGset), len(TFset)]) + a = list(range(1, 23)) + a.append("X") for i in a: - chrN='chr'+str(i) - TF_TG = pd.read_csv(GRNdir+'Primary_TF_TG_'+chrN+'.txt',sep='\t') - TF_TG = TF_TG[TF_TG['TF'].isin(TFset)] - TF_TG = TF_TG[TF_TG['TG'].isin(TGset)] + chrN = "chr" + str(i) + TF_TG = pd.read_csv(GRNdir + "Primary_TF_TG_" + chrN + ".txt", sep="\t") + TF_TG = TF_TG[TF_TG["TF"].isin(TFset)] + TF_TG = TF_TG[TF_TG["TG"].isin(TGset)] col_dict = {col: i for i, col in enumerate(TFset)} row_dict = {row: i for i, row in enumerate(TGset)} TF_TG["col_index"] = TF_TG["TF"].map(col_dict) @@ -1103,86 +1380,105 @@ def load_TF_TG( GRNdir, TFset,TGset): col_indices = TF_TG["col_index"].tolist() row_indices = TF_TG["row_index"].tolist() values = TF_TG["score"].tolist() - sparse_S = coo_matrix((values, (row_indices, col_indices)),shape=( len(TGset),len(TFset))) - idx=list(set(row_indices)) - TGset1=TGset[idx] - TF_TG=sparse_S.toarray() - TF_TG=pd.DataFrame(TF_TG,index=TGset,columns=TFset) - TF_TG=TF_TG.loc[TGset1] - TF_TG_all[idx,:]=TF_TG.values - TF_TG_all=pd.DataFrame(TF_TG_all,index=TGset,columns=TFset) + sparse_S = coo_matrix( + (values, (row_indices, col_indices)), shape=(len(TGset), len(TFset)) + ) + idx = list(set(row_indices)) + TGset1 = TGset[idx] + TF_TG = sparse_S.toarray() + TF_TG = pd.DataFrame(TF_TG, index=TGset, columns=TFset) + TF_TG = TF_TG.loc[TGset1] + TF_TG_all[idx, :] = TF_TG.values + TF_TG_all = pd.DataFrame(TF_TG_all, index=TGset, columns=TFset) return TF_TG_all -def trans_reg(GRNdir,method,outdir,genome): + +def trans_reg(GRNdir, method, outdir, genome): import ast - import numpy as np - import pandas as pd from scipy.sparse import coo_matrix, csc_matrix - print('Generating trans-regulatory netowrk ...') - if method=='baseline': - Binding=pd.read_csv(outdir+'cell_population_TF_RE_binding.txt',sep='\t',index_col=0) - cis=load_cis(Binding,'',outdir) - TFset=Binding.columns - TGset=cis.columns - TF_TG=load_TF_TG(GRNdir, TFset,TGset) - S=np.matmul(Binding.values.T, cis.values).T*(TF_TG.values.T).T - S=pd.DataFrame(S, index=TGset,columns=TFset) - elif method=='LINGER': - chrom=['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') - S=pd.DataFrame([]) + + print("Generating trans-regulatory netowrk ...") + if method == "baseline": + Binding = pd.read_csv( + outdir + "cell_population_TF_RE_binding.txt", sep="\t", index_col=0 + ) + cis = load_cis(Binding, "", outdir) + TFset = Binding.columns + TGset = cis.columns + TF_TG = load_TF_TG(GRNdir, TFset, TGset) + S = np.matmul(Binding.values.T, cis.values).T * (TF_TG.values.T).T + S = pd.DataFrame(S, index=TGset, columns=TFset) + elif method == "LINGER": + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") + S = pd.DataFrame([]) results = [] for i in tqdm(range(23)): - chrN=chrom[i] - temp=trans_shap(chrN,outdir) + chrN = chrom[i] + temp = trans_shap(chrN, outdir) results.append(temp) - S = pd.concat(results, axis = 0, join="outer") - elif method=='scNN': - Exp,Opn,Target,RE_TGlink=load_data_scNN(GRNdir,genome) - RE_TGlink=pd.read_csv(outdir+'RE_TGlink.txt',sep='\t',header=0) - RE_TGlink.columns=[0,1,'chr'] - #chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] - chrlist=RE_TGlink['chr'].unique() - REName=Opn.index - geneName=Target.index - TFName=Exp.index - S=pd.DataFrame([]) + S = pd.concat(results, axis=0, join="outer") + elif method == "scNN": + Exp, Opn, Target, RE_TGlink = load_data_scNN(GRNdir, genome) + RE_TGlink = pd.read_csv(outdir + "RE_TGlink.txt", sep="\t", header=0) + RE_TGlink.columns = [0, 1, "chr"] + # chrall=[RE_TGlink[0][i][0].split(':')[0] for i in range(RE_TGlink.shape[0])] + chrlist = RE_TGlink["chr"].unique() + REName = Opn.index + geneName = Target.index + TFName = Exp.index + S = pd.DataFrame([]) results = [] for i in tqdm(range(len(chrlist))): - chrN=chrlist[i] - RE_TGlink1=RE_TGlink[RE_TGlink['chr']==chrN] - temp=trans_shap_scNN(chrN,outdir,RE_TGlink1,REName,TFName) + chrN = chrlist[i] + RE_TGlink1 = RE_TGlink[RE_TGlink["chr"] == chrN] + temp = trans_shap_scNN(chrN, outdir, RE_TGlink1, REName, TFName) results.append(temp) - S=pd.concat(results,axis=0,join='outer') - print('Saving trans-regulatory netowrk ...') - S.to_csv(outdir+'cell_population_trans_regulatory.txt',sep='\t') + S = pd.concat(results, axis=0, join="outer") + print("Saving trans-regulatory netowrk ...") + S.to_csv(outdir + "cell_population_trans_regulatory.txt", sep="\t") + + +def cell_type_specific_trans_reg(GRNdir, adata_RNA, celltype, outdir): -def cell_type_specific_trans_reg(GRNdir,adata_RNA,celltype,outdir): - import numpy as np - import pandas as pd from scipy.sparse import coo_matrix, csc_matrix - label=adata_RNA.obs['label'].values.tolist() - labelset=list(set(label)) - if celltype=='all': + + label = adata_RNA.obs["label"].values.tolist() + labelset = list(set(label)) + if celltype == "all": for label0 in labelset: - Binding=pd.read_csv(outdir+'cell_type_specific_TF_RE_binding_'+str(label0)+'.txt',sep='\t',index_col=0) - label0=str(label0) - cis=load_cis(Binding,label0,outdir) - TFset=Binding.columns - TGset=cis.columns - #TF_TG=load_TF_TG(GRNdir, TFset,TGset) - S=np.matmul(Binding.values.T, cis.values).T#*(TF_TG.values.T).T - S=pd.DataFrame(S, index=TGset,columns=TFset) - S.to_csv(outdir+'cell_type_specific_trans_regulatory_'+str(label0)+'.txt',sep='\t') + Binding = pd.read_csv( + outdir + "cell_type_specific_TF_RE_binding_" + str(label0) + ".txt", + sep="\t", + index_col=0, + ) + label0 = str(label0) + cis = load_cis(Binding, label0, outdir) + TFset = Binding.columns + TGset = cis.columns + # TF_TG=load_TF_TG(GRNdir, TFset,TGset) + S = np.matmul(Binding.values.T, cis.values).T # *(TF_TG.values.T).T + S = pd.DataFrame(S, index=TGset, columns=TFset) + S.to_csv( + outdir + "cell_type_specific_trans_regulatory_" + str(label0) + ".txt", + sep="\t", + ) else: - Binding=pd.read_csv(outdir+'cell_type_specific_TF_RE_binding_'+celltype+'.txt',sep='\t',index_col=0) - cis=load_cis(Binding,celltype,outdir) - TFset=Binding.columns - TGset=cis.columns - #TF_TG=load_TF_TG(GRNdir, TFset,TGset) - S=np.matmul(Binding.values.T, cis.values).T#*(TF_TG.values.T).T - S=pd.DataFrame(S, index=TGset,columns=TFset) - S.to_csv(outdir+'cell_type_specific_trans_regulatory_'+celltype+'.txt',sep='\t') + Binding = pd.read_csv( + outdir + "cell_type_specific_TF_RE_binding_" + celltype + ".txt", + sep="\t", + index_col=0, + ) + cis = load_cis(Binding, celltype, outdir) + TFset = Binding.columns + TGset = cis.columns + # TF_TG=load_TF_TG(GRNdir, TFset,TGset) + S = np.matmul(Binding.values.T, cis.values).T # *(TF_TG.values.T).T + S = pd.DataFrame(S, index=TGset, columns=TFset) + S.to_csv( + outdir + "cell_type_specific_trans_regulatory_" + celltype + ".txt", + sep="\t", + ) diff --git a/code/lingergrn-1.106/LingerGRN/LingerGRN.py b/code/lingergrn-1.106/LingerGRN/LingerGRN.py index f523e77..d3405e7 100644 --- a/code/lingergrn-1.106/LingerGRN/LingerGRN.py +++ b/code/lingergrn-1.106/LingerGRN/LingerGRN.py @@ -1 +1 @@ -__version__ = "1.106" \ No newline at end of file +__version__ = "1.106" diff --git a/code/lingergrn-1.106/LingerGRN/TF_activity.py b/code/lingergrn-1.106/LingerGRN/TF_activity.py index 4f9e0d3..50d231a 100644 --- a/code/lingergrn-1.106/LingerGRN/TF_activity.py +++ b/code/lingergrn-1.106/LingerGRN/TF_activity.py @@ -1,29 +1,33 @@ +import numpy as np +import pandas as pd +from scipy.stats import rankdata + # def quantile_normalize(df): # rank_mean = df.stack().groupby(df.rank(method='first').stack().astype(int)).mean() # return df.rank(method='min').stack().astype(int).map(rank_mean).unstack() # -from scipy.stats import rankdata -# approx 4x faster to use np and scipy over pandas methods +# approx 4x faster to use np and scipy over pandas methods - Arnav G. # tie_break: how ties betw values are broken for finding the means, if true then ties are assigned unique vals based on appearance # order. This is the same as the original function. False: ties are arbitrarily assigned values (faster). def quantile_normalize(df, tie_break: bool = True): arr = df.values if tie_break: - sort_order = rankdata(arr, method='ordinal', axis=0).astype(int) - 1 + sort_order = rankdata(arr, method="ordinal", axis=0).astype(int) - 1 sorted_arr = arr[sort_order, np.arange(arr.shape[1])] else: - sorted_arr = np.sort(arr, axis = 0) - means = np.mean(sorted_arr, axis = -1) + sorted_arr = np.sort(arr, axis=0) + means = np.mean(sorted_arr, axis=-1) # mask = np.argsort(np.argsort(arr, axis = 0), axis = 0) - mask = (rankdata(arr, method="min", axis = 0).astype(int)-1) + mask = rankdata(arr, method="min", axis=0).astype(int) - 1 - return pd.DataFrame(means[mask], columns = df.columns, index = df.index) + return pd.DataFrame(means[mask], columns=df.columns, index=df.index) +def bulk_reg(outdir, GRNdir, genome, chrN): + from scipy.sparse import coo_matrix -def bulk_reg(outdir,GRNdir,genome,chrN): from LingerGRN.LL_net import ( load_RE_TG, load_RE_TG_distance, @@ -31,42 +35,49 @@ def bulk_reg(outdir,GRNdir,genome,chrN): load_TF_RE, load_TFbinding, ) - from scipy.sparse import coo_matrix - O_overlap, N_overlap,O_overlap_u,N_overlap_u,O_overlap_hg19_u=load_region(outdir,GRNdir,genome,chrN) - TFbinding=load_TFbinding(GRNdir,O_overlap,O_overlap_u,O_overlap_hg19_u,chrN) - mat=load_TF_RE(GRNdir,chrN,O_overlap,O_overlap_u,O_overlap_hg19_u) + + O_overlap, N_overlap, O_overlap_u, N_overlap_u, O_overlap_hg19_u = load_region( + outdir, GRNdir, genome, chrN + ) + TFbinding = load_TFbinding(GRNdir, O_overlap, O_overlap_u, O_overlap_hg19_u, chrN) + mat = load_TF_RE(GRNdir, chrN, O_overlap, O_overlap_u, O_overlap_hg19_u) TFoverlap = list(set(mat.columns) & set(TFbinding.columns)) mat = mat[TFoverlap] TFbinding = TFbinding[TFoverlap] - mat_m=np.mean(mat.values[mat>0]) + mat_m = np.mean(mat.values[mat > 0]) mat = mat / mat_m - mat.values[mat.values<0]=0 + mat.values[mat.values < 0] = 0 TFbinding = TFbinding / TFbinding.mean(axis=1).mean() - S = np.log(mat+TFbinding+0.1) - S.index=N_overlap + S = np.log(mat + TFbinding + 0.1) + S.index = N_overlap S = S.groupby(S.index).max() - sparse_S,TGset=load_RE_TG(GRNdir,chrN,O_overlap_u,O_overlap_hg19_u,O_overlap) - sparse_S+=0.1 - sparse_dis=load_RE_TG_distance(GRNdir,chrN,O_overlap_hg19_u,O_overlap_u,O_overlap,TGset) - Score=sparse_S.multiply(sparse_dis.values) - Score=pd.DataFrame(Score.values,index=N_overlap,columns=TGset) - Score=Score.groupby(Score.index).max() - data = Score.values[Score.values!=0] + sparse_S, TGset = load_RE_TG(GRNdir, chrN, O_overlap_u, O_overlap_hg19_u, O_overlap) + sparse_S += 0.1 + sparse_dis = load_RE_TG_distance( + GRNdir, chrN, O_overlap_hg19_u, O_overlap_u, O_overlap, TGset + ) + Score = sparse_S.multiply(sparse_dis.values) + Score = pd.DataFrame(Score.values, index=N_overlap, columns=TGset) + Score = Score.groupby(Score.index).max() + data = Score.values[Score.values != 0] rows, cols = np.nonzero(Score.values) - coo = coo_matrix((data,(rows,cols)),shape=Score.shape) - combined = np.zeros([len(data),3], dtype=object) - combined[:,0]=Score.index[coo.row] - combined[:,1]=np.array(TGset)[coo.col] - combined[:,2]=coo.data - combined=pd.DataFrame(combined) - return S,combined -def TF_RE2m(result_RE_TG,REset): + coo = coo_matrix((data, (rows, cols)), shape=Score.shape) + combined = np.zeros([len(data), 3], dtype=object) + combined[:, 0] = Score.index[coo.row] + combined[:, 1] = np.array(TGset)[coo.col] + combined[:, 2] = coo.data + combined = pd.DataFrame(combined) + return S, combined + + +def TF_RE2m(result_RE_TG, REset): from scipy.sparse import coo_matrix - TGset=result_RE_TG['TG'].unique() - #REset=result_RE_TG.['TG'].unique() + + TGset = result_RE_TG["TG"].unique() + # REset=result_RE_TG.['TG'].unique() col_dict = {col: i for i, col in enumerate(TGset)} row_dict = {row: i for i, row in enumerate(REset)} -# Map the column names and row names to integer indices in the DataFrame + # Map the column names and row names to integer indices in the DataFrame result_RE_TG["col_index"] = result_RE_TG["TG"].map(col_dict) result_RE_TG["row_index"] = result_RE_TG["RE"].map(row_dict) # Extract the column indices, row indices, and values from the DataFrame @@ -74,183 +85,217 @@ def TF_RE2m(result_RE_TG,REset): row_indices = result_RE_TG["row_index"].tolist() values = result_RE_TG["Score"].tolist() # Create the sparse matrix using coo_matrix - sparse_S = coo_matrix((values, (row_indices, col_indices)),shape=(len(REset), len(TGset))) + sparse_S = coo_matrix( + (values, (row_indices, col_indices)), shape=(len(REset), len(TGset)) + ) sparse_S.colnames = TGset sparse_S.rownames = REset - cis=sparse_S.toarray() - cis=pd.DataFrame(cis,index=REset,columns=TGset) + cis = sparse_S.toarray() + cis = pd.DataFrame(cis, index=REset, columns=TGset) return cis -import numpy as np -import pandas as pd + + import scipy.io as sio -def regulon(outdir,adata_RNA,GRNdir,network,genome): -# Load data from MATLAB .mat files - if network=='cell population': - trans_reg = pd.read_csv(outdir+'cell_population_trans_regulatory.txt',sep='\t',index_col=0) -# Apply quantile normalization to 'trans_reg_n' - elif network=='general': +def regulon(outdir, adata_RNA, GRNdir, network, genome): + # Load data from MATLAB .mat files + if network == "cell population": + trans_reg = pd.read_csv( + outdir + "cell_population_trans_regulatory.txt", sep="\t", index_col=0 + ) + # Apply quantile normalization to 'trans_reg_n' + elif network == "general": from tqdm import tqdm - chrom=['chr'+str(i+1) for i in range(22)] - chrom.append('chrX') - result_TF_RE=pd.DataFrame([]) - result_RE_TG=pd.DataFrame([]) + + chrom = ["chr" + str(i + 1) for i in range(22)] + chrom.append("chrX") + result_TF_RE = pd.DataFrame([]) + result_RE_TG = pd.DataFrame([]) tf_res = [] re_tgs = [] for i in tqdm(range(23)): - chrN=chrom[i] - TF_RE,RE_TG=bulk_reg(outdir,GRNdir,genome,chrN) - RE_TG.columns=['RE','TG','Score'] + chrN = chrom[i] + TF_RE, RE_TG = bulk_reg(outdir, GRNdir, genome, chrN) + RE_TG.columns = ["RE", "TG", "Score"] tf_res.append(TF_RE) re_tgs.append(RE_TG) - result_RE_TG=pd.concat(re_tgs,axis=0,join='outer') - result_TF_RE=pd.concat(tf_res,axis=0,join='outer') + result_RE_TG = pd.concat(re_tgs, axis=0, join="outer") + result_TF_RE = pd.concat(tf_res, axis=0, join="outer") - TFset=result_TF_RE.columns - REset=result_TF_RE.index - cis=TF_RE2m(result_RE_TG,REset) - TGset=cis.columns + TFset = result_TF_RE.columns + REset = result_TF_RE.index + cis = TF_RE2m(result_RE_TG, REset) + TGset = cis.columns from LL_net import load_TF_TG - TF_TG=load_TF_TG(GRNdir, TFset,TGset) - trans_reg=np.matmul(result_TF_RE.values.T, cis.values).T*(TF_TG.values) - trans_reg=pd.DataFrame(trans_reg, index=TGset,columns=TFset) + + TF_TG = load_TF_TG(GRNdir, TFset, TGset) + trans_reg = np.matmul(result_TF_RE.values.T, cis.values).T * (TF_TG.values) + trans_reg = pd.DataFrame(trans_reg, index=TGset, columns=TFset) else: - trans_reg = pd.read_csv(outdir+'cell_type_specific_trans_regulatory_'+network+'.txt',sep='\t',index_col=0) - #RNA=pd.read_csv(outdir+RNA_file,sep='\t',index_col=0) - RNA=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) - gene_overlap=list(set(trans_reg.index)&set(RNA.index)) - RNA=RNA.loc[gene_overlap] - trans_reg=trans_reg.loc[gene_overlap] - row=trans_reg.sum(axis=1).values[:,np.newaxis]+trans_reg.sum(axis=1).mean()*0.000001 - colsum=trans_reg.sum(axis=0).values[:,np.newaxis]+trans_reg.sum(axis=0).mean()*0.000001 + trans_reg = pd.read_csv( + outdir + "cell_type_specific_trans_regulatory_" + network + ".txt", + sep="\t", + index_col=0, + ) + # RNA=pd.read_csv(outdir+RNA_file,sep='\t',index_col=0) + RNA = pd.DataFrame( + adata_RNA.X.toarray().T, + index=adata_RNA.var["gene_ids"].values, + columns=adata_RNA.obs["barcode"].values, + ) + gene_overlap = list(set(trans_reg.index) & set(RNA.index)) + RNA = RNA.loc[gene_overlap] + trans_reg = trans_reg.loc[gene_overlap] + row = ( + trans_reg.sum(axis=1).values[:, np.newaxis] + + trans_reg.sum(axis=1).mean() * 0.000001 + ) + colsum = ( + trans_reg.sum(axis=0).values[:, np.newaxis] + + trans_reg.sum(axis=0).mean() * 0.000001 + ) # data_norm=trans_reg.values/(row*colsum.T)*row.sum() - E=(row*colsum.T)/row.sum() + E = (row * colsum.T) / row.sum() - #trans_reg=trans_reg/trans_reg.sum(axis=0) - #trans_reg_norm = quantile_normalize(trans_reg.T) - RNA=RNA/RNA.sum(axis=0) #neccessary - RNA_norm=quantile_normalize(RNA) - #RNA_norm=RNA - e_rna = np.dot(E.T,RNA_norm.values) - regulon=(np.dot(trans_reg.values.T,RNA_norm.values)-e_rna)/ e_rna - regulon=pd.DataFrame(regulon,index=trans_reg.columns,columns=RNA_norm.columns) + # trans_reg=trans_reg/trans_reg.sum(axis=0) + # trans_reg_norm = quantile_normalize(trans_reg.T) + RNA = RNA / RNA.sum(axis=0) # neccessary + RNA_norm = quantile_normalize(RNA) + # RNA_norm=RNA + e_rna = np.dot(E.T, RNA_norm.values) + regulon = (np.dot(trans_reg.values.T, RNA_norm.values) - e_rna) / e_rna + regulon = pd.DataFrame(regulon, index=trans_reg.columns, columns=RNA_norm.columns) return regulon -def master_regulator(regulon_score,adata_RNA,celltype): - import pandas as pd + + +def master_regulator(regulon_score, adata_RNA, celltype): import statsmodels.stats.multitest as smm from scipy import stats - #label=pd.read_csv(outdir+labels,sep='\t',header=None) - #label = label.astype(str) - #label.columns=['celltype'] - label=pd.DataFrame(adata_RNA.obs['label'].values.tolist(),columns=['celltype']) + + # label=pd.read_csv(outdir+labels,sep='\t',header=None) + # label = label.astype(str) + # label.columns=['celltype'] + label = pd.DataFrame(adata_RNA.obs["label"].values.tolist(), columns=["celltype"]) label = label.astype(str) - if celltype in label['celltype'].values: -# Assuming X and Y are DataFrames with multiple variables/columns - idx=regulon_score.columns[np.isin(label['celltype'],celltype)] - X=regulon_score[idx] - idx=regulon_score.columns[np.isin(label['celltype'],celltype)==0] - Y=regulon_score[idx] -# Initialize an empty DataFrame to store the t-test results - t_test_results = np.zeros((regulon_score.shape[0],2)) - # Iterate over the columns/variables in X and Y + if celltype in label["celltype"].values: + # Assuming X and Y are DataFrames with multiple variables/columns + idx = regulon_score.columns[np.isin(label["celltype"], celltype)] + X = regulon_score[idx] + idx = regulon_score.columns[np.isin(label["celltype"], celltype) == 0] + Y = regulon_score[idx] + # Initialize an empty DataFrame to store the t-test results + t_test_results = np.zeros((regulon_score.shape[0], 2)) + # Iterate over the columns/variables in X and Y t_stat, p_value = stats.ttest_ind(X, Y, axis=1, alternative="greater") t_test_results = pd.DataFrame( - {"t_stat": t_stat, "p_value": p_value}, - index=regulon_score.index + {"t_stat": t_stat, "p_value": p_value}, index=regulon_score.index ) t_test_results.fillna({"t_stat": 0, "p_value": 1}, inplace=True) - t_test_results["adj_p"] = smm.multipletests(t_test_results["p_value"], method="fdr_bh")[1] - elif celltype=='all': - label_set=np.array(list(set(label['celltype'].values))) - t_test_results = np.zeros((regulon_score.shape[0],3*len(label_set))) + t_test_results["adj_p"] = smm.multipletests( + t_test_results["p_value"], method="fdr_bh" + )[1] + elif celltype == "all": + label_set = np.array(list(set(label["celltype"].values))) + t_test_results = np.zeros((regulon_score.shape[0], 3 * len(label_set))) res = [] for j in range(len(label_set)): - idx=regulon_score.columns[np.isin(label['celltype'],label_set[j])] - X=regulon_score[idx] - idx=regulon_score.columns[np.isin(label['celltype'],label_set[j])==0] - Y=regulon_score[idx] -# Initialize an empty DataFrame to store the t-test results - # Iterate over the columns/variables in X and Y - t_stat, p_value = stats.ttest_ind(X, Y,alternative='greater', axis=1) + idx = regulon_score.columns[np.isin(label["celltype"], label_set[j])] + X = regulon_score[idx] + idx = regulon_score.columns[np.isin(label["celltype"], label_set[j]) == 0] + Y = regulon_score[idx] + # Initialize an empty DataFrame to store the t-test results + # Iterate over the columns/variables in X and Y + t_stat, p_value = stats.ttest_ind(X, Y, alternative="greater", axis=1) t_test_results = pd.DataFrame( - {"t_stat": t_stat, "p_value": p_value}, - index=regulon_score.index + {"t_stat": t_stat, "p_value": p_value}, index=regulon_score.index ) t_test_results.fillna({"t_stat": 0, "p_value": 1}, inplace=True) - t_test_results["adj_p"] = smm.multipletests(t_test_results["p_value"], method="fdr_bh")[1] + t_test_results["adj_p"] = smm.multipletests( + t_test_results["p_value"], method="fdr_bh" + )[1] res.append(t_test_results) - # for i in range(X.shape[0]): - # row= regulon_score.index[i] - # # Perform the t-test between X and Y for the current variable - # t_stat, p_value = stats.ttest_ind(X.loc[row], Y.loc[row],alternative='greater') - # # Append the results to the DataFrame - # p_value = np.nan_to_num(p_value, nan=1) - # t_test_results[i,3*j+1] = p_value - # t_test_results[i,3*j] = t_stat - # t_test_results[:,3*j+2]=smm.multipletests(t_test_results[:,3*j+1], method='fdr_bh')[1] - t_test_results = pd.concat(res, axis = 1) + # for i in range(X.shape[0]): + # row= regulon_score.index[i] + # # Perform the t-test between X and Y for the current variable + # t_stat, p_value = stats.ttest_ind(X.loc[row], Y.loc[row],alternative='greater') + # # Append the results to the DataFrame + # p_value = np.nan_to_num(p_value, nan=1) + # t_test_results[i,3*j+1] = p_value + # t_test_results[i,3*j] = t_stat + # t_test_results[:,3*j+2]=smm.multipletests(t_test_results[:,3*j+1], method='fdr_bh')[1] + t_test_results = pd.concat(res, axis=1) # t_test_results=pd.DataFrame(t_test_results,index=regulon_score.index) - col=[0 for kk in range(len(label_set)*3)] + col = [0 for kk in range(len(label_set) * 3)] for j in range(len(label_set)): - col[3*j]=label_set[j]+'_t_stat' - col[3*j+1]=label_set[j]+'_p_value' - col[3*j+2]=label_set[j]+'_adj_p' - t_test_results.columns=col + col[3 * j] = label_set[j] + "_t_stat" + col[3 * j + 1] = label_set[j] + "_p_value" + col[3 * j + 2] = label_set[j] + "_adj_p" + t_test_results.columns = col return t_test_results -def heatmap_cluster(regulon_score,adata_RNA,save,outdir): + + +def heatmap_cluster(regulon_score, adata_RNA, save, outdir): import matplotlib.pyplot as plt - import numpy as np import seaborn as sns from scipy.stats import zscore -# Generate random data for the heatmap - Vars=regulon_score.var(axis=1) - regulon_score=regulon_score.loc[regulon_score.index[Vars>0]] + + # Generate random data for the heatmap + Vars = regulon_score.var(axis=1) + regulon_score = regulon_score.loc[regulon_score.index[Vars > 0]] z_scores = zscore(regulon_score, axis=1) - z_scores1=z_scores.values - label=adata_RNA.obs['label'].values - labelset=set(label) - idx=0 + z_scores1 = z_scores.values + label = adata_RNA.obs["label"].values + labelset = set(label) + idx = 0 for labeltemp in labelset: - index=(label==labeltemp) - z_scores1[:,idx:idx+index.sum()]=z_scores[z_scores.columns[index]].values - idx=idx+index.sum() - z_scores1[z_scores1<-2]=-2 - z_scores1[z_scores1>2]=2 -# Set up the figure size + index = label == labeltemp + z_scores1[:, idx : idx + index.sum()] = z_scores[z_scores.columns[index]].values + idx = idx + index.sum() + z_scores1[z_scores1 < -2] = -2 + z_scores1[z_scores1 > 2] = 2 + # Set up the figure size plt.figure(figsize=(8, 6)) sns.clustermap(z_scores1, row_cluster=True, col_cluster=False) - plt.xlabel('Columns') - plt.ylabel('Rows') - if save==True: - plt.savefig(outdir+"heatmap_activity.png", format='png', bbox_inches='tight') + plt.xlabel("Columns") + plt.ylabel("Rows") + if save == True: + plt.savefig(outdir + "heatmap_activity.png", format="png", bbox_inches="tight") # Finally, display the plot plt.show() -def box_comp(TFName,adata_RNA,celltype1,celltype2,datatype,regulon_score,save,outdir): + +def box_comp( + TFName, adata_RNA, celltype1, celltype2, datatype, regulon_score, save, outdir +): import numpy as np - data=np.zeros(regulon_score.shape[1]) - if datatype=='activity': - TFexp=regulon_score.loc[TFName].values - if datatype=='expression': - data0=pd.DataFrame(adata_RNA.X.toarray().T,index=adata_RNA.var['gene_ids'].values,columns=adata_RNA.obs['barcode'].values) - TFexp=data0.loc[TFName].values - label=adata_RNA.obs['label'].values - if type(label[0]) in [np.int64,np.int32,np.float64,np.float32]: - label=[str(label[i]) for i in range(len(label))] - if celltype1=='Others': - G2=TFexp[np.array(label)==celltype2] - G1=TFexp[np.array(label)!=celltype2] - elif celltype2=='Others': - G1=TFexp[np.array(label)==celltype1] - G2=TFexp[np.array(label)!=celltype1] + + data = np.zeros(regulon_score.shape[1]) + if datatype == "activity": + TFexp = regulon_score.loc[TFName].values + if datatype == "expression": + data0 = pd.DataFrame( + adata_RNA.X.toarray().T, + index=adata_RNA.var["gene_ids"].values, + columns=adata_RNA.obs["barcode"].values, + ) + TFexp = data0.loc[TFName].values + label = adata_RNA.obs["label"].values + if type(label[0]) in [np.int64, np.int32, np.float64, np.float32]: + label = [str(label[i]) for i in range(len(label))] + if celltype1 == "Others": + G2 = TFexp[np.array(label) == celltype2] + G1 = TFexp[np.array(label) != celltype2] + elif celltype2 == "Others": + G1 = TFexp[np.array(label) == celltype1] + G2 = TFexp[np.array(label) != celltype1] else: - G1=TFexp[np.array(label)==celltype1] - G2=TFexp[np.array(label)==celltype2] + G1 = TFexp[np.array(label) == celltype1] + G2 = TFexp[np.array(label) == celltype2] import matplotlib.pyplot as plt - import numpy as np import seaborn as sns + # Combine the vectors into a single list data = [G1, G2] # Set up the figure size and style @@ -262,12 +307,25 @@ def box_comp(TFName,adata_RNA,celltype1,celltype2,datatype,regulon_score,save,ou sns.violinplot(data=data, palette=colors) # Add box plots to the violin plot sns.boxplot(data=data, color="white", width=0.15) -# Customize the plot - plt.xlabel('Violins') - plt.ylabel('Values') -# Rename x-axis labels + # Customize the plot + plt.xlabel("Violins") + plt.ylabel("Values") + # Rename x-axis labels plt.xticks([0, 1], [celltype1, celltype2]) - if save==True: - plt.savefig(outdir+"box_plot_"+TFName+'_'+datatype+'_'+celltype1+'_'+celltype2+".png", format='png', bbox_inches='tight') -# Finally, display the plot + if save == True: + plt.savefig( + outdir + + "box_plot_" + + TFName + + "_" + + datatype + + "_" + + celltype1 + + "_" + + celltype2 + + ".png", + format="png", + bbox_inches="tight", + ) + # Finally, display the plot plt.show() diff --git a/code/lingergrn-1.106/LingerGRN/perturb.py b/code/lingergrn-1.106/LingerGRN/perturb.py index c4f81b8..064c071 100644 --- a/code/lingergrn-1.106/LingerGRN/perturb.py +++ b/code/lingergrn-1.106/LingerGRN/perturb.py @@ -1,6 +1,7 @@ def generate_colors(N): import matplotlib.colors as mcolors import seaborn as sns + """ Generate N visually appealing colors using seaborn color palette. @@ -13,222 +14,304 @@ def generate_colors(N): color_palette = sns.color_palette("husl", N) colors = [mcolors.rgb2hex(color_palette[i]) for i in range(N)] return colors -def load_data_ptb(Input_dir,outdir,GRNdir): - import pandas as pd + + +def load_data_ptb(Input_dir, outdir, GRNdir): import numpy as np + import pandas as pd import torch - ATAC_file='ATAC.txt' - idx_file=outdir+'index.txt' - RNA_file='RNA.txt' - label_file='label.txt' - TFName=outdir+'TFName.txt' + + ATAC_file = "ATAC.txt" + idx_file = outdir + "index.txt" + RNA_file = "RNA.txt" + label_file = "label.txt" + TFName = outdir + "TFName.txt" from LingerGRN import pseudo_bulk - RNA=pd.read_csv(Input_dir+RNA_file,sep='\t',index_col=0) - ATAC=pd.read_csv(Input_dir+ATAC_file,sep='\t',index_col=0) + + RNA = pd.read_csv(Input_dir + RNA_file, sep="\t", index_col=0) + ATAC = pd.read_csv(Input_dir + ATAC_file, sep="\t", index_col=0) RNA = np.log2(1 + RNA) from sklearn.impute import KNNImputer - K=int(np.floor(np.sqrt(RNA.shape[1]))) + + K = int(np.floor(np.sqrt(RNA.shape[1]))) imputer = KNNImputer(n_neighbors=K) -# RNA row is genes col is cells + # RNA row is genes col is cells TG_filter1 = imputer.fit_transform(RNA.values.T) - TG_filter1=pd.DataFrame(TG_filter1.T,columns=RNA.columns,index=RNA.index) - RE_filter1 = imputer.fit_transform(np.log2(1+ATAC.values.T)) - RE_filter1=pd.DataFrame(RE_filter1.T,columns=ATAC.columns,index=ATAC.index) - #Opn=pd.read_csv(Opn_file,header=0,sep='\t',index_col=0) - #Opn=Opn.values - idx=pd.read_csv(idx_file,header=None,sep='\t') - #Target=pd.read_csv(geneexp_file,header=0,sep='\t',index_col=0) - genename=pd.read_csv(outdir+'Symbol.txt',sep='\t',header=None) - genename=genename[0].values - TFname=pd.read_csv(outdir+'TFName.txt',sep='\t',header=None) - TFname=TFname[0].values - Exp=TG_filter1.loc[TFname].values - Target=TG_filter1.loc[genename].values - Opn=RE_filter1.values - chrall=[str(i+1) for i in range(22)] - chrall.append('X') - data_merge=pd.read_csv(outdir+'data_merge.txt',sep='\t',index_col=0) - return chrall,data_merge,Exp,Opn,Target,idx,TFname -def LINGER_simulation(ii,gene_chr,TFindex,Exp,REindex,Opn,netall,index_all): - import warnings + TG_filter1 = pd.DataFrame(TG_filter1.T, columns=RNA.columns, index=RNA.index) + RE_filter1 = imputer.fit_transform(np.log2(1 + ATAC.values.T)) + RE_filter1 = pd.DataFrame(RE_filter1.T, columns=ATAC.columns, index=ATAC.index) + # Opn=pd.read_csv(Opn_file,header=0,sep='\t',index_col=0) + # Opn=Opn.values + idx = pd.read_csv(idx_file, header=None, sep="\t") + # Target=pd.read_csv(geneexp_file,header=0,sep='\t',index_col=0) + genename = pd.read_csv(outdir + "Symbol.txt", sep="\t", header=None) + genename = genename[0].values + TFname = pd.read_csv(outdir + "TFName.txt", sep="\t", header=None) + TFname = TFname[0].values + Exp = TG_filter1.loc[TFname].values + Target = TG_filter1.loc[genename].values + Opn = RE_filter1.values + chrall = [str(i + 1) for i in range(22)] + chrall.append("X") + data_merge = pd.read_csv(outdir + "data_merge.txt", sep="\t", index_col=0) + return chrall, data_merge, Exp, Opn, Target, idx, TFname + + +def LINGER_simulation(ii, gene_chr, TFindex, Exp, REindex, Opn, netall, index_all): import time - import LingerGRN - from tqdm import tqdm - import torch - import pandas as pd + import warnings + import numpy as np - eps=1e-6 - gene_idx=gene_chr['id_s'].values[ii]-1 - TFidxtemp=TFindex[gene_idx] - TFidxtemp=TFidxtemp.split('_') - TFidxtemp=[int(TFidxtemp[k])+1 for k in range(len(TFidxtemp))] - TFtemp=Exp[np.array(TFidxtemp)-1,:] - REidxtemp=REindex[gene_idx] - REidxtemp=str(REidxtemp).split('_') - if (len(REidxtemp)==1)&(REidxtemp[0]=='nan'): - REidxtemp=[] - inputs=TFtemp+1-1 + import pandas as pd + import torch + from tqdm import tqdm + + import LingerGRN + + eps = 1e-6 + gene_idx = gene_chr["id_s"].values[ii] - 1 + TFidxtemp = TFindex[gene_idx] + TFidxtemp = TFidxtemp.split("_") + TFidxtemp = [int(TFidxtemp[k]) + 1 for k in range(len(TFidxtemp))] + TFtemp = Exp[np.array(TFidxtemp) - 1, :] + REidxtemp = REindex[gene_idx] + REidxtemp = str(REidxtemp).split("_") + if (len(REidxtemp) == 1) & (REidxtemp[0] == "nan"): + REidxtemp = [] + inputs = TFtemp + 1 - 1 else: - REidxtemp=[int(REidxtemp[k])+1 for k in range(len(REidxtemp))] - REtemp=Opn[np.array(REidxtemp)-1,:] - inputs=np.vstack((TFtemp, REtemp)) - inputs = torch.tensor(inputs,dtype=torch.float32) + REidxtemp = [int(REidxtemp[k]) + 1 for k in range(len(REidxtemp))] + REtemp = Opn[np.array(REidxtemp) - 1, :] + inputs = np.vstack((TFtemp, REtemp)) + inputs = torch.tensor(inputs, dtype=torch.float32) mean = inputs.mean(dim=1) std = inputs.std(dim=1) - inputs = (inputs.T - mean) / (std+eps) - inputs=inputs.T - num_nodes=inputs.shape[0] + inputs = (inputs.T - mean) / (std + eps) + inputs = inputs.T + num_nodes = inputs.shape[0] loaded_net = netall[index_all] X_tr = inputs.T y_pred = loaded_net(X_tr) return y_pred -def get_simulation(outdir,chrall,data_merge,GRNdir,Exp,Opn,Target,idx): - import warnings + + +def get_simulation(outdir, chrall, data_merge, GRNdir, Exp, Opn, Target, idx): import time - import LingerGRN - from tqdm import tqdm - import torch - import pandas as pd + import warnings + import numpy as np - output=np.zeros(Target.shape) + import pandas as pd + import torch + from tqdm import tqdm + + import LingerGRN + + output = np.zeros(Target.shape) for i in range(23): - chr='chr'+chrall[i] + chr = "chr" + chrall[i] print(chr) - gene_chr=data_merge[data_merge['chr']==chr] - N=len(gene_chr) - netall=torch.load(outdir+'net_'+chr+'.pt') - idx_file1=GRNdir+chr+'_index.txt' - idx_file_all=GRNdir+chr+'_index_all.txt' - idxRE_all=pd.read_csv(idx_file_all,header=None,sep='\t') - gene_chr=data_merge[data_merge['chr']==chr] - N=len(gene_chr) - TFindex=idx.values[:,2] - REindex=idx.values[:,1] + gene_chr = data_merge[data_merge["chr"] == chr] + N = len(gene_chr) + netall = torch.load(outdir + "net_" + chr + ".pt") + idx_file1 = GRNdir + chr + "_index.txt" + idx_file_all = GRNdir + chr + "_index_all.txt" + idxRE_all = pd.read_csv(idx_file_all, header=None, sep="\t") + gene_chr = data_merge[data_merge["chr"] == chr] + N = len(gene_chr) + TFindex = idx.values[:, 2] + REindex = idx.values[:, 1] for ii in tqdm(range(N)): - index_all=gene_chr.index[ii] + index_all = gene_chr.index[ii] if index_all in netall.keys(): - res=LINGER_simulation(ii,gene_chr,TFindex,Exp,REindex,Opn,netall,index_all) - output[index_all,:]=res.detach().numpy().reshape(-1,) - output1=pd.DataFrame(output,index=data_merge.loc[range(Target.shape[0])]['Symbol']) + res = LINGER_simulation( + ii, gene_chr, TFindex, Exp, REindex, Opn, netall, index_all + ) + output[index_all, :] = ( + res.detach() + .numpy() + .reshape( + -1, + ) + ) + output1 = pd.DataFrame( + output, index=data_merge.loc[range(Target.shape[0])]["Symbol"] + ) return output1 -def umap_embedding(outdir,Target,original,perturb,Input_dir): - import umap + + +def umap_embedding(outdir, Target, original, perturb, Input_dir): import scanpy as sc -# Assuming you have loaded or created an AnnData object named 'adata' -# Create and train the UMAP model + import umap + + # Assuming you have loaded or created an AnnData object named 'adata' + # Create and train the UMAP model from sklearn.decomposition import PCA import numpy as np import pandas as pd -#RNA=pd.read_csv(Input_dir+'RNA.txt',header=0,index_col=0,sep='\t') - Symbol=pd.read_csv(outdir+'Symbol.txt',header=None,sep='\t') -#sampleall=RNA.columns - RNA=pd.DataFrame(Target,index=Symbol[0].values) -# Assuming your feature * sample matrix is stored in a variable "matrix" -# Step 1: Calculate the variance across the samples for each feature + from sklearn.decomposition import PCA + + # RNA=pd.read_csv(Input_dir+'RNA.txt',header=0,index_col=0,sep='\t') + Symbol = pd.read_csv(outdir + "Symbol.txt", header=None, sep="\t") + # sampleall=RNA.columns + RNA = pd.DataFrame(Target, index=Symbol[0].values) + # Assuming your feature * sample matrix is stored in a variable "matrix" + # Step 1: Calculate the variance across the samples for each feature variance = np.var(RNA.values, axis=1) -# Step 2: Sort the features based on the variance in descending order + # Step 2: Sort the features based on the variance in descending order sorted_indices = np.argsort(variance)[::-1] -# Step 3: Select the top 2000 features + # Step 3: Select the top 2000 features top_2000_features = sorted_indices[:2000] -# Assuming you have loaded or created an AnnData object named 'adata' -# Perform PCA using the 'arpack' solver - pca = PCA(svd_solver='arpack') - pca.fit(RNA.values[top_2000_features,:].T) - pca_result = pca.fit_transform(RNA.values[top_2000_features,:].T) - pca_result=pca_result[:,1:20] - original1=original.loc[RNA.index[top_2000_features]].values - original1[original1<0]=0 - original1=np.log2(1+original1) - #original1=original1 - perturb1=perturb.loc[RNA.index[top_2000_features]].values - perturb1[perturb1<0]=0 - perturb1=np.log2(perturb1+1) - #perturb1=perturb1 - O_PCA=pca.fit_transform(original1.T) - P_PCA=pca.fit_transform(perturb1.T) - O_PCA=O_PCA[:,1:20] - P_PCA=P_PCA[:,1:20] + # Assuming you have loaded or created an AnnData object named 'adata' + # Perform PCA using the 'arpack' solver + pca = PCA(svd_solver="arpack") + pca.fit(RNA.values[top_2000_features, :].T) + pca_result = pca.fit_transform(RNA.values[top_2000_features, :].T) + pca_result = pca_result[:, 1:20] + original1 = original.loc[RNA.index[top_2000_features]].values + original1[original1 < 0] = 0 + original1 = np.log2(1 + original1) + # original1=original1 + perturb1 = perturb.loc[RNA.index[top_2000_features]].values + perturb1[perturb1 < 0] = 0 + perturb1 = np.log2(perturb1 + 1) + # perturb1=perturb1 + O_PCA = pca.fit_transform(original1.T) + P_PCA = pca.fit_transform(perturb1.T) + O_PCA = O_PCA[:, 1:20] + P_PCA = P_PCA[:, 1:20] umap_model = umap.UMAP(n_components=2) umap_model.fit(pca_result) O_cell_umap = umap_model.transform(O_PCA) P_cell_umap = umap_model.transform(P_PCA) embedding = umap_model.transform(pca_result) - D=P_cell_umap-O_cell_umap - return embedding,D + D = P_cell_umap - O_cell_umap + return embedding, D + # Assuming you have a continuous value stored in `continuous_values` # Define the color for small values (e.g., white) and the color for higher values -def diff_umap(TFko,TFName,save,outdir,embedding,perturb,original,Input_dir): - import seaborn as sns +def diff_umap(TFko, TFName, save, outdir, embedding, perturb, original, Input_dir): import matplotlib.colors as mcolors - import pandas as pd - import numpy as np import matplotlib.pyplot as plt - label=pd.read_csv(Input_dir+'label.txt',sep='\t',header=None) - label=label[0].values + import numpy as np + import pandas as pd + import seaborn as sns + + label = pd.read_csv(Input_dir + "label.txt", sep="\t", header=None) + label = label[0].values from matplotlib.colors import LinearSegmentedColormap - zero_color = 'white' - positive_color = 'orange' - negative_color = 'blue' -# Define the colormap with a white-to-orange-to-blue gradient + + zero_color = "white" + positive_color = "orange" + negative_color = "blue" + # Define the colormap with a white-to-orange-to-blue gradient cmap_colors = [negative_color, zero_color, positive_color] -# Define the colors for each cluster - sns.set(style='white') + # Define the colors for each cluster + sns.set(style="white") fig, ax = plt.subplots(figsize=(4, 4)) - continuous_values=perturb.loc[TFName].values-original.loc[TFName].values -#continuous_values[continuous_values<0]=0 - cmap = LinearSegmentedColormap.from_list('custom_cmap', cmap_colors) -# Create a scatter plot with colored dots based on the cluster annotations - plt.scatter(embedding[:,0], embedding[:,1], c=continuous_values,cmap=cmap,s=2, - vmin=-np.abs(continuous_values).max(), vmax=np.abs(continuous_values).max()) - anno=label + continuous_values = perturb.loc[TFName].values - original.loc[TFName].values + # continuous_values[continuous_values<0]=0 + cmap = LinearSegmentedColormap.from_list("custom_cmap", cmap_colors) + # Create a scatter plot with colored dots based on the cluster annotations + plt.scatter( + embedding[:, 0], + embedding[:, 1], + c=continuous_values, + cmap=cmap, + s=2, + vmin=-np.abs(continuous_values).max(), + vmax=np.abs(continuous_values).max(), + ) + anno = label unique_clusters = np.unique(anno) for cluster in unique_clusters: indices = np.where(anno == cluster) - cluster_center = (np.mean(embedding[indices, 0]), np.mean(embedding[indices, 1])) - plt.text(cluster_center[0], cluster_center[1], f'Cluster {cluster}', fontsize=10, ha='center', va='center') - plt.colorbar() - plt.xlabel('Umap 1') - plt.ylabel('Umap 2') - if save==True: - plt.savefig(outdir+TFko+"_KO_Diff_exp_Umap_"+TFName+".png", format='png', bbox_inches='tight') -# Add arrows to indicate gene expression changes + cluster_center = ( + np.mean(embedding[indices, 0]), + np.mean(embedding[indices, 1]), + ) + plt.text( + cluster_center[0], + cluster_center[1], + f"Cluster {cluster}", + fontsize=10, + ha="center", + va="center", + ) + plt.colorbar() + plt.xlabel("Umap 1") + plt.ylabel("Umap 2") + if save == True: + plt.savefig( + outdir + TFko + "_KO_Diff_exp_Umap_" + TFName + ".png", + format="png", + bbox_inches="tight", + ) + # Add arrows to indicate gene expression changes plt.show() plt.close() + + # Define the colors for each cluster -def Umap_direct(TFko,Input_dir,embedding,D,save,outdir): - import seaborn as sns +def Umap_direct(TFko, Input_dir, embedding, D, save, outdir): import matplotlib.colors as mcolors - import pandas as pd + import matplotlib.pyplot as plt import numpy as np - import matplotlib.pyplot as plt - label=pd.read_csv(Input_dir+'label.txt',sep='\t',header=None) - label=label[0].values - N=len(np.unique(label)) + import pandas as pd + import seaborn as sns + + label = pd.read_csv(Input_dir + "label.txt", sep="\t", header=None) + label = label[0].values + N = len(np.unique(label)) colors = generate_colors(N) - sns.set(style='white') - label1=label.copy() - anno=label + sns.set(style="white") + label1 = label.copy() + anno = label unique_clusters = np.unique(anno) - D[np.abs(D[:,0])0).sum(axis=1)>=1) + D[np.abs(D[:, 0]) < np.abs(D).mean(axis=0)[0], 0] = 0 + D[np.abs(D[:, 1]) < np.abs(D).mean(axis=0)[1], 1] = 0 + idx = (np.abs(D) > 0).sum(axis=1) >= 1 if type(label[0]) is str: for i in range(N): - label1[label==unique_clusters[i]]=i + label1[label == unique_clusters[i]] = i fig, ax = plt.subplots(figsize=(4, 4)) - continuous_values=[colors[i] for i in label1] -# Create a scatter plot with colored dots based on the cluster annotations - plt.scatter(embedding[:,0], embedding[:,1], c=continuous_values, s=2) + continuous_values = [colors[i] for i in label1] + # Create a scatter plot with colored dots based on the cluster annotations + plt.scatter(embedding[:, 0], embedding[:, 1], c=continuous_values, s=2) for cluster in unique_clusters: indices = np.where(anno == cluster) - cluster_center = (np.mean(embedding[indices, 0]), np.mean(embedding[indices, 1])) - plt.text(cluster_center[0], cluster_center[1], f'Cluster {cluster}', fontsize=10, ha='center', va='center') -# Add arrows to indicate gene expression changes - ax.quiver(embedding[idx,0], embedding[idx, 1], - 2*D[idx,0], # Assuming gene_index is the index of the gene you are interested in - 2*D[idx,1], # Assuming gene_index+1 is the index of another gene for the y-component - scale=30, scale_units='inches', alpha=0.5) - if save==True: - plt.savefig(outdir+TFko+"_KO_Differentiation_Umap.png", format='png', bbox_inches='tight') - plt.show() \ No newline at end of file + cluster_center = ( + np.mean(embedding[indices, 0]), + np.mean(embedding[indices, 1]), + ) + plt.text( + cluster_center[0], + cluster_center[1], + f"Cluster {cluster}", + fontsize=10, + ha="center", + va="center", + ) + # Add arrows to indicate gene expression changes + ax.quiver( + embedding[idx, 0], + embedding[idx, 1], + 2 + * D[ + idx, 0 + ], # Assuming gene_index is the index of the gene you are interested in + 2 + * D[ + idx, 1 + ], # Assuming gene_index+1 is the index of another gene for the y-component + scale=30, + scale_units="inches", + alpha=0.5, + ) + if save == True: + plt.savefig( + outdir + TFko + "_KO_Differentiation_Umap.png", + format="png", + bbox_inches="tight", + ) + plt.show() diff --git a/code/lingergrn-1.106/LingerGRN/preprocess.py b/code/lingergrn-1.106/LingerGRN/preprocess.py index 85c3fc8..ae4e3a9 100644 --- a/code/lingergrn-1.106/LingerGRN/preprocess.py +++ b/code/lingergrn-1.106/LingerGRN/preprocess.py @@ -1,328 +1,451 @@ import os + +# from LingerGRN.immupute_dis import immupute_dis +# import LingerGRN.pseudo_bulk as pseudo_bulk +import subprocess + import numpy as np import pandas as pd -#from LingerGRN.immupute_dis import immupute_dis -#import LingerGRN.pseudo_bulk as pseudo_bulk -import subprocess from tqdm import tqdm -def list2mat(df,i_n,j_n,x_n): + + +def list2mat(df, i_n, j_n, x_n): TFs = df[j_n].unique() REs = df[i_n].unique() -#Initialize matrix as numpy array -#Map row and col indices for lookup - row_map = {r:i for i,r in enumerate(REs)} - col_map = {c:i for i,c in enumerate(TFs)} + # Initialize matrix as numpy array + # Map row and col indices for lookup + row_map = {r: i for i, r in enumerate(REs)} + col_map = {c: i for i, c in enumerate(TFs)} row_indices = np.array([row_map[row] for row in df[i_n]]) col_indices = np.array([col_map[col] for col in df[j_n]]) from scipy.sparse import coo_matrix - matrix = coo_matrix((df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs))) - mat=coo_matrix.toarray(matrix) - return mat,REs,TFs + matrix = coo_matrix( + (df[x_n], (row_indices, col_indices)), shape=(len(REs), len(TFs)) + ) + mat = coo_matrix.toarray(matrix) + return mat, REs, TFs -def gene_expression(GRNdir,TG_pseudobulk,outdir): - gene = pd.read_csv(GRNdir+'bulk_gene_all.txt') - gene.columns=['gene'] - #gene=gene['gene'] - d1 = np.isin(TG_pseudobulk.index, gene['gene'].values) - List = TG_pseudobulk.index[d1] - A = np.log2(1 + TG_pseudobulk.loc[List]) - #Write Exp.txt and Symbol.txt - pd.DataFrame(A).to_csv(outdir+'Exp.txt',sep='\t',index=False,header=False) - pd.DataFrame(List).to_csv(outdir+'Symbol.txt', sep='\t', header=False, index=False) - pd.DataFrame(A.columns).to_csv(outdir+'Col.txt',sep='\t',index=False,header=False) - return List,A +def gene_expression(GRNdir, TG_pseudobulk, outdir): + gene = pd.read_csv(GRNdir + "bulk_gene_all.txt") + gene.columns = ["gene"] + # gene=gene['gene'] + d1 = np.isin(TG_pseudobulk.index, gene["gene"].values) + List = TG_pseudobulk.index[d1] + A = np.log2(1 + TG_pseudobulk.loc[List]) + # Write Exp.txt and Symbol.txt + pd.DataFrame(A).to_csv(outdir + "Exp.txt", sep="\t", index=False, header=False) + pd.DataFrame(List).to_csv( + outdir + "Symbol.txt", sep="\t", header=False, index=False + ) + pd.DataFrame(A.columns).to_csv( + outdir + "Col.txt", sep="\t", index=False, header=False + ) + return List, A -def TF_expression(TFName,List,Match2,A,outdir): - d= np.isin(TFName,List) +def TF_expression(TFName, List, Match2, A, outdir): + d = np.isin(TFName, List) TFName = TFName[d] - List_idx=pd.DataFrame(range(len(List)),index=List) - f=List_idx.loc[TFName][0].values + List_idx = pd.DataFrame(range(len(List)), index=List) + f = List_idx.loc[TFName][0].values TF = A.values[f, :] Match2 = Match2[np.isin(Match2[:, 1], TFName)] d = np.isin(TFName, Match2[:, 1]) TFName = TFName[d] TF = TF[d, :] - pd.DataFrame(TF).to_csv(outdir+'TFexp.txt', sep='\t', header=False, index=False) - pd.DataFrame(TFName).to_csv(outdir+'TFName.txt', sep='\t', header=False, index=False) - return TFName + pd.DataFrame(TF).to_csv(outdir + "TFexp.txt", sep="\t", header=False, index=False) + pd.DataFrame(TFName).to_csv( + outdir + "TFName.txt", sep="\t", header=False, index=False + ) + return TFName + -def index_generate(choosL_i,merged_s,merged_b,TFName): +def index_generate(choosL_i, merged_s, merged_b, TFName): if choosL_i in merged_s.index: - REid = merged_s.loc[choosL_i]['id_s'] - REid_b = merged_b.loc[choosL_i]['id_b'] + REid = merged_s.loc[choosL_i]["id_s"] + REid_b = merged_b.loc[choosL_i]["id_b"] else: - REid='' - REid_b='' + REid = "" + REid_b = "" TFName_1 = np.delete(TFName, np.where(TFName == choosL_i)) TFid = np.where(np.isin(TFName, TFName_1))[0] - RE_s = '_'.join(map(str, REid)) - TF_s = '_'.join(map(str, TFid)) - RE_b = '_'.join(map(str, REid_b)) + RE_s = "_".join(map(str, REid)) + TF_s = "_".join(map(str, TFid)) + RE_b = "_".join(map(str, REid_b)) return choosL_i, RE_s, TF_s, RE_b -def load_corr_RE_TG(List,Element_name,Element_name_bulk,outdir): - Element_gene = pd.read_csv(outdir+"hg19_Peak_hg19_gene_u.txt", delimiter="\t", header=None) - choosL = List - index_ElementName=pd.DataFrame(np.arange(0, len(Element_name)),index=Element_name) - index_Element_name_bulk=pd.DataFrame(np.arange(0, len(Element_name_bulk)),index=Element_name_bulk) - index_Element_name_bulk=index_Element_name_bulk.groupby(index_Element_name_bulk.index).min() - Element_gene.columns=['Element_name_b','Element_name_s','TG'] - Element_gene['value']=1 - #RE_all_s=Element_gene['Element_name_s'].unique() - #RE_all_b=Element_gene['Element_name_b'].unique() - #index_RE_all_s=pd.DataFrame(np.arange(0, len(RE_all_s)),index=RE_all_s) - #index_RE_all_b=pd.DataFrame(np.arange(0, len(RE_all_b)),index=RE_all_b) - Element_gene['id_s']=index_ElementName.loc[Element_gene['Element_name_s']][0].values - Element_gene['id_b']=index_Element_name_bulk.loc[Element_gene['Element_name_b']][0].values - merged_s = Element_gene.groupby('TG')['id_s'].agg(list).reset_index() - merged_b = Element_gene.groupby('TG')['id_b'].agg(list).reset_index() - merged_s = merged_s.set_index('TG') - merged_b =merged_b.set_index('TG') - #index_ElementName1=index_ElementName.loc[RE_all_s][0].values - #index_Element_name_bulk1=index_Element_name_bulk.loc[RE_all_b][0].values - return merged_s,merged_b - -def load_motifbinding_chr(chrN,GRNdir,motifWeight,outdir): - Motif_binding_temp=pd.read_csv(GRNdir+'MotifTarget_Matrix_'+chrN+'.txt',sep='\t',index_col=0) - REs=Motif_binding_temp.index - march_hg19_Regrion=pd.read_csv(outdir+'MotifTarget_hg19_hg38_'+chrN+'.txt',sep='\t',header=None) - REoverlap=list(set(march_hg19_Regrion[1].values)) - Motif_binding_temp1=Motif_binding_temp.loc[REoverlap] - REs=Motif_binding_temp1.index - Motif_binding_temp=np.zeros([march_hg19_Regrion.shape[0],Motif_binding_temp.shape[1]]) - Motif_binding_temp=Motif_binding_temp1.loc[march_hg19_Regrion[1].values].values - Motif_binding_temp=pd.DataFrame(Motif_binding_temp,index=march_hg19_Regrion[0].values,columns=Motif_binding_temp1.columns) - Motif_binding_temp1=Motif_binding_temp.groupby(Motif_binding_temp.index).max() - motifoverlap=list(set(Motif_binding_temp1.columns)&set(motifWeight.index)) - Motif_binding_temp1=Motif_binding_temp1[motifoverlap] - motifWeight=motifWeight.loc[Motif_binding_temp1.columns] + +def load_corr_RE_TG(List, Element_name, Element_name_bulk, outdir): + Element_gene = pd.read_csv( + outdir + "hg19_Peak_hg19_gene_u.txt", delimiter="\t", header=None + ) + choosL = List + index_ElementName = pd.DataFrame( + np.arange(0, len(Element_name)), index=Element_name + ) + index_Element_name_bulk = pd.DataFrame( + np.arange(0, len(Element_name_bulk)), index=Element_name_bulk + ) + index_Element_name_bulk = index_Element_name_bulk.groupby( + index_Element_name_bulk.index + ).min() + Element_gene.columns = ["Element_name_b", "Element_name_s", "TG"] + Element_gene["value"] = 1 + # RE_all_s=Element_gene['Element_name_s'].unique() + # RE_all_b=Element_gene['Element_name_b'].unique() + # index_RE_all_s=pd.DataFrame(np.arange(0, len(RE_all_s)),index=RE_all_s) + # index_RE_all_b=pd.DataFrame(np.arange(0, len(RE_all_b)),index=RE_all_b) + Element_gene["id_s"] = index_ElementName.loc[Element_gene["Element_name_s"]][ + 0 + ].values + Element_gene["id_b"] = index_Element_name_bulk.loc[Element_gene["Element_name_b"]][ + 0 + ].values + merged_s = Element_gene.groupby("TG")["id_s"].agg(list).reset_index() + merged_b = Element_gene.groupby("TG")["id_b"].agg(list).reset_index() + merged_s = merged_s.set_index("TG") + merged_b = merged_b.set_index("TG") + # index_ElementName1=index_ElementName.loc[RE_all_s][0].values + # index_Element_name_bulk1=index_Element_name_bulk.loc[RE_all_b][0].values + return merged_s, merged_b + + +def load_motifbinding_chr(chrN, GRNdir, motifWeight, outdir): + Motif_binding_temp = pd.read_csv( + GRNdir + "MotifTarget_Matrix_" + chrN + ".txt", sep="\t", index_col=0 + ) + REs = Motif_binding_temp.index + march_hg19_Regrion = pd.read_csv( + outdir + "MotifTarget_hg19_hg38_" + chrN + ".txt", sep="\t", header=None + ) + REoverlap = list(set(march_hg19_Regrion[1].values)) + Motif_binding_temp1 = Motif_binding_temp.loc[REoverlap] + REs = Motif_binding_temp1.index + Motif_binding_temp = np.zeros( + [march_hg19_Regrion.shape[0], Motif_binding_temp.shape[1]] + ) + Motif_binding_temp = Motif_binding_temp1.loc[march_hg19_Regrion[1].values].values + Motif_binding_temp = pd.DataFrame( + Motif_binding_temp, + index=march_hg19_Regrion[0].values, + columns=Motif_binding_temp1.columns, + ) + Motif_binding_temp1 = Motif_binding_temp.groupby(Motif_binding_temp.index).max() + motifoverlap = list(set(Motif_binding_temp1.columns) & set(motifWeight.index)) + Motif_binding_temp1 = Motif_binding_temp1[motifoverlap] + motifWeight = motifWeight.loc[Motif_binding_temp1.columns] Motif_binding = np.diag(1.0 / (motifWeight.T + 0.1)) * Motif_binding_temp1.values.T Motif_binding = np.log1p(Motif_binding) return Motif_binding_temp1 -def load_TFbinding(GRNdir,motifWeight,Match2,TFName,Element_name,outdir): + +def load_TFbinding(GRNdir, motifWeight, Match2, TFName, Element_name, outdir): from tqdm import tqdm - motif_binding=pd.DataFrame() - chrall=['chr'+str(i+1) for i in range(22)] - chrall.append('chrX') + + motif_binding = pd.DataFrame() + chrall = ["chr" + str(i + 1) for i in range(22)] + chrall.append("chrX") for chrN in tqdm(chrall): - Motif_binding_temp1=load_motifbinding_chr(chrN,GRNdir,motifWeight,outdir) - motif_binding=pd.concat([motif_binding,Motif_binding_temp1],join='outer',axis=0) - motif_binding=motif_binding.fillna(0) - motif_binding=motif_binding.groupby(motif_binding.index).max() - motifoverlap=list(set(motif_binding.columns)&set(motifWeight.index)) - Match2=Match2[np.isin(Match2[:, 0],motifoverlap), :] + Motif_binding_temp1 = load_motifbinding_chr(chrN, GRNdir, motifWeight, outdir) + motif_binding = pd.concat( + [motif_binding, Motif_binding_temp1], join="outer", axis=0 + ) + motif_binding = motif_binding.fillna(0) + motif_binding = motif_binding.groupby(motif_binding.index).max() + motifoverlap = list(set(motif_binding.columns) & set(motifWeight.index)) + Match2 = Match2[np.isin(Match2[:, 0], motifoverlap), :] TF_binding_temp = np.zeros((len(TFName), len(Element_name))) - Motif_binding=np.zeros((motif_binding.shape[1], len(Element_name))) - Element_name_idx=pd.DataFrame(range(len(Element_name)),index=Element_name) - idx=Element_name_idx.loc[motif_binding.index][0].values - Motif_binding=np.zeros((motif_binding.shape[1], len(Element_name))) - Motif_binding[:,idx]=motif_binding.loc[Element_name[idx]].values.T - Motif_binding=pd.DataFrame(Motif_binding,index=motif_binding.columns,columns=Element_name) - Match2=Match2[np.isin(Match2[:, 1],TFName), :] - Motif_binding=Motif_binding.loc[Match2[:, 0]] - Motif_binding.index=Match2[:, 1] - TF_binding=Motif_binding.groupby(Motif_binding.index).sum() + Motif_binding = np.zeros((motif_binding.shape[1], len(Element_name))) + Element_name_idx = pd.DataFrame(range(len(Element_name)), index=Element_name) + idx = Element_name_idx.loc[motif_binding.index][0].values + Motif_binding = np.zeros((motif_binding.shape[1], len(Element_name))) + Motif_binding[:, idx] = motif_binding.loc[Element_name[idx]].values.T + Motif_binding = pd.DataFrame( + Motif_binding, index=motif_binding.columns, columns=Element_name + ) + Match2 = Match2[np.isin(Match2[:, 1], TFName), :] + Motif_binding = Motif_binding.loc[Match2[:, 0]] + Motif_binding.index = Match2[:, 1] + TF_binding = Motif_binding.groupby(Motif_binding.index).sum() a = np.sum(TF_binding.values, axis=1) - a[a == 0] =1 - TF_binding_n = np.diag(1.0 / a) @TF_binding.values - TF_binding_n=pd.DataFrame(TF_binding_n.T,index=Element_name,columns=TF_binding.index) - TF_binding=np.zeros((len(Element_name),len(TFName))) - idx=np.isin(TFName,TF_binding_n.columns) - TF_binding[:,idx]=TF_binding_n[TFName[idx]].values - TF_binding=pd.DataFrame(TF_binding,index=Element_name,columns=TFName) - TF_binding.to_csv(outdir+'TF_binding.txt',sep='\t',index=None,header=None) - -def extract_overlap_regions(genome,GRNdir,outdir,method): - import pybedtools - import pandas as pd + a[a == 0] = 1 + TF_binding_n = np.diag(1.0 / a) @ TF_binding.values + TF_binding_n = pd.DataFrame( + TF_binding_n.T, index=Element_name, columns=TF_binding.index + ) + TF_binding = np.zeros((len(Element_name), len(TFName))) + idx = np.isin(TFName, TF_binding_n.columns) + TF_binding[:, idx] = TF_binding_n[TFName[idx]].values + TF_binding = pd.DataFrame(TF_binding, index=Element_name, columns=TFName) + TF_binding.to_csv(outdir + "TF_binding.txt", sep="\t", index=None, header=None) + + +def extract_overlap_regions(genome, GRNdir, outdir, method): import os + + import pandas as pd + import pybedtools + os.makedirs(outdir, exist_ok=True) - input_file = 'data/Peaks.txt' - output_file = outdir+'Region.bed' -# Read the input file - df = pd.read_csv(input_file, sep='\t',header=None) - chromosomes = [item.split(':')[0] for item in df[0].values] -# Drop the first row -# Replace ':' and '-' with tabs - df = df.replace({':': '\t', '-': '\t'}, regex=True) - chrall=['chr'+str(i+1) for i in range(23)]+['chrX'] - df=df[pd.DataFrame(chromosomes)[0].isin(chrall).values] + input_file = "data/Peaks.txt" + output_file = outdir + "Region.bed" + # Read the input file + df = pd.read_csv(input_file, sep="\t", header=None) + chromosomes = [item.split(":")[0] for item in df[0].values] + # Drop the first row + # Replace ':' and '-' with tabs + df = df.replace({":": "\t", "-": "\t"}, regex=True) + chrall = ["chr" + str(i + 1) for i in range(23)] + ["chrX"] + df = df[pd.DataFrame(chromosomes)[0].isin(chrall).values] df.to_csv(output_file, index=None, header=None) - if method=='LINGER': - if genome=='hg38': - a = pybedtools.example_bedtool(outdir+'Region.bed') - b = pybedtools.example_bedtool(GRNdir+'hg38_hg19_pair.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'temp.bed') - a_with_b=pd.read_csv(outdir+'temp.bed',sep='\t',header=None) - a_with_b[[6,7,8,0,1,2]].to_csv(outdir+'match_hg19_peak.bed',sep='\t',header=None,index=None) - if genome=='hg19': - a = pybedtools.example_bedtool(outdir+'Region.bed') - b = pybedtools.example_bedtool(GRNdir+'hg19_hg38_pair.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'temp.bed') - a_with_b=pd.read_csv(outdir+'temp.bed',sep='\t',header=None) - a_with_b[[6,7,8,0,1,2]].to_csv(outdir+'match_hg19_peak.bed',sep='\t',header=None,index=None) - a = pybedtools.example_bedtool(outdir+'match_hg19_peak.bed') - b = pybedtools.example_bedtool(GRNdir+'RE_gene_corr_hg19.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'temp.bed') - a_with_b=pd.read_csv(outdir+'temp.bed',sep='\t',header=None) - a_with_b=a_with_b[(a_with_b[1].values==a_with_b[7].values)&(a_with_b[2].values==a_with_b[8].values)] - a_with_b_n = pd.DataFrame({ - 'column1': a_with_b[0] + ':' + a_with_b[1].astype(str) + '-' + a_with_b[2].astype(str), - 'column2': a_with_b[3] + ':' + a_with_b[4].astype(str) + '-' + a_with_b[5].astype(str), - 'column3': a_with_b[9]}) - a_with_b_n=a_with_b_n.drop_duplicates() - a_with_b_n.to_csv(outdir+'hg19_Peak_hg19_gene_u.txt',sep='\t',header=None,index=None) - chr_all=['chr'+str(i+1) for i in range(22)] - chr_all.append('chrX') + if method == "LINGER": + if genome == "hg38": + a = pybedtools.example_bedtool(outdir + "Region.bed") + b = pybedtools.example_bedtool(GRNdir + "hg38_hg19_pair.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "temp.bed") + a_with_b = pd.read_csv(outdir + "temp.bed", sep="\t", header=None) + a_with_b[[6, 7, 8, 0, 1, 2]].to_csv( + outdir + "match_hg19_peak.bed", sep="\t", header=None, index=None + ) + if genome == "hg19": + a = pybedtools.example_bedtool(outdir + "Region.bed") + b = pybedtools.example_bedtool(GRNdir + "hg19_hg38_pair.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "temp.bed") + a_with_b = pd.read_csv(outdir + "temp.bed", sep="\t", header=None) + a_with_b[[6, 7, 8, 0, 1, 2]].to_csv( + outdir + "match_hg19_peak.bed", sep="\t", header=None, index=None + ) + a = pybedtools.example_bedtool(outdir + "match_hg19_peak.bed") + b = pybedtools.example_bedtool(GRNdir + "RE_gene_corr_hg19.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "temp.bed") + a_with_b = pd.read_csv(outdir + "temp.bed", sep="\t", header=None) + a_with_b = a_with_b[ + (a_with_b[1].values == a_with_b[7].values) + & (a_with_b[2].values == a_with_b[8].values) + ] + a_with_b_n = pd.DataFrame( + { + "column1": a_with_b[0] + + ":" + + a_with_b[1].astype(str) + + "-" + + a_with_b[2].astype(str), + "column2": a_with_b[3] + + ":" + + a_with_b[4].astype(str) + + "-" + + a_with_b[5].astype(str), + "column3": a_with_b[9], + } + ) + a_with_b_n = a_with_b_n.drop_duplicates() + a_with_b_n.to_csv( + outdir + "hg19_Peak_hg19_gene_u.txt", sep="\t", header=None, index=None + ) + chr_all = ["chr" + str(i + 1) for i in range(22)] + chr_all.append("chrX") for chrtemp in chr_all: - a = pybedtools.example_bedtool(outdir+'match_hg19_peak.bed') - b = pybedtools.example_bedtool(GRNdir+'MotifTarget_matrix_'+chrtemp+'.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'temp.bed') - a_with_b=pd.read_csv(outdir+'temp.bed',sep='\t',header=None) - a_with_b=a_with_b[(a_with_b[1].values==a_with_b[7].values)&(a_with_b[2].values==a_with_b[8].values)] - a_with_b_n = pd.DataFrame({ - 'column1': a_with_b[3] + ':' + a_with_b[4].astype(str) + '-' + a_with_b[5].astype(str), - 'column2': a_with_b[6] + ':' + a_with_b[7].astype(str) + '-' + a_with_b[8].astype(str)}) - a_with_b_n=a_with_b_n.drop_duplicates() - a_with_b_n.to_csv(outdir+'MotifTarget_hg19_hg38_'+chrtemp+'.txt',sep='\t',header=None,index=None) - a = pybedtools.example_bedtool(GRNdir+genome+'_Peaks_'+chrtemp+'.bed') - b = pybedtools.example_bedtool(outdir+'Region.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'Region_overlap_'+chrtemp+'.bed') - if method=='baseline': - chr_all=['chr'+str(i+1) for i in range(22)] - chr_all.append('chrX') + a = pybedtools.example_bedtool(outdir + "match_hg19_peak.bed") + b = pybedtools.example_bedtool( + GRNdir + "MotifTarget_matrix_" + chrtemp + ".bed" + ) + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "temp.bed") + a_with_b = pd.read_csv(outdir + "temp.bed", sep="\t", header=None) + a_with_b = a_with_b[ + (a_with_b[1].values == a_with_b[7].values) + & (a_with_b[2].values == a_with_b[8].values) + ] + a_with_b_n = pd.DataFrame( + { + "column1": a_with_b[3] + + ":" + + a_with_b[4].astype(str) + + "-" + + a_with_b[5].astype(str), + "column2": a_with_b[6] + + ":" + + a_with_b[7].astype(str) + + "-" + + a_with_b[8].astype(str), + } + ) + a_with_b_n = a_with_b_n.drop_duplicates() + a_with_b_n.to_csv( + outdir + "MotifTarget_hg19_hg38_" + chrtemp + ".txt", + sep="\t", + header=None, + index=None, + ) + a = pybedtools.example_bedtool( + GRNdir + genome + "_Peaks_" + chrtemp + ".bed" + ) + b = pybedtools.example_bedtool(outdir + "Region.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "Region_overlap_" + chrtemp + ".bed") + if method == "baseline": + chr_all = ["chr" + str(i + 1) for i in range(22)] + chr_all.append("chrX") for chrtemp in chr_all: - a = pybedtools.example_bedtool(GRNdir+genome+'_Peaks_'+chrtemp+'.bed') - b = pybedtools.example_bedtool(outdir+'Region.bed') - a_with_b = a.intersect(b, wa=True,wb=True) - a_with_b.saveas(outdir+'Region_overlap_'+chrtemp+'.bed') - - -def preprocess(TG_pseudobulk,RE_pseudobulk,GRNdir,genome,method,outdir): - #package_dir = os.path.dirname(os.path.abspath(__file__)) - if method=='LINGER': - extract_overlap_regions(genome,GRNdir,outdir,method) - print('Mapping gene expression...') - TFName = pd.read_csv(GRNdir+'TFName.txt',header=None) - TFName.columns=['TFName'] - TFName=TFName['TFName'].values - Match2=pd.read_csv(GRNdir+'Match2.txt',sep='\t') - Match2=Match2.values - List,A=gene_expression(GRNdir,TG_pseudobulk,outdir) - print('Generate TF expression...') - TFName=TF_expression(TFName,List,Match2,A,outdir) - print('Generate RE chromatin accessibility...') - RE_pseudobulk.to_csv(outdir+'Openness.txt',sep='\t',header=None,index=None) - print('Generate TF binding...') - Element_name_bulk = pd.read_csv(GRNdir+'all_hg19.txt', delimiter="\t", header=None) - Element_name_bulk=Element_name_bulk[0].values + a = pybedtools.example_bedtool( + GRNdir + genome + "_Peaks_" + chrtemp + ".bed" + ) + b = pybedtools.example_bedtool(outdir + "Region.bed") + a_with_b = a.intersect(b, wa=True, wb=True) + a_with_b.saveas(outdir + "Region_overlap_" + chrtemp + ".bed") + + +def preprocess(TG_pseudobulk, RE_pseudobulk, GRNdir, genome, method, outdir): + # package_dir = os.path.dirname(os.path.abspath(__file__)) + if method == "LINGER": + extract_overlap_regions(genome, GRNdir, outdir, method) + print("Mapping gene expression...") + TFName = pd.read_csv(GRNdir + "TFName.txt", header=None) + TFName.columns = ["TFName"] + TFName = TFName["TFName"].values + Match2 = pd.read_csv(GRNdir + "Match2.txt", sep="\t") + Match2 = Match2.values + List, A = gene_expression(GRNdir, TG_pseudobulk, outdir) + print("Generate TF expression...") + TFName = TF_expression(TFName, List, Match2, A, outdir) + print("Generate RE chromatin accessibility...") + RE_pseudobulk.to_csv(outdir + "Openness.txt", sep="\t", header=None, index=None) + print("Generate TF binding...") + Element_name_bulk = pd.read_csv( + GRNdir + "all_hg19.txt", delimiter="\t", header=None + ) + Element_name_bulk = Element_name_bulk[0].values Element_name = RE_pseudobulk.index - motifWeight=pd.read_csv(GRNdir+'motifWeight.txt',index_col=0,sep='\t') - load_TFbinding(GRNdir,motifWeight,Match2,TFName,Element_name,outdir) - print('Generate Index...') - #Read hg19_Peak_hg19_gene_u.txt - merged_s,merged_b=load_corr_RE_TG(List,Element_name,Element_name_bulk,outdir) + motifWeight = pd.read_csv(GRNdir + "motifWeight.txt", index_col=0, sep="\t") + load_TFbinding(GRNdir, motifWeight, Match2, TFName, Element_name, outdir) + print("Generate Index...") + # Read hg19_Peak_hg19_gene_u.txt + merged_s, merged_b = load_corr_RE_TG( + List, Element_name, Element_name_bulk, outdir + ) from tqdm import tqdm - #Assuming you have imported the necessary libraries and defined the variables - #Create a progress bar for the loop - choosL=List - out=np.empty([len(choosL),4], dtype=object) + + # Assuming you have imported the necessary libraries and defined the variables + # Create a progress bar for the loop + choosL = List + out = np.empty([len(choosL), 4], dtype=object) for i in tqdm(range(len(choosL))): choosL_i = choosL[i] - out[i, :] = index_generate(choosL_i,merged_s,merged_b,TFName) - pd.DataFrame(out).to_csv(outdir+'index.txt', sep='\t', header=None, index=None) - elif method=='baseline': - print('Overlap the regions with bulk data ...') - #script_path = os.path.join( "extract_overlap_regions_baseline.sh") - #subprocess.run(["sh", script_path, GRNdir, genome,outdir,workdir]) - extract_overlap_regions(genome,GRNdir,outdir,method) + out[i, :] = index_generate(choosL_i, merged_s, merged_b, TFName) + pd.DataFrame(out).to_csv( + outdir + "index.txt", sep="\t", header=None, index=None + ) + elif method == "baseline": + print("Overlap the regions with bulk data ...") + # script_path = os.path.join( "extract_overlap_regions_baseline.sh") + # subprocess.run(["sh", script_path, GRNdir, genome,outdir,workdir]) + extract_overlap_regions(genome, GRNdir, outdir, method) else: - print('Method:' +method+ 'is not found! Please set method as baseline or LINGER') + print( + "Method:" + method + "is not found! Please set method as baseline or LINGER" + ) + -import scanpy as sc -#set some figure parameters for nice display inside jupyternotebooks. -import scipy -import pandas as pd import anndata import numpy as np -from scipy.sparse import coo_matrix -from scipy.sparse import csc_matrix -def get_adata(matrix,features,barcodes,label): +import pandas as pd +import scanpy as sc + +# set some figure parameters for nice display inside jupyternotebooks. +import scipy +from scipy.sparse import coo_matrix, csc_matrix + + +def get_adata(matrix, features, barcodes, label): ### generate the anndata - matrix.data=matrix.data.astype(np.float32) - adata=anndata.AnnData(X= csc_matrix(matrix.T)) - adata.var['gene_ids']=features[1].values - adata.obs['barcode']=barcodes[0].values - if len(barcodes[0].values[0].split("-"))==2: - adata.obs['sample'] = [int(string.split("-")[1]) for string in barcodes[0].values] + matrix.data = matrix.data.astype(np.float32) + adata = anndata.AnnData(X=csc_matrix(matrix.T)) + adata.var["gene_ids"] = features[1].values + adata.obs["barcode"] = barcodes[0].values + if len(barcodes[0].values[0].split("-")) == 2: + adata.obs["sample"] = [ + int(string.split("-")[1]) for string in barcodes[0].values + ] else: - adata.obs['sample'] = 1 - rows_to_select=features[features[2]=='Gene Expression'].index - adata_RNA = adata[:,rows_to_select] - rows_to_select=features[features[2]=='Peaks'].index - adata_ATAC = adata[:,rows_to_select] -### if you have the label (cell type annotation) - idx=adata_RNA.obs['barcode'].isin(label['barcode_use'].values) - adata_RNA=adata_RNA[idx] - adata_ATAC=adata_ATAC[idx] - label.index=label['barcode_use'] - adata_RNA.obs['label']=label.loc[adata_RNA.obs['barcode']]['label'].values - #barcode_indices = np.where(np.isin(adata_RNA.obs['barcode'].values, label['barcode_use'].values))[0] - #adata_ATAC = adata_ATAC[barcode_indices, :] - adata_ATAC.obs['label']=label.loc[adata_ATAC.obs['barcode']]['label'].values + adata.obs["sample"] = 1 + rows_to_select = features[features[2] == "Gene Expression"].index + adata_RNA = adata[:, rows_to_select] + rows_to_select = features[features[2] == "Peaks"].index + adata_ATAC = adata[:, rows_to_select] + ### if you have the label (cell type annotation) + idx = adata_RNA.obs["barcode"].isin(label["barcode_use"].values) + adata_RNA = adata_RNA[idx] + adata_ATAC = adata_ATAC[idx] + label.index = label["barcode_use"] + adata_RNA.obs["label"] = label.loc[adata_RNA.obs["barcode"]]["label"].values + # barcode_indices = np.where(np.isin(adata_RNA.obs['barcode'].values, label['barcode_use'].values))[0] + # adata_ATAC = adata_ATAC[barcode_indices, :] + adata_ATAC.obs["label"] = label.loc[adata_ATAC.obs["barcode"]]["label"].values adata_RNA.var["mt"] = adata_RNA.var_names.str.startswith("MT-") sc.pp.calculate_qc_metrics( - adata_RNA, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True -) + adata_RNA, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True + ) adata_RNA = adata_RNA[adata_RNA.obs.pct_counts_mt < 5, :].copy() - adata_RNA.var.index=adata_RNA.var['gene_ids'].values + adata_RNA.var.index = adata_RNA.var["gene_ids"].values adata_RNA.var_names_make_unique() - adata_RNA.var['gene_ids']=adata_RNA.var.index - selected_barcode=list(set(adata_RNA.obs['barcode'].values)&set(adata_ATAC.obs['barcode'].values)) - barcode_idx=pd.DataFrame(range(adata_RNA.shape[0]), index=adata_RNA.obs['barcode'].values) + adata_RNA.var["gene_ids"] = adata_RNA.var.index + selected_barcode = list( + set(adata_RNA.obs["barcode"].values) & set(adata_ATAC.obs["barcode"].values) + ) + barcode_idx = pd.DataFrame( + range(adata_RNA.shape[0]), index=adata_RNA.obs["barcode"].values + ) adata_RNA = adata_RNA[barcode_idx.loc[selected_barcode][0]] - barcode_idx=pd.DataFrame(range(adata_ATAC.shape[0]), index=adata_ATAC.obs['barcode'].values) + barcode_idx = pd.DataFrame( + range(adata_ATAC.shape[0]), index=adata_ATAC.obs["barcode"].values + ) adata_ATAC = adata_ATAC[barcode_idx.loc[selected_barcode][0]] - return adata_RNA,adata_ATAC + return adata_RNA, adata_ATAC -def get_adata_h5(adata_RNA,adata_ATAC,label): +def get_adata_h5(adata_RNA, adata_ATAC, label): ### generate the anndata - if len(adata_RNA.obs['barcode'].values[0].split("-"))==2: - adata_RNA.obs['sample'] = [int(string.split("-")[1]) for string in adata_RNA.obs['barcode'].values] - adata_ATAC.obs['sample'] = [int(string.split("-")[1]) for string in adata_ATAC.obs['barcode'].values] + if len(adata_RNA.obs["barcode"].values[0].split("-")) == 2: + adata_RNA.obs["sample"] = [ + int(string.split("-")[1]) for string in adata_RNA.obs["barcode"].values + ] + adata_ATAC.obs["sample"] = [ + int(string.split("-")[1]) for string in adata_ATAC.obs["barcode"].values + ] else: - adata_RNA.obs['sample'] = 1 - adata_ATAC.obs['sample'] = 1 -### if you have the label (cell type annotation) - idx=adata_RNA.obs['barcode'].isin(label['barcode_use'].values) - adata_RNA=adata_RNA[idx] - adata_ATAC=adata_ATAC[idx] - label.index=label['barcode_use'] - adata_RNA.obs['label']=label.loc[adata_RNA.obs['barcode']]['label'].values - #barcode_indices = np.where(np.isin(adata_RNA.obs['barcode'].values, label['barcode_use'].values))[0] - #adata_ATAC = adata_ATAC[barcode_indices, :] - adata_ATAC.obs['label']=label.loc[adata_ATAC.obs['barcode']]['label'].values + adata_RNA.obs["sample"] = 1 + adata_ATAC.obs["sample"] = 1 + ### if you have the label (cell type annotation) + idx = adata_RNA.obs["barcode"].isin(label["barcode_use"].values) + adata_RNA = adata_RNA[idx] + adata_ATAC = adata_ATAC[idx] + label.index = label["barcode_use"] + adata_RNA.obs["label"] = label.loc[adata_RNA.obs["barcode"]]["label"].values + # barcode_indices = np.where(np.isin(adata_RNA.obs['barcode'].values, label['barcode_use'].values))[0] + # adata_ATAC = adata_ATAC[barcode_indices, :] + adata_ATAC.obs["label"] = label.loc[adata_ATAC.obs["barcode"]]["label"].values adata_RNA.var["mt"] = adata_RNA.var_names.str.startswith("MT-") sc.pp.calculate_qc_metrics( - adata_RNA, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True -) + adata_RNA, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True + ) adata_RNA = adata_RNA[adata_RNA.obs.pct_counts_mt < 5, :].copy() - adata_RNA.var.index=adata_RNA.var['gene_ids'].values + adata_RNA.var.index = adata_RNA.var["gene_ids"].values adata_RNA.var_names_make_unique() - adata_RNA.var['gene_ids']=adata_RNA.var.index - selected_barcode=list(set(adata_RNA.obs['barcode'].values)&set(adata_ATAC.obs['barcode'].values)) - barcode_idx=pd.DataFrame(range(adata_RNA.shape[0]), index=adata_RNA.obs['barcode'].values) + adata_RNA.var["gene_ids"] = adata_RNA.var.index + selected_barcode = list( + set(adata_RNA.obs["barcode"].values) & set(adata_ATAC.obs["barcode"].values) + ) + barcode_idx = pd.DataFrame( + range(adata_RNA.shape[0]), index=adata_RNA.obs["barcode"].values + ) adata_RNA = adata_RNA[barcode_idx.loc[selected_barcode][0]] - barcode_idx=pd.DataFrame(range(adata_ATAC.shape[0]), index=adata_ATAC.obs['barcode'].values) + barcode_idx = pd.DataFrame( + range(adata_ATAC.shape[0]), index=adata_ATAC.obs["barcode"].values + ) adata_ATAC = adata_ATAC[barcode_idx.loc[selected_barcode][0]] - return adata_RNA,adata_ATAC \ No newline at end of file + return adata_RNA, adata_ATAC diff --git a/code/lingergrn-1.106/LingerGRN/pseudo_bulk.py b/code/lingergrn-1.106/LingerGRN/pseudo_bulk.py index 32abcb1..17de0da 100644 --- a/code/lingergrn-1.106/LingerGRN/pseudo_bulk.py +++ b/code/lingergrn-1.106/LingerGRN/pseudo_bulk.py @@ -1,90 +1,103 @@ +import random + import numpy as np import pandas as pd -import random import scanpy as sc -#from LingerGRN.immupute_dis import immupute_dis + + +# from LingerGRN.immupute_dis import immupute_dis def tfidf(ATAC): O = 1 * (ATAC > 0) - tf1 = O / (np.ones((O.shape[0], 1)) * np.log(1 + np.sum(O, axis=0))[np.newaxis,:]) + tf1 = O / (np.ones((O.shape[0], 1)) * np.log(1 + np.sum(O, axis=0))[np.newaxis, :]) idf = np.log(1 + O.shape[1] / (1 + np.sum(O > 0, axis=1))) O1 = tf1 * (idf[:, np.newaxis] * np.ones((1, O.shape[1]))) O1[np.isnan(O1)] = 0 RE = O1.T return RE -def find_neighbors(adata_RNA,adata_ATAC): + + +def find_neighbors(adata_RNA, adata_ATAC): import scanpy as sc + K = 20 - #sc.tl.pca(adata_RNA, svd_solver="arpack") + # sc.tl.pca(adata_RNA, svd_solver="arpack") sc.pp.normalize_total(adata_RNA, target_sum=1e4) sc.pp.log1p(adata_RNA) sc.pp.highly_variable_genes(adata_RNA, min_mean=0.0125, max_mean=3, min_disp=0.5) - adata_RNA.raw=adata_RNA + adata_RNA.raw = adata_RNA adata_RNA = adata_RNA[:, adata_RNA.var.highly_variable] sc.pp.scale(adata_RNA, max_value=10) - sc.tl.pca(adata_RNA, n_comps=15,svd_solver="arpack") - pca_RNA=adata_RNA.obsm['X_pca'] + sc.tl.pca(adata_RNA, n_comps=15, svd_solver="arpack") + pca_RNA = adata_RNA.obsm["X_pca"] sc.pp.log1p(adata_ATAC) sc.pp.highly_variable_genes(adata_ATAC, min_mean=0.0125, max_mean=3, min_disp=0.5) - adata_ATAC.raw=adata_ATAC + adata_ATAC.raw = adata_ATAC adata_ATAC = adata_ATAC[:, adata_ATAC.var.highly_variable] sc.pp.scale(adata_ATAC, max_value=10, zero_center=True) - sc.tl.pca(adata_ATAC, n_comps=15,svd_solver="arpack") - pca_ATAC=adata_ATAC.obsm['X_pca'] - pca = np.concatenate((pca_RNA,pca_ATAC), axis=1) - adata_RNA.obsm['pca']=pca - adata_ATAC.obsm['pca']=pca - #sc.pp.neighbors(adata_RNA, n_neighbors=K, n_pcs=30,use_rep='pca') - return adata_RNA,adata_ATAC + sc.tl.pca(adata_ATAC, n_comps=15, svd_solver="arpack") + pca_ATAC = adata_ATAC.obsm["X_pca"] + pca = np.concatenate((pca_RNA, pca_ATAC), axis=1) + adata_RNA.obsm["pca"] = pca + adata_ATAC.obsm["pca"] = pca + # sc.pp.neighbors(adata_RNA, n_neighbors=K, n_pcs=30,use_rep='pca') + return adata_RNA, adata_ATAC -def pseudo_bulk(adata_RNA,adata_ATAC,singlepseudobulk): +def pseudo_bulk(adata_RNA, adata_ATAC, singlepseudobulk): K = 20 - #sc.tl.pca(adata_RNA, svd_solver="arpack") + # sc.tl.pca(adata_RNA, svd_solver="arpack") sc.pp.normalize_total(adata_RNA, target_sum=1e4) sc.pp.log1p(adata_RNA) - sc.pp.filter_genes(adata_RNA, min_cells=3) + sc.pp.filter_genes(adata_RNA, min_cells=3) sc.pp.highly_variable_genes(adata_RNA, min_mean=0.0125, max_mean=3, min_disp=0.5) - adata_RNA.raw=adata_RNA + adata_RNA.raw = adata_RNA adata_RNA = adata_RNA[:, adata_RNA.var.highly_variable] sc.pp.scale(adata_RNA, max_value=10) - sc.tl.pca(adata_RNA, n_comps=15,svd_solver="arpack") - pca_RNA=adata_RNA.obsm['X_pca'] + sc.tl.pca(adata_RNA, n_comps=15, svd_solver="arpack") + pca_RNA = adata_RNA.obsm["X_pca"] sc.pp.log1p(adata_ATAC) - sc.pp.filter_genes(adata_ATAC, min_cells=3) + sc.pp.filter_genes(adata_ATAC, min_cells=3) sc.pp.highly_variable_genes(adata_ATAC, min_mean=0.0125, max_mean=3, min_disp=0.5) - adata_ATAC.raw=adata_ATAC + adata_ATAC.raw = adata_ATAC adata_ATAC = adata_ATAC[:, adata_ATAC.var.highly_variable] sc.pp.scale(adata_ATAC, max_value=10, zero_center=True) - sc.tl.pca(adata_ATAC, n_comps=15,svd_solver="arpack") - pca_ATAC=adata_ATAC.obsm['X_pca'] - pca = np.concatenate((pca_RNA,pca_ATAC), axis=1) - adata_RNA.obsm['pca']=pca - adata_ATAC.obsm['pca']=pca - sc.pp.neighbors(adata_RNA, n_neighbors=K, n_pcs=30,use_rep='pca') - connectivities=(adata_RNA.obsp['distances']>0) + sc.tl.pca(adata_ATAC, n_comps=15, svd_solver="arpack") + pca_ATAC = adata_ATAC.obsm["X_pca"] + pca = np.concatenate((pca_RNA, pca_ATAC), axis=1) + adata_RNA.obsm["pca"] = pca + adata_ATAC.obsm["pca"] = pca + sc.pp.neighbors(adata_RNA, n_neighbors=K, n_pcs=30, use_rep="pca") + connectivities = adata_RNA.obsp["distances"] > 0 import random - label=pd.DataFrame(adata_RNA.obs['label']) - label.columns=['label'] - label.index=adata_RNA.obs['barcode'].tolist() - #label=label['label'].values - cluster=list(set(label['label'].values)) - allindex=[] + + label = pd.DataFrame(adata_RNA.obs["label"]) + label.columns = ["label"] + label.index = adata_RNA.obs["barcode"].tolist() + # label=label['label'].values + cluster = list(set(label["label"].values)) + allindex = [] np.random.seed(42) # Set seed for reproducibility for i in range(len(cluster)): - temp=label[label['label']==cluster[i]].index - N = len(temp) # Total number of elements - if N>=10: - m = int(np.floor(np.sqrt(N)))+1 # Number of elements to sample - if singlepseudobulk>0: - m=1 + temp = label[label["label"] == cluster[i]].index + N = len(temp) # Total number of elements + if N >= 10: + m = int(np.floor(np.sqrt(N))) + 1 # Number of elements to sample + if singlepseudobulk > 0: + m = 1 sampled_elements = random.sample(range(N), m) - temp=temp[sampled_elements] - allindex=allindex+temp.tolist() - connectivities=pd.DataFrame(connectivities.toarray(),index=adata_RNA.obs['barcode'].tolist()) - connectivities=connectivities.loc[allindex].values - A=(connectivities @ adata_RNA.raw.X.toarray()) - TG_filter1=A/(K-1) - RE_filter1=(connectivities @ adata_ATAC.raw.X.toarray())/(K-1) - TG_filter1=pd.DataFrame(TG_filter1.T,columns=allindex,index=adata_RNA.raw.var['gene_ids'].tolist()) - RE_filter1=pd.DataFrame(RE_filter1.T,columns=allindex,index=adata_ATAC.raw.var['gene_ids'].tolist()) - return TG_filter1,RE_filter1 + temp = temp[sampled_elements] + allindex = allindex + temp.tolist() + connectivities = pd.DataFrame( + connectivities.toarray(), index=adata_RNA.obs["barcode"].tolist() + ) + connectivities = connectivities.loc[allindex].values + A = connectivities @ adata_RNA.raw.X.toarray() + TG_filter1 = A / (K - 1) + RE_filter1 = (connectivities @ adata_ATAC.raw.X.toarray()) / (K - 1) + TG_filter1 = pd.DataFrame( + TG_filter1.T, columns=allindex, index=adata_RNA.raw.var["gene_ids"].tolist() + ) + RE_filter1 = pd.DataFrame( + RE_filter1.T, columns=allindex, index=adata_ATAC.raw.var["gene_ids"].tolist() + ) + return TG_filter1, RE_filter1 diff --git a/code/lingergrn-1.106/setup.py b/code/lingergrn-1.106/setup.py index 7719fdd..28819af 100644 --- a/code/lingergrn-1.106/setup.py +++ b/code/lingergrn-1.106/setup.py @@ -1,13 +1,28 @@ from setuptools import setup setup( - name='LingerGRN', - version='1.106', - description='Gene regulatory network inference', - author='Kaya Yuan', - author_email='qyyuan33@gmail.com', - packages=['LingerGRN'], - license = "MIT", - url='https://github.com/Durenlab/LINGER', - install_requires=['torch', 'scipy==1.11.3', 'numpy==1.24.3', 'pandas==2.0.3', 'shap==0.42.0', 'scikit-learn==1.3.0', 'joblib==1.3.2','matplotlib==3.8.0','seaborn==0.13.0','statsmodels==0.14.1','umap-learn','scanpy==1.9.5','anndata==0.9.2','pybedtools==0.10.0'], + name="LingerGRN", + version="1.106", + description="Gene regulatory network inference", + author="Kaya Yuan", + author_email="qyyuan33@gmail.com", + packages=["LingerGRN"], + license="MIT", + url="https://github.com/Durenlab/LINGER", + install_requires=[ + "torch", + "scipy==1.11.3", + "numpy==1.24.3", + "pandas==2.0.3", + "shap==0.42.0", + "scikit-learn==1.3.0", + "joblib==1.3.2", + "matplotlib==3.8.0", + "seaborn==0.13.0", + "statsmodels==0.14.1", + "umap-learn", + "scanpy==1.9.5", + "anndata==0.9.2", + "pybedtools==0.10.0", + ], ) From 037431757c26c61a6b62022c8245e020c8d1c268 Mon Sep 17 00:00:00 2001 From: Arnav G <73305591+arnavg115@users.noreply.github.com> Date: Mon, 9 Mar 2026 15:18:23 -0700 Subject: [PATCH 3/4] pd.concat optimization --- code/lingergrn-1.106/LingerGRN/Compare.py | 14 +++++++++++--- code/lingergrn-1.106/LingerGRN/LINGER_tr.py | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/code/lingergrn-1.106/LingerGRN/Compare.py b/code/lingergrn-1.106/LingerGRN/Compare.py index f0706fb..72995f2 100644 --- a/code/lingergrn-1.106/LingerGRN/Compare.py +++ b/code/lingergrn-1.106/LingerGRN/Compare.py @@ -314,6 +314,7 @@ def driver_score(expression, aud_idx, GRN, outdir, adjust_method, corr_method): ) E = E + E.mean().mean() * 10 ** (-1) expression = expression / E + c_res,p_res, q_res = [],[],[] for i in range(len(allcelltype)): print("cell type " + allcelltype[i]) aud_idx1 = aud_idx.reset_index() @@ -333,14 +334,21 @@ def driver_score(expression, aud_idx, GRN, outdir, adjust_method, corr_method): print(np.isnan(FC).sum()) c, cp = correlation_FC(np.log(FC[0]).values, reg, corr_method) # idx=pd.DataFrame(range(expression.shape[0]),index=expression.index) - C_result = pd.concat([C_result, c], axis=1) - P_result = pd.concat([P_result, cp], axis=1) + c_res.append(c) + p_res.append[cp] + # C_result = pd.concat([C_result, c], axis=1) + # P_result = pd.concat([P_result, cp], axis=1) cp = cp.fillna(1) adjusted_p_values = pd.DataFrame( multipletests(cp[0].values, method=adjust_method)[1], index=c.index ) # print(adjusted_p_values) - Q_result = pd.concat([Q_result, adjusted_p_values], axis=1) + q_res.append(adjusted_p_values) + # Q_result = pd.concat([Q_result, adjusted_p_values], axis=1) + C_result = pd.concat(c_res, axis = 1) + P_result = pd.concat(p_res, axis = 1) + Q_result = pd.concat(q_res, axis = 1) + C_result.columns = allcelltype P_result.columns = allcelltype Q_result.columns = allcelltype diff --git a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py index 00e0df9..1d56da6 100644 --- a/code/lingergrn-1.106/LingerGRN/LINGER_tr.py +++ b/code/lingergrn-1.106/LingerGRN/LINGER_tr.py @@ -260,13 +260,15 @@ def get_TSS(GRNdir, genome, TSS_dis): def load_data(GRNdir, outdir): gene_all = pd.DataFrame([]) + gene_all_res = [] for i in range(22): chr = "chr" + str(i + 1) gene_file = GRNdir + chr + "_gene.txt" data0 = pd.read_csv(gene_file, sep="\t", header=None) data0["chr"] = chr data0["id_b"] = data0.index + 1 - gene_all = pd.concat([gene_all, data0]) + gene_all_res.append(data0) + gene_all = pd.concat(gene_all_res) chr = "chrX" gene_file = GRNdir + chr + "_gene.txt" data0 = pd.read_csv(gene_file, sep="\t", header=None) From 4a49c407616c15087d110bf050b50a747d71ca57 Mon Sep 17 00:00:00 2001 From: Arnav G <73305591+arnavg115@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:09:16 -0700 Subject: [PATCH 4/4] module t-test optimization --- code/lingergrn-1.106/LingerGRN/Compare.py | 39 ++++++++--------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/code/lingergrn-1.106/LingerGRN/Compare.py b/code/lingergrn-1.106/LingerGRN/Compare.py index 72995f2..97e63b9 100644 --- a/code/lingergrn-1.106/LingerGRN/Compare.py +++ b/code/lingergrn-1.106/LingerGRN/Compare.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd from scipy import stats +import matplotlib.pyplot as plt def assignLabel(W, p): @@ -17,9 +18,6 @@ def assignLabel(W, p): return S_gene, W2 -import matplotlib.pyplot as plt -import numpy as np - def qq_pval(p1, names, celltype): p1[np.isnan(p1)] = 1 @@ -48,8 +46,6 @@ def qq_pval(p1, names, celltype): class Module_obj: def __init__(self): - import numpy as np - import pandas as pd self.S_TG = pd.DataFrame() # Initialize A.x as an empty DataFrame self.pvalue_all = pd.DataFrame() # Initialize A.y as an empty list @@ -62,25 +58,24 @@ def diff_Module(Exp_TG, metadata, S_TG, K): celltype = metadata["celltype"].unique().tolist() pvalue_all = np.zeros((K, len(celltype))) tvalue_all = np.zeros((K, len(celltype))) - from scipy import stats from statsmodels.stats.multitest import multipletests - - for k in range(len(celltype)): - temp = Exp_TG.iloc[:, metadata["celltype"].values == celltype[k]] - aud_idxtemp = metadata[(metadata["celltype"].values == celltype[k])][ - "group" - ].values + from scipy.stats import ttest_ind + # compute masks ahead of time + celltype_masks = {ct: metadata["celltype"].values == ct for ct in celltype} + # access group values once + group_values = metadata['group'].values + for k, ct in enumerate(celltype): + mask = celltype_masks[ct] + temp = Exp_TG.iloc[:, mask] + aud_idxtemp = group_values[mask] Exp_mean = stats.zscore(temp.T).T.groupby(S_TG["Module"].values).mean() Exp_mean = Exp_mean.loc[range(1, K + 1)] X = Exp_mean.values[:, (aud_idxtemp == 1)] Y = Exp_mean.values[:, (aud_idxtemp == 0)] - p_values = np.zeros((K,)) - t_values = np.zeros((K,)) - from scipy.stats import ttest_ind - for i in range(K): - t_values[i], p_values[i] = ttest_ind(X[i], Y[i]) - # p_values = np.nan_to_num(p_values, nan=1) + # utilize vectorized ttest which is faster than the iterative variant + p_values, t_values = ttest_ind(X, Y, axis = 1) + pvalue_all[:, k] = p_values tvalue_all[:, k] = t_values @@ -130,8 +125,6 @@ def GWAS_Module_enrich(S_TG, TGset, GWASgene, K): def Module_trans(outdir, metadata, TG_pseudobulk, K, GWASfile=None): - import numpy as np - from scipy import stats print("loading GRN......") trans_reg = pd.read_csv( @@ -222,9 +215,6 @@ def remove_covariate(TG_pseudobulk, aud_idx, celltype): return Exp_norm -import numpy as np -import pandas as pd - def runWGCNA(celltypetemp, TG_pseudobulk_all, metadata): metadata_temp = metadata[metadata["celltype"].isin([celltypetemp])] @@ -259,7 +249,6 @@ def runWGCNA(celltypetemp, TG_pseudobulk_all, metadata): def correlation_FC(x, y, method): - import numpy as np from scipy import stats # Loop through each column of y and calculate correlation with x @@ -279,7 +268,6 @@ def correlation_FC(x, y, method): def driver_score(expression, aud_idx, GRN, outdir, adjust_method, corr_method): print("loading GRN......") - import numpy as np from statsmodels.stats.multitest import multipletests allcelltype = aud_idx["celltype"].unique() @@ -356,7 +344,6 @@ def driver_score(expression, aud_idx, GRN, outdir, adjust_method, corr_method): def driver_result(C_result, Q_result, K): - import pandas as pd # Create a sample DataFrame of size 100x7 # Rank all values in the DataFrame