# Filepaths and Hard-coded Defaults
proj_root = "/users/kcochran/projects/procap_models/"
sequence_path = proj_root + "genomes/GRCh38_no_alt_analysis_set_GCA_000001405.15.fasta"
chrom_sizes = proj_root + "genomes/hg38.chrom.sizes.withrRNA"
in_window = 2114
out_window = 1000
# stuff to get from config file
with open("2022-07-02_16-51-15_run1_modisco_config.txt") as config_f:
config_dict = {line.split()[0] : line.strip().split()[1] for line in config_f}
modisco_out_path = config_dict["modisco_out_path"]
scoring_type = config_dict["scoring_type"]
score_center_size = int(config_dict["score_center_size"])
profile_display_center_size = int(config_dict["profile_display_center_size"])
# digest what's in config file
assay_type, model_type, cell, accession, modisco_dir_base = modisco_out_path.split("/")[-5:]
ts_part1, ts_part2, run_str, _ = modisco_dir_base.split("_")
timestamp = ts_part1 + "_" + ts_part2
run = int(run_str.replace("run", ""))
print(modisco_out_path)
print("cell_type:", cell, accession)
print("timestamp:", timestamp)
print("run:", run)
print("scoring_type:", scoring_type)
print("score_center_size:", score_center_size)
print("profile_display_center_size:", profile_display_center_size)
/users/kcochran//projects/procap_models/modisco_out/procap/bpnetlite_basic_v2/A673/ENCSR046BCI/2022-07-02_16-51-15_run1_modisco cell_type: A673 ENCSR046BCI timestamp: 2022-07-02_16-51-15 run: 1 scoring_type: counts score_center_size: 1000 profile_display_center_size: 400
data_dir = proj_root + "/data/procap/processed/" + cell + "/" + accession + "/"
plus_bw_path = data_dir + "final.5prime.pos.bigWig"
minus_bw_path = data_dir + "final.5prime.neg.bigWig"
val_peak_path = data_dir + "peaks_uni_and_bi_train_and_val.bed.gz"
val_save_dir = proj_root + "model_out/" + assay_type + "/" + model_type + "/" + cell + "/" + accession + "/"
val_save_path = val_save_dir + timestamp + "_run" + str(run) + "_train_and_val"
attr_save_path = val_save_dir.replace("model_out", "deepshap_out") + timestamp + "_run" + str(run) + "_deepshap"
if not modisco_out_path.endswith("/"):
modisco_out_path = modisco_out_path + "/"
# task-specific filepaths
import os
assert scoring_type in ["profile", "counts"], scoring_type
if scoring_type == "profile":
scores_path = attr_save_path + "_prof.npy"
onehot_scores_path = attr_save_path + "_prof_onehot.npy"
modisco_obj_path = modisco_out_path + "results_allChroms_prof_slice500.hdf5"
seqlet_path = modisco_out_path + "seqlets_prof.txt"
else:
scores_path = attr_save_path + "_count.npy"
onehot_scores_path = attr_save_path + "_count_onehot.npy"
modisco_obj_path = modisco_out_path + "results_allChroms_count_slice500.hdf5"
seqlet_path = modisco_out_path + "seqlets_count.txt"
assert(os.path.exists(scores_path)), scores_path
assert(os.path.exists(onehot_scores_path)), onehot_scores_path
# Imports, Plotting Defaults
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
plot_params = {
"figure.titlesize": 22,
"axes.titlesize": 22,
"axes.labelsize": 20,
"legend.fontsize": 18,
"xtick.labelsize": 16,
"ytick.labelsize": 16,
"font.weight": "bold"
}
plt.rcParams.update(plot_params)
from IPython.display import display
import tqdm
tqdm.tqdm_notebook()
import numpy as np
from view_modisco_results_utils import *
/users/kcochran/miniconda3/envs/procap/lib/python3.7/site-packages/ipykernel_launcher.py:19: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0 Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
# Load in True Profiles and Sequences
import sys
sys.path.append('../1_train_models')
from data_loading import extract_peaks
one_hot_seqs, true_profs = extract_peaks(sequence_path,
plus_bw_path, minus_bw_path, val_peak_path, in_window, out_window,
max_jitter=0, verbose=True)
one_hot_seqs = one_hot_seqs.swapaxes(1,2)
one_hot_seqs = one_hot_seqs[:, (in_window // 2 - score_center_size // 2):(in_window // 2 + score_center_size // 2), :]
Reading FASTA: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [00:17<00:00, 1.39it/s] Loading Peaks: 44667it [01:37, 458.93it/s]
# Load in Coordinates of Examples
coords = load_coords(val_peak_path, in_window)
# Import SHAP scores, predicted profiles
hyp_scores = np.load(scores_path).swapaxes(1,2)
hyp_scores = hyp_scores[:, (in_window // 2 - score_center_size // 2):(in_window // 2 + score_center_size // 2), :]
pred_profs = np.exp(np.load(val_save_path + ".profs.npy"))
# Load modisco results object
tfm_obj = import_tfmodisco_results(modisco_obj_path, hyp_scores, one_hot_seqs)
motif_pfms, motif_hcwms, motif_cwms, \
motif_pfms_short, num_seqlets, \
motif_seqlets, num_metaclusters = plot_all_metaclusters(tfm_obj, one_hot_seqs, hyp_scores,
true_profs, pred_profs, coords,
in_window, out_window,
score_center_size,
profile_display_center_size)
11554 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
10125 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
7962 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
7566 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
4257 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
4145 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
4078 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
4051 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
2843 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
2209 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
2153 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
1938 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
1431 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
1346 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
1045 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
690 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
354 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
94 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
58 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
32 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
181 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
83 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
38 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
33 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
25 seqlets
Sequence (PFM) | |
Hypothetical contributions (hCWM) | |
Actual contributions (CWM) |
run_and_plot_tomtom(modisco_out_path, motif_pfms, motif_hcwms, motif_pfms_short, num_metaclusters)
The output directory '/users/kcochran//projects/procap_models/modisco_out/procap/bpnetlite_basic_v2/A673/ENCSR046BCI/2022-07-02_16-51-15_run1_modisco/tomtom' already exists. Its contents will be overwritten. Processing query 1 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.984216 Processing query 2 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.972927 Processing query 3 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=1 Processing query 4 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.954329 Processing query 5 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.990124 Processing query 6 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.996547 Processing query 7 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.970079 Processing query 8 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.998346 Processing query 9 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.9855 Processing query 10 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.9855 Processing query 11 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=1 Processing query 12 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=1 Processing query 13 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.990509 Processing query 14 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=1 Processing query 15 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.973178 Processing query 16 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.986785 Processing query 17 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.979456 Processing query 18 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.954591 Processing query 19 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.978637 Processing query 20 out of 20 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.994235 Can't locate HtmlMonolithWr.pm: /root/lib/perl/HtmlMonolithWr.pm: Permission denied at /software/meme/4.11.2/bin/tomtom_xml_to_html line 48. Warning: tomtom_xml_to_html exited abnormally and may have failed to create HTML output.
Motif ID | q-val | PWM |
---|---|---|
MA0916.1-Ets21C | 4.38346e-05 | |
MA0076.2-ELK4 | 4.38346e-05 | |
MA1484.1-ETS2 | 9.51595e-05 | |
MA0764.3-ETV4 | 0.000298849 | |
MA1940.1-ETV2::DRGX | 0.000298849 |
Motif ID | q-val | PWM |
---|---|---|
MA1513.1-KLF15 | 5.16776e-05 | |
MA0516.3-SP2 | 0.00014776 | |
MA0685.2-SP4 | 0.00014776 | |
MA0742.2-KLF12 | 0.00014776 | |
MA0079.5-SP1 | 0.000163251 |
Motif ID | q-val | PWM |
---|---|---|
MA0314.2-HAP3 | 2.10161e-05 | |
MA1644.1-NFYC | 3.11483e-05 | |
MA0060.3-NFYA | 3.11483e-05 | |
MA0502.2-NFYB | 0.0162431 | |
MA0316.1-HAP5 | 0.0168161 |
Motif ID | q-val | PWM |
---|---|---|
MA1475.1-CREB3L4 | 0.00638728 | |
MA0286.1-CST6 | 0.00638728 | |
MA1449.1-hlh-30 | 0.00638728 | |
MA0605.2-ATF3 | 0.00638728 | |
MA1336.1-TGA3 | 0.00638728 |
Motif ID | q-val | PWM |
---|---|---|
MA0474.3-Erg | 0.000772132 | |
MA0598.3-EHF | 0.000772132 | |
MA0062.3-GABPA | 0.000772132 | |
MA0761.2-ETV1 | 0.000772132 | |
MA1992.1-Ikzf3 | 0.000772132 |
Motif ID | q-val | PWM |
---|---|---|
MA1651.1-ZFP42 | 0.00286028 | |
MA0748.2-YY2 | 0.00501397 | |
MA0095.3-Yy1 | 0.0399096 | |
MA1927.1-YY1-2 | 0.0399096 | |
MA1460.1-pho | 0.0652459 |
Motif ID | q-val | PWM |
---|---|---|
MA0535.1-Mad | 0.0206967 | |
MA1818.1-Zm00001d052229 | 0.0206967 | |
MA1821.1-Zm00001d020595 | 0.0206967 | |
MA1832.1-Zm00001d002364 | 0.0206967 | |
MA1819.1-Zm00001d005892 | 0.0206967 |
Motif ID | q-val | PWM |
---|---|---|
MA0506.2-Nrf1 | 0.000292231 | |
MA1412.1-TSAR2 | 0.387092 |
Motif ID | q-val | PWM |
---|---|---|
MA1988.1-Atf3 | 1.24713e-05 | |
MA0478.1-FOSL2 | 7.43694e-05 | |
MA1128.1-FOSL1::JUN | 0.00013221 | |
MA0491.2-JUND | 0.00013221 | |
MA0490.2-JUNB | 0.00013221 |
Motif ID | q-val | PWM |
---|---|---|
MA0833.2-ATF4 | 8.75194e-05 | |
MA1636.1-CEBPG | 0.000602773 | |
MA0488.1-JUN | 0.0121305 | |
MA0025.2-NFIL3 | 0.0334403 | |
MA1702.1-Pdp1 | 0.0379851 |
Motif ID | q-val | PWM |
---|---|---|
MA1573.2-Thap11 | 9.94946e-09 | |
MA0088.2-ZNF143 | 0.0146411 | |
MA1716.1-ZNF76 | 0.026512 | |
MA1625.1-Stat5b | 0.396635 | |
MA0525.2-TP63 | 0.453823 |
Motif ID | q-val | PWM |
---|---|---|
MA0527.1-ZBTB33 | 4.19642e-05 |
Motif ID | q-val | PWM |
---|---|---|
MA1929.1-CTCF | 2.79368e-11 | |
MA0139.1-CTCF | 3.56202e-11 | |
MA1930.1-CTCF | 4.30118e-10 | |
MA1102.2-CTCFL | 6.62009e-06 | |
MA0531.1-CTCF | 1.94341e-05 |
Motif ID | q-val | PWM |
---|---|---|
MA0149.1-EWSR1-FLI1 | 6.79976e-12 |
Motif ID | q-val | PWM |
---|---|---|
MA0108.2-TBP | 7.23214e-06 | |
MA1961.1-PATZ1 | 0.000140317 | |
MA1820.1-Zm00001d024324 | 0.000140317 | |
MA1819.1-Zm00001d005892 | 0.000280103 | |
MA1817.1-Zm00001d020267 | 0.000313799 |
Motif ID | q-val | PWM |
---|---|---|
MA1933.1-ELK1::SREBF2 | 0.000430502 | |
MA0076.2-ELK4 | 0.00262661 | |
MA1939.1-ERF::SREBF2 | 0.00262661 | |
MA0475.2-FLI1 | 0.0041689 | |
MA0156.3-FEV | 0.0041689 |
Motif ID | q-val | PWM |
---|---|---|
MA0108.2-TBP | 0.021844 | |
MA1819.1-Zm00001d005892 | 0.021844 | |
MA1832.1-Zm00001d002364 | 0.0548352 | |
MA1821.1-Zm00001d020595 | 0.0553189 | |
MA1961.1-PATZ1 | 0.0637054 |
Motif ID | q-val | PWM |
---|---|---|
MA1833.1-Zm00001d049364 | 3.24576e-06 | |
MA2022.1-LOB | 3.55384e-05 | |
MA1817.1-Zm00001d020267 | 3.80009e-05 | |
MA1819.1-Zm00001d005892 | 6.2985e-05 | |
MA1820.1-Zm00001d024324 | 6.84977e-05 |
Motif ID | q-val | PWM |
---|---|---|
MA1513.1-KLF15 | 0.00820478 | |
MA0516.3-SP2 | 0.00820478 | |
MA0742.2-KLF12 | 0.00820478 | |
MA1511.2-KLF10 | 0.00820478 | |
MA0747.1-SP8 | 0.00820478 |
Motif ID | q-val | PWM |
---|---|---|
MA0591.1-Bach1::Mafk | 0.0557221 | |
MA1448.1-fos-1 | 0.0557221 | |
MA0150.2-Nfe2l2 | 0.0684371 | |
MA0478.1-FOSL2 | 0.0684371 | |
MA1461.1-sv | 0.0950865 |
The output directory '/users/kcochran//projects/procap_models/modisco_out/procap/bpnetlite_basic_v2/A673/ENCSR046BCI/2022-07-02_16-51-15_run1_modisco/tomtom' already exists. Its contents will be overwritten. Processing query 1 out of 5 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.971101 Processing query 2 out of 5 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.929491 Processing query 3 out of 5 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.936018 Processing query 4 out of 5 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.955741 Processing query 5 out of 5 Estimating pi_0 from all 3912 observed p-values. Estimating pi_0. Estimated pi_0=0.94647 Can't locate HtmlMonolithWr.pm: /root/lib/perl/HtmlMonolithWr.pm: Permission denied at /software/meme/4.11.2/bin/tomtom_xml_to_html line 48. Warning: tomtom_xml_to_html exited abnormally and may have failed to create HTML output.
Motif ID | q-val | PWM |
---|---|---|
MA0829.2-SREBF1 | 0.305851 | |
MA0830.2-TCF4 | 0.305851 | |
MA0828.2-SREBF2 | 0.305851 | |
MA0004.1-Arnt | 0.305851 | |
MA0499.2-MYOD1 | 0.305851 |
Motif ID | q-val | PWM |
---|---|---|
MA1053.1-ERF109 | 0.0174079 | |
MA0997.1-ERF069 | 0.0174079 | |
MA2022.1-LOB | 0.0174079 | |
MA1102.2-CTCFL | 0.0174079 | |
MA0975.1-CRF2 | 0.0174079 |
Motif ID | q-val | PWM |
---|---|---|
MA1880.1-Hey | 3.524e-05 | |
MA1819.1-Zm00001d005892 | 0.0033764 | |
MA1820.1-Zm00001d024324 | 0.00864863 | |
MA1877.1-Hes-b | 0.00864863 | |
MA1818.1-Zm00001d052229 | 0.0185633 |
Motif ID | q-val | PWM |
---|---|---|
MA1819.1-Zm00001d005892 | 0.00140206 | |
MA1832.1-Zm00001d002364 | 0.00242122 | |
MA1821.1-Zm00001d020595 | 0.00261271 | |
MA1820.1-Zm00001d024324 | 0.00541994 | |
MA2022.1-LOB | 0.00627875 |
Motif ID | q-val | PWM |
---|---|---|
MA0375.1-RSC30 | 0.0356011 | |
MA0374.1-RSC3 | 0.0356011 | |
MA1930.1-CTCF | 0.0356011 | |
MA0449.1-h | 0.0647462 | |
MA1650.1-ZBTB14 | 0.0763549 |