Purpose of the notebook is to generate reports via saved images

import os
import sys
sys.path.append("/oak/stanford/groups/akundaje/manyu/C2H2_ZNF_project/utils/modisco_utils")
sys.path.append(os.path.abspath("/oak/stanford/groups/akundaje/manyu/softwares/tfmodisco_tf_models/src/"))
sys.path.append(os.path.abspath("/oak/stanford/groups/akundaje/manyu/softwares/tfmodisco_tf_models/notebooks/reports/"))
from tfmodisco.run_tfmodisco import import_shap_scores, import_tfmodisco_results
from motif.read_motifs import pfm_info_content, trim_motif_by_ic
import motif.moods as moods
import plot.viz_sequence as viz_sequence
from util import figure_to_vdom_image, import_peak_table
import h5py
import pandas as pd
import numpy as np
import modisco
import sklearn.decomposition
import umap
import matplotlib.pyplot as plt
import matplotlib.font_manager as font_manager
import vdom.helpers as vdomh
from IPython.display import display
import tqdm
import io
import base64
import urllib
import deepdish as dd
import shutil
from vdom.helpers import h1, p, img, div, b

Main class to generate reports in HTML form using the saved images and h5 files

from shutil import copyfile
def get_read_handle(h5_file):
    '''Copy to temp with a hashed name and give the read handle'''
    name = h5_file.split('/')[-1]
    #copy_file = '/users/manyu/temp/{}_{}'.format(str(hash(h5_file)),name)
    copy_file = '/srv/scratch/manyu/tmp/{}_{}'.format(str(hash(h5_file)),name)
    
    if os.path.exists(copy_file):
        os.remove(copy_file)
    copyfile(h5_file,copy_file)
    f = h5py.File(copy_file,'r')
    return(f)


def close_h5file_read_handle(h5_file,read_handle):
    '''Removes the copied h5 file in temp and the read handle to avoid bloating'''
    read_handle.close()
    name = h5_file.split('/')[-1]
    #copy_file = '/users/manyu/temp/{}_{}'.format(str(hash(h5_file)),name)
    copy_file = '/srv/scratch/manyu/tmp/{}_{}'.format(str(hash(h5_file)),name)
    if os.path.exists(copy_file):
        os.remove(copy_file)
    
    
    
  

    

class ReportGenerator():
    def __init__(self,tf,dataset):
        self.dataset = dataset
        self.tf = tf
        # Path to saved modisco results on /oak. Images and h5 files
        self.reports_base = '/oak/stanford/groups/akundaje/manyu/C2H2_ZNF_project/train_profile_models_2020/modisco_reports'
        self.results_dir = '{}/{}/{}'.format(self.reports_base,dataset,tf)
        
        ## Paths to the saved modisco report results to copy/copied on Mitra: Images and h5 files
        self.modisco_reports_results_mitra = '/srv/www/kundaje/manyu/C2H2_ZNF_project/modisco_report_saved_information'
        self.mitra_results_dataset_dir = '{}/{}'.format(self.modisco_reports_results_mitra,self.dataset)
        if not os.path.exists(self.mitra_results_dataset_dir):
            os.makedirs(self.mitra_results_dataset_dir)
        self.save_results_mitra_dir  = '{}/{}'.format(self.mitra_results_dataset_dir,self.tf)
        
        self.main_patterns_dir = '{}/main_patterns'.format(self.save_results_mitra_dir)
        self.main_patterns = '{}/all_motifs.h5'.format(self.main_patterns_dir)
        
        self.subpatterns_dir  = '{}/sub_patterns'.format(self.save_results_mitra_dir)
        self.subpatterns = '{}/all_motif_subclusters.h5'.format(self.subpatterns_dir)
        
        
        ## Paths to the modisco reports html on lab cluster
        self.modisco_reports_base = '/srv/www/kundaje/manyu/C2H2_ZNF_project/modisco_reports'
        self.modisco_reports_dataset_base = '{}/{}'.format(self.modisco_reports_base,self.dataset)
        if not os.path.exists(self.modisco_reports_dataset_base):
            os.makedirs(self.modisco_reports_dataset_base)
        self.modisco_report_filename = '{}/{}.html'.format(self.modisco_reports_dataset_base,self.tf)
        
        ## Path to webpages and weblinks to images(needs weblinks to load; not local links)
        self.mitra_web_base = 'http://mitra.stanford.edu/kundaje/manyu/C2H2_ZNF_project/'
        self.mitra_web_results_base = '{}/modisco_report_saved_information/{}/{}'.format(self.mitra_web_base,self.dataset,self.tf)
        self.mitra_web_results_main_patterns = '{}/main_patterns'.format(self.mitra_web_results_base)
        self.mitra_web_results_sub_patterns = '{}/sub_patterns'.format(self.mitra_web_results_base)
        self.mitra_web_results_profiles = '{}/profiles_dir'.format(self.mitra_web_results_base)
        self.mitra_web_results_b1h_aln = '{}/b1h_aligments'.format(self.mitra_web_results_base)
        
        ##Do we need this here?
        #self.mitra_web_reports_base = '{}/modisco_reports'.format(self.mitra_web_base)
        
        
    
    def copy_to_mitra(self):
        assert(os.path.exists(self.results_dir))
        if os.path.exists(self.save_results_mitra_dir):
            shutil.rmtree(self.save_results_mitra_dir)
        shutil.copytree(self.results_dir,self.save_results_mitra_dir)
        print('Finished copying files to Mitra')
        permissions_cmd = 'chmod -R 777 {}'.format(self.save_results_mitra_dir)
        os.system(permissions_cmd)
        
        
        
    def show_subcluster_table(self):
      
        assert(os.path.exists(self.main_patterns))
        
        ## Opening a read handle for h5. Remmeber to close it also
        try:
            patterns_read_handle = get_read_handle(self.main_patterns)
        except Exception as e:
            print(e)
        
        try:
            sub_patterns_read_handle = get_read_handle(self.subpatterns)
        except Exception as e:
            print(e)

        num_patterns  = len(patterns_read_handle.keys())

     
        
        for pattern_i in range(num_patterns):
            
            display(vdomh.h3("Pattern {}/{}".format(str(pattern_i + 1), num_patterns)))
        
            

            # Sort this out:
            colgroup = vdomh.colgroup(
                vdomh.col(style={"width": "5%"}),
                vdomh.col(style={"width": "5%"}),
                vdomh.col(style={"width": "50%"}),
                vdomh.col(style={"width": "40%"})
            )
            header = vdomh.thead(
                vdomh.tr(
                    vdomh.th("Subpattern", style={"text-align": "center"}),
                    vdomh.th("Seqlets", style={"text-align": "center"}),
                    vdomh.th("Embeddings", style={"text-align": "center"}),
                    vdomh.th("CWM", style={"text-align": "center"}),
                    vdomh.th("PFM", style={"text-align": "center"})
                )
            )
            
            
            ## Fill in the paths and seqlets of the main pattern
            total_seqlets = patterns_read_handle[str(pattern_i)]['n_seqlets'].value
            emb_agg_path = '{}/{}_subcluster_agg.png'.format(self.mitra_web_results_sub_patterns,str(pattern_i))
            pwm_path = '{}/{}_pfm_full.png'.format(self.mitra_web_results_main_patterns,str(pattern_i))
            cwm_path = '{}/{}_cwm_full.png'.format(self.mitra_web_results_main_patterns,str(pattern_i))
            
            
            table_rows = [vdomh.tr(
                vdomh.td("Agg."),
                vdomh.td(str(total_seqlets)),
                vdomh.td(img(src=emb_agg_path)),
                vdomh.td(img(src=cwm_path)),
                vdomh.td(img(src=pwm_path))
            )]

            
            total_sub_pats = len(sub_patterns_read_handle[str(pattern_i)].keys())
            for subpat_i in range(total_sub_pats):
                emb_path = '{}/{}_subcluster_{}.png'.format(self.mitra_web_results_sub_patterns,str(pattern_i),str(subpat_i))
            
                total_seqlets = sub_patterns_read_handle[str(pattern_i)]['subcluster_{}'.format(str(subpat_i))]['n_seqlets'].value
                pwm_path = '{}/{}_subcluster_{}_pfm_trimmed.png'.format(self.mitra_web_results_sub_patterns,str(pattern_i),str(subpat_i))
                cwm_path = '{}/{}_subcluster_{}_cwm_trimmed.png'.format(self.mitra_web_results_sub_patterns,str(pattern_i),str(subpat_i))
                table_rows.append(vdomh.tr(
                vdomh.td('{}'.format(str(subpat_i))),
                vdomh.td(str(total_seqlets)),
                vdomh.td(img(src=emb_path)),
                vdomh.td(img(src=cwm_path)),
                vdomh.td(img(src=pwm_path))
            ))
            table = vdomh.table(header,vdomh.tbody(*table_rows))
            
            display(table)
            
        #Remove unnecessary temp files
        close_h5file_read_handle(self.main_patterns,patterns_read_handle)
        close_h5file_read_handle(self.subpatterns,sub_patterns_read_handle)
    
    
    def show_footprints_and_peak_dist(self):
        try:
            patterns_read_handle = get_read_handle(self.main_patterns)
        except Exception as e:
            print(e)
        
        num_patterns  = len(patterns_read_handle.keys())
        for pattern_i in range(num_patterns):
            
            display(vdomh.h3("Pattern {}/{} Footprint".format(str(pattern_i + 1), num_patterns)))
            footprint_plot = '{}/profiles_pattern_{}.png'.format(self.mitra_web_results_profiles,str(pattern_i))
            motif_distplot  = '{}/motif_distances_pattern_{}.png'.format(self.mitra_web_results_profiles,str(pattern_i))
            display(img(src=footprint_plot))
            display(vdomh.h3("Pattern {}/{} Motif Distance Distribution".format(str(pattern_i + 1), num_patterns)))
            display(img(src=motif_distplot))
    
    
    def show_b1h_alignments(self):
        try:
            patterns_read_handle = get_read_handle(self.main_patterns)
        except Exception as e:
            print(e)
        


        num_patterns  = len(patterns_read_handle.keys())
        header = vdomh.thead(
            vdomh.tr(
                vdomh.th("Pattern", style={"text-align": "center"}),
                vdomh.th("N-Seqlets", style={"text-align": "center"}),
                vdomh.th("Alignment", style={"text-align": "center"})

            )
        )

         
        display(vdomh.h2("B1H alignments of all patterns"))
        table_rows = []
        for pattern_i in range(num_patterns):
            
            
        
            total_seqlets = patterns_read_handle[str(pattern_i)]['n_seqlets'].value
            alignment_img = '{}/pattern_{}_aligned.png'.format(self.mitra_web_results_b1h_aln,str(pattern_i))
            

            table_rows.append(vdomh.tr(
            vdomh.td('{}'.format(str(pattern_i+1))),
            vdomh.td('{}'.format(str(total_seqlets))),
            vdomh.td(img(src=alignment_img)),

            ))
        close_h5file_read_handle(self.main_patterns,patterns_read_handle)
        table = vdomh.table(header,vdomh.tbody(*table_rows))
        display(table)

# Define parameters/fetch arguments
dataset = os.environ["DATASET"]
tf = os.environ["TF"]

# tf = 'ENCSR904JEY'
# dataset = 'ENCODE_new'

vdomh.h1('Running {} in dataset: {}'.format(tf,dataset))

Running ENCSR370NFS in dataset: ENCODE_new

a = ReportGenerator(tf=tf,dataset=dataset)

Show motifs and submotifs and relevant embeddings

a.show_subcluster_table()

Pattern 1/38

/users/manyu/.local/lib/python3.6/site-packages/ipykernel_launcher.py:127: H5pyDeprecationWarning: dataset.value has been deprecated. Use dataset[()] instead.
/users/manyu/.local/lib/python3.6/site-packages/ipykernel_launcher.py:146: H5pyDeprecationWarning: dataset.value has been deprecated. Use dataset[()] instead.

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	919
0	229
1	208
2	122
3	103
4	68
5	65
6	59
7	40
8	17
9	4
10	2
11	2

Pattern 2/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	381
0	97
1	64
2	58
3	45
4	33
5	25
6	20
7	19
8	9
9	4
10	3
11	2
12	2

Pattern 3/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	285
0	62
1	47
2	43
3	37
4	34
5	32
6	12
7	7
8	6
9	5

Pattern 4/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	252
0	80
1	50
2	39
3	32
4	15
5	13
6	7
7	6
8	4
9	4
10	2

Pattern 5/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	258
0	56
1	44
2	34
3	30
4	21
5	19
6	11
7	9
8	6
9	5
10	5
11	4
12	4
13	3
14	3
15	2
16	2

Pattern 6/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	176
0	50
1	47
2	45
3	19
4	7
5	6
6	2

Pattern 7/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	167
0	41
1	40
2	31
3	28
4	14
5	7
6	3
7	3

Pattern 8/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	170
0	39
1	39
2	31
3	25
4	15
5	14
6	4
7	3

Pattern 9/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	161
0	49
1	48
2	13
3	12
4	12
5	11
6	8
7	6
8	2

Pattern 10/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	144
0	46
1	40
2	19
3	17
4	15
5	5
6	2

Pattern 11/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	147
0	50
1	44
2	34
3	14
4	3
5	2

Pattern 12/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	138
0	32
1	32
2	26
3	22
4	13
5	7
6	4
7	2

Pattern 13/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	102
0	24
1	22
2	22
3	20
4	6
5	6
6	2

Pattern 14/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	103
0	32
1	23
2	20
3	10
4	7
5	7
6	2
7	2

Pattern 15/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	101
0	27
1	20
2	15
3	14
4	11
5	8
6	6

Pattern 16/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	120
0	30
1	24
2	20
3	15
4	14
5	13
6	4

Pattern 17/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	91
0	31
1	22
2	17
3	10
4	6
5	3
6	2

Pattern 18/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	121
0	28
1	26
2	25
3	13
4	9
5	9
6	9
7	2

Pattern 19/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	84
0	26
1	21
2	20
3	10
4	3
5	2
6	2

Pattern 20/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	97
0	33
1	31
2	20
3	9
4	4

Pattern 21/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	114
0	28
1	25
2	21
3	12
4	10
5	6
6	6
7	4
8	2

Pattern 22/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	56
0	17
1	16
2	11
3	7
4	5

Pattern 23/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	70
0	17
1	16
2	10
3	10
4	9
5	6
6	2

Pattern 24/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	62
0	23
1	19
2	16
3	4

Pattern 25/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	108
0	23
1	23
2	21
3	16
4	9
5	6
6	5
7	5

Pattern 26/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	109
0	38
1	37
2	27
3	4
4	3

Pattern 27/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	68
0	23
1	17
2	16
3	12

Pattern 28/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	56
0	14
1	14
2	10
3	9
4	9

Pattern 29/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	61
0	15
1	14
2	11
3	11
4	10

Pattern 30/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	64
0	16
1	16
2	11
3	9
4	7
5	3
6	2

Pattern 31/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	85
0	24
1	16
2	14
3	10
4	9
5	7
6	3
7	2

Pattern 32/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	55
0	23
1	17
2	15

Pattern 33/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	47
0	25
1	18
2	4

Pattern 34/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	82
0	23
1	21
2	18
3	10
4	7
5	3

Pattern 35/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	71
0	39
1	28
2	2
3	2

Pattern 36/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	56
0	31
1	14
2	7
3	2
4	2

Pattern 37/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	67
0	28
1	20
2	11
3	4
4	4

Pattern 38/38

Subpattern	Seqlets	Embeddings	CWM	PFM
Agg.	73
0	23
1	22
2	13
3	9
4	4
5	2

Show footprints and motif distributions within peaks

a.show_footprints_and_peak_dist()

Pattern 1/38 Footprint

Pattern 1/38 Motif Distance Distribution

Pattern 2/38 Footprint

Pattern 2/38 Motif Distance Distribution

Pattern 3/38 Footprint

Pattern 3/38 Motif Distance Distribution

Pattern 4/38 Footprint

Pattern 4/38 Motif Distance Distribution

Pattern 5/38 Footprint

Pattern 5/38 Motif Distance Distribution

Pattern 6/38 Footprint

Pattern 6/38 Motif Distance Distribution

Pattern 7/38 Footprint

Pattern 7/38 Motif Distance Distribution

Pattern 8/38 Footprint

Pattern 8/38 Motif Distance Distribution

Pattern 9/38 Footprint

Pattern 9/38 Motif Distance Distribution

Pattern 10/38 Footprint

Pattern 10/38 Motif Distance Distribution

Pattern 11/38 Footprint

Pattern 11/38 Motif Distance Distribution

Pattern 12/38 Footprint

Pattern 12/38 Motif Distance Distribution

Pattern 13/38 Footprint

Pattern 13/38 Motif Distance Distribution

Pattern 14/38 Footprint

Pattern 14/38 Motif Distance Distribution

Pattern 15/38 Footprint

Pattern 15/38 Motif Distance Distribution

Pattern 16/38 Footprint

Pattern 16/38 Motif Distance Distribution

Pattern 17/38 Footprint

Pattern 17/38 Motif Distance Distribution

Pattern 18/38 Footprint

Pattern 18/38 Motif Distance Distribution

Pattern 19/38 Footprint

Pattern 19/38 Motif Distance Distribution

Pattern 20/38 Footprint

Pattern 20/38 Motif Distance Distribution

Pattern 21/38 Footprint

Pattern 21/38 Motif Distance Distribution

Pattern 22/38 Footprint

Pattern 22/38 Motif Distance Distribution

Pattern 23/38 Footprint

Pattern 23/38 Motif Distance Distribution

Pattern 24/38 Footprint

Pattern 24/38 Motif Distance Distribution

Pattern 25/38 Footprint

Pattern 25/38 Motif Distance Distribution

Pattern 26/38 Footprint

Pattern 26/38 Motif Distance Distribution

Pattern 27/38 Footprint

Pattern 27/38 Motif Distance Distribution

Pattern 28/38 Footprint

Pattern 28/38 Motif Distance Distribution

Pattern 29/38 Footprint

Pattern 29/38 Motif Distance Distribution

Pattern 30/38 Footprint

Pattern 30/38 Motif Distance Distribution

Pattern 31/38 Footprint

Pattern 31/38 Motif Distance Distribution

Pattern 32/38 Footprint

Pattern 32/38 Motif Distance Distribution

Pattern 33/38 Footprint

Pattern 33/38 Motif Distance Distribution

Pattern 34/38 Footprint

Pattern 34/38 Motif Distance Distribution

Pattern 35/38 Footprint

Pattern 35/38 Motif Distance Distribution

Pattern 36/38 Footprint

Pattern 36/38 Motif Distance Distribution

Pattern 37/38 Footprint

Pattern 37/38 Motif Distance Distribution

Pattern 38/38 Footprint

Pattern 38/38 Motif Distance Distribution

Show B1H alignments of main motifs

a.show_b1h_alignments()

B1H alignments of all patterns

/users/manyu/.local/lib/python3.6/site-packages/ipykernel_launcher.py:207: H5pyDeprecationWarning: dataset.value has been deprecated. Use dataset[()] instead.

Pattern	N-Seqlets	Alignment
1	919
2	381
3	285
4	252
5	258
6	176
7	167
8	170
9	161
10	144
11	147
12	138
13	102
14	103
15	101
16	120
17	91
18	121
19	84
20	97
21	114
22	56
23	70
24	62
25	108
26	109
27	68
28	56
29	61
30	64
31	85
32	55
33	47
34	82
35	71
36	56
37	67
38	73