Nc@s-ddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z m Z dZdefdYZdfdYZd efd YZd fd YZd ZdZdZdZdZdZdddZdS(iN(tarray(tgetReverseComplementtgetConfigParsertgetConfigOptions3.0tReadDatasetErrorcBseZRS((t__name__t __module__(((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRst ReadCountercBsDeZdZdZdZiddZdZdZdZRS(igtbothcCs||_||_dS(N(tmultiReadIDCounttsense(tselfR R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt__init__s cCsk|j|}|dkr7|jtd|7_n0t|jrX|jd7_n|jd7_dS(Nig?(tgetReadMultiplicitytmultiReadCounttfloatt isSpliceEntrytcigartspliceReadCountt uniqReadCount(R treadt multiplicity((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt__call__s  cCsNt|}d|j|f}y|j|}Wntk rId}nX|S(Ns%s%si(tgetPairedReadNumberSuffixtqnameR tKeyError(R RtpairReadSuffixtreadIDR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR 's   cCs/t|}d|j|f}|jj|S(Ns%s%s(RRR thas_key(R RRR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytisMulti2s ( RRRRRR RR R(((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRs tMaxCoordFindercBs#eZdZdZdZdZRS(icCs||j|r*t|j|j|_nNt|jr`t|jt|j|j|_nt|j|j|_dS(N( Rtmaxt maxMultiStarttposRRtgetSpliceRightStarttmaxSpliceStarttmaxUniqueStart(R R((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR>s '(RRR$R R#R(((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR9st ReadDatasetcBseZdZedeeedZdZdZdZdZ dZ dZ d Z d Z d Zd Zd ZddddZddZdZdZdZddZddZdZdZdZedZdZdZeeedZdd ZeeeddeeeeeeeeeeddMdMd!eeded"Z!eddedd#Z"ed$d%Z#ded&Z$ed'Z%d(Z&eeddeeeeeeeeeedMdMdd)Z'd*Z(dMdMdMeeeed+d,Z)ddMdMd+d-Z*dddd.Z+ddddedd/Z,dMdMdMd+ed0Z-dMdMdMd+ed1Z.dddded2Z/eeeed!d3Z0dMeed4Z1d!d!eed5ed+d6d7Z2d8Z3dd9Z4d:Z5d;Z6d<Z7d=Z8eeed+d>Z9eeed?Z:d@Z;eeedAZ<dBZ=dCdDZ>edEZ?edFZ@dGZAddHZBdIZCdedJZDddKZEddLZFRS(NsC Class for storing reads from experiments. Assumes that custom scripts will translate incoming data into a format that can be inserted into the class using the insert* methods. Default class subtype ('DNA') includes tables for unique and multireads, whereas 'RNA' subtype also includes a splices table. tDNAcCsZd|_d|_d|_t|_t|_d|_d|_d|_ |j |d|_ t j |d|_d|_d|_idd6|_|r|d krtdn|r||_|j|jn|jd|_y|jd|_Wn-tk r*|jdttfgnX|j|_|rV|j|||ndS( s creates an rds datafile if initialize is set to true, otherwise will append to existing tables. datasetType can be either 'DNA' or 'RNA'. ttrbtRNAtdataTypeR&s8failed to initialize: datasetType must be 'DNA' or 'RNA't rdsVersionN(R&R)(tdbcontmemconR*tcurrentRDSVersionR+tFalset memBackedtmemChromt memCursort cachedDBFiletgetMultiReadIDCountstmultiReadIDCountstpysamtSamfiletbamfiletNonetreadsizettotalReadWeighttmetadataRtinitializeTablesRtinsertMetadataRtgetFullReadCounttfullReadCountst printRDSInfo(R tdatafilet initializet datasetTypetverbosetcachet reportCount((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR Qs4             cCs=d}|jj}x$|D]}||jj|7}qW|S(s return the number of reads in the bam file This is not the same as the original as the original returned total weight. i(R8t referencestcount(R RIRHt reference((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt__len__s   c Csytj||}Wn#tk r;dGHtjdnXi}xk|jdtD]W}t|}d|j|f}y||cd7d}tg|D]\}}||kr|^q|_dS(Nii(ii(tsumR:(R Rttaketoptlength((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRscCsdS(s6 returns 0 as cache is going to be deprecated i((R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRbscCs;|rt|j}nt|j}|j|S(sS returns a sorted list of distinct chromosomes in bam file reference list. (tlisttgetFullReferenceNamestgetShortReferenceNamesR_(R t fullChromR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetChromosomess  cCst|jjS(s4 returns a set of bam file reference names. (tsetR8RH(R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRscCs\t}|jj}x@|D]8}|d}t|jdkr|j|qqW|S(sV returns a set of bam file reference names after removing first 3 characters. ii(RR8RHtlentstriptadd(R RRHt chromosomet shortName((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRs    cCstd}t}|jjd|d||r:|j}n|rUt|j|}n|rpt|j|}n|S(sI returns the maximum coordinate for reads on a given chromosome. iRJtcallback(RR8RPR$RR#R (R tchromtdoUniqstdoMultit doSplicestmaxCoordt coordFinder((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetMaxCoordinates  tstartcCsgd}d|||f}|j}y+|j|t|jdd}Wnd|GHnX|S(Nis)select max(%s) from %s where chrom = '%s's,couldn't retrieve coordMax for chromosome %s(RRxR[R(R RRt startFieldRt sqlStatementR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetMaxStartCoordinateInTable,s   ic$Cs|jjd|d|d|}i}|r|dkrg|D]B}t|js@it|jd6t|jd6dd6^q@|| %ds%s < %disreadID LIKE 's%'smismatch != ''RRs sense = '%s's and swhere %s(RR(R1RtstringtjoinR(R RRRRRRRRRZt startTexttstopTextt whereClausetflagLikeClauset readIDClausetwhereStatementt whereQuery((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetReadWhereQueryws:      c Cs|g}|r|jdn|s5|jdn|rK|jdn|ra|jdn|rw|jdntj|d}|S(NRR RRRt,(RRR( R t baseSelectRRRRRt selectClauset selectQuery((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetReadSelectQuerys cCsRg}|rdg}n|dkr?| r?|jd|ntj|}|S(NsGROUP BY start, senseisLIMIT %d(RRR(R RRRtgroupByt groupQuery((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetReadGroupQuerys c#Cs|jjd|d|d|}i}d}x|D]}t|jsOq4nt|j}|j}|r|jj|j}n|jj|jd}| r|}d|kr| r|j d\}}n|}n|}|j dkr|j |jnt |j|j |j\}}}}it|d6}t||d= %ss%s <= %ss and swhere %ss0select count(distinct chrom+%s+sense) from %s %ssselect sum(weight) from %s %s( R1RR`RRRRRxtfetchoneR[(R RRRRtrestricttdistinctRRRIRRRtresult((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetTableEntryCounts0         c Cs+|jd|d|d|d|}|jS(NRRRR (RR(R RRRR R R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytgetSplicesCounts$c Cs+|jd|d|d|d|}|jS(NRRRR (RR(R RRRR R R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt getUniqsCounts$c Cs+|jd|d|d|d|}|jS(NRRRR (RR(R RRRR R R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt getMultiCount)s$c Csg}|r|jdn|r2|jdn|rH|jdnt|dkrotj|d}nd}d}|dkrd|}nd||f} |j} | j| | j} |rg| D]} | djd d^qSg| D]} | d^qSd S( s get readID's. sselect readID from uniqssselect readID from multisselect readID from splicesis union R'sLIMIT %ds%s group by readID %sRN(RRRRRRxRR( R RRYRtpairedRRzt selectPartt limitParttsqlQueryRRtx((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt getReadIDs0s(     (c Cs|j}|r|g}n|j}|ji}xy|D]q}|rYd|GHng||<|rJ|jdkrJ|jdtd|dtdtdt}|j} x| D]} || d} | d } | d } | d }| d }| d }|jd}x6|D].}d|kr)qnt|}|dkrs|d}||d}t |d|d!}nR|dkrt |d}t ||d}|t |d|d!d}nt | t | d}d}t |t |kr| |d}n||}||}||j | |||gqWqWn|j dtd|dtdt}||jkrqAnx)||D]}|d}|d }|d }|jd}x|D]}d|krqnt|}|dkr-|d}||d}t |d|d!}nR|dkrt |d}t ||d}|t |d|d!d}n||d}||j ||||gqWqWqAW|S(sB returns the uniq and spliced mismatches in a dictionary. s%getting mismatches from chromosome %sR)RRRRRiRRRR RRtNRiRR( RRR_R*RRQRRRRR[RRR(R tmischromREt useSplicestreadlent hitChromListtsnpDictRt spliceDictt spliceIDListtspliceIDt spliceEntrytstartpostlefthalft rightstartR RtspMismatchListRt change_lent change_fromt change_baset change_post firsthalft secondhalft change_atthitDictt readEntryRt mismatchList((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRRs~       *             "  ($         ")g?ic Cs|j} yt| d} Wntk r9d} nX| d} d|} i}t| |dscCsZdt||f}|dkr9|dt|7}n|jj||jjdS(sM update a metadata field given the original value and the new value. s.update metadata set value='%s' where name='%s'R's and value='%s' N(R`R,RxR(R RdtnewValuet originalValueRz((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytupdateMetadatas  cCs$|jjd||jjdS(su inserts a list of (readID, chrom, start, stop, sense, weight, flag, mismatch) into the uniqs table. sninsert into uniqs(ID, readID, chrom, start, stop, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?)N(R,t executemanyR(R RJ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt insertUniqsscCs$|jjd||jjdS(su inserts a list of (readID, chrom, start, stop, sense, weight, flag, mismatch) into the multi table. sninsert into multi(ID, readID, chrom, start, stop, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?)N(R,ROR(R RJ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt insertMultiscCs$|jjd||jjdS(s inserts a list of (readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) into the splices table. sinsert into splices(ID, readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?,?,?)N(R,ROR(R RJ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt insertSplices"scCs$|jjd||jjdS(s inserts a list of (readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) into the multisplices table. sinsert into multisplices(ID, readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?,?,?)N(R,ROR(R RJ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytinsertMultisplices+scCsd}|dkrd}n|r;|jjd||n|r[|jjd||n|jdkr|r|jjd|||jjd||n|jjd S( s6 update reads on file database in a list region of regions for a chromosome to have a new flag. regionsList must have 4 fields per region of the form (flag, chrom, start, stop) or, with sense set to '+' or '-', 5 fields per region of the form (flag, chrom, start, stop, sense). R'Rs and sense = ? sGUPDATE uniqs SET flag = ? where chrom = ? and start >= ? and start < ? sGUPDATE multi SET flag = ? where chrom = ? and start >= ? and start < ? R)s\UPDATE splices SET flag = flag || ' L:' || ? where chrom = ? and startL >= ? and startL < ? s\UPDATE splices SET flag = flag || ' R:' || ? where chrom = ? and startR >= ? and startR < ? N(R,ROR*R(R t regionsListRRYRR R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt flagReads4s  cCsk|r|jd|n|r2|jd|n|jdkrZ|rZ|jd|n|jjdS(s4 set the flag fields in the entire dataset. RRYR)RN(t setFlagsInDBR*R,R(R RRRYR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytsetFlagsKscCs|jjd||fdS(s8 set the flag field for every entry in a table. sUPDATE %s SET flag = '%s'N(R,Rx(R RR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRV[scCs|jd|||dS(sn reset the flag fields in the entire dataset to clear. Useful for rerunning an analysis from scratch. R'N(RW(R RRYR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt resetFlagsbscCs|jjd|dS(NsIUPDATE multi SET weight = ? where chrom = ? and start = ? and readID = ? (R,RO(R treadList((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytreweighMultireadsistONcCs/y|jjd|Wnd|GHnXdS(NsPRAGMA SYNCHRONOUS = %ss-warning: couldn't set PRAGMA SYNCHRONOUS = %s(R,Rx(R tvalue((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytsetSynchronousPragmamscCsdS(N((R RFRj((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt setDBcache{scCs3|j}|j||r/|j}|SdS(N(RRxR(R t statementt returnResultsRR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRxs    cCs7|j||jr&|jjn |jjdS(N(RxR0R-RR,(R R_((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR~s  cCs||jkr"|j|n|jd|jjddGH|jjddGH|jjddGH|jjdd GH|jd kr|jjd d GH|jjd dGH|jjddGHn|jj|jddS(s Builds the file indeces for the main tables. Cache is the number of 1.5 kb pages to keep in memory. 100000 pages translates into 150MB of RAM, which is our default. tOFFs-CREATE INDEX uPosIndex on uniqs(chrom, start)sbuilt uPosIndexs(CREATE INDEX uChromIndex on uniqs(chrom)sbuilt uChromIndexs-CREATE INDEX mPosIndex on multi(chrom, start)sbuilt mPosIndexs(CREATE INDEX mChromIndex on multi(chrom)sbuilt mChromIndexR)s0CREATE INDEX sPosIndex on splices(chrom, startL)sbuilt sPosIndexs1CREATE INDEX sPosIndex2 on splices(chrom, startR)sbuilt sPosIndex2s*CREATE INDEX sChromIndex on splices(chrom)sbuilt sChromIndexR[N(RbR^R]R,RxR*R(R RF((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt buildIndexs(  cCsy|jd|jjd|jjd|jjd|jjd|jdkr|jjdy|jjdWnnX|jjd n|jjWn d GHnX|jd d S( s5 drops the file indices for the main tables. RasDROP INDEX uPosIndexsDROP INDEX uChromIndexsDROP INDEX mPosIndexsDROP INDEX mChromIndexR)sDROP INDEX sPosIndexsDROP INDEX sPosIndex2sDROP INDEX sChromIndexsproblem dropping indexR[N(R]R,RxR*R(R ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyt dropIndexs"  cCsd|_tjd|_|j|j|jj}d}|dkrkd|GHd|}||_n d|_|jjd|jjd|jjd|jd}g}x)|D]!}|j|d |d fqW|jj d ||j d ||j d ||j dkr7|j |n|r|dkr|jjd|jjd|j dkr|jjd|jjdqq|jjd|jjd|j dkr|jjd|jjdqnt |_tj|j_|jjdS(s makes a copy of the dataset into memory for faster access. Can be restricted to a "full" chromosome. Can also build the memory indices. R's:memory:s memSync %ss where chrom = '%s' sPRAGMA temp_store = MEMORYsPRAGMA CACHE_SIZE = 1000000sdelete from metadatas select name, value from metadataRR\s.insert into metadata(name, value) values (?,?)RRYR)s&CREATE INDEX uPosIndex on uniqs(start)s&CREATE INDEX mPosIndex on multi(start)s*CREATE INDEX sPosLIndex on splices(startL)s*CREATE INDEX sPosRIndex on splices(startR)s-CREATE INDEX uPosIndex on uniqs(chrom, start)s-CREATE INDEX mPosIndex on multi(chrom, start)s1CREATE INDEX sPosLIndex on splices(chrom, startL)s1CREATE INDEX sPosRIndex on splices(chrom, startR)N(R-tsqlitetconnectR=R,RR1RxRROtcopyDBEntriesToMemoryR*tcopySpliceDBEntriesToMemoryRQR0tRowt row_factoryR(R RRHRt whereclauseRtresults2R((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pytmemSyncsJ         c Cs|jj}|jd||f}g}x_|D]W}|j|d|dt|dt|d|d|d|d|d fq5W|jjd ||dS( NsKselect chrom, start, stop, sense, weight, flag, mismatch, readID from %s %sRRRRR RRRskinsert into %s(ID, readID, chrom, start, stop, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?)(R,RRxRR[R-RO(R RyRRt sourceEntriestdestinationEntriesR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRfs  Uc Cs|jj}|jd|}g}xy|D]q}|j|d|dt|dt|dt|dt|d|d|d |d |d f q/W|jjd |dS( Nsaselect chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch, readID from splices %sRRRRRRR RRRsinsert into splices(ID, readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?,?,?)(R,RRxRR[R-RO(R RRRmRnR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRgs O N(GRRt__doc__R/RQR RKR4R;R^RARaRsRtR]R{R|RRRRRR=RRRRbRRRRRR9RRRRRRRR?RRR RRRRRRRIR>RNRPRQRRRSRURWRVRXRZR]R^RxR~RbRcRlRfRg(((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR%Hs A           9     "  -  `     4  "Q  V       4 cCs|rd}nd}|S(NRR((treverseR ((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRs cCs7t}x*|D]"\}}|dkr t}Pq q W|S(Ni(R/RQ(tcigarTupleListtisSplicet operationR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRs  cCs[d}xN|D]F\}}|dkrIt||}t||}|S||7}q WdS(Nii(R[(RRqtoffsetRsRRR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR"$s c Csvd}xS|D]K\}}|dkrNt||}t||}d}q ||7}q W||}||||fS(Nii(R[( RR:RqRtRsRRRR((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyR1s   cCs>d}t|sdS|jr(d}n|jr:d}n|S(NR's/1s/2(t isPairedReadtis_read1tis_read2(Rt readSuffix((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRAs     cCs|jo|jp|jS(N(tis_proper_pairRvRw(R((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRuNsR'Rc Cs,g}d}d}tjd|}tjd|}xtt|D]}||} |t||}tj| |dkrqInyx|r||} nd} |dkrt| } t| } nt|d} |jd| | | f|d7}WqIt k rd SXqIWtj |d S( Nt^is\d+s\d+([ACGTN]|\^[ACGTN]+)RRis%s%d%sR'R( tretfindallR4RR[RtfindRRt IndexErrorR( Rt querySequenceR toutputtdeletionMarkertpositiontlengthstmismatchSequencest mismatchEntryRtgenomicNucleotidet#erange1BasedElandCompatiblePosition((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyRRs.      (tsqlite3RdRRkRnRuR{RNR6Rt commoncodeRRRR.t ExceptionRRRR%RRR"RRRuR(((sF/woldlab/castor/data00/home/georgi/code/erange-4.0a-BAM/ReadDataset.pyts6        &