3%Nc@sddlZddlZddlZddlZddlZddlmZddlmZm Z m Z dZ de fdYZ dd dYZdS( iN(tarray(tgetReverseComplementtgetConfigParsertgetConfigOptions2.1tReadDatasetErrorcBseZRS((t__name__t __module__(((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR st ReadDatasetcBseZdZedeeedZdZdZdZdZ dZ dZ d Z d Z d Zd d d dZd dZdZdZdZdZddZd dZdZdZdedZeeedZddZeeed d eeeeeeeeeed dddeed edZd ed Zed!Zd"Zeed d eeeeeeeeeeddd d#Z d d d eeeed$d%Z!d d d d&Z"d d d d edd'Z#d d d d ed(Z$d d d d ed)Z%d d d d ed*Z&eeeedd+Z'dDeed,Z)ddeed-ed$d.d/Z*d0Z+d d1Z,d2Z-d3Z.d4Z/d5Z0eeed$d6Z1eeed7Z2eeed8Z3d9Z4d:d;Z5ed<Z6ed=Z7d>Z8dd?Z9d@Z:d edAZ;d dBZ<d dCZ=RS(EsC Class for storing reads from experiments. Assumes that custom scripts will translate incoming data into a format that can be inserted into the class using the insert* methods. Default class subtype ('DNA') includes tables for unique and multireads, whereas 'RNA' subtype also includes a splices table. tDNAc Csd|_d|_d|_t|_t|_d|_d|_d|_ |ri|d krit dn|r|r}dGHn|j ||j }n|}t j ||_t j|j_|jjd|r||_|j|jn|jd}|d|_y |jd}|d|_WnIy |jdttfgWqwtk rrd GHd |_qwXnX|r|j|||nd S( s creates an rds datafile if initialize is set to true, otherwise will append to existing tables. datasetType can be either 'DNA' or 'RNA'. tRtRNAs8failed to initialize: datasetType must be 'DNA' or 'RNA's caching ....sPRAGMA temp_store = MEMORYtdataTypet rdsVersions&could not add rdsVersion - read-only ?spre-1.0N(sDNAsRNA(tdbcontmemconR tcurrentRDSVersionR tFalset memBackedtmemChromt memCursort cachedDBFileRtcacheDBtsqlitetconnecttRowt row_factorytexecutetinitializeTablest getMetadatatinsertMetadatatfloattIOErrort printRDSInfo( tselftdatafilet initializet datasetTypetverbosetcachet reportCounttdbFiletmetadata((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt__init__sF              cCsN|j}||j7}|jdkr>||j7}nt|}|S(s; return the number of usable reads in the dataset. R (t getUniqsCountt getMultiCountR tgetSplicesCounttint(R!ttotal((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt__len__Js   cCs |jdkr|jndS(s2 cleanup copy in local cache, if present. R N(Rt uncacheDB(R!((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt__del__XscCs|rd|GHn d|GH|j}dGH|j}|jx#|D]}d|d||GHqIW|r| r|jnd|jGH|jrdGHndGHdS(NsINITIALIZED dataset %ss dataset %ss metadata:s sdefault cache size is %d pagess found indexs not indexed(RtkeystsorttprintReadCountstgetDefaultCacheSizethasIndex(R!R"R'R#R)t pnameListtpname((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR _s         cCsm|j}|j}|jdkr9d||fGHn0|jdkri|j}d|||fGHndS(NRs" %d unique reads and %d multireadsR s4 %d unique reads, %d spliced reads and %d multireads(R+R,R R-(R!tucounttmcounttscount((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR5vs   cCsTt}t|dddd}|t_dtj|_tj||jdS(s+ copy geneinfoDB to a local cache. tgeneraltcistematic_temptdefaults/tmps%s.dbN(RRttempfilettempdirtmktempRtshutiltcopyfile(R!tfilenamet configParsertcisTemp((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRs   cCstj|j|dS(s+ copy geneinfoDB to a local cache. N(RCRDR(R!RE((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt saveCacheDBscCsI|jdkrEytj|jWnd|jGHnXd|_ndS(s- delete geneinfoDB from local cache. R scould not delete %sN(RtostremovetcachedDB(R!((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR1s cCs!d||f}|j|dS(s: attach another database file to the readDataset. sattach '%s' as %sN(R(R!REtdbNametstmt((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytattachDBscCsd|}|j|dS(s4 detach a database file to the readDataset. s detach %sN(R(R!RLRM((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytdetachDBs t*R cCsGd|||||f}|dkr6|d|7}n|j|dS(s import into current RDS the table (with columns destcolumns, with default all columns) from the database file asname, using the column specification of ascolumns (default all). s&insert into %s %s select %s from %s.%sR s where flag = '%s' N(t executeCommit(R!RLttablet ascolumnst destcolumnstflaggedRM((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt importFromDBs cCstg}|j}|dkr+d|}nd|}|j||j}x|D]}|j|dqUW|S(sB get a list of table names in a particular database file. R s%s.s3select name from %ssqlite_master where type='table'tname(t getSqlCursorRtfetchalltappend(R!RLt resultListtsqlRMtresultstrow((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt getTabless       cCs(|jr|j}n |j}|S(N(Rt getMemCursort getFileCursor(R!R\((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRXs  cCs |jjS(s] returns a cursor to memory database for low-level (SQL) access to the data. (Rtcursor(R!((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR`scCs |jjS(s[ returns a cursor to file database for low-level (SQL) access to the data. (R Rb(R!((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRascCs3d}t|j|dtdd}|dkS(s= return True if the RDS file has at least one index. s5select count(*) from sqlite_master where type='index't returnResultsi(R.RtTrue(R!RMtcount((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR7s#icCs|jd||jd|jd|jd}d|}|jd||jd||jdkrd }d|}|jd |d }d|}|jd |n|jd S( s} creates table schema in a database connection, which is typically a database file or an in-memory database. sPRAGMA DEFAULT_CACHE_SIZE = %ds3create table metadata (name varchar, value varchar)s,insert into metadata values('dataType','%s')sstart int, stop intsw(ID INTEGER PRIMARY KEY, readID varchar, chrom varchar, %s, sense varchar, weight real, flag varchar, mismatch varchar)screate table uniqs %sscreate table multi %sR s,startL int, stopL int, startR int, stopR intscreate table splices %sscreate table multisplices %sN(RR tcommit(R!t dbConnectionR&tpositionSchemat tableSchema((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRs    c Csd}i}|dkr%d|}n|j}|jd||j}x|D]}|d}|d}||kr||||%jdd}'nd|'krh| rh|%jd\}'}$n|'|#krg| |'<|'}#|'}&qnit |!dd6}(|rt |!d|(d 1 and count(chrom) > 1) unionsfrom(sG) group by chrom, start having ( count(start) = 1 and count(chrom) = 1)sorder by startsorder by readID, startsorder by chrom, startiRitsenseitweightR treadIDR}is::t/tstoptflagtmismatchtpairIDN(tgetReadWhereQuerytgetReadSelectQuerytgetReadGroupQueryRZRlRmRtNoneRRR RXRR.RRRRy()R!tbothEndstnoSenseRR}Rt withWeighttwithFlagt withMismatchtwithIDt withChromt withPairIDRRtfindallOptimizet readIDDicttreadLikeRRtlimitt hasMismatchtflagLiketstrandt combine5pt whereQueryt selectQueryt groupQueryRMt subSelecttsqlStmtR\tsqlQueryRqR^t currentChromt currentReadIDRRtdictKeyt theReadIDtnewrow((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt getReadsDictqs $                J             c Cs| rd} d} n d} d} g} |dkrV||jkrV| jd|n|dkr|rtjd|dgd} | j| q| jd |n|d kr| jd | |fn|d kr| jd | |fnt|d kr0tjd|dgd}| j|n|rF| jdn|dkrf| jd|nt| d krtj| d}d|}nd}|S(NtstartLtstopRRRR s chrom = '%s's flag LIKE "%s%"s flag = '%s'is%s > %ds%s < %disreadID LIKE 's%'smismatch != ''t+t-s sense = '%s's and swhere %s(RR(RRZRlRmR~(R!R}RRRRRRRtsplicet startTexttstopTextRptflagLikeClauset readIDClausetwhereStatementR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRs:      c Cs|g}|r|jdn|s5|jdn|rK|jdn|ra|jdn|rw|jdntj|d}|S(NRRRRRt,(RZRlRm( R!t baseSelectRRRRRt selectClauseR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR"s cCsRg}|rdg}n|dkr?| r?|jd|ntj|}|S(NsGROUP BY start, senseisLIMIT %d(RZRlRm(R!RRRtgroupByR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR9s c" Cs|j|||||| |dt}d}|j|||||}|j}d||f}|j|d}d}i}x|D]}d}|d}|r|d}n|dd}| r||krg||<|}|}nX| rEd |kr|jd \}}n|}||krEg||<|}|}qEnit|d d 6}t|d |d = %ss%s <= %ss and swhere %ss0select count(distinct chrom+%s+sense) from %s %ssselect sum(weight) from %s %s( RRZRnR~RlRmRXRtfetchoneR.(R!RRR}RRRtdistinctRRpReRRR\tresult((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytgetTableEntryCounts0         c Cs"|jd|||||ddS(s9 returns the number of row in the splices table. RRR(R(R!R}RRRR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR-scCs|jd|||||S(sD returns the number of distinct readIDs in the uniqs table. R|(R(R!R}RRRR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR+scCs|jd|||||S(sA returns the total weight of readIDs in the multi table. R(R(R!R}RRRR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR,sc Csg}|r|jdn|r2|jdn|rH|jdnt|dkrotj|d}nd}d}|dkrd|}nd||f} |j} | j| | j} |rg| D]} | djd d^qSg| D]} | d^qSd S( s get readID's. sselect readID from uniqssselect readID from multisselect readID from splicesis union R sLIMIT %ds%s group by readID %sRN(RZR~RlRmRXRRYRy( R!R|RRtpairedRRMt selectPartt limitPartRR\Rtx((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt getReadIDss(     (c Cs|j}|r|g}n|j}|ji}x|D]}}|rYd|GHng||<|rP|jdkrP|jdtd|dtdtdt}|j} x| D]} || d} | d } | d } | d }| d }| d }|jd}x<|D]4}d|kr)qnt|}|dkrs|d}||d}t |d|d!}nX|dkrt |dg}t ||dg}|t |d|d!d}nt | t | d}d}t |t |kr| |d}n||}||}||j | |||gqWqWn|j dtd|dtdt}||jkrqAnx/||D]#}|d}|d }|d }|jd}x|D]}d|krqnt|}|dkr3|d}||d}t |d|d!}nX|dkrt |dg}t ||dg}|t |d|d!d}n||d}||j ||||gqWqWqAW|S(sB returns the uniq and spliced mismatches in a dictionary. s%getting mismatches from chromosome %sR RR}RRRiRRRRRRtNRiRR( R{RR4R RRdR3RyR~R.RRZR(R!tmischromR%t useSplicestreadlent hitChromListtsnpDicttachromt spliceDictt spliceIDListtspliceIDt spliceEntrytstartpostlefthalft rightstartRt mismatchestspMismatchListRt change_lent change_fromt change_baset change_post firsthalft secondhalft change_atthitDictt readEntryRt mismatchList((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt getMismatchess~       *             "  ($         ")g?ic Cs|j} yt| d} Wntk r9d} nX| d} d|} i}t| |d= ? and start < ? sGUPDATE multi SET flag = ? where chrom = ? and start >= ? and start < ? R s\UPDATE splices SET flag = flag || ' L:' || ? where chrom = ? and startL >= ? and startL < ? s\UPDATE splices SET flag = flag || ' R:' || ? where chrom = ? and startR >= ? and startR < ? N(R RR Rf(R!t regionsListR|RRRR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt flagReadss  cCsw|r|jjd|n|r:|jjd|n|jdkrf|rf|jjd|n|jjdS(s4 set the flag fields in the entire dataset. sUPDATE uniqs SET flag = '%s'sUPDATE multi SET flag = '%s'R sUPDATE splices SET flag = '%s'N(R RR Rf(R!RR|RR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytsetFlagsscCs|jd|||dS(sn reset the flag fields in the entire dataset to clear. Useful for rerunning an analysis from scratch. R N(R(R!R|RR((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt resetFlagsscCs|jjd|dS(NsIUPDATE multi SET weight = ? where chrom = ? and start = ? and readID = ? (R R(R!treadList((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytreweighMultireadsstONcCs/y|jjd|Wnd|GHnXdS(NsPRAGMA SYNCHRONOUS = %ss-warning: couldn't set PRAGMA SYNCHRONOUS = %s(R R(R!Rj((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytsetSynchronousPragma scCs5|jjd||r1|jjd|ndS(NsPRAGMA CACHE_SIZE = %dsPRAGMA DEFAULT_CACHE_SIZE = %d(R R(R!R&R?((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt setDBcache'scCs3|j}|j||r/|j}|SdS(N(RXRRY(R!RRcR\R((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR-s    cCs7|j||jr&|jjn |jjdS(N(RRRRfR (R!R((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRQ6s  cCs||jkr"|j|n|jd|jjddGH|jjddGH|jjddGH|jjdd GH|jd kr|jjd d GH|jjd dGH|jjddGHn|jj|jddS(s Builds the file indeces for the main tables. Cache is the number of 1.5 kb pages to keep in memory. 100000 pages translates into 150MB of RAM, which is our default. tOFFs-CREATE INDEX uPosIndex on uniqs(chrom, start)sbuilt uPosIndexs(CREATE INDEX uChromIndex on uniqs(chrom)sbuilt uChromIndexs-CREATE INDEX mPosIndex on multi(chrom, start)sbuilt mPosIndexs(CREATE INDEX mChromIndex on multi(chrom)sbuilt mChromIndexR s0CREATE INDEX sPosIndex on splices(chrom, startL)sbuilt sPosIndexs1CREATE INDEX sPosIndex2 on splices(chrom, startR)sbuilt sPosIndex2s*CREATE INDEX sChromIndex on splices(chrom)sbuilt sChromIndexR#N(R6R%R$R RR Rf(R!R&((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt buildIndex?s(  cCsy|jd|jjd|jjd|jjd|jjd|jdkr|jjdy|jjdWnnX|jjd n|jjWn d GHnX|jd d S( s5 drops the file indices for the main tables. R&sDROP INDEX uPosIndexsDROP INDEX uChromIndexsDROP INDEX mPosIndexsDROP INDEX mChromIndexR sDROP INDEX sPosIndexsDROP INDEX sPosIndex2sDROP INDEX sChromIndexsproblem dropping indexR#N(R$R RR Rf(R!((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyt dropIndex\s"  cCsd|_tjd|_|j|j|jj}d}|dkrkd|GHd|}||_n d|_|jjd|jjd|jjd|jd}g}x)|D]!}|j|d |d fqW|jj d ||j d ||j d ||j dkr7|j |n|r|dkr|jjd|jjd|j dkr|jjd|jjdqq|jjd|jjd|j dkr|jjd|jjdqnt |_tj|j_|jjdS(s makes a copy of the dataset into memory for faster access. Can be restricted to a "full" chromosome. Can also build the memory indices. R s:memory:s memSync %ss where chrom = '%s' sPRAGMA temp_store = MEMORYsPRAGMA CACHE_SIZE = 1000000sdelete from metadatas select name, value from metadataRWRjs.insert into metadata(name, value) values (?,?)R|RR s&CREATE INDEX uPosIndex on uniqs(start)s&CREATE INDEX mPosIndex on multi(start)s*CREATE INDEX sPosLIndex on splices(startL)s*CREATE INDEX sPosRIndex on splices(startR)s-CREATE INDEX uPosIndex on uniqs(chrom, start)s-CREATE INDEX mPosIndex on multi(chrom, start)s1CREATE INDEX sPosLIndex on splices(chrom, startL)s1CREATE INDEX sPosRIndex on splices(chrom, startR)N(RRRRR RbRRRZRtcopyDBEntriesToMemoryR tcopySpliceDBEntriesToMemoryRdRRRRf(R!R}RuRbt whereclauseR]tresults2R^((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pytmemSyncvsJ         c Cs|jj}|jd||f}g}x_|D]W}|j|d|dt|dt|d|d|d|d|d fq5W|jjd ||dS( NsKselect chrom, start, stop, sense, weight, flag, mismatch, readID from %s %sRR}RRRRRRskinsert into %s(ID, readID, chrom, start, stop, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?)(R RbRRZR.RR(R!RLRpRbt sourceEntriestdestinationEntriesR^((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR)s  Uc Cs|jj}|jd|}g}xy|D]q}|j|d|dt|dt|dt|dt|d|d|d |d |d f q/W|jjd |dS( Nsaselect chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch, readID from splices %sRR}RRRRRRRRsinsert into splices(ID, readID, chrom, startL, stopL, startR, stopR, sense, weight, flag, mismatch) values (NULL,?,?,?,?,?,?,?,?,?,?)(R RbRRZR.RR(R!RpRbR.R/R^((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyR*s O N(>RRt__doc__RRdR*R0R2R R5RRHR1RNRORVR_RXR`RaR7RRR{R6RRRRRRRRRRRR-R+R,RRRRRRRRRRRRR R"R$R%RRQR'R(R-R)R*(((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyRs 1           !   ,     N#)!P  M          4 ((tsqlite3RRlR@RCRIRt commoncodeRRRRt ExceptionRR(((s=/woldlab/castor/data00/home/georgi/erange-4.0a/ReadDataset.pyts